1use std::{
2 collections::{HashMap, HashSet, VecDeque},
3 error::Error,
4 fmt, io,
5 path::PathBuf,
6 process::{ExitStatus, Stdio},
7 sync::{Arc, Mutex, OnceLock},
8 time::{Duration, SystemTime, UNIX_EPOCH},
9};
10
11use cortexkit_log::Retention;
12use serde_json::Value;
13use sha2::{Digest, Sha256};
14use subc_control::{
15 ClientControlPush, LiveSpawn, ModuleProtocol, RouteCloseReason, SpawnCursor, SpawnEvent,
16 SpawnEventKind, SpawnSnapshot, SupervisorHealthStatus, TerminalDisposition, TerminalExitKind,
17};
18use subc_protocol::{
19 manifest::{SelfSignalKind, SignalAnchor},
20 session::{
21 HealthReport, HealthStatus, ModuleControlCommand, ModuleControlRequest,
22 MODULE_CONTROL_OP_HEALTH_CHECK,
23 },
24 Flags, FrameType, Principal, Priority, SUBC_LAUNCH_NONCE_ENV, SUBC_MODULE_ID_ENV,
25};
26use tokio::{
27 process::{Child, Command},
28 sync::{mpsc, oneshot, watch, Mutex as AsyncMutex},
29 task::JoinHandle,
30 time::{sleep, sleep_until, timeout, timeout_at, Instant},
31};
32use tracing::{debug, error, info, warn};
33
34use crate::{
35 child_roster::ChildRoster,
36 daemon_config::{
37 CAPTURE_KEEP_ENV, CAPTURE_MAX_AGE_DAYS_ENV, CAPTURE_MAX_FILE_MB_ENV, CK_LOG_ENV,
38 },
39 forwarding::{
40 CloseReason, ForwardingError, ForwardingTable, GoodbyeTarget, ModuleControlRpcOutcome,
41 ModuleDrainTarget, PendingModuleControlRpc,
42 },
43 provenance::{ExecutableIdentityProbe, SpawnedFileIdentity},
44 registry::{ConnectionId, RegistrationEndReason, RegistryError},
45 stderr_tail::{
46 pump_stderr_to, pump_stdout_to, ChildOutputSink, StderrRing, StderrTailConfig,
47 StderrTailSnapshot,
48 },
49 terminal_ring::{TerminalHistorySnapshot, TerminalRecord, TerminalRing, TerminalRingConfig},
50 Frame, FrameSink, Registry,
51};
52
53#[cfg(not(windows))]
54use crate::provenance::spawned_file_identity;
55
56#[path = "supervise_swap.rs"]
57mod swap;
58
59#[path = "supervise_activity.rs"]
60mod activity;
61
62mod slow_retry;
63use slow_retry::{FailedRetry, FailureContext};
64
65#[cfg(all(test, unix))]
66mod slow_retry_tests;
67
68pub const SUBC_ARG: &str = "--subc";
74
75const DEFAULT_MAX_RESTARTS: u32 = 3;
76const DEFAULT_BACKOFF: Duration = Duration::from_millis(100);
77const DEFAULT_MAX_BACKOFF: Duration = Duration::from_secs(30);
78const DEFAULT_RESTART_WINDOW: Duration = Duration::from_secs(600);
82pub const DEFAULT_DRAIN_TIMEOUT: Duration = Duration::from_secs(30);
93pub(crate) const DEFAULT_REGISTER_TIMEOUT: Duration = Duration::from_secs(120);
94const REGISTRY_RELEASE_TIMEOUT: Duration = Duration::from_secs(1);
95const REGISTRY_RELEASE_POLL: Duration = Duration::from_millis(10);
96const DRAIN_BUSY_REPROBE_INTERVAL: Duration = Duration::from_millis(250);
108const STDERR_PUMP_DRAIN_TIMEOUT: Duration = Duration::from_millis(250);
131pub const SPAWN_EVENT_RING_CAPACITY: usize = 4096;
133const SPAWN_SUBSCRIBER_BUFFER: usize = SPAWN_EVENT_RING_CAPACITY + 1;
134pub(crate) const SPAWN_SUBSCRIBER_LAGGED_CODE: &str = "spawn_subscriber_lagged";
139
140#[cfg(target_os = "macos")]
141struct PrivacyExec {
142 reader: tokio::io::unix::AsyncFd<std::io::PipeReader>,
143 deadline: tokio::time::Instant,
144 expected: Option<subc_os::FileIdentity>,
145 trampoline: Option<subc_os::FileIdentity>,
146 script: bool,
147 module_id: String,
148 #[cfg(test)]
149 confirmation_pause: Option<(String, Option<tokio::net::TcpStream>)>,
150}
151
152#[cfg(all(test, target_os = "macos"))]
153pub(crate) fn test_privacy_trampoline() -> PathBuf {
154 let path = std::env::current_exe()
157 .unwrap()
158 .parent()
159 .unwrap()
160 .parent()
161 .unwrap()
162 .join("privacy-trampoline-fixture");
163 assert!(
167 path.exists(),
168 "privacy-trampoline-fixture not built at {}: run `cargo build -p subc-daemon \
169 --bins --features test-support` or `cargo test -p subc-daemon` first",
170 path.display()
171 );
172 path
173}
174
175#[cfg(target_os = "macos")]
176fn probe_privacy_trampoline(path: &std::path::Path) -> Result<(), String> {
177 use std::io::Read;
178 let mut probe = std::process::Command::new(path)
179 .args(["__disclaim-exec", "--probe"])
180 .stdin(Stdio::null())
181 .stdout(Stdio::piped())
182 .stderr(Stdio::piped())
183 .spawn()
184 .map_err(|error| {
185 format!(
186 "privacy trampoline probe failed for {}: {error}",
187 path.display()
188 )
189 })?;
190 let deadline = std::time::Instant::now() + Duration::from_secs(5);
191 let status = loop {
192 match probe.try_wait() {
193 Ok(Some(status)) => break status,
194 Ok(None) if std::time::Instant::now() < deadline => {
195 std::thread::sleep(Duration::from_millis(5))
196 }
197 result => {
198 let _ = probe.kill();
199 let _ = probe.wait();
200 return Err(format!(
201 "privacy trampoline probe failed or timed out for {}: {result:?}",
202 path.display()
203 ));
204 }
205 }
206 };
207 let mut answer = String::new();
208 if let Some(stdout) = probe.stdout.take() {
209 let _ = stdout.take(256).read_to_string(&mut answer);
210 }
211 if status.success() && answer.trim() == subc_os::privacy_identity::TRAMPOLINE_PROBE {
212 return Ok(());
213 }
214 let mut diagnostic = String::new();
215 if let Some(stderr) = probe.stderr.take() {
216 let _ = stderr.take(1024).read_to_string(&mut diagnostic);
217 }
218 let cause = diagnostic
219 .trim()
220 .strip_prefix("ck-subc: own privacy identity refused: ")
221 .unwrap_or("binary does not implement the privacy trampoline protocol");
222 Err(format!(
223 "{cause}: probe of {} exited {status}",
224 path.display()
225 ))
226}
227
228#[cfg(target_os = "macos")]
229fn privacy_command(
230 spec: &ModuleSpec,
231 roster: &ChildRoster,
232) -> Result<
233 (
234 Command,
235 Option<PrivacyExec>,
236 subc_os::privacy_identity::ExecAcknowledgement,
237 ),
238 SuperviseError,
239> {
240 let failure = |cause: String| {
241 warn!(module_id = %spec.module_id, %cause, "privacy identity trampoline refused module spawn");
242 SuperviseError::Spawn {
243 program: spec.program.clone(),
244 source: io::Error::other(cause),
245 cgroup_path: None,
246 }
247 };
248 let trampoline = roster.privacy_trampoline().map_err(failure)?;
249 let program = if spec.program.components().count() == 1 && !spec.program.is_absolute() {
254 let path = spec
255 .env
256 .iter()
257 .find(|(key, _)| key == "PATH")
258 .map(|(_, value)| std::ffi::OsString::from(value))
259 .or_else(|| std::env::var_os("PATH"))
260 .unwrap_or_else(|| "/usr/bin:/bin".into());
261 std::env::split_paths(&path)
262 .map(|dir| dir.join(&spec.program))
263 .find(|path| path.is_file())
264 .unwrap_or_else(|| spec.program.clone())
265 } else {
266 spec.program.clone()
267 };
268 let expected = subc_os::file_identity(&program);
269 let trampoline_image = subc_os::file_identity(&trampoline);
270 let script = {
271 use std::io::Read;
272 let mut prefix = [0u8; 2];
273 std::fs::File::open(&program)
274 .is_ok_and(|mut file| file.read_exact(&mut prefix).is_ok() && &prefix == b"#!")
275 };
276 if expected.is_none() || expected == subc_os::file_identity(&trampoline) {
277 return Err(failure(
278 "privacy identity module executable is missing or is the trampoline itself".to_string(),
279 ));
280 }
281 let (reader, ack) = subc_os::privacy_identity::ExecAcknowledgement::pipe()
282 .map_err(|error| failure(error.to_string()))?;
283 let reader =
284 tokio::io::unix::AsyncFd::new(reader).map_err(|error| failure(error.to_string()))?;
285 let mut command = Command::new(&trampoline);
286 command
287 .arg("__disclaim-exec")
288 .arg(ack.fd().to_string())
289 .arg(&program);
290 ack.install(command.as_std_mut());
291 Ok((
292 command,
293 Some(PrivacyExec {
294 reader,
295 deadline: tokio::time::Instant::now() + Duration::from_secs(5),
296 expected,
297 trampoline: trampoline_image,
298 script,
299 module_id: spec.module_id.clone(),
300 #[cfg(test)]
301 confirmation_pause: spec
302 .env
303 .iter()
304 .find(|(key, _)| key == "SUBC_TEST_BEFORE_EXEC_CONFIRM")
305 .map(|(_, address)| (address.clone(), None)),
306 }),
307 ack,
308 ))
309}
310
311struct SupervisedChild {
312 subtree_target: crate::subtree_resources::Target,
313 child: Child,
314 exec: crate::activity::ExecOwner,
315 #[cfg(windows)]
316 nonce_delivery: Option<subc_os::LaunchNoncePipeDelivery>,
317 #[cfg(target_os = "macos")]
318 privacy_exec: Option<PrivacyExec>,
319 #[cfg(target_os = "macos")]
326 report_ready: Arc<OnceLock<()>>,
327 spawn_failure: Option<String>,
330 protocol: ModuleProtocol,
335 #[cfg(target_os = "linux")]
341 module_id: String,
342 #[cfg(target_os = "linux")]
343 cgroup_placement: Option<subc_cgroup::Placement>,
344 #[cfg(windows)]
366 job: Option<Arc<Mutex<subc_jobobject::JobObject>>>,
367 stdout_pump: Option<JoinHandle<()>>,
368 stderr_pump: Option<StderrPump>,
369 stderr_ring: Arc<Mutex<StderrRing>>,
370 spawned_at: Instant,
373 registration_deadline: Option<Instant>,
376 registration_watch: Option<RegistrationWatch>,
377 spawned_at_ms: u64,
378 spawned_from: PathBuf,
379 launch_spec_digest: LaunchSpecDigest,
380 spawned_file_identity: Option<SpawnedFileIdentity>,
381 process_start_time: Option<u64>,
382 process_identity: Option<ProcessIdentity>,
383 pid: u32,
384 roster_guard: Option<crate::child_roster::RosterGuard>,
387}
388
389impl SupervisedChild {
390 fn registered_since_spawn(&self) -> bool {
391 self.registration_watch
392 .as_ref()
393 .is_some_and(|watch| *watch.changes.borrow() != watch.baseline)
394 }
395
396 fn id(&self) -> Option<u32> {
397 Some(self.pid)
398 }
399
400 fn process_identity(&self) -> Option<ProcessIdentity> {
401 self.process_identity
402 }
403
404 async fn wait(&mut self) -> io::Result<ExitStatus> {
405 #[cfg(target_os = "macos")]
406 self.confirm_privacy_exec().await;
407 let result = self.child.wait().await;
415 #[cfg(target_os = "linux")]
416 if result.is_ok() {
417 if let Some(placement) = self.cgroup_placement.as_ref() {
418 cleanup_reaped_cgroup(placement, &self.module_id).await;
419 self.cgroup_placement = None;
422 }
423 }
424 result
425 }
426
427 #[cfg(target_os = "macos")]
428 async fn confirm_privacy_exec(&mut self) {
429 let Some(pending) = &mut self.privacy_exec else {
430 return;
431 };
432 let result = tokio::time::timeout_at(pending.deadline, async {
433 let mut record = Vec::new();
434 loop {
435 let mut ready = pending.reader.readable().await?;
436 let read = ready.try_io(|reader| {
437 use std::io::Read;
438 let mut reader = reader.get_ref();
439 let mut buffer = [0u8; 256];
440 reader.read(&mut buffer).map(|count| (count, buffer))
441 });
442 match read {
443 Ok(Ok((0, _))) => return Ok::<_, io::Error>(record),
444 Ok(Ok((count, buffer))) => {
445 if record.len() + count > 1024 {
446 return Err(io::Error::other(
447 "privacy exec refusal record is too long",
448 ));
449 }
450 record.extend_from_slice(&buffer[..count]);
451 }
452 Ok(Err(error)) => return Err(error),
453 Err(_) => continue,
454 }
455 }
456 })
457 .await;
458 #[cfg(test)]
459 let result = if let Some((address, stream)) = &mut pending.confirmation_pause {
460 use tokio::io::{AsyncReadExt, AsyncWriteExt};
461 if stream.is_none() {
462 *stream = Some(
463 tokio::net::TcpStream::connect(address.as_str())
464 .await
465 .unwrap(),
466 );
467 stream.as_mut().unwrap().write_all(b"R").await.unwrap();
468 stream
469 .as_mut()
470 .unwrap()
471 .write_all(&self.pid.to_ne_bytes())
472 .await
473 .unwrap();
474 }
475 let mut reply = [0];
476 stream
477 .as_mut()
478 .unwrap()
479 .read_exact(&mut reply)
480 .await
481 .unwrap();
482 pending.confirmation_pause = None;
483 if &reply == b"F" {
484 Ok(Err(io::Error::other("injected late exec refusal")))
485 } else {
486 assert_eq!(&reply, b"X");
487 result
488 }
489 } else {
490 result
491 };
492 let pending = self.privacy_exec.as_ref().expect("pending exec");
495 let cause = match result {
496 Err(_) => Some("privacy identity trampoline did not exec within 5s".to_string()),
497 Ok(Err(error)) => Some(format!(
498 "privacy identity exec acknowledgement failed: {error}"
499 )),
500 Ok(Ok(record)) if !record.is_empty() => Some(
501 std::str::from_utf8(&record)
502 .ok()
503 .and_then(|record| {
504 record
505 .trim()
506 .strip_prefix(subc_os::privacy_identity::EXEC_REFUSAL_TAG)
507 })
508 .filter(|cause| !cause.is_empty())
509 .unwrap_or("invalid privacy exec refusal record")
510 .to_string(),
511 ),
512 Ok(Ok(_)) => match self.child.try_wait() {
513 Ok(Some(_status)) => None,
517 Err(error) => Some(format!("privacy identity trampoline wait failed: {error}")),
518 Ok(None) => {
519 let image = observe_spawned_image(self.pid);
520 if let Some(image) = image.filter(|image| {
521 image.executable.is_some()
522 && image.executable != pending.trampoline
523 && (image.executable == pending.expected || pending.script)
524 }) {
525 if let Err(error) = self
526 .roster_guard
527 .as_ref()
528 .map(|guard| guard.confirm_executable(image))
529 .transpose()
530 {
531 Some(format!(
532 "could not durably record confirmed module image: {error}"
533 ))
534 } else {
535 let _ = self.report_ready.set(());
536 info!(module_id = %pending.module_id, pid = self.pid,
537 "module spawned with own privacy identity (responsibility disclaimed)");
538 None
539 }
540 } else if image.is_none()
541 || image.is_some_and(|image| image.executable.is_none())
542 {
543 match tokio::time::timeout_at(pending.deadline, self.child.wait()).await {
550 Ok(Ok(_status)) => None,
551 Ok(Err(error)) => Some(format!("privacy identity trampoline wait failed: {error}")),
552 Err(_) => Some("privacy identity module executable remained unreadable until the 5s exec deadline".to_string()),
553 }
554 } else {
555 Some("privacy identity trampoline executable mismatch: expected the module program, not ck-subc".to_string())
556 }
557 }
558 },
559 };
560 let pending = self.privacy_exec.take().expect("pending exec");
561 self.exec.0.finish(if cause.is_some() {
564 crate::activity::ExecState::Refused
565 } else {
566 crate::activity::ExecState::CommittedExec
567 });
568 if let Some(cause) = cause {
569 warn!(module_id = %pending.module_id, pid = self.pid, %cause, "privacy identity trampoline refused module spawn");
570 self.spawn_failure = Some(cause);
571 if let Some(pid) = rustix::process::Pid::from_raw(self.pid as i32) {
574 let _ = rustix::process::kill_process_group(pid, rustix::process::Signal::KILL);
575 }
576 let _ = self.child.start_kill();
577 }
578 }
579
580 fn release_roster(&mut self) {
584 self.roster_guard = None;
585 }
586
587 fn start_kill(&mut self) -> io::Result<()> {
600 #[cfg(windows)]
601 if let Some(job) = &self.job {
602 if let Err(error) = job.lock().unwrap_or_else(|p| p.into_inner()).terminate() {
603 debug!(
604 error = %error,
605 "job termination failed; the direct-child kill still owns the outcome"
606 );
607 }
608 }
609 #[cfg(target_os = "linux")]
610 kill_module_cgroup(self.cgroup_placement.as_ref(), &self.module_id);
611 self.child.start_kill()
612 }
613
614 async fn drain_stderr(&mut self, module_id: &str) {
615 if let Some(pump) = self.stdout_pump.take() {
616 settle_stdout_pump(module_id, pump, STDERR_PUMP_DRAIN_TIMEOUT).await;
617 }
618
619 let Some(pump) = self.stderr_pump.take() else {
620 return;
621 };
622 settle_stderr_pump(
623 module_id,
624 &self.stderr_ring,
625 pump,
626 STDERR_PUMP_DRAIN_TIMEOUT,
627 )
628 .await;
629 }
630}
631
632async fn settle_stdout_pump(module_id: &str, mut pump: JoinHandle<()>, bound: Duration) {
636 match timeout(bound, &mut pump).await {
637 Ok(Ok(())) => {}
638 Ok(Err(error)) => {
639 warn!(module_id, error = %error, "stdout pump ended unexpectedly");
640 }
641 Err(_) => {
642 drop(pump);
643 warn!(
644 module_id,
645 waited = ?bound,
646 "stdout pipe of the exited process has not drained; its reader keeps running without delaying the restart"
647 );
648 }
649 }
650}
651
652struct StderrPump {
655 task: JoinHandle<()>,
656 generation: u64,
657}
658
659async fn settle_stderr_pump(
665 module_id: &str,
666 ring: &Arc<Mutex<StderrRing>>,
667 pump: StderrPump,
668 bound: Duration,
669) {
670 let lock = || ring.lock().unwrap_or_else(|poisoned| poisoned.into_inner());
671 let StderrPump {
672 mut task,
673 generation,
674 } = pump;
675 lock().retire_pump(generation);
676 match timeout(bound, &mut task).await {
677 Ok(Ok(())) => {}
678 Ok(Err(err)) => {
679 let mut ring = lock();
680 ring.mark_incomplete_from(generation, format!("stderr pump ended unexpectedly: {err}"));
681 ring.finish_pump(generation);
682 warn!(module_id, error = %err, "stderr pump ended before clean EOF");
683 }
684 Err(_) => {
685 drop(task);
687 lock().mark_pump_late(
688 generation,
689 format!(
690 "stderr of the exited process had not reached EOF {bound:?} after it was \
691 retired (a descendant may still hold the pipe open); lines it still \
692 writes are kept in that process's section"
693 ),
694 );
695 warn!(
696 module_id,
697 waited = ?bound,
698 "stderr pipe of the exited process is still open; its reader keeps running without delaying the restart"
699 );
700 }
701 }
702}
703
704fn registration_release_events() -> &'static watch::Sender<u64> {
705 static EVENTS: OnceLock<watch::Sender<u64>> = OnceLock::new();
706 EVENTS.get_or_init(|| {
707 let (sender, _receiver) = watch::channel(0);
708 sender
709 })
710}
711
712pub(crate) fn notify_registration_release() {
713 let events = registration_release_events();
714 let next_generation = (*events.borrow()).wrapping_add(1);
715 events.send_replace(next_generation);
716}
717
718#[derive(Debug, Clone, PartialEq, Eq)]
720pub struct ModuleSpec {
721 pub module_id: String,
722 pub program: PathBuf,
723 pub args: Vec<String>,
724 pub env: Vec<(String, String)>,
725 pub reserved: bool,
730 pub reserved_prefixes: Vec<String>,
735 pub protocol: ModuleProtocol,
754 pub overlap: ModuleOverlap,
759}
760
761#[derive(Clone, PartialEq, Eq)]
764pub(crate) struct LaunchSpecDigest([u8; 32]);
765
766impl fmt::Debug for LaunchSpecDigest {
767 fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
768 f.write_str("LaunchSpecDigest([redacted])")
769 }
770}
771
772impl LaunchSpecDigest {
773 pub(crate) fn from_spec(spec: &ModuleSpec) -> Self {
774 fn field(hash: &mut Sha256, bytes: &[u8]) {
775 hash.update((bytes.len() as u64).to_le_bytes());
776 hash.update(bytes);
777 }
778 let mut hash = Sha256::new();
779 if let Some(program) = spec.program.to_str() {
780 field(&mut hash, b"utf8-program");
781 field(&mut hash, program.as_bytes());
782 } else {
783 #[cfg(unix)]
786 {
787 use std::os::unix::ffi::OsStrExt;
788 field(&mut hash, b"unix-program");
789 field(&mut hash, spec.program.as_os_str().as_bytes());
790 }
791 #[cfg(windows)]
792 {
793 use std::os::windows::ffi::OsStrExt;
794 field(&mut hash, b"windows-program");
795 let bytes = spec
796 .program
797 .as_os_str()
798 .encode_wide()
799 .flat_map(u16::to_le_bytes)
800 .collect::<Vec<_>>();
801 field(&mut hash, &bytes);
802 }
803 #[cfg(not(any(unix, windows)))]
804 field(&mut hash, spec.program.to_string_lossy().as_bytes());
805 }
806 hash.update((spec.args.len() as u64).to_le_bytes());
807 for arg in &spec.args {
808 field(&mut hash, arg.as_bytes());
809 }
810 let mut env = spec.env.iter().collect::<Vec<_>>();
811 env.sort_unstable();
812 hash.update((env.len() as u64).to_le_bytes());
813 for (key, value) in env {
814 field(&mut hash, key.as_bytes());
815 field(&mut hash, value.as_bytes());
816 }
817 field(
818 &mut hash,
819 match spec.protocol {
820 ModuleProtocol::Subc => b"subc",
821 ModuleProtocol::None => b"none",
822 },
823 );
824 Self(hash.finalize().into())
825 }
826}
827
828#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)]
835pub enum ModuleOverlap {
836 #[default]
838 Exclusive,
839 Safe,
851}
852
853impl ModuleOverlap {
854 pub fn as_str(self) -> &'static str {
855 match self {
856 Self::Exclusive => "exclusive",
857 Self::Safe => "safe",
858 }
859 }
860}
861
862pub const SUBC_SPAWN_ROLE_ENV: &str = "SUBC_SPAWN_ROLE";
872pub const SPAWN_ROLE_SWAP_CANDIDATE: &str = "swap_candidate";
874pub const DEFAULT_SWAP_READY_TIMEOUT: Duration = Duration::from_secs(100);
879
880#[derive(Debug, Clone, Copy, PartialEq, Eq)]
898pub struct RestartPolicy {
899 pub max_restarts: u32,
900 pub backoff: Duration,
903 pub max_backoff: Duration,
905 pub window: Duration,
909}
910
911impl RestartPolicy {
912 pub fn new(max_restarts: u32, backoff: Duration) -> Self {
916 Self {
917 max_restarts,
918 backoff,
919 max_backoff: DEFAULT_MAX_BACKOFF,
920 window: DEFAULT_RESTART_WINDOW,
921 }
922 }
923
924 pub fn with_max_backoff(mut self, max_backoff: Duration) -> Self {
925 self.max_backoff = max_backoff;
926 self
927 }
928
929 pub fn with_window(mut self, window: Duration) -> Self {
930 self.window = window;
931 self
932 }
933
934 fn delay_for_restart(&self, restart_in_window: u32) -> Duration {
939 if self.backoff.is_zero() || self.max_backoff.is_zero() {
940 return Duration::ZERO;
941 }
942
943 let mut delay = self.backoff;
944 for _ in 0..restart_in_window {
945 if delay >= self.max_backoff {
946 return self.max_backoff;
947 }
948 delay = delay
949 .checked_mul(10)
950 .unwrap_or(self.max_backoff)
951 .min(self.max_backoff);
952 }
953 delay.min(self.max_backoff)
954 }
955
956 fn budget_exhausted_detail(&self) -> String {
961 format!(
962 "crash budget exhausted: max_restarts={} within window_secs={}",
963 self.max_restarts,
964 self.window.as_secs()
965 )
966 }
967}
968
969impl Default for RestartPolicy {
970 fn default() -> Self {
971 Self {
972 max_restarts: DEFAULT_MAX_RESTARTS,
973 backoff: DEFAULT_BACKOFF,
974 max_backoff: DEFAULT_MAX_BACKOFF,
975 window: DEFAULT_RESTART_WINDOW,
976 }
977 }
978}
979
980#[derive(Debug, Clone, Copy, PartialEq, Eq)]
981struct CrashRestartSchedule {
982 restart_in_window: u32,
983 delay: Duration,
984}
985
986fn daemon_will_restart(
993 state: &mut SupervisorSnapshot,
994 policy: &RestartPolicy,
995 now: Instant,
996) -> bool {
997 state.enabled && state.crash_restarts_in_window(policy.window, now) < policy.max_restarts
998}
999
1000const DEFAULT_HEALTH_CADENCE: Duration = Duration::from_secs(30);
1001const DEFAULT_HEALTH_DEADLINE: Duration = Duration::from_secs(5);
1002const DEFAULT_HEALTH_FAILURE_THRESHOLD: u32 = 3;
1003const MAX_HEALTH_METRICS_BYTES: usize = 16 * 1024;
1004
1005#[derive(Debug, Clone, Copy, PartialEq, Eq)]
1006pub enum HealthAction {
1007 Report,
1008 Restart,
1009 Alert,
1010}
1011
1012impl fmt::Display for HealthAction {
1013 fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
1014 f.write_str(match self {
1015 Self::Report => "report",
1016 Self::Restart => "restart",
1017 Self::Alert => "alert",
1018 })
1019 }
1020}
1021
1022#[derive(Debug, Clone, PartialEq, Eq)]
1023pub struct HealthConfig {
1024 pub http: Option<String>,
1028 pub cadence: Duration,
1029 pub deadline: Duration,
1030 pub failure_threshold: u32,
1031 pub on_degraded: HealthAction,
1032 pub on_failing: HealthAction,
1033 pub critical: bool,
1034}
1035
1036impl Default for HealthConfig {
1037 fn default() -> Self {
1038 Self {
1039 http: None,
1040 cadence: DEFAULT_HEALTH_CADENCE,
1041 deadline: DEFAULT_HEALTH_DEADLINE,
1042 failure_threshold: DEFAULT_HEALTH_FAILURE_THRESHOLD,
1043 on_degraded: HealthAction::Report,
1044 on_failing: HealthAction::Report,
1045 critical: false,
1046 }
1047 }
1048}
1049
1050#[derive(Debug, Clone, PartialEq)]
1068pub struct ModuleHealthStatus {
1069 pub status: SupervisorHealthStatus,
1070 pub last_probe_ms: Option<u64>,
1071 pub detail: Option<String>,
1072 pub metrics: Option<Value>,
1073 pub consecutive_failures: u32,
1074 pub late_answer_count: u64,
1077 pub last_late_answer_latency_ms: Option<u64>,
1079 pub last_action: Option<String>,
1080 pub last_action_ms: Option<u64>,
1084}
1085
1086impl Default for ModuleHealthStatus {
1087 fn default() -> Self {
1088 Self {
1089 status: SupervisorHealthStatus::Unknown,
1090 last_probe_ms: None,
1091 detail: None,
1092 metrics: None,
1093 consecutive_failures: 0,
1094 late_answer_count: 0,
1095 last_late_answer_latency_ms: None,
1096 last_action: None,
1097 last_action_ms: None,
1098 }
1099 }
1100}
1101
1102#[derive(Debug, Clone, Copy, PartialEq, Eq)]
1104pub enum ModuleState {
1105 Starting,
1106 Running,
1107 Unresponsive,
1108 Restarting,
1109 Draining,
1110 Stopped,
1111 Failed,
1112 Disabled,
1113}
1114
1115impl fmt::Display for ModuleState {
1116 fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
1117 f.write_str(match self {
1118 Self::Starting => "starting",
1119 Self::Running => "running",
1120 Self::Unresponsive => "unresponsive",
1121 Self::Restarting => "restarting",
1122 Self::Draining => "draining",
1123 Self::Stopped => "stopped",
1124 Self::Failed => "failed",
1125 Self::Disabled => "disabled",
1126 })
1127 }
1128}
1129
1130#[derive(Debug, Clone, Copy, PartialEq, Eq)]
1132pub enum ExitKind {
1133 Clean,
1134 Crash,
1135 DeliberateSeverance,
1136}
1137
1138impl From<ExitKind> for TerminalExitKind {
1139 fn from(kind: ExitKind) -> Self {
1140 match kind {
1141 ExitKind::Clean => Self::Clean,
1142 ExitKind::Crash => Self::Crash,
1143 ExitKind::DeliberateSeverance => Self::DeliberateSeverance,
1144 }
1145 }
1146}
1147
1148#[derive(Debug, Clone, Copy, PartialEq, Eq)]
1151pub(crate) struct ProcessIdentity {
1152 pub(crate) pid: u32,
1153 pub(crate) start_time: u64,
1154}
1155
1156#[derive(Debug, Clone, PartialEq, Eq)]
1158pub struct ExitReport {
1159 pub kind: ExitKind,
1160 pub code: Option<i32>,
1161 pub signal: Option<i32>,
1162 pub at_ms: u64,
1163}
1164
1165#[derive(Debug, Clone, PartialEq)]
1168pub struct ModuleStatus {
1169 pub module_id: String,
1170 pub waiting_for_previous_pid: Option<u32>,
1172 pub waiting_for_previous_detail: Option<String>,
1174 pub launch_nonce_source: Option<String>,
1179 pub state: ModuleState,
1180 pub enabled: bool,
1181 pub process_alive: bool,
1182 pub registration_active: bool,
1183 pub protocol: ModuleProtocol,
1188 pub live: bool,
1199 pub restart_count: u32,
1203 pub lifetime_restarts: u32,
1207 pub spawn_generation: u64,
1208 pub max_restarts: u32,
1213 pub restart_window: Duration,
1217 pub drain_timeout: Duration,
1221 pub restart_backoff: Duration,
1222 pub restart_max_backoff: Duration,
1223 pub pid: Option<u32>,
1228 pub spawned_at_ms: Option<u64>,
1229 pub spawned_from: Option<PathBuf>,
1230 pub(crate) launch_spec_digest: Option<LaunchSpecDigest>,
1231 pub process_start_time: Option<u64>,
1232 pub last_exit: Option<ExitReport>,
1233 pub health: ModuleHealthStatus,
1234}
1235
1236#[cfg(test)]
1237#[derive(Debug, Clone, Copy, PartialEq)]
1238struct ActorSelectCheckpoint {
1239 generation: u64,
1240 turn: u64,
1241 registered_connection: Option<ConnectionId>,
1242 next_probe_at: Option<Instant>,
1243 wake_after: Option<Duration>,
1244}
1245
1246#[derive(Debug, Clone, PartialEq)]
1247struct SupervisorSnapshot {
1248 subtree_target: crate::subtree_resources::Target,
1249 failed_retry: FailedRetry,
1250 exec_outcome: Option<Arc<crate::activity::ExecOutcome>>,
1251 #[cfg(windows)]
1252 pipe_consumed: Option<Arc<OnceLock<()>>>,
1253 #[cfg(test)]
1255 actor_turns: u64,
1256 #[cfg(test)]
1259 actor_select: Option<ActorSelectCheckpoint>,
1260 state: ModuleState,
1261 enabled: bool,
1262 process_alive: bool,
1263 spawned_protocol: Option<ModuleProtocol>,
1264 spawn_failure: Option<String>,
1265 waiting_for_previous_pid: Option<u32>,
1266 waiting_for_previous_detail: Option<String>,
1267 crash_restarts: VecDeque<Instant>,
1273 lifetime_restarts: u32,
1274 spawn_generation: u64,
1283 pid: Option<u32>,
1284 #[cfg(target_os = "macos")]
1285 report_ready: Option<Arc<OnceLock<()>>>,
1286 reaped_pid: Option<u32>,
1288 respawn_pending: bool,
1290 coalesced_restart_pending: bool,
1292 spawned_at_ms: Option<u64>,
1293 spawned_from: Option<PathBuf>,
1294 launch_spec_digest: Option<LaunchSpecDigest>,
1295 spawned_file_identity: Option<SpawnedFileIdentity>,
1296 process_start_time: Option<u64>,
1297 deliberate_severance: Option<ProcessIdentity>,
1298 last_exit: Option<ExitReport>,
1299 drain_disposition_detail: Option<String>,
1301 health: ModuleHealthStatus,
1302 in_alternate_slot: bool,
1307 draining_to_replace: bool,
1314 configuration_updated_since_spawn: bool,
1320}
1321
1322impl SupervisorSnapshot {
1323 fn reported_pid(&self) -> Option<u32> {
1328 #[cfg(target_os = "macos")]
1329 if self
1330 .report_ready
1331 .as_ref()
1332 .is_some_and(|ready| ready.get().is_none())
1333 {
1334 return None;
1335 }
1336 self.pid
1337 }
1338
1339 fn starting() -> Self {
1340 Self::new(ModuleState::Starting, true)
1341 }
1342
1343 fn disabled() -> Self {
1344 Self::new(ModuleState::Disabled, false)
1345 }
1346
1347 fn crash_restarts_in_window(&mut self, window: Duration, now: Instant) -> u32 {
1351 while let Some(oldest) = self.crash_restarts.front() {
1352 if now.duration_since(*oldest) > window {
1353 self.crash_restarts.pop_front();
1354 } else {
1355 break;
1356 }
1357 }
1358 u32::try_from(self.crash_restarts.len()).unwrap_or(u32::MAX)
1359 }
1360
1361 fn record_crash_restart(&mut self, policy: &RestartPolicy, now: Instant) {
1367 self.crash_restarts.push_back(now);
1368 while self.crash_restarts.len() > policy.max_restarts as usize {
1369 self.crash_restarts.pop_front();
1370 }
1371 self.lifetime_restarts += 1;
1372 }
1373
1374 fn next_crash_restart(
1378 &mut self,
1379 policy: &RestartPolicy,
1380 now: Instant,
1381 ) -> Option<CrashRestartSchedule> {
1382 let restart_in_window = self.crash_restarts_in_window(policy.window, now);
1383 if restart_in_window >= policy.max_restarts {
1384 return None;
1385 }
1386 self.record_crash_restart(policy, now);
1387 Some(CrashRestartSchedule {
1388 restart_in_window,
1389 delay: policy.delay_for_restart(restart_in_window),
1390 })
1391 }
1392
1393 fn clear_crash_restarts(&mut self) {
1398 self.crash_restarts.clear();
1399 }
1400
1401 fn new(state: ModuleState, enabled: bool) -> Self {
1402 Self {
1403 subtree_target: crate::subtree_resources::Target::Unavailable("not_running"),
1404 failed_retry: FailedRetry::default(),
1405 exec_outcome: None,
1406 #[cfg(test)]
1407 actor_turns: 0,
1408 #[cfg(test)]
1409 actor_select: None,
1410 state,
1411 enabled,
1412 process_alive: false,
1413 #[cfg(windows)]
1414 pipe_consumed: None,
1415 spawned_protocol: None,
1416 spawn_failure: None,
1417 waiting_for_previous_pid: None,
1418 waiting_for_previous_detail: None,
1419 crash_restarts: VecDeque::new(),
1420 lifetime_restarts: 0,
1421 spawn_generation: 0,
1422 pid: None,
1423 #[cfg(target_os = "macos")]
1424 report_ready: None,
1425 reaped_pid: None,
1426 respawn_pending: false,
1427 coalesced_restart_pending: false,
1428 spawned_at_ms: None,
1429 spawned_from: None,
1430 launch_spec_digest: None,
1431 spawned_file_identity: None,
1432 process_start_time: None,
1433 deliberate_severance: None,
1434 last_exit: None,
1435 drain_disposition_detail: None,
1436 health: ModuleHealthStatus::default(),
1437 in_alternate_slot: false,
1438 draining_to_replace: false,
1439 configuration_updated_since_spawn: false,
1440 }
1441 }
1442}
1443
1444type SharedSnapshot = Arc<Mutex<SupervisorSnapshot>>;
1445
1446type SpawnSubscriberKey = (ConnectionId, u64);
1447
1448#[derive(Debug)]
1449struct SpawnSubscriber {
1450 version: u8,
1451 frames: mpsc::Sender<Frame>,
1452 lagged: Option<oneshot::Sender<SpawnCursor>>,
1456}
1457
1458#[derive(Debug)]
1459struct SpawnEventState {
1460 daemon_incarnation: String,
1461 seq: u64,
1462 capacity: usize,
1463 live: HashMap<String, LiveSpawn>,
1464 generations: HashMap<String, u64>,
1465 events: VecDeque<SpawnEvent>,
1466 subscribers: HashMap<SpawnSubscriberKey, SpawnSubscriber>,
1467}
1468
1469impl Default for SpawnEventState {
1470 fn default() -> Self {
1471 Self {
1472 daemon_incarnation: "unconfigured".to_string(),
1473 seq: 0,
1474 capacity: SPAWN_EVENT_RING_CAPACITY,
1475 live: HashMap::new(),
1476 generations: HashMap::new(),
1477 events: VecDeque::new(),
1478 subscribers: HashMap::new(),
1479 }
1480 }
1481}
1482
1483#[derive(Debug, Clone, Default)]
1484struct SpawnEventFeed(
1485 Arc<Mutex<SpawnEventState>>,
1486 #[cfg(test)] Arc<Mutex<Option<std::net::TcpStream>>>,
1487);
1488
1489#[derive(Debug, Clone, PartialEq, Eq)]
1490pub(crate) enum SpawnSubscribeRefusal {
1491 ForeignIncarnation { current: String },
1492 TooOld { oldest: SpawnCursor },
1493 Frame(String),
1494}
1495
1496impl SpawnEventFeed {
1497 fn configure_incarnation(&self, daemon_incarnation: String) {
1498 let mut state = self.0.lock().unwrap_or_else(|p| p.into_inner());
1499 state.daemon_incarnation = daemon_incarnation;
1500 state.seq = 0;
1501 state.live.clear();
1502 state.generations.clear();
1503 state.events.clear();
1504 state.subscribers.clear();
1505 }
1506
1507 fn cursor(state: &SpawnEventState) -> SpawnCursor {
1508 SpawnCursor {
1509 daemon_incarnation: state.daemon_incarnation.clone(),
1510 seq: state.seq,
1511 }
1512 }
1513
1514 fn snapshot(&self) -> SpawnSnapshot {
1515 let state = self.0.lock().unwrap_or_else(|p| p.into_inner());
1516 let mut live = state.live.values().cloned().collect::<Vec<_>>();
1517 live.sort_by(|left, right| left.module_id.cmp(&right.module_id));
1518 SpawnSnapshot {
1519 cursor: Self::cursor(&state),
1520 ring_bound: state.capacity as u64,
1521 live,
1522 }
1523 }
1524
1525 fn emit_spawned(&self, module_id: &str, pid: u32, spawned_at_ms: u64) -> u64 {
1526 let mut state = self.0.lock().unwrap_or_else(|p| p.into_inner());
1527 let generation = state
1528 .generations
1529 .get(module_id)
1530 .copied()
1531 .unwrap_or(0)
1532 .checked_add(1)
1533 .expect("spawn generation exhausted");
1534 state.generations.insert(module_id.to_string(), generation);
1535 let live = LiveSpawn {
1536 module_id: module_id.to_string(),
1537 spawn_generation: generation,
1538 pid,
1539 spawned_at_ms,
1540 };
1541 state.live.insert(module_id.to_string(), live);
1542 Self::emit_locked(
1543 &mut state,
1544 SpawnEventKind::Spawned,
1545 module_id.to_string(),
1546 generation,
1547 pid,
1548 None,
1549 None,
1550 );
1551 generation
1552 }
1553
1554 fn emit_exited(&self, module_id: &str, exit_code: Option<i32>, exit_signal: Option<i32>) {
1555 let mut state = self.0.lock().unwrap_or_else(|p| p.into_inner());
1556 let Some(live) = state.live.remove(module_id) else {
1557 warn!(
1558 module_id,
1559 "terminal record had no live spawn event identity"
1560 );
1561 return;
1562 };
1563 Self::emit_locked(
1564 &mut state,
1565 SpawnEventKind::Exited,
1566 module_id.to_string(),
1567 live.spawn_generation,
1568 live.pid,
1569 exit_code,
1570 exit_signal,
1571 );
1572 }
1573
1574 fn emit_superseded_exited(
1581 &self,
1582 module_id: &str,
1583 spawn_generation: u64,
1584 pid: u32,
1585 exit_code: Option<i32>,
1586 exit_signal: Option<i32>,
1587 ) {
1588 let mut state = self.0.lock().unwrap_or_else(|p| p.into_inner());
1589 if state
1590 .live
1591 .get(module_id)
1592 .is_some_and(|live| live.spawn_generation == spawn_generation)
1593 {
1594 state.live.remove(module_id);
1595 }
1596 Self::emit_locked(
1597 &mut state,
1598 SpawnEventKind::Exited,
1599 module_id.to_string(),
1600 spawn_generation,
1601 pid,
1602 exit_code,
1603 exit_signal,
1604 );
1605 }
1606
1607 #[allow(clippy::too_many_arguments)]
1608 fn emit_locked(
1609 state: &mut SpawnEventState,
1610 kind: SpawnEventKind,
1611 module_id: String,
1612 spawn_generation: u64,
1613 pid: u32,
1614 exit_code: Option<i32>,
1615 exit_signal: Option<i32>,
1616 ) {
1617 state.seq = state
1618 .seq
1619 .checked_add(1)
1620 .expect("spawn event sequence exhausted");
1621 let event = SpawnEvent {
1622 cursor: Self::cursor(state),
1623 kind,
1624 module_id,
1625 spawn_generation,
1626 pid,
1627 exit_code,
1628 exit_signal,
1629 };
1630 state.events.push_back(event.clone());
1631 while state.events.len() > state.capacity {
1632 state.events.pop_front();
1633 }
1634 let body = match serde_json::to_vec(&event) {
1635 Ok(body) => body,
1636 Err(error) => {
1637 error!(%error, "failed to serialize supervisor spawn event");
1638 return;
1639 }
1640 };
1641 state.subscribers.retain(|(connection_id, corr), subscriber| {
1642 let frame = Frame::build_with_version(
1643 subscriber.version,
1644 FrameType::StreamData,
1645 control_flags(),
1646 0,
1647 0,
1648 *corr,
1649 body.clone(),
1650 );
1651 match frame {
1652 Ok(frame) => {
1653 if subscriber.frames.try_send(frame).is_ok() {
1654 true
1655 } else {
1656 warn!(connection_id = connection_id.get(), corr, "dropping lagged supervisor spawn subscriber");
1657 if let Some(lagged) = subscriber.lagged.take() {
1658 let _ = lagged.send(event.cursor.clone());
1659 }
1660 false
1661 }
1662 }
1663 Err(error) => {
1664 warn!(connection_id = connection_id.get(), corr, %error, "dropping supervisor spawn subscriber after frame build failure");
1665 false
1666 }
1667 }
1668 });
1669 }
1670
1671 fn subscribe(
1672 &self,
1673 connection_id: ConnectionId,
1674 corr: u64,
1675 version: u8,
1676 since: Option<SpawnCursor>,
1677 sink: FrameSink,
1678 ) -> Result<(), SpawnSubscribeRefusal> {
1679 let (frames, mut receiver) = mpsc::channel(SPAWN_SUBSCRIBER_BUFFER);
1680 let (lagged, mut lagged_rx) = oneshot::channel::<SpawnCursor>();
1681 {
1682 let mut state = self.0.lock().unwrap_or_else(|p| p.into_inner());
1683 let replay = if let Some(since) = since {
1684 if since.daemon_incarnation != state.daemon_incarnation {
1685 return Err(SpawnSubscribeRefusal::ForeignIncarnation {
1686 current: state.daemon_incarnation.clone(),
1687 });
1688 }
1689 if let Some(oldest) = state.events.front().map(|event| event.cursor.clone()) {
1690 if since.seq < oldest.seq.saturating_sub(1) {
1691 return Err(SpawnSubscribeRefusal::TooOld { oldest });
1692 }
1693 }
1694 state
1695 .events
1696 .iter()
1697 .filter(|event| event.cursor.seq > since.seq)
1698 .cloned()
1699 .collect::<Vec<_>>()
1700 } else {
1701 Vec::new()
1702 };
1703 for event in replay {
1704 let body = serde_json::to_vec(&event)
1705 .map_err(|error| SpawnSubscribeRefusal::Frame(error.to_string()))?;
1706 let frame = Frame::build_with_version(
1707 version,
1708 FrameType::StreamData,
1709 control_flags(),
1710 0,
1711 0,
1712 corr,
1713 body,
1714 )
1715 .map_err(|error| SpawnSubscribeRefusal::Frame(error.to_string()))?;
1716 frames
1717 .try_send(frame)
1718 .map_err(|error| SpawnSubscribeRefusal::Frame(error.to_string()))?;
1719 }
1720 state.subscribers.insert(
1721 (connection_id, corr),
1722 SpawnSubscriber {
1723 version,
1724 frames,
1725 lagged: Some(lagged),
1726 },
1727 );
1728 }
1729 tokio::spawn(async move {
1740 while let Some(frame) = receiver.recv().await {
1741 if sink.send(frame).await.is_err() {
1742 return;
1743 }
1744 }
1745 let Ok(first_undelivered) = lagged_rx.try_recv() else {
1746 return;
1747 };
1748 match spawn_subscriber_lagged_frame(version, corr, first_undelivered) {
1749 Ok(frame) => {
1750 let _ = sink.send(frame).await;
1751 }
1752 Err(error) => {
1753 error!(%error, corr, "failed to build lagged spawn subscriber terminal frame");
1754 }
1755 }
1756 });
1757 Ok(())
1758 }
1759
1760 fn cancel(&self, connection_id: ConnectionId, corr: u64) -> bool {
1761 let Some(subscriber) = self
1762 .0
1763 .lock()
1764 .unwrap_or_else(|p| p.into_inner())
1765 .subscribers
1766 .remove(&(connection_id, corr))
1767 else {
1768 return false;
1769 };
1770 if let Ok(frame) = Frame::build_with_version(
1771 subscriber.version,
1772 FrameType::StreamEnd,
1773 control_flags(),
1774 0,
1775 0,
1776 corr,
1777 Vec::new(),
1778 ) {
1779 tokio::spawn(async move {
1780 let _ = subscriber.frames.send(frame).await;
1781 });
1782 }
1783 true
1784 }
1785
1786 fn remove_connection(&self, connection_id: ConnectionId) {
1787 self.0
1788 .lock()
1789 .unwrap_or_else(|p| p.into_inner())
1790 .subscribers
1791 .retain(|(subscriber_connection, _), _| *subscriber_connection != connection_id);
1792 }
1793
1794 #[cfg(any(test, feature = "test-support"))]
1795 fn set_capacity(&self, capacity: usize) {
1796 self.0.lock().unwrap_or_else(|p| p.into_inner()).capacity = capacity;
1797 }
1798
1799 #[cfg(any(test, feature = "test-support"))]
1800 fn subscriber_count(&self) -> usize {
1801 self.0
1802 .lock()
1803 .unwrap_or_else(|p| p.into_inner())
1804 .subscribers
1805 .len()
1806 }
1807}
1808
1809fn spawn_subscriber_lagged_frame(
1812 version: u8,
1813 corr: u64,
1814 first_undelivered: SpawnCursor,
1815) -> Result<Frame, String> {
1816 let body = serde_json::to_vec(&subc_protocol::ErrorBody {
1818 code: SPAWN_SUBSCRIBER_LAGGED_CODE.to_string(),
1819 message: "spawn subscriber fell behind and was dropped; resubscribe from the last cursor received"
1820 .to_string(),
1821 detail: Some(serde_json::json!({
1822 "first_undelivered_cursor": first_undelivered
1823 })),
1824 })
1825 .map_err(|error| error.to_string())?;
1826 Frame::build_with_version(
1827 version,
1828 FrameType::Error,
1829 crate::router::daemon_error_flags(),
1830 0,
1831 0,
1832 corr,
1833 body,
1834 )
1835 .map_err(|error| error.to_string())
1836}
1837
1838pub trait ModuleProcessLiveness: Send + Sync {
1839 fn process_live(&self, module_id: &str) -> Option<bool>;
1840
1841 fn process_replacing(&self, _module_id: &str) -> bool {
1847 false
1848 }
1849}
1850
1851#[derive(Debug, Clone, Default)]
1853pub struct SupervisorProcessLiveness {
1854 snapshots: Arc<Mutex<HashMap<String, SharedSnapshot>>>,
1855}
1856
1857impl SupervisorProcessLiveness {
1858 pub fn new() -> Self {
1859 Self::default()
1860 }
1861
1862 fn track(&self, module_id: String, snapshot: SharedSnapshot) {
1863 let mut snapshots = self
1864 .snapshots
1865 .lock()
1866 .unwrap_or_else(|poisoned| poisoned.into_inner());
1867 snapshots.insert(module_id, snapshot);
1868 }
1869
1870 fn untrack_if_current(&self, module_id: &str, snapshot: &SharedSnapshot) {
1871 let mut snapshots = self
1872 .snapshots
1873 .lock()
1874 .unwrap_or_else(|poisoned| poisoned.into_inner());
1875 let is_current = snapshots
1876 .get(module_id)
1877 .map(|tracked| Arc::ptr_eq(tracked, snapshot))
1878 .unwrap_or(false);
1879 if is_current {
1880 snapshots.remove(module_id);
1881 }
1882 }
1883}
1884
1885impl ModuleProcessLiveness for SupervisorProcessLiveness {
1886 fn process_live(&self, module_id: &str) -> Option<bool> {
1887 let snapshot = {
1888 let snapshots = self
1889 .snapshots
1890 .lock()
1891 .unwrap_or_else(|poisoned| poisoned.into_inner());
1892 snapshots.get(module_id).cloned()
1893 }?;
1894 let snapshot = snapshot
1895 .lock()
1896 .unwrap_or_else(|poisoned| poisoned.into_inner());
1897 Some(snapshot.state == ModuleState::Running && snapshot.process_alive)
1898 }
1899
1900 fn process_replacing(&self, module_id: &str) -> bool {
1901 let Some(snapshot) = self
1902 .snapshots
1903 .lock()
1904 .unwrap_or_else(|poisoned| poisoned.into_inner())
1905 .get(module_id)
1906 .cloned()
1907 else {
1908 return false;
1909 };
1910 let snapshot = snapshot
1911 .lock()
1912 .unwrap_or_else(|poisoned| poisoned.into_inner());
1913 snapshot.enabled
1914 && match snapshot.state {
1915 ModuleState::Restarting => true,
1916 ModuleState::Draining => snapshot.draining_to_replace,
1917 ModuleState::Starting
1918 | ModuleState::Running
1919 | ModuleState::Unresponsive
1920 | ModuleState::Stopped
1921 | ModuleState::Failed
1922 | ModuleState::Disabled => false,
1923 }
1924 }
1925}
1926
1927#[cfg(test)]
1928#[derive(Debug, Default)]
1929struct ReloadExitRecordGate {
1930 reached: tokio::sync::Notify,
1931 resume: tokio::sync::Notify,
1932}
1933
1934#[derive(Debug, Clone, Copy)]
1935enum RespawnKind {
1936 Spawn,
1937 Reload,
1938}
1939
1940#[derive(Debug, Clone, Copy)]
1941struct PendingRespawn {
1942 deadline: Instant,
1943 kind: RespawnKind,
1944 daemon_initiated: bool,
1945}
1946
1947type ReloadReply = oneshot::Sender<Result<(), SuperviseError>>;
1948
1949#[derive(Debug, Clone)]
1950struct SupervisorRuntimeConfig {
1951 registry: Arc<Registry>,
1952 recovery_is_daemon: bool,
1954 start_requester: Option<Principal>,
1955 pending_activity_restart: bool,
1956 provenance_probe: ExecutableIdentityProbe,
1957 scheduled_respawn: Arc<Mutex<Option<PendingRespawn>>>,
1959 deferred_reload_reply: Arc<Mutex<Option<ReloadReply>>>,
1961 restart_policy: RestartPolicy,
1962 register_timeout: Duration,
1963 drain_timeout: Duration,
1966 effective_drain_timeout: Arc<Mutex<Duration>>,
1969 default_drain_timeout: Duration,
1972 health: HealthConfig,
1973 connection_file_path: Option<PathBuf>,
1974 capture_logs_dir: Option<PathBuf>,
1975 forwarding: Option<Arc<ForwardingTable>>,
1976 supervisor_handle: Option<SupervisorHandle>,
1979 stderr_ring: Arc<Mutex<StderrRing>>,
1986 terminal_ring: Arc<Mutex<TerminalRing>>,
1987 spawn_events: SpawnEventFeed,
1988 child_roster: ChildRoster,
1989 #[cfg(target_os = "linux")]
1990 cgroup_placement: Option<subc_cgroup::Placement>,
1991 #[cfg(test)]
1992 test_seed_stale_facts_before_enable_spawn: bool,
1993 #[cfg(test)]
1994 test_reload_exit_record_gate: Option<Arc<ReloadExitRecordGate>>,
1995}
1996
1997impl SupervisorRuntimeConfig {
1998 fn failure_context(&self, daemon_initiated: bool) -> FailureContext {
1999 FailureContext {
2000 daemon_initiated,
2001 shutdown: self.child_roster.shutdown_flag(),
2002 }
2003 }
2004}
2005
2006#[derive(Debug, Clone, PartialEq, Eq)]
2007struct SupervisedConfiguration {
2008 spec: ModuleSpec,
2009 health: HealthConfig,
2010}
2011
2012#[derive(Debug, Clone, Default)]
2018pub struct SupervisorHandle {
2019 restart_intent: Arc<Mutex<DaemonRestartIntent>>,
2020 activity: Arc<Mutex<Option<crate::activity::ActivityLedger>>>,
2021 activity_images: Arc<Mutex<HashMap<String, activity::ActivityImage>>>,
2022 modules: Arc<Mutex<HashMap<String, SupervisedModule>>>,
2023 configured_ids: Arc<Mutex<HashSet<String>>>,
2035 spawn_events: SpawnEventFeed,
2036 reserved_nonces: Arc<Mutex<HashMap<String, Option<String>>>>,
2047 removal_tombstones: Arc<Mutex<HashMap<String, u64>>>,
2053 spawn_nonces: Arc<Mutex<HashMap<String, String>>>,
2057 reserved_prefix_owners: Arc<Mutex<HashMap<String, String>>>,
2065 swaps: Arc<Mutex<HashMap<String, OpenSwap>>>,
2071 promotion_observer: PromotionObserverSlot,
2073 operation_lock: Arc<AsyncMutex<()>>,
2077}
2078
2079pub(crate) trait SwapPromotionObserver: Send + Sync {
2088 fn swap_promoted(&self, registration: &crate::registry::ModuleRegistration);
2089}
2090
2091#[derive(Clone, Default)]
2095struct PromotionObserverSlot(Arc<Mutex<Option<std::sync::Weak<dyn SwapPromotionObserver>>>>);
2096
2097impl fmt::Debug for PromotionObserverSlot {
2098 fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
2099 f.write_str("PromotionObserverSlot")
2100 }
2101}
2102
2103#[derive(Debug, Clone)]
2105struct OpenSwap {
2106 candidate_nonce: String,
2109 incumbent_nonce: Option<String>,
2114 candidate_admitted: bool,
2118}
2119
2120#[derive(Debug, Clone, Copy, PartialEq, Eq)]
2123pub(crate) enum SwapHelloAdmission {
2124 NotSwapping,
2127 Candidate,
2129 Refused,
2132}
2133
2134#[derive(Debug, Clone, PartialEq, Eq)]
2135pub(crate) enum ReservedHelloRejection {
2136 Exact {
2137 module_id: String,
2138 },
2139 Prefix {
2140 prefix: String,
2141 owner_module_id: String,
2142 },
2143}
2144
2145#[derive(Debug)]
2146struct DaemonRestartIntent {
2147 expires: Option<Duration>,
2148 window: Duration,
2149}
2150
2151impl Default for DaemonRestartIntent {
2152 fn default() -> Self {
2153 Self {
2154 expires: None,
2155 window: Duration::from_secs(120),
2156 }
2157 }
2158}
2159
2160impl SupervisorHandle {
2161 pub(crate) fn announce_daemon_restart(&self) -> u64 {
2162 let mut intent = self
2163 .restart_intent
2164 .lock()
2165 .unwrap_or_else(|p| p.into_inner());
2166 intent.expires = Some(subc_uptime::suspend_inclusive_now() + intent.window);
2167 unix_ms_now().saturating_add(intent.window.as_millis() as u64)
2168 }
2169
2170 #[cfg(feature = "test-support")]
2171 pub(crate) fn set_shutdown_intent_window(&self, window: Duration) {
2172 self.restart_intent
2173 .lock()
2174 .unwrap_or_else(|p| p.into_inner())
2175 .window = window;
2176 }
2177
2178 #[cfg(unix)]
2179 fn take_daemon_shutdown_reason(&self) -> RouteCloseReason {
2180 let expiry = self
2181 .restart_intent
2182 .lock()
2183 .unwrap_or_else(|p| p.into_inner())
2184 .expires
2185 .take();
2186 if expiry.is_some_and(|expiry| subc_uptime::suspend_inclusive_now() < expiry) {
2187 RouteCloseReason::Restart
2188 } else {
2189 RouteCloseReason::Disable
2190 }
2191 }
2192
2193 pub fn new() -> Self {
2194 Self::default()
2195 }
2196
2197 pub(crate) fn spawn_snapshot(&self) -> SpawnSnapshot {
2198 self.spawn_events.snapshot()
2199 }
2200
2201 pub(crate) fn subscribe_spawns(
2202 &self,
2203 connection_id: ConnectionId,
2204 corr: u64,
2205 version: u8,
2206 since: Option<SpawnCursor>,
2207 sink: FrameSink,
2208 ) -> Result<(), SpawnSubscribeRefusal> {
2209 self.spawn_events
2210 .subscribe(connection_id, corr, version, since, sink)
2211 }
2212
2213 pub(crate) fn cancel_spawn_subscription(&self, connection_id: ConnectionId, corr: u64) -> bool {
2214 self.spawn_events.cancel(connection_id, corr)
2215 }
2216
2217 pub(crate) fn remove_spawn_subscribers(&self, connection_id: ConnectionId) {
2218 self.spawn_events.remove_connection(connection_id);
2219 }
2220
2221 #[cfg(any(test, feature = "test-support"))]
2222 pub fn set_spawn_event_capacity_for_test(&self, capacity: usize) {
2223 assert!(capacity > 0, "spawn event capacity must be non-zero");
2224 self.spawn_events.set_capacity(capacity);
2225 }
2226
2227 #[cfg(any(test, feature = "test-support"))]
2228 pub fn spawn_subscriber_count_for_test(&self) -> usize {
2229 self.spawn_events.subscriber_count()
2230 }
2231
2232 pub fn set_spawn_nonce(&self, module_id: &str, nonce: String) {
2235 self.spawn_nonces
2236 .lock()
2237 .unwrap_or_else(|poisoned| poisoned.into_inner())
2238 .insert(module_id.to_string(), nonce);
2239 }
2240
2241 pub fn set_reserved_nonce(&self, module_id: &str, nonce: String) {
2244 self.reserved_nonces
2245 .lock()
2246 .unwrap_or_else(|poisoned| poisoned.into_inner())
2247 .insert(module_id.to_string(), Some(nonce));
2248 }
2249
2250 pub fn set_reserved_prefixes(&self, owner_module_id: &str, prefixes: &[String]) {
2252 let mut owners = self
2253 .reserved_prefix_owners
2254 .lock()
2255 .unwrap_or_else(|poisoned| poisoned.into_inner());
2256 owners.retain(|_, owner| owner != owner_module_id);
2257 for prefix in prefixes {
2258 owners.insert(prefix.clone(), owner_module_id.to_string());
2259 }
2260 }
2261
2262 #[cfg(test)]
2264 pub(crate) fn spawn_nonce(&self, module_id: &str) -> Option<String> {
2265 self.spawn_nonces
2266 .lock()
2267 .unwrap_or_else(|poisoned| poisoned.into_inner())
2268 .get(module_id)
2269 .cloned()
2270 }
2271
2272 fn apply_identity_configuration(&self, spec: &ModuleSpec) {
2273 self.set_reserved_prefixes(&spec.module_id, &spec.reserved_prefixes);
2274 let spawn_nonce = self
2275 .spawn_nonces
2276 .lock()
2277 .unwrap_or_else(|poisoned| poisoned.into_inner())
2278 .get(&spec.module_id)
2279 .cloned();
2280 let mut reserved_nonces = self
2281 .reserved_nonces
2282 .lock()
2283 .unwrap_or_else(|poisoned| poisoned.into_inner());
2284 if spec.reserved {
2285 reserved_nonces.insert(spec.module_id.clone(), spawn_nonce);
2290 }
2291 drop(reserved_nonces);
2292 self.removal_tombstones
2296 .lock()
2297 .unwrap_or_else(|poisoned| poisoned.into_inner())
2298 .remove(&spec.module_id);
2299 }
2300
2301 pub fn reserved_hello_authorized(&self, module_id: &str, presented: Option<&str>) -> bool {
2306 self.reserved_hello_verification(module_id, presented)
2307 .is_ok()
2308 }
2309
2310 pub(crate) fn reserved_hello_verification(
2313 &self,
2314 module_id: &str,
2315 presented: Option<&str>,
2316 ) -> Result<bool, ReservedHelloRejection> {
2317 let nonces = self
2318 .reserved_nonces
2319 .lock()
2320 .unwrap_or_else(|poisoned| poisoned.into_inner());
2321 if let Some(expected) = nonces.get(module_id) {
2322 let authorized = match expected {
2326 Some(expected) => {
2327 presented.is_some_and(|p| constant_time_eq(expected.as_bytes(), p.as_bytes()))
2328 }
2329 None => false,
2330 };
2331 if authorized {
2332 return Ok(true);
2333 }
2334 return Err(ReservedHelloRejection::Exact {
2335 module_id: module_id.to_string(),
2336 });
2337 }
2338 drop(nonces);
2339
2340 let matched_prefix = self
2341 .reserved_prefix_owners
2342 .lock()
2343 .unwrap_or_else(|poisoned| poisoned.into_inner())
2344 .iter()
2345 .filter(|(prefix, _)| module_id.starts_with(prefix.as_str()))
2346 .max_by_key(|(prefix, _)| prefix.len())
2347 .map(|(prefix, owner)| (prefix.clone(), owner.clone()));
2348 let Some((prefix, owner_module_id)) = matched_prefix else {
2349 return Ok(false);
2350 };
2351
2352 let authorized = presented.is_some_and(|presented| {
2353 self.spawn_nonces
2354 .lock()
2355 .unwrap_or_else(|poisoned| poisoned.into_inner())
2356 .get(&owner_module_id)
2357 .is_some_and(|expected| constant_time_eq(expected.as_bytes(), presented.as_bytes()))
2358 || self.swap_nonce_matches(&owner_module_id, presented)
2361 });
2362 if authorized {
2363 Ok(true)
2364 } else {
2365 Err(ReservedHelloRejection::Prefix {
2366 prefix,
2367 owner_module_id,
2368 })
2369 }
2370 }
2371
2372 pub fn spawned_consumer_authorized(&self, module_id: &str, presented: &str) -> bool {
2377 if presented.is_empty() {
2378 return false;
2379 }
2380 let nonces = self
2381 .spawn_nonces
2382 .lock()
2383 .unwrap_or_else(|poisoned| poisoned.into_inner());
2384 let current = nonces
2385 .get(module_id)
2386 .is_some_and(|expected| constant_time_eq(expected.as_bytes(), presented.as_bytes()));
2387 drop(nonces);
2388 current || self.swap_nonce_matches(module_id, presented)
2393 }
2394
2395 fn swap_nonce_matches(&self, module_id: &str, presented: &str) -> bool {
2397 let swaps = self
2398 .swaps
2399 .lock()
2400 .unwrap_or_else(|poisoned| poisoned.into_inner());
2401 swaps.get(module_id).is_some_and(|swap| {
2402 constant_time_eq(swap.candidate_nonce.as_bytes(), presented.as_bytes())
2403 || swap.incumbent_nonce.as_deref().is_some_and(|incumbent| {
2404 constant_time_eq(incumbent.as_bytes(), presented.as_bytes())
2405 })
2406 })
2407 }
2408
2409 pub(crate) fn open_swap(&self, module_id: &str, candidate_nonce: String) {
2412 let incumbent_nonce = self
2413 .spawn_nonces
2414 .lock()
2415 .unwrap_or_else(|poisoned| poisoned.into_inner())
2416 .get(module_id)
2417 .cloned();
2418 self.swaps
2419 .lock()
2420 .unwrap_or_else(|poisoned| poisoned.into_inner())
2421 .insert(
2422 module_id.to_string(),
2423 OpenSwap {
2424 candidate_nonce,
2425 incumbent_nonce,
2426 candidate_admitted: false,
2427 },
2428 );
2429 }
2430
2431 pub(crate) fn close_swap(&self, module_id: &str) {
2434 self.swaps
2435 .lock()
2436 .unwrap_or_else(|poisoned| poisoned.into_inner())
2437 .remove(module_id);
2438 }
2439
2440 pub(crate) fn set_swap_promotion_observer(
2443 &self,
2444 observer: std::sync::Weak<dyn SwapPromotionObserver>,
2445 ) {
2446 *self
2447 .promotion_observer
2448 .0
2449 .lock()
2450 .unwrap_or_else(|poisoned| poisoned.into_inner()) = Some(observer);
2451 }
2452
2453 fn notify_swap_promoted(&self, registration: &crate::registry::ModuleRegistration) {
2456 let observer = self
2457 .promotion_observer
2458 .0
2459 .lock()
2460 .unwrap_or_else(|poisoned| poisoned.into_inner())
2461 .as_ref()
2462 .and_then(std::sync::Weak::upgrade);
2463 if let Some(observer) = observer {
2464 observer.swap_promoted(registration);
2465 }
2466 }
2467
2468 pub(crate) fn swap_open(&self, module_id: &str) -> bool {
2470 self.swaps
2471 .lock()
2472 .unwrap_or_else(|poisoned| poisoned.into_inner())
2473 .contains_key(module_id)
2474 }
2475
2476 fn promote_swap_nonce(&self, module_id: &str, reserved: bool) {
2481 let candidate_nonce = self
2482 .swaps
2483 .lock()
2484 .unwrap_or_else(|poisoned| poisoned.into_inner())
2485 .get(module_id)
2486 .map(|swap| swap.candidate_nonce.clone());
2487 let Some(nonce) = candidate_nonce else {
2488 return;
2489 };
2490 self.set_spawn_nonce(module_id, nonce.clone());
2491 if reserved {
2492 self.set_reserved_nonce(module_id, nonce);
2493 }
2494 }
2495
2496 pub(crate) fn swap_hello_admission(
2511 &self,
2512 module_id: &str,
2513 presented: Option<&str>,
2514 ) -> SwapHelloAdmission {
2515 let swaps = self
2516 .swaps
2517 .lock()
2518 .unwrap_or_else(|poisoned| poisoned.into_inner());
2519 let Some(swap) = swaps.get(module_id) else {
2520 return SwapHelloAdmission::NotSwapping;
2521 };
2522 let Some(presented) = presented else {
2523 return SwapHelloAdmission::Refused;
2524 };
2525 if constant_time_eq(swap.candidate_nonce.as_bytes(), presented.as_bytes()) {
2526 return if swap.candidate_admitted {
2527 SwapHelloAdmission::Refused
2528 } else {
2529 SwapHelloAdmission::Candidate
2530 };
2531 }
2532 if swap
2533 .incumbent_nonce
2534 .as_deref()
2535 .is_some_and(|incumbent| constant_time_eq(incumbent.as_bytes(), presented.as_bytes()))
2536 {
2537 return SwapHelloAdmission::NotSwapping;
2538 }
2539 SwapHelloAdmission::Refused
2540 }
2541
2542 pub(crate) fn mark_swap_candidate_admitted(&self, module_id: &str) {
2545 if let Some(swap) = self
2546 .swaps
2547 .lock()
2548 .unwrap_or_else(|poisoned| poisoned.into_inner())
2549 .get_mut(module_id)
2550 {
2551 swap.candidate_admitted = true;
2552 }
2553 }
2554
2555 pub fn spawn_launch_nonce_for(&self, module_id: &str) -> Option<String> {
2557 self.spawn_nonces
2558 .lock()
2559 .unwrap_or_else(|poisoned| poisoned.into_inner())
2560 .get(module_id)
2561 .cloned()
2562 }
2563
2564 pub fn reserved_launch_nonce_for(&self, module_id: &str) -> Option<String> {
2566 self.reserved_nonces
2567 .lock()
2568 .unwrap_or_else(|poisoned| poisoned.into_inner())
2569 .get(module_id)
2570 .cloned()
2571 .flatten()
2572 }
2573
2574 pub fn insert(&self, module: SupervisedModule) -> Option<SupervisedModule> {
2575 self.mark_configured(module.module_id());
2579 let mut modules = self
2580 .modules
2581 .lock()
2582 .unwrap_or_else(|poisoned| poisoned.into_inner());
2583 modules.insert(module.module_id().to_string(), module)
2584 }
2585
2586 fn mark_configured(&self, module_id: &str) {
2590 self.configured_ids
2591 .lock()
2592 .unwrap_or_else(|poisoned| poisoned.into_inner())
2593 .insert(module_id.to_string());
2594 }
2595
2596 fn unmark_configured_unless_rostered(&self, module_id: &str) {
2600 let modules = self
2601 .modules
2602 .lock()
2603 .unwrap_or_else(|poisoned| poisoned.into_inner());
2604 if !modules.contains_key(module_id) {
2605 self.configured_ids
2606 .lock()
2607 .unwrap_or_else(|poisoned| poisoned.into_inner())
2608 .remove(module_id);
2609 }
2610 }
2611
2612 pub(crate) fn is_configured(&self, module_id: &str) -> bool {
2620 self.configured_ids
2621 .lock()
2622 .unwrap_or_else(|poisoned| poisoned.into_inner())
2623 .contains(module_id)
2624 }
2625
2626 pub fn get(&self, module_id: &str) -> Option<SupervisedModule> {
2627 let modules = self
2628 .modules
2629 .lock()
2630 .unwrap_or_else(|poisoned| poisoned.into_inner());
2631 modules.get(module_id).cloned()
2632 }
2633
2634 pub(crate) fn record_late_health_answer(
2635 &self,
2636 module_id: &str,
2637 latency_ms: u64,
2638 ) -> Result<bool, SuperviseError> {
2639 let Some(module) = self.get(module_id) else {
2640 return Ok(false);
2641 };
2642 update_snapshot(&module.inner.snapshot, Some(module_id), |state| {
2643 state.health.late_answer_count = state.health.late_answer_count.saturating_add(1);
2644 state.health.last_late_answer_latency_ms = Some(latency_ms);
2645 state.health.consecutive_failures = 0;
2653 })?;
2654 Ok(true)
2655 }
2656
2657 pub fn record_deliberate_severance(&self, module_id: &str) -> Result<bool, SuperviseError> {
2663 let Some(module) = self.get(module_id) else {
2664 return Ok(false);
2665 };
2666 let snapshot = lock_snapshot(&module.inner.snapshot)?;
2667 let Some((pid, start_time)) = snapshot.pid.zip(snapshot.process_start_time) else {
2668 return Ok(false);
2669 };
2670 drop(snapshot);
2671 module.record_deliberate_severance(ProcessIdentity { pid, start_time })
2672 }
2673
2674 pub fn list(&self) -> Vec<SupervisedModule> {
2675 let modules = self
2676 .modules
2677 .lock()
2678 .unwrap_or_else(|poisoned| poisoned.into_inner());
2679 let mut modules = modules.values().cloned().collect::<Vec<_>>();
2680 modules.sort_by(|left, right| left.module_id().cmp(right.module_id()));
2681 modules
2682 }
2683
2684 pub(crate) fn retire(&self, module_id: &str) -> Option<SupervisedModule> {
2685 self.activity_images
2686 .lock()
2687 .unwrap_or_else(|p| p.into_inner())
2688 .remove(module_id);
2689 self.spawn_nonces
2690 .lock()
2691 .unwrap_or_else(|poisoned| poisoned.into_inner())
2692 .remove(module_id);
2693 self.close_swap(module_id);
2694 let mut reserved_nonces = self
2695 .reserved_nonces
2696 .lock()
2697 .unwrap_or_else(|poisoned| poisoned.into_inner());
2698 if reserved_nonces.contains_key(module_id) {
2699 reserved_nonces.insert(module_id.to_string(), None);
2702 }
2703 drop(reserved_nonces);
2704 self.reserved_prefix_owners
2705 .lock()
2706 .unwrap_or_else(|poisoned| poisoned.into_inner())
2707 .retain(|_, owner| owner != module_id);
2708 let removed = self
2709 .modules
2710 .lock()
2711 .unwrap_or_else(|poisoned| poisoned.into_inner())
2712 .remove(module_id);
2713 self.configured_ids
2714 .lock()
2715 .unwrap_or_else(|poisoned| poisoned.into_inner())
2716 .remove(module_id);
2717 removed
2718 }
2719
2720 pub(crate) fn record_rescan_removal(&self, module_id: &str) {
2723 self.removal_tombstones
2724 .lock()
2725 .unwrap_or_else(|poisoned| poisoned.into_inner())
2726 .insert(module_id.to_string(), unix_ms_now());
2727 }
2728
2729 pub(crate) fn removal_tombstone_age_ms(&self, module_id: &str) -> Option<u64> {
2731 self.removal_tombstones
2732 .lock()
2733 .unwrap_or_else(|poisoned| poisoned.into_inner())
2734 .get(module_id)
2735 .copied()
2736 .map(|removed_at_ms| unix_ms_now().saturating_sub(removed_at_ms))
2737 }
2738
2739 pub(crate) fn release_retained_reserved_gate(&self, module_id: &str) -> bool {
2744 if self.get(module_id).is_some() {
2745 return false;
2746 }
2747 let mut reserved_nonces = self
2748 .reserved_nonces
2749 .lock()
2750 .unwrap_or_else(|poisoned| poisoned.into_inner());
2751 if !matches!(reserved_nonces.get(module_id), Some(None)) {
2752 return false;
2753 }
2754 reserved_nonces.remove(module_id);
2755 true
2756 }
2757
2758 pub(crate) fn operation_lock(&self) -> Arc<AsyncMutex<()>> {
2759 Arc::clone(&self.operation_lock)
2760 }
2761}
2762
2763#[derive(Debug, Clone)]
2765pub struct Supervisor {
2766 registry: Arc<Registry>,
2767 restart_policy: RestartPolicy,
2768 drain_timeout: Duration,
2769 connection_file_path: Option<PathBuf>,
2770 capture_logs_dir: Option<PathBuf>,
2771 forwarding: Option<Arc<ForwardingTable>>,
2772 process_liveness: Arc<SupervisorProcessLiveness>,
2773 supervisor_handle: Option<SupervisorHandle>,
2774 health: HealthConfig,
2775 daemon_start_clock: crate::clock::StartClock,
2776 terminal_journal: Option<Arc<crate::terminal_journal::TerminalJournal>>,
2777 spawn_events: SpawnEventFeed,
2778 provenance_probe: ExecutableIdentityProbe,
2779 child_roster: ChildRoster,
2782 #[cfg(target_os = "linux")]
2783 cgroup_placement: Option<subc_cgroup::Placement>,
2784 #[cfg(test)]
2785 test_after_first_spawn: AfterFirstSpawnHook,
2786}
2787
2788pub(crate) struct RegisteredConfiguredModule {
2792 spec: ModuleSpec,
2793 runtime: SupervisorRuntimeConfig,
2794 module: SupervisedModule,
2795 commands: mpsc::Receiver<SupervisorCommand>,
2796}
2797
2798#[cfg(test)]
2804#[derive(Clone, Default)]
2805struct AfterFirstSpawnHook(Option<AfterFirstSpawnFn>);
2806
2807#[cfg(test)]
2808type AfterFirstSpawnFn = Arc<dyn Fn(&str) + Send + Sync>;
2809
2810#[cfg(test)]
2811impl fmt::Debug for AfterFirstSpawnHook {
2812 fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
2813 f.write_str("AfterFirstSpawnHook")
2814 }
2815}
2816
2817#[cfg(test)]
2818impl AfterFirstSpawnHook {
2819 fn run(&self, module_id: &str) {
2820 if let Some(hook) = &self.0 {
2821 hook(module_id);
2822 }
2823 }
2824}
2825
2826impl Supervisor {
2827 #[cfg(test)]
2828 pub(crate) fn new_for_test(registry: Arc<Registry>, policy: RestartPolicy) -> Self {
2829 let supervisor = Self::new(registry, policy);
2830 #[cfg(target_os = "macos")]
2831 let supervisor = supervisor.with_privacy_trampoline(test_privacy_trampoline());
2832 supervisor
2833 }
2834 pub fn with_privacy_trampoline(self, path: impl Into<PathBuf>) -> Self {
2839 let path = path.into();
2840 #[cfg(target_os = "macos")]
2841 {
2842 let result = probe_privacy_trampoline(&path).map(|()| path);
2843 if let Err(cause) = &result {
2844 error!(%cause, "macOS privacy identity unavailable; supervised launches will refuse");
2845 }
2846 self.child_roster.set_privacy_trampoline(result);
2847 }
2848 #[cfg(not(target_os = "macos"))]
2849 let _ = path;
2850 self
2851 }
2852 pub(crate) fn daemon_shutdown_flag(&self) -> crate::child_roster::DaemonShutdownFlag {
2855 self.child_roster.shutdown_flag()
2856 }
2857
2858 #[cfg(unix)]
2869 pub(crate) fn begin_daemon_shutdown(&self) {
2870 self.child_roster.close();
2871 if let Some(journal) = &self.terminal_journal {
2872 journal.stamp_shutdown();
2873 }
2874 }
2875
2876 #[cfg(unix)]
2881 pub(crate) async fn drain_for_daemon_shutdown(&self) -> Result<(), SuperviseError> {
2882 const NOTICE_BUDGET: Duration = Duration::from_millis(500);
2883 const DRAIN_BUDGET: Duration = Duration::from_secs(2);
2884 let Some(forwarding) = &self.forwarding else {
2885 return Ok(());
2886 };
2887 let notice_started = Instant::now();
2888 let modules = forwarding
2889 .begin_daemon_drain()
2890 .map_err(SuperviseError::Forwarding)?;
2891 let deadline_ms =
2892 unix_ms_now().saturating_add((NOTICE_BUDGET + DRAIN_BUDGET).as_millis() as u64);
2893 let shutdown_reason = self
2894 .supervisor_handle
2895 .as_ref()
2896 .map(SupervisorHandle::take_daemon_shutdown_reason)
2897 .unwrap_or(RouteCloseReason::Disable);
2898 let mut modules_told = 0;
2899 let mut notices = tokio::task::JoinSet::new();
2900 let mut drains = Vec::new();
2901 for connection in forwarding
2902 .module_connections()
2903 .map_err(SuperviseError::Forwarding)?
2904 {
2905 let target = ModuleDrainTarget {
2906 endpoint: connection.endpoint,
2907 sink: connection.sink,
2908 negotiated_ver: connection.negotiated_ver,
2909 abandoned_bindings: Vec::new(),
2910 excluded_subscriptions: 0,
2911 };
2912 notices.spawn(async move {
2913 (
2914 true,
2915 send_module_draining(
2916 &connection.module_id,
2917 shutdown_reason,
2918 deadline_ms,
2919 &target,
2920 true,
2921 )
2922 .await,
2923 )
2924 });
2925 }
2926 for (module_id, endpoint) in modules {
2927 let Some(target) = forwarding
2928 .begin_endpoint_drain(endpoint, RouteCloseReason::Restart)
2929 .map_err(SuperviseError::Forwarding)?
2930 else {
2931 continue;
2932 };
2933 let routes = forwarding
2934 .endpoint_routes(target.endpoint)
2935 .map_err(SuperviseError::Forwarding)?;
2936 let mut recipients = Vec::new();
2939 let mut clients: Vec<(crate::forwarding::GoodbyeTarget, Vec<u16>)> = Vec::new();
2940 for route in routes {
2941 let client = route.goodbye_target;
2942 if let Some((_, channels)) = clients
2943 .iter_mut()
2944 .find(|(existing, _)| existing.connection_id == client.connection_id)
2945 {
2946 channels.push(client.channel);
2947 } else {
2948 let channel = client.channel;
2949 clients.push((client, vec![channel]));
2950 }
2951 }
2952 for (client, mut channels) in clients {
2953 channels.sort_unstable();
2954 channels.dedup();
2955 let closing = serde_json::to_vec(&ClientControlPush::RouteClosing {
2956 module_id: module_id.clone(),
2957 channels,
2958 reason: RouteCloseReason::Restart,
2959 })
2960 .expect("route closing serializes");
2961 recipients.push((client.sink, client.negotiated_ver, closing));
2962 }
2963 for (sink, version, body) in recipients {
2964 notices.spawn(async move {
2965 let frame = Frame::build_with_version(
2966 version,
2967 FrameType::Push,
2968 control_flags(),
2969 0,
2970 0,
2971 0,
2972 body,
2973 )
2974 .expect("bounded lifecycle notice frame builds");
2975 (false, sink.send_flushed(frame).await.is_ok())
2976 });
2977 }
2978 let gauges = declared_busy_gauges(&self.registry, &module_id)?;
2979 drains.push((module_id, target.endpoint, gauges));
2980 }
2981 let notice_deadline = Instant::now() + NOTICE_BUDGET;
2984 while let Ok(Some(result)) = timeout_at(notice_deadline, notices.join_next()).await {
2985 match result {
2986 Ok((true, true)) => modules_told += 1,
2987 Ok((false, true)) => {}
2988 result => warn!(?result, "daemon shutdown notice delivery failed"),
2989 }
2990 }
2991 notices.abort_all();
2992 info!(
2993 ?shutdown_reason,
2994 modules_told, "daemon shutdown module notices delivered"
2995 );
2996 let notice_elapsed = notice_started.elapsed();
2997 let drain_started = Instant::now();
2998 let deadline = drain_started + DRAIN_BUDGET;
2999 let mut waits = tokio::task::JoinSet::new();
3000 for (module_id, endpoint, gauges) in drains {
3001 let forwarding = Arc::clone(forwarding);
3002 let mut runtime = self.runtime_config();
3003 runtime.health.cadence = Duration::from_millis(100);
3004 waits.spawn(async move {
3005 let wait_result = wait_for_forwarding_quiescence(
3006 &forwarding,
3007 &module_id,
3008 &runtime,
3009 endpoint,
3010 deadline,
3011 &gauges,
3012 DrainScope::Endpoint(endpoint),
3013 &mut DrainGaugeTrace::new(&gauges),
3014 )
3015 .await;
3016 forwarding
3020 .record_daemon_drain_result(
3021 endpoint,
3022 drained_after_quiescence_wait(&wait_result),
3023 )
3024 .map_err(SuperviseError::Forwarding)?;
3025 wait_result
3026 });
3027 }
3028 while let Ok(Some(result)) = timeout_at(deadline, waits.join_next()).await {
3029 if !matches!(result, Ok(Ok(true))) {
3030 warn!(?result, "daemon shutdown drain did not reach quiescence");
3031 }
3032 }
3033 info!(
3036 notice_elapsed_ms = notice_elapsed.as_millis() as u64,
3037 drain_elapsed_ms = drain_started.elapsed().as_millis() as u64,
3038 "daemon shutdown forwarding drain complete"
3039 );
3040 Ok(())
3041 }
3042
3043 #[cfg(unix)]
3055 pub(crate) async fn end_children_for_daemon_shutdown(
3056 &self,
3057 already_escalated: bool,
3058 escalate: impl std::future::Future<Output = ()>,
3059 ) {
3060 tokio::pin!(escalate);
3061 let mut escalated = already_escalated;
3062 if let Some(forwarding) = &self.forwarding {
3063 let reason = CloseReason::new(
3064 "daemon_shutdown",
3065 "the daemon is exiting after its shutdown notice and drain",
3066 );
3067 if escalated {
3068 send_module_goodbyes_for_daemon_shutdown(forwarding, &reason, false).await;
3071 } else {
3072 tokio::select! {
3073 biased;
3074 _ = escalate.as_mut() => {
3075 info!("second SIGTERM: abandoning module GOODBYE delivery");
3076 escalated = true;
3077 }
3078 _ = send_module_goodbyes_for_daemon_shutdown(forwarding, &reason, true) => {}
3079 }
3080 }
3081 let closed = forwarding.close_all_connections(&reason);
3082 debug!(closed, "closed established connections for daemon shutdown");
3083 }
3084 let escalated_here = escalated && !already_escalated;
3088 let remaining_escalate = async move {
3089 if escalated_here {
3090 std::future::pending::<()>().await;
3091 } else {
3092 escalate.await;
3093 }
3094 };
3095 crate::child_roster::end_children_for_daemon_shutdown(
3096 &self.child_roster,
3097 escalated,
3098 remaining_escalate,
3099 )
3100 .await;
3101 }
3102
3103 pub fn new(registry: Arc<Registry>, restart_policy: RestartPolicy) -> Self {
3104 Self {
3105 registry,
3106 restart_policy,
3107 drain_timeout: DEFAULT_DRAIN_TIMEOUT,
3108 connection_file_path: None,
3109 capture_logs_dir: None,
3110 forwarding: None,
3111 process_liveness: Arc::new(SupervisorProcessLiveness::default()),
3112 supervisor_handle: None,
3113 health: HealthConfig::default(),
3114 daemon_start_clock: crate::clock::StartClock::capture(),
3115 terminal_journal: None,
3116 spawn_events: SpawnEventFeed::default(),
3117 provenance_probe: ExecutableIdentityProbe::default(),
3118 child_roster: ChildRoster::default(),
3119 #[cfg(target_os = "linux")]
3120 cgroup_placement: None,
3121 #[cfg(test)]
3122 test_after_first_spawn: AfterFirstSpawnHook::default(),
3123 }
3124 }
3125
3126 pub fn with_drain_timeout(mut self, drain_timeout: Duration) -> Self {
3127 self.drain_timeout = drain_timeout;
3128 self
3129 }
3130
3131 pub(crate) fn default_drain_timeout(&self) -> Duration {
3133 self.drain_timeout
3134 }
3135
3136 pub fn with_process_liveness(
3137 mut self,
3138 process_liveness: Arc<SupervisorProcessLiveness>,
3139 ) -> Self {
3140 self.process_liveness = process_liveness;
3141 self
3142 }
3143
3144 pub fn with_connection_file_path(mut self, connection_file_path: impl Into<PathBuf>) -> Self {
3145 self.connection_file_path = Some(connection_file_path.into());
3146 self
3147 }
3148
3149 pub fn with_capture_logs_dir(mut self, logs_dir: impl Into<PathBuf>) -> Self {
3151 self.capture_logs_dir = Some(logs_dir.into());
3152 self
3153 }
3154
3155 pub fn with_daemon_incarnation(self, daemon_incarnation: String) -> Self {
3158 self.spawn_events.configure_incarnation(daemon_incarnation);
3162 self
3163 }
3164
3165 pub fn with_terminal_journal(self, path: PathBuf, daemon_incarnation: String) -> Self {
3168 let mut this = self.with_daemon_incarnation(daemon_incarnation.clone());
3169 this.terminal_journal = Some(Arc::new(crate::terminal_journal::TerminalJournal::open(
3170 path,
3171 daemon_incarnation,
3172 )));
3173 this
3174 }
3175
3176 pub fn with_forwarding(mut self, forwarding: Arc<ForwardingTable>) -> Self {
3177 self.forwarding = Some(forwarding);
3178 self
3179 }
3180
3181 pub fn with_handle(mut self, supervisor_handle: SupervisorHandle) -> Self {
3182 self.spawn_events = supervisor_handle.spawn_events.clone();
3183 self.supervisor_handle = Some(supervisor_handle);
3184 self
3185 }
3186
3187 pub fn with_health_config(mut self, health: HealthConfig) -> Self {
3188 self.health = health;
3189 self
3190 }
3191
3192 pub fn with_live_children_record(self, path: impl Into<PathBuf>) -> Self {
3196 self.child_roster.record_to(path.into());
3197 self
3198 }
3199
3200 #[cfg(target_os = "linux")]
3201 pub fn with_cgroup_placement(
3202 mut self,
3203 cgroup_placement: Option<subc_cgroup::Placement>,
3204 ) -> Self {
3205 self.cgroup_placement = cgroup_placement;
3206 self
3207 }
3208
3209 pub fn spawn(&self, spec: ModuleSpec) -> Result<SupervisedModule, SuperviseError> {
3215 validate_spec(&spec)?;
3216 self.establish_identity(&spec);
3217
3218 let runtime = self.runtime_config();
3219 let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::starting()));
3220 let activity_spawn = runtime.begin_activity_spawn(&spec.module_id);
3221 let registration_watch = watch_registration(&self.registry, &spec);
3222 let spawned = spawn_child(
3223 &spec,
3224 runtime.connection_file_path.as_deref(),
3225 self.supervisor_handle.as_ref(),
3226 &runtime.stderr_ring,
3227 runtime.capture_logs_dir.as_deref(),
3228 &runtime.child_roster,
3229 #[cfg(target_os = "linux")]
3230 runtime.cgroup_placement.as_ref(),
3231 );
3232 #[cfg(test)]
3233 self.test_after_first_spawn.run(&spec.module_id);
3234 let mut child = match spawned {
3235 Ok(child) => child,
3236 Err(err) => {
3237 self.abandon_unrostered(&spec.module_id);
3240 return Err(err);
3241 }
3242 };
3243 arm_registration_deadline(&mut child, runtime.register_timeout, registration_watch);
3244 self.mark_first_process_running(&spec, &runtime, &snapshot, &mut child)?;
3245 if let Some(spawn) = activity_spawn {
3246 spawn.complete(&snapshot);
3247 }
3248
3249 Ok(self.supervised_module(spec, runtime, snapshot, Some(child)))
3250 }
3251
3252 fn establish_identity(&self, spec: &ModuleSpec) {
3265 if let Some(supervisor_handle) = &self.supervisor_handle {
3266 supervisor_handle.apply_identity_configuration(spec);
3267 supervisor_handle.mark_configured(&spec.module_id);
3268 }
3269 }
3270
3271 fn abandon_unrostered(&self, module_id: &str) {
3274 if let Some(supervisor_handle) = &self.supervisor_handle {
3275 supervisor_handle.unmark_configured_unless_rostered(module_id);
3276 }
3277 }
3278
3279 fn mark_first_process_running(
3282 &self,
3283 spec: &ModuleSpec,
3284 runtime: &SupervisorRuntimeConfig,
3285 snapshot: &SharedSnapshot,
3286 child: &mut SupervisedChild,
3287 ) -> Result<(), SuperviseError> {
3288 if let Err(err) = set_running(snapshot, child, &spec.module_id, &runtime.spawn_events) {
3289 self.abandon_unrostered(&spec.module_id);
3290 return Err(err);
3291 }
3292 self.process_liveness
3293 .track(spec.module_id.clone(), Arc::clone(snapshot));
3294 Ok(())
3295 }
3296
3297 pub fn supervise_configured(
3303 &self,
3304 spec: ModuleSpec,
3305 enabled: bool,
3306 ) -> Result<SupervisedModule, SuperviseError> {
3307 self.supervise_configured_requested(
3308 spec,
3309 enabled,
3310 self.health.clone(),
3311 None,
3312 self.restart_policy,
3313 DEFAULT_REGISTER_TIMEOUT,
3314 None,
3315 )
3316 }
3317
3318 pub fn supervise_configured_with_health(
3324 &self,
3325 spec: ModuleSpec,
3326 enabled: bool,
3327 health: HealthConfig,
3328 drain_timeout_ms: Option<u64>,
3329 restart_policy: RestartPolicy,
3330 ) -> Result<SupervisedModule, SuperviseError> {
3331 self.supervise_configured_requested(
3332 spec,
3333 enabled,
3334 health,
3335 drain_timeout_ms,
3336 restart_policy,
3337 DEFAULT_REGISTER_TIMEOUT,
3338 None,
3339 )
3340 }
3341
3342 #[allow(clippy::too_many_arguments)]
3343 pub(crate) fn supervise_configured_requested(
3344 &self,
3345 spec: ModuleSpec,
3346 enabled: bool,
3347 health: HealthConfig,
3348 drain_timeout_ms: Option<u64>,
3349 restart_policy: RestartPolicy,
3350 register_timeout: Duration,
3351 requester: Option<Principal>,
3352 ) -> Result<SupervisedModule, SuperviseError> {
3353 let registered = self.register_configured_with_health(
3354 spec,
3355 enabled,
3356 health,
3357 drain_timeout_ms,
3358 restart_policy,
3359 register_timeout,
3360 )?;
3361 let daemon_initiated = requester.is_none();
3362 self.start_registered_configured(registered, requester, daemon_initiated)
3363 }
3364
3365 pub(crate) fn register_configured_with_health(
3370 &self,
3371 spec: ModuleSpec,
3372 enabled: bool,
3373 health: HealthConfig,
3374 drain_timeout_ms: Option<u64>,
3375 restart_policy: RestartPolicy,
3376 register_timeout: Duration,
3377 ) -> Result<RegisteredConfiguredModule, SuperviseError> {
3378 validate_spec(&spec)?;
3379 self.establish_identity(&spec);
3380
3381 let mut runtime = self.runtime_config();
3382 runtime.health = health;
3383 runtime.restart_policy = restart_policy;
3384 runtime.register_timeout = register_timeout;
3385 if let Some(ms) = drain_timeout_ms {
3386 runtime.drain_timeout = Duration::from_millis(ms);
3387 *runtime
3388 .effective_drain_timeout
3389 .lock()
3390 .unwrap_or_else(|poisoned| poisoned.into_inner()) = runtime.drain_timeout;
3391 }
3392 let snapshot = if enabled {
3393 SupervisorSnapshot::starting()
3394 } else {
3395 SupervisorSnapshot::disabled()
3396 };
3397 let snapshot = {
3398 let mut snapshot = snapshot;
3399 if enabled {
3400 if let Some((pid, reason)) = runtime.child_roster.previous_instance(&spec.module_id)
3401 {
3402 snapshot.waiting_for_previous_pid = Some(pid);
3403 snapshot.waiting_for_previous_detail = Some(reason.waiting_message(pid));
3404 }
3405 }
3406 snapshot
3407 };
3408 let (module, commands) =
3409 self.register_module(&spec, &runtime, Arc::new(Mutex::new(snapshot)));
3410 Ok(RegisteredConfiguredModule {
3411 spec,
3412 runtime,
3413 module,
3414 commands,
3415 })
3416 }
3417
3418 pub(crate) fn start_registered_configured(
3421 &self,
3422 registered: RegisteredConfiguredModule,
3423 requester: Option<Principal>,
3424 daemon_initiated: bool,
3425 ) -> Result<SupervisedModule, SuperviseError> {
3426 let RegisteredConfiguredModule {
3427 spec,
3428 mut runtime,
3429 module,
3430 commands,
3431 } = registered;
3432 runtime.start_requester = requester;
3433 runtime.recovery_is_daemon = daemon_initiated;
3434 let snapshot = &module.inner.snapshot;
3435 let should_spawn = {
3436 let state = lock_snapshot(snapshot)?;
3437 state.enabled && state.waiting_for_previous_pid.is_none()
3438 };
3439 let child = if should_spawn {
3440 let activity_spawn = runtime.begin_activity_spawn(&spec.module_id);
3441 let registration_watch = watch_registration(&self.registry, &spec);
3442 let spawned = spawn_child(
3443 &spec,
3444 runtime.connection_file_path.as_deref(),
3445 self.supervisor_handle.as_ref(),
3446 &runtime.stderr_ring,
3447 runtime.capture_logs_dir.as_deref(),
3448 &runtime.child_roster,
3449 #[cfg(target_os = "linux")]
3450 runtime.cgroup_placement.as_ref(),
3451 );
3452 #[cfg(test)]
3453 self.test_after_first_spawn.run(&spec.module_id);
3454 match spawned {
3455 Ok(mut child) => {
3456 arm_registration_deadline(
3457 &mut child,
3458 runtime.register_timeout,
3459 registration_watch,
3460 );
3461 self.mark_first_process_running(&spec, &runtime, snapshot, &mut child)?;
3462 if let Some(spawn) = activity_spawn {
3463 spawn.complete(snapshot);
3464 }
3465 Some(child)
3466 }
3467 Err(err) => {
3468 if runtime.health.critical {
3469 error!(
3470 module_id = %spec.module_id,
3471 program = %spec.program.display(),
3472 error = %err,
3473 "critical configured module failed to spawn; marking failed and alerting"
3474 );
3475 } else {
3476 error!(
3477 module_id = %spec.module_id,
3478 program = %spec.program.display(),
3479 error = %err,
3480 "configured module failed to spawn; marking failed and continuing"
3481 );
3482 }
3483 fail_snapshot(
3484 snapshot,
3485 Some(&spec.module_id),
3486 None,
3487 &runtime.failure_context(daemon_initiated),
3488 );
3489 None
3490 }
3491 }
3492 } else {
3493 None
3494 };
3495 self.start_monitor(spec, runtime, &module, child, commands);
3496 Ok(module)
3497 }
3498
3499 fn runtime_config(&self) -> SupervisorRuntimeConfig {
3500 let effective_drain_timeout = Arc::new(Mutex::new(self.drain_timeout));
3501 SupervisorRuntimeConfig {
3502 registry: self.registry.clone(),
3503 recovery_is_daemon: true,
3504 start_requester: None,
3505 pending_activity_restart: false,
3506 provenance_probe: self.provenance_probe.clone(),
3507 scheduled_respawn: Arc::default(),
3508 deferred_reload_reply: Arc::default(),
3509 restart_policy: self.restart_policy,
3510 register_timeout: DEFAULT_REGISTER_TIMEOUT,
3511 drain_timeout: self.drain_timeout,
3512 child_roster: self
3515 .child_roster
3516 .for_module(Arc::clone(&effective_drain_timeout)),
3517 effective_drain_timeout,
3518 default_drain_timeout: self.drain_timeout,
3519 health: self.health.clone(),
3520 connection_file_path: self.connection_file_path.clone(),
3521 capture_logs_dir: self.capture_logs_dir.clone(),
3522 forwarding: self.forwarding.clone(),
3523 supervisor_handle: self.supervisor_handle.clone(),
3524 stderr_ring: Arc::new(Mutex::new(StderrRing::new(StderrTailConfig::default()))),
3525 terminal_ring: Arc::new(Mutex::new(
3526 TerminalRing::new(
3527 TerminalRingConfig::default(),
3528 self.daemon_start_clock.started_at_ms(),
3529 )
3530 .with_start_clock(self.daemon_start_clock)
3531 .with_journal(self.terminal_journal.clone())
3532 .with_daemon_shutdown(self.child_roster.shutdown_flag()),
3533 )),
3534 spawn_events: self.spawn_events.clone(),
3535 #[cfg(target_os = "linux")]
3536 cgroup_placement: self.cgroup_placement.clone(),
3537 #[cfg(test)]
3538 test_seed_stale_facts_before_enable_spawn: false,
3539 #[cfg(test)]
3540 test_reload_exit_record_gate: None,
3541 }
3542 }
3543
3544 fn supervised_module(
3545 &self,
3546 spec: ModuleSpec,
3547 runtime: SupervisorRuntimeConfig,
3548 snapshot: SharedSnapshot,
3549 child: Option<SupervisedChild>,
3550 ) -> SupervisedModule {
3551 let (module, commands) = self.register_module(&spec, &runtime, snapshot);
3552 self.start_monitor(spec, runtime, &module, child, commands);
3553 module
3554 }
3555
3556 fn register_module(
3557 &self,
3558 spec: &ModuleSpec,
3559 runtime: &SupervisorRuntimeConfig,
3560 snapshot: SharedSnapshot,
3561 ) -> (SupervisedModule, mpsc::Receiver<SupervisorCommand>) {
3562 let configuration = Arc::new(Mutex::new(SupervisedConfiguration {
3563 spec: spec.clone(),
3564 health: runtime.health.clone(),
3565 }));
3566 let stderr_ring = Arc::clone(&runtime.stderr_ring);
3567 let terminal_ring = Arc::clone(&runtime.terminal_ring);
3568 let restart_policy = runtime.restart_policy;
3572 let effective_drain_timeout = Arc::clone(&runtime.effective_drain_timeout);
3573 let (tx, rx) = mpsc::channel(4);
3574 let module_id = spec.module_id.clone();
3575 let module = SupervisedModule {
3576 inner: Arc::new(SupervisedModuleInner {
3577 module_id: module_id.clone(),
3578 registry: Arc::clone(&self.registry),
3579 snapshot,
3580 configuration,
3581 stderr_ring,
3582 terminal_ring,
3583 commands: tx,
3584 monitor: Mutex::new(None),
3585 restart_policy,
3586 effective_drain_timeout,
3587 provenance_probe: self.provenance_probe.clone(),
3588 }),
3589 };
3590 if let Some(supervisor_handle) = &self.supervisor_handle {
3593 supervisor_handle.insert(module.clone());
3594 }
3595 (module, rx)
3596 }
3597
3598 fn start_monitor(
3599 &self,
3600 spec: ModuleSpec,
3601 runtime: SupervisorRuntimeConfig,
3602 module: &SupervisedModule,
3603 child: Option<SupervisedChild>,
3604 commands: mpsc::Receiver<SupervisorCommand>,
3605 ) {
3606 let monitor = tokio::spawn(supervise_loop(
3607 spec,
3608 runtime,
3609 Arc::clone(&self.registry),
3610 Arc::clone(&self.process_liveness),
3611 Arc::clone(&module.inner.snapshot),
3612 child,
3613 commands,
3614 ));
3615 *module
3616 .inner
3617 .monitor
3618 .lock()
3619 .unwrap_or_else(|p| p.into_inner()) = Some(monitor);
3620 }
3621}
3622
3623impl Default for Supervisor {
3624 fn default() -> Self {
3625 Self::new(Arc::new(Registry::default()), RestartPolicy::default())
3626 }
3627}
3628
3629#[derive(Clone)]
3631pub struct SupervisedModule {
3632 inner: Arc<SupervisedModuleInner>,
3633}
3634
3635struct SupervisedModuleInner {
3636 module_id: String,
3637 registry: Arc<Registry>,
3638 snapshot: SharedSnapshot,
3639 configuration: Arc<Mutex<SupervisedConfiguration>>,
3640 stderr_ring: Arc<Mutex<StderrRing>>,
3641 terminal_ring: Arc<Mutex<TerminalRing>>,
3642 commands: mpsc::Sender<SupervisorCommand>,
3643 monitor: Mutex<Option<JoinHandle<()>>>,
3644 restart_policy: RestartPolicy,
3648 effective_drain_timeout: Arc<Mutex<Duration>>,
3649 provenance_probe: ExecutableIdentityProbe,
3650}
3651
3652impl fmt::Debug for SupervisedModule {
3653 fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
3654 f.debug_struct("SupervisedModule")
3655 .field("module_id", &self.inner.module_id)
3656 .field("status", &self.status())
3657 .finish_non_exhaustive()
3658 }
3659}
3660
3661impl SupervisedModule {
3662 pub fn module_id(&self) -> &str {
3663 &self.inner.module_id
3664 }
3665
3666 #[cfg(test)]
3670 pub(crate) fn record_health_probe_failure_for_test(
3671 &self,
3672 detail: &str,
3673 ) -> Result<(), SuperviseError> {
3674 update_snapshot(&self.inner.snapshot, Some(&self.inner.module_id), |state| {
3675 state.health.consecutive_failures = state.health.consecutive_failures.saturating_add(1);
3676 state.health.detail = Some(detail.to_string());
3677 })
3678 }
3679
3680 pub fn state(&self) -> Result<ModuleState, SuperviseError> {
3681 Ok(lock_snapshot(&self.inner.snapshot)?.state)
3682 }
3683
3684 pub(crate) fn registration_end_reason(
3685 &self,
3686 ) -> Result<Option<RegistrationEndReason>, SuperviseError> {
3687 let snapshot = lock_snapshot(&self.inner.snapshot)?;
3688 Ok(match snapshot.state {
3689 ModuleState::Draining if snapshot.draining_to_replace => {
3690 Some(RegistrationEndReason::SupervisorRestart)
3691 }
3692 ModuleState::Draining => Some(RegistrationEndReason::SupervisorStop),
3693 ModuleState::Restarting if snapshot.draining_to_replace => {
3694 Some(RegistrationEndReason::SupervisorRestart)
3695 }
3696 _ => None,
3697 })
3698 }
3699
3700 pub fn stderr_tail(
3707 &self,
3708 max_lines: Option<usize>,
3709 max_bytes: Option<usize>,
3710 ) -> StderrTailSnapshot {
3711 self.inner
3712 .stderr_ring
3713 .lock()
3714 .unwrap_or_else(|poisoned| poisoned.into_inner())
3715 .snapshot(max_lines, max_bytes)
3716 }
3717
3718 pub fn terminal_history(&self) -> TerminalHistorySnapshot {
3723 self.inner
3724 .terminal_ring
3725 .lock()
3726 .unwrap_or_else(|poisoned| poisoned.into_inner())
3727 .snapshot()
3728 }
3729
3730 pub fn durable_terminal_history(&self) -> subc_control::TerminalHistory {
3735 durable_terminal_history_of(&self.inner.terminal_ring, &self.inner.module_id)
3736 }
3737
3738 pub(crate) async fn read_durable_terminal_history(
3743 &self,
3744 ) -> Result<subc_control::TerminalHistory, tokio::task::JoinError> {
3745 let terminal_ring = Arc::clone(&self.inner.terminal_ring);
3746 let module_id = self.inner.module_id.clone();
3747 tokio::task::spawn_blocking(move || durable_terminal_history_of(&terminal_ring, &module_id))
3748 .await
3749 }
3750
3751 pub fn status(&self) -> Result<ModuleStatus, SuperviseError> {
3752 self.status_with_snapshot_lock(&self.inner.snapshot, None)
3753 .map(|(status, _)| status)
3754 }
3755
3756 pub(crate) fn next_retry_at_ms(&self) -> Result<Option<u64>, SuperviseError> {
3757 let state = lock_snapshot(&self.inner.snapshot)?;
3758 Ok(state
3759 .failed_retry
3760 .pending
3761 .filter(|_| state.state == ModuleState::Failed && state.enabled)
3762 .map(|retry| {
3763 unix_ms_now().saturating_add(
3764 u64::try_from(
3765 retry
3766 .deadline
3767 .saturating_duration_since(Instant::now())
3768 .as_millis(),
3769 )
3770 .unwrap_or(u64::MAX),
3771 )
3772 }))
3773 }
3774
3775 pub(crate) fn record_deliberate_severance(
3776 &self,
3777 identity: ProcessIdentity,
3778 ) -> Result<bool, SuperviseError> {
3779 let mut snapshot = lock_snapshot(&self.inner.snapshot)?;
3780 if snapshot.pid != Some(identity.pid)
3781 || snapshot.process_start_time != Some(identity.start_time)
3782 {
3783 return Ok(false);
3784 }
3785 snapshot.deliberate_severance = Some(identity);
3786 Ok(true)
3787 }
3788
3789 pub(crate) fn status_for_control(
3794 &self,
3795 caller: &'static str,
3796 ) -> Result<ModuleStatus, SuperviseError> {
3797 self.status_with_snapshot_lock(&self.inner.snapshot, Some(caller))
3798 .map(|(status, _)| status)
3799 }
3800
3801 fn status_with_snapshot_lock(
3802 &self,
3803 snapshot: &SharedSnapshot,
3804 caller: Option<&'static str>,
3805 ) -> Result<(ModuleStatus, Option<SpawnedFileIdentity>), SuperviseError> {
3806 let mut guard = match caller {
3807 Some(caller) => lock_snapshot_for_control(snapshot, &self.inner.module_id, caller)?,
3808 None => lock_snapshot(snapshot)?,
3809 };
3810 let restart_count =
3813 guard.crash_restarts_in_window(self.inner.restart_policy.window, Instant::now());
3814 let snapshot = guard.clone();
3815 drop(guard);
3816 let drain_timeout = *self.inner.effective_drain_timeout.lock().map_err(|_| {
3817 SuperviseError::StatePoisoned {
3818 module_id: Some(self.inner.module_id.clone()),
3819 }
3820 })?;
3821 let registration_active = self
3822 .inner
3823 .registry
3824 .get_module(&self.inner.module_id)
3825 .map_err(SuperviseError::Registry)?
3826 .is_some();
3827 let protocol = snapshot
3828 .spawned_protocol
3829 .unwrap_or(self.declared_protocol()?);
3830 let running_process =
3831 snapshot.enabled && snapshot.state == ModuleState::Running && snapshot.process_alive;
3832 let live = match protocol {
3838 ModuleProtocol::Subc => running_process && registration_active,
3839 ModuleProtocol::None => running_process,
3840 };
3841
3842 Ok((
3843 ModuleStatus {
3844 waiting_for_previous_pid: snapshot.waiting_for_previous_pid,
3845 waiting_for_previous_detail: snapshot.waiting_for_previous_detail.clone(),
3846 module_id: self.inner.module_id.clone(),
3847 launch_nonce_source: observed_launch_nonce_source(&snapshot),
3848 state: snapshot.state,
3849 enabled: snapshot.enabled,
3850 process_alive: snapshot.process_alive,
3851 registration_active,
3852 protocol,
3853 live,
3854 restart_count,
3855 lifetime_restarts: snapshot.lifetime_restarts,
3856 spawn_generation: snapshot.spawn_generation,
3857 max_restarts: self.inner.restart_policy.max_restarts,
3858 restart_window: self.inner.restart_policy.window,
3859 drain_timeout,
3860 restart_backoff: self.inner.restart_policy.backoff,
3861 restart_max_backoff: self.inner.restart_policy.max_backoff,
3862 pid: snapshot.reported_pid(),
3863 spawned_at_ms: snapshot.spawned_at_ms,
3864 spawned_from: snapshot.spawned_from,
3865 launch_spec_digest: snapshot.launch_spec_digest,
3866 process_start_time: snapshot.process_start_time,
3867 last_exit: snapshot.last_exit,
3868 health: snapshot.health,
3869 },
3870 snapshot.spawned_file_identity.clone(),
3871 ))
3872 }
3873
3874 #[cfg(test)]
3875 pub(crate) fn hold_snapshot_for_test(
3876 &self,
3877 acquired: std::sync::mpsc::Sender<()>,
3878 hold: Duration,
3879 ) -> std::thread::JoinHandle<()> {
3880 let snapshot = Arc::clone(&self.inner.snapshot);
3881 std::thread::spawn(move || {
3882 let _guard = snapshot.lock().expect("test snapshot lock is not poisoned");
3883 acquired
3884 .send(())
3885 .expect("test receiver waits for snapshot lock");
3886 std::thread::sleep(hold);
3887 })
3888 }
3889
3890 pub(crate) async fn status_and_running_image_agreement(
3896 &self,
3897 ) -> Result<(ModuleStatus, subc_control::RunningImageAgreement), SuperviseError> {
3898 let (status, identity) = self.status_with_snapshot_lock(&self.inner.snapshot, None)?;
3899 let image = self
3900 .inner
3901 .provenance_probe
3902 .observe(
3903 status.pid,
3904 status.spawned_from.as_deref(),
3905 identity,
3906 status.process_start_time,
3907 )
3908 .await;
3909 Ok((status, image))
3910 }
3911
3912 pub(crate) async fn running_image_agreement(&self) -> subc_control::RunningImageAgreement {
3913 let snapshot = match lock_snapshot(&self.inner.snapshot) {
3914 Ok(snapshot) => snapshot.clone(),
3915 Err(_) => {
3916 return subc_control::RunningImageAgreement::Unavailable {
3917 reason: subc_control::RunningImageUnavailableReason::NotRunning,
3918 };
3919 }
3920 };
3921 self.inner
3922 .provenance_probe
3923 .observe(
3924 snapshot.reported_pid(),
3925 snapshot.spawned_from.as_deref(),
3926 snapshot.spawned_file_identity.clone(),
3927 snapshot.process_start_time,
3928 )
3929 .await
3930 }
3931
3932 pub(crate) fn child_resource_usage(&self) -> subc_control::ChildResourceUsage {
3935 #[cfg(windows)]
3936 {
3937 let image = lock_snapshot(&self.inner.snapshot)
3938 .ok()
3939 .and_then(|snapshot| {
3940 snapshot
3941 .spawned_file_identity
3942 .as_ref()
3943 .and_then(|identity| identity.windows_image.clone())
3944 });
3945 match image {
3946 Some(image) => {
3947 crate::child_resources::read_windows_handle(image.process(), image.start_time())
3948 }
3949 None => subc_control::ChildResourceUsage::Unavailable {
3950 reason: subc_control::ChildResourceUnavailableReason::NotRunning,
3951 },
3952 }
3953 }
3954 #[cfg(not(windows))]
3955 {
3956 let (pid, start_time) = match lock_snapshot(&self.inner.snapshot) {
3957 Ok(snapshot) => (snapshot.reported_pid(), snapshot.process_start_time),
3958 Err(_) => {
3959 return subc_control::ChildResourceUsage::Unavailable {
3960 reason: subc_control::ChildResourceUnavailableReason::Unreadable,
3961 }
3962 }
3963 };
3964 crate::child_resources::read(pid, start_time)
3965 }
3966 }
3967
3968 pub(crate) fn subtree_target(&self) -> crate::subtree_resources::Target {
3969 self.inner
3970 .snapshot
3971 .try_lock()
3972 .map(|snapshot| snapshot.subtree_target.clone())
3973 .unwrap_or(crate::subtree_resources::Target::Unavailable("read_failed"))
3974 }
3975
3976 #[cfg(test)]
3977 pub(crate) fn set_subtree_target_for_test(&self, target: crate::subtree_resources::Target) {
3978 self.inner.snapshot.lock().unwrap().subtree_target = target;
3979 }
3980
3981 pub(crate) fn will_recover_after_connection_loss(&self) -> Result<bool, SuperviseError> {
3982 let mut snapshot = lock_snapshot(&self.inner.snapshot)?;
3983 Ok(match snapshot.state {
3984 ModuleState::Restarting => true,
3985 ModuleState::Failed | ModuleState::Disabled => false,
3986 _ if snapshot.enabled && snapshot.respawn_pending => true,
3987 _ => daemon_will_restart(&mut snapshot, &self.inner.restart_policy, Instant::now()),
3988 })
3989 }
3990
3991 #[cfg(test)]
3992 pub(crate) fn is_warming(&self) -> Result<bool, SuperviseError> {
3993 self.is_warming_with_snapshot_lock(None)
3994 }
3995
3996 pub(crate) fn is_warming_for_control(
3997 &self,
3998 caller: &'static str,
3999 ) -> Result<bool, SuperviseError> {
4000 self.is_warming_with_snapshot_lock(Some(caller))
4001 }
4002
4003 fn is_warming_with_snapshot_lock(
4004 &self,
4005 caller: Option<&'static str>,
4006 ) -> Result<bool, SuperviseError> {
4007 let snapshot = match caller {
4008 Some(caller) => {
4009 lock_snapshot_for_control(&self.inner.snapshot, &self.inner.module_id, caller)?
4010 }
4011 None => lock_snapshot(&self.inner.snapshot)?,
4012 }
4013 .clone();
4014 Ok(matches!(
4015 snapshot.state,
4016 ModuleState::Starting | ModuleState::Running | ModuleState::Restarting
4017 ))
4018 }
4019
4020 pub async fn drain(&self) -> Result<(), SuperviseError> {
4022 self.stop().await
4023 }
4024
4025 pub(crate) async fn retire_requested(
4026 &self,
4027 requester: Option<Principal>,
4028 ) -> Result<(), SuperviseError> {
4029 match self.state()? {
4030 ModuleState::Stopped => return Ok(()),
4031 ModuleState::Failed
4032 if lock_snapshot(&self.inner.snapshot)?
4033 .failed_retry
4034 .pending
4035 .is_none() =>
4036 {
4037 return Ok(())
4038 }
4039 ModuleState::Failed
4040 | ModuleState::Starting
4041 | ModuleState::Running
4042 | ModuleState::Unresponsive
4043 | ModuleState::Restarting
4044 | ModuleState::Draining
4045 | ModuleState::Disabled => {}
4046 }
4047
4048 let (reply_tx, reply_rx) = oneshot::channel();
4049 self.inner
4050 .commands
4051 .send(SupervisorCommand::Retire {
4052 requester,
4053 reply: reply_tx,
4054 })
4055 .await
4056 .map_err(|_| SuperviseError::CommandClosed {
4057 module_id: self.inner.module_id.clone(),
4058 })?;
4059 reply_rx.await.map_err(|_| SuperviseError::CommandClosed {
4060 module_id: self.inner.module_id.clone(),
4061 })?
4062 }
4063
4064 pub async fn stop(&self) -> Result<(), SuperviseError> {
4065 self.stop_requested(None).await
4066 }
4067
4068 pub(crate) async fn stop_requested(
4069 &self,
4070 requester: Option<Principal>,
4071 ) -> Result<(), SuperviseError> {
4072 match self.state()? {
4073 ModuleState::Stopped => return Ok(()),
4074 ModuleState::Failed
4075 if lock_snapshot(&self.inner.snapshot)?
4076 .failed_retry
4077 .pending
4078 .is_none() =>
4079 {
4080 return Ok(())
4081 }
4082 ModuleState::Failed
4083 | ModuleState::Starting
4084 | ModuleState::Running
4085 | ModuleState::Unresponsive
4086 | ModuleState::Restarting
4087 | ModuleState::Draining
4088 | ModuleState::Disabled => {}
4089 }
4090
4091 let (reply_tx, reply_rx) = oneshot::channel();
4092 self.inner
4093 .commands
4094 .send(SupervisorCommand::Drain {
4095 requester,
4096 reply: reply_tx,
4097 })
4098 .await
4099 .map_err(|_| SuperviseError::CommandClosed {
4100 module_id: self.inner.module_id.clone(),
4101 })?;
4102 reply_rx.await.map_err(|_| SuperviseError::CommandClosed {
4103 module_id: self.inner.module_id.clone(),
4104 })?
4105 }
4106
4107 pub async fn restart(&self, drain_timeout_ms: Option<u64>) -> Result<(), SuperviseError> {
4108 self.restart_requested(drain_timeout_ms, None).await
4109 }
4110
4111 pub(crate) async fn restart_requested(
4112 &self,
4113 drain_timeout_ms: Option<u64>,
4114 requester: Option<Principal>,
4115 ) -> Result<(), SuperviseError> {
4116 let queued_at = Instant::now();
4117 let received_at_generation = lock_snapshot(&self.inner.snapshot)?.spawn_generation;
4118 let (reply_tx, reply_rx) = oneshot::channel();
4119 self.inner
4120 .commands
4121 .send(SupervisorCommand::Restart {
4122 requester,
4123 drain_timeout_ms,
4124 received_at_generation,
4125 queued_at,
4126 reply: reply_tx,
4127 })
4128 .await
4129 .map_err(|_| SuperviseError::CommandClosed {
4130 module_id: self.inner.module_id.clone(),
4131 })?;
4132 reply_rx.await.map_err(|_| SuperviseError::CommandClosed {
4133 module_id: self.inner.module_id.clone(),
4134 })?
4135 }
4136
4137 pub async fn swap(&self, ready_timeout: Option<Duration>) -> Result<(), SuperviseError> {
4142 self.swap_requested(ready_timeout, None).await
4143 }
4144
4145 pub(crate) async fn swap_requested(
4146 &self,
4147 ready_timeout: Option<Duration>,
4148 requester: Option<Principal>,
4149 ) -> Result<(), SuperviseError> {
4150 let (reply_tx, reply_rx) = oneshot::channel();
4151 self.inner
4152 .commands
4153 .send(SupervisorCommand::Swap {
4154 requester,
4155 ready_timeout,
4156 reply: reply_tx,
4157 })
4158 .await
4159 .map_err(|_| SuperviseError::CommandClosed {
4160 module_id: self.inner.module_id.clone(),
4161 })?;
4162 reply_rx.await.map_err(|_| SuperviseError::CommandClosed {
4163 module_id: self.inner.module_id.clone(),
4164 })?
4165 }
4166
4167 pub async fn reload(&self) -> Result<(), SuperviseError> {
4168 self.reload_requested(None).await
4169 }
4170
4171 pub(crate) async fn reload_requested(
4172 &self,
4173 requester: Option<Principal>,
4174 ) -> Result<(), SuperviseError> {
4175 let (reply_tx, reply_rx) = oneshot::channel();
4176 self.inner
4177 .commands
4178 .send(SupervisorCommand::Reload {
4179 requester,
4180 reply: reply_tx,
4181 })
4182 .await
4183 .map_err(|_| SuperviseError::CommandClosed {
4184 module_id: self.inner.module_id.clone(),
4185 })?;
4186 reply_rx.await.map_err(|_| SuperviseError::CommandClosed {
4187 module_id: self.inner.module_id.clone(),
4188 })?
4189 }
4190
4191 pub async fn set_enabled(&self, enabled: bool) -> Result<bool, SuperviseError> {
4192 self.set_enabled_requested(enabled, None).await
4193 }
4194
4195 pub(crate) async fn set_enabled_requested(
4196 &self,
4197 enabled: bool,
4198 requester: Option<Principal>,
4199 ) -> Result<bool, SuperviseError> {
4200 let (reply_tx, reply_rx) = oneshot::channel();
4201 self.inner
4202 .commands
4203 .send(SupervisorCommand::SetEnabled {
4204 requester,
4205 enabled,
4206 reply: reply_tx,
4207 })
4208 .await
4209 .map_err(|_| SuperviseError::CommandClosed {
4210 module_id: self.inner.module_id.clone(),
4211 })?;
4212 reply_rx.await.map_err(|_| SuperviseError::CommandClosed {
4213 module_id: self.inner.module_id.clone(),
4214 })?
4215 }
4216
4217 pub(crate) fn declared_protocol(&self) -> Result<ModuleProtocol, SuperviseError> {
4221 let configured = self
4222 .inner
4223 .configuration
4224 .lock()
4225 .map_err(|_| SuperviseError::StatePoisoned {
4226 module_id: Some(self.inner.module_id.clone()),
4227 })?
4228 .spec
4229 .protocol;
4230 let state = lock_snapshot(&self.inner.snapshot)?;
4231 Ok(state.spawned_protocol.unwrap_or(configured))
4232 }
4233
4234 pub(crate) fn configuration(&self) -> Result<(ModuleSpec, HealthConfig), SuperviseError> {
4235 let configuration =
4236 self.inner
4237 .configuration
4238 .lock()
4239 .map_err(|_| SuperviseError::StatePoisoned {
4240 module_id: Some(self.inner.module_id.clone()),
4241 })?;
4242 Ok((configuration.spec.clone(), configuration.health.clone()))
4243 }
4244
4245 #[cfg(any(test, feature = "test-support"))]
4249 pub async fn update_spec_for_test(&self, spec: ModuleSpec) -> Result<(), SuperviseError> {
4250 let (_, health) = self.configuration()?;
4251 let drain_timeout_ms = u64::try_from(
4252 self.inner
4253 .effective_drain_timeout
4254 .lock()
4255 .unwrap_or_else(|poisoned| poisoned.into_inner())
4256 .as_millis(),
4257 )
4258 .ok();
4259 self.update_configuration(spec, health, drain_timeout_ms)
4260 .await
4261 }
4262
4263 pub(crate) async fn update_configuration(
4264 &self,
4265 spec: ModuleSpec,
4266 health: HealthConfig,
4267 drain_timeout_ms: Option<u64>,
4268 ) -> Result<(), SuperviseError> {
4269 if spec.module_id != self.inner.module_id {
4270 return Err(SuperviseError::InvalidSpec {
4271 reason: "a supervised module's module_id cannot be changed".to_string(),
4272 });
4273 }
4274 validate_spec(&spec)?;
4275 let (reply_tx, reply_rx) = oneshot::channel();
4276 self.inner
4277 .commands
4278 .send(SupervisorCommand::UpdateConfiguration {
4279 spec: spec.clone(),
4280 health: health.clone(),
4281 drain_timeout_ms,
4282 reply: reply_tx,
4283 })
4284 .await
4285 .map_err(|_| SuperviseError::CommandClosed {
4286 module_id: self.inner.module_id.clone(),
4287 })?;
4288 reply_rx.await.map_err(|_| SuperviseError::CommandClosed {
4289 module_id: self.inner.module_id.clone(),
4290 })?;
4291 let mut configuration =
4292 self.inner
4293 .configuration
4294 .lock()
4295 .map_err(|_| SuperviseError::StatePoisoned {
4296 module_id: Some(self.inner.module_id.clone()),
4297 })?;
4298 configuration.spec = spec;
4299 configuration.health = health;
4300 Ok(())
4301 }
4302}
4303
4304impl Drop for SupervisedModuleInner {
4305 fn drop(&mut self) {
4306 let Ok(mut monitor) = self.monitor.lock() else {
4307 return;
4308 };
4309 if let Some(monitor) = monitor.as_ref().filter(|monitor| !monitor.is_finished()) {
4310 let _ = update_snapshot(&self.snapshot, Some(&self.module_id), |state| {
4311 state.state = ModuleState::Stopped;
4312 clear_current_process_facts(state);
4313 });
4314 monitor.abort();
4315 }
4316 let _ = monitor.take();
4317 }
4318}
4319
4320#[derive(Debug)]
4321enum SupervisorCommand {
4322 Drain {
4323 requester: Option<Principal>,
4324 reply: oneshot::Sender<Result<(), SuperviseError>>,
4325 },
4326 Retire {
4327 requester: Option<Principal>,
4328 reply: oneshot::Sender<Result<(), SuperviseError>>,
4329 },
4330 Restart {
4331 requester: Option<Principal>,
4332 drain_timeout_ms: Option<u64>,
4337 received_at_generation: u64,
4341 queued_at: Instant,
4344 reply: oneshot::Sender<Result<(), SuperviseError>>,
4345 },
4346 Reload {
4347 requester: Option<Principal>,
4348 reply: oneshot::Sender<Result<(), SuperviseError>>,
4349 },
4350 SetEnabled {
4351 requester: Option<Principal>,
4352 enabled: bool,
4353 reply: oneshot::Sender<Result<bool, SuperviseError>>,
4354 },
4355 UpdateConfiguration {
4356 spec: ModuleSpec,
4357 health: HealthConfig,
4358 drain_timeout_ms: Option<u64>,
4361 reply: oneshot::Sender<()>,
4362 },
4363 Swap {
4364 requester: Option<Principal>,
4365 ready_timeout: Option<Duration>,
4368 reply: oneshot::Sender<Result<(), SuperviseError>>,
4370 },
4371}
4372
4373#[derive(Debug)]
4374pub enum SuperviseError {
4375 InvalidSpec {
4376 reason: String,
4377 },
4378 Spawn {
4379 program: PathBuf,
4380 source: io::Error,
4381 cgroup_path: Option<PathBuf>,
4382 },
4383 Cgroup {
4384 module_id: String,
4385 source: io::Error,
4386 },
4387 LaunchNonce {
4390 reason: String,
4391 },
4392 Wait {
4393 module_id: String,
4394 source: io::Error,
4395 },
4396 Kill {
4397 module_id: String,
4398 source: io::Error,
4399 },
4400 Forwarding(ForwardingError),
4401 Registry(RegistryError),
4402 ReloadUnavailable {
4403 module_id: String,
4404 reason: String,
4405 },
4406 Disabled {
4411 module_id: String,
4412 },
4413 ReloadFailed {
4414 module_id: String,
4415 reason: String,
4416 },
4417 RegistrationStillActive {
4418 module_id: String,
4419 waited: Duration,
4420 },
4421 StatePoisoned {
4422 module_id: Option<String>,
4423 },
4424 CommandClosed {
4425 module_id: String,
4426 },
4427 SwapInProgress {
4431 module_id: String,
4432 },
4433 SwapRefused {
4435 module_id: String,
4436 reason: SwapRefusal,
4437 },
4438 SwapFailed {
4442 module_id: String,
4443 arm: SwapFailureArm,
4444 detail: String,
4445 candidate_exit: Option<ExitReport>,
4448 },
4449}
4450
4451#[derive(Debug, Clone, Copy, PartialEq, Eq)]
4453pub enum SwapRefusal {
4454 OverlapExclusive,
4456 NotRegistered,
4459 ProtocolNone,
4462 NotConfigured,
4465 AlreadySwapping,
4467}
4468
4469impl SwapRefusal {
4470 pub fn as_str(self) -> &'static str {
4471 match self {
4472 Self::OverlapExclusive => "overlap_exclusive",
4473 Self::NotRegistered => "not_registered",
4474 Self::ProtocolNone => "protocol_none",
4475 Self::NotConfigured => "not_configured",
4476 Self::AlreadySwapping => "already_swapping",
4477 }
4478 }
4479}
4480
4481#[derive(Debug, Clone, Copy, PartialEq, Eq)]
4484pub enum SwapFailureArm {
4485 SpawnFailed,
4487 NeverRegistered,
4489 NeverReady,
4491 CandidateExited,
4493 CandidateUnhealthy,
4495 Interrupted,
4499 CutoverLost,
4504}
4505
4506impl SwapFailureArm {
4507 pub fn as_str(self) -> &'static str {
4508 match self {
4509 Self::SpawnFailed => "spawn_failed",
4510 Self::NeverRegistered => "never_registered",
4511 Self::NeverReady => "never_ready",
4512 Self::CandidateExited => "candidate_exited",
4513 Self::CandidateUnhealthy => "candidate_unhealthy",
4514 Self::Interrupted => "interrupted",
4515 Self::CutoverLost => "cutover_lost",
4516 }
4517 }
4518}
4519
4520impl fmt::Display for SuperviseError {
4521 fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
4522 match self {
4523 Self::InvalidSpec { reason } => write!(f, "invalid module spec: {reason}"),
4524 Self::Spawn {
4525 program,
4526 source,
4527 cgroup_path: Some(cgroup_path),
4528 } => write!(
4529 f,
4530 "failed to place module in cgroup '{}' while spawning '{}': {source}",
4531 cgroup_path.display(),
4532 program.display()
4533 ),
4534 Self::Spawn {
4535 program,
4536 source,
4537 cgroup_path: None,
4538 } => write!(
4539 f,
4540 "failed to spawn module '{}': {source}",
4541 program.display()
4542 ),
4543 Self::Cgroup { module_id, source } => {
4544 write!(
4545 f,
4546 "failed to prepare cgroup for module '{module_id}': {source}"
4547 )
4548 }
4549 Self::LaunchNonce { reason } => {
4550 write!(
4551 f,
4552 "failed to generate reserved-module launch nonce: {reason}"
4553 )
4554 }
4555 Self::Wait { module_id, source } => {
4556 write!(f, "failed to wait for module '{module_id}': {source}")
4557 }
4558 Self::Kill { module_id, source } => {
4559 write!(f, "failed to kill module '{module_id}': {source}")
4560 }
4561 Self::Forwarding(err) => write!(f, "forwarding error: {err}"),
4562 Self::Registry(err) => write!(f, "registry error: {err}"),
4563 Self::ReloadUnavailable { module_id, reason } => {
4564 write!(f, "reload unavailable for module '{module_id}': {reason}")
4565 }
4566 Self::Disabled { module_id } => {
4567 write!(
4568 f,
4569 "module '{module_id}' is disabled; enable it before restart or reload"
4570 )
4571 }
4572 Self::ReloadFailed { module_id, reason } => {
4573 write!(f, "reload failed for module '{module_id}': {reason}")
4574 }
4575 Self::RegistrationStillActive { module_id, waited } => write!(
4576 f,
4577 "module '{module_id}' registration remained active after waiting {waited:?}"
4578 ),
4579 Self::StatePoisoned { module_id } => match module_id {
4580 Some(module_id) => {
4581 write!(f, "supervisor state for module '{module_id}' was poisoned")
4582 }
4583 None => write!(f, "supervisor state was poisoned"),
4584 },
4585 Self::CommandClosed { module_id } => {
4586 write!(
4587 f,
4588 "supervisor command channel for module '{module_id}' is closed"
4589 )
4590 }
4591 Self::SwapInProgress { module_id } => write!(
4592 f,
4593 "module '{module_id}' is being swapped; retry once the swap has cut over or failed, or stop the module to abort the swap"
4594 ),
4595 Self::SwapRefused { module_id, reason } => match reason {
4596 SwapRefusal::OverlapExclusive => write!(
4597 f,
4598 "module '{module_id}' is declared overlap: \"exclusive\" (the default): two processes of it must not run at once, so it cannot be swapped; use a plain restart, or declare overlap: \"safe\" in its config if it really tolerates a second process"
4599 ),
4600 SwapRefusal::NotRegistered => write!(
4601 f,
4602 "module '{module_id}' is not registered, so there is no serving process to keep while a replacement warms; use a plain restart"
4603 ),
4604 SwapRefusal::ProtocolNone => write!(
4605 f,
4606 "module '{module_id}' is protocol: \"none\" and never registers, so a swap could never see its replacement become ready; use a plain restart"
4607 ),
4608 SwapRefusal::NotConfigured => write!(
4609 f,
4610 "module '{module_id}' cannot be swapped: the supervisor was built without the forwarding table or shared handle a swap needs"
4611 ),
4612 SwapRefusal::AlreadySwapping => {
4613 write!(f, "module '{module_id}' is already being swapped")
4614 }
4615 },
4616 Self::SwapFailed {
4617 module_id,
4618 arm,
4619 detail,
4620 ..
4621 } => write!(
4622 f,
4623 "swap of module '{module_id}' failed ({}): {detail}; the running process was left serving",
4624 arm.as_str()
4625 ),
4626 }
4627 }
4628}
4629
4630impl Error for SuperviseError {
4631 fn source(&self) -> Option<&(dyn Error + 'static)> {
4632 match self {
4633 Self::Spawn { source, .. }
4634 | Self::Cgroup { source, .. }
4635 | Self::Wait { source, .. }
4636 | Self::Kill { source, .. } => Some(source),
4637 Self::Forwarding(err) => Some(err),
4638 Self::Registry(err) => Some(err),
4639 Self::LaunchNonce { .. }
4640 | Self::InvalidSpec { .. }
4641 | Self::ReloadUnavailable { .. }
4642 | Self::Disabled { .. }
4643 | Self::ReloadFailed { .. }
4644 | Self::RegistrationStillActive { .. }
4645 | Self::StatePoisoned { .. }
4646 | Self::CommandClosed { .. }
4647 | Self::SwapInProgress { .. }
4648 | Self::SwapRefused { .. }
4649 | Self::SwapFailed { .. } => None,
4650 }
4651 }
4652}
4653
4654pub(crate) fn validate_spec(spec: &ModuleSpec) -> Result<(), SuperviseError> {
4655 if spec.module_id.trim().is_empty() {
4656 return Err(SuperviseError::InvalidSpec {
4657 reason: "module_id must not be empty".to_string(),
4658 });
4659 }
4660
4661 Ok(())
4662}
4663
4664#[derive(Debug, Default)]
4665struct HealthProbeRuntime {
4666 spawn_generation: Option<u64>,
4667 configured_health: Option<HealthConfig>,
4668 registered_connection: Option<crate::ConnectionId>,
4669 advertised: bool,
4670 next_probe_at: Option<Instant>,
4671 probe_index: u64,
4672}
4673
4674fn running_protocol(spec: &ModuleSpec, snapshot: &SharedSnapshot) -> ModuleProtocol {
4675 lock_snapshot(snapshot)
4676 .ok()
4677 .and_then(|state| state.spawned_protocol)
4678 .unwrap_or(spec.protocol)
4679}
4680
4681impl HealthProbeRuntime {
4682 fn refresh_registration(
4683 &mut self,
4684 spec: &ModuleSpec,
4685 runtime: &SupervisorRuntimeConfig,
4686 registry: &Registry,
4687 snapshot: &SharedSnapshot,
4688 ) {
4689 let generation = lock_snapshot(snapshot)
4690 .ok()
4691 .map(|state| state.spawn_generation);
4692 if self.spawn_generation != generation {
4693 *self = Self {
4697 spawn_generation: generation,
4698 ..Self::default()
4699 };
4700 }
4701 if self.configured_health.as_ref() != Some(&runtime.health) {
4702 self.configured_health = Some(runtime.health.clone());
4703 self.next_probe_at = None;
4704 self.registered_connection = None;
4705 self.probe_index = 0;
4706 }
4707 if running_protocol(spec, snapshot) == ModuleProtocol::None {
4711 self.registered_connection = None;
4712 self.advertised = runtime.health.http.is_some();
4713 if !self.advertised {
4714 self.next_probe_at = None;
4715 let _ = update_snapshot(snapshot, Some(&spec.module_id), |state| {
4716 let unknown = ModuleHealthStatus::default();
4717 if state.health != unknown {
4718 state.health = unknown;
4719 }
4720 });
4721 } else if self.next_probe_at.is_none() {
4722 self.next_probe_at = Some(
4723 Instant::now()
4724 + jittered_health_delay(&spec.module_id, 0, runtime.health.cadence),
4725 );
4726 }
4727 return;
4728 }
4729
4730 let registration = match registry.get_module(&spec.module_id) {
4731 Ok(registration) => registration,
4732 Err(err) => {
4733 warn!(module_id = %spec.module_id, error = %err, "health prober could not read registry");
4734 self.advertised = false;
4735 self.next_probe_at = None;
4736 return;
4737 }
4738 };
4739
4740 let Some(registration) = registration else {
4741 self.registered_connection = None;
4742 self.advertised = false;
4743 self.next_probe_at = None;
4744 return;
4745 };
4746
4747 let advertised = registration
4748 .control_ops
4749 .iter()
4750 .any(|op| op == MODULE_CONTROL_OP_HEALTH_CHECK);
4751 if !advertised {
4752 self.registered_connection = Some(registration.connection_id);
4753 self.advertised = false;
4754 self.next_probe_at = None;
4755 let _ = update_snapshot(snapshot, Some(&spec.module_id), |state| {
4756 if state.health.status != SupervisorHealthStatus::Unknown
4757 || state.health.consecutive_failures != 0
4758 || state.health.last_probe_ms.is_some()
4759 || state.health.detail.is_some()
4760 || state.health.metrics.is_some()
4761 {
4762 state.health.status = SupervisorHealthStatus::Unknown;
4763 state.health.consecutive_failures = 0;
4764 state.health.last_probe_ms = None;
4765 state.health.detail = None;
4766 state.health.metrics = None;
4767 }
4768 });
4769 return;
4770 }
4771
4772 let reregistered = self.registered_connection != Some(registration.connection_id);
4773 self.registered_connection = Some(registration.connection_id);
4774 self.advertised = true;
4775 if reregistered || self.next_probe_at.is_none() {
4776 self.probe_index = 0;
4777 self.next_probe_at = Some(
4778 Instant::now() + jittered_health_delay(&spec.module_id, 0, runtime.health.cadence),
4779 );
4780 let _ = update_snapshot(snapshot, Some(&spec.module_id), |state| {
4781 state.health.status = SupervisorHealthStatus::Unknown;
4782 state.health.consecutive_failures = 0;
4783 state.health.detail = None;
4784 state.health.metrics = None;
4785 });
4786 }
4787 }
4788
4789 fn wake_at(&self) -> Option<Instant> {
4790 if !self.advertised {
4791 return None;
4792 }
4793 self.next_probe_at
4794 }
4795
4796 fn due(&self) -> bool {
4797 self.advertised
4798 && self
4799 .next_probe_at
4800 .is_some_and(|next| Instant::now() >= next)
4801 }
4802
4803 fn schedule_next(&mut self, spec: &ModuleSpec, cadence: Duration) {
4804 self.probe_index = self.probe_index.wrapping_add(1);
4805 self.next_probe_at = Some(
4806 Instant::now() + jittered_health_delay(&spec.module_id, self.probe_index, cadence),
4807 );
4808 }
4809}
4810
4811#[derive(Debug)]
4846enum HealthProbeEvidence {
4847 LaneDead,
4849 NoAnswer,
4851 BadAnswer,
4853 Misconfigured,
4855}
4856
4857#[derive(Debug)]
4858struct HealthProbeError {
4859 evidence: HealthProbeEvidence,
4860 message: String,
4861}
4862
4863impl HealthProbeError {
4864 fn lane_dead(message: impl Into<String>) -> Self {
4865 Self::with(HealthProbeEvidence::LaneDead, message)
4866 }
4867
4868 fn no_answer(message: impl Into<String>) -> Self {
4869 Self::with(HealthProbeEvidence::NoAnswer, message)
4870 }
4871
4872 fn bad_answer(message: impl Into<String>) -> Self {
4873 Self::with(HealthProbeEvidence::BadAnswer, message)
4874 }
4875
4876 fn misconfigured(message: impl Into<String>) -> Self {
4877 Self::with(HealthProbeEvidence::Misconfigured, message)
4878 }
4879
4880 fn with(evidence: HealthProbeEvidence, message: impl Into<String>) -> Self {
4881 Self {
4882 evidence,
4883 message: message.into(),
4884 }
4885 }
4886
4887 #[allow(dead_code)]
4901 fn is_proof_of_death(&self) -> bool {
4902 matches!(self.evidence, HealthProbeEvidence::LaneDead)
4903 }
4904
4905 fn label(&self) -> &'static str {
4913 match self.evidence {
4914 HealthProbeEvidence::LaneDead => "lane-dead",
4915 HealthProbeEvidence::NoAnswer => "no-answer",
4916 HealthProbeEvidence::BadAnswer => "bad-answer",
4917 HealthProbeEvidence::Misconfigured => "daemon-misconfigured",
4918 }
4919 }
4920}
4921
4922impl fmt::Display for HealthProbeError {
4923 fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
4924 f.write_str(&self.message)
4925 }
4926}
4927
4928async fn run_health_probe_cycle(
4929 spec: &ModuleSpec,
4930 runtime: &SupervisorRuntimeConfig,
4931 registry: &Registry,
4932 process_liveness: &SupervisorProcessLiveness,
4933 snapshot: &SharedSnapshot,
4934 child: &mut Option<SupervisedChild>,
4935) {
4936 let now_ms = unix_ms_now();
4937 let http = (running_protocol(spec, snapshot) == ModuleProtocol::None)
4938 .then_some(runtime.health.http.as_deref())
4939 .flatten();
4940 let result = match http {
4941 Some(url) => probe_http_health(url, runtime.health.deadline).await,
4942 None => probe_module_health(&spec.module_id, runtime, None).await,
4943 };
4944 match result {
4945 Ok(report) => {
4946 handle_health_report(
4947 spec,
4948 runtime,
4949 registry,
4950 process_liveness,
4951 snapshot,
4952 child,
4953 report,
4954 now_ms,
4955 )
4956 .await;
4957 }
4958 Err(err) => {
4959 if http.is_some() {
4960 let _ = update_snapshot(snapshot, Some(&spec.module_id), |state| {
4961 state.health.status = SupervisorHealthStatus::Failing;
4962 });
4963 }
4964 handle_health_probe_failure(
4965 spec,
4966 runtime,
4967 registry,
4968 process_liveness,
4969 snapshot,
4970 child,
4971 err,
4972 now_ms,
4973 )
4974 .await;
4975 }
4976 }
4977}
4978
4979pub(crate) struct HttpProbeTarget<'a> {
4980 address: std::net::SocketAddr,
4981 localhost: bool,
4982 authority: &'a str,
4983 path: String,
4984}
4985
4986pub(crate) fn parse_http_probe_url(url: &str) -> Result<HttpProbeTarget<'_>, String> {
4989 if url.bytes().any(|b| b <= b' ' || b == 127) || url.contains('#') {
4990 return Err("must not contain whitespace, controls, or a fragment".into());
4991 }
4992 let rest = url
4993 .strip_prefix("http://")
4994 .ok_or("must use plain http://")?;
4995 let split = rest.find(['/', '?']).unwrap_or(rest.len());
4996 let (authority, suffix) = rest.split_at(split);
4997 let (host, port) = if let Some(rest) = authority.strip_prefix("[::1]") {
4998 ("::1", rest)
4999 } else {
5000 let split = authority.find(':').unwrap_or(authority.len());
5001 authority.split_at(split)
5002 };
5003 let ip: std::net::IpAddr = match host {
5004 "127.0.0.1" | "localhost" => std::net::Ipv4Addr::LOCALHOST.into(),
5005 "::1" => std::net::Ipv6Addr::LOCALHOST.into(),
5006 _ => return Err("host must be 127.0.0.1, [::1], or localhost".into()),
5007 };
5008 let port = if port.is_empty() {
5009 80
5010 } else {
5011 port.strip_prefix(':')
5012 .and_then(|p| p.parse::<u16>().ok())
5013 .filter(|p| *p > 0)
5014 .ok_or("must have a valid nonzero TCP port")?
5015 };
5016 let path = if suffix.is_empty() {
5017 "/".into()
5018 } else if suffix.starts_with('?') {
5019 format!("/{suffix}")
5020 } else {
5021 suffix.into()
5022 };
5023 Ok(HttpProbeTarget {
5024 address: std::net::SocketAddr::new(ip, port),
5025 localhost: host == "localhost",
5026 authority,
5027 path,
5028 })
5029}
5030
5031async fn probe_http_health(
5032 url: &str,
5033 deadline: Duration,
5034) -> Result<HealthReport, HealthProbeError> {
5035 use tokio::io::{AsyncReadExt, AsyncWriteExt, BufReader};
5036 let target = parse_http_probe_url(url).map_err(HealthProbeError::misconfigured)?;
5037 let mut response_status = String::new();
5040 let mut body = Vec::new();
5041 let probe = async {
5042 let connection = match tokio::net::TcpStream::connect(target.address).await {
5045 Err(_) if target.localhost => {
5046 tokio::net::TcpStream::connect((
5047 std::net::Ipv6Addr::LOCALHOST,
5048 target.address.port(),
5049 ))
5050 .await
5051 }
5052 result => result,
5053 };
5054 let mut stream = connection.map_err(|error| {
5055 HealthProbeError::no_answer(format!("HTTP connect failed: {error}"))
5056 })?;
5057 stream.write_all(format!("GET {} HTTP/1.1\r\nHost: {}\r\nAccept-Encoding: identity\r\nConnection: close\r\n\r\n", target.path, target.authority).as_bytes()).await
5058 .map_err(|error| HealthProbeError::no_answer(format!("HTTP write failed: {error}")))?;
5059 let mut reader = BufReader::new(stream);
5060 let mut budget = 16 * 1024;
5061 let status = http_line(&mut reader, &mut budget).await?;
5062 let mut words = status.split_ascii_whitespace();
5063 let version = words.next();
5064 let code = words
5065 .next()
5066 .filter(|word| word.len() == 3)
5067 .and_then(|word| word.parse::<u16>().ok());
5068 if !matches!(version, Some("HTTP/1.1" | "HTTP/1.0"))
5069 || !code.is_some_and(|code| (100..600).contains(&code))
5070 {
5071 return Err(HealthProbeError::bad_answer(format!(
5072 "invalid HTTP status: {status}"
5073 )));
5074 }
5075 let code = code.expect("validated status code");
5076 response_status = status.clone();
5077 let mut length = None;
5078 let mut chunked = false;
5079 loop {
5080 let line = http_line(&mut reader, &mut budget).await?;
5081 if line.is_empty() {
5082 break;
5083 }
5084 if let Some((name, value)) = line.split_once(':') {
5085 if name.eq_ignore_ascii_case("content-length") {
5086 length = Some(value.trim().parse::<u64>().map_err(|_| {
5087 HealthProbeError::bad_answer("invalid HTTP Content-Length")
5088 })?);
5089 } else if name.eq_ignore_ascii_case("transfer-encoding") {
5090 chunked = value.trim().eq_ignore_ascii_case("chunked");
5091 }
5092 }
5093 }
5094 if chunked {
5095 while body.len() < 200 {
5096 let line = http_line(&mut reader, &mut budget).await?;
5097 let size = u64::from_str_radix(line.split(';').next().unwrap_or("").trim(), 16)
5098 .map_err(|_| HealthProbeError::bad_answer("invalid HTTP chunk size"))?;
5099 if size == 0 {
5100 break;
5101 }
5102 let count = size.min((200 - body.len()) as u64) as usize;
5103 let start = body.len();
5104 (&mut reader)
5105 .take(count as u64)
5106 .read_to_end(&mut body)
5107 .await
5108 .map_err(|error| {
5109 HealthProbeError::no_answer(format!("HTTP body read failed: {error}"))
5110 })?;
5111 if body.len() - start != count {
5112 return Err(HealthProbeError::bad_answer("truncated HTTP chunk"));
5113 }
5114 if size > count as u64 || body.len() == 200 {
5115 break;
5116 }
5117 if !http_line(&mut reader, &mut budget).await?.is_empty() {
5118 return Err(HealthProbeError::bad_answer("invalid HTTP chunk delimiter"));
5119 }
5120 }
5121 } else {
5122 reader
5123 .take(length.unwrap_or(200).min(200))
5124 .read_to_end(&mut body)
5125 .await
5126 .map_err(|error| {
5127 HealthProbeError::no_answer(format!("HTTP body read failed: {error}"))
5128 })?;
5129 }
5130 if (200..300).contains(&code) {
5131 Ok(HealthReport::ok())
5132 } else {
5133 Err(HealthProbeError::bad_answer(
5134 "HTTP health endpoint returned non-2xx",
5135 ))
5136 }
5137 };
5138 let mut result = timeout(deadline, probe).await.unwrap_or_else(|_| {
5139 Err(HealthProbeError::no_answer(format!(
5140 "HTTP probe timed out after {deadline:?}"
5141 )))
5142 });
5143 if let Err(error) = &mut result {
5144 if !response_status.is_empty() {
5145 error.message = format!(
5146 "{}; {response_status}: {}",
5147 error.message,
5148 String::from_utf8_lossy(&body)
5149 );
5150 }
5151 }
5152 result
5153}
5154
5155async fn http_line(
5156 reader: &mut tokio::io::BufReader<tokio::net::TcpStream>,
5157 remaining: &mut usize,
5158) -> Result<String, HealthProbeError> {
5159 use tokio::io::{AsyncBufReadExt, AsyncReadExt};
5160 let mut line = Vec::new();
5161 (&mut *reader)
5162 .take(*remaining as u64)
5163 .read_until(b'\n', &mut line)
5164 .await
5165 .map_err(|error| {
5166 HealthProbeError::no_answer(format!("HTTP header read failed: {error}"))
5167 })?;
5168 *remaining -= line.len();
5169 if !line.ends_with(b"\r\n") {
5170 return Err(HealthProbeError::bad_answer(
5171 "HTTP headers are incomplete or exceed 16 KiB",
5172 ));
5173 }
5174 line.truncate(line.len() - 2);
5175 String::from_utf8(line).map_err(|_| HealthProbeError::bad_answer("HTTP header is not UTF-8"))
5176}
5177
5178async fn probe_module_health(
5179 module_id: &str,
5180 runtime: &SupervisorRuntimeConfig,
5181 drain_deadline: Option<Instant>,
5182) -> Result<HealthReport, HealthProbeError> {
5183 let Some(forwarding) = runtime.forwarding.as_ref() else {
5184 return Err(HealthProbeError::misconfigured(
5185 "supervisor was not configured with a forwarding table",
5186 ));
5187 };
5188 let probe_started_at = Instant::now();
5189 let mut deadline = probe_started_at + runtime.health.deadline;
5190 if let Some(drain_deadline) = drain_deadline {
5191 deadline = deadline.min(drain_deadline);
5192 }
5193 let pending = if drain_deadline.is_some() {
5194 forwarding.begin_drain_health_probe_rpc_for(
5195 module_id,
5196 MODULE_CONTROL_OP_HEALTH_CHECK,
5197 probe_started_at,
5198 deadline,
5199 )
5200 } else {
5201 forwarding.begin_health_probe_rpc_for(
5202 module_id,
5203 MODULE_CONTROL_OP_HEALTH_CHECK,
5204 probe_started_at,
5205 deadline,
5206 )
5207 }
5208 .map_err(|err| {
5209 HealthProbeError::lane_dead(format!("failed to begin health.check RPC: {err}"))
5212 })?;
5213 await_health_probe(forwarding, pending, deadline, runtime.health.deadline).await
5214}
5215
5216async fn probe_endpoint_health(
5223 endpoint: crate::ModuleEndpointId,
5224 runtime: &SupervisorRuntimeConfig,
5225 deadline_cap: Option<Instant>,
5226) -> Result<HealthReport, HealthProbeError> {
5227 let Some(forwarding) = runtime.forwarding.as_ref() else {
5228 return Err(HealthProbeError::misconfigured(
5229 "supervisor was not configured with a forwarding table",
5230 ));
5231 };
5232 let probe_started_at = Instant::now();
5233 let mut deadline = probe_started_at + runtime.health.deadline;
5234 if let Some(cap) = deadline_cap {
5235 deadline = deadline.min(cap);
5236 }
5237 let pending = forwarding
5238 .begin_endpoint_health_probe_rpc_for(
5239 endpoint,
5240 MODULE_CONTROL_OP_HEALTH_CHECK,
5241 probe_started_at,
5242 deadline,
5243 )
5244 .map_err(|err| {
5245 HealthProbeError::lane_dead(format!("failed to begin health.check RPC: {err}"))
5246 })?;
5247 await_health_probe(forwarding, pending, deadline, runtime.health.deadline).await
5248}
5249
5250async fn await_health_probe(
5252 forwarding: &ForwardingTable,
5253 pending: PendingModuleControlRpc,
5254 deadline: Instant,
5255 probe_budget: Duration,
5256) -> Result<HealthReport, HealthProbeError> {
5257 let PendingModuleControlRpc {
5258 endpoint,
5259 module_sink,
5260 negotiated_ver,
5261 corr,
5262 receiver,
5263 } = pending;
5264 let body = serde_json::to_vec(&ModuleControlRequest::HealthCheck {}).map_err(|err| {
5265 HealthProbeError::misconfigured(format!("failed to encode health.check: {err}"))
5266 })?;
5267 let frame = Frame::build_with_version(
5268 negotiated_ver,
5269 FrameType::Request,
5270 control_flags(),
5271 0,
5272 0,
5273 corr,
5274 body,
5275 )
5276 .map_err(|err| {
5277 HealthProbeError::misconfigured(format!("failed to build health.check frame: {err}"))
5278 })?;
5279
5280 match timeout_at(deadline, module_sink.send(frame)).await {
5286 Ok(Ok(())) => {}
5287 Ok(Err(err)) => {
5288 let _ = forwarding.cancel_module_control_rpc(endpoint, corr);
5289 return Err(HealthProbeError::lane_dead(format!(
5292 "failed to send health.check: {err}"
5293 )));
5294 }
5295 Err(_elapsed) => {
5296 let _ = forwarding.cancel_module_control_rpc(endpoint, corr);
5297 return Err(HealthProbeError::no_answer(
5301 "health.check send timed out before enqueue (module egress full)",
5302 ));
5303 }
5304 }
5305
5306 match timeout_at(deadline, receiver).await {
5307 Ok(Ok(ModuleControlRpcOutcome::Response(response))) => {
5311 response.health_report().ok_or_else(|| {
5312 HealthProbeError::bad_answer("health.check RPC returned a non-health response")
5313 })
5314 }
5315 Ok(Ok(ModuleControlRpcOutcome::Rejected(body))) => Err(HealthProbeError::bad_answer(
5316 format!("health.check rejected: {}", body.message),
5317 )),
5318 Ok(Ok(ModuleControlRpcOutcome::ModuleGone(message))) => {
5319 Err(HealthProbeError::lane_dead(message))
5320 }
5321 Ok(Ok(ModuleControlRpcOutcome::MalformedResponse(message))) => {
5322 Err(HealthProbeError::bad_answer(message))
5323 }
5324 Ok(Ok(ModuleControlRpcOutcome::UnexpectedOp { expected, actual })) => {
5325 Err(HealthProbeError::bad_answer(format!(
5326 "expected module-control op '{expected}', got '{actual}'"
5327 )))
5328 }
5329 Ok(Ok(ModuleControlRpcOutcome::DeadlineElapsed)) => Err(HealthProbeError::bad_answer(
5333 "module answered health.check after its daemon deadline",
5334 )),
5335 Ok(Err(_)) => Err(HealthProbeError::misconfigured(
5336 "health.check waiter was canceled before the module responded",
5337 )),
5338 Err(_) => {
5339 let _ = forwarding.tombstone_health_probe_rpc(endpoint, corr);
5340 Err(HealthProbeError::no_answer(format!(
5341 "module did not answer health.check within {probe_budget:?}"
5342 )))
5343 }
5344 }
5345}
5346
5347#[allow(clippy::too_many_arguments)]
5348async fn handle_health_report(
5349 spec: &ModuleSpec,
5350 runtime: &SupervisorRuntimeConfig,
5351 registry: &Registry,
5352 process_liveness: &SupervisorProcessLiveness,
5353 snapshot: &SharedSnapshot,
5354 child: &mut Option<SupervisedChild>,
5355 report: HealthReport,
5356 now_ms: u64,
5357) {
5358 let status = supervisor_health_status(report.status);
5359 let detail = report.detail.clone();
5360 let metrics = truncate_health_metrics(report.metrics);
5361 let _ = update_snapshot(snapshot, Some(&spec.module_id), |state| {
5362 state.health.status = status;
5363 state.health.last_probe_ms = Some(now_ms);
5364 state.health.detail = detail.clone();
5365 state.health.metrics = metrics.clone();
5366 state.health.consecutive_failures = 0;
5367 state.failed_retry.observe_health(
5368 report.status == HealthStatus::Ok,
5369 runtime.restart_policy.window,
5370 );
5371 });
5372
5373 let action = match report.status {
5374 HealthStatus::Ok => return,
5375 HealthStatus::Degraded => runtime.health.on_degraded,
5376 HealthStatus::Failing => runtime.health.on_failing,
5377 };
5378 apply_l3_health_action(
5379 spec,
5380 runtime,
5381 registry,
5382 process_liveness,
5383 snapshot,
5384 child,
5385 status,
5386 detail.as_deref(),
5387 action,
5388 now_ms,
5389 )
5390 .await;
5391}
5392
5393#[allow(clippy::too_many_arguments)]
5394async fn handle_health_probe_failure(
5395 spec: &ModuleSpec,
5396 runtime: &SupervisorRuntimeConfig,
5397 registry: &Registry,
5398 process_liveness: &SupervisorProcessLiveness,
5399 snapshot: &SharedSnapshot,
5400 child: &mut Option<SupervisedChild>,
5401 err: HealthProbeError,
5402 now_ms: u64,
5403) {
5404 let threshold = runtime.health.failure_threshold.max(1);
5405 let mut failures = 0;
5406 let detail = format!("[{}] {err}", err.label());
5411 let _ = update_snapshot(snapshot, Some(&spec.module_id), |state| {
5412 state
5413 .failed_retry
5414 .observe_health(false, runtime.restart_policy.window);
5415 if state.spawned_protocol.unwrap_or(spec.protocol) == ModuleProtocol::Subc {
5418 state.health.status = SupervisorHealthStatus::Unknown;
5419 }
5420 state.health.last_probe_ms = Some(now_ms);
5421 state.health.consecutive_failures = state.health.consecutive_failures.saturating_add(1);
5422 state.health.detail = Some(detail.clone());
5423 state.health.metrics = None;
5424 failures = state.health.consecutive_failures;
5425 });
5426
5427 if failures < threshold {
5428 warn!(
5429 module_id = %spec.module_id,
5430 consecutive_failures = failures,
5431 threshold,
5432 evidence = err.label(),
5433 detail = %detail,
5434 "health.check probe failed"
5435 );
5436 return;
5437 }
5438
5439 let _ = update_snapshot(snapshot, Some(&spec.module_id), |state| {
5440 state.state = ModuleState::Unresponsive;
5441 state.health.status = SupervisorHealthStatus::Unresponsive;
5442 });
5443 if runtime.health.critical {
5447 error!(
5448 module_id = %spec.module_id,
5449 status = "unresponsive",
5450 evidence = err.label(),
5451 detail = %detail,
5452 "critical module health alert"
5453 );
5454 } else {
5455 warn!(
5456 module_id = %spec.module_id,
5457 status = "unresponsive",
5458 evidence = err.label(),
5459 detail = %detail,
5460 "module health threshold breached"
5461 );
5462 }
5463 if let Err(err) = health_restart_child(
5464 spec,
5465 runtime,
5466 registry,
5467 process_liveness,
5468 snapshot,
5469 child,
5470 SupervisorHealthStatus::Unresponsive,
5471 Some(&detail),
5472 now_ms,
5473 )
5474 .await
5475 {
5476 error!(module_id = %spec.module_id, error = %err, "health-triggered restart failed");
5477 }
5478}
5479
5480#[allow(clippy::too_many_arguments)]
5481async fn apply_l3_health_action(
5482 spec: &ModuleSpec,
5483 runtime: &SupervisorRuntimeConfig,
5484 registry: &Registry,
5485 process_liveness: &SupervisorProcessLiveness,
5486 snapshot: &SharedSnapshot,
5487 child: &mut Option<SupervisedChild>,
5488 status: SupervisorHealthStatus,
5489 detail: Option<&str>,
5490 action: HealthAction,
5491 now_ms: u64,
5492) {
5493 record_health_action(snapshot, &spec.module_id, action.to_string(), now_ms);
5494 match action {
5495 HealthAction::Report => {
5496 info!(
5497 module_id = %spec.module_id,
5498 status = ?status,
5499 detail,
5500 "module reported non-ok health"
5501 );
5502 }
5503 HealthAction::Alert => {
5504 error!(
5505 module_id = %spec.module_id,
5506 status = ?status,
5507 detail,
5508 "module health alert"
5509 );
5510 }
5511 HealthAction::Restart => {
5512 if let Err(err) = health_restart_child(
5513 spec,
5514 runtime,
5515 registry,
5516 process_liveness,
5517 snapshot,
5518 child,
5519 status,
5520 detail,
5521 now_ms,
5522 )
5523 .await
5524 {
5525 error!(module_id = %spec.module_id, error = %err, "health-triggered restart failed");
5526 }
5527 }
5528 }
5529}
5530
5531#[allow(clippy::too_many_arguments)]
5532async fn health_restart_child(
5533 spec: &ModuleSpec,
5534 runtime: &SupervisorRuntimeConfig,
5535 registry: &Registry,
5536 process_liveness: &SupervisorProcessLiveness,
5537 snapshot: &SharedSnapshot,
5538 child: &mut Option<SupervisedChild>,
5539 status: SupervisorHealthStatus,
5540 detail: Option<&str>,
5541 now_ms: u64,
5542) -> Result<(), SuperviseError> {
5543 let (enabled, schedule) = {
5544 let mut state = lock_snapshot(snapshot)?;
5545 let enabled = state.enabled;
5546 let schedule = if enabled {
5547 state.next_crash_restart(&runtime.restart_policy, Instant::now())
5548 } else {
5549 None
5550 };
5551 (enabled, schedule)
5552 };
5553
5554 if !enabled {
5555 return Err(SuperviseError::Disabled {
5556 module_id: spec.module_id.clone(),
5557 });
5558 }
5559
5560 if schedule.is_none() {
5561 record_health_action(snapshot, &spec.module_id, "failed".to_string(), now_ms);
5562 error!(
5563 module_id = %spec.module_id,
5564 status = ?status,
5565 detail,
5566 max_restarts = runtime.restart_policy.max_restarts,
5567 window_secs = runtime.restart_policy.window.as_secs(),
5568 reason = %runtime.restart_policy.budget_exhausted_detail(),
5569 "health restart budget exhausted; marking module failed"
5570 );
5571 let stop_notice = begin_forwarding_drain_if_configured(
5572 spec,
5573 runtime,
5574 registry,
5575 snapshot,
5576 Some(true),
5577 RouteCloseReason::Disable,
5578 )
5579 .await?;
5580 update_snapshot(snapshot, Some(&spec.module_id), |state| {
5581 state.drain_disposition_detail = Some(runtime.restart_policy.budget_exhausted_detail());
5582 })?;
5583 let result = drain_optional_child(
5584 &spec.module_id,
5585 spec.protocol,
5586 stop_notice,
5587 registry,
5588 runtime.forwarding.as_deref(),
5589 snapshot,
5590 &runtime.terminal_ring,
5591 &runtime.spawn_events,
5592 child,
5593 runtime.drain_timeout,
5594 ModuleState::Failed,
5595 Some(true),
5596 &runtime.failure_context(true),
5597 )
5598 .await;
5599 update_snapshot(snapshot, Some(&spec.module_id), |state| {
5600 if state.state == ModuleState::Failed && !runtime.child_roster.is_closed() {
5601 state.failed_retry.schedule(&spec.module_id);
5602 }
5603 })?;
5604 result?;
5605 process_liveness.untrack_if_current(&spec.module_id, snapshot);
5606 return Ok(());
5607 }
5608
5609 let schedule = schedule.expect("a health restart must have a crash-restart schedule");
5610 let mut restart_count = 0;
5611 update_snapshot(snapshot, Some(&spec.module_id), |state| {
5612 restart_count = state.crash_restarts.len();
5613 state.state = ModuleState::Unresponsive;
5614 state.health.status = status;
5615 state.health.last_action = Some(HealthAction::Restart.to_string());
5616 state.health.last_action_ms = Some(now_ms);
5617 })?;
5618 warn!(
5619 module_id = %spec.module_id,
5620 status = ?status,
5621 detail,
5622 restart_count,
5623 restart_in_window = schedule.restart_in_window,
5624 delay_ms = schedule.delay.as_millis() as u64,
5625 "health-triggered module restart"
5626 );
5627
5628 let stop_notice = begin_forwarding_drain_if_configured(
5629 spec,
5630 runtime,
5631 registry,
5632 snapshot,
5633 Some(true),
5634 RouteCloseReason::Restart,
5635 )
5636 .await?;
5637 drain_optional_child(
5638 &spec.module_id,
5639 spec.protocol,
5640 stop_notice,
5641 registry,
5642 runtime.forwarding.as_deref(),
5643 snapshot,
5644 &runtime.terminal_ring,
5645 &runtime.spawn_events,
5646 child,
5647 runtime.drain_timeout,
5648 ModuleState::Restarting,
5649 Some(true),
5650 &runtime.failure_context(true),
5651 )
5652 .await?;
5653 schedule_respawn(
5654 runtime,
5655 snapshot,
5656 &spec.module_id,
5657 schedule.delay,
5658 RespawnKind::Spawn,
5659 true,
5660 )
5661}
5662
5663fn cancel_deferred_reload(runtime: &SupervisorRuntimeConfig, module_id: &str, reason: &str) {
5664 if let Some(reply) = runtime
5665 .deferred_reload_reply
5666 .lock()
5667 .unwrap_or_else(|p| p.into_inner())
5668 .take()
5669 {
5670 let _ = reply.send(Err(SuperviseError::ReloadFailed {
5671 module_id: module_id.to_string(),
5672 reason: reason.to_string(),
5673 }));
5674 }
5675}
5676
5677fn schedule_respawn(
5678 runtime: &SupervisorRuntimeConfig,
5679 snapshot: &SharedSnapshot,
5680 module_id: &str,
5681 delay: Duration,
5682 kind: RespawnKind,
5683 daemon_initiated: bool,
5684) -> Result<(), SuperviseError> {
5685 cancel_deferred_reload(runtime, module_id, "respawn superseded by another restart");
5686 update_snapshot(snapshot, Some(module_id), |state| {
5687 state.respawn_pending = true
5688 })?;
5689 *runtime
5690 .scheduled_respawn
5691 .lock()
5692 .unwrap_or_else(|p| p.into_inner()) = Some(PendingRespawn {
5693 deadline: Instant::now() + delay,
5694 kind,
5695 daemon_initiated,
5696 });
5697 Ok(())
5698}
5699
5700fn record_health_action(snapshot: &SharedSnapshot, module_id: &str, action: String, now_ms: u64) {
5701 let _ = update_snapshot(snapshot, Some(module_id), |state| {
5702 state.health.last_action = Some(action);
5703 state.health.last_action_ms = Some(now_ms);
5704 });
5705}
5706
5707fn supervisor_health_status(status: HealthStatus) -> SupervisorHealthStatus {
5708 match status {
5709 HealthStatus::Ok => SupervisorHealthStatus::Ok,
5710 HealthStatus::Degraded => SupervisorHealthStatus::Degraded,
5711 HealthStatus::Failing => SupervisorHealthStatus::Failing,
5712 }
5713}
5714
5715fn truncate_health_metrics(metrics: Option<Value>) -> Option<Value> {
5727 let metrics = metrics?;
5728 match serde_json::to_vec(&metrics) {
5729 Ok(encoded) if encoded.len() > MAX_HEALTH_METRICS_BYTES => Some(serde_json::json!({
5730 "truncated": true,
5731 "original_bytes": encoded.len(),
5732 })),
5733 Ok(_) | Err(_) => Some(metrics),
5734 }
5735}
5736
5737fn jittered_health_delay(module_id: &str, probe_index: u64, cadence: Duration) -> Duration {
5743 if cadence.is_zero() {
5744 return Duration::ZERO;
5745 }
5746 let cadence_ms = cadence.as_millis() as u64;
5747 if cadence_ms == 0 {
5763 return cadence;
5764 }
5765 let jitter_span = (cadence_ms / 10).max(1);
5780 let hash = module_id.as_bytes().iter().fold(
5781 probe_index.wrapping_mul(0x9E37_79B9_7F4A_7C15),
5782 |acc, byte| {
5783 acc.wrapping_mul(1099511628211)
5784 .wrapping_add(u64::from(*byte))
5785 },
5786 );
5787 cadence + Duration::from_millis(hash % jitter_span)
5788}
5789
5790#[cfg(test)]
5791mod tests {
5792 use super::*;
5793
5794 #[test]
5795 fn readding_a_module_clears_its_rescan_removal_tombstone() {
5796 let handle = SupervisorHandle::new();
5797 let module_id = "readded-tombstone";
5798 handle.record_rescan_removal(module_id);
5799 assert!(handle.removal_tombstone_age_ms(module_id).is_some());
5800
5801 handle.apply_identity_configuration(&ModuleSpec {
5802 module_id: module_id.to_string(),
5803 program: PathBuf::from("/test/module"),
5804 args: Vec::new(),
5805 env: Vec::new(),
5806 reserved: false,
5807 reserved_prefixes: Vec::new(),
5808 protocol: ModuleProtocol::Subc,
5809 overlap: Default::default(),
5810 });
5811
5812 assert!(
5813 handle.removal_tombstone_age_ms(module_id).is_none(),
5814 "a re-added module must not retain a stale removal tombstone"
5815 );
5816 }
5817
5818 #[derive(Debug, PartialEq, Eq)]
5821 struct OwnerInSpawnWindow {
5822 module_id: String,
5823 configured: bool,
5824 on_roster: bool,
5825 admission_refusal: Option<&'static str>,
5826 }
5827
5828 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
5839 async fn a_new_module_is_configured_before_its_first_process_can_register() {
5840 use crate::scopes::ScopeTable;
5841 use subc_protocol::{error_codes, scope::ScopeSelector, Principal};
5842
5843 let dir = cortexkit_test_support::ScratchDir::new("configured-before-spawn");
5844 let stub = |module_id: &str, program: PathBuf| ModuleSpec {
5845 module_id: module_id.to_string(),
5846 program,
5847 args: Vec::new(),
5848 env: ["XDG_DATA_HOME", "XDG_RUNTIME_DIR", "XDG_CONFIG_HOME"]
5849 .into_iter()
5850 .map(|key| (key.to_string(), dir.path().display().to_string()))
5851 .chain([("FAKE_AFT_NEVER_CONNECT".to_string(), "1".to_string())])
5852 .collect(),
5853 reserved: false,
5854 reserved_prefixes: Vec::new(),
5855 protocol: ModuleProtocol::Subc,
5856 overlap: Default::default(),
5857 };
5858 let live = super::terminal_history_tests::fake_aft_stub_path();
5859 let missing = dir.path().join("definitely-missing-module");
5860
5861 let handle = SupervisorHandle::new();
5862 let mut supervisor =
5863 Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default())
5864 .with_handle(handle.clone());
5865 let observed = Arc::new(Mutex::new(Vec::<OwnerInSpawnWindow>::new()));
5866 let hook_handle = handle.clone();
5867 let hook_observed = Arc::clone(&observed);
5868 supervisor.test_after_first_spawn = AfterFirstSpawnHook(Some(Arc::new(move |module_id| {
5869 let configured = hook_handle.is_configured(module_id);
5872 let selector = ScopeSelector::new(
5873 Principal::Reserved {
5874 module_id: module_id.to_string(),
5875 },
5876 "s".to_string(),
5877 Some(1),
5878 );
5879 let carrier = Principal::Reserved {
5880 module_id: "carrier".to_string(),
5881 };
5882 let admission_refusal = match ScopeTable::new(Vec::<String>::new())
5883 .admit(&carrier, module_id, &selector, configured)
5884 {
5885 Ok(_) => None,
5886 Err(refusal) => Some(refusal.code),
5887 };
5888 hook_observed.lock().unwrap().push(OwnerInSpawnWindow {
5889 module_id: module_id.to_string(),
5890 configured,
5891 on_roster: hook_handle.get(module_id).is_some(),
5892 admission_refusal,
5893 });
5894 })));
5895
5896 let plain = supervisor.spawn(stub("plain", live.clone())).unwrap();
5897 let configured = supervisor
5898 .supervise_configured(stub("configured", live.clone()), true)
5899 .unwrap();
5900 let with_health = supervisor
5901 .supervise_configured_with_health(
5902 stub("with-health", live.clone()),
5903 true,
5904 HealthConfig::default(),
5905 None,
5906 RestartPolicy::default(),
5907 )
5908 .unwrap();
5909 let failed = supervisor
5912 .supervise_configured_with_health(
5913 stub("failed-spawn", missing.clone()),
5914 true,
5915 HealthConfig::default(),
5916 None,
5917 RestartPolicy::default(),
5918 )
5919 .unwrap();
5920 assert!(supervisor.spawn(stub("spawn-error", missing)).is_err());
5923
5924 let expected = [
5925 "plain",
5926 "configured",
5927 "with-health",
5928 "failed-spawn",
5929 "spawn-error",
5930 ]
5931 .into_iter()
5932 .map(|module_id| OwnerInSpawnWindow {
5933 module_id: module_id.to_string(),
5934 configured: true,
5935 on_roster: !matches!(module_id, "plain" | "spawn-error"),
5936 admission_refusal: Some(error_codes::SCOPE_NOT_SYNCED),
5937 })
5938 .collect::<Vec<_>>();
5939 assert_eq!(*observed.lock().unwrap(), expected);
5940
5941 for module_id in ["plain", "configured", "with-health", "failed-spawn"] {
5942 assert!(
5943 handle.get(module_id).is_some(),
5944 "{module_id} is on the roster"
5945 );
5946 assert!(
5947 handle.is_configured(module_id),
5948 "{module_id} stays configured"
5949 );
5950 }
5951 assert!(handle.get("spawn-error").is_none());
5952 assert!(
5953 !handle.is_configured("spawn-error"),
5954 "a plain spawn that failed must not leave its module marked configured"
5955 );
5956
5957 handle.retire("failed-spawn");
5959 assert!(!handle.is_configured("failed-spawn"));
5960
5961 for module in [plain, configured, with_health] {
5962 module.stop().await.unwrap();
5963 }
5964 drop(failed);
5965 }
5966
5967 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
5974 async fn a_configured_module_reads_starting_while_its_first_process_already_runs() {
5975 let dir = cortexkit_test_support::ScratchDir::new("starting-while-spawned");
5976 let spec = ModuleSpec {
5977 module_id: "spawned-not-admitted".to_string(),
5978 program: super::terminal_history_tests::fake_aft_stub_path(),
5979 args: Vec::new(),
5980 env: ["XDG_DATA_HOME", "XDG_RUNTIME_DIR", "XDG_CONFIG_HOME"]
5981 .into_iter()
5982 .map(|key| (key.to_string(), dir.path().display().to_string()))
5983 .chain([("FAKE_AFT_NEVER_CONNECT".to_string(), "1".to_string())])
5984 .collect(),
5985 reserved: false,
5986 reserved_prefixes: Vec::new(),
5987 protocol: ModuleProtocol::Subc,
5988 overlap: Default::default(),
5989 };
5990 let handle = SupervisorHandle::new();
5991 let mut supervisor =
5992 Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default())
5993 .with_handle(handle.clone());
5994 let observed = Arc::new(Mutex::new(None));
5995 let hook_handle = handle.clone();
5996 let hook_observed = Arc::clone(&observed);
5997 supervisor.test_after_first_spawn = AfterFirstSpawnHook(Some(Arc::new(move |module_id| {
5998 let state = hook_handle
5999 .get(module_id)
6000 .expect("a configured module is on the roster before it is spawned")
6001 .state()
6002 .unwrap();
6003 *hook_observed.lock().unwrap() = Some(state);
6004 })));
6005
6006 let module = supervisor.supervise_configured(spec, true).unwrap();
6007 assert_eq!(
6008 *observed.lock().unwrap(),
6009 Some(ModuleState::Starting),
6010 "between the spawn and its admission the supervisor still reports starting"
6011 );
6012 assert_eq!(module.state().unwrap(), ModuleState::Running);
6013 module.stop().await.unwrap();
6014 }
6015
6016 fn stale_process_snapshot(state: ModuleState, enabled: bool) -> SharedSnapshot {
6017 let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::new(state, enabled)));
6018 update_snapshot(&snapshot, Some("stale-process-facts"), |snapshot| {
6019 snapshot.process_alive = true;
6020 snapshot.pid = Some(41);
6021 snapshot.spawned_at_ms = Some(42);
6022 snapshot.spawned_from = Some(PathBuf::from("/spawned/module"));
6023 snapshot.launch_spec_digest = Some(LaunchSpecDigest([0xA5; 32]));
6024 snapshot.spawned_file_identity = Some(SpawnedFileIdentity {
6025 device: 43,
6026 inode: 44,
6027 #[cfg(windows)]
6028 windows_image: None,
6029 });
6030 })
6031 .unwrap();
6032 snapshot
6033 }
6034
6035 fn assert_snapshot_process_facts_cleared(snapshot: &SharedSnapshot) {
6036 let snapshot = lock_snapshot(snapshot).unwrap();
6037 assert!(!snapshot.process_alive);
6038 assert_eq!(snapshot.pid, None);
6039 assert_eq!(snapshot.spawned_at_ms, None);
6040 assert_eq!(snapshot.spawned_from, None);
6041 assert_eq!(snapshot.launch_spec_digest, None);
6042 assert_eq!(snapshot.spawned_file_identity, None);
6043 }
6044
6045 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
6046 async fn failed_enable_spawn_clears_preexisting_current_process_facts() {
6047 let supervisor =
6048 Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default());
6049 let mut runtime = supervisor.runtime_config();
6050 runtime.test_seed_stale_facts_before_enable_spawn = true;
6051 let snapshot = stale_process_snapshot(ModuleState::Disabled, false);
6052 let mut child = None;
6053 let spec = ModuleSpec {
6054 module_id: "failed-enable-clears-facts".to_string(),
6055 program: PathBuf::from("/definitely/missing/failed-enable-module"),
6056 args: Vec::new(),
6057 env: Vec::new(),
6058 reserved: false,
6059 reserved_prefixes: Vec::new(),
6060 protocol: ModuleProtocol::Subc,
6061 overlap: Default::default(),
6062 };
6063
6064 let result = set_child_enabled(
6065 &spec,
6066 &runtime,
6067 &supervisor.registry,
6068 &supervisor.process_liveness,
6069 &snapshot,
6070 &mut child,
6071 true,
6072 &runtime.failure_context(false),
6073 )
6074 .await;
6075
6076 assert!(matches!(result, Err(SuperviseError::Spawn { .. })));
6077 assert_eq!(lock_snapshot(&snapshot).unwrap().state, ModuleState::Failed);
6078 assert_snapshot_process_facts_cleared(&snapshot);
6079 }
6080
6081 #[tokio::test]
6082 async fn start_revives_stranded_restarting_but_not_pending_backoff() {
6083 let supervisor =
6084 Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default());
6085 let runtime = supervisor.runtime_config();
6086 let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::new(
6087 ModuleState::Restarting,
6088 true,
6089 )));
6090 let spec = ModuleSpec {
6091 module_id: "start-stranded-restarting".to_string(),
6092 program: super::terminal_history_tests::fake_aft_stub_path(),
6093 args: Vec::new(),
6094 env: vec![("FAKE_AFT_NEVER_CONNECT".to_string(), "1".to_string())],
6095 reserved: false,
6096 reserved_prefixes: Vec::new(),
6097 protocol: ModuleProtocol::None,
6098 overlap: Default::default(),
6099 };
6100 let mut child = None;
6101 lock_snapshot(&snapshot).unwrap().respawn_pending = true;
6102 assert!(!super::set_child_enabled(
6103 &spec,
6104 &runtime,
6105 &Registry::default(),
6106 &supervisor.process_liveness,
6107 &snapshot,
6108 &mut child,
6109 true,
6110 &runtime.failure_context(false),
6111 )
6112 .await
6113 .unwrap());
6114 assert!(child.is_none());
6115 lock_snapshot(&snapshot).unwrap().respawn_pending = false;
6116 assert!(super::set_child_enabled(
6117 &spec,
6118 &runtime,
6119 &Registry::default(),
6120 &supervisor.process_liveness,
6121 &snapshot,
6122 &mut child,
6123 true,
6124 &runtime.failure_context(false),
6125 )
6126 .await
6127 .unwrap());
6128 assert_eq!(
6129 lock_snapshot(&snapshot).unwrap().state,
6130 ModuleState::Running
6131 );
6132 let mut child = child.unwrap();
6133 child.start_kill().unwrap();
6134 child.wait().await.unwrap();
6135 }
6136
6137 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
6138 async fn failed_reload_spawn_clears_current_process_facts() {
6139 let supervisor =
6140 Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default());
6141 let mut runtime = supervisor.runtime_config();
6142 runtime.restart_policy = RestartPolicy::new(0, Duration::ZERO);
6143 let snapshot = stale_process_snapshot(ModuleState::Running, true);
6144 let mut child = None;
6145 let spec = ModuleSpec {
6146 module_id: "failed-reload-clears-facts".to_string(),
6147 program: PathBuf::from("/unused/failed-reload-module"),
6148 args: Vec::new(),
6149 env: Vec::new(),
6150 reserved: false,
6151 reserved_prefixes: Vec::new(),
6152 protocol: ModuleProtocol::Subc,
6153 overlap: Default::default(),
6154 };
6155
6156 let result = handle_reload_spawn_failure(
6157 &spec,
6158 &runtime,
6159 &supervisor.process_liveness,
6160 &snapshot,
6161 &mut child,
6162 "forced reload spawn failure".to_string(),
6163 )
6164 .await;
6165
6166 assert!(matches!(result, Err(SuperviseError::ReloadFailed { .. })));
6167 assert_eq!(lock_snapshot(&snapshot).unwrap().state, ModuleState::Failed);
6168 assert_snapshot_process_facts_cleared(&snapshot);
6169 }
6170
6171 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
6172 async fn dropping_a_module_with_an_active_monitor_clears_current_process_facts() {
6173 let supervisor =
6174 Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default());
6175 let snapshot = stale_process_snapshot(ModuleState::Running, true);
6176 let module = supervisor.supervised_module(
6177 ModuleSpec {
6178 module_id: "drop-clears-facts".to_string(),
6179 program: PathBuf::from("/unused/drop-module"),
6180 args: Vec::new(),
6181 env: Vec::new(),
6182 reserved: false,
6183 reserved_prefixes: Vec::new(),
6184 protocol: ModuleProtocol::Subc,
6185 overlap: Default::default(),
6186 },
6187 supervisor.runtime_config(),
6188 Arc::clone(&snapshot),
6189 None,
6190 );
6191 assert!(!module
6192 .inner
6193 .monitor
6194 .lock()
6195 .unwrap()
6196 .as_ref()
6197 .unwrap()
6198 .is_finished());
6199
6200 drop(module);
6201
6202 assert_eq!(
6203 lock_snapshot(&snapshot).unwrap().state,
6204 ModuleState::Stopped
6205 );
6206 assert_snapshot_process_facts_cleared(&snapshot);
6207 }
6208
6209 #[cfg(unix)]
6210 #[tokio::test]
6211 async fn rescan_preserves_running_protocol_until_respawn() {
6212 let dir = cortexkit_test_support::ScratchDir::new("rescan-protocol");
6213 let initial = ModuleSpec {
6214 module_id: "rescan-protocol".into(),
6215 program: PathBuf::from("/bin/sleep"),
6216 args: vec!["60".into()],
6217 env: ["XDG_DATA_HOME", "XDG_RUNTIME_DIR", "XDG_CONFIG_HOME"]
6218 .into_iter()
6219 .map(|key| (key.into(), dir.path().to_string_lossy().into_owned()))
6220 .collect(),
6221 reserved: false,
6222 reserved_prefixes: vec![],
6223 protocol: ModuleProtocol::None,
6224 overlap: Default::default(),
6225 };
6226 let supervisor =
6227 Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default());
6228 let module = supervisor.spawn(initial.clone()).unwrap();
6229 assert!(module.status().unwrap().live);
6230 let mut next = initial;
6231 next.protocol = ModuleProtocol::Subc;
6232 module
6233 .update_configuration(next.clone(), HealthConfig::default(), None)
6234 .await
6235 .unwrap();
6236 assert!(
6237 module.status().unwrap().live,
6238 "rescan must not require HELLO from the old non-wire process"
6239 );
6240 let runtime = supervisor.runtime_config();
6241 let action = on_child_exit(
6242 &next,
6243 RestartPolicy::default(),
6244 &supervisor.registry,
6245 &module.inner.snapshot,
6246 &runtime.terminal_ring,
6247 &runtime.spawn_events,
6248 &runtime.child_roster,
6249 ExitReport {
6250 kind: ExitKind::Clean,
6251 code: Some(0),
6252 signal: None,
6253 at_ms: unix_ms_now(),
6254 },
6255 true,
6256 )
6257 .await;
6258 assert!(
6259 matches!(action, NextAction::Restart { .. }),
6260 "the old non-wire process's clean exit must restart"
6261 );
6262 module.drain().await.unwrap();
6263 }
6264
6265 #[cfg(unix)]
6266 #[tokio::test]
6267 async fn orphan_identity_matches_path_names_and_shebang_interpreters() {
6268 use std::os::unix::fs::PermissionsExt;
6269 let dir = cortexkit_test_support::ScratchDir::new("spawn-image-identity");
6270 let script = dir.join("module.sh");
6271 std::fs::write(&script, "#!/bin/sh\nwhile :; do sleep 0.1; done\n").unwrap();
6272 std::fs::set_permissions(&script, std::fs::Permissions::from_mode(0o700)).unwrap();
6273 let record_path = dir.join("live-children.json");
6274 let supervisor =
6275 Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default())
6276 .with_live_children_record(&record_path);
6277 for (program, args) in [
6278 (PathBuf::from("sleep"), vec!["60".into()]),
6279 (script, vec![]),
6280 ] {
6281 let spec = ModuleSpec {
6282 module_id: "image-identity".into(),
6283 program: program.clone(),
6284 args,
6285 env: ["XDG_DATA_HOME", "XDG_RUNTIME_DIR", "XDG_CONFIG_HOME"]
6286 .into_iter()
6287 .map(|key| (key.into(), dir.to_string_lossy().into_owned()))
6288 .collect(),
6289 reserved: false,
6290 reserved_prefixes: vec![],
6291 protocol: ModuleProtocol::None,
6292 overlap: Default::default(),
6293 };
6294 let module = supervisor.spawn(spec).unwrap();
6295 #[cfg(target_os = "macos")]
6296 {
6297 let deadline = Instant::now() + Duration::from_secs(5);
6300 while crate::live_children::read_record(&record_path)
6301 .unwrap()
6302 .iter()
6303 .all(|entry| entry.executable.is_none())
6304 {
6305 assert!(Instant::now() < deadline, "module image was not confirmed");
6306 tokio::time::sleep(Duration::from_millis(5)).await;
6307 }
6308 }
6309 let entry = crate::live_children::read_record(&record_path)
6310 .unwrap()
6311 .pop()
6312 .unwrap();
6313 let observed = subc_os::Process::open(entry.pid)
6314 .unwrap()
6315 .unwrap()
6316 .observe()
6317 .unwrap();
6318 let verdict = crate::live_children::identity_verdict(&entry, entry.pid, &observed);
6319 module.drain().await.unwrap();
6320 assert_eq!(
6321 verdict,
6322 crate::live_children::IdentityVerdict::Matches,
6323 "program {program:?}: recorded {entry:?}, observed {observed:?}"
6324 );
6325 }
6326 }
6327
6328 #[cfg(unix)]
6329 fn http_fixture(
6330 dir: &std::path::Path,
6331 url: &str,
6332 threshold: u32,
6333 ) -> crate::daemon_config::ConfiguredModule {
6334 let path = dir.join("subc.jsonc");
6335 std::fs::write(&path, serde_json::json!({"version": 1, "modules": {"http-process": {
6336 "program": "/bin/sleep", "args": ["60"], "protocol": "none",
6337 "env": {"XDG_DATA_HOME":dir,"XDG_RUNTIME_DIR":dir,"XDG_CONFIG_HOME":dir},
6338 "health": {"http":url,"cadence_ms":20,"deadline_ms":200,"failure_threshold":threshold},
6339 "restart": {"backoff_ms":1,"max_backoff_ms":1}, "drain_timeout_ms":10
6340 }}}).to_string()).unwrap();
6341 crate::daemon_config::load(&path)
6342 .unwrap()
6343 .unwrap()
6344 .modules
6345 .pop()
6346 .unwrap()
6347 }
6348
6349 #[cfg(unix)]
6350 async fn wait_http_health(module: &SupervisedModule, status: SupervisorHealthStatus) {
6351 timeout(Duration::from_secs(5), async {
6352 loop {
6353 if module.status().unwrap().health.status == status {
6354 break;
6355 }
6356 sleep(Duration::from_millis(5)).await;
6357 }
6358 })
6359 .await
6360 .unwrap_or_else(|_| {
6361 panic!(
6362 "expected {status:?}, got {:?}",
6363 module.status().unwrap().health
6364 )
6365 });
6366 }
6367
6368 #[cfg(unix)]
6369 #[tokio::test]
6370 async fn http_health_status_flips_ok_failing_ok() {
6371 use tokio::io::{AsyncReadExt, AsyncWriteExt};
6372 let dir = cortexkit_test_support::ScratchDir::new("http-status-flips");
6373 let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
6374 let status = Arc::new(std::sync::atomic::AtomicU16::new(200));
6375 let serving_status = status.clone();
6376 let url = format!("http://{}/healthz", listener.local_addr().unwrap());
6377 let server = tokio::spawn(async move {
6378 loop {
6379 let (mut stream, _) = listener.accept().await.unwrap();
6380 let mut request = [0u8; 2048];
6381 let count = stream.read(&mut request).await.unwrap();
6382 assert!(count > 0, "a probe must send an HTTP request");
6383 let code = serving_status.load(std::sync::atomic::Ordering::SeqCst);
6384 let body = if code == 200 {
6385 "ready"
6386 } else {
6387 "{\"status\":\"unavailable\",\"error\":\"scratch failure\"}"
6388 };
6389 let response = format!("HTTP/1.1 {code} scratch\r\nContent-Length: {}\r\nConnection: close\r\n\r\n{body}", body.len());
6390 let _ = stream.write_all(response.as_bytes()).await;
6391 }
6392 });
6393 let configured = http_fixture(&dir, &url, 1000);
6394 let module =
6395 Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default())
6396 .supervise_configured_with_health(
6397 configured.module_spec(),
6398 true,
6399 configured.health,
6400 configured.drain_timeout_ms,
6401 configured.restart,
6402 )
6403 .unwrap();
6404 wait_http_health(&module, SupervisorHealthStatus::Ok).await;
6405 status.store(503, std::sync::atomic::Ordering::SeqCst);
6406 wait_http_health(&module, SupervisorHealthStatus::Failing).await;
6407 assert!(module
6408 .status()
6409 .unwrap()
6410 .health
6411 .detail
6412 .unwrap()
6413 .contains("scratch failure"));
6414 status.store(200, std::sync::atomic::Ordering::SeqCst);
6415 wait_http_health(&module, SupervisorHealthStatus::Ok).await;
6416 assert_eq!(module.status().unwrap().health.consecutive_failures, 0);
6417 let before = module.status().unwrap();
6418 let (spec, mut health) = module.configuration().unwrap();
6419 health.http = None;
6420 module
6421 .update_configuration(spec.clone(), health.clone(), Some(10))
6422 .await
6423 .unwrap();
6424 wait_http_health(&module, SupervisorHealthStatus::Unknown).await;
6425 health.http = Some(url);
6426 module
6427 .update_configuration(spec, health, Some(10))
6428 .await
6429 .unwrap();
6430 wait_http_health(&module, SupervisorHealthStatus::Ok).await;
6431 assert_eq!(
6432 module.status().unwrap().pid,
6433 before.pid,
6434 "changing a probe must apply live, not restart its process"
6435 );
6436 let (spec, mut health) = module.configuration().unwrap();
6437 health.failure_threshold = 2;
6438 module
6439 .update_configuration(spec, health, Some(10))
6440 .await
6441 .unwrap();
6442 status.store(503, std::sync::atomic::Ordering::SeqCst);
6443 timeout(Duration::from_secs(5), async {
6444 while module.status().unwrap().spawn_generation == before.spawn_generation {
6445 sleep(Duration::from_millis(5)).await;
6446 }
6447 })
6448 .await
6449 .expect("sustained HTTP 503 responses must trigger the health restart policy");
6450 module.drain().await.unwrap();
6451 server.abort();
6452 }
6453
6454 #[cfg(unix)]
6455 #[tokio::test]
6456 async fn http_health_replacement_recovers_after_one_restart() {
6457 use tokio::io::{AsyncReadExt, AsyncWriteExt};
6458 let dir = cortexkit_test_support::ScratchDir::new("http-replacement-recovers");
6459 let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
6460 let url = format!("http://{}/healthz", listener.local_addr().unwrap());
6461 let mut configured = http_fixture(&dir, &url, 2);
6462 configured.restart.max_restarts = 2;
6463 let module =
6464 Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default())
6465 .supervise_configured_with_health(
6466 configured.module_spec(),
6467 true,
6468 configured.health,
6469 configured.drain_timeout_ms,
6470 configured.restart,
6471 )
6472 .unwrap();
6473 let first_generation = module.status().unwrap().spawn_generation;
6474 let observed = module.clone();
6475 let (healthy_tx, mut healthy_rx) = mpsc::channel(8);
6476 let server = tokio::spawn(async move {
6477 let mut generation = first_generation;
6478 let mut probes = 0;
6479 loop {
6480 let (mut stream, _) = listener.accept().await.unwrap();
6481 let mut request = [0; 2048];
6482 assert!(stream.read(&mut request).await.unwrap() > 0);
6483 let current = observed.status().unwrap().spawn_generation;
6484 if current != generation {
6485 generation = current;
6486 probes = 0;
6487 }
6488 probes += 1;
6489 let healthy = generation != first_generation && probes > 1;
6492 let code = if healthy { 200 } else { 503 };
6493 let response = format!(
6494 "HTTP/1.1 {code} scratch\r\nContent-Length: 0\r\nConnection: close\r\n\r\n"
6495 );
6496 let _ = stream.write_all(response.as_bytes()).await;
6497 if healthy {
6498 healthy_tx.send(()).await.unwrap();
6499 }
6500 }
6501 });
6502 timeout(Duration::from_secs(5), async {
6503 let mut healthy_probes = 0;
6504 loop {
6505 if module.state().unwrap() == ModuleState::Failed {
6506 break;
6507 }
6508 while healthy_rx.try_recv().is_ok() {
6509 healthy_probes += 1;
6510 }
6511 if healthy_probes >= 3
6512 && module.status().unwrap().health.status == SupervisorHealthStatus::Ok
6513 {
6514 break;
6515 }
6516 sleep(Duration::from_millis(5)).await;
6517 }
6518 })
6519 .await
6520 .expect("the replacement must settle healthy without exhausting its restart budget");
6521 let status = module.status().unwrap();
6522 module.drain().await.unwrap();
6523 server.abort();
6524 assert_eq!(status.state, ModuleState::Running);
6525 assert_eq!(status.health.status, SupervisorHealthStatus::Ok);
6526 assert_eq!(status.health.consecutive_failures, 0);
6527 assert_eq!(status.lifetime_restarts, 1);
6528 assert_eq!(status.restart_count, 1);
6529 }
6530
6531 #[cfg(unix)]
6532 #[tokio::test]
6533 async fn http_health_sustained_failure_exhausts_restart_budget() {
6534 use std::collections::BTreeMap;
6535 use tokio::io::{AsyncReadExt, AsyncWriteExt};
6536 let dir = cortexkit_test_support::ScratchDir::new("http-health-budget");
6537 let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
6538 let url = format!("http://{}/healthz", listener.local_addr().unwrap());
6539 let mut configured = http_fixture(&dir, &url, 2);
6540 configured.restart.max_restarts = 2;
6541 let module =
6542 Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default())
6543 .supervise_configured_with_health(
6544 configured.module_spec(),
6545 true,
6546 configured.health,
6547 configured.drain_timeout_ms,
6548 configured.restart,
6549 )
6550 .unwrap();
6551 let observed = module.clone();
6552 let probes = Arc::new(Mutex::new(BTreeMap::<u64, u32>::new()));
6553 let served = probes.clone();
6554 let server = tokio::spawn(async move {
6555 loop {
6556 let (mut stream, _) = listener.accept().await.unwrap();
6557 let mut request = [0; 2048];
6558 assert!(stream.read(&mut request).await.unwrap() > 0);
6559 *served
6560 .lock()
6561 .unwrap()
6562 .entry(observed.status().unwrap().spawn_generation)
6563 .or_default() += 1;
6564 let _ = stream
6565 .write_all(b"HTTP/1.1 503 Unavailable\r\nContent-Length: 0\r\nConnection: close\r\n\r\n")
6566 .await;
6567 }
6568 });
6569 timeout(Duration::from_secs(5), async {
6570 while module.state().unwrap() != ModuleState::Failed {
6571 sleep(Duration::from_millis(5)).await;
6572 }
6573 })
6574 .await
6575 .expect("sustained failures must still exhaust the health restart budget");
6576 let status = module.status().unwrap();
6577 server.abort();
6578 assert_eq!(status.restart_count, 2);
6579 assert_eq!(status.lifetime_restarts, 2);
6580 assert!(!status.process_alive);
6581 assert_eq!(
6582 probes.lock().unwrap().values().copied().collect::<Vec<_>>(),
6583 vec![2, 2, 2]
6584 );
6585 }
6586
6587 #[cfg(unix)]
6588 #[tokio::test]
6589 async fn spawn_clears_the_previous_process_health() {
6590 let dir = cortexkit_test_support::ScratchDir::new("spawn-fresh-health");
6591 let mut configured = http_fixture(&dir, "http://127.0.0.1:1/healthz", 2);
6592 configured.health.cadence = Duration::from_secs(3600);
6593 let module =
6594 Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default())
6595 .supervise_configured_with_health(
6596 configured.module_spec(),
6597 false,
6598 configured.health,
6599 configured.drain_timeout_ms,
6600 configured.restart,
6601 )
6602 .unwrap();
6603 update_snapshot(&module.inner.snapshot, Some(module.module_id()), |state| {
6604 state.health.status = SupervisorHealthStatus::Unresponsive;
6605 state.health.consecutive_failures = 3;
6606 state.health.detail = Some("old process refused connections".into());
6607 state.health.metrics = Some(serde_json::json!({"old": true}));
6608 state.health.last_probe_ms = Some(123);
6609 state.health.last_action = Some("restart".into());
6610 state.health.last_action_ms = Some(124);
6611 })
6612 .unwrap();
6613 module.set_enabled(true).await.unwrap();
6614 let health = module.status().unwrap().health;
6615 module.drain().await.unwrap();
6616 assert_eq!(health.consecutive_failures, 0);
6617 assert_eq!(health.status, SupervisorHealthStatus::Unknown);
6618 assert_eq!(health.detail, None);
6619 assert_eq!(health.metrics, None);
6620 assert_eq!(health.last_probe_ms, None);
6621 assert_eq!(health.last_action.as_deref(), Some("restart"));
6622 assert_eq!(health.last_action_ms, Some(124));
6623 }
6624
6625 #[cfg(unix)]
6626 #[tokio::test]
6627 async fn http_health_no_listener_restarts_after_consecutive_failures() {
6628 let dir = cortexkit_test_support::ScratchDir::new("http-refused");
6629 let unused = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
6630 let url = format!("http://{}/healthz", unused.local_addr().unwrap());
6631 drop(unused);
6632 let configured = http_fixture(&dir, &url, 2);
6633 let module =
6634 Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default())
6635 .supervise_configured_with_health(
6636 configured.module_spec(),
6637 true,
6638 configured.health,
6639 configured.drain_timeout_ms,
6640 configured.restart,
6641 )
6642 .unwrap();
6643 let before = module.status().unwrap().spawn_generation;
6644 timeout(Duration::from_secs(5), async {
6645 loop {
6646 let status = module.status().unwrap();
6647 if status.spawn_generation > before {
6648 assert!(status.lifetime_restarts > 0);
6649 break;
6650 }
6651 sleep(Duration::from_millis(5)).await;
6652 }
6653 })
6654 .await
6655 .expect("sustained HTTP refusal must trigger the health restart policy");
6656 module.drain().await.unwrap();
6657 }
6658
6659 #[cfg(unix)]
6660 #[tokio::test]
6661 async fn http_health_timeout_honours_deadline() {
6662 let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
6663 let url = format!("http://{}/healthz", listener.local_addr().unwrap());
6664 let server = tokio::spawn(async move {
6665 let _held = listener.accept().await.unwrap();
6666 std::future::pending::<()>().await;
6667 });
6668 let error = timeout(
6669 Duration::from_secs(1),
6670 probe_http_health(&url, Duration::from_millis(10)),
6671 )
6672 .await
6673 .expect("the probe must enforce its own deadline")
6674 .unwrap_err();
6675 server.abort();
6676 assert!(error.to_string().contains("timed out"));
6677 }
6678
6679 #[cfg(unix)]
6680 #[tokio::test]
6681 async fn http_health_localhost_reaches_an_ipv6_only_listener() {
6682 use tokio::io::{AsyncReadExt, AsyncWriteExt};
6683 let listener = tokio::net::TcpListener::bind("[::1]:0").await.unwrap();
6684 let url = format!(
6685 "http://localhost:{}/healthz",
6686 listener.local_addr().unwrap().port()
6687 );
6688 let server = tokio::spawn(async move {
6689 let (mut stream, _) = listener.accept().await.unwrap();
6690 let mut request = [0u8; 2048];
6691 assert!(stream.read(&mut request).await.unwrap() > 0);
6692 stream
6693 .write_all(b"HTTP/1.1 200 OK\r\nContent-Length: 2\r\n\r\nok")
6694 .await
6695 .unwrap();
6696 });
6697 assert_eq!(
6702 probe_http_health(&url, Duration::from_secs(10))
6703 .await
6704 .unwrap()
6705 .status,
6706 HealthStatus::Ok
6707 );
6708 server.await.unwrap();
6709 }
6710
6711 #[cfg(unix)]
6712 #[tokio::test]
6713 async fn http_health_timeout_keeps_partial_status_and_body() {
6714 use tokio::io::{AsyncReadExt, AsyncWriteExt};
6715 let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
6716 let url = format!("http://{}/healthz", listener.local_addr().unwrap());
6717 let server = tokio::spawn(async move {
6718 let (mut stream, _) = listener.accept().await.unwrap();
6719 let mut request = [0u8; 2048];
6720 assert!(stream.read(&mut request).await.unwrap() > 0);
6721 stream
6722 .write_all(
6723 b"HTTP/1.1 503 Unavailable\r\nContent-Length: 100\r\n\r\npartial diagnostic",
6724 )
6725 .await
6726 .unwrap();
6727 std::future::pending::<()>().await;
6728 });
6729 let error = probe_http_health(&url, Duration::from_secs(1))
6730 .await
6731 .unwrap_err()
6732 .to_string();
6733 server.abort();
6734 assert!(
6735 error.contains("timed out")
6736 && error.contains("503 Unavailable")
6737 && error.contains("partial diagnostic"),
6738 "{error}"
6739 );
6740 }
6741
6742 #[cfg(unix)]
6743 #[tokio::test]
6744 async fn http_health_diagnostic_body_is_bounded_to_200_bytes() {
6745 use tokio::io::{AsyncReadExt, AsyncWriteExt};
6746 let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
6747 let url = format!("http://{}/healthz", listener.local_addr().unwrap());
6748 let server = tokio::spawn(async move {
6749 let (mut stream, _) = listener.accept().await.unwrap();
6750 let mut request = [0u8; 2048];
6751 assert!(stream.read(&mut request).await.unwrap() > 0);
6752 let body = format!("{}not-in-diagnostic", "x".repeat(400));
6753 let response = format!(
6754 "HTTP/1.1 503 Unavailable\r\nContent-Length: {}\r\n\r\n{body}",
6755 body.len()
6756 );
6757 stream.write_all(response.as_bytes()).await.unwrap();
6758 });
6759 let error = probe_http_health(&url, Duration::from_secs(1))
6760 .await
6761 .unwrap_err()
6762 .to_string();
6763 server.await.unwrap();
6764 assert!(error.ends_with(&"x".repeat(200)), "{error}");
6765 assert_eq!(error.split(": ").last().unwrap().len(), 200);
6766 assert!(!error.contains("not-in-diagnostic"));
6767 }
6768
6769 #[cfg(unix)]
6770 #[tokio::test]
6771 async fn http_health_real_nats_server_monitoring() {
6772 if std::process::Command::new("nats-server")
6773 .arg("--version")
6774 .env("XDG_DATA_HOME", std::env::temp_dir())
6775 .env("XDG_RUNTIME_DIR", std::env::temp_dir())
6776 .env("XDG_CONFIG_HOME", std::env::temp_dir())
6777 .output()
6778 .is_err()
6779 {
6780 eprintln!(
6781 "skipped: http_health_real_nats_server_monitoring (nats-server is not on PATH)"
6782 );
6783 return;
6784 }
6785 let dir = cortexkit_test_support::ScratchDir::new("nats-http-monitoring");
6786 let monitor = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
6787 let port = monitor.local_addr().unwrap().port();
6788 let client = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
6789 let client_port = client.local_addr().unwrap().port();
6790 let config = dir.join("server.conf");
6791 std::fs::write(&config, format!("listen: 127.0.0.1:{client_port}\nhttp: 127.0.0.1:{port}\njetstream {{ store_dir: \"{}\" }}\n", dir.join("jetstream").display())).unwrap();
6792 let mut configured = http_fixture(&dir, &format!("http://127.0.0.1:{port}/healthz"), 1000);
6793 configured.program = PathBuf::from("nats-server");
6794 configured.args = vec!["-c".into(), config.to_string_lossy().into_owned()];
6795 drop(monitor);
6796 drop(client);
6797 let module =
6798 Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default())
6799 .supervise_configured_with_health(
6800 configured.module_spec(),
6801 true,
6802 configured.health,
6803 configured.drain_timeout_ms,
6804 configured.restart,
6805 )
6806 .unwrap();
6807 wait_http_health(&module, SupervisorHealthStatus::Ok).await;
6808 module.drain().await.unwrap();
6809 }
6810
6811 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
6812 async fn configuration_update_does_not_replace_captured_running_process_facts() {
6813 let supervisor =
6814 Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default());
6815 let snapshot = stale_process_snapshot(ModuleState::Running, true);
6816 let initial = ModuleSpec {
6817 module_id: "rescan-preserves-spawn-facts".to_string(),
6818 program: PathBuf::from("/spawned/module"),
6819 args: Vec::new(),
6820 env: Vec::new(),
6821 reserved: false,
6822 reserved_prefixes: Vec::new(),
6823 protocol: ModuleProtocol::Subc,
6824 overlap: Default::default(),
6825 };
6826 let module = supervisor.supervised_module(
6827 initial.clone(),
6828 supervisor.runtime_config(),
6829 snapshot,
6830 None,
6831 );
6832 let before = module.status().unwrap();
6833 let mut replacement = initial;
6834 replacement.program = PathBuf::from("/rescanned/replacement-module");
6835
6836 module
6837 .update_configuration(replacement, HealthConfig::default(), None)
6838 .await
6839 .unwrap();
6840
6841 let after = module.status().unwrap();
6842 assert_eq!(after.pid, before.pid);
6843 assert_eq!(after.spawned_at_ms, before.spawned_at_ms);
6844 assert_eq!(after.spawned_from, before.spawned_from);
6845 drop(module);
6846 }
6847}
6848
6849fn unix_ms_now() -> u64 {
6850 SystemTime::now()
6851 .duration_since(UNIX_EPOCH)
6852 .map(|duration| duration.as_millis().min(u128::from(u64::MAX)) as u64)
6853 .unwrap_or(0)
6854}
6855
6856async fn supervise_loop(
6857 mut spec: ModuleSpec,
6858 mut runtime: SupervisorRuntimeConfig,
6859 registry: Arc<Registry>,
6860 process_liveness: Arc<SupervisorProcessLiveness>,
6861 snapshot: SharedSnapshot,
6862 mut child: Option<SupervisedChild>,
6863 mut commands: mpsc::Receiver<SupervisorCommand>,
6864) {
6865 let mut health_probe = HealthProbeRuntime::default();
6866 let mut registration_changes = match registry.subscribe_module_changes(&spec.module_id) {
6870 Ok(changes) => Some(changes),
6871 Err(err) => {
6872 warn!(module_id = %spec.module_id, error = %err, "health prober could not subscribe to registry");
6875 None
6876 }
6877 };
6878 let mut pending_respawn: Option<PendingRespawn> = None;
6882 let mut requeued: VecDeque<SupervisorCommand> = VecDeque::new();
6885 let mut previous_instance_recheck = tokio::time::interval_at(
6886 Instant::now() + Duration::from_secs(5),
6887 Duration::from_secs(5),
6888 );
6889 previous_instance_recheck.set_missed_tick_behavior(tokio::time::MissedTickBehavior::Skip);
6890 loop {
6891 #[cfg(test)]
6892 {
6893 lock_snapshot(&snapshot).unwrap().actor_select = None;
6894 }
6895 #[cfg(target_os = "macos")]
6896 if let Some(active) = child.as_mut() {
6897 active.confirm_privacy_exec().await;
6898 }
6899 if let Some(scheduled) = runtime
6900 .scheduled_respawn
6901 .lock()
6902 .unwrap_or_else(|p| p.into_inner())
6903 .take()
6904 {
6905 runtime.recovery_is_daemon = scheduled.daemon_initiated;
6906 pending_respawn = Some(scheduled);
6907 }
6908 if pending_respawn.is_some() && (child.is_some() || !respawn_still_pending(&snapshot)) {
6909 pending_respawn = None;
6910 cancel_deferred_reload(
6911 &runtime,
6912 &spec.module_id,
6913 "respawn cancelled by a supervisor command",
6914 );
6915 }
6916 if child.is_none()
6917 && pending_respawn.is_none()
6918 && lock_snapshot(&snapshot).is_ok_and(|state| state.waiting_for_previous_pid.is_none())
6919 {
6920 cancel_deferred_reload(
6921 &runtime,
6922 &spec.module_id,
6923 "respawn cancelled before a replacement was spawned",
6924 );
6925 let mut stranded = false;
6926 let _ = update_snapshot(&snapshot, Some(&spec.module_id), |state| {
6927 state.respawn_pending = false;
6928 state.coalesced_restart_pending = false;
6929 if matches!(
6930 state.state,
6931 ModuleState::Restarting
6932 | ModuleState::Starting
6933 | ModuleState::Draining
6934 | ModuleState::Unresponsive
6935 ) {
6936 error!(module_id = %spec.module_id, state = ?state.state,
6937 "supervision operation ended without a child or pending respawn; marking failed so start can retry");
6938 stranded = true;
6939 }
6940 });
6941 if stranded {
6942 fail_snapshot(
6943 &snapshot,
6944 Some(&spec.module_id),
6945 None,
6946 &runtime.failure_context(runtime.recovery_is_daemon),
6947 );
6948 }
6949 }
6950 if let Some(command) = requeued.pop_front() {
6951 if !handle_supervisor_command(
6952 command,
6953 &mut spec,
6954 &mut runtime,
6955 ®istry,
6956 &process_liveness,
6957 &snapshot,
6958 &mut child,
6959 &mut commands,
6960 &mut requeued,
6961 )
6962 .await
6963 {
6964 return;
6965 }
6966 if child.is_some() || !respawn_still_pending(&snapshot) {
6967 pending_respawn = None;
6968 let _ = update_snapshot(&snapshot, Some(&spec.module_id), |state| {
6969 state.respawn_pending = false
6970 });
6971 }
6972 continue;
6973 }
6974 if child.is_some() {
6975 runtime.recovery_is_daemon = true;
6976 #[cfg(test)]
6977 {
6978 lock_snapshot(&snapshot).unwrap().actor_turns += 1;
6979 }
6980 health_probe.refresh_registration(&spec, &runtime, ®istry, &snapshot);
6981 let active = child.as_mut().expect("child checked above");
6982 if runtime.child_roster.is_closed() || active.registered_since_spawn() {
6983 active.registration_deadline = None;
6984 }
6985 let registration_deadline = active.registration_deadline;
6986 let no_checks = runtime.health.http.is_none()
6987 && (running_protocol(&spec, &snapshot) == ModuleProtocol::None
6988 || (health_probe.registered_connection.is_some() && !health_probe.advertised));
6989 let uptime_reset = lock_snapshot(&snapshot).ok().and_then(|mut state| {
6990 state
6991 .failed_retry
6992 .uptime_reset_at(no_checks, runtime.restart_policy.window)
6993 });
6994 let wake_at = match (health_probe.wake_at(), uptime_reset) {
6995 (probe, Some(reset)) => Some(probe.map_or(reset, |probe| probe.min(reset))),
6996 (probe, None) => probe,
6997 };
6998 let wake_at = match (wake_at, registration_deadline) {
6999 (Some(wake), Some(register)) => Some(wake.min(register)),
7000 (wake, register) => wake.or(register),
7001 };
7002 #[cfg(test)]
7003 let wake_after = wake_at.map(|at| at.saturating_duration_since(Instant::now()));
7004 let probe_sleep = supervisor_sleep(wake_at);
7005 tokio::pin!(probe_sleep);
7006 let wire_child = running_protocol(&spec, &snapshot) == ModuleProtocol::Subc;
7011 #[cfg(test)]
7012 {
7013 let registration_pending = wire_child
7014 && registration_changes
7015 .as_ref()
7016 .is_some_and(|changes| changes.has_changed().unwrap());
7017 let mut state = lock_snapshot(&snapshot).unwrap();
7018 state.actor_select = (!registration_pending).then_some(ActorSelectCheckpoint {
7019 generation: state.spawn_generation,
7020 turn: state.actor_turns,
7021 registered_connection: health_probe.registered_connection,
7022 next_probe_at: health_probe.next_probe_at,
7023 wake_after,
7024 });
7025 }
7026 let active_child = child.as_mut().expect("child checked above");
7027 tokio::select! {
7028 wait_result = active_child.wait() => {
7029 if let Ok(mut state) = lock_snapshot(&snapshot) {
7030 state.failed_retry.uptime_reset_at(no_checks, runtime.restart_policy.window);
7031 }
7032 #[cfg(test)]
7033 { lock_snapshot(&snapshot).unwrap().actor_select = None; }
7034 let exit_report = match wait_result {
7043 Ok(status) => classify_reaped_child_exit(&snapshot, active_child, &status),
7044 Err(err) => {
7045 active_child.drain_stderr(&spec.module_id).await;
7046 fail_after_wait_error(
7047 &spec.module_id,
7048 &snapshot,
7049 &runtime.terminal_ring,
7050 &runtime.spawn_events,
7051 format!("failed to wait for supervised module: {err}"),
7052 &runtime.failure_context(true),
7053 );
7054 untrack_if_registration_released(
7055 &process_liveness,
7056 ®istry,
7057 &spec.module_id,
7058 &snapshot,
7059 );
7060 error!(module_id = %spec.module_id, error = %err, "failed to wait for supervised module");
7061 child = None;
7062 continue;
7063 }
7064 };
7065 active_child.drain_stderr(&spec.module_id).await;
7066
7067 let next = on_child_exit(
7068 &spec,
7069 runtime.restart_policy,
7070 ®istry,
7071 &snapshot,
7072 &runtime.terminal_ring,
7073 &runtime.spawn_events,
7074 &runtime.child_roster,
7075 exit_report,
7076 true,
7077 ).await;
7078 runtime.start_requester = None;
7079 runtime.pending_activity_restart = false;
7080 active_child.release_roster();
7083 match next {
7084 NextAction::Stop { registration_released } => {
7085 if registration_released {
7086 process_liveness.untrack_if_current(&spec.module_id, &snapshot);
7087 }
7088 child = None;
7089 }
7090 NextAction::Restart { schedule } => {
7091 let delay = schedule.map_or(
7092 runtime.restart_policy.delay_for_restart(0),
7093 |schedule| schedule.delay,
7094 );
7095 if let Some(schedule) = schedule {
7096 log_crash_respawn(&spec.module_id, schedule);
7097 }
7098 child = None;
7106 pending_respawn = Some(PendingRespawn { deadline: Instant::now() + delay, kind: RespawnKind::Spawn, daemon_initiated: true });
7107 let _ = update_snapshot(&snapshot, Some(&spec.module_id), |state| state.respawn_pending = true);
7108 }
7109 }
7110 }
7111 command = commands.recv() => {
7112 #[cfg(test)]
7113 { lock_snapshot(&snapshot).unwrap().actor_select = None; }
7114 let Some(command) = command else {
7115 return;
7116 };
7117 if !handle_supervisor_command(
7118 command,
7119 &mut spec,
7120 &mut runtime,
7121 ®istry,
7122 &process_liveness,
7123 &snapshot,
7124 &mut child,
7125 &mut commands,
7126 &mut requeued,
7127 ).await {
7128 return;
7129 }
7130 }
7131 _ = async {
7132 match registration_changes.as_mut() {
7133 Some(changes) => { let _ = changes.changed().await; }
7134 None => std::future::pending().await,
7135 }
7136 }, if wire_child => {
7137 #[cfg(test)]
7138 { lock_snapshot(&snapshot).unwrap().actor_select = None; }
7139 }
7140 _ = &mut probe_sleep => {
7141 #[cfg(test)]
7142 { lock_snapshot(&snapshot).unwrap().actor_select = None; }
7143 if registration_deadline.is_some_and(|deadline| Instant::now() >= deadline) {
7144 active_child.registration_deadline = None;
7147 if !runtime.child_roster.is_closed()
7148 && !active_child.registered_since_spawn()
7149 && active_child.child.try_wait().is_ok_and(|status| status.is_none())
7150 {
7151 warn!(module_id = %spec.module_id, pid = active_child.pid,
7152 wait_secs = runtime.register_timeout.as_secs(),
7153 "supervised module registration_timeout: first HELLO deadline expired; stopping child");
7154 active_child.spawn_failure = Some("registration_timeout".into());
7155 if let Err(error) = stop_unregistered_child(active_child).await {
7156 error!(module_id = %spec.module_id, %error, "failed to stop unregistered child");
7157 fail_after_wait_error(&spec.module_id, &snapshot, &runtime.terminal_ring,
7158 &runtime.spawn_events, format!("registration_timeout: {error}"),
7159 &runtime.failure_context(true));
7160 child = None;
7161 }
7162 continue;
7165 }
7166 }
7167 if health_probe.due() {
7168 run_health_probe_cycle(
7169 &spec,
7170 &runtime,
7171 ®istry,
7172 &process_liveness,
7173 &snapshot,
7174 &mut child,
7175 ).await;
7176 if child.is_none() {
7177 runtime.start_requester = None;
7178 runtime.pending_activity_restart = false;
7179 }
7180 if child.is_some() {
7181 health_probe.schedule_next(&spec, runtime.health.cadence);
7182 }
7183 }
7184 }
7185 }
7186 } else if let Some(pending) = pending_respawn {
7187 tokio::select! {
7188 _ = sleep_until(pending.deadline) => {
7189 pending_respawn = None;
7190 let _ = update_snapshot(&snapshot, Some(&spec.module_id), |state| state.respawn_pending = false);
7191 if !respawn_still_pending(&snapshot) {
7195 continue;
7196 }
7197 if runtime.child_roster.is_closed() {
7202 let _ = update_snapshot(&snapshot, Some(&spec.module_id), |state| {
7203 state.state = ModuleState::Stopped;
7204 });
7205 debug!(module_id = %spec.module_id, "crash respawn cancelled by daemon shutdown");
7206 continue;
7207 }
7208 if let Err(err) = release_dead_registration(
7209 ®istry,
7210 runtime.forwarding.as_deref(),
7211 &snapshot,
7212 &spec.module_id,
7213 &runtime.failure_context(runtime.recovery_is_daemon),
7214 ).await {
7215 error!(module_id = %spec.module_id, error = %err, "registration did not release before restart");
7216 continue;
7217 }
7218
7219 if matches!(pending.kind, RespawnKind::Reload) {
7220 let reply = runtime.deferred_reload_reply.lock().unwrap_or_else(|p| p.into_inner()).take();
7221 let result = finish_reload_child(&spec, &runtime, ®istry, &process_liveness, &snapshot, &mut child).await;
7222 if result.is_ok() && runtime.pending_activity_restart {
7223 runtime.record_activity_transition("subc.module_restarted", &spec.module_id, runtime.start_requester.clone());
7224 }
7225 runtime.pending_activity_restart = false;
7226 if let Some(reply) = reply { let _ = reply.send(result); }
7227 continue;
7228 }
7229 process_liveness.track(spec.module_id.clone(), Arc::clone(&snapshot));
7230 match spawn_and_mark_running(&spec, &runtime, &snapshot) {
7231 Ok(next_child) => {
7232 child = Some(next_child);
7233 if runtime.pending_activity_restart {
7234 runtime.record_activity_transition("subc.module_restarted", &spec.module_id, runtime.start_requester.clone());
7235 runtime.pending_activity_restart = false;
7236 }
7237 debug!(module_id = %spec.module_id, "supervised module restarted after crash");
7238 }
7239 Err(err) => {
7240 fail_snapshot(&snapshot, Some(&spec.module_id), None, &runtime.failure_context(runtime.recovery_is_daemon));
7241 process_liveness.untrack_if_current(&spec.module_id, &snapshot);
7242 error!(module_id = %spec.module_id, error = %err, "failed to restart supervised module");
7243 }
7244 }
7245 }
7246 command = commands.recv() => {
7247 let Some(command) = command else {
7248 return;
7249 };
7250 if !handle_supervisor_command(
7251 command,
7252 &mut spec,
7253 &mut runtime,
7254 ®istry,
7255 &process_liveness,
7256 &snapshot,
7257 &mut child,
7258 &mut commands,
7259 &mut requeued,
7260 ).await {
7261 return;
7262 }
7263 if child.is_some() || !respawn_still_pending(&snapshot) {
7268 pending_respawn = None;
7269 let _ = update_snapshot(&snapshot, Some(&spec.module_id), |state| state.respawn_pending = false);
7270 }
7271 }
7272 }
7273 } else {
7274 let slow_retry = lock_snapshot(&snapshot).ok().and_then(|state| {
7275 (state.enabled && state.state == ModuleState::Failed)
7276 .then_some(state.failed_retry.pending)
7277 .flatten()
7278 });
7279 let command = if lock_snapshot(&snapshot)
7280 .is_ok_and(|state| state.waiting_for_previous_pid.is_some())
7281 {
7282 tokio::select! {
7283 command = commands.recv() => command,
7284 _ = previous_instance_recheck.tick() => {
7285 if runtime.child_roster.previous_instance_pid(&spec.module_id).is_none() {
7286 let enabled = lock_snapshot(&snapshot).is_ok_and(|state| state.enabled);
7287 if enabled && !runtime.child_roster.is_closed() {
7288 process_liveness.track(spec.module_id.clone(), Arc::clone(&snapshot));
7289 match spawn_and_mark_running(&spec, &runtime, &snapshot) {
7290 Ok(next_child) => child = Some(next_child),
7291 Err(error) => {
7292 fail_snapshot(&snapshot, Some(&spec.module_id), None, &runtime.failure_context(true));
7293 error!(module_id = %spec.module_id, %error, "failed to start module after its previous instance exited");
7294 }
7295 }
7296 } else {
7297 let _ = update_snapshot(&snapshot, Some(&spec.module_id), |state| {
7298 state.waiting_for_previous_pid = None;
7299 state.waiting_for_previous_detail = None;
7300 });
7301 }
7302 }
7303 continue;
7304 }
7305 }
7306 } else {
7307 tokio::select! {
7308 biased;
7309 command = commands.recv() => command,
7310 _ = async {
7311 match slow_retry {
7312 Some(retry) => sleep_until(retry.deadline).await,
7313 None => std::future::pending().await,
7314 }
7315 } => {
7316 let retry = slow_retry.expect("only a scheduled slow retry can wake");
7317 let _ = update_snapshot(&snapshot, Some(&spec.module_id), |state| {
7318 state.failed_retry.pending = None;
7319 });
7320 if runtime.child_roster.is_closed() {
7321 continue;
7322 }
7323 info!(module_id = %spec.module_id, attempt = retry.attempt,
7324 delay_secs = retry.delay.as_secs(), "starting failed module retry");
7325 if let Err(error) = set_child_enabled(
7326 &spec, &runtime, ®istry, &process_liveness, &snapshot, &mut child, true,
7327 &runtime.failure_context(true),
7328 ).await {
7329 error!(module_id = %spec.module_id, %error, "failed module retry could not start");
7330 }
7331 continue;
7332 }
7333 }
7334 };
7335 let Some(command) = command else {
7336 return;
7337 };
7338 if !handle_supervisor_command(
7339 command,
7340 &mut spec,
7341 &mut runtime,
7342 ®istry,
7343 &process_liveness,
7344 &snapshot,
7345 &mut child,
7346 &mut commands,
7347 &mut requeued,
7348 )
7349 .await
7350 {
7351 return;
7352 }
7353 }
7354 }
7355}
7356
7357async fn supervisor_sleep(wake_at: Option<Instant>) {
7360 match wake_at {
7361 Some(deadline) => sleep_until(deadline).await,
7362 None => std::future::pending().await,
7363 }
7364}
7365
7366fn log_crash_respawn(module_id: &str, schedule: CrashRestartSchedule) {
7367 info!(
7368 module_id,
7369 restart_in_window = schedule.restart_in_window,
7370 delay_ms = schedule.delay.as_millis() as u64,
7371 "respawning after crash"
7372 );
7373}
7374
7375fn respawn_still_pending(snapshot: &SharedSnapshot) -> bool {
7381 matches!(
7382 lock_snapshot(snapshot),
7383 Ok(state) if state.enabled && state.state == ModuleState::Restarting
7384 )
7385}
7386
7387enum NextAction {
7388 Stop {
7389 registration_released: bool,
7390 },
7391 Restart {
7392 schedule: Option<CrashRestartSchedule>,
7393 },
7394}
7395
7396#[allow(clippy::too_many_arguments)]
7397async fn handle_supervisor_command(
7398 command: SupervisorCommand,
7399 spec: &mut ModuleSpec,
7400 runtime: &mut SupervisorRuntimeConfig,
7401 registry: &Arc<Registry>,
7402 process_liveness: &SupervisorProcessLiveness,
7403 snapshot: &SharedSnapshot,
7404 child: &mut Option<SupervisedChild>,
7405 commands: &mut mpsc::Receiver<SupervisorCommand>,
7406 requeued: &mut VecDeque<SupervisorCommand>,
7407) -> bool {
7408 if !matches!(command, SupervisorCommand::UpdateConfiguration { .. }) {
7409 runtime.recovery_is_daemon = false;
7410 }
7411 if matches!(
7412 command,
7413 SupervisorCommand::Drain { .. }
7414 | SupervisorCommand::Retire { .. }
7415 | SupervisorCommand::SetEnabled { .. }
7416 | SupervisorCommand::Restart { .. }
7417 ) {
7418 let _ = update_snapshot(snapshot, Some(&spec.module_id), |state| {
7419 state.failed_retry.cancel()
7420 });
7421 }
7422 let command =
7423 if let Some((pid, reason)) = runtime.child_roster.previous_instance(&spec.module_id) {
7424 match command {
7425 SupervisorCommand::Restart { reply, .. }
7426 | SupervisorCommand::Reload { reply, .. }
7427 | SupervisorCommand::Swap { reply, .. } => {
7428 let _ = reply.send(Err(previous_instance_error(spec, pid, reason)));
7429 return true;
7430 }
7431 SupervisorCommand::SetEnabled {
7432 enabled: true,
7433 reply,
7434 ..
7435 } => {
7436 let _ = reply.send(Err(previous_instance_error(spec, pid, reason)));
7437 return true;
7438 }
7439 other => other,
7440 }
7441 } else {
7442 command
7443 };
7444 match command {
7445 SupervisorCommand::Drain { requester, reply } => {
7446 let result = drain_optional_child(
7449 &spec.module_id,
7450 spec.protocol,
7451 StopNotice::NotSent,
7452 registry,
7453 runtime.forwarding.as_deref(),
7454 snapshot,
7455 &runtime.terminal_ring,
7456 &runtime.spawn_events,
7457 child,
7458 runtime.drain_timeout,
7459 ModuleState::Stopped,
7460 None,
7461 &FailureContext::operator(),
7462 )
7463 .await;
7464 let registration_released = result.is_ok();
7465 if registration_released {
7466 runtime.record_activity_transition(
7467 "subc.module_stopped",
7468 &spec.module_id,
7469 requester,
7470 );
7471 }
7472 let _ = reply.send(result);
7473 if registration_released {
7474 process_liveness.untrack_if_current(&spec.module_id, snapshot);
7475 }
7476 false
7477 }
7478 SupervisorCommand::Retire { requester, reply } => {
7479 let result = async {
7480 let stop_notice = begin_forwarding_drain_if_configured(
7481 spec,
7482 runtime,
7483 registry,
7484 snapshot,
7485 None,
7486 RouteCloseReason::Disable,
7487 )
7488 .await?;
7489 drain_optional_child(
7490 &spec.module_id,
7491 spec.protocol,
7492 stop_notice,
7493 registry,
7494 runtime.forwarding.as_deref(),
7495 snapshot,
7496 &runtime.terminal_ring,
7497 &runtime.spawn_events,
7498 child,
7499 runtime.drain_timeout,
7500 ModuleState::Stopped,
7501 None,
7502 &FailureContext::operator(),
7503 )
7504 .await
7505 }
7506 .await;
7507 let registration_released = result.is_ok();
7508 if registration_released {
7509 runtime.record_activity_transition(
7510 "subc.module_stopped",
7511 &spec.module_id,
7512 requester,
7513 );
7514 }
7515 let _ = reply.send(result);
7516 if registration_released {
7517 process_liveness.untrack_if_current(&spec.module_id, snapshot);
7518 }
7519 false
7520 }
7521 SupervisorCommand::Restart {
7522 requester,
7523 drain_timeout_ms,
7524 received_at_generation,
7525 queued_at,
7526 reply,
7527 } => {
7528 info!(
7532 module_id = %spec.module_id,
7533 queued_ms = u64::try_from(queued_at.elapsed().as_millis()).unwrap_or(u64::MAX),
7534 "restart command dequeued"
7535 );
7536 let validation = match lock_snapshot(snapshot) {
7548 Ok(state) if !state.enabled => Err(SuperviseError::Disabled {
7549 module_id: spec.module_id.clone(),
7550 }),
7551 Ok(_) => Ok(()),
7552 Err(err) => Err(err),
7553 };
7554 let initiated = validation.is_ok();
7555 let _ = reply.send(validation);
7556 let satisfied_by_generation = if initiated
7570 && child
7571 .as_ref()
7572 .is_some_and(|child| child.spawned_at > queued_at)
7573 {
7574 lock_snapshot(snapshot).ok().and_then(|state| {
7575 (!state.configuration_updated_since_spawn).then_some(state.spawn_generation)
7576 })
7577 } else {
7578 None
7579 };
7580 let satisfied_by_pending = initiated
7581 && child.is_none()
7582 && lock_snapshot(snapshot).ok().is_some_and(|mut state| {
7583 let pending = state.respawn_pending && !state.configuration_updated_since_spawn;
7587 if pending {
7588 state.coalesced_restart_pending = true;
7589 }
7590 pending
7591 });
7592 if satisfied_by_pending {
7593 debug!(module_id = %spec.module_id, "restart coalesced into the pending replacement");
7594 } else if let Some(generation) = satisfied_by_generation {
7595 info!(
7596 module_id = %spec.module_id,
7597 received_at_generation,
7598 "restart already satisfied by generation {generation}; not restarting again"
7599 );
7600 } else if initiated {
7601 let drain_timeout = drain_timeout_ms
7604 .map(Duration::from_millis)
7605 .unwrap_or(runtime.drain_timeout);
7606 if let Err(err) = restart_child(
7607 spec,
7608 runtime,
7609 registry,
7610 process_liveness,
7611 snapshot,
7612 child,
7613 drain_timeout,
7614 )
7615 .await
7616 {
7617 warn!(
7618 module_id = %spec.module_id,
7619 error = %err,
7620 "operator restart failed after initiation ack; module state carries the outcome"
7621 );
7622 let _ = update_snapshot(snapshot, Some(&spec.module_id), |state| {
7623 state.state = ModuleState::Failed;
7624 clear_current_process_facts(state);
7625 });
7626 } else {
7627 runtime.start_requester = requester;
7628 runtime.pending_activity_restart = true;
7629 }
7630 }
7631 true
7632 }
7633 SupervisorCommand::Reload { requester, reply } => {
7634 let result =
7635 reload_child(spec, runtime, registry, process_liveness, snapshot, child).await;
7636 if result.is_ok() {
7637 runtime.start_requester = requester;
7638 runtime.pending_activity_restart = true;
7639 }
7640 if result.is_ok()
7641 && runtime
7642 .scheduled_respawn
7643 .lock()
7644 .unwrap_or_else(|p| p.into_inner())
7645 .as_ref()
7646 .is_some_and(|pending| matches!(pending.kind, RespawnKind::Reload))
7647 {
7648 *runtime
7649 .deferred_reload_reply
7650 .lock()
7651 .unwrap_or_else(|p| p.into_inner()) = Some(reply);
7652 } else {
7653 let _ = reply.send(result);
7654 }
7655 true
7656 }
7657 SupervisorCommand::SetEnabled {
7658 enabled,
7659 requester,
7660 reply,
7661 } => {
7662 let previous_requester = runtime.start_requester.clone();
7663 if enabled {
7664 runtime.start_requester = requester.clone();
7665 }
7666 let result = set_child_enabled(
7667 spec,
7668 runtime,
7669 registry,
7670 process_liveness,
7671 snapshot,
7672 child,
7673 enabled,
7674 &runtime.failure_context(false),
7675 )
7676 .await;
7677 if matches!(result, Ok(true)) {
7678 runtime.pending_activity_restart = false;
7679 if enabled {
7680 runtime.start_requester = requester;
7681 } else {
7682 runtime.start_requester = None;
7683 runtime.record_activity_transition(
7684 "subc.module_disabled",
7685 &spec.module_id,
7686 requester,
7687 );
7688 }
7689 } else {
7690 runtime.start_requester = previous_requester;
7691 }
7692 let _ = reply.send(result);
7693 true
7694 }
7695 SupervisorCommand::UpdateConfiguration {
7696 spec: next_spec,
7697 health,
7698 drain_timeout_ms,
7699 reply,
7700 } => {
7701 if let Some(handle) = &runtime.supervisor_handle {
7702 handle.apply_identity_configuration(&next_spec);
7703 }
7704 *spec = next_spec;
7705 let _ = update_snapshot(snapshot, Some(&spec.module_id), |state| {
7706 state.configuration_updated_since_spawn = true;
7707 });
7708 let health_changed = runtime.health != health;
7709 runtime.health = health;
7710 if health_changed {
7713 let _ = update_snapshot(snapshot, Some(&spec.module_id), |state| {
7714 state.health = ModuleHealthStatus::default();
7715 });
7716 }
7717 runtime.drain_timeout = drain_timeout_ms
7718 .map(Duration::from_millis)
7719 .unwrap_or(runtime.default_drain_timeout);
7720 *runtime
7721 .effective_drain_timeout
7722 .lock()
7723 .unwrap_or_else(|poisoned| poisoned.into_inner()) = runtime.drain_timeout;
7724 let _ = reply.send(());
7725 true
7726 }
7727 SupervisorCommand::Swap {
7728 requester,
7729 ready_timeout,
7730 reply,
7731 } => {
7732 let mut swap_runtime = runtime.clone();
7733 swap_runtime.start_requester = requester;
7734 let before = lock_snapshot(snapshot)
7735 .ok()
7736 .map(|state| state.spawn_generation);
7737 let end = swap::run_swap(
7738 spec,
7739 &swap_runtime,
7740 registry,
7741 process_liveness,
7742 snapshot,
7743 child,
7744 commands,
7745 ready_timeout.unwrap_or(DEFAULT_SWAP_READY_TIMEOUT),
7746 reply,
7747 )
7748 .await;
7749 if before
7750 != lock_snapshot(snapshot)
7751 .ok()
7752 .map(|state| state.spawn_generation)
7753 {
7754 runtime.start_requester = swap_runtime.start_requester;
7755 }
7756 requeued.extend(end.requeue);
7757 true
7758 }
7759 }
7760}
7761
7762async fn restart_child(
7763 spec: &ModuleSpec,
7764 runtime: &SupervisorRuntimeConfig,
7765 registry: &Registry,
7766 process_liveness: &SupervisorProcessLiveness,
7767 snapshot: &SharedSnapshot,
7768 child: &mut Option<SupervisedChild>,
7769 drain_timeout: Duration,
7770) -> Result<(), SuperviseError> {
7771 if !lock_snapshot(snapshot)?.enabled {
7773 return Err(SuperviseError::Disabled {
7774 module_id: spec.module_id.clone(),
7775 });
7776 }
7777 let stop_notice = begin_forwarding_drain_with_timeout(
7778 spec,
7779 runtime,
7780 registry,
7781 snapshot,
7782 None,
7783 RouteCloseReason::Restart,
7784 drain_timeout,
7785 )
7786 .await?;
7787
7788 if child.is_some() {
7789 drain_optional_child(
7790 &spec.module_id,
7791 spec.protocol,
7792 stop_notice,
7793 registry,
7794 runtime.forwarding.as_deref(),
7795 snapshot,
7796 &runtime.terminal_ring,
7797 &runtime.spawn_events,
7798 child,
7799 drain_timeout,
7800 ModuleState::Restarting,
7801 Some(true),
7802 &FailureContext::operator(),
7803 )
7804 .await?;
7805 } else {
7806 update_snapshot(snapshot, Some(&spec.module_id), |state| {
7807 state.enabled = true;
7808 state.state = ModuleState::Restarting;
7809 clear_current_process_facts(state);
7810 })?;
7811 release_dead_registration(
7812 registry,
7813 runtime.forwarding.as_deref(),
7814 snapshot,
7815 &spec.module_id,
7816 &runtime.failure_context(false),
7817 )
7818 .await?;
7819 }
7820
7821 reset_restart_count(snapshot, &spec.module_id)?;
7822 process_liveness.track(spec.module_id.clone(), Arc::clone(snapshot));
7823 schedule_respawn(
7824 runtime,
7825 snapshot,
7826 &spec.module_id,
7827 runtime.restart_policy.backoff,
7828 RespawnKind::Spawn,
7829 false,
7830 )
7831}
7832
7833async fn reload_child(
7834 spec: &ModuleSpec,
7835 runtime: &SupervisorRuntimeConfig,
7836 registry: &Registry,
7837 process_liveness: &SupervisorProcessLiveness,
7838 snapshot: &SharedSnapshot,
7839 child: &mut Option<SupervisedChild>,
7840) -> Result<(), SuperviseError> {
7841 if !lock_snapshot(snapshot)?.enabled {
7843 return Err(SuperviseError::Disabled {
7844 module_id: spec.module_id.clone(),
7845 });
7846 }
7847 let stop_notice = begin_forwarding_drain(
7848 spec,
7849 runtime,
7850 registry,
7851 snapshot,
7852 Some(true),
7853 RouteCloseReason::Reload,
7854 )
7855 .await?;
7856
7857 if child.is_some() {
7858 drain_optional_child(
7859 &spec.module_id,
7860 spec.protocol,
7861 stop_notice,
7862 registry,
7863 runtime.forwarding.as_deref(),
7864 snapshot,
7865 &runtime.terminal_ring,
7866 &runtime.spawn_events,
7867 child,
7868 runtime.drain_timeout,
7869 ModuleState::Restarting,
7870 Some(true),
7871 &FailureContext::operator(),
7872 )
7873 .await?;
7874 } else {
7875 update_snapshot(snapshot, Some(&spec.module_id), |state| {
7876 state.enabled = true;
7877 state.state = ModuleState::Restarting;
7878 clear_current_process_facts(state);
7879 })?;
7880 release_dead_registration(
7881 registry,
7882 runtime.forwarding.as_deref(),
7883 snapshot,
7884 &spec.module_id,
7885 &runtime.failure_context(false),
7886 )
7887 .await?;
7888 }
7889
7890 reset_restart_count(snapshot, &spec.module_id)?;
7891 process_liveness.track(spec.module_id.clone(), Arc::clone(snapshot));
7892 schedule_respawn(
7893 runtime,
7894 snapshot,
7895 &spec.module_id,
7896 runtime.restart_policy.backoff,
7897 RespawnKind::Reload,
7898 false,
7899 )
7900}
7901
7902async fn finish_reload_child(
7903 spec: &ModuleSpec,
7904 runtime: &SupervisorRuntimeConfig,
7905 registry: &Registry,
7906 process_liveness: &SupervisorProcessLiveness,
7907 snapshot: &SharedSnapshot,
7908 child: &mut Option<SupervisedChild>,
7909) -> Result<(), SuperviseError> {
7910 process_liveness.track(spec.module_id.clone(), Arc::clone(snapshot));
7911 let next_child = match spawn_and_mark_running(spec, runtime, snapshot) {
7912 Ok(next_child) => next_child,
7913 Err(err) => {
7914 return handle_reload_spawn_failure(
7915 spec,
7916 runtime,
7917 process_liveness,
7918 snapshot,
7919 child,
7920 format!("new child failed to spawn: {err}"),
7921 )
7922 .await;
7923 }
7924 };
7925 *child = Some(next_child);
7926
7927 let wait_outcome = {
7928 let active_child = child.as_mut().expect("new reload child was just stored");
7929 wait_for_registration_after_reload(
7930 registry,
7931 &spec.module_id,
7932 snapshot,
7933 active_child,
7934 REGISTRY_RELEASE_TIMEOUT,
7935 )
7936 .await?
7937 };
7938
7939 match wait_outcome {
7940 RegistrationWaitOutcome::Registered => {
7941 child
7942 .as_mut()
7943 .expect("registered reload child")
7944 .registration_deadline = None;
7945 debug!(module_id = %spec.module_id, "supervised module reloaded and registered");
7946 Ok(())
7947 }
7948 RegistrationWaitOutcome::Exited(exit_report) => {
7949 if let Some(active_child) = child.as_mut() {
7950 active_child.drain_stderr(&spec.module_id).await;
7951 }
7952 let mut exited_child = child.take().expect("exited reload child is still stored");
7955 #[cfg(test)]
7956 if let Some(gate) = &runtime.test_reload_exit_record_gate {
7957 gate.reached.notify_one();
7958 gate.resume.notified().await;
7959 }
7960 let result = handle_reload_child_registration_failure(
7961 spec,
7962 runtime,
7963 registry,
7964 process_liveness,
7965 snapshot,
7966 child,
7967 ReloadRegistrationFailure {
7968 exit_report: registration_failure_exit_report(exit_report),
7969 reason: exited_child
7970 .spawn_failure
7971 .clone()
7972 .unwrap_or_else(|| "new child exited before registering".to_string()),
7973 },
7974 )
7975 .await;
7976 exited_child.release_roster();
7977 result
7978 }
7979 RegistrationWaitOutcome::TimedOut => {
7980 let mut timed_out_child = child
7981 .take()
7982 .expect("timed-out reload child is still running");
7983 timed_out_child
7984 .start_kill()
7985 .map_err(|source| SuperviseError::Kill {
7986 module_id: spec.module_id.clone(),
7987 source,
7988 })?;
7989 let status = timed_out_child
7990 .wait()
7991 .await
7992 .map_err(|source| SuperviseError::Wait {
7993 module_id: spec.module_id.clone(),
7994 source,
7995 })?;
7996 timed_out_child.drain_stderr(&spec.module_id).await;
7997 handle_reload_child_registration_failure(
7998 spec,
7999 runtime,
8000 registry,
8001 process_liveness,
8002 snapshot,
8003 child,
8004 ReloadRegistrationFailure {
8005 exit_report: registration_failure_exit_report(classify_reaped_child_exit(
8006 snapshot,
8007 &timed_out_child,
8008 &status,
8009 )),
8010 reason: format!(
8011 "new child did not register within {:?}",
8012 REGISTRY_RELEASE_TIMEOUT
8013 ),
8014 },
8015 )
8016 .await
8017 }
8018 }
8019}
8020
8021#[allow(clippy::too_many_arguments)]
8022async fn set_child_enabled(
8023 spec: &ModuleSpec,
8024 runtime: &SupervisorRuntimeConfig,
8025 registry: &Registry,
8026 process_liveness: &SupervisorProcessLiveness,
8027 snapshot: &SharedSnapshot,
8028 child: &mut Option<SupervisedChild>,
8029 enabled: bool,
8030 failure: &FailureContext,
8031) -> Result<bool, SuperviseError> {
8032 let (current_enabled, current_state, respawn_pending) = {
8033 let state = lock_snapshot(snapshot)?;
8034 (state.enabled, state.state, state.respawn_pending)
8035 };
8036 let revive_terminal = enabled
8044 && current_enabled
8045 && child.is_none()
8046 && (matches!(current_state, ModuleState::Failed | ModuleState::Stopped)
8047 || (current_state == ModuleState::Restarting && !respawn_pending));
8048 if current_enabled == enabled && !revive_terminal {
8049 return Ok(false);
8050 }
8051
8052 if enabled {
8053 update_snapshot(snapshot, Some(&spec.module_id), |state| {
8054 state.enabled = true;
8055 state.state = ModuleState::Starting;
8056 clear_current_process_facts(state);
8057 })?;
8058 #[cfg(test)]
8059 if runtime.test_seed_stale_facts_before_enable_spawn {
8060 update_snapshot(snapshot, Some(&spec.module_id), |state| {
8061 state.process_alive = true;
8062 state.pid = Some(41);
8063 state.spawned_at_ms = Some(42);
8064 state.spawned_from = Some(PathBuf::from("/spawned/module"));
8065 state.spawned_file_identity = Some(SpawnedFileIdentity {
8066 device: 43,
8067 inode: 44,
8068 #[cfg(windows)]
8069 windows_image: None,
8070 });
8071 })?;
8072 }
8073 release_dead_registration(
8074 registry,
8075 runtime.forwarding.as_deref(),
8076 snapshot,
8077 &spec.module_id,
8078 failure,
8079 )
8080 .await?;
8081 reset_restart_count(snapshot, &spec.module_id)?;
8082 process_liveness.track(spec.module_id.clone(), Arc::clone(snapshot));
8083 let next_child = match spawn_and_mark_running(spec, runtime, snapshot) {
8084 Ok(next_child) => next_child,
8085 Err(err) => {
8086 fail_snapshot(snapshot, Some(&spec.module_id), None, failure);
8087 process_liveness.untrack_if_current(&spec.module_id, snapshot);
8088 return Err(err);
8089 }
8090 };
8091 *child = Some(next_child);
8092 debug!(module_id = %spec.module_id, "supervised module enabled");
8093 Ok(true)
8094 } else {
8095 let stop_notice = begin_forwarding_drain_if_configured(
8096 spec,
8097 runtime,
8098 registry,
8099 snapshot,
8100 Some(false),
8101 RouteCloseReason::Disable,
8102 )
8103 .await?;
8104 drain_optional_child(
8105 &spec.module_id,
8106 spec.protocol,
8107 stop_notice,
8108 registry,
8109 runtime.forwarding.as_deref(),
8110 snapshot,
8111 &runtime.terminal_ring,
8112 &runtime.spawn_events,
8113 child,
8114 runtime.drain_timeout,
8115 ModuleState::Disabled,
8116 Some(false),
8117 failure,
8118 )
8119 .await?;
8120 debug!(module_id = %spec.module_id, "supervised module disabled");
8121 Ok(true)
8122 }
8123}
8124
8125#[allow(clippy::too_many_arguments)]
8126async fn on_child_exit(
8127 spec: &ModuleSpec,
8128 policy: RestartPolicy,
8129 registry: &Registry,
8130 snapshot: &SharedSnapshot,
8131 terminal_ring: &Arc<Mutex<TerminalRing>>,
8132 spawn_events: &SpawnEventFeed,
8133 roster: &ChildRoster,
8134 exit_report: ExitReport,
8135 daemon_initiated: bool,
8136) -> NextAction {
8137 if roster.is_closed() {
8143 return on_child_exit_during_daemon_shutdown(
8144 spec,
8145 registry,
8146 snapshot,
8147 terminal_ring,
8148 spawn_events,
8149 exit_report,
8150 )
8151 .await;
8152 }
8153 let unrequested_clean_exit_of_protocol_none = exit_report.kind == ExitKind::Clean
8169 && running_protocol(spec, snapshot) == ModuleProtocol::None;
8170 match exit_report.kind {
8171 ExitKind::Clean if !unrequested_clean_exit_of_protocol_none => {
8172 info!(
8173 module_id = %spec.module_id,
8174 exit_code = ?exit_report.code,
8175 exit_signal = ?exit_report.signal,
8176 "supervised module exited cleanly"
8177 );
8178 if let Err(err) = publish_after_terminal_record(
8179 &spec.module_id,
8180 || {
8181 record_terminal(
8182 &spec.module_id,
8183 terminal_ring,
8184 spawn_events,
8185 &exit_report,
8186 TerminalDisposition::Stopped,
8187 )
8188 },
8189 || {
8190 update_snapshot(snapshot, Some(&spec.module_id), |state| {
8191 state.state = ModuleState::Stopped;
8192 clear_current_process_facts(state);
8193 state.last_exit = Some(exit_report.clone());
8194 })
8195 },
8196 ) {
8197 error!(module_id = %spec.module_id, error = %err, "failed to publish clean module exit");
8198 }
8199 let registration_released = match wait_for_registration_release(
8200 registry,
8201 &spec.module_id,
8202 REGISTRY_RELEASE_TIMEOUT,
8203 )
8204 .await
8205 {
8206 Ok(()) => true,
8207 Err(err) => {
8208 warn!(module_id = %spec.module_id, error = %err, "registration still active after clean exit");
8209 false
8210 }
8211 };
8212 NextAction::Stop {
8213 registration_released,
8214 }
8215 }
8216 ExitKind::Clean | ExitKind::Crash => {
8217 if unrequested_clean_exit_of_protocol_none {
8218 warn!(
8219 module_id = %spec.module_id,
8220 exit_code = ?exit_report.code,
8221 exit_signal = ?exit_report.signal,
8222 "protocol-none module exited cleanly without a stop request; handling it as a crash"
8223 );
8224 } else {
8225 warn!(
8226 module_id = %spec.module_id,
8227 exit_code = ?exit_report.code,
8228 exit_signal = ?exit_report.signal,
8229 "supervised module exited abnormally (crash)"
8230 );
8231 }
8232 let mut restart_schedule = None;
8233 let mut disposition = TerminalDisposition::Disabled;
8234 let mut final_state = ModuleState::Disabled;
8235 let mut disposition_detail = lock_snapshot(snapshot)
8239 .ok()
8240 .and_then(|mut state| state.spawn_failure.take());
8241 let now = Instant::now();
8242 if let Err(err) = update_snapshot(snapshot, Some(&spec.module_id), |state| {
8243 if state.enabled {
8244 if let Some(schedule) = state.next_crash_restart(&policy, now) {
8245 state.respawn_pending = true;
8251 final_state = ModuleState::Restarting;
8252 restart_schedule = Some(schedule);
8253 disposition = TerminalDisposition::Restarting;
8254 } else {
8255 final_state = ModuleState::Failed;
8256 disposition = TerminalDisposition::Failed;
8257 let budget = policy.budget_exhausted_detail();
8258 disposition_detail =
8259 Some(disposition_detail.take().map_or_else(
8260 || budget.clone(),
8261 |cause| format!("{cause}; {budget}"),
8262 ));
8263 }
8264 } else {
8265 disposition = TerminalDisposition::Disabled;
8266 }
8267 }) {
8268 error!(module_id = %spec.module_id, error = %err, "failed to record crashed module exit");
8269 return NextAction::Stop {
8270 registration_released: false,
8271 };
8272 }
8273 if disposition == TerminalDisposition::Failed {
8274 error!(
8279 module_id = %spec.module_id,
8280 max_restarts = policy.max_restarts,
8281 window_secs = policy.window.as_secs(),
8282 "module stopped: {}",
8283 policy.budget_exhausted_detail()
8284 );
8285 }
8286 if let Err(err) = publish_after_terminal_record(
8287 &spec.module_id,
8288 || {
8289 record_terminal_with_detail(
8290 &spec.module_id,
8291 terminal_ring,
8292 spawn_events,
8293 &exit_report,
8294 disposition,
8295 disposition_detail,
8296 )
8297 },
8298 || {
8299 update_snapshot(snapshot, Some(&spec.module_id), |state| {
8300 state.state = final_state;
8301 clear_current_process_facts(state);
8302 if final_state == ModuleState::Failed
8303 && daemon_initiated
8304 && !roster.is_closed()
8305 {
8306 state.failed_retry.schedule(&spec.module_id);
8307 }
8308 state.last_exit = Some(exit_report.clone());
8309 })
8310 },
8311 ) {
8312 error!(module_id = %spec.module_id, error = %err, "failed to publish crashed module exit");
8313 }
8314
8315 if let Some(schedule) = restart_schedule {
8316 NextAction::Restart {
8317 schedule: Some(schedule),
8318 }
8319 } else {
8320 let registration_released = match wait_for_registration_release(
8321 registry,
8322 &spec.module_id,
8323 REGISTRY_RELEASE_TIMEOUT,
8324 )
8325 .await
8326 {
8327 Ok(()) => true,
8328 Err(err) => {
8329 warn!(module_id = %spec.module_id, error = %err, "registration still active after failed module");
8330 false
8331 }
8332 };
8333 NextAction::Stop {
8334 registration_released,
8335 }
8336 }
8337 }
8338 ExitKind::DeliberateSeverance => {
8339 warn!(
8340 module_id = %spec.module_id,
8341 exit_code = ?exit_report.code,
8342 exit_signal = ?exit_report.signal,
8343 "supervised module exited after deliberate connection severance"
8344 );
8345 let should_restart = match lock_snapshot(snapshot) {
8346 Ok(state) => state.enabled,
8347 Err(err) => {
8348 error!(module_id = %spec.module_id, error = %err, "failed to read deliberately severed module state");
8349 return NextAction::Stop {
8350 registration_released: false,
8351 };
8352 }
8353 };
8354 let final_state = if should_restart {
8355 ModuleState::Restarting
8356 } else {
8357 ModuleState::Disabled
8358 };
8359 if let Err(err) = publish_after_terminal_record(
8360 &spec.module_id,
8361 || {
8362 record_terminal(
8363 &spec.module_id,
8364 terminal_ring,
8365 spawn_events,
8366 &exit_report,
8367 terminal_disposition(final_state),
8368 )
8369 },
8370 || {
8371 update_snapshot(snapshot, Some(&spec.module_id), |state| {
8372 state.state = final_state;
8373 clear_current_process_facts(state);
8374 state.last_exit = Some(exit_report.clone());
8375 state.lifetime_restarts += 1;
8376 })
8377 },
8378 ) {
8379 error!(module_id = %spec.module_id, error = %err, "failed to publish deliberately severed module exit");
8380 return NextAction::Stop {
8381 registration_released: false,
8382 };
8383 }
8384
8385 if should_restart {
8386 NextAction::Restart { schedule: None }
8387 } else {
8388 let registration_released = match wait_for_registration_release(
8389 registry,
8390 &spec.module_id,
8391 REGISTRY_RELEASE_TIMEOUT,
8392 )
8393 .await
8394 {
8395 Ok(()) => true,
8396 Err(err) => {
8397 warn!(module_id = %spec.module_id, error = %err, "registration still active after deliberately severed module exit");
8398 false
8399 }
8400 };
8401 NextAction::Stop {
8402 registration_released,
8403 }
8404 }
8405 }
8406 }
8407}
8408
8409async fn on_child_exit_during_daemon_shutdown(
8410 spec: &ModuleSpec,
8411 registry: &Registry,
8412 snapshot: &SharedSnapshot,
8413 terminal_ring: &Arc<Mutex<TerminalRing>>,
8414 spawn_events: &SpawnEventFeed,
8415 exit_report: ExitReport,
8416) -> NextAction {
8417 info!(
8418 module_id = %spec.module_id,
8419 exit_code = ?exit_report.code,
8420 exit_signal = ?exit_report.signal,
8421 exit_kind = ?exit_report.kind,
8422 "supervised module exited during daemon shutdown; not restarting it"
8423 );
8424 if let Err(err) = publish_after_terminal_record(
8425 &spec.module_id,
8426 || {
8427 record_terminal(
8428 &spec.module_id,
8429 terminal_ring,
8430 spawn_events,
8431 &exit_report,
8432 TerminalDisposition::DaemonShutdown,
8433 )
8434 },
8435 || {
8436 update_snapshot(snapshot, Some(&spec.module_id), |state| {
8437 state.state = ModuleState::Stopped;
8438 clear_current_process_facts(state);
8439 state.last_exit = Some(exit_report.clone());
8440 })
8441 },
8442 ) {
8443 error!(module_id = %spec.module_id, error = %err, "failed to publish module exit during daemon shutdown");
8444 }
8445 let registration_released =
8446 wait_for_registration_release(registry, &spec.module_id, REGISTRY_RELEASE_TIMEOUT)
8447 .await
8448 .is_ok();
8449 NextAction::Stop {
8450 registration_released,
8451 }
8452}
8453
8454fn publish_after_terminal_record<T>(
8459 module_id: &str,
8460 record: impl FnOnce(),
8461 publish: impl FnOnce() -> T,
8462) -> T {
8463 let recorded = std::panic::catch_unwind(std::panic::AssertUnwindSafe(record));
8464 let published = publish();
8465 if let Err(panic) = recorded {
8466 error!(
8467 module_id,
8468 "terminal recording panicked; published final state before resuming unwind"
8469 );
8470 std::panic::resume_unwind(panic);
8471 }
8472 published
8473}
8474
8475fn fail_after_wait_error(
8476 module_id: &str,
8477 snapshot: &SharedSnapshot,
8478 terminal_ring: &Arc<Mutex<TerminalRing>>,
8479 spawn_events: &SpawnEventFeed,
8480 detail: String,
8481 failure: &FailureContext,
8482) {
8483 let report = wait_error_exit_report();
8484 publish_after_terminal_record(
8485 module_id,
8486 || {
8487 record_terminal_with_detail(
8488 module_id,
8489 terminal_ring,
8490 spawn_events,
8491 &report,
8492 TerminalDisposition::Failed,
8493 Some(detail),
8494 )
8495 },
8496 || fail_snapshot(snapshot, Some(module_id), Some(report.clone()), failure),
8497 );
8498}
8499
8500#[cfg(test)]
8501fn record_wait_error_terminal(
8502 module_id: &str,
8503 terminal_ring: &Arc<Mutex<TerminalRing>>,
8504 spawn_events: &SpawnEventFeed,
8505) {
8506 record_terminal(
8507 module_id,
8508 terminal_ring,
8509 spawn_events,
8510 &wait_error_exit_report(),
8511 TerminalDisposition::Failed,
8512 );
8513}
8514
8515fn record_terminal(
8516 module_id: &str,
8517 terminal_ring: &Arc<Mutex<TerminalRing>>,
8518 spawn_events: &SpawnEventFeed,
8519 exit_report: &ExitReport,
8520 disposition: TerminalDisposition,
8521) {
8522 record_terminal_with_detail(
8523 module_id,
8524 terminal_ring,
8525 spawn_events,
8526 exit_report,
8527 disposition,
8528 None,
8529 );
8530}
8531
8532fn durable_terminal_history_of(
8536 terminal_ring: &Mutex<TerminalRing>,
8537 module_id: &str,
8538) -> subc_control::TerminalHistory {
8539 let read = terminal_ring
8540 .lock()
8541 .unwrap_or_else(|p| p.into_inner())
8542 .capture_durable_history();
8543 read.read(module_id)
8544}
8545
8546fn record_terminal_with_detail(
8547 module_id: &str,
8548 terminal_ring: &Arc<Mutex<TerminalRing>>,
8549 spawn_events: &SpawnEventFeed,
8550 exit_report: &ExitReport,
8551 disposition: TerminalDisposition,
8552 disposition_detail: Option<String>,
8553) {
8554 #[cfg(test)]
8555 terminal_history_tests::before_terminal_record(spawn_events);
8556 spawn_events.emit_exited(module_id, exit_report.code, exit_report.signal);
8557 let record = TerminalRecord {
8558 exit_code: exit_report.code,
8559 exit_signal: exit_report.signal,
8560 at_ms: exit_report.at_ms,
8561 disposition,
8562 exit_kind: exit_report.kind.into(),
8563 disposition_detail,
8564 };
8565 terminal_ring
8566 .lock()
8567 .unwrap_or_else(|poisoned| poisoned.into_inner())
8568 .record_exit(module_id, record);
8569}
8570
8571fn untrack_if_registration_released(
8572 process_liveness: &SupervisorProcessLiveness,
8573 registry: &Registry,
8574 module_id: &str,
8575 snapshot: &SharedSnapshot,
8576) {
8577 match registry.get_module(module_id) {
8578 Ok(None) => process_liveness.untrack_if_current(module_id, snapshot),
8579 Ok(Some(_)) => {}
8580 Err(err) => {
8581 warn!(module_id, error = %err, "could not determine whether supervisor liveness can be untracked");
8582 }
8583 }
8584}
8585
8586#[cfg(test)]
8600fn apply_wire_spawn_args(
8601 command: &mut Command,
8602 spec: &ModuleSpec,
8603 connection_file_path: Option<&std::path::Path>,
8604 handle: Option<&SupervisorHandle>,
8605) -> Result<Option<NonceHandoff>, SuperviseError> {
8606 apply_wire_spawn_args_for_role(
8607 command,
8608 spec,
8609 connection_file_path,
8610 handle,
8611 SpawnRole::Plain,
8612 )
8613}
8614
8615#[cfg(unix)]
8620type NonceHandoff = subc_os::LaunchNonceHandoff;
8621#[cfg(windows)]
8622type NonceHandoff = subc_os::LaunchNoncePipeHandoff;
8623#[cfg(not(any(unix, windows)))]
8624type NonceHandoff = std::convert::Infallible;
8625
8626fn apply_wire_spawn_args_for_role(
8647 command: &mut Command,
8648 spec: &ModuleSpec,
8649 connection_file_path: Option<&std::path::Path>,
8650 handle: Option<&SupervisorHandle>,
8651 role: SpawnRole,
8652) -> Result<Option<NonceHandoff>, SuperviseError> {
8653 command.env(SUBC_MODULE_ID_ENV, &spec.module_id);
8654 command.env_remove(subc_os::LAUNCH_NONCE_FD_ENV);
8659 #[cfg(windows)]
8660 {
8661 command.env_remove(subc_os::LAUNCH_NONCE_PIPE_ENV);
8662 command.env_remove(subc_os::LAUNCH_NONCE_PIPE_FALLBACK_ENV);
8663 }
8664 command.env_remove(SUBC_LAUNCH_NONCE_ENV);
8666 if spec.protocol == ModuleProtocol::None {
8667 return Ok(None);
8668 }
8669 if let Some(connection_file_path) = connection_file_path {
8670 command.arg(SUBC_ARG).arg(connection_file_path);
8671 }
8672
8673 let nonce = generate_launch_nonce()?;
8677 if let Some(handle) = handle {
8678 match role {
8679 SpawnRole::Plain => {
8680 handle.set_spawn_nonce(&spec.module_id, nonce.clone());
8681 if spec.reserved {
8682 handle.set_reserved_nonce(&spec.module_id, nonce.clone());
8683 }
8684 }
8685 SpawnRole::SwapCandidate => handle.open_swap(&spec.module_id, nonce.clone()),
8686 }
8687 }
8688 #[cfg(unix)]
8689 let handoff = {
8690 let handoff =
8691 subc_os::LaunchNonceHandoff::new(&nonce).map_err(|source| SuperviseError::Spawn {
8692 program: spec.program.clone(),
8693 source,
8694 cgroup_path: None,
8695 })?;
8696 command.env(subc_os::LAUNCH_NONCE_FD_ENV, handoff.fd_env_value());
8697 Some(handoff)
8698 };
8699 #[cfg(windows)]
8700 let handoff = prepare_windows_nonce_pipe(command, spec, &nonce);
8701 #[cfg(not(any(unix, windows)))]
8702 let handoff = None;
8703 #[cfg(not(unix))]
8707 command.env(SUBC_LAUNCH_NONCE_ENV, nonce);
8708 Ok(handoff)
8709}
8710
8711fn apply_child_env(command: &mut Command, spec: &ModuleSpec) {
8712 command.env_remove(CK_LOG_ENV);
8713 command.env_remove(SUBC_SPAWN_ROLE_ENV);
8720 for (key, value) in &spec.env {
8721 if matches!(
8725 key.as_str(),
8726 CAPTURE_MAX_FILE_MB_ENV | CAPTURE_KEEP_ENV | CAPTURE_MAX_AGE_DAYS_ENV
8727 ) || key == SUBC_SPAWN_ROLE_ENV
8728 {
8729 continue;
8730 }
8731 command.env(key, value);
8732 }
8733}
8734
8735#[cfg(windows)]
8736fn prepare_windows_nonce_pipe(
8737 command: &mut Command,
8738 spec: &ModuleSpec,
8739 nonce: &str,
8740) -> Option<NonceHandoff> {
8741 match subc_os::LaunchNoncePipeHandoff::new(nonce) {
8742 Ok(handoff) => {
8743 command.env(subc_os::LAUNCH_NONCE_PIPE_ENV, handoff.name());
8744 command.env(subc_os::LAUNCH_NONCE_PIPE_FALLBACK_ENV, "env");
8749 Some(handoff)
8750 }
8751 Err(error) => {
8752 warn!(module_id = %spec.module_id, %error, "could not prepare launch nonce pipe; retaining the environment handoff");
8753 None
8754 }
8755 }
8756}
8757
8758#[cfg(windows)]
8759fn start_windows_nonce_pipe(
8760 handoff: Option<NonceHandoff>,
8761 pid: u32,
8762 spec: &ModuleSpec,
8763) -> Option<subc_os::LaunchNoncePipeDelivery> {
8764 handoff.and_then(|handoff| match handoff.serve(pid, REGISTRY_RELEASE_TIMEOUT) {
8768 Ok(delivery) => Some(delivery),
8769 Err(error) => {
8770 warn!(module_id = %spec.module_id, %error, "could not serve launch nonce pipe; retaining the environment handoff");
8771 None
8772 }
8773 })
8774}
8775
8776fn observed_launch_nonce_source(snapshot: &SupervisorSnapshot) -> Option<String> {
8777 #[cfg(windows)]
8778 if snapshot.process_alive && snapshot.spawned_protocol == Some(ModuleProtocol::Subc) {
8779 return Some(
8780 if snapshot
8781 .pipe_consumed
8782 .as_ref()
8783 .is_some_and(|consumed| consumed.get().is_some())
8784 {
8785 "pipe"
8786 } else {
8787 "env"
8788 }
8789 .to_owned(),
8790 );
8791 }
8792 let _ = snapshot;
8793 None
8794}
8795
8796#[derive(Debug, Clone, Copy, PartialEq, Eq)]
8799enum SpawnRole {
8800 Plain,
8801 SwapCandidate,
8802}
8803
8804fn apply_spawn_role(command: &mut Command, role: SpawnRole) {
8807 if role == SpawnRole::SwapCandidate {
8808 command.env(SUBC_SPAWN_ROLE_ENV, SPAWN_ROLE_SWAP_CANDIDATE);
8809 }
8810}
8811
8812fn previous_instance_error(
8813 spec: &ModuleSpec,
8814 pid: u32,
8815 reason: crate::live_children::RetainedReason,
8816) -> SuperviseError {
8817 SuperviseError::Spawn {
8818 program: spec.program.clone(),
8819 source: io::Error::new(io::ErrorKind::WouldBlock, reason.waiting_message(pid)),
8820 cgroup_path: None,
8821 }
8822}
8823
8824fn spawn_child(
8825 spec: &ModuleSpec,
8826 connection_file_path: Option<&std::path::Path>,
8827 handle: Option<&SupervisorHandle>,
8828 ring: &Arc<Mutex<StderrRing>>,
8829 capture_logs_dir: Option<&std::path::Path>,
8830 roster: &ChildRoster,
8831 #[cfg(target_os = "linux")] cgroup_placement: Option<&subc_cgroup::Placement>,
8832) -> Result<SupervisedChild, SuperviseError> {
8833 spawn_child_in_slot(
8834 spec,
8835 connection_file_path,
8836 handle,
8837 ring,
8838 capture_logs_dir,
8839 roster,
8840 #[cfg(target_os = "linux")]
8841 cgroup_placement,
8842 SpawnRole::Plain,
8843 false,
8844 )
8845}
8846
8847#[allow(clippy::too_many_arguments)]
8860fn spawn_child_in_slot(
8861 spec: &ModuleSpec,
8862 connection_file_path: Option<&std::path::Path>,
8863 handle: Option<&SupervisorHandle>,
8864 ring: &Arc<Mutex<StderrRing>>,
8865 capture_logs_dir: Option<&std::path::Path>,
8866 roster: &ChildRoster,
8867 #[cfg(target_os = "linux")] cgroup_placement: Option<&subc_cgroup::Placement>,
8868 role: SpawnRole,
8869 alternate_slot: bool,
8870) -> Result<SupervisedChild, SuperviseError> {
8871 if let Some((pid, reason)) = roster.previous_instance(&spec.module_id) {
8872 return Err(previous_instance_error(spec, pid, reason));
8873 }
8874 if roster.is_closed() {
8875 return Err(SuperviseError::Spawn {
8876 program: spec.program.clone(),
8877 source: io::Error::other("the daemon is shutting down; not starting a new process"),
8878 cgroup_path: None,
8879 });
8880 }
8881 #[cfg(target_os = "linux")]
8882 let cgroup_name = {
8883 if cgroup_placement.is_none() {
8888 swap::cgroup_name(&spec.module_id, alternate_slot)
8889 } else {
8890 let nonce = generate_launch_nonce()?;
8891 let suffix = u128::from_str_radix(&nonce[..32], 16).expect("hex launch nonce");
8892 let mut end = spec.module_id.len().min(64);
8895 while !spec.module_id.is_char_boundary(end) {
8896 end -= 1;
8897 }
8898 format!(
8899 "{}_{suffix}",
8900 swap::cgroup_name(&spec.module_id[..end], alternate_slot)
8901 )
8902 }
8903 };
8904 #[cfg(not(target_os = "linux"))]
8905 let _ = alternate_slot;
8906 #[cfg(target_os = "macos")]
8907 let (mut command, privacy_exec, exec_ack) = privacy_command(spec, roster)?;
8908 #[cfg(windows)]
8909 let image_capture = windows_spawn_capture(spec).map_err(|source| SuperviseError::Spawn {
8910 program: spec.program.clone(),
8911 source,
8912 cgroup_path: None,
8913 })?;
8914 #[cfg(windows)]
8915 let mut command = Command::new(image_capture.path());
8916 #[cfg(not(any(target_os = "macos", windows)))]
8917 let mut command = Command::new(&spec.program);
8918 command.args(&spec.args);
8919 apply_child_env(&mut command, spec);
8949 apply_spawn_role(&mut command, role);
8950 let nonce_handoff =
8951 apply_wire_spawn_args_for_role(&mut command, spec, connection_file_path, handle, role)?;
8952
8953 #[cfg(target_os = "linux")]
8954 let cgroup_path = cgroup_placement
8955 .map(|placement| placement.module_path(&cgroup_name))
8956 .transpose()
8957 .map_err(|source| SuperviseError::Cgroup {
8958 module_id: spec.module_id.clone(),
8959 source,
8960 })?;
8961 #[cfg(not(target_os = "linux"))]
8962 let cgroup_path: Option<PathBuf> = None;
8963 #[cfg(target_os = "linux")]
8964 if let Some(path) = &cgroup_path {
8965 if let Err(error) = apply_cgroup_placement(&mut command, spec, path) {
8966 if let Some(placement) = cgroup_placement {
8967 remove_module_cgroup(placement, &cgroup_name);
8968 }
8969 return Err(error);
8970 }
8971 }
8972
8973 let output_sink = if let Some(logs_dir) = capture_logs_dir {
8974 let path = logs_dir.join(format!("{}.stderr.log", spec.module_id));
8975 match ChildOutputSink::open(&path, capture_retention(spec)) {
8976 Ok(sink) => sink,
8977 Err(error) => {
8978 warn!(
8979 module_id = %spec.module_id,
8980 path = %path.display(),
8981 error = %error,
8982 "could not open child output capture file; forwarding to stderr"
8983 );
8984 ChildOutputSink::Stderr
8985 }
8986 }
8987 } else {
8988 ChildOutputSink::Stderr
8989 };
8990
8991 command.stdout(Stdio::piped());
8992 command.stderr(Stdio::piped());
8993 command.kill_on_drop(true);
8994 #[cfg(unix)]
9011 command.process_group(0);
9012 command.stdin(Stdio::null());
9013 #[cfg(unix)]
9017 if let Some(handoff) = nonce_handoff {
9018 handoff.install_last(command.as_std_mut());
9019 }
9020 #[cfg(not(any(unix, windows)))]
9021 let _ = nonce_handoff;
9022
9023 #[cfg(windows)]
9028 {
9029 subc_jobobject::ensure_root_job().map_err(|source| SuperviseError::Spawn {
9030 program: spec.program.clone(),
9031 source,
9032 cgroup_path: None,
9033 })?;
9034 subc_jobobject::suspend_on_create_async(&mut command);
9035 }
9036 let spawned_at = Instant::now();
9040 let mut child = match command.spawn() {
9041 Ok(child) => child,
9042 Err(source) => {
9043 #[cfg(target_os = "linux")]
9044 if let Some(placement) = cgroup_placement {
9045 remove_module_cgroup(placement, &cgroup_name);
9046 }
9047 return Err(SuperviseError::Spawn {
9048 program: spec.program.clone(),
9049 source,
9050 cgroup_path,
9051 });
9052 }
9053 };
9054 #[cfg(test)]
9055 spawn_record_tests::after_create(&child, spec);
9056 #[cfg(target_os = "macos")]
9061 drop(exec_ack);
9062
9063 #[cfg(windows)]
9068 let windows_image = match image_capture.bind_tokio(&child) {
9069 Ok(image) => Arc::new(image),
9070 Err(source) => {
9071 kill_and_reap_rejected_child(&mut child);
9072 return Err(SuperviseError::Spawn {
9073 program: spec.program.clone(),
9074 source,
9075 cgroup_path: None,
9076 });
9077 }
9078 };
9079 #[cfg(windows)]
9082 let job = contain_spawned_child(&mut child, spec)?;
9083 let spawned_at_ms = unix_ms_now();
9084 #[cfg(not(windows))]
9085 let spawned_from = spec.program.clone();
9086 #[cfg(windows)]
9087 let spawned_from = windows_image.path().to_path_buf();
9088 #[cfg(not(windows))]
9089 let spawned_file_identity = spawned_file_identity(&spawned_from);
9090 #[cfg(windows)]
9091 let spawned_file_identity = Some(SpawnedFileIdentity {
9092 device: 0,
9093 inode: 0,
9094 windows_image: Some(Arc::clone(&windows_image)),
9095 });
9096 let pid = child.id().ok_or_else(|| SuperviseError::Spawn {
9097 program: spec.program.clone(),
9098 source: io::Error::other("spawned child exposed no live pid"),
9099 cgroup_path: cgroup_path.clone(),
9100 })?;
9101 #[cfg(windows)]
9102 let nonce_delivery = start_windows_nonce_pipe(nonce_handoff, pid, spec);
9103 #[cfg(not(windows))]
9104 let process_start_time = crate::provenance::process_start_time(pid);
9105 #[cfg(windows)]
9106 let process_start_time = Some(windows_image.start_time());
9107 let process_identity = process_start_time.map(|start_time| ProcessIdentity { pid, start_time });
9108 #[cfg(all(test, target_os = "macos"))]
9109 privacy_exec_boundary_tests::before_image_sample(spec, pid);
9110 #[cfg(not(windows))]
9115 let recorded_image = observe_spawned_image(pid);
9116 #[cfg(target_os = "macos")]
9120 let recorded_image = if privacy_exec.is_some() {
9121 None
9122 } else {
9123 recorded_image
9124 };
9125 #[cfg(target_os = "linux")]
9126 let recorded_cgroup_name = cgroup_path.as_ref().map(|_| cgroup_name.clone());
9127 #[cfg(not(target_os = "linux"))]
9128 let recorded_cgroup_name = None;
9129 let roster_guard = roster.admit(
9130 spec.module_id.clone(),
9131 pid,
9132 spec.protocol,
9133 process_start_time,
9134 crate::child_roster::RecordedIdentity {
9135 #[cfg(not(windows))]
9136 start_time: recorded_image.map(|image| image.start_time),
9137 #[cfg(windows)]
9138 start_time: process_start_time,
9139 #[cfg(not(windows))]
9140 executable: recorded_image
9141 .and_then(|image| image.executable)
9142 .map(crate::live_children::ExecutableIdentity::from),
9143 #[cfg(windows)]
9144 executable: Some(crate::live_children::ExecutableIdentity::from(
9145 windows_image.identity(),
9146 )),
9147 cgroup_name: recorded_cgroup_name,
9148 #[cfg(target_os = "linux")]
9149 cgroup_placement: cgroup_placement.cloned(),
9150 },
9151 );
9152 let roster_guard = match roster_guard {
9153 Ok(guard) => guard,
9154 Err(failure) => {
9155 return failure.with_cleanup(|source| {
9156 #[cfg(windows)]
9157 if let Some(job) = &job {
9158 let _ = job.terminate();
9159 }
9160 #[cfg(target_os = "linux")]
9161 kill_module_cgroup(cgroup_placement, &cgroup_name);
9162 kill_and_reap_rejected_child(&mut child);
9163 #[cfg(target_os = "linux")]
9164 if let Some(placement) = cgroup_placement {
9165 remove_module_cgroup(placement, &cgroup_name);
9166 }
9167 Err(SuperviseError::Spawn {
9168 program: spec.program.clone(),
9169 source: io::Error::other(format!(
9170 "could not durably record module child: {source}"
9171 )),
9172 cgroup_path,
9173 })
9174 });
9175 }
9176 };
9177 if roster.is_closed() {
9182 #[cfg(target_os = "linux")]
9184 kill_module_cgroup(cgroup_placement, &cgroup_name);
9185 if let Err(error) = child.start_kill() {
9186 debug!(module_id = %spec.module_id, pid, %error, "kill of a process spawned during daemon shutdown failed; it may already have exited");
9187 }
9188 #[cfg(target_os = "linux")]
9189 if let Some(placement) = cgroup_placement {
9190 while matches!(child.try_wait(), Ok(None)) {
9194 std::thread::yield_now();
9195 }
9196 if matches!(
9197 subc_cgroup::kill_module(Some(placement), &cgroup_name),
9198 subc_cgroup::KillOutcome::Killed
9199 ) {
9200 if let Ok(path) = placement.module_path(&cgroup_name) {
9201 while std::fs::read_to_string(path.join("cgroup.events"))
9202 .ok()
9203 .is_some_and(|events| events.lines().any(|line| line == "populated 1"))
9204 {
9205 std::thread::yield_now();
9206 }
9207 }
9208 }
9209 remove_module_cgroup(placement, &cgroup_name);
9210 }
9211 drop(roster_guard);
9212 return Err(SuperviseError::Spawn {
9213 program: spec.program.clone(),
9214 source: io::Error::other(
9215 "the daemon began shutting down while this process was starting; ended it",
9216 ),
9217 cgroup_path,
9218 });
9219 }
9220
9221 #[cfg(windows)]
9222 if let Err(error) = resume_suspended_child(pid, spec) {
9223 kill_and_reap_rejected_child(&mut child);
9224 return Err(error);
9225 }
9226
9227 let stdout_pump = match child.stdout.take() {
9228 Some(stdout) => Some(tokio::spawn(pump_stdout_to(stdout, output_sink.clone()))),
9229 None => {
9230 warn!(
9231 module_id = %spec.module_id,
9232 "spawned child exposed no stdout pipe; file capture will be incomplete"
9233 );
9234 None
9235 }
9236 };
9237 let stderr_pump = match child.stderr.take() {
9238 Some(stderr) => {
9239 let generation = ring
9240 .lock()
9241 .unwrap_or_else(|poisoned| poisoned.into_inner())
9242 .begin_process();
9243 Some(StderrPump {
9244 task: tokio::spawn(pump_stderr_to(
9245 stderr,
9246 Arc::clone(ring),
9247 generation,
9248 output_sink,
9249 )),
9250 generation,
9251 })
9252 }
9253 None => {
9254 ring.lock()
9258 .unwrap_or_else(|poisoned| poisoned.into_inner())
9259 .mark_not_captured("stderr pipe was not available on spawn");
9260 warn!(
9261 module_id = %spec.module_id,
9262 "spawned child exposed no stderr pipe; tail will be unavailable"
9263 );
9264 None
9265 }
9266 };
9267
9268 let exec = Arc::new(crate::activity::ExecOutcome::new());
9269 #[cfg(target_os = "macos")]
9270 let pending_exec = privacy_exec.is_some();
9271 #[cfg(not(target_os = "macos"))]
9272 let pending_exec = false;
9273 if !pending_exec {
9274 exec.finish(crate::activity::ExecState::CommittedExec);
9275 }
9276 #[cfg(windows)]
9277 let job = job.map(|job| Arc::new(Mutex::new(job)));
9278 #[cfg(windows)]
9279 let subtree_target = job
9280 .as_ref()
9281 .map(|job| crate::subtree_resources::Target::Job(Arc::downgrade(job)))
9282 .unwrap_or(crate::subtree_resources::Target::Unavailable("no_job"));
9283 #[cfg(target_os = "linux")]
9284 let subtree_target = cgroup_path
9285 .map(crate::subtree_resources::Target::Cgroup)
9286 .unwrap_or(crate::subtree_resources::Target::Unavailable("no_cgroup"));
9287 #[cfg(not(any(target_os = "linux", windows)))]
9288 let subtree_target = crate::subtree_resources::Target::Unavailable("unsupported_platform");
9289 Ok(SupervisedChild {
9290 subtree_target,
9291 child,
9292 exec: crate::activity::ExecOwner(exec),
9293 #[cfg(windows)]
9294 nonce_delivery,
9295 protocol: spec.protocol,
9296 #[cfg(target_os = "linux")]
9297 module_id: cgroup_name,
9298 #[cfg(target_os = "linux")]
9299 cgroup_placement: cgroup_placement.cloned(),
9300 #[cfg(windows)]
9301 job,
9302 stdout_pump,
9303 stderr_pump,
9304 stderr_ring: Arc::clone(ring),
9305 spawned_at,
9306 registration_deadline: None,
9307 registration_watch: None,
9308 spawned_at_ms,
9309 spawned_from,
9310 launch_spec_digest: LaunchSpecDigest::from_spec(spec),
9311 spawned_file_identity,
9312 process_start_time,
9313 process_identity,
9314 pid,
9315 roster_guard: Some(roster_guard),
9316 #[cfg(target_os = "macos")]
9317 privacy_exec,
9318 #[cfg(target_os = "macos")]
9319 report_ready: Arc::new(OnceLock::new()),
9320 spawn_failure: None,
9321 })
9322}
9323
9324#[cfg(test)]
9325mod spawn_record_tests {
9326 use super::*;
9327
9328 thread_local! {
9329 static LAST_PID: std::cell::Cell<Option<u32>> = const { std::cell::Cell::new(None) };
9330 }
9331
9332 pub(super) fn after_create(child: &Child, _spec: &ModuleSpec) {
9333 LAST_PID.set(child.id());
9334 #[cfg(windows)]
9335 if let Some((_, address)) = _spec
9336 .env
9337 .iter()
9338 .find(|(key, _)| key == "SUBC_TEST_AFTER_CREATE")
9339 {
9340 use std::io::{Read, Write};
9341 let mut stream = std::net::TcpStream::connect(address).unwrap();
9342 stream
9343 .write_all(&child.id().unwrap().to_ne_bytes())
9344 .unwrap();
9345 let mut release = [0];
9346 stream.read_exact(&mut release).unwrap();
9347 }
9348 }
9349
9350 #[cfg(windows)]
9351 pub(super) fn assignment_failpoint(spec: &ModuleSpec) -> io::Result<()> {
9352 if spec
9353 .env
9354 .iter()
9355 .any(|(key, _)| key == "SUBC_TEST_ASSIGNMENT_FAILURE")
9356 {
9357 Err(io::Error::other("injected module job assignment failure"))
9358 } else {
9359 Ok(())
9360 }
9361 }
9362
9363 fn spec() -> ModuleSpec {
9364 ModuleSpec {
9365 module_id: "spawn-record-worker".into(),
9366 #[cfg(unix)]
9367 program: "/bin/sleep".into(),
9368 #[cfg(windows)]
9369 program: std::env::current_exe().unwrap(),
9370 #[cfg(unix)]
9371 args: vec!["60".into()],
9372 #[cfg(windows)]
9373 args: vec![
9374 "--exact".into(),
9375 "supervise::spawn_record_tests::windows_store_touch_fixture".into(),
9376 "--nocapture".into(),
9377 ],
9378 env: vec![],
9379 reserved: false,
9380 reserved_prefixes: vec![],
9381 protocol: ModuleProtocol::None,
9382 overlap: Default::default(),
9383 }
9384 }
9385
9386 #[tokio::test]
9387 async fn record_sync_failure_fails_spawn_and_reaps_child() {
9388 let home = cortexkit_test_support::ScratchDir::new("record-sync-failure");
9389 let supervisor =
9390 Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default())
9391 .with_live_children_record(home.join("live-children.json"));
9392 crate::live_children::fail_next_file_sync();
9393 let result = supervisor.spawn(spec());
9394 let failed = result.is_err();
9395 if let Ok(module) = result {
9396 module.stop().await.unwrap();
9397 }
9398 assert!(failed, "record failure must fail spawn");
9399 let pid = LAST_PID.get().expect("the real OS child was created");
9400 assert!(
9401 subc_os::Process::open(pid).unwrap().is_none(),
9402 "child {pid} survived rejected admission or was not reaped"
9403 );
9404 }
9405
9406 #[cfg(target_os = "macos")]
9407 #[test]
9408 #[ignore = "manual on-device latency measurement"]
9409 fn durable_record_write_latency_50() {
9410 let home = cortexkit_test_support::ScratchDir::new("record-latency");
9411 let path = home.join("live-children.json");
9412 let children = vec![crate::live_children::LiveChild {
9413 module_id: "latency-worker".into(),
9414 pid: std::process::id(),
9415 protocol: ModuleProtocol::None,
9416 start_time: Some(1),
9417 executable: None,
9418 cgroup_name: None,
9419 }];
9420 let mut samples = Vec::new();
9421 for _ in 0..50 {
9422 let start = std::time::Instant::now();
9423 crate::live_children::write_record(&path, &children).unwrap();
9424 samples.push(start.elapsed());
9425 }
9426 samples.sort();
9427 eprintln!(
9428 "durable_record_write_latency_50: n=50 median={:?} p95={:?}",
9429 (samples[24] + samples[25]) / 2,
9430 samples[47]
9431 );
9432 }
9433
9434 #[cfg(windows)]
9435 #[test]
9436 fn windows_store_touch_fixture() {
9437 if let Some(path) = std::env::var_os("SUBC_TEST_STORE_TOUCH") {
9438 std::fs::write(path, b"module ran").unwrap();
9439 }
9440 }
9441
9442 #[cfg(windows)]
9443 #[tokio::test]
9444 async fn windows_daemon_fixture() {
9445 let Ok(address) = std::env::var("SUBC_TEST_DAEMON_BARRIER") else {
9446 return;
9447 };
9448 let mut spec = spec();
9449 spec.env.push(("SUBC_TEST_AFTER_CREATE".into(), address));
9450 let supervisor =
9451 Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default());
9452 let _module = supervisor.spawn(spec).unwrap();
9453 }
9454
9455 #[cfg(windows)]
9456 #[test]
9457 fn windows_daemon_death_before_module_job_assignment_leaves_no_child() {
9458 use std::io::Read;
9459 let listener = std::net::TcpListener::bind("127.0.0.1:0").unwrap();
9460 let mut command = std::process::Command::new(std::env::current_exe().unwrap());
9463 command
9464 .args([
9465 "--exact",
9466 "supervise::spawn_record_tests::windows_daemon_fixture",
9467 "--nocapture",
9468 ])
9469 .env(
9470 "SUBC_TEST_DAEMON_BARRIER",
9471 listener.local_addr().unwrap().to_string(),
9472 );
9473 let mut daemon = subc_jobobject::spawn_contained(&mut command).unwrap();
9474 listener.set_nonblocking(true).unwrap();
9475 let deadline = std::time::Instant::now() + Duration::from_secs(30);
9476 let mut stream = loop {
9477 if let Ok((stream, _)) = listener.accept() {
9478 break stream;
9479 }
9480 assert!(
9481 daemon.child.try_wait().unwrap().is_none(),
9482 "daemon failed before rendezvous"
9483 );
9484 assert!(
9485 std::time::Instant::now() < deadline,
9486 "daemon did not reach CreateProcess failpoint"
9487 );
9488 std::thread::yield_now();
9489 };
9490 stream
9491 .set_read_timeout(Some(Duration::from_secs(30)))
9492 .unwrap();
9493 let mut pid = [0; 4];
9494 stream.read_exact(&mut pid).unwrap();
9495 let pid = u32::from_ne_bytes(pid);
9496 assert!(subc_jobobject::process_exists(pid));
9497 daemon.child.kill().unwrap();
9500 daemon.child.wait().unwrap();
9501 let exited = subc_jobobject::wait_for_process_exit(pid, Duration::from_secs(10));
9502 if !exited {
9503 daemon.job.terminate().unwrap();
9504 }
9505 assert!(exited, "suspended child {pid} outlived hard-killed daemon");
9506 }
9507
9508 #[cfg(windows)]
9509 #[tokio::test]
9510 async fn windows_containment_failure_never_resumes() {
9511 let home = cortexkit_test_support::ScratchDir::new("assignment-refusal");
9512 let sentinel = home.join("store-touched");
9513 let mut spec = spec();
9514 spec.env
9515 .push(("SUBC_TEST_ASSIGNMENT_FAILURE".into(), "1".into()));
9516 spec.env.push((
9517 "SUBC_TEST_STORE_TOUCH".into(),
9518 sentinel.display().to_string(),
9519 ));
9520 let supervisor =
9521 Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default());
9522 let result = supervisor.spawn(spec);
9523 if let Ok(module) = result {
9526 let _ = tokio::time::timeout(Duration::from_secs(10), async {
9527 while !sentinel.exists() && module.state().unwrap() == ModuleState::Running {
9528 tokio::task::yield_now().await;
9529 }
9530 })
9531 .await;
9532 let _ = module.stop().await;
9533 panic!(
9534 "assignment failure resumed a module; store touched={}",
9535 sentinel.exists()
9536 );
9537 }
9538 assert!(
9539 !sentinel.exists(),
9540 "refused module executed store-touch sentinel"
9541 );
9542 assert!(!subc_jobobject::process_exists(LAST_PID.get().unwrap()));
9543 }
9544
9545 #[cfg(windows)]
9546 #[tokio::test]
9547 async fn windows_nested_daemon_fixture() {
9548 let Some(sentinel) = std::env::var_os("SUBC_TEST_NESTED_STORE") else {
9549 return;
9550 };
9551 let mut spec = spec();
9552 spec.env.push((
9553 "SUBC_TEST_STORE_TOUCH".into(),
9554 sentinel.to_str().unwrap().into(),
9555 ));
9556 let supervisor =
9557 Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default());
9558 let module = supervisor
9559 .spawn(spec)
9560 .expect("per-module job must nest below daemon and host jobs");
9561 tokio::time::timeout(Duration::from_secs(10), async {
9562 while !PathBuf::from(&sentinel).exists() {
9563 tokio::task::yield_now().await;
9564 }
9565 })
9566 .await
9567 .unwrap();
9568 module.stop().await.unwrap();
9569 }
9570
9571 #[cfg(windows)]
9572 #[test]
9573 fn windows_module_job_nests_under_host_job() {
9574 let home = cortexkit_test_support::ScratchDir::new("nested-jobs");
9575 let sentinel = home.join("nested-store-touched");
9576 let mut command = std::process::Command::new(std::env::current_exe().unwrap());
9577 command
9578 .args([
9579 "--exact",
9580 "supervise::spawn_record_tests::windows_nested_daemon_fixture",
9581 "--nocapture",
9582 ])
9583 .env("SUBC_TEST_NESTED_STORE", &sentinel);
9584 let mut daemon = subc_jobobject::spawn_contained(&mut command).unwrap();
9585 assert!(daemon.child.wait().unwrap().success());
9586 assert!(sentinel.exists(), "nested protocol:none module never ran");
9587 }
9588}
9589
9590#[cfg(target_os = "linux")]
9591pub(crate) fn kill_module_cgroup(placement: Option<&subc_cgroup::Placement>, module_id: &str) {
9592 use subc_cgroup::KillOutcome;
9593 match subc_cgroup::kill_module(placement, module_id) {
9594 KillOutcome::Killed => {}
9595 KillOutcome::NotPlaced | KillOutcome::Unsupported => {
9596 debug!(
9597 module_id,
9598 "cgroup tree kill unavailable; using direct-child kill"
9599 );
9600 }
9601 KillOutcome::IoError { path, error } => {
9602 warn!(module_id, path = %path.display(), %error, "cgroup tree kill failed; using direct-child kill");
9603 }
9604 }
9605}
9606
9607#[cfg(windows)]
9615fn contain_spawned_child(
9616 child: &mut Child,
9617 spec: &ModuleSpec,
9618) -> Result<Option<subc_jobobject::JobObject>, SuperviseError> {
9619 let assignment = || {
9620 let job = subc_jobobject::JobObject::new()?;
9621 #[cfg(test)]
9622 spawn_record_tests::assignment_failpoint(spec)?;
9623 job.assign(child)?;
9624 Ok::<_, io::Error>(job)
9625 };
9626 match assignment() {
9627 Ok(job) => Ok(Some(job)),
9628 Err(source) => {
9629 kill_and_reap_rejected_child(child);
9630 Err(SuperviseError::Spawn {
9631 program: spec.program.clone(),
9632 source: io::Error::other(format!(
9633 "Windows module job containment failed: {source}"
9634 )),
9635 cgroup_path: None,
9636 })
9637 }
9638 }
9639}
9640
9641fn kill_and_reap_rejected_child(child: &mut Child) {
9644 #[cfg(unix)]
9645 if let Some(pid) = child
9646 .id()
9647 .and_then(|pid| rustix::process::Pid::from_raw(pid as i32))
9648 {
9649 let _ = rustix::process::kill_process_group(pid, rustix::process::Signal::KILL);
9650 }
9651 let _ = child.start_kill();
9652 while matches!(child.try_wait(), Ok(None)) {
9653 std::thread::yield_now();
9654 }
9655}
9656
9657#[cfg(windows)]
9662fn resume_suspended_child(pid: u32, spec: &ModuleSpec) -> Result<(), SuperviseError> {
9663 if let Err(source) = subc_jobobject::resume_main_thread(pid) {
9664 let _ = std::process::Command::new("taskkill.exe")
9668 .args(["/PID", &pid.to_string(), "/T", "/F"])
9669 .stdin(Stdio::null())
9670 .stdout(Stdio::null())
9671 .stderr(Stdio::null())
9672 .status();
9673 return Err(SuperviseError::Spawn {
9674 program: spec.program.clone(),
9675 source,
9676 cgroup_path: None,
9677 });
9678 }
9679 Ok(())
9680}
9681
9682#[cfg(target_os = "linux")]
9683fn remove_module_cgroup(placement: &subc_cgroup::Placement, module_id: &str) {
9684 match placement.remove_module(module_id) {
9685 Ok(()) => debug!(module_id, "removed module cgroup after process exit"),
9686 Err(error) => warn!(
9687 module_id,
9688 error = %error,
9689 "could not remove module cgroup after process exit; continuing teardown"
9690 ),
9691 }
9692}
9693
9694#[cfg(target_os = "linux")]
9695const REAPED_CGROUP_TERM_GRACE: Duration = Duration::from_secs(2);
9696#[cfg(target_os = "linux")]
9697const REAPED_CGROUP_FREEZE_TIMEOUT: Duration = Duration::from_millis(100);
9698
9699#[cfg(target_os = "linux")]
9700async fn cleanup_reaped_cgroup(placement: &subc_cgroup::Placement, module_id: &str) {
9701 let path = placement.module_path(module_id).ok();
9702 if path.as_ref().is_some_and(|path| {
9703 std::fs::read_to_string(path.join("cgroup.events"))
9704 .ok()
9705 .is_some_and(|events| events.lines().any(|line| line == "populated 0"))
9706 }) {
9707 remove_module_cgroup(placement, module_id);
9708 return;
9709 }
9710
9711 if terminate_reaped_cgroup(placement, module_id).await {
9714 if let Some(path) = &path {
9715 let deadline = Instant::now() + REAPED_CGROUP_TERM_GRACE;
9716 while cgroup_is_populated(path) && Instant::now() < deadline {
9717 sleep_until((Instant::now() + Duration::from_millis(1)).min(deadline)).await;
9718 }
9719 }
9720 }
9721
9722 if matches!(
9726 subc_cgroup::kill_module(Some(placement), module_id),
9727 subc_cgroup::KillOutcome::Killed
9728 ) {
9729 if let Some(path) = &path {
9730 while cgroup_is_populated(path) {
9731 sleep(Duration::from_millis(1)).await;
9732 }
9733 }
9734 }
9735 remove_module_cgroup(placement, module_id);
9736}
9737
9738#[cfg(target_os = "linux")]
9739fn cgroup_is_populated(path: &std::path::Path) -> bool {
9740 std::fs::read_to_string(path.join("cgroup.events"))
9741 .ok()
9742 .is_some_and(|events| events.lines().any(|line| line == "populated 1"))
9743}
9744
9745#[cfg(target_os = "linux")]
9746async fn terminate_reaped_cgroup(placement: &subc_cgroup::Placement, module_id: &str) -> bool {
9747 let frozen =
9748 match subc_cgroup::freeze_module(Some(placement), module_id, REAPED_CGROUP_FREEZE_TIMEOUT)
9749 .await
9750 {
9751 subc_cgroup::FreezeOutcome::Frozen(frozen) => frozen,
9752 outcome => {
9753 tracing::debug!(
9754 module_id,
9755 ?outcome,
9756 "cgroup freeze failed; skipping SIGTERM grace"
9757 );
9758 return false;
9759 }
9760 };
9761 let listed = match frozen.processes() {
9762 subc_cgroup::ProcsOutcome::Listed(pids) => {
9763 for pid in pids.into_iter().filter(|pid| *pid > 1) {
9764 if let Some(pid) = rustix::process::Pid::from_raw(pid) {
9765 if let Err(error) =
9766 rustix::process::kill_process(pid, rustix::process::Signal::TERM)
9767 {
9768 tracing::debug!(module_id, %pid, %error, "leftover cgroup SIGTERM failed");
9769 }
9770 }
9771 }
9772 true
9773 }
9774 outcome => {
9775 tracing::debug!(
9776 module_id,
9777 ?outcome,
9778 "cgroup process listing failed; skipping SIGTERM grace"
9779 );
9780 false
9781 }
9782 };
9783 match frozen.thaw() {
9784 subc_cgroup::ThawOutcome::Thawed => listed,
9785 outcome => {
9786 tracing::debug!(
9787 module_id,
9788 ?outcome,
9789 "cgroup thaw failed; proceeding to cgroup.kill"
9790 );
9791 false
9792 }
9793 }
9794}
9795
9796#[cfg(target_os = "linux")]
9797fn apply_cgroup_placement(
9798 command: &mut Command,
9799 spec: &ModuleSpec,
9800 path: &std::path::Path,
9801) -> Result<(), SuperviseError> {
9802 subc_cgroup::apply(command, path).map_err(|source| SuperviseError::Cgroup {
9803 module_id: spec.module_id.clone(),
9804 source,
9805 })
9806}
9807
9808fn capture_retention(spec: &ModuleSpec) -> Retention {
9809 let defaults = Retention::default();
9810 let value = |name: &str| {
9811 spec.env
9812 .iter()
9813 .rev()
9814 .find_map(|(key, value)| (key == name).then_some(value.as_str()))
9815 };
9816 Retention {
9817 max_file_mb: value(CAPTURE_MAX_FILE_MB_ENV)
9818 .and_then(|value| value.parse().ok())
9819 .unwrap_or(defaults.max_file_mb),
9820 keep: value(CAPTURE_KEEP_ENV)
9821 .and_then(|value| value.parse().ok())
9822 .unwrap_or(defaults.keep),
9823 max_age_days: value(CAPTURE_MAX_AGE_DAYS_ENV)
9824 .and_then(|value| value.parse().ok())
9825 .unwrap_or(defaults.max_age_days),
9826 }
9827}
9828
9829fn generate_launch_nonce() -> Result<String, SuperviseError> {
9832 let mut bytes = [0u8; 32];
9833 getrandom::getrandom(&mut bytes).map_err(|source| SuperviseError::LaunchNonce {
9834 reason: source.to_string(),
9835 })?;
9836 let mut hex = String::with_capacity(64);
9837 for b in bytes {
9838 use std::fmt::Write;
9839 let _ = write!(hex, "{b:02x}");
9840 }
9841 Ok(hex)
9842}
9843
9844fn constant_time_eq(a: &[u8], b: &[u8]) -> bool {
9847 if a.len() != b.len() {
9848 return false;
9849 }
9850 let mut diff = 0u8;
9851 for (x, y) in a.iter().zip(b.iter()) {
9852 diff |= x ^ y;
9853 }
9854 diff == 0
9855}
9856
9857#[cfg(not(windows))]
9861fn observe_spawned_image(pid: u32) -> Option<subc_os::Observation> {
9862 let process = subc_os::Process::open(pid).ok().flatten()?;
9863 #[cfg(target_os = "linux")]
9864 return sample_after_linux_exec(
9865 || {
9866 let stat = std::fs::read_to_string(format!("/proc/{pid}/stat")).ok()?;
9867 let committed = linux_exec_committed(&stat);
9868 #[cfg(test)]
9869 spawn_image_tests::report_exec_state(committed);
9870 committed
9871 },
9872 || process.observe(),
9873 );
9874 #[cfg(not(target_os = "linux"))]
9875 process.observe()
9876}
9877
9878#[cfg(target_os = "linux")]
9882fn linux_exec_committed(stat: &str) -> Option<bool> {
9883 const PF_FORKNOEXEC: u64 = 0x40;
9884 let flags = stat.rsplit_once(')')?.1.split_whitespace().nth(6)?;
9885 Some(flags.parse::<u64>().ok()? & PF_FORKNOEXEC == 0)
9886}
9887
9888#[cfg(target_os = "linux")]
9889fn sample_after_linux_exec(
9890 mut exec_committed: impl FnMut() -> Option<bool>,
9891 observe: impl FnOnce() -> Option<subc_os::Observation>,
9892) -> Option<subc_os::Observation> {
9893 let deadline = std::time::Instant::now() + Duration::from_secs(1);
9894 while !exec_committed()? {
9895 if std::time::Instant::now() >= deadline {
9899 return None;
9900 }
9901 std::thread::sleep(Duration::from_micros(50));
9902 }
9903 observe()
9904}
9905
9906#[cfg(all(test, target_os = "linux"))]
9907mod spawn_image_tests {
9908 use super::*;
9909 use std::cell::{Cell, RefCell};
9910
9911 enum ExecEvent {
9912 Uncommitted,
9913 Image(Option<subc_os::Observation>),
9914 }
9915
9916 thread_local! {
9917 static EXEC_EVENTS: RefCell<Option<std::sync::mpsc::Sender<ExecEvent>>> = const { RefCell::new(None) };
9918 }
9919
9920 pub(super) fn report_exec_state(committed: Option<bool>) {
9921 if committed == Some(false) {
9922 EXEC_EVENTS.with(|events| {
9923 if let Some(events) = events.borrow_mut().take() {
9924 let _ = events.send(ExecEvent::Uncommitted);
9925 }
9926 });
9927 }
9928 }
9929
9930 #[test]
9931 fn linux_spawn_observation_waits_for_a_paused_fork_to_exec() {
9932 use std::io::{Read, Write};
9933 use std::os::fd::AsRawFd;
9934
9935 let descriptor = std::fs::File::open("/dev/null").unwrap();
9936 let mut command = std::process::Command::new("/bin/sleep");
9937 command.arg("30");
9938 let (mut ready, mut release) =
9939 subc_os::fork_exec_test::pause_before_exec(&mut command, descriptor.as_raw_fd())
9940 .unwrap();
9941 let spawn = std::thread::spawn(move || command.spawn());
9942 let mut message = [0u8; 8];
9943 ready.read_exact(&mut message).unwrap();
9944 let pid = i32::from_ne_bytes(message[..4].try_into().unwrap()) as u32;
9945 let before = std::fs::read_to_string(format!("/proc/{pid}/stat"));
9946 let inherited = subc_os::Process::open(pid).unwrap().unwrap().observe();
9947 let (events, received) = std::sync::mpsc::channel();
9948 let observer = std::thread::spawn(move || {
9949 EXEC_EVENTS.with(|slot| *slot.borrow_mut() = Some(events.clone()));
9950 let image = observe_spawned_image(pid);
9951 let _ = events.send(ExecEvent::Image(image));
9952 });
9953 let first = received.recv_timeout(Duration::from_secs(5));
9957 release.write_all(&[1]).unwrap();
9958 let mut child = spawn.join().unwrap().unwrap();
9959 let after = match first {
9960 Ok(ExecEvent::Uncommitted) => match received.recv_timeout(Duration::from_secs(5)) {
9961 Ok(ExecEvent::Image(image)) => image,
9962 _ => None,
9963 },
9964 Ok(ExecEvent::Image(image)) => image,
9965 Err(_) => None,
9966 };
9967 observer.join().unwrap();
9968 child.kill().unwrap();
9969 child.wait().unwrap();
9970
9971 assert_eq!(linux_exec_committed(&before.unwrap()), Some(false));
9972 assert_eq!(
9973 inherited.unwrap().executable,
9974 subc_os::file_identity(&std::env::current_exe().unwrap())
9975 );
9976 assert_eq!(
9977 after.unwrap().executable,
9978 subc_os::file_identity(std::path::Path::new("/bin/sleep"))
9979 );
9980 }
9981
9982 #[test]
9983 fn spawned_image_is_not_sampled_until_linux_exec_commits() {
9984 let committed = Cell::new(false);
9985 let inherited = subc_os::Observation {
9986 start_time: 23,
9987 executable: Some(subc_os::FileIdentity {
9988 device: 29,
9989 inode: 31,
9990 }),
9991 };
9992 let module = subc_os::Observation {
9993 start_time: 23,
9994 executable: Some(subc_os::FileIdentity {
9995 device: 29,
9996 inode: 37,
9997 }),
9998 };
9999 let polls = Cell::new(0);
10000 let image = sample_after_linux_exec(
10001 || {
10002 polls.set(polls.get() + 1);
10003 committed.set(polls.get() == 2);
10006 Some(committed.get())
10007 },
10008 || Some(if committed.get() { module } else { inherited }),
10009 );
10010 assert_eq!(image, Some(module));
10011 assert_eq!(polls.get(), 2);
10012 }
10013
10014 #[test]
10015 fn linux_exec_flag_is_read_from_flags_after_the_last_parenthesis() {
10016 assert_eq!(
10017 linux_exec_committed("17 (a ) b) R 1 2 3 4 5 4194368"),
10018 Some(false)
10019 );
10020 assert_eq!(
10021 linux_exec_committed("17 (a ) b) R 1 2 3 4 5 4194304"),
10022 Some(true)
10023 );
10024 assert_eq!(linux_exec_committed("17 (a) R 1 2 3 4 5 invalid"), None);
10025 assert_eq!(linux_exec_committed("17 (a) R"), None);
10026 }
10027
10028 #[test]
10029 fn unreadable_exec_state_never_samples_an_inherited_image() {
10030 assert_eq!(
10031 sample_after_linux_exec(
10032 || None,
10033 || panic!("must not sample before exec confirmation")
10034 ),
10035 None
10036 );
10037 }
10038
10039 #[test]
10040 fn uncommitted_exec_times_out_without_sampling_an_inherited_image() {
10041 assert_eq!(
10042 sample_after_linux_exec(
10043 || Some(false),
10044 || panic!("must not sample after an exec timeout")
10045 ),
10046 None
10047 );
10048 }
10049}
10050
10051#[cfg(windows)]
10054fn windows_spawn_capture(spec: &ModuleSpec) -> io::Result<subc_os::ExecutableCapture> {
10055 let mut program = spec.program.clone();
10056 if program.is_absolute() || program.components().count() > 1 {
10057 match subc_os::ExecutableCapture::open(&program) {
10060 Ok(capture) => return Ok(capture),
10061 Err(error)
10062 if error.kind() == io::ErrorKind::NotFound && program.extension().is_none() => {}
10063 Err(error) => return Err(error),
10064 }
10065 program.set_extension("exe");
10066 return subc_os::ExecutableCapture::open(&program);
10067 }
10068 if program.extension().is_none() {
10069 program.set_extension("exe");
10070 }
10071 let custom_path = spec
10072 .env
10073 .iter()
10074 .rev()
10075 .find(|(key, _)| key.eq_ignore_ascii_case("PATH"));
10076 let mut directories = Vec::new();
10077 if let Some(parent) = std::env::current_exe()?.parent() {
10080 directories.push(parent.to_path_buf());
10081 }
10082 directories.push(std::env::current_dir()?);
10083 if let Some(windows) = std::env::var_os("SystemRoot") {
10084 let windows = PathBuf::from(windows);
10085 directories.push(windows.join("System32"));
10086 directories.push(windows.clone());
10087 }
10088 let path = custom_path
10089 .map(|(_, value)| std::ffi::OsString::from(value))
10090 .or_else(|| std::env::var_os("PATH"));
10091 if let Some(path) = path {
10092 directories.extend(std::env::split_paths(&path));
10093 }
10094 for directory in directories {
10095 match subc_os::ExecutableCapture::open(&directory.join(&program)) {
10096 Ok(capture) => return Ok(capture),
10097 Err(error) if error.kind() == io::ErrorKind::NotFound => continue,
10098 Err(error) => return Err(error),
10099 }
10100 }
10101 Err(io::Error::new(
10102 io::ErrorKind::NotFound,
10103 "resolved executable not found",
10104 ))
10105}
10106
10107fn spawn_and_mark_running(
10108 spec: &ModuleSpec,
10109 runtime: &SupervisorRuntimeConfig,
10110 snapshot: &SharedSnapshot,
10111) -> Result<SupervisedChild, SuperviseError> {
10112 let activity_spawn = runtime.begin_activity_spawn(&spec.module_id);
10113 let registration_watch = watch_registration(&runtime.registry, spec);
10114 let child = spawn_child(
10115 spec,
10116 runtime.connection_file_path.as_deref(),
10117 runtime.supervisor_handle.as_ref(),
10118 &runtime.stderr_ring,
10119 runtime.capture_logs_dir.as_deref(),
10120 &runtime.child_roster,
10121 #[cfg(target_os = "linux")]
10122 runtime.cgroup_placement.as_ref(),
10123 )?;
10124 let mut child = child;
10125 arm_registration_deadline(&mut child, runtime.register_timeout, registration_watch);
10126 set_running(snapshot, &child, &spec.module_id, &runtime.spawn_events)?;
10127 if let Some(spawn) = activity_spawn {
10128 spawn.complete(snapshot);
10129 }
10130 Ok(child)
10131}
10132
10133struct RegistrationWatch {
10134 changes: watch::Receiver<u64>,
10135 baseline: u64,
10136}
10137
10138fn watch_registration(registry: &Registry, spec: &ModuleSpec) -> Option<RegistrationWatch> {
10142 if spec.protocol != ModuleProtocol::Subc {
10143 return None;
10144 }
10145 let changes = registry.subscribe_module_changes(&spec.module_id).ok()?;
10146 let baseline = *changes.borrow();
10147 Some(RegistrationWatch { changes, baseline })
10148}
10149
10150fn arm_registration_deadline(
10151 child: &mut SupervisedChild,
10152 budget: Duration,
10153 registration_watch: Option<RegistrationWatch>,
10154) {
10155 child.registration_deadline =
10156 (child.protocol == ModuleProtocol::Subc).then_some(child.spawned_at + budget);
10157 child.registration_watch = registration_watch;
10158}
10159
10160async fn stop_unregistered_child(child: &mut SupervisedChild) -> io::Result<()> {
10161 #[cfg(unix)]
10162 {
10163 let group = unreaped_registration_group(child)?;
10166 let _ = rustix::process::kill_process_group(group, rustix::process::Signal::TERM);
10167 sleep(crate::child_roster::TERM_TO_KILL).await;
10172 let group = unreaped_registration_group(child)?;
10173 let _ = rustix::process::kill_process_group(group, rustix::process::Signal::KILL);
10174 }
10175 child.start_kill()?;
10178 child.wait().await?;
10179 Ok(())
10180}
10181
10182#[cfg(unix)]
10183fn unreaped_registration_group(child: &SupervisedChild) -> io::Result<rustix::process::Pid> {
10184 let group = i32::try_from(child.pid)
10187 .ok()
10188 .filter(|pid| *pid > 1)
10189 .and_then(rustix::process::Pid::from_raw)
10190 .ok_or_else(|| {
10191 io::Error::new(
10192 io::ErrorKind::InvalidInput,
10193 "invalid supervised process group",
10194 )
10195 })?;
10196 if child.child.id() != Some(child.pid) {
10197 return Err(io::Error::new(
10198 io::ErrorKind::InvalidInput,
10199 "refusing process-group signal after supervised child was reaped",
10200 ));
10201 }
10202 Ok(group)
10203}
10204
10205enum RegistrationWaitOutcome {
10206 Registered,
10207 Exited(ExitReport),
10208 TimedOut,
10209}
10210
10211struct ReloadRegistrationFailure {
10212 exit_report: ExitReport,
10213 reason: String,
10214}
10215
10216#[derive(Debug, Clone, Copy, PartialEq, Eq)]
10217enum BusyGaugeObservation {
10218 Quiescent,
10219 Busy,
10220 Omitted,
10221}
10222
10223#[derive(Debug, Clone, PartialEq, Eq)]
10225enum GaugeVerdict {
10226 Zero,
10228 NonZero(u64),
10230 NotACount(String),
10234 Missing,
10236 ProbeFailed(&'static str),
10239}
10240
10241impl GaugeVerdict {
10242 fn reads_quiet(&self) -> bool {
10243 matches!(self, Self::Zero)
10244 }
10245}
10246
10247impl fmt::Display for GaugeVerdict {
10248 fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
10249 match self {
10250 Self::Zero => f.write_str("zero"),
10251 Self::NonZero(value) => write!(f, "non_zero({value})"),
10252 Self::NotACount(value) => write!(f, "not_a_count({value})"),
10253 Self::Missing => f.write_str("missing_from_health_body"),
10254 Self::ProbeFailed(kind) => write!(f, "probe_failed({kind})"),
10255 }
10256 }
10257}
10258
10259const GAUGE_VALUE_TEXT_LIMIT: usize = 32;
10262
10263fn gauge_verdicts(metrics: Option<&Value>, gauges: &[String]) -> Vec<GaugeVerdict> {
10265 let metrics = metrics.and_then(Value::as_object);
10266 gauges
10267 .iter()
10268 .map(
10269 |gauge| match metrics.and_then(|metrics| metrics.get(gauge)) {
10270 None => GaugeVerdict::Missing,
10271 Some(value) => match value.as_u64() {
10272 Some(0) => GaugeVerdict::Zero,
10273 Some(count) => GaugeVerdict::NonZero(count),
10274 None => {
10275 let mut text = value.to_string();
10276 if text.len() > GAUGE_VALUE_TEXT_LIMIT {
10277 let mut end = GAUGE_VALUE_TEXT_LIMIT;
10278 while !text.is_char_boundary(end) {
10279 end -= 1;
10280 }
10281 text.truncate(end);
10282 text.push_str("...");
10283 }
10284 GaugeVerdict::NotACount(text)
10285 }
10286 },
10287 },
10288 )
10289 .collect()
10290}
10291
10292fn busy_gauge_observation(verdicts: &[GaugeVerdict]) -> BusyGaugeObservation {
10295 if verdicts
10296 .iter()
10297 .any(|verdict| matches!(verdict, GaugeVerdict::Missing))
10298 {
10299 BusyGaugeObservation::Omitted
10300 } else if verdicts.iter().all(GaugeVerdict::reads_quiet) {
10301 BusyGaugeObservation::Quiescent
10302 } else {
10303 BusyGaugeObservation::Busy
10304 }
10305}
10306
10307#[derive(Debug, Clone, Default, PartialEq, Eq)]
10312struct GaugeTrace {
10313 latest: Option<(GaugeVerdict, Duration)>,
10314 last_value: Option<(GaugeVerdict, Duration)>,
10315}
10316
10317#[derive(Debug, Clone, Default, PartialEq, Eq)]
10323struct DrainGaugeTrace {
10324 probes: u32,
10326 gauges: Vec<(String, GaugeTrace)>,
10328}
10329
10330impl DrainGaugeTrace {
10331 fn new(gauges: &[String]) -> Self {
10332 Self {
10333 probes: 0,
10334 gauges: gauges
10335 .iter()
10336 .map(|gauge| (gauge.clone(), GaugeTrace::default()))
10337 .collect(),
10338 }
10339 }
10340
10341 fn record(&mut self, verdicts: &[GaugeVerdict], at: Duration) {
10343 self.probes = self.probes.saturating_add(1);
10344 for ((_, trace), verdict) in self.gauges.iter_mut().zip(verdicts) {
10345 trace.latest = Some((verdict.clone(), at));
10346 if matches!(
10347 verdict,
10348 GaugeVerdict::Zero | GaugeVerdict::NonZero(_) | GaugeVerdict::NotACount(_)
10349 ) {
10350 trace.last_value = Some((verdict.clone(), at));
10351 }
10352 }
10353 }
10354}
10355
10356impl fmt::Display for DrainGaugeTrace {
10361 fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
10362 if self.gauges.is_empty() {
10363 return f.write_str("none declared");
10364 }
10365 for (index, (name, trace)) in self.gauges.iter().enumerate() {
10366 if index > 0 {
10367 f.write_str(" | ")?;
10368 }
10369 write!(f, "{name}: ")?;
10370 match &trace.latest {
10371 Some((verdict, at)) => write!(f, "{verdict} @{}ms", at.as_millis())?,
10372 None => f.write_str("never probed")?,
10373 }
10374 match &trace.last_value {
10375 Some((value, at)) => write!(f, "; last value {value} @{}ms", at.as_millis())?,
10376 None => f.write_str("; no value ever read")?,
10377 }
10378 }
10379 Ok(())
10380 }
10381}
10382
10383pub(crate) fn declared_busy_gauges(
10384 registry: &Registry,
10385 module_id: &str,
10386) -> Result<Vec<String>, SuperviseError> {
10387 busy_gauges_of(
10388 registry
10389 .get_module(module_id)
10390 .map_err(SuperviseError::Registry)?,
10391 )
10392}
10393
10394fn declared_busy_gauges_for_connection(
10398 registry: &Registry,
10399 connection_id: ConnectionId,
10400) -> Result<Vec<String>, SuperviseError> {
10401 busy_gauges_of(
10402 registry
10403 .get_module_by_connection(connection_id)
10404 .map_err(SuperviseError::Registry)?,
10405 )
10406}
10407
10408fn busy_gauges_of(
10409 registration: Option<crate::registry::ModuleRegistration>,
10410) -> Result<Vec<String>, SuperviseError> {
10411 let Some(registration) = registration else {
10412 return Ok(Vec::new());
10413 };
10414 let Some(self_signals) = registration.manifest.self_signals else {
10415 return Ok(Vec::new());
10416 };
10417
10418 let mut gauges = Vec::new();
10419 for declaration in self_signals {
10420 if declaration.kind != SelfSignalKind::Busy {
10421 continue;
10422 }
10423 match declaration.anchored_to {
10424 SignalAnchor::HealthGauges { gauges: declared } if !declared.is_empty() => {
10425 gauges.extend(declared)
10426 }
10427 _ => {
10428 gauges.push(String::new());
10431 }
10432 }
10433 }
10434 Ok(gauges)
10435}
10436
10437#[allow(clippy::too_many_arguments)]
10446async fn wait_for_forwarding_quiescence(
10447 forwarding: &ForwardingTable,
10448 module_id: &str,
10449 runtime: &SupervisorRuntimeConfig,
10450 endpoint: crate::ModuleEndpointId,
10451 deadline: Instant,
10452 busy_gauges: &[String],
10453 scope: DrainScope,
10454 trace: &mut DrainGaugeTrace,
10455) -> Result<bool, SuperviseError> {
10456 let wait_started = Instant::now();
10457 let mut gauges_quiescent = busy_gauges.is_empty();
10458 let mut next_probe_at = wait_started;
10459 let mut omission_reported = false;
10460
10461 loop {
10462 let now = Instant::now();
10463 if !busy_gauges.is_empty() && now >= next_probe_at && now < deadline {
10464 let report = match scope {
10465 DrainScope::Active => probe_module_health(module_id, runtime, Some(deadline)).await,
10466 DrainScope::Endpoint(endpoint) => {
10467 probe_endpoint_health(endpoint, runtime, Some(deadline)).await
10468 }
10469 };
10470 let verdicts = match &report {
10471 Ok(report) => gauge_verdicts(report.metrics.as_ref(), busy_gauges),
10472 Err(err) => vec![GaugeVerdict::ProbeFailed(err.label()); busy_gauges.len()],
10473 };
10474 trace.record(&verdicts, wait_started.elapsed());
10475 gauges_quiescent = match report {
10476 Ok(_) => match busy_gauge_observation(&verdicts) {
10477 BusyGaugeObservation::Quiescent => true,
10478 BusyGaugeObservation::Busy => false,
10479 BusyGaugeObservation::Omitted => {
10480 if !omission_reported {
10483 forwarding
10484 .counters()
10485 .increment_drains_with_undeclared_gauge();
10486 let missing = busy_gauges
10487 .iter()
10488 .zip(&verdicts)
10489 .filter(|(_, verdict)| matches!(verdict, GaugeVerdict::Missing))
10490 .map(|(gauge, _)| gauge.as_str())
10491 .collect::<Vec<_>>()
10492 .join(",");
10493 warn!(
10494 module_id,
10495 missing_gauges = %missing,
10496 "drain health.check answer lacks a declared busy gauge; the drain treats the module as busy until it reports it"
10497 );
10498 omission_reported = true;
10499 }
10500 false
10501 }
10502 },
10503 Err(err) => {
10504 warn!(
10505 module_id,
10506 error = %err,
10507 "drain health.check did not produce declared busy gauges; treating module as busy"
10508 );
10509 false
10510 }
10511 };
10512 next_probe_at = Instant::now() + drain_busy_reprobe_interval();
10515 }
10516
10517 let in_flight = forwarding
10518 .endpoint_in_flight_count(endpoint)
10519 .map_err(SuperviseError::Forwarding)?;
10520 if in_flight == 0 && gauges_quiescent {
10521 return Ok(true);
10522 }
10523
10524 let now = Instant::now();
10525 if now >= deadline {
10526 return Ok(false);
10527 }
10528 let mut wait = deadline
10529 .saturating_duration_since(now)
10530 .min(REGISTRY_RELEASE_POLL);
10531 if !busy_gauges.is_empty() {
10532 wait = wait.min(next_probe_at.saturating_duration_since(now));
10533 }
10534 sleep(wait).await;
10535 }
10536}
10537
10538fn drain_busy_reprobe_interval() -> Duration {
10542 DRAIN_BUSY_REPROBE_INTERVAL.max(REGISTRY_RELEASE_POLL)
10543}
10544
10545fn warn_drain_timed_out(
10560 module_id: &str,
10561 drain_timeout: Duration,
10562 reason: RouteCloseReason,
10563 holdouts: &crate::forwarding::DrainHoldouts,
10564 gauges: &DrainGaugeTrace,
10565) {
10566 warn!(
10567 module_id = %module_id,
10568 waited = ?drain_timeout,
10569 ?reason,
10570 held_requests = holdouts.requests,
10571 held_routes = holdouts.routes,
10572 total_routes = holdouts.total_routes,
10573 top_connections = ?holdouts.top_connections,
10574 held = %holdouts
10577 .held
10578 .iter()
10579 .map(|(channel, corr)| format!("{channel}:{corr}"))
10580 .collect::<Vec<_>>()
10581 .join(","),
10582 gauge_probes = gauges.probes,
10583 busy_gauges = %gauges,
10584 "route drain timed out before request quiescence; forcing teardown"
10585 );
10586}
10587
10588fn drained_after_quiescence_wait(wait_result: &Result<bool, SuperviseError>) -> bool {
10596 match wait_result {
10597 Ok(drained) => *drained,
10598 Err(_) => false,
10599 }
10600}
10601
10602fn send_route_goodbyes(forwarding: &ForwardingTable, released_routes: Vec<GoodbyeTarget>) {
10603 for released in released_routes {
10604 let frame = match Frame::build_with_version(
10605 released.negotiated_ver,
10606 FrameType::Goodbye,
10607 control_flags(),
10608 released.channel,
10609 released.epoch,
10610 0,
10611 Vec::new(),
10612 ) {
10613 Ok(frame) => frame,
10614 Err(err) => {
10615 warn!(
10616 route_channel = released.channel,
10617 error = %err,
10618 "failed to build supervisor drain route GOODBYE frame"
10619 );
10620 continue;
10621 }
10622 };
10623 if !released.close_on_delivery_failure() {
10624 crate::forwarding::send_module_route_goodbye(
10625 &forwarding.counters(),
10626 &released.sink,
10627 frame,
10628 released.module_id.as_deref(),
10629 "supervisor drain",
10630 );
10631 continue;
10632 }
10633 if let Err(err) = released.sink.try_send(frame) {
10634 warn!(
10635 target_connection_id = released.connection_id.get(),
10636 route_channel = released.channel,
10637 error = %err,
10638 "supervisor drain route GOODBYE was not delivered to client; closing target connection"
10639 );
10640 let _ = forwarding.escalate_client_delivery_failure(
10641 released.connection_id,
10642 released.channel,
10643 released.epoch,
10644 CloseReason::new(
10645 "route_goodbye_delivery_failed",
10646 format!(
10647 "failed to enqueue supervisor drain route GOODBYE for channel {}: {err}",
10648 released.channel
10649 ),
10650 ),
10651 crate::forwarding::UndeliveredFrame {
10652 module_id: released.module_id.as_deref(),
10653 sink: &released.sink,
10654 },
10655 );
10656 }
10657 }
10658}
10659
10660async fn send_module_draining(
10661 module_id: &str,
10662 reason: RouteCloseReason,
10663 deadline_ms: u64,
10664 target: &ModuleDrainTarget,
10665 _wait_for_flush: bool,
10666) -> bool {
10667 let body = match serde_json::to_vec(&ModuleControlCommand::Draining {
10668 reason,
10669 deadline_ms,
10670 }) {
10671 Ok(body) => body,
10672 Err(err) => {
10673 warn!(
10674 module_id,
10675 error = %err,
10676 "failed to encode module draining command"
10677 );
10678 return false;
10679 }
10680 };
10681 let frame = match Frame::build_with_version(
10682 target.negotiated_ver,
10683 FrameType::Push,
10684 control_flags(),
10685 0,
10686 0,
10687 0,
10688 body,
10689 ) {
10690 Ok(frame) => frame,
10691 Err(err) => {
10692 warn!(
10693 module_id,
10694 error = %err,
10695 "failed to build module draining command frame"
10696 );
10697 return false;
10698 }
10699 };
10700 #[cfg(unix)]
10701 if _wait_for_flush {
10702 return target.sink.send_flushed(frame).await.is_ok();
10703 }
10704 if let Err(err) = target.sink.try_send(frame) {
10705 warn!(
10706 module_id,
10707 target_connection_id = target.endpoint.connection_id.get(),
10708 error = %err,
10709 "module draining command was not delivered to peer"
10710 );
10711 return false;
10712 }
10713 true
10714}
10715
10716fn module_goodbye_frame(module_id: &str, negotiated_ver: u8) -> Option<Frame> {
10718 match Frame::build_with_version(
10719 negotiated_ver,
10720 FrameType::Goodbye,
10721 control_flags(),
10722 0,
10723 0,
10724 0,
10725 Vec::new(),
10726 ) {
10727 Ok(frame) => Some(frame),
10728 Err(err) => {
10729 warn!(
10730 module_id,
10731 error = %err,
10732 "failed to build module GOODBYE frame"
10733 );
10734 None
10735 }
10736 }
10737}
10738
10739#[cfg(unix)]
10753async fn send_module_goodbyes_for_daemon_shutdown(
10754 forwarding: &Arc<ForwardingTable>,
10755 reason: &CloseReason,
10756 wait_for_flush: bool,
10757) {
10758 const GOODBYE_BUDGET: Duration = Duration::from_millis(500);
10759 let targets = match forwarding.module_connections() {
10760 Ok(targets) => targets,
10761 Err(err) => {
10762 warn!(error = %err, "could not list module connections for shutdown GOODBYE");
10763 return;
10764 }
10765 };
10766 let deadline = Instant::now() + GOODBYE_BUDGET;
10767 let mut sends = tokio::task::JoinSet::new();
10768 for target in targets {
10769 let Some(frame) = module_goodbye_frame(&target.module_id, target.negotiated_ver) else {
10770 continue;
10771 };
10772 if !wait_for_flush {
10773 if let Err(err) = target.sink.try_send(frame) {
10774 debug!(
10775 module_id = %target.module_id,
10776 error = %err,
10777 "shutdown module GOODBYE was not queued"
10778 );
10779 }
10780 continue;
10781 }
10782 let forwarding = Arc::clone(forwarding);
10783 let reason = reason.clone();
10784 sends.spawn(async move {
10785 match timeout_at(deadline, target.sink.send_flushed(frame)).await {
10786 Ok(Ok(())) => {}
10787 Ok(Err(err)) => debug!(
10788 module_id = %target.module_id,
10789 error = %err,
10790 "module connection closed before its shutdown GOODBYE was written"
10791 ),
10792 Err(_) => warn!(
10793 module_id = %target.module_id,
10794 budget = ?GOODBYE_BUDGET,
10795 "shutdown module GOODBYE was not written within its budget; closing anyway"
10796 ),
10797 }
10798 forwarding.request_connection_close(target.endpoint.connection_id, reason);
10799 });
10800 }
10801 while sends.join_next().await.is_some() {}
10803}
10804
10805fn send_module_goodbye(module_id: &str, forwarding: &ForwardingTable, target: &ModuleDrainTarget) {
10806 let Some(frame) = module_goodbye_frame(module_id, target.negotiated_ver) else {
10807 return;
10808 };
10809 if let Err(err) = target.sink.try_send(frame) {
10810 warn!(
10811 module_id,
10812 target_connection_id = target.endpoint.connection_id.get(),
10813 error = %err,
10814 "supervisor drain module GOODBYE was not delivered to peer; closing module connection"
10815 );
10816 forwarding.request_connection_close(
10817 target.endpoint.connection_id,
10818 CloseReason::new(
10819 "module_goodbye_delivery_failed",
10820 format!("failed to enqueue supervisor drain module GOODBYE for module '{module_id}': {err}"),
10821 ),
10822 );
10823 }
10824}
10825
10826#[derive(Clone, Copy)]
10827struct ForwardingDrainContext<'a> {
10828 spec: &'a ModuleSpec,
10829 runtime: &'a SupervisorRuntimeConfig,
10830 registry: &'a Registry,
10831 scope: DrainScope,
10832}
10833
10834#[derive(Debug, Clone, Copy, PartialEq, Eq)]
10836enum DrainScope {
10837 Active,
10840 Endpoint(crate::ModuleEndpointId),
10845}
10846
10847#[derive(Debug, Clone, Copy, PartialEq, Eq)]
10855enum StopNotice {
10856 SentOverConnection,
10859 NoConnection,
10863 NotSent,
10867}
10868
10869async fn begin_forwarding_drain(
10870 spec: &ModuleSpec,
10871 runtime: &SupervisorRuntimeConfig,
10872 registry: &Registry,
10873 snapshot: &SharedSnapshot,
10874 enabled: Option<bool>,
10875 reason: RouteCloseReason,
10876) -> Result<StopNotice, SuperviseError> {
10877 let Some(forwarding) = runtime.forwarding.as_ref() else {
10878 return Err(SuperviseError::ReloadUnavailable {
10879 module_id: spec.module_id.clone(),
10880 reason: "supervisor was not configured with a forwarding table".to_string(),
10881 });
10882 };
10883
10884 begin_forwarding_drain_with(
10885 forwarding,
10886 ForwardingDrainContext {
10887 spec,
10888 runtime,
10889 registry,
10890 scope: DrainScope::Active,
10891 },
10892 snapshot,
10893 enabled,
10894 reason,
10895 runtime.drain_timeout,
10896 )
10897 .await
10898}
10899
10900async fn begin_forwarding_drain_if_configured(
10901 spec: &ModuleSpec,
10902 runtime: &SupervisorRuntimeConfig,
10903 registry: &Registry,
10904 snapshot: &SharedSnapshot,
10905 enabled: Option<bool>,
10906 reason: RouteCloseReason,
10907) -> Result<StopNotice, SuperviseError> {
10908 begin_forwarding_drain_with_timeout(
10909 spec,
10910 runtime,
10911 registry,
10912 snapshot,
10913 enabled,
10914 reason,
10915 runtime.drain_timeout,
10916 )
10917 .await
10918}
10919
10920async fn begin_forwarding_drain_with_timeout(
10924 spec: &ModuleSpec,
10925 runtime: &SupervisorRuntimeConfig,
10926 registry: &Registry,
10927 snapshot: &SharedSnapshot,
10928 enabled: Option<bool>,
10929 reason: RouteCloseReason,
10930 drain_timeout: Duration,
10931) -> Result<StopNotice, SuperviseError> {
10932 let Some(forwarding) = runtime.forwarding.as_ref() else {
10933 return Ok(StopNotice::NotSent);
10934 };
10935
10936 begin_forwarding_drain_with(
10937 forwarding,
10938 ForwardingDrainContext {
10939 spec,
10940 runtime,
10941 registry,
10942 scope: DrainScope::Active,
10943 },
10944 snapshot,
10945 enabled,
10946 reason,
10947 drain_timeout,
10948 )
10949 .await
10950}
10951
10952async fn begin_forwarding_drain_with(
10953 forwarding: &ForwardingTable,
10954 context: ForwardingDrainContext<'_>,
10955 snapshot: &SharedSnapshot,
10956 enabled: Option<bool>,
10957 reason: RouteCloseReason,
10958 drain_timeout: Duration,
10959) -> Result<StopNotice, SuperviseError> {
10960 let ForwardingDrainContext {
10961 spec,
10962 runtime,
10963 registry,
10964 scope,
10965 } = context;
10966 debug_assert_ne!(reason, RouteCloseReason::Crash);
10967 let terminal = matches!(reason, RouteCloseReason::Disable);
10968 let drain_started_at = Instant::now();
10969 let drain_deadline = drain_started_at + drain_timeout;
10970 let deadline_ms =
10971 unix_ms_now().saturating_add(u64::try_from(drain_timeout.as_millis()).unwrap_or(u64::MAX));
10972 let busy_gauges = match scope {
10973 DrainScope::Active => declared_busy_gauges(registry, &spec.module_id)?,
10974 DrainScope::Endpoint(endpoint) => {
10975 declared_busy_gauges_for_connection(registry, endpoint.connection_id)?
10976 }
10977 };
10978
10979 let gate_started = Instant::now();
10982 let drain_target = match scope {
10983 DrainScope::Active => forwarding.begin_module_drain(&spec.module_id, reason),
10984 DrainScope::Endpoint(endpoint) => forwarding.begin_endpoint_drain(endpoint, reason),
10985 }
10986 .map_err(SuperviseError::Forwarding)?;
10987 info!(
10992 module_id = %spec.module_id,
10993 ?reason,
10994 gate_ms = u64::try_from(gate_started.elapsed().as_millis()).unwrap_or(u64::MAX),
10995 connected = drain_target.is_some(),
10996 "module drain began; route admission closed"
10997 );
10998 if scope == DrainScope::Active {
10999 update_snapshot(snapshot, Some(&spec.module_id), |state| {
11000 state.state = ModuleState::Draining;
11001 state.draining_to_replace =
11002 matches!(reason, RouteCloseReason::Restart | RouteCloseReason::Reload);
11003 if let Some(enabled) = enabled {
11004 state.enabled = enabled;
11005 }
11006 })?;
11007 }
11008
11009 let Some(target) = drain_target.as_ref() else {
11010 return Ok(StopNotice::NoConnection);
11014 };
11015 {
11016 send_module_draining(&spec.module_id, reason, deadline_ms, target, false).await;
11017 #[cfg(feature = "test-support")]
11018 forwarding.wait_before_route_closing_for_test().await;
11019 let routes = forwarding
11020 .endpoint_routes(target.endpoint)
11021 .map_err(SuperviseError::Forwarding)?;
11022 let routes_notified = routes.len();
11023 crate::control::send_route_control_pushes(
11024 forwarding,
11025 routes.clone(),
11026 ClientControlPush::RouteClosing {
11027 module_id: spec.module_id.clone(),
11028 channels: Vec::new(),
11029 reason,
11030 },
11031 );
11032 send_route_goodbyes(forwarding, target.abandoned_bindings.clone());
11033
11034 let mut gauge_trace = DrainGaugeTrace::new(&busy_gauges);
11040 let wait_result = wait_for_forwarding_quiescence(
11041 forwarding,
11042 &spec.module_id,
11043 runtime,
11044 target.endpoint,
11045 drain_deadline,
11046 &busy_gauges,
11047 scope,
11048 &mut gauge_trace,
11049 )
11050 .await;
11051 let drained = drained_after_quiescence_wait(&wait_result);
11052 if let Err(err) = &wait_result {
11053 error!(
11054 module_id = %spec.module_id,
11055 ?reason,
11056 error = %err,
11057 "forwarding quiescence wait failed after route.closing; forcing route.closed(drained: false) so the client is not left waiting on an unfulfilled promise"
11058 );
11059 } else if !drained {
11060 let holdouts = forwarding
11062 .endpoint_drain_holdouts(target.endpoint)
11063 .unwrap_or_default();
11064 warn_drain_timed_out(
11065 &spec.module_id,
11066 drain_timeout,
11067 reason,
11068 &holdouts,
11069 &gauge_trace,
11070 );
11071 }
11072 crate::control::send_route_control_pushes(
11073 forwarding,
11074 routes,
11075 ClientControlPush::RouteClosed {
11076 module_id: spec.module_id.clone(),
11077 channels: Vec::new(),
11078 reason,
11079 drained,
11080 abandoned: target.abandoned_bindings.len() as u32,
11081 excluded_subscriptions: target.excluded_subscriptions,
11082 terminal: Some(terminal),
11083 },
11084 );
11085 wait_result?;
11086
11087 let released_routes = match forwarding.release_module_endpoint_routes(target.endpoint) {
11093 Ok(routes) => routes,
11094 Err(err) => {
11095 warn!(
11096 module_id = %spec.module_id,
11097 ?reason,
11098 error = %err,
11099 "failed to release module endpoint routes after route.closed; module GOODBYE will still be sent"
11100 );
11101 send_module_goodbye(&spec.module_id, forwarding, target);
11102 return Err(SuperviseError::Forwarding(err));
11103 }
11104 };
11105 let route_goodbye_count = released_routes.len();
11106 send_route_goodbyes(forwarding, released_routes);
11107 send_module_goodbye(&spec.module_id, forwarding, target);
11108
11109 info!(
11115 module_id = %spec.module_id,
11116 ?reason,
11117 routes_notified,
11118 route_goodbyes = route_goodbye_count,
11119 abandoned_reservations = target.abandoned_bindings.len(),
11120 excluded_subscriptions = target.excluded_subscriptions,
11121 drained,
11122 "module drain complete; consumers notified via route.closing/route.closed pushes and per-route GOODBYE frames"
11123 );
11124 }
11125
11126 Ok(StopNotice::SentOverConnection)
11127}
11128
11129async fn wait_for_registration_after_reload(
11132 registry: &Registry,
11133 module_id: &str,
11134 snapshot: &SharedSnapshot,
11135 child: &mut SupervisedChild,
11136 wait: Duration,
11137) -> Result<RegistrationWaitOutcome, SuperviseError> {
11138 wait_for_slot_registration(
11139 registry,
11140 crate::registry::RegistrationSlot::Active(module_id),
11141 module_id,
11142 snapshot,
11143 child,
11144 wait,
11145 )
11146 .await
11147}
11148
11149async fn wait_for_slot_registration(
11157 registry: &Registry,
11158 slot: crate::registry::RegistrationSlot<'_>,
11159 module_id: &str,
11160 snapshot: &SharedSnapshot,
11161 child: &mut SupervisedChild,
11162 wait: Duration,
11163) -> Result<RegistrationWaitOutcome, SuperviseError> {
11164 let deadline = Instant::now() + wait;
11165 loop {
11166 if registry
11167 .registration(slot)
11168 .map_err(SuperviseError::Registry)?
11169 .is_some()
11170 {
11171 return Ok(RegistrationWaitOutcome::Registered);
11172 }
11173
11174 let now = Instant::now();
11175 if now >= deadline {
11176 return Ok(RegistrationWaitOutcome::TimedOut);
11177 }
11178 let remaining = deadline.saturating_duration_since(now);
11179 let poll = remaining.min(REGISTRY_RELEASE_POLL);
11180
11181 tokio::select! {
11182 wait_result = child.wait() => {
11183 let status = wait_result.map_err(|source| SuperviseError::Wait {
11184 module_id: module_id.to_string(),
11185 source,
11186 })?;
11187 return Ok(RegistrationWaitOutcome::Exited(classify_reaped_child_exit(
11188 snapshot,
11189 child,
11190 &status,
11191 )));
11192 }
11193 _ = sleep(poll) => {}
11194 }
11195 }
11196}
11197
11198fn registration_failure_exit_report(mut exit_report: ExitReport) -> ExitReport {
11199 if exit_report.kind != ExitKind::DeliberateSeverance {
11203 exit_report.kind = ExitKind::Crash;
11204 }
11205 exit_report
11206}
11207
11208async fn handle_reload_child_registration_failure(
11209 spec: &ModuleSpec,
11210 runtime: &SupervisorRuntimeConfig,
11211 registry: &Registry,
11212 process_liveness: &SupervisorProcessLiveness,
11213 snapshot: &SharedSnapshot,
11214 _child: &mut Option<SupervisedChild>,
11215 failure: ReloadRegistrationFailure,
11216) -> Result<(), SuperviseError> {
11217 let ReloadRegistrationFailure {
11218 exit_report,
11219 reason,
11220 } = failure;
11221 match on_child_exit(
11222 spec,
11223 runtime.restart_policy,
11224 registry,
11225 snapshot,
11226 &runtime.terminal_ring,
11227 &runtime.spawn_events,
11228 &runtime.child_roster,
11229 exit_report,
11230 runtime.recovery_is_daemon,
11231 )
11232 .await
11233 {
11234 NextAction::Stop {
11235 registration_released,
11236 } => {
11237 if registration_released {
11238 process_liveness.untrack_if_current(&spec.module_id, snapshot);
11239 }
11240 }
11241 NextAction::Restart { schedule } => {
11242 let delay = schedule.map_or(runtime.restart_policy.delay_for_restart(0), |schedule| {
11243 schedule.delay
11244 });
11245 if let Some(schedule) = schedule {
11246 log_crash_respawn(&spec.module_id, schedule);
11247 }
11248 schedule_respawn(
11249 runtime,
11250 snapshot,
11251 &spec.module_id,
11252 delay,
11253 RespawnKind::Spawn,
11254 true,
11255 )?;
11256 }
11257 }
11258 Err(SuperviseError::ReloadFailed {
11259 module_id: spec.module_id.clone(),
11260 reason,
11261 })
11262}
11263
11264async fn handle_reload_spawn_failure(
11265 spec: &ModuleSpec,
11266 runtime: &SupervisorRuntimeConfig,
11267 process_liveness: &SupervisorProcessLiveness,
11268 snapshot: &SharedSnapshot,
11269 _child: &mut Option<SupervisedChild>,
11270 reason: String,
11271) -> Result<(), SuperviseError> {
11272 let now = Instant::now();
11273 let mut schedule = None;
11274 update_snapshot(snapshot, Some(&spec.module_id), |state| {
11275 clear_current_process_facts(state);
11276 if state.enabled {
11277 schedule = state.next_crash_restart(&runtime.restart_policy, now);
11278 state.state = if schedule.is_some() {
11279 ModuleState::Restarting
11280 } else {
11281 if runtime
11282 .failure_context(runtime.recovery_is_daemon)
11283 .may_retry()
11284 {
11285 state.failed_retry.schedule(&spec.module_id);
11286 }
11287 ModuleState::Failed
11288 };
11289 } else {
11290 state.state = ModuleState::Disabled;
11291 }
11292 })?;
11293 if let Some(schedule) = schedule {
11294 schedule_respawn(
11295 runtime,
11296 snapshot,
11297 &spec.module_id,
11298 schedule.delay,
11299 RespawnKind::Spawn,
11300 true,
11301 )?;
11302 } else {
11303 process_liveness.untrack_if_current(&spec.module_id, snapshot);
11304 }
11305 Err(SuperviseError::ReloadFailed {
11306 module_id: spec.module_id.clone(),
11307 reason,
11308 })
11309}
11310
11311fn control_flags() -> Flags {
11312 Flags::new(false, Priority::Passive, false)
11313}
11314
11315#[allow(clippy::too_many_arguments)]
11316async fn drain_optional_child(
11317 module_id: &str,
11318 protocol: ModuleProtocol,
11319 stop_notice: StopNotice,
11320 registry: &Registry,
11321 forwarding: Option<&ForwardingTable>,
11322 snapshot: &SharedSnapshot,
11323 terminal_ring: &Arc<Mutex<TerminalRing>>,
11324 spawn_events: &SpawnEventFeed,
11325 child: &mut Option<SupervisedChild>,
11326 drain_timeout: Duration,
11327 final_state: ModuleState,
11328 enabled: Option<bool>,
11329 failure: &FailureContext,
11330) -> Result<(), SuperviseError> {
11331 if let Some(child) = child.take() {
11332 drain_child_to_state(
11333 module_id,
11334 protocol,
11335 stop_notice,
11336 registry,
11337 forwarding,
11338 snapshot,
11339 terminal_ring,
11340 spawn_events,
11341 child,
11342 drain_timeout,
11343 final_state,
11344 enabled,
11345 failure,
11346 )
11347 .await
11348 } else {
11349 update_snapshot(snapshot, Some(module_id), |state| {
11350 state.state = final_state;
11351 if final_state == ModuleState::Failed {
11352 state.failed_retry.failed_at = Some(Instant::now());
11353 }
11354 if let Some(enabled) = enabled {
11355 state.enabled = enabled;
11356 }
11357 clear_current_process_facts(state);
11358 })?;
11359 release_dead_registration(registry, forwarding, snapshot, module_id, failure).await
11360 }
11361}
11362
11363#[allow(clippy::too_many_arguments)]
11364async fn drain_child_to_state(
11365 module_id: &str,
11366 _protocol: ModuleProtocol,
11367 stop_notice: StopNotice,
11368 registry: &Registry,
11369 forwarding: Option<&ForwardingTable>,
11370 snapshot: &SharedSnapshot,
11371 terminal_ring: &Arc<Mutex<TerminalRing>>,
11372 spawn_events: &SpawnEventFeed,
11373 mut child: SupervisedChild,
11374 drain_timeout: Duration,
11375 final_state: ModuleState,
11376 enabled: Option<bool>,
11377 failure: &FailureContext,
11378) -> Result<(), SuperviseError> {
11379 let protocol = child.protocol;
11380 update_snapshot(snapshot, Some(module_id), |state| {
11381 state.state = ModuleState::Draining;
11382 state.draining_to_replace = final_state == ModuleState::Restarting;
11383 if let Some(enabled) = enabled {
11384 state.enabled = enabled;
11385 }
11386 })?;
11387
11388 if stop_notice != StopNotice::SentOverConnection {
11399 if protocol == ModuleProtocol::Subc && stop_notice == StopNotice::NoConnection {
11400 info!(
11401 module_id,
11402 pid = child.pid,
11403 budget_ms = u64::try_from(drain_timeout.as_millis()).unwrap_or(u64::MAX),
11404 "module has no connection yet; requesting stop by signal"
11405 );
11406 }
11407 request_graceful_stop(module_id, &child);
11408 }
11409
11410 let exit_report = match timeout(drain_timeout, child.wait()).await {
11411 Ok(Ok(status)) => classify_reaped_child_exit(snapshot, &child, &status),
11412 Ok(Err(source)) => {
11413 fail_after_wait_error(
11414 module_id,
11415 snapshot,
11416 terminal_ring,
11417 spawn_events,
11418 format!("failed to wait during drain: {source}"),
11419 failure,
11420 );
11421 return Err(SuperviseError::Wait {
11422 module_id: module_id.to_string(),
11423 source,
11424 });
11425 }
11426 Err(_) => {
11427 warn!(
11440 module_id,
11441 pid = child.pid,
11442 budget_ms = u64::try_from(drain_timeout.as_millis()).unwrap_or(u64::MAX),
11443 reason = ?final_state,
11444 ?stop_notice,
11445 "drain budget expired before the module exited; killing it"
11446 );
11447 child.start_kill().map_err(|source| {
11448 fail_after_wait_error(
11449 module_id,
11450 snapshot,
11451 terminal_ring,
11452 spawn_events,
11453 format!("failed to kill during drain: {source}"),
11454 failure,
11455 );
11456 SuperviseError::Kill {
11457 module_id: module_id.to_string(),
11458 source,
11459 }
11460 })?;
11461 let status = child.wait().await.map_err(|source| {
11462 fail_after_wait_error(
11463 module_id,
11464 snapshot,
11465 terminal_ring,
11466 spawn_events,
11467 format!("failed to wait after drain kill: {source}"),
11468 failure,
11469 );
11470 SuperviseError::Wait {
11471 module_id: module_id.to_string(),
11472 source,
11473 }
11474 })?;
11475 classify_reaped_child_exit(snapshot, &child, &status)
11476 }
11477 };
11478
11479 let detail = lock_snapshot(snapshot)?.drain_disposition_detail.take();
11480 publish_after_terminal_record(
11481 module_id,
11482 || {
11483 record_terminal_with_detail(
11484 module_id,
11485 terminal_ring,
11486 spawn_events,
11487 &exit_report,
11488 terminal_disposition(final_state),
11489 detail,
11490 )
11491 },
11492 || {
11493 update_snapshot(snapshot, Some(module_id), |state| {
11494 state.state = final_state;
11495 if final_state == ModuleState::Failed {
11496 state.failed_retry.failed_at = Some(Instant::now());
11497 }
11498 if let Some(enabled) = enabled {
11499 state.enabled = enabled;
11500 }
11501 clear_current_process_facts(state);
11502 state.last_exit = Some(exit_report.clone());
11503 if exit_report.kind == ExitKind::DeliberateSeverance {
11504 state.lifetime_restarts += 1;
11505 }
11506 })
11507 },
11508 )?;
11509 child.drain_stderr(module_id).await;
11510
11511 release_dead_registration(registry, forwarding, snapshot, module_id, failure).await
11512}
11513
11514#[cfg(unix)]
11534fn request_graceful_stop(module_id: &str, child: &SupervisedChild) {
11535 let Some(pid) = child
11536 .id()
11537 .and_then(|pid| i32::try_from(pid).ok())
11538 .and_then(rustix::process::Pid::from_raw)
11539 else {
11540 debug!(
11541 module_id,
11542 "no pid to signal for teardown; falling through to the drain wait"
11543 );
11544 return;
11545 };
11546 match rustix::process::kill_process(pid, rustix::process::Signal::TERM) {
11547 Ok(()) => debug!(
11548 module_id,
11549 "sent SIGTERM to a module nothing else asked to stop"
11550 ),
11551 Err(err) => debug!(
11552 module_id,
11553 error = %err,
11554 "SIGTERM to module failed; the drain wait and kill still apply"
11555 ),
11556 }
11557}
11558
11559#[cfg(not(unix))]
11567fn request_graceful_stop(module_id: &str, _child: &SupervisedChild) {
11568 debug!(
11569 module_id,
11570 "no graceful stop signal exists on this platform; teardown of a module nothing asked to stop waits, then kills"
11571 );
11572}
11573
11574fn terminal_disposition(final_state: ModuleState) -> TerminalDisposition {
11575 match final_state {
11576 ModuleState::Stopped => TerminalDisposition::Stopped,
11577 ModuleState::Disabled => TerminalDisposition::Disabled,
11578 ModuleState::Restarting => TerminalDisposition::Restarting,
11579 ModuleState::Failed => TerminalDisposition::Failed,
11580 ModuleState::Starting
11581 | ModuleState::Running
11582 | ModuleState::Unresponsive
11583 | ModuleState::Draining => {
11584 unreachable!("terminal exits only finish in terminal or restarting states")
11585 }
11586 }
11587}
11588
11589async fn release_dead_registration(
11597 registry: &Registry,
11598 forwarding: Option<&ForwardingTable>,
11599 snapshot: &SharedSnapshot,
11600 module_id: &str,
11601 failure: &FailureContext,
11602) -> Result<(), SuperviseError> {
11603 let result = async {
11604 let registration = registry
11605 .get_module(module_id)
11606 .map_err(SuperviseError::Registry)?;
11607 match wait_for_registration_release(registry, module_id, REGISTRY_RELEASE_TIMEOUT).await {
11608 Ok(()) => return Ok(()),
11609 Err(SuperviseError::RegistrationStillActive { .. }) => {}
11610 Err(err) => return Err(err),
11611 }
11612 let pid = lock_snapshot(snapshot)?.reaped_pid;
11613 if let (Some(registration), Some(forwarding), Some(pid)) = (registration, forwarding, pid) {
11614 warn!(
11615 module_id,
11616 pid,
11617 connection_id = registration.connection_id.get(),
11618 "reaped module registration outlived release grace; closing dead connection"
11619 );
11620 forwarding.request_connection_close(
11621 registration.connection_id,
11622 CloseReason::new(
11623 "supervised_process_reaped",
11624 format!("module '{module_id}' pid {pid} exited"),
11625 ),
11626 );
11627 wait_for_slot_registration_release(
11628 registry,
11629 crate::registry::RegistrationSlot::Connection(registration.connection_id),
11630 REGISTRY_RELEASE_TIMEOUT,
11631 )
11632 .await?;
11633 }
11634 wait_for_registration_release(registry, module_id, Duration::ZERO).await
11635 }
11636 .await;
11637 if let Err(err) = &result {
11638 fail_snapshot(snapshot, Some(module_id), None, failure);
11639 error!(module_id, error = %err, "registration release failed after child exit; module is failed and start can retry");
11640 }
11641 result
11642}
11643
11644async fn wait_for_registration_release(
11647 registry: &Registry,
11648 module_id: &str,
11649 wait: Duration,
11650) -> Result<(), SuperviseError> {
11651 wait_for_slot_registration_release(
11652 registry,
11653 crate::registry::RegistrationSlot::Active(module_id),
11654 wait,
11655 )
11656 .await
11657}
11658
11659async fn wait_for_slot_registration_release(
11667 registry: &Registry,
11668 slot: crate::registry::RegistrationSlot<'_>,
11669 wait: Duration,
11670) -> Result<(), SuperviseError> {
11671 let deadline = Instant::now() + wait;
11672 let mut release_events = registration_release_events().subscribe();
11673 let still_active = |registration: &crate::registry::ModuleRegistration| {
11674 SuperviseError::RegistrationStillActive {
11675 module_id: registration.manifest.module_id.clone(),
11676 waited: wait,
11677 }
11678 };
11679 loop {
11680 let _observed_generation = *release_events.borrow_and_update();
11681 let Some(registration) = registry
11682 .registration(slot)
11683 .map_err(SuperviseError::Registry)?
11684 else {
11685 return Ok(());
11686 };
11687
11688 let now = Instant::now();
11689 if now >= deadline {
11690 return Err(still_active(®istration));
11691 }
11692
11693 let remaining = deadline.saturating_duration_since(now);
11694 match timeout(remaining, release_events.changed()).await {
11695 Ok(Ok(())) | Ok(Err(_)) => {}
11696 Err(_) => return Err(still_active(®istration)),
11697 }
11698 }
11699}
11700
11701#[cfg(test)]
11702mod slot_registration_wait_tests {
11703 use super::*;
11704 use crate::registry::{ConnectionId, RegistrationSlot};
11705 use subc_protocol::manifest::ModuleManifest;
11706
11707 #[tokio::test]
11708 async fn enable_release_failure_is_failed_and_a_second_enable_retries() {
11709 let registry = Arc::new(Registry::default());
11710 let supervisor = Supervisor::new_for_test(Arc::clone(®istry), RestartPolicy::default());
11711 let runtime = supervisor.runtime_config();
11712 let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::disabled()));
11713 let spec = ModuleSpec {
11714 module_id: "enable-stale-registration".to_string(),
11715 program: PathBuf::from("/missing/enable-retry-test"),
11716 args: Vec::new(),
11717 env: Vec::new(),
11718 reserved: false,
11719 reserved_prefixes: Vec::new(),
11720 protocol: ModuleProtocol::Subc,
11721 overlap: Default::default(),
11722 };
11723 let connection = ConnectionId::new(90);
11724 registry
11725 .register_with_control_ops(
11726 ModuleManifest::builder(&spec.module_id, "0.1.0").build(),
11727 1,
11728 connection,
11729 Vec::new(),
11730 )
11731 .unwrap();
11732 let mut child = None;
11733 let err = set_child_enabled(
11734 &spec,
11735 &runtime,
11736 ®istry,
11737 &supervisor.process_liveness,
11738 &snapshot,
11739 &mut child,
11740 true,
11741 &runtime.failure_context(false),
11742 )
11743 .await
11744 .unwrap_err();
11745 assert!(matches!(
11746 err,
11747 SuperviseError::RegistrationStillActive { .. }
11748 ));
11749 assert_eq!(lock_snapshot(&snapshot).unwrap().state, ModuleState::Failed);
11750 assert!(child.is_none());
11751 registry.deregister_connection(connection).unwrap();
11752 let err = set_child_enabled(
11753 &spec,
11754 &runtime,
11755 ®istry,
11756 &supervisor.process_liveness,
11757 &snapshot,
11758 &mut child,
11759 true,
11760 &runtime.failure_context(false),
11761 )
11762 .await
11763 .unwrap_err();
11764 assert!(
11765 matches!(err, SuperviseError::Spawn { .. }),
11766 "second enable must attempt a spawn: {err}"
11767 );
11768 assert_eq!(lock_snapshot(&snapshot).unwrap().state, ModuleState::Failed);
11769 }
11770
11771 const INCUMBENT: u64 = 1;
11772 const CANDIDATE: u64 = 2;
11773
11774 fn swapped_registry() -> Arc<Registry> {
11775 let registry = Arc::new(Registry::default());
11776 let manifest = ModuleManifest::builder("m", "0.1.0").build();
11777 registry
11778 .register_with_control_ops(
11779 manifest.clone(),
11780 1,
11781 ConnectionId::new(INCUMBENT),
11782 Vec::new(),
11783 )
11784 .unwrap();
11785 registry
11786 .register_candidate_with_control_ops(
11787 manifest,
11788 1,
11789 ConnectionId::new(CANDIDATE),
11790 Vec::new(),
11791 )
11792 .unwrap();
11793 registry
11794 }
11795
11796 #[tokio::test]
11800 async fn incumbent_release_is_awaited_by_connection_not_by_module_id() {
11801 let registry = swapped_registry();
11802 registry.promote_candidate("m").unwrap().unwrap();
11803
11804 assert!(matches!(
11805 wait_for_registration_release(®istry, "m", Duration::from_millis(50)).await,
11806 Err(SuperviseError::RegistrationStillActive { .. })
11807 ));
11808
11809 assert!(matches!(
11811 wait_for_slot_registration_release(
11812 ®istry,
11813 RegistrationSlot::Connection(ConnectionId::new(INCUMBENT)),
11814 Duration::from_millis(50),
11815 )
11816 .await,
11817 Err(SuperviseError::RegistrationStillActive { .. })
11818 ));
11819
11820 let releaser = Arc::clone(®istry);
11821 let release = tokio::spawn(async move {
11822 sleep(Duration::from_millis(20)).await;
11823 releaser
11824 .deregister_connection(ConnectionId::new(INCUMBENT))
11825 .unwrap();
11826 notify_registration_release();
11827 });
11828 wait_for_slot_registration_release(
11829 ®istry,
11830 RegistrationSlot::Connection(ConnectionId::new(INCUMBENT)),
11831 Duration::from_secs(5),
11832 )
11833 .await
11834 .expect("the incumbent's own registration is released");
11835 release.await.unwrap();
11836 assert!(registry.get_module("m").unwrap().is_some());
11837 }
11838
11839 #[tokio::test]
11842 async fn candidate_slot_wait_ignores_the_incumbents_registration() {
11843 let registry = swapped_registry();
11844 assert!(matches!(
11845 wait_for_slot_registration_release(
11846 ®istry,
11847 RegistrationSlot::Candidate("m"),
11848 Duration::from_millis(50),
11849 )
11850 .await,
11851 Err(SuperviseError::RegistrationStillActive { .. })
11852 ));
11853 registry
11854 .deregister_connection(ConnectionId::new(CANDIDATE))
11855 .unwrap();
11856 wait_for_slot_registration_release(
11857 ®istry,
11858 RegistrationSlot::Candidate("m"),
11859 Duration::from_millis(50),
11860 )
11861 .await
11862 .expect("a candidate slot with no candidate is released");
11863 assert!(registry
11864 .registration(RegistrationSlot::Active("m"))
11865 .unwrap()
11866 .is_some());
11867 }
11868}
11869
11870fn classify_exit(status: &ExitStatus) -> ExitReport {
11871 ExitReport {
11872 kind: if status.success() {
11873 ExitKind::Clean
11874 } else {
11875 ExitKind::Crash
11876 },
11877 code: status.code(),
11878 signal: exit_signal(status),
11879 at_ms: unix_ms_now(),
11880 }
11881}
11882
11883fn wait_error_exit_report() -> ExitReport {
11889 ExitReport {
11890 kind: ExitKind::Crash,
11891 code: None,
11892 signal: None,
11893 at_ms: unix_ms_now(),
11894 }
11895}
11896
11897#[cfg(unix)]
11898fn exit_signal(status: &ExitStatus) -> Option<i32> {
11899 use std::os::unix::process::ExitStatusExt;
11900
11901 status.signal()
11902}
11903
11904#[cfg(not(unix))]
11905fn exit_signal(_status: &ExitStatus) -> Option<i32> {
11906 None
11907}
11908
11909fn reset_restart_count(snapshot: &SharedSnapshot, module_id: &str) -> Result<(), SuperviseError> {
11915 update_snapshot(snapshot, Some(module_id), |state| {
11916 state.clear_crash_restarts();
11917 })
11918}
11919
11920fn set_running(
11921 snapshot: &SharedSnapshot,
11922 child: &SupervisedChild,
11923 module_id: &str,
11924 spawn_events: &SpawnEventFeed,
11925) -> Result<(), SuperviseError> {
11926 let mut state = snapshot.lock().map_err(|_| SuperviseError::StatePoisoned {
11927 module_id: Some(module_id.to_string()),
11928 })?;
11929 state.spawn_generation = spawn_events.emit_spawned(module_id, child.pid, child.spawned_at_ms);
11930 if std::mem::take(&mut state.coalesced_restart_pending) {
11931 let generation = state.spawn_generation;
11932 info!(module_id, "restart already satisfied by generation {generation}; coalesced pending request completed");
11933 }
11934 state.drain_disposition_detail = None;
11935 state.spawn_failure = None;
11936 state.waiting_for_previous_pid = None;
11937 state.waiting_for_previous_detail = None;
11938 state.in_alternate_slot = false;
11941 state.configuration_updated_since_spawn = false;
11942 state.spawned_protocol = Some(child.protocol);
11943 state.state = ModuleState::Running;
11944 state.enabled = true;
11945 state.process_alive = true;
11946 state.pid = child.id();
11947 state.subtree_target = child.subtree_target.clone();
11948 state.health.status = SupervisorHealthStatus::Unknown;
11951 state.health.consecutive_failures = 0;
11952 state.health.detail = None;
11953 state.health.metrics = None;
11954 state.health.last_probe_ms = None;
11955 state.failed_retry.healthy_since = None;
11956 state.failed_retry.failed_at = None;
11957 state.failed_retry.running_since = Some(child.spawned_at);
11958 #[cfg(windows)]
11959 {
11960 state.pipe_consumed = child
11961 .nonce_delivery
11962 .as_ref()
11963 .map(|delivery| delivery.consumed());
11964 }
11965 #[cfg(target_os = "macos")]
11966 {
11967 state.report_ready = Some(Arc::clone(&child.report_ready));
11968 }
11969 state.spawned_at_ms = Some(child.spawned_at_ms);
11970 state.exec_outcome = Some(Arc::clone(&child.exec.0));
11971 state.spawned_from = Some(child.spawned_from.clone());
11972 state.launch_spec_digest = Some(child.launch_spec_digest.clone());
11973 state.spawned_file_identity = child.spawned_file_identity.clone();
11974 state.process_start_time = child.process_start_time;
11975 Ok(())
11976}
11977
11978fn clear_current_process_facts(state: &mut SupervisorSnapshot) {
11979 state.subtree_target = crate::subtree_resources::Target::Unavailable("not_running");
11980 state.failed_retry.healthy_since = None;
11981 state.failed_retry.running_since = None;
11982 state.waiting_for_previous_pid = None;
11983 state.waiting_for_previous_detail = None;
11984 state.process_alive = false;
11985 state.spawned_protocol = None;
11986 state.pid = None;
11987 #[cfg(windows)]
11988 {
11989 state.pipe_consumed = None;
11990 }
11991 #[cfg(target_os = "macos")]
11992 {
11993 state.report_ready = None;
11994 }
11995 state.spawned_at_ms = None;
11996 state.exec_outcome = None;
11997 state.spawned_from = None;
11998 state.launch_spec_digest = None;
11999 state.spawned_file_identity = None;
12000 state.process_start_time = None;
12001 state.deliberate_severance = None;
12002}
12003
12004#[cfg(test)]
12005fn record_deliberate_severance(
12006 snapshot: &SharedSnapshot,
12007 identity: ProcessIdentity,
12008) -> Result<(), SuperviseError> {
12009 update_snapshot(snapshot, None, |state| {
12010 state.deliberate_severance = Some(identity);
12011 })
12012}
12013
12014fn apply_deliberate_severance_marker(
12015 snapshot: &SharedSnapshot,
12016 exited_identity: Option<ProcessIdentity>,
12017 mut exit_report: ExitReport,
12018) -> ExitReport {
12019 let marker = lock_snapshot(snapshot)
12020 .ok()
12021 .and_then(|mut state| state.deliberate_severance.take());
12022 if marker.is_some() && marker == exited_identity {
12023 exit_report.kind = ExitKind::DeliberateSeverance;
12024 }
12025 exit_report
12026}
12027
12028fn classify_reaped_child_exit(
12029 snapshot: &SharedSnapshot,
12030 child: &SupervisedChild,
12031 status: &ExitStatus,
12032) -> ExitReport {
12033 let _ = update_snapshot(snapshot, None, |state| {
12034 state.reaped_pid = Some(child.pid);
12035 state.spawn_failure = child.spawn_failure.clone();
12036 });
12037 if child.spawn_failure.as_deref() == Some("registration_timeout") {
12038 registration_failure_exit_report(classify_exit(status))
12039 } else {
12040 apply_deliberate_severance_marker(snapshot, child.process_identity(), classify_exit(status))
12041 }
12042}
12043
12044fn fail_snapshot(
12045 snapshot: &SharedSnapshot,
12046 module_id: Option<&str>,
12047 last_exit: Option<ExitReport>,
12048 failure: &FailureContext,
12049) {
12050 if let Err(err) = update_snapshot(snapshot, module_id, |state| {
12051 state.state = ModuleState::Failed;
12052 clear_current_process_facts(state);
12053 if failure.may_retry() && state.enabled {
12054 state.failed_retry.schedule(module_id.unwrap_or("unknown"));
12055 } else if failure.shutdown.is_set() {
12056 state.failed_retry.cancel();
12057 }
12058 if let Some(last_exit) = last_exit {
12059 state.last_exit = Some(last_exit);
12060 }
12061 }) {
12062 error!(error = %err, "failed to mark supervisor state failed");
12063 }
12064}
12065
12066fn update_snapshot(
12067 snapshot: &SharedSnapshot,
12068 module_id: Option<&str>,
12069 update: impl FnOnce(&mut SupervisorSnapshot),
12070) -> Result<(), SuperviseError> {
12071 let mut state = snapshot.lock().map_err(|_| SuperviseError::StatePoisoned {
12072 module_id: module_id.map(ToOwned::to_owned),
12073 })?;
12074 update(&mut state);
12075 Ok(())
12076}
12077
12078const SLOW_SNAPSHOT_LOCK_THRESHOLD: Duration = Duration::from_millis(250);
12079
12080fn lock_snapshot_for_control<'a>(
12081 snapshot: &'a SharedSnapshot,
12082 module_id: &str,
12083 caller: &'static str,
12084) -> Result<std::sync::MutexGuard<'a, SupervisorSnapshot>, SuperviseError> {
12085 let started_at = Instant::now();
12086 let guard = lock_snapshot(snapshot)?;
12087 let waited = started_at.elapsed();
12088 if waited >= SLOW_SNAPSHOT_LOCK_THRESHOLD {
12089 warn!(
12090 module_id = %module_id,
12091 waited_ms = waited.as_millis() as u64,
12092 caller = %caller,
12093 "slow snapshot lock"
12094 );
12095 }
12096 Ok(guard)
12097}
12098
12099fn lock_snapshot(
12100 snapshot: &SharedSnapshot,
12101) -> Result<std::sync::MutexGuard<'_, SupervisorSnapshot>, SuperviseError> {
12102 snapshot
12103 .lock()
12104 .map_err(|_| SuperviseError::StatePoisoned { module_id: None })
12105}
12106
12107#[cfg(test)]
12108mod terminal_history_tests {
12109 use std::{
12110 path::PathBuf,
12111 sync::Arc,
12112 time::{Duration, Instant},
12113 };
12114
12115 use tokio::{sync::mpsc, time::sleep};
12116
12117 use super::{
12118 apply_deliberate_severance_marker, daemon_will_restart, drain_child_to_state,
12119 drained_after_quiescence_wait, handle_reload_spawn_failure, health_restart_child,
12120 lock_snapshot, on_child_exit, record_deliberate_severance, record_wait_error_terminal,
12121 reset_restart_count, spawn_and_mark_running, update_snapshot, wait_error_exit_report,
12122 ExitKind, ExitReport, ModuleProtocol, ModuleSpec, ModuleState, NextAction, ProcessIdentity,
12123 RestartPolicy, SpawnEventKind, StopNotice, SuperviseError, SupervisedConfiguration,
12124 SupervisedModule, SupervisedModuleInner, Supervisor, SupervisorHandle,
12125 SupervisorHealthStatus, SupervisorSnapshot,
12126 };
12127 use super::Instant as ClockInstant;
12132 use crate::{
12133 registry::Registry,
12134 terminal_ring::{TerminalRing, TerminalRingConfig},
12135 };
12136 use std::sync::Mutex;
12137 use subc_control::TerminalDisposition;
12138
12139 pub(super) fn fake_aft_stub_path() -> PathBuf {
12144 let mut path = std::env::current_exe().expect("current_exe available in tests");
12145 path.pop();
12146 path.pop();
12147 path.push(if cfg!(windows) {
12148 "fake-aft-stub.exe"
12149 } else {
12150 "fake-aft-stub"
12151 });
12152 assert!(
12153 path.exists(),
12154 "fake-aft-stub not built at {}: run `cargo test -p subc-core` (which builds \
12155 [[bin]] targets) rather than `cargo test -p subc-core --lib` (which does not)",
12156 path.display()
12157 );
12158 path
12159 }
12160
12161 #[test]
12162 fn reserved_never_spawned_refuses_every_hello() {
12163 let supervisor = SupervisorHandle::default();
12168 supervisor.apply_identity_configuration(&ModuleSpec {
12169 module_id: "never-spawned".to_string(),
12170 program: PathBuf::from("/usr/bin/false"),
12171 args: Vec::new(),
12172 env: Vec::new(),
12173 reserved: true,
12174 reserved_prefixes: Vec::new(),
12175 protocol: ModuleProtocol::Subc,
12176 overlap: Default::default(),
12177 });
12178 assert!(
12179 supervisor
12180 .reserved_hello_verification("never-spawned", Some("any-forged-nonce"))
12181 .is_err(),
12182 "forged nonce must refuse on a reserved never-spawned id"
12183 );
12184 assert!(
12185 supervisor
12186 .reserved_hello_verification("never-spawned", None)
12187 .is_err(),
12188 "absent nonce must refuse on a reserved never-spawned id"
12189 );
12190 supervisor.set_spawn_nonce("never-spawned", "minted".to_string());
12192 supervisor.apply_identity_configuration(&ModuleSpec {
12193 module_id: "never-spawned".to_string(),
12194 program: PathBuf::from("/usr/bin/false"),
12195 args: Vec::new(),
12196 env: Vec::new(),
12197 reserved: true,
12198 reserved_prefixes: Vec::new(),
12199 protocol: ModuleProtocol::Subc,
12200 overlap: Default::default(),
12201 });
12202 assert!(supervisor
12203 .reserved_hello_verification("never-spawned", Some("minted"))
12204 .is_ok());
12205 assert!(supervisor
12206 .reserved_hello_verification("never-spawned", Some("forged"))
12207 .is_err());
12208 }
12209
12210 fn seed_crash_restarts(state: &mut SupervisorSnapshot, count: u32) {
12213 let now = ClockInstant::now();
12214 for _ in 0..count {
12215 state.crash_restarts.push_back(now);
12216 }
12217 }
12218
12219 fn age_oldest_crash_restart_out_of_window(state: &mut SupervisorSnapshot, window: Duration) {
12223 let aged = state
12224 .crash_restarts
12225 .front()
12226 .expect("a crash restart must be recorded before it can be aged")
12227 .checked_sub(window + Duration::from_secs(1))
12228 .expect("the test clock is far enough from its origin to age an instant");
12229 state.crash_restarts[0] = aged;
12230 }
12231
12232 fn snapshot_with_restarts(enabled: bool, count: u32) -> SupervisorSnapshot {
12233 let mut state = SupervisorSnapshot::new(ModuleState::Running, enabled);
12234 seed_crash_restarts(&mut state, count);
12235 state
12236 }
12237
12238 #[test]
12239 fn daemon_owned_recovery_predicate_uses_the_pre_increment_budget() {
12240 let policy = RestartPolicy::new(3, Duration::ZERO);
12241 let now = ClockInstant::now();
12242 assert!(daemon_will_restart(
12243 &mut snapshot_with_restarts(true, 2),
12244 &policy,
12245 now
12246 ));
12247 assert!(!daemon_will_restart(
12248 &mut snapshot_with_restarts(true, 3),
12249 &policy,
12250 now
12251 ));
12252 assert!(!daemon_will_restart(
12253 &mut snapshot_with_restarts(false, 0),
12254 &policy,
12255 now
12256 ));
12257 }
12258
12259 #[test]
12260 fn crash_restart_backoff_escalates_with_in_window_count() {
12261 let policy = RestartPolicy::new(4, Duration::from_millis(100))
12262 .with_max_backoff(Duration::from_secs(30));
12263 let now = ClockInstant::now();
12264 let mut state = SupervisorSnapshot::new(ModuleState::Running, true);
12265 let schedules = (0..4)
12266 .map(|_| {
12267 state
12268 .next_crash_restart(&policy, now)
12269 .expect("the test policy allows four crash restarts")
12270 })
12271 .collect::<Vec<_>>();
12272
12273 assert_eq!(
12274 schedules
12275 .iter()
12276 .map(|schedule| schedule.restart_in_window)
12277 .collect::<Vec<_>>(),
12278 vec![0, 1, 2, 3]
12279 );
12280 assert_eq!(
12281 schedules
12282 .iter()
12283 .map(|schedule| schedule.delay)
12284 .collect::<Vec<_>>(),
12285 vec![
12286 Duration::from_millis(100),
12287 Duration::from_secs(1),
12288 Duration::from_secs(10),
12289 Duration::from_secs(30),
12290 ]
12291 );
12292 }
12293
12294 #[test]
12295 fn crash_restart_backoff_resets_after_ring_clear() {
12296 let policy = RestartPolicy::new(3, Duration::from_millis(100));
12297 let now = ClockInstant::now();
12298 let mut state = SupervisorSnapshot::new(ModuleState::Running, true);
12299 assert_eq!(
12300 state.next_crash_restart(&policy, now).unwrap().delay,
12301 Duration::from_millis(100)
12302 );
12303 assert_eq!(
12304 state.next_crash_restart(&policy, now).unwrap().delay,
12305 Duration::from_secs(1)
12306 );
12307
12308 state.clear_crash_restarts();
12309 let schedule = state
12310 .next_crash_restart(&policy, now)
12311 .expect("a cleared ring must allow another restart");
12312 assert_eq!(schedule.restart_in_window, 0);
12313 assert_eq!(schedule.delay, Duration::from_millis(100));
12314 }
12315
12316 #[test]
12317 fn crash_restart_backoff_ignores_aged_restarts() {
12318 let policy = RestartPolicy::new(3, Duration::from_millis(100));
12319 let now = ClockInstant::now();
12320 let mut state = SupervisorSnapshot::new(ModuleState::Running, true);
12321 state
12322 .next_crash_restart(&policy, now)
12323 .expect("the first restart is allowed");
12324 state
12325 .next_crash_restart(&policy, now)
12326 .expect("the second restart is allowed");
12327 state.crash_restarts[0] = now
12328 .checked_sub(policy.window + Duration::from_secs(1))
12329 .expect("the fake clock can age a restart past the window");
12330
12331 let schedule = state
12332 .next_crash_restart(&policy, now)
12333 .expect("an aged restart must release its slot");
12334 assert_eq!(schedule.restart_in_window, 1);
12335 assert_eq!(schedule.delay, Duration::from_secs(1));
12336 assert_eq!(state.crash_restarts.len(), 2);
12337 }
12338
12339 #[test]
12343 fn a_budget_spent_before_the_window_no_longer_refuses() {
12344 let policy = RestartPolicy::new(3, Duration::ZERO);
12345 let mut state = snapshot_with_restarts(true, 3);
12346 let now = ClockInstant::now();
12347 assert!(!daemon_will_restart(&mut state, &policy, now));
12348
12349 assert!(daemon_will_restart(
12350 &mut state,
12351 &policy,
12352 now + policy.window + Duration::from_secs(1)
12353 ));
12354 assert!(
12355 state.crash_restarts.is_empty(),
12356 "reading the budget must drop the instants that left the window"
12357 );
12358 }
12359
12360 fn module_with_recovery_snapshot(
12361 state: ModuleState,
12362 enabled: bool,
12363 restart_count: u32,
12364 ) -> SupervisedModule {
12365 let registry = Arc::new(Registry::default());
12366 let supervisor =
12367 Supervisor::new_for_test(Arc::clone(®istry), RestartPolicy::new(3, Duration::ZERO));
12368 let runtime = supervisor.runtime_config();
12369 let spec = ModuleSpec {
12370 module_id: "recovery-snapshot".to_string(),
12371 program: fake_aft_stub_path(),
12372 args: Vec::new(),
12373 env: Vec::new(),
12374 reserved: false,
12375 reserved_prefixes: Vec::new(),
12376 protocol: ModuleProtocol::Subc,
12377 overlap: Default::default(),
12378 };
12379 let mut snapshot = SupervisorSnapshot::new(state, enabled);
12380 seed_crash_restarts(&mut snapshot, restart_count);
12381 let (commands, _rx) = mpsc::channel(4);
12384 SupervisedModule {
12385 inner: Arc::new(SupervisedModuleInner {
12386 module_id: spec.module_id.clone(),
12387 registry,
12388 snapshot: Arc::new(Mutex::new(snapshot)),
12389 configuration: Arc::new(Mutex::new(SupervisedConfiguration {
12390 spec,
12391 health: runtime.health,
12392 })),
12393 stderr_ring: runtime.stderr_ring,
12394 terminal_ring: runtime.terminal_ring,
12395 commands,
12396 monitor: Mutex::new(None),
12397 restart_policy: runtime.restart_policy,
12398 effective_drain_timeout: runtime.effective_drain_timeout,
12399 provenance_probe: supervisor.provenance_probe.clone(),
12400 }),
12401 }
12402 }
12403
12404 #[cfg(target_os = "linux")]
12405 #[tokio::test]
12406 async fn no_cgroup_placement_does_not_block_fake_aft_stub_spawn() {
12407 let supervisor =
12408 Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default())
12409 .with_cgroup_placement(None);
12410 let result = supervisor.spawn(ModuleSpec {
12411 module_id: "no-cgroup-placement".to_string(),
12412 program: fake_aft_stub_path(),
12413 args: Vec::new(),
12414 env: Vec::new(),
12415 reserved: false,
12416 reserved_prefixes: Vec::new(),
12417 protocol: ModuleProtocol::Subc,
12418 overlap: Default::default(),
12419 });
12420
12421 assert!(
12422 result.is_ok(),
12423 "no delegation must not turn an otherwise valid spawn into a failure: {result:?}"
12424 );
12425 }
12426
12427 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
12428 async fn undecided_snapshot_uses_shared_restart_predicate() {
12429 assert!(module_with_recovery_snapshot(ModuleState::Running, true, 2)
12430 .will_recover_after_connection_loss()
12431 .unwrap());
12432 assert!(
12433 !module_with_recovery_snapshot(ModuleState::Running, true, 3)
12434 .will_recover_after_connection_loss()
12435 .unwrap()
12436 );
12437 }
12438
12439 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
12440 async fn restarting_snapshot_at_exhausted_budget_is_non_terminal() {
12441 assert!(
12442 module_with_recovery_snapshot(ModuleState::Restarting, true, 3)
12443 .will_recover_after_connection_loss()
12444 .unwrap()
12445 );
12446 }
12447
12448 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
12449 async fn terminal_phase_snapshots_are_terminal_before_budget_exhaustion() {
12450 assert!(!module_with_recovery_snapshot(ModuleState::Failed, true, 0)
12451 .will_recover_after_connection_loss()
12452 .unwrap());
12453 assert!(
12454 !module_with_recovery_snapshot(ModuleState::Disabled, true, 0)
12455 .will_recover_after_connection_loss()
12456 .unwrap()
12457 );
12458 }
12459
12460 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
12461 async fn warming_snapshot_is_limited_to_startup_phases() {
12462 for state in [
12463 ModuleState::Starting,
12464 ModuleState::Running,
12465 ModuleState::Restarting,
12466 ] {
12467 assert!(
12468 module_with_recovery_snapshot(state, true, 0)
12469 .is_warming()
12470 .unwrap(),
12471 "{state:?} should be warming"
12472 );
12473 }
12474 for state in [
12475 ModuleState::Unresponsive,
12476 ModuleState::Draining,
12477 ModuleState::Stopped,
12478 ModuleState::Failed,
12479 ModuleState::Disabled,
12480 ] {
12481 assert!(
12482 !module_with_recovery_snapshot(state, true, 0)
12483 .is_warming()
12484 .unwrap(),
12485 "{state:?} should not be warming"
12486 );
12487 }
12488 }
12489
12490 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
12491 async fn terminal_history_survives_respawn_and_keeps_both_crashes_in_order() {
12492 let registry = Arc::new(Registry::default());
12493 let supervisor =
12494 Supervisor::new_for_test(Arc::clone(®istry), RestartPolicy::new(1, Duration::ZERO));
12495 let module = supervisor
12496 .spawn(ModuleSpec {
12497 module_id: "terminal-history".to_string(),
12498 program: fake_aft_stub_path(),
12499 args: Vec::new(),
12500 env: vec![("FAKE_AFT_EXIT_CODE".to_string(), "23".to_string())],
12501 reserved: false,
12502 reserved_prefixes: Vec::new(),
12503 protocol: ModuleProtocol::Subc,
12504 overlap: Default::default(),
12505 })
12506 .unwrap();
12507
12508 let deadline = Instant::now() + Duration::from_secs(5);
12509 loop {
12510 let history = module.terminal_history();
12511 if history.entries.len() == 2 {
12512 assert_eq!(module.status().unwrap().state, ModuleState::Failed);
12513 assert_eq!(history.dropped, 0);
12514 assert_eq!(
12515 history
12516 .entries
12517 .iter()
12518 .map(|entry| entry.exit_code)
12519 .collect::<Vec<_>>(),
12520 vec![Some(23), Some(23)]
12521 );
12522 assert!(history.entries[0].at_ms <= history.entries[1].at_ms);
12523 return;
12524 }
12525 assert!(
12526 Instant::now() < deadline,
12527 "module did not retain two terminal exits: {history:?}"
12528 );
12529 sleep(Duration::from_millis(10)).await;
12530 }
12531 }
12532
12533 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
12537 async fn disable_during_crash_backoff_cancels_pending_respawn() {
12538 let backoff = Duration::from_secs(2);
12539 let supervisor = Supervisor::new_for_test(
12540 Arc::new(Registry::default()),
12541 RestartPolicy::new(10, backoff),
12542 );
12543 let module = supervisor
12544 .spawn(ModuleSpec {
12545 module_id: "disable-during-backoff".to_string(),
12546 program: fake_aft_stub_path(),
12547 args: Vec::new(),
12548 env: vec![("FAKE_AFT_EXIT_CODE".to_string(), "23".to_string())],
12549 reserved: false,
12550 reserved_prefixes: Vec::new(),
12551 protocol: ModuleProtocol::Subc,
12552 overlap: Default::default(),
12553 })
12554 .unwrap();
12555
12556 let deadline = Instant::now() + Duration::from_secs(5);
12558 loop {
12559 if module.status().unwrap().state == ModuleState::Restarting {
12560 break;
12561 }
12562 assert!(
12563 Instant::now() < deadline,
12564 "module never entered the crash backoff"
12565 );
12566 sleep(Duration::from_millis(10)).await;
12567 }
12568
12569 let started = Instant::now();
12570 module.set_enabled(false).await.unwrap();
12571 let waited = started.elapsed();
12572
12573 assert!(
12574 waited < backoff / 2,
12575 "disable waited {waited:?} behind the {backoff:?} crash backoff; the operator command must preempt the pending respawn"
12576 );
12577 assert_eq!(module.status().unwrap().state, ModuleState::Disabled);
12578
12579 sleep(backoff + Duration::from_millis(500)).await;
12581 let status = module.status().unwrap();
12582 assert_eq!(status.state, ModuleState::Disabled);
12583 assert_eq!(
12584 status.spawn_generation, 1,
12585 "module respawned after the operator disabled it"
12586 );
12587 }
12588
12589 #[cfg(unix)]
12592 fn protocol_none_sigterm_exits_clean_spec(
12593 module_id: &str,
12594 dir: &std::path::Path,
12595 ) -> (ModuleSpec, PathBuf, PathBuf) {
12596 let ready = dir.join("ready");
12597 let marker = dir.join("sigterm");
12598 let spec = ModuleSpec {
12599 module_id: module_id.to_string(),
12600 program: fake_aft_stub_path(),
12601 args: Vec::new(),
12602 env: vec![
12603 ("FAKE_AFT_NEVER_CONNECT".to_string(), "1".to_string()),
12604 (
12605 "FAKE_AFT_SIGTERM_MARKER_PATH".to_string(),
12606 marker.display().to_string(),
12607 ),
12608 (
12609 "FAKE_AFT_NEVER_CONNECT_READY_PATH".to_string(),
12610 ready.display().to_string(),
12611 ),
12612 ],
12613 reserved: false,
12614 reserved_prefixes: Vec::new(),
12615 protocol: ModuleProtocol::None,
12616 overlap: Default::default(),
12617 };
12618 (spec, ready, marker)
12619 }
12620
12621 #[cfg(unix)]
12625 async fn wait_for_file(path: &std::path::Path) {
12626 let deadline = Instant::now() + Duration::from_secs(10);
12627 while !path.exists() {
12628 assert!(
12629 Instant::now() < deadline,
12630 "{} never appeared",
12631 path.display()
12632 );
12633 sleep(Duration::from_millis(10)).await;
12634 }
12635 }
12636
12637 #[cfg(unix)]
12641 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
12642 async fn protocol_none_unrequested_clean_exit_restarts_as_a_crash() {
12643 let dir = cortexkit_test_support::ScratchDir::new("none-unrequested-clean-exit");
12644 let (spec, ready, marker) =
12645 protocol_none_sigterm_exits_clean_spec("none-unrequested-clean-exit", dir.path());
12646 let supervisor = Supervisor::new_for_test(
12647 Arc::new(Registry::default()),
12648 RestartPolicy::new(3, Duration::ZERO),
12649 );
12650 let module = supervisor.spawn(spec).unwrap();
12651 wait_for_file(&ready).await;
12652 let deadline = Instant::now() + Duration::from_secs(10);
12656 let first_pid = loop {
12657 let status = module.status().unwrap();
12658 if status.state == ModuleState::Running {
12659 if let Some(pid) = status.pid {
12660 break pid;
12661 }
12662 }
12663 assert!(
12664 Instant::now() < deadline,
12665 "a running module must report its pid after exec confirmation: {status:?}"
12666 );
12667 sleep(Duration::from_millis(10)).await;
12668 };
12669
12670 rustix::process::kill_process(
12671 rustix::process::Pid::from_raw(i32::try_from(first_pid).unwrap()).unwrap(),
12672 rustix::process::Signal::TERM,
12673 )
12674 .unwrap();
12675
12676 let deadline = Instant::now() + Duration::from_secs(10);
12677 let respawned = loop {
12678 let status = module.status().unwrap();
12679 if status.state == ModuleState::Running
12680 && status.pid.is_some_and(|pid| pid != first_pid)
12681 {
12682 break status;
12683 }
12684 assert!(
12685 Instant::now() < deadline,
12686 "protocol-none module was not respawned after an unrequested clean exit: {status:?}"
12687 );
12688 sleep(Duration::from_millis(10)).await;
12689 };
12690 assert_eq!(respawned.spawn_generation, 2);
12691 assert!(
12692 marker.exists(),
12693 "the child must have exited through its SIGTERM handler (exit 0), or this proves nothing about clean exits"
12694 );
12695
12696 let history = module.terminal_history();
12697 assert_eq!(history.entries.len(), 1, "{history:?}");
12698 let entry = &history.entries[0];
12699 assert_eq!(entry.exit_code, Some(0));
12700 assert_eq!(entry.exit_kind, subc_control::TerminalExitKind::Clean);
12701 assert_eq!(entry.disposition, TerminalDisposition::Restarting);
12702
12703 module.stop().await.unwrap();
12704 }
12705
12706 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
12710 async fn protocol_none_repeated_clean_exits_exhaust_the_restart_budget() {
12711 let supervisor = Supervisor::new_for_test(
12712 Arc::new(Registry::default()),
12713 RestartPolicy::new(1, Duration::ZERO),
12714 );
12715 let module = supervisor
12716 .spawn(ModuleSpec {
12717 module_id: "none-clean-exit-budget".to_string(),
12718 program: fake_aft_stub_path(),
12719 args: Vec::new(),
12720 env: vec![("FAKE_AFT_EXIT_CODE".to_string(), "0".to_string())],
12721 reserved: false,
12722 reserved_prefixes: Vec::new(),
12723 protocol: ModuleProtocol::None,
12724 overlap: Default::default(),
12725 })
12726 .unwrap();
12727
12728 let deadline = Instant::now() + Duration::from_secs(10);
12731 loop {
12732 let status = module.status().unwrap();
12733 if status.state == ModuleState::Failed {
12734 break;
12735 }
12736 assert!(
12737 Instant::now() < deadline,
12738 "module never exhausted its budget: {status:?} {:?}",
12739 module.terminal_history()
12740 );
12741 sleep(Duration::from_millis(10)).await;
12742 }
12743 let history = module.terminal_history();
12744 assert_eq!(
12745 history
12746 .entries
12747 .iter()
12748 .map(|entry| (entry.exit_code, entry.disposition.clone()))
12749 .collect::<Vec<_>>(),
12750 vec![
12751 (Some(0), TerminalDisposition::Restarting),
12752 (Some(0), TerminalDisposition::Failed),
12753 ]
12754 );
12755 let detail = history.entries[1]
12756 .disposition_detail
12757 .as_deref()
12758 .expect("a budget failure names the budget");
12759 assert!(detail.contains("max_restarts=1"), "{detail}");
12760 assert_eq!(module.status().unwrap().spawn_generation, 2);
12761 }
12762
12763 #[test]
12764 fn restart_budget_failure_is_published_after_its_terminal_record() {
12765 use std::io::{Read, Write};
12766 let supervisor = Supervisor::new_for_test(
12767 Arc::new(Registry::default()),
12768 RestartPolicy::new(0, Duration::ZERO),
12769 );
12770 let runtime = supervisor.runtime_config();
12771 let spec = ModuleSpec {
12772 protocol: ModuleProtocol::None,
12773 ..windowed_crash_spec("budget-publication-order")
12774 };
12775 let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::new(
12776 ModuleState::Running,
12777 true,
12778 )));
12779 lock_snapshot(&snapshot).unwrap().process_alive = true;
12780 let reaped_snapshot = snapshot.clone();
12781 let mut barrier = terminal_record_barrier(&runtime.spawn_events);
12782 let history = runtime.terminal_ring.clone();
12783 let reap = std::thread::spawn(move || {
12784 tokio::runtime::Builder::new_current_thread()
12785 .enable_all()
12786 .build()
12787 .unwrap()
12788 .block_on(on_child_exit(
12789 &spec,
12790 runtime.restart_policy,
12791 &supervisor.registry,
12792 &reaped_snapshot,
12793 &runtime.terminal_ring,
12794 &runtime.spawn_events,
12795 &runtime.child_roster,
12796 ExitReport {
12797 kind: ExitKind::Clean,
12798 code: Some(0),
12799 signal: None,
12800 at_ms: 1,
12801 },
12802 true,
12803 ))
12804 });
12805 let mut reached = [0];
12806 barrier.read_exact(&mut reached).unwrap();
12807 assert_eq!(&reached, b"R");
12808 let (before_state, before_alive) = {
12809 let state = lock_snapshot(&snapshot).unwrap();
12810 (state.state, state.process_alive)
12811 };
12812 let before_history = history.lock().unwrap().snapshot();
12813 barrier.write_all(b"X").unwrap();
12814 assert!(matches!(reap.join().unwrap(), NextAction::Stop { .. }));
12815 assert!(before_history.entries.is_empty());
12816 assert_ne!(
12817 before_state,
12818 ModuleState::Failed,
12819 "Failed was visible before its terminal record could be written"
12820 );
12821 assert!(before_alive);
12822 assert_eq!(lock_snapshot(&snapshot).unwrap().state, ModuleState::Failed);
12823 let history = history.lock().unwrap().snapshot();
12824 assert_eq!(history.entries.len(), 1);
12825 assert_eq!(history.entries[0].exit_code, Some(0));
12826 assert_eq!(history.entries[0].disposition, TerminalDisposition::Failed);
12827 }
12828
12829 #[test]
12830 fn crash_restart_at_budget_limit_remains_recoverable_while_recording() {
12831 use std::io::{Read, Write};
12832
12833 let supervisor = Supervisor::new_for_test(
12834 Arc::new(Registry::default()),
12835 RestartPolicy::new(3, Duration::ZERO),
12836 );
12837 let mut runtime = supervisor.runtime_config();
12838 let module = module_with_recovery_snapshot(ModuleState::Running, true, 2);
12839 let snapshot = module.inner.snapshot.clone();
12840 lock_snapshot(&snapshot).unwrap().process_alive = true;
12841 runtime.terminal_ring = module.inner.terminal_ring.clone();
12842 let spec = windowed_crash_spec(module.module_id());
12843 let mut barrier = terminal_record_barrier(&runtime.spawn_events);
12844 let reaped_snapshot = snapshot.clone();
12845 let reap = std::thread::spawn(move || {
12846 tokio::runtime::Builder::new_current_thread()
12847 .enable_all()
12848 .build()
12849 .unwrap()
12850 .block_on(on_child_exit(
12851 &spec,
12852 runtime.restart_policy,
12853 &supervisor.registry,
12854 &reaped_snapshot,
12855 &runtime.terminal_ring,
12856 &runtime.spawn_events,
12857 &runtime.child_roster,
12858 ExitReport {
12859 kind: ExitKind::Crash,
12860 code: Some(2),
12861 signal: None,
12862 at_ms: 1,
12863 },
12864 true,
12865 ))
12866 });
12867
12868 let mut reached = [0];
12872 barrier.read_exact(&mut reached).unwrap();
12873 assert_eq!(&reached, b"R");
12874 let before_status = module.status().unwrap();
12875 let before_history = module.terminal_history();
12876 let before_recovers = module.will_recover_after_connection_loss().unwrap();
12877 barrier.write_all(b"X").unwrap();
12878 assert!(matches!(reap.join().unwrap(), NextAction::Restart { .. }));
12879
12880 assert_eq!(before_status.state, ModuleState::Running);
12881 assert!(before_status.process_alive);
12882 assert_eq!(before_status.restart_count, 3);
12883 assert!(before_history.entries.is_empty());
12884 assert!(
12885 before_recovers,
12886 "connection cleanup would report a terminal close for a scheduled crash restart"
12887 );
12888 let restarting = module.status().unwrap();
12889 assert_eq!(restarting.state, ModuleState::Restarting);
12890 assert_eq!(restarting.restart_count, 3);
12891 assert!(module.will_recover_after_connection_loss().unwrap());
12892 let history = module.terminal_history();
12893 assert_eq!(history.entries.len(), 1);
12894 assert_eq!(
12895 history.entries[0].disposition,
12896 TerminalDisposition::Restarting
12897 );
12898
12899 update_snapshot(&snapshot, Some(module.module_id()), |state| {
12902 state.state = ModuleState::Running;
12903 state.respawn_pending = false;
12904 })
12905 .unwrap();
12906 assert!(!module.will_recover_after_connection_loss().unwrap());
12907 update_snapshot(&snapshot, Some(module.module_id()), |state| {
12908 state.enabled = false;
12909 state.respawn_pending = true;
12910 })
12911 .unwrap();
12912 assert!(!module.will_recover_after_connection_loss().unwrap());
12913 }
12914
12915 fn terminal_record_barrier(events: &super::SpawnEventFeed) -> std::net::TcpStream {
12916 use std::net::{TcpListener, TcpStream};
12917 let listener = TcpListener::bind("127.0.0.1:0").unwrap();
12918 let writer = TcpStream::connect(listener.local_addr().unwrap()).unwrap();
12919 let (barrier, _) = listener.accept().unwrap();
12920 for stream in [&writer, &barrier] {
12921 stream
12922 .set_read_timeout(Some(Duration::from_secs(10)))
12923 .unwrap();
12924 stream
12925 .set_write_timeout(Some(Duration::from_secs(10)))
12926 .unwrap();
12927 }
12928 *events.1.lock().unwrap() = Some(writer);
12929 barrier
12930 }
12931
12932 pub(super) fn before_terminal_record(events: &super::SpawnEventFeed) {
12933 use std::io::{Read, Write};
12934 let barrier = events.1.lock().unwrap().take();
12935 if let Some(mut barrier) = barrier {
12936 barrier.write_all(b"R").unwrap();
12937 let mut release = [0];
12938 barrier.read_exact(&mut release).unwrap();
12939 assert_eq!(&release, b"X");
12940 }
12941 }
12942
12943 #[test]
12944 fn health_restart_budget_failure_is_published_after_its_terminal_record() {
12945 use std::io::{Read, Write};
12946
12947 let supervisor = Supervisor::new_for_test(
12948 Arc::new(Registry::default()),
12949 RestartPolicy::new(0, Duration::ZERO),
12950 );
12951 let mut runtime = supervisor.runtime_config();
12952 let module = module_with_recovery_snapshot(ModuleState::Running, true, 0);
12953 let snapshot = module.inner.snapshot.clone();
12954 runtime.terminal_ring = module.inner.terminal_ring.clone();
12955 let spec = ModuleSpec {
12956 module_id: module.module_id().into(),
12957 program: fake_aft_stub_path(),
12958 protocol: ModuleProtocol::None,
12959 env: vec![("FAKE_AFT_EXIT_CODE".into(), "17".into())],
12960 ..windowed_crash_spec(module.module_id())
12961 };
12962 let mut barrier = terminal_record_barrier(&runtime.spawn_events);
12963 let reap = std::thread::spawn(move || {
12964 tokio::runtime::Builder::new_current_thread()
12965 .enable_all()
12966 .build()
12967 .unwrap()
12968 .block_on(async {
12969 let mut child =
12970 Some(spawn_and_mark_running(&spec, &runtime, &snapshot).unwrap());
12971 health_restart_child(
12972 &spec,
12973 &runtime,
12974 &supervisor.registry,
12975 &supervisor.process_liveness,
12976 &snapshot,
12977 &mut child,
12978 SupervisorHealthStatus::Failing,
12979 Some("health probe did not answer"),
12980 1,
12981 )
12982 .await
12983 .unwrap();
12984 assert!(child.is_none());
12985 });
12986 });
12987 let mut reached = [0];
12992 barrier.read_exact(&mut reached).unwrap();
12993 assert_eq!(&reached, b"R");
12994 let before_status = module.status().unwrap();
12995 let before_history = module.terminal_history();
12996 barrier.write_all(b"X").unwrap();
12997 reap.join().unwrap();
12998 assert!(before_history.entries.is_empty());
12999 assert_ne!(
13000 before_status.state,
13001 ModuleState::Failed,
13002 "Failed was visible before its terminal record could be written"
13003 );
13004 assert!(
13005 before_status.process_alive,
13006 "process death was visible before its terminal record could be written"
13007 );
13008 let failed = module.status().unwrap();
13009 assert_eq!(failed.state, ModuleState::Failed);
13010 assert!(!failed.process_alive);
13011 let history = module.terminal_history();
13012 assert_eq!(history.entries.len(), 1);
13013 assert_eq!(history.entries[0].disposition, TerminalDisposition::Failed);
13014 assert_eq!(
13015 history.entries[0].disposition_detail.as_deref(),
13016 Some("crash budget exhausted: max_restarts=0 within window_secs=600")
13017 );
13018 }
13019
13020 #[test]
13021 fn observed_exits_publish_process_death_after_terminal_history() {
13022 use std::io::{Read, Write};
13023 for (kind, enabled, shutdown, wait_error, expected_state) in [
13024 (ExitKind::Clean, true, false, false, ModuleState::Stopped),
13025 (ExitKind::Crash, true, false, false, ModuleState::Restarting),
13026 (ExitKind::Crash, false, false, false, ModuleState::Disabled),
13027 (
13028 ExitKind::DeliberateSeverance,
13029 true,
13030 false,
13031 false,
13032 ModuleState::Restarting,
13033 ),
13034 #[cfg(unix)]
13035 (ExitKind::Crash, true, true, false, ModuleState::Stopped),
13036 (ExitKind::Crash, true, false, true, ModuleState::Failed),
13037 ] {
13038 let supervisor = Supervisor::new_for_test(
13039 Arc::new(Registry::default()),
13040 RestartPolicy::new(1, Duration::ZERO),
13041 );
13042 let runtime = supervisor.runtime_config();
13043 #[cfg(unix)]
13044 if shutdown {
13045 runtime.child_roster.close();
13046 }
13047 #[cfg(not(unix))]
13048 let _ = shutdown;
13049 let spec = windowed_crash_spec("exit-publication-order");
13050 let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::new(
13051 ModuleState::Running,
13052 enabled,
13053 )));
13054 lock_snapshot(&snapshot).unwrap().process_alive = true;
13055 let reaped_snapshot = snapshot.clone();
13056 let history = runtime.terminal_ring.clone();
13057 let mut barrier = terminal_record_barrier(&runtime.spawn_events);
13058 let reap = std::thread::spawn(move || {
13059 if wait_error {
13060 super::fail_after_wait_error(
13061 &spec.module_id,
13062 &reaped_snapshot,
13063 &runtime.terminal_ring,
13064 &runtime.spawn_events,
13065 "wait failed".into(),
13066 &runtime.failure_context(true),
13067 );
13068 } else {
13069 tokio::runtime::Builder::new_current_thread()
13070 .enable_all()
13071 .build()
13072 .unwrap()
13073 .block_on(on_child_exit(
13074 &spec,
13075 runtime.restart_policy,
13076 &supervisor.registry,
13077 &reaped_snapshot,
13078 &runtime.terminal_ring,
13079 &runtime.spawn_events,
13080 &runtime.child_roster,
13081 ExitReport {
13082 kind,
13083 code: Some(0),
13084 signal: None,
13085 at_ms: 1,
13086 },
13087 true,
13088 ));
13089 }
13090 });
13091 let mut reached = [0];
13092 barrier.read_exact(&mut reached).unwrap();
13093 assert_eq!(&reached, b"R");
13094 let before_alive = lock_snapshot(&snapshot).unwrap().process_alive;
13095 let before_history = history.lock().unwrap().snapshot();
13096 barrier.write_all(b"X").unwrap();
13097 reap.join().unwrap();
13098 assert!(
13099 before_alive,
13100 "{kind:?}: death was published before recording"
13101 );
13102 assert!(before_history.entries.is_empty());
13103 assert_eq!(lock_snapshot(&snapshot).unwrap().state, expected_state);
13104 assert!(!lock_snapshot(&snapshot).unwrap().process_alive);
13105 assert_eq!(history.lock().unwrap().snapshot().entries.len(), 1);
13106 }
13107 }
13108
13109 #[tokio::test]
13110 async fn health_restart_budget_failure_remains_failed_when_journal_append_fails() {
13111 let dir = cortexkit_test_support::ScratchDir::new("health-budget-journal-failure");
13112 let path = dir.join("terminals.jsonl");
13113 std::fs::create_dir(&path).unwrap();
13114 let supervisor = Supervisor::new_for_test(
13115 Arc::new(Registry::default()),
13116 RestartPolicy::new(0, Duration::ZERO),
13117 )
13118 .with_terminal_journal(path, "health-budget-journal-failure".into());
13119 let runtime = supervisor.runtime_config();
13120 let spec = ModuleSpec {
13121 program: fake_aft_stub_path(),
13122 protocol: ModuleProtocol::None,
13123 env: vec![("FAKE_AFT_EXIT_CODE".into(), "17".into())],
13124 ..windowed_crash_spec("health-budget-journal-failure")
13125 };
13126 let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::starting()));
13127 let mut child = Some(spawn_and_mark_running(&spec, &runtime, &snapshot).unwrap());
13128 health_restart_child(
13129 &spec,
13130 &runtime,
13131 &supervisor.registry,
13132 &supervisor.process_liveness,
13133 &snapshot,
13134 &mut child,
13135 SupervisorHealthStatus::Failing,
13136 None,
13137 1,
13138 )
13139 .await
13140 .unwrap();
13141 assert_eq!(lock_snapshot(&snapshot).unwrap().state, ModuleState::Failed);
13142 assert!(!lock_snapshot(&snapshot).unwrap().process_alive);
13143 let history = runtime
13144 .terminal_ring
13145 .lock()
13146 .unwrap()
13147 .durable_history(&spec.module_id);
13148 assert!(history.journal_write_failures > 0);
13149 assert_eq!(history.entries.len(), 1);
13150 assert_eq!(history.entries[0].disposition, TerminalDisposition::Failed);
13151 }
13152
13153 #[test]
13154 fn health_restart_budget_failure_remains_failed_when_exit_recording_panics() {
13155 let supervisor = Supervisor::new_for_test(
13156 Arc::new(Registry::default()),
13157 RestartPolicy::new(0, Duration::ZERO),
13158 );
13159 let runtime = supervisor.runtime_config();
13160 let spec = ModuleSpec {
13161 program: fake_aft_stub_path(),
13162 protocol: ModuleProtocol::None,
13163 env: vec![("FAKE_AFT_EXIT_CODE".into(), "17".into())],
13164 ..windowed_crash_spec("health-budget-recording-panic")
13165 };
13166 let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::starting()));
13167 let result = std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| {
13168 tokio::runtime::Builder::new_current_thread()
13169 .enable_all()
13170 .build()
13171 .unwrap()
13172 .block_on(async {
13173 let mut child =
13174 Some(spawn_and_mark_running(&spec, &runtime, &snapshot).unwrap());
13175 runtime.spawn_events.0.lock().unwrap().seq = u64::MAX;
13176 health_restart_child(
13177 &spec,
13178 &runtime,
13179 &supervisor.registry,
13180 &supervisor.process_liveness,
13181 &snapshot,
13182 &mut child,
13183 SupervisorHealthStatus::Failing,
13184 None,
13185 1,
13186 )
13187 .await
13188 .unwrap();
13189 });
13190 }));
13191 let panic = result.expect_err("recording must still unwind");
13192 assert_eq!(
13193 panic.downcast_ref::<String>().map(String::as_str),
13194 Some("spawn event sequence exhausted")
13195 );
13196 let state = lock_snapshot(&snapshot).unwrap();
13197 assert_eq!(state.state, ModuleState::Failed);
13198 assert!(!state.process_alive);
13199 assert!(state.last_exit.is_some());
13200 }
13201
13202 #[tokio::test]
13203 async fn restart_budget_failure_remains_failed_when_journal_append_fails() {
13204 let dir = cortexkit_test_support::ScratchDir::new("budget-journal-failure");
13205 let path = dir.join("terminals.jsonl");
13206 std::fs::create_dir(&path).unwrap();
13207 let supervisor = Supervisor::new_for_test(
13208 Arc::new(Registry::default()),
13209 RestartPolicy::new(0, Duration::ZERO),
13210 )
13211 .with_terminal_journal(path, "budget-journal-failure".into());
13212 let runtime = supervisor.runtime_config();
13213 let spec = windowed_crash_spec("budget-journal-failure");
13214 let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::starting()));
13215 assert!(matches!(
13216 on_child_exit(
13217 &spec,
13218 runtime.restart_policy,
13219 &supervisor.registry,
13220 &snapshot,
13221 &runtime.terminal_ring,
13222 &runtime.spawn_events,
13223 &runtime.child_roster,
13224 crash_exit_report(1),
13225 true,
13226 )
13227 .await,
13228 NextAction::Stop { .. }
13229 ));
13230 assert_eq!(lock_snapshot(&snapshot).unwrap().state, ModuleState::Failed);
13231 let history = runtime
13232 .terminal_ring
13233 .lock()
13234 .unwrap()
13235 .durable_history(&spec.module_id);
13236 assert!(history.journal_write_failures > 0);
13237 assert_eq!(history.entries.len(), 1);
13238 assert_eq!(history.entries[0].disposition, TerminalDisposition::Failed);
13239 }
13240
13241 #[test]
13242 fn restart_budget_failure_remains_failed_when_exit_recording_panics() {
13243 let supervisor = Supervisor::new_for_test(
13244 Arc::new(Registry::default()),
13245 RestartPolicy::new(0, Duration::ZERO),
13246 );
13247 let runtime = supervisor.runtime_config();
13248 let spec = windowed_crash_spec("budget-recording-panic");
13249 let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::starting()));
13250 runtime.spawn_events.emit_spawned(&spec.module_id, 1, 1);
13251 runtime.spawn_events.0.lock().unwrap().seq = u64::MAX;
13254 let result = std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| {
13255 tokio::runtime::Builder::new_current_thread()
13256 .enable_all()
13257 .build()
13258 .unwrap()
13259 .block_on(on_child_exit(
13260 &spec,
13261 runtime.restart_policy,
13262 &supervisor.registry,
13263 &snapshot,
13264 &runtime.terminal_ring,
13265 &runtime.spawn_events,
13266 &runtime.child_roster,
13267 crash_exit_report(1),
13268 true,
13269 ))
13270 }));
13271 let panic = result.err().expect("recording must still unwind");
13272 assert_eq!(
13273 panic.downcast_ref::<String>().map(String::as_str),
13274 Some("spawn event sequence exhausted")
13275 );
13276 assert_eq!(lock_snapshot(&snapshot).unwrap().state, ModuleState::Failed);
13277 }
13278
13279 #[cfg(unix)]
13282 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
13283 async fn protocol_none_requested_stop_and_disable_do_not_respawn() {
13284 for disable in [false, true] {
13285 let label = if disable {
13286 "none-requested-disable"
13287 } else {
13288 "none-requested-stop"
13289 };
13290 let dir = cortexkit_test_support::ScratchDir::new(label);
13291 let (spec, ready, marker) = protocol_none_sigterm_exits_clean_spec(label, dir.path());
13292 let supervisor = Supervisor::new_for_test(
13293 Arc::new(Registry::default()),
13294 RestartPolicy::new(3, Duration::ZERO),
13295 );
13296 let module = supervisor.spawn(spec).unwrap();
13297 wait_for_file(&ready).await;
13298
13299 if disable {
13300 module.set_enabled(false).await.unwrap();
13301 } else {
13302 module.stop().await.unwrap();
13303 }
13304 assert!(
13305 marker.exists(),
13306 "{label}: the child must have left through its SIGTERM handler with exit 0"
13307 );
13308
13309 sleep(Duration::from_millis(500)).await;
13312 let status = module.status().unwrap();
13313 let expected = if disable {
13314 ModuleState::Disabled
13315 } else {
13316 ModuleState::Stopped
13317 };
13318 assert_eq!(status.state, expected, "{label}");
13319 assert_eq!(
13320 status.spawn_generation, 1,
13321 "{label}: respawned after a requested stop"
13322 );
13323 let history = module.terminal_history();
13324 assert_eq!(history.entries.len(), 1, "{label}: {history:?}");
13325 assert_eq!(history.entries[0].exit_code, Some(0), "{label}");
13326 assert_ne!(
13327 history.entries[0].disposition,
13328 TerminalDisposition::Restarting,
13329 "{label}"
13330 );
13331 }
13332 }
13333
13334 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
13337 async fn subc_wire_clean_exit_is_still_a_stop() {
13338 let supervisor = Supervisor::new_for_test(
13339 Arc::new(Registry::default()),
13340 RestartPolicy::new(3, Duration::ZERO),
13341 );
13342 let module = supervisor
13343 .spawn(ModuleSpec {
13344 module_id: "wire-clean-exit".to_string(),
13345 program: fake_aft_stub_path(),
13346 args: Vec::new(),
13347 env: vec![("FAKE_AFT_EXIT_CODE".to_string(), "0".to_string())],
13348 reserved: false,
13349 reserved_prefixes: Vec::new(),
13350 protocol: ModuleProtocol::Subc,
13351 overlap: Default::default(),
13352 })
13353 .unwrap();
13354
13355 let deadline = Instant::now() + Duration::from_secs(10);
13356 while module.terminal_history().entries.is_empty() {
13357 assert!(Instant::now() < deadline, "module never exited");
13358 sleep(Duration::from_millis(10)).await;
13359 }
13360 sleep(Duration::from_millis(500)).await;
13362 let status = module.status().unwrap();
13363 assert_eq!(status.state, ModuleState::Stopped);
13364 assert_eq!(status.spawn_generation, 1);
13365 let history = module.terminal_history();
13366 assert_eq!(history.entries.len(), 1, "{history:?}");
13367 assert_eq!(history.entries[0].exit_code, Some(0));
13368 assert_eq!(history.entries[0].disposition, TerminalDisposition::Stopped);
13369 }
13370
13371 #[cfg(unix)]
13372 #[tokio::test]
13373 async fn reload_exit_keeps_roster_until_terminal_is_recorded_during_shutdown() {
13374 let dir = cortexkit_test_support::ScratchDir::new("reload-roster-terminal-order");
13375 let record = dir.join("live-children.json");
13376 let supervisor = Supervisor::new_for_test(
13377 Arc::new(Registry::default()),
13378 RestartPolicy::new(0, Duration::ZERO),
13379 );
13380 let mut runtime = supervisor.runtime_config();
13381 runtime.child_roster.record_to(record.clone());
13382 let gate = Arc::new(super::ReloadExitRecordGate::default());
13383 runtime.test_reload_exit_record_gate = Some(Arc::clone(&gate));
13384 let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::starting()));
13385 let spec = ModuleSpec {
13386 module_id: "reload-exit-roster".into(),
13387 program: fake_aft_stub_path(),
13388 args: Vec::new(),
13389 env: vec![("FAKE_AFT_EXIT_CODE".into(), "23".into())],
13390 reserved: false,
13391 reserved_prefixes: Vec::new(),
13392 protocol: ModuleProtocol::Subc,
13393 overlap: Default::default(),
13394 };
13395 let mut child = None;
13396 let reload = super::finish_reload_child(
13397 &spec,
13398 &runtime,
13399 &supervisor.registry,
13400 &supervisor.process_liveness,
13401 &snapshot,
13402 &mut child,
13403 );
13404 tokio::pin!(reload);
13405 tokio::select! {
13406 result = &mut reload => panic!("reload missed the before-record gate: {result:?}"),
13407 _ = gate.reached.notified() => {}
13408 }
13409 assert!(runtime
13410 .terminal_ring
13411 .lock()
13412 .unwrap()
13413 .snapshot()
13414 .entries
13415 .is_empty());
13416 assert_eq!(
13417 crate::live_children::read_record(&record).unwrap().len(),
13418 1,
13419 "shutdown must still wait for the reaped child until its terminal record exists"
13420 );
13421 runtime.child_roster.close();
13422 gate.resume.notify_one();
13423 assert!(reload.await.is_err());
13424 assert!(crate::live_children::read_record(&record)
13425 .unwrap()
13426 .is_empty());
13427 let history = runtime.terminal_ring.lock().unwrap().snapshot();
13428 assert_eq!(history.entries.len(), 1);
13429 assert_eq!(
13430 history.entries[0].disposition,
13431 TerminalDisposition::DaemonShutdown
13432 );
13433 }
13434
13435 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
13439 async fn every_restart_increment_path_advances_lifetime_count() {
13440 let supervisor = Supervisor::new_for_test(
13441 Arc::new(Registry::default()),
13442 RestartPolicy::new(1, Duration::ZERO),
13443 );
13444 let runtime = supervisor.runtime_config();
13445 let spec = ModuleSpec {
13446 module_id: "lifetime-increment-path".to_string(),
13447 program: PathBuf::from("/unused/lifetime-increment-path"),
13448 args: Vec::new(),
13449 env: Vec::new(),
13450 reserved: false,
13451 reserved_prefixes: Vec::new(),
13452 protocol: ModuleProtocol::Subc,
13453 overlap: Default::default(),
13454 };
13455
13456 let crash_snapshot = Arc::new(Mutex::new(SupervisorSnapshot::starting()));
13457 assert!(matches!(
13458 on_child_exit(
13459 &spec,
13460 runtime.restart_policy,
13461 &supervisor.registry,
13462 &crash_snapshot,
13463 &runtime.terminal_ring,
13464 &runtime.spawn_events,
13465 &runtime.child_roster,
13466 ExitReport {
13467 kind: ExitKind::Crash,
13468 code: Some(1),
13469 signal: None,
13470 at_ms: 1,
13471 },
13472 true,
13473 )
13474 .await,
13475 NextAction::Restart { schedule: _ }
13476 ));
13477 let (crash_restarts, crash_lifetime) = {
13478 let state = lock_snapshot(&crash_snapshot).unwrap();
13479 (state.crash_restarts.len(), state.lifetime_restarts)
13480 };
13481 assert_eq!(crash_restarts, 1);
13482 assert_eq!(crash_lifetime, 1);
13483
13484 let health_snapshot = Arc::new(Mutex::new(SupervisorSnapshot::starting()));
13485 let mut health_child = None;
13486 assert!(matches!(
13487 health_restart_child(
13488 &spec,
13489 &runtime,
13490 &supervisor.registry,
13491 &supervisor.process_liveness,
13492 &health_snapshot,
13493 &mut health_child,
13494 SupervisorHealthStatus::Failing,
13495 None,
13496 2,
13497 )
13498 .await,
13499 Ok(())
13500 ));
13501 assert!(health_child.is_none());
13502 assert!(lock_snapshot(&health_snapshot).unwrap().respawn_pending);
13503 assert!(runtime.scheduled_respawn.lock().unwrap().take().is_some());
13504 let (health_restarts, health_lifetime) = {
13505 let state = lock_snapshot(&health_snapshot).unwrap();
13506 (state.crash_restarts.len(), state.lifetime_restarts)
13507 };
13508 assert_eq!(health_restarts, 1);
13509 assert_eq!(health_lifetime, 1);
13510
13511 let reload_snapshot = Arc::new(Mutex::new(SupervisorSnapshot::starting()));
13512 let mut reload_child = None;
13513 assert!(matches!(
13514 handle_reload_spawn_failure(
13515 &spec,
13516 &runtime,
13517 &supervisor.process_liveness,
13518 &reload_snapshot,
13519 &mut reload_child,
13520 "forced reload spawn failure".to_string(),
13521 )
13522 .await,
13523 Err(SuperviseError::ReloadFailed { .. })
13524 ));
13525 let (reload_restarts, reload_lifetime) = {
13526 let state = lock_snapshot(&reload_snapshot).unwrap();
13527 (state.crash_restarts.len(), state.lifetime_restarts)
13528 };
13529 assert_eq!(reload_restarts, 1);
13530 assert_eq!(reload_lifetime, 1);
13531 }
13532
13533 #[tokio::test]
13534 async fn deliberately_severed_live_child_records_lifetime_without_spending_restart_budget() {
13535 let supervisor = Supervisor::new_for_test(
13536 Arc::new(Registry::default()),
13537 RestartPolicy::new(3, Duration::ZERO),
13538 );
13539 let runtime = supervisor.runtime_config();
13540 let spec = ModuleSpec {
13541 module_id: "deliberately-severed".to_string(),
13542 program: PathBuf::from("/unused/deliberately-severed"),
13543 args: Vec::new(),
13544 env: Vec::new(),
13545 reserved: false,
13546 reserved_prefixes: Vec::new(),
13547 protocol: ModuleProtocol::Subc,
13548 overlap: Default::default(),
13549 };
13550 let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::starting()));
13551 let process = ProcessIdentity {
13552 pid: 41,
13553 start_time: 101,
13554 };
13555 record_deliberate_severance(&snapshot, process).unwrap();
13556 let exit_report = apply_deliberate_severance_marker(
13557 &snapshot,
13558 Some(process),
13559 ExitReport {
13560 kind: ExitKind::Crash,
13561 code: Some(1),
13562 signal: None,
13563 at_ms: 1,
13564 },
13565 );
13566 assert_eq!(exit_report.kind, ExitKind::DeliberateSeverance);
13567
13568 assert!(matches!(
13569 on_child_exit(
13570 &spec,
13571 runtime.restart_policy,
13572 &supervisor.registry,
13573 &snapshot,
13574 &runtime.terminal_ring,
13575 &runtime.spawn_events,
13576 &runtime.child_roster,
13577 exit_report,
13578 true,
13579 )
13580 .await,
13581 NextAction::Restart { schedule: _ }
13582 ));
13583 let state = lock_snapshot(&snapshot).unwrap();
13584 assert_eq!(state.lifetime_restarts, 1);
13585 assert_eq!(state.crash_restarts.len(), 0);
13586 }
13587
13588 #[tokio::test]
13589 async fn genuine_crash_spends_restart_budget_and_records_lifetime() {
13590 let supervisor = Supervisor::new_for_test(
13591 Arc::new(Registry::default()),
13592 RestartPolicy::new(3, Duration::ZERO),
13593 );
13594 let runtime = supervisor.runtime_config();
13595 let spec = ModuleSpec {
13596 module_id: "genuine-crash".to_string(),
13597 program: PathBuf::from("/unused/genuine-crash"),
13598 args: Vec::new(),
13599 env: Vec::new(),
13600 reserved: false,
13601 reserved_prefixes: Vec::new(),
13602 protocol: ModuleProtocol::Subc,
13603 overlap: Default::default(),
13604 };
13605 let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::starting()));
13606
13607 assert!(matches!(
13608 on_child_exit(
13609 &spec,
13610 runtime.restart_policy,
13611 &supervisor.registry,
13612 &snapshot,
13613 &runtime.terminal_ring,
13614 &runtime.spawn_events,
13615 &runtime.child_roster,
13616 ExitReport {
13617 kind: ExitKind::Crash,
13618 code: Some(1),
13619 signal: None,
13620 at_ms: 1,
13621 },
13622 true,
13623 )
13624 .await,
13625 NextAction::Restart { schedule: _ }
13626 ));
13627 let state = lock_snapshot(&snapshot).unwrap();
13628 assert_eq!(state.lifetime_restarts, 1);
13629 assert_eq!(state.crash_restarts.len(), 1);
13630 }
13631
13632 fn crash_exit_report(at_ms: u64) -> ExitReport {
13633 ExitReport {
13634 kind: ExitKind::Crash,
13635 code: Some(1),
13636 signal: None,
13637 at_ms,
13638 }
13639 }
13640
13641 fn windowed_crash_spec(module_id: &str) -> ModuleSpec {
13642 ModuleSpec {
13643 module_id: module_id.to_string(),
13644 program: PathBuf::from("/unused").join(module_id),
13645 args: Vec::new(),
13646 env: Vec::new(),
13647 reserved: false,
13648 reserved_prefixes: Vec::new(),
13649 protocol: ModuleProtocol::Subc,
13650 overlap: Default::default(),
13651 }
13652 }
13653
13654 #[tokio::test]
13660 async fn three_crashes_inside_the_window_stop_the_module_and_name_the_window() {
13661 let (logs, _guard) = crate::router::test_log::log_capture(tracing::Level::ERROR);
13662 let supervisor = Supervisor::new_for_test(
13663 Arc::new(Registry::default()),
13664 RestartPolicy::new(2, Duration::ZERO),
13665 );
13666 let runtime = supervisor.runtime_config();
13667 let spec = windowed_crash_spec("crash-loop-in-window");
13668 let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::starting()));
13669
13670 for attempt in 1..=2 {
13671 assert!(
13672 matches!(
13673 on_child_exit(
13674 &spec,
13675 runtime.restart_policy,
13676 &supervisor.registry,
13677 &snapshot,
13678 &runtime.terminal_ring,
13679 &runtime.spawn_events,
13680 &runtime.child_roster,
13681 crash_exit_report(attempt),
13682 true,
13683 )
13684 .await,
13685 NextAction::Restart { schedule: _ }
13686 ),
13687 "crash {attempt} is inside the budget and must respawn"
13688 );
13689 }
13690
13691 assert!(matches!(
13692 on_child_exit(
13693 &spec,
13694 runtime.restart_policy,
13695 &supervisor.registry,
13696 &snapshot,
13697 &runtime.terminal_ring,
13698 &runtime.spawn_events,
13699 &runtime.child_roster,
13700 crash_exit_report(3),
13701 true,
13702 )
13703 .await,
13704 NextAction::Stop { .. }
13705 ));
13706
13707 {
13708 let state = lock_snapshot(&snapshot).unwrap();
13709 assert_eq!(state.state, ModuleState::Failed);
13710 assert_eq!(state.crash_restarts.len(), 2);
13711 assert_eq!(state.lifetime_restarts, 2);
13712 }
13713
13714 let history = runtime
13715 .terminal_ring
13716 .lock()
13717 .expect("terminal ring is not poisoned")
13718 .snapshot();
13719 let last = history
13720 .entries
13721 .last()
13722 .expect("the refused crash is retained");
13723 assert_eq!(last.disposition, TerminalDisposition::Failed);
13724 assert_eq!(
13725 last.disposition_detail.as_deref(),
13726 Some("crash budget exhausted: max_restarts=2 within window_secs=600")
13727 );
13728
13729 let captured = crate::router::test_log::captured_logs(&logs);
13730 assert!(
13731 captured.contains("crash budget exhausted: max_restarts=2 within window_secs=600"),
13732 "the stop must be logged with its window: {captured}"
13733 );
13734 }
13735
13736 #[tokio::test]
13744 async fn a_crash_older_than_the_window_frees_its_slot_for_a_later_crash() {
13745 let supervisor = Supervisor::new_for_test(
13746 Arc::new(Registry::default()),
13747 RestartPolicy::new(2, Duration::ZERO),
13748 );
13749 let runtime = supervisor.runtime_config();
13750 let spec = windowed_crash_spec("crash-across-windows");
13751 let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::starting()));
13752
13753 for attempt in 1..=2 {
13754 assert!(matches!(
13755 on_child_exit(
13756 &spec,
13757 runtime.restart_policy,
13758 &supervisor.registry,
13759 &snapshot,
13760 &runtime.terminal_ring,
13761 &runtime.spawn_events,
13762 &runtime.child_roster,
13763 crash_exit_report(attempt),
13764 true,
13765 )
13766 .await,
13767 NextAction::Restart { schedule: _ }
13768 ));
13769 }
13770
13771 update_snapshot(&snapshot, Some(&spec.module_id), |state| {
13774 age_oldest_crash_restart_out_of_window(state, runtime.restart_policy.window);
13775 })
13776 .unwrap();
13777
13778 assert!(
13779 matches!(
13780 on_child_exit(
13781 &spec,
13782 runtime.restart_policy,
13783 &supervisor.registry,
13784 &snapshot,
13785 &runtime.terminal_ring,
13786 &runtime.spawn_events,
13787 &runtime.child_roster,
13788 crash_exit_report(3),
13789 true,
13790 )
13791 .await,
13792 NextAction::Restart { schedule: _ }
13793 ),
13794 "a crash older than the window must not hold a budget slot"
13795 );
13796
13797 let state = lock_snapshot(&snapshot).unwrap();
13798 assert_eq!(state.state, ModuleState::Restarting);
13799 assert_eq!(
13800 state.crash_restarts.len(),
13801 2,
13802 "the aged instant is dropped and the new one takes its place"
13803 );
13804 assert_eq!(
13805 state.lifetime_restarts, 3,
13806 "the ledger counts every restart, including the ones the window forgot"
13807 );
13808 }
13809
13810 #[tokio::test]
13815 async fn an_operator_restart_clears_the_ring_and_leaves_the_ledger_alone() {
13816 let supervisor = Supervisor::new_for_test(
13817 Arc::new(Registry::default()),
13818 RestartPolicy::new(2, Duration::ZERO),
13819 );
13820 let runtime = supervisor.runtime_config();
13821 let spec = windowed_crash_spec("operator-cleared-budget");
13822 let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::starting()));
13823
13824 for attempt in 1..=2 {
13825 assert!(matches!(
13826 on_child_exit(
13827 &spec,
13828 runtime.restart_policy,
13829 &supervisor.registry,
13830 &snapshot,
13831 &runtime.terminal_ring,
13832 &runtime.spawn_events,
13833 &runtime.child_roster,
13834 crash_exit_report(attempt),
13835 true,
13836 )
13837 .await,
13838 NextAction::Restart { schedule: _ }
13839 ));
13840 }
13841
13842 reset_restart_count(&snapshot, &spec.module_id).unwrap();
13843 {
13844 let state = lock_snapshot(&snapshot).unwrap();
13845 assert!(
13846 state.crash_restarts.is_empty(),
13847 "an operator restart returns the full budget"
13848 );
13849 assert_eq!(
13850 state.lifetime_restarts, 2,
13851 "clearing the budget must not unmake the crashes"
13852 );
13853 }
13854
13855 assert!(
13856 matches!(
13857 on_child_exit(
13858 &spec,
13859 runtime.restart_policy,
13860 &supervisor.registry,
13861 &snapshot,
13862 &runtime.terminal_ring,
13863 &runtime.spawn_events,
13864 &runtime.child_roster,
13865 crash_exit_report(3),
13866 true,
13867 )
13868 .await,
13869 NextAction::Restart { schedule: _ }
13870 ),
13871 "the cleared budget must be spendable again"
13872 );
13873 let state = lock_snapshot(&snapshot).unwrap();
13874 assert_eq!(state.crash_restarts.len(), 1);
13875 assert_eq!(state.lifetime_restarts, 3);
13876 }
13877
13878 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
13879 async fn severance_marker_for_a_dead_child_does_not_label_its_successor() {
13880 let severed = ProcessIdentity {
13881 pid: 41,
13882 start_time: 101,
13883 };
13884 let successor = ProcessIdentity {
13885 pid: 41,
13886 start_time: 202,
13887 };
13888 let module = module_with_recovery_snapshot(ModuleState::Running, true, 0);
13889 update_snapshot(&module.inner.snapshot, Some("recovery-snapshot"), |state| {
13890 state.pid = Some(successor.pid);
13891 state.process_start_time = Some(successor.start_time);
13892 })
13893 .unwrap();
13894 assert!(!module.record_deliberate_severance(severed).unwrap());
13895
13896 let exit_report = apply_deliberate_severance_marker(
13897 &module.inner.snapshot,
13898 Some(successor),
13899 ExitReport {
13900 kind: ExitKind::Crash,
13901 code: Some(1),
13902 signal: None,
13903 at_ms: 1,
13904 },
13905 );
13906
13907 assert_eq!(exit_report.kind, ExitKind::Crash);
13908 }
13909
13910 #[tokio::test]
13911 async fn drain_reap_marks_deliberate_severance_and_records_lifetime_without_budget() {
13912 let registry = Registry::default();
13913 let supervisor = Supervisor::new_for_test(
13914 Arc::new(Registry::default()),
13915 RestartPolicy::new(3, Duration::ZERO),
13916 );
13917 let runtime = supervisor.runtime_config();
13918 let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::starting()));
13919 let spec = ModuleSpec {
13920 module_id: "drain-deliberate-severance".to_string(),
13921 program: fake_aft_stub_path(),
13922 args: Vec::new(),
13923 env: vec![("FAKE_AFT_EXIT_CODE".to_string(), "23".to_string())],
13924 reserved: false,
13925 reserved_prefixes: Vec::new(),
13926 protocol: ModuleProtocol::Subc,
13927 overlap: Default::default(),
13928 };
13929 let mut child = spawn_and_mark_running(&spec, &runtime, &snapshot).unwrap();
13930 let process = ProcessIdentity {
13931 pid: 41,
13932 start_time: 101,
13933 };
13934 child.process_identity = Some(process);
13935 update_snapshot(&snapshot, Some(&spec.module_id), |state| {
13936 state.pid = Some(process.pid);
13937 state.process_start_time = Some(process.start_time);
13938 })
13939 .unwrap();
13940 record_deliberate_severance(&snapshot, process).unwrap();
13941
13942 drain_child_to_state(
13943 &spec.module_id,
13944 spec.protocol,
13945 StopNotice::SentOverConnection,
13948 ®istry,
13949 None,
13950 &snapshot,
13951 &runtime.terminal_ring,
13952 &runtime.spawn_events,
13953 child,
13954 Duration::from_secs(1),
13955 ModuleState::Stopped,
13956 Some(false),
13957 &super::FailureContext::operator(),
13958 )
13959 .await
13960 .unwrap();
13961
13962 let state = lock_snapshot(&snapshot).unwrap();
13963 assert_eq!(
13964 state.last_exit.as_ref().map(|exit| exit.kind),
13965 Some(ExitKind::DeliberateSeverance)
13966 );
13967 assert_eq!(state.lifetime_restarts, 1);
13968 assert_eq!(state.crash_restarts.len(), 0);
13969 drop(state);
13970 let history = runtime.terminal_ring.lock().unwrap().snapshot();
13971 assert_eq!(
13972 history.entries[0].exit_kind,
13973 subc_control::TerminalExitKind::DeliberateSeverance
13974 );
13975 }
13976
13977 #[tokio::test]
13978 async fn ordinary_drain_reap_does_not_record_a_lifetime_restart() {
13979 let registry = Registry::default();
13980 let supervisor = Supervisor::new_for_test(
13981 Arc::new(Registry::default()),
13982 RestartPolicy::new(3, Duration::ZERO),
13983 );
13984 let runtime = supervisor.runtime_config();
13985 let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::starting()));
13986 let spec = ModuleSpec {
13987 module_id: "ordinary-drain".to_string(),
13988 program: fake_aft_stub_path(),
13989 args: Vec::new(),
13990 env: vec![("FAKE_AFT_EXIT_CODE".to_string(), "23".to_string())],
13991 reserved: false,
13992 reserved_prefixes: Vec::new(),
13993 protocol: ModuleProtocol::Subc,
13994 overlap: Default::default(),
13995 };
13996 let child = spawn_and_mark_running(&spec, &runtime, &snapshot).unwrap();
13997
13998 drain_child_to_state(
13999 &spec.module_id,
14000 spec.protocol,
14001 StopNotice::SentOverConnection,
14004 ®istry,
14005 None,
14006 &snapshot,
14007 &runtime.terminal_ring,
14008 &runtime.spawn_events,
14009 child,
14010 Duration::from_secs(1),
14011 ModuleState::Stopped,
14012 Some(false),
14013 &super::FailureContext::operator(),
14014 )
14015 .await
14016 .unwrap();
14017
14018 let state = lock_snapshot(&snapshot).unwrap();
14019 assert_eq!(
14020 state.last_exit.as_ref().map(|exit| exit.kind),
14021 Some(ExitKind::Crash)
14022 );
14023 assert_eq!(state.lifetime_restarts, 0);
14024 assert_eq!(state.crash_restarts.len(), 0);
14025 }
14026
14027 #[test]
14028 fn fatal_connection_teardown_cannot_arm_a_marker_for_a_surviving_process() {
14029 assert!(!include_str!("server.rs")
14035 .contains("router.record_deliberate_connection_severance(ctx.connection_id)"));
14036 }
14037
14038 #[test]
14045 fn drained_after_quiescence_wait_passes_ok_through_and_forces_false_on_err() {
14046 assert!(drained_after_quiescence_wait(&Ok(true)));
14047 assert!(!drained_after_quiescence_wait(&Ok(false)));
14048 assert!(!drained_after_quiescence_wait(&Err(
14049 SuperviseError::StatePoisoned { module_id: None }
14050 )));
14051 }
14052
14053 #[test]
14062 fn wait_error_exit_report_records_a_failed_terminal_with_no_code_or_signal() {
14063 let ring = Arc::new(Mutex::new(TerminalRing::new(
14064 TerminalRingConfig::default(),
14065 0,
14066 )));
14067 record_wait_error_terminal("wait-error", &ring, &super::SpawnEventFeed::default());
14068
14069 let snapshot = ring.lock().unwrap().snapshot();
14070 assert_eq!(snapshot.entries.len(), 1);
14071 let entry = &snapshot.entries[0];
14072 assert_eq!(entry.exit_code, None);
14073 assert_eq!(entry.exit_signal, None);
14074 assert_eq!(entry.disposition, TerminalDisposition::Failed);
14075 }
14076
14077 #[test]
14078 fn wait_error_exit_path_preserves_spawn_event_density() {
14079 let feed = super::SpawnEventFeed::default();
14080 feed.configure_incarnation("wait-error-density".to_string());
14081 feed.emit_spawned("wait-error", 41, 1);
14082 let ring = Arc::new(Mutex::new(TerminalRing::new(
14083 TerminalRingConfig::default(),
14084 0,
14085 )));
14086
14087 record_wait_error_terminal("wait-error", &ring, &feed);
14088 feed.emit_spawned("after-wait-error", 42, 2);
14089
14090 let state = feed.0.lock().unwrap();
14091 let sequences = state
14092 .events
14093 .iter()
14094 .map(|event| event.cursor.seq)
14095 .collect::<Vec<_>>();
14096 assert_eq!(sequences, vec![1, 2, 3]);
14097 assert_eq!(state.events[1].kind, SpawnEventKind::Exited);
14098 assert_eq!(state.events[1].exit_code, None);
14099 assert_eq!(state.events[1].exit_signal, None);
14100 }
14101
14102 #[test]
14106 fn wait_error_exit_report_is_classified_as_a_crash() {
14107 assert_eq!(wait_error_exit_report().kind, ExitKind::Crash);
14108 }
14109}
14110
14111#[cfg(all(test, unix))]
14112mod restart_ordering_tests {
14113 use super::*;
14114 use std::io::Write;
14115 use tokio::io::AsyncReadExt;
14116
14117 #[test]
14118 #[ignore = "child process fixture"]
14119 fn serving_child() {
14120 let Ok(address_file) = std::env::var("RESTART_TEST_ADDRESS_FILE") else {
14121 return;
14122 };
14123 let listener = std::net::TcpListener::bind("127.0.0.1:0").unwrap();
14124 std::fs::write(address_file, listener.local_addr().unwrap().to_string()).unwrap();
14125 for stream in listener.incoming() {
14126 writeln!(stream.unwrap(), "{}", std::process::id()).unwrap();
14127 }
14128 }
14129
14130 async fn restart_before_publication(requests: usize) {
14131 let home = cortexkit_test_support::ScratchDir::new("restart-before-publication");
14132 let address_file = home.path().join("address");
14133 let spec = ModuleSpec {
14134 module_id: "restart-serving-child".into(),
14135 program: std::env::current_exe().unwrap(),
14136 args: vec![
14137 "--exact".into(),
14138 "supervise::restart_ordering_tests::serving_child".into(),
14139 "--ignored".into(),
14140 "--nocapture".into(),
14141 ],
14142 env: vec![(
14143 "RESTART_TEST_ADDRESS_FILE".into(),
14144 address_file.display().to_string(),
14145 )],
14146 reserved: false,
14147 reserved_prefixes: vec![],
14148 protocol: ModuleProtocol::None,
14149 overlap: Default::default(),
14150 };
14151 let mut supervisor =
14152 Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default());
14153 let (spawned_tx, spawned_rx) = oneshot::channel();
14154 let spawned_tx = Mutex::new(Some(spawned_tx));
14155 let (release_tx, release_rx) = std::sync::mpsc::channel();
14156 let release_rx = Mutex::new(release_rx);
14157 supervisor.test_after_first_spawn = AfterFirstSpawnHook(Some(Arc::new(move |_| {
14160 spawned_tx.lock().unwrap().take().unwrap().send(()).unwrap();
14161 release_rx
14162 .lock()
14163 .unwrap()
14164 .recv_timeout(Duration::from_secs(20))
14165 .unwrap();
14166 })));
14167 let registered = supervisor
14168 .register_configured_with_health(
14169 spec.clone(),
14170 true,
14171 HealthConfig::default(),
14172 None,
14173 RestartPolicy::new(3, Duration::ZERO),
14174 DEFAULT_REGISTER_TIMEOUT,
14175 )
14176 .unwrap();
14177 let module = registered.module.clone();
14178 let startup = tokio::task::spawn_blocking(move || {
14179 supervisor.start_registered_configured(registered, None, true)
14180 });
14181 spawned_rx.await.unwrap();
14182 let old_pid = tokio::time::timeout(Duration::from_secs(10), async {
14183 let address = loop {
14184 if let Ok(address) = std::fs::read_to_string(&address_file) {
14185 if !address.is_empty() {
14186 break address;
14187 }
14188 }
14189 tokio::time::sleep(Duration::from_millis(1)).await;
14190 };
14191 let mut stream = tokio::net::TcpStream::connect(address).await.unwrap();
14192 let mut reply = String::new();
14193 stream.read_to_string(&mut reply).await.unwrap();
14194 reply.trim().parse::<u32>().unwrap()
14195 })
14196 .await
14197 .expect("child must serve before generation publication");
14198 assert_eq!(
14199 lock_snapshot(&module.inner.snapshot)
14200 .unwrap()
14201 .spawn_generation,
14202 0
14203 );
14204 let mut replies = Vec::new();
14205 for _ in 0..requests {
14206 let mut restart = Box::pin(module.restart(Some(0)));
14207 std::future::poll_fn(|cx| {
14210 assert!(std::future::Future::poll(restart.as_mut(), cx).is_pending());
14211 std::task::Poll::Ready(())
14212 })
14213 .await;
14214 replies.push(restart);
14215 }
14216 release_tx.send(()).unwrap();
14217 startup.await.unwrap().unwrap();
14218 for reply in replies {
14219 reply.await.unwrap();
14220 }
14221 let replacement = tokio::time::timeout(Duration::from_secs(10), async {
14224 loop {
14225 if lock_snapshot(&module.inner.snapshot)
14226 .unwrap()
14227 .spawn_generation
14228 > 1
14229 {
14230 break;
14231 }
14232 tokio::time::sleep(Duration::from_millis(1)).await;
14233 }
14234 })
14235 .await;
14236 module.update_spec_for_test(spec).await.unwrap();
14240 let (generation, pid) = {
14241 let state = lock_snapshot(&module.inner.snapshot).unwrap();
14242 (state.spawn_generation, state.pid)
14243 };
14244 module.stop().await.unwrap();
14245 replacement.expect("restart must publish a replacement for the already-serving child");
14246 assert_ne!(
14247 pid,
14248 Some(old_pid),
14249 "a serving child predating the restart must be replaced"
14250 );
14251 assert_eq!(generation, 2, "queued restarts must share one replacement");
14252 }
14253
14254 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
14255 async fn restart_received_after_serving_before_generation_publication_replaces_child() {
14256 restart_before_publication(1).await;
14257 }
14258
14259 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
14260 async fn restart_queued_behind_restart_does_not_kill_its_replacement() {
14261 restart_before_publication(2).await;
14262 }
14263
14264 #[tokio::test]
14265 async fn restart_queued_before_replacement_spawn_keeps_the_new_child() {
14266 let supervisor =
14267 Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default());
14268 let registered = supervisor
14269 .register_configured_with_health(
14270 ModuleSpec {
14271 module_id: "queued-before-replacement".into(),
14272 program: "/bin/sleep".into(),
14277 args: vec!["600".into()],
14278 env: vec![],
14279 reserved: false,
14280 reserved_prefixes: vec![],
14281 protocol: ModuleProtocol::None,
14282 overlap: Default::default(),
14283 },
14284 true,
14285 HealthConfig::default(),
14286 None,
14287 RestartPolicy::default(),
14288 DEFAULT_REGISTER_TIMEOUT,
14289 )
14290 .unwrap();
14291 let RegisteredConfiguredModule {
14292 mut spec,
14293 mut runtime,
14294 module,
14295 mut commands,
14296 } = registered;
14297 let snapshot = &module.inner.snapshot;
14298 let mut restart = Box::pin(module.restart(Some(0)));
14299 std::future::poll_fn(|cx| {
14300 assert!(std::future::Future::poll(restart.as_mut(), cx).is_pending());
14301 std::task::Poll::Ready(())
14302 })
14303 .await;
14304 let mut child = Some(spawn_and_mark_running(&spec, &runtime, snapshot).unwrap());
14307 let replacement_pid = child.as_ref().unwrap().id();
14308 let command = commands.recv().await.unwrap();
14309 assert!(
14310 handle_supervisor_command(
14311 command,
14312 &mut spec,
14313 &mut runtime,
14314 &supervisor.registry,
14315 &supervisor.process_liveness,
14316 snapshot,
14317 &mut child,
14318 &mut commands,
14319 &mut VecDeque::new(),
14320 )
14321 .await
14322 );
14323 restart.await.unwrap();
14324 let kept_pid = child.as_ref().and_then(SupervisedChild::id);
14325 let pending = lock_snapshot(snapshot).unwrap().respawn_pending;
14326 if let Some(mut child) = child {
14327 child.start_kill().unwrap();
14328 child.wait().await.unwrap();
14329 }
14330 assert_eq!(
14331 kept_pid, replacement_pid,
14332 "a newer replacement must survive"
14333 );
14334 assert!(
14335 !pending,
14336 "a satisfied restart must not schedule another spawn"
14337 );
14338 }
14339}
14340
14341#[cfg(all(test, unix))]
14342mod health_event_tests {
14343 use super::*;
14344 use cortexkit_test_support::ScratchDir;
14345 use subc_protocol::{manifest::Concurrency, session::ModuleControlResponse};
14346
14347 struct Actor {
14348 module: SupervisedModule,
14349 registry: Arc<Registry>,
14350 forwarding: Arc<ForwardingTable>,
14351 spec: ModuleSpec,
14352 health: HealthConfig,
14353 rx: mpsc::Receiver<crate::router::OutboundFrame>,
14354 roster: ChildRoster,
14355 _home: ScratchDir,
14356 }
14357
14358 async fn wait_for<T>(reason: &str, mut observe: impl FnMut() -> Option<T>) -> T {
14359 let deadline = std::time::Instant::now() + Duration::from_secs(10);
14363 let tick = Instant::now();
14364 loop {
14365 if let Some(value) = observe() {
14366 assert_eq!(Instant::now(), tick, "{reason} must not wait for a timer");
14367 return value;
14368 }
14369 assert!(
14370 std::time::Instant::now() < deadline,
14371 "timed out waiting for {reason}"
14372 );
14373 tokio::task::yield_now().await;
14374 }
14375 }
14376
14377 impl Actor {
14378 async fn start(protocol: ModuleProtocol, cadence: Duration) -> Self {
14379 Self::start_with_policy(
14380 protocol,
14381 cadence,
14382 DEFAULT_REGISTER_TIMEOUT,
14383 RestartPolicy::new(3, Duration::ZERO),
14384 )
14385 .await
14386 }
14387
14388 async fn start_with_policy(
14389 protocol: ModuleProtocol,
14390 cadence: Duration,
14391 register_timeout: Duration,
14392 policy: RestartPolicy,
14393 ) -> Self {
14394 let home = ScratchDir::new("health-events");
14395 let registry = Arc::new(Registry::default());
14396 let forwarding = Arc::new(ForwardingTable::default());
14397 let health = HealthConfig {
14398 cadence,
14399 deadline: Duration::from_secs(3600),
14400 ..HealthConfig::default()
14401 };
14402 let supervisor = Supervisor::new_for_test(registry.clone(), policy)
14403 .with_forwarding(forwarding.clone())
14404 .with_health_config(health.clone());
14405 let spec = ModuleSpec {
14406 module_id: "event-health-child".into(),
14407 program: PathBuf::from("/bin/sleep"),
14415 args: vec!["600".into()],
14418 env: ["XDG_DATA_HOME", "XDG_RUNTIME_DIR", "XDG_CONFIG_HOME"]
14419 .into_iter()
14420 .map(|key| (key.into(), home.path().display().to_string()))
14421 .collect(),
14422 reserved: false,
14423 reserved_prefixes: vec![],
14424 protocol,
14425 overlap: Default::default(),
14426 };
14427 let module = supervisor
14428 .supervise_configured_requested(
14429 spec.clone(),
14430 true,
14431 health.clone(),
14432 Some(0),
14433 policy,
14434 register_timeout,
14435 None,
14436 )
14437 .unwrap();
14438 let (_, rx) = mpsc::channel(8);
14439 let actor = Self {
14440 module,
14441 registry,
14442 forwarding,
14443 spec,
14444 health,
14445 rx,
14446 roster: supervisor.child_roster.clone(),
14447 _home: home,
14448 };
14449 actor
14450 .wait_for_select(|checkpoint| checkpoint.generation > 0)
14451 .await;
14452 actor
14453 }
14454
14455 async fn expire_registration_wait(&self, remaining: Duration) {
14456 tokio::time::advance(remaining).await;
14457 wait_for("registration timeout TERM grace", || {
14458 lock_snapshot(&self.module.inner.snapshot)
14459 .unwrap()
14460 .actor_select
14461 .is_none()
14462 .then_some(())
14463 })
14464 .await;
14465 tokio::time::advance(crate::child_roster::TERM_TO_KILL).await;
14466 }
14467
14468 fn turns(&self) -> u64 {
14469 lock_snapshot(&self.module.inner.snapshot)
14470 .unwrap()
14471 .actor_turns
14472 }
14473
14474 async fn wait_for_select(
14475 &self,
14476 expected: impl Fn(&ActorSelectCheckpoint) -> bool,
14477 ) -> ActorSelectCheckpoint {
14478 wait_for("actor select checkpoint", || {
14479 let state = lock_snapshot(&self.module.inner.snapshot).unwrap();
14480 state.actor_select.filter(|checkpoint| {
14481 checkpoint.generation == state.spawn_generation
14482 && state.state == ModuleState::Running
14483 && state.process_alive
14484 && state.reported_pid().is_some()
14485 && expected(checkpoint)
14486 })
14487 })
14488 .await
14489 }
14490
14491 fn hello(&mut self, connection: u64, health: bool) {
14492 let manifest =
14493 subc_protocol::manifest::ModuleManifest::builder(&self.spec.module_id, "1.0.0")
14494 .protocol_ver(subc_protocol::PROTOCOL_VERSION)
14495 .build();
14496 let connection = ConnectionId::new(connection);
14497 let (tx, rx) = mpsc::channel(8);
14498 self.rx = rx;
14499 self.forwarding
14500 .register_module_connection(
14501 connection,
14502 self.spec.module_id.clone(),
14503 subc_protocol::PROTOCOL_VERSION,
14504 Concurrency::ModuleManaged,
14505 FrameSink::new(tx),
14506 )
14507 .unwrap();
14508 self.registry
14509 .register_with_control_ops(
14510 manifest,
14511 subc_protocol::PROTOCOL_VERSION,
14512 connection,
14513 if health {
14514 vec![MODULE_CONTROL_OP_HEALTH_CHECK.into()]
14515 } else {
14516 vec![]
14517 },
14518 )
14519 .unwrap();
14520 }
14521
14522 async fn probe(&mut self) -> crate::router::OutboundFrame {
14523 let frame = wait_for("outbound health probe", || match self.rx.try_recv() {
14524 Ok(frame) => Some(frame),
14525 Err(mpsc::error::TryRecvError::Empty) => None,
14526 Err(mpsc::error::TryRecvError::Disconnected) => panic!("health peer disconnected"),
14527 })
14528 .await;
14529 let body: Value = serde_json::from_slice(&frame.body).unwrap();
14530 assert_eq!(body["op"], MODULE_CONTROL_OP_HEALTH_CHECK);
14531 frame
14532 }
14533
14534 fn no_probe(&mut self) {
14535 assert!(matches!(
14536 self.rx.try_recv(),
14537 Err(mpsc::error::TryRecvError::Empty)
14538 ));
14539 }
14540
14541 fn answer(&self, connection: u64, frame: crate::router::OutboundFrame) {
14542 self.forwarding
14543 .complete_module_control_rpc(
14544 ConnectionId::new(connection),
14545 frame.header.corr,
14546 Some(MODULE_CONTROL_OP_HEALTH_CHECK),
14547 ModuleControlRpcOutcome::Response(ModuleControlResponse::HealthCheck {
14548 status: HealthStatus::Ok,
14549 detail: None,
14550 metrics: None,
14551 }),
14552 )
14553 .unwrap();
14554 }
14555 }
14556
14557 #[tokio::test(start_paused = true)]
14558 async fn registration_timeout_kills_at_120_seconds_and_respawns() {
14559 let actor = Actor::start(ModuleProtocol::Subc, Duration::from_secs(30)).await;
14560 let initial = actor.module.status().unwrap();
14561 tokio::time::advance(Duration::from_secs(119)).await;
14562 tokio::task::yield_now().await;
14563 assert_eq!(actor.module.status().unwrap().pid, initial.pid);
14564 assert!(actor.module.terminal_history().entries.is_empty());
14565 actor.expire_registration_wait(Duration::from_secs(1)).await;
14566 actor
14567 .wait_for_select(|at| at.generation > initial.spawn_generation)
14568 .await;
14569 let status = actor.module.status().unwrap();
14570 assert_ne!(status.pid, initial.pid);
14571 assert!(!cortexkit_test_support::process_alive(
14572 initial.pid.unwrap() as i32
14573 ));
14574 assert_eq!(status.restart_count, 1);
14575 assert_eq!(status.last_exit.unwrap().kind, ExitKind::Crash);
14576 let entries = actor.module.terminal_history().entries;
14577 assert_eq!(entries.len(), 1);
14578 assert_eq!(
14579 entries[0].disposition_detail.as_deref(),
14580 Some("registration_timeout")
14581 );
14582 actor.module.stop().await.unwrap();
14583 }
14584
14585 #[tokio::test(start_paused = true)]
14586 async fn registration_timeout_is_cancelled_by_hello_at_119_seconds() {
14587 let mut actor = Actor::start(ModuleProtocol::Subc, Duration::from_secs(30)).await;
14588 let pid = actor.module.status().unwrap().pid;
14589 tokio::time::advance(Duration::from_secs(119)).await;
14590 actor.hello(1, false);
14591 let registered = actor
14592 .wait_for_select(|at| at.registered_connection.is_some())
14593 .await;
14594 assert_eq!(
14595 registered.wake_after, None,
14596 "HELLO must cancel the deadline"
14597 );
14598 tokio::time::advance(Duration::from_secs(300)).await;
14599 tokio::task::yield_now().await;
14600 assert_eq!(actor.module.status().unwrap().pid, pid);
14601 assert!(actor.module.terminal_history().entries.is_empty());
14602 actor
14604 .registry
14605 .deregister_connection(ConnectionId::new(1))
14606 .unwrap();
14607 actor
14608 .wait_for_select(|at| at.registered_connection.is_none())
14609 .await;
14610 tokio::time::advance(Duration::from_secs(300)).await;
14611 tokio::task::yield_now().await;
14612 assert_eq!(actor.module.status().unwrap().pid, pid);
14613 assert!(actor.module.terminal_history().entries.is_empty());
14614 actor.module.stop().await.unwrap();
14615
14616 let mut transient = Actor::start(ModuleProtocol::Subc, Duration::from_secs(30)).await;
14617 let pid = transient.module.status().unwrap().pid;
14618 tokio::time::advance(Duration::from_secs(119)).await;
14619 let turns = transient.turns();
14620 transient.hello(1, false);
14623 transient
14624 .registry
14625 .deregister_connection(ConnectionId::new(1))
14626 .unwrap();
14627 let parked = transient.wait_for_select(|at| at.turn > turns).await;
14628 assert_eq!(
14629 parked.wake_after, None,
14630 "transient HELLO must cancel the deadline"
14631 );
14632 tokio::time::advance(Duration::from_secs(300)).await;
14633 tokio::task::yield_now().await;
14634 assert_eq!(transient.module.status().unwrap().pid, pid);
14635 assert!(transient.module.terminal_history().entries.is_empty());
14636 transient.module.stop().await.unwrap();
14637 }
14638
14639 #[tokio::test(start_paused = true)]
14640 async fn registration_timeouts_exhaust_budget_and_schedule_slow_retry() {
14641 let actor = Actor::start_with_policy(
14642 ModuleProtocol::Subc,
14643 Duration::from_secs(30),
14644 Duration::from_secs(10),
14645 RestartPolicy::new(1, Duration::ZERO),
14646 )
14647 .await;
14648 let generation = actor.module.status().unwrap().spawn_generation;
14649 actor
14650 .expire_registration_wait(Duration::from_secs(10))
14651 .await;
14652 actor.wait_for_select(|at| at.generation > generation).await;
14653 actor
14654 .expire_registration_wait(Duration::from_secs(10))
14655 .await;
14656 let retry = wait_for("failed retry schedule", || {
14657 let state = lock_snapshot(&actor.module.inner.snapshot).unwrap();
14658 (state.state == ModuleState::Failed)
14659 .then_some(state.failed_retry.pending)
14660 .flatten()
14661 })
14662 .await;
14663 assert_eq!(retry.delay, Duration::from_secs(600));
14664 assert_eq!(retry.attempt, 1);
14665 let entries = actor.module.terminal_history().entries;
14666 assert_eq!(entries.len(), 2);
14667 assert_eq!(entries[1].disposition, TerminalDisposition::Failed);
14668 assert!(entries[1]
14669 .disposition_detail
14670 .as_deref()
14671 .unwrap()
14672 .contains("registration_timeout"));
14673 tokio::time::advance(Duration::from_secs(599)).await;
14674 tokio::task::yield_now().await;
14675 assert_eq!(actor.module.state().unwrap(), ModuleState::Failed);
14676 tokio::time::advance(Duration::from_secs(1)).await;
14677 actor
14678 .wait_for_select(|at| at.generation > generation + 1)
14679 .await;
14680 actor
14682 .expire_registration_wait(Duration::from_secs(10))
14683 .await;
14684 wait_for("slow retry registration timeout", || {
14685 (actor.module.terminal_history().entries.len() == 3).then_some(())
14686 })
14687 .await;
14688 actor.module.stop().await.unwrap();
14689 }
14690
14691 #[tokio::test(start_paused = true)]
14692 async fn registration_timeout_honours_ten_second_override() {
14693 let actor = Actor::start_with_policy(
14694 ModuleProtocol::Subc,
14695 Duration::from_secs(30),
14696 Duration::from_secs(10),
14697 RestartPolicy::new(3, Duration::ZERO),
14698 )
14699 .await;
14700 let generation = actor.module.status().unwrap().spawn_generation;
14701 tokio::time::advance(Duration::from_secs(9)).await;
14702 tokio::task::yield_now().await;
14703 assert!(actor.module.terminal_history().entries.is_empty());
14704 actor.expire_registration_wait(Duration::from_secs(1)).await;
14705 actor.wait_for_select(|at| at.generation > generation).await;
14706 assert_eq!(actor.module.terminal_history().entries.len(), 1);
14707 actor.module.stop().await.unwrap();
14708 }
14709
14710 #[tokio::test]
14711 async fn registration_timeout_kills_term_ignoring_helper_before_reaping_leader() {
14712 use std::os::unix::process::{CommandExt, ExitStatusExt};
14713
14714 struct Helper(std::process::Child);
14715 impl Drop for Helper {
14716 fn drop(&mut self) {
14717 if matches!(self.0.try_wait(), Ok(None)) {
14720 let _ = self.0.kill();
14721 let _ = self.0.wait();
14722 }
14723 }
14724 }
14725
14726 let home = ScratchDir::new("registration-timeout-helper");
14727 let supervisor = Supervisor::new_for_test(
14728 Arc::new(Registry::default()),
14729 RestartPolicy::new(0, Duration::ZERO),
14730 );
14731 let spec = ModuleSpec {
14732 module_id: "registration-timeout-group".into(),
14733 program: "/bin/sleep".into(),
14734 args: vec!["120".into()],
14735 env: ["XDG_DATA_HOME", "XDG_RUNTIME_DIR", "XDG_CONFIG_HOME"]
14736 .into_iter()
14737 .map(|key| (key.into(), home.path().display().to_string()))
14738 .collect(),
14739 reserved: false,
14740 reserved_prefixes: vec![],
14741 protocol: ModuleProtocol::Subc,
14742 overlap: Default::default(),
14743 };
14744 let module = supervisor
14745 .supervise_configured_requested(
14746 spec,
14747 true,
14748 HealthConfig::default(),
14749 None,
14750 RestartPolicy::new(0, Duration::ZERO),
14751 Duration::from_secs(10),
14752 None,
14753 )
14754 .unwrap();
14755 let leader = timeout(Duration::from_secs(5), async {
14756 loop {
14757 if let Some(pid) = module.status().unwrap().pid {
14758 break pid;
14759 }
14760 sleep(Duration::from_millis(5)).await;
14761 }
14762 })
14763 .await
14764 .expect("supervised group leader must be reported before the deadline");
14765 assert!(leader > 1);
14766 let ready = home.join("helper.ready");
14767 let mut helper = Helper(
14770 std::process::Command::new("/bin/sh")
14771 .args([
14772 "-c",
14773 "trap '' TERM; : > \"$1\"; exec /bin/sleep 120",
14774 "helper",
14775 ])
14776 .arg(&ready)
14777 .process_group(i32::try_from(leader).unwrap())
14778 .stdout(std::process::Stdio::null())
14779 .stderr(std::process::Stdio::null())
14780 .spawn()
14781 .unwrap(),
14782 );
14783 timeout(Duration::from_secs(5), async {
14784 while !ready.exists() {
14785 assert!(helper.0.try_wait().unwrap().is_none());
14786 sleep(Duration::from_millis(5)).await;
14787 }
14788 })
14789 .await
14790 .expect("helper must install TERM ignore before the registration deadline");
14791 let helper_pid =
14792 rustix::process::Pid::from_raw(i32::try_from(helper.0.id()).unwrap()).unwrap();
14793 rustix::process::kill_process(helper_pid, rustix::process::Signal::TERM).unwrap();
14794 sleep(Duration::from_millis(20)).await;
14795 assert!(
14796 helper.0.try_wait().unwrap().is_none(),
14797 "helper must ignore TERM"
14798 );
14799
14800 timeout(Duration::from_secs(20), async {
14801 while module.state().unwrap() != ModuleState::Failed {
14802 sleep(Duration::from_millis(5)).await;
14803 }
14804 })
14805 .await
14806 .expect("registration timeout must stop and reap the group leader");
14807 let helper_exit = timeout(Duration::from_secs(3), async {
14808 loop {
14809 if let Some(status) = helper.0.try_wait().unwrap() {
14810 break status;
14811 }
14812 sleep(Duration::from_millis(5)).await;
14813 }
14814 })
14815 .await
14816 .expect("TERM-ignoring helper survived the registration timeout's group KILL");
14817 assert_eq!(helper_exit.signal(), Some(9));
14818 assert!(!cortexkit_test_support::process_alive(
14819 i32::try_from(leader).unwrap()
14820 ));
14821 assert_eq!(
14822 module.status().unwrap().last_exit.unwrap().kind,
14823 ExitKind::Crash
14824 );
14825 assert!(module.terminal_history().entries[0]
14826 .disposition_detail
14827 .as_deref()
14828 .unwrap()
14829 .contains("registration_timeout"));
14830 module.stop().await.unwrap();
14831 }
14832
14833 #[tokio::test(start_paused = true)]
14834 async fn registration_timeout_does_not_count_an_earlier_exit() {
14835 let actor = Actor::start_with_policy(
14836 ModuleProtocol::Subc,
14837 Duration::from_secs(30),
14838 DEFAULT_REGISTER_TIMEOUT,
14839 RestartPolicy::new(0, Duration::ZERO),
14840 )
14841 .await;
14842 let pid = actor.module.status().unwrap().pid.unwrap();
14843 rustix::process::kill_process(
14844 rustix::process::Pid::from_raw(pid as i32).unwrap(),
14845 rustix::process::Signal::TERM,
14846 )
14847 .unwrap();
14848 wait_for("early exit", || {
14849 (actor.module.state().unwrap() == ModuleState::Failed).then_some(())
14850 })
14851 .await;
14852 tokio::time::advance(Duration::from_secs(121)).await;
14853 tokio::task::yield_now().await;
14854 let entries = actor.module.terminal_history().entries;
14855 assert_eq!(entries.len(), 1);
14856 assert!(!entries[0]
14857 .disposition_detail
14858 .as_deref()
14859 .unwrap()
14860 .contains("registration_timeout"));
14861 assert_eq!(actor.module.status().unwrap().lifetime_restarts, 0);
14862 actor.module.stop().await.unwrap();
14863 }
14864
14865 #[tokio::test(start_paused = true)]
14866 async fn registration_timeout_is_cancelled_by_daemon_shutdown() {
14867 let actor = Actor::start(ModuleProtocol::Subc, Duration::from_secs(30)).await;
14868 let pid = actor.module.status().unwrap().pid;
14869 tokio::time::advance(Duration::from_secs(119)).await;
14870 actor.roster.close();
14871 let turns = actor.turns();
14872 tokio::time::advance(Duration::from_secs(300)).await;
14873 let parked = actor.wait_for_select(|at| at.turn > turns).await;
14874 assert_eq!(parked.wake_after, None);
14875 assert_eq!(actor.module.status().unwrap().pid, pid);
14876 assert!(actor.module.terminal_history().entries.is_empty());
14877 actor.module.stop().await.unwrap();
14878 let status = actor.module.status().unwrap();
14879 assert_eq!(status.restart_count, 0);
14880 assert_eq!(
14881 actor.module.terminal_history().entries[0].disposition,
14882 TerminalDisposition::DaemonShutdown
14883 );
14884 }
14885
14886 #[tokio::test(start_paused = true)]
14887 async fn no_health_children_park_for_sixty_seconds() {
14888 let none = Actor::start(ModuleProtocol::None, Duration::from_secs(30)).await;
14889 let unregistered = Actor::start(ModuleProtocol::Subc, Duration::from_secs(30)).await;
14890 let mut unadvertised = Actor::start(ModuleProtocol::Subc, Duration::from_secs(30)).await;
14891 unadvertised.hello(1, false);
14892 for _ in 0..6000 {
14897 tokio::time::advance(Duration::from_millis(10)).await;
14898 tokio::task::yield_now().await;
14899 }
14900 for actor in [&none, &unregistered, &unadvertised] {
14901 assert!(
14902 actor.turns() <= 5,
14903 "no-health actor woke {} times",
14904 actor.turns()
14905 );
14906 assert_eq!(actor.module.state().unwrap(), ModuleState::Running);
14907 }
14908 }
14909
14910 #[tokio::test(start_paused = true)]
14911 async fn late_hello_starts_a_due_probe_in_the_notification_tick() {
14912 let mut actor = Actor::start(ModuleProtocol::Subc, Duration::ZERO).await;
14916 let tick = Instant::now();
14917 actor.hello(1, true);
14918 actor.probe().await;
14919 assert_eq!(Instant::now(), tick, "HELLO must not wait for a poll");
14920 }
14921
14922 #[tokio::test(start_paused = true)]
14923 async fn hello_keeps_the_jittered_cadence_and_catalog_updates_do_not_reset_it() {
14924 let mut actor = Actor::start(ModuleProtocol::Subc, Duration::from_secs(30)).await;
14925 let hello_tick = Instant::now();
14926 actor.hello(1, true);
14927 let armed = actor
14928 .wait_for_select(|checkpoint| {
14929 checkpoint.registered_connection == Some(ConnectionId::new(1))
14930 && checkpoint.next_probe_at.is_some()
14931 })
14932 .await;
14933 let delay = jittered_health_delay(&actor.spec.module_id, 0, actor.health.cadence);
14934 assert!((Duration::from_secs(30)..Duration::from_secs(33)).contains(&delay));
14935 let first_deadline = hello_tick + delay;
14936 assert_eq!(armed.next_probe_at, Some(first_deadline));
14937 tokio::time::advance(Duration::from_secs(29)).await;
14938 let turns = actor.turns();
14939 actor
14940 .registry
14941 .replace_catalog_for_connection(ConnectionId::new(1), vec![], None, Some(true))
14942 .unwrap();
14943 let updated = actor
14944 .wait_for_select(|checkpoint| checkpoint.turn > turns)
14945 .await;
14946 assert_eq!(
14947 updated.next_probe_at,
14948 Some(first_deadline),
14949 "catalog update must not reset cadence"
14950 );
14951 actor.no_probe();
14952 tokio::time::advance(first_deadline - Instant::now()).await;
14953 let frame = actor.probe().await;
14954 actor.answer(1, frame);
14955 let next = jittered_health_delay(&actor.spec.module_id, 1, actor.health.cadence);
14956 let next_deadline = Instant::now() + next;
14957 let rearmed = actor
14958 .wait_for_select(|checkpoint| {
14959 checkpoint
14960 .next_probe_at
14961 .is_some_and(|deadline| deadline > first_deadline)
14962 })
14963 .await;
14964 assert_eq!(rearmed.next_probe_at, Some(next_deadline));
14965 tokio::time::advance(next - Duration::from_nanos(1)).await;
14966 actor.no_probe();
14967 tokio::time::advance(Duration::from_nanos(1)).await;
14968 actor.probe().await;
14969 }
14970
14971 #[tokio::test(start_paused = true)]
14972 async fn disconnect_disarms_and_reconnect_rearms_health() {
14973 let mut actor = Actor::start(ModuleProtocol::Subc, Duration::from_secs(30)).await;
14974 actor.hello(1, true);
14975 actor
14976 .wait_for_select(|checkpoint| {
14977 checkpoint.registered_connection == Some(ConnectionId::new(1))
14978 && checkpoint.next_probe_at.is_some()
14979 })
14980 .await;
14981 let turns = actor.turns();
14982 let tick = Instant::now();
14983 actor
14984 .registry
14985 .deregister_connection(ConnectionId::new(1))
14986 .unwrap();
14987 let disarmed = actor
14988 .wait_for_select(|checkpoint| {
14989 checkpoint.turn > turns && checkpoint.registered_connection.is_none()
14990 })
14991 .await;
14992 assert_eq!(disarmed.next_probe_at, None);
14993 assert_eq!(disarmed.wake_after, None);
14994 assert_eq!(
14995 Instant::now(),
14996 tick,
14997 "disconnect must disarm in the notification tick"
14998 );
14999 tokio::time::advance(Duration::from_secs(60)).await;
15000 actor.no_probe();
15001 let reconnect_tick = Instant::now();
15002 actor.hello(2, true);
15003 let rearmed = actor
15004 .wait_for_select(|checkpoint| {
15005 checkpoint.registered_connection == Some(ConnectionId::new(2))
15006 && checkpoint.next_probe_at.is_some()
15007 })
15008 .await;
15009 let delay = jittered_health_delay(&actor.spec.module_id, 0, actor.health.cadence);
15010 assert_eq!(rearmed.next_probe_at, Some(reconnect_tick + delay));
15011 tokio::time::advance(delay).await;
15012 actor.probe().await;
15013 }
15014
15015 #[tokio::test(start_paused = true)]
15016 async fn reregistration_without_health_disarms_the_old_schedule() {
15017 let mut actor = Actor::start(ModuleProtocol::Subc, Duration::from_secs(30)).await;
15018 actor.hello(1, true);
15019 actor
15020 .wait_for_select(|checkpoint| {
15021 checkpoint.registered_connection == Some(ConnectionId::new(1))
15022 && checkpoint.next_probe_at.is_some()
15023 })
15024 .await;
15025 actor
15026 .registry
15027 .deregister_connection(ConnectionId::new(1))
15028 .unwrap();
15029 actor.hello(2, false);
15030 let disarmed = actor
15031 .wait_for_select(|checkpoint| {
15032 checkpoint.registered_connection == Some(ConnectionId::new(2))
15033 })
15034 .await;
15035 assert_eq!(disarmed.next_probe_at, None);
15036 assert_eq!(disarmed.wake_after, None);
15037 tokio::time::advance(Duration::from_secs(60)).await;
15038 actor.no_probe();
15039 }
15040
15041 #[tokio::test(start_paused = true)]
15042 async fn restart_waits_for_its_own_hello_before_rearming_health() {
15043 let mut actor = Actor::start(ModuleProtocol::Subc, Duration::from_secs(30)).await;
15044 actor.hello(1, true);
15045 let armed = actor
15046 .wait_for_select(|checkpoint| {
15047 checkpoint.registered_connection == Some(ConnectionId::new(1))
15048 && checkpoint.next_probe_at.is_some()
15049 })
15050 .await;
15051 actor
15055 .registry
15056 .deregister_connection(ConnectionId::new(1))
15057 .unwrap();
15058 actor.module.restart(Some(0)).await.unwrap();
15059 let parked = actor
15060 .wait_for_select(|checkpoint| {
15061 checkpoint.generation > armed.generation
15062 && checkpoint.registered_connection.is_none()
15063 && checkpoint.next_probe_at.is_none()
15064 })
15065 .await;
15066 assert_eq!(parked.wake_after, Some(DEFAULT_REGISTER_TIMEOUT));
15067 actor.rx.close();
15068 while actor.rx.try_recv().is_ok() {}
15071 let turns = actor.turns();
15072 tokio::time::advance(Duration::from_secs(60)).await;
15073 let after = actor.turns();
15074 eprintln!(
15075 "restart park turns: before={turns} after={after} delta={}",
15076 after - turns
15077 );
15078 assert_eq!(after, turns, "replacement must park before HELLO");
15079 let hello_tick = Instant::now();
15080 actor.hello(2, true);
15081 let rearmed = actor
15082 .wait_for_select(|checkpoint| {
15083 checkpoint.registered_connection == Some(ConnectionId::new(2))
15084 && checkpoint.next_probe_at.is_some()
15085 })
15086 .await;
15087 let delay = jittered_health_delay(&actor.spec.module_id, 0, actor.health.cadence);
15088 assert_eq!(rearmed.next_probe_at, Some(hello_tick + delay));
15089 tokio::time::advance(delay).await;
15090 actor.probe().await;
15091 }
15092
15093 #[tokio::test(start_paused = true)]
15094 async fn http_probe_waits_a_full_cadence_for_each_spawn() {
15095 use tokio::{
15096 io::{AsyncReadExt, AsyncWriteExt},
15097 net::TcpListener,
15098 };
15099 let mut actor = Actor::start(ModuleProtocol::None, Duration::from_secs(30)).await;
15100 let listener = TcpListener::bind("127.0.0.1:0").await.unwrap();
15101 actor.health.http = Some(format!("http://{}/healthz", listener.local_addr().unwrap()));
15102 let (started_tx, mut started_rx) = mpsc::channel(8);
15103 let peer = tokio::spawn(async move {
15104 loop {
15105 let (mut socket, _) = listener.accept().await.unwrap();
15106 let mut request = [0; 1024];
15107 assert!(socket.read(&mut request).await.unwrap() > 0);
15108 started_tx.send(Instant::now()).await.unwrap();
15109 socket
15110 .write_all(b"HTTP/1.1 200 OK\r\nContent-Length: 0\r\n\r\n")
15111 .await
15112 .unwrap();
15113 }
15114 });
15115 actor
15116 .module
15117 .update_configuration(actor.spec.clone(), actor.health.clone(), None)
15118 .await
15119 .unwrap();
15120 let first_tick = Instant::now();
15121 let first = actor
15122 .wait_for_select(|checkpoint| checkpoint.next_probe_at.is_some())
15123 .await;
15124 assert!(first.next_probe_at.unwrap() >= first_tick + actor.health.cadence);
15125 assert!(started_rx.try_recv().is_err());
15126
15127 tokio::time::advance(
15130 first.next_probe_at.unwrap() - Instant::now() - Duration::from_secs(1),
15131 )
15132 .await;
15133 actor.module.restart(Some(0)).await.unwrap();
15134 let spawn_tick = Instant::now();
15135 let replacement = actor
15136 .wait_for_select(|checkpoint| checkpoint.generation > first.generation)
15137 .await;
15138 let deadline = replacement.next_probe_at.unwrap();
15139 assert!(
15140 deadline >= spawn_tick + actor.health.cadence,
15141 "a replacement must not inherit its predecessor's probe deadline"
15142 );
15143 tokio::time::advance(actor.health.cadence - Duration::from_nanos(1)).await;
15144 assert!(
15145 started_rx.try_recv().is_err(),
15146 "no HTTP probe during startup grace"
15147 );
15148 tokio::time::advance(deadline - Instant::now()).await;
15149 let landed = wait_for("first replacement HTTP probe", || {
15150 started_rx.try_recv().ok()
15151 })
15152 .await;
15153 assert!(landed >= spawn_tick + actor.health.cadence);
15154 wait_for("healthy replacement", || {
15155 (actor.module.status().unwrap().health.status == SupervisorHealthStatus::Ok)
15156 .then_some(())
15157 })
15158 .await;
15159 peer.abort();
15160 }
15161
15162 #[tokio::test(start_paused = true)]
15163 async fn rescan_adds_and_removes_http_health_without_polling() {
15164 use tokio::{
15165 io::{AsyncReadExt, AsyncWriteExt},
15166 net::TcpListener,
15167 };
15168 let mut actor = Actor::start(ModuleProtocol::None, Duration::from_secs(30)).await;
15169 let listener = TcpListener::bind("127.0.0.1:0").await.unwrap();
15170 actor.health.http = Some(format!("http://{}/health", listener.local_addr().unwrap()));
15171 actor.health.cadence = Duration::from_millis(1);
15173 let (started_tx, mut started_rx) = mpsc::channel(8);
15174 let peer = tokio::spawn(async move {
15175 loop {
15176 let (mut socket, _) = listener.accept().await.unwrap();
15177 let mut request = [0; 1024];
15178 assert!(socket.read(&mut request).await.unwrap() > 0);
15179 started_tx.send(()).await.unwrap();
15180 socket
15181 .write_all(b"HTTP/1.1 200 OK\r\nContent-Length: 0\r\n\r\n")
15182 .await
15183 .unwrap();
15184 }
15185 });
15186 let tick = Instant::now();
15187 let turns = actor.turns();
15188 actor
15189 .module
15190 .update_configuration(actor.spec.clone(), actor.health.clone(), None)
15191 .await
15192 .unwrap();
15193 let armed = actor
15194 .wait_for_select(|checkpoint| {
15195 checkpoint.turn > turns && checkpoint.next_probe_at.is_some()
15196 })
15197 .await;
15198 assert_eq!(armed.next_probe_at, Some(tick + Duration::from_millis(1)));
15199 assert_eq!(Instant::now(), tick);
15200 tokio::time::advance(Duration::from_millis(1)).await;
15201 let mut started = false;
15202 wait_for("successful HTTP probe", || {
15203 started |= started_rx.try_recv().is_ok();
15204 (started && actor.module.status().unwrap().health.status == SupervisorHealthStatus::Ok)
15205 .then_some(())
15206 })
15207 .await;
15208 assert!(started, "added HTTP check must start at its first deadline");
15209 assert_eq!(
15210 actor.module.status().unwrap().health.status,
15211 SupervisorHealthStatus::Ok
15212 );
15213 assert_eq!(Instant::now(), tick + Duration::from_millis(1));
15214 actor.health.http = None;
15215 let turns = actor.turns();
15216 actor
15217 .module
15218 .update_configuration(actor.spec.clone(), actor.health.clone(), None)
15219 .await
15220 .unwrap();
15221 let parked = actor
15222 .wait_for_select(|checkpoint| {
15223 checkpoint.turn > turns && checkpoint.next_probe_at.is_none()
15224 })
15225 .await;
15226 assert_eq!(parked.wake_after, None);
15227 let turns = actor.turns();
15228 tokio::time::advance(Duration::from_secs(60)).await;
15229 assert_eq!(
15230 actor.turns(),
15231 turns,
15232 "removed HTTP check must park the actor"
15233 );
15234 assert!(
15235 started_rx.try_recv().is_err(),
15236 "removed HTTP check must stay disarmed"
15237 );
15238 assert_eq!(
15239 actor.module.status().unwrap().health,
15240 ModuleHealthStatus::default()
15241 );
15242 peer.abort();
15243 }
15244}
15245
15246#[cfg(test)]
15247mod drain_busy_reprobe_tests {
15248 use super::*;
15252 use std::collections::BTreeMap;
15253 use std::sync::Mutex as StdMutex;
15254 use subc_protocol::{manifest::Concurrency, session::ModuleControlResponse};
15255
15256 const MODULE_ID: &str = "drain-reprobe-module";
15257 const CONNECTION: u64 = 7;
15258 const GAUGE: &str = "runs_in_flight";
15259 const CADENCE: Duration = Duration::from_secs(30);
15262 const CEILING: Duration = Duration::from_secs(30);
15263
15264 #[derive(Clone, Copy)]
15266 enum Answer {
15267 BusyUntil { until: Instant, latency: Duration },
15270 OmitsGauge,
15272 Silent,
15274 }
15275
15276 struct Drain {
15277 forwarding: Arc<ForwardingTable>,
15278 runtime: SupervisorRuntimeConfig,
15279 endpoint: crate::ModuleEndpointId,
15280 probes: Arc<StdMutex<Vec<Instant>>>,
15282 responder: tokio::task::JoinHandle<()>,
15283 }
15284
15285 impl Drain {
15286 fn start(answer: Answer) -> Self {
15287 let registry = Arc::new(Registry::default());
15288 let forwarding = Arc::new(ForwardingTable::default());
15289 let health = HealthConfig {
15290 cadence: CADENCE,
15291 ..HealthConfig::default()
15292 };
15293 let runtime = Supervisor::new_for_test(registry, RestartPolicy::new(3, Duration::ZERO))
15294 .with_forwarding(forwarding.clone())
15295 .with_health_config(health)
15296 .runtime_config();
15297 let (tx, mut rx) = mpsc::channel::<crate::router::OutboundFrame>(8);
15298 let endpoint = forwarding
15299 .register_module_connection(
15300 ConnectionId::new(CONNECTION),
15301 MODULE_ID.to_string(),
15302 subc_protocol::PROTOCOL_VERSION,
15303 Concurrency::ModuleManaged,
15304 FrameSink::new(tx),
15305 )
15306 .unwrap();
15307 let probes = Arc::new(StdMutex::new(Vec::new()));
15308 let responder = tokio::spawn({
15309 let forwarding = forwarding.clone();
15310 let probes = probes.clone();
15311 async move {
15312 while let Some(frame) = rx.recv().await {
15313 let body: Value = serde_json::from_slice(&frame.body).unwrap();
15314 assert_eq!(body["op"], MODULE_CONTROL_OP_HEALTH_CHECK);
15315 probes.lock().unwrap().push(Instant::now());
15316 let metrics = match answer {
15317 Answer::BusyUntil { until, latency } => {
15318 sleep(latency).await;
15319 let busy = u64::from(Instant::now() < until);
15320 serde_json::json!({ GAUGE: busy })
15321 }
15322 Answer::OmitsGauge => serde_json::json!({ "queue_depth": 0 }),
15323 Answer::Silent => continue,
15324 };
15325 let _ = forwarding.complete_module_control_rpc(
15328 ConnectionId::new(CONNECTION),
15329 frame.header.corr,
15330 Some(MODULE_CONTROL_OP_HEALTH_CHECK),
15331 ModuleControlRpcOutcome::Response(ModuleControlResponse::HealthCheck {
15332 status: HealthStatus::Ok,
15333 detail: None,
15334 metrics: Some(metrics),
15335 }),
15336 );
15337 }
15338 }
15339 });
15340 Self {
15341 forwarding,
15342 runtime,
15343 endpoint,
15344 probes,
15345 responder,
15346 }
15347 }
15348
15349 async fn run(&self, ceiling: Duration) -> (bool, Duration) {
15352 let (drained, elapsed, _) = self.run_traced(ceiling).await;
15353 (drained, elapsed)
15354 }
15355
15356 async fn run_traced(&self, ceiling: Duration) -> (bool, Duration, DrainGaugeTrace) {
15358 let started = Instant::now();
15359 let mut trace = DrainGaugeTrace::new(&[GAUGE.to_string()]);
15360 let drained = wait_for_forwarding_quiescence(
15361 &self.forwarding,
15362 MODULE_ID,
15363 &self.runtime,
15364 self.endpoint,
15365 started + ceiling,
15366 &[GAUGE.to_string()],
15367 DrainScope::Active,
15368 &mut trace,
15369 )
15370 .await
15371 .unwrap();
15372 (drained, started.elapsed(), trace)
15373 }
15374
15375 fn probes(&self) -> Vec<Instant> {
15376 self.probes.lock().unwrap().clone()
15377 }
15378 }
15379
15380 impl Drop for Drain {
15381 fn drop(&mut self) {
15382 self.responder.abort();
15383 }
15384 }
15385
15386 #[tokio::test(start_paused = true)]
15387 async fn busy_at_first_probe_then_quiet_drains_long_before_the_ceiling() {
15388 let drain = Drain::start(Answer::BusyUntil {
15389 until: Instant::now() + Duration::from_secs(1),
15390 latency: Duration::ZERO,
15391 });
15392 let (drained, elapsed) = drain.run(CEILING).await;
15393 assert!(
15394 drained,
15395 "gauges that read zero during the drain must let it finish drained, \
15396 not wait out the {CADENCE:?} health cadence (elapsed {elapsed:?})"
15397 );
15398 assert!(
15399 elapsed < Duration::from_secs(3),
15400 "drain took {elapsed:?} after the gauges went quiet at 1s"
15401 );
15402 let probes = drain.probes();
15403 assert!(
15404 probes.len() >= 2,
15405 "the module was busy at the first probe, so a quiet verdict needs a second: {probes:?}"
15406 );
15407 }
15408
15409 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
15414 async fn an_instant_busy_first_answer_does_not_stop_later_probes_in_real_time() {
15415 let drain = Drain::start(Answer::BusyUntil {
15416 until: Instant::now() + Duration::from_millis(600),
15417 latency: Duration::ZERO,
15418 });
15419 let (drained, elapsed, trace) = drain.run_traced(CEILING).await;
15420 assert!(drained, "the gauge fell at 600ms: {trace}");
15421 assert!(
15422 elapsed < Duration::from_secs(3),
15423 "drain took {elapsed:?} after the gauge fell at 600ms: {trace}"
15424 );
15425 let probes = drain.probes();
15426 assert!(
15427 probes.len() >= 2 && trace.probes as usize == probes.len(),
15428 "the drain must re-probe after the first busy answer: {probes:?}, {trace}"
15429 );
15430 }
15431
15432 #[tokio::test(start_paused = true)]
15433 async fn busy_throughout_ends_undrained_at_the_ceiling_with_bounded_probes() {
15434 let started = Instant::now();
15435 let drain = Drain::start(Answer::BusyUntil {
15436 until: started + Duration::from_secs(3600),
15437 latency: Duration::ZERO,
15438 });
15439 let (drained, elapsed) = drain.run(CEILING).await;
15440 assert!(!drained, "a module busy throughout must not drain");
15441 assert!(
15442 elapsed >= CEILING && elapsed < CEILING + drain_busy_reprobe_interval(),
15443 "a busy module is held until the ceiling and no longer: {elapsed:?}"
15444 );
15445 let probes = drain.probes();
15448 let max_probes = (CEILING.as_millis() / drain_busy_reprobe_interval().as_millis()) + 1;
15449 assert!(
15450 probes.len() as u128 <= max_probes,
15451 "{} probes in a {CEILING:?} drain exceeds the {max_probes} the interval allows",
15452 probes.len()
15453 );
15454 for pair in probes.windows(2) {
15455 assert!(
15456 pair[1] - pair[0] >= drain_busy_reprobe_interval(),
15457 "consecutive probes {:?} apart",
15458 pair[1] - pair[0]
15459 );
15460 }
15461 }
15462
15463 #[tokio::test(start_paused = true)]
15464 async fn a_slow_probe_is_never_overlapped_by_the_next() {
15465 let latency = Duration::from_secs(1);
15468 let drain = Drain::start(Answer::BusyUntil {
15469 until: Instant::now() + Duration::from_secs(3600),
15470 latency,
15471 });
15472 let (drained, _) = drain.run(Duration::from_secs(5)).await;
15473 assert!(!drained);
15474 let probes = drain.probes();
15475 assert!(probes.len() >= 2, "the drain must re-probe: {probes:?}");
15476 for pair in probes.windows(2) {
15477 assert!(
15478 pair[1] - pair[0] >= latency + drain_busy_reprobe_interval(),
15479 "a probe started {:?} after the previous one, before it could finish",
15480 pair[1] - pair[0]
15481 );
15482 }
15483 }
15484
15485 #[tokio::test(start_paused = true)]
15486 async fn an_unanswered_probe_cannot_hold_the_drain_past_its_ceiling() {
15487 let drain = Drain::start(Answer::Silent);
15488 let ceiling = Duration::from_secs(2);
15489 let (drained, elapsed) = drain.run(ceiling).await;
15490 assert!(!drained, "an unanswered probe counts as busy");
15491 assert!(
15492 elapsed >= ceiling && elapsed < ceiling + drain_busy_reprobe_interval(),
15493 "the probe deadline is capped at the drain ceiling: {elapsed:?}"
15494 );
15495 assert_eq!(
15496 drain.probes().len(),
15497 1,
15498 "the unanswered probe holds the only slot until the ceiling"
15499 );
15500 }
15501
15502 #[derive(Clone, Default)]
15504 struct LogCapture {
15505 events: Arc<StdMutex<Vec<BTreeMap<String, String>>>>,
15506 }
15507
15508 impl<S: tracing::Subscriber> tracing_subscriber::Layer<S> for LogCapture {
15509 fn on_event(
15510 &self,
15511 event: &tracing::Event<'_>,
15512 _context: tracing_subscriber::layer::Context<'_, S>,
15513 ) {
15514 struct Fields(BTreeMap<String, String>);
15515 impl tracing::field::Visit for Fields {
15516 fn record_debug(&mut self, field: &tracing::field::Field, value: &dyn fmt::Debug) {
15517 self.0
15518 .insert(field.name().to_string(), format!("{value:?}"));
15519 }
15520 }
15521 let mut fields = Fields(BTreeMap::new());
15522 event.record(&mut fields);
15523 self.events.lock().unwrap().push(fields.0);
15524 }
15525 }
15526
15527 impl LogCapture {
15528 fn install(&self) -> tracing::subscriber::DefaultGuard {
15529 use tracing_subscriber::layer::SubscriberExt;
15530 tracing::subscriber::set_default(tracing_subscriber::registry().with(self.clone()))
15531 }
15532
15533 fn with_message(&self, needle: &str) -> Vec<BTreeMap<String, String>> {
15534 self.events
15535 .lock()
15536 .unwrap()
15537 .iter()
15538 .filter(|fields| fields.get("message").is_some_and(|m| m.contains(needle)))
15539 .cloned()
15540 .collect()
15541 }
15542 }
15543
15544 #[tokio::test(start_paused = true)]
15549 async fn a_missing_declared_gauge_is_named_on_the_drain_timeout_line() {
15550 let capture = LogCapture::default();
15551 let _guard = capture.install();
15552 let drain = Drain::start(Answer::OmitsGauge);
15553 let (drained, _, trace) = drain.run_traced(Duration::from_secs(2)).await;
15554 assert!(!drained, "a gauge never reported cannot read quiet");
15555 assert!(trace.probes >= 2, "the drain must re-probe: {trace:?}");
15556
15557 let missing = capture.with_message("lacks a declared busy gauge");
15558 assert_eq!(missing.len(), 1, "one warning per drain: {missing:?}");
15559 assert_eq!(
15560 missing[0].get("missing_gauges").map(String::as_str),
15561 Some(GAUGE)
15562 );
15563
15564 warn_drain_timed_out(
15565 MODULE_ID,
15566 Duration::from_secs(2),
15567 RouteCloseReason::Restart,
15568 &Default::default(),
15569 &trace,
15570 );
15571 let timeouts = capture.with_message("route drain timed out");
15572 assert_eq!(timeouts.len(), 1, "{timeouts:?}");
15573 let gauges = &timeouts[0]["busy_gauges"];
15574 assert!(
15575 gauges.starts_with(&format!("{GAUGE}: missing_from_health_body @")),
15576 "the timeout line must name the gauge and why it read busy: {gauges}"
15577 );
15578 assert!(
15579 gauges.ends_with("; no value ever read"),
15580 "no value was ever read for the gauge: {gauges}"
15581 );
15582 assert_eq!(timeouts[0]["gauge_probes"], trace.probes.to_string());
15583 }
15584
15585 #[tokio::test(start_paused = true)]
15589 async fn the_drain_trace_names_a_busy_value_and_an_unanswered_probe() {
15590 let busy = Drain::start(Answer::BusyUntil {
15591 until: Instant::now() + Duration::from_secs(3600),
15592 latency: Duration::ZERO,
15593 });
15594 let (_, _, trace) = busy.run_traced(Duration::from_secs(1)).await;
15595 let rendered = trace.to_string();
15596 assert!(
15597 rendered.starts_with(&format!("{GAUGE}: non_zero(1) @"))
15598 && rendered.contains("; last value non_zero(1) @"),
15599 "{rendered}"
15600 );
15601
15602 let silent = Drain::start(Answer::Silent);
15603 let (_, _, trace) = silent.run_traced(Duration::from_secs(1)).await;
15604 assert_eq!(
15607 trace.to_string(),
15608 format!("{GAUGE}: probe_failed(no-answer) @1000ms; no value ever read")
15609 );
15610 assert_eq!(trace.probes, 1);
15611 }
15612
15613 #[test]
15615 fn a_gauge_that_is_not_a_count_is_busy_and_shown_as_sent() {
15616 let gauges = [GAUGE.to_string()];
15617 let verdicts = gauge_verdicts(Some(&serde_json::json!({ GAUGE: 0.0 })), &gauges);
15618 assert_eq!(verdicts, vec![GaugeVerdict::NotACount("0.0".to_string())]);
15619 assert_eq!(
15620 busy_gauge_observation(&verdicts),
15621 BusyGaugeObservation::Busy
15622 );
15623 let zero = gauge_verdicts(Some(&serde_json::json!({ GAUGE: 0 })), &gauges);
15624 assert_eq!(
15625 busy_gauge_observation(&zero),
15626 BusyGaugeObservation::Quiescent
15627 );
15628 let none = gauge_verdicts(None, &gauges);
15629 assert_eq!(busy_gauge_observation(&none), BusyGaugeObservation::Omitted);
15630 }
15631}
15632
15633#[cfg(test)]
15634mod health_evidence_tests {
15635 use super::{HealthProbeError, HealthProbeEvidence};
15636 use std::collections::HashSet;
15637
15638 #[test]
15646 fn only_a_dead_lane_is_proof_of_death() {
15647 assert!(HealthProbeError::lane_dead("gone").is_proof_of_death());
15648 assert!(!HealthProbeError::no_answer("timed out").is_proof_of_death());
15652 assert!(!HealthProbeError::bad_answer("garbage").is_proof_of_death());
15653 assert!(!HealthProbeError::misconfigured("no table").is_proof_of_death());
15654 }
15655
15656 #[test]
15662 fn every_evidence_class_has_a_distinct_label() {
15663 let labels = [
15664 HealthProbeError::lane_dead("").label(),
15665 HealthProbeError::no_answer("").label(),
15666 HealthProbeError::bad_answer("").label(),
15667 HealthProbeError::misconfigured("").label(),
15668 ];
15669 let unique: HashSet<_> = labels.iter().collect();
15670 assert_eq!(unique.len(), labels.len(), "labels collided: {labels:?}");
15671 }
15672
15673 #[test]
15679 fn classification_preserves_the_original_message() {
15680 let err = HealthProbeError::no_answer("module did not answer within 5s");
15681 assert_eq!(err.to_string(), "module did not answer within 5s");
15682 assert!(matches!(err.evidence, HealthProbeEvidence::NoAnswer));
15683 }
15684}
15685
15686#[cfg(test)]
15687mod health_tombstone_tests {
15688 use std::{path::PathBuf, sync::Arc, time::Duration};
15689
15690 use subc_protocol::{
15691 manifest::Concurrency,
15692 session::{HealthStatus, ModuleControlResponse},
15693 };
15694 use tokio::sync::mpsc;
15695
15696 use super::{
15697 probe_module_health, HealthAction, HealthConfig, HealthProbeEvidence, ModuleProtocol,
15698 ModuleSpec, RestartPolicy, Supervisor, SupervisorRuntimeConfig,
15699 };
15700 use crate::{
15701 control::ControlHandler,
15702 forwarding::{ForwardingTable, ModuleControlRpcCompletion, ModuleControlRpcOutcome},
15703 registry::{ConnectionId, Registry},
15704 router::FrameSink,
15705 };
15706
15707 struct ProbeHarness {
15708 spec: ModuleSpec,
15709 runtime: SupervisorRuntimeConfig,
15710 forwarding: Arc<ForwardingTable>,
15711 module_connection: ConnectionId,
15712 module_rx: mpsc::Receiver<crate::router::OutboundFrame>,
15713 handler: ControlHandler,
15714 module: super::SupervisedModule,
15715 }
15716
15717 fn probe_harness() -> ProbeHarness {
15718 let registry = Arc::new(Registry::default());
15719 let forwarding = Arc::new(ForwardingTable::default());
15720 let supervisor_handle = super::SupervisorHandle::new();
15721 let health = HealthConfig {
15722 http: None,
15723 cadence: Duration::from_secs(30),
15724 deadline: Duration::from_secs(5),
15725 failure_threshold: 3,
15726 on_degraded: HealthAction::Report,
15727 on_failing: HealthAction::Report,
15728 critical: false,
15729 };
15730 let supervisor = Supervisor::new_for_test(Arc::clone(®istry), RestartPolicy::default())
15731 .with_forwarding(Arc::clone(&forwarding))
15732 .with_handle(supervisor_handle.clone())
15733 .with_health_config(health);
15734 let spec = ModuleSpec {
15735 module_id: "late-health-module".to_string(),
15736 program: PathBuf::from("disabled-module"),
15737 args: Vec::new(),
15738 env: Vec::new(),
15739 reserved: false,
15740 reserved_prefixes: Vec::new(),
15741 protocol: ModuleProtocol::Subc,
15742 overlap: Default::default(),
15743 };
15744 let module = supervisor
15745 .supervise_configured(spec.clone(), false)
15746 .unwrap();
15747 let runtime = supervisor.runtime_config();
15748 let handler = ControlHandler::with_forwarding(registry, Arc::clone(&forwarding))
15749 .with_supervisor(supervisor_handle);
15750 let module_connection = ConnectionId::new(700);
15751 let (module_tx, module_rx) = mpsc::channel(8);
15752 forwarding
15753 .register_module_connection(
15754 module_connection,
15755 spec.module_id.clone(),
15756 subc_protocol::PROTOCOL_VERSION,
15757 Concurrency::ModuleManaged,
15758 FrameSink::new(module_tx),
15759 )
15760 .unwrap();
15761
15762 ProbeHarness {
15763 spec,
15764 runtime,
15765 forwarding,
15766 module_connection,
15767 module_rx,
15768 handler,
15769 module,
15770 }
15771 }
15772
15773 async fn finish_after(
15774 harness: &mut ProbeHarness,
15775 stall: Duration,
15776 ) -> ModuleControlRpcCompletion {
15777 assert!(stall > harness.runtime.health.deadline);
15778 let deadline = harness.runtime.health.deadline;
15779 let probe = probe_module_health(&harness.spec.module_id, &harness.runtime, None);
15780 let answer = async {
15781 let frame = harness.module_rx.recv().await.expect("health.check frame");
15782 tokio::time::advance(deadline).await;
15783 tokio::task::yield_now().await;
15784 tokio::time::advance(stall - deadline).await;
15785 harness
15786 .forwarding
15787 .complete_module_control_rpc(
15788 harness.module_connection,
15789 frame.header.corr,
15790 Some("health.check"),
15791 ModuleControlRpcOutcome::Response(ModuleControlResponse::HealthCheck {
15792 status: HealthStatus::Ok,
15793 detail: None,
15794 metrics: None,
15795 }),
15796 )
15797 .unwrap()
15798 };
15799 let (probe_result, completion) = tokio::join!(probe, answer);
15800 let err = probe_result.expect_err("probe must miss its deadline");
15801 assert!(matches!(err.evidence, HealthProbeEvidence::NoAnswer));
15802 completion
15803 }
15804
15805 async fn time_out_without_answer(harness: &mut ProbeHarness) {
15806 let deadline = harness.runtime.health.deadline;
15807 let probe = probe_module_health(&harness.spec.module_id, &harness.runtime, None);
15808 let exhaust_deadline = async {
15809 let _frame = harness.module_rx.recv().await.expect("health.check frame");
15810 tokio::time::advance(deadline).await;
15811 tokio::task::yield_now().await;
15812 };
15813 let (probe_result, ()) = tokio::join!(probe, exhaust_deadline);
15814 let err = probe_result.expect_err("probe must miss its deadline");
15815 assert!(matches!(err.evidence, HealthProbeEvidence::NoAnswer));
15816 }
15817
15818 async fn run_probe_cycle(harness: &mut ProbeHarness, answer: bool) {
15819 let registry = Arc::clone(&harness.module.inner.registry);
15820 let snapshot = Arc::clone(&harness.module.inner.snapshot);
15821 let process_liveness = super::SupervisorProcessLiveness::default();
15822 let mut child = None;
15823 let cycle = super::run_health_probe_cycle(
15824 &harness.spec,
15825 &harness.runtime,
15826 ®istry,
15827 &process_liveness,
15828 &snapshot,
15829 &mut child,
15830 );
15831 let peer = async {
15832 let frame = harness.module_rx.recv().await.expect("health.check frame");
15833 if answer {
15834 harness
15835 .forwarding
15836 .complete_module_control_rpc(
15837 harness.module_connection,
15838 frame.header.corr,
15839 Some("health.check"),
15840 ModuleControlRpcOutcome::Response(ModuleControlResponse::HealthCheck {
15841 status: HealthStatus::Ok,
15842 detail: None,
15843 metrics: Some(serde_json::json!({"ready": true})),
15844 }),
15845 )
15846 .unwrap();
15847 } else {
15848 tokio::time::advance(harness.runtime.health.deadline).await;
15849 tokio::task::yield_now().await;
15850 }
15851 };
15852 tokio::join!(cycle, peer);
15853 }
15854
15855 #[tokio::test(start_paused = true)]
15856 async fn unanswered_probe_is_unknown_until_threshold_and_ok_report_recovers() {
15857 let mut harness = probe_harness();
15858 let monitor = harness.module.inner.monitor.lock().unwrap().take().unwrap();
15862 monitor.abort();
15863 let _ = monitor.await;
15864 super::update_snapshot(&harness.module.inner.snapshot, None, |state| {
15865 state.enabled = true;
15866 state.state = super::ModuleState::Running;
15867 state.process_alive = true;
15868 })
15869 .unwrap();
15870
15871 run_probe_cycle(&mut harness, true).await;
15872 assert_eq!(
15873 harness.module.status().unwrap().health.status,
15874 super::SupervisorHealthStatus::Ok
15875 );
15876
15877 for failures in 1..harness.runtime.health.failure_threshold {
15878 run_probe_cycle(&mut harness, false).await;
15879 let status = harness.module.status().unwrap();
15880 assert_eq!(status.health.status, super::SupervisorHealthStatus::Unknown);
15881 assert_eq!(status.health.consecutive_failures, failures);
15882 assert!(status.health.last_probe_ms.is_some());
15883 assert!(status.health.detail.unwrap().starts_with("[no-answer]"));
15884 assert!(status.health.metrics.is_none());
15885 assert_eq!(status.state, super::ModuleState::Running);
15886 assert!(status.process_alive);
15887 assert_eq!(status.restart_count, 0);
15888 assert_eq!(status.lifetime_restarts, 0);
15889 assert!(status.health.last_action.is_none());
15890 assert!(harness.runtime.scheduled_respawn.lock().unwrap().is_none());
15891 }
15892
15893 run_probe_cycle(&mut harness, true).await;
15894 let recovered = harness.module.status().unwrap();
15895 assert_eq!(recovered.health.status, super::SupervisorHealthStatus::Ok);
15896 assert_eq!(recovered.health.consecutive_failures, 0);
15897 assert!(recovered.health.detail.is_none());
15898 assert_eq!(
15899 recovered.health.metrics,
15900 Some(serde_json::json!({"ready": true}))
15901 );
15902 assert_eq!(recovered.lifetime_restarts, 0);
15903
15904 for failures in 1..=harness.runtime.health.failure_threshold {
15905 run_probe_cycle(&mut harness, false).await;
15906 let status = harness.module.status().unwrap();
15907 assert_eq!(status.health.consecutive_failures, failures);
15908 if failures < harness.runtime.health.failure_threshold {
15909 assert_eq!(status.health.status, super::SupervisorHealthStatus::Unknown);
15910 assert_eq!(status.state, super::ModuleState::Running);
15911 assert_eq!(status.lifetime_restarts, 0);
15912 assert!(harness.runtime.scheduled_respawn.lock().unwrap().is_none());
15913 } else {
15914 assert_eq!(
15915 status.health.status,
15916 super::SupervisorHealthStatus::Unresponsive
15917 );
15918 assert_eq!(status.state, super::ModuleState::Restarting);
15919 assert_eq!(status.restart_count, 1);
15920 assert_eq!(status.lifetime_restarts, 1);
15921 assert_eq!(status.health.last_action.as_deref(), Some("restart"));
15922 assert!(harness.runtime.scheduled_respawn.lock().unwrap().is_some());
15923 }
15924 }
15925 }
15926
15927 #[tokio::test(start_paused = true)]
15928 async fn late_health_answers_record_start_anchored_latency_for_two_stalls() {
15929 let mut harness = probe_harness();
15930
15931 let first = finish_after(&mut harness, Duration::from_secs(8)).await;
15932 let first_latency = match &first {
15933 ModuleControlRpcCompletion::LateHealthAnswer { latency, .. } => *latency,
15934 other => panic!("late answer was not retained: {other:?}"),
15935 };
15936 assert!(harness.handler.observe_module_control_completion(first));
15937
15938 let second = finish_after(&mut harness, Duration::from_secs(11)).await;
15939 let second_latency = match &second {
15940 ModuleControlRpcCompletion::LateHealthAnswer { latency, .. } => *latency,
15941 other => panic!("late answer was not retained: {other:?}"),
15942 };
15943 assert!(harness.handler.observe_module_control_completion(second));
15944
15945 assert_eq!(first_latency, Duration::from_secs(8));
15946 assert_eq!(
15947 second_latency - first_latency,
15948 Duration::from_secs(3),
15949 "latency must grow linearly with the additional stall"
15950 );
15951 let health = harness.module.status().unwrap().health;
15952 assert_eq!(health.late_answer_count, 2);
15953 assert_eq!(health.last_late_answer_latency_ms, Some(11_000));
15954 }
15955
15956 #[tokio::test(start_paused = true)]
15964 async fn late_answer_clears_the_consecutive_failure_streak() {
15965 let mut harness = probe_harness();
15966
15967 time_out_without_answer(&mut harness).await;
15969 harness
15970 .module
15971 .record_health_probe_failure_for_test("[no-answer] test miss")
15972 .unwrap();
15973 assert_eq!(
15974 harness.module.status().unwrap().health.consecutive_failures,
15975 1,
15976 "precondition: the miss must be on the streak before the late answer"
15977 );
15978
15979 let late = finish_after(&mut harness, Duration::from_secs(9)).await;
15981 assert!(matches!(
15982 late,
15983 ModuleControlRpcCompletion::LateHealthAnswer { .. }
15984 ));
15985 assert!(harness.handler.observe_module_control_completion(late));
15986
15987 let health = harness.module.status().unwrap().health;
15988 assert_eq!(
15989 health.consecutive_failures, 0,
15990 "a late answer is an answer: the streak must reset"
15991 );
15992 assert_eq!(health.late_answer_count, 1);
15993 }
15994
15995 #[tokio::test(start_paused = true)]
15996 async fn repeated_serial_probe_cycles_keep_one_tombstone_per_endpoint() {
15997 let mut harness = probe_harness();
15998
15999 for _ in 0..20 {
16000 time_out_without_answer(&mut harness).await;
16001 assert_eq!(
16002 harness.forwarding.health_probe_tombstone_count().unwrap(),
16003 1
16004 );
16005 }
16006 }
16007}
16008
16009#[cfg(test)]
16010mod child_env_tests {
16011 #[cfg(windows)]
16012 use super::{
16013 clear_current_process_facts, observed_launch_nonce_source, ModuleState, SupervisorSnapshot,
16014 };
16015 #[cfg(windows)]
16016 use std::sync::{Arc, OnceLock};
16017 #[cfg(windows)]
16018 #[test]
16019 fn windows_wire_spawn_prepares_pipe_and_keeps_phase_one_env() {
16020 let spec = spec(Vec::new());
16021 let mut command = Command::new("unused");
16022 let handoff = apply_wire_spawn_args(&mut command, &spec, None, None)
16023 .unwrap()
16024 .unwrap();
16025 let name = command
16026 .as_std()
16027 .get_envs()
16028 .find(|(key, _)| *key == OsStr::new(subc_os::launch_nonce::LAUNCH_NONCE_PIPE_ENV))
16029 .and_then(|(_, value)| value)
16030 .unwrap();
16031 assert_eq!(name, OsStr::new(handoff.name()));
16032 assert!(handoff.name().starts_with(r"\\.\pipe\subc-launch-"));
16033 assert!(command.as_std().get_envs().any(|(key, value)| key
16034 == OsStr::new(subc_os::LAUNCH_NONCE_PIPE_FALLBACK_ENV)
16035 && value == Some(OsStr::new("env"))));
16036 assert!(command
16037 .as_std()
16038 .get_envs()
16039 .any(|(key, value)| key == OsStr::new(SUBC_LAUNCH_NONCE_ENV) && value.is_some()));
16040 }
16041
16042 #[cfg(windows)]
16043 #[test]
16044 fn windows_source_reporting_tracks_consumption_and_resets_per_spawn() {
16045 let mut snapshot = SupervisorSnapshot::new(ModuleState::Running, true);
16046 snapshot.process_alive = true;
16047 snapshot.spawned_protocol = Some(ModuleProtocol::Subc);
16048 assert_eq!(
16049 observed_launch_nonce_source(&snapshot).as_deref(),
16050 Some("env")
16051 );
16052 let consumed = Arc::new(OnceLock::new());
16053 snapshot.pipe_consumed = Some(consumed.clone());
16054 assert_eq!(
16055 observed_launch_nonce_source(&snapshot).as_deref(),
16056 Some("env")
16057 );
16058 consumed.set(()).unwrap();
16059 assert_eq!(
16060 observed_launch_nonce_source(&snapshot).as_deref(),
16061 Some("pipe")
16062 );
16063 snapshot.pipe_consumed = Some(Arc::new(OnceLock::new()));
16064 assert_eq!(
16065 observed_launch_nonce_source(&snapshot).as_deref(),
16066 Some("env")
16067 );
16068 clear_current_process_facts(&mut snapshot);
16069 assert_eq!(observed_launch_nonce_source(&snapshot), None);
16070 }
16071 use super::{
16072 apply_child_env, apply_spawn_role, apply_wire_spawn_args, LaunchSpecDigest, ModuleProtocol,
16073 ModuleSpec, SpawnRole, SupervisorHandle, SPAWN_ROLE_SWAP_CANDIDATE, SUBC_ARG,
16074 SUBC_LAUNCH_NONCE_ENV, SUBC_MODULE_ID_ENV, SUBC_SPAWN_ROLE_ENV,
16075 };
16076 use std::{ffi::OsStr, path::PathBuf};
16077 use tokio::process::Command;
16078
16079 fn spec(env: Vec<(String, String)>) -> ModuleSpec {
16080 ModuleSpec {
16081 module_id: "env-plan".to_string(),
16082 program: PathBuf::from("/nonexistent"),
16083 args: Vec::new(),
16084 env,
16085 reserved: false,
16086 reserved_prefixes: Vec::new(),
16087 protocol: ModuleProtocol::Subc,
16088 overlap: Default::default(),
16089 }
16090 }
16091
16092 #[test]
16093 fn configured_launch_spec_digest_is_canonical_and_excludes_injected_inputs() {
16094 let base = spec(vec![("A".into(), "B=C".into()), ("Z".into(), "v".into())]);
16095 let digest = LaunchSpecDigest::from_spec(&base);
16096 let mut reordered = base.clone();
16097 reordered.env.reverse();
16098 reordered.module_id = "different-id".into();
16099 reordered.reserved = true;
16100 reordered.overlap = Default::default();
16101 assert_eq!(digest, LaunchSpecDigest::from_spec(&reordered));
16102 let mut delimiter_collision = base.clone();
16103 delimiter_collision.env[0] = ("A=B".into(), "C".into());
16104 assert_ne!(digest, LaunchSpecDigest::from_spec(&delimiter_collision));
16105 let mut changed = base.clone();
16106 changed.protocol = ModuleProtocol::None;
16107 assert_ne!(digest, LaunchSpecDigest::from_spec(&changed));
16108 changed = base.clone();
16109 changed.args.push("--flag".into());
16110 assert_ne!(digest, LaunchSpecDigest::from_spec(&changed));
16111 changed = base.clone();
16112 changed.program = PathBuf::from("/other");
16113 assert_ne!(digest, LaunchSpecDigest::from_spec(&changed));
16114 changed = base.clone();
16115 changed.env[1].1 = "other".into();
16116 assert_ne!(digest, LaunchSpecDigest::from_spec(&changed));
16117 assert!(!format!("{digest:?}").contains("B=C"));
16118 assert!(!format!("{digest:?}").contains("[u8"));
16119 }
16120
16121 #[cfg(unix)]
16122 #[test]
16123 fn native_program_bytes_do_not_collapse_through_lossy_utf8() {
16124 use std::os::unix::ffi::OsStringExt;
16125 let mut left = spec(Vec::new());
16126 left.program = std::ffi::OsString::from_vec(vec![b'/', 0xff]).into();
16127 let mut right = left.clone();
16128 right.program = std::ffi::OsString::from_vec(vec![b'/', 0xfe]).into();
16129 assert_eq!(
16130 left.program.to_string_lossy(),
16131 right.program.to_string_lossy()
16132 );
16133 assert_ne!(
16134 LaunchSpecDigest::from_spec(&left),
16135 LaunchSpecDigest::from_spec(&right)
16136 );
16137 }
16138
16139 #[test]
16153 fn ambient_ck_log_is_removed_and_a_configured_one_survives() {
16154 let mut command = Command::new("/nonexistent");
16155 apply_child_env(&mut command, &spec(Vec::new()));
16156 let removed = command
16157 .as_std()
16158 .get_envs()
16159 .any(|(key, value)| key == OsStr::new("CK_LOG") && value.is_none());
16160 assert!(
16161 removed,
16162 "ambient CK_LOG must be explicitly removed for an unconfigured module"
16163 );
16164
16165 let mut configured = Command::new("/nonexistent");
16166 apply_child_env(
16167 &mut configured,
16168 &spec(vec![("CK_LOG".to_string(), "debug".to_string())]),
16169 );
16170 let effective = configured
16171 .as_std()
16172 .get_envs()
16173 .filter(|(key, _)| *key == OsStr::new("CK_LOG"))
16174 .last()
16175 .map(|(_, value)| value.map(|v| v.to_string_lossy().into_owned()));
16176 assert_eq!(
16177 effective,
16178 Some(Some("debug".to_string())),
16179 "a module's configured CK_LOG must survive the ambient removal"
16180 );
16181 }
16182
16183 #[test]
16192 fn protocol_none_spawn_carries_no_subc_argument_and_no_nonce() {
16193 let connection_file = std::path::Path::new("/run/subc-connection.json");
16194 let handle = SupervisorHandle::new();
16195
16196 let mut none_spec = spec(Vec::new());
16197 none_spec.protocol = ModuleProtocol::None;
16198 let mut none = Command::new("/nonexistent");
16199 let none_handoff =
16200 apply_wire_spawn_args(&mut none, &none_spec, Some(connection_file), Some(&handle))
16201 .expect("protocol-none spawn args apply");
16202 assert!(
16203 none_handoff.is_none(),
16204 "protocol:none spawn must not receive a nonce descriptor"
16205 );
16206 #[cfg(windows)]
16207 assert!(!none.as_std().get_envs().any(|(key, value)| key
16208 == OsStr::new(subc_os::LAUNCH_NONCE_PIPE_FALLBACK_ENV)
16209 && value.is_some()));
16210 assert!(
16211 !none.as_std().get_envs().any(|(key, value)| key
16212 == OsStr::new(subc_os::LAUNCH_NONCE_FD_ENV)
16213 && value.is_some()),
16214 "protocol:none spawn must not name a nonce descriptor"
16215 );
16216 let none_args: Vec<String> = none
16217 .as_std()
16218 .get_args()
16219 .map(|a| a.to_string_lossy().into_owned())
16220 .collect();
16221 assert!(
16222 !none_args.iter().any(|a| a == SUBC_ARG),
16223 "protocol:none argv must not carry --subc; got {none_args:?}"
16224 );
16225 let none_has_nonce = none
16226 .as_std()
16227 .get_envs()
16228 .any(|(key, value)| key == OsStr::new(SUBC_LAUNCH_NONCE_ENV) && value.is_some());
16229 assert!(
16230 !none_has_nonce,
16231 "protocol:none spawn must not receive a launch nonce"
16232 );
16233 let none_has_module_id = none
16234 .as_std()
16235 .get_envs()
16236 .any(|(key, value)| key == OsStr::new(SUBC_MODULE_ID_ENV) && value.is_some());
16237 assert!(
16238 none_has_module_id,
16239 "SUBC_MODULE_ID is inert and stays on every path"
16240 );
16241 assert!(
16242 handle.spawn_nonce(&none_spec.module_id).is_none(),
16243 "no nonce record for a process that will never present one"
16244 );
16245
16246 let wire_spec = spec(Vec::new());
16248 let mut wire = Command::new("/nonexistent");
16249 let wire_handoff =
16250 apply_wire_spawn_args(&mut wire, &wire_spec, Some(connection_file), Some(&handle))
16251 .expect("subc-wire spawn args apply");
16252 let wire_fd_env = wire
16253 .as_std()
16254 .get_envs()
16255 .find(|(key, _)| *key == OsStr::new(subc_os::LAUNCH_NONCE_FD_ENV))
16256 .and_then(|(_, value)| value.map(|v| v.to_string_lossy().into_owned()));
16257 #[cfg(unix)]
16258 assert_eq!(
16259 wire_fd_env,
16260 Some(wire_handoff.expect("a descriptor handoff").fd_env_value()),
16261 "a subc-wire spawn names the pipe it will receive at descriptor 3"
16262 );
16263 #[cfg(windows)]
16264 assert!(wire_handoff.is_some() && wire_fd_env.is_none());
16265 #[cfg(not(any(unix, windows)))]
16266 assert!(wire_handoff.is_none() && wire_fd_env.is_none());
16267 let wire_args: Vec<String> = wire
16268 .as_std()
16269 .get_args()
16270 .map(|a| a.to_string_lossy().into_owned())
16271 .collect();
16272 assert_eq!(
16273 wire_args,
16274 vec![
16275 SUBC_ARG.to_string(),
16276 connection_file.to_string_lossy().into_owned()
16277 ],
16278 "a subc-wire spawn still carries --subc <path>"
16279 );
16280 assert_eq!(
16281 wire.as_std()
16282 .get_envs()
16283 .any(|(key, value)| key == OsStr::new(SUBC_LAUNCH_NONCE_ENV) && value.is_some()),
16284 !cfg!(unix),
16285 "only Windows supplies the environment nonce"
16286 );
16287 assert!(handle.spawn_nonce(&wire_spec.module_id).is_some());
16288 }
16289
16290 #[test]
16300 fn plain_spawn_removes_the_spawn_role_even_when_the_spec_sets_it() {
16301 let role = |command: &Command| {
16302 command
16303 .as_std()
16304 .get_envs()
16305 .filter(|(key, _)| *key == OsStr::new(SUBC_SPAWN_ROLE_ENV))
16306 .last()
16307 .map(|(_, value)| value.map(|v| v.to_string_lossy().into_owned()))
16308 };
16309 let forged = spec(vec![(
16310 SUBC_SPAWN_ROLE_ENV.to_string(),
16311 SPAWN_ROLE_SWAP_CANDIDATE.to_string(),
16312 )]);
16313
16314 let mut plain = Command::new("/nonexistent");
16315 apply_child_env(&mut plain, &forged);
16316 apply_spawn_role(&mut plain, SpawnRole::Plain);
16317 assert_eq!(
16318 role(&plain),
16319 Some(None),
16320 "a plain spawn must remove SUBC_SPAWN_ROLE, whatever the spec says"
16321 );
16322
16323 let mut candidate = Command::new("/nonexistent");
16324 apply_child_env(&mut candidate, &spec(Vec::new()));
16325 apply_spawn_role(&mut candidate, SpawnRole::SwapCandidate);
16326 assert_eq!(
16327 role(&candidate),
16328 Some(Some(SPAWN_ROLE_SWAP_CANDIDATE.to_string()))
16329 );
16330 }
16331
16332 #[test]
16338 fn daemon_private_capture_keys_are_not_passed_to_the_child() {
16339 let mut command = Command::new("/nonexistent");
16340 apply_child_env(
16341 &mut command,
16342 &spec(vec![
16343 (super::CAPTURE_KEEP_ENV.to_string(), "5".to_string()),
16344 ("KEPT".to_string(), "yes".to_string()),
16345 ]),
16346 );
16347 let keys: Vec<String> = command
16348 .as_std()
16349 .get_envs()
16350 .filter(|(_, value)| value.is_some())
16351 .map(|(key, _)| key.to_string_lossy().into_owned())
16352 .collect();
16353 assert!(keys.contains(&"KEPT".to_string()), "got {keys:?}");
16354 assert!(
16355 !keys.contains(&super::CAPTURE_KEEP_ENV.to_string()),
16356 "daemon-private capture key leaked to the child: {keys:?}"
16357 );
16358 }
16359}
16360
16361#[cfg(test)]
16362mod jitter_tests {
16363 use super::jittered_health_delay;
16364 use std::{collections::HashSet, time::Duration};
16365
16366 const FLEET: [&str; 14] = [
16375 "aft",
16376 "alfonso-core",
16377 "magic-context",
16378 "broca",
16379 "thalamus",
16380 "quota",
16381 "engram",
16382 "plexus",
16383 "cerebellum",
16384 "astrocyte",
16385 "synapse",
16386 "subc-mcp",
16387 "cortexkit-credentials",
16388 "subc-federation",
16389 ];
16390
16391 #[test]
16399 fn probe_delays_disperse_across_the_fleet() {
16400 let cadence = Duration::from_secs(30);
16401 let delays: HashSet<Duration> = FLEET
16402 .iter()
16403 .map(|id| jittered_health_delay(id, 0, cadence))
16404 .collect();
16405 assert_eq!(
16406 delays.len(),
16407 FLEET.len(),
16408 "every supervised module must land on its own probe offset"
16409 );
16410 }
16411
16412 #[test]
16418 fn jitter_only_delays_and_stays_within_one_tenth_of_cadence() {
16419 let cadence = Duration::from_secs(30);
16420 let span = cadence / 10;
16421 for id in FLEET {
16422 for probe_index in 0..8 {
16423 let delay = jittered_health_delay(id, probe_index, cadence);
16424 assert!(
16425 delay >= cadence,
16426 "{id}#{probe_index}: jitter must not shorten the cadence"
16427 );
16428 assert!(
16429 delay < cadence + span,
16430 "{id}#{probe_index}: jitter must stay inside one tenth of the cadence"
16431 );
16432 }
16433 }
16434 }
16435
16436 #[test]
16442 fn a_module_offset_is_stable_across_restarts() {
16443 let cadence = Duration::from_secs(30);
16444 for id in FLEET {
16445 assert_eq!(
16446 jittered_health_delay(id, 0, cadence),
16447 jittered_health_delay(id, 0, cadence),
16448 "{id}: the same module and probe index must produce the same offset"
16449 );
16450 }
16451 }
16452
16453 #[test]
16455 fn zero_cadence_yields_zero_delay() {
16456 assert_eq!(
16457 jittered_health_delay("aft", 0, Duration::ZERO),
16458 Duration::ZERO
16459 );
16460 }
16461}
16462
16463#[cfg(all(test, target_os = "linux"))]
16464mod cgroup_placement_tests {
16465 use super::{
16466 apply_cgroup_placement, remove_module_cgroup, LaunchSpecDigest, ModuleProtocol, ModuleSpec,
16467 SuperviseError, SupervisedChild,
16468 };
16469 use crate::stderr_tail::{StderrRing, StderrTailConfig};
16470 use cortexkit_test_support::ScratchDir;
16471 use std::{
16472 fs, io,
16473 path::{Path, PathBuf},
16474 sync::{Arc, Mutex},
16475 };
16476 use tokio::process::Command;
16477
16478 #[tokio::test]
16479 async fn unique_spawn_cgroups_clean_up_across_restarts_and_shutdown() {
16480 use super::*;
16481 let dir = ScratchDir::new("unique-spawn-cgroups");
16482 let root = PathBuf::from(format!(
16483 "/sys/fs/cgroup/subc-unique-test-{}-{}",
16484 std::process::id(),
16485 unix_ms_now()
16486 ));
16487 if let Err(error) = fs::create_dir(&root) {
16488 assert!(
16489 std::env::var_os("SUBC_REQUIRE_CGROUP_TEST").is_none(),
16490 "required cgroup test cannot execute: {error}"
16491 );
16492 eprintln!("SKIP unique_spawn_cgroups_clean_up_across_restarts_and_shutdown: {error}");
16493 return;
16494 }
16495 let placement = subc_cgroup::prepare_at(&root).unwrap().unwrap();
16496 let group_count = || {
16497 fs::read_dir(root.join("subc-modules"))
16498 .unwrap()
16499 .map(|entry| entry.unwrap().file_type().unwrap())
16500 .filter(|kind| kind.is_dir())
16501 .count()
16502 };
16503 let supervisor =
16504 Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default())
16505 .with_cgroup_placement(Some(placement.clone()));
16506 let runtime = supervisor.runtime_config();
16507 let mut spec = ModuleSpec {
16508 module_id: "unique-spawn".into(),
16509 program: PathBuf::from("/bin/sleep"),
16510 args: vec!["60".into()],
16511 env: ["XDG_DATA_HOME", "XDG_RUNTIME_DIR", "XDG_CONFIG_HOME"]
16512 .into_iter()
16513 .map(|key| (key.into(), dir.to_string_lossy().into_owned()))
16514 .collect(),
16515 reserved: false,
16516 reserved_prefixes: vec![],
16517 protocol: ModuleProtocol::None,
16518 overlap: Default::default(),
16519 };
16520 let spawn = |spec: &ModuleSpec| {
16521 spawn_child(
16522 spec,
16523 None,
16524 None,
16525 &runtime.stderr_ring,
16526 None,
16527 &runtime.child_roster,
16528 Some(&placement),
16529 )
16530 .unwrap()
16531 };
16532 let mut live = spawn(&spec);
16533 for _ in 0..3 {
16534 let next = spawn(&spec);
16536 assert_ne!(live.module_id, next.module_id);
16537 live.start_kill().unwrap();
16538 live.wait().await.unwrap();
16539 live = next;
16540 assert!(
16541 live.child.try_wait().unwrap().is_none(),
16542 "retiring the old slot must not kill the replacement"
16543 );
16544 assert_eq!(
16545 group_count(),
16546 1,
16547 "only the live spawn's cgroup should remain"
16548 );
16549 }
16550 supervisor.begin_daemon_shutdown();
16551 let reap = tokio::spawn(async move {
16552 live.wait().await.unwrap();
16553 });
16554 supervisor
16555 .end_children_for_daemon_shutdown(false, std::future::pending())
16556 .await;
16557 reap.await.unwrap();
16558 assert_eq!(group_count(), 0);
16559 spec.program = PathBuf::from("/bin/true");
16561 spec.args.clear();
16562 let fresh_roster = ChildRoster::default();
16563 let mut short = spawn_child(
16564 &spec,
16565 None,
16566 None,
16567 &runtime.stderr_ring,
16568 None,
16569 &fresh_roster,
16570 Some(&placement),
16571 )
16572 .unwrap();
16573 short.wait().await.unwrap();
16574 assert_eq!(group_count(), 0);
16575 spec.module_id = "_".repeat(255);
16576 let mut long_id = spawn_child(
16577 &spec,
16578 None,
16579 None,
16580 &runtime.stderr_ring,
16581 None,
16582 &fresh_roster,
16583 Some(&placement),
16584 )
16585 .unwrap();
16586 long_id.wait().await.unwrap();
16587 assert_eq!(
16588 group_count(),
16589 0,
16590 "valid long module IDs must not exceed cgroup NAME_MAX"
16591 );
16592 fs::remove_dir(root.join("subc-modules")).unwrap();
16593 fs::remove_dir(root).unwrap();
16594 eprintln!("EXECUTED unique_spawn_cgroups_clean_up_across_restarts_and_shutdown");
16595 }
16596
16597 #[test]
16598 fn failed_parent_cgroup_open_is_a_cgroup_supervision_error() {
16599 let path = Path::new("/definitely-missing-subc-cgroup");
16600 let mut command = Command::new("true");
16601 let error = apply_cgroup_placement(
16602 &mut command,
16603 &ModuleSpec {
16604 module_id: "broken-cgroup".to_string(),
16605 program: PathBuf::from("true"),
16606 args: Vec::new(),
16607 env: Vec::new(),
16608 reserved: false,
16609 reserved_prefixes: Vec::new(),
16610 protocol: ModuleProtocol::Subc,
16611 overlap: Default::default(),
16612 },
16613 path,
16614 )
16615 .expect_err("a parent cgroup open failure must reject the supervised spawn");
16616 let reason = error.to_string();
16617
16618 assert!(
16619 matches!(error, SuperviseError::Cgroup { .. }),
16620 "parent cgroup open must be reported as a cgroup supervision error: {reason}"
16621 );
16622 assert!(
16623 reason.contains("/definitely-missing-subc-cgroup/cgroup.procs"),
16624 "parent cgroup open failure must name cgroup.procs: {reason}"
16625 );
16626 }
16627
16628 #[tokio::test]
16629 async fn reaping_a_child_removes_its_empty_module_cgroup() {
16630 let root = ScratchDir::new("supervisor-reap-cgroup");
16631 fs::write(root.join("cgroup.procs"), b"").expect("write scratch cgroup marker");
16632 let placement = subc_cgroup::prepare_at(&root)
16633 .expect("prepare scratch cgroup root")
16634 .expect("scratch root has a cgroup.procs marker");
16635 let module_id = "reaped-module";
16636 let module = placement
16637 .module_path(module_id)
16638 .expect("create scratch module cgroup");
16639 let child = Command::new("true")
16640 .env("XDG_DATA_HOME", root.path())
16641 .env("XDG_RUNTIME_DIR", root.path())
16642 .env("XDG_CONFIG_HOME", root.path())
16643 .spawn()
16644 .expect("spawn short-lived child");
16645 let pid = child.id().expect("spawned child has pid");
16646 let mut child = SupervisedChild {
16647 subtree_target: crate::subtree_resources::Target::Unavailable("not_running"),
16648 child,
16649 exec: crate::activity::ExecOwner(Arc::new(crate::activity::ExecOutcome::new())),
16650 protocol: ModuleProtocol::Subc,
16651 module_id: module_id.to_string(),
16652 cgroup_placement: Some(placement),
16653 stdout_pump: None,
16654 stderr_pump: None,
16655 stderr_ring: Arc::new(Mutex::new(StderrRing::new(StderrTailConfig::default()))),
16656 spawned_at: tokio::time::Instant::now(),
16657 registration_deadline: None,
16658 registration_watch: None,
16659 spawned_at_ms: 0,
16660 spawned_from: PathBuf::from("true"),
16661 launch_spec_digest: LaunchSpecDigest([0; 32]),
16662 spawned_file_identity: None,
16663 process_start_time: None,
16664 process_identity: None,
16665 pid,
16666 roster_guard: None,
16667 #[cfg(target_os = "macos")]
16668 privacy_exec: None,
16669 spawn_failure: None,
16670 };
16671
16672 child.wait().await.expect("reap short-lived child");
16673
16674 assert!(
16675 !module.exists(),
16676 "reaping the supervised child must remove its empty cgroup"
16677 );
16678 }
16679
16680 #[test]
16681 fn non_empty_cgroup_removal_is_reported_without_blocking_teardown() {
16682 let root = ScratchDir::new("supervisor-non-empty-cgroup");
16683 fs::write(root.join("cgroup.procs"), b"").expect("write scratch cgroup marker");
16684 let placement = subc_cgroup::prepare_at(&root)
16685 .expect("prepare scratch cgroup root")
16686 .expect("scratch root has a cgroup.procs marker");
16687 let module = placement
16688 .module_path("surviving-module")
16689 .expect("create scratch module cgroup");
16690 fs::write(module.join("surviving-process"), b"still present")
16691 .expect("make scratch cgroup non-empty");
16692 let (logs, _guard) = crate::router::test_log::log_capture(tracing::Level::WARN);
16693
16694 remove_module_cgroup(&placement, "surviving-module");
16695
16696 let logs = crate::router::test_log::captured_logs(&logs);
16697 assert!(
16698 module.exists(),
16699 "failed removal must leave the cgroup intact"
16700 );
16701 assert!(
16702 logs.contains("could not remove module cgroup after process exit; continuing teardown")
16703 && logs.contains("surviving-module"),
16704 "best-effort removal must report the failure without returning it: {logs}"
16705 );
16706 }
16707
16708 #[test]
16709 fn cgroup_pre_exec_spawn_failure_names_the_cgroup_path() {
16710 let cgroup_path = PathBuf::from("/sys/fs/cgroup/subc-modules/broken-module");
16711 let reason = SuperviseError::Spawn {
16712 program: PathBuf::from("/bin/true"),
16713 source: io::Error::from_raw_os_error(13),
16714 cgroup_path: Some(cgroup_path.clone()),
16715 }
16716 .to_string();
16717
16718 assert!(
16719 reason.contains(&cgroup_path.display().to_string()),
16720 "a pre_exec spawn failure must name the cgroup path: {reason}"
16721 );
16722 }
16723}
16724
16725#[cfg(test)]
16726mod spawn_subscriber_lag_tests {
16727 use super::*;
16728
16729 #[tokio::test]
16734 async fn lagged_spawn_subscriber_receives_a_terminal_lagged_error_after_its_queued_frames() {
16735 let feed = SpawnEventFeed::default();
16736 feed.configure_incarnation("lag-incarnation".to_string());
16737 let (tx, mut rx) = mpsc::channel(1);
16740 feed.subscribe(ConnectionId::new(1), 7, 1, None, FrameSink::new(tx))
16741 .expect("subscribe");
16742 let emitted = SPAWN_SUBSCRIBER_BUFFER + 16;
16743 for index in 0..emitted {
16744 feed.emit_spawned(&format!("lag-module-{index}"), 1000, 0);
16745 tokio::task::yield_now().await;
16748 }
16749 assert_eq!(
16750 feed.subscriber_count(),
16751 0,
16752 "the lagged subscriber must be removed"
16753 );
16754
16755 let mut data = Vec::new();
16756 let mut last = None;
16757 loop {
16758 let next = tokio::time::timeout(Duration::from_secs(5), rx.recv())
16759 .await
16760 .expect("the forwarder must finish once the subscriber is dropped");
16761 let Some(outbound) = next else { break };
16762 let frame = outbound.frame;
16763 if frame.header.ty == FrameType::StreamData {
16764 assert!(last.is_none(), "no data may follow the terminal frame");
16765 let event: SpawnEvent = serde_json::from_slice(&frame.body).unwrap();
16766 data.push(event.cursor.seq);
16767 } else {
16768 assert!(last.is_none(), "exactly one terminal frame");
16769 last = Some(frame);
16770 }
16771 }
16772 assert!(!data.is_empty(), "queued frames drain before the terminal");
16773 for pair in data.windows(2) {
16774 assert_eq!(
16775 pair[1],
16776 pair[0] + 1,
16777 "queued frames arrive dense and in order"
16778 );
16779 }
16780 let terminal = last.expect("a lagged subscriber must receive a terminal frame");
16781 assert_eq!(terminal.header.ty, FrameType::Error);
16782 assert_eq!(terminal.header.corr, 7);
16783 let body: subc_protocol::ErrorBody = serde_json::from_slice(&terminal.body).unwrap();
16784 assert_eq!(body.code, SPAWN_SUBSCRIBER_LAGGED_CODE);
16785 let detail = body.detail.expect("lagged error carries detail");
16786 assert_eq!(
16787 detail["first_undelivered_cursor"]["seq"],
16788 data.last().unwrap() + 1,
16789 "the named cursor is the first event the subscriber did not receive"
16790 );
16791 assert_eq!(
16792 detail["first_undelivered_cursor"]["daemon_incarnation"],
16793 "lag-incarnation"
16794 );
16795 }
16796}
16797
16798#[cfg(test)]
16799mod terminal_history_read_concurrency_tests {
16800 use super::*;
16801 use crate::terminal_journal::read_pause;
16802 use cortexkit_test_support::ScratchDir;
16803 use std::sync::mpsc as std_mpsc;
16804
16805 fn journaled_ring(
16806 journal: &Arc<crate::terminal_journal::TerminalJournal>,
16807 ) -> Arc<Mutex<TerminalRing>> {
16808 Arc::new(Mutex::new(
16809 TerminalRing::new(TerminalRingConfig::default(), 1)
16810 .with_journal(Some(Arc::clone(journal))),
16811 ))
16812 }
16813
16814 fn crash(at_ms: u64) -> ExitReport {
16815 ExitReport {
16816 kind: ExitKind::Crash,
16817 code: Some(1),
16818 signal: None,
16819 at_ms,
16820 }
16821 }
16822
16823 fn record_within(
16826 module_id: &'static str,
16827 ring: &Arc<Mutex<TerminalRing>>,
16828 at_ms: u64,
16829 bound: Duration,
16830 ) -> bool {
16831 let ring = Arc::clone(ring);
16832 let (done, done_rx) = std_mpsc::channel();
16833 std::thread::spawn(move || {
16834 record_terminal(
16835 module_id,
16836 &ring,
16837 &SpawnEventFeed::default(),
16838 &crash(at_ms),
16839 TerminalDisposition::Restarting,
16840 );
16841 let _ = done.send(());
16842 });
16843 done_rx.recv_timeout(bound).is_ok()
16844 }
16845
16846 #[test]
16851 fn exits_recorded_during_a_paused_history_read_are_not_blocked_or_half_merged() {
16852 let dir = ScratchDir::new("terminal-history-concurrent-read");
16853 let path = dir.join("terminals.jsonl");
16854 let journal = Arc::new(crate::terminal_journal::TerminalJournal::open(
16855 path.clone(),
16856 "daemon".into(),
16857 ));
16858 let reader_ring = journaled_ring(&journal);
16859 let other_ring = journaled_ring(&journal);
16860 assert!(record_within(
16861 "reader-module",
16862 &reader_ring,
16863 10,
16864 Duration::from_secs(5)
16865 ));
16866
16867 let (started, release) = read_pause::install(&path);
16868 let reading = {
16869 let ring = Arc::clone(&reader_ring);
16870 std::thread::spawn(move || durable_terminal_history_of(&ring, "reader-module"))
16871 };
16872 started
16873 .recv_timeout(Duration::from_secs(5))
16874 .expect("the history read reached its pause");
16875
16876 let bound = Duration::from_secs(1);
16877 assert!(
16878 record_within("other-module", &other_ring, 20, bound),
16879 "another module's exit waited on a history read (journal writer held)"
16880 );
16881 assert!(
16882 record_within("reader-module", &reader_ring, 30, bound),
16883 "the read module's own exit waited on its history read (ring held)"
16884 );
16885
16886 drop(release);
16887 let paused = reading.join().unwrap();
16888 assert_eq!(
16889 paused.entries.iter().map(|e| e.at_ms).collect::<Vec<_>>(),
16890 vec![10],
16891 "an exit recorded after the read began lands in neither half of it"
16892 );
16893 assert_eq!(paused.journal_skipped_lines, 0);
16894 assert_eq!(paused.journal_read_errors, 0);
16895
16896 let after = durable_terminal_history_of(&reader_ring, "reader-module");
16897 assert_eq!(
16898 after.entries.iter().map(|e| e.at_ms).collect::<Vec<_>>(),
16899 vec![10, 30],
16900 "the next read merges ring and journal with no duplicate"
16901 );
16902 assert_eq!(after.journal_skipped_lines, 0);
16903 }
16904}
16905
16906#[cfg(test)]
16911mod stderr_settle_tests {
16912 use std::{
16913 future::Future,
16914 io,
16915 pin::Pin,
16916 sync::{Arc, Mutex},
16917 task::{Context, Poll},
16918 time::Duration,
16919 };
16920
16921 use tokio::{
16922 io::{AsyncRead, ReadBuf},
16923 sync::oneshot,
16924 time::Instant,
16925 };
16926
16927 use super::{settle_stderr_pump, settle_stdout_pump, StderrPump};
16928 use crate::stderr_tail::{
16929 pump_stderr_to, pump_stdout_to, split_capture_stamp, untimed, CaptureState,
16930 ChildOutputSink, OutputSink, StderrRing, StderrTailConfig, TailEntry,
16931 };
16932
16933 const BOUND: Duration = Duration::from_millis(250);
16934
16935 struct HeldReader {
16939 before: Option<Vec<u8>>,
16940 gate: Option<oneshot::Receiver<()>>,
16941 after: io::Cursor<Vec<u8>>,
16942 }
16943
16944 impl AsyncRead for HeldReader {
16945 fn poll_read(
16946 mut self: Pin<&mut Self>,
16947 cx: &mut Context<'_>,
16948 buf: &mut ReadBuf<'_>,
16949 ) -> Poll<io::Result<()>> {
16950 if let Some(bytes) = self.before.take() {
16951 buf.put_slice(&bytes);
16952 return Poll::Ready(Ok(()));
16953 }
16954 if let Some(gate) = self.gate.as_mut() {
16955 match Pin::new(gate).poll(cx) {
16956 Poll::Pending => return Poll::Pending,
16957 Poll::Ready(_) => self.gate = None,
16958 }
16959 }
16960 Pin::new(&mut self.after).poll_read(cx, buf)
16961 }
16962 }
16963
16964 struct DiscardSink;
16965
16966 impl OutputSink for DiscardSink {
16967 fn write_line(&mut self, _line: &[u8]) {}
16968 }
16969
16970 fn line(text: &str) -> TailEntry {
16971 TailEntry::Line {
16972 text: text.to_string(),
16973 truncated: false,
16974 at_ms: None,
16975 }
16976 }
16977
16978 fn lock(ring: &Arc<Mutex<StderrRing>>) -> std::sync::MutexGuard<'_, StderrRing> {
16979 ring.lock().unwrap()
16980 }
16981
16982 fn held_pump(
16986 ring: &Arc<Mutex<StderrRing>>,
16987 before: &str,
16988 after: &str,
16989 ) -> (StderrPump, oneshot::Sender<()>) {
16990 let generation = lock(ring).begin_process();
16991 let (release, gate) = oneshot::channel();
16992 let reader = HeldReader {
16993 before: Some(before.as_bytes().to_vec()),
16994 gate: Some(gate),
16995 after: io::Cursor::new(after.as_bytes().to_vec()),
16996 };
16997 let task = tokio::spawn(pump_stderr_to(
16998 reader,
16999 Arc::clone(ring),
17000 generation,
17001 DiscardSink,
17002 ));
17003 (StderrPump { task, generation }, release)
17004 }
17005
17006 async fn wait_until(ring: &Arc<Mutex<StderrRing>>, done: impl Fn(&StderrRing) -> bool) {
17007 for _ in 0..1000 {
17008 if done(&lock(ring)) {
17009 return;
17010 }
17011 tokio::time::sleep(Duration::from_millis(1)).await;
17012 }
17013 panic!(
17014 "ring never reached the expected state: {:?}",
17015 lock(ring).snapshot(None, None)
17016 );
17017 }
17018
17019 #[tokio::test(start_paused = true)]
17020 async fn a_crash_line_the_reader_had_not_reached_by_the_bound_is_kept_before_the_restart() {
17021 let ring = Arc::new(Mutex::new(StderrRing::new(StderrTailConfig::default())));
17022 let (pump, release) = held_pump(&ring, "booting\n", "config error: missing storage\n");
17023
17024 settle_stderr_pump("crasher", &ring, pump, BOUND).await;
17025 let before_release = lock(&ring).snapshot(None, None);
17026 assert!(
17027 matches!(before_release.capture, CaptureState::Incomplete { .. }),
17028 "a reader that has not reached EOF cannot claim a whole tail: {before_release:?}"
17029 );
17030
17031 let next = lock(&ring).begin_process();
17034 lock(&ring).push_line_from(next, "next process booting");
17035 release.send(()).unwrap();
17036 wait_until(&ring, |ring| {
17037 ring.snapshot(None, None).capture == CaptureState::Captured
17038 })
17039 .await;
17040
17041 assert_eq!(
17042 untimed(lock(&ring).snapshot(None, None).entries),
17043 vec![
17044 line("booting"),
17045 line("config error: missing storage"),
17046 TailEntry::ProcessStart,
17047 line("next process booting"),
17048 ],
17049 "the crash's last line must survive a slow reader and stay in the crashed process's section"
17050 );
17051 }
17052
17053 #[tokio::test(start_paused = true)]
17054 async fn a_pipe_held_open_by_a_descendant_reads_incomplete_without_delaying_the_restart_past_the_bound(
17055 ) {
17056 let ring = Arc::new(Mutex::new(StderrRing::new(StderrTailConfig::default())));
17057 let (pump, _held) = held_pump(&ring, "parent exiting\n", "");
17060
17061 let started = Instant::now();
17062 settle_stderr_pump("orphaning", &ring, pump, BOUND).await;
17063 assert_eq!(
17064 started.elapsed(),
17065 BOUND,
17066 "the restart must wait exactly the bound for a pipe that stays open, no longer"
17067 );
17068
17069 let next = lock(&ring).begin_process();
17070 lock(&ring).push_line_from(next, "next process booting");
17071 tokio::time::sleep(Duration::from_secs(60)).await;
17072
17073 let snapshot = lock(&ring).snapshot(None, None);
17074 match &snapshot.capture {
17075 CaptureState::Incomplete { reason } => assert!(
17076 reason.contains("had not reached EOF") && reason.contains("250ms"),
17077 "the reason must say what is missing and after how long: {reason}"
17078 ),
17079 other => panic!("expected Incomplete while the pipe is held open, got {other:?}"),
17080 }
17081 assert_eq!(
17082 untimed(snapshot.entries),
17083 vec![
17084 line("parent exiting"),
17085 TailEntry::ProcessStart,
17086 line("next process booting"),
17087 ]
17088 );
17089 }
17090
17091 #[tokio::test(start_paused = true)]
17092 async fn a_reader_that_reaches_eof_within_the_bound_leaves_the_tail_captured() {
17093 let ring = Arc::new(Mutex::new(StderrRing::new(StderrTailConfig::default())));
17094 let (pump, release) = held_pump(&ring, "one\n", "two\n");
17095 release.send(()).unwrap();
17096
17097 settle_stderr_pump("clean", &ring, pump, BOUND).await;
17098
17099 let snapshot = lock(&ring).snapshot(None, None);
17100 assert_eq!(snapshot.capture, CaptureState::Captured);
17101 assert_eq!(untimed(snapshot.entries), vec![line("one"), line("two")]);
17102 }
17103
17104 #[tokio::test(start_paused = true)]
17105 async fn stdout_buffered_past_the_exit_drain_bound_reaches_the_capture_file() {
17106 const BURST: usize = 150;
17107 let capture = cortexkit_test_support::ScratchDir::new("stdout-exit-drain");
17108 let path = capture.join("two-pipe.stderr.log");
17109 let sink = ChildOutputSink::open(&path, cortexkit_log::Retention::default()).unwrap();
17110 let ring = Arc::new(Mutex::new(StderrRing::new(StderrTailConfig::default())));
17111 let generation = lock(&ring).begin_process();
17112 let stdout = (0..BURST)
17113 .map(|i| format!("out-{i:04}-{}\n", "o".repeat(64)))
17114 .collect::<String>();
17115 let stderr = (0..BURST)
17116 .map(|i| format!("err-{i:04}-{}\n", "e".repeat(64)))
17117 .collect::<String>();
17118 let first_chunk = stdout.find('\n').unwrap() + 1 + "out-0001-".len();
17123 let (release, gate) = oneshot::channel();
17124 let reader = HeldReader {
17125 before: Some(stdout.as_bytes()[..first_chunk].to_vec()),
17126 gate: Some(gate),
17127 after: io::Cursor::new(stdout.as_bytes()[first_chunk..stdout.len() - 1].to_vec()),
17128 };
17129 let (finished, completion) = oneshot::channel();
17130 let stdout_sink = sink.clone();
17131 let pump = tokio::spawn(async move {
17132 pump_stdout_to(reader, stdout_sink).await;
17133 finished.send(()).unwrap();
17134 });
17135 pump_stderr_to(io::Cursor::new(stderr.into_bytes()), ring, generation, sink).await;
17136
17137 let started = Instant::now();
17138 settle_stdout_pump("two-pipe", pump, BOUND).await;
17139 assert_eq!(
17140 started.elapsed(),
17141 BOUND,
17142 "a late reader must not hold up restart"
17143 );
17144 assert_eq!(
17145 std::fs::read_to_string(&path).unwrap().lines().count(),
17146 BURST + 1
17147 );
17148
17149 release
17150 .send(())
17151 .expect("stdout reader must survive the exit drain bound");
17152 tokio::time::timeout(Duration::from_secs(1), completion)
17153 .await
17154 .expect("released stdout reader must reach EOF")
17155 .expect("stdout capture must finish at EOF, not be cancelled");
17156 let contents = std::fs::read_to_string(&path).unwrap();
17157 let mut lines = contents
17158 .lines()
17159 .map(|line| split_capture_stamp(line).expect("capture stamp").1)
17160 .collect::<Vec<_>>();
17161 let mut expected = (0..BURST)
17162 .flat_map(|i| {
17163 [
17164 format!("out-{i:04}-{}", "o".repeat(64)),
17165 format!("err-{i:04}-{}", "e".repeat(64)),
17166 ]
17167 })
17168 .collect::<Vec<_>>();
17169 lines.sort_unstable();
17170 expected.sort_unstable();
17171 assert_eq!(
17172 lines, expected,
17173 "both pipes must retain every complete line exactly once"
17174 );
17175 }
17176}
17177
17178#[cfg(all(test, windows))]
17192mod job_containment_tests {
17193 use super::*;
17194 use cortexkit_test_support::ScratchDir;
17195 use std::{
17196 path::{Path, PathBuf},
17197 sync::{Arc, Mutex},
17198 time::{Duration, Instant},
17199 };
17200
17201 fn stub_path() -> PathBuf {
17207 let mut path = std::env::current_exe().expect("current_exe available in tests");
17208 path.pop();
17209 path.pop();
17210 path.push("fake-aft-stub.exe");
17211 assert!(
17212 path.exists(),
17213 "fake-aft-stub not built at {}: run `cargo test -p subc-core` (which builds \
17214 [[bin]] targets) rather than `cargo test -p subc-core --lib` (which does not)",
17215 path.display()
17216 );
17217 path
17218 }
17219
17220 fn read_grandchild_pid(path: &Path) -> u32 {
17222 let deadline = Instant::now() + Duration::from_secs(10);
17223 loop {
17224 if let Ok(contents) = std::fs::read_to_string(path) {
17225 if let Ok(pid) = contents.trim().parse() {
17226 return pid;
17227 }
17228 }
17229 assert!(
17230 Instant::now() < deadline,
17231 "the stub never recorded a grandchild pid at {}",
17232 path.display()
17233 );
17234 std::thread::sleep(Duration::from_millis(10));
17235 }
17236 }
17237
17238 struct Fixture {
17241 _dir: ScratchDir,
17242 module_id: String,
17243 grandchild: u32,
17244 child: Option<SupervisedChild>,
17245 registry: Arc<Registry>,
17246 snapshot: Arc<Mutex<SupervisorSnapshot>>,
17247 terminal_ring: Arc<Mutex<TerminalRing>>,
17248 spawn_events: SpawnEventFeed,
17249 }
17250
17251 fn fixture(label: &str, module_id: &str) -> Fixture {
17252 let dir = ScratchDir::new(label);
17253 let pid_file = dir.join("grandchild.pid");
17254 let supervisor = Supervisor::new_for_test(
17255 Arc::new(Registry::default()),
17256 RestartPolicy::new(3, Duration::ZERO),
17257 );
17258 let runtime = supervisor.runtime_config();
17259 let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::starting()));
17260 let spec = ModuleSpec {
17261 module_id: module_id.to_string(),
17262 program: stub_path(),
17263 args: Vec::new(),
17267 env: vec![
17268 ("FAKE_AFT_NEVER_CONNECT".to_string(), "1".to_string()),
17269 (
17270 "FAKE_AFT_GRANDCHILD_PID_FILE".to_string(),
17271 pid_file.display().to_string(),
17272 ),
17273 ],
17274 reserved: false,
17275 reserved_prefixes: Vec::new(),
17276 protocol: ModuleProtocol::Subc,
17277 overlap: Default::default(),
17278 };
17279 let child = spawn_and_mark_running(&spec, &runtime, &snapshot)
17280 .expect("spawn the supervised fixture");
17281 let grandchild = read_grandchild_pid(&pid_file);
17282 Fixture {
17283 _dir: dir,
17284 module_id: module_id.to_string(),
17285 grandchild,
17286 child: Some(child),
17287 registry: Arc::new(Registry::default()),
17288 snapshot,
17289 terminal_ring: Arc::clone(&runtime.terminal_ring),
17290 spawn_events: SpawnEventFeed::default(),
17291 }
17292 }
17293
17294 impl Fixture {
17295 async fn drain(&mut self) {
17297 let child = self
17298 .child
17299 .take()
17300 .expect("the fixture child is still present");
17301 drain_child_to_state(
17302 &self.module_id,
17303 ModuleProtocol::Subc,
17304 StopNotice::NotSent,
17307 &self.registry,
17308 None,
17309 &self.snapshot,
17310 &self.terminal_ring,
17311 &self.spawn_events,
17312 child,
17313 Duration::from_millis(500),
17314 ModuleState::Stopped,
17315 Some(false),
17316 &FailureContext::operator(),
17317 )
17318 .await
17319 .expect("drain the supervised fixture");
17320 }
17321 }
17322
17323 #[tokio::test]
17329 async fn teardown_reaps_the_grandchild() {
17330 let mut fixture = fixture("teardown-grandchild", "tree-teardown");
17331 let grandchild = fixture.grandchild;
17332
17333 assert!(
17334 subc_jobobject::process_exists(grandchild),
17335 "grandchild {grandchild} must be alive before teardown, or this proves nothing"
17336 );
17337
17338 fixture.drain().await;
17339
17340 assert!(
17341 subc_jobobject::wait_for_process_exit(grandchild, Duration::from_secs(10)),
17342 "grandchild {grandchild} outlived module teardown: the tree was not contained"
17343 );
17344 }
17345
17346 #[test]
17359 fn an_uncontained_grandchild_survives_a_direct_child_kill() {
17360 let dir = ScratchDir::new("teardown-uncontained");
17361 let pid_file = dir.join("grandchild.pid");
17362 let mut child = std::process::Command::new(stub_path())
17363 .env("FAKE_AFT_NEVER_CONNECT", "1")
17364 .env(
17365 "FAKE_AFT_GRANDCHILD_PID_FILE",
17366 pid_file.display().to_string(),
17367 )
17368 .stdin(std::process::Stdio::null())
17369 .stdout(std::process::Stdio::null())
17370 .stderr(std::process::Stdio::null())
17371 .spawn()
17372 .expect("spawn the uncontained fixture");
17373 let grandchild = read_grandchild_pid(&pid_file);
17374
17375 child.kill().expect("kill the direct child");
17377 let _ = child.wait();
17378
17379 assert!(
17380 subc_jobobject::process_exists(grandchild),
17381 "grandchild {grandchild} died with the direct child, so this control no longer \
17382 distinguishes contained from uncontained teardown and the regression test is \
17383 passing vacuously"
17384 );
17385
17386 kill_tree(grandchild);
17389 }
17390
17391 #[tokio::test]
17400 async fn dropping_containment_reaps_the_grandchild() {
17401 let mut fixture = fixture("drop-containment", "tree-drop");
17402 let grandchild = fixture.grandchild;
17403
17404 assert!(subc_jobobject::process_exists(grandchild));
17405
17406 fixture.child.as_mut().expect("child present").job = None;
17408
17409 assert!(
17410 subc_jobobject::wait_for_process_exit(grandchild, Duration::from_secs(10)),
17411 "grandchild {grandchild} survived the containment handle closing, so a daemon \
17412 crash would leave the tree behind"
17413 );
17414 }
17415
17416 fn kill_tree(pid: u32) {
17418 let _ = std::process::Command::new("taskkill.exe")
17419 .args(["/PID", &pid.to_string(), "/T", "/F"])
17420 .stdin(std::process::Stdio::null())
17421 .stdout(std::process::Stdio::null())
17422 .stderr(std::process::Stdio::null())
17423 .status();
17424 assert!(
17425 subc_jobobject::wait_for_process_exit(pid, Duration::from_secs(10)),
17426 "could not clean up grandchild {pid}"
17427 );
17428 }
17429}
17430
17431#[cfg(test)]
17432mod privacy_trampoline_configuration_tests {
17433 #[tokio::test]
17434 #[cfg_attr(not(target_os = "macos"), ignore = "requires macOS privacy trampoline")]
17435 async fn macos_spawn_without_a_configured_trampoline_refuses_by_name() {
17436 #[cfg(target_os = "macos")]
17437 {
17438 let supervisor = super::Supervisor::new(
17439 std::sync::Arc::new(crate::Registry::default()),
17440 super::RestartPolicy::default(),
17441 );
17442 let error = supervisor.spawn(spec()).unwrap_err();
17443 assert!(
17444 error
17445 .to_string()
17446 .contains("no privacy trampoline configured"),
17447 "{error}"
17448 );
17449 }
17450 }
17451
17452 #[tokio::test]
17453 #[cfg_attr(not(target_os = "macos"), ignore = "requires macOS privacy trampoline")]
17454 async fn macos_wrong_trampoline_is_rejected_by_the_startup_probe() {
17455 #[cfg(target_os = "macos")]
17456 {
17457 let supervisor = super::Supervisor::new(
17458 std::sync::Arc::new(crate::Registry::default()),
17459 super::RestartPolicy::default(),
17460 )
17461 .with_privacy_trampoline(std::env::current_exe().unwrap());
17462 let error = supervisor.spawn(spec()).unwrap_err();
17463 assert!(
17464 error
17465 .to_string()
17466 .contains("binary does not implement the privacy trampoline protocol"),
17467 "{error}"
17468 );
17469 }
17470 }
17471
17472 #[cfg(target_os = "macos")]
17473 fn spec() -> super::ModuleSpec {
17474 super::ModuleSpec {
17475 module_id: "privacy-configuration".into(),
17476 program: "/bin/sleep".into(),
17477 args: vec!["30".into()],
17478 env: vec![],
17479 reserved: false,
17480 reserved_prefixes: vec![],
17481 protocol: subc_control::ModuleProtocol::None,
17482 overlap: super::ModuleOverlap::Exclusive,
17483 }
17484 }
17485}
17486
17487#[cfg(test)]
17488mod privacy_exec_boundary_tests {
17489 #[cfg(target_os = "macos")]
17490 use super::*;
17491 #[cfg(target_os = "macos")]
17492 use std::{
17493 io::{Read, Write},
17494 net::{TcpListener, TcpStream},
17495 };
17496
17497 #[cfg(target_os = "macos")]
17500 pub(super) fn before_image_sample(spec: &ModuleSpec, pid: u32) {
17501 if let Some((_, path)) = spec
17502 .env
17503 .iter()
17504 .find(|(key, _)| key == "SUBC_TEST_PRIVACY_SAMPLE_BARRIER")
17505 {
17506 let mut barrier = TcpStream::connect(path).unwrap();
17507 barrier
17508 .set_read_timeout(Some(Duration::from_secs(30)))
17509 .unwrap();
17510 barrier.write_all(&pid.to_ne_bytes()).unwrap();
17511 let mut release = [0];
17512 barrier.read_exact(&mut release).unwrap();
17513 assert_eq!(&release, b"X");
17514 }
17515 }
17516
17517 #[cfg(target_os = "macos")]
17518 fn spec(program: &str, args: &[&str]) -> ModuleSpec {
17519 ModuleSpec {
17520 module_id: "privacy-boundary".into(),
17521 program: program.into(),
17522 args: args.iter().map(|arg| (*arg).into()).collect(),
17523 env: vec![],
17524 reserved: false,
17525 reserved_prefixes: vec![],
17526 protocol: ModuleProtocol::None,
17527 overlap: ModuleOverlap::Exclusive,
17528 }
17529 }
17530
17531 #[cfg(target_os = "macos")]
17532 async fn accept(listener: TcpListener) -> TcpStream {
17533 let listener = tokio::net::TcpListener::from_std({
17535 listener.set_nonblocking(true).unwrap();
17536 listener
17537 })
17538 .unwrap();
17539 let (stream, _) = tokio::time::timeout(Duration::from_secs(30), listener.accept())
17540 .await
17541 .unwrap()
17542 .unwrap();
17543 let stream = stream.into_std().unwrap();
17544 stream.set_nonblocking(false).unwrap();
17545 stream
17546 .set_read_timeout(Some(Duration::from_secs(30)))
17547 .unwrap();
17548 stream
17549 }
17550
17551 #[tokio::test]
17552 #[cfg_attr(not(target_os = "macos"), ignore = "requires macOS privacy trampoline")]
17553 async fn macos_roster_withholds_a_nonnull_trampoline_image_until_exec_confirmation() {
17554 #[cfg(target_os = "macos")]
17555 {
17556 let root = cortexkit_test_support::ScratchDir::new("privacy-roster-barrier");
17557 let exec_listener = TcpListener::bind("127.0.0.1:0").unwrap();
17560 let sample_listener = TcpListener::bind("127.0.0.1:0").unwrap();
17561 let record = root.join("live-children.json");
17562 let supervisor =
17563 Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default())
17564 .with_live_children_record(&record);
17565 let runtime = supervisor.runtime_config();
17566 let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::starting()));
17567 let mut spec = spec("/bin/sleep", &["30"]);
17568 spec.env = vec![
17569 (
17570 "SUBC_TEST_PRIVACY_EXEC_BARRIER".into(),
17571 exec_listener.local_addr().unwrap().to_string(),
17572 ),
17573 (
17574 "SUBC_TEST_PRIVACY_SAMPLE_BARRIER".into(),
17575 sample_listener.local_addr().unwrap().to_string(),
17576 ),
17577 ];
17578 let spawn = tokio::task::spawn_blocking(move || {
17579 spawn_and_mark_running(&spec, &runtime, &snapshot).unwrap()
17580 });
17581 let mut sample = accept(sample_listener).await;
17582 let mut pid = [0; 4];
17583 sample.read_exact(&mut pid).unwrap();
17584 let pid = u32::from_ne_bytes(pid);
17585 let mut exec = accept(exec_listener).await;
17586 let mut ready = [0];
17587 exec.read_exact(&mut ready).unwrap();
17588 assert_eq!(&ready, b"R");
17589 let trampoline = subc_os::file_identity(&test_privacy_trampoline()).unwrap();
17592 assert_eq!(
17593 observe_spawned_image(pid).unwrap().executable,
17594 Some(trampoline)
17595 );
17596 sample.write_all(b"X").unwrap();
17597 let mut child = spawn.await.unwrap();
17598 let early = crate::live_children::read_record(&record).unwrap();
17599 assert_eq!(early.len(), 1);
17600 assert_eq!(early[0].pid, pid);
17601 assert_eq!(
17602 early[0].executable, None,
17603 "unconfirmed trampoline image entered the roster"
17604 );
17605 assert!(child.report_ready.get().is_none());
17606 child.privacy_exec.as_mut().unwrap().deadline =
17609 tokio::time::Instant::now() + Duration::from_secs(30);
17610 exec.write_all(b"X").unwrap();
17611 child.confirm_privacy_exec().await;
17612 assert_eq!(child.spawn_failure, None);
17613 assert!(child.report_ready.get().is_some());
17614 let confirmed = crate::live_children::read_record(&record).unwrap();
17615 let module = subc_os::file_identity(std::path::Path::new("/bin/sleep")).unwrap();
17616 assert_ne!(module, trampoline);
17617 assert_eq!(confirmed[0].executable, Some(module.into()));
17618 child.start_kill().unwrap();
17619 child.wait().await.unwrap();
17620 child.release_roster();
17621 }
17622 }
17623
17624 #[tokio::test]
17625 #[cfg_attr(not(target_os = "macos"), ignore = "requires macOS privacy trampoline")]
17626 async fn macos_already_exited_121_in_try_wait_is_a_module_exit_not_a_trampoline_refusal() {
17627 #[cfg(target_os = "macos")]
17628 {
17629 let registry = Arc::new(Registry::default());
17630 let policy = RestartPolicy::new(0, Duration::ZERO);
17631 let supervisor = Supervisor::new_for_test(Arc::clone(®istry), policy);
17632 let runtime = supervisor.runtime_config();
17633 let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::starting()));
17634 let spec = spec("/bin/sh", &["-c", "exit 121"]);
17635 let mut child = spawn_and_mark_running(&spec, &runtime, &snapshot).unwrap();
17639 let pid = child.pid;
17640 tokio::task::spawn_blocking(move || {
17641 subc_os::fork_exec_test::wait_for_child_exit_without_reaping(pid)
17642 })
17643 .await
17644 .unwrap()
17645 .unwrap();
17646 child.privacy_exec.as_mut().unwrap().deadline =
17647 tokio::time::Instant::now() + Duration::from_secs(30);
17648 let status = child.wait().await.unwrap();
17649 assert_eq!(status.code(), Some(121));
17650 assert!(child.privacy_exec.is_none());
17651 assert!(
17652 child.report_ready.get().is_none(),
17653 "an exited module must not publish a live pid"
17654 );
17655 let report = classify_reaped_child_exit(&snapshot, &child, &status);
17656 on_child_exit(
17657 &spec,
17658 policy,
17659 ®istry,
17660 &snapshot,
17661 &runtime.terminal_ring,
17662 &runtime.spawn_events,
17663 &runtime.child_roster,
17664 report,
17665 true,
17666 )
17667 .await;
17668 let state = lock_snapshot(&snapshot).unwrap();
17669 assert_eq!(state.state, ModuleState::Failed);
17670 assert_eq!(state.last_exit.as_ref().unwrap().code, Some(121));
17671 assert_eq!(state.reported_pid(), None);
17672 drop(state);
17673 let history = runtime.terminal_ring.lock().unwrap().snapshot();
17674 assert_eq!(history.entries.len(), 1);
17675 let terminal = &history.entries[0];
17676 assert_eq!(terminal.exit_code, Some(121));
17677 assert_eq!(terminal.exit_kind, subc_control::TerminalExitKind::Crash);
17678 assert_eq!(terminal.disposition, TerminalDisposition::Failed);
17679 assert_eq!(
17680 terminal.disposition_detail.as_deref(),
17681 Some(policy.budget_exhausted_detail().as_str()),
17682 "module exit 121 was classified as a trampoline refusal: {terminal:?}"
17683 );
17684 assert_eq!(child.spawn_failure, None);
17685 child.release_roster();
17686 }
17687 }
17688}
17689
17690#[cfg(all(test, unix))]
17694mod launch_nonce_descriptor_tests {
17695 use super::{ChildRoster, ModuleProtocol, ModuleSpec, SupervisorHandle};
17696 use crate::stderr_tail::{StderrRing, StderrTailConfig};
17697 use cortexkit_test_support::ScratchDir;
17698 use std::{
17699 path::PathBuf,
17700 sync::{Arc, Mutex},
17701 time::{Duration, Instant},
17702 };
17703
17704 async fn probe(role: super::SpawnRole) {
17705 let scratch = ScratchDir::new("launch-nonce-descriptor");
17706 let fd_copy = scratch.join("from-descriptor");
17707 let env_copy = scratch.join("environment");
17708 let script = format!(
17709 "env > '{env}.tmp' && mv '{env}.tmp' '{env}'; cat <&3 > '{fd}.tmp' && mv '{fd}.tmp' '{fd}'; sleep 30",
17710 fd = fd_copy.display(), env = env_copy.display(),
17711 );
17712 let xdg = |name: &str| (name.to_string(), scratch.join(name).display().to_string());
17713 let spec = ModuleSpec {
17714 module_id: "nonce-descriptor-probe".to_string(),
17715 program: PathBuf::from("/bin/sh"),
17716 args: vec!["-c".to_string(), script],
17717 env: vec![
17718 xdg("XDG_DATA_HOME"),
17719 xdg("XDG_RUNTIME_DIR"),
17720 xdg("XDG_CONFIG_HOME"),
17721 ("SUBC_LAUNCH_NONCE".to_string(), "must-not-leak".to_string()),
17722 ],
17723 reserved: true,
17724 reserved_prefixes: Vec::new(),
17725 protocol: ModuleProtocol::Subc,
17726 overlap: Default::default(),
17727 };
17728 let handle = SupervisorHandle::new();
17729 let ring = Arc::new(Mutex::new(StderrRing::new(StderrTailConfig::default())));
17730 let roster = ChildRoster::default();
17731 #[cfg(target_os = "macos")]
17732 {
17733 let path = super::test_privacy_trampoline();
17734 roster.set_privacy_trampoline(super::probe_privacy_trampoline(&path).map(|()| path));
17735 }
17736 let child = super::spawn_child_in_slot(
17737 &spec,
17738 None,
17739 Some(&handle),
17740 &ring,
17741 None,
17742 &roster,
17743 #[cfg(target_os = "linux")]
17744 None,
17745 role,
17746 matches!(role, super::SpawnRole::SwapCandidate),
17747 )
17748 .expect("spawn probe");
17749 let deadline = Instant::now() + Duration::from_secs(10);
17750 while !(fd_copy.exists() && env_copy.exists()) {
17751 assert!(Instant::now() < deadline, "probe never wrote its copies");
17752 tokio::time::sleep(Duration::from_millis(20)).await;
17753 }
17754 let nonce = std::fs::read_to_string(fd_copy).unwrap();
17755 assert!(!nonce.is_empty());
17756 let environment = std::fs::read_to_string(env_copy).unwrap();
17757 assert!(environment
17758 .lines()
17759 .any(|line| line.starts_with("SUBC_LAUNCH_NONCE_FD=3:")));
17760 let copy = environment
17761 .lines()
17762 .find_map(|line| line.strip_prefix("SUBC_LAUNCH_NONCE="));
17763 assert_eq!(
17764 copy, None,
17765 "Unix children must never receive the environment nonce"
17766 );
17767 if matches!(role, super::SpawnRole::Plain) {
17768 assert_eq!(
17769 handle.spawn_nonce(&spec.module_id).as_deref(),
17770 Some(nonce.as_str())
17771 );
17772 }
17773 drop(child);
17774 }
17775
17776 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
17777 async fn a_spawned_module_receives_its_nonce_only_on_descriptor_3() {
17778 probe(super::SpawnRole::Plain).await;
17779 }
17780
17781 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
17782 async fn swap_candidate_receives_its_nonce_only_on_descriptor_3() {
17783 probe(super::SpawnRole::SwapCandidate).await;
17784 }
17785}
17786
17787#[cfg(all(test, target_os = "linux"))]
17788#[path = "supervise/cgroup_reap_tests.rs"]
17789mod cgroup_reap_tests;
17790
17791#[cfg(all(test, target_os = "linux"))]
17792mod cgroup_containment_tests {
17793 use super::*;
17794 use cortexkit_test_support::ScratchDir;
17795
17796 #[tokio::test]
17797 async fn linux_subtree_memory_covers_module_children() {
17798 let dir = ScratchDir::new("subtree-memory");
17799 let root = PathBuf::from(format!(
17800 "/sys/fs/cgroup/subc-memory-test-{}-{}",
17801 std::process::id(),
17802 unix_ms_now()
17803 ));
17804 if let Err(error) = std::fs::create_dir(&root) {
17805 assert!(
17806 std::env::var_os("SUBC_REQUIRE_CGROUP_TEST").is_none(),
17807 "required cgroup test cannot execute: {error}"
17808 );
17809 eprintln!("SKIP linux_subtree_memory_covers_module_children: {error}");
17810 return;
17811 }
17812 let placement = subc_cgroup::prepare_at(&root)
17813 .unwrap()
17814 .expect("delegated cgroup");
17815 let supervisor = Supervisor::new_for_test(
17816 Arc::new(Registry::default()),
17817 RestartPolicy::new(3, Duration::ZERO),
17818 )
17819 .with_cgroup_placement(Some(placement));
17820 let ready = dir.join("ready");
17821 let module = supervisor
17822 .spawn(ModuleSpec {
17823 module_id: "memory-tree".into(),
17824 program: PathBuf::from("/bin/sh"),
17825 args: vec![
17826 "-c".into(),
17827 "sleep 600 & sleep 600 & sleep 600 & echo ready > \"$1\"; wait".into(),
17828 "fixture".into(),
17829 ready.display().to_string(),
17830 ],
17831 env: ["XDG_DATA_HOME", "XDG_RUNTIME_DIR", "XDG_CONFIG_HOME"]
17832 .into_iter()
17833 .map(|key| (key.into(), dir.display().to_string()))
17834 .collect(),
17835 reserved: false,
17836 reserved_prefixes: Vec::new(),
17837 protocol: ModuleProtocol::None,
17838 overlap: Default::default(),
17839 })
17840 .unwrap();
17841 let deadline = tokio::time::Instant::now() + Duration::from_secs(10);
17842 while !ready.exists() {
17843 assert!(
17844 tokio::time::Instant::now() < deadline,
17845 "children never started"
17846 );
17847 tokio::time::sleep(Duration::from_millis(10)).await;
17848 }
17849 let readings = crate::subtree_resources::read(vec![module.subtree_target()]).await;
17850 let reading = &readings[0];
17851 assert!(
17852 reading.process_count.is_some_and(|count| count >= 4),
17853 "{reading:?}"
17854 );
17855 assert!(
17856 reading.memory_bytes.is_some_and(|bytes| bytes > 0),
17857 "{reading:?}"
17858 );
17859 module.stop().await.unwrap();
17860 drop(module);
17861 std::fs::remove_dir(root.join("subc-modules")).unwrap();
17862 std::fs::remove_dir(&root).unwrap();
17863 eprintln!("EXECUTED linux_subtree_memory_covers_module_children: {reading:?}");
17864 }
17865
17866 fn running(pid: u32) -> bool {
17867 std::fs::read_to_string(format!("/proc/{pid}/stat"))
17869 .ok()
17870 .and_then(|stat| {
17871 stat.rsplit_once(") ")
17872 .map(|(_, rest)| rest.starts_with('Z'))
17873 })
17874 .is_some_and(|zombie| !zombie)
17875 }
17876
17877 #[tokio::test]
17878 async fn linux_teardown_reaps_the_grandchild() {
17879 teardown_tree("linux_teardown_reaps_the_grandchild", false).await;
17880 }
17881
17882 #[tokio::test]
17883 async fn linux_shutdown_straggler_reaps_the_grandchild() {
17884 teardown_tree("linux_shutdown_straggler_reaps_the_grandchild", true).await;
17885 }
17886
17887 async fn teardown_tree(test_name: &str, shutdown: bool) {
17888 let dir = ScratchDir::new(test_name);
17889 let root = PathBuf::from(format!(
17890 "/sys/fs/cgroup/subc-tree-test-{test_name}-{}-{}",
17891 std::process::id(),
17892 unix_ms_now()
17893 ));
17894 if let Err(error) = std::fs::create_dir(&root) {
17895 assert!(
17896 std::env::var_os("SUBC_REQUIRE_CGROUP_TEST").is_none(),
17897 "required cgroup test cannot execute: {error}"
17898 );
17899 eprintln!(
17900 "SKIP {test_name}: no writable delegated cgroup at {}: {error}",
17901 root.display()
17902 );
17903 return;
17904 }
17905 let placement = subc_cgroup::prepare_at(&root)
17906 .expect("prepare isolated kernel cgroup")
17907 .expect("isolated cgroup is delegated");
17908 let module_id = "tree-teardown";
17909 let module = placement
17910 .module_path(module_id)
17911 .expect("create isolated module cgroup");
17912 if !module.join("cgroup.kill").exists() {
17913 std::fs::remove_dir(&module).unwrap();
17914 std::fs::remove_dir(root.join("subc-modules")).unwrap();
17915 std::fs::remove_dir(&root).unwrap();
17916 assert!(
17917 std::env::var_os("SUBC_REQUIRE_CGROUP_TEST").is_none(),
17918 "required cgroup.kill interface unavailable"
17919 );
17920 eprintln!("SKIP {test_name}: cgroup.kill unavailable (kernel < 5.14)");
17921 return;
17922 }
17923 let supervisor = Supervisor::new_for_test(
17924 Arc::new(Registry::default()),
17925 RestartPolicy::new(3, Duration::ZERO),
17926 )
17927 .with_cgroup_placement(Some(placement));
17928 let mut runtime = supervisor.runtime_config();
17929 runtime.child_roster = runtime
17930 .child_roster
17931 .for_module(Arc::new(Mutex::new(Duration::from_millis(100))));
17932 let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::starting()));
17933 let pid_file = dir.join("grandchild.pid");
17934 let spec = ModuleSpec {
17935 module_id: module_id.to_string(),
17936 program: PathBuf::from("/bin/sh"),
17937 args: vec![
17938 "-c".into(),
17939 "trap '' TERM; sleep 600 & echo $! > \"$1\"; wait".into(),
17940 "fixture".into(),
17941 pid_file.display().to_string(),
17942 ],
17943 env: ["XDG_DATA_HOME", "XDG_RUNTIME_DIR", "XDG_CONFIG_HOME"]
17944 .into_iter()
17945 .map(|key| (key.to_string(), dir.display().to_string()))
17946 .collect(),
17947 reserved: false,
17948 reserved_prefixes: Vec::new(),
17949 protocol: ModuleProtocol::None,
17950 overlap: Default::default(),
17951 };
17952 let child =
17953 spawn_and_mark_running(&spec, &runtime, &snapshot).expect("spawn supervised tree");
17954 let deadline = tokio::time::Instant::now() + Duration::from_secs(10);
17955 let grandchild: u32 = loop {
17956 if let Ok(contents) = std::fs::read_to_string(&pid_file) {
17957 if let Ok(pid) = contents.trim().parse() {
17958 break pid;
17959 }
17960 }
17961 assert!(
17962 tokio::time::Instant::now() < deadline,
17963 "grandchild pid was not recorded"
17964 );
17965 tokio::time::sleep(Duration::from_millis(10)).await;
17966 };
17967 assert!(
17968 running(grandchild),
17969 "grandchild must be alive before teardown"
17970 );
17971 if shutdown {
17972 let mut child = child;
17973 crate::child_roster::end_children_for_daemon_shutdown(
17974 &runtime.child_roster,
17975 false,
17976 std::future::pending(),
17977 )
17978 .await;
17979 child.wait().await.expect("reap shutdown straggler");
17980 } else {
17981 drain_child_to_state(
17982 module_id,
17983 ModuleProtocol::None,
17984 StopNotice::NotSent,
17985 &Registry::default(),
17986 None,
17987 &snapshot,
17988 &runtime.terminal_ring,
17989 &SpawnEventFeed::default(),
17990 child,
17991 Duration::from_millis(100),
17992 ModuleState::Stopped,
17993 Some(false),
17994 &FailureContext::operator(),
17995 )
17996 .await
17997 .expect("real supervisor teardown");
17998 }
17999 let deadline = tokio::time::Instant::now() + Duration::from_secs(5);
18000 while running(grandchild) && tokio::time::Instant::now() < deadline {
18001 tokio::time::sleep(Duration::from_millis(10)).await;
18002 }
18003 let survived = running(grandchild);
18004 if survived {
18006 let pid = rustix::process::Pid::from_raw(grandchild as i32).unwrap();
18007 let _ = rustix::process::kill_process(pid, rustix::process::Signal::KILL);
18008 tokio::time::sleep(Duration::from_millis(100)).await;
18009 }
18010 if module.exists() {
18011 std::fs::remove_dir(&module).expect("remove empty module cgroup");
18012 }
18013 std::fs::remove_dir(root.join("subc-modules")).unwrap();
18014 std::fs::remove_dir(&root).unwrap();
18015 assert!(
18016 !survived,
18017 "grandchild {grandchild} outlived module teardown"
18018 );
18019 eprintln!("EXECUTED {test_name}: grandchild {grandchild} killed in isolated cgroup");
18020 }
18021}
18022
18023#[cfg(all(test, unix))]
18026mod record_write_stall_tests {
18027 use super::*;
18028 use std::sync::atomic::{AtomicBool, Ordering};
18029 use subc_transport::{authenticate_client, ConnectionInfo, Endpoint, SCHEMA_VERSION};
18030
18031 const MODULE_AUTH_DEADLINE: Duration = Duration::from_secs(2);
18034 const HELD_RECORD_WRITE: Duration = Duration::from_secs(3);
18038
18039 fn sleeping_spec() -> ModuleSpec {
18051 ModuleSpec {
18052 module_id: "record-write-stall".into(),
18053 program: "/bin/sleep".into(),
18054 args: vec!["600".into()],
18055 env: vec![],
18056 reserved: false,
18057 reserved_prefixes: vec![],
18058 protocol: ModuleProtocol::None,
18059 overlap: Default::default(),
18060 }
18061 }
18062
18063 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
18064 async fn module_auth_completes_while_the_restart_waits_on_the_record_write() {
18065 let home = cortexkit_test_support::ScratchDir::new("record-write-stall");
18066 let supervisor =
18067 Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default())
18068 .with_live_children_record(home.join("live-children.json"));
18069 let module = supervisor.spawn(sleeping_spec()).unwrap();
18070 let old_pid = tokio::time::timeout(Duration::from_secs(10), async {
18075 loop {
18076 if let Some(pid) = lock_snapshot(&module.inner.snapshot)
18077 .unwrap()
18078 .reported_pid()
18079 {
18080 return pid;
18081 }
18082 tokio::time::sleep(Duration::from_millis(10)).await;
18083 }
18084 })
18085 .await
18086 .expect("the first process was admitted as the module");
18087
18088 let key = vec![0x42; 32];
18091 let daemon_id = [0x24; 16];
18092 let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
18093 let port = listener.local_addr().unwrap().port();
18094 let server = tokio::spawn(crate::server::serve_listener(
18095 listener,
18096 Arc::new(crate::router::Router::with_default_self_handler()),
18097 crate::server::ServerAuth::with_limits(
18098 key.clone(),
18099 daemon_id,
18100 "test",
18101 crate::server::DEFAULT_AUTH_DEADLINE,
18102 4,
18103 ),
18104 ));
18105
18106 let (held_tx, held_rx) = std::sync::mpsc::channel();
18109 let armed = AtomicBool::new(true);
18110 supervisor
18111 .child_roster
18112 .set_record_write_hook_for_test(move |live| {
18113 let admitting_replacement = live.values().any(|entry| entry.pid != old_pid);
18114 if admitting_replacement && armed.swap(false, Ordering::SeqCst) {
18115 let _ = held_tx.send(());
18116 std::thread::sleep(HELD_RECORD_WRITE);
18117 }
18118 });
18119
18120 let module_process = std::thread::spawn(move || {
18123 held_rx
18124 .recv_timeout(Duration::from_secs(30))
18125 .expect("the restart reached the replacement's record write");
18126 let conn = ConnectionInfo {
18127 schema: SCHEMA_VERSION,
18128 wire_version: None,
18129 endpoints: vec![Endpoint {
18130 host: "127.0.0.1".to_owned(),
18131 port,
18132 }],
18133 key,
18134 daemon_id,
18135 pid: std::process::id(),
18136 daemon_ver: "test".to_owned(),
18137 };
18138 tokio::runtime::Builder::new_current_thread()
18139 .enable_all()
18140 .build()
18141 .unwrap()
18142 .block_on(async {
18143 let started = Instant::now();
18144 let mut stream = tokio::net::TcpStream::connect(("127.0.0.1", port))
18145 .await
18146 .unwrap();
18147 let result =
18148 authenticate_client(&mut stream, &conn, MODULE_AUTH_DEADLINE).await;
18149 (result, started.elapsed())
18150 })
18151 });
18152
18153 module.restart(Some(0)).await.unwrap();
18154 let (auth, elapsed) = tokio::task::spawn_blocking(move || module_process.join().unwrap())
18155 .await
18156 .unwrap();
18157 assert!(
18158 auth.is_ok(),
18159 "the daemon must answer a module's handshake while a restart waits on the \
18160 live-children record write; got {auth:?} after {elapsed:?}"
18161 );
18162
18163 tokio::time::timeout(Duration::from_secs(20), async {
18164 while lock_snapshot(&module.inner.snapshot)
18165 .unwrap()
18166 .reported_pid()
18167 .is_none_or(|pid| pid == old_pid)
18168 {
18169 tokio::time::sleep(Duration::from_millis(10)).await;
18170 }
18171 })
18172 .await
18173 .expect("the restart published its replacement");
18174 module.stop().await.unwrap();
18175 server.abort();
18176 }
18177}