1use std::{
2 collections::{HashMap, HashSet, VecDeque},
3 error::Error,
4 fmt, io,
5 path::PathBuf,
6 process::{ExitStatus, Stdio},
7 sync::{Arc, Mutex, OnceLock},
8 time::{Duration, SystemTime, UNIX_EPOCH},
9};
10
11use cortexkit_log::Retention;
12use serde_json::Value;
13use sha2::{Digest, Sha256};
14use subc_control::{
15 ClientControlPush, LiveSpawn, ModuleProtocol, RouteCloseReason, SpawnCursor, SpawnEvent,
16 SpawnEventKind, SpawnSnapshot, SupervisorHealthStatus, TerminalDisposition, TerminalExitKind,
17};
18use subc_protocol::{
19 manifest::{SelfSignalKind, SignalAnchor},
20 session::{
21 HealthReport, HealthStatus, ModuleControlCommand, ModuleControlRequest,
22 MODULE_CONTROL_OP_HEALTH_CHECK,
23 },
24 Flags, FrameType, Principal, Priority, SUBC_LAUNCH_NONCE_ENV, SUBC_MODULE_ID_ENV,
25};
26use tokio::{
27 process::{Child, Command},
28 sync::{mpsc, oneshot, watch, Mutex as AsyncMutex},
29 task::JoinHandle,
30 time::{sleep, sleep_until, timeout, timeout_at, Instant},
31};
32use tracing::{debug, error, info, warn};
33
34use crate::{
35 child_roster::ChildRoster,
36 daemon_config::{
37 CAPTURE_KEEP_ENV, CAPTURE_MAX_AGE_DAYS_ENV, CAPTURE_MAX_FILE_MB_ENV, CK_LOG_ENV,
38 },
39 forwarding::{
40 CloseReason, ForwardingError, ForwardingTable, GoodbyeTarget, ModuleControlRpcOutcome,
41 ModuleDrainTarget, PendingModuleControlRpc,
42 },
43 provenance::{ExecutableIdentityProbe, SpawnedFileIdentity},
44 registry::{ConnectionId, RegistrationEndReason, RegistryError},
45 stderr_tail::{
46 pump_stderr_to, pump_stdout_to, ChildOutputSink, StderrRing, StderrTailConfig,
47 StderrTailSnapshot,
48 },
49 terminal_ring::{TerminalHistorySnapshot, TerminalRecord, TerminalRing, TerminalRingConfig},
50 Frame, FrameSink, Registry,
51};
52
53#[cfg(not(windows))]
54use crate::provenance::spawned_file_identity;
55
56#[path = "supervise_swap.rs"]
57mod swap;
58
59#[path = "supervise_activity.rs"]
60mod activity;
61
62pub const SUBC_ARG: &str = "--subc";
68
69const DEFAULT_MAX_RESTARTS: u32 = 3;
70const DEFAULT_BACKOFF: Duration = Duration::from_millis(100);
71const DEFAULT_MAX_BACKOFF: Duration = Duration::from_secs(30);
72const DEFAULT_RESTART_WINDOW: Duration = Duration::from_secs(600);
76pub const DEFAULT_DRAIN_TIMEOUT: Duration = Duration::from_secs(30);
87const REGISTRY_RELEASE_TIMEOUT: Duration = Duration::from_secs(1);
88const REGISTRY_RELEASE_POLL: Duration = Duration::from_millis(10);
89const DRAIN_BUSY_REPROBE_INTERVAL: Duration = Duration::from_millis(250);
101const STDERR_PUMP_DRAIN_TIMEOUT: Duration = Duration::from_millis(250);
124pub const SPAWN_EVENT_RING_CAPACITY: usize = 4096;
126const SPAWN_SUBSCRIBER_BUFFER: usize = SPAWN_EVENT_RING_CAPACITY + 1;
127pub(crate) const SPAWN_SUBSCRIBER_LAGGED_CODE: &str = "spawn_subscriber_lagged";
132
133#[cfg(target_os = "macos")]
134struct PrivacyExec {
135 reader: tokio::io::unix::AsyncFd<std::io::PipeReader>,
136 deadline: tokio::time::Instant,
137 expected: Option<subc_os::FileIdentity>,
138 trampoline: Option<subc_os::FileIdentity>,
139 script: bool,
140 module_id: String,
141 #[cfg(test)]
142 confirmation_pause: Option<(String, Option<tokio::net::TcpStream>)>,
143}
144
145#[cfg(all(test, target_os = "macos"))]
146pub(crate) fn test_privacy_trampoline() -> PathBuf {
147 let path = std::env::current_exe()
150 .unwrap()
151 .parent()
152 .unwrap()
153 .parent()
154 .unwrap()
155 .join("privacy-trampoline-fixture");
156 assert!(
160 path.exists(),
161 "privacy-trampoline-fixture not built at {}: run `cargo build -p subc-daemon \
162 --bins --features test-support` or `cargo test -p subc-daemon` first",
163 path.display()
164 );
165 path
166}
167
168#[cfg(target_os = "macos")]
169fn probe_privacy_trampoline(path: &std::path::Path) -> Result<(), String> {
170 use std::io::Read;
171 let mut probe = std::process::Command::new(path)
172 .args(["__disclaim-exec", "--probe"])
173 .stdin(Stdio::null())
174 .stdout(Stdio::piped())
175 .stderr(Stdio::piped())
176 .spawn()
177 .map_err(|error| {
178 format!(
179 "privacy trampoline probe failed for {}: {error}",
180 path.display()
181 )
182 })?;
183 let deadline = std::time::Instant::now() + Duration::from_secs(5);
184 let status = loop {
185 match probe.try_wait() {
186 Ok(Some(status)) => break status,
187 Ok(None) if std::time::Instant::now() < deadline => {
188 std::thread::sleep(Duration::from_millis(5))
189 }
190 result => {
191 let _ = probe.kill();
192 let _ = probe.wait();
193 return Err(format!(
194 "privacy trampoline probe failed or timed out for {}: {result:?}",
195 path.display()
196 ));
197 }
198 }
199 };
200 let mut answer = String::new();
201 if let Some(stdout) = probe.stdout.take() {
202 let _ = stdout.take(256).read_to_string(&mut answer);
203 }
204 if status.success() && answer.trim() == subc_os::privacy_identity::TRAMPOLINE_PROBE {
205 return Ok(());
206 }
207 let mut diagnostic = String::new();
208 if let Some(stderr) = probe.stderr.take() {
209 let _ = stderr.take(1024).read_to_string(&mut diagnostic);
210 }
211 let cause = diagnostic
212 .trim()
213 .strip_prefix("ck-subc: own privacy identity refused: ")
214 .unwrap_or("binary does not implement the privacy trampoline protocol");
215 Err(format!(
216 "{cause}: probe of {} exited {status}",
217 path.display()
218 ))
219}
220
221#[cfg(target_os = "macos")]
222fn privacy_command(
223 spec: &ModuleSpec,
224 roster: &ChildRoster,
225) -> Result<
226 (
227 Command,
228 Option<PrivacyExec>,
229 subc_os::privacy_identity::ExecAcknowledgement,
230 ),
231 SuperviseError,
232> {
233 let failure = |cause: String| {
234 warn!(module_id = %spec.module_id, %cause, "privacy identity trampoline refused module spawn");
235 SuperviseError::Spawn {
236 program: spec.program.clone(),
237 source: io::Error::other(cause),
238 cgroup_path: None,
239 }
240 };
241 let trampoline = roster.privacy_trampoline().map_err(failure)?;
242 let program = if spec.program.components().count() == 1 && !spec.program.is_absolute() {
247 let path = spec
248 .env
249 .iter()
250 .find(|(key, _)| key == "PATH")
251 .map(|(_, value)| std::ffi::OsString::from(value))
252 .or_else(|| std::env::var_os("PATH"))
253 .unwrap_or_else(|| "/usr/bin:/bin".into());
254 std::env::split_paths(&path)
255 .map(|dir| dir.join(&spec.program))
256 .find(|path| path.is_file())
257 .unwrap_or_else(|| spec.program.clone())
258 } else {
259 spec.program.clone()
260 };
261 let expected = subc_os::file_identity(&program);
262 let trampoline_image = subc_os::file_identity(&trampoline);
263 let script = {
264 use std::io::Read;
265 let mut prefix = [0u8; 2];
266 std::fs::File::open(&program)
267 .is_ok_and(|mut file| file.read_exact(&mut prefix).is_ok() && &prefix == b"#!")
268 };
269 if expected.is_none() || expected == subc_os::file_identity(&trampoline) {
270 return Err(failure(
271 "privacy identity module executable is missing or is the trampoline itself".to_string(),
272 ));
273 }
274 let (reader, ack) = subc_os::privacy_identity::ExecAcknowledgement::pipe()
275 .map_err(|error| failure(error.to_string()))?;
276 let reader =
277 tokio::io::unix::AsyncFd::new(reader).map_err(|error| failure(error.to_string()))?;
278 let mut command = Command::new(&trampoline);
279 command
280 .arg("__disclaim-exec")
281 .arg(ack.fd().to_string())
282 .arg(&program);
283 ack.install(command.as_std_mut());
284 Ok((
285 command,
286 Some(PrivacyExec {
287 reader,
288 deadline: tokio::time::Instant::now() + Duration::from_secs(5),
289 expected,
290 trampoline: trampoline_image,
291 script,
292 module_id: spec.module_id.clone(),
293 #[cfg(test)]
294 confirmation_pause: spec
295 .env
296 .iter()
297 .find(|(key, _)| key == "SUBC_TEST_BEFORE_EXEC_CONFIRM")
298 .map(|(_, address)| (address.clone(), None)),
299 }),
300 ack,
301 ))
302}
303
304struct SupervisedChild {
305 child: Child,
306 exec: crate::activity::ExecOwner,
307 #[cfg(windows)]
308 nonce_delivery: Option<subc_os::LaunchNoncePipeDelivery>,
309 #[cfg(target_os = "macos")]
310 privacy_exec: Option<PrivacyExec>,
311 #[cfg(target_os = "macos")]
318 report_ready: Arc<OnceLock<()>>,
319 spawn_failure: Option<String>,
321 protocol: ModuleProtocol,
326 #[cfg(target_os = "linux")]
332 module_id: String,
333 #[cfg(target_os = "linux")]
334 cgroup_placement: Option<subc_cgroup::Placement>,
335 #[cfg(windows)]
357 job: Option<subc_jobobject::JobObject>,
358 stdout_pump: Option<JoinHandle<()>>,
359 stderr_pump: Option<StderrPump>,
360 stderr_ring: Arc<Mutex<StderrRing>>,
361 spawned_at: Instant,
364 spawned_at_ms: u64,
365 spawned_from: PathBuf,
366 launch_spec_digest: LaunchSpecDigest,
367 spawned_file_identity: Option<SpawnedFileIdentity>,
368 process_start_time: Option<u64>,
369 process_identity: Option<ProcessIdentity>,
370 pid: u32,
371 roster_guard: Option<crate::child_roster::RosterGuard>,
374}
375
376impl SupervisedChild {
377 fn id(&self) -> Option<u32> {
378 Some(self.pid)
379 }
380
381 fn process_identity(&self) -> Option<ProcessIdentity> {
382 self.process_identity
383 }
384
385 async fn wait(&mut self) -> io::Result<ExitStatus> {
386 #[cfg(target_os = "macos")]
387 self.confirm_privacy_exec().await;
388 let result = self.child.wait().await;
396 #[cfg(target_os = "linux")]
397 if result.is_ok() {
398 if let Some(placement) = self.cgroup_placement.as_ref() {
399 cleanup_reaped_cgroup(placement, &self.module_id).await;
400 self.cgroup_placement = None;
403 }
404 }
405 result
406 }
407
408 #[cfg(target_os = "macos")]
409 async fn confirm_privacy_exec(&mut self) {
410 let Some(pending) = &mut self.privacy_exec else {
411 return;
412 };
413 let result = tokio::time::timeout_at(pending.deadline, async {
414 let mut record = Vec::new();
415 loop {
416 let mut ready = pending.reader.readable().await?;
417 let read = ready.try_io(|reader| {
418 use std::io::Read;
419 let mut reader = reader.get_ref();
420 let mut buffer = [0u8; 256];
421 reader.read(&mut buffer).map(|count| (count, buffer))
422 });
423 match read {
424 Ok(Ok((0, _))) => return Ok::<_, io::Error>(record),
425 Ok(Ok((count, buffer))) => {
426 if record.len() + count > 1024 {
427 return Err(io::Error::other(
428 "privacy exec refusal record is too long",
429 ));
430 }
431 record.extend_from_slice(&buffer[..count]);
432 }
433 Ok(Err(error)) => return Err(error),
434 Err(_) => continue,
435 }
436 }
437 })
438 .await;
439 #[cfg(test)]
440 let result = if let Some((address, stream)) = &mut pending.confirmation_pause {
441 use tokio::io::{AsyncReadExt, AsyncWriteExt};
442 if stream.is_none() {
443 *stream = Some(
444 tokio::net::TcpStream::connect(address.as_str())
445 .await
446 .unwrap(),
447 );
448 stream.as_mut().unwrap().write_all(b"R").await.unwrap();
449 stream
450 .as_mut()
451 .unwrap()
452 .write_all(&self.pid.to_ne_bytes())
453 .await
454 .unwrap();
455 }
456 let mut reply = [0];
457 stream
458 .as_mut()
459 .unwrap()
460 .read_exact(&mut reply)
461 .await
462 .unwrap();
463 pending.confirmation_pause = None;
464 if &reply == b"F" {
465 Ok(Err(io::Error::other("injected late exec refusal")))
466 } else {
467 assert_eq!(&reply, b"X");
468 result
469 }
470 } else {
471 result
472 };
473 let pending = self.privacy_exec.as_ref().expect("pending exec");
476 let cause = match result {
477 Err(_) => Some("privacy identity trampoline did not exec within 5s".to_string()),
478 Ok(Err(error)) => Some(format!(
479 "privacy identity exec acknowledgement failed: {error}"
480 )),
481 Ok(Ok(record)) if !record.is_empty() => Some(
482 std::str::from_utf8(&record)
483 .ok()
484 .and_then(|record| {
485 record
486 .trim()
487 .strip_prefix(subc_os::privacy_identity::EXEC_REFUSAL_TAG)
488 })
489 .filter(|cause| !cause.is_empty())
490 .unwrap_or("invalid privacy exec refusal record")
491 .to_string(),
492 ),
493 Ok(Ok(_)) => match self.child.try_wait() {
494 Ok(Some(_status)) => None,
498 Err(error) => Some(format!("privacy identity trampoline wait failed: {error}")),
499 Ok(None) => {
500 let image = observe_spawned_image(self.pid);
501 if let Some(image) = image.filter(|image| {
502 image.executable.is_some()
503 && image.executable != pending.trampoline
504 && (image.executable == pending.expected || pending.script)
505 }) {
506 if let Err(error) = self
507 .roster_guard
508 .as_ref()
509 .map(|guard| guard.confirm_executable(image))
510 .transpose()
511 {
512 Some(format!(
513 "could not durably record confirmed module image: {error}"
514 ))
515 } else {
516 let _ = self.report_ready.set(());
517 info!(module_id = %pending.module_id, pid = self.pid,
518 "module spawned with own privacy identity (responsibility disclaimed)");
519 None
520 }
521 } else if image.is_none()
522 || image.is_some_and(|image| image.executable.is_none())
523 {
524 match tokio::time::timeout_at(pending.deadline, self.child.wait()).await {
531 Ok(Ok(_status)) => None,
532 Ok(Err(error)) => Some(format!("privacy identity trampoline wait failed: {error}")),
533 Err(_) => Some("privacy identity module executable remained unreadable until the 5s exec deadline".to_string()),
534 }
535 } else {
536 Some("privacy identity trampoline executable mismatch: expected the module program, not ck-subc".to_string())
537 }
538 }
539 },
540 };
541 let pending = self.privacy_exec.take().expect("pending exec");
542 self.exec.0.finish(if cause.is_some() {
545 crate::activity::ExecState::Refused
546 } else {
547 crate::activity::ExecState::CommittedExec
548 });
549 if let Some(cause) = cause {
550 warn!(module_id = %pending.module_id, pid = self.pid, %cause, "privacy identity trampoline refused module spawn");
551 self.spawn_failure = Some(cause);
552 if let Some(pid) = rustix::process::Pid::from_raw(self.pid as i32) {
555 let _ = rustix::process::kill_process_group(pid, rustix::process::Signal::KILL);
556 }
557 let _ = self.child.start_kill();
558 }
559 }
560
561 fn release_roster(&mut self) {
565 self.roster_guard = None;
566 }
567
568 fn start_kill(&mut self) -> io::Result<()> {
581 #[cfg(windows)]
582 if let Some(job) = &self.job {
583 if let Err(error) = job.terminate() {
584 debug!(
585 error = %error,
586 "job termination failed; the direct-child kill still owns the outcome"
587 );
588 }
589 }
590 #[cfg(target_os = "linux")]
591 kill_module_cgroup(self.cgroup_placement.as_ref(), &self.module_id);
592 self.child.start_kill()
593 }
594
595 async fn drain_stderr(&mut self, module_id: &str) {
596 if let Some(pump) = self.stdout_pump.take() {
597 settle_stdout_pump(module_id, pump, STDERR_PUMP_DRAIN_TIMEOUT).await;
598 }
599
600 let Some(pump) = self.stderr_pump.take() else {
601 return;
602 };
603 settle_stderr_pump(
604 module_id,
605 &self.stderr_ring,
606 pump,
607 STDERR_PUMP_DRAIN_TIMEOUT,
608 )
609 .await;
610 }
611}
612
613async fn settle_stdout_pump(module_id: &str, mut pump: JoinHandle<()>, bound: Duration) {
617 match timeout(bound, &mut pump).await {
618 Ok(Ok(())) => {}
619 Ok(Err(error)) => {
620 warn!(module_id, error = %error, "stdout pump ended unexpectedly");
621 }
622 Err(_) => {
623 drop(pump);
624 warn!(
625 module_id,
626 waited = ?bound,
627 "stdout pipe of the exited process has not drained; its reader keeps running without delaying the restart"
628 );
629 }
630 }
631}
632
633struct StderrPump {
636 task: JoinHandle<()>,
637 generation: u64,
638}
639
640async fn settle_stderr_pump(
646 module_id: &str,
647 ring: &Arc<Mutex<StderrRing>>,
648 pump: StderrPump,
649 bound: Duration,
650) {
651 let lock = || ring.lock().unwrap_or_else(|poisoned| poisoned.into_inner());
652 let StderrPump {
653 mut task,
654 generation,
655 } = pump;
656 lock().retire_pump(generation);
657 match timeout(bound, &mut task).await {
658 Ok(Ok(())) => {}
659 Ok(Err(err)) => {
660 let mut ring = lock();
661 ring.mark_incomplete_from(generation, format!("stderr pump ended unexpectedly: {err}"));
662 ring.finish_pump(generation);
663 warn!(module_id, error = %err, "stderr pump ended before clean EOF");
664 }
665 Err(_) => {
666 drop(task);
668 lock().mark_pump_late(
669 generation,
670 format!(
671 "stderr of the exited process had not reached EOF {bound:?} after it was \
672 retired (a descendant may still hold the pipe open); lines it still \
673 writes are kept in that process's section"
674 ),
675 );
676 warn!(
677 module_id,
678 waited = ?bound,
679 "stderr pipe of the exited process is still open; its reader keeps running without delaying the restart"
680 );
681 }
682 }
683}
684
685fn registration_release_events() -> &'static watch::Sender<u64> {
686 static EVENTS: OnceLock<watch::Sender<u64>> = OnceLock::new();
687 EVENTS.get_or_init(|| {
688 let (sender, _receiver) = watch::channel(0);
689 sender
690 })
691}
692
693pub(crate) fn notify_registration_release() {
694 let events = registration_release_events();
695 let next_generation = (*events.borrow()).wrapping_add(1);
696 events.send_replace(next_generation);
697}
698
699#[derive(Debug, Clone, PartialEq, Eq)]
701pub struct ModuleSpec {
702 pub module_id: String,
703 pub program: PathBuf,
704 pub args: Vec<String>,
705 pub env: Vec<(String, String)>,
706 pub reserved: bool,
711 pub reserved_prefixes: Vec<String>,
716 pub protocol: ModuleProtocol,
735 pub overlap: ModuleOverlap,
740}
741
742#[derive(Clone, PartialEq, Eq)]
745pub(crate) struct LaunchSpecDigest([u8; 32]);
746
747impl fmt::Debug for LaunchSpecDigest {
748 fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
749 f.write_str("LaunchSpecDigest([redacted])")
750 }
751}
752
753impl LaunchSpecDigest {
754 pub(crate) fn from_spec(spec: &ModuleSpec) -> Self {
755 fn field(hash: &mut Sha256, bytes: &[u8]) {
756 hash.update((bytes.len() as u64).to_le_bytes());
757 hash.update(bytes);
758 }
759 let mut hash = Sha256::new();
760 if let Some(program) = spec.program.to_str() {
761 field(&mut hash, b"utf8-program");
762 field(&mut hash, program.as_bytes());
763 } else {
764 #[cfg(unix)]
767 {
768 use std::os::unix::ffi::OsStrExt;
769 field(&mut hash, b"unix-program");
770 field(&mut hash, spec.program.as_os_str().as_bytes());
771 }
772 #[cfg(windows)]
773 {
774 use std::os::windows::ffi::OsStrExt;
775 field(&mut hash, b"windows-program");
776 let bytes = spec
777 .program
778 .as_os_str()
779 .encode_wide()
780 .flat_map(u16::to_le_bytes)
781 .collect::<Vec<_>>();
782 field(&mut hash, &bytes);
783 }
784 #[cfg(not(any(unix, windows)))]
785 field(&mut hash, spec.program.to_string_lossy().as_bytes());
786 }
787 hash.update((spec.args.len() as u64).to_le_bytes());
788 for arg in &spec.args {
789 field(&mut hash, arg.as_bytes());
790 }
791 let mut env = spec.env.iter().collect::<Vec<_>>();
792 env.sort_unstable();
793 hash.update((env.len() as u64).to_le_bytes());
794 for (key, value) in env {
795 field(&mut hash, key.as_bytes());
796 field(&mut hash, value.as_bytes());
797 }
798 field(
799 &mut hash,
800 match spec.protocol {
801 ModuleProtocol::Subc => b"subc",
802 ModuleProtocol::None => b"none",
803 },
804 );
805 Self(hash.finalize().into())
806 }
807}
808
809#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)]
816pub enum ModuleOverlap {
817 #[default]
819 Exclusive,
820 Safe,
832}
833
834impl ModuleOverlap {
835 pub fn as_str(self) -> &'static str {
836 match self {
837 Self::Exclusive => "exclusive",
838 Self::Safe => "safe",
839 }
840 }
841}
842
843pub const SUBC_SPAWN_ROLE_ENV: &str = "SUBC_SPAWN_ROLE";
853pub const SPAWN_ROLE_SWAP_CANDIDATE: &str = "swap_candidate";
855pub const DEFAULT_SWAP_READY_TIMEOUT: Duration = Duration::from_secs(100);
860
861#[derive(Debug, Clone, Copy, PartialEq, Eq)]
879pub struct RestartPolicy {
880 pub max_restarts: u32,
881 pub backoff: Duration,
884 pub max_backoff: Duration,
886 pub window: Duration,
890}
891
892impl RestartPolicy {
893 pub fn new(max_restarts: u32, backoff: Duration) -> Self {
897 Self {
898 max_restarts,
899 backoff,
900 max_backoff: DEFAULT_MAX_BACKOFF,
901 window: DEFAULT_RESTART_WINDOW,
902 }
903 }
904
905 pub fn with_max_backoff(mut self, max_backoff: Duration) -> Self {
906 self.max_backoff = max_backoff;
907 self
908 }
909
910 pub fn with_window(mut self, window: Duration) -> Self {
911 self.window = window;
912 self
913 }
914
915 fn delay_for_restart(&self, restart_in_window: u32) -> Duration {
920 if self.backoff.is_zero() || self.max_backoff.is_zero() {
921 return Duration::ZERO;
922 }
923
924 let mut delay = self.backoff;
925 for _ in 0..restart_in_window {
926 if delay >= self.max_backoff {
927 return self.max_backoff;
928 }
929 delay = delay
930 .checked_mul(10)
931 .unwrap_or(self.max_backoff)
932 .min(self.max_backoff);
933 }
934 delay.min(self.max_backoff)
935 }
936
937 fn budget_exhausted_detail(&self) -> String {
942 format!(
943 "crash budget exhausted: max_restarts={} within window_secs={}",
944 self.max_restarts,
945 self.window.as_secs()
946 )
947 }
948}
949
950impl Default for RestartPolicy {
951 fn default() -> Self {
952 Self {
953 max_restarts: DEFAULT_MAX_RESTARTS,
954 backoff: DEFAULT_BACKOFF,
955 max_backoff: DEFAULT_MAX_BACKOFF,
956 window: DEFAULT_RESTART_WINDOW,
957 }
958 }
959}
960
961#[derive(Debug, Clone, Copy, PartialEq, Eq)]
962struct CrashRestartSchedule {
963 restart_in_window: u32,
964 delay: Duration,
965}
966
967fn daemon_will_restart(
974 state: &mut SupervisorSnapshot,
975 policy: &RestartPolicy,
976 now: Instant,
977) -> bool {
978 state.enabled && state.crash_restarts_in_window(policy.window, now) < policy.max_restarts
979}
980
981const DEFAULT_HEALTH_CADENCE: Duration = Duration::from_secs(30);
982const DEFAULT_HEALTH_DEADLINE: Duration = Duration::from_secs(5);
983const DEFAULT_HEALTH_FAILURE_THRESHOLD: u32 = 3;
984const MAX_HEALTH_METRICS_BYTES: usize = 16 * 1024;
985
986#[derive(Debug, Clone, Copy, PartialEq, Eq)]
987pub enum HealthAction {
988 Report,
989 Restart,
990 Alert,
991}
992
993impl fmt::Display for HealthAction {
994 fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
995 f.write_str(match self {
996 Self::Report => "report",
997 Self::Restart => "restart",
998 Self::Alert => "alert",
999 })
1000 }
1001}
1002
1003#[derive(Debug, Clone, PartialEq, Eq)]
1004pub struct HealthConfig {
1005 pub http: Option<String>,
1009 pub cadence: Duration,
1010 pub deadline: Duration,
1011 pub failure_threshold: u32,
1012 pub on_degraded: HealthAction,
1013 pub on_failing: HealthAction,
1014 pub critical: bool,
1015}
1016
1017impl Default for HealthConfig {
1018 fn default() -> Self {
1019 Self {
1020 http: None,
1021 cadence: DEFAULT_HEALTH_CADENCE,
1022 deadline: DEFAULT_HEALTH_DEADLINE,
1023 failure_threshold: DEFAULT_HEALTH_FAILURE_THRESHOLD,
1024 on_degraded: HealthAction::Report,
1025 on_failing: HealthAction::Report,
1026 critical: false,
1027 }
1028 }
1029}
1030
1031#[derive(Debug, Clone, PartialEq)]
1049pub struct ModuleHealthStatus {
1050 pub status: SupervisorHealthStatus,
1051 pub last_probe_ms: Option<u64>,
1052 pub detail: Option<String>,
1053 pub metrics: Option<Value>,
1054 pub consecutive_failures: u32,
1055 pub late_answer_count: u64,
1058 pub last_late_answer_latency_ms: Option<u64>,
1060 pub last_action: Option<String>,
1061 pub last_action_ms: Option<u64>,
1065}
1066
1067impl Default for ModuleHealthStatus {
1068 fn default() -> Self {
1069 Self {
1070 status: SupervisorHealthStatus::Unknown,
1071 last_probe_ms: None,
1072 detail: None,
1073 metrics: None,
1074 consecutive_failures: 0,
1075 late_answer_count: 0,
1076 last_late_answer_latency_ms: None,
1077 last_action: None,
1078 last_action_ms: None,
1079 }
1080 }
1081}
1082
1083#[derive(Debug, Clone, Copy, PartialEq, Eq)]
1085pub enum ModuleState {
1086 Starting,
1087 Running,
1088 Unresponsive,
1089 Restarting,
1090 Draining,
1091 Stopped,
1092 Failed,
1093 Disabled,
1094}
1095
1096impl fmt::Display for ModuleState {
1097 fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
1098 f.write_str(match self {
1099 Self::Starting => "starting",
1100 Self::Running => "running",
1101 Self::Unresponsive => "unresponsive",
1102 Self::Restarting => "restarting",
1103 Self::Draining => "draining",
1104 Self::Stopped => "stopped",
1105 Self::Failed => "failed",
1106 Self::Disabled => "disabled",
1107 })
1108 }
1109}
1110
1111#[derive(Debug, Clone, Copy, PartialEq, Eq)]
1113pub enum ExitKind {
1114 Clean,
1115 Crash,
1116 DeliberateSeverance,
1117}
1118
1119impl From<ExitKind> for TerminalExitKind {
1120 fn from(kind: ExitKind) -> Self {
1121 match kind {
1122 ExitKind::Clean => Self::Clean,
1123 ExitKind::Crash => Self::Crash,
1124 ExitKind::DeliberateSeverance => Self::DeliberateSeverance,
1125 }
1126 }
1127}
1128
1129#[derive(Debug, Clone, Copy, PartialEq, Eq)]
1132pub(crate) struct ProcessIdentity {
1133 pub(crate) pid: u32,
1134 pub(crate) start_time: u64,
1135}
1136
1137#[derive(Debug, Clone, PartialEq, Eq)]
1139pub struct ExitReport {
1140 pub kind: ExitKind,
1141 pub code: Option<i32>,
1142 pub signal: Option<i32>,
1143 pub at_ms: u64,
1144}
1145
1146#[derive(Debug, Clone, PartialEq)]
1149pub struct ModuleStatus {
1150 pub module_id: String,
1151 pub waiting_for_previous_pid: Option<u32>,
1153 pub waiting_for_previous_detail: Option<String>,
1155 pub launch_nonce_source: Option<String>,
1160 pub state: ModuleState,
1161 pub enabled: bool,
1162 pub process_alive: bool,
1163 pub registration_active: bool,
1164 pub protocol: ModuleProtocol,
1169 pub live: bool,
1180 pub restart_count: u32,
1184 pub lifetime_restarts: u32,
1188 pub spawn_generation: u64,
1189 pub max_restarts: u32,
1194 pub restart_window: Duration,
1198 pub drain_timeout: Duration,
1202 pub restart_backoff: Duration,
1203 pub restart_max_backoff: Duration,
1204 pub pid: Option<u32>,
1209 pub spawned_at_ms: Option<u64>,
1210 pub spawned_from: Option<PathBuf>,
1211 pub(crate) launch_spec_digest: Option<LaunchSpecDigest>,
1212 pub process_start_time: Option<u64>,
1213 pub last_exit: Option<ExitReport>,
1214 pub health: ModuleHealthStatus,
1215}
1216
1217#[cfg(test)]
1218#[derive(Debug, Clone, Copy, PartialEq)]
1219struct ActorSelectCheckpoint {
1220 generation: u64,
1221 turn: u64,
1222 registered_connection: Option<ConnectionId>,
1223 next_probe_at: Option<Instant>,
1224 wake_after: Option<Duration>,
1225}
1226
1227#[derive(Debug, Clone, PartialEq)]
1228struct SupervisorSnapshot {
1229 exec_outcome: Option<Arc<crate::activity::ExecOutcome>>,
1230 #[cfg(windows)]
1231 pipe_consumed: Option<Arc<OnceLock<()>>>,
1232 #[cfg(test)]
1234 actor_turns: u64,
1235 #[cfg(test)]
1238 actor_select: Option<ActorSelectCheckpoint>,
1239 state: ModuleState,
1240 enabled: bool,
1241 process_alive: bool,
1242 spawned_protocol: Option<ModuleProtocol>,
1243 spawn_failure: Option<String>,
1244 waiting_for_previous_pid: Option<u32>,
1245 waiting_for_previous_detail: Option<String>,
1246 crash_restarts: VecDeque<Instant>,
1252 lifetime_restarts: u32,
1253 spawn_generation: u64,
1262 pid: Option<u32>,
1263 #[cfg(target_os = "macos")]
1264 report_ready: Option<Arc<OnceLock<()>>>,
1265 reaped_pid: Option<u32>,
1267 respawn_pending: bool,
1269 coalesced_restart_pending: bool,
1271 spawned_at_ms: Option<u64>,
1272 spawned_from: Option<PathBuf>,
1273 launch_spec_digest: Option<LaunchSpecDigest>,
1274 spawned_file_identity: Option<SpawnedFileIdentity>,
1275 process_start_time: Option<u64>,
1276 deliberate_severance: Option<ProcessIdentity>,
1277 last_exit: Option<ExitReport>,
1278 drain_disposition_detail: Option<String>,
1280 health: ModuleHealthStatus,
1281 in_alternate_slot: bool,
1286 draining_to_replace: bool,
1293 configuration_updated_since_spawn: bool,
1299}
1300
1301impl SupervisorSnapshot {
1302 fn reported_pid(&self) -> Option<u32> {
1307 #[cfg(target_os = "macos")]
1308 if self
1309 .report_ready
1310 .as_ref()
1311 .is_some_and(|ready| ready.get().is_none())
1312 {
1313 return None;
1314 }
1315 self.pid
1316 }
1317
1318 fn starting() -> Self {
1319 Self::new(ModuleState::Starting, true)
1320 }
1321
1322 fn disabled() -> Self {
1323 Self::new(ModuleState::Disabled, false)
1324 }
1325
1326 fn crash_restarts_in_window(&mut self, window: Duration, now: Instant) -> u32 {
1330 while let Some(oldest) = self.crash_restarts.front() {
1331 if now.duration_since(*oldest) > window {
1332 self.crash_restarts.pop_front();
1333 } else {
1334 break;
1335 }
1336 }
1337 u32::try_from(self.crash_restarts.len()).unwrap_or(u32::MAX)
1338 }
1339
1340 fn record_crash_restart(&mut self, policy: &RestartPolicy, now: Instant) {
1346 self.crash_restarts.push_back(now);
1347 while self.crash_restarts.len() > policy.max_restarts as usize {
1348 self.crash_restarts.pop_front();
1349 }
1350 self.lifetime_restarts += 1;
1351 }
1352
1353 fn next_crash_restart(
1357 &mut self,
1358 policy: &RestartPolicy,
1359 now: Instant,
1360 ) -> Option<CrashRestartSchedule> {
1361 let restart_in_window = self.crash_restarts_in_window(policy.window, now);
1362 if restart_in_window >= policy.max_restarts {
1363 return None;
1364 }
1365 self.record_crash_restart(policy, now);
1366 Some(CrashRestartSchedule {
1367 restart_in_window,
1368 delay: policy.delay_for_restart(restart_in_window),
1369 })
1370 }
1371
1372 fn clear_crash_restarts(&mut self) {
1377 self.crash_restarts.clear();
1378 }
1379
1380 fn new(state: ModuleState, enabled: bool) -> Self {
1381 Self {
1382 exec_outcome: None,
1383 #[cfg(test)]
1384 actor_turns: 0,
1385 #[cfg(test)]
1386 actor_select: None,
1387 state,
1388 enabled,
1389 process_alive: false,
1390 #[cfg(windows)]
1391 pipe_consumed: None,
1392 spawned_protocol: None,
1393 spawn_failure: None,
1394 waiting_for_previous_pid: None,
1395 waiting_for_previous_detail: None,
1396 crash_restarts: VecDeque::new(),
1397 lifetime_restarts: 0,
1398 spawn_generation: 0,
1399 pid: None,
1400 #[cfg(target_os = "macos")]
1401 report_ready: None,
1402 reaped_pid: None,
1403 respawn_pending: false,
1404 coalesced_restart_pending: false,
1405 spawned_at_ms: None,
1406 spawned_from: None,
1407 launch_spec_digest: None,
1408 spawned_file_identity: None,
1409 process_start_time: None,
1410 deliberate_severance: None,
1411 last_exit: None,
1412 drain_disposition_detail: None,
1413 health: ModuleHealthStatus::default(),
1414 in_alternate_slot: false,
1415 draining_to_replace: false,
1416 configuration_updated_since_spawn: false,
1417 }
1418 }
1419}
1420
1421type SharedSnapshot = Arc<Mutex<SupervisorSnapshot>>;
1422
1423type SpawnSubscriberKey = (ConnectionId, u64);
1424
1425#[derive(Debug)]
1426struct SpawnSubscriber {
1427 version: u8,
1428 frames: mpsc::Sender<Frame>,
1429 lagged: Option<oneshot::Sender<SpawnCursor>>,
1433}
1434
1435#[derive(Debug)]
1436struct SpawnEventState {
1437 daemon_incarnation: String,
1438 seq: u64,
1439 capacity: usize,
1440 live: HashMap<String, LiveSpawn>,
1441 generations: HashMap<String, u64>,
1442 events: VecDeque<SpawnEvent>,
1443 subscribers: HashMap<SpawnSubscriberKey, SpawnSubscriber>,
1444}
1445
1446impl Default for SpawnEventState {
1447 fn default() -> Self {
1448 Self {
1449 daemon_incarnation: "unconfigured".to_string(),
1450 seq: 0,
1451 capacity: SPAWN_EVENT_RING_CAPACITY,
1452 live: HashMap::new(),
1453 generations: HashMap::new(),
1454 events: VecDeque::new(),
1455 subscribers: HashMap::new(),
1456 }
1457 }
1458}
1459
1460#[derive(Debug, Clone, Default)]
1461struct SpawnEventFeed(
1462 Arc<Mutex<SpawnEventState>>,
1463 #[cfg(test)] Arc<Mutex<Option<std::net::TcpStream>>>,
1464);
1465
1466#[derive(Debug, Clone, PartialEq, Eq)]
1467pub(crate) enum SpawnSubscribeRefusal {
1468 ForeignIncarnation { current: String },
1469 TooOld { oldest: SpawnCursor },
1470 Frame(String),
1471}
1472
1473impl SpawnEventFeed {
1474 fn configure_incarnation(&self, daemon_incarnation: String) {
1475 let mut state = self.0.lock().unwrap_or_else(|p| p.into_inner());
1476 state.daemon_incarnation = daemon_incarnation;
1477 state.seq = 0;
1478 state.live.clear();
1479 state.generations.clear();
1480 state.events.clear();
1481 state.subscribers.clear();
1482 }
1483
1484 fn cursor(state: &SpawnEventState) -> SpawnCursor {
1485 SpawnCursor {
1486 daemon_incarnation: state.daemon_incarnation.clone(),
1487 seq: state.seq,
1488 }
1489 }
1490
1491 fn snapshot(&self) -> SpawnSnapshot {
1492 let state = self.0.lock().unwrap_or_else(|p| p.into_inner());
1493 let mut live = state.live.values().cloned().collect::<Vec<_>>();
1494 live.sort_by(|left, right| left.module_id.cmp(&right.module_id));
1495 SpawnSnapshot {
1496 cursor: Self::cursor(&state),
1497 ring_bound: state.capacity as u64,
1498 live,
1499 }
1500 }
1501
1502 fn emit_spawned(&self, module_id: &str, pid: u32, spawned_at_ms: u64) -> u64 {
1503 let mut state = self.0.lock().unwrap_or_else(|p| p.into_inner());
1504 let generation = state
1505 .generations
1506 .get(module_id)
1507 .copied()
1508 .unwrap_or(0)
1509 .checked_add(1)
1510 .expect("spawn generation exhausted");
1511 state.generations.insert(module_id.to_string(), generation);
1512 let live = LiveSpawn {
1513 module_id: module_id.to_string(),
1514 spawn_generation: generation,
1515 pid,
1516 spawned_at_ms,
1517 };
1518 state.live.insert(module_id.to_string(), live);
1519 Self::emit_locked(
1520 &mut state,
1521 SpawnEventKind::Spawned,
1522 module_id.to_string(),
1523 generation,
1524 pid,
1525 None,
1526 None,
1527 );
1528 generation
1529 }
1530
1531 fn emit_exited(&self, module_id: &str, exit_code: Option<i32>, exit_signal: Option<i32>) {
1532 let mut state = self.0.lock().unwrap_or_else(|p| p.into_inner());
1533 let Some(live) = state.live.remove(module_id) else {
1534 warn!(
1535 module_id,
1536 "terminal record had no live spawn event identity"
1537 );
1538 return;
1539 };
1540 Self::emit_locked(
1541 &mut state,
1542 SpawnEventKind::Exited,
1543 module_id.to_string(),
1544 live.spawn_generation,
1545 live.pid,
1546 exit_code,
1547 exit_signal,
1548 );
1549 }
1550
1551 fn emit_superseded_exited(
1558 &self,
1559 module_id: &str,
1560 spawn_generation: u64,
1561 pid: u32,
1562 exit_code: Option<i32>,
1563 exit_signal: Option<i32>,
1564 ) {
1565 let mut state = self.0.lock().unwrap_or_else(|p| p.into_inner());
1566 if state
1567 .live
1568 .get(module_id)
1569 .is_some_and(|live| live.spawn_generation == spawn_generation)
1570 {
1571 state.live.remove(module_id);
1572 }
1573 Self::emit_locked(
1574 &mut state,
1575 SpawnEventKind::Exited,
1576 module_id.to_string(),
1577 spawn_generation,
1578 pid,
1579 exit_code,
1580 exit_signal,
1581 );
1582 }
1583
1584 #[allow(clippy::too_many_arguments)]
1585 fn emit_locked(
1586 state: &mut SpawnEventState,
1587 kind: SpawnEventKind,
1588 module_id: String,
1589 spawn_generation: u64,
1590 pid: u32,
1591 exit_code: Option<i32>,
1592 exit_signal: Option<i32>,
1593 ) {
1594 state.seq = state
1595 .seq
1596 .checked_add(1)
1597 .expect("spawn event sequence exhausted");
1598 let event = SpawnEvent {
1599 cursor: Self::cursor(state),
1600 kind,
1601 module_id,
1602 spawn_generation,
1603 pid,
1604 exit_code,
1605 exit_signal,
1606 };
1607 state.events.push_back(event.clone());
1608 while state.events.len() > state.capacity {
1609 state.events.pop_front();
1610 }
1611 let body = match serde_json::to_vec(&event) {
1612 Ok(body) => body,
1613 Err(error) => {
1614 error!(%error, "failed to serialize supervisor spawn event");
1615 return;
1616 }
1617 };
1618 state.subscribers.retain(|(connection_id, corr), subscriber| {
1619 let frame = Frame::build_with_version(
1620 subscriber.version,
1621 FrameType::StreamData,
1622 control_flags(),
1623 0,
1624 0,
1625 *corr,
1626 body.clone(),
1627 );
1628 match frame {
1629 Ok(frame) => {
1630 if subscriber.frames.try_send(frame).is_ok() {
1631 true
1632 } else {
1633 warn!(connection_id = connection_id.get(), corr, "dropping lagged supervisor spawn subscriber");
1634 if let Some(lagged) = subscriber.lagged.take() {
1635 let _ = lagged.send(event.cursor.clone());
1636 }
1637 false
1638 }
1639 }
1640 Err(error) => {
1641 warn!(connection_id = connection_id.get(), corr, %error, "dropping supervisor spawn subscriber after frame build failure");
1642 false
1643 }
1644 }
1645 });
1646 }
1647
1648 fn subscribe(
1649 &self,
1650 connection_id: ConnectionId,
1651 corr: u64,
1652 version: u8,
1653 since: Option<SpawnCursor>,
1654 sink: FrameSink,
1655 ) -> Result<(), SpawnSubscribeRefusal> {
1656 let (frames, mut receiver) = mpsc::channel(SPAWN_SUBSCRIBER_BUFFER);
1657 let (lagged, mut lagged_rx) = oneshot::channel::<SpawnCursor>();
1658 {
1659 let mut state = self.0.lock().unwrap_or_else(|p| p.into_inner());
1660 let replay = if let Some(since) = since {
1661 if since.daemon_incarnation != state.daemon_incarnation {
1662 return Err(SpawnSubscribeRefusal::ForeignIncarnation {
1663 current: state.daemon_incarnation.clone(),
1664 });
1665 }
1666 if let Some(oldest) = state.events.front().map(|event| event.cursor.clone()) {
1667 if since.seq < oldest.seq.saturating_sub(1) {
1668 return Err(SpawnSubscribeRefusal::TooOld { oldest });
1669 }
1670 }
1671 state
1672 .events
1673 .iter()
1674 .filter(|event| event.cursor.seq > since.seq)
1675 .cloned()
1676 .collect::<Vec<_>>()
1677 } else {
1678 Vec::new()
1679 };
1680 for event in replay {
1681 let body = serde_json::to_vec(&event)
1682 .map_err(|error| SpawnSubscribeRefusal::Frame(error.to_string()))?;
1683 let frame = Frame::build_with_version(
1684 version,
1685 FrameType::StreamData,
1686 control_flags(),
1687 0,
1688 0,
1689 corr,
1690 body,
1691 )
1692 .map_err(|error| SpawnSubscribeRefusal::Frame(error.to_string()))?;
1693 frames
1694 .try_send(frame)
1695 .map_err(|error| SpawnSubscribeRefusal::Frame(error.to_string()))?;
1696 }
1697 state.subscribers.insert(
1698 (connection_id, corr),
1699 SpawnSubscriber {
1700 version,
1701 frames,
1702 lagged: Some(lagged),
1703 },
1704 );
1705 }
1706 tokio::spawn(async move {
1717 while let Some(frame) = receiver.recv().await {
1718 if sink.send(frame).await.is_err() {
1719 return;
1720 }
1721 }
1722 let Ok(first_undelivered) = lagged_rx.try_recv() else {
1723 return;
1724 };
1725 match spawn_subscriber_lagged_frame(version, corr, first_undelivered) {
1726 Ok(frame) => {
1727 let _ = sink.send(frame).await;
1728 }
1729 Err(error) => {
1730 error!(%error, corr, "failed to build lagged spawn subscriber terminal frame");
1731 }
1732 }
1733 });
1734 Ok(())
1735 }
1736
1737 fn cancel(&self, connection_id: ConnectionId, corr: u64) -> bool {
1738 let Some(subscriber) = self
1739 .0
1740 .lock()
1741 .unwrap_or_else(|p| p.into_inner())
1742 .subscribers
1743 .remove(&(connection_id, corr))
1744 else {
1745 return false;
1746 };
1747 if let Ok(frame) = Frame::build_with_version(
1748 subscriber.version,
1749 FrameType::StreamEnd,
1750 control_flags(),
1751 0,
1752 0,
1753 corr,
1754 Vec::new(),
1755 ) {
1756 tokio::spawn(async move {
1757 let _ = subscriber.frames.send(frame).await;
1758 });
1759 }
1760 true
1761 }
1762
1763 fn remove_connection(&self, connection_id: ConnectionId) {
1764 self.0
1765 .lock()
1766 .unwrap_or_else(|p| p.into_inner())
1767 .subscribers
1768 .retain(|(subscriber_connection, _), _| *subscriber_connection != connection_id);
1769 }
1770
1771 #[cfg(any(test, feature = "test-support"))]
1772 fn set_capacity(&self, capacity: usize) {
1773 self.0.lock().unwrap_or_else(|p| p.into_inner()).capacity = capacity;
1774 }
1775
1776 #[cfg(any(test, feature = "test-support"))]
1777 fn subscriber_count(&self) -> usize {
1778 self.0
1779 .lock()
1780 .unwrap_or_else(|p| p.into_inner())
1781 .subscribers
1782 .len()
1783 }
1784}
1785
1786fn spawn_subscriber_lagged_frame(
1789 version: u8,
1790 corr: u64,
1791 first_undelivered: SpawnCursor,
1792) -> Result<Frame, String> {
1793 let body = serde_json::to_vec(&subc_protocol::ErrorBody {
1794 code: SPAWN_SUBSCRIBER_LAGGED_CODE.to_string(),
1795 message: "spawn subscriber fell behind and was dropped; resubscribe from the last cursor received"
1796 .to_string(),
1797 detail: Some(serde_json::json!({
1798 "first_undelivered_cursor": first_undelivered
1799 })),
1800 })
1801 .map_err(|error| error.to_string())?;
1802 Frame::build_with_version(version, FrameType::Error, control_flags(), 0, 0, corr, body)
1803 .map_err(|error| error.to_string())
1804}
1805
1806pub trait ModuleProcessLiveness: Send + Sync {
1807 fn process_live(&self, module_id: &str) -> Option<bool>;
1808
1809 fn process_replacing(&self, _module_id: &str) -> bool {
1815 false
1816 }
1817}
1818
1819#[derive(Debug, Clone, Default)]
1821pub struct SupervisorProcessLiveness {
1822 snapshots: Arc<Mutex<HashMap<String, SharedSnapshot>>>,
1823}
1824
1825impl SupervisorProcessLiveness {
1826 pub fn new() -> Self {
1827 Self::default()
1828 }
1829
1830 fn track(&self, module_id: String, snapshot: SharedSnapshot) {
1831 let mut snapshots = self
1832 .snapshots
1833 .lock()
1834 .unwrap_or_else(|poisoned| poisoned.into_inner());
1835 snapshots.insert(module_id, snapshot);
1836 }
1837
1838 fn untrack_if_current(&self, module_id: &str, snapshot: &SharedSnapshot) {
1839 let mut snapshots = self
1840 .snapshots
1841 .lock()
1842 .unwrap_or_else(|poisoned| poisoned.into_inner());
1843 let is_current = snapshots
1844 .get(module_id)
1845 .map(|tracked| Arc::ptr_eq(tracked, snapshot))
1846 .unwrap_or(false);
1847 if is_current {
1848 snapshots.remove(module_id);
1849 }
1850 }
1851}
1852
1853impl ModuleProcessLiveness for SupervisorProcessLiveness {
1854 fn process_live(&self, module_id: &str) -> Option<bool> {
1855 let snapshot = {
1856 let snapshots = self
1857 .snapshots
1858 .lock()
1859 .unwrap_or_else(|poisoned| poisoned.into_inner());
1860 snapshots.get(module_id).cloned()
1861 }?;
1862 let snapshot = snapshot
1863 .lock()
1864 .unwrap_or_else(|poisoned| poisoned.into_inner());
1865 Some(snapshot.state == ModuleState::Running && snapshot.process_alive)
1866 }
1867
1868 fn process_replacing(&self, module_id: &str) -> bool {
1869 let Some(snapshot) = self
1870 .snapshots
1871 .lock()
1872 .unwrap_or_else(|poisoned| poisoned.into_inner())
1873 .get(module_id)
1874 .cloned()
1875 else {
1876 return false;
1877 };
1878 let snapshot = snapshot
1879 .lock()
1880 .unwrap_or_else(|poisoned| poisoned.into_inner());
1881 snapshot.enabled
1882 && match snapshot.state {
1883 ModuleState::Restarting => true,
1884 ModuleState::Draining => snapshot.draining_to_replace,
1885 ModuleState::Starting
1886 | ModuleState::Running
1887 | ModuleState::Unresponsive
1888 | ModuleState::Stopped
1889 | ModuleState::Failed
1890 | ModuleState::Disabled => false,
1891 }
1892 }
1893}
1894
1895#[cfg(test)]
1896#[derive(Debug, Default)]
1897struct ReloadExitRecordGate {
1898 reached: tokio::sync::Notify,
1899 resume: tokio::sync::Notify,
1900}
1901
1902#[derive(Debug, Clone, Copy)]
1903enum RespawnKind {
1904 Spawn,
1905 Reload,
1906}
1907
1908#[derive(Debug, Clone, Copy)]
1909struct PendingRespawn {
1910 deadline: Instant,
1911 kind: RespawnKind,
1912}
1913
1914type ReloadReply = oneshot::Sender<Result<(), SuperviseError>>;
1915
1916#[derive(Debug, Clone)]
1917struct SupervisorRuntimeConfig {
1918 start_requester: Option<Principal>,
1919 pending_activity_restart: bool,
1920 provenance_probe: ExecutableIdentityProbe,
1921 scheduled_respawn: Arc<Mutex<Option<PendingRespawn>>>,
1923 deferred_reload_reply: Arc<Mutex<Option<ReloadReply>>>,
1925 restart_policy: RestartPolicy,
1926 drain_timeout: Duration,
1929 effective_drain_timeout: Arc<Mutex<Duration>>,
1932 default_drain_timeout: Duration,
1935 health: HealthConfig,
1936 connection_file_path: Option<PathBuf>,
1937 capture_logs_dir: Option<PathBuf>,
1938 forwarding: Option<Arc<ForwardingTable>>,
1939 supervisor_handle: Option<SupervisorHandle>,
1942 stderr_ring: Arc<Mutex<StderrRing>>,
1949 terminal_ring: Arc<Mutex<TerminalRing>>,
1950 spawn_events: SpawnEventFeed,
1951 child_roster: ChildRoster,
1952 #[cfg(target_os = "linux")]
1953 cgroup_placement: Option<subc_cgroup::Placement>,
1954 #[cfg(test)]
1955 test_seed_stale_facts_before_enable_spawn: bool,
1956 #[cfg(test)]
1957 test_reload_exit_record_gate: Option<Arc<ReloadExitRecordGate>>,
1958}
1959
1960#[derive(Debug, Clone, PartialEq, Eq)]
1961struct SupervisedConfiguration {
1962 spec: ModuleSpec,
1963 health: HealthConfig,
1964}
1965
1966#[derive(Debug, Clone, Default)]
1972pub struct SupervisorHandle {
1973 activity: Arc<Mutex<Option<crate::activity::ActivityLedger>>>,
1974 activity_images: Arc<Mutex<HashMap<String, activity::ActivityImage>>>,
1975 modules: Arc<Mutex<HashMap<String, SupervisedModule>>>,
1976 configured_ids: Arc<Mutex<HashSet<String>>>,
1988 spawn_events: SpawnEventFeed,
1989 reserved_nonces: Arc<Mutex<HashMap<String, Option<String>>>>,
2000 removal_tombstones: Arc<Mutex<HashMap<String, u64>>>,
2006 spawn_nonces: Arc<Mutex<HashMap<String, String>>>,
2010 reserved_prefix_owners: Arc<Mutex<HashMap<String, String>>>,
2018 swaps: Arc<Mutex<HashMap<String, OpenSwap>>>,
2024 promotion_observer: PromotionObserverSlot,
2026 operation_lock: Arc<AsyncMutex<()>>,
2030}
2031
2032pub(crate) trait SwapPromotionObserver: Send + Sync {
2041 fn swap_promoted(&self, registration: &crate::registry::ModuleRegistration);
2042}
2043
2044#[derive(Clone, Default)]
2048struct PromotionObserverSlot(Arc<Mutex<Option<std::sync::Weak<dyn SwapPromotionObserver>>>>);
2049
2050impl fmt::Debug for PromotionObserverSlot {
2051 fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
2052 f.write_str("PromotionObserverSlot")
2053 }
2054}
2055
2056#[derive(Debug, Clone)]
2058struct OpenSwap {
2059 candidate_nonce: String,
2062 incumbent_nonce: Option<String>,
2067 candidate_admitted: bool,
2071}
2072
2073#[derive(Debug, Clone, Copy, PartialEq, Eq)]
2076pub(crate) enum SwapHelloAdmission {
2077 NotSwapping,
2080 Candidate,
2082 Refused,
2085}
2086
2087#[derive(Debug, Clone, PartialEq, Eq)]
2088pub(crate) enum ReservedHelloRejection {
2089 Exact {
2090 module_id: String,
2091 },
2092 Prefix {
2093 prefix: String,
2094 owner_module_id: String,
2095 },
2096}
2097
2098impl SupervisorHandle {
2099 pub fn new() -> Self {
2100 Self::default()
2101 }
2102
2103 pub(crate) fn spawn_snapshot(&self) -> SpawnSnapshot {
2104 self.spawn_events.snapshot()
2105 }
2106
2107 pub(crate) fn subscribe_spawns(
2108 &self,
2109 connection_id: ConnectionId,
2110 corr: u64,
2111 version: u8,
2112 since: Option<SpawnCursor>,
2113 sink: FrameSink,
2114 ) -> Result<(), SpawnSubscribeRefusal> {
2115 self.spawn_events
2116 .subscribe(connection_id, corr, version, since, sink)
2117 }
2118
2119 pub(crate) fn cancel_spawn_subscription(&self, connection_id: ConnectionId, corr: u64) -> bool {
2120 self.spawn_events.cancel(connection_id, corr)
2121 }
2122
2123 pub(crate) fn remove_spawn_subscribers(&self, connection_id: ConnectionId) {
2124 self.spawn_events.remove_connection(connection_id);
2125 }
2126
2127 #[cfg(any(test, feature = "test-support"))]
2128 pub fn set_spawn_event_capacity_for_test(&self, capacity: usize) {
2129 assert!(capacity > 0, "spawn event capacity must be non-zero");
2130 self.spawn_events.set_capacity(capacity);
2131 }
2132
2133 #[cfg(any(test, feature = "test-support"))]
2134 pub fn spawn_subscriber_count_for_test(&self) -> usize {
2135 self.spawn_events.subscriber_count()
2136 }
2137
2138 pub fn set_spawn_nonce(&self, module_id: &str, nonce: String) {
2141 self.spawn_nonces
2142 .lock()
2143 .unwrap_or_else(|poisoned| poisoned.into_inner())
2144 .insert(module_id.to_string(), nonce);
2145 }
2146
2147 pub fn set_reserved_nonce(&self, module_id: &str, nonce: String) {
2150 self.reserved_nonces
2151 .lock()
2152 .unwrap_or_else(|poisoned| poisoned.into_inner())
2153 .insert(module_id.to_string(), Some(nonce));
2154 }
2155
2156 pub fn set_reserved_prefixes(&self, owner_module_id: &str, prefixes: &[String]) {
2158 let mut owners = self
2159 .reserved_prefix_owners
2160 .lock()
2161 .unwrap_or_else(|poisoned| poisoned.into_inner());
2162 owners.retain(|_, owner| owner != owner_module_id);
2163 for prefix in prefixes {
2164 owners.insert(prefix.clone(), owner_module_id.to_string());
2165 }
2166 }
2167
2168 #[cfg(test)]
2170 pub(crate) fn spawn_nonce(&self, module_id: &str) -> Option<String> {
2171 self.spawn_nonces
2172 .lock()
2173 .unwrap_or_else(|poisoned| poisoned.into_inner())
2174 .get(module_id)
2175 .cloned()
2176 }
2177
2178 fn apply_identity_configuration(&self, spec: &ModuleSpec) {
2179 self.set_reserved_prefixes(&spec.module_id, &spec.reserved_prefixes);
2180 let spawn_nonce = self
2181 .spawn_nonces
2182 .lock()
2183 .unwrap_or_else(|poisoned| poisoned.into_inner())
2184 .get(&spec.module_id)
2185 .cloned();
2186 let mut reserved_nonces = self
2187 .reserved_nonces
2188 .lock()
2189 .unwrap_or_else(|poisoned| poisoned.into_inner());
2190 if spec.reserved {
2191 reserved_nonces.insert(spec.module_id.clone(), spawn_nonce);
2196 }
2197 drop(reserved_nonces);
2198 self.removal_tombstones
2202 .lock()
2203 .unwrap_or_else(|poisoned| poisoned.into_inner())
2204 .remove(&spec.module_id);
2205 }
2206
2207 pub fn reserved_hello_authorized(&self, module_id: &str, presented: Option<&str>) -> bool {
2212 self.reserved_hello_rejection(module_id, presented)
2213 .is_none()
2214 }
2215
2216 pub(crate) fn reserved_hello_rejection(
2217 &self,
2218 module_id: &str,
2219 presented: Option<&str>,
2220 ) -> Option<ReservedHelloRejection> {
2221 let nonces = self
2222 .reserved_nonces
2223 .lock()
2224 .unwrap_or_else(|poisoned| poisoned.into_inner());
2225 if let Some(expected) = nonces.get(module_id) {
2226 let authorized = match expected {
2230 Some(expected) => {
2231 presented.is_some_and(|p| constant_time_eq(expected.as_bytes(), p.as_bytes()))
2232 }
2233 None => false,
2234 };
2235 if authorized {
2236 return None;
2237 }
2238 return Some(ReservedHelloRejection::Exact {
2239 module_id: module_id.to_string(),
2240 });
2241 }
2242 drop(nonces);
2243
2244 let matched_prefix = self
2245 .reserved_prefix_owners
2246 .lock()
2247 .unwrap_or_else(|poisoned| poisoned.into_inner())
2248 .iter()
2249 .filter(|(prefix, _)| module_id.starts_with(prefix.as_str()))
2250 .max_by_key(|(prefix, _)| prefix.len())
2251 .map(|(prefix, owner)| (prefix.clone(), owner.clone()));
2252 let (prefix, owner_module_id) = matched_prefix?;
2253
2254 let authorized = presented.is_some_and(|presented| {
2255 self.spawn_nonces
2256 .lock()
2257 .unwrap_or_else(|poisoned| poisoned.into_inner())
2258 .get(&owner_module_id)
2259 .is_some_and(|expected| constant_time_eq(expected.as_bytes(), presented.as_bytes()))
2260 || self.swap_nonce_matches(&owner_module_id, presented)
2263 });
2264 if authorized {
2265 None
2266 } else {
2267 Some(ReservedHelloRejection::Prefix {
2268 prefix,
2269 owner_module_id,
2270 })
2271 }
2272 }
2273
2274 pub fn spawned_consumer_authorized(&self, module_id: &str, presented: &str) -> bool {
2279 if presented.is_empty() {
2280 return false;
2281 }
2282 let nonces = self
2283 .spawn_nonces
2284 .lock()
2285 .unwrap_or_else(|poisoned| poisoned.into_inner());
2286 let current = nonces
2287 .get(module_id)
2288 .is_some_and(|expected| constant_time_eq(expected.as_bytes(), presented.as_bytes()));
2289 drop(nonces);
2290 current || self.swap_nonce_matches(module_id, presented)
2295 }
2296
2297 fn swap_nonce_matches(&self, module_id: &str, presented: &str) -> bool {
2299 let swaps = self
2300 .swaps
2301 .lock()
2302 .unwrap_or_else(|poisoned| poisoned.into_inner());
2303 swaps.get(module_id).is_some_and(|swap| {
2304 constant_time_eq(swap.candidate_nonce.as_bytes(), presented.as_bytes())
2305 || swap.incumbent_nonce.as_deref().is_some_and(|incumbent| {
2306 constant_time_eq(incumbent.as_bytes(), presented.as_bytes())
2307 })
2308 })
2309 }
2310
2311 pub(crate) fn open_swap(&self, module_id: &str, candidate_nonce: String) {
2314 let incumbent_nonce = self
2315 .spawn_nonces
2316 .lock()
2317 .unwrap_or_else(|poisoned| poisoned.into_inner())
2318 .get(module_id)
2319 .cloned();
2320 self.swaps
2321 .lock()
2322 .unwrap_or_else(|poisoned| poisoned.into_inner())
2323 .insert(
2324 module_id.to_string(),
2325 OpenSwap {
2326 candidate_nonce,
2327 incumbent_nonce,
2328 candidate_admitted: false,
2329 },
2330 );
2331 }
2332
2333 pub(crate) fn close_swap(&self, module_id: &str) {
2336 self.swaps
2337 .lock()
2338 .unwrap_or_else(|poisoned| poisoned.into_inner())
2339 .remove(module_id);
2340 }
2341
2342 pub(crate) fn set_swap_promotion_observer(
2345 &self,
2346 observer: std::sync::Weak<dyn SwapPromotionObserver>,
2347 ) {
2348 *self
2349 .promotion_observer
2350 .0
2351 .lock()
2352 .unwrap_or_else(|poisoned| poisoned.into_inner()) = Some(observer);
2353 }
2354
2355 fn notify_swap_promoted(&self, registration: &crate::registry::ModuleRegistration) {
2358 let observer = self
2359 .promotion_observer
2360 .0
2361 .lock()
2362 .unwrap_or_else(|poisoned| poisoned.into_inner())
2363 .as_ref()
2364 .and_then(std::sync::Weak::upgrade);
2365 if let Some(observer) = observer {
2366 observer.swap_promoted(registration);
2367 }
2368 }
2369
2370 pub(crate) fn swap_open(&self, module_id: &str) -> bool {
2372 self.swaps
2373 .lock()
2374 .unwrap_or_else(|poisoned| poisoned.into_inner())
2375 .contains_key(module_id)
2376 }
2377
2378 fn promote_swap_nonce(&self, module_id: &str, reserved: bool) {
2383 let candidate_nonce = self
2384 .swaps
2385 .lock()
2386 .unwrap_or_else(|poisoned| poisoned.into_inner())
2387 .get(module_id)
2388 .map(|swap| swap.candidate_nonce.clone());
2389 let Some(nonce) = candidate_nonce else {
2390 return;
2391 };
2392 self.set_spawn_nonce(module_id, nonce.clone());
2393 if reserved {
2394 self.set_reserved_nonce(module_id, nonce);
2395 }
2396 }
2397
2398 pub(crate) fn swap_hello_admission(
2413 &self,
2414 module_id: &str,
2415 presented: Option<&str>,
2416 ) -> SwapHelloAdmission {
2417 let swaps = self
2418 .swaps
2419 .lock()
2420 .unwrap_or_else(|poisoned| poisoned.into_inner());
2421 let Some(swap) = swaps.get(module_id) else {
2422 return SwapHelloAdmission::NotSwapping;
2423 };
2424 let Some(presented) = presented else {
2425 return SwapHelloAdmission::Refused;
2426 };
2427 if constant_time_eq(swap.candidate_nonce.as_bytes(), presented.as_bytes()) {
2428 return if swap.candidate_admitted {
2429 SwapHelloAdmission::Refused
2430 } else {
2431 SwapHelloAdmission::Candidate
2432 };
2433 }
2434 if swap
2435 .incumbent_nonce
2436 .as_deref()
2437 .is_some_and(|incumbent| constant_time_eq(incumbent.as_bytes(), presented.as_bytes()))
2438 {
2439 return SwapHelloAdmission::NotSwapping;
2440 }
2441 SwapHelloAdmission::Refused
2442 }
2443
2444 pub(crate) fn mark_swap_candidate_admitted(&self, module_id: &str) {
2447 if let Some(swap) = self
2448 .swaps
2449 .lock()
2450 .unwrap_or_else(|poisoned| poisoned.into_inner())
2451 .get_mut(module_id)
2452 {
2453 swap.candidate_admitted = true;
2454 }
2455 }
2456
2457 pub fn spawn_launch_nonce_for(&self, module_id: &str) -> Option<String> {
2459 self.spawn_nonces
2460 .lock()
2461 .unwrap_or_else(|poisoned| poisoned.into_inner())
2462 .get(module_id)
2463 .cloned()
2464 }
2465
2466 pub fn reserved_launch_nonce_for(&self, module_id: &str) -> Option<String> {
2468 self.reserved_nonces
2469 .lock()
2470 .unwrap_or_else(|poisoned| poisoned.into_inner())
2471 .get(module_id)
2472 .cloned()
2473 .flatten()
2474 }
2475
2476 pub fn insert(&self, module: SupervisedModule) -> Option<SupervisedModule> {
2477 self.mark_configured(module.module_id());
2481 let mut modules = self
2482 .modules
2483 .lock()
2484 .unwrap_or_else(|poisoned| poisoned.into_inner());
2485 modules.insert(module.module_id().to_string(), module)
2486 }
2487
2488 fn mark_configured(&self, module_id: &str) {
2492 self.configured_ids
2493 .lock()
2494 .unwrap_or_else(|poisoned| poisoned.into_inner())
2495 .insert(module_id.to_string());
2496 }
2497
2498 fn unmark_configured_unless_rostered(&self, module_id: &str) {
2502 let modules = self
2503 .modules
2504 .lock()
2505 .unwrap_or_else(|poisoned| poisoned.into_inner());
2506 if !modules.contains_key(module_id) {
2507 self.configured_ids
2508 .lock()
2509 .unwrap_or_else(|poisoned| poisoned.into_inner())
2510 .remove(module_id);
2511 }
2512 }
2513
2514 pub(crate) fn is_configured(&self, module_id: &str) -> bool {
2522 self.configured_ids
2523 .lock()
2524 .unwrap_or_else(|poisoned| poisoned.into_inner())
2525 .contains(module_id)
2526 }
2527
2528 pub fn get(&self, module_id: &str) -> Option<SupervisedModule> {
2529 let modules = self
2530 .modules
2531 .lock()
2532 .unwrap_or_else(|poisoned| poisoned.into_inner());
2533 modules.get(module_id).cloned()
2534 }
2535
2536 pub(crate) fn record_late_health_answer(
2537 &self,
2538 module_id: &str,
2539 latency_ms: u64,
2540 ) -> Result<bool, SuperviseError> {
2541 let Some(module) = self.get(module_id) else {
2542 return Ok(false);
2543 };
2544 update_snapshot(&module.inner.snapshot, Some(module_id), |state| {
2545 state.health.late_answer_count = state.health.late_answer_count.saturating_add(1);
2546 state.health.last_late_answer_latency_ms = Some(latency_ms);
2547 state.health.consecutive_failures = 0;
2555 })?;
2556 Ok(true)
2557 }
2558
2559 pub fn record_deliberate_severance(&self, module_id: &str) -> Result<bool, SuperviseError> {
2565 let Some(module) = self.get(module_id) else {
2566 return Ok(false);
2567 };
2568 let snapshot = lock_snapshot(&module.inner.snapshot)?;
2569 let Some((pid, start_time)) = snapshot.pid.zip(snapshot.process_start_time) else {
2570 return Ok(false);
2571 };
2572 drop(snapshot);
2573 module.record_deliberate_severance(ProcessIdentity { pid, start_time })
2574 }
2575
2576 pub fn list(&self) -> Vec<SupervisedModule> {
2577 let modules = self
2578 .modules
2579 .lock()
2580 .unwrap_or_else(|poisoned| poisoned.into_inner());
2581 let mut modules = modules.values().cloned().collect::<Vec<_>>();
2582 modules.sort_by(|left, right| left.module_id().cmp(right.module_id()));
2583 modules
2584 }
2585
2586 pub(crate) fn retire(&self, module_id: &str) -> Option<SupervisedModule> {
2587 self.activity_images
2588 .lock()
2589 .unwrap_or_else(|p| p.into_inner())
2590 .remove(module_id);
2591 self.spawn_nonces
2592 .lock()
2593 .unwrap_or_else(|poisoned| poisoned.into_inner())
2594 .remove(module_id);
2595 self.close_swap(module_id);
2596 let mut reserved_nonces = self
2597 .reserved_nonces
2598 .lock()
2599 .unwrap_or_else(|poisoned| poisoned.into_inner());
2600 if reserved_nonces.contains_key(module_id) {
2601 reserved_nonces.insert(module_id.to_string(), None);
2604 }
2605 drop(reserved_nonces);
2606 self.reserved_prefix_owners
2607 .lock()
2608 .unwrap_or_else(|poisoned| poisoned.into_inner())
2609 .retain(|_, owner| owner != module_id);
2610 let removed = self
2611 .modules
2612 .lock()
2613 .unwrap_or_else(|poisoned| poisoned.into_inner())
2614 .remove(module_id);
2615 self.configured_ids
2616 .lock()
2617 .unwrap_or_else(|poisoned| poisoned.into_inner())
2618 .remove(module_id);
2619 removed
2620 }
2621
2622 pub(crate) fn record_rescan_removal(&self, module_id: &str) {
2625 self.removal_tombstones
2626 .lock()
2627 .unwrap_or_else(|poisoned| poisoned.into_inner())
2628 .insert(module_id.to_string(), unix_ms_now());
2629 }
2630
2631 pub(crate) fn removal_tombstone_age_ms(&self, module_id: &str) -> Option<u64> {
2633 self.removal_tombstones
2634 .lock()
2635 .unwrap_or_else(|poisoned| poisoned.into_inner())
2636 .get(module_id)
2637 .copied()
2638 .map(|removed_at_ms| unix_ms_now().saturating_sub(removed_at_ms))
2639 }
2640
2641 pub(crate) fn release_retained_reserved_gate(&self, module_id: &str) -> bool {
2646 if self.get(module_id).is_some() {
2647 return false;
2648 }
2649 let mut reserved_nonces = self
2650 .reserved_nonces
2651 .lock()
2652 .unwrap_or_else(|poisoned| poisoned.into_inner());
2653 if !matches!(reserved_nonces.get(module_id), Some(None)) {
2654 return false;
2655 }
2656 reserved_nonces.remove(module_id);
2657 true
2658 }
2659
2660 pub(crate) fn operation_lock(&self) -> Arc<AsyncMutex<()>> {
2661 Arc::clone(&self.operation_lock)
2662 }
2663}
2664
2665#[derive(Debug, Clone)]
2667pub struct Supervisor {
2668 registry: Arc<Registry>,
2669 restart_policy: RestartPolicy,
2670 drain_timeout: Duration,
2671 connection_file_path: Option<PathBuf>,
2672 capture_logs_dir: Option<PathBuf>,
2673 forwarding: Option<Arc<ForwardingTable>>,
2674 process_liveness: Arc<SupervisorProcessLiveness>,
2675 supervisor_handle: Option<SupervisorHandle>,
2676 health: HealthConfig,
2677 daemon_start_clock: crate::clock::StartClock,
2678 terminal_journal: Option<Arc<crate::terminal_journal::TerminalJournal>>,
2679 spawn_events: SpawnEventFeed,
2680 provenance_probe: ExecutableIdentityProbe,
2681 child_roster: ChildRoster,
2684 #[cfg(target_os = "linux")]
2685 cgroup_placement: Option<subc_cgroup::Placement>,
2686 #[cfg(test)]
2687 test_after_first_spawn: AfterFirstSpawnHook,
2688}
2689
2690pub(crate) struct RegisteredConfiguredModule {
2694 spec: ModuleSpec,
2695 runtime: SupervisorRuntimeConfig,
2696 module: SupervisedModule,
2697 commands: mpsc::Receiver<SupervisorCommand>,
2698}
2699
2700#[cfg(test)]
2706#[derive(Clone, Default)]
2707struct AfterFirstSpawnHook(Option<AfterFirstSpawnFn>);
2708
2709#[cfg(test)]
2710type AfterFirstSpawnFn = Arc<dyn Fn(&str) + Send + Sync>;
2711
2712#[cfg(test)]
2713impl fmt::Debug for AfterFirstSpawnHook {
2714 fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
2715 f.write_str("AfterFirstSpawnHook")
2716 }
2717}
2718
2719#[cfg(test)]
2720impl AfterFirstSpawnHook {
2721 fn run(&self, module_id: &str) {
2722 if let Some(hook) = &self.0 {
2723 hook(module_id);
2724 }
2725 }
2726}
2727
2728impl Supervisor {
2729 #[cfg(test)]
2730 pub(crate) fn new_for_test(registry: Arc<Registry>, policy: RestartPolicy) -> Self {
2731 let supervisor = Self::new(registry, policy);
2732 #[cfg(target_os = "macos")]
2733 let supervisor = supervisor.with_privacy_trampoline(test_privacy_trampoline());
2734 supervisor
2735 }
2736 pub fn with_privacy_trampoline(self, path: impl Into<PathBuf>) -> Self {
2741 let path = path.into();
2742 #[cfg(target_os = "macos")]
2743 {
2744 let result = probe_privacy_trampoline(&path).map(|()| path);
2745 if let Err(cause) = &result {
2746 error!(%cause, "macOS privacy identity unavailable; supervised launches will refuse");
2747 }
2748 self.child_roster.set_privacy_trampoline(result);
2749 }
2750 #[cfg(not(target_os = "macos"))]
2751 let _ = path;
2752 self
2753 }
2754 pub(crate) fn daemon_shutdown_flag(&self) -> crate::child_roster::DaemonShutdownFlag {
2757 self.child_roster.shutdown_flag()
2758 }
2759
2760 #[cfg(unix)]
2771 pub(crate) fn begin_daemon_shutdown(&self) {
2772 self.child_roster.close();
2773 if let Some(journal) = &self.terminal_journal {
2774 journal.stamp_shutdown();
2775 }
2776 }
2777
2778 #[cfg(unix)]
2782 pub(crate) async fn drain_for_daemon_shutdown(&self) -> Result<(), SuperviseError> {
2783 const NOTICE_BUDGET: Duration = Duration::from_millis(500);
2784 const DRAIN_BUDGET: Duration = Duration::from_secs(2);
2785 let Some(forwarding) = &self.forwarding else {
2786 return Ok(());
2787 };
2788 let modules = forwarding
2789 .begin_daemon_drain()
2790 .map_err(SuperviseError::Forwarding)?;
2791 let deadline_ms =
2792 unix_ms_now().saturating_add((NOTICE_BUDGET + DRAIN_BUDGET).as_millis() as u64);
2793 let mut notices = tokio::task::JoinSet::new();
2794 let mut drains = Vec::new();
2795 for (module_id, endpoint) in modules {
2796 let Some(target) = forwarding
2797 .begin_endpoint_drain(endpoint, RouteCloseReason::Restart)
2798 .map_err(SuperviseError::Forwarding)?
2799 else {
2800 continue;
2801 };
2802 let routes = forwarding
2803 .endpoint_routes(target.endpoint)
2804 .map_err(SuperviseError::Forwarding)?;
2805 let command = serde_json::to_vec(&ModuleControlCommand::Draining {
2811 reason: RouteCloseReason::Restart,
2812 deadline_ms,
2813 })
2814 .expect("module draining serializes");
2815 let mut recipients = vec![(target.sink.clone(), target.negotiated_ver, command)];
2816 let mut clients: Vec<(crate::forwarding::GoodbyeTarget, Vec<u16>)> = Vec::new();
2817 for route in routes {
2818 let client = route.goodbye_target;
2819 if let Some((_, channels)) = clients
2820 .iter_mut()
2821 .find(|(existing, _)| existing.connection_id == client.connection_id)
2822 {
2823 channels.push(client.channel);
2824 } else {
2825 let channel = client.channel;
2826 clients.push((client, vec![channel]));
2827 }
2828 }
2829 for (client, mut channels) in clients {
2830 channels.sort_unstable();
2831 channels.dedup();
2832 let closing = serde_json::to_vec(&ClientControlPush::RouteClosing {
2833 module_id: module_id.clone(),
2834 channels,
2835 reason: RouteCloseReason::Restart,
2836 })
2837 .expect("route closing serializes");
2838 recipients.push((client.sink, client.negotiated_ver, closing));
2839 }
2840 for (sink, version, body) in recipients {
2841 notices.spawn(async move {
2842 let frame = Frame::build_with_version(
2843 version,
2844 FrameType::Push,
2845 control_flags(),
2846 0,
2847 0,
2848 0,
2849 body,
2850 )
2851 .expect("bounded lifecycle notice frame builds");
2852 sink.send_flushed(frame).await
2853 });
2854 }
2855 let gauges = declared_busy_gauges(&self.registry, &module_id)?;
2856 drains.push((module_id, target.endpoint, gauges));
2857 }
2858 let notice_deadline = Instant::now() + NOTICE_BUDGET;
2861 while let Ok(Some(result)) = timeout_at(notice_deadline, notices.join_next()).await {
2862 if !matches!(result, Ok(Ok(()))) {
2863 warn!(?result, "daemon shutdown notice delivery failed");
2864 }
2865 }
2866 notices.abort_all();
2867 let deadline = Instant::now() + DRAIN_BUDGET;
2868 let mut waits = tokio::task::JoinSet::new();
2869 for (module_id, endpoint, gauges) in drains {
2870 let forwarding = Arc::clone(forwarding);
2871 let mut runtime = self.runtime_config();
2872 runtime.health.cadence = Duration::from_millis(100);
2873 waits.spawn(async move {
2874 let wait_result = wait_for_forwarding_quiescence(
2875 &forwarding,
2876 &module_id,
2877 &runtime,
2878 endpoint,
2879 deadline,
2880 &gauges,
2881 DrainScope::Endpoint(endpoint),
2882 &mut DrainGaugeTrace::new(&gauges),
2883 )
2884 .await;
2885 forwarding
2889 .record_daemon_drain_result(
2890 endpoint,
2891 drained_after_quiescence_wait(&wait_result),
2892 )
2893 .map_err(SuperviseError::Forwarding)?;
2894 wait_result
2895 });
2896 }
2897 while let Ok(Some(result)) = timeout_at(deadline, waits.join_next()).await {
2898 if !matches!(result, Ok(Ok(true))) {
2899 warn!(?result, "daemon shutdown drain did not reach quiescence");
2900 }
2901 }
2902 Ok(())
2903 }
2904
2905 #[cfg(unix)]
2917 pub(crate) async fn end_children_for_daemon_shutdown(
2918 &self,
2919 already_escalated: bool,
2920 escalate: impl std::future::Future<Output = ()>,
2921 ) {
2922 tokio::pin!(escalate);
2923 let mut escalated = already_escalated;
2924 if let Some(forwarding) = &self.forwarding {
2925 let reason = CloseReason::new(
2926 "daemon_shutdown",
2927 "the daemon is exiting after its shutdown notice and drain",
2928 );
2929 if escalated {
2930 send_module_goodbyes_for_daemon_shutdown(forwarding, &reason, false).await;
2933 } else {
2934 tokio::select! {
2935 biased;
2936 _ = escalate.as_mut() => {
2937 info!("second SIGTERM: abandoning module GOODBYE delivery");
2938 escalated = true;
2939 }
2940 _ = send_module_goodbyes_for_daemon_shutdown(forwarding, &reason, true) => {}
2941 }
2942 }
2943 let closed = forwarding.close_all_connections(&reason);
2944 debug!(closed, "closed established connections for daemon shutdown");
2945 }
2946 let escalated_here = escalated && !already_escalated;
2950 let remaining_escalate = async move {
2951 if escalated_here {
2952 std::future::pending::<()>().await;
2953 } else {
2954 escalate.await;
2955 }
2956 };
2957 crate::child_roster::end_children_for_daemon_shutdown(
2958 &self.child_roster,
2959 escalated,
2960 remaining_escalate,
2961 )
2962 .await;
2963 }
2964
2965 pub fn new(registry: Arc<Registry>, restart_policy: RestartPolicy) -> Self {
2966 Self {
2967 registry,
2968 restart_policy,
2969 drain_timeout: DEFAULT_DRAIN_TIMEOUT,
2970 connection_file_path: None,
2971 capture_logs_dir: None,
2972 forwarding: None,
2973 process_liveness: Arc::new(SupervisorProcessLiveness::default()),
2974 supervisor_handle: None,
2975 health: HealthConfig::default(),
2976 daemon_start_clock: crate::clock::StartClock::capture(),
2977 terminal_journal: None,
2978 spawn_events: SpawnEventFeed::default(),
2979 provenance_probe: ExecutableIdentityProbe::default(),
2980 child_roster: ChildRoster::default(),
2981 #[cfg(target_os = "linux")]
2982 cgroup_placement: None,
2983 #[cfg(test)]
2984 test_after_first_spawn: AfterFirstSpawnHook::default(),
2985 }
2986 }
2987
2988 pub fn with_drain_timeout(mut self, drain_timeout: Duration) -> Self {
2989 self.drain_timeout = drain_timeout;
2990 self
2991 }
2992
2993 pub fn with_process_liveness(
2994 mut self,
2995 process_liveness: Arc<SupervisorProcessLiveness>,
2996 ) -> Self {
2997 self.process_liveness = process_liveness;
2998 self
2999 }
3000
3001 pub fn with_connection_file_path(mut self, connection_file_path: impl Into<PathBuf>) -> Self {
3002 self.connection_file_path = Some(connection_file_path.into());
3003 self
3004 }
3005
3006 pub fn with_capture_logs_dir(mut self, logs_dir: impl Into<PathBuf>) -> Self {
3008 self.capture_logs_dir = Some(logs_dir.into());
3009 self
3010 }
3011
3012 pub fn with_daemon_incarnation(self, daemon_incarnation: String) -> Self {
3015 self.spawn_events.configure_incarnation(daemon_incarnation);
3019 self
3020 }
3021
3022 pub fn with_terminal_journal(self, path: PathBuf, daemon_incarnation: String) -> Self {
3025 let mut this = self.with_daemon_incarnation(daemon_incarnation.clone());
3026 this.terminal_journal = Some(Arc::new(crate::terminal_journal::TerminalJournal::open(
3027 path,
3028 daemon_incarnation,
3029 )));
3030 this
3031 }
3032
3033 pub fn with_forwarding(mut self, forwarding: Arc<ForwardingTable>) -> Self {
3034 self.forwarding = Some(forwarding);
3035 self
3036 }
3037
3038 pub fn with_handle(mut self, supervisor_handle: SupervisorHandle) -> Self {
3039 self.spawn_events = supervisor_handle.spawn_events.clone();
3040 self.supervisor_handle = Some(supervisor_handle);
3041 self
3042 }
3043
3044 pub fn with_health_config(mut self, health: HealthConfig) -> Self {
3045 self.health = health;
3046 self
3047 }
3048
3049 pub fn with_live_children_record(self, path: impl Into<PathBuf>) -> Self {
3053 self.child_roster.record_to(path.into());
3054 self
3055 }
3056
3057 #[cfg(target_os = "linux")]
3058 pub fn with_cgroup_placement(
3059 mut self,
3060 cgroup_placement: Option<subc_cgroup::Placement>,
3061 ) -> Self {
3062 self.cgroup_placement = cgroup_placement;
3063 self
3064 }
3065
3066 pub fn spawn(&self, spec: ModuleSpec) -> Result<SupervisedModule, SuperviseError> {
3072 validate_spec(&spec)?;
3073 self.establish_identity(&spec);
3074
3075 let runtime = self.runtime_config();
3076 let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::starting()));
3077 let activity_spawn = runtime.begin_activity_spawn(&spec.module_id);
3078 let spawned = spawn_child(
3079 &spec,
3080 runtime.connection_file_path.as_deref(),
3081 self.supervisor_handle.as_ref(),
3082 &runtime.stderr_ring,
3083 runtime.capture_logs_dir.as_deref(),
3084 &runtime.child_roster,
3085 #[cfg(target_os = "linux")]
3086 runtime.cgroup_placement.as_ref(),
3087 );
3088 #[cfg(test)]
3089 self.test_after_first_spawn.run(&spec.module_id);
3090 let child = match spawned {
3091 Ok(child) => child,
3092 Err(err) => {
3093 self.abandon_unrostered(&spec.module_id);
3096 return Err(err);
3097 }
3098 };
3099 self.mark_first_process_running(&spec, &runtime, &snapshot, &child)?;
3100 if let Some(spawn) = activity_spawn {
3101 spawn.complete(&snapshot);
3102 }
3103
3104 Ok(self.supervised_module(spec, runtime, snapshot, Some(child)))
3105 }
3106
3107 fn establish_identity(&self, spec: &ModuleSpec) {
3120 if let Some(supervisor_handle) = &self.supervisor_handle {
3121 supervisor_handle.apply_identity_configuration(spec);
3122 supervisor_handle.mark_configured(&spec.module_id);
3123 }
3124 }
3125
3126 fn abandon_unrostered(&self, module_id: &str) {
3129 if let Some(supervisor_handle) = &self.supervisor_handle {
3130 supervisor_handle.unmark_configured_unless_rostered(module_id);
3131 }
3132 }
3133
3134 fn mark_first_process_running(
3137 &self,
3138 spec: &ModuleSpec,
3139 runtime: &SupervisorRuntimeConfig,
3140 snapshot: &SharedSnapshot,
3141 child: &SupervisedChild,
3142 ) -> Result<(), SuperviseError> {
3143 if let Err(err) = set_running(snapshot, child, &spec.module_id, &runtime.spawn_events) {
3144 self.abandon_unrostered(&spec.module_id);
3145 return Err(err);
3146 }
3147 self.process_liveness
3148 .track(spec.module_id.clone(), Arc::clone(snapshot));
3149 Ok(())
3150 }
3151
3152 pub fn supervise_configured(
3158 &self,
3159 spec: ModuleSpec,
3160 enabled: bool,
3161 ) -> Result<SupervisedModule, SuperviseError> {
3162 self.supervise_configured_requested(
3163 spec,
3164 enabled,
3165 self.health.clone(),
3166 None,
3167 self.restart_policy,
3168 None,
3169 )
3170 }
3171
3172 pub fn supervise_configured_with_health(
3178 &self,
3179 spec: ModuleSpec,
3180 enabled: bool,
3181 health: HealthConfig,
3182 drain_timeout_ms: Option<u64>,
3183 restart_policy: RestartPolicy,
3184 ) -> Result<SupervisedModule, SuperviseError> {
3185 self.supervise_configured_requested(
3186 spec,
3187 enabled,
3188 health,
3189 drain_timeout_ms,
3190 restart_policy,
3191 None,
3192 )
3193 }
3194
3195 pub(crate) fn supervise_configured_requested(
3196 &self,
3197 spec: ModuleSpec,
3198 enabled: bool,
3199 health: HealthConfig,
3200 drain_timeout_ms: Option<u64>,
3201 restart_policy: RestartPolicy,
3202 requester: Option<Principal>,
3203 ) -> Result<SupervisedModule, SuperviseError> {
3204 let registered = self.register_configured_with_health(
3205 spec,
3206 enabled,
3207 health,
3208 drain_timeout_ms,
3209 restart_policy,
3210 )?;
3211 self.start_registered_configured(registered, requester)
3212 }
3213
3214 pub(crate) fn register_configured_with_health(
3219 &self,
3220 spec: ModuleSpec,
3221 enabled: bool,
3222 health: HealthConfig,
3223 drain_timeout_ms: Option<u64>,
3224 restart_policy: RestartPolicy,
3225 ) -> Result<RegisteredConfiguredModule, SuperviseError> {
3226 validate_spec(&spec)?;
3227 self.establish_identity(&spec);
3228
3229 let mut runtime = self.runtime_config();
3230 runtime.health = health;
3231 runtime.restart_policy = restart_policy;
3232 if let Some(ms) = drain_timeout_ms {
3233 runtime.drain_timeout = Duration::from_millis(ms);
3234 *runtime
3235 .effective_drain_timeout
3236 .lock()
3237 .unwrap_or_else(|poisoned| poisoned.into_inner()) = runtime.drain_timeout;
3238 }
3239 let snapshot = if enabled {
3240 SupervisorSnapshot::starting()
3241 } else {
3242 SupervisorSnapshot::disabled()
3243 };
3244 let snapshot = {
3245 let mut snapshot = snapshot;
3246 if enabled {
3247 if let Some((pid, reason)) = runtime.child_roster.previous_instance(&spec.module_id)
3248 {
3249 snapshot.waiting_for_previous_pid = Some(pid);
3250 snapshot.waiting_for_previous_detail = Some(reason.waiting_message(pid));
3251 }
3252 }
3253 snapshot
3254 };
3255 let (module, commands) =
3256 self.register_module(&spec, &runtime, Arc::new(Mutex::new(snapshot)));
3257 Ok(RegisteredConfiguredModule {
3258 spec,
3259 runtime,
3260 module,
3261 commands,
3262 })
3263 }
3264
3265 pub(crate) fn start_registered_configured(
3268 &self,
3269 registered: RegisteredConfiguredModule,
3270 requester: Option<Principal>,
3271 ) -> Result<SupervisedModule, SuperviseError> {
3272 let RegisteredConfiguredModule {
3273 spec,
3274 mut runtime,
3275 module,
3276 commands,
3277 } = registered;
3278 runtime.start_requester = requester;
3279 let snapshot = &module.inner.snapshot;
3280 let should_spawn = {
3281 let state = lock_snapshot(snapshot)?;
3282 state.enabled && state.waiting_for_previous_pid.is_none()
3283 };
3284 let child = if should_spawn {
3285 let activity_spawn = runtime.begin_activity_spawn(&spec.module_id);
3286 let spawned = spawn_child(
3287 &spec,
3288 runtime.connection_file_path.as_deref(),
3289 self.supervisor_handle.as_ref(),
3290 &runtime.stderr_ring,
3291 runtime.capture_logs_dir.as_deref(),
3292 &runtime.child_roster,
3293 #[cfg(target_os = "linux")]
3294 runtime.cgroup_placement.as_ref(),
3295 );
3296 #[cfg(test)]
3297 self.test_after_first_spawn.run(&spec.module_id);
3298 match spawned {
3299 Ok(child) => {
3300 self.mark_first_process_running(&spec, &runtime, snapshot, &child)?;
3301 if let Some(spawn) = activity_spawn {
3302 spawn.complete(snapshot);
3303 }
3304 Some(child)
3305 }
3306 Err(err) => {
3307 if runtime.health.critical {
3308 error!(
3309 module_id = %spec.module_id,
3310 program = %spec.program.display(),
3311 error = %err,
3312 "critical configured module failed to spawn; marking failed and alerting"
3313 );
3314 } else {
3315 error!(
3316 module_id = %spec.module_id,
3317 program = %spec.program.display(),
3318 error = %err,
3319 "configured module failed to spawn; marking failed and continuing"
3320 );
3321 }
3322 update_snapshot(snapshot, Some(&spec.module_id), |state| {
3323 state.state = ModuleState::Failed;
3324 clear_current_process_facts(state);
3325 })?;
3326 None
3327 }
3328 }
3329 } else {
3330 None
3331 };
3332 self.start_monitor(spec, runtime, &module, child, commands);
3333 Ok(module)
3334 }
3335
3336 fn runtime_config(&self) -> SupervisorRuntimeConfig {
3337 let effective_drain_timeout = Arc::new(Mutex::new(self.drain_timeout));
3338 SupervisorRuntimeConfig {
3339 start_requester: None,
3340 pending_activity_restart: false,
3341 provenance_probe: self.provenance_probe.clone(),
3342 scheduled_respawn: Arc::default(),
3343 deferred_reload_reply: Arc::default(),
3344 restart_policy: self.restart_policy,
3345 drain_timeout: self.drain_timeout,
3346 child_roster: self
3349 .child_roster
3350 .for_module(Arc::clone(&effective_drain_timeout)),
3351 effective_drain_timeout,
3352 default_drain_timeout: self.drain_timeout,
3353 health: self.health.clone(),
3354 connection_file_path: self.connection_file_path.clone(),
3355 capture_logs_dir: self.capture_logs_dir.clone(),
3356 forwarding: self.forwarding.clone(),
3357 supervisor_handle: self.supervisor_handle.clone(),
3358 stderr_ring: Arc::new(Mutex::new(StderrRing::new(StderrTailConfig::default()))),
3359 terminal_ring: Arc::new(Mutex::new(
3360 TerminalRing::new(
3361 TerminalRingConfig::default(),
3362 self.daemon_start_clock.started_at_ms(),
3363 )
3364 .with_start_clock(self.daemon_start_clock)
3365 .with_journal(self.terminal_journal.clone())
3366 .with_daemon_shutdown(self.child_roster.shutdown_flag()),
3367 )),
3368 spawn_events: self.spawn_events.clone(),
3369 #[cfg(target_os = "linux")]
3370 cgroup_placement: self.cgroup_placement.clone(),
3371 #[cfg(test)]
3372 test_seed_stale_facts_before_enable_spawn: false,
3373 #[cfg(test)]
3374 test_reload_exit_record_gate: None,
3375 }
3376 }
3377
3378 fn supervised_module(
3379 &self,
3380 spec: ModuleSpec,
3381 runtime: SupervisorRuntimeConfig,
3382 snapshot: SharedSnapshot,
3383 child: Option<SupervisedChild>,
3384 ) -> SupervisedModule {
3385 let (module, commands) = self.register_module(&spec, &runtime, snapshot);
3386 self.start_monitor(spec, runtime, &module, child, commands);
3387 module
3388 }
3389
3390 fn register_module(
3391 &self,
3392 spec: &ModuleSpec,
3393 runtime: &SupervisorRuntimeConfig,
3394 snapshot: SharedSnapshot,
3395 ) -> (SupervisedModule, mpsc::Receiver<SupervisorCommand>) {
3396 let configuration = Arc::new(Mutex::new(SupervisedConfiguration {
3397 spec: spec.clone(),
3398 health: runtime.health.clone(),
3399 }));
3400 let stderr_ring = Arc::clone(&runtime.stderr_ring);
3401 let terminal_ring = Arc::clone(&runtime.terminal_ring);
3402 let restart_policy = runtime.restart_policy;
3406 let effective_drain_timeout = Arc::clone(&runtime.effective_drain_timeout);
3407 let (tx, rx) = mpsc::channel(4);
3408 let module_id = spec.module_id.clone();
3409 let module = SupervisedModule {
3410 inner: Arc::new(SupervisedModuleInner {
3411 module_id: module_id.clone(),
3412 registry: Arc::clone(&self.registry),
3413 snapshot,
3414 configuration,
3415 stderr_ring,
3416 terminal_ring,
3417 commands: tx,
3418 monitor: Mutex::new(None),
3419 restart_policy,
3420 effective_drain_timeout,
3421 provenance_probe: self.provenance_probe.clone(),
3422 }),
3423 };
3424 if let Some(supervisor_handle) = &self.supervisor_handle {
3427 supervisor_handle.insert(module.clone());
3428 }
3429 (module, rx)
3430 }
3431
3432 fn start_monitor(
3433 &self,
3434 spec: ModuleSpec,
3435 runtime: SupervisorRuntimeConfig,
3436 module: &SupervisedModule,
3437 child: Option<SupervisedChild>,
3438 commands: mpsc::Receiver<SupervisorCommand>,
3439 ) {
3440 let monitor = tokio::spawn(supervise_loop(
3441 spec,
3442 runtime,
3443 Arc::clone(&self.registry),
3444 Arc::clone(&self.process_liveness),
3445 Arc::clone(&module.inner.snapshot),
3446 child,
3447 commands,
3448 ));
3449 *module
3450 .inner
3451 .monitor
3452 .lock()
3453 .unwrap_or_else(|p| p.into_inner()) = Some(monitor);
3454 }
3455}
3456
3457impl Default for Supervisor {
3458 fn default() -> Self {
3459 Self::new(Arc::new(Registry::default()), RestartPolicy::default())
3460 }
3461}
3462
3463#[derive(Clone)]
3465pub struct SupervisedModule {
3466 inner: Arc<SupervisedModuleInner>,
3467}
3468
3469struct SupervisedModuleInner {
3470 module_id: String,
3471 registry: Arc<Registry>,
3472 snapshot: SharedSnapshot,
3473 configuration: Arc<Mutex<SupervisedConfiguration>>,
3474 stderr_ring: Arc<Mutex<StderrRing>>,
3475 terminal_ring: Arc<Mutex<TerminalRing>>,
3476 commands: mpsc::Sender<SupervisorCommand>,
3477 monitor: Mutex<Option<JoinHandle<()>>>,
3478 restart_policy: RestartPolicy,
3482 effective_drain_timeout: Arc<Mutex<Duration>>,
3483 provenance_probe: ExecutableIdentityProbe,
3484}
3485
3486impl fmt::Debug for SupervisedModule {
3487 fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
3488 f.debug_struct("SupervisedModule")
3489 .field("module_id", &self.inner.module_id)
3490 .field("status", &self.status())
3491 .finish_non_exhaustive()
3492 }
3493}
3494
3495impl SupervisedModule {
3496 pub fn module_id(&self) -> &str {
3497 &self.inner.module_id
3498 }
3499
3500 #[cfg(test)]
3504 pub(crate) fn record_health_probe_failure_for_test(
3505 &self,
3506 detail: &str,
3507 ) -> Result<(), SuperviseError> {
3508 update_snapshot(&self.inner.snapshot, Some(&self.inner.module_id), |state| {
3509 state.health.consecutive_failures = state.health.consecutive_failures.saturating_add(1);
3510 state.health.detail = Some(detail.to_string());
3511 })
3512 }
3513
3514 pub fn state(&self) -> Result<ModuleState, SuperviseError> {
3515 Ok(lock_snapshot(&self.inner.snapshot)?.state)
3516 }
3517
3518 pub(crate) fn registration_end_reason(
3519 &self,
3520 ) -> Result<Option<RegistrationEndReason>, SuperviseError> {
3521 let snapshot = lock_snapshot(&self.inner.snapshot)?;
3522 Ok(match snapshot.state {
3523 ModuleState::Draining if snapshot.draining_to_replace => {
3524 Some(RegistrationEndReason::SupervisorRestart)
3525 }
3526 ModuleState::Draining => Some(RegistrationEndReason::SupervisorStop),
3527 ModuleState::Restarting if snapshot.draining_to_replace => {
3528 Some(RegistrationEndReason::SupervisorRestart)
3529 }
3530 _ => None,
3531 })
3532 }
3533
3534 pub fn stderr_tail(
3541 &self,
3542 max_lines: Option<usize>,
3543 max_bytes: Option<usize>,
3544 ) -> StderrTailSnapshot {
3545 self.inner
3546 .stderr_ring
3547 .lock()
3548 .unwrap_or_else(|poisoned| poisoned.into_inner())
3549 .snapshot(max_lines, max_bytes)
3550 }
3551
3552 pub fn terminal_history(&self) -> TerminalHistorySnapshot {
3557 self.inner
3558 .terminal_ring
3559 .lock()
3560 .unwrap_or_else(|poisoned| poisoned.into_inner())
3561 .snapshot()
3562 }
3563
3564 pub fn durable_terminal_history(&self) -> subc_control::TerminalHistory {
3569 durable_terminal_history_of(&self.inner.terminal_ring, &self.inner.module_id)
3570 }
3571
3572 pub(crate) async fn read_durable_terminal_history(
3577 &self,
3578 ) -> Result<subc_control::TerminalHistory, tokio::task::JoinError> {
3579 let terminal_ring = Arc::clone(&self.inner.terminal_ring);
3580 let module_id = self.inner.module_id.clone();
3581 tokio::task::spawn_blocking(move || durable_terminal_history_of(&terminal_ring, &module_id))
3582 .await
3583 }
3584
3585 pub fn status(&self) -> Result<ModuleStatus, SuperviseError> {
3586 self.status_with_snapshot_lock(&self.inner.snapshot, None)
3587 .map(|(status, _)| status)
3588 }
3589
3590 pub(crate) fn record_deliberate_severance(
3591 &self,
3592 identity: ProcessIdentity,
3593 ) -> Result<bool, SuperviseError> {
3594 let mut snapshot = lock_snapshot(&self.inner.snapshot)?;
3595 if snapshot.pid != Some(identity.pid)
3596 || snapshot.process_start_time != Some(identity.start_time)
3597 {
3598 return Ok(false);
3599 }
3600 snapshot.deliberate_severance = Some(identity);
3601 Ok(true)
3602 }
3603
3604 pub(crate) fn status_for_control(
3609 &self,
3610 caller: &'static str,
3611 ) -> Result<ModuleStatus, SuperviseError> {
3612 self.status_with_snapshot_lock(&self.inner.snapshot, Some(caller))
3613 .map(|(status, _)| status)
3614 }
3615
3616 fn status_with_snapshot_lock(
3617 &self,
3618 snapshot: &SharedSnapshot,
3619 caller: Option<&'static str>,
3620 ) -> Result<(ModuleStatus, Option<SpawnedFileIdentity>), SuperviseError> {
3621 let mut guard = match caller {
3622 Some(caller) => lock_snapshot_for_control(snapshot, &self.inner.module_id, caller)?,
3623 None => lock_snapshot(snapshot)?,
3624 };
3625 let restart_count =
3628 guard.crash_restarts_in_window(self.inner.restart_policy.window, Instant::now());
3629 let snapshot = guard.clone();
3630 drop(guard);
3631 let drain_timeout = *self.inner.effective_drain_timeout.lock().map_err(|_| {
3632 SuperviseError::StatePoisoned {
3633 module_id: Some(self.inner.module_id.clone()),
3634 }
3635 })?;
3636 let registration_active = self
3637 .inner
3638 .registry
3639 .get_module(&self.inner.module_id)
3640 .map_err(SuperviseError::Registry)?
3641 .is_some();
3642 let protocol = snapshot
3643 .spawned_protocol
3644 .unwrap_or(self.declared_protocol()?);
3645 let running_process =
3646 snapshot.enabled && snapshot.state == ModuleState::Running && snapshot.process_alive;
3647 let live = match protocol {
3653 ModuleProtocol::Subc => running_process && registration_active,
3654 ModuleProtocol::None => running_process,
3655 };
3656
3657 Ok((
3658 ModuleStatus {
3659 waiting_for_previous_pid: snapshot.waiting_for_previous_pid,
3660 waiting_for_previous_detail: snapshot.waiting_for_previous_detail.clone(),
3661 module_id: self.inner.module_id.clone(),
3662 launch_nonce_source: observed_launch_nonce_source(&snapshot),
3663 state: snapshot.state,
3664 enabled: snapshot.enabled,
3665 process_alive: snapshot.process_alive,
3666 registration_active,
3667 protocol,
3668 live,
3669 restart_count,
3670 lifetime_restarts: snapshot.lifetime_restarts,
3671 spawn_generation: snapshot.spawn_generation,
3672 max_restarts: self.inner.restart_policy.max_restarts,
3673 restart_window: self.inner.restart_policy.window,
3674 drain_timeout,
3675 restart_backoff: self.inner.restart_policy.backoff,
3676 restart_max_backoff: self.inner.restart_policy.max_backoff,
3677 pid: snapshot.reported_pid(),
3678 spawned_at_ms: snapshot.spawned_at_ms,
3679 spawned_from: snapshot.spawned_from,
3680 launch_spec_digest: snapshot.launch_spec_digest,
3681 process_start_time: snapshot.process_start_time,
3682 last_exit: snapshot.last_exit,
3683 health: snapshot.health,
3684 },
3685 snapshot.spawned_file_identity.clone(),
3686 ))
3687 }
3688
3689 #[cfg(test)]
3690 pub(crate) fn hold_snapshot_for_test(
3691 &self,
3692 acquired: std::sync::mpsc::Sender<()>,
3693 hold: Duration,
3694 ) -> std::thread::JoinHandle<()> {
3695 let snapshot = Arc::clone(&self.inner.snapshot);
3696 std::thread::spawn(move || {
3697 let _guard = snapshot.lock().expect("test snapshot lock is not poisoned");
3698 acquired
3699 .send(())
3700 .expect("test receiver waits for snapshot lock");
3701 std::thread::sleep(hold);
3702 })
3703 }
3704
3705 pub(crate) async fn status_and_running_image_agreement(
3711 &self,
3712 ) -> Result<(ModuleStatus, subc_control::RunningImageAgreement), SuperviseError> {
3713 let (status, identity) = self.status_with_snapshot_lock(&self.inner.snapshot, None)?;
3714 let image = self
3715 .inner
3716 .provenance_probe
3717 .observe(
3718 status.pid,
3719 status.spawned_from.as_deref(),
3720 identity,
3721 status.process_start_time,
3722 )
3723 .await;
3724 Ok((status, image))
3725 }
3726
3727 pub(crate) async fn running_image_agreement(&self) -> subc_control::RunningImageAgreement {
3728 let snapshot = match lock_snapshot(&self.inner.snapshot) {
3729 Ok(snapshot) => snapshot.clone(),
3730 Err(_) => {
3731 return subc_control::RunningImageAgreement::Unavailable {
3732 reason: subc_control::RunningImageUnavailableReason::NotRunning,
3733 };
3734 }
3735 };
3736 self.inner
3737 .provenance_probe
3738 .observe(
3739 snapshot.reported_pid(),
3740 snapshot.spawned_from.as_deref(),
3741 snapshot.spawned_file_identity.clone(),
3742 snapshot.process_start_time,
3743 )
3744 .await
3745 }
3746
3747 pub(crate) fn child_resource_usage(&self) -> subc_control::ChildResourceUsage {
3750 #[cfg(windows)]
3751 {
3752 let image = lock_snapshot(&self.inner.snapshot)
3753 .ok()
3754 .and_then(|snapshot| {
3755 snapshot
3756 .spawned_file_identity
3757 .as_ref()
3758 .and_then(|identity| identity.windows_image.clone())
3759 });
3760 match image {
3761 Some(image) => {
3762 crate::child_resources::read_windows_handle(image.process(), image.start_time())
3763 }
3764 None => subc_control::ChildResourceUsage::Unavailable {
3765 reason: subc_control::ChildResourceUnavailableReason::NotRunning,
3766 },
3767 }
3768 }
3769 #[cfg(not(windows))]
3770 {
3771 let (pid, start_time) = match lock_snapshot(&self.inner.snapshot) {
3772 Ok(snapshot) => (snapshot.reported_pid(), snapshot.process_start_time),
3773 Err(_) => {
3774 return subc_control::ChildResourceUsage::Unavailable {
3775 reason: subc_control::ChildResourceUnavailableReason::Unreadable,
3776 }
3777 }
3778 };
3779 crate::child_resources::read(pid, start_time)
3780 }
3781 }
3782
3783 pub(crate) fn will_recover_after_connection_loss(&self) -> Result<bool, SuperviseError> {
3784 let mut snapshot = lock_snapshot(&self.inner.snapshot)?;
3785 Ok(match snapshot.state {
3786 ModuleState::Restarting => true,
3787 ModuleState::Failed | ModuleState::Disabled => false,
3788 _ if snapshot.enabled && snapshot.respawn_pending => true,
3789 _ => daemon_will_restart(&mut snapshot, &self.inner.restart_policy, Instant::now()),
3790 })
3791 }
3792
3793 #[cfg(test)]
3794 pub(crate) fn is_warming(&self) -> Result<bool, SuperviseError> {
3795 self.is_warming_with_snapshot_lock(None)
3796 }
3797
3798 pub(crate) fn is_warming_for_control(
3799 &self,
3800 caller: &'static str,
3801 ) -> Result<bool, SuperviseError> {
3802 self.is_warming_with_snapshot_lock(Some(caller))
3803 }
3804
3805 fn is_warming_with_snapshot_lock(
3806 &self,
3807 caller: Option<&'static str>,
3808 ) -> Result<bool, SuperviseError> {
3809 let snapshot = match caller {
3810 Some(caller) => {
3811 lock_snapshot_for_control(&self.inner.snapshot, &self.inner.module_id, caller)?
3812 }
3813 None => lock_snapshot(&self.inner.snapshot)?,
3814 }
3815 .clone();
3816 Ok(matches!(
3817 snapshot.state,
3818 ModuleState::Starting | ModuleState::Running | ModuleState::Restarting
3819 ))
3820 }
3821
3822 pub async fn drain(&self) -> Result<(), SuperviseError> {
3824 self.stop().await
3825 }
3826
3827 pub(crate) async fn retire_requested(
3828 &self,
3829 requester: Option<Principal>,
3830 ) -> Result<(), SuperviseError> {
3831 match self.state()? {
3832 ModuleState::Stopped | ModuleState::Failed => return Ok(()),
3833 ModuleState::Starting
3834 | ModuleState::Running
3835 | ModuleState::Unresponsive
3836 | ModuleState::Restarting
3837 | ModuleState::Draining
3838 | ModuleState::Disabled => {}
3839 }
3840
3841 let (reply_tx, reply_rx) = oneshot::channel();
3842 self.inner
3843 .commands
3844 .send(SupervisorCommand::Retire {
3845 requester,
3846 reply: reply_tx,
3847 })
3848 .await
3849 .map_err(|_| SuperviseError::CommandClosed {
3850 module_id: self.inner.module_id.clone(),
3851 })?;
3852 reply_rx.await.map_err(|_| SuperviseError::CommandClosed {
3853 module_id: self.inner.module_id.clone(),
3854 })?
3855 }
3856
3857 pub async fn stop(&self) -> Result<(), SuperviseError> {
3858 self.stop_requested(None).await
3859 }
3860
3861 pub(crate) async fn stop_requested(
3862 &self,
3863 requester: Option<Principal>,
3864 ) -> Result<(), SuperviseError> {
3865 match self.state()? {
3866 ModuleState::Stopped | ModuleState::Failed => return Ok(()),
3867 ModuleState::Starting
3868 | ModuleState::Running
3869 | ModuleState::Unresponsive
3870 | ModuleState::Restarting
3871 | ModuleState::Draining
3872 | ModuleState::Disabled => {}
3873 }
3874
3875 let (reply_tx, reply_rx) = oneshot::channel();
3876 self.inner
3877 .commands
3878 .send(SupervisorCommand::Drain {
3879 requester,
3880 reply: reply_tx,
3881 })
3882 .await
3883 .map_err(|_| SuperviseError::CommandClosed {
3884 module_id: self.inner.module_id.clone(),
3885 })?;
3886 reply_rx.await.map_err(|_| SuperviseError::CommandClosed {
3887 module_id: self.inner.module_id.clone(),
3888 })?
3889 }
3890
3891 pub async fn restart(&self, drain_timeout_ms: Option<u64>) -> Result<(), SuperviseError> {
3892 self.restart_requested(drain_timeout_ms, None).await
3893 }
3894
3895 pub(crate) async fn restart_requested(
3896 &self,
3897 drain_timeout_ms: Option<u64>,
3898 requester: Option<Principal>,
3899 ) -> Result<(), SuperviseError> {
3900 let queued_at = Instant::now();
3901 let received_at_generation = lock_snapshot(&self.inner.snapshot)?.spawn_generation;
3902 let (reply_tx, reply_rx) = oneshot::channel();
3903 self.inner
3904 .commands
3905 .send(SupervisorCommand::Restart {
3906 requester,
3907 drain_timeout_ms,
3908 received_at_generation,
3909 queued_at,
3910 reply: reply_tx,
3911 })
3912 .await
3913 .map_err(|_| SuperviseError::CommandClosed {
3914 module_id: self.inner.module_id.clone(),
3915 })?;
3916 reply_rx.await.map_err(|_| SuperviseError::CommandClosed {
3917 module_id: self.inner.module_id.clone(),
3918 })?
3919 }
3920
3921 pub async fn swap(&self, ready_timeout: Option<Duration>) -> Result<(), SuperviseError> {
3926 self.swap_requested(ready_timeout, None).await
3927 }
3928
3929 pub(crate) async fn swap_requested(
3930 &self,
3931 ready_timeout: Option<Duration>,
3932 requester: Option<Principal>,
3933 ) -> Result<(), SuperviseError> {
3934 let (reply_tx, reply_rx) = oneshot::channel();
3935 self.inner
3936 .commands
3937 .send(SupervisorCommand::Swap {
3938 requester,
3939 ready_timeout,
3940 reply: reply_tx,
3941 })
3942 .await
3943 .map_err(|_| SuperviseError::CommandClosed {
3944 module_id: self.inner.module_id.clone(),
3945 })?;
3946 reply_rx.await.map_err(|_| SuperviseError::CommandClosed {
3947 module_id: self.inner.module_id.clone(),
3948 })?
3949 }
3950
3951 pub async fn reload(&self) -> Result<(), SuperviseError> {
3952 self.reload_requested(None).await
3953 }
3954
3955 pub(crate) async fn reload_requested(
3956 &self,
3957 requester: Option<Principal>,
3958 ) -> Result<(), SuperviseError> {
3959 let (reply_tx, reply_rx) = oneshot::channel();
3960 self.inner
3961 .commands
3962 .send(SupervisorCommand::Reload {
3963 requester,
3964 reply: reply_tx,
3965 })
3966 .await
3967 .map_err(|_| SuperviseError::CommandClosed {
3968 module_id: self.inner.module_id.clone(),
3969 })?;
3970 reply_rx.await.map_err(|_| SuperviseError::CommandClosed {
3971 module_id: self.inner.module_id.clone(),
3972 })?
3973 }
3974
3975 pub async fn set_enabled(&self, enabled: bool) -> Result<bool, SuperviseError> {
3976 self.set_enabled_requested(enabled, None).await
3977 }
3978
3979 pub(crate) async fn set_enabled_requested(
3980 &self,
3981 enabled: bool,
3982 requester: Option<Principal>,
3983 ) -> Result<bool, SuperviseError> {
3984 let (reply_tx, reply_rx) = oneshot::channel();
3985 self.inner
3986 .commands
3987 .send(SupervisorCommand::SetEnabled {
3988 requester,
3989 enabled,
3990 reply: reply_tx,
3991 })
3992 .await
3993 .map_err(|_| SuperviseError::CommandClosed {
3994 module_id: self.inner.module_id.clone(),
3995 })?;
3996 reply_rx.await.map_err(|_| SuperviseError::CommandClosed {
3997 module_id: self.inner.module_id.clone(),
3998 })?
3999 }
4000
4001 pub(crate) fn declared_protocol(&self) -> Result<ModuleProtocol, SuperviseError> {
4005 let configured = self
4006 .inner
4007 .configuration
4008 .lock()
4009 .map_err(|_| SuperviseError::StatePoisoned {
4010 module_id: Some(self.inner.module_id.clone()),
4011 })?
4012 .spec
4013 .protocol;
4014 let state = lock_snapshot(&self.inner.snapshot)?;
4015 Ok(state.spawned_protocol.unwrap_or(configured))
4016 }
4017
4018 pub(crate) fn configuration(&self) -> Result<(ModuleSpec, HealthConfig), SuperviseError> {
4019 let configuration =
4020 self.inner
4021 .configuration
4022 .lock()
4023 .map_err(|_| SuperviseError::StatePoisoned {
4024 module_id: Some(self.inner.module_id.clone()),
4025 })?;
4026 Ok((configuration.spec.clone(), configuration.health.clone()))
4027 }
4028
4029 #[cfg(any(test, feature = "test-support"))]
4033 pub async fn update_spec_for_test(&self, spec: ModuleSpec) -> Result<(), SuperviseError> {
4034 let (_, health) = self.configuration()?;
4035 let drain_timeout_ms = u64::try_from(
4036 self.inner
4037 .effective_drain_timeout
4038 .lock()
4039 .unwrap_or_else(|poisoned| poisoned.into_inner())
4040 .as_millis(),
4041 )
4042 .ok();
4043 self.update_configuration(spec, health, drain_timeout_ms)
4044 .await
4045 }
4046
4047 pub(crate) async fn update_configuration(
4048 &self,
4049 spec: ModuleSpec,
4050 health: HealthConfig,
4051 drain_timeout_ms: Option<u64>,
4052 ) -> Result<(), SuperviseError> {
4053 if spec.module_id != self.inner.module_id {
4054 return Err(SuperviseError::InvalidSpec {
4055 reason: "a supervised module's module_id cannot be changed".to_string(),
4056 });
4057 }
4058 validate_spec(&spec)?;
4059 let (reply_tx, reply_rx) = oneshot::channel();
4060 self.inner
4061 .commands
4062 .send(SupervisorCommand::UpdateConfiguration {
4063 spec: spec.clone(),
4064 health: health.clone(),
4065 drain_timeout_ms,
4066 reply: reply_tx,
4067 })
4068 .await
4069 .map_err(|_| SuperviseError::CommandClosed {
4070 module_id: self.inner.module_id.clone(),
4071 })?;
4072 reply_rx.await.map_err(|_| SuperviseError::CommandClosed {
4073 module_id: self.inner.module_id.clone(),
4074 })?;
4075 let mut configuration =
4076 self.inner
4077 .configuration
4078 .lock()
4079 .map_err(|_| SuperviseError::StatePoisoned {
4080 module_id: Some(self.inner.module_id.clone()),
4081 })?;
4082 configuration.spec = spec;
4083 configuration.health = health;
4084 Ok(())
4085 }
4086}
4087
4088impl Drop for SupervisedModuleInner {
4089 fn drop(&mut self) {
4090 let Ok(mut monitor) = self.monitor.lock() else {
4091 return;
4092 };
4093 if let Some(monitor) = monitor.as_ref().filter(|monitor| !monitor.is_finished()) {
4094 let _ = update_snapshot(&self.snapshot, Some(&self.module_id), |state| {
4095 state.state = ModuleState::Stopped;
4096 clear_current_process_facts(state);
4097 });
4098 monitor.abort();
4099 }
4100 let _ = monitor.take();
4101 }
4102}
4103
4104#[derive(Debug)]
4105enum SupervisorCommand {
4106 Drain {
4107 requester: Option<Principal>,
4108 reply: oneshot::Sender<Result<(), SuperviseError>>,
4109 },
4110 Retire {
4111 requester: Option<Principal>,
4112 reply: oneshot::Sender<Result<(), SuperviseError>>,
4113 },
4114 Restart {
4115 requester: Option<Principal>,
4116 drain_timeout_ms: Option<u64>,
4121 received_at_generation: u64,
4125 queued_at: Instant,
4128 reply: oneshot::Sender<Result<(), SuperviseError>>,
4129 },
4130 Reload {
4131 requester: Option<Principal>,
4132 reply: oneshot::Sender<Result<(), SuperviseError>>,
4133 },
4134 SetEnabled {
4135 requester: Option<Principal>,
4136 enabled: bool,
4137 reply: oneshot::Sender<Result<bool, SuperviseError>>,
4138 },
4139 UpdateConfiguration {
4140 spec: ModuleSpec,
4141 health: HealthConfig,
4142 drain_timeout_ms: Option<u64>,
4145 reply: oneshot::Sender<()>,
4146 },
4147 Swap {
4148 requester: Option<Principal>,
4149 ready_timeout: Option<Duration>,
4152 reply: oneshot::Sender<Result<(), SuperviseError>>,
4154 },
4155}
4156
4157#[derive(Debug)]
4158pub enum SuperviseError {
4159 InvalidSpec {
4160 reason: String,
4161 },
4162 Spawn {
4163 program: PathBuf,
4164 source: io::Error,
4165 cgroup_path: Option<PathBuf>,
4166 },
4167 Cgroup {
4168 module_id: String,
4169 source: io::Error,
4170 },
4171 LaunchNonce {
4174 reason: String,
4175 },
4176 Wait {
4177 module_id: String,
4178 source: io::Error,
4179 },
4180 Kill {
4181 module_id: String,
4182 source: io::Error,
4183 },
4184 Forwarding(ForwardingError),
4185 Registry(RegistryError),
4186 ReloadUnavailable {
4187 module_id: String,
4188 reason: String,
4189 },
4190 Disabled {
4195 module_id: String,
4196 },
4197 ReloadFailed {
4198 module_id: String,
4199 reason: String,
4200 },
4201 RegistrationStillActive {
4202 module_id: String,
4203 waited: Duration,
4204 },
4205 StatePoisoned {
4206 module_id: Option<String>,
4207 },
4208 CommandClosed {
4209 module_id: String,
4210 },
4211 SwapInProgress {
4215 module_id: String,
4216 },
4217 SwapRefused {
4219 module_id: String,
4220 reason: SwapRefusal,
4221 },
4222 SwapFailed {
4226 module_id: String,
4227 arm: SwapFailureArm,
4228 detail: String,
4229 candidate_exit: Option<ExitReport>,
4232 },
4233}
4234
4235#[derive(Debug, Clone, Copy, PartialEq, Eq)]
4237pub enum SwapRefusal {
4238 OverlapExclusive,
4240 NotRegistered,
4243 ProtocolNone,
4246 NotConfigured,
4249 AlreadySwapping,
4251}
4252
4253impl SwapRefusal {
4254 pub fn as_str(self) -> &'static str {
4255 match self {
4256 Self::OverlapExclusive => "overlap_exclusive",
4257 Self::NotRegistered => "not_registered",
4258 Self::ProtocolNone => "protocol_none",
4259 Self::NotConfigured => "not_configured",
4260 Self::AlreadySwapping => "already_swapping",
4261 }
4262 }
4263}
4264
4265#[derive(Debug, Clone, Copy, PartialEq, Eq)]
4268pub enum SwapFailureArm {
4269 SpawnFailed,
4271 NeverRegistered,
4273 NeverReady,
4275 CandidateExited,
4277 CandidateUnhealthy,
4279 Interrupted,
4283 CutoverLost,
4288}
4289
4290impl SwapFailureArm {
4291 pub fn as_str(self) -> &'static str {
4292 match self {
4293 Self::SpawnFailed => "spawn_failed",
4294 Self::NeverRegistered => "never_registered",
4295 Self::NeverReady => "never_ready",
4296 Self::CandidateExited => "candidate_exited",
4297 Self::CandidateUnhealthy => "candidate_unhealthy",
4298 Self::Interrupted => "interrupted",
4299 Self::CutoverLost => "cutover_lost",
4300 }
4301 }
4302}
4303
4304impl fmt::Display for SuperviseError {
4305 fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
4306 match self {
4307 Self::InvalidSpec { reason } => write!(f, "invalid module spec: {reason}"),
4308 Self::Spawn {
4309 program,
4310 source,
4311 cgroup_path: Some(cgroup_path),
4312 } => write!(
4313 f,
4314 "failed to place module in cgroup '{}' while spawning '{}': {source}",
4315 cgroup_path.display(),
4316 program.display()
4317 ),
4318 Self::Spawn {
4319 program,
4320 source,
4321 cgroup_path: None,
4322 } => write!(
4323 f,
4324 "failed to spawn module '{}': {source}",
4325 program.display()
4326 ),
4327 Self::Cgroup { module_id, source } => {
4328 write!(
4329 f,
4330 "failed to prepare cgroup for module '{module_id}': {source}"
4331 )
4332 }
4333 Self::LaunchNonce { reason } => {
4334 write!(
4335 f,
4336 "failed to generate reserved-module launch nonce: {reason}"
4337 )
4338 }
4339 Self::Wait { module_id, source } => {
4340 write!(f, "failed to wait for module '{module_id}': {source}")
4341 }
4342 Self::Kill { module_id, source } => {
4343 write!(f, "failed to kill module '{module_id}': {source}")
4344 }
4345 Self::Forwarding(err) => write!(f, "forwarding error: {err}"),
4346 Self::Registry(err) => write!(f, "registry error: {err}"),
4347 Self::ReloadUnavailable { module_id, reason } => {
4348 write!(f, "reload unavailable for module '{module_id}': {reason}")
4349 }
4350 Self::Disabled { module_id } => {
4351 write!(
4352 f,
4353 "module '{module_id}' is disabled; enable it before restart or reload"
4354 )
4355 }
4356 Self::ReloadFailed { module_id, reason } => {
4357 write!(f, "reload failed for module '{module_id}': {reason}")
4358 }
4359 Self::RegistrationStillActive { module_id, waited } => write!(
4360 f,
4361 "module '{module_id}' registration remained active after waiting {waited:?}"
4362 ),
4363 Self::StatePoisoned { module_id } => match module_id {
4364 Some(module_id) => {
4365 write!(f, "supervisor state for module '{module_id}' was poisoned")
4366 }
4367 None => write!(f, "supervisor state was poisoned"),
4368 },
4369 Self::CommandClosed { module_id } => {
4370 write!(
4371 f,
4372 "supervisor command channel for module '{module_id}' is closed"
4373 )
4374 }
4375 Self::SwapInProgress { module_id } => write!(
4376 f,
4377 "module '{module_id}' is being swapped; retry once the swap has cut over or failed, or stop the module to abort the swap"
4378 ),
4379 Self::SwapRefused { module_id, reason } => match reason {
4380 SwapRefusal::OverlapExclusive => write!(
4381 f,
4382 "module '{module_id}' is declared overlap: \"exclusive\" (the default): two processes of it must not run at once, so it cannot be swapped; use a plain restart, or declare overlap: \"safe\" in its config if it really tolerates a second process"
4383 ),
4384 SwapRefusal::NotRegistered => write!(
4385 f,
4386 "module '{module_id}' is not registered, so there is no serving process to keep while a replacement warms; use a plain restart"
4387 ),
4388 SwapRefusal::ProtocolNone => write!(
4389 f,
4390 "module '{module_id}' is protocol: \"none\" and never registers, so a swap could never see its replacement become ready; use a plain restart"
4391 ),
4392 SwapRefusal::NotConfigured => write!(
4393 f,
4394 "module '{module_id}' cannot be swapped: the supervisor was built without the forwarding table or shared handle a swap needs"
4395 ),
4396 SwapRefusal::AlreadySwapping => {
4397 write!(f, "module '{module_id}' is already being swapped")
4398 }
4399 },
4400 Self::SwapFailed {
4401 module_id,
4402 arm,
4403 detail,
4404 ..
4405 } => write!(
4406 f,
4407 "swap of module '{module_id}' failed ({}): {detail}; the running process was left serving",
4408 arm.as_str()
4409 ),
4410 }
4411 }
4412}
4413
4414impl Error for SuperviseError {
4415 fn source(&self) -> Option<&(dyn Error + 'static)> {
4416 match self {
4417 Self::Spawn { source, .. }
4418 | Self::Cgroup { source, .. }
4419 | Self::Wait { source, .. }
4420 | Self::Kill { source, .. } => Some(source),
4421 Self::Forwarding(err) => Some(err),
4422 Self::Registry(err) => Some(err),
4423 Self::LaunchNonce { .. }
4424 | Self::InvalidSpec { .. }
4425 | Self::ReloadUnavailable { .. }
4426 | Self::Disabled { .. }
4427 | Self::ReloadFailed { .. }
4428 | Self::RegistrationStillActive { .. }
4429 | Self::StatePoisoned { .. }
4430 | Self::CommandClosed { .. }
4431 | Self::SwapInProgress { .. }
4432 | Self::SwapRefused { .. }
4433 | Self::SwapFailed { .. } => None,
4434 }
4435 }
4436}
4437
4438pub(crate) fn validate_spec(spec: &ModuleSpec) -> Result<(), SuperviseError> {
4439 if spec.module_id.trim().is_empty() {
4440 return Err(SuperviseError::InvalidSpec {
4441 reason: "module_id must not be empty".to_string(),
4442 });
4443 }
4444
4445 Ok(())
4446}
4447
4448#[derive(Debug, Default)]
4449struct HealthProbeRuntime {
4450 configured_health: Option<HealthConfig>,
4451 registered_connection: Option<crate::ConnectionId>,
4452 advertised: bool,
4453 next_probe_at: Option<Instant>,
4454 probe_index: u64,
4455}
4456
4457fn running_protocol(spec: &ModuleSpec, snapshot: &SharedSnapshot) -> ModuleProtocol {
4458 lock_snapshot(snapshot)
4459 .ok()
4460 .and_then(|state| state.spawned_protocol)
4461 .unwrap_or(spec.protocol)
4462}
4463
4464impl HealthProbeRuntime {
4465 fn refresh_registration(
4466 &mut self,
4467 spec: &ModuleSpec,
4468 runtime: &SupervisorRuntimeConfig,
4469 registry: &Registry,
4470 snapshot: &SharedSnapshot,
4471 ) {
4472 if self.configured_health.as_ref() != Some(&runtime.health) {
4473 self.configured_health = Some(runtime.health.clone());
4474 self.next_probe_at = None;
4475 self.registered_connection = None;
4476 self.probe_index = 0;
4477 }
4478 if running_protocol(spec, snapshot) == ModuleProtocol::None {
4482 self.registered_connection = None;
4483 self.advertised = runtime.health.http.is_some();
4484 if !self.advertised {
4485 self.next_probe_at = None;
4486 let _ = update_snapshot(snapshot, Some(&spec.module_id), |state| {
4487 let unknown = ModuleHealthStatus::default();
4488 if state.health != unknown {
4489 state.health = unknown;
4490 }
4491 });
4492 } else if self.next_probe_at.is_none() {
4493 self.next_probe_at = Some(
4494 Instant::now()
4495 + jittered_health_delay(&spec.module_id, 0, runtime.health.cadence),
4496 );
4497 }
4498 return;
4499 }
4500
4501 let registration = match registry.get_module(&spec.module_id) {
4502 Ok(registration) => registration,
4503 Err(err) => {
4504 warn!(module_id = %spec.module_id, error = %err, "health prober could not read registry");
4505 self.advertised = false;
4506 self.next_probe_at = None;
4507 return;
4508 }
4509 };
4510
4511 let Some(registration) = registration else {
4512 self.registered_connection = None;
4513 self.advertised = false;
4514 self.next_probe_at = None;
4515 return;
4516 };
4517
4518 let advertised = registration
4519 .control_ops
4520 .iter()
4521 .any(|op| op == MODULE_CONTROL_OP_HEALTH_CHECK);
4522 if !advertised {
4523 self.registered_connection = Some(registration.connection_id);
4524 self.advertised = false;
4525 self.next_probe_at = None;
4526 let _ = update_snapshot(snapshot, Some(&spec.module_id), |state| {
4527 if state.health.status != SupervisorHealthStatus::Unknown
4528 || state.health.consecutive_failures != 0
4529 || state.health.last_probe_ms.is_some()
4530 || state.health.detail.is_some()
4531 || state.health.metrics.is_some()
4532 {
4533 state.health.status = SupervisorHealthStatus::Unknown;
4534 state.health.consecutive_failures = 0;
4535 state.health.last_probe_ms = None;
4536 state.health.detail = None;
4537 state.health.metrics = None;
4538 }
4539 });
4540 return;
4541 }
4542
4543 let reregistered = self.registered_connection != Some(registration.connection_id);
4544 self.registered_connection = Some(registration.connection_id);
4545 self.advertised = true;
4546 if reregistered || self.next_probe_at.is_none() {
4547 self.probe_index = 0;
4548 self.next_probe_at = Some(
4549 Instant::now() + jittered_health_delay(&spec.module_id, 0, runtime.health.cadence),
4550 );
4551 let _ = update_snapshot(snapshot, Some(&spec.module_id), |state| {
4552 state.health.status = SupervisorHealthStatus::Unknown;
4553 state.health.consecutive_failures = 0;
4554 state.health.detail = None;
4555 state.health.metrics = None;
4556 });
4557 }
4558 }
4559
4560 fn wake_after(&self) -> Option<Duration> {
4561 if !self.advertised {
4562 return None;
4563 }
4564 self.next_probe_at
4565 .map(|next| next.saturating_duration_since(Instant::now()))
4566 }
4567
4568 fn due(&self) -> bool {
4569 self.advertised
4570 && self
4571 .next_probe_at
4572 .is_some_and(|next| Instant::now() >= next)
4573 }
4574
4575 fn schedule_next(&mut self, spec: &ModuleSpec, cadence: Duration) {
4576 self.probe_index = self.probe_index.wrapping_add(1);
4577 self.next_probe_at = Some(
4578 Instant::now() + jittered_health_delay(&spec.module_id, self.probe_index, cadence),
4579 );
4580 }
4581}
4582
4583#[derive(Debug)]
4618enum HealthProbeEvidence {
4619 LaneDead,
4621 NoAnswer,
4623 BadAnswer,
4625 Misconfigured,
4627}
4628
4629#[derive(Debug)]
4630struct HealthProbeError {
4631 evidence: HealthProbeEvidence,
4632 message: String,
4633}
4634
4635impl HealthProbeError {
4636 fn lane_dead(message: impl Into<String>) -> Self {
4637 Self::with(HealthProbeEvidence::LaneDead, message)
4638 }
4639
4640 fn no_answer(message: impl Into<String>) -> Self {
4641 Self::with(HealthProbeEvidence::NoAnswer, message)
4642 }
4643
4644 fn bad_answer(message: impl Into<String>) -> Self {
4645 Self::with(HealthProbeEvidence::BadAnswer, message)
4646 }
4647
4648 fn misconfigured(message: impl Into<String>) -> Self {
4649 Self::with(HealthProbeEvidence::Misconfigured, message)
4650 }
4651
4652 fn with(evidence: HealthProbeEvidence, message: impl Into<String>) -> Self {
4653 Self {
4654 evidence,
4655 message: message.into(),
4656 }
4657 }
4658
4659 #[allow(dead_code)]
4673 fn is_proof_of_death(&self) -> bool {
4674 matches!(self.evidence, HealthProbeEvidence::LaneDead)
4675 }
4676
4677 fn label(&self) -> &'static str {
4685 match self.evidence {
4686 HealthProbeEvidence::LaneDead => "lane-dead",
4687 HealthProbeEvidence::NoAnswer => "no-answer",
4688 HealthProbeEvidence::BadAnswer => "bad-answer",
4689 HealthProbeEvidence::Misconfigured => "daemon-misconfigured",
4690 }
4691 }
4692}
4693
4694impl fmt::Display for HealthProbeError {
4695 fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
4696 f.write_str(&self.message)
4697 }
4698}
4699
4700async fn run_health_probe_cycle(
4701 spec: &ModuleSpec,
4702 runtime: &SupervisorRuntimeConfig,
4703 registry: &Registry,
4704 process_liveness: &SupervisorProcessLiveness,
4705 snapshot: &SharedSnapshot,
4706 child: &mut Option<SupervisedChild>,
4707) {
4708 let now_ms = unix_ms_now();
4709 let http = (running_protocol(spec, snapshot) == ModuleProtocol::None)
4710 .then_some(runtime.health.http.as_deref())
4711 .flatten();
4712 let result = match http {
4713 Some(url) => probe_http_health(url, runtime.health.deadline).await,
4714 None => probe_module_health(&spec.module_id, runtime, None).await,
4715 };
4716 match result {
4717 Ok(report) => {
4718 handle_health_report(
4719 spec,
4720 runtime,
4721 registry,
4722 process_liveness,
4723 snapshot,
4724 child,
4725 report,
4726 now_ms,
4727 )
4728 .await;
4729 }
4730 Err(err) => {
4731 if http.is_some() {
4732 let _ = update_snapshot(snapshot, Some(&spec.module_id), |state| {
4733 state.health.status = SupervisorHealthStatus::Failing;
4734 });
4735 }
4736 handle_health_probe_failure(
4737 spec,
4738 runtime,
4739 registry,
4740 process_liveness,
4741 snapshot,
4742 child,
4743 err,
4744 now_ms,
4745 )
4746 .await;
4747 }
4748 }
4749}
4750
4751pub(crate) struct HttpProbeTarget<'a> {
4752 address: std::net::SocketAddr,
4753 localhost: bool,
4754 authority: &'a str,
4755 path: String,
4756}
4757
4758pub(crate) fn parse_http_probe_url(url: &str) -> Result<HttpProbeTarget<'_>, String> {
4761 if url.bytes().any(|b| b <= b' ' || b == 127) || url.contains('#') {
4762 return Err("must not contain whitespace, controls, or a fragment".into());
4763 }
4764 let rest = url
4765 .strip_prefix("http://")
4766 .ok_or("must use plain http://")?;
4767 let split = rest.find(['/', '?']).unwrap_or(rest.len());
4768 let (authority, suffix) = rest.split_at(split);
4769 let (host, port) = if let Some(rest) = authority.strip_prefix("[::1]") {
4770 ("::1", rest)
4771 } else {
4772 let split = authority.find(':').unwrap_or(authority.len());
4773 authority.split_at(split)
4774 };
4775 let ip: std::net::IpAddr = match host {
4776 "127.0.0.1" | "localhost" => std::net::Ipv4Addr::LOCALHOST.into(),
4777 "::1" => std::net::Ipv6Addr::LOCALHOST.into(),
4778 _ => return Err("host must be 127.0.0.1, [::1], or localhost".into()),
4779 };
4780 let port = if port.is_empty() {
4781 80
4782 } else {
4783 port.strip_prefix(':')
4784 .and_then(|p| p.parse::<u16>().ok())
4785 .filter(|p| *p > 0)
4786 .ok_or("must have a valid nonzero TCP port")?
4787 };
4788 let path = if suffix.is_empty() {
4789 "/".into()
4790 } else if suffix.starts_with('?') {
4791 format!("/{suffix}")
4792 } else {
4793 suffix.into()
4794 };
4795 Ok(HttpProbeTarget {
4796 address: std::net::SocketAddr::new(ip, port),
4797 localhost: host == "localhost",
4798 authority,
4799 path,
4800 })
4801}
4802
4803async fn probe_http_health(
4804 url: &str,
4805 deadline: Duration,
4806) -> Result<HealthReport, HealthProbeError> {
4807 use tokio::io::{AsyncReadExt, AsyncWriteExt, BufReader};
4808 let target = parse_http_probe_url(url).map_err(HealthProbeError::misconfigured)?;
4809 let mut response_status = String::new();
4812 let mut body = Vec::new();
4813 let probe = async {
4814 let connection = match tokio::net::TcpStream::connect(target.address).await {
4817 Err(_) if target.localhost => {
4818 tokio::net::TcpStream::connect((
4819 std::net::Ipv6Addr::LOCALHOST,
4820 target.address.port(),
4821 ))
4822 .await
4823 }
4824 result => result,
4825 };
4826 let mut stream = connection.map_err(|error| {
4827 HealthProbeError::no_answer(format!("HTTP connect failed: {error}"))
4828 })?;
4829 stream.write_all(format!("GET {} HTTP/1.1\r\nHost: {}\r\nAccept-Encoding: identity\r\nConnection: close\r\n\r\n", target.path, target.authority).as_bytes()).await
4830 .map_err(|error| HealthProbeError::no_answer(format!("HTTP write failed: {error}")))?;
4831 let mut reader = BufReader::new(stream);
4832 let mut budget = 16 * 1024;
4833 let status = http_line(&mut reader, &mut budget).await?;
4834 let mut words = status.split_ascii_whitespace();
4835 let version = words.next();
4836 let code = words
4837 .next()
4838 .filter(|word| word.len() == 3)
4839 .and_then(|word| word.parse::<u16>().ok());
4840 if !matches!(version, Some("HTTP/1.1" | "HTTP/1.0"))
4841 || !code.is_some_and(|code| (100..600).contains(&code))
4842 {
4843 return Err(HealthProbeError::bad_answer(format!(
4844 "invalid HTTP status: {status}"
4845 )));
4846 }
4847 let code = code.expect("validated status code");
4848 response_status = status.clone();
4849 let mut length = None;
4850 let mut chunked = false;
4851 loop {
4852 let line = http_line(&mut reader, &mut budget).await?;
4853 if line.is_empty() {
4854 break;
4855 }
4856 if let Some((name, value)) = line.split_once(':') {
4857 if name.eq_ignore_ascii_case("content-length") {
4858 length = Some(value.trim().parse::<u64>().map_err(|_| {
4859 HealthProbeError::bad_answer("invalid HTTP Content-Length")
4860 })?);
4861 } else if name.eq_ignore_ascii_case("transfer-encoding") {
4862 chunked = value.trim().eq_ignore_ascii_case("chunked");
4863 }
4864 }
4865 }
4866 if chunked {
4867 while body.len() < 200 {
4868 let line = http_line(&mut reader, &mut budget).await?;
4869 let size = u64::from_str_radix(line.split(';').next().unwrap_or("").trim(), 16)
4870 .map_err(|_| HealthProbeError::bad_answer("invalid HTTP chunk size"))?;
4871 if size == 0 {
4872 break;
4873 }
4874 let count = size.min((200 - body.len()) as u64) as usize;
4875 let start = body.len();
4876 (&mut reader)
4877 .take(count as u64)
4878 .read_to_end(&mut body)
4879 .await
4880 .map_err(|error| {
4881 HealthProbeError::no_answer(format!("HTTP body read failed: {error}"))
4882 })?;
4883 if body.len() - start != count {
4884 return Err(HealthProbeError::bad_answer("truncated HTTP chunk"));
4885 }
4886 if size > count as u64 || body.len() == 200 {
4887 break;
4888 }
4889 if !http_line(&mut reader, &mut budget).await?.is_empty() {
4890 return Err(HealthProbeError::bad_answer("invalid HTTP chunk delimiter"));
4891 }
4892 }
4893 } else {
4894 reader
4895 .take(length.unwrap_or(200).min(200))
4896 .read_to_end(&mut body)
4897 .await
4898 .map_err(|error| {
4899 HealthProbeError::no_answer(format!("HTTP body read failed: {error}"))
4900 })?;
4901 }
4902 if (200..300).contains(&code) {
4903 Ok(HealthReport::ok())
4904 } else {
4905 Err(HealthProbeError::bad_answer(
4906 "HTTP health endpoint returned non-2xx",
4907 ))
4908 }
4909 };
4910 let mut result = timeout(deadline, probe).await.unwrap_or_else(|_| {
4911 Err(HealthProbeError::no_answer(format!(
4912 "HTTP probe timed out after {deadline:?}"
4913 )))
4914 });
4915 if let Err(error) = &mut result {
4916 if !response_status.is_empty() {
4917 error.message = format!(
4918 "{}; {response_status}: {}",
4919 error.message,
4920 String::from_utf8_lossy(&body)
4921 );
4922 }
4923 }
4924 result
4925}
4926
4927async fn http_line(
4928 reader: &mut tokio::io::BufReader<tokio::net::TcpStream>,
4929 remaining: &mut usize,
4930) -> Result<String, HealthProbeError> {
4931 use tokio::io::{AsyncBufReadExt, AsyncReadExt};
4932 let mut line = Vec::new();
4933 (&mut *reader)
4934 .take(*remaining as u64)
4935 .read_until(b'\n', &mut line)
4936 .await
4937 .map_err(|error| {
4938 HealthProbeError::no_answer(format!("HTTP header read failed: {error}"))
4939 })?;
4940 *remaining -= line.len();
4941 if !line.ends_with(b"\r\n") {
4942 return Err(HealthProbeError::bad_answer(
4943 "HTTP headers are incomplete or exceed 16 KiB",
4944 ));
4945 }
4946 line.truncate(line.len() - 2);
4947 String::from_utf8(line).map_err(|_| HealthProbeError::bad_answer("HTTP header is not UTF-8"))
4948}
4949
4950async fn probe_module_health(
4951 module_id: &str,
4952 runtime: &SupervisorRuntimeConfig,
4953 drain_deadline: Option<Instant>,
4954) -> Result<HealthReport, HealthProbeError> {
4955 let Some(forwarding) = runtime.forwarding.as_ref() else {
4956 return Err(HealthProbeError::misconfigured(
4957 "supervisor was not configured with a forwarding table",
4958 ));
4959 };
4960 let probe_started_at = Instant::now();
4961 let mut deadline = probe_started_at + runtime.health.deadline;
4962 if let Some(drain_deadline) = drain_deadline {
4963 deadline = deadline.min(drain_deadline);
4964 }
4965 let pending = if drain_deadline.is_some() {
4966 forwarding.begin_drain_health_probe_rpc_for(
4967 module_id,
4968 MODULE_CONTROL_OP_HEALTH_CHECK,
4969 probe_started_at,
4970 deadline,
4971 )
4972 } else {
4973 forwarding.begin_health_probe_rpc_for(
4974 module_id,
4975 MODULE_CONTROL_OP_HEALTH_CHECK,
4976 probe_started_at,
4977 deadline,
4978 )
4979 }
4980 .map_err(|err| {
4981 HealthProbeError::lane_dead(format!("failed to begin health.check RPC: {err}"))
4984 })?;
4985 await_health_probe(forwarding, pending, deadline, runtime.health.deadline).await
4986}
4987
4988async fn probe_endpoint_health(
4995 endpoint: crate::ModuleEndpointId,
4996 runtime: &SupervisorRuntimeConfig,
4997 deadline_cap: Option<Instant>,
4998) -> Result<HealthReport, HealthProbeError> {
4999 let Some(forwarding) = runtime.forwarding.as_ref() else {
5000 return Err(HealthProbeError::misconfigured(
5001 "supervisor was not configured with a forwarding table",
5002 ));
5003 };
5004 let probe_started_at = Instant::now();
5005 let mut deadline = probe_started_at + runtime.health.deadline;
5006 if let Some(cap) = deadline_cap {
5007 deadline = deadline.min(cap);
5008 }
5009 let pending = forwarding
5010 .begin_endpoint_health_probe_rpc_for(
5011 endpoint,
5012 MODULE_CONTROL_OP_HEALTH_CHECK,
5013 probe_started_at,
5014 deadline,
5015 )
5016 .map_err(|err| {
5017 HealthProbeError::lane_dead(format!("failed to begin health.check RPC: {err}"))
5018 })?;
5019 await_health_probe(forwarding, pending, deadline, runtime.health.deadline).await
5020}
5021
5022async fn await_health_probe(
5024 forwarding: &ForwardingTable,
5025 pending: PendingModuleControlRpc,
5026 deadline: Instant,
5027 probe_budget: Duration,
5028) -> Result<HealthReport, HealthProbeError> {
5029 let PendingModuleControlRpc {
5030 endpoint,
5031 module_sink,
5032 negotiated_ver,
5033 corr,
5034 receiver,
5035 } = pending;
5036 let body = serde_json::to_vec(&ModuleControlRequest::HealthCheck {}).map_err(|err| {
5037 HealthProbeError::misconfigured(format!("failed to encode health.check: {err}"))
5038 })?;
5039 let frame = Frame::build_with_version(
5040 negotiated_ver,
5041 FrameType::Request,
5042 control_flags(),
5043 0,
5044 0,
5045 corr,
5046 body,
5047 )
5048 .map_err(|err| {
5049 HealthProbeError::misconfigured(format!("failed to build health.check frame: {err}"))
5050 })?;
5051
5052 match timeout_at(deadline, module_sink.send(frame)).await {
5058 Ok(Ok(())) => {}
5059 Ok(Err(err)) => {
5060 let _ = forwarding.cancel_module_control_rpc(endpoint, corr);
5061 return Err(HealthProbeError::lane_dead(format!(
5064 "failed to send health.check: {err}"
5065 )));
5066 }
5067 Err(_elapsed) => {
5068 let _ = forwarding.cancel_module_control_rpc(endpoint, corr);
5069 return Err(HealthProbeError::no_answer(
5073 "health.check send timed out before enqueue (module egress full)",
5074 ));
5075 }
5076 }
5077
5078 match timeout_at(deadline, receiver).await {
5079 Ok(Ok(ModuleControlRpcOutcome::Response(response))) => {
5083 response.health_report().ok_or_else(|| {
5084 HealthProbeError::bad_answer("health.check RPC returned a non-health response")
5085 })
5086 }
5087 Ok(Ok(ModuleControlRpcOutcome::Rejected(body))) => Err(HealthProbeError::bad_answer(
5088 format!("health.check rejected: {}", body.message),
5089 )),
5090 Ok(Ok(ModuleControlRpcOutcome::ModuleGone(message))) => {
5091 Err(HealthProbeError::lane_dead(message))
5092 }
5093 Ok(Ok(ModuleControlRpcOutcome::MalformedResponse(message))) => {
5094 Err(HealthProbeError::bad_answer(message))
5095 }
5096 Ok(Ok(ModuleControlRpcOutcome::UnexpectedOp { expected, actual })) => {
5097 Err(HealthProbeError::bad_answer(format!(
5098 "expected module-control op '{expected}', got '{actual}'"
5099 )))
5100 }
5101 Ok(Ok(ModuleControlRpcOutcome::DeadlineElapsed)) => Err(HealthProbeError::bad_answer(
5105 "module answered health.check after its daemon deadline",
5106 )),
5107 Ok(Err(_)) => Err(HealthProbeError::misconfigured(
5108 "health.check waiter was canceled before the module responded",
5109 )),
5110 Err(_) => {
5111 let _ = forwarding.tombstone_health_probe_rpc(endpoint, corr);
5112 Err(HealthProbeError::no_answer(format!(
5113 "module did not answer health.check within {probe_budget:?}"
5114 )))
5115 }
5116 }
5117}
5118
5119#[allow(clippy::too_many_arguments)]
5120async fn handle_health_report(
5121 spec: &ModuleSpec,
5122 runtime: &SupervisorRuntimeConfig,
5123 registry: &Registry,
5124 process_liveness: &SupervisorProcessLiveness,
5125 snapshot: &SharedSnapshot,
5126 child: &mut Option<SupervisedChild>,
5127 report: HealthReport,
5128 now_ms: u64,
5129) {
5130 let status = supervisor_health_status(report.status);
5131 let detail = report.detail.clone();
5132 let metrics = truncate_health_metrics(report.metrics);
5133 let _ = update_snapshot(snapshot, Some(&spec.module_id), |state| {
5134 state.health.status = status;
5135 state.health.last_probe_ms = Some(now_ms);
5136 state.health.detail = detail.clone();
5137 state.health.metrics = metrics.clone();
5138 state.health.consecutive_failures = 0;
5139 });
5140
5141 let action = match report.status {
5142 HealthStatus::Ok => return,
5143 HealthStatus::Degraded => runtime.health.on_degraded,
5144 HealthStatus::Failing => runtime.health.on_failing,
5145 };
5146 apply_l3_health_action(
5147 spec,
5148 runtime,
5149 registry,
5150 process_liveness,
5151 snapshot,
5152 child,
5153 status,
5154 detail.as_deref(),
5155 action,
5156 now_ms,
5157 )
5158 .await;
5159}
5160
5161#[allow(clippy::too_many_arguments)]
5162async fn handle_health_probe_failure(
5163 spec: &ModuleSpec,
5164 runtime: &SupervisorRuntimeConfig,
5165 registry: &Registry,
5166 process_liveness: &SupervisorProcessLiveness,
5167 snapshot: &SharedSnapshot,
5168 child: &mut Option<SupervisedChild>,
5169 err: HealthProbeError,
5170 now_ms: u64,
5171) {
5172 let threshold = runtime.health.failure_threshold.max(1);
5173 let mut failures = 0;
5174 let detail = format!("[{}] {err}", err.label());
5179 let _ = update_snapshot(snapshot, Some(&spec.module_id), |state| {
5180 if state.spawned_protocol.unwrap_or(spec.protocol) == ModuleProtocol::Subc {
5183 state.health.status = SupervisorHealthStatus::Unknown;
5184 }
5185 state.health.last_probe_ms = Some(now_ms);
5186 state.health.consecutive_failures = state.health.consecutive_failures.saturating_add(1);
5187 state.health.detail = Some(detail.clone());
5188 state.health.metrics = None;
5189 failures = state.health.consecutive_failures;
5190 });
5191
5192 if failures < threshold {
5193 warn!(
5194 module_id = %spec.module_id,
5195 consecutive_failures = failures,
5196 threshold,
5197 evidence = err.label(),
5198 detail = %detail,
5199 "health.check probe failed"
5200 );
5201 return;
5202 }
5203
5204 let _ = update_snapshot(snapshot, Some(&spec.module_id), |state| {
5205 state.state = ModuleState::Unresponsive;
5206 state.health.status = SupervisorHealthStatus::Unresponsive;
5207 });
5208 if runtime.health.critical {
5212 error!(
5213 module_id = %spec.module_id,
5214 status = "unresponsive",
5215 evidence = err.label(),
5216 detail = %detail,
5217 "critical module health alert"
5218 );
5219 } else {
5220 warn!(
5221 module_id = %spec.module_id,
5222 status = "unresponsive",
5223 evidence = err.label(),
5224 detail = %detail,
5225 "module health threshold breached"
5226 );
5227 }
5228 if let Err(err) = health_restart_child(
5229 spec,
5230 runtime,
5231 registry,
5232 process_liveness,
5233 snapshot,
5234 child,
5235 SupervisorHealthStatus::Unresponsive,
5236 Some(&detail),
5237 now_ms,
5238 )
5239 .await
5240 {
5241 error!(module_id = %spec.module_id, error = %err, "health-triggered restart failed");
5242 }
5243}
5244
5245#[allow(clippy::too_many_arguments)]
5246async fn apply_l3_health_action(
5247 spec: &ModuleSpec,
5248 runtime: &SupervisorRuntimeConfig,
5249 registry: &Registry,
5250 process_liveness: &SupervisorProcessLiveness,
5251 snapshot: &SharedSnapshot,
5252 child: &mut Option<SupervisedChild>,
5253 status: SupervisorHealthStatus,
5254 detail: Option<&str>,
5255 action: HealthAction,
5256 now_ms: u64,
5257) {
5258 record_health_action(snapshot, &spec.module_id, action.to_string(), now_ms);
5259 match action {
5260 HealthAction::Report => {
5261 info!(
5262 module_id = %spec.module_id,
5263 status = ?status,
5264 detail,
5265 "module reported non-ok health"
5266 );
5267 }
5268 HealthAction::Alert => {
5269 error!(
5270 module_id = %spec.module_id,
5271 status = ?status,
5272 detail,
5273 "module health alert"
5274 );
5275 }
5276 HealthAction::Restart => {
5277 if let Err(err) = health_restart_child(
5278 spec,
5279 runtime,
5280 registry,
5281 process_liveness,
5282 snapshot,
5283 child,
5284 status,
5285 detail,
5286 now_ms,
5287 )
5288 .await
5289 {
5290 error!(module_id = %spec.module_id, error = %err, "health-triggered restart failed");
5291 }
5292 }
5293 }
5294}
5295
5296#[allow(clippy::too_many_arguments)]
5297async fn health_restart_child(
5298 spec: &ModuleSpec,
5299 runtime: &SupervisorRuntimeConfig,
5300 registry: &Registry,
5301 process_liveness: &SupervisorProcessLiveness,
5302 snapshot: &SharedSnapshot,
5303 child: &mut Option<SupervisedChild>,
5304 status: SupervisorHealthStatus,
5305 detail: Option<&str>,
5306 now_ms: u64,
5307) -> Result<(), SuperviseError> {
5308 let (enabled, schedule) = {
5309 let mut state = lock_snapshot(snapshot)?;
5310 let enabled = state.enabled;
5311 let schedule = if enabled {
5312 state.next_crash_restart(&runtime.restart_policy, Instant::now())
5313 } else {
5314 None
5315 };
5316 (enabled, schedule)
5317 };
5318
5319 if !enabled {
5320 return Err(SuperviseError::Disabled {
5321 module_id: spec.module_id.clone(),
5322 });
5323 }
5324
5325 if schedule.is_none() {
5326 record_health_action(snapshot, &spec.module_id, "failed".to_string(), now_ms);
5327 error!(
5328 module_id = %spec.module_id,
5329 status = ?status,
5330 detail,
5331 max_restarts = runtime.restart_policy.max_restarts,
5332 window_secs = runtime.restart_policy.window.as_secs(),
5333 reason = %runtime.restart_policy.budget_exhausted_detail(),
5334 "health restart budget exhausted; marking module failed"
5335 );
5336 let stop_notice = begin_forwarding_drain_if_configured(
5337 spec,
5338 runtime,
5339 registry,
5340 snapshot,
5341 Some(true),
5342 RouteCloseReason::Disable,
5343 )
5344 .await?;
5345 update_snapshot(snapshot, Some(&spec.module_id), |state| {
5346 state.drain_disposition_detail = Some(runtime.restart_policy.budget_exhausted_detail());
5347 })?;
5348 drain_optional_child(
5349 &spec.module_id,
5350 spec.protocol,
5351 stop_notice,
5352 registry,
5353 runtime.forwarding.as_deref(),
5354 snapshot,
5355 &runtime.terminal_ring,
5356 &runtime.spawn_events,
5357 child,
5358 runtime.drain_timeout,
5359 ModuleState::Failed,
5360 Some(true),
5361 )
5362 .await?;
5363 process_liveness.untrack_if_current(&spec.module_id, snapshot);
5364 return Ok(());
5365 }
5366
5367 let schedule = schedule.expect("a health restart must have a crash-restart schedule");
5368 let mut restart_count = 0;
5369 update_snapshot(snapshot, Some(&spec.module_id), |state| {
5370 restart_count = state.crash_restarts.len();
5371 state.state = ModuleState::Unresponsive;
5372 state.health.status = status;
5373 state.health.last_action = Some(HealthAction::Restart.to_string());
5374 state.health.last_action_ms = Some(now_ms);
5375 })?;
5376 warn!(
5377 module_id = %spec.module_id,
5378 status = ?status,
5379 detail,
5380 restart_count,
5381 restart_in_window = schedule.restart_in_window,
5382 delay_ms = schedule.delay.as_millis() as u64,
5383 "health-triggered module restart"
5384 );
5385
5386 let stop_notice = begin_forwarding_drain_if_configured(
5387 spec,
5388 runtime,
5389 registry,
5390 snapshot,
5391 Some(true),
5392 RouteCloseReason::Restart,
5393 )
5394 .await?;
5395 drain_optional_child(
5396 &spec.module_id,
5397 spec.protocol,
5398 stop_notice,
5399 registry,
5400 runtime.forwarding.as_deref(),
5401 snapshot,
5402 &runtime.terminal_ring,
5403 &runtime.spawn_events,
5404 child,
5405 runtime.drain_timeout,
5406 ModuleState::Restarting,
5407 Some(true),
5408 )
5409 .await?;
5410 schedule_respawn(
5411 runtime,
5412 snapshot,
5413 &spec.module_id,
5414 schedule.delay,
5415 RespawnKind::Spawn,
5416 )
5417}
5418
5419fn cancel_deferred_reload(runtime: &SupervisorRuntimeConfig, module_id: &str, reason: &str) {
5420 if let Some(reply) = runtime
5421 .deferred_reload_reply
5422 .lock()
5423 .unwrap_or_else(|p| p.into_inner())
5424 .take()
5425 {
5426 let _ = reply.send(Err(SuperviseError::ReloadFailed {
5427 module_id: module_id.to_string(),
5428 reason: reason.to_string(),
5429 }));
5430 }
5431}
5432
5433fn schedule_respawn(
5434 runtime: &SupervisorRuntimeConfig,
5435 snapshot: &SharedSnapshot,
5436 module_id: &str,
5437 delay: Duration,
5438 kind: RespawnKind,
5439) -> Result<(), SuperviseError> {
5440 cancel_deferred_reload(runtime, module_id, "respawn superseded by another restart");
5441 update_snapshot(snapshot, Some(module_id), |state| {
5442 state.respawn_pending = true
5443 })?;
5444 *runtime
5445 .scheduled_respawn
5446 .lock()
5447 .unwrap_or_else(|p| p.into_inner()) = Some(PendingRespawn {
5448 deadline: Instant::now() + delay,
5449 kind,
5450 });
5451 Ok(())
5452}
5453
5454fn record_health_action(snapshot: &SharedSnapshot, module_id: &str, action: String, now_ms: u64) {
5455 let _ = update_snapshot(snapshot, Some(module_id), |state| {
5456 state.health.last_action = Some(action);
5457 state.health.last_action_ms = Some(now_ms);
5458 });
5459}
5460
5461fn supervisor_health_status(status: HealthStatus) -> SupervisorHealthStatus {
5462 match status {
5463 HealthStatus::Ok => SupervisorHealthStatus::Ok,
5464 HealthStatus::Degraded => SupervisorHealthStatus::Degraded,
5465 HealthStatus::Failing => SupervisorHealthStatus::Failing,
5466 }
5467}
5468
5469fn truncate_health_metrics(metrics: Option<Value>) -> Option<Value> {
5481 let metrics = metrics?;
5482 match serde_json::to_vec(&metrics) {
5483 Ok(encoded) if encoded.len() > MAX_HEALTH_METRICS_BYTES => Some(serde_json::json!({
5484 "truncated": true,
5485 "original_bytes": encoded.len(),
5486 })),
5487 Ok(_) | Err(_) => Some(metrics),
5488 }
5489}
5490
5491fn jittered_health_delay(module_id: &str, probe_index: u64, cadence: Duration) -> Duration {
5497 if cadence.is_zero() {
5498 return Duration::ZERO;
5499 }
5500 let cadence_ms = cadence.as_millis() as u64;
5501 if cadence_ms == 0 {
5517 return cadence;
5518 }
5519 let jitter_span = (cadence_ms / 10).max(1);
5534 let hash = module_id.as_bytes().iter().fold(
5535 probe_index.wrapping_mul(0x9E37_79B9_7F4A_7C15),
5536 |acc, byte| {
5537 acc.wrapping_mul(1099511628211)
5538 .wrapping_add(u64::from(*byte))
5539 },
5540 );
5541 cadence + Duration::from_millis(hash % jitter_span)
5542}
5543
5544#[cfg(test)]
5545mod tests {
5546 use super::*;
5547
5548 #[test]
5549 fn readding_a_module_clears_its_rescan_removal_tombstone() {
5550 let handle = SupervisorHandle::new();
5551 let module_id = "readded-tombstone";
5552 handle.record_rescan_removal(module_id);
5553 assert!(handle.removal_tombstone_age_ms(module_id).is_some());
5554
5555 handle.apply_identity_configuration(&ModuleSpec {
5556 module_id: module_id.to_string(),
5557 program: PathBuf::from("/test/module"),
5558 args: Vec::new(),
5559 env: Vec::new(),
5560 reserved: false,
5561 reserved_prefixes: Vec::new(),
5562 protocol: ModuleProtocol::Subc,
5563 overlap: Default::default(),
5564 });
5565
5566 assert!(
5567 handle.removal_tombstone_age_ms(module_id).is_none(),
5568 "a re-added module must not retain a stale removal tombstone"
5569 );
5570 }
5571
5572 #[derive(Debug, PartialEq, Eq)]
5575 struct OwnerInSpawnWindow {
5576 module_id: String,
5577 configured: bool,
5578 on_roster: bool,
5579 admission_refusal: Option<&'static str>,
5580 }
5581
5582 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
5593 async fn a_new_module_is_configured_before_its_first_process_can_register() {
5594 use crate::scopes::ScopeTable;
5595 use subc_protocol::{error_codes, scope::ScopeSelector, Principal};
5596
5597 let dir = cortexkit_test_support::ScratchDir::new("configured-before-spawn");
5598 let stub = |module_id: &str, program: PathBuf| ModuleSpec {
5599 module_id: module_id.to_string(),
5600 program,
5601 args: Vec::new(),
5602 env: ["XDG_DATA_HOME", "XDG_RUNTIME_DIR", "XDG_CONFIG_HOME"]
5603 .into_iter()
5604 .map(|key| (key.to_string(), dir.path().display().to_string()))
5605 .chain([("FAKE_AFT_NEVER_CONNECT".to_string(), "1".to_string())])
5606 .collect(),
5607 reserved: false,
5608 reserved_prefixes: Vec::new(),
5609 protocol: ModuleProtocol::Subc,
5610 overlap: Default::default(),
5611 };
5612 let live = super::terminal_history_tests::fake_aft_stub_path();
5613 let missing = dir.path().join("definitely-missing-module");
5614
5615 let handle = SupervisorHandle::new();
5616 let mut supervisor =
5617 Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default())
5618 .with_handle(handle.clone());
5619 let observed = Arc::new(Mutex::new(Vec::<OwnerInSpawnWindow>::new()));
5620 let hook_handle = handle.clone();
5621 let hook_observed = Arc::clone(&observed);
5622 supervisor.test_after_first_spawn = AfterFirstSpawnHook(Some(Arc::new(move |module_id| {
5623 let configured = hook_handle.is_configured(module_id);
5626 let selector = ScopeSelector::new(
5627 Principal::Reserved {
5628 module_id: module_id.to_string(),
5629 },
5630 "s".to_string(),
5631 Some(1),
5632 );
5633 let carrier = Principal::Reserved {
5634 module_id: "carrier".to_string(),
5635 };
5636 let admission_refusal = match ScopeTable::new(Vec::<String>::new())
5637 .admit(&carrier, module_id, &selector, configured)
5638 {
5639 Ok(_) => None,
5640 Err(refusal) => Some(refusal.code),
5641 };
5642 hook_observed.lock().unwrap().push(OwnerInSpawnWindow {
5643 module_id: module_id.to_string(),
5644 configured,
5645 on_roster: hook_handle.get(module_id).is_some(),
5646 admission_refusal,
5647 });
5648 })));
5649
5650 let plain = supervisor.spawn(stub("plain", live.clone())).unwrap();
5651 let configured = supervisor
5652 .supervise_configured(stub("configured", live.clone()), true)
5653 .unwrap();
5654 let with_health = supervisor
5655 .supervise_configured_with_health(
5656 stub("with-health", live.clone()),
5657 true,
5658 HealthConfig::default(),
5659 None,
5660 RestartPolicy::default(),
5661 )
5662 .unwrap();
5663 let failed = supervisor
5666 .supervise_configured_with_health(
5667 stub("failed-spawn", missing.clone()),
5668 true,
5669 HealthConfig::default(),
5670 None,
5671 RestartPolicy::default(),
5672 )
5673 .unwrap();
5674 assert!(supervisor.spawn(stub("spawn-error", missing)).is_err());
5677
5678 let expected = [
5679 "plain",
5680 "configured",
5681 "with-health",
5682 "failed-spawn",
5683 "spawn-error",
5684 ]
5685 .into_iter()
5686 .map(|module_id| OwnerInSpawnWindow {
5687 module_id: module_id.to_string(),
5688 configured: true,
5689 on_roster: !matches!(module_id, "plain" | "spawn-error"),
5690 admission_refusal: Some(error_codes::SCOPE_NOT_SYNCED),
5691 })
5692 .collect::<Vec<_>>();
5693 assert_eq!(*observed.lock().unwrap(), expected);
5694
5695 for module_id in ["plain", "configured", "with-health", "failed-spawn"] {
5696 assert!(
5697 handle.get(module_id).is_some(),
5698 "{module_id} is on the roster"
5699 );
5700 assert!(
5701 handle.is_configured(module_id),
5702 "{module_id} stays configured"
5703 );
5704 }
5705 assert!(handle.get("spawn-error").is_none());
5706 assert!(
5707 !handle.is_configured("spawn-error"),
5708 "a plain spawn that failed must not leave its module marked configured"
5709 );
5710
5711 handle.retire("failed-spawn");
5713 assert!(!handle.is_configured("failed-spawn"));
5714
5715 for module in [plain, configured, with_health] {
5716 module.stop().await.unwrap();
5717 }
5718 drop(failed);
5719 }
5720
5721 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
5728 async fn a_configured_module_reads_starting_while_its_first_process_already_runs() {
5729 let dir = cortexkit_test_support::ScratchDir::new("starting-while-spawned");
5730 let spec = ModuleSpec {
5731 module_id: "spawned-not-admitted".to_string(),
5732 program: super::terminal_history_tests::fake_aft_stub_path(),
5733 args: Vec::new(),
5734 env: ["XDG_DATA_HOME", "XDG_RUNTIME_DIR", "XDG_CONFIG_HOME"]
5735 .into_iter()
5736 .map(|key| (key.to_string(), dir.path().display().to_string()))
5737 .chain([("FAKE_AFT_NEVER_CONNECT".to_string(), "1".to_string())])
5738 .collect(),
5739 reserved: false,
5740 reserved_prefixes: Vec::new(),
5741 protocol: ModuleProtocol::Subc,
5742 overlap: Default::default(),
5743 };
5744 let handle = SupervisorHandle::new();
5745 let mut supervisor =
5746 Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default())
5747 .with_handle(handle.clone());
5748 let observed = Arc::new(Mutex::new(None));
5749 let hook_handle = handle.clone();
5750 let hook_observed = Arc::clone(&observed);
5751 supervisor.test_after_first_spawn = AfterFirstSpawnHook(Some(Arc::new(move |module_id| {
5752 let state = hook_handle
5753 .get(module_id)
5754 .expect("a configured module is on the roster before it is spawned")
5755 .state()
5756 .unwrap();
5757 *hook_observed.lock().unwrap() = Some(state);
5758 })));
5759
5760 let module = supervisor.supervise_configured(spec, true).unwrap();
5761 assert_eq!(
5762 *observed.lock().unwrap(),
5763 Some(ModuleState::Starting),
5764 "between the spawn and its admission the supervisor still reports starting"
5765 );
5766 assert_eq!(module.state().unwrap(), ModuleState::Running);
5767 module.stop().await.unwrap();
5768 }
5769
5770 fn stale_process_snapshot(state: ModuleState, enabled: bool) -> SharedSnapshot {
5771 let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::new(state, enabled)));
5772 update_snapshot(&snapshot, Some("stale-process-facts"), |snapshot| {
5773 snapshot.process_alive = true;
5774 snapshot.pid = Some(41);
5775 snapshot.spawned_at_ms = Some(42);
5776 snapshot.spawned_from = Some(PathBuf::from("/spawned/module"));
5777 snapshot.launch_spec_digest = Some(LaunchSpecDigest([0xA5; 32]));
5778 snapshot.spawned_file_identity = Some(SpawnedFileIdentity {
5779 device: 43,
5780 inode: 44,
5781 #[cfg(windows)]
5782 windows_image: None,
5783 });
5784 })
5785 .unwrap();
5786 snapshot
5787 }
5788
5789 fn assert_snapshot_process_facts_cleared(snapshot: &SharedSnapshot) {
5790 let snapshot = lock_snapshot(snapshot).unwrap();
5791 assert!(!snapshot.process_alive);
5792 assert_eq!(snapshot.pid, None);
5793 assert_eq!(snapshot.spawned_at_ms, None);
5794 assert_eq!(snapshot.spawned_from, None);
5795 assert_eq!(snapshot.launch_spec_digest, None);
5796 assert_eq!(snapshot.spawned_file_identity, None);
5797 }
5798
5799 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
5800 async fn failed_enable_spawn_clears_preexisting_current_process_facts() {
5801 let supervisor =
5802 Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default());
5803 let mut runtime = supervisor.runtime_config();
5804 runtime.test_seed_stale_facts_before_enable_spawn = true;
5805 let snapshot = stale_process_snapshot(ModuleState::Disabled, false);
5806 let mut child = None;
5807 let spec = ModuleSpec {
5808 module_id: "failed-enable-clears-facts".to_string(),
5809 program: PathBuf::from("/definitely/missing/failed-enable-module"),
5810 args: Vec::new(),
5811 env: Vec::new(),
5812 reserved: false,
5813 reserved_prefixes: Vec::new(),
5814 protocol: ModuleProtocol::Subc,
5815 overlap: Default::default(),
5816 };
5817
5818 let result = set_child_enabled(
5819 &spec,
5820 &runtime,
5821 &supervisor.registry,
5822 &supervisor.process_liveness,
5823 &snapshot,
5824 &mut child,
5825 true,
5826 )
5827 .await;
5828
5829 assert!(matches!(result, Err(SuperviseError::Spawn { .. })));
5830 assert_eq!(lock_snapshot(&snapshot).unwrap().state, ModuleState::Failed);
5831 assert_snapshot_process_facts_cleared(&snapshot);
5832 }
5833
5834 #[tokio::test]
5835 async fn start_revives_stranded_restarting_but_not_pending_backoff() {
5836 let supervisor =
5837 Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default());
5838 let runtime = supervisor.runtime_config();
5839 let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::new(
5840 ModuleState::Restarting,
5841 true,
5842 )));
5843 let spec = ModuleSpec {
5844 module_id: "start-stranded-restarting".to_string(),
5845 program: super::terminal_history_tests::fake_aft_stub_path(),
5846 args: Vec::new(),
5847 env: vec![("FAKE_AFT_NEVER_CONNECT".to_string(), "1".to_string())],
5848 reserved: false,
5849 reserved_prefixes: Vec::new(),
5850 protocol: ModuleProtocol::None,
5851 overlap: Default::default(),
5852 };
5853 let mut child = None;
5854 lock_snapshot(&snapshot).unwrap().respawn_pending = true;
5855 assert!(!super::set_child_enabled(
5856 &spec,
5857 &runtime,
5858 &Registry::default(),
5859 &supervisor.process_liveness,
5860 &snapshot,
5861 &mut child,
5862 true
5863 )
5864 .await
5865 .unwrap());
5866 assert!(child.is_none());
5867 lock_snapshot(&snapshot).unwrap().respawn_pending = false;
5868 assert!(super::set_child_enabled(
5869 &spec,
5870 &runtime,
5871 &Registry::default(),
5872 &supervisor.process_liveness,
5873 &snapshot,
5874 &mut child,
5875 true
5876 )
5877 .await
5878 .unwrap());
5879 assert_eq!(
5880 lock_snapshot(&snapshot).unwrap().state,
5881 ModuleState::Running
5882 );
5883 let mut child = child.unwrap();
5884 child.start_kill().unwrap();
5885 child.wait().await.unwrap();
5886 }
5887
5888 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
5889 async fn failed_reload_spawn_clears_current_process_facts() {
5890 let supervisor =
5891 Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default());
5892 let mut runtime = supervisor.runtime_config();
5893 runtime.restart_policy = RestartPolicy::new(0, Duration::ZERO);
5894 let snapshot = stale_process_snapshot(ModuleState::Running, true);
5895 let mut child = None;
5896 let spec = ModuleSpec {
5897 module_id: "failed-reload-clears-facts".to_string(),
5898 program: PathBuf::from("/unused/failed-reload-module"),
5899 args: Vec::new(),
5900 env: Vec::new(),
5901 reserved: false,
5902 reserved_prefixes: Vec::new(),
5903 protocol: ModuleProtocol::Subc,
5904 overlap: Default::default(),
5905 };
5906
5907 let result = handle_reload_spawn_failure(
5908 &spec,
5909 &runtime,
5910 &supervisor.process_liveness,
5911 &snapshot,
5912 &mut child,
5913 "forced reload spawn failure".to_string(),
5914 )
5915 .await;
5916
5917 assert!(matches!(result, Err(SuperviseError::ReloadFailed { .. })));
5918 assert_eq!(lock_snapshot(&snapshot).unwrap().state, ModuleState::Failed);
5919 assert_snapshot_process_facts_cleared(&snapshot);
5920 }
5921
5922 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
5923 async fn dropping_a_module_with_an_active_monitor_clears_current_process_facts() {
5924 let supervisor =
5925 Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default());
5926 let snapshot = stale_process_snapshot(ModuleState::Running, true);
5927 let module = supervisor.supervised_module(
5928 ModuleSpec {
5929 module_id: "drop-clears-facts".to_string(),
5930 program: PathBuf::from("/unused/drop-module"),
5931 args: Vec::new(),
5932 env: Vec::new(),
5933 reserved: false,
5934 reserved_prefixes: Vec::new(),
5935 protocol: ModuleProtocol::Subc,
5936 overlap: Default::default(),
5937 },
5938 supervisor.runtime_config(),
5939 Arc::clone(&snapshot),
5940 None,
5941 );
5942 assert!(!module
5943 .inner
5944 .monitor
5945 .lock()
5946 .unwrap()
5947 .as_ref()
5948 .unwrap()
5949 .is_finished());
5950
5951 drop(module);
5952
5953 assert_eq!(
5954 lock_snapshot(&snapshot).unwrap().state,
5955 ModuleState::Stopped
5956 );
5957 assert_snapshot_process_facts_cleared(&snapshot);
5958 }
5959
5960 #[cfg(unix)]
5961 #[tokio::test]
5962 async fn rescan_preserves_running_protocol_until_respawn() {
5963 let dir = cortexkit_test_support::ScratchDir::new("rescan-protocol");
5964 let initial = ModuleSpec {
5965 module_id: "rescan-protocol".into(),
5966 program: PathBuf::from("/bin/sleep"),
5967 args: vec!["60".into()],
5968 env: ["XDG_DATA_HOME", "XDG_RUNTIME_DIR", "XDG_CONFIG_HOME"]
5969 .into_iter()
5970 .map(|key| (key.into(), dir.path().to_string_lossy().into_owned()))
5971 .collect(),
5972 reserved: false,
5973 reserved_prefixes: vec![],
5974 protocol: ModuleProtocol::None,
5975 overlap: Default::default(),
5976 };
5977 let supervisor =
5978 Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default());
5979 let module = supervisor.spawn(initial.clone()).unwrap();
5980 assert!(module.status().unwrap().live);
5981 let mut next = initial;
5982 next.protocol = ModuleProtocol::Subc;
5983 module
5984 .update_configuration(next.clone(), HealthConfig::default(), None)
5985 .await
5986 .unwrap();
5987 assert!(
5988 module.status().unwrap().live,
5989 "rescan must not require HELLO from the old non-wire process"
5990 );
5991 let runtime = supervisor.runtime_config();
5992 let action = on_child_exit(
5993 &next,
5994 RestartPolicy::default(),
5995 &supervisor.registry,
5996 &module.inner.snapshot,
5997 &runtime.terminal_ring,
5998 &runtime.spawn_events,
5999 &runtime.child_roster,
6000 ExitReport {
6001 kind: ExitKind::Clean,
6002 code: Some(0),
6003 signal: None,
6004 at_ms: unix_ms_now(),
6005 },
6006 )
6007 .await;
6008 assert!(
6009 matches!(action, NextAction::Restart { .. }),
6010 "the old non-wire process's clean exit must restart"
6011 );
6012 module.drain().await.unwrap();
6013 }
6014
6015 #[cfg(unix)]
6016 #[tokio::test]
6017 async fn orphan_identity_matches_path_names_and_shebang_interpreters() {
6018 use std::os::unix::fs::PermissionsExt;
6019 let dir = cortexkit_test_support::ScratchDir::new("spawn-image-identity");
6020 let script = dir.join("module.sh");
6021 std::fs::write(&script, "#!/bin/sh\nwhile :; do sleep 0.1; done\n").unwrap();
6022 std::fs::set_permissions(&script, std::fs::Permissions::from_mode(0o700)).unwrap();
6023 let record_path = dir.join("live-children.json");
6024 let supervisor =
6025 Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default())
6026 .with_live_children_record(&record_path);
6027 for (program, args) in [
6028 (PathBuf::from("sleep"), vec!["60".into()]),
6029 (script, vec![]),
6030 ] {
6031 let spec = ModuleSpec {
6032 module_id: "image-identity".into(),
6033 program: program.clone(),
6034 args,
6035 env: ["XDG_DATA_HOME", "XDG_RUNTIME_DIR", "XDG_CONFIG_HOME"]
6036 .into_iter()
6037 .map(|key| (key.into(), dir.to_string_lossy().into_owned()))
6038 .collect(),
6039 reserved: false,
6040 reserved_prefixes: vec![],
6041 protocol: ModuleProtocol::None,
6042 overlap: Default::default(),
6043 };
6044 let module = supervisor.spawn(spec).unwrap();
6045 #[cfg(target_os = "macos")]
6046 {
6047 let deadline = Instant::now() + Duration::from_secs(5);
6050 while crate::live_children::read_record(&record_path)
6051 .unwrap()
6052 .iter()
6053 .all(|entry| entry.executable.is_none())
6054 {
6055 assert!(Instant::now() < deadline, "module image was not confirmed");
6056 tokio::time::sleep(Duration::from_millis(5)).await;
6057 }
6058 }
6059 let entry = crate::live_children::read_record(&record_path)
6060 .unwrap()
6061 .pop()
6062 .unwrap();
6063 let observed = subc_os::Process::open(entry.pid)
6064 .unwrap()
6065 .unwrap()
6066 .observe()
6067 .unwrap();
6068 let verdict = crate::live_children::identity_verdict(&entry, entry.pid, &observed);
6069 module.drain().await.unwrap();
6070 assert_eq!(
6071 verdict,
6072 crate::live_children::IdentityVerdict::Matches,
6073 "program {program:?}: recorded {entry:?}, observed {observed:?}"
6074 );
6075 }
6076 }
6077
6078 #[cfg(unix)]
6079 fn http_fixture(
6080 dir: &std::path::Path,
6081 url: &str,
6082 threshold: u32,
6083 ) -> crate::daemon_config::ConfiguredModule {
6084 let path = dir.join("subc.jsonc");
6085 std::fs::write(&path, serde_json::json!({"version": 1, "modules": {"http-process": {
6086 "program": "/bin/sleep", "args": ["60"], "protocol": "none",
6087 "env": {"XDG_DATA_HOME":dir,"XDG_RUNTIME_DIR":dir,"XDG_CONFIG_HOME":dir},
6088 "health": {"http":url,"cadence_ms":20,"deadline_ms":200,"failure_threshold":threshold},
6089 "restart": {"backoff_ms":1,"max_backoff_ms":1}, "drain_timeout_ms":10
6090 }}}).to_string()).unwrap();
6091 crate::daemon_config::load(&path)
6092 .unwrap()
6093 .unwrap()
6094 .modules
6095 .pop()
6096 .unwrap()
6097 }
6098
6099 #[cfg(unix)]
6100 async fn wait_http_health(module: &SupervisedModule, status: SupervisorHealthStatus) {
6101 timeout(Duration::from_secs(5), async {
6102 loop {
6103 if module.status().unwrap().health.status == status {
6104 break;
6105 }
6106 sleep(Duration::from_millis(5)).await;
6107 }
6108 })
6109 .await
6110 .unwrap_or_else(|_| {
6111 panic!(
6112 "expected {status:?}, got {:?}",
6113 module.status().unwrap().health
6114 )
6115 });
6116 }
6117
6118 #[cfg(unix)]
6119 #[tokio::test]
6120 async fn http_health_status_flips_ok_failing_ok() {
6121 use tokio::io::{AsyncReadExt, AsyncWriteExt};
6122 let dir = cortexkit_test_support::ScratchDir::new("http-status-flips");
6123 let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
6124 let status = Arc::new(std::sync::atomic::AtomicU16::new(200));
6125 let serving_status = status.clone();
6126 let url = format!("http://{}/healthz", listener.local_addr().unwrap());
6127 let server = tokio::spawn(async move {
6128 loop {
6129 let (mut stream, _) = listener.accept().await.unwrap();
6130 let mut request = [0u8; 2048];
6131 let count = stream.read(&mut request).await.unwrap();
6132 assert!(count > 0, "a probe must send an HTTP request");
6133 let code = serving_status.load(std::sync::atomic::Ordering::SeqCst);
6134 let body = if code == 200 {
6135 "ready"
6136 } else {
6137 "{\"status\":\"unavailable\",\"error\":\"scratch failure\"}"
6138 };
6139 let response = format!("HTTP/1.1 {code} scratch\r\nContent-Length: {}\r\nConnection: close\r\n\r\n{body}", body.len());
6140 let _ = stream.write_all(response.as_bytes()).await;
6141 }
6142 });
6143 let configured = http_fixture(&dir, &url, 1000);
6144 let module =
6145 Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default())
6146 .supervise_configured_with_health(
6147 configured.module_spec(),
6148 true,
6149 configured.health,
6150 configured.drain_timeout_ms,
6151 configured.restart,
6152 )
6153 .unwrap();
6154 wait_http_health(&module, SupervisorHealthStatus::Ok).await;
6155 status.store(503, std::sync::atomic::Ordering::SeqCst);
6156 wait_http_health(&module, SupervisorHealthStatus::Failing).await;
6157 assert!(module
6158 .status()
6159 .unwrap()
6160 .health
6161 .detail
6162 .unwrap()
6163 .contains("scratch failure"));
6164 status.store(200, std::sync::atomic::Ordering::SeqCst);
6165 wait_http_health(&module, SupervisorHealthStatus::Ok).await;
6166 assert_eq!(module.status().unwrap().health.consecutive_failures, 0);
6167 let before = module.status().unwrap();
6168 let (spec, mut health) = module.configuration().unwrap();
6169 health.http = None;
6170 module
6171 .update_configuration(spec.clone(), health.clone(), Some(10))
6172 .await
6173 .unwrap();
6174 wait_http_health(&module, SupervisorHealthStatus::Unknown).await;
6175 health.http = Some(url);
6176 module
6177 .update_configuration(spec, health, Some(10))
6178 .await
6179 .unwrap();
6180 wait_http_health(&module, SupervisorHealthStatus::Ok).await;
6181 assert_eq!(
6182 module.status().unwrap().pid,
6183 before.pid,
6184 "changing a probe must apply live, not restart its process"
6185 );
6186 let (spec, mut health) = module.configuration().unwrap();
6187 health.failure_threshold = 2;
6188 module
6189 .update_configuration(spec, health, Some(10))
6190 .await
6191 .unwrap();
6192 status.store(503, std::sync::atomic::Ordering::SeqCst);
6193 timeout(Duration::from_secs(5), async {
6194 while module.status().unwrap().spawn_generation == before.spawn_generation {
6195 sleep(Duration::from_millis(5)).await;
6196 }
6197 })
6198 .await
6199 .expect("sustained HTTP 503 responses must trigger the health restart policy");
6200 module.drain().await.unwrap();
6201 server.abort();
6202 }
6203
6204 #[cfg(unix)]
6205 #[tokio::test]
6206 async fn http_health_no_listener_restarts_after_consecutive_failures() {
6207 let dir = cortexkit_test_support::ScratchDir::new("http-refused");
6208 let unused = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
6209 let url = format!("http://{}/healthz", unused.local_addr().unwrap());
6210 drop(unused);
6211 let configured = http_fixture(&dir, &url, 2);
6212 let module =
6213 Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default())
6214 .supervise_configured_with_health(
6215 configured.module_spec(),
6216 true,
6217 configured.health,
6218 configured.drain_timeout_ms,
6219 configured.restart,
6220 )
6221 .unwrap();
6222 let before = module.status().unwrap().spawn_generation;
6223 timeout(Duration::from_secs(5), async {
6224 loop {
6225 let status = module.status().unwrap();
6226 if status.spawn_generation > before {
6227 assert!(status.lifetime_restarts > 0);
6228 break;
6229 }
6230 sleep(Duration::from_millis(5)).await;
6231 }
6232 })
6233 .await
6234 .expect("sustained HTTP refusal must trigger the health restart policy");
6235 module.drain().await.unwrap();
6236 }
6237
6238 #[cfg(unix)]
6239 #[tokio::test]
6240 async fn http_health_timeout_honours_deadline() {
6241 let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
6242 let url = format!("http://{}/healthz", listener.local_addr().unwrap());
6243 let server = tokio::spawn(async move {
6244 let _held = listener.accept().await.unwrap();
6245 std::future::pending::<()>().await;
6246 });
6247 let error = timeout(
6248 Duration::from_secs(1),
6249 probe_http_health(&url, Duration::from_millis(10)),
6250 )
6251 .await
6252 .expect("the probe must enforce its own deadline")
6253 .unwrap_err();
6254 server.abort();
6255 assert!(error.to_string().contains("timed out"));
6256 }
6257
6258 #[cfg(unix)]
6259 #[tokio::test]
6260 async fn http_health_localhost_reaches_an_ipv6_only_listener() {
6261 use tokio::io::{AsyncReadExt, AsyncWriteExt};
6262 let listener = tokio::net::TcpListener::bind("[::1]:0").await.unwrap();
6263 let url = format!(
6264 "http://localhost:{}/healthz",
6265 listener.local_addr().unwrap().port()
6266 );
6267 let server = tokio::spawn(async move {
6268 let (mut stream, _) = listener.accept().await.unwrap();
6269 let mut request = [0u8; 2048];
6270 assert!(stream.read(&mut request).await.unwrap() > 0);
6271 stream
6272 .write_all(b"HTTP/1.1 200 OK\r\nContent-Length: 2\r\n\r\nok")
6273 .await
6274 .unwrap();
6275 });
6276 assert_eq!(
6281 probe_http_health(&url, Duration::from_secs(10))
6282 .await
6283 .unwrap()
6284 .status,
6285 HealthStatus::Ok
6286 );
6287 server.await.unwrap();
6288 }
6289
6290 #[cfg(unix)]
6291 #[tokio::test]
6292 async fn http_health_timeout_keeps_partial_status_and_body() {
6293 use tokio::io::{AsyncReadExt, AsyncWriteExt};
6294 let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
6295 let url = format!("http://{}/healthz", listener.local_addr().unwrap());
6296 let server = tokio::spawn(async move {
6297 let (mut stream, _) = listener.accept().await.unwrap();
6298 let mut request = [0u8; 2048];
6299 assert!(stream.read(&mut request).await.unwrap() > 0);
6300 stream
6301 .write_all(
6302 b"HTTP/1.1 503 Unavailable\r\nContent-Length: 100\r\n\r\npartial diagnostic",
6303 )
6304 .await
6305 .unwrap();
6306 std::future::pending::<()>().await;
6307 });
6308 let error = probe_http_health(&url, Duration::from_secs(1))
6309 .await
6310 .unwrap_err()
6311 .to_string();
6312 server.abort();
6313 assert!(
6314 error.contains("timed out")
6315 && error.contains("503 Unavailable")
6316 && error.contains("partial diagnostic"),
6317 "{error}"
6318 );
6319 }
6320
6321 #[cfg(unix)]
6322 #[tokio::test]
6323 async fn http_health_diagnostic_body_is_bounded_to_200_bytes() {
6324 use tokio::io::{AsyncReadExt, AsyncWriteExt};
6325 let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
6326 let url = format!("http://{}/healthz", listener.local_addr().unwrap());
6327 let server = tokio::spawn(async move {
6328 let (mut stream, _) = listener.accept().await.unwrap();
6329 let mut request = [0u8; 2048];
6330 assert!(stream.read(&mut request).await.unwrap() > 0);
6331 let body = format!("{}not-in-diagnostic", "x".repeat(400));
6332 let response = format!(
6333 "HTTP/1.1 503 Unavailable\r\nContent-Length: {}\r\n\r\n{body}",
6334 body.len()
6335 );
6336 stream.write_all(response.as_bytes()).await.unwrap();
6337 });
6338 let error = probe_http_health(&url, Duration::from_secs(1))
6339 .await
6340 .unwrap_err()
6341 .to_string();
6342 server.await.unwrap();
6343 assert!(error.ends_with(&"x".repeat(200)), "{error}");
6344 assert_eq!(error.split(": ").last().unwrap().len(), 200);
6345 assert!(!error.contains("not-in-diagnostic"));
6346 }
6347
6348 #[cfg(unix)]
6349 #[tokio::test]
6350 async fn http_health_real_nats_server_monitoring() {
6351 if std::process::Command::new("nats-server")
6352 .arg("--version")
6353 .env("XDG_DATA_HOME", std::env::temp_dir())
6354 .env("XDG_RUNTIME_DIR", std::env::temp_dir())
6355 .env("XDG_CONFIG_HOME", std::env::temp_dir())
6356 .output()
6357 .is_err()
6358 {
6359 eprintln!(
6360 "skipped: http_health_real_nats_server_monitoring (nats-server is not on PATH)"
6361 );
6362 return;
6363 }
6364 let dir = cortexkit_test_support::ScratchDir::new("nats-http-monitoring");
6365 let monitor = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
6366 let port = monitor.local_addr().unwrap().port();
6367 let client = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
6368 let client_port = client.local_addr().unwrap().port();
6369 let config = dir.join("server.conf");
6370 std::fs::write(&config, format!("listen: 127.0.0.1:{client_port}\nhttp: 127.0.0.1:{port}\njetstream {{ store_dir: \"{}\" }}\n", dir.join("jetstream").display())).unwrap();
6371 let mut configured = http_fixture(&dir, &format!("http://127.0.0.1:{port}/healthz"), 1000);
6372 configured.program = PathBuf::from("nats-server");
6373 configured.args = vec!["-c".into(), config.to_string_lossy().into_owned()];
6374 drop(monitor);
6375 drop(client);
6376 let module =
6377 Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default())
6378 .supervise_configured_with_health(
6379 configured.module_spec(),
6380 true,
6381 configured.health,
6382 configured.drain_timeout_ms,
6383 configured.restart,
6384 )
6385 .unwrap();
6386 wait_http_health(&module, SupervisorHealthStatus::Ok).await;
6387 module.drain().await.unwrap();
6388 }
6389
6390 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
6391 async fn configuration_update_does_not_replace_captured_running_process_facts() {
6392 let supervisor =
6393 Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default());
6394 let snapshot = stale_process_snapshot(ModuleState::Running, true);
6395 let initial = ModuleSpec {
6396 module_id: "rescan-preserves-spawn-facts".to_string(),
6397 program: PathBuf::from("/spawned/module"),
6398 args: Vec::new(),
6399 env: Vec::new(),
6400 reserved: false,
6401 reserved_prefixes: Vec::new(),
6402 protocol: ModuleProtocol::Subc,
6403 overlap: Default::default(),
6404 };
6405 let module = supervisor.supervised_module(
6406 initial.clone(),
6407 supervisor.runtime_config(),
6408 snapshot,
6409 None,
6410 );
6411 let before = module.status().unwrap();
6412 let mut replacement = initial;
6413 replacement.program = PathBuf::from("/rescanned/replacement-module");
6414
6415 module
6416 .update_configuration(replacement, HealthConfig::default(), None)
6417 .await
6418 .unwrap();
6419
6420 let after = module.status().unwrap();
6421 assert_eq!(after.pid, before.pid);
6422 assert_eq!(after.spawned_at_ms, before.spawned_at_ms);
6423 assert_eq!(after.spawned_from, before.spawned_from);
6424 drop(module);
6425 }
6426}
6427
6428fn unix_ms_now() -> u64 {
6429 SystemTime::now()
6430 .duration_since(UNIX_EPOCH)
6431 .map(|duration| duration.as_millis().min(u128::from(u64::MAX)) as u64)
6432 .unwrap_or(0)
6433}
6434
6435async fn supervise_loop(
6436 mut spec: ModuleSpec,
6437 mut runtime: SupervisorRuntimeConfig,
6438 registry: Arc<Registry>,
6439 process_liveness: Arc<SupervisorProcessLiveness>,
6440 snapshot: SharedSnapshot,
6441 mut child: Option<SupervisedChild>,
6442 mut commands: mpsc::Receiver<SupervisorCommand>,
6443) {
6444 let mut health_probe = HealthProbeRuntime::default();
6445 let mut registration_changes = match registry.subscribe_module_changes(&spec.module_id) {
6449 Ok(changes) => Some(changes),
6450 Err(err) => {
6451 warn!(module_id = %spec.module_id, error = %err, "health prober could not subscribe to registry");
6454 None
6455 }
6456 };
6457 let mut pending_respawn: Option<PendingRespawn> = None;
6461 let mut requeued: VecDeque<SupervisorCommand> = VecDeque::new();
6464 let mut previous_instance_recheck = tokio::time::interval_at(
6465 Instant::now() + Duration::from_secs(5),
6466 Duration::from_secs(5),
6467 );
6468 previous_instance_recheck.set_missed_tick_behavior(tokio::time::MissedTickBehavior::Skip);
6469 loop {
6470 #[cfg(test)]
6471 {
6472 lock_snapshot(&snapshot).unwrap().actor_select = None;
6473 }
6474 #[cfg(target_os = "macos")]
6475 if let Some(active) = child.as_mut() {
6476 active.confirm_privacy_exec().await;
6477 }
6478 if let Some(scheduled) = runtime
6479 .scheduled_respawn
6480 .lock()
6481 .unwrap_or_else(|p| p.into_inner())
6482 .take()
6483 {
6484 pending_respawn = Some(scheduled);
6485 }
6486 if pending_respawn.is_some() && (child.is_some() || !respawn_still_pending(&snapshot)) {
6487 pending_respawn = None;
6488 cancel_deferred_reload(
6489 &runtime,
6490 &spec.module_id,
6491 "respawn cancelled by a supervisor command",
6492 );
6493 }
6494 if child.is_none()
6495 && pending_respawn.is_none()
6496 && lock_snapshot(&snapshot).is_ok_and(|state| state.waiting_for_previous_pid.is_none())
6497 {
6498 cancel_deferred_reload(
6499 &runtime,
6500 &spec.module_id,
6501 "respawn cancelled before a replacement was spawned",
6502 );
6503 let _ = update_snapshot(&snapshot, Some(&spec.module_id), |state| {
6504 state.respawn_pending = false;
6505 state.coalesced_restart_pending = false;
6506 if matches!(
6507 state.state,
6508 ModuleState::Restarting
6509 | ModuleState::Starting
6510 | ModuleState::Draining
6511 | ModuleState::Unresponsive
6512 ) {
6513 error!(module_id = %spec.module_id, state = ?state.state,
6514 "supervision operation ended without a child or pending respawn; marking failed so start can retry");
6515 state.state = ModuleState::Failed;
6516 clear_current_process_facts(state);
6517 }
6518 });
6519 }
6520 if let Some(command) = requeued.pop_front() {
6521 if !handle_supervisor_command(
6522 command,
6523 &mut spec,
6524 &mut runtime,
6525 ®istry,
6526 &process_liveness,
6527 &snapshot,
6528 &mut child,
6529 &mut commands,
6530 &mut requeued,
6531 )
6532 .await
6533 {
6534 return;
6535 }
6536 if child.is_some() || !respawn_still_pending(&snapshot) {
6537 pending_respawn = None;
6538 let _ = update_snapshot(&snapshot, Some(&spec.module_id), |state| {
6539 state.respawn_pending = false
6540 });
6541 }
6542 continue;
6543 }
6544 if child.is_some() {
6545 #[cfg(test)]
6546 {
6547 lock_snapshot(&snapshot).unwrap().actor_turns += 1;
6548 }
6549 health_probe.refresh_registration(&spec, &runtime, ®istry, &snapshot);
6550 let wake_after = health_probe.wake_after();
6551 let probe_sleep = async move {
6552 match wake_after {
6553 Some(delay) => sleep(delay).await,
6554 None => std::future::pending().await,
6555 }
6556 };
6557 tokio::pin!(probe_sleep);
6558 let wire_child = running_protocol(&spec, &snapshot) == ModuleProtocol::Subc;
6563 #[cfg(test)]
6564 {
6565 let registration_pending = wire_child
6566 && registration_changes
6567 .as_ref()
6568 .is_some_and(|changes| changes.has_changed().unwrap());
6569 let mut state = lock_snapshot(&snapshot).unwrap();
6570 state.actor_select = (!registration_pending).then_some(ActorSelectCheckpoint {
6571 generation: state.spawn_generation,
6572 turn: state.actor_turns,
6573 registered_connection: health_probe.registered_connection,
6574 next_probe_at: health_probe.next_probe_at,
6575 wake_after,
6576 });
6577 }
6578 let active_child = child.as_mut().expect("child checked above");
6579 tokio::select! {
6580 wait_result = active_child.wait() => {
6581 #[cfg(test)]
6582 { lock_snapshot(&snapshot).unwrap().actor_select = None; }
6583 let exit_report = match wait_result {
6592 Ok(status) => classify_reaped_child_exit(&snapshot, active_child, &status),
6593 Err(err) => {
6594 active_child.drain_stderr(&spec.module_id).await;
6595 fail_after_wait_error(
6596 &spec.module_id,
6597 &snapshot,
6598 &runtime.terminal_ring,
6599 &runtime.spawn_events,
6600 format!("failed to wait for supervised module: {err}"),
6601 );
6602 untrack_if_registration_released(
6603 &process_liveness,
6604 ®istry,
6605 &spec.module_id,
6606 &snapshot,
6607 );
6608 error!(module_id = %spec.module_id, error = %err, "failed to wait for supervised module");
6609 child = None;
6610 continue;
6611 }
6612 };
6613 active_child.drain_stderr(&spec.module_id).await;
6614
6615 let next = on_child_exit(
6616 &spec,
6617 runtime.restart_policy,
6618 ®istry,
6619 &snapshot,
6620 &runtime.terminal_ring,
6621 &runtime.spawn_events,
6622 &runtime.child_roster,
6623 exit_report,
6624 ).await;
6625 runtime.start_requester = None;
6626 runtime.pending_activity_restart = false;
6627 active_child.release_roster();
6630 match next {
6631 NextAction::Stop { registration_released } => {
6632 if registration_released {
6633 process_liveness.untrack_if_current(&spec.module_id, &snapshot);
6634 }
6635 child = None;
6636 }
6637 NextAction::Restart { schedule } => {
6638 let delay = schedule.map_or(
6639 runtime.restart_policy.delay_for_restart(0),
6640 |schedule| schedule.delay,
6641 );
6642 if let Some(schedule) = schedule {
6643 log_crash_respawn(&spec.module_id, schedule);
6644 }
6645 child = None;
6653 pending_respawn = Some(PendingRespawn { deadline: Instant::now() + delay, kind: RespawnKind::Spawn });
6654 let _ = update_snapshot(&snapshot, Some(&spec.module_id), |state| state.respawn_pending = true);
6655 }
6656 }
6657 }
6658 command = commands.recv() => {
6659 #[cfg(test)]
6660 { lock_snapshot(&snapshot).unwrap().actor_select = None; }
6661 let Some(command) = command else {
6662 return;
6663 };
6664 if !handle_supervisor_command(
6665 command,
6666 &mut spec,
6667 &mut runtime,
6668 ®istry,
6669 &process_liveness,
6670 &snapshot,
6671 &mut child,
6672 &mut commands,
6673 &mut requeued,
6674 ).await {
6675 return;
6676 }
6677 }
6678 _ = async {
6679 match registration_changes.as_mut() {
6680 Some(changes) => { let _ = changes.changed().await; }
6681 None => std::future::pending().await,
6682 }
6683 }, if wire_child => {
6684 #[cfg(test)]
6685 { lock_snapshot(&snapshot).unwrap().actor_select = None; }
6686 }
6687 _ = &mut probe_sleep => {
6688 #[cfg(test)]
6689 { lock_snapshot(&snapshot).unwrap().actor_select = None; }
6690 if health_probe.due() {
6691 run_health_probe_cycle(
6692 &spec,
6693 &runtime,
6694 ®istry,
6695 &process_liveness,
6696 &snapshot,
6697 &mut child,
6698 ).await;
6699 if child.is_none() {
6700 runtime.start_requester = None;
6701 runtime.pending_activity_restart = false;
6702 }
6703 if child.is_some() {
6704 health_probe.schedule_next(&spec, runtime.health.cadence);
6705 }
6706 }
6707 }
6708 }
6709 } else if let Some(pending) = pending_respawn {
6710 tokio::select! {
6711 _ = sleep_until(pending.deadline) => {
6712 pending_respawn = None;
6713 let _ = update_snapshot(&snapshot, Some(&spec.module_id), |state| state.respawn_pending = false);
6714 if !respawn_still_pending(&snapshot) {
6718 continue;
6719 }
6720 if runtime.child_roster.is_closed() {
6725 let _ = update_snapshot(&snapshot, Some(&spec.module_id), |state| {
6726 state.state = ModuleState::Stopped;
6727 });
6728 debug!(module_id = %spec.module_id, "crash respawn cancelled by daemon shutdown");
6729 continue;
6730 }
6731 if let Err(err) = release_dead_registration(
6732 ®istry,
6733 runtime.forwarding.as_deref(),
6734 &snapshot,
6735 &spec.module_id,
6736 ).await {
6737 fail_snapshot(&snapshot, Some(&spec.module_id), None);
6738 error!(module_id = %spec.module_id, error = %err, "registration did not release before restart");
6739 continue;
6740 }
6741
6742 if matches!(pending.kind, RespawnKind::Reload) {
6743 let reply = runtime.deferred_reload_reply.lock().unwrap_or_else(|p| p.into_inner()).take();
6744 let result = finish_reload_child(&spec, &runtime, ®istry, &process_liveness, &snapshot, &mut child).await;
6745 if result.is_ok() && runtime.pending_activity_restart {
6746 runtime.record_activity_transition("subc.module_restarted", &spec.module_id, runtime.start_requester.clone());
6747 }
6748 runtime.pending_activity_restart = false;
6749 if let Some(reply) = reply { let _ = reply.send(result); }
6750 continue;
6751 }
6752 process_liveness.track(spec.module_id.clone(), Arc::clone(&snapshot));
6753 match spawn_and_mark_running(&spec, &runtime, &snapshot) {
6754 Ok(next_child) => {
6755 child = Some(next_child);
6756 if runtime.pending_activity_restart {
6757 runtime.record_activity_transition("subc.module_restarted", &spec.module_id, runtime.start_requester.clone());
6758 runtime.pending_activity_restart = false;
6759 }
6760 debug!(module_id = %spec.module_id, "supervised module restarted after crash");
6761 }
6762 Err(err) => {
6763 fail_snapshot(&snapshot, Some(&spec.module_id), None);
6764 process_liveness.untrack_if_current(&spec.module_id, &snapshot);
6765 error!(module_id = %spec.module_id, error = %err, "failed to restart supervised module");
6766 }
6767 }
6768 }
6769 command = commands.recv() => {
6770 let Some(command) = command else {
6771 return;
6772 };
6773 if !handle_supervisor_command(
6774 command,
6775 &mut spec,
6776 &mut runtime,
6777 ®istry,
6778 &process_liveness,
6779 &snapshot,
6780 &mut child,
6781 &mut commands,
6782 &mut requeued,
6783 ).await {
6784 return;
6785 }
6786 if child.is_some() || !respawn_still_pending(&snapshot) {
6791 pending_respawn = None;
6792 let _ = update_snapshot(&snapshot, Some(&spec.module_id), |state| state.respawn_pending = false);
6793 }
6794 }
6795 }
6796 } else {
6797 let command = if lock_snapshot(&snapshot)
6798 .is_ok_and(|state| state.waiting_for_previous_pid.is_some())
6799 {
6800 tokio::select! {
6801 command = commands.recv() => command,
6802 _ = previous_instance_recheck.tick() => {
6803 if runtime.child_roster.previous_instance_pid(&spec.module_id).is_none() {
6804 let enabled = lock_snapshot(&snapshot).is_ok_and(|state| state.enabled);
6805 if enabled && !runtime.child_roster.is_closed() {
6806 process_liveness.track(spec.module_id.clone(), Arc::clone(&snapshot));
6807 match spawn_and_mark_running(&spec, &runtime, &snapshot) {
6808 Ok(next_child) => child = Some(next_child),
6809 Err(error) => {
6810 fail_snapshot(&snapshot, Some(&spec.module_id), None);
6811 error!(module_id = %spec.module_id, %error, "failed to start module after its previous instance exited");
6812 }
6813 }
6814 } else {
6815 let _ = update_snapshot(&snapshot, Some(&spec.module_id), |state| {
6816 state.waiting_for_previous_pid = None;
6817 state.waiting_for_previous_detail = None;
6818 });
6819 }
6820 }
6821 continue;
6822 }
6823 }
6824 } else {
6825 commands.recv().await
6826 };
6827 let Some(command) = command else {
6828 return;
6829 };
6830 if !handle_supervisor_command(
6831 command,
6832 &mut spec,
6833 &mut runtime,
6834 ®istry,
6835 &process_liveness,
6836 &snapshot,
6837 &mut child,
6838 &mut commands,
6839 &mut requeued,
6840 )
6841 .await
6842 {
6843 return;
6844 }
6845 }
6846 }
6847}
6848
6849fn log_crash_respawn(module_id: &str, schedule: CrashRestartSchedule) {
6850 info!(
6851 module_id,
6852 restart_in_window = schedule.restart_in_window,
6853 delay_ms = schedule.delay.as_millis() as u64,
6854 "respawning after crash"
6855 );
6856}
6857
6858fn respawn_still_pending(snapshot: &SharedSnapshot) -> bool {
6864 matches!(
6865 lock_snapshot(snapshot),
6866 Ok(state) if state.enabled && state.state == ModuleState::Restarting
6867 )
6868}
6869
6870enum NextAction {
6871 Stop {
6872 registration_released: bool,
6873 },
6874 Restart {
6875 schedule: Option<CrashRestartSchedule>,
6876 },
6877}
6878
6879#[allow(clippy::too_many_arguments)]
6880async fn handle_supervisor_command(
6881 command: SupervisorCommand,
6882 spec: &mut ModuleSpec,
6883 runtime: &mut SupervisorRuntimeConfig,
6884 registry: &Arc<Registry>,
6885 process_liveness: &SupervisorProcessLiveness,
6886 snapshot: &SharedSnapshot,
6887 child: &mut Option<SupervisedChild>,
6888 commands: &mut mpsc::Receiver<SupervisorCommand>,
6889 requeued: &mut VecDeque<SupervisorCommand>,
6890) -> bool {
6891 let command =
6892 if let Some((pid, reason)) = runtime.child_roster.previous_instance(&spec.module_id) {
6893 match command {
6894 SupervisorCommand::Restart { reply, .. }
6895 | SupervisorCommand::Reload { reply, .. }
6896 | SupervisorCommand::Swap { reply, .. } => {
6897 let _ = reply.send(Err(previous_instance_error(spec, pid, reason)));
6898 return true;
6899 }
6900 SupervisorCommand::SetEnabled {
6901 enabled: true,
6902 reply,
6903 ..
6904 } => {
6905 let _ = reply.send(Err(previous_instance_error(spec, pid, reason)));
6906 return true;
6907 }
6908 other => other,
6909 }
6910 } else {
6911 command
6912 };
6913 match command {
6914 SupervisorCommand::Drain { requester, reply } => {
6915 let result = drain_optional_child(
6918 &spec.module_id,
6919 spec.protocol,
6920 StopNotice::NotSent,
6921 registry,
6922 runtime.forwarding.as_deref(),
6923 snapshot,
6924 &runtime.terminal_ring,
6925 &runtime.spawn_events,
6926 child,
6927 runtime.drain_timeout,
6928 ModuleState::Stopped,
6929 None,
6930 )
6931 .await;
6932 let registration_released = result.is_ok();
6933 if registration_released {
6934 runtime.record_activity_transition(
6935 "subc.module_stopped",
6936 &spec.module_id,
6937 requester,
6938 );
6939 }
6940 let _ = reply.send(result);
6941 if registration_released {
6942 process_liveness.untrack_if_current(&spec.module_id, snapshot);
6943 }
6944 false
6945 }
6946 SupervisorCommand::Retire { requester, reply } => {
6947 let result = async {
6948 let stop_notice = begin_forwarding_drain_if_configured(
6949 spec,
6950 runtime,
6951 registry,
6952 snapshot,
6953 None,
6954 RouteCloseReason::Disable,
6955 )
6956 .await?;
6957 drain_optional_child(
6958 &spec.module_id,
6959 spec.protocol,
6960 stop_notice,
6961 registry,
6962 runtime.forwarding.as_deref(),
6963 snapshot,
6964 &runtime.terminal_ring,
6965 &runtime.spawn_events,
6966 child,
6967 runtime.drain_timeout,
6968 ModuleState::Stopped,
6969 None,
6970 )
6971 .await
6972 }
6973 .await;
6974 let registration_released = result.is_ok();
6975 if registration_released {
6976 runtime.record_activity_transition(
6977 "subc.module_stopped",
6978 &spec.module_id,
6979 requester,
6980 );
6981 }
6982 let _ = reply.send(result);
6983 if registration_released {
6984 process_liveness.untrack_if_current(&spec.module_id, snapshot);
6985 }
6986 false
6987 }
6988 SupervisorCommand::Restart {
6989 requester,
6990 drain_timeout_ms,
6991 received_at_generation,
6992 queued_at,
6993 reply,
6994 } => {
6995 info!(
6999 module_id = %spec.module_id,
7000 queued_ms = u64::try_from(queued_at.elapsed().as_millis()).unwrap_or(u64::MAX),
7001 "restart command dequeued"
7002 );
7003 let validation = match lock_snapshot(snapshot) {
7015 Ok(state) if !state.enabled => Err(SuperviseError::Disabled {
7016 module_id: spec.module_id.clone(),
7017 }),
7018 Ok(_) => Ok(()),
7019 Err(err) => Err(err),
7020 };
7021 let initiated = validation.is_ok();
7022 let _ = reply.send(validation);
7023 let satisfied_by_generation = if initiated
7037 && child
7038 .as_ref()
7039 .is_some_and(|child| child.spawned_at > queued_at)
7040 {
7041 lock_snapshot(snapshot).ok().and_then(|state| {
7042 (!state.configuration_updated_since_spawn).then_some(state.spawn_generation)
7043 })
7044 } else {
7045 None
7046 };
7047 let satisfied_by_pending = initiated
7048 && child.is_none()
7049 && lock_snapshot(snapshot).ok().is_some_and(|mut state| {
7050 let pending = state.respawn_pending && !state.configuration_updated_since_spawn;
7054 if pending {
7055 state.coalesced_restart_pending = true;
7056 }
7057 pending
7058 });
7059 if satisfied_by_pending {
7060 debug!(module_id = %spec.module_id, "restart coalesced into the pending replacement");
7061 } else if let Some(generation) = satisfied_by_generation {
7062 info!(
7063 module_id = %spec.module_id,
7064 received_at_generation,
7065 "restart already satisfied by generation {generation}; not restarting again"
7066 );
7067 } else if initiated {
7068 let drain_timeout = drain_timeout_ms
7071 .map(Duration::from_millis)
7072 .unwrap_or(runtime.drain_timeout);
7073 if let Err(err) = restart_child(
7074 spec,
7075 runtime,
7076 registry,
7077 process_liveness,
7078 snapshot,
7079 child,
7080 drain_timeout,
7081 )
7082 .await
7083 {
7084 warn!(
7085 module_id = %spec.module_id,
7086 error = %err,
7087 "operator restart failed after initiation ack; module state carries the outcome"
7088 );
7089 let _ = update_snapshot(snapshot, Some(&spec.module_id), |state| {
7090 state.state = ModuleState::Failed;
7091 clear_current_process_facts(state);
7092 });
7093 } else {
7094 runtime.start_requester = requester;
7095 runtime.pending_activity_restart = true;
7096 }
7097 }
7098 true
7099 }
7100 SupervisorCommand::Reload { requester, reply } => {
7101 let result =
7102 reload_child(spec, runtime, registry, process_liveness, snapshot, child).await;
7103 if result.is_ok() {
7104 runtime.start_requester = requester;
7105 runtime.pending_activity_restart = true;
7106 }
7107 if result.is_ok()
7108 && runtime
7109 .scheduled_respawn
7110 .lock()
7111 .unwrap_or_else(|p| p.into_inner())
7112 .as_ref()
7113 .is_some_and(|pending| matches!(pending.kind, RespawnKind::Reload))
7114 {
7115 *runtime
7116 .deferred_reload_reply
7117 .lock()
7118 .unwrap_or_else(|p| p.into_inner()) = Some(reply);
7119 } else {
7120 let _ = reply.send(result);
7121 }
7122 true
7123 }
7124 SupervisorCommand::SetEnabled {
7125 enabled,
7126 requester,
7127 reply,
7128 } => {
7129 let previous_requester = runtime.start_requester.clone();
7130 if enabled {
7131 runtime.start_requester = requester.clone();
7132 }
7133 let result = set_child_enabled(
7134 spec,
7135 runtime,
7136 registry,
7137 process_liveness,
7138 snapshot,
7139 child,
7140 enabled,
7141 )
7142 .await;
7143 if matches!(result, Ok(true)) {
7144 runtime.pending_activity_restart = false;
7145 if enabled {
7146 runtime.start_requester = requester;
7147 } else {
7148 runtime.start_requester = None;
7149 runtime.record_activity_transition(
7150 "subc.module_disabled",
7151 &spec.module_id,
7152 requester,
7153 );
7154 }
7155 } else {
7156 runtime.start_requester = previous_requester;
7157 }
7158 let _ = reply.send(result);
7159 true
7160 }
7161 SupervisorCommand::UpdateConfiguration {
7162 spec: next_spec,
7163 health,
7164 drain_timeout_ms,
7165 reply,
7166 } => {
7167 if let Some(handle) = &runtime.supervisor_handle {
7168 handle.apply_identity_configuration(&next_spec);
7169 }
7170 *spec = next_spec;
7171 let _ = update_snapshot(snapshot, Some(&spec.module_id), |state| {
7172 state.configuration_updated_since_spawn = true;
7173 });
7174 let health_changed = runtime.health != health;
7175 runtime.health = health;
7176 if health_changed {
7179 let _ = update_snapshot(snapshot, Some(&spec.module_id), |state| {
7180 state.health = ModuleHealthStatus::default();
7181 });
7182 }
7183 runtime.drain_timeout = drain_timeout_ms
7184 .map(Duration::from_millis)
7185 .unwrap_or(runtime.default_drain_timeout);
7186 *runtime
7187 .effective_drain_timeout
7188 .lock()
7189 .unwrap_or_else(|poisoned| poisoned.into_inner()) = runtime.drain_timeout;
7190 let _ = reply.send(());
7191 true
7192 }
7193 SupervisorCommand::Swap {
7194 requester,
7195 ready_timeout,
7196 reply,
7197 } => {
7198 let mut swap_runtime = runtime.clone();
7199 swap_runtime.start_requester = requester;
7200 let before = lock_snapshot(snapshot)
7201 .ok()
7202 .map(|state| state.spawn_generation);
7203 let end = swap::run_swap(
7204 spec,
7205 &swap_runtime,
7206 registry,
7207 process_liveness,
7208 snapshot,
7209 child,
7210 commands,
7211 ready_timeout.unwrap_or(DEFAULT_SWAP_READY_TIMEOUT),
7212 reply,
7213 )
7214 .await;
7215 if before
7216 != lock_snapshot(snapshot)
7217 .ok()
7218 .map(|state| state.spawn_generation)
7219 {
7220 runtime.start_requester = swap_runtime.start_requester;
7221 }
7222 requeued.extend(end.requeue);
7223 true
7224 }
7225 }
7226}
7227
7228async fn restart_child(
7229 spec: &ModuleSpec,
7230 runtime: &SupervisorRuntimeConfig,
7231 registry: &Registry,
7232 process_liveness: &SupervisorProcessLiveness,
7233 snapshot: &SharedSnapshot,
7234 child: &mut Option<SupervisedChild>,
7235 drain_timeout: Duration,
7236) -> Result<(), SuperviseError> {
7237 if !lock_snapshot(snapshot)?.enabled {
7239 return Err(SuperviseError::Disabled {
7240 module_id: spec.module_id.clone(),
7241 });
7242 }
7243 let stop_notice = begin_forwarding_drain_with_timeout(
7244 spec,
7245 runtime,
7246 registry,
7247 snapshot,
7248 None,
7249 RouteCloseReason::Restart,
7250 drain_timeout,
7251 )
7252 .await?;
7253
7254 if child.is_some() {
7255 drain_optional_child(
7256 &spec.module_id,
7257 spec.protocol,
7258 stop_notice,
7259 registry,
7260 runtime.forwarding.as_deref(),
7261 snapshot,
7262 &runtime.terminal_ring,
7263 &runtime.spawn_events,
7264 child,
7265 drain_timeout,
7266 ModuleState::Restarting,
7267 Some(true),
7268 )
7269 .await?;
7270 } else {
7271 update_snapshot(snapshot, Some(&spec.module_id), |state| {
7272 state.enabled = true;
7273 state.state = ModuleState::Restarting;
7274 clear_current_process_facts(state);
7275 })?;
7276 release_dead_registration(
7277 registry,
7278 runtime.forwarding.as_deref(),
7279 snapshot,
7280 &spec.module_id,
7281 )
7282 .await?;
7283 }
7284
7285 reset_restart_count(snapshot, &spec.module_id)?;
7286 process_liveness.track(spec.module_id.clone(), Arc::clone(snapshot));
7287 schedule_respawn(
7288 runtime,
7289 snapshot,
7290 &spec.module_id,
7291 runtime.restart_policy.backoff,
7292 RespawnKind::Spawn,
7293 )
7294}
7295
7296async fn reload_child(
7297 spec: &ModuleSpec,
7298 runtime: &SupervisorRuntimeConfig,
7299 registry: &Registry,
7300 process_liveness: &SupervisorProcessLiveness,
7301 snapshot: &SharedSnapshot,
7302 child: &mut Option<SupervisedChild>,
7303) -> Result<(), SuperviseError> {
7304 if !lock_snapshot(snapshot)?.enabled {
7306 return Err(SuperviseError::Disabled {
7307 module_id: spec.module_id.clone(),
7308 });
7309 }
7310 let stop_notice = begin_forwarding_drain(
7311 spec,
7312 runtime,
7313 registry,
7314 snapshot,
7315 Some(true),
7316 RouteCloseReason::Reload,
7317 )
7318 .await?;
7319
7320 if child.is_some() {
7321 drain_optional_child(
7322 &spec.module_id,
7323 spec.protocol,
7324 stop_notice,
7325 registry,
7326 runtime.forwarding.as_deref(),
7327 snapshot,
7328 &runtime.terminal_ring,
7329 &runtime.spawn_events,
7330 child,
7331 runtime.drain_timeout,
7332 ModuleState::Restarting,
7333 Some(true),
7334 )
7335 .await?;
7336 } else {
7337 update_snapshot(snapshot, Some(&spec.module_id), |state| {
7338 state.enabled = true;
7339 state.state = ModuleState::Restarting;
7340 clear_current_process_facts(state);
7341 })?;
7342 release_dead_registration(
7343 registry,
7344 runtime.forwarding.as_deref(),
7345 snapshot,
7346 &spec.module_id,
7347 )
7348 .await?;
7349 }
7350
7351 reset_restart_count(snapshot, &spec.module_id)?;
7352 process_liveness.track(spec.module_id.clone(), Arc::clone(snapshot));
7353 schedule_respawn(
7354 runtime,
7355 snapshot,
7356 &spec.module_id,
7357 runtime.restart_policy.backoff,
7358 RespawnKind::Reload,
7359 )
7360}
7361
7362async fn finish_reload_child(
7363 spec: &ModuleSpec,
7364 runtime: &SupervisorRuntimeConfig,
7365 registry: &Registry,
7366 process_liveness: &SupervisorProcessLiveness,
7367 snapshot: &SharedSnapshot,
7368 child: &mut Option<SupervisedChild>,
7369) -> Result<(), SuperviseError> {
7370 process_liveness.track(spec.module_id.clone(), Arc::clone(snapshot));
7371 let next_child = match spawn_and_mark_running(spec, runtime, snapshot) {
7372 Ok(next_child) => next_child,
7373 Err(err) => {
7374 return handle_reload_spawn_failure(
7375 spec,
7376 runtime,
7377 process_liveness,
7378 snapshot,
7379 child,
7380 format!("new child failed to spawn: {err}"),
7381 )
7382 .await;
7383 }
7384 };
7385 *child = Some(next_child);
7386
7387 let wait_outcome = {
7388 let active_child = child.as_mut().expect("new reload child was just stored");
7389 wait_for_registration_after_reload(
7390 registry,
7391 &spec.module_id,
7392 snapshot,
7393 active_child,
7394 REGISTRY_RELEASE_TIMEOUT,
7395 )
7396 .await?
7397 };
7398
7399 match wait_outcome {
7400 RegistrationWaitOutcome::Registered => {
7401 debug!(module_id = %spec.module_id, "supervised module reloaded and registered");
7402 Ok(())
7403 }
7404 RegistrationWaitOutcome::Exited(exit_report) => {
7405 if let Some(active_child) = child.as_mut() {
7406 active_child.drain_stderr(&spec.module_id).await;
7407 }
7408 let mut exited_child = child.take().expect("exited reload child is still stored");
7411 #[cfg(test)]
7412 if let Some(gate) = &runtime.test_reload_exit_record_gate {
7413 gate.reached.notify_one();
7414 gate.resume.notified().await;
7415 }
7416 let result = handle_reload_child_registration_failure(
7417 spec,
7418 runtime,
7419 registry,
7420 process_liveness,
7421 snapshot,
7422 child,
7423 ReloadRegistrationFailure {
7424 exit_report: registration_failure_exit_report(exit_report),
7425 reason: exited_child
7426 .spawn_failure
7427 .clone()
7428 .unwrap_or_else(|| "new child exited before registering".to_string()),
7429 },
7430 )
7431 .await;
7432 exited_child.release_roster();
7433 result
7434 }
7435 RegistrationWaitOutcome::TimedOut => {
7436 let mut timed_out_child = child
7437 .take()
7438 .expect("timed-out reload child is still running");
7439 timed_out_child
7440 .start_kill()
7441 .map_err(|source| SuperviseError::Kill {
7442 module_id: spec.module_id.clone(),
7443 source,
7444 })?;
7445 let status = timed_out_child
7446 .wait()
7447 .await
7448 .map_err(|source| SuperviseError::Wait {
7449 module_id: spec.module_id.clone(),
7450 source,
7451 })?;
7452 timed_out_child.drain_stderr(&spec.module_id).await;
7453 handle_reload_child_registration_failure(
7454 spec,
7455 runtime,
7456 registry,
7457 process_liveness,
7458 snapshot,
7459 child,
7460 ReloadRegistrationFailure {
7461 exit_report: registration_failure_exit_report(classify_reaped_child_exit(
7462 snapshot,
7463 &timed_out_child,
7464 &status,
7465 )),
7466 reason: format!(
7467 "new child did not register within {:?}",
7468 REGISTRY_RELEASE_TIMEOUT
7469 ),
7470 },
7471 )
7472 .await
7473 }
7474 }
7475}
7476
7477async fn set_child_enabled(
7478 spec: &ModuleSpec,
7479 runtime: &SupervisorRuntimeConfig,
7480 registry: &Registry,
7481 process_liveness: &SupervisorProcessLiveness,
7482 snapshot: &SharedSnapshot,
7483 child: &mut Option<SupervisedChild>,
7484 enabled: bool,
7485) -> Result<bool, SuperviseError> {
7486 let (current_enabled, current_state, respawn_pending) = {
7487 let state = lock_snapshot(snapshot)?;
7488 (state.enabled, state.state, state.respawn_pending)
7489 };
7490 let revive_terminal = enabled
7498 && current_enabled
7499 && child.is_none()
7500 && (matches!(current_state, ModuleState::Failed | ModuleState::Stopped)
7501 || (current_state == ModuleState::Restarting && !respawn_pending));
7502 if current_enabled == enabled && !revive_terminal {
7503 return Ok(false);
7504 }
7505
7506 if enabled {
7507 update_snapshot(snapshot, Some(&spec.module_id), |state| {
7508 state.enabled = true;
7509 state.state = ModuleState::Starting;
7510 clear_current_process_facts(state);
7511 })?;
7512 #[cfg(test)]
7513 if runtime.test_seed_stale_facts_before_enable_spawn {
7514 update_snapshot(snapshot, Some(&spec.module_id), |state| {
7515 state.process_alive = true;
7516 state.pid = Some(41);
7517 state.spawned_at_ms = Some(42);
7518 state.spawned_from = Some(PathBuf::from("/spawned/module"));
7519 state.spawned_file_identity = Some(SpawnedFileIdentity {
7520 device: 43,
7521 inode: 44,
7522 #[cfg(windows)]
7523 windows_image: None,
7524 });
7525 })?;
7526 }
7527 release_dead_registration(
7528 registry,
7529 runtime.forwarding.as_deref(),
7530 snapshot,
7531 &spec.module_id,
7532 )
7533 .await?;
7534 reset_restart_count(snapshot, &spec.module_id)?;
7535 process_liveness.track(spec.module_id.clone(), Arc::clone(snapshot));
7536 let next_child = match spawn_and_mark_running(spec, runtime, snapshot) {
7537 Ok(next_child) => next_child,
7538 Err(err) => {
7539 if let Err(state_err) = update_snapshot(snapshot, Some(&spec.module_id), |state| {
7540 state.state = ModuleState::Failed;
7541 clear_current_process_facts(state);
7542 }) {
7543 error!(module_id = %spec.module_id, error = %state_err, "failed to record enable spawn failure");
7544 }
7545 process_liveness.untrack_if_current(&spec.module_id, snapshot);
7546 return Err(err);
7547 }
7548 };
7549 *child = Some(next_child);
7550 debug!(module_id = %spec.module_id, "supervised module enabled");
7551 Ok(true)
7552 } else {
7553 let stop_notice = begin_forwarding_drain_if_configured(
7554 spec,
7555 runtime,
7556 registry,
7557 snapshot,
7558 Some(false),
7559 RouteCloseReason::Disable,
7560 )
7561 .await?;
7562 drain_optional_child(
7563 &spec.module_id,
7564 spec.protocol,
7565 stop_notice,
7566 registry,
7567 runtime.forwarding.as_deref(),
7568 snapshot,
7569 &runtime.terminal_ring,
7570 &runtime.spawn_events,
7571 child,
7572 runtime.drain_timeout,
7573 ModuleState::Disabled,
7574 Some(false),
7575 )
7576 .await?;
7577 debug!(module_id = %spec.module_id, "supervised module disabled");
7578 Ok(true)
7579 }
7580}
7581
7582#[allow(clippy::too_many_arguments)]
7583async fn on_child_exit(
7584 spec: &ModuleSpec,
7585 policy: RestartPolicy,
7586 registry: &Registry,
7587 snapshot: &SharedSnapshot,
7588 terminal_ring: &Arc<Mutex<TerminalRing>>,
7589 spawn_events: &SpawnEventFeed,
7590 roster: &ChildRoster,
7591 exit_report: ExitReport,
7592) -> NextAction {
7593 if roster.is_closed() {
7599 return on_child_exit_during_daemon_shutdown(
7600 spec,
7601 registry,
7602 snapshot,
7603 terminal_ring,
7604 spawn_events,
7605 exit_report,
7606 )
7607 .await;
7608 }
7609 let unrequested_clean_exit_of_protocol_none = exit_report.kind == ExitKind::Clean
7624 && running_protocol(spec, snapshot) == ModuleProtocol::None;
7625 match exit_report.kind {
7626 ExitKind::Clean if !unrequested_clean_exit_of_protocol_none => {
7627 info!(
7628 module_id = %spec.module_id,
7629 exit_code = ?exit_report.code,
7630 exit_signal = ?exit_report.signal,
7631 "supervised module exited cleanly"
7632 );
7633 if let Err(err) = publish_after_terminal_record(
7634 &spec.module_id,
7635 || {
7636 record_terminal(
7637 &spec.module_id,
7638 terminal_ring,
7639 spawn_events,
7640 &exit_report,
7641 TerminalDisposition::Stopped,
7642 )
7643 },
7644 || {
7645 update_snapshot(snapshot, Some(&spec.module_id), |state| {
7646 state.state = ModuleState::Stopped;
7647 clear_current_process_facts(state);
7648 state.last_exit = Some(exit_report.clone());
7649 })
7650 },
7651 ) {
7652 error!(module_id = %spec.module_id, error = %err, "failed to publish clean module exit");
7653 }
7654 let registration_released = match wait_for_registration_release(
7655 registry,
7656 &spec.module_id,
7657 REGISTRY_RELEASE_TIMEOUT,
7658 )
7659 .await
7660 {
7661 Ok(()) => true,
7662 Err(err) => {
7663 warn!(module_id = %spec.module_id, error = %err, "registration still active after clean exit");
7664 false
7665 }
7666 };
7667 NextAction::Stop {
7668 registration_released,
7669 }
7670 }
7671 ExitKind::Clean | ExitKind::Crash => {
7672 if unrequested_clean_exit_of_protocol_none {
7673 warn!(
7674 module_id = %spec.module_id,
7675 exit_code = ?exit_report.code,
7676 exit_signal = ?exit_report.signal,
7677 "protocol-none module exited cleanly without a stop request; handling it as a crash"
7678 );
7679 } else {
7680 warn!(
7681 module_id = %spec.module_id,
7682 exit_code = ?exit_report.code,
7683 exit_signal = ?exit_report.signal,
7684 "supervised module exited abnormally (crash)"
7685 );
7686 }
7687 let mut restart_schedule = None;
7688 let mut disposition = TerminalDisposition::Disabled;
7689 let mut final_state = ModuleState::Disabled;
7690 let mut disposition_detail = lock_snapshot(snapshot)
7694 .ok()
7695 .and_then(|mut state| state.spawn_failure.take());
7696 let now = Instant::now();
7697 if let Err(err) = update_snapshot(snapshot, Some(&spec.module_id), |state| {
7698 if state.enabled {
7699 if let Some(schedule) = state.next_crash_restart(&policy, now) {
7700 state.respawn_pending = true;
7706 final_state = ModuleState::Restarting;
7707 restart_schedule = Some(schedule);
7708 disposition = TerminalDisposition::Restarting;
7709 } else {
7710 final_state = ModuleState::Failed;
7711 disposition = TerminalDisposition::Failed;
7712 let budget = policy.budget_exhausted_detail();
7713 disposition_detail =
7714 Some(disposition_detail.take().map_or_else(
7715 || budget.clone(),
7716 |cause| format!("{cause}; {budget}"),
7717 ));
7718 }
7719 } else {
7720 disposition = TerminalDisposition::Disabled;
7721 }
7722 }) {
7723 error!(module_id = %spec.module_id, error = %err, "failed to record crashed module exit");
7724 return NextAction::Stop {
7725 registration_released: false,
7726 };
7727 }
7728 if disposition == TerminalDisposition::Failed {
7729 error!(
7734 module_id = %spec.module_id,
7735 max_restarts = policy.max_restarts,
7736 window_secs = policy.window.as_secs(),
7737 "module stopped: {}",
7738 policy.budget_exhausted_detail()
7739 );
7740 }
7741 if let Err(err) = publish_after_terminal_record(
7742 &spec.module_id,
7743 || {
7744 record_terminal_with_detail(
7745 &spec.module_id,
7746 terminal_ring,
7747 spawn_events,
7748 &exit_report,
7749 disposition,
7750 disposition_detail,
7751 )
7752 },
7753 || {
7754 update_snapshot(snapshot, Some(&spec.module_id), |state| {
7755 state.state = final_state;
7756 clear_current_process_facts(state);
7757 state.last_exit = Some(exit_report.clone());
7758 })
7759 },
7760 ) {
7761 error!(module_id = %spec.module_id, error = %err, "failed to publish crashed module exit");
7762 }
7763
7764 if let Some(schedule) = restart_schedule {
7765 NextAction::Restart {
7766 schedule: Some(schedule),
7767 }
7768 } else {
7769 let registration_released = match wait_for_registration_release(
7770 registry,
7771 &spec.module_id,
7772 REGISTRY_RELEASE_TIMEOUT,
7773 )
7774 .await
7775 {
7776 Ok(()) => true,
7777 Err(err) => {
7778 warn!(module_id = %spec.module_id, error = %err, "registration still active after failed module");
7779 false
7780 }
7781 };
7782 NextAction::Stop {
7783 registration_released,
7784 }
7785 }
7786 }
7787 ExitKind::DeliberateSeverance => {
7788 warn!(
7789 module_id = %spec.module_id,
7790 exit_code = ?exit_report.code,
7791 exit_signal = ?exit_report.signal,
7792 "supervised module exited after deliberate connection severance"
7793 );
7794 let should_restart = match lock_snapshot(snapshot) {
7795 Ok(state) => state.enabled,
7796 Err(err) => {
7797 error!(module_id = %spec.module_id, error = %err, "failed to read deliberately severed module state");
7798 return NextAction::Stop {
7799 registration_released: false,
7800 };
7801 }
7802 };
7803 let final_state = if should_restart {
7804 ModuleState::Restarting
7805 } else {
7806 ModuleState::Disabled
7807 };
7808 if let Err(err) = publish_after_terminal_record(
7809 &spec.module_id,
7810 || {
7811 record_terminal(
7812 &spec.module_id,
7813 terminal_ring,
7814 spawn_events,
7815 &exit_report,
7816 terminal_disposition(final_state),
7817 )
7818 },
7819 || {
7820 update_snapshot(snapshot, Some(&spec.module_id), |state| {
7821 state.state = final_state;
7822 clear_current_process_facts(state);
7823 state.last_exit = Some(exit_report.clone());
7824 state.lifetime_restarts += 1;
7825 })
7826 },
7827 ) {
7828 error!(module_id = %spec.module_id, error = %err, "failed to publish deliberately severed module exit");
7829 return NextAction::Stop {
7830 registration_released: false,
7831 };
7832 }
7833
7834 if should_restart {
7835 NextAction::Restart { schedule: None }
7836 } else {
7837 let registration_released = match wait_for_registration_release(
7838 registry,
7839 &spec.module_id,
7840 REGISTRY_RELEASE_TIMEOUT,
7841 )
7842 .await
7843 {
7844 Ok(()) => true,
7845 Err(err) => {
7846 warn!(module_id = %spec.module_id, error = %err, "registration still active after deliberately severed module exit");
7847 false
7848 }
7849 };
7850 NextAction::Stop {
7851 registration_released,
7852 }
7853 }
7854 }
7855 }
7856}
7857
7858async fn on_child_exit_during_daemon_shutdown(
7859 spec: &ModuleSpec,
7860 registry: &Registry,
7861 snapshot: &SharedSnapshot,
7862 terminal_ring: &Arc<Mutex<TerminalRing>>,
7863 spawn_events: &SpawnEventFeed,
7864 exit_report: ExitReport,
7865) -> NextAction {
7866 info!(
7867 module_id = %spec.module_id,
7868 exit_code = ?exit_report.code,
7869 exit_signal = ?exit_report.signal,
7870 exit_kind = ?exit_report.kind,
7871 "supervised module exited during daemon shutdown; not restarting it"
7872 );
7873 if let Err(err) = publish_after_terminal_record(
7874 &spec.module_id,
7875 || {
7876 record_terminal(
7877 &spec.module_id,
7878 terminal_ring,
7879 spawn_events,
7880 &exit_report,
7881 TerminalDisposition::DaemonShutdown,
7882 )
7883 },
7884 || {
7885 update_snapshot(snapshot, Some(&spec.module_id), |state| {
7886 state.state = ModuleState::Stopped;
7887 clear_current_process_facts(state);
7888 state.last_exit = Some(exit_report.clone());
7889 })
7890 },
7891 ) {
7892 error!(module_id = %spec.module_id, error = %err, "failed to publish module exit during daemon shutdown");
7893 }
7894 let registration_released =
7895 wait_for_registration_release(registry, &spec.module_id, REGISTRY_RELEASE_TIMEOUT)
7896 .await
7897 .is_ok();
7898 NextAction::Stop {
7899 registration_released,
7900 }
7901}
7902
7903fn publish_after_terminal_record<T>(
7908 module_id: &str,
7909 record: impl FnOnce(),
7910 publish: impl FnOnce() -> T,
7911) -> T {
7912 let recorded = std::panic::catch_unwind(std::panic::AssertUnwindSafe(record));
7913 let published = publish();
7914 if let Err(panic) = recorded {
7915 error!(
7916 module_id,
7917 "terminal recording panicked; published final state before resuming unwind"
7918 );
7919 std::panic::resume_unwind(panic);
7920 }
7921 published
7922}
7923
7924fn fail_after_wait_error(
7925 module_id: &str,
7926 snapshot: &SharedSnapshot,
7927 terminal_ring: &Arc<Mutex<TerminalRing>>,
7928 spawn_events: &SpawnEventFeed,
7929 detail: String,
7930) {
7931 let report = wait_error_exit_report();
7932 publish_after_terminal_record(
7933 module_id,
7934 || {
7935 record_terminal_with_detail(
7936 module_id,
7937 terminal_ring,
7938 spawn_events,
7939 &report,
7940 TerminalDisposition::Failed,
7941 Some(detail),
7942 )
7943 },
7944 || fail_snapshot(snapshot, Some(module_id), Some(report.clone())),
7945 );
7946}
7947
7948#[cfg(test)]
7949fn record_wait_error_terminal(
7950 module_id: &str,
7951 terminal_ring: &Arc<Mutex<TerminalRing>>,
7952 spawn_events: &SpawnEventFeed,
7953) {
7954 record_terminal(
7955 module_id,
7956 terminal_ring,
7957 spawn_events,
7958 &wait_error_exit_report(),
7959 TerminalDisposition::Failed,
7960 );
7961}
7962
7963fn record_terminal(
7964 module_id: &str,
7965 terminal_ring: &Arc<Mutex<TerminalRing>>,
7966 spawn_events: &SpawnEventFeed,
7967 exit_report: &ExitReport,
7968 disposition: TerminalDisposition,
7969) {
7970 record_terminal_with_detail(
7971 module_id,
7972 terminal_ring,
7973 spawn_events,
7974 exit_report,
7975 disposition,
7976 None,
7977 );
7978}
7979
7980fn durable_terminal_history_of(
7984 terminal_ring: &Mutex<TerminalRing>,
7985 module_id: &str,
7986) -> subc_control::TerminalHistory {
7987 let read = terminal_ring
7988 .lock()
7989 .unwrap_or_else(|p| p.into_inner())
7990 .capture_durable_history();
7991 read.read(module_id)
7992}
7993
7994fn record_terminal_with_detail(
7995 module_id: &str,
7996 terminal_ring: &Arc<Mutex<TerminalRing>>,
7997 spawn_events: &SpawnEventFeed,
7998 exit_report: &ExitReport,
7999 disposition: TerminalDisposition,
8000 disposition_detail: Option<String>,
8001) {
8002 #[cfg(test)]
8003 terminal_history_tests::before_terminal_record(spawn_events);
8004 spawn_events.emit_exited(module_id, exit_report.code, exit_report.signal);
8005 let record = TerminalRecord {
8006 exit_code: exit_report.code,
8007 exit_signal: exit_report.signal,
8008 at_ms: exit_report.at_ms,
8009 disposition,
8010 exit_kind: exit_report.kind.into(),
8011 disposition_detail,
8012 };
8013 terminal_ring
8014 .lock()
8015 .unwrap_or_else(|poisoned| poisoned.into_inner())
8016 .record_exit(module_id, record);
8017}
8018
8019fn untrack_if_registration_released(
8020 process_liveness: &SupervisorProcessLiveness,
8021 registry: &Registry,
8022 module_id: &str,
8023 snapshot: &SharedSnapshot,
8024) {
8025 match registry.get_module(module_id) {
8026 Ok(None) => process_liveness.untrack_if_current(module_id, snapshot),
8027 Ok(Some(_)) => {}
8028 Err(err) => {
8029 warn!(module_id, error = %err, "could not determine whether supervisor liveness can be untracked");
8030 }
8031 }
8032}
8033
8034#[cfg(test)]
8048fn apply_wire_spawn_args(
8049 command: &mut Command,
8050 spec: &ModuleSpec,
8051 connection_file_path: Option<&std::path::Path>,
8052 handle: Option<&SupervisorHandle>,
8053) -> Result<Option<NonceHandoff>, SuperviseError> {
8054 apply_wire_spawn_args_for_role(
8055 command,
8056 spec,
8057 connection_file_path,
8058 handle,
8059 SpawnRole::Plain,
8060 )
8061}
8062
8063#[cfg(unix)]
8068type NonceHandoff = subc_os::LaunchNonceHandoff;
8069#[cfg(windows)]
8070type NonceHandoff = subc_os::LaunchNoncePipeHandoff;
8071#[cfg(not(any(unix, windows)))]
8072type NonceHandoff = std::convert::Infallible;
8073
8074fn apply_wire_spawn_args_for_role(
8095 command: &mut Command,
8096 spec: &ModuleSpec,
8097 connection_file_path: Option<&std::path::Path>,
8098 handle: Option<&SupervisorHandle>,
8099 role: SpawnRole,
8100) -> Result<Option<NonceHandoff>, SuperviseError> {
8101 command.env(SUBC_MODULE_ID_ENV, &spec.module_id);
8102 command.env_remove(subc_os::LAUNCH_NONCE_FD_ENV);
8107 #[cfg(windows)]
8108 {
8109 command.env_remove(subc_os::LAUNCH_NONCE_PIPE_ENV);
8110 command.env_remove(subc_os::LAUNCH_NONCE_PIPE_FALLBACK_ENV);
8111 }
8112 command.env_remove(SUBC_LAUNCH_NONCE_ENV);
8114 if spec.protocol == ModuleProtocol::None {
8115 return Ok(None);
8116 }
8117 if let Some(connection_file_path) = connection_file_path {
8118 command.arg(SUBC_ARG).arg(connection_file_path);
8119 }
8120
8121 let nonce = generate_launch_nonce()?;
8125 if let Some(handle) = handle {
8126 match role {
8127 SpawnRole::Plain => {
8128 handle.set_spawn_nonce(&spec.module_id, nonce.clone());
8129 if spec.reserved {
8130 handle.set_reserved_nonce(&spec.module_id, nonce.clone());
8131 }
8132 }
8133 SpawnRole::SwapCandidate => handle.open_swap(&spec.module_id, nonce.clone()),
8134 }
8135 }
8136 #[cfg(unix)]
8137 let handoff = {
8138 let handoff =
8139 subc_os::LaunchNonceHandoff::new(&nonce).map_err(|source| SuperviseError::Spawn {
8140 program: spec.program.clone(),
8141 source,
8142 cgroup_path: None,
8143 })?;
8144 command.env(subc_os::LAUNCH_NONCE_FD_ENV, handoff.fd_env_value());
8145 Some(handoff)
8146 };
8147 #[cfg(windows)]
8148 let handoff = prepare_windows_nonce_pipe(command, spec, &nonce);
8149 #[cfg(not(any(unix, windows)))]
8150 let handoff = None;
8151 #[cfg(not(unix))]
8155 command.env(SUBC_LAUNCH_NONCE_ENV, nonce);
8156 Ok(handoff)
8157}
8158
8159fn apply_child_env(command: &mut Command, spec: &ModuleSpec) {
8160 command.env_remove(CK_LOG_ENV);
8161 command.env_remove(SUBC_SPAWN_ROLE_ENV);
8168 for (key, value) in &spec.env {
8169 if matches!(
8173 key.as_str(),
8174 CAPTURE_MAX_FILE_MB_ENV | CAPTURE_KEEP_ENV | CAPTURE_MAX_AGE_DAYS_ENV
8175 ) || key == SUBC_SPAWN_ROLE_ENV
8176 {
8177 continue;
8178 }
8179 command.env(key, value);
8180 }
8181}
8182
8183#[cfg(windows)]
8184fn prepare_windows_nonce_pipe(
8185 command: &mut Command,
8186 spec: &ModuleSpec,
8187 nonce: &str,
8188) -> Option<NonceHandoff> {
8189 match subc_os::LaunchNoncePipeHandoff::new(nonce) {
8190 Ok(handoff) => {
8191 command.env(subc_os::LAUNCH_NONCE_PIPE_ENV, handoff.name());
8192 command.env(subc_os::LAUNCH_NONCE_PIPE_FALLBACK_ENV, "env");
8197 Some(handoff)
8198 }
8199 Err(error) => {
8200 warn!(module_id = %spec.module_id, %error, "could not prepare launch nonce pipe; retaining the environment handoff");
8201 None
8202 }
8203 }
8204}
8205
8206#[cfg(windows)]
8207fn start_windows_nonce_pipe(
8208 handoff: Option<NonceHandoff>,
8209 pid: u32,
8210 spec: &ModuleSpec,
8211) -> Option<subc_os::LaunchNoncePipeDelivery> {
8212 handoff.and_then(|handoff| match handoff.serve(pid, REGISTRY_RELEASE_TIMEOUT) {
8216 Ok(delivery) => Some(delivery),
8217 Err(error) => {
8218 warn!(module_id = %spec.module_id, %error, "could not serve launch nonce pipe; retaining the environment handoff");
8219 None
8220 }
8221 })
8222}
8223
8224fn observed_launch_nonce_source(snapshot: &SupervisorSnapshot) -> Option<String> {
8225 #[cfg(windows)]
8226 if snapshot.process_alive && snapshot.spawned_protocol == Some(ModuleProtocol::Subc) {
8227 return Some(
8228 if snapshot
8229 .pipe_consumed
8230 .as_ref()
8231 .is_some_and(|consumed| consumed.get().is_some())
8232 {
8233 "pipe"
8234 } else {
8235 "env"
8236 }
8237 .to_owned(),
8238 );
8239 }
8240 let _ = snapshot;
8241 None
8242}
8243
8244#[derive(Debug, Clone, Copy, PartialEq, Eq)]
8247enum SpawnRole {
8248 Plain,
8249 SwapCandidate,
8250}
8251
8252fn apply_spawn_role(command: &mut Command, role: SpawnRole) {
8255 if role == SpawnRole::SwapCandidate {
8256 command.env(SUBC_SPAWN_ROLE_ENV, SPAWN_ROLE_SWAP_CANDIDATE);
8257 }
8258}
8259
8260fn previous_instance_error(
8261 spec: &ModuleSpec,
8262 pid: u32,
8263 reason: crate::live_children::RetainedReason,
8264) -> SuperviseError {
8265 SuperviseError::Spawn {
8266 program: spec.program.clone(),
8267 source: io::Error::new(io::ErrorKind::WouldBlock, reason.waiting_message(pid)),
8268 cgroup_path: None,
8269 }
8270}
8271
8272fn spawn_child(
8273 spec: &ModuleSpec,
8274 connection_file_path: Option<&std::path::Path>,
8275 handle: Option<&SupervisorHandle>,
8276 ring: &Arc<Mutex<StderrRing>>,
8277 capture_logs_dir: Option<&std::path::Path>,
8278 roster: &ChildRoster,
8279 #[cfg(target_os = "linux")] cgroup_placement: Option<&subc_cgroup::Placement>,
8280) -> Result<SupervisedChild, SuperviseError> {
8281 spawn_child_in_slot(
8282 spec,
8283 connection_file_path,
8284 handle,
8285 ring,
8286 capture_logs_dir,
8287 roster,
8288 #[cfg(target_os = "linux")]
8289 cgroup_placement,
8290 SpawnRole::Plain,
8291 false,
8292 )
8293}
8294
8295#[allow(clippy::too_many_arguments)]
8308fn spawn_child_in_slot(
8309 spec: &ModuleSpec,
8310 connection_file_path: Option<&std::path::Path>,
8311 handle: Option<&SupervisorHandle>,
8312 ring: &Arc<Mutex<StderrRing>>,
8313 capture_logs_dir: Option<&std::path::Path>,
8314 roster: &ChildRoster,
8315 #[cfg(target_os = "linux")] cgroup_placement: Option<&subc_cgroup::Placement>,
8316 role: SpawnRole,
8317 alternate_slot: bool,
8318) -> Result<SupervisedChild, SuperviseError> {
8319 if let Some((pid, reason)) = roster.previous_instance(&spec.module_id) {
8320 return Err(previous_instance_error(spec, pid, reason));
8321 }
8322 if roster.is_closed() {
8323 return Err(SuperviseError::Spawn {
8324 program: spec.program.clone(),
8325 source: io::Error::other("the daemon is shutting down; not starting a new process"),
8326 cgroup_path: None,
8327 });
8328 }
8329 #[cfg(target_os = "linux")]
8330 let cgroup_name = {
8331 if cgroup_placement.is_none() {
8336 swap::cgroup_name(&spec.module_id, alternate_slot)
8337 } else {
8338 let nonce = generate_launch_nonce()?;
8339 let suffix = u128::from_str_radix(&nonce[..32], 16).expect("hex launch nonce");
8340 let mut end = spec.module_id.len().min(64);
8343 while !spec.module_id.is_char_boundary(end) {
8344 end -= 1;
8345 }
8346 format!(
8347 "{}_{suffix}",
8348 swap::cgroup_name(&spec.module_id[..end], alternate_slot)
8349 )
8350 }
8351 };
8352 #[cfg(not(target_os = "linux"))]
8353 let _ = alternate_slot;
8354 #[cfg(target_os = "macos")]
8355 let (mut command, privacy_exec, exec_ack) = privacy_command(spec, roster)?;
8356 #[cfg(windows)]
8357 let image_capture = windows_spawn_capture(spec).map_err(|source| SuperviseError::Spawn {
8358 program: spec.program.clone(),
8359 source,
8360 cgroup_path: None,
8361 })?;
8362 #[cfg(windows)]
8363 let mut command = Command::new(image_capture.path());
8364 #[cfg(not(any(target_os = "macos", windows)))]
8365 let mut command = Command::new(&spec.program);
8366 command.args(&spec.args);
8367 apply_child_env(&mut command, spec);
8397 apply_spawn_role(&mut command, role);
8398 let nonce_handoff =
8399 apply_wire_spawn_args_for_role(&mut command, spec, connection_file_path, handle, role)?;
8400
8401 #[cfg(target_os = "linux")]
8402 let cgroup_path = cgroup_placement
8403 .map(|placement| placement.module_path(&cgroup_name))
8404 .transpose()
8405 .map_err(|source| SuperviseError::Cgroup {
8406 module_id: spec.module_id.clone(),
8407 source,
8408 })?;
8409 #[cfg(not(target_os = "linux"))]
8410 let cgroup_path: Option<PathBuf> = None;
8411 #[cfg(target_os = "linux")]
8412 if let Some(path) = &cgroup_path {
8413 if let Err(error) = apply_cgroup_placement(&mut command, spec, path) {
8414 if let Some(placement) = cgroup_placement {
8415 remove_module_cgroup(placement, &cgroup_name);
8416 }
8417 return Err(error);
8418 }
8419 }
8420
8421 let output_sink = if let Some(logs_dir) = capture_logs_dir {
8422 let path = logs_dir.join(format!("{}.stderr.log", spec.module_id));
8423 match ChildOutputSink::open(&path, capture_retention(spec)) {
8424 Ok(sink) => sink,
8425 Err(error) => {
8426 warn!(
8427 module_id = %spec.module_id,
8428 path = %path.display(),
8429 error = %error,
8430 "could not open child output capture file; forwarding to stderr"
8431 );
8432 ChildOutputSink::Stderr
8433 }
8434 }
8435 } else {
8436 ChildOutputSink::Stderr
8437 };
8438
8439 command.stdout(Stdio::piped());
8440 command.stderr(Stdio::piped());
8441 command.kill_on_drop(true);
8442 #[cfg(unix)]
8459 command.process_group(0);
8460 command.stdin(Stdio::null());
8461 #[cfg(unix)]
8465 if let Some(handoff) = nonce_handoff {
8466 handoff.install_last(command.as_std_mut());
8467 }
8468 #[cfg(not(any(unix, windows)))]
8469 let _ = nonce_handoff;
8470
8471 #[cfg(windows)]
8476 {
8477 subc_jobobject::ensure_root_job().map_err(|source| SuperviseError::Spawn {
8478 program: spec.program.clone(),
8479 source,
8480 cgroup_path: None,
8481 })?;
8482 subc_jobobject::suspend_on_create_async(&mut command);
8483 }
8484 let spawned_at = Instant::now();
8488 let mut child = match command.spawn() {
8489 Ok(child) => child,
8490 Err(source) => {
8491 #[cfg(target_os = "linux")]
8492 if let Some(placement) = cgroup_placement {
8493 remove_module_cgroup(placement, &cgroup_name);
8494 }
8495 return Err(SuperviseError::Spawn {
8496 program: spec.program.clone(),
8497 source,
8498 cgroup_path,
8499 });
8500 }
8501 };
8502 #[cfg(test)]
8503 spawn_record_tests::after_create(&child, spec);
8504 #[cfg(target_os = "macos")]
8509 drop(exec_ack);
8510
8511 #[cfg(windows)]
8516 let windows_image = match image_capture.bind_tokio(&child) {
8517 Ok(image) => Arc::new(image),
8518 Err(source) => {
8519 kill_and_reap_rejected_child(&mut child);
8520 return Err(SuperviseError::Spawn {
8521 program: spec.program.clone(),
8522 source,
8523 cgroup_path: None,
8524 });
8525 }
8526 };
8527 #[cfg(windows)]
8530 let job = contain_spawned_child(&mut child, spec)?;
8531 let spawned_at_ms = unix_ms_now();
8532 #[cfg(not(windows))]
8533 let spawned_from = spec.program.clone();
8534 #[cfg(windows)]
8535 let spawned_from = windows_image.path().to_path_buf();
8536 #[cfg(not(windows))]
8537 let spawned_file_identity = spawned_file_identity(&spawned_from);
8538 #[cfg(windows)]
8539 let spawned_file_identity = Some(SpawnedFileIdentity {
8540 device: 0,
8541 inode: 0,
8542 windows_image: Some(Arc::clone(&windows_image)),
8543 });
8544 let pid = child.id().ok_or_else(|| SuperviseError::Spawn {
8545 program: spec.program.clone(),
8546 source: io::Error::other("spawned child exposed no live pid"),
8547 cgroup_path: cgroup_path.clone(),
8548 })?;
8549 #[cfg(windows)]
8550 let nonce_delivery = start_windows_nonce_pipe(nonce_handoff, pid, spec);
8551 #[cfg(not(windows))]
8552 let process_start_time = crate::provenance::process_start_time(pid);
8553 #[cfg(windows)]
8554 let process_start_time = Some(windows_image.start_time());
8555 let process_identity = process_start_time.map(|start_time| ProcessIdentity { pid, start_time });
8556 #[cfg(all(test, target_os = "macos"))]
8557 privacy_exec_boundary_tests::before_image_sample(spec, pid);
8558 #[cfg(not(windows))]
8563 let recorded_image = observe_spawned_image(pid);
8564 #[cfg(target_os = "macos")]
8568 let recorded_image = if privacy_exec.is_some() {
8569 None
8570 } else {
8571 recorded_image
8572 };
8573 #[cfg(target_os = "linux")]
8574 let recorded_cgroup_name = cgroup_path.as_ref().map(|_| cgroup_name.clone());
8575 #[cfg(not(target_os = "linux"))]
8576 let recorded_cgroup_name = None;
8577 let roster_guard = roster.admit(
8578 spec.module_id.clone(),
8579 pid,
8580 spec.protocol,
8581 process_start_time,
8582 crate::child_roster::RecordedIdentity {
8583 #[cfg(not(windows))]
8584 start_time: recorded_image.map(|image| image.start_time),
8585 #[cfg(windows)]
8586 start_time: process_start_time,
8587 #[cfg(not(windows))]
8588 executable: recorded_image
8589 .and_then(|image| image.executable)
8590 .map(crate::live_children::ExecutableIdentity::from),
8591 #[cfg(windows)]
8592 executable: Some(crate::live_children::ExecutableIdentity::from(
8593 windows_image.identity(),
8594 )),
8595 cgroup_name: recorded_cgroup_name,
8596 #[cfg(target_os = "linux")]
8597 cgroup_placement: cgroup_placement.cloned(),
8598 },
8599 );
8600 let roster_guard = match roster_guard {
8601 Ok(guard) => guard,
8602 Err(source) => {
8603 #[cfg(windows)]
8604 if let Some(job) = &job {
8605 let _ = job.terminate();
8606 }
8607 #[cfg(target_os = "linux")]
8608 kill_module_cgroup(cgroup_placement, &cgroup_name);
8609 kill_and_reap_rejected_child(&mut child);
8610 #[cfg(target_os = "linux")]
8611 if let Some(placement) = cgroup_placement {
8612 remove_module_cgroup(placement, &cgroup_name);
8613 }
8614 return Err(SuperviseError::Spawn {
8615 program: spec.program.clone(),
8616 source: io::Error::other(format!(
8617 "could not durably record module child: {source}"
8618 )),
8619 cgroup_path,
8620 });
8621 }
8622 };
8623 if roster.is_closed() {
8632 #[cfg(target_os = "linux")]
8634 kill_module_cgroup(cgroup_placement, &cgroup_name);
8635 if let Err(error) = child.start_kill() {
8636 debug!(module_id = %spec.module_id, pid, %error, "kill of a process spawned during daemon shutdown failed; it may already have exited");
8637 }
8638 #[cfg(target_os = "linux")]
8639 if let Some(placement) = cgroup_placement {
8640 while matches!(child.try_wait(), Ok(None)) {
8644 std::thread::yield_now();
8645 }
8646 if matches!(
8647 subc_cgroup::kill_module(Some(placement), &cgroup_name),
8648 subc_cgroup::KillOutcome::Killed
8649 ) {
8650 if let Ok(path) = placement.module_path(&cgroup_name) {
8651 while std::fs::read_to_string(path.join("cgroup.events"))
8652 .ok()
8653 .is_some_and(|events| events.lines().any(|line| line == "populated 1"))
8654 {
8655 std::thread::yield_now();
8656 }
8657 }
8658 }
8659 remove_module_cgroup(placement, &cgroup_name);
8660 }
8661 drop(roster_guard);
8662 return Err(SuperviseError::Spawn {
8663 program: spec.program.clone(),
8664 source: io::Error::other(
8665 "the daemon began shutting down while this process was starting; ended it",
8666 ),
8667 cgroup_path,
8668 });
8669 }
8670
8671 #[cfg(windows)]
8672 if let Err(error) = resume_suspended_child(pid, spec) {
8673 kill_and_reap_rejected_child(&mut child);
8674 return Err(error);
8675 }
8676
8677 let stdout_pump = match child.stdout.take() {
8678 Some(stdout) => Some(tokio::spawn(pump_stdout_to(stdout, output_sink.clone()))),
8679 None => {
8680 warn!(
8681 module_id = %spec.module_id,
8682 "spawned child exposed no stdout pipe; file capture will be incomplete"
8683 );
8684 None
8685 }
8686 };
8687 let stderr_pump = match child.stderr.take() {
8688 Some(stderr) => {
8689 let generation = ring
8690 .lock()
8691 .unwrap_or_else(|poisoned| poisoned.into_inner())
8692 .begin_process();
8693 Some(StderrPump {
8694 task: tokio::spawn(pump_stderr_to(
8695 stderr,
8696 Arc::clone(ring),
8697 generation,
8698 output_sink,
8699 )),
8700 generation,
8701 })
8702 }
8703 None => {
8704 ring.lock()
8708 .unwrap_or_else(|poisoned| poisoned.into_inner())
8709 .mark_not_captured("stderr pipe was not available on spawn");
8710 warn!(
8711 module_id = %spec.module_id,
8712 "spawned child exposed no stderr pipe; tail will be unavailable"
8713 );
8714 None
8715 }
8716 };
8717
8718 let exec = Arc::new(crate::activity::ExecOutcome::new());
8719 #[cfg(target_os = "macos")]
8720 let pending_exec = privacy_exec.is_some();
8721 #[cfg(not(target_os = "macos"))]
8722 let pending_exec = false;
8723 if !pending_exec {
8724 exec.finish(crate::activity::ExecState::CommittedExec);
8725 }
8726 Ok(SupervisedChild {
8727 child,
8728 exec: crate::activity::ExecOwner(exec),
8729 #[cfg(windows)]
8730 nonce_delivery,
8731 protocol: spec.protocol,
8732 #[cfg(target_os = "linux")]
8733 module_id: cgroup_name,
8734 #[cfg(target_os = "linux")]
8735 cgroup_placement: cgroup_placement.cloned(),
8736 #[cfg(windows)]
8737 job,
8738 stdout_pump,
8739 stderr_pump,
8740 stderr_ring: Arc::clone(ring),
8741 spawned_at,
8742 spawned_at_ms,
8743 spawned_from,
8744 launch_spec_digest: LaunchSpecDigest::from_spec(spec),
8745 spawned_file_identity,
8746 process_start_time,
8747 process_identity,
8748 pid,
8749 roster_guard: Some(roster_guard),
8750 #[cfg(target_os = "macos")]
8751 privacy_exec,
8752 #[cfg(target_os = "macos")]
8753 report_ready: Arc::new(OnceLock::new()),
8754 spawn_failure: None,
8755 })
8756}
8757
8758#[cfg(test)]
8759mod spawn_record_tests {
8760 use super::*;
8761
8762 thread_local! {
8763 static LAST_PID: std::cell::Cell<Option<u32>> = const { std::cell::Cell::new(None) };
8764 }
8765
8766 pub(super) fn after_create(child: &Child, _spec: &ModuleSpec) {
8767 LAST_PID.set(child.id());
8768 #[cfg(windows)]
8769 if let Some((_, address)) = _spec
8770 .env
8771 .iter()
8772 .find(|(key, _)| key == "SUBC_TEST_AFTER_CREATE")
8773 {
8774 use std::io::{Read, Write};
8775 let mut stream = std::net::TcpStream::connect(address).unwrap();
8776 stream
8777 .write_all(&child.id().unwrap().to_ne_bytes())
8778 .unwrap();
8779 let mut release = [0];
8780 stream.read_exact(&mut release).unwrap();
8781 }
8782 }
8783
8784 #[cfg(windows)]
8785 pub(super) fn assignment_failpoint(spec: &ModuleSpec) -> io::Result<()> {
8786 if spec
8787 .env
8788 .iter()
8789 .any(|(key, _)| key == "SUBC_TEST_ASSIGNMENT_FAILURE")
8790 {
8791 Err(io::Error::other("injected module job assignment failure"))
8792 } else {
8793 Ok(())
8794 }
8795 }
8796
8797 fn spec() -> ModuleSpec {
8798 ModuleSpec {
8799 module_id: "spawn-record-worker".into(),
8800 #[cfg(unix)]
8801 program: "/bin/sleep".into(),
8802 #[cfg(windows)]
8803 program: std::env::current_exe().unwrap(),
8804 #[cfg(unix)]
8805 args: vec!["60".into()],
8806 #[cfg(windows)]
8807 args: vec![
8808 "--exact".into(),
8809 "supervise::spawn_record_tests::windows_store_touch_fixture".into(),
8810 "--nocapture".into(),
8811 ],
8812 env: vec![],
8813 reserved: false,
8814 reserved_prefixes: vec![],
8815 protocol: ModuleProtocol::None,
8816 overlap: Default::default(),
8817 }
8818 }
8819
8820 #[tokio::test]
8821 async fn record_sync_failure_fails_spawn_and_reaps_child() {
8822 let home = cortexkit_test_support::ScratchDir::new("record-sync-failure");
8823 let supervisor =
8824 Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default())
8825 .with_live_children_record(home.join("live-children.json"));
8826 crate::live_children::fail_next_file_sync();
8827 let result = supervisor.spawn(spec());
8828 let failed = result.is_err();
8829 if let Ok(module) = result {
8830 module.stop().await.unwrap();
8831 }
8832 assert!(failed, "record failure must fail spawn");
8833 let pid = LAST_PID.get().expect("the real OS child was created");
8834 assert!(
8835 subc_os::Process::open(pid).unwrap().is_none(),
8836 "child {pid} survived rejected admission or was not reaped"
8837 );
8838 }
8839
8840 #[cfg(target_os = "macos")]
8841 #[test]
8842 #[ignore = "manual on-device latency measurement"]
8843 fn durable_record_write_latency_50() {
8844 let home = cortexkit_test_support::ScratchDir::new("record-latency");
8845 let path = home.join("live-children.json");
8846 let children = vec![crate::live_children::LiveChild {
8847 module_id: "latency-worker".into(),
8848 pid: std::process::id(),
8849 protocol: ModuleProtocol::None,
8850 start_time: Some(1),
8851 executable: None,
8852 cgroup_name: None,
8853 }];
8854 let mut samples = Vec::new();
8855 for _ in 0..50 {
8856 let start = std::time::Instant::now();
8857 crate::live_children::write_record(&path, &children).unwrap();
8858 samples.push(start.elapsed());
8859 }
8860 samples.sort();
8861 eprintln!(
8862 "durable_record_write_latency_50: n=50 median={:?} p95={:?}",
8863 (samples[24] + samples[25]) / 2,
8864 samples[47]
8865 );
8866 }
8867
8868 #[cfg(windows)]
8869 #[test]
8870 fn windows_store_touch_fixture() {
8871 if let Some(path) = std::env::var_os("SUBC_TEST_STORE_TOUCH") {
8872 std::fs::write(path, b"module ran").unwrap();
8873 }
8874 }
8875
8876 #[cfg(windows)]
8877 #[tokio::test]
8878 async fn windows_daemon_fixture() {
8879 let Ok(address) = std::env::var("SUBC_TEST_DAEMON_BARRIER") else {
8880 return;
8881 };
8882 let mut spec = spec();
8883 spec.env.push(("SUBC_TEST_AFTER_CREATE".into(), address));
8884 let supervisor =
8885 Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default());
8886 let _module = supervisor.spawn(spec).unwrap();
8887 }
8888
8889 #[cfg(windows)]
8890 #[test]
8891 fn windows_daemon_death_before_module_job_assignment_leaves_no_child() {
8892 use std::io::Read;
8893 let listener = std::net::TcpListener::bind("127.0.0.1:0").unwrap();
8894 let mut command = std::process::Command::new(std::env::current_exe().unwrap());
8897 command
8898 .args([
8899 "--exact",
8900 "supervise::spawn_record_tests::windows_daemon_fixture",
8901 "--nocapture",
8902 ])
8903 .env(
8904 "SUBC_TEST_DAEMON_BARRIER",
8905 listener.local_addr().unwrap().to_string(),
8906 );
8907 let mut daemon = subc_jobobject::spawn_contained(&mut command).unwrap();
8908 listener.set_nonblocking(true).unwrap();
8909 let deadline = std::time::Instant::now() + Duration::from_secs(30);
8910 let mut stream = loop {
8911 if let Ok((stream, _)) = listener.accept() {
8912 break stream;
8913 }
8914 assert!(
8915 daemon.child.try_wait().unwrap().is_none(),
8916 "daemon failed before rendezvous"
8917 );
8918 assert!(
8919 std::time::Instant::now() < deadline,
8920 "daemon did not reach CreateProcess failpoint"
8921 );
8922 std::thread::yield_now();
8923 };
8924 stream
8925 .set_read_timeout(Some(Duration::from_secs(30)))
8926 .unwrap();
8927 let mut pid = [0; 4];
8928 stream.read_exact(&mut pid).unwrap();
8929 let pid = u32::from_ne_bytes(pid);
8930 assert!(subc_jobobject::process_exists(pid));
8931 daemon.child.kill().unwrap();
8934 daemon.child.wait().unwrap();
8935 let exited = subc_jobobject::wait_for_process_exit(pid, Duration::from_secs(10));
8936 if !exited {
8937 daemon.job.terminate().unwrap();
8938 }
8939 assert!(exited, "suspended child {pid} outlived hard-killed daemon");
8940 }
8941
8942 #[cfg(windows)]
8943 #[tokio::test]
8944 async fn windows_containment_failure_never_resumes() {
8945 let home = cortexkit_test_support::ScratchDir::new("assignment-refusal");
8946 let sentinel = home.join("store-touched");
8947 let mut spec = spec();
8948 spec.env
8949 .push(("SUBC_TEST_ASSIGNMENT_FAILURE".into(), "1".into()));
8950 spec.env.push((
8951 "SUBC_TEST_STORE_TOUCH".into(),
8952 sentinel.display().to_string(),
8953 ));
8954 let supervisor =
8955 Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default());
8956 let result = supervisor.spawn(spec);
8957 if let Ok(module) = result {
8960 let _ = tokio::time::timeout(Duration::from_secs(10), async {
8961 while !sentinel.exists() && module.state().unwrap() == ModuleState::Running {
8962 tokio::task::yield_now().await;
8963 }
8964 })
8965 .await;
8966 let _ = module.stop().await;
8967 panic!(
8968 "assignment failure resumed a module; store touched={}",
8969 sentinel.exists()
8970 );
8971 }
8972 assert!(
8973 !sentinel.exists(),
8974 "refused module executed store-touch sentinel"
8975 );
8976 assert!(!subc_jobobject::process_exists(LAST_PID.get().unwrap()));
8977 }
8978
8979 #[cfg(windows)]
8980 #[tokio::test]
8981 async fn windows_nested_daemon_fixture() {
8982 let Some(sentinel) = std::env::var_os("SUBC_TEST_NESTED_STORE") else {
8983 return;
8984 };
8985 let mut spec = spec();
8986 spec.env.push((
8987 "SUBC_TEST_STORE_TOUCH".into(),
8988 sentinel.to_str().unwrap().into(),
8989 ));
8990 let supervisor =
8991 Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default());
8992 let module = supervisor
8993 .spawn(spec)
8994 .expect("per-module job must nest below daemon and host jobs");
8995 tokio::time::timeout(Duration::from_secs(10), async {
8996 while !PathBuf::from(&sentinel).exists() {
8997 tokio::task::yield_now().await;
8998 }
8999 })
9000 .await
9001 .unwrap();
9002 module.stop().await.unwrap();
9003 }
9004
9005 #[cfg(windows)]
9006 #[test]
9007 fn windows_module_job_nests_under_host_job() {
9008 let home = cortexkit_test_support::ScratchDir::new("nested-jobs");
9009 let sentinel = home.join("nested-store-touched");
9010 let mut command = std::process::Command::new(std::env::current_exe().unwrap());
9011 command
9012 .args([
9013 "--exact",
9014 "supervise::spawn_record_tests::windows_nested_daemon_fixture",
9015 "--nocapture",
9016 ])
9017 .env("SUBC_TEST_NESTED_STORE", &sentinel);
9018 let mut daemon = subc_jobobject::spawn_contained(&mut command).unwrap();
9019 assert!(daemon.child.wait().unwrap().success());
9020 assert!(sentinel.exists(), "nested protocol:none module never ran");
9021 }
9022}
9023
9024#[cfg(target_os = "linux")]
9025pub(crate) fn kill_module_cgroup(placement: Option<&subc_cgroup::Placement>, module_id: &str) {
9026 use subc_cgroup::KillOutcome;
9027 match subc_cgroup::kill_module(placement, module_id) {
9028 KillOutcome::Killed => {}
9029 KillOutcome::NotPlaced | KillOutcome::Unsupported => {
9030 debug!(
9031 module_id,
9032 "cgroup tree kill unavailable; using direct-child kill"
9033 );
9034 }
9035 KillOutcome::IoError { path, error } => {
9036 warn!(module_id, path = %path.display(), %error, "cgroup tree kill failed; using direct-child kill");
9037 }
9038 }
9039}
9040
9041#[cfg(windows)]
9049fn contain_spawned_child(
9050 child: &mut Child,
9051 spec: &ModuleSpec,
9052) -> Result<Option<subc_jobobject::JobObject>, SuperviseError> {
9053 let assignment = || {
9054 let job = subc_jobobject::JobObject::new()?;
9055 #[cfg(test)]
9056 spawn_record_tests::assignment_failpoint(spec)?;
9057 job.assign(child)?;
9058 Ok::<_, io::Error>(job)
9059 };
9060 match assignment() {
9061 Ok(job) => Ok(Some(job)),
9062 Err(source) => {
9063 kill_and_reap_rejected_child(child);
9064 Err(SuperviseError::Spawn {
9065 program: spec.program.clone(),
9066 source: io::Error::other(format!(
9067 "Windows module job containment failed: {source}"
9068 )),
9069 cgroup_path: None,
9070 })
9071 }
9072 }
9073}
9074
9075fn kill_and_reap_rejected_child(child: &mut Child) {
9078 #[cfg(unix)]
9079 if let Some(pid) = child
9080 .id()
9081 .and_then(|pid| rustix::process::Pid::from_raw(pid as i32))
9082 {
9083 let _ = rustix::process::kill_process_group(pid, rustix::process::Signal::KILL);
9084 }
9085 let _ = child.start_kill();
9086 while matches!(child.try_wait(), Ok(None)) {
9087 std::thread::yield_now();
9088 }
9089}
9090
9091#[cfg(windows)]
9096fn resume_suspended_child(pid: u32, spec: &ModuleSpec) -> Result<(), SuperviseError> {
9097 if let Err(source) = subc_jobobject::resume_main_thread(pid) {
9098 let _ = std::process::Command::new("taskkill.exe")
9102 .args(["/PID", &pid.to_string(), "/T", "/F"])
9103 .stdin(Stdio::null())
9104 .stdout(Stdio::null())
9105 .stderr(Stdio::null())
9106 .status();
9107 return Err(SuperviseError::Spawn {
9108 program: spec.program.clone(),
9109 source,
9110 cgroup_path: None,
9111 });
9112 }
9113 Ok(())
9114}
9115
9116#[cfg(target_os = "linux")]
9117fn remove_module_cgroup(placement: &subc_cgroup::Placement, module_id: &str) {
9118 match placement.remove_module(module_id) {
9119 Ok(()) => debug!(module_id, "removed module cgroup after process exit"),
9120 Err(error) => warn!(
9121 module_id,
9122 error = %error,
9123 "could not remove module cgroup after process exit; continuing teardown"
9124 ),
9125 }
9126}
9127
9128#[cfg(target_os = "linux")]
9129async fn cleanup_reaped_cgroup(placement: &subc_cgroup::Placement, module_id: &str) {
9130 if matches!(
9134 subc_cgroup::kill_module(Some(placement), module_id),
9135 subc_cgroup::KillOutcome::Killed
9136 ) {
9137 if let Ok(path) = placement.module_path(module_id) {
9138 while std::fs::read_to_string(path.join("cgroup.events"))
9139 .ok()
9140 .is_some_and(|events| events.lines().any(|line| line == "populated 1"))
9141 {
9142 sleep(Duration::from_millis(1)).await;
9143 }
9144 }
9145 }
9146 remove_module_cgroup(placement, module_id);
9147}
9148
9149#[cfg(target_os = "linux")]
9150fn apply_cgroup_placement(
9151 command: &mut Command,
9152 spec: &ModuleSpec,
9153 path: &std::path::Path,
9154) -> Result<(), SuperviseError> {
9155 subc_cgroup::apply(command, path).map_err(|source| SuperviseError::Cgroup {
9156 module_id: spec.module_id.clone(),
9157 source,
9158 })
9159}
9160
9161fn capture_retention(spec: &ModuleSpec) -> Retention {
9162 let defaults = Retention::default();
9163 let value = |name: &str| {
9164 spec.env
9165 .iter()
9166 .rev()
9167 .find_map(|(key, value)| (key == name).then_some(value.as_str()))
9168 };
9169 Retention {
9170 max_file_mb: value(CAPTURE_MAX_FILE_MB_ENV)
9171 .and_then(|value| value.parse().ok())
9172 .unwrap_or(defaults.max_file_mb),
9173 keep: value(CAPTURE_KEEP_ENV)
9174 .and_then(|value| value.parse().ok())
9175 .unwrap_or(defaults.keep),
9176 max_age_days: value(CAPTURE_MAX_AGE_DAYS_ENV)
9177 .and_then(|value| value.parse().ok())
9178 .unwrap_or(defaults.max_age_days),
9179 }
9180}
9181
9182fn generate_launch_nonce() -> Result<String, SuperviseError> {
9185 let mut bytes = [0u8; 32];
9186 getrandom::getrandom(&mut bytes).map_err(|source| SuperviseError::LaunchNonce {
9187 reason: source.to_string(),
9188 })?;
9189 let mut hex = String::with_capacity(64);
9190 for b in bytes {
9191 use std::fmt::Write;
9192 let _ = write!(hex, "{b:02x}");
9193 }
9194 Ok(hex)
9195}
9196
9197fn constant_time_eq(a: &[u8], b: &[u8]) -> bool {
9200 if a.len() != b.len() {
9201 return false;
9202 }
9203 let mut diff = 0u8;
9204 for (x, y) in a.iter().zip(b.iter()) {
9205 diff |= x ^ y;
9206 }
9207 diff == 0
9208}
9209
9210#[cfg(not(windows))]
9214fn observe_spawned_image(pid: u32) -> Option<subc_os::Observation> {
9215 let process = subc_os::Process::open(pid).ok().flatten()?;
9216 #[cfg(target_os = "linux")]
9217 return sample_after_linux_exec(
9218 || {
9219 let stat = std::fs::read_to_string(format!("/proc/{pid}/stat")).ok()?;
9220 let committed = linux_exec_committed(&stat);
9221 #[cfg(test)]
9222 spawn_image_tests::report_exec_state(committed);
9223 committed
9224 },
9225 || process.observe(),
9226 );
9227 #[cfg(not(target_os = "linux"))]
9228 process.observe()
9229}
9230
9231#[cfg(target_os = "linux")]
9235fn linux_exec_committed(stat: &str) -> Option<bool> {
9236 const PF_FORKNOEXEC: u64 = 0x40;
9237 let flags = stat.rsplit_once(')')?.1.split_whitespace().nth(6)?;
9238 Some(flags.parse::<u64>().ok()? & PF_FORKNOEXEC == 0)
9239}
9240
9241#[cfg(target_os = "linux")]
9242fn sample_after_linux_exec(
9243 mut exec_committed: impl FnMut() -> Option<bool>,
9244 observe: impl FnOnce() -> Option<subc_os::Observation>,
9245) -> Option<subc_os::Observation> {
9246 let deadline = std::time::Instant::now() + Duration::from_secs(1);
9247 while !exec_committed()? {
9248 if std::time::Instant::now() >= deadline {
9252 return None;
9253 }
9254 std::thread::sleep(Duration::from_micros(50));
9255 }
9256 observe()
9257}
9258
9259#[cfg(all(test, target_os = "linux"))]
9260mod spawn_image_tests {
9261 use super::*;
9262 use std::cell::{Cell, RefCell};
9263
9264 enum ExecEvent {
9265 Uncommitted,
9266 Image(Option<subc_os::Observation>),
9267 }
9268
9269 thread_local! {
9270 static EXEC_EVENTS: RefCell<Option<std::sync::mpsc::Sender<ExecEvent>>> = const { RefCell::new(None) };
9271 }
9272
9273 pub(super) fn report_exec_state(committed: Option<bool>) {
9274 if committed == Some(false) {
9275 EXEC_EVENTS.with(|events| {
9276 if let Some(events) = events.borrow_mut().take() {
9277 let _ = events.send(ExecEvent::Uncommitted);
9278 }
9279 });
9280 }
9281 }
9282
9283 #[test]
9284 fn linux_spawn_observation_waits_for_a_paused_fork_to_exec() {
9285 use std::io::{Read, Write};
9286 use std::os::fd::AsRawFd;
9287
9288 let descriptor = std::fs::File::open("/dev/null").unwrap();
9289 let mut command = std::process::Command::new("/bin/sleep");
9290 command.arg("30");
9291 let (mut ready, mut release) =
9292 subc_os::fork_exec_test::pause_before_exec(&mut command, descriptor.as_raw_fd())
9293 .unwrap();
9294 let spawn = std::thread::spawn(move || command.spawn());
9295 let mut message = [0u8; 8];
9296 ready.read_exact(&mut message).unwrap();
9297 let pid = i32::from_ne_bytes(message[..4].try_into().unwrap()) as u32;
9298 let before = std::fs::read_to_string(format!("/proc/{pid}/stat"));
9299 let inherited = subc_os::Process::open(pid).unwrap().unwrap().observe();
9300 let (events, received) = std::sync::mpsc::channel();
9301 let observer = std::thread::spawn(move || {
9302 EXEC_EVENTS.with(|slot| *slot.borrow_mut() = Some(events.clone()));
9303 let image = observe_spawned_image(pid);
9304 let _ = events.send(ExecEvent::Image(image));
9305 });
9306 let first = received.recv_timeout(Duration::from_secs(5));
9310 release.write_all(&[1]).unwrap();
9311 let mut child = spawn.join().unwrap().unwrap();
9312 let after = match first {
9313 Ok(ExecEvent::Uncommitted) => match received.recv_timeout(Duration::from_secs(5)) {
9314 Ok(ExecEvent::Image(image)) => image,
9315 _ => None,
9316 },
9317 Ok(ExecEvent::Image(image)) => image,
9318 Err(_) => None,
9319 };
9320 observer.join().unwrap();
9321 child.kill().unwrap();
9322 child.wait().unwrap();
9323
9324 assert_eq!(linux_exec_committed(&before.unwrap()), Some(false));
9325 assert_eq!(
9326 inherited.unwrap().executable,
9327 subc_os::file_identity(&std::env::current_exe().unwrap())
9328 );
9329 assert_eq!(
9330 after.unwrap().executable,
9331 subc_os::file_identity(std::path::Path::new("/bin/sleep"))
9332 );
9333 }
9334
9335 #[test]
9336 fn spawned_image_is_not_sampled_until_linux_exec_commits() {
9337 let committed = Cell::new(false);
9338 let inherited = subc_os::Observation {
9339 start_time: 23,
9340 executable: Some(subc_os::FileIdentity {
9341 device: 29,
9342 inode: 31,
9343 }),
9344 };
9345 let module = subc_os::Observation {
9346 start_time: 23,
9347 executable: Some(subc_os::FileIdentity {
9348 device: 29,
9349 inode: 37,
9350 }),
9351 };
9352 let polls = Cell::new(0);
9353 let image = sample_after_linux_exec(
9354 || {
9355 polls.set(polls.get() + 1);
9356 committed.set(polls.get() == 2);
9359 Some(committed.get())
9360 },
9361 || Some(if committed.get() { module } else { inherited }),
9362 );
9363 assert_eq!(image, Some(module));
9364 assert_eq!(polls.get(), 2);
9365 }
9366
9367 #[test]
9368 fn linux_exec_flag_is_read_from_flags_after_the_last_parenthesis() {
9369 assert_eq!(
9370 linux_exec_committed("17 (a ) b) R 1 2 3 4 5 4194368"),
9371 Some(false)
9372 );
9373 assert_eq!(
9374 linux_exec_committed("17 (a ) b) R 1 2 3 4 5 4194304"),
9375 Some(true)
9376 );
9377 assert_eq!(linux_exec_committed("17 (a) R 1 2 3 4 5 invalid"), None);
9378 assert_eq!(linux_exec_committed("17 (a) R"), None);
9379 }
9380
9381 #[test]
9382 fn unreadable_exec_state_never_samples_an_inherited_image() {
9383 assert_eq!(
9384 sample_after_linux_exec(
9385 || None,
9386 || panic!("must not sample before exec confirmation")
9387 ),
9388 None
9389 );
9390 }
9391
9392 #[test]
9393 fn uncommitted_exec_times_out_without_sampling_an_inherited_image() {
9394 assert_eq!(
9395 sample_after_linux_exec(
9396 || Some(false),
9397 || panic!("must not sample after an exec timeout")
9398 ),
9399 None
9400 );
9401 }
9402}
9403
9404#[cfg(windows)]
9407fn windows_spawn_capture(spec: &ModuleSpec) -> io::Result<subc_os::ExecutableCapture> {
9408 let mut program = spec.program.clone();
9409 if program.is_absolute() || program.components().count() > 1 {
9410 match subc_os::ExecutableCapture::open(&program) {
9413 Ok(capture) => return Ok(capture),
9414 Err(error)
9415 if error.kind() == io::ErrorKind::NotFound && program.extension().is_none() => {}
9416 Err(error) => return Err(error),
9417 }
9418 program.set_extension("exe");
9419 return subc_os::ExecutableCapture::open(&program);
9420 }
9421 if program.extension().is_none() {
9422 program.set_extension("exe");
9423 }
9424 let custom_path = spec
9425 .env
9426 .iter()
9427 .rev()
9428 .find(|(key, _)| key.eq_ignore_ascii_case("PATH"));
9429 let mut directories = Vec::new();
9430 if let Some(parent) = std::env::current_exe()?.parent() {
9433 directories.push(parent.to_path_buf());
9434 }
9435 directories.push(std::env::current_dir()?);
9436 if let Some(windows) = std::env::var_os("SystemRoot") {
9437 let windows = PathBuf::from(windows);
9438 directories.push(windows.join("System32"));
9439 directories.push(windows.clone());
9440 }
9441 let path = custom_path
9442 .map(|(_, value)| std::ffi::OsString::from(value))
9443 .or_else(|| std::env::var_os("PATH"));
9444 if let Some(path) = path {
9445 directories.extend(std::env::split_paths(&path));
9446 }
9447 for directory in directories {
9448 match subc_os::ExecutableCapture::open(&directory.join(&program)) {
9449 Ok(capture) => return Ok(capture),
9450 Err(error) if error.kind() == io::ErrorKind::NotFound => continue,
9451 Err(error) => return Err(error),
9452 }
9453 }
9454 Err(io::Error::new(
9455 io::ErrorKind::NotFound,
9456 "resolved executable not found",
9457 ))
9458}
9459
9460fn spawn_and_mark_running(
9461 spec: &ModuleSpec,
9462 runtime: &SupervisorRuntimeConfig,
9463 snapshot: &SharedSnapshot,
9464) -> Result<SupervisedChild, SuperviseError> {
9465 let activity_spawn = runtime.begin_activity_spawn(&spec.module_id);
9466 let child = spawn_child(
9467 spec,
9468 runtime.connection_file_path.as_deref(),
9469 runtime.supervisor_handle.as_ref(),
9470 &runtime.stderr_ring,
9471 runtime.capture_logs_dir.as_deref(),
9472 &runtime.child_roster,
9473 #[cfg(target_os = "linux")]
9474 runtime.cgroup_placement.as_ref(),
9475 )?;
9476 set_running(snapshot, &child, &spec.module_id, &runtime.spawn_events)?;
9477 if let Some(spawn) = activity_spawn {
9478 spawn.complete(snapshot);
9479 }
9480 Ok(child)
9481}
9482
9483enum RegistrationWaitOutcome {
9484 Registered,
9485 Exited(ExitReport),
9486 TimedOut,
9487}
9488
9489struct ReloadRegistrationFailure {
9490 exit_report: ExitReport,
9491 reason: String,
9492}
9493
9494#[derive(Debug, Clone, Copy, PartialEq, Eq)]
9495enum BusyGaugeObservation {
9496 Quiescent,
9497 Busy,
9498 Omitted,
9499}
9500
9501#[derive(Debug, Clone, PartialEq, Eq)]
9503enum GaugeVerdict {
9504 Zero,
9506 NonZero(u64),
9508 NotACount(String),
9512 Missing,
9514 ProbeFailed(&'static str),
9517}
9518
9519impl GaugeVerdict {
9520 fn reads_quiet(&self) -> bool {
9521 matches!(self, Self::Zero)
9522 }
9523}
9524
9525impl fmt::Display for GaugeVerdict {
9526 fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
9527 match self {
9528 Self::Zero => f.write_str("zero"),
9529 Self::NonZero(value) => write!(f, "non_zero({value})"),
9530 Self::NotACount(value) => write!(f, "not_a_count({value})"),
9531 Self::Missing => f.write_str("missing_from_health_body"),
9532 Self::ProbeFailed(kind) => write!(f, "probe_failed({kind})"),
9533 }
9534 }
9535}
9536
9537const GAUGE_VALUE_TEXT_LIMIT: usize = 32;
9540
9541fn gauge_verdicts(metrics: Option<&Value>, gauges: &[String]) -> Vec<GaugeVerdict> {
9543 let metrics = metrics.and_then(Value::as_object);
9544 gauges
9545 .iter()
9546 .map(
9547 |gauge| match metrics.and_then(|metrics| metrics.get(gauge)) {
9548 None => GaugeVerdict::Missing,
9549 Some(value) => match value.as_u64() {
9550 Some(0) => GaugeVerdict::Zero,
9551 Some(count) => GaugeVerdict::NonZero(count),
9552 None => {
9553 let mut text = value.to_string();
9554 if text.len() > GAUGE_VALUE_TEXT_LIMIT {
9555 let mut end = GAUGE_VALUE_TEXT_LIMIT;
9556 while !text.is_char_boundary(end) {
9557 end -= 1;
9558 }
9559 text.truncate(end);
9560 text.push_str("...");
9561 }
9562 GaugeVerdict::NotACount(text)
9563 }
9564 },
9565 },
9566 )
9567 .collect()
9568}
9569
9570fn busy_gauge_observation(verdicts: &[GaugeVerdict]) -> BusyGaugeObservation {
9573 if verdicts
9574 .iter()
9575 .any(|verdict| matches!(verdict, GaugeVerdict::Missing))
9576 {
9577 BusyGaugeObservation::Omitted
9578 } else if verdicts.iter().all(GaugeVerdict::reads_quiet) {
9579 BusyGaugeObservation::Quiescent
9580 } else {
9581 BusyGaugeObservation::Busy
9582 }
9583}
9584
9585#[derive(Debug, Clone, Default, PartialEq, Eq)]
9590struct GaugeTrace {
9591 latest: Option<(GaugeVerdict, Duration)>,
9592 last_value: Option<(GaugeVerdict, Duration)>,
9593}
9594
9595#[derive(Debug, Clone, Default, PartialEq, Eq)]
9601struct DrainGaugeTrace {
9602 probes: u32,
9604 gauges: Vec<(String, GaugeTrace)>,
9606}
9607
9608impl DrainGaugeTrace {
9609 fn new(gauges: &[String]) -> Self {
9610 Self {
9611 probes: 0,
9612 gauges: gauges
9613 .iter()
9614 .map(|gauge| (gauge.clone(), GaugeTrace::default()))
9615 .collect(),
9616 }
9617 }
9618
9619 fn record(&mut self, verdicts: &[GaugeVerdict], at: Duration) {
9621 self.probes = self.probes.saturating_add(1);
9622 for ((_, trace), verdict) in self.gauges.iter_mut().zip(verdicts) {
9623 trace.latest = Some((verdict.clone(), at));
9624 if matches!(
9625 verdict,
9626 GaugeVerdict::Zero | GaugeVerdict::NonZero(_) | GaugeVerdict::NotACount(_)
9627 ) {
9628 trace.last_value = Some((verdict.clone(), at));
9629 }
9630 }
9631 }
9632}
9633
9634impl fmt::Display for DrainGaugeTrace {
9639 fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
9640 if self.gauges.is_empty() {
9641 return f.write_str("none declared");
9642 }
9643 for (index, (name, trace)) in self.gauges.iter().enumerate() {
9644 if index > 0 {
9645 f.write_str(" | ")?;
9646 }
9647 write!(f, "{name}: ")?;
9648 match &trace.latest {
9649 Some((verdict, at)) => write!(f, "{verdict} @{}ms", at.as_millis())?,
9650 None => f.write_str("never probed")?,
9651 }
9652 match &trace.last_value {
9653 Some((value, at)) => write!(f, "; last value {value} @{}ms", at.as_millis())?,
9654 None => f.write_str("; no value ever read")?,
9655 }
9656 }
9657 Ok(())
9658 }
9659}
9660
9661fn declared_busy_gauges(
9662 registry: &Registry,
9663 module_id: &str,
9664) -> Result<Vec<String>, SuperviseError> {
9665 busy_gauges_of(
9666 registry
9667 .get_module(module_id)
9668 .map_err(SuperviseError::Registry)?,
9669 )
9670}
9671
9672fn declared_busy_gauges_for_connection(
9676 registry: &Registry,
9677 connection_id: ConnectionId,
9678) -> Result<Vec<String>, SuperviseError> {
9679 busy_gauges_of(
9680 registry
9681 .get_module_by_connection(connection_id)
9682 .map_err(SuperviseError::Registry)?,
9683 )
9684}
9685
9686fn busy_gauges_of(
9687 registration: Option<crate::registry::ModuleRegistration>,
9688) -> Result<Vec<String>, SuperviseError> {
9689 let Some(registration) = registration else {
9690 return Ok(Vec::new());
9691 };
9692 let Some(self_signals) = registration.manifest.self_signals else {
9693 return Ok(Vec::new());
9694 };
9695
9696 let mut gauges = Vec::new();
9697 for declaration in self_signals {
9698 if declaration.kind != SelfSignalKind::Busy {
9699 continue;
9700 }
9701 match declaration.anchored_to {
9702 SignalAnchor::HealthGauges { gauges: declared } if !declared.is_empty() => {
9703 gauges.extend(declared)
9704 }
9705 _ => {
9706 gauges.push(String::new());
9709 }
9710 }
9711 }
9712 Ok(gauges)
9713}
9714
9715#[allow(clippy::too_many_arguments)]
9724async fn wait_for_forwarding_quiescence(
9725 forwarding: &ForwardingTable,
9726 module_id: &str,
9727 runtime: &SupervisorRuntimeConfig,
9728 endpoint: crate::ModuleEndpointId,
9729 deadline: Instant,
9730 busy_gauges: &[String],
9731 scope: DrainScope,
9732 trace: &mut DrainGaugeTrace,
9733) -> Result<bool, SuperviseError> {
9734 let wait_started = Instant::now();
9735 let mut gauges_quiescent = busy_gauges.is_empty();
9736 let mut next_probe_at = wait_started;
9737 let mut omission_reported = false;
9738
9739 loop {
9740 let now = Instant::now();
9741 if !busy_gauges.is_empty() && now >= next_probe_at && now < deadline {
9742 let report = match scope {
9743 DrainScope::Active => probe_module_health(module_id, runtime, Some(deadline)).await,
9744 DrainScope::Endpoint(endpoint) => {
9745 probe_endpoint_health(endpoint, runtime, Some(deadline)).await
9746 }
9747 };
9748 let verdicts = match &report {
9749 Ok(report) => gauge_verdicts(report.metrics.as_ref(), busy_gauges),
9750 Err(err) => vec![GaugeVerdict::ProbeFailed(err.label()); busy_gauges.len()],
9751 };
9752 trace.record(&verdicts, wait_started.elapsed());
9753 gauges_quiescent = match report {
9754 Ok(_) => match busy_gauge_observation(&verdicts) {
9755 BusyGaugeObservation::Quiescent => true,
9756 BusyGaugeObservation::Busy => false,
9757 BusyGaugeObservation::Omitted => {
9758 if !omission_reported {
9761 forwarding
9762 .counters()
9763 .increment_drains_with_undeclared_gauge();
9764 let missing = busy_gauges
9765 .iter()
9766 .zip(&verdicts)
9767 .filter(|(_, verdict)| matches!(verdict, GaugeVerdict::Missing))
9768 .map(|(gauge, _)| gauge.as_str())
9769 .collect::<Vec<_>>()
9770 .join(",");
9771 warn!(
9772 module_id,
9773 missing_gauges = %missing,
9774 "drain health.check answer lacks a declared busy gauge; the drain treats the module as busy until it reports it"
9775 );
9776 omission_reported = true;
9777 }
9778 false
9779 }
9780 },
9781 Err(err) => {
9782 warn!(
9783 module_id,
9784 error = %err,
9785 "drain health.check did not produce declared busy gauges; treating module as busy"
9786 );
9787 false
9788 }
9789 };
9790 next_probe_at = Instant::now() + drain_busy_reprobe_interval();
9793 }
9794
9795 let in_flight = forwarding
9796 .endpoint_in_flight_count(endpoint)
9797 .map_err(SuperviseError::Forwarding)?;
9798 if in_flight == 0 && gauges_quiescent {
9799 return Ok(true);
9800 }
9801
9802 let now = Instant::now();
9803 if now >= deadline {
9804 return Ok(false);
9805 }
9806 let mut wait = deadline
9807 .saturating_duration_since(now)
9808 .min(REGISTRY_RELEASE_POLL);
9809 if !busy_gauges.is_empty() {
9810 wait = wait.min(next_probe_at.saturating_duration_since(now));
9811 }
9812 sleep(wait).await;
9813 }
9814}
9815
9816fn drain_busy_reprobe_interval() -> Duration {
9820 DRAIN_BUSY_REPROBE_INTERVAL.max(REGISTRY_RELEASE_POLL)
9821}
9822
9823fn warn_drain_timed_out(
9838 module_id: &str,
9839 drain_timeout: Duration,
9840 reason: RouteCloseReason,
9841 holdouts: &crate::forwarding::DrainHoldouts,
9842 gauges: &DrainGaugeTrace,
9843) {
9844 warn!(
9845 module_id = %module_id,
9846 waited = ?drain_timeout,
9847 ?reason,
9848 held_requests = holdouts.requests,
9849 held_routes = holdouts.routes,
9850 total_routes = holdouts.total_routes,
9851 top_connections = ?holdouts.top_connections,
9852 held = %holdouts
9855 .held
9856 .iter()
9857 .map(|(channel, corr)| format!("{channel}:{corr}"))
9858 .collect::<Vec<_>>()
9859 .join(","),
9860 gauge_probes = gauges.probes,
9861 busy_gauges = %gauges,
9862 "route drain timed out before request quiescence; forcing teardown"
9863 );
9864}
9865
9866fn drained_after_quiescence_wait(wait_result: &Result<bool, SuperviseError>) -> bool {
9874 match wait_result {
9875 Ok(drained) => *drained,
9876 Err(_) => false,
9877 }
9878}
9879
9880fn send_route_goodbyes(forwarding: &ForwardingTable, released_routes: Vec<GoodbyeTarget>) {
9881 for released in released_routes {
9882 let frame = match Frame::build_with_version(
9883 released.negotiated_ver,
9884 FrameType::Goodbye,
9885 control_flags(),
9886 released.channel,
9887 released.epoch,
9888 0,
9889 Vec::new(),
9890 ) {
9891 Ok(frame) => frame,
9892 Err(err) => {
9893 warn!(
9894 route_channel = released.channel,
9895 error = %err,
9896 "failed to build supervisor drain route GOODBYE frame"
9897 );
9898 continue;
9899 }
9900 };
9901 if !released.close_on_delivery_failure() {
9902 crate::forwarding::send_module_route_goodbye(
9903 &forwarding.counters(),
9904 &released.sink,
9905 frame,
9906 released.module_id.as_deref(),
9907 "supervisor drain",
9908 );
9909 continue;
9910 }
9911 if let Err(err) = released.sink.try_send(frame) {
9912 warn!(
9913 target_connection_id = released.connection_id.get(),
9914 route_channel = released.channel,
9915 error = %err,
9916 "supervisor drain route GOODBYE was not delivered to client; closing target connection"
9917 );
9918 let _ = forwarding.escalate_client_delivery_failure(
9919 released.connection_id,
9920 released.channel,
9921 released.epoch,
9922 CloseReason::new(
9923 "route_goodbye_delivery_failed",
9924 format!(
9925 "failed to enqueue supervisor drain route GOODBYE for channel {}: {err}",
9926 released.channel
9927 ),
9928 ),
9929 crate::forwarding::UndeliveredFrame {
9930 module_id: released.module_id.as_deref(),
9931 sink: &released.sink,
9932 },
9933 );
9934 }
9935 }
9936}
9937
9938fn send_module_draining(
9939 module_id: &str,
9940 reason: RouteCloseReason,
9941 deadline_ms: u64,
9942 target: &ModuleDrainTarget,
9943) {
9944 let body = match serde_json::to_vec(&ModuleControlCommand::Draining {
9945 reason,
9946 deadline_ms,
9947 }) {
9948 Ok(body) => body,
9949 Err(err) => {
9950 warn!(
9951 module_id,
9952 error = %err,
9953 "failed to encode module draining command"
9954 );
9955 return;
9956 }
9957 };
9958 let frame = match Frame::build_with_version(
9959 target.negotiated_ver,
9960 FrameType::Push,
9961 control_flags(),
9962 0,
9963 0,
9964 0,
9965 body,
9966 ) {
9967 Ok(frame) => frame,
9968 Err(err) => {
9969 warn!(
9970 module_id,
9971 error = %err,
9972 "failed to build module draining command frame"
9973 );
9974 return;
9975 }
9976 };
9977 if let Err(err) = target.sink.try_send(frame) {
9978 warn!(
9979 module_id,
9980 target_connection_id = target.endpoint.connection_id.get(),
9981 error = %err,
9982 "module draining command was not delivered to peer"
9983 );
9984 }
9985}
9986
9987fn module_goodbye_frame(module_id: &str, negotiated_ver: u8) -> Option<Frame> {
9989 match Frame::build_with_version(
9990 negotiated_ver,
9991 FrameType::Goodbye,
9992 control_flags(),
9993 0,
9994 0,
9995 0,
9996 Vec::new(),
9997 ) {
9998 Ok(frame) => Some(frame),
9999 Err(err) => {
10000 warn!(
10001 module_id,
10002 error = %err,
10003 "failed to build module GOODBYE frame"
10004 );
10005 None
10006 }
10007 }
10008}
10009
10010#[cfg(unix)]
10024async fn send_module_goodbyes_for_daemon_shutdown(
10025 forwarding: &Arc<ForwardingTable>,
10026 reason: &CloseReason,
10027 wait_for_flush: bool,
10028) {
10029 const GOODBYE_BUDGET: Duration = Duration::from_millis(500);
10030 let targets = match forwarding.module_connections() {
10031 Ok(targets) => targets,
10032 Err(err) => {
10033 warn!(error = %err, "could not list module connections for shutdown GOODBYE");
10034 return;
10035 }
10036 };
10037 let deadline = Instant::now() + GOODBYE_BUDGET;
10038 let mut sends = tokio::task::JoinSet::new();
10039 for target in targets {
10040 let Some(frame) = module_goodbye_frame(&target.module_id, target.negotiated_ver) else {
10041 continue;
10042 };
10043 if !wait_for_flush {
10044 if let Err(err) = target.sink.try_send(frame) {
10045 debug!(
10046 module_id = %target.module_id,
10047 error = %err,
10048 "shutdown module GOODBYE was not queued"
10049 );
10050 }
10051 continue;
10052 }
10053 let forwarding = Arc::clone(forwarding);
10054 let reason = reason.clone();
10055 sends.spawn(async move {
10056 match timeout_at(deadline, target.sink.send_flushed(frame)).await {
10057 Ok(Ok(())) => {}
10058 Ok(Err(err)) => debug!(
10059 module_id = %target.module_id,
10060 error = %err,
10061 "module connection closed before its shutdown GOODBYE was written"
10062 ),
10063 Err(_) => warn!(
10064 module_id = %target.module_id,
10065 budget = ?GOODBYE_BUDGET,
10066 "shutdown module GOODBYE was not written within its budget; closing anyway"
10067 ),
10068 }
10069 forwarding.request_connection_close(target.endpoint.connection_id, reason);
10070 });
10071 }
10072 while sends.join_next().await.is_some() {}
10074}
10075
10076fn send_module_goodbye(module_id: &str, forwarding: &ForwardingTable, target: &ModuleDrainTarget) {
10077 let Some(frame) = module_goodbye_frame(module_id, target.negotiated_ver) else {
10078 return;
10079 };
10080 if let Err(err) = target.sink.try_send(frame) {
10081 warn!(
10082 module_id,
10083 target_connection_id = target.endpoint.connection_id.get(),
10084 error = %err,
10085 "supervisor drain module GOODBYE was not delivered to peer; closing module connection"
10086 );
10087 forwarding.request_connection_close(
10088 target.endpoint.connection_id,
10089 CloseReason::new(
10090 "module_goodbye_delivery_failed",
10091 format!("failed to enqueue supervisor drain module GOODBYE for module '{module_id}': {err}"),
10092 ),
10093 );
10094 }
10095}
10096
10097#[derive(Clone, Copy)]
10098struct ForwardingDrainContext<'a> {
10099 spec: &'a ModuleSpec,
10100 runtime: &'a SupervisorRuntimeConfig,
10101 registry: &'a Registry,
10102 scope: DrainScope,
10103}
10104
10105#[derive(Debug, Clone, Copy, PartialEq, Eq)]
10107enum DrainScope {
10108 Active,
10111 Endpoint(crate::ModuleEndpointId),
10116}
10117
10118#[derive(Debug, Clone, Copy, PartialEq, Eq)]
10126enum StopNotice {
10127 SentOverConnection,
10130 NoConnection,
10134 NotSent,
10138}
10139
10140async fn begin_forwarding_drain(
10141 spec: &ModuleSpec,
10142 runtime: &SupervisorRuntimeConfig,
10143 registry: &Registry,
10144 snapshot: &SharedSnapshot,
10145 enabled: Option<bool>,
10146 reason: RouteCloseReason,
10147) -> Result<StopNotice, SuperviseError> {
10148 let Some(forwarding) = runtime.forwarding.as_ref() else {
10149 return Err(SuperviseError::ReloadUnavailable {
10150 module_id: spec.module_id.clone(),
10151 reason: "supervisor was not configured with a forwarding table".to_string(),
10152 });
10153 };
10154
10155 begin_forwarding_drain_with(
10156 forwarding,
10157 ForwardingDrainContext {
10158 spec,
10159 runtime,
10160 registry,
10161 scope: DrainScope::Active,
10162 },
10163 snapshot,
10164 enabled,
10165 reason,
10166 runtime.drain_timeout,
10167 )
10168 .await
10169}
10170
10171async fn begin_forwarding_drain_if_configured(
10172 spec: &ModuleSpec,
10173 runtime: &SupervisorRuntimeConfig,
10174 registry: &Registry,
10175 snapshot: &SharedSnapshot,
10176 enabled: Option<bool>,
10177 reason: RouteCloseReason,
10178) -> Result<StopNotice, SuperviseError> {
10179 begin_forwarding_drain_with_timeout(
10180 spec,
10181 runtime,
10182 registry,
10183 snapshot,
10184 enabled,
10185 reason,
10186 runtime.drain_timeout,
10187 )
10188 .await
10189}
10190
10191async fn begin_forwarding_drain_with_timeout(
10195 spec: &ModuleSpec,
10196 runtime: &SupervisorRuntimeConfig,
10197 registry: &Registry,
10198 snapshot: &SharedSnapshot,
10199 enabled: Option<bool>,
10200 reason: RouteCloseReason,
10201 drain_timeout: Duration,
10202) -> Result<StopNotice, SuperviseError> {
10203 let Some(forwarding) = runtime.forwarding.as_ref() else {
10204 return Ok(StopNotice::NotSent);
10205 };
10206
10207 begin_forwarding_drain_with(
10208 forwarding,
10209 ForwardingDrainContext {
10210 spec,
10211 runtime,
10212 registry,
10213 scope: DrainScope::Active,
10214 },
10215 snapshot,
10216 enabled,
10217 reason,
10218 drain_timeout,
10219 )
10220 .await
10221}
10222
10223async fn begin_forwarding_drain_with(
10224 forwarding: &ForwardingTable,
10225 context: ForwardingDrainContext<'_>,
10226 snapshot: &SharedSnapshot,
10227 enabled: Option<bool>,
10228 reason: RouteCloseReason,
10229 drain_timeout: Duration,
10230) -> Result<StopNotice, SuperviseError> {
10231 let ForwardingDrainContext {
10232 spec,
10233 runtime,
10234 registry,
10235 scope,
10236 } = context;
10237 debug_assert_ne!(reason, RouteCloseReason::Crash);
10238 let terminal = matches!(reason, RouteCloseReason::Disable);
10239 let drain_started_at = Instant::now();
10240 let drain_deadline = drain_started_at + drain_timeout;
10241 let deadline_ms =
10242 unix_ms_now().saturating_add(u64::try_from(drain_timeout.as_millis()).unwrap_or(u64::MAX));
10243 let busy_gauges = match scope {
10244 DrainScope::Active => declared_busy_gauges(registry, &spec.module_id)?,
10245 DrainScope::Endpoint(endpoint) => {
10246 declared_busy_gauges_for_connection(registry, endpoint.connection_id)?
10247 }
10248 };
10249
10250 let gate_started = Instant::now();
10253 let drain_target = match scope {
10254 DrainScope::Active => forwarding.begin_module_drain(&spec.module_id, reason),
10255 DrainScope::Endpoint(endpoint) => forwarding.begin_endpoint_drain(endpoint, reason),
10256 }
10257 .map_err(SuperviseError::Forwarding)?;
10258 info!(
10263 module_id = %spec.module_id,
10264 ?reason,
10265 gate_ms = u64::try_from(gate_started.elapsed().as_millis()).unwrap_or(u64::MAX),
10266 connected = drain_target.is_some(),
10267 "module drain began; route admission closed"
10268 );
10269 if scope == DrainScope::Active {
10270 update_snapshot(snapshot, Some(&spec.module_id), |state| {
10271 state.state = ModuleState::Draining;
10272 state.draining_to_replace =
10273 matches!(reason, RouteCloseReason::Restart | RouteCloseReason::Reload);
10274 if let Some(enabled) = enabled {
10275 state.enabled = enabled;
10276 }
10277 })?;
10278 }
10279
10280 let Some(target) = drain_target.as_ref() else {
10281 return Ok(StopNotice::NoConnection);
10285 };
10286 {
10287 send_module_draining(&spec.module_id, reason, deadline_ms, target);
10288 #[cfg(feature = "test-support")]
10289 forwarding.wait_before_route_closing_for_test().await;
10290 let routes = forwarding
10291 .endpoint_routes(target.endpoint)
10292 .map_err(SuperviseError::Forwarding)?;
10293 let routes_notified = routes.len();
10294 crate::control::send_route_control_pushes(
10295 forwarding,
10296 routes.clone(),
10297 ClientControlPush::RouteClosing {
10298 module_id: spec.module_id.clone(),
10299 channels: Vec::new(),
10300 reason,
10301 },
10302 );
10303 send_route_goodbyes(forwarding, target.abandoned_bindings.clone());
10304
10305 let mut gauge_trace = DrainGaugeTrace::new(&busy_gauges);
10311 let wait_result = wait_for_forwarding_quiescence(
10312 forwarding,
10313 &spec.module_id,
10314 runtime,
10315 target.endpoint,
10316 drain_deadline,
10317 &busy_gauges,
10318 scope,
10319 &mut gauge_trace,
10320 )
10321 .await;
10322 let drained = drained_after_quiescence_wait(&wait_result);
10323 if let Err(err) = &wait_result {
10324 error!(
10325 module_id = %spec.module_id,
10326 ?reason,
10327 error = %err,
10328 "forwarding quiescence wait failed after route.closing; forcing route.closed(drained: false) so the client is not left waiting on an unfulfilled promise"
10329 );
10330 } else if !drained {
10331 let holdouts = forwarding
10333 .endpoint_drain_holdouts(target.endpoint)
10334 .unwrap_or_default();
10335 warn_drain_timed_out(
10336 &spec.module_id,
10337 drain_timeout,
10338 reason,
10339 &holdouts,
10340 &gauge_trace,
10341 );
10342 }
10343 crate::control::send_route_control_pushes(
10344 forwarding,
10345 routes,
10346 ClientControlPush::RouteClosed {
10347 module_id: spec.module_id.clone(),
10348 channels: Vec::new(),
10349 reason,
10350 drained,
10351 abandoned: target.abandoned_bindings.len() as u32,
10352 excluded_subscriptions: target.excluded_subscriptions,
10353 terminal: Some(terminal),
10354 },
10355 );
10356 wait_result?;
10357
10358 let released_routes = match forwarding.release_module_endpoint_routes(target.endpoint) {
10364 Ok(routes) => routes,
10365 Err(err) => {
10366 warn!(
10367 module_id = %spec.module_id,
10368 ?reason,
10369 error = %err,
10370 "failed to release module endpoint routes after route.closed; module GOODBYE will still be sent"
10371 );
10372 send_module_goodbye(&spec.module_id, forwarding, target);
10373 return Err(SuperviseError::Forwarding(err));
10374 }
10375 };
10376 let route_goodbye_count = released_routes.len();
10377 send_route_goodbyes(forwarding, released_routes);
10378 send_module_goodbye(&spec.module_id, forwarding, target);
10379
10380 info!(
10386 module_id = %spec.module_id,
10387 ?reason,
10388 routes_notified,
10389 route_goodbyes = route_goodbye_count,
10390 abandoned_reservations = target.abandoned_bindings.len(),
10391 excluded_subscriptions = target.excluded_subscriptions,
10392 drained,
10393 "module drain complete; consumers notified via route.closing/route.closed pushes and per-route GOODBYE frames"
10394 );
10395 }
10396
10397 Ok(StopNotice::SentOverConnection)
10398}
10399
10400async fn wait_for_registration_after_reload(
10403 registry: &Registry,
10404 module_id: &str,
10405 snapshot: &SharedSnapshot,
10406 child: &mut SupervisedChild,
10407 wait: Duration,
10408) -> Result<RegistrationWaitOutcome, SuperviseError> {
10409 wait_for_slot_registration(
10410 registry,
10411 crate::registry::RegistrationSlot::Active(module_id),
10412 module_id,
10413 snapshot,
10414 child,
10415 wait,
10416 )
10417 .await
10418}
10419
10420async fn wait_for_slot_registration(
10428 registry: &Registry,
10429 slot: crate::registry::RegistrationSlot<'_>,
10430 module_id: &str,
10431 snapshot: &SharedSnapshot,
10432 child: &mut SupervisedChild,
10433 wait: Duration,
10434) -> Result<RegistrationWaitOutcome, SuperviseError> {
10435 let deadline = Instant::now() + wait;
10436 loop {
10437 if registry
10438 .registration(slot)
10439 .map_err(SuperviseError::Registry)?
10440 .is_some()
10441 {
10442 return Ok(RegistrationWaitOutcome::Registered);
10443 }
10444
10445 let now = Instant::now();
10446 if now >= deadline {
10447 return Ok(RegistrationWaitOutcome::TimedOut);
10448 }
10449 let remaining = deadline.saturating_duration_since(now);
10450 let poll = remaining.min(REGISTRY_RELEASE_POLL);
10451
10452 tokio::select! {
10453 wait_result = child.wait() => {
10454 let status = wait_result.map_err(|source| SuperviseError::Wait {
10455 module_id: module_id.to_string(),
10456 source,
10457 })?;
10458 return Ok(RegistrationWaitOutcome::Exited(classify_reaped_child_exit(
10459 snapshot,
10460 child,
10461 &status,
10462 )));
10463 }
10464 _ = sleep(poll) => {}
10465 }
10466 }
10467}
10468
10469fn registration_failure_exit_report(mut exit_report: ExitReport) -> ExitReport {
10470 if exit_report.kind != ExitKind::DeliberateSeverance {
10473 exit_report.kind = ExitKind::Crash;
10474 }
10475 exit_report
10476}
10477
10478async fn handle_reload_child_registration_failure(
10479 spec: &ModuleSpec,
10480 runtime: &SupervisorRuntimeConfig,
10481 registry: &Registry,
10482 process_liveness: &SupervisorProcessLiveness,
10483 snapshot: &SharedSnapshot,
10484 _child: &mut Option<SupervisedChild>,
10485 failure: ReloadRegistrationFailure,
10486) -> Result<(), SuperviseError> {
10487 let ReloadRegistrationFailure {
10488 exit_report,
10489 reason,
10490 } = failure;
10491 match on_child_exit(
10492 spec,
10493 runtime.restart_policy,
10494 registry,
10495 snapshot,
10496 &runtime.terminal_ring,
10497 &runtime.spawn_events,
10498 &runtime.child_roster,
10499 exit_report,
10500 )
10501 .await
10502 {
10503 NextAction::Stop {
10504 registration_released,
10505 } => {
10506 if registration_released {
10507 process_liveness.untrack_if_current(&spec.module_id, snapshot);
10508 }
10509 }
10510 NextAction::Restart { schedule } => {
10511 let delay = schedule.map_or(runtime.restart_policy.delay_for_restart(0), |schedule| {
10512 schedule.delay
10513 });
10514 if let Some(schedule) = schedule {
10515 log_crash_respawn(&spec.module_id, schedule);
10516 }
10517 schedule_respawn(
10518 runtime,
10519 snapshot,
10520 &spec.module_id,
10521 delay,
10522 RespawnKind::Spawn,
10523 )?;
10524 }
10525 }
10526 Err(SuperviseError::ReloadFailed {
10527 module_id: spec.module_id.clone(),
10528 reason,
10529 })
10530}
10531
10532async fn handle_reload_spawn_failure(
10533 spec: &ModuleSpec,
10534 runtime: &SupervisorRuntimeConfig,
10535 process_liveness: &SupervisorProcessLiveness,
10536 snapshot: &SharedSnapshot,
10537 _child: &mut Option<SupervisedChild>,
10538 reason: String,
10539) -> Result<(), SuperviseError> {
10540 let now = Instant::now();
10541 let mut schedule = None;
10542 update_snapshot(snapshot, Some(&spec.module_id), |state| {
10543 clear_current_process_facts(state);
10544 if state.enabled {
10545 schedule = state.next_crash_restart(&runtime.restart_policy, now);
10546 state.state = if schedule.is_some() {
10547 ModuleState::Restarting
10548 } else {
10549 ModuleState::Failed
10550 };
10551 } else {
10552 state.state = ModuleState::Disabled;
10553 }
10554 })?;
10555 if let Some(schedule) = schedule {
10556 schedule_respawn(
10557 runtime,
10558 snapshot,
10559 &spec.module_id,
10560 schedule.delay,
10561 RespawnKind::Spawn,
10562 )?;
10563 } else {
10564 process_liveness.untrack_if_current(&spec.module_id, snapshot);
10565 }
10566 Err(SuperviseError::ReloadFailed {
10567 module_id: spec.module_id.clone(),
10568 reason,
10569 })
10570}
10571
10572fn control_flags() -> Flags {
10573 Flags::new(false, Priority::Passive, false)
10574}
10575
10576#[allow(clippy::too_many_arguments)]
10577async fn drain_optional_child(
10578 module_id: &str,
10579 protocol: ModuleProtocol,
10580 stop_notice: StopNotice,
10581 registry: &Registry,
10582 forwarding: Option<&ForwardingTable>,
10583 snapshot: &SharedSnapshot,
10584 terminal_ring: &Arc<Mutex<TerminalRing>>,
10585 spawn_events: &SpawnEventFeed,
10586 child: &mut Option<SupervisedChild>,
10587 drain_timeout: Duration,
10588 final_state: ModuleState,
10589 enabled: Option<bool>,
10590) -> Result<(), SuperviseError> {
10591 if let Some(child) = child.take() {
10592 drain_child_to_state(
10593 module_id,
10594 protocol,
10595 stop_notice,
10596 registry,
10597 forwarding,
10598 snapshot,
10599 terminal_ring,
10600 spawn_events,
10601 child,
10602 drain_timeout,
10603 final_state,
10604 enabled,
10605 )
10606 .await
10607 } else {
10608 update_snapshot(snapshot, Some(module_id), |state| {
10609 state.state = final_state;
10610 if let Some(enabled) = enabled {
10611 state.enabled = enabled;
10612 }
10613 clear_current_process_facts(state);
10614 })?;
10615 release_dead_registration(registry, forwarding, snapshot, module_id).await
10616 }
10617}
10618
10619#[allow(clippy::too_many_arguments)]
10620async fn drain_child_to_state(
10621 module_id: &str,
10622 _protocol: ModuleProtocol,
10623 stop_notice: StopNotice,
10624 registry: &Registry,
10625 forwarding: Option<&ForwardingTable>,
10626 snapshot: &SharedSnapshot,
10627 terminal_ring: &Arc<Mutex<TerminalRing>>,
10628 spawn_events: &SpawnEventFeed,
10629 mut child: SupervisedChild,
10630 drain_timeout: Duration,
10631 final_state: ModuleState,
10632 enabled: Option<bool>,
10633) -> Result<(), SuperviseError> {
10634 let protocol = child.protocol;
10635 update_snapshot(snapshot, Some(module_id), |state| {
10636 state.state = ModuleState::Draining;
10637 state.draining_to_replace = final_state == ModuleState::Restarting;
10638 if let Some(enabled) = enabled {
10639 state.enabled = enabled;
10640 }
10641 })?;
10642
10643 if stop_notice != StopNotice::SentOverConnection {
10654 if protocol == ModuleProtocol::Subc && stop_notice == StopNotice::NoConnection {
10655 info!(
10656 module_id,
10657 pid = child.pid,
10658 budget_ms = u64::try_from(drain_timeout.as_millis()).unwrap_or(u64::MAX),
10659 "module has no connection yet; requesting stop by signal"
10660 );
10661 }
10662 request_graceful_stop(module_id, &child);
10663 }
10664
10665 let exit_report = match timeout(drain_timeout, child.wait()).await {
10666 Ok(Ok(status)) => classify_reaped_child_exit(snapshot, &child, &status),
10667 Ok(Err(source)) => {
10668 fail_after_wait_error(
10669 module_id,
10670 snapshot,
10671 terminal_ring,
10672 spawn_events,
10673 format!("failed to wait during drain: {source}"),
10674 );
10675 return Err(SuperviseError::Wait {
10676 module_id: module_id.to_string(),
10677 source,
10678 });
10679 }
10680 Err(_) => {
10681 warn!(
10694 module_id,
10695 pid = child.pid,
10696 budget_ms = u64::try_from(drain_timeout.as_millis()).unwrap_or(u64::MAX),
10697 reason = ?final_state,
10698 ?stop_notice,
10699 "drain budget expired before the module exited; killing it"
10700 );
10701 child.start_kill().map_err(|source| {
10702 fail_after_wait_error(
10703 module_id,
10704 snapshot,
10705 terminal_ring,
10706 spawn_events,
10707 format!("failed to kill during drain: {source}"),
10708 );
10709 SuperviseError::Kill {
10710 module_id: module_id.to_string(),
10711 source,
10712 }
10713 })?;
10714 let status = child.wait().await.map_err(|source| {
10715 fail_after_wait_error(
10716 module_id,
10717 snapshot,
10718 terminal_ring,
10719 spawn_events,
10720 format!("failed to wait after drain kill: {source}"),
10721 );
10722 SuperviseError::Wait {
10723 module_id: module_id.to_string(),
10724 source,
10725 }
10726 })?;
10727 classify_reaped_child_exit(snapshot, &child, &status)
10728 }
10729 };
10730
10731 let detail = lock_snapshot(snapshot)?.drain_disposition_detail.take();
10732 publish_after_terminal_record(
10733 module_id,
10734 || {
10735 record_terminal_with_detail(
10736 module_id,
10737 terminal_ring,
10738 spawn_events,
10739 &exit_report,
10740 terminal_disposition(final_state),
10741 detail,
10742 )
10743 },
10744 || {
10745 update_snapshot(snapshot, Some(module_id), |state| {
10746 state.state = final_state;
10747 if let Some(enabled) = enabled {
10748 state.enabled = enabled;
10749 }
10750 clear_current_process_facts(state);
10751 state.last_exit = Some(exit_report.clone());
10752 if exit_report.kind == ExitKind::DeliberateSeverance {
10753 state.lifetime_restarts += 1;
10754 }
10755 })
10756 },
10757 )?;
10758 child.drain_stderr(module_id).await;
10759
10760 release_dead_registration(registry, forwarding, snapshot, module_id).await
10761}
10762
10763#[cfg(unix)]
10783fn request_graceful_stop(module_id: &str, child: &SupervisedChild) {
10784 let Some(pid) = child
10785 .id()
10786 .and_then(|pid| i32::try_from(pid).ok())
10787 .and_then(rustix::process::Pid::from_raw)
10788 else {
10789 debug!(
10790 module_id,
10791 "no pid to signal for teardown; falling through to the drain wait"
10792 );
10793 return;
10794 };
10795 match rustix::process::kill_process(pid, rustix::process::Signal::TERM) {
10796 Ok(()) => debug!(
10797 module_id,
10798 "sent SIGTERM to a module nothing else asked to stop"
10799 ),
10800 Err(err) => debug!(
10801 module_id,
10802 error = %err,
10803 "SIGTERM to module failed; the drain wait and kill still apply"
10804 ),
10805 }
10806}
10807
10808#[cfg(not(unix))]
10816fn request_graceful_stop(module_id: &str, _child: &SupervisedChild) {
10817 debug!(
10818 module_id,
10819 "no graceful stop signal exists on this platform; teardown of a module nothing asked to stop waits, then kills"
10820 );
10821}
10822
10823fn terminal_disposition(final_state: ModuleState) -> TerminalDisposition {
10824 match final_state {
10825 ModuleState::Stopped => TerminalDisposition::Stopped,
10826 ModuleState::Disabled => TerminalDisposition::Disabled,
10827 ModuleState::Restarting => TerminalDisposition::Restarting,
10828 ModuleState::Failed => TerminalDisposition::Failed,
10829 ModuleState::Starting
10830 | ModuleState::Running
10831 | ModuleState::Unresponsive
10832 | ModuleState::Draining => {
10833 unreachable!("terminal exits only finish in terminal or restarting states")
10834 }
10835 }
10836}
10837
10838async fn release_dead_registration(
10846 registry: &Registry,
10847 forwarding: Option<&ForwardingTable>,
10848 snapshot: &SharedSnapshot,
10849 module_id: &str,
10850) -> Result<(), SuperviseError> {
10851 let result = async {
10852 let registration = registry
10853 .get_module(module_id)
10854 .map_err(SuperviseError::Registry)?;
10855 match wait_for_registration_release(registry, module_id, REGISTRY_RELEASE_TIMEOUT).await {
10856 Ok(()) => return Ok(()),
10857 Err(SuperviseError::RegistrationStillActive { .. }) => {}
10858 Err(err) => return Err(err),
10859 }
10860 let pid = lock_snapshot(snapshot)?.reaped_pid;
10861 if let (Some(registration), Some(forwarding), Some(pid)) = (registration, forwarding, pid) {
10862 warn!(
10863 module_id,
10864 pid,
10865 connection_id = registration.connection_id.get(),
10866 "reaped module registration outlived release grace; closing dead connection"
10867 );
10868 forwarding.request_connection_close(
10869 registration.connection_id,
10870 CloseReason::new(
10871 "supervised_process_reaped",
10872 format!("module '{module_id}' pid {pid} exited"),
10873 ),
10874 );
10875 wait_for_slot_registration_release(
10876 registry,
10877 crate::registry::RegistrationSlot::Connection(registration.connection_id),
10878 REGISTRY_RELEASE_TIMEOUT,
10879 )
10880 .await?;
10881 }
10882 wait_for_registration_release(registry, module_id, Duration::ZERO).await
10883 }
10884 .await;
10885 if let Err(err) = &result {
10886 fail_snapshot(snapshot, Some(module_id), None);
10887 error!(module_id, error = %err, "registration release failed after child exit; module is failed and start can retry");
10888 }
10889 result
10890}
10891
10892async fn wait_for_registration_release(
10895 registry: &Registry,
10896 module_id: &str,
10897 wait: Duration,
10898) -> Result<(), SuperviseError> {
10899 wait_for_slot_registration_release(
10900 registry,
10901 crate::registry::RegistrationSlot::Active(module_id),
10902 wait,
10903 )
10904 .await
10905}
10906
10907async fn wait_for_slot_registration_release(
10915 registry: &Registry,
10916 slot: crate::registry::RegistrationSlot<'_>,
10917 wait: Duration,
10918) -> Result<(), SuperviseError> {
10919 let deadline = Instant::now() + wait;
10920 let mut release_events = registration_release_events().subscribe();
10921 let still_active = |registration: &crate::registry::ModuleRegistration| {
10922 SuperviseError::RegistrationStillActive {
10923 module_id: registration.manifest.module_id.clone(),
10924 waited: wait,
10925 }
10926 };
10927 loop {
10928 let _observed_generation = *release_events.borrow_and_update();
10929 let Some(registration) = registry
10930 .registration(slot)
10931 .map_err(SuperviseError::Registry)?
10932 else {
10933 return Ok(());
10934 };
10935
10936 let now = Instant::now();
10937 if now >= deadline {
10938 return Err(still_active(®istration));
10939 }
10940
10941 let remaining = deadline.saturating_duration_since(now);
10942 match timeout(remaining, release_events.changed()).await {
10943 Ok(Ok(())) | Ok(Err(_)) => {}
10944 Err(_) => return Err(still_active(®istration)),
10945 }
10946 }
10947}
10948
10949#[cfg(test)]
10950mod slot_registration_wait_tests {
10951 use super::*;
10952 use crate::registry::{ConnectionId, RegistrationSlot};
10953 use subc_protocol::manifest::ModuleManifest;
10954
10955 #[tokio::test]
10956 async fn enable_release_failure_is_failed_and_a_second_enable_retries() {
10957 let registry = Arc::new(Registry::default());
10958 let supervisor = Supervisor::new_for_test(Arc::clone(®istry), RestartPolicy::default());
10959 let runtime = supervisor.runtime_config();
10960 let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::disabled()));
10961 let spec = ModuleSpec {
10962 module_id: "enable-stale-registration".to_string(),
10963 program: PathBuf::from("/missing/enable-retry-test"),
10964 args: Vec::new(),
10965 env: Vec::new(),
10966 reserved: false,
10967 reserved_prefixes: Vec::new(),
10968 protocol: ModuleProtocol::Subc,
10969 overlap: Default::default(),
10970 };
10971 let connection = ConnectionId::new(90);
10972 registry
10973 .register_with_control_ops(
10974 ModuleManifest::builder(&spec.module_id, "0.1.0").build(),
10975 1,
10976 connection,
10977 Vec::new(),
10978 )
10979 .unwrap();
10980 let mut child = None;
10981 let err = set_child_enabled(
10982 &spec,
10983 &runtime,
10984 ®istry,
10985 &supervisor.process_liveness,
10986 &snapshot,
10987 &mut child,
10988 true,
10989 )
10990 .await
10991 .unwrap_err();
10992 assert!(matches!(
10993 err,
10994 SuperviseError::RegistrationStillActive { .. }
10995 ));
10996 assert_eq!(lock_snapshot(&snapshot).unwrap().state, ModuleState::Failed);
10997 assert!(child.is_none());
10998 registry.deregister_connection(connection).unwrap();
10999 let err = set_child_enabled(
11000 &spec,
11001 &runtime,
11002 ®istry,
11003 &supervisor.process_liveness,
11004 &snapshot,
11005 &mut child,
11006 true,
11007 )
11008 .await
11009 .unwrap_err();
11010 assert!(
11011 matches!(err, SuperviseError::Spawn { .. }),
11012 "second enable must attempt a spawn: {err}"
11013 );
11014 assert_eq!(lock_snapshot(&snapshot).unwrap().state, ModuleState::Failed);
11015 }
11016
11017 const INCUMBENT: u64 = 1;
11018 const CANDIDATE: u64 = 2;
11019
11020 fn swapped_registry() -> Arc<Registry> {
11021 let registry = Arc::new(Registry::default());
11022 let manifest = ModuleManifest::builder("m", "0.1.0").build();
11023 registry
11024 .register_with_control_ops(
11025 manifest.clone(),
11026 1,
11027 ConnectionId::new(INCUMBENT),
11028 Vec::new(),
11029 )
11030 .unwrap();
11031 registry
11032 .register_candidate_with_control_ops(
11033 manifest,
11034 1,
11035 ConnectionId::new(CANDIDATE),
11036 Vec::new(),
11037 )
11038 .unwrap();
11039 registry
11040 }
11041
11042 #[tokio::test]
11046 async fn incumbent_release_is_awaited_by_connection_not_by_module_id() {
11047 let registry = swapped_registry();
11048 registry.promote_candidate("m").unwrap().unwrap();
11049
11050 assert!(matches!(
11051 wait_for_registration_release(®istry, "m", Duration::from_millis(50)).await,
11052 Err(SuperviseError::RegistrationStillActive { .. })
11053 ));
11054
11055 assert!(matches!(
11057 wait_for_slot_registration_release(
11058 ®istry,
11059 RegistrationSlot::Connection(ConnectionId::new(INCUMBENT)),
11060 Duration::from_millis(50),
11061 )
11062 .await,
11063 Err(SuperviseError::RegistrationStillActive { .. })
11064 ));
11065
11066 let releaser = Arc::clone(®istry);
11067 let release = tokio::spawn(async move {
11068 sleep(Duration::from_millis(20)).await;
11069 releaser
11070 .deregister_connection(ConnectionId::new(INCUMBENT))
11071 .unwrap();
11072 notify_registration_release();
11073 });
11074 wait_for_slot_registration_release(
11075 ®istry,
11076 RegistrationSlot::Connection(ConnectionId::new(INCUMBENT)),
11077 Duration::from_secs(5),
11078 )
11079 .await
11080 .expect("the incumbent's own registration is released");
11081 release.await.unwrap();
11082 assert!(registry.get_module("m").unwrap().is_some());
11083 }
11084
11085 #[tokio::test]
11088 async fn candidate_slot_wait_ignores_the_incumbents_registration() {
11089 let registry = swapped_registry();
11090 assert!(matches!(
11091 wait_for_slot_registration_release(
11092 ®istry,
11093 RegistrationSlot::Candidate("m"),
11094 Duration::from_millis(50),
11095 )
11096 .await,
11097 Err(SuperviseError::RegistrationStillActive { .. })
11098 ));
11099 registry
11100 .deregister_connection(ConnectionId::new(CANDIDATE))
11101 .unwrap();
11102 wait_for_slot_registration_release(
11103 ®istry,
11104 RegistrationSlot::Candidate("m"),
11105 Duration::from_millis(50),
11106 )
11107 .await
11108 .expect("a candidate slot with no candidate is released");
11109 assert!(registry
11110 .registration(RegistrationSlot::Active("m"))
11111 .unwrap()
11112 .is_some());
11113 }
11114}
11115
11116fn classify_exit(status: &ExitStatus) -> ExitReport {
11117 ExitReport {
11118 kind: if status.success() {
11119 ExitKind::Clean
11120 } else {
11121 ExitKind::Crash
11122 },
11123 code: status.code(),
11124 signal: exit_signal(status),
11125 at_ms: unix_ms_now(),
11126 }
11127}
11128
11129fn wait_error_exit_report() -> ExitReport {
11135 ExitReport {
11136 kind: ExitKind::Crash,
11137 code: None,
11138 signal: None,
11139 at_ms: unix_ms_now(),
11140 }
11141}
11142
11143#[cfg(unix)]
11144fn exit_signal(status: &ExitStatus) -> Option<i32> {
11145 use std::os::unix::process::ExitStatusExt;
11146
11147 status.signal()
11148}
11149
11150#[cfg(not(unix))]
11151fn exit_signal(_status: &ExitStatus) -> Option<i32> {
11152 None
11153}
11154
11155fn reset_restart_count(snapshot: &SharedSnapshot, module_id: &str) -> Result<(), SuperviseError> {
11161 update_snapshot(snapshot, Some(module_id), |state| {
11162 state.clear_crash_restarts();
11163 })
11164}
11165
11166fn set_running(
11167 snapshot: &SharedSnapshot,
11168 child: &SupervisedChild,
11169 module_id: &str,
11170 spawn_events: &SpawnEventFeed,
11171) -> Result<(), SuperviseError> {
11172 let mut state = snapshot.lock().map_err(|_| SuperviseError::StatePoisoned {
11173 module_id: Some(module_id.to_string()),
11174 })?;
11175 state.spawn_generation = spawn_events.emit_spawned(module_id, child.pid, child.spawned_at_ms);
11176 if std::mem::take(&mut state.coalesced_restart_pending) {
11177 let generation = state.spawn_generation;
11178 info!(module_id, "restart already satisfied by generation {generation}; coalesced pending request completed");
11179 }
11180 state.drain_disposition_detail = None;
11181 state.spawn_failure = None;
11182 state.waiting_for_previous_pid = None;
11183 state.waiting_for_previous_detail = None;
11184 state.in_alternate_slot = false;
11187 state.configuration_updated_since_spawn = false;
11188 state.spawned_protocol = Some(child.protocol);
11189 state.state = ModuleState::Running;
11190 state.enabled = true;
11191 state.process_alive = true;
11192 state.pid = child.id();
11193 #[cfg(windows)]
11194 {
11195 state.pipe_consumed = child
11196 .nonce_delivery
11197 .as_ref()
11198 .map(|delivery| delivery.consumed());
11199 }
11200 #[cfg(target_os = "macos")]
11201 {
11202 state.report_ready = Some(Arc::clone(&child.report_ready));
11203 }
11204 state.spawned_at_ms = Some(child.spawned_at_ms);
11205 state.exec_outcome = Some(Arc::clone(&child.exec.0));
11206 state.spawned_from = Some(child.spawned_from.clone());
11207 state.launch_spec_digest = Some(child.launch_spec_digest.clone());
11208 state.spawned_file_identity = child.spawned_file_identity.clone();
11209 state.process_start_time = child.process_start_time;
11210 Ok(())
11211}
11212
11213fn clear_current_process_facts(state: &mut SupervisorSnapshot) {
11214 state.waiting_for_previous_pid = None;
11215 state.waiting_for_previous_detail = None;
11216 state.process_alive = false;
11217 state.spawned_protocol = None;
11218 state.pid = None;
11219 #[cfg(windows)]
11220 {
11221 state.pipe_consumed = None;
11222 }
11223 #[cfg(target_os = "macos")]
11224 {
11225 state.report_ready = None;
11226 }
11227 state.spawned_at_ms = None;
11228 state.exec_outcome = None;
11229 state.spawned_from = None;
11230 state.launch_spec_digest = None;
11231 state.spawned_file_identity = None;
11232 state.process_start_time = None;
11233 state.deliberate_severance = None;
11234}
11235
11236#[cfg(test)]
11237fn record_deliberate_severance(
11238 snapshot: &SharedSnapshot,
11239 identity: ProcessIdentity,
11240) -> Result<(), SuperviseError> {
11241 update_snapshot(snapshot, None, |state| {
11242 state.deliberate_severance = Some(identity);
11243 })
11244}
11245
11246fn apply_deliberate_severance_marker(
11247 snapshot: &SharedSnapshot,
11248 exited_identity: Option<ProcessIdentity>,
11249 mut exit_report: ExitReport,
11250) -> ExitReport {
11251 let marker = lock_snapshot(snapshot)
11252 .ok()
11253 .and_then(|mut state| state.deliberate_severance.take());
11254 if marker.is_some() && marker == exited_identity {
11255 exit_report.kind = ExitKind::DeliberateSeverance;
11256 }
11257 exit_report
11258}
11259
11260fn classify_reaped_child_exit(
11261 snapshot: &SharedSnapshot,
11262 child: &SupervisedChild,
11263 status: &ExitStatus,
11264) -> ExitReport {
11265 let _ = update_snapshot(snapshot, None, |state| {
11266 state.reaped_pid = Some(child.pid);
11267 state.spawn_failure = child.spawn_failure.clone();
11268 });
11269 apply_deliberate_severance_marker(snapshot, child.process_identity(), classify_exit(status))
11270}
11271
11272fn fail_snapshot(
11273 snapshot: &SharedSnapshot,
11274 module_id: Option<&str>,
11275 last_exit: Option<ExitReport>,
11276) {
11277 if let Err(err) = update_snapshot(snapshot, module_id, |state| {
11278 state.state = ModuleState::Failed;
11279 clear_current_process_facts(state);
11280 if let Some(last_exit) = last_exit {
11281 state.last_exit = Some(last_exit);
11282 }
11283 }) {
11284 error!(error = %err, "failed to mark supervisor state failed");
11285 }
11286}
11287
11288fn update_snapshot(
11289 snapshot: &SharedSnapshot,
11290 module_id: Option<&str>,
11291 update: impl FnOnce(&mut SupervisorSnapshot),
11292) -> Result<(), SuperviseError> {
11293 let mut state = snapshot.lock().map_err(|_| SuperviseError::StatePoisoned {
11294 module_id: module_id.map(ToOwned::to_owned),
11295 })?;
11296 update(&mut state);
11297 Ok(())
11298}
11299
11300const SLOW_SNAPSHOT_LOCK_THRESHOLD: Duration = Duration::from_millis(250);
11301
11302fn lock_snapshot_for_control<'a>(
11303 snapshot: &'a SharedSnapshot,
11304 module_id: &str,
11305 caller: &'static str,
11306) -> Result<std::sync::MutexGuard<'a, SupervisorSnapshot>, SuperviseError> {
11307 let started_at = Instant::now();
11308 let guard = lock_snapshot(snapshot)?;
11309 let waited = started_at.elapsed();
11310 if waited >= SLOW_SNAPSHOT_LOCK_THRESHOLD {
11311 warn!(
11312 module_id = %module_id,
11313 waited_ms = waited.as_millis() as u64,
11314 caller = %caller,
11315 "slow snapshot lock"
11316 );
11317 }
11318 Ok(guard)
11319}
11320
11321fn lock_snapshot(
11322 snapshot: &SharedSnapshot,
11323) -> Result<std::sync::MutexGuard<'_, SupervisorSnapshot>, SuperviseError> {
11324 snapshot
11325 .lock()
11326 .map_err(|_| SuperviseError::StatePoisoned { module_id: None })
11327}
11328
11329#[cfg(test)]
11330mod terminal_history_tests {
11331 use std::{
11332 path::PathBuf,
11333 sync::Arc,
11334 time::{Duration, Instant},
11335 };
11336
11337 use tokio::{sync::mpsc, time::sleep};
11338
11339 use super::{
11340 apply_deliberate_severance_marker, daemon_will_restart, drain_child_to_state,
11341 drained_after_quiescence_wait, handle_reload_spawn_failure, health_restart_child,
11342 lock_snapshot, on_child_exit, record_deliberate_severance, record_wait_error_terminal,
11343 reset_restart_count, spawn_and_mark_running, update_snapshot, wait_error_exit_report,
11344 ExitKind, ExitReport, ModuleProtocol, ModuleSpec, ModuleState, NextAction, ProcessIdentity,
11345 RestartPolicy, SpawnEventKind, StopNotice, SuperviseError, SupervisedConfiguration,
11346 SupervisedModule, SupervisedModuleInner, Supervisor, SupervisorHandle,
11347 SupervisorHealthStatus, SupervisorSnapshot,
11348 };
11349 use super::Instant as ClockInstant;
11354 use crate::{
11355 registry::Registry,
11356 terminal_ring::{TerminalRing, TerminalRingConfig},
11357 };
11358 use std::sync::Mutex;
11359 use subc_control::TerminalDisposition;
11360
11361 pub(super) fn fake_aft_stub_path() -> PathBuf {
11366 let mut path = std::env::current_exe().expect("current_exe available in tests");
11367 path.pop();
11368 path.pop();
11369 path.push(if cfg!(windows) {
11370 "fake-aft-stub.exe"
11371 } else {
11372 "fake-aft-stub"
11373 });
11374 assert!(
11375 path.exists(),
11376 "fake-aft-stub not built at {}: run `cargo test -p subc-core` (which builds \
11377 [[bin]] targets) rather than `cargo test -p subc-core --lib` (which does not)",
11378 path.display()
11379 );
11380 path
11381 }
11382
11383 #[test]
11384 fn reserved_never_spawned_refuses_every_hello() {
11385 let supervisor = SupervisorHandle::default();
11390 supervisor.apply_identity_configuration(&ModuleSpec {
11391 module_id: "never-spawned".to_string(),
11392 program: PathBuf::from("/usr/bin/false"),
11393 args: Vec::new(),
11394 env: Vec::new(),
11395 reserved: true,
11396 reserved_prefixes: Vec::new(),
11397 protocol: ModuleProtocol::Subc,
11398 overlap: Default::default(),
11399 });
11400 assert!(
11401 supervisor
11402 .reserved_hello_rejection("never-spawned", Some("any-forged-nonce"))
11403 .is_some(),
11404 "forged nonce must refuse on a reserved never-spawned id"
11405 );
11406 assert!(
11407 supervisor
11408 .reserved_hello_rejection("never-spawned", None)
11409 .is_some(),
11410 "absent nonce must refuse on a reserved never-spawned id"
11411 );
11412 supervisor.set_spawn_nonce("never-spawned", "minted".to_string());
11414 supervisor.apply_identity_configuration(&ModuleSpec {
11415 module_id: "never-spawned".to_string(),
11416 program: PathBuf::from("/usr/bin/false"),
11417 args: Vec::new(),
11418 env: Vec::new(),
11419 reserved: true,
11420 reserved_prefixes: Vec::new(),
11421 protocol: ModuleProtocol::Subc,
11422 overlap: Default::default(),
11423 });
11424 assert!(supervisor
11425 .reserved_hello_rejection("never-spawned", Some("minted"))
11426 .is_none());
11427 assert!(supervisor
11428 .reserved_hello_rejection("never-spawned", Some("forged"))
11429 .is_some());
11430 }
11431
11432 fn seed_crash_restarts(state: &mut SupervisorSnapshot, count: u32) {
11435 let now = ClockInstant::now();
11436 for _ in 0..count {
11437 state.crash_restarts.push_back(now);
11438 }
11439 }
11440
11441 fn age_oldest_crash_restart_out_of_window(state: &mut SupervisorSnapshot, window: Duration) {
11445 let aged = state
11446 .crash_restarts
11447 .front()
11448 .expect("a crash restart must be recorded before it can be aged")
11449 .checked_sub(window + Duration::from_secs(1))
11450 .expect("the test clock is far enough from its origin to age an instant");
11451 state.crash_restarts[0] = aged;
11452 }
11453
11454 fn snapshot_with_restarts(enabled: bool, count: u32) -> SupervisorSnapshot {
11455 let mut state = SupervisorSnapshot::new(ModuleState::Running, enabled);
11456 seed_crash_restarts(&mut state, count);
11457 state
11458 }
11459
11460 #[test]
11461 fn daemon_owned_recovery_predicate_uses_the_pre_increment_budget() {
11462 let policy = RestartPolicy::new(3, Duration::ZERO);
11463 let now = ClockInstant::now();
11464 assert!(daemon_will_restart(
11465 &mut snapshot_with_restarts(true, 2),
11466 &policy,
11467 now
11468 ));
11469 assert!(!daemon_will_restart(
11470 &mut snapshot_with_restarts(true, 3),
11471 &policy,
11472 now
11473 ));
11474 assert!(!daemon_will_restart(
11475 &mut snapshot_with_restarts(false, 0),
11476 &policy,
11477 now
11478 ));
11479 }
11480
11481 #[test]
11482 fn crash_restart_backoff_escalates_with_in_window_count() {
11483 let policy = RestartPolicy::new(4, Duration::from_millis(100))
11484 .with_max_backoff(Duration::from_secs(30));
11485 let now = ClockInstant::now();
11486 let mut state = SupervisorSnapshot::new(ModuleState::Running, true);
11487 let schedules = (0..4)
11488 .map(|_| {
11489 state
11490 .next_crash_restart(&policy, now)
11491 .expect("the test policy allows four crash restarts")
11492 })
11493 .collect::<Vec<_>>();
11494
11495 assert_eq!(
11496 schedules
11497 .iter()
11498 .map(|schedule| schedule.restart_in_window)
11499 .collect::<Vec<_>>(),
11500 vec![0, 1, 2, 3]
11501 );
11502 assert_eq!(
11503 schedules
11504 .iter()
11505 .map(|schedule| schedule.delay)
11506 .collect::<Vec<_>>(),
11507 vec![
11508 Duration::from_millis(100),
11509 Duration::from_secs(1),
11510 Duration::from_secs(10),
11511 Duration::from_secs(30),
11512 ]
11513 );
11514 }
11515
11516 #[test]
11517 fn crash_restart_backoff_resets_after_ring_clear() {
11518 let policy = RestartPolicy::new(3, Duration::from_millis(100));
11519 let now = ClockInstant::now();
11520 let mut state = SupervisorSnapshot::new(ModuleState::Running, true);
11521 assert_eq!(
11522 state.next_crash_restart(&policy, now).unwrap().delay,
11523 Duration::from_millis(100)
11524 );
11525 assert_eq!(
11526 state.next_crash_restart(&policy, now).unwrap().delay,
11527 Duration::from_secs(1)
11528 );
11529
11530 state.clear_crash_restarts();
11531 let schedule = state
11532 .next_crash_restart(&policy, now)
11533 .expect("a cleared ring must allow another restart");
11534 assert_eq!(schedule.restart_in_window, 0);
11535 assert_eq!(schedule.delay, Duration::from_millis(100));
11536 }
11537
11538 #[test]
11539 fn crash_restart_backoff_ignores_aged_restarts() {
11540 let policy = RestartPolicy::new(3, Duration::from_millis(100));
11541 let now = ClockInstant::now();
11542 let mut state = SupervisorSnapshot::new(ModuleState::Running, true);
11543 state
11544 .next_crash_restart(&policy, now)
11545 .expect("the first restart is allowed");
11546 state
11547 .next_crash_restart(&policy, now)
11548 .expect("the second restart is allowed");
11549 state.crash_restarts[0] = now
11550 .checked_sub(policy.window + Duration::from_secs(1))
11551 .expect("the fake clock can age a restart past the window");
11552
11553 let schedule = state
11554 .next_crash_restart(&policy, now)
11555 .expect("an aged restart must release its slot");
11556 assert_eq!(schedule.restart_in_window, 1);
11557 assert_eq!(schedule.delay, Duration::from_secs(1));
11558 assert_eq!(state.crash_restarts.len(), 2);
11559 }
11560
11561 #[test]
11565 fn a_budget_spent_before_the_window_no_longer_refuses() {
11566 let policy = RestartPolicy::new(3, Duration::ZERO);
11567 let mut state = snapshot_with_restarts(true, 3);
11568 let now = ClockInstant::now();
11569 assert!(!daemon_will_restart(&mut state, &policy, now));
11570
11571 assert!(daemon_will_restart(
11572 &mut state,
11573 &policy,
11574 now + policy.window + Duration::from_secs(1)
11575 ));
11576 assert!(
11577 state.crash_restarts.is_empty(),
11578 "reading the budget must drop the instants that left the window"
11579 );
11580 }
11581
11582 fn module_with_recovery_snapshot(
11583 state: ModuleState,
11584 enabled: bool,
11585 restart_count: u32,
11586 ) -> SupervisedModule {
11587 let registry = Arc::new(Registry::default());
11588 let supervisor =
11589 Supervisor::new_for_test(Arc::clone(®istry), RestartPolicy::new(3, Duration::ZERO));
11590 let runtime = supervisor.runtime_config();
11591 let spec = ModuleSpec {
11592 module_id: "recovery-snapshot".to_string(),
11593 program: fake_aft_stub_path(),
11594 args: Vec::new(),
11595 env: Vec::new(),
11596 reserved: false,
11597 reserved_prefixes: Vec::new(),
11598 protocol: ModuleProtocol::Subc,
11599 overlap: Default::default(),
11600 };
11601 let mut snapshot = SupervisorSnapshot::new(state, enabled);
11602 seed_crash_restarts(&mut snapshot, restart_count);
11603 let (commands, _rx) = mpsc::channel(4);
11606 SupervisedModule {
11607 inner: Arc::new(SupervisedModuleInner {
11608 module_id: spec.module_id.clone(),
11609 registry,
11610 snapshot: Arc::new(Mutex::new(snapshot)),
11611 configuration: Arc::new(Mutex::new(SupervisedConfiguration {
11612 spec,
11613 health: runtime.health,
11614 })),
11615 stderr_ring: runtime.stderr_ring,
11616 terminal_ring: runtime.terminal_ring,
11617 commands,
11618 monitor: Mutex::new(None),
11619 restart_policy: runtime.restart_policy,
11620 effective_drain_timeout: runtime.effective_drain_timeout,
11621 provenance_probe: supervisor.provenance_probe.clone(),
11622 }),
11623 }
11624 }
11625
11626 #[cfg(target_os = "linux")]
11627 #[tokio::test]
11628 async fn no_cgroup_placement_does_not_block_fake_aft_stub_spawn() {
11629 let supervisor =
11630 Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default())
11631 .with_cgroup_placement(None);
11632 let result = supervisor.spawn(ModuleSpec {
11633 module_id: "no-cgroup-placement".to_string(),
11634 program: fake_aft_stub_path(),
11635 args: Vec::new(),
11636 env: Vec::new(),
11637 reserved: false,
11638 reserved_prefixes: Vec::new(),
11639 protocol: ModuleProtocol::Subc,
11640 overlap: Default::default(),
11641 });
11642
11643 assert!(
11644 result.is_ok(),
11645 "no delegation must not turn an otherwise valid spawn into a failure: {result:?}"
11646 );
11647 }
11648
11649 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
11650 async fn undecided_snapshot_uses_shared_restart_predicate() {
11651 assert!(module_with_recovery_snapshot(ModuleState::Running, true, 2)
11652 .will_recover_after_connection_loss()
11653 .unwrap());
11654 assert!(
11655 !module_with_recovery_snapshot(ModuleState::Running, true, 3)
11656 .will_recover_after_connection_loss()
11657 .unwrap()
11658 );
11659 }
11660
11661 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
11662 async fn restarting_snapshot_at_exhausted_budget_is_non_terminal() {
11663 assert!(
11664 module_with_recovery_snapshot(ModuleState::Restarting, true, 3)
11665 .will_recover_after_connection_loss()
11666 .unwrap()
11667 );
11668 }
11669
11670 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
11671 async fn terminal_phase_snapshots_are_terminal_before_budget_exhaustion() {
11672 assert!(!module_with_recovery_snapshot(ModuleState::Failed, true, 0)
11673 .will_recover_after_connection_loss()
11674 .unwrap());
11675 assert!(
11676 !module_with_recovery_snapshot(ModuleState::Disabled, true, 0)
11677 .will_recover_after_connection_loss()
11678 .unwrap()
11679 );
11680 }
11681
11682 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
11683 async fn warming_snapshot_is_limited_to_startup_phases() {
11684 for state in [
11685 ModuleState::Starting,
11686 ModuleState::Running,
11687 ModuleState::Restarting,
11688 ] {
11689 assert!(
11690 module_with_recovery_snapshot(state, true, 0)
11691 .is_warming()
11692 .unwrap(),
11693 "{state:?} should be warming"
11694 );
11695 }
11696 for state in [
11697 ModuleState::Unresponsive,
11698 ModuleState::Draining,
11699 ModuleState::Stopped,
11700 ModuleState::Failed,
11701 ModuleState::Disabled,
11702 ] {
11703 assert!(
11704 !module_with_recovery_snapshot(state, true, 0)
11705 .is_warming()
11706 .unwrap(),
11707 "{state:?} should not be warming"
11708 );
11709 }
11710 }
11711
11712 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
11713 async fn terminal_history_survives_respawn_and_keeps_both_crashes_in_order() {
11714 let registry = Arc::new(Registry::default());
11715 let supervisor =
11716 Supervisor::new_for_test(Arc::clone(®istry), RestartPolicy::new(1, Duration::ZERO));
11717 let module = supervisor
11718 .spawn(ModuleSpec {
11719 module_id: "terminal-history".to_string(),
11720 program: fake_aft_stub_path(),
11721 args: Vec::new(),
11722 env: vec![("FAKE_AFT_EXIT_CODE".to_string(), "23".to_string())],
11723 reserved: false,
11724 reserved_prefixes: Vec::new(),
11725 protocol: ModuleProtocol::Subc,
11726 overlap: Default::default(),
11727 })
11728 .unwrap();
11729
11730 let deadline = Instant::now() + Duration::from_secs(5);
11731 loop {
11732 let history = module.terminal_history();
11733 if history.entries.len() == 2 {
11734 assert_eq!(module.status().unwrap().state, ModuleState::Failed);
11735 assert_eq!(history.dropped, 0);
11736 assert_eq!(
11737 history
11738 .entries
11739 .iter()
11740 .map(|entry| entry.exit_code)
11741 .collect::<Vec<_>>(),
11742 vec![Some(23), Some(23)]
11743 );
11744 assert!(history.entries[0].at_ms <= history.entries[1].at_ms);
11745 return;
11746 }
11747 assert!(
11748 Instant::now() < deadline,
11749 "module did not retain two terminal exits: {history:?}"
11750 );
11751 sleep(Duration::from_millis(10)).await;
11752 }
11753 }
11754
11755 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
11759 async fn disable_during_crash_backoff_cancels_pending_respawn() {
11760 let backoff = Duration::from_secs(2);
11761 let supervisor = Supervisor::new_for_test(
11762 Arc::new(Registry::default()),
11763 RestartPolicy::new(10, backoff),
11764 );
11765 let module = supervisor
11766 .spawn(ModuleSpec {
11767 module_id: "disable-during-backoff".to_string(),
11768 program: fake_aft_stub_path(),
11769 args: Vec::new(),
11770 env: vec![("FAKE_AFT_EXIT_CODE".to_string(), "23".to_string())],
11771 reserved: false,
11772 reserved_prefixes: Vec::new(),
11773 protocol: ModuleProtocol::Subc,
11774 overlap: Default::default(),
11775 })
11776 .unwrap();
11777
11778 let deadline = Instant::now() + Duration::from_secs(5);
11780 loop {
11781 if module.status().unwrap().state == ModuleState::Restarting {
11782 break;
11783 }
11784 assert!(
11785 Instant::now() < deadline,
11786 "module never entered the crash backoff"
11787 );
11788 sleep(Duration::from_millis(10)).await;
11789 }
11790
11791 let started = Instant::now();
11792 module.set_enabled(false).await.unwrap();
11793 let waited = started.elapsed();
11794
11795 assert!(
11796 waited < backoff / 2,
11797 "disable waited {waited:?} behind the {backoff:?} crash backoff; the operator command must preempt the pending respawn"
11798 );
11799 assert_eq!(module.status().unwrap().state, ModuleState::Disabled);
11800
11801 sleep(backoff + Duration::from_millis(500)).await;
11803 let status = module.status().unwrap();
11804 assert_eq!(status.state, ModuleState::Disabled);
11805 assert_eq!(
11806 status.spawn_generation, 1,
11807 "module respawned after the operator disabled it"
11808 );
11809 }
11810
11811 #[cfg(unix)]
11814 fn protocol_none_sigterm_exits_clean_spec(
11815 module_id: &str,
11816 dir: &std::path::Path,
11817 ) -> (ModuleSpec, PathBuf, PathBuf) {
11818 let ready = dir.join("ready");
11819 let marker = dir.join("sigterm");
11820 let spec = ModuleSpec {
11821 module_id: module_id.to_string(),
11822 program: fake_aft_stub_path(),
11823 args: Vec::new(),
11824 env: vec![
11825 ("FAKE_AFT_NEVER_CONNECT".to_string(), "1".to_string()),
11826 (
11827 "FAKE_AFT_SIGTERM_MARKER_PATH".to_string(),
11828 marker.display().to_string(),
11829 ),
11830 (
11831 "FAKE_AFT_NEVER_CONNECT_READY_PATH".to_string(),
11832 ready.display().to_string(),
11833 ),
11834 ],
11835 reserved: false,
11836 reserved_prefixes: Vec::new(),
11837 protocol: ModuleProtocol::None,
11838 overlap: Default::default(),
11839 };
11840 (spec, ready, marker)
11841 }
11842
11843 #[cfg(unix)]
11847 async fn wait_for_file(path: &std::path::Path) {
11848 let deadline = Instant::now() + Duration::from_secs(10);
11849 while !path.exists() {
11850 assert!(
11851 Instant::now() < deadline,
11852 "{} never appeared",
11853 path.display()
11854 );
11855 sleep(Duration::from_millis(10)).await;
11856 }
11857 }
11858
11859 #[cfg(unix)]
11863 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
11864 async fn protocol_none_unrequested_clean_exit_restarts_as_a_crash() {
11865 let dir = cortexkit_test_support::ScratchDir::new("none-unrequested-clean-exit");
11866 let (spec, ready, marker) =
11867 protocol_none_sigterm_exits_clean_spec("none-unrequested-clean-exit", dir.path());
11868 let supervisor = Supervisor::new_for_test(
11869 Arc::new(Registry::default()),
11870 RestartPolicy::new(3, Duration::ZERO),
11871 );
11872 let module = supervisor.spawn(spec).unwrap();
11873 wait_for_file(&ready).await;
11874 let deadline = Instant::now() + Duration::from_secs(10);
11878 let first_pid = loop {
11879 let status = module.status().unwrap();
11880 if status.state == ModuleState::Running {
11881 if let Some(pid) = status.pid {
11882 break pid;
11883 }
11884 }
11885 assert!(
11886 Instant::now() < deadline,
11887 "a running module must report its pid after exec confirmation: {status:?}"
11888 );
11889 sleep(Duration::from_millis(10)).await;
11890 };
11891
11892 rustix::process::kill_process(
11893 rustix::process::Pid::from_raw(i32::try_from(first_pid).unwrap()).unwrap(),
11894 rustix::process::Signal::TERM,
11895 )
11896 .unwrap();
11897
11898 let deadline = Instant::now() + Duration::from_secs(10);
11899 let respawned = loop {
11900 let status = module.status().unwrap();
11901 if status.state == ModuleState::Running
11902 && status.pid.is_some_and(|pid| pid != first_pid)
11903 {
11904 break status;
11905 }
11906 assert!(
11907 Instant::now() < deadline,
11908 "protocol-none module was not respawned after an unrequested clean exit: {status:?}"
11909 );
11910 sleep(Duration::from_millis(10)).await;
11911 };
11912 assert_eq!(respawned.spawn_generation, 2);
11913 assert!(
11914 marker.exists(),
11915 "the child must have exited through its SIGTERM handler (exit 0), or this proves nothing about clean exits"
11916 );
11917
11918 let history = module.terminal_history();
11919 assert_eq!(history.entries.len(), 1, "{history:?}");
11920 let entry = &history.entries[0];
11921 assert_eq!(entry.exit_code, Some(0));
11922 assert_eq!(entry.exit_kind, subc_control::TerminalExitKind::Clean);
11923 assert_eq!(entry.disposition, TerminalDisposition::Restarting);
11924
11925 module.stop().await.unwrap();
11926 }
11927
11928 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
11932 async fn protocol_none_repeated_clean_exits_exhaust_the_restart_budget() {
11933 let supervisor = Supervisor::new_for_test(
11934 Arc::new(Registry::default()),
11935 RestartPolicy::new(1, Duration::ZERO),
11936 );
11937 let module = supervisor
11938 .spawn(ModuleSpec {
11939 module_id: "none-clean-exit-budget".to_string(),
11940 program: fake_aft_stub_path(),
11941 args: Vec::new(),
11942 env: vec![("FAKE_AFT_EXIT_CODE".to_string(), "0".to_string())],
11943 reserved: false,
11944 reserved_prefixes: Vec::new(),
11945 protocol: ModuleProtocol::None,
11946 overlap: Default::default(),
11947 })
11948 .unwrap();
11949
11950 let deadline = Instant::now() + Duration::from_secs(10);
11953 loop {
11954 let status = module.status().unwrap();
11955 if status.state == ModuleState::Failed {
11956 break;
11957 }
11958 assert!(
11959 Instant::now() < deadline,
11960 "module never exhausted its budget: {status:?} {:?}",
11961 module.terminal_history()
11962 );
11963 sleep(Duration::from_millis(10)).await;
11964 }
11965 let history = module.terminal_history();
11966 assert_eq!(
11967 history
11968 .entries
11969 .iter()
11970 .map(|entry| (entry.exit_code, entry.disposition.clone()))
11971 .collect::<Vec<_>>(),
11972 vec![
11973 (Some(0), TerminalDisposition::Restarting),
11974 (Some(0), TerminalDisposition::Failed),
11975 ]
11976 );
11977 let detail = history.entries[1]
11978 .disposition_detail
11979 .as_deref()
11980 .expect("a budget failure names the budget");
11981 assert!(detail.contains("max_restarts=1"), "{detail}");
11982 assert_eq!(module.status().unwrap().spawn_generation, 2);
11983 }
11984
11985 #[test]
11986 fn restart_budget_failure_is_published_after_its_terminal_record() {
11987 use std::io::{Read, Write};
11988 let supervisor = Supervisor::new_for_test(
11989 Arc::new(Registry::default()),
11990 RestartPolicy::new(0, Duration::ZERO),
11991 );
11992 let runtime = supervisor.runtime_config();
11993 let spec = ModuleSpec {
11994 protocol: ModuleProtocol::None,
11995 ..windowed_crash_spec("budget-publication-order")
11996 };
11997 let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::new(
11998 ModuleState::Running,
11999 true,
12000 )));
12001 lock_snapshot(&snapshot).unwrap().process_alive = true;
12002 let reaped_snapshot = snapshot.clone();
12003 let mut barrier = terminal_record_barrier(&runtime.spawn_events);
12004 let history = runtime.terminal_ring.clone();
12005 let reap = std::thread::spawn(move || {
12006 tokio::runtime::Builder::new_current_thread()
12007 .enable_all()
12008 .build()
12009 .unwrap()
12010 .block_on(on_child_exit(
12011 &spec,
12012 runtime.restart_policy,
12013 &supervisor.registry,
12014 &reaped_snapshot,
12015 &runtime.terminal_ring,
12016 &runtime.spawn_events,
12017 &runtime.child_roster,
12018 ExitReport {
12019 kind: ExitKind::Clean,
12020 code: Some(0),
12021 signal: None,
12022 at_ms: 1,
12023 },
12024 ))
12025 });
12026 let mut reached = [0];
12027 barrier.read_exact(&mut reached).unwrap();
12028 assert_eq!(&reached, b"R");
12029 let (before_state, before_alive) = {
12030 let state = lock_snapshot(&snapshot).unwrap();
12031 (state.state, state.process_alive)
12032 };
12033 let before_history = history.lock().unwrap().snapshot();
12034 barrier.write_all(b"X").unwrap();
12035 assert!(matches!(reap.join().unwrap(), NextAction::Stop { .. }));
12036 assert!(before_history.entries.is_empty());
12037 assert_ne!(
12038 before_state,
12039 ModuleState::Failed,
12040 "Failed was visible before its terminal record could be written"
12041 );
12042 assert!(before_alive);
12043 assert_eq!(lock_snapshot(&snapshot).unwrap().state, ModuleState::Failed);
12044 let history = history.lock().unwrap().snapshot();
12045 assert_eq!(history.entries.len(), 1);
12046 assert_eq!(history.entries[0].exit_code, Some(0));
12047 assert_eq!(history.entries[0].disposition, TerminalDisposition::Failed);
12048 }
12049
12050 #[test]
12051 fn crash_restart_at_budget_limit_remains_recoverable_while_recording() {
12052 use std::io::{Read, Write};
12053
12054 let supervisor = Supervisor::new_for_test(
12055 Arc::new(Registry::default()),
12056 RestartPolicy::new(3, Duration::ZERO),
12057 );
12058 let mut runtime = supervisor.runtime_config();
12059 let module = module_with_recovery_snapshot(ModuleState::Running, true, 2);
12060 let snapshot = module.inner.snapshot.clone();
12061 lock_snapshot(&snapshot).unwrap().process_alive = true;
12062 runtime.terminal_ring = module.inner.terminal_ring.clone();
12063 let spec = windowed_crash_spec(module.module_id());
12064 let mut barrier = terminal_record_barrier(&runtime.spawn_events);
12065 let reaped_snapshot = snapshot.clone();
12066 let reap = std::thread::spawn(move || {
12067 tokio::runtime::Builder::new_current_thread()
12068 .enable_all()
12069 .build()
12070 .unwrap()
12071 .block_on(on_child_exit(
12072 &spec,
12073 runtime.restart_policy,
12074 &supervisor.registry,
12075 &reaped_snapshot,
12076 &runtime.terminal_ring,
12077 &runtime.spawn_events,
12078 &runtime.child_roster,
12079 ExitReport {
12080 kind: ExitKind::Crash,
12081 code: Some(2),
12082 signal: None,
12083 at_ms: 1,
12084 },
12085 ))
12086 });
12087
12088 let mut reached = [0];
12092 barrier.read_exact(&mut reached).unwrap();
12093 assert_eq!(&reached, b"R");
12094 let before_status = module.status().unwrap();
12095 let before_history = module.terminal_history();
12096 let before_recovers = module.will_recover_after_connection_loss().unwrap();
12097 barrier.write_all(b"X").unwrap();
12098 assert!(matches!(reap.join().unwrap(), NextAction::Restart { .. }));
12099
12100 assert_eq!(before_status.state, ModuleState::Running);
12101 assert!(before_status.process_alive);
12102 assert_eq!(before_status.restart_count, 3);
12103 assert!(before_history.entries.is_empty());
12104 assert!(
12105 before_recovers,
12106 "connection cleanup would report a terminal close for a scheduled crash restart"
12107 );
12108 let restarting = module.status().unwrap();
12109 assert_eq!(restarting.state, ModuleState::Restarting);
12110 assert_eq!(restarting.restart_count, 3);
12111 assert!(module.will_recover_after_connection_loss().unwrap());
12112 let history = module.terminal_history();
12113 assert_eq!(history.entries.len(), 1);
12114 assert_eq!(
12115 history.entries[0].disposition,
12116 TerminalDisposition::Restarting
12117 );
12118
12119 update_snapshot(&snapshot, Some(module.module_id()), |state| {
12122 state.state = ModuleState::Running;
12123 state.respawn_pending = false;
12124 })
12125 .unwrap();
12126 assert!(!module.will_recover_after_connection_loss().unwrap());
12127 update_snapshot(&snapshot, Some(module.module_id()), |state| {
12128 state.enabled = false;
12129 state.respawn_pending = true;
12130 })
12131 .unwrap();
12132 assert!(!module.will_recover_after_connection_loss().unwrap());
12133 }
12134
12135 fn terminal_record_barrier(events: &super::SpawnEventFeed) -> std::net::TcpStream {
12136 use std::net::{TcpListener, TcpStream};
12137 let listener = TcpListener::bind("127.0.0.1:0").unwrap();
12138 let writer = TcpStream::connect(listener.local_addr().unwrap()).unwrap();
12139 let (barrier, _) = listener.accept().unwrap();
12140 for stream in [&writer, &barrier] {
12141 stream
12142 .set_read_timeout(Some(Duration::from_secs(10)))
12143 .unwrap();
12144 stream
12145 .set_write_timeout(Some(Duration::from_secs(10)))
12146 .unwrap();
12147 }
12148 *events.1.lock().unwrap() = Some(writer);
12149 barrier
12150 }
12151
12152 pub(super) fn before_terminal_record(events: &super::SpawnEventFeed) {
12153 use std::io::{Read, Write};
12154 let barrier = events.1.lock().unwrap().take();
12155 if let Some(mut barrier) = barrier {
12156 barrier.write_all(b"R").unwrap();
12157 let mut release = [0];
12158 barrier.read_exact(&mut release).unwrap();
12159 assert_eq!(&release, b"X");
12160 }
12161 }
12162
12163 #[test]
12164 fn health_restart_budget_failure_is_published_after_its_terminal_record() {
12165 use std::io::{Read, Write};
12166
12167 let supervisor = Supervisor::new_for_test(
12168 Arc::new(Registry::default()),
12169 RestartPolicy::new(0, Duration::ZERO),
12170 );
12171 let mut runtime = supervisor.runtime_config();
12172 let module = module_with_recovery_snapshot(ModuleState::Running, true, 0);
12173 let snapshot = module.inner.snapshot.clone();
12174 runtime.terminal_ring = module.inner.terminal_ring.clone();
12175 let spec = ModuleSpec {
12176 module_id: module.module_id().into(),
12177 program: fake_aft_stub_path(),
12178 protocol: ModuleProtocol::None,
12179 env: vec![("FAKE_AFT_EXIT_CODE".into(), "17".into())],
12180 ..windowed_crash_spec(module.module_id())
12181 };
12182 let mut barrier = terminal_record_barrier(&runtime.spawn_events);
12183 let reap = std::thread::spawn(move || {
12184 tokio::runtime::Builder::new_current_thread()
12185 .enable_all()
12186 .build()
12187 .unwrap()
12188 .block_on(async {
12189 let mut child =
12190 Some(spawn_and_mark_running(&spec, &runtime, &snapshot).unwrap());
12191 health_restart_child(
12192 &spec,
12193 &runtime,
12194 &supervisor.registry,
12195 &supervisor.process_liveness,
12196 &snapshot,
12197 &mut child,
12198 SupervisorHealthStatus::Failing,
12199 Some("health probe did not answer"),
12200 1,
12201 )
12202 .await
12203 .unwrap();
12204 assert!(child.is_none());
12205 });
12206 });
12207 let mut reached = [0];
12212 barrier.read_exact(&mut reached).unwrap();
12213 assert_eq!(&reached, b"R");
12214 let before_status = module.status().unwrap();
12215 let before_history = module.terminal_history();
12216 barrier.write_all(b"X").unwrap();
12217 reap.join().unwrap();
12218 assert!(before_history.entries.is_empty());
12219 assert_ne!(
12220 before_status.state,
12221 ModuleState::Failed,
12222 "Failed was visible before its terminal record could be written"
12223 );
12224 assert!(
12225 before_status.process_alive,
12226 "process death was visible before its terminal record could be written"
12227 );
12228 let failed = module.status().unwrap();
12229 assert_eq!(failed.state, ModuleState::Failed);
12230 assert!(!failed.process_alive);
12231 let history = module.terminal_history();
12232 assert_eq!(history.entries.len(), 1);
12233 assert_eq!(history.entries[0].disposition, TerminalDisposition::Failed);
12234 assert_eq!(
12235 history.entries[0].disposition_detail.as_deref(),
12236 Some("crash budget exhausted: max_restarts=0 within window_secs=600")
12237 );
12238 }
12239
12240 #[test]
12241 fn observed_exits_publish_process_death_after_terminal_history() {
12242 use std::io::{Read, Write};
12243 for (kind, enabled, shutdown, wait_error, expected_state) in [
12244 (ExitKind::Clean, true, false, false, ModuleState::Stopped),
12245 (ExitKind::Crash, true, false, false, ModuleState::Restarting),
12246 (ExitKind::Crash, false, false, false, ModuleState::Disabled),
12247 (
12248 ExitKind::DeliberateSeverance,
12249 true,
12250 false,
12251 false,
12252 ModuleState::Restarting,
12253 ),
12254 #[cfg(unix)]
12255 (ExitKind::Crash, true, true, false, ModuleState::Stopped),
12256 (ExitKind::Crash, true, false, true, ModuleState::Failed),
12257 ] {
12258 let supervisor = Supervisor::new_for_test(
12259 Arc::new(Registry::default()),
12260 RestartPolicy::new(1, Duration::ZERO),
12261 );
12262 let runtime = supervisor.runtime_config();
12263 #[cfg(unix)]
12264 if shutdown {
12265 runtime.child_roster.close();
12266 }
12267 #[cfg(not(unix))]
12268 let _ = shutdown;
12269 let spec = windowed_crash_spec("exit-publication-order");
12270 let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::new(
12271 ModuleState::Running,
12272 enabled,
12273 )));
12274 lock_snapshot(&snapshot).unwrap().process_alive = true;
12275 let reaped_snapshot = snapshot.clone();
12276 let history = runtime.terminal_ring.clone();
12277 let mut barrier = terminal_record_barrier(&runtime.spawn_events);
12278 let reap = std::thread::spawn(move || {
12279 if wait_error {
12280 super::fail_after_wait_error(
12281 &spec.module_id,
12282 &reaped_snapshot,
12283 &runtime.terminal_ring,
12284 &runtime.spawn_events,
12285 "wait failed".into(),
12286 );
12287 } else {
12288 tokio::runtime::Builder::new_current_thread()
12289 .enable_all()
12290 .build()
12291 .unwrap()
12292 .block_on(on_child_exit(
12293 &spec,
12294 runtime.restart_policy,
12295 &supervisor.registry,
12296 &reaped_snapshot,
12297 &runtime.terminal_ring,
12298 &runtime.spawn_events,
12299 &runtime.child_roster,
12300 ExitReport {
12301 kind,
12302 code: Some(0),
12303 signal: None,
12304 at_ms: 1,
12305 },
12306 ));
12307 }
12308 });
12309 let mut reached = [0];
12310 barrier.read_exact(&mut reached).unwrap();
12311 assert_eq!(&reached, b"R");
12312 let before_alive = lock_snapshot(&snapshot).unwrap().process_alive;
12313 let before_history = history.lock().unwrap().snapshot();
12314 barrier.write_all(b"X").unwrap();
12315 reap.join().unwrap();
12316 assert!(
12317 before_alive,
12318 "{kind:?}: death was published before recording"
12319 );
12320 assert!(before_history.entries.is_empty());
12321 assert_eq!(lock_snapshot(&snapshot).unwrap().state, expected_state);
12322 assert!(!lock_snapshot(&snapshot).unwrap().process_alive);
12323 assert_eq!(history.lock().unwrap().snapshot().entries.len(), 1);
12324 }
12325 }
12326
12327 #[tokio::test]
12328 async fn health_restart_budget_failure_remains_failed_when_journal_append_fails() {
12329 let dir = cortexkit_test_support::ScratchDir::new("health-budget-journal-failure");
12330 let path = dir.join("terminals.jsonl");
12331 std::fs::create_dir(&path).unwrap();
12332 let supervisor = Supervisor::new_for_test(
12333 Arc::new(Registry::default()),
12334 RestartPolicy::new(0, Duration::ZERO),
12335 )
12336 .with_terminal_journal(path, "health-budget-journal-failure".into());
12337 let runtime = supervisor.runtime_config();
12338 let spec = ModuleSpec {
12339 program: fake_aft_stub_path(),
12340 protocol: ModuleProtocol::None,
12341 env: vec![("FAKE_AFT_EXIT_CODE".into(), "17".into())],
12342 ..windowed_crash_spec("health-budget-journal-failure")
12343 };
12344 let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::starting()));
12345 let mut child = Some(spawn_and_mark_running(&spec, &runtime, &snapshot).unwrap());
12346 health_restart_child(
12347 &spec,
12348 &runtime,
12349 &supervisor.registry,
12350 &supervisor.process_liveness,
12351 &snapshot,
12352 &mut child,
12353 SupervisorHealthStatus::Failing,
12354 None,
12355 1,
12356 )
12357 .await
12358 .unwrap();
12359 assert_eq!(lock_snapshot(&snapshot).unwrap().state, ModuleState::Failed);
12360 assert!(!lock_snapshot(&snapshot).unwrap().process_alive);
12361 let history = runtime
12362 .terminal_ring
12363 .lock()
12364 .unwrap()
12365 .durable_history(&spec.module_id);
12366 assert!(history.journal_write_failures > 0);
12367 assert_eq!(history.entries.len(), 1);
12368 assert_eq!(history.entries[0].disposition, TerminalDisposition::Failed);
12369 }
12370
12371 #[test]
12372 fn health_restart_budget_failure_remains_failed_when_exit_recording_panics() {
12373 let supervisor = Supervisor::new_for_test(
12374 Arc::new(Registry::default()),
12375 RestartPolicy::new(0, Duration::ZERO),
12376 );
12377 let runtime = supervisor.runtime_config();
12378 let spec = ModuleSpec {
12379 program: fake_aft_stub_path(),
12380 protocol: ModuleProtocol::None,
12381 env: vec![("FAKE_AFT_EXIT_CODE".into(), "17".into())],
12382 ..windowed_crash_spec("health-budget-recording-panic")
12383 };
12384 let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::starting()));
12385 let result = std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| {
12386 tokio::runtime::Builder::new_current_thread()
12387 .enable_all()
12388 .build()
12389 .unwrap()
12390 .block_on(async {
12391 let mut child =
12392 Some(spawn_and_mark_running(&spec, &runtime, &snapshot).unwrap());
12393 runtime.spawn_events.0.lock().unwrap().seq = u64::MAX;
12394 health_restart_child(
12395 &spec,
12396 &runtime,
12397 &supervisor.registry,
12398 &supervisor.process_liveness,
12399 &snapshot,
12400 &mut child,
12401 SupervisorHealthStatus::Failing,
12402 None,
12403 1,
12404 )
12405 .await
12406 .unwrap();
12407 });
12408 }));
12409 let panic = result.expect_err("recording must still unwind");
12410 assert_eq!(
12411 panic.downcast_ref::<String>().map(String::as_str),
12412 Some("spawn event sequence exhausted")
12413 );
12414 let state = lock_snapshot(&snapshot).unwrap();
12415 assert_eq!(state.state, ModuleState::Failed);
12416 assert!(!state.process_alive);
12417 assert!(state.last_exit.is_some());
12418 }
12419
12420 #[tokio::test]
12421 async fn restart_budget_failure_remains_failed_when_journal_append_fails() {
12422 let dir = cortexkit_test_support::ScratchDir::new("budget-journal-failure");
12423 let path = dir.join("terminals.jsonl");
12424 std::fs::create_dir(&path).unwrap();
12425 let supervisor = Supervisor::new_for_test(
12426 Arc::new(Registry::default()),
12427 RestartPolicy::new(0, Duration::ZERO),
12428 )
12429 .with_terminal_journal(path, "budget-journal-failure".into());
12430 let runtime = supervisor.runtime_config();
12431 let spec = windowed_crash_spec("budget-journal-failure");
12432 let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::starting()));
12433 assert!(matches!(
12434 on_child_exit(
12435 &spec,
12436 runtime.restart_policy,
12437 &supervisor.registry,
12438 &snapshot,
12439 &runtime.terminal_ring,
12440 &runtime.spawn_events,
12441 &runtime.child_roster,
12442 crash_exit_report(1),
12443 )
12444 .await,
12445 NextAction::Stop { .. }
12446 ));
12447 assert_eq!(lock_snapshot(&snapshot).unwrap().state, ModuleState::Failed);
12448 let history = runtime
12449 .terminal_ring
12450 .lock()
12451 .unwrap()
12452 .durable_history(&spec.module_id);
12453 assert!(history.journal_write_failures > 0);
12454 assert_eq!(history.entries.len(), 1);
12455 assert_eq!(history.entries[0].disposition, TerminalDisposition::Failed);
12456 }
12457
12458 #[test]
12459 fn restart_budget_failure_remains_failed_when_exit_recording_panics() {
12460 let supervisor = Supervisor::new_for_test(
12461 Arc::new(Registry::default()),
12462 RestartPolicy::new(0, Duration::ZERO),
12463 );
12464 let runtime = supervisor.runtime_config();
12465 let spec = windowed_crash_spec("budget-recording-panic");
12466 let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::starting()));
12467 runtime.spawn_events.emit_spawned(&spec.module_id, 1, 1);
12468 runtime.spawn_events.0.lock().unwrap().seq = u64::MAX;
12471 let result = std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| {
12472 tokio::runtime::Builder::new_current_thread()
12473 .enable_all()
12474 .build()
12475 .unwrap()
12476 .block_on(on_child_exit(
12477 &spec,
12478 runtime.restart_policy,
12479 &supervisor.registry,
12480 &snapshot,
12481 &runtime.terminal_ring,
12482 &runtime.spawn_events,
12483 &runtime.child_roster,
12484 crash_exit_report(1),
12485 ))
12486 }));
12487 let panic = result.err().expect("recording must still unwind");
12488 assert_eq!(
12489 panic.downcast_ref::<String>().map(String::as_str),
12490 Some("spawn event sequence exhausted")
12491 );
12492 assert_eq!(lock_snapshot(&snapshot).unwrap().state, ModuleState::Failed);
12493 }
12494
12495 #[cfg(unix)]
12498 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
12499 async fn protocol_none_requested_stop_and_disable_do_not_respawn() {
12500 for disable in [false, true] {
12501 let label = if disable {
12502 "none-requested-disable"
12503 } else {
12504 "none-requested-stop"
12505 };
12506 let dir = cortexkit_test_support::ScratchDir::new(label);
12507 let (spec, ready, marker) = protocol_none_sigterm_exits_clean_spec(label, dir.path());
12508 let supervisor = Supervisor::new_for_test(
12509 Arc::new(Registry::default()),
12510 RestartPolicy::new(3, Duration::ZERO),
12511 );
12512 let module = supervisor.spawn(spec).unwrap();
12513 wait_for_file(&ready).await;
12514
12515 if disable {
12516 module.set_enabled(false).await.unwrap();
12517 } else {
12518 module.stop().await.unwrap();
12519 }
12520 assert!(
12521 marker.exists(),
12522 "{label}: the child must have left through its SIGTERM handler with exit 0"
12523 );
12524
12525 sleep(Duration::from_millis(500)).await;
12528 let status = module.status().unwrap();
12529 let expected = if disable {
12530 ModuleState::Disabled
12531 } else {
12532 ModuleState::Stopped
12533 };
12534 assert_eq!(status.state, expected, "{label}");
12535 assert_eq!(
12536 status.spawn_generation, 1,
12537 "{label}: respawned after a requested stop"
12538 );
12539 let history = module.terminal_history();
12540 assert_eq!(history.entries.len(), 1, "{label}: {history:?}");
12541 assert_eq!(history.entries[0].exit_code, Some(0), "{label}");
12542 assert_ne!(
12543 history.entries[0].disposition,
12544 TerminalDisposition::Restarting,
12545 "{label}"
12546 );
12547 }
12548 }
12549
12550 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
12553 async fn subc_wire_clean_exit_is_still_a_stop() {
12554 let supervisor = Supervisor::new_for_test(
12555 Arc::new(Registry::default()),
12556 RestartPolicy::new(3, Duration::ZERO),
12557 );
12558 let module = supervisor
12559 .spawn(ModuleSpec {
12560 module_id: "wire-clean-exit".to_string(),
12561 program: fake_aft_stub_path(),
12562 args: Vec::new(),
12563 env: vec![("FAKE_AFT_EXIT_CODE".to_string(), "0".to_string())],
12564 reserved: false,
12565 reserved_prefixes: Vec::new(),
12566 protocol: ModuleProtocol::Subc,
12567 overlap: Default::default(),
12568 })
12569 .unwrap();
12570
12571 let deadline = Instant::now() + Duration::from_secs(10);
12572 while module.terminal_history().entries.is_empty() {
12573 assert!(Instant::now() < deadline, "module never exited");
12574 sleep(Duration::from_millis(10)).await;
12575 }
12576 sleep(Duration::from_millis(500)).await;
12578 let status = module.status().unwrap();
12579 assert_eq!(status.state, ModuleState::Stopped);
12580 assert_eq!(status.spawn_generation, 1);
12581 let history = module.terminal_history();
12582 assert_eq!(history.entries.len(), 1, "{history:?}");
12583 assert_eq!(history.entries[0].exit_code, Some(0));
12584 assert_eq!(history.entries[0].disposition, TerminalDisposition::Stopped);
12585 }
12586
12587 #[cfg(unix)]
12588 #[tokio::test]
12589 async fn reload_exit_keeps_roster_until_terminal_is_recorded_during_shutdown() {
12590 let dir = cortexkit_test_support::ScratchDir::new("reload-roster-terminal-order");
12591 let record = dir.join("live-children.json");
12592 let supervisor = Supervisor::new_for_test(
12593 Arc::new(Registry::default()),
12594 RestartPolicy::new(0, Duration::ZERO),
12595 );
12596 let mut runtime = supervisor.runtime_config();
12597 runtime.child_roster.record_to(record.clone());
12598 let gate = Arc::new(super::ReloadExitRecordGate::default());
12599 runtime.test_reload_exit_record_gate = Some(Arc::clone(&gate));
12600 let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::starting()));
12601 let spec = ModuleSpec {
12602 module_id: "reload-exit-roster".into(),
12603 program: fake_aft_stub_path(),
12604 args: Vec::new(),
12605 env: vec![("FAKE_AFT_EXIT_CODE".into(), "23".into())],
12606 reserved: false,
12607 reserved_prefixes: Vec::new(),
12608 protocol: ModuleProtocol::Subc,
12609 overlap: Default::default(),
12610 };
12611 let mut child = None;
12612 let reload = super::finish_reload_child(
12613 &spec,
12614 &runtime,
12615 &supervisor.registry,
12616 &supervisor.process_liveness,
12617 &snapshot,
12618 &mut child,
12619 );
12620 tokio::pin!(reload);
12621 tokio::select! {
12622 result = &mut reload => panic!("reload missed the before-record gate: {result:?}"),
12623 _ = gate.reached.notified() => {}
12624 }
12625 assert!(runtime
12626 .terminal_ring
12627 .lock()
12628 .unwrap()
12629 .snapshot()
12630 .entries
12631 .is_empty());
12632 assert_eq!(
12633 crate::live_children::read_record(&record).unwrap().len(),
12634 1,
12635 "shutdown must still wait for the reaped child until its terminal record exists"
12636 );
12637 runtime.child_roster.close();
12638 gate.resume.notify_one();
12639 assert!(reload.await.is_err());
12640 assert!(crate::live_children::read_record(&record)
12641 .unwrap()
12642 .is_empty());
12643 let history = runtime.terminal_ring.lock().unwrap().snapshot();
12644 assert_eq!(history.entries.len(), 1);
12645 assert_eq!(
12646 history.entries[0].disposition,
12647 TerminalDisposition::DaemonShutdown
12648 );
12649 }
12650
12651 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
12655 async fn every_restart_increment_path_advances_lifetime_count() {
12656 let supervisor = Supervisor::new_for_test(
12657 Arc::new(Registry::default()),
12658 RestartPolicy::new(1, Duration::ZERO),
12659 );
12660 let runtime = supervisor.runtime_config();
12661 let spec = ModuleSpec {
12662 module_id: "lifetime-increment-path".to_string(),
12663 program: PathBuf::from("/unused/lifetime-increment-path"),
12664 args: Vec::new(),
12665 env: Vec::new(),
12666 reserved: false,
12667 reserved_prefixes: Vec::new(),
12668 protocol: ModuleProtocol::Subc,
12669 overlap: Default::default(),
12670 };
12671
12672 let crash_snapshot = Arc::new(Mutex::new(SupervisorSnapshot::starting()));
12673 assert!(matches!(
12674 on_child_exit(
12675 &spec,
12676 runtime.restart_policy,
12677 &supervisor.registry,
12678 &crash_snapshot,
12679 &runtime.terminal_ring,
12680 &runtime.spawn_events,
12681 &runtime.child_roster,
12682 ExitReport {
12683 kind: ExitKind::Crash,
12684 code: Some(1),
12685 signal: None,
12686 at_ms: 1,
12687 },
12688 )
12689 .await,
12690 NextAction::Restart { schedule: _ }
12691 ));
12692 let (crash_restarts, crash_lifetime) = {
12693 let state = lock_snapshot(&crash_snapshot).unwrap();
12694 (state.crash_restarts.len(), state.lifetime_restarts)
12695 };
12696 assert_eq!(crash_restarts, 1);
12697 assert_eq!(crash_lifetime, 1);
12698
12699 let health_snapshot = Arc::new(Mutex::new(SupervisorSnapshot::starting()));
12700 let mut health_child = None;
12701 assert!(matches!(
12702 health_restart_child(
12703 &spec,
12704 &runtime,
12705 &supervisor.registry,
12706 &supervisor.process_liveness,
12707 &health_snapshot,
12708 &mut health_child,
12709 SupervisorHealthStatus::Failing,
12710 None,
12711 2,
12712 )
12713 .await,
12714 Ok(())
12715 ));
12716 assert!(health_child.is_none());
12717 assert!(lock_snapshot(&health_snapshot).unwrap().respawn_pending);
12718 assert!(runtime.scheduled_respawn.lock().unwrap().take().is_some());
12719 let (health_restarts, health_lifetime) = {
12720 let state = lock_snapshot(&health_snapshot).unwrap();
12721 (state.crash_restarts.len(), state.lifetime_restarts)
12722 };
12723 assert_eq!(health_restarts, 1);
12724 assert_eq!(health_lifetime, 1);
12725
12726 let reload_snapshot = Arc::new(Mutex::new(SupervisorSnapshot::starting()));
12727 let mut reload_child = None;
12728 assert!(matches!(
12729 handle_reload_spawn_failure(
12730 &spec,
12731 &runtime,
12732 &supervisor.process_liveness,
12733 &reload_snapshot,
12734 &mut reload_child,
12735 "forced reload spawn failure".to_string(),
12736 )
12737 .await,
12738 Err(SuperviseError::ReloadFailed { .. })
12739 ));
12740 let (reload_restarts, reload_lifetime) = {
12741 let state = lock_snapshot(&reload_snapshot).unwrap();
12742 (state.crash_restarts.len(), state.lifetime_restarts)
12743 };
12744 assert_eq!(reload_restarts, 1);
12745 assert_eq!(reload_lifetime, 1);
12746 }
12747
12748 #[tokio::test]
12749 async fn deliberately_severed_live_child_records_lifetime_without_spending_restart_budget() {
12750 let supervisor = Supervisor::new_for_test(
12751 Arc::new(Registry::default()),
12752 RestartPolicy::new(3, Duration::ZERO),
12753 );
12754 let runtime = supervisor.runtime_config();
12755 let spec = ModuleSpec {
12756 module_id: "deliberately-severed".to_string(),
12757 program: PathBuf::from("/unused/deliberately-severed"),
12758 args: Vec::new(),
12759 env: Vec::new(),
12760 reserved: false,
12761 reserved_prefixes: Vec::new(),
12762 protocol: ModuleProtocol::Subc,
12763 overlap: Default::default(),
12764 };
12765 let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::starting()));
12766 let process = ProcessIdentity {
12767 pid: 41,
12768 start_time: 101,
12769 };
12770 record_deliberate_severance(&snapshot, process).unwrap();
12771 let exit_report = apply_deliberate_severance_marker(
12772 &snapshot,
12773 Some(process),
12774 ExitReport {
12775 kind: ExitKind::Crash,
12776 code: Some(1),
12777 signal: None,
12778 at_ms: 1,
12779 },
12780 );
12781 assert_eq!(exit_report.kind, ExitKind::DeliberateSeverance);
12782
12783 assert!(matches!(
12784 on_child_exit(
12785 &spec,
12786 runtime.restart_policy,
12787 &supervisor.registry,
12788 &snapshot,
12789 &runtime.terminal_ring,
12790 &runtime.spawn_events,
12791 &runtime.child_roster,
12792 exit_report,
12793 )
12794 .await,
12795 NextAction::Restart { schedule: _ }
12796 ));
12797 let state = lock_snapshot(&snapshot).unwrap();
12798 assert_eq!(state.lifetime_restarts, 1);
12799 assert_eq!(state.crash_restarts.len(), 0);
12800 }
12801
12802 #[tokio::test]
12803 async fn genuine_crash_spends_restart_budget_and_records_lifetime() {
12804 let supervisor = Supervisor::new_for_test(
12805 Arc::new(Registry::default()),
12806 RestartPolicy::new(3, Duration::ZERO),
12807 );
12808 let runtime = supervisor.runtime_config();
12809 let spec = ModuleSpec {
12810 module_id: "genuine-crash".to_string(),
12811 program: PathBuf::from("/unused/genuine-crash"),
12812 args: Vec::new(),
12813 env: Vec::new(),
12814 reserved: false,
12815 reserved_prefixes: Vec::new(),
12816 protocol: ModuleProtocol::Subc,
12817 overlap: Default::default(),
12818 };
12819 let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::starting()));
12820
12821 assert!(matches!(
12822 on_child_exit(
12823 &spec,
12824 runtime.restart_policy,
12825 &supervisor.registry,
12826 &snapshot,
12827 &runtime.terminal_ring,
12828 &runtime.spawn_events,
12829 &runtime.child_roster,
12830 ExitReport {
12831 kind: ExitKind::Crash,
12832 code: Some(1),
12833 signal: None,
12834 at_ms: 1,
12835 },
12836 )
12837 .await,
12838 NextAction::Restart { schedule: _ }
12839 ));
12840 let state = lock_snapshot(&snapshot).unwrap();
12841 assert_eq!(state.lifetime_restarts, 1);
12842 assert_eq!(state.crash_restarts.len(), 1);
12843 }
12844
12845 fn crash_exit_report(at_ms: u64) -> ExitReport {
12846 ExitReport {
12847 kind: ExitKind::Crash,
12848 code: Some(1),
12849 signal: None,
12850 at_ms,
12851 }
12852 }
12853
12854 fn windowed_crash_spec(module_id: &str) -> ModuleSpec {
12855 ModuleSpec {
12856 module_id: module_id.to_string(),
12857 program: PathBuf::from("/unused").join(module_id),
12858 args: Vec::new(),
12859 env: Vec::new(),
12860 reserved: false,
12861 reserved_prefixes: Vec::new(),
12862 protocol: ModuleProtocol::Subc,
12863 overlap: Default::default(),
12864 }
12865 }
12866
12867 #[tokio::test]
12873 async fn three_crashes_inside_the_window_stop_the_module_and_name_the_window() {
12874 let (logs, _guard) = crate::router::test_log::log_capture(tracing::Level::ERROR);
12875 let supervisor = Supervisor::new_for_test(
12876 Arc::new(Registry::default()),
12877 RestartPolicy::new(2, Duration::ZERO),
12878 );
12879 let runtime = supervisor.runtime_config();
12880 let spec = windowed_crash_spec("crash-loop-in-window");
12881 let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::starting()));
12882
12883 for attempt in 1..=2 {
12884 assert!(
12885 matches!(
12886 on_child_exit(
12887 &spec,
12888 runtime.restart_policy,
12889 &supervisor.registry,
12890 &snapshot,
12891 &runtime.terminal_ring,
12892 &runtime.spawn_events,
12893 &runtime.child_roster,
12894 crash_exit_report(attempt),
12895 )
12896 .await,
12897 NextAction::Restart { schedule: _ }
12898 ),
12899 "crash {attempt} is inside the budget and must respawn"
12900 );
12901 }
12902
12903 assert!(matches!(
12904 on_child_exit(
12905 &spec,
12906 runtime.restart_policy,
12907 &supervisor.registry,
12908 &snapshot,
12909 &runtime.terminal_ring,
12910 &runtime.spawn_events,
12911 &runtime.child_roster,
12912 crash_exit_report(3),
12913 )
12914 .await,
12915 NextAction::Stop { .. }
12916 ));
12917
12918 {
12919 let state = lock_snapshot(&snapshot).unwrap();
12920 assert_eq!(state.state, ModuleState::Failed);
12921 assert_eq!(state.crash_restarts.len(), 2);
12922 assert_eq!(state.lifetime_restarts, 2);
12923 }
12924
12925 let history = runtime
12926 .terminal_ring
12927 .lock()
12928 .expect("terminal ring is not poisoned")
12929 .snapshot();
12930 let last = history
12931 .entries
12932 .last()
12933 .expect("the refused crash is retained");
12934 assert_eq!(last.disposition, TerminalDisposition::Failed);
12935 assert_eq!(
12936 last.disposition_detail.as_deref(),
12937 Some("crash budget exhausted: max_restarts=2 within window_secs=600")
12938 );
12939
12940 let captured = crate::router::test_log::captured_logs(&logs);
12941 assert!(
12942 captured.contains("crash budget exhausted: max_restarts=2 within window_secs=600"),
12943 "the stop must be logged with its window: {captured}"
12944 );
12945 }
12946
12947 #[tokio::test]
12955 async fn a_crash_older_than_the_window_frees_its_slot_for_a_later_crash() {
12956 let supervisor = Supervisor::new_for_test(
12957 Arc::new(Registry::default()),
12958 RestartPolicy::new(2, Duration::ZERO),
12959 );
12960 let runtime = supervisor.runtime_config();
12961 let spec = windowed_crash_spec("crash-across-windows");
12962 let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::starting()));
12963
12964 for attempt in 1..=2 {
12965 assert!(matches!(
12966 on_child_exit(
12967 &spec,
12968 runtime.restart_policy,
12969 &supervisor.registry,
12970 &snapshot,
12971 &runtime.terminal_ring,
12972 &runtime.spawn_events,
12973 &runtime.child_roster,
12974 crash_exit_report(attempt),
12975 )
12976 .await,
12977 NextAction::Restart { schedule: _ }
12978 ));
12979 }
12980
12981 update_snapshot(&snapshot, Some(&spec.module_id), |state| {
12984 age_oldest_crash_restart_out_of_window(state, runtime.restart_policy.window);
12985 })
12986 .unwrap();
12987
12988 assert!(
12989 matches!(
12990 on_child_exit(
12991 &spec,
12992 runtime.restart_policy,
12993 &supervisor.registry,
12994 &snapshot,
12995 &runtime.terminal_ring,
12996 &runtime.spawn_events,
12997 &runtime.child_roster,
12998 crash_exit_report(3),
12999 )
13000 .await,
13001 NextAction::Restart { schedule: _ }
13002 ),
13003 "a crash older than the window must not hold a budget slot"
13004 );
13005
13006 let state = lock_snapshot(&snapshot).unwrap();
13007 assert_eq!(state.state, ModuleState::Restarting);
13008 assert_eq!(
13009 state.crash_restarts.len(),
13010 2,
13011 "the aged instant is dropped and the new one takes its place"
13012 );
13013 assert_eq!(
13014 state.lifetime_restarts, 3,
13015 "the ledger counts every restart, including the ones the window forgot"
13016 );
13017 }
13018
13019 #[tokio::test]
13024 async fn an_operator_restart_clears_the_ring_and_leaves_the_ledger_alone() {
13025 let supervisor = Supervisor::new_for_test(
13026 Arc::new(Registry::default()),
13027 RestartPolicy::new(2, Duration::ZERO),
13028 );
13029 let runtime = supervisor.runtime_config();
13030 let spec = windowed_crash_spec("operator-cleared-budget");
13031 let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::starting()));
13032
13033 for attempt in 1..=2 {
13034 assert!(matches!(
13035 on_child_exit(
13036 &spec,
13037 runtime.restart_policy,
13038 &supervisor.registry,
13039 &snapshot,
13040 &runtime.terminal_ring,
13041 &runtime.spawn_events,
13042 &runtime.child_roster,
13043 crash_exit_report(attempt),
13044 )
13045 .await,
13046 NextAction::Restart { schedule: _ }
13047 ));
13048 }
13049
13050 reset_restart_count(&snapshot, &spec.module_id).unwrap();
13051 {
13052 let state = lock_snapshot(&snapshot).unwrap();
13053 assert!(
13054 state.crash_restarts.is_empty(),
13055 "an operator restart returns the full budget"
13056 );
13057 assert_eq!(
13058 state.lifetime_restarts, 2,
13059 "clearing the budget must not unmake the crashes"
13060 );
13061 }
13062
13063 assert!(
13064 matches!(
13065 on_child_exit(
13066 &spec,
13067 runtime.restart_policy,
13068 &supervisor.registry,
13069 &snapshot,
13070 &runtime.terminal_ring,
13071 &runtime.spawn_events,
13072 &runtime.child_roster,
13073 crash_exit_report(3),
13074 )
13075 .await,
13076 NextAction::Restart { schedule: _ }
13077 ),
13078 "the cleared budget must be spendable again"
13079 );
13080 let state = lock_snapshot(&snapshot).unwrap();
13081 assert_eq!(state.crash_restarts.len(), 1);
13082 assert_eq!(state.lifetime_restarts, 3);
13083 }
13084
13085 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
13086 async fn severance_marker_for_a_dead_child_does_not_label_its_successor() {
13087 let severed = ProcessIdentity {
13088 pid: 41,
13089 start_time: 101,
13090 };
13091 let successor = ProcessIdentity {
13092 pid: 41,
13093 start_time: 202,
13094 };
13095 let module = module_with_recovery_snapshot(ModuleState::Running, true, 0);
13096 update_snapshot(&module.inner.snapshot, Some("recovery-snapshot"), |state| {
13097 state.pid = Some(successor.pid);
13098 state.process_start_time = Some(successor.start_time);
13099 })
13100 .unwrap();
13101 assert!(!module.record_deliberate_severance(severed).unwrap());
13102
13103 let exit_report = apply_deliberate_severance_marker(
13104 &module.inner.snapshot,
13105 Some(successor),
13106 ExitReport {
13107 kind: ExitKind::Crash,
13108 code: Some(1),
13109 signal: None,
13110 at_ms: 1,
13111 },
13112 );
13113
13114 assert_eq!(exit_report.kind, ExitKind::Crash);
13115 }
13116
13117 #[tokio::test]
13118 async fn drain_reap_marks_deliberate_severance_and_records_lifetime_without_budget() {
13119 let registry = Registry::default();
13120 let supervisor = Supervisor::new_for_test(
13121 Arc::new(Registry::default()),
13122 RestartPolicy::new(3, Duration::ZERO),
13123 );
13124 let runtime = supervisor.runtime_config();
13125 let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::starting()));
13126 let spec = ModuleSpec {
13127 module_id: "drain-deliberate-severance".to_string(),
13128 program: fake_aft_stub_path(),
13129 args: Vec::new(),
13130 env: vec![("FAKE_AFT_EXIT_CODE".to_string(), "23".to_string())],
13131 reserved: false,
13132 reserved_prefixes: Vec::new(),
13133 protocol: ModuleProtocol::Subc,
13134 overlap: Default::default(),
13135 };
13136 let mut child = spawn_and_mark_running(&spec, &runtime, &snapshot).unwrap();
13137 let process = ProcessIdentity {
13138 pid: 41,
13139 start_time: 101,
13140 };
13141 child.process_identity = Some(process);
13142 update_snapshot(&snapshot, Some(&spec.module_id), |state| {
13143 state.pid = Some(process.pid);
13144 state.process_start_time = Some(process.start_time);
13145 })
13146 .unwrap();
13147 record_deliberate_severance(&snapshot, process).unwrap();
13148
13149 drain_child_to_state(
13150 &spec.module_id,
13151 spec.protocol,
13152 StopNotice::SentOverConnection,
13155 ®istry,
13156 None,
13157 &snapshot,
13158 &runtime.terminal_ring,
13159 &runtime.spawn_events,
13160 child,
13161 Duration::from_secs(1),
13162 ModuleState::Stopped,
13163 Some(false),
13164 )
13165 .await
13166 .unwrap();
13167
13168 let state = lock_snapshot(&snapshot).unwrap();
13169 assert_eq!(
13170 state.last_exit.as_ref().map(|exit| exit.kind),
13171 Some(ExitKind::DeliberateSeverance)
13172 );
13173 assert_eq!(state.lifetime_restarts, 1);
13174 assert_eq!(state.crash_restarts.len(), 0);
13175 drop(state);
13176 let history = runtime.terminal_ring.lock().unwrap().snapshot();
13177 assert_eq!(
13178 history.entries[0].exit_kind,
13179 subc_control::TerminalExitKind::DeliberateSeverance
13180 );
13181 }
13182
13183 #[tokio::test]
13184 async fn ordinary_drain_reap_does_not_record_a_lifetime_restart() {
13185 let registry = Registry::default();
13186 let supervisor = Supervisor::new_for_test(
13187 Arc::new(Registry::default()),
13188 RestartPolicy::new(3, Duration::ZERO),
13189 );
13190 let runtime = supervisor.runtime_config();
13191 let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::starting()));
13192 let spec = ModuleSpec {
13193 module_id: "ordinary-drain".to_string(),
13194 program: fake_aft_stub_path(),
13195 args: Vec::new(),
13196 env: vec![("FAKE_AFT_EXIT_CODE".to_string(), "23".to_string())],
13197 reserved: false,
13198 reserved_prefixes: Vec::new(),
13199 protocol: ModuleProtocol::Subc,
13200 overlap: Default::default(),
13201 };
13202 let child = spawn_and_mark_running(&spec, &runtime, &snapshot).unwrap();
13203
13204 drain_child_to_state(
13205 &spec.module_id,
13206 spec.protocol,
13207 StopNotice::SentOverConnection,
13210 ®istry,
13211 None,
13212 &snapshot,
13213 &runtime.terminal_ring,
13214 &runtime.spawn_events,
13215 child,
13216 Duration::from_secs(1),
13217 ModuleState::Stopped,
13218 Some(false),
13219 )
13220 .await
13221 .unwrap();
13222
13223 let state = lock_snapshot(&snapshot).unwrap();
13224 assert_eq!(
13225 state.last_exit.as_ref().map(|exit| exit.kind),
13226 Some(ExitKind::Crash)
13227 );
13228 assert_eq!(state.lifetime_restarts, 0);
13229 assert_eq!(state.crash_restarts.len(), 0);
13230 }
13231
13232 #[test]
13233 fn fatal_connection_teardown_cannot_arm_a_marker_for_a_surviving_process() {
13234 assert!(!include_str!("server.rs")
13240 .contains("router.record_deliberate_connection_severance(ctx.connection_id)"));
13241 }
13242
13243 #[test]
13250 fn drained_after_quiescence_wait_passes_ok_through_and_forces_false_on_err() {
13251 assert!(drained_after_quiescence_wait(&Ok(true)));
13252 assert!(!drained_after_quiescence_wait(&Ok(false)));
13253 assert!(!drained_after_quiescence_wait(&Err(
13254 SuperviseError::StatePoisoned { module_id: None }
13255 )));
13256 }
13257
13258 #[test]
13267 fn wait_error_exit_report_records_a_failed_terminal_with_no_code_or_signal() {
13268 let ring = Arc::new(Mutex::new(TerminalRing::new(
13269 TerminalRingConfig::default(),
13270 0,
13271 )));
13272 record_wait_error_terminal("wait-error", &ring, &super::SpawnEventFeed::default());
13273
13274 let snapshot = ring.lock().unwrap().snapshot();
13275 assert_eq!(snapshot.entries.len(), 1);
13276 let entry = &snapshot.entries[0];
13277 assert_eq!(entry.exit_code, None);
13278 assert_eq!(entry.exit_signal, None);
13279 assert_eq!(entry.disposition, TerminalDisposition::Failed);
13280 }
13281
13282 #[test]
13283 fn wait_error_exit_path_preserves_spawn_event_density() {
13284 let feed = super::SpawnEventFeed::default();
13285 feed.configure_incarnation("wait-error-density".to_string());
13286 feed.emit_spawned("wait-error", 41, 1);
13287 let ring = Arc::new(Mutex::new(TerminalRing::new(
13288 TerminalRingConfig::default(),
13289 0,
13290 )));
13291
13292 record_wait_error_terminal("wait-error", &ring, &feed);
13293 feed.emit_spawned("after-wait-error", 42, 2);
13294
13295 let state = feed.0.lock().unwrap();
13296 let sequences = state
13297 .events
13298 .iter()
13299 .map(|event| event.cursor.seq)
13300 .collect::<Vec<_>>();
13301 assert_eq!(sequences, vec![1, 2, 3]);
13302 assert_eq!(state.events[1].kind, SpawnEventKind::Exited);
13303 assert_eq!(state.events[1].exit_code, None);
13304 assert_eq!(state.events[1].exit_signal, None);
13305 }
13306
13307 #[test]
13311 fn wait_error_exit_report_is_classified_as_a_crash() {
13312 assert_eq!(wait_error_exit_report().kind, ExitKind::Crash);
13313 }
13314}
13315
13316#[cfg(all(test, unix))]
13317mod restart_ordering_tests {
13318 use super::*;
13319 use std::io::Write;
13320 use tokio::io::AsyncReadExt;
13321
13322 #[test]
13323 #[ignore = "child process fixture"]
13324 fn serving_child() {
13325 let Ok(address_file) = std::env::var("RESTART_TEST_ADDRESS_FILE") else {
13326 return;
13327 };
13328 let listener = std::net::TcpListener::bind("127.0.0.1:0").unwrap();
13329 std::fs::write(address_file, listener.local_addr().unwrap().to_string()).unwrap();
13330 for stream in listener.incoming() {
13331 writeln!(stream.unwrap(), "{}", std::process::id()).unwrap();
13332 }
13333 }
13334
13335 async fn restart_before_publication(requests: usize) {
13336 let home = cortexkit_test_support::ScratchDir::new("restart-before-publication");
13337 let address_file = home.path().join("address");
13338 let spec = ModuleSpec {
13339 module_id: "restart-serving-child".into(),
13340 program: std::env::current_exe().unwrap(),
13341 args: vec![
13342 "--exact".into(),
13343 "supervise::restart_ordering_tests::serving_child".into(),
13344 "--ignored".into(),
13345 "--nocapture".into(),
13346 ],
13347 env: vec![(
13348 "RESTART_TEST_ADDRESS_FILE".into(),
13349 address_file.display().to_string(),
13350 )],
13351 reserved: false,
13352 reserved_prefixes: vec![],
13353 protocol: ModuleProtocol::None,
13354 overlap: Default::default(),
13355 };
13356 let mut supervisor =
13357 Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default());
13358 let (spawned_tx, spawned_rx) = oneshot::channel();
13359 let spawned_tx = Mutex::new(Some(spawned_tx));
13360 let (release_tx, release_rx) = std::sync::mpsc::channel();
13361 let release_rx = Mutex::new(release_rx);
13362 supervisor.test_after_first_spawn = AfterFirstSpawnHook(Some(Arc::new(move |_| {
13365 spawned_tx.lock().unwrap().take().unwrap().send(()).unwrap();
13366 release_rx
13367 .lock()
13368 .unwrap()
13369 .recv_timeout(Duration::from_secs(20))
13370 .unwrap();
13371 })));
13372 let registered = supervisor
13373 .register_configured_with_health(
13374 spec.clone(),
13375 true,
13376 HealthConfig::default(),
13377 None,
13378 RestartPolicy::new(3, Duration::ZERO),
13379 )
13380 .unwrap();
13381 let module = registered.module.clone();
13382 let startup = tokio::task::spawn_blocking(move || {
13383 supervisor.start_registered_configured(registered, None)
13384 });
13385 spawned_rx.await.unwrap();
13386 let old_pid = tokio::time::timeout(Duration::from_secs(10), async {
13387 let address = loop {
13388 if let Ok(address) = std::fs::read_to_string(&address_file) {
13389 if !address.is_empty() {
13390 break address;
13391 }
13392 }
13393 tokio::time::sleep(Duration::from_millis(1)).await;
13394 };
13395 let mut stream = tokio::net::TcpStream::connect(address).await.unwrap();
13396 let mut reply = String::new();
13397 stream.read_to_string(&mut reply).await.unwrap();
13398 reply.trim().parse::<u32>().unwrap()
13399 })
13400 .await
13401 .expect("child must serve before generation publication");
13402 assert_eq!(
13403 lock_snapshot(&module.inner.snapshot)
13404 .unwrap()
13405 .spawn_generation,
13406 0
13407 );
13408 let mut replies = Vec::new();
13409 for _ in 0..requests {
13410 let mut restart = Box::pin(module.restart(Some(0)));
13411 std::future::poll_fn(|cx| {
13414 assert!(std::future::Future::poll(restart.as_mut(), cx).is_pending());
13415 std::task::Poll::Ready(())
13416 })
13417 .await;
13418 replies.push(restart);
13419 }
13420 release_tx.send(()).unwrap();
13421 startup.await.unwrap().unwrap();
13422 for reply in replies {
13423 reply.await.unwrap();
13424 }
13425 let replacement = tokio::time::timeout(Duration::from_secs(10), async {
13428 loop {
13429 if lock_snapshot(&module.inner.snapshot)
13430 .unwrap()
13431 .spawn_generation
13432 > 1
13433 {
13434 break;
13435 }
13436 tokio::time::sleep(Duration::from_millis(1)).await;
13437 }
13438 })
13439 .await;
13440 module.update_spec_for_test(spec).await.unwrap();
13444 let (generation, pid) = {
13445 let state = lock_snapshot(&module.inner.snapshot).unwrap();
13446 (state.spawn_generation, state.pid)
13447 };
13448 module.stop().await.unwrap();
13449 replacement.expect("restart must publish a replacement for the already-serving child");
13450 assert_ne!(
13451 pid,
13452 Some(old_pid),
13453 "a serving child predating the restart must be replaced"
13454 );
13455 assert_eq!(generation, 2, "queued restarts must share one replacement");
13456 }
13457
13458 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
13459 async fn restart_received_after_serving_before_generation_publication_replaces_child() {
13460 restart_before_publication(1).await;
13461 }
13462
13463 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
13464 async fn restart_queued_behind_restart_does_not_kill_its_replacement() {
13465 restart_before_publication(2).await;
13466 }
13467
13468 #[tokio::test]
13469 async fn restart_queued_before_replacement_spawn_keeps_the_new_child() {
13470 let supervisor =
13471 Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default());
13472 let registered = supervisor
13473 .register_configured_with_health(
13474 ModuleSpec {
13475 module_id: "queued-before-replacement".into(),
13476 program: "/bin/sh".into(),
13477 args: vec!["-c".into(), "exec sleep 600".into()],
13478 env: vec![],
13479 reserved: false,
13480 reserved_prefixes: vec![],
13481 protocol: ModuleProtocol::None,
13482 overlap: Default::default(),
13483 },
13484 true,
13485 HealthConfig::default(),
13486 None,
13487 RestartPolicy::default(),
13488 )
13489 .unwrap();
13490 let RegisteredConfiguredModule {
13491 mut spec,
13492 mut runtime,
13493 module,
13494 mut commands,
13495 } = registered;
13496 let snapshot = &module.inner.snapshot;
13497 let mut restart = Box::pin(module.restart(Some(0)));
13498 std::future::poll_fn(|cx| {
13499 assert!(std::future::Future::poll(restart.as_mut(), cx).is_pending());
13500 std::task::Poll::Ready(())
13501 })
13502 .await;
13503 let mut child = Some(spawn_and_mark_running(&spec, &runtime, snapshot).unwrap());
13506 let replacement_pid = child.as_ref().unwrap().id();
13507 let command = commands.recv().await.unwrap();
13508 assert!(
13509 handle_supervisor_command(
13510 command,
13511 &mut spec,
13512 &mut runtime,
13513 &supervisor.registry,
13514 &supervisor.process_liveness,
13515 snapshot,
13516 &mut child,
13517 &mut commands,
13518 &mut VecDeque::new(),
13519 )
13520 .await
13521 );
13522 restart.await.unwrap();
13523 let kept_pid = child.as_ref().and_then(SupervisedChild::id);
13524 let pending = lock_snapshot(snapshot).unwrap().respawn_pending;
13525 if let Some(mut child) = child {
13526 child.start_kill().unwrap();
13527 child.wait().await.unwrap();
13528 }
13529 assert_eq!(
13530 kept_pid, replacement_pid,
13531 "a newer replacement must survive"
13532 );
13533 assert!(
13534 !pending,
13535 "a satisfied restart must not schedule another spawn"
13536 );
13537 }
13538}
13539
13540#[cfg(all(test, unix))]
13541mod health_event_tests {
13542 use super::*;
13543 use cortexkit_test_support::ScratchDir;
13544 use subc_protocol::{manifest::Concurrency, session::ModuleControlResponse};
13545
13546 struct Actor {
13547 module: SupervisedModule,
13548 registry: Arc<Registry>,
13549 forwarding: Arc<ForwardingTable>,
13550 spec: ModuleSpec,
13551 health: HealthConfig,
13552 rx: mpsc::Receiver<crate::router::OutboundFrame>,
13553 _home: ScratchDir,
13554 }
13555
13556 async fn wait_for<T>(reason: &str, mut observe: impl FnMut() -> Option<T>) -> T {
13557 let deadline = std::time::Instant::now() + Duration::from_secs(10);
13561 let tick = Instant::now();
13562 loop {
13563 if let Some(value) = observe() {
13564 assert_eq!(Instant::now(), tick, "{reason} must not wait for a timer");
13565 return value;
13566 }
13567 assert!(
13568 std::time::Instant::now() < deadline,
13569 "timed out waiting for {reason}"
13570 );
13571 tokio::task::yield_now().await;
13572 }
13573 }
13574
13575 impl Actor {
13576 async fn start(protocol: ModuleProtocol, cadence: Duration) -> Self {
13577 let home = ScratchDir::new("health-events");
13578 let registry = Arc::new(Registry::default());
13579 let forwarding = Arc::new(ForwardingTable::default());
13580 let health = HealthConfig {
13581 cadence,
13582 deadline: Duration::from_secs(3600),
13583 ..HealthConfig::default()
13584 };
13585 let supervisor =
13586 Supervisor::new_for_test(registry.clone(), RestartPolicy::new(3, Duration::ZERO))
13587 .with_forwarding(forwarding.clone())
13588 .with_health_config(health.clone());
13589 let spec = ModuleSpec {
13590 module_id: "event-health-child".into(),
13591 program: PathBuf::from("/bin/sh"),
13592 args: vec!["-c".into(), "exec sleep 600".into()],
13595 env: ["XDG_DATA_HOME", "XDG_RUNTIME_DIR", "XDG_CONFIG_HOME"]
13596 .into_iter()
13597 .map(|key| (key.into(), home.path().display().to_string()))
13598 .collect(),
13599 reserved: false,
13600 reserved_prefixes: vec![],
13601 protocol,
13602 overlap: Default::default(),
13603 };
13604 let module = supervisor.spawn(spec.clone()).unwrap();
13605 let (_, rx) = mpsc::channel(8);
13606 let actor = Self {
13607 module,
13608 registry,
13609 forwarding,
13610 spec,
13611 health,
13612 rx,
13613 _home: home,
13614 };
13615 actor
13616 .wait_for_select(|checkpoint| checkpoint.generation > 0)
13617 .await;
13618 actor
13619 }
13620
13621 fn turns(&self) -> u64 {
13622 lock_snapshot(&self.module.inner.snapshot)
13623 .unwrap()
13624 .actor_turns
13625 }
13626
13627 async fn wait_for_select(
13628 &self,
13629 expected: impl Fn(&ActorSelectCheckpoint) -> bool,
13630 ) -> ActorSelectCheckpoint {
13631 wait_for("actor select checkpoint", || {
13632 let state = lock_snapshot(&self.module.inner.snapshot).unwrap();
13633 state.actor_select.filter(|checkpoint| {
13634 checkpoint.generation == state.spawn_generation
13635 && state.state == ModuleState::Running
13636 && state.process_alive
13637 && state.reported_pid().is_some()
13638 && expected(checkpoint)
13639 })
13640 })
13641 .await
13642 }
13643
13644 fn hello(&mut self, connection: u64, health: bool) {
13645 let manifest =
13646 subc_protocol::manifest::ModuleManifest::builder(&self.spec.module_id, "1.0.0")
13647 .protocol_ver(subc_protocol::PROTOCOL_VERSION)
13648 .build();
13649 let connection = ConnectionId::new(connection);
13650 let (tx, rx) = mpsc::channel(8);
13651 self.rx = rx;
13652 self.forwarding
13653 .register_module_connection(
13654 connection,
13655 self.spec.module_id.clone(),
13656 subc_protocol::PROTOCOL_VERSION,
13657 Concurrency::ModuleManaged,
13658 FrameSink::new(tx),
13659 )
13660 .unwrap();
13661 self.registry
13662 .register_with_control_ops(
13663 manifest,
13664 subc_protocol::PROTOCOL_VERSION,
13665 connection,
13666 if health {
13667 vec![MODULE_CONTROL_OP_HEALTH_CHECK.into()]
13668 } else {
13669 vec![]
13670 },
13671 )
13672 .unwrap();
13673 }
13674
13675 async fn probe(&mut self) -> crate::router::OutboundFrame {
13676 let frame = wait_for("outbound health probe", || match self.rx.try_recv() {
13677 Ok(frame) => Some(frame),
13678 Err(mpsc::error::TryRecvError::Empty) => None,
13679 Err(mpsc::error::TryRecvError::Disconnected) => panic!("health peer disconnected"),
13680 })
13681 .await;
13682 let body: Value = serde_json::from_slice(&frame.body).unwrap();
13683 assert_eq!(body["op"], MODULE_CONTROL_OP_HEALTH_CHECK);
13684 frame
13685 }
13686
13687 fn no_probe(&mut self) {
13688 assert!(matches!(
13689 self.rx.try_recv(),
13690 Err(mpsc::error::TryRecvError::Empty)
13691 ));
13692 }
13693
13694 fn answer(&self, connection: u64, frame: crate::router::OutboundFrame) {
13695 self.forwarding
13696 .complete_module_control_rpc(
13697 ConnectionId::new(connection),
13698 frame.header.corr,
13699 Some(MODULE_CONTROL_OP_HEALTH_CHECK),
13700 ModuleControlRpcOutcome::Response(ModuleControlResponse::HealthCheck {
13701 status: HealthStatus::Ok,
13702 detail: None,
13703 metrics: None,
13704 }),
13705 )
13706 .unwrap();
13707 }
13708 }
13709
13710 #[tokio::test(start_paused = true)]
13711 async fn no_health_children_park_for_sixty_seconds() {
13712 let none = Actor::start(ModuleProtocol::None, Duration::from_secs(30)).await;
13713 let unregistered = Actor::start(ModuleProtocol::Subc, Duration::from_secs(30)).await;
13714 let mut unadvertised = Actor::start(ModuleProtocol::Subc, Duration::from_secs(30)).await;
13715 unadvertised.hello(1, false);
13716 for _ in 0..6000 {
13721 tokio::time::advance(Duration::from_millis(10)).await;
13722 tokio::task::yield_now().await;
13723 }
13724 for actor in [&none, &unregistered, &unadvertised] {
13725 assert!(
13726 actor.turns() <= 5,
13727 "no-health actor woke {} times",
13728 actor.turns()
13729 );
13730 assert_eq!(actor.module.state().unwrap(), ModuleState::Running);
13731 }
13732 }
13733
13734 #[tokio::test(start_paused = true)]
13735 async fn late_hello_starts_a_due_probe_in_the_notification_tick() {
13736 let mut actor = Actor::start(ModuleProtocol::Subc, Duration::ZERO).await;
13740 let tick = Instant::now();
13741 actor.hello(1, true);
13742 actor.probe().await;
13743 assert_eq!(Instant::now(), tick, "HELLO must not wait for a poll");
13744 }
13745
13746 #[tokio::test(start_paused = true)]
13747 async fn hello_keeps_the_jittered_cadence_and_catalog_updates_do_not_reset_it() {
13748 let mut actor = Actor::start(ModuleProtocol::Subc, Duration::from_secs(30)).await;
13749 let hello_tick = Instant::now();
13750 actor.hello(1, true);
13751 let armed = actor
13752 .wait_for_select(|checkpoint| {
13753 checkpoint.registered_connection == Some(ConnectionId::new(1))
13754 && checkpoint.next_probe_at.is_some()
13755 })
13756 .await;
13757 let delay = jittered_health_delay(&actor.spec.module_id, 0, actor.health.cadence);
13758 assert!((Duration::from_secs(30)..Duration::from_secs(33)).contains(&delay));
13759 let first_deadline = hello_tick + delay;
13760 assert_eq!(armed.next_probe_at, Some(first_deadline));
13761 tokio::time::advance(Duration::from_secs(29)).await;
13762 let turns = actor.turns();
13763 actor
13764 .registry
13765 .replace_catalog_for_connection(ConnectionId::new(1), vec![], None, Some(true))
13766 .unwrap();
13767 let updated = actor
13768 .wait_for_select(|checkpoint| checkpoint.turn > turns)
13769 .await;
13770 assert_eq!(
13771 updated.next_probe_at,
13772 Some(first_deadline),
13773 "catalog update must not reset cadence"
13774 );
13775 actor.no_probe();
13776 tokio::time::advance(first_deadline - Instant::now()).await;
13777 let frame = actor.probe().await;
13778 actor.answer(1, frame);
13779 let next = jittered_health_delay(&actor.spec.module_id, 1, actor.health.cadence);
13780 let next_deadline = Instant::now() + next;
13781 let rearmed = actor
13782 .wait_for_select(|checkpoint| {
13783 checkpoint
13784 .next_probe_at
13785 .is_some_and(|deadline| deadline > first_deadline)
13786 })
13787 .await;
13788 assert_eq!(rearmed.next_probe_at, Some(next_deadline));
13789 tokio::time::advance(next - Duration::from_nanos(1)).await;
13790 actor.no_probe();
13791 tokio::time::advance(Duration::from_nanos(1)).await;
13792 actor.probe().await;
13793 }
13794
13795 #[tokio::test(start_paused = true)]
13796 async fn disconnect_disarms_and_reconnect_rearms_health() {
13797 let mut actor = Actor::start(ModuleProtocol::Subc, Duration::from_secs(30)).await;
13798 actor.hello(1, true);
13799 actor
13800 .wait_for_select(|checkpoint| {
13801 checkpoint.registered_connection == Some(ConnectionId::new(1))
13802 && checkpoint.next_probe_at.is_some()
13803 })
13804 .await;
13805 let turns = actor.turns();
13806 let tick = Instant::now();
13807 actor
13808 .registry
13809 .deregister_connection(ConnectionId::new(1))
13810 .unwrap();
13811 let disarmed = actor
13812 .wait_for_select(|checkpoint| {
13813 checkpoint.turn > turns && checkpoint.registered_connection.is_none()
13814 })
13815 .await;
13816 assert_eq!(disarmed.next_probe_at, None);
13817 assert_eq!(disarmed.wake_after, None);
13818 assert_eq!(
13819 Instant::now(),
13820 tick,
13821 "disconnect must disarm in the notification tick"
13822 );
13823 tokio::time::advance(Duration::from_secs(60)).await;
13824 actor.no_probe();
13825 let reconnect_tick = Instant::now();
13826 actor.hello(2, true);
13827 let rearmed = actor
13828 .wait_for_select(|checkpoint| {
13829 checkpoint.registered_connection == Some(ConnectionId::new(2))
13830 && checkpoint.next_probe_at.is_some()
13831 })
13832 .await;
13833 let delay = jittered_health_delay(&actor.spec.module_id, 0, actor.health.cadence);
13834 assert_eq!(rearmed.next_probe_at, Some(reconnect_tick + delay));
13835 tokio::time::advance(delay).await;
13836 actor.probe().await;
13837 }
13838
13839 #[tokio::test(start_paused = true)]
13840 async fn reregistration_without_health_disarms_the_old_schedule() {
13841 let mut actor = Actor::start(ModuleProtocol::Subc, Duration::from_secs(30)).await;
13842 actor.hello(1, true);
13843 actor
13844 .wait_for_select(|checkpoint| {
13845 checkpoint.registered_connection == Some(ConnectionId::new(1))
13846 && checkpoint.next_probe_at.is_some()
13847 })
13848 .await;
13849 actor
13850 .registry
13851 .deregister_connection(ConnectionId::new(1))
13852 .unwrap();
13853 actor.hello(2, false);
13854 let disarmed = actor
13855 .wait_for_select(|checkpoint| {
13856 checkpoint.registered_connection == Some(ConnectionId::new(2))
13857 })
13858 .await;
13859 assert_eq!(disarmed.next_probe_at, None);
13860 assert_eq!(disarmed.wake_after, None);
13861 tokio::time::advance(Duration::from_secs(60)).await;
13862 actor.no_probe();
13863 }
13864
13865 #[tokio::test(start_paused = true)]
13866 async fn restart_waits_for_its_own_hello_before_rearming_health() {
13867 let mut actor = Actor::start(ModuleProtocol::Subc, Duration::from_secs(30)).await;
13868 actor.hello(1, true);
13869 let armed = actor
13870 .wait_for_select(|checkpoint| {
13871 checkpoint.registered_connection == Some(ConnectionId::new(1))
13872 && checkpoint.next_probe_at.is_some()
13873 })
13874 .await;
13875 actor
13879 .registry
13880 .deregister_connection(ConnectionId::new(1))
13881 .unwrap();
13882 actor.module.restart(Some(0)).await.unwrap();
13883 let parked = actor
13884 .wait_for_select(|checkpoint| {
13885 checkpoint.generation > armed.generation
13886 && checkpoint.registered_connection.is_none()
13887 && checkpoint.next_probe_at.is_none()
13888 })
13889 .await;
13890 assert_eq!(parked.wake_after, None);
13891 actor.rx.close();
13892 while actor.rx.try_recv().is_ok() {}
13895 let turns = actor.turns();
13896 tokio::time::advance(Duration::from_secs(60)).await;
13897 let after = actor.turns();
13898 eprintln!(
13899 "restart park turns: before={turns} after={after} delta={}",
13900 after - turns
13901 );
13902 assert_eq!(after, turns, "replacement must park before HELLO");
13903 let hello_tick = Instant::now();
13904 actor.hello(2, true);
13905 let rearmed = actor
13906 .wait_for_select(|checkpoint| {
13907 checkpoint.registered_connection == Some(ConnectionId::new(2))
13908 && checkpoint.next_probe_at.is_some()
13909 })
13910 .await;
13911 let delay = jittered_health_delay(&actor.spec.module_id, 0, actor.health.cadence);
13912 assert_eq!(rearmed.next_probe_at, Some(hello_tick + delay));
13913 tokio::time::advance(delay).await;
13914 actor.probe().await;
13915 }
13916
13917 #[tokio::test(start_paused = true)]
13918 async fn rescan_adds_and_removes_http_health_without_polling() {
13919 use tokio::{
13920 io::{AsyncReadExt, AsyncWriteExt},
13921 net::TcpListener,
13922 };
13923 let mut actor = Actor::start(ModuleProtocol::None, Duration::from_secs(30)).await;
13924 let listener = TcpListener::bind("127.0.0.1:0").await.unwrap();
13925 actor.health.http = Some(format!("http://{}/health", listener.local_addr().unwrap()));
13926 actor.health.cadence = Duration::from_millis(1);
13928 let (started_tx, mut started_rx) = mpsc::channel(8);
13929 let peer = tokio::spawn(async move {
13930 loop {
13931 let (mut socket, _) = listener.accept().await.unwrap();
13932 let mut request = [0; 1024];
13933 assert!(socket.read(&mut request).await.unwrap() > 0);
13934 started_tx.send(()).await.unwrap();
13935 socket
13936 .write_all(b"HTTP/1.1 200 OK\r\nContent-Length: 0\r\n\r\n")
13937 .await
13938 .unwrap();
13939 }
13940 });
13941 let tick = Instant::now();
13942 let turns = actor.turns();
13943 actor
13944 .module
13945 .update_configuration(actor.spec.clone(), actor.health.clone(), None)
13946 .await
13947 .unwrap();
13948 let armed = actor
13949 .wait_for_select(|checkpoint| {
13950 checkpoint.turn > turns && checkpoint.next_probe_at.is_some()
13951 })
13952 .await;
13953 assert_eq!(armed.next_probe_at, Some(tick + Duration::from_millis(1)));
13954 assert_eq!(Instant::now(), tick);
13955 tokio::time::advance(Duration::from_millis(1)).await;
13956 let mut started = false;
13957 wait_for("successful HTTP probe", || {
13958 started |= started_rx.try_recv().is_ok();
13959 (started && actor.module.status().unwrap().health.status == SupervisorHealthStatus::Ok)
13960 .then_some(())
13961 })
13962 .await;
13963 assert!(started, "added HTTP check must start at its first deadline");
13964 assert_eq!(
13965 actor.module.status().unwrap().health.status,
13966 SupervisorHealthStatus::Ok
13967 );
13968 assert_eq!(Instant::now(), tick + Duration::from_millis(1));
13969 actor.health.http = None;
13970 let turns = actor.turns();
13971 actor
13972 .module
13973 .update_configuration(actor.spec.clone(), actor.health.clone(), None)
13974 .await
13975 .unwrap();
13976 let parked = actor
13977 .wait_for_select(|checkpoint| {
13978 checkpoint.turn > turns && checkpoint.next_probe_at.is_none()
13979 })
13980 .await;
13981 assert_eq!(parked.wake_after, None);
13982 let turns = actor.turns();
13983 tokio::time::advance(Duration::from_secs(60)).await;
13984 assert_eq!(
13985 actor.turns(),
13986 turns,
13987 "removed HTTP check must park the actor"
13988 );
13989 assert!(
13990 started_rx.try_recv().is_err(),
13991 "removed HTTP check must stay disarmed"
13992 );
13993 assert_eq!(
13994 actor.module.status().unwrap().health,
13995 ModuleHealthStatus::default()
13996 );
13997 peer.abort();
13998 }
13999}
14000
14001#[cfg(test)]
14002mod drain_busy_reprobe_tests {
14003 use super::*;
14007 use std::collections::BTreeMap;
14008 use std::sync::Mutex as StdMutex;
14009 use subc_protocol::{manifest::Concurrency, session::ModuleControlResponse};
14010
14011 const MODULE_ID: &str = "drain-reprobe-module";
14012 const CONNECTION: u64 = 7;
14013 const GAUGE: &str = "runs_in_flight";
14014 const CADENCE: Duration = Duration::from_secs(30);
14017 const CEILING: Duration = Duration::from_secs(30);
14018
14019 #[derive(Clone, Copy)]
14021 enum Answer {
14022 BusyUntil { until: Instant, latency: Duration },
14025 OmitsGauge,
14027 Silent,
14029 }
14030
14031 struct Drain {
14032 forwarding: Arc<ForwardingTable>,
14033 runtime: SupervisorRuntimeConfig,
14034 endpoint: crate::ModuleEndpointId,
14035 probes: Arc<StdMutex<Vec<Instant>>>,
14037 responder: tokio::task::JoinHandle<()>,
14038 }
14039
14040 impl Drain {
14041 fn start(answer: Answer) -> Self {
14042 let registry = Arc::new(Registry::default());
14043 let forwarding = Arc::new(ForwardingTable::default());
14044 let health = HealthConfig {
14045 cadence: CADENCE,
14046 ..HealthConfig::default()
14047 };
14048 let runtime = Supervisor::new_for_test(registry, RestartPolicy::new(3, Duration::ZERO))
14049 .with_forwarding(forwarding.clone())
14050 .with_health_config(health)
14051 .runtime_config();
14052 let (tx, mut rx) = mpsc::channel::<crate::router::OutboundFrame>(8);
14053 let endpoint = forwarding
14054 .register_module_connection(
14055 ConnectionId::new(CONNECTION),
14056 MODULE_ID.to_string(),
14057 subc_protocol::PROTOCOL_VERSION,
14058 Concurrency::ModuleManaged,
14059 FrameSink::new(tx),
14060 )
14061 .unwrap();
14062 let probes = Arc::new(StdMutex::new(Vec::new()));
14063 let responder = tokio::spawn({
14064 let forwarding = forwarding.clone();
14065 let probes = probes.clone();
14066 async move {
14067 while let Some(frame) = rx.recv().await {
14068 let body: Value = serde_json::from_slice(&frame.body).unwrap();
14069 assert_eq!(body["op"], MODULE_CONTROL_OP_HEALTH_CHECK);
14070 probes.lock().unwrap().push(Instant::now());
14071 let metrics = match answer {
14072 Answer::BusyUntil { until, latency } => {
14073 sleep(latency).await;
14074 let busy = u64::from(Instant::now() < until);
14075 serde_json::json!({ GAUGE: busy })
14076 }
14077 Answer::OmitsGauge => serde_json::json!({ "queue_depth": 0 }),
14078 Answer::Silent => continue,
14079 };
14080 let _ = forwarding.complete_module_control_rpc(
14083 ConnectionId::new(CONNECTION),
14084 frame.header.corr,
14085 Some(MODULE_CONTROL_OP_HEALTH_CHECK),
14086 ModuleControlRpcOutcome::Response(ModuleControlResponse::HealthCheck {
14087 status: HealthStatus::Ok,
14088 detail: None,
14089 metrics: Some(metrics),
14090 }),
14091 );
14092 }
14093 }
14094 });
14095 Self {
14096 forwarding,
14097 runtime,
14098 endpoint,
14099 probes,
14100 responder,
14101 }
14102 }
14103
14104 async fn run(&self, ceiling: Duration) -> (bool, Duration) {
14107 let (drained, elapsed, _) = self.run_traced(ceiling).await;
14108 (drained, elapsed)
14109 }
14110
14111 async fn run_traced(&self, ceiling: Duration) -> (bool, Duration, DrainGaugeTrace) {
14113 let started = Instant::now();
14114 let mut trace = DrainGaugeTrace::new(&[GAUGE.to_string()]);
14115 let drained = wait_for_forwarding_quiescence(
14116 &self.forwarding,
14117 MODULE_ID,
14118 &self.runtime,
14119 self.endpoint,
14120 started + ceiling,
14121 &[GAUGE.to_string()],
14122 DrainScope::Active,
14123 &mut trace,
14124 )
14125 .await
14126 .unwrap();
14127 (drained, started.elapsed(), trace)
14128 }
14129
14130 fn probes(&self) -> Vec<Instant> {
14131 self.probes.lock().unwrap().clone()
14132 }
14133 }
14134
14135 impl Drop for Drain {
14136 fn drop(&mut self) {
14137 self.responder.abort();
14138 }
14139 }
14140
14141 #[tokio::test(start_paused = true)]
14142 async fn busy_at_first_probe_then_quiet_drains_long_before_the_ceiling() {
14143 let drain = Drain::start(Answer::BusyUntil {
14144 until: Instant::now() + Duration::from_secs(1),
14145 latency: Duration::ZERO,
14146 });
14147 let (drained, elapsed) = drain.run(CEILING).await;
14148 assert!(
14149 drained,
14150 "gauges that read zero during the drain must let it finish drained, \
14151 not wait out the {CADENCE:?} health cadence (elapsed {elapsed:?})"
14152 );
14153 assert!(
14154 elapsed < Duration::from_secs(3),
14155 "drain took {elapsed:?} after the gauges went quiet at 1s"
14156 );
14157 let probes = drain.probes();
14158 assert!(
14159 probes.len() >= 2,
14160 "the module was busy at the first probe, so a quiet verdict needs a second: {probes:?}"
14161 );
14162 }
14163
14164 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
14169 async fn an_instant_busy_first_answer_does_not_stop_later_probes_in_real_time() {
14170 let drain = Drain::start(Answer::BusyUntil {
14171 until: Instant::now() + Duration::from_millis(600),
14172 latency: Duration::ZERO,
14173 });
14174 let (drained, elapsed, trace) = drain.run_traced(CEILING).await;
14175 assert!(drained, "the gauge fell at 600ms: {trace}");
14176 assert!(
14177 elapsed < Duration::from_secs(3),
14178 "drain took {elapsed:?} after the gauge fell at 600ms: {trace}"
14179 );
14180 let probes = drain.probes();
14181 assert!(
14182 probes.len() >= 2 && trace.probes as usize == probes.len(),
14183 "the drain must re-probe after the first busy answer: {probes:?}, {trace}"
14184 );
14185 }
14186
14187 #[tokio::test(start_paused = true)]
14188 async fn busy_throughout_ends_undrained_at_the_ceiling_with_bounded_probes() {
14189 let started = Instant::now();
14190 let drain = Drain::start(Answer::BusyUntil {
14191 until: started + Duration::from_secs(3600),
14192 latency: Duration::ZERO,
14193 });
14194 let (drained, elapsed) = drain.run(CEILING).await;
14195 assert!(!drained, "a module busy throughout must not drain");
14196 assert!(
14197 elapsed >= CEILING && elapsed < CEILING + drain_busy_reprobe_interval(),
14198 "a busy module is held until the ceiling and no longer: {elapsed:?}"
14199 );
14200 let probes = drain.probes();
14203 let max_probes = (CEILING.as_millis() / drain_busy_reprobe_interval().as_millis()) + 1;
14204 assert!(
14205 probes.len() as u128 <= max_probes,
14206 "{} probes in a {CEILING:?} drain exceeds the {max_probes} the interval allows",
14207 probes.len()
14208 );
14209 for pair in probes.windows(2) {
14210 assert!(
14211 pair[1] - pair[0] >= drain_busy_reprobe_interval(),
14212 "consecutive probes {:?} apart",
14213 pair[1] - pair[0]
14214 );
14215 }
14216 }
14217
14218 #[tokio::test(start_paused = true)]
14219 async fn a_slow_probe_is_never_overlapped_by_the_next() {
14220 let latency = Duration::from_secs(1);
14223 let drain = Drain::start(Answer::BusyUntil {
14224 until: Instant::now() + Duration::from_secs(3600),
14225 latency,
14226 });
14227 let (drained, _) = drain.run(Duration::from_secs(5)).await;
14228 assert!(!drained);
14229 let probes = drain.probes();
14230 assert!(probes.len() >= 2, "the drain must re-probe: {probes:?}");
14231 for pair in probes.windows(2) {
14232 assert!(
14233 pair[1] - pair[0] >= latency + drain_busy_reprobe_interval(),
14234 "a probe started {:?} after the previous one, before it could finish",
14235 pair[1] - pair[0]
14236 );
14237 }
14238 }
14239
14240 #[tokio::test(start_paused = true)]
14241 async fn an_unanswered_probe_cannot_hold_the_drain_past_its_ceiling() {
14242 let drain = Drain::start(Answer::Silent);
14243 let ceiling = Duration::from_secs(2);
14244 let (drained, elapsed) = drain.run(ceiling).await;
14245 assert!(!drained, "an unanswered probe counts as busy");
14246 assert!(
14247 elapsed >= ceiling && elapsed < ceiling + drain_busy_reprobe_interval(),
14248 "the probe deadline is capped at the drain ceiling: {elapsed:?}"
14249 );
14250 assert_eq!(
14251 drain.probes().len(),
14252 1,
14253 "the unanswered probe holds the only slot until the ceiling"
14254 );
14255 }
14256
14257 #[derive(Clone, Default)]
14259 struct LogCapture {
14260 events: Arc<StdMutex<Vec<BTreeMap<String, String>>>>,
14261 }
14262
14263 impl<S: tracing::Subscriber> tracing_subscriber::Layer<S> for LogCapture {
14264 fn on_event(
14265 &self,
14266 event: &tracing::Event<'_>,
14267 _context: tracing_subscriber::layer::Context<'_, S>,
14268 ) {
14269 struct Fields(BTreeMap<String, String>);
14270 impl tracing::field::Visit for Fields {
14271 fn record_debug(&mut self, field: &tracing::field::Field, value: &dyn fmt::Debug) {
14272 self.0
14273 .insert(field.name().to_string(), format!("{value:?}"));
14274 }
14275 }
14276 let mut fields = Fields(BTreeMap::new());
14277 event.record(&mut fields);
14278 self.events.lock().unwrap().push(fields.0);
14279 }
14280 }
14281
14282 impl LogCapture {
14283 fn install(&self) -> tracing::subscriber::DefaultGuard {
14284 use tracing_subscriber::layer::SubscriberExt;
14285 tracing::subscriber::set_default(tracing_subscriber::registry().with(self.clone()))
14286 }
14287
14288 fn with_message(&self, needle: &str) -> Vec<BTreeMap<String, String>> {
14289 self.events
14290 .lock()
14291 .unwrap()
14292 .iter()
14293 .filter(|fields| fields.get("message").is_some_and(|m| m.contains(needle)))
14294 .cloned()
14295 .collect()
14296 }
14297 }
14298
14299 #[tokio::test(start_paused = true)]
14304 async fn a_missing_declared_gauge_is_named_on_the_drain_timeout_line() {
14305 let capture = LogCapture::default();
14306 let _guard = capture.install();
14307 let drain = Drain::start(Answer::OmitsGauge);
14308 let (drained, _, trace) = drain.run_traced(Duration::from_secs(2)).await;
14309 assert!(!drained, "a gauge never reported cannot read quiet");
14310 assert!(trace.probes >= 2, "the drain must re-probe: {trace:?}");
14311
14312 let missing = capture.with_message("lacks a declared busy gauge");
14313 assert_eq!(missing.len(), 1, "one warning per drain: {missing:?}");
14314 assert_eq!(
14315 missing[0].get("missing_gauges").map(String::as_str),
14316 Some(GAUGE)
14317 );
14318
14319 warn_drain_timed_out(
14320 MODULE_ID,
14321 Duration::from_secs(2),
14322 RouteCloseReason::Restart,
14323 &Default::default(),
14324 &trace,
14325 );
14326 let timeouts = capture.with_message("route drain timed out");
14327 assert_eq!(timeouts.len(), 1, "{timeouts:?}");
14328 let gauges = &timeouts[0]["busy_gauges"];
14329 assert!(
14330 gauges.starts_with(&format!("{GAUGE}: missing_from_health_body @")),
14331 "the timeout line must name the gauge and why it read busy: {gauges}"
14332 );
14333 assert!(
14334 gauges.ends_with("; no value ever read"),
14335 "no value was ever read for the gauge: {gauges}"
14336 );
14337 assert_eq!(timeouts[0]["gauge_probes"], trace.probes.to_string());
14338 }
14339
14340 #[tokio::test(start_paused = true)]
14344 async fn the_drain_trace_names_a_busy_value_and_an_unanswered_probe() {
14345 let busy = Drain::start(Answer::BusyUntil {
14346 until: Instant::now() + Duration::from_secs(3600),
14347 latency: Duration::ZERO,
14348 });
14349 let (_, _, trace) = busy.run_traced(Duration::from_secs(1)).await;
14350 let rendered = trace.to_string();
14351 assert!(
14352 rendered.starts_with(&format!("{GAUGE}: non_zero(1) @"))
14353 && rendered.contains("; last value non_zero(1) @"),
14354 "{rendered}"
14355 );
14356
14357 let silent = Drain::start(Answer::Silent);
14358 let (_, _, trace) = silent.run_traced(Duration::from_secs(1)).await;
14359 assert_eq!(
14362 trace.to_string(),
14363 format!("{GAUGE}: probe_failed(no-answer) @1000ms; no value ever read")
14364 );
14365 assert_eq!(trace.probes, 1);
14366 }
14367
14368 #[test]
14370 fn a_gauge_that_is_not_a_count_is_busy_and_shown_as_sent() {
14371 let gauges = [GAUGE.to_string()];
14372 let verdicts = gauge_verdicts(Some(&serde_json::json!({ GAUGE: 0.0 })), &gauges);
14373 assert_eq!(verdicts, vec![GaugeVerdict::NotACount("0.0".to_string())]);
14374 assert_eq!(
14375 busy_gauge_observation(&verdicts),
14376 BusyGaugeObservation::Busy
14377 );
14378 let zero = gauge_verdicts(Some(&serde_json::json!({ GAUGE: 0 })), &gauges);
14379 assert_eq!(
14380 busy_gauge_observation(&zero),
14381 BusyGaugeObservation::Quiescent
14382 );
14383 let none = gauge_verdicts(None, &gauges);
14384 assert_eq!(busy_gauge_observation(&none), BusyGaugeObservation::Omitted);
14385 }
14386}
14387
14388#[cfg(test)]
14389mod health_evidence_tests {
14390 use super::{HealthProbeError, HealthProbeEvidence};
14391 use std::collections::HashSet;
14392
14393 #[test]
14401 fn only_a_dead_lane_is_proof_of_death() {
14402 assert!(HealthProbeError::lane_dead("gone").is_proof_of_death());
14403 assert!(!HealthProbeError::no_answer("timed out").is_proof_of_death());
14407 assert!(!HealthProbeError::bad_answer("garbage").is_proof_of_death());
14408 assert!(!HealthProbeError::misconfigured("no table").is_proof_of_death());
14409 }
14410
14411 #[test]
14417 fn every_evidence_class_has_a_distinct_label() {
14418 let labels = [
14419 HealthProbeError::lane_dead("").label(),
14420 HealthProbeError::no_answer("").label(),
14421 HealthProbeError::bad_answer("").label(),
14422 HealthProbeError::misconfigured("").label(),
14423 ];
14424 let unique: HashSet<_> = labels.iter().collect();
14425 assert_eq!(unique.len(), labels.len(), "labels collided: {labels:?}");
14426 }
14427
14428 #[test]
14434 fn classification_preserves_the_original_message() {
14435 let err = HealthProbeError::no_answer("module did not answer within 5s");
14436 assert_eq!(err.to_string(), "module did not answer within 5s");
14437 assert!(matches!(err.evidence, HealthProbeEvidence::NoAnswer));
14438 }
14439}
14440
14441#[cfg(test)]
14442mod health_tombstone_tests {
14443 use std::{path::PathBuf, sync::Arc, time::Duration};
14444
14445 use subc_protocol::{
14446 manifest::Concurrency,
14447 session::{HealthStatus, ModuleControlResponse},
14448 };
14449 use tokio::sync::mpsc;
14450
14451 use super::{
14452 probe_module_health, HealthAction, HealthConfig, HealthProbeEvidence, ModuleProtocol,
14453 ModuleSpec, RestartPolicy, Supervisor, SupervisorRuntimeConfig,
14454 };
14455 use crate::{
14456 control::ControlHandler,
14457 forwarding::{ForwardingTable, ModuleControlRpcCompletion, ModuleControlRpcOutcome},
14458 registry::{ConnectionId, Registry},
14459 router::FrameSink,
14460 };
14461
14462 struct ProbeHarness {
14463 spec: ModuleSpec,
14464 runtime: SupervisorRuntimeConfig,
14465 forwarding: Arc<ForwardingTable>,
14466 module_connection: ConnectionId,
14467 module_rx: mpsc::Receiver<crate::router::OutboundFrame>,
14468 handler: ControlHandler,
14469 module: super::SupervisedModule,
14470 }
14471
14472 fn probe_harness() -> ProbeHarness {
14473 let registry = Arc::new(Registry::default());
14474 let forwarding = Arc::new(ForwardingTable::default());
14475 let supervisor_handle = super::SupervisorHandle::new();
14476 let health = HealthConfig {
14477 http: None,
14478 cadence: Duration::from_secs(30),
14479 deadline: Duration::from_secs(5),
14480 failure_threshold: 3,
14481 on_degraded: HealthAction::Report,
14482 on_failing: HealthAction::Report,
14483 critical: false,
14484 };
14485 let supervisor = Supervisor::new_for_test(Arc::clone(®istry), RestartPolicy::default())
14486 .with_forwarding(Arc::clone(&forwarding))
14487 .with_handle(supervisor_handle.clone())
14488 .with_health_config(health);
14489 let spec = ModuleSpec {
14490 module_id: "late-health-module".to_string(),
14491 program: PathBuf::from("disabled-module"),
14492 args: Vec::new(),
14493 env: Vec::new(),
14494 reserved: false,
14495 reserved_prefixes: Vec::new(),
14496 protocol: ModuleProtocol::Subc,
14497 overlap: Default::default(),
14498 };
14499 let module = supervisor
14500 .supervise_configured(spec.clone(), false)
14501 .unwrap();
14502 let runtime = supervisor.runtime_config();
14503 let handler = ControlHandler::with_forwarding(registry, Arc::clone(&forwarding))
14504 .with_supervisor(supervisor_handle);
14505 let module_connection = ConnectionId::new(700);
14506 let (module_tx, module_rx) = mpsc::channel(8);
14507 forwarding
14508 .register_module_connection(
14509 module_connection,
14510 spec.module_id.clone(),
14511 subc_protocol::PROTOCOL_VERSION,
14512 Concurrency::ModuleManaged,
14513 FrameSink::new(module_tx),
14514 )
14515 .unwrap();
14516
14517 ProbeHarness {
14518 spec,
14519 runtime,
14520 forwarding,
14521 module_connection,
14522 module_rx,
14523 handler,
14524 module,
14525 }
14526 }
14527
14528 async fn finish_after(
14529 harness: &mut ProbeHarness,
14530 stall: Duration,
14531 ) -> ModuleControlRpcCompletion {
14532 assert!(stall > harness.runtime.health.deadline);
14533 let deadline = harness.runtime.health.deadline;
14534 let probe = probe_module_health(&harness.spec.module_id, &harness.runtime, None);
14535 let answer = async {
14536 let frame = harness.module_rx.recv().await.expect("health.check frame");
14537 tokio::time::advance(deadline).await;
14538 tokio::task::yield_now().await;
14539 tokio::time::advance(stall - deadline).await;
14540 harness
14541 .forwarding
14542 .complete_module_control_rpc(
14543 harness.module_connection,
14544 frame.header.corr,
14545 Some("health.check"),
14546 ModuleControlRpcOutcome::Response(ModuleControlResponse::HealthCheck {
14547 status: HealthStatus::Ok,
14548 detail: None,
14549 metrics: None,
14550 }),
14551 )
14552 .unwrap()
14553 };
14554 let (probe_result, completion) = tokio::join!(probe, answer);
14555 let err = probe_result.expect_err("probe must miss its deadline");
14556 assert!(matches!(err.evidence, HealthProbeEvidence::NoAnswer));
14557 completion
14558 }
14559
14560 async fn time_out_without_answer(harness: &mut ProbeHarness) {
14561 let deadline = harness.runtime.health.deadline;
14562 let probe = probe_module_health(&harness.spec.module_id, &harness.runtime, None);
14563 let exhaust_deadline = async {
14564 let _frame = harness.module_rx.recv().await.expect("health.check frame");
14565 tokio::time::advance(deadline).await;
14566 tokio::task::yield_now().await;
14567 };
14568 let (probe_result, ()) = tokio::join!(probe, exhaust_deadline);
14569 let err = probe_result.expect_err("probe must miss its deadline");
14570 assert!(matches!(err.evidence, HealthProbeEvidence::NoAnswer));
14571 }
14572
14573 async fn run_probe_cycle(harness: &mut ProbeHarness, answer: bool) {
14574 let registry = Arc::clone(&harness.module.inner.registry);
14575 let snapshot = Arc::clone(&harness.module.inner.snapshot);
14576 let process_liveness = super::SupervisorProcessLiveness::default();
14577 let mut child = None;
14578 let cycle = super::run_health_probe_cycle(
14579 &harness.spec,
14580 &harness.runtime,
14581 ®istry,
14582 &process_liveness,
14583 &snapshot,
14584 &mut child,
14585 );
14586 let peer = async {
14587 let frame = harness.module_rx.recv().await.expect("health.check frame");
14588 if answer {
14589 harness
14590 .forwarding
14591 .complete_module_control_rpc(
14592 harness.module_connection,
14593 frame.header.corr,
14594 Some("health.check"),
14595 ModuleControlRpcOutcome::Response(ModuleControlResponse::HealthCheck {
14596 status: HealthStatus::Ok,
14597 detail: None,
14598 metrics: Some(serde_json::json!({"ready": true})),
14599 }),
14600 )
14601 .unwrap();
14602 } else {
14603 tokio::time::advance(harness.runtime.health.deadline).await;
14604 tokio::task::yield_now().await;
14605 }
14606 };
14607 tokio::join!(cycle, peer);
14608 }
14609
14610 #[tokio::test(start_paused = true)]
14611 async fn unanswered_probe_is_unknown_until_threshold_and_ok_report_recovers() {
14612 let mut harness = probe_harness();
14613 let monitor = harness.module.inner.monitor.lock().unwrap().take().unwrap();
14617 monitor.abort();
14618 let _ = monitor.await;
14619 super::update_snapshot(&harness.module.inner.snapshot, None, |state| {
14620 state.enabled = true;
14621 state.state = super::ModuleState::Running;
14622 state.process_alive = true;
14623 })
14624 .unwrap();
14625
14626 run_probe_cycle(&mut harness, true).await;
14627 assert_eq!(
14628 harness.module.status().unwrap().health.status,
14629 super::SupervisorHealthStatus::Ok
14630 );
14631
14632 for failures in 1..harness.runtime.health.failure_threshold {
14633 run_probe_cycle(&mut harness, false).await;
14634 let status = harness.module.status().unwrap();
14635 assert_eq!(status.health.status, super::SupervisorHealthStatus::Unknown);
14636 assert_eq!(status.health.consecutive_failures, failures);
14637 assert!(status.health.last_probe_ms.is_some());
14638 assert!(status.health.detail.unwrap().starts_with("[no-answer]"));
14639 assert!(status.health.metrics.is_none());
14640 assert_eq!(status.state, super::ModuleState::Running);
14641 assert!(status.process_alive);
14642 assert_eq!(status.restart_count, 0);
14643 assert_eq!(status.lifetime_restarts, 0);
14644 assert!(status.health.last_action.is_none());
14645 assert!(harness.runtime.scheduled_respawn.lock().unwrap().is_none());
14646 }
14647
14648 run_probe_cycle(&mut harness, true).await;
14649 let recovered = harness.module.status().unwrap();
14650 assert_eq!(recovered.health.status, super::SupervisorHealthStatus::Ok);
14651 assert_eq!(recovered.health.consecutive_failures, 0);
14652 assert!(recovered.health.detail.is_none());
14653 assert_eq!(
14654 recovered.health.metrics,
14655 Some(serde_json::json!({"ready": true}))
14656 );
14657 assert_eq!(recovered.lifetime_restarts, 0);
14658
14659 for failures in 1..=harness.runtime.health.failure_threshold {
14660 run_probe_cycle(&mut harness, false).await;
14661 let status = harness.module.status().unwrap();
14662 assert_eq!(status.health.consecutive_failures, failures);
14663 if failures < harness.runtime.health.failure_threshold {
14664 assert_eq!(status.health.status, super::SupervisorHealthStatus::Unknown);
14665 assert_eq!(status.state, super::ModuleState::Running);
14666 assert_eq!(status.lifetime_restarts, 0);
14667 assert!(harness.runtime.scheduled_respawn.lock().unwrap().is_none());
14668 } else {
14669 assert_eq!(
14670 status.health.status,
14671 super::SupervisorHealthStatus::Unresponsive
14672 );
14673 assert_eq!(status.state, super::ModuleState::Restarting);
14674 assert_eq!(status.restart_count, 1);
14675 assert_eq!(status.lifetime_restarts, 1);
14676 assert_eq!(status.health.last_action.as_deref(), Some("restart"));
14677 assert!(harness.runtime.scheduled_respawn.lock().unwrap().is_some());
14678 }
14679 }
14680 }
14681
14682 #[tokio::test(start_paused = true)]
14683 async fn late_health_answers_record_start_anchored_latency_for_two_stalls() {
14684 let mut harness = probe_harness();
14685
14686 let first = finish_after(&mut harness, Duration::from_secs(8)).await;
14687 let first_latency = match &first {
14688 ModuleControlRpcCompletion::LateHealthAnswer { latency, .. } => *latency,
14689 other => panic!("late answer was not retained: {other:?}"),
14690 };
14691 assert!(harness.handler.observe_module_control_completion(first));
14692
14693 let second = finish_after(&mut harness, Duration::from_secs(11)).await;
14694 let second_latency = match &second {
14695 ModuleControlRpcCompletion::LateHealthAnswer { latency, .. } => *latency,
14696 other => panic!("late answer was not retained: {other:?}"),
14697 };
14698 assert!(harness.handler.observe_module_control_completion(second));
14699
14700 assert_eq!(first_latency, Duration::from_secs(8));
14701 assert_eq!(
14702 second_latency - first_latency,
14703 Duration::from_secs(3),
14704 "latency must grow linearly with the additional stall"
14705 );
14706 let health = harness.module.status().unwrap().health;
14707 assert_eq!(health.late_answer_count, 2);
14708 assert_eq!(health.last_late_answer_latency_ms, Some(11_000));
14709 }
14710
14711 #[tokio::test(start_paused = true)]
14719 async fn late_answer_clears_the_consecutive_failure_streak() {
14720 let mut harness = probe_harness();
14721
14722 time_out_without_answer(&mut harness).await;
14724 harness
14725 .module
14726 .record_health_probe_failure_for_test("[no-answer] test miss")
14727 .unwrap();
14728 assert_eq!(
14729 harness.module.status().unwrap().health.consecutive_failures,
14730 1,
14731 "precondition: the miss must be on the streak before the late answer"
14732 );
14733
14734 let late = finish_after(&mut harness, Duration::from_secs(9)).await;
14736 assert!(matches!(
14737 late,
14738 ModuleControlRpcCompletion::LateHealthAnswer { .. }
14739 ));
14740 assert!(harness.handler.observe_module_control_completion(late));
14741
14742 let health = harness.module.status().unwrap().health;
14743 assert_eq!(
14744 health.consecutive_failures, 0,
14745 "a late answer is an answer: the streak must reset"
14746 );
14747 assert_eq!(health.late_answer_count, 1);
14748 }
14749
14750 #[tokio::test(start_paused = true)]
14751 async fn repeated_serial_probe_cycles_keep_one_tombstone_per_endpoint() {
14752 let mut harness = probe_harness();
14753
14754 for _ in 0..20 {
14755 time_out_without_answer(&mut harness).await;
14756 assert_eq!(
14757 harness.forwarding.health_probe_tombstone_count().unwrap(),
14758 1
14759 );
14760 }
14761 }
14762}
14763
14764#[cfg(test)]
14765mod child_env_tests {
14766 #[cfg(windows)]
14767 use super::{
14768 clear_current_process_facts, observed_launch_nonce_source, ModuleState, SupervisorSnapshot,
14769 };
14770 #[cfg(windows)]
14771 use std::sync::{Arc, OnceLock};
14772 #[cfg(windows)]
14773 #[test]
14774 fn windows_wire_spawn_prepares_pipe_and_keeps_phase_one_env() {
14775 let spec = spec(Vec::new());
14776 let mut command = Command::new("unused");
14777 let handoff = apply_wire_spawn_args(&mut command, &spec, None, None)
14778 .unwrap()
14779 .unwrap();
14780 let name = command
14781 .as_std()
14782 .get_envs()
14783 .find(|(key, _)| *key == OsStr::new(subc_os::launch_nonce::LAUNCH_NONCE_PIPE_ENV))
14784 .and_then(|(_, value)| value)
14785 .unwrap();
14786 assert_eq!(name, OsStr::new(handoff.name()));
14787 assert!(handoff.name().starts_with(r"\\.\pipe\subc-launch-"));
14788 assert!(command.as_std().get_envs().any(|(key, value)| key
14789 == OsStr::new(subc_os::LAUNCH_NONCE_PIPE_FALLBACK_ENV)
14790 && value == Some(OsStr::new("env"))));
14791 assert!(command
14792 .as_std()
14793 .get_envs()
14794 .any(|(key, value)| key == OsStr::new(SUBC_LAUNCH_NONCE_ENV) && value.is_some()));
14795 }
14796
14797 #[cfg(windows)]
14798 #[test]
14799 fn windows_source_reporting_tracks_consumption_and_resets_per_spawn() {
14800 let mut snapshot = SupervisorSnapshot::new(ModuleState::Running, true);
14801 snapshot.process_alive = true;
14802 snapshot.spawned_protocol = Some(ModuleProtocol::Subc);
14803 assert_eq!(
14804 observed_launch_nonce_source(&snapshot).as_deref(),
14805 Some("env")
14806 );
14807 let consumed = Arc::new(OnceLock::new());
14808 snapshot.pipe_consumed = Some(consumed.clone());
14809 assert_eq!(
14810 observed_launch_nonce_source(&snapshot).as_deref(),
14811 Some("env")
14812 );
14813 consumed.set(()).unwrap();
14814 assert_eq!(
14815 observed_launch_nonce_source(&snapshot).as_deref(),
14816 Some("pipe")
14817 );
14818 snapshot.pipe_consumed = Some(Arc::new(OnceLock::new()));
14819 assert_eq!(
14820 observed_launch_nonce_source(&snapshot).as_deref(),
14821 Some("env")
14822 );
14823 clear_current_process_facts(&mut snapshot);
14824 assert_eq!(observed_launch_nonce_source(&snapshot), None);
14825 }
14826 use super::{
14827 apply_child_env, apply_spawn_role, apply_wire_spawn_args, LaunchSpecDigest, ModuleProtocol,
14828 ModuleSpec, SpawnRole, SupervisorHandle, SPAWN_ROLE_SWAP_CANDIDATE, SUBC_ARG,
14829 SUBC_LAUNCH_NONCE_ENV, SUBC_MODULE_ID_ENV, SUBC_SPAWN_ROLE_ENV,
14830 };
14831 use std::{ffi::OsStr, path::PathBuf};
14832 use tokio::process::Command;
14833
14834 fn spec(env: Vec<(String, String)>) -> ModuleSpec {
14835 ModuleSpec {
14836 module_id: "env-plan".to_string(),
14837 program: PathBuf::from("/nonexistent"),
14838 args: Vec::new(),
14839 env,
14840 reserved: false,
14841 reserved_prefixes: Vec::new(),
14842 protocol: ModuleProtocol::Subc,
14843 overlap: Default::default(),
14844 }
14845 }
14846
14847 #[test]
14848 fn configured_launch_spec_digest_is_canonical_and_excludes_injected_inputs() {
14849 let base = spec(vec![("A".into(), "B=C".into()), ("Z".into(), "v".into())]);
14850 let digest = LaunchSpecDigest::from_spec(&base);
14851 let mut reordered = base.clone();
14852 reordered.env.reverse();
14853 reordered.module_id = "different-id".into();
14854 reordered.reserved = true;
14855 reordered.overlap = Default::default();
14856 assert_eq!(digest, LaunchSpecDigest::from_spec(&reordered));
14857 let mut delimiter_collision = base.clone();
14858 delimiter_collision.env[0] = ("A=B".into(), "C".into());
14859 assert_ne!(digest, LaunchSpecDigest::from_spec(&delimiter_collision));
14860 let mut changed = base.clone();
14861 changed.protocol = ModuleProtocol::None;
14862 assert_ne!(digest, LaunchSpecDigest::from_spec(&changed));
14863 changed = base.clone();
14864 changed.args.push("--flag".into());
14865 assert_ne!(digest, LaunchSpecDigest::from_spec(&changed));
14866 changed = base.clone();
14867 changed.program = PathBuf::from("/other");
14868 assert_ne!(digest, LaunchSpecDigest::from_spec(&changed));
14869 changed = base.clone();
14870 changed.env[1].1 = "other".into();
14871 assert_ne!(digest, LaunchSpecDigest::from_spec(&changed));
14872 assert!(!format!("{digest:?}").contains("B=C"));
14873 assert!(!format!("{digest:?}").contains("[u8"));
14874 }
14875
14876 #[cfg(unix)]
14877 #[test]
14878 fn native_program_bytes_do_not_collapse_through_lossy_utf8() {
14879 use std::os::unix::ffi::OsStringExt;
14880 let mut left = spec(Vec::new());
14881 left.program = std::ffi::OsString::from_vec(vec![b'/', 0xff]).into();
14882 let mut right = left.clone();
14883 right.program = std::ffi::OsString::from_vec(vec![b'/', 0xfe]).into();
14884 assert_eq!(
14885 left.program.to_string_lossy(),
14886 right.program.to_string_lossy()
14887 );
14888 assert_ne!(
14889 LaunchSpecDigest::from_spec(&left),
14890 LaunchSpecDigest::from_spec(&right)
14891 );
14892 }
14893
14894 #[test]
14908 fn ambient_ck_log_is_removed_and_a_configured_one_survives() {
14909 let mut command = Command::new("/nonexistent");
14910 apply_child_env(&mut command, &spec(Vec::new()));
14911 let removed = command
14912 .as_std()
14913 .get_envs()
14914 .any(|(key, value)| key == OsStr::new("CK_LOG") && value.is_none());
14915 assert!(
14916 removed,
14917 "ambient CK_LOG must be explicitly removed for an unconfigured module"
14918 );
14919
14920 let mut configured = Command::new("/nonexistent");
14921 apply_child_env(
14922 &mut configured,
14923 &spec(vec![("CK_LOG".to_string(), "debug".to_string())]),
14924 );
14925 let effective = configured
14926 .as_std()
14927 .get_envs()
14928 .filter(|(key, _)| *key == OsStr::new("CK_LOG"))
14929 .last()
14930 .map(|(_, value)| value.map(|v| v.to_string_lossy().into_owned()));
14931 assert_eq!(
14932 effective,
14933 Some(Some("debug".to_string())),
14934 "a module's configured CK_LOG must survive the ambient removal"
14935 );
14936 }
14937
14938 #[test]
14947 fn protocol_none_spawn_carries_no_subc_argument_and_no_nonce() {
14948 let connection_file = std::path::Path::new("/run/subc-connection.json");
14949 let handle = SupervisorHandle::new();
14950
14951 let mut none_spec = spec(Vec::new());
14952 none_spec.protocol = ModuleProtocol::None;
14953 let mut none = Command::new("/nonexistent");
14954 let none_handoff =
14955 apply_wire_spawn_args(&mut none, &none_spec, Some(connection_file), Some(&handle))
14956 .expect("protocol-none spawn args apply");
14957 assert!(
14958 none_handoff.is_none(),
14959 "protocol:none spawn must not receive a nonce descriptor"
14960 );
14961 #[cfg(windows)]
14962 assert!(!none.as_std().get_envs().any(|(key, value)| key
14963 == OsStr::new(subc_os::LAUNCH_NONCE_PIPE_FALLBACK_ENV)
14964 && value.is_some()));
14965 assert!(
14966 !none.as_std().get_envs().any(|(key, value)| key
14967 == OsStr::new(subc_os::LAUNCH_NONCE_FD_ENV)
14968 && value.is_some()),
14969 "protocol:none spawn must not name a nonce descriptor"
14970 );
14971 let none_args: Vec<String> = none
14972 .as_std()
14973 .get_args()
14974 .map(|a| a.to_string_lossy().into_owned())
14975 .collect();
14976 assert!(
14977 !none_args.iter().any(|a| a == SUBC_ARG),
14978 "protocol:none argv must not carry --subc; got {none_args:?}"
14979 );
14980 let none_has_nonce = none
14981 .as_std()
14982 .get_envs()
14983 .any(|(key, value)| key == OsStr::new(SUBC_LAUNCH_NONCE_ENV) && value.is_some());
14984 assert!(
14985 !none_has_nonce,
14986 "protocol:none spawn must not receive a launch nonce"
14987 );
14988 let none_has_module_id = none
14989 .as_std()
14990 .get_envs()
14991 .any(|(key, value)| key == OsStr::new(SUBC_MODULE_ID_ENV) && value.is_some());
14992 assert!(
14993 none_has_module_id,
14994 "SUBC_MODULE_ID is inert and stays on every path"
14995 );
14996 assert!(
14997 handle.spawn_nonce(&none_spec.module_id).is_none(),
14998 "no nonce record for a process that will never present one"
14999 );
15000
15001 let wire_spec = spec(Vec::new());
15003 let mut wire = Command::new("/nonexistent");
15004 let wire_handoff =
15005 apply_wire_spawn_args(&mut wire, &wire_spec, Some(connection_file), Some(&handle))
15006 .expect("subc-wire spawn args apply");
15007 let wire_fd_env = wire
15008 .as_std()
15009 .get_envs()
15010 .find(|(key, _)| *key == OsStr::new(subc_os::LAUNCH_NONCE_FD_ENV))
15011 .and_then(|(_, value)| value.map(|v| v.to_string_lossy().into_owned()));
15012 #[cfg(unix)]
15013 assert_eq!(
15014 wire_fd_env,
15015 Some(wire_handoff.expect("a descriptor handoff").fd_env_value()),
15016 "a subc-wire spawn names the pipe it will receive at descriptor 3"
15017 );
15018 #[cfg(windows)]
15019 assert!(wire_handoff.is_some() && wire_fd_env.is_none());
15020 #[cfg(not(any(unix, windows)))]
15021 assert!(wire_handoff.is_none() && wire_fd_env.is_none());
15022 let wire_args: Vec<String> = wire
15023 .as_std()
15024 .get_args()
15025 .map(|a| a.to_string_lossy().into_owned())
15026 .collect();
15027 assert_eq!(
15028 wire_args,
15029 vec![
15030 SUBC_ARG.to_string(),
15031 connection_file.to_string_lossy().into_owned()
15032 ],
15033 "a subc-wire spawn still carries --subc <path>"
15034 );
15035 assert_eq!(
15036 wire.as_std()
15037 .get_envs()
15038 .any(|(key, value)| key == OsStr::new(SUBC_LAUNCH_NONCE_ENV) && value.is_some()),
15039 !cfg!(unix),
15040 "only Windows supplies the environment nonce"
15041 );
15042 assert!(handle.spawn_nonce(&wire_spec.module_id).is_some());
15043 }
15044
15045 #[test]
15055 fn plain_spawn_removes_the_spawn_role_even_when_the_spec_sets_it() {
15056 let role = |command: &Command| {
15057 command
15058 .as_std()
15059 .get_envs()
15060 .filter(|(key, _)| *key == OsStr::new(SUBC_SPAWN_ROLE_ENV))
15061 .last()
15062 .map(|(_, value)| value.map(|v| v.to_string_lossy().into_owned()))
15063 };
15064 let forged = spec(vec![(
15065 SUBC_SPAWN_ROLE_ENV.to_string(),
15066 SPAWN_ROLE_SWAP_CANDIDATE.to_string(),
15067 )]);
15068
15069 let mut plain = Command::new("/nonexistent");
15070 apply_child_env(&mut plain, &forged);
15071 apply_spawn_role(&mut plain, SpawnRole::Plain);
15072 assert_eq!(
15073 role(&plain),
15074 Some(None),
15075 "a plain spawn must remove SUBC_SPAWN_ROLE, whatever the spec says"
15076 );
15077
15078 let mut candidate = Command::new("/nonexistent");
15079 apply_child_env(&mut candidate, &spec(Vec::new()));
15080 apply_spawn_role(&mut candidate, SpawnRole::SwapCandidate);
15081 assert_eq!(
15082 role(&candidate),
15083 Some(Some(SPAWN_ROLE_SWAP_CANDIDATE.to_string()))
15084 );
15085 }
15086
15087 #[test]
15093 fn daemon_private_capture_keys_are_not_passed_to_the_child() {
15094 let mut command = Command::new("/nonexistent");
15095 apply_child_env(
15096 &mut command,
15097 &spec(vec![
15098 (super::CAPTURE_KEEP_ENV.to_string(), "5".to_string()),
15099 ("KEPT".to_string(), "yes".to_string()),
15100 ]),
15101 );
15102 let keys: Vec<String> = command
15103 .as_std()
15104 .get_envs()
15105 .filter(|(_, value)| value.is_some())
15106 .map(|(key, _)| key.to_string_lossy().into_owned())
15107 .collect();
15108 assert!(keys.contains(&"KEPT".to_string()), "got {keys:?}");
15109 assert!(
15110 !keys.contains(&super::CAPTURE_KEEP_ENV.to_string()),
15111 "daemon-private capture key leaked to the child: {keys:?}"
15112 );
15113 }
15114}
15115
15116#[cfg(test)]
15117mod jitter_tests {
15118 use super::jittered_health_delay;
15119 use std::{collections::HashSet, time::Duration};
15120
15121 const FLEET: [&str; 14] = [
15130 "aft",
15131 "alfonso-core",
15132 "magic-context",
15133 "broca",
15134 "thalamus",
15135 "quota",
15136 "engram",
15137 "plexus",
15138 "cerebellum",
15139 "astrocyte",
15140 "synapse",
15141 "subc-mcp",
15142 "cortexkit-credentials",
15143 "subc-federation",
15144 ];
15145
15146 #[test]
15154 fn probe_delays_disperse_across_the_fleet() {
15155 let cadence = Duration::from_secs(30);
15156 let delays: HashSet<Duration> = FLEET
15157 .iter()
15158 .map(|id| jittered_health_delay(id, 0, cadence))
15159 .collect();
15160 assert_eq!(
15161 delays.len(),
15162 FLEET.len(),
15163 "every supervised module must land on its own probe offset"
15164 );
15165 }
15166
15167 #[test]
15173 fn jitter_only_delays_and_stays_within_one_tenth_of_cadence() {
15174 let cadence = Duration::from_secs(30);
15175 let span = cadence / 10;
15176 for id in FLEET {
15177 for probe_index in 0..8 {
15178 let delay = jittered_health_delay(id, probe_index, cadence);
15179 assert!(
15180 delay >= cadence,
15181 "{id}#{probe_index}: jitter must not shorten the cadence"
15182 );
15183 assert!(
15184 delay < cadence + span,
15185 "{id}#{probe_index}: jitter must stay inside one tenth of the cadence"
15186 );
15187 }
15188 }
15189 }
15190
15191 #[test]
15197 fn a_module_offset_is_stable_across_restarts() {
15198 let cadence = Duration::from_secs(30);
15199 for id in FLEET {
15200 assert_eq!(
15201 jittered_health_delay(id, 0, cadence),
15202 jittered_health_delay(id, 0, cadence),
15203 "{id}: the same module and probe index must produce the same offset"
15204 );
15205 }
15206 }
15207
15208 #[test]
15210 fn zero_cadence_yields_zero_delay() {
15211 assert_eq!(
15212 jittered_health_delay("aft", 0, Duration::ZERO),
15213 Duration::ZERO
15214 );
15215 }
15216}
15217
15218#[cfg(all(test, target_os = "linux"))]
15219mod cgroup_placement_tests {
15220 use super::{
15221 apply_cgroup_placement, remove_module_cgroup, LaunchSpecDigest, ModuleProtocol, ModuleSpec,
15222 SuperviseError, SupervisedChild,
15223 };
15224 use crate::stderr_tail::{StderrRing, StderrTailConfig};
15225 use cortexkit_test_support::ScratchDir;
15226 use std::{
15227 fs, io,
15228 path::{Path, PathBuf},
15229 sync::{Arc, Mutex},
15230 };
15231 use tokio::process::Command;
15232
15233 #[tokio::test]
15234 async fn unique_spawn_cgroups_clean_up_across_restarts_and_shutdown() {
15235 use super::*;
15236 let dir = ScratchDir::new("unique-spawn-cgroups");
15237 let root = PathBuf::from(format!(
15238 "/sys/fs/cgroup/subc-unique-test-{}-{}",
15239 std::process::id(),
15240 unix_ms_now()
15241 ));
15242 if let Err(error) = fs::create_dir(&root) {
15243 assert!(
15244 std::env::var_os("SUBC_REQUIRE_CGROUP_TEST").is_none(),
15245 "required cgroup test cannot execute: {error}"
15246 );
15247 eprintln!("SKIP unique_spawn_cgroups_clean_up_across_restarts_and_shutdown: {error}");
15248 return;
15249 }
15250 let placement = subc_cgroup::prepare_at(&root).unwrap().unwrap();
15251 let group_count = || {
15252 fs::read_dir(root.join("subc-modules"))
15253 .unwrap()
15254 .map(|entry| entry.unwrap().file_type().unwrap())
15255 .filter(|kind| kind.is_dir())
15256 .count()
15257 };
15258 let supervisor =
15259 Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default())
15260 .with_cgroup_placement(Some(placement.clone()));
15261 let runtime = supervisor.runtime_config();
15262 let mut spec = ModuleSpec {
15263 module_id: "unique-spawn".into(),
15264 program: PathBuf::from("/bin/sleep"),
15265 args: vec!["60".into()],
15266 env: ["XDG_DATA_HOME", "XDG_RUNTIME_DIR", "XDG_CONFIG_HOME"]
15267 .into_iter()
15268 .map(|key| (key.into(), dir.to_string_lossy().into_owned()))
15269 .collect(),
15270 reserved: false,
15271 reserved_prefixes: vec![],
15272 protocol: ModuleProtocol::None,
15273 overlap: Default::default(),
15274 };
15275 let spawn = |spec: &ModuleSpec| {
15276 spawn_child(
15277 spec,
15278 None,
15279 None,
15280 &runtime.stderr_ring,
15281 None,
15282 &runtime.child_roster,
15283 Some(&placement),
15284 )
15285 .unwrap()
15286 };
15287 let mut live = spawn(&spec);
15288 for _ in 0..3 {
15289 let next = spawn(&spec);
15291 assert_ne!(live.module_id, next.module_id);
15292 live.start_kill().unwrap();
15293 live.wait().await.unwrap();
15294 live = next;
15295 assert!(
15296 live.child.try_wait().unwrap().is_none(),
15297 "retiring the old slot must not kill the replacement"
15298 );
15299 assert_eq!(
15300 group_count(),
15301 1,
15302 "only the live spawn's cgroup should remain"
15303 );
15304 }
15305 supervisor.begin_daemon_shutdown();
15306 let reap = tokio::spawn(async move {
15307 live.wait().await.unwrap();
15308 });
15309 supervisor
15310 .end_children_for_daemon_shutdown(false, std::future::pending())
15311 .await;
15312 reap.await.unwrap();
15313 assert_eq!(group_count(), 0);
15314 spec.program = PathBuf::from("/bin/true");
15316 spec.args.clear();
15317 let fresh_roster = ChildRoster::default();
15318 let mut short = spawn_child(
15319 &spec,
15320 None,
15321 None,
15322 &runtime.stderr_ring,
15323 None,
15324 &fresh_roster,
15325 Some(&placement),
15326 )
15327 .unwrap();
15328 short.wait().await.unwrap();
15329 assert_eq!(group_count(), 0);
15330 spec.module_id = "_".repeat(255);
15331 let mut long_id = spawn_child(
15332 &spec,
15333 None,
15334 None,
15335 &runtime.stderr_ring,
15336 None,
15337 &fresh_roster,
15338 Some(&placement),
15339 )
15340 .unwrap();
15341 long_id.wait().await.unwrap();
15342 assert_eq!(
15343 group_count(),
15344 0,
15345 "valid long module IDs must not exceed cgroup NAME_MAX"
15346 );
15347 fs::remove_dir(root.join("subc-modules")).unwrap();
15348 fs::remove_dir(root).unwrap();
15349 eprintln!("EXECUTED unique_spawn_cgroups_clean_up_across_restarts_and_shutdown");
15350 }
15351
15352 #[test]
15353 fn failed_parent_cgroup_open_is_a_cgroup_supervision_error() {
15354 let path = Path::new("/definitely-missing-subc-cgroup");
15355 let mut command = Command::new("true");
15356 let error = apply_cgroup_placement(
15357 &mut command,
15358 &ModuleSpec {
15359 module_id: "broken-cgroup".to_string(),
15360 program: PathBuf::from("true"),
15361 args: Vec::new(),
15362 env: Vec::new(),
15363 reserved: false,
15364 reserved_prefixes: Vec::new(),
15365 protocol: ModuleProtocol::Subc,
15366 overlap: Default::default(),
15367 },
15368 path,
15369 )
15370 .expect_err("a parent cgroup open failure must reject the supervised spawn");
15371 let reason = error.to_string();
15372
15373 assert!(
15374 matches!(error, SuperviseError::Cgroup { .. }),
15375 "parent cgroup open must be reported as a cgroup supervision error: {reason}"
15376 );
15377 assert!(
15378 reason.contains("/definitely-missing-subc-cgroup/cgroup.procs"),
15379 "parent cgroup open failure must name cgroup.procs: {reason}"
15380 );
15381 }
15382
15383 #[tokio::test]
15384 async fn reaping_a_child_removes_its_empty_module_cgroup() {
15385 let root = ScratchDir::new("supervisor-reap-cgroup");
15386 fs::write(root.join("cgroup.procs"), b"").expect("write scratch cgroup marker");
15387 let placement = subc_cgroup::prepare_at(&root)
15388 .expect("prepare scratch cgroup root")
15389 .expect("scratch root has a cgroup.procs marker");
15390 let module_id = "reaped-module";
15391 let module = placement
15392 .module_path(module_id)
15393 .expect("create scratch module cgroup");
15394 let child = Command::new("true")
15395 .env("XDG_DATA_HOME", root.path())
15396 .env("XDG_RUNTIME_DIR", root.path())
15397 .env("XDG_CONFIG_HOME", root.path())
15398 .spawn()
15399 .expect("spawn short-lived child");
15400 let pid = child.id().expect("spawned child has pid");
15401 let mut child = SupervisedChild {
15402 child,
15403 exec: crate::activity::ExecOwner(Arc::new(crate::activity::ExecOutcome::new())),
15404 protocol: ModuleProtocol::Subc,
15405 module_id: module_id.to_string(),
15406 cgroup_placement: Some(placement),
15407 stdout_pump: None,
15408 stderr_pump: None,
15409 stderr_ring: Arc::new(Mutex::new(StderrRing::new(StderrTailConfig::default()))),
15410 spawned_at: tokio::time::Instant::now(),
15411 spawned_at_ms: 0,
15412 spawned_from: PathBuf::from("true"),
15413 launch_spec_digest: LaunchSpecDigest([0; 32]),
15414 spawned_file_identity: None,
15415 process_start_time: None,
15416 process_identity: None,
15417 pid,
15418 roster_guard: None,
15419 #[cfg(target_os = "macos")]
15420 privacy_exec: None,
15421 spawn_failure: None,
15422 };
15423
15424 child.wait().await.expect("reap short-lived child");
15425
15426 assert!(
15427 !module.exists(),
15428 "reaping the supervised child must remove its empty cgroup"
15429 );
15430 }
15431
15432 #[test]
15433 fn non_empty_cgroup_removal_is_reported_without_blocking_teardown() {
15434 let root = ScratchDir::new("supervisor-non-empty-cgroup");
15435 fs::write(root.join("cgroup.procs"), b"").expect("write scratch cgroup marker");
15436 let placement = subc_cgroup::prepare_at(&root)
15437 .expect("prepare scratch cgroup root")
15438 .expect("scratch root has a cgroup.procs marker");
15439 let module = placement
15440 .module_path("surviving-module")
15441 .expect("create scratch module cgroup");
15442 fs::write(module.join("surviving-process"), b"still present")
15443 .expect("make scratch cgroup non-empty");
15444 let (logs, _guard) = crate::router::test_log::log_capture(tracing::Level::WARN);
15445
15446 remove_module_cgroup(&placement, "surviving-module");
15447
15448 let logs = crate::router::test_log::captured_logs(&logs);
15449 assert!(
15450 module.exists(),
15451 "failed removal must leave the cgroup intact"
15452 );
15453 assert!(
15454 logs.contains("could not remove module cgroup after process exit; continuing teardown")
15455 && logs.contains("surviving-module"),
15456 "best-effort removal must report the failure without returning it: {logs}"
15457 );
15458 }
15459
15460 #[test]
15461 fn cgroup_pre_exec_spawn_failure_names_the_cgroup_path() {
15462 let cgroup_path = PathBuf::from("/sys/fs/cgroup/subc-modules/broken-module");
15463 let reason = SuperviseError::Spawn {
15464 program: PathBuf::from("/bin/true"),
15465 source: io::Error::from_raw_os_error(13),
15466 cgroup_path: Some(cgroup_path.clone()),
15467 }
15468 .to_string();
15469
15470 assert!(
15471 reason.contains(&cgroup_path.display().to_string()),
15472 "a pre_exec spawn failure must name the cgroup path: {reason}"
15473 );
15474 }
15475}
15476
15477#[cfg(test)]
15478mod spawn_subscriber_lag_tests {
15479 use super::*;
15480
15481 #[tokio::test]
15486 async fn lagged_spawn_subscriber_receives_a_terminal_lagged_error_after_its_queued_frames() {
15487 let feed = SpawnEventFeed::default();
15488 feed.configure_incarnation("lag-incarnation".to_string());
15489 let (tx, mut rx) = mpsc::channel(1);
15492 feed.subscribe(ConnectionId::new(1), 7, 1, None, FrameSink::new(tx))
15493 .expect("subscribe");
15494 let emitted = SPAWN_SUBSCRIBER_BUFFER + 16;
15495 for index in 0..emitted {
15496 feed.emit_spawned(&format!("lag-module-{index}"), 1000, 0);
15497 tokio::task::yield_now().await;
15500 }
15501 assert_eq!(
15502 feed.subscriber_count(),
15503 0,
15504 "the lagged subscriber must be removed"
15505 );
15506
15507 let mut data = Vec::new();
15508 let mut last = None;
15509 loop {
15510 let next = tokio::time::timeout(Duration::from_secs(5), rx.recv())
15511 .await
15512 .expect("the forwarder must finish once the subscriber is dropped");
15513 let Some(outbound) = next else { break };
15514 let frame = outbound.frame;
15515 if frame.header.ty == FrameType::StreamData {
15516 assert!(last.is_none(), "no data may follow the terminal frame");
15517 let event: SpawnEvent = serde_json::from_slice(&frame.body).unwrap();
15518 data.push(event.cursor.seq);
15519 } else {
15520 assert!(last.is_none(), "exactly one terminal frame");
15521 last = Some(frame);
15522 }
15523 }
15524 assert!(!data.is_empty(), "queued frames drain before the terminal");
15525 for pair in data.windows(2) {
15526 assert_eq!(
15527 pair[1],
15528 pair[0] + 1,
15529 "queued frames arrive dense and in order"
15530 );
15531 }
15532 let terminal = last.expect("a lagged subscriber must receive a terminal frame");
15533 assert_eq!(terminal.header.ty, FrameType::Error);
15534 assert_eq!(terminal.header.corr, 7);
15535 let body: subc_protocol::ErrorBody = serde_json::from_slice(&terminal.body).unwrap();
15536 assert_eq!(body.code, SPAWN_SUBSCRIBER_LAGGED_CODE);
15537 let detail = body.detail.expect("lagged error carries detail");
15538 assert_eq!(
15539 detail["first_undelivered_cursor"]["seq"],
15540 data.last().unwrap() + 1,
15541 "the named cursor is the first event the subscriber did not receive"
15542 );
15543 assert_eq!(
15544 detail["first_undelivered_cursor"]["daemon_incarnation"],
15545 "lag-incarnation"
15546 );
15547 }
15548}
15549
15550#[cfg(test)]
15551mod terminal_history_read_concurrency_tests {
15552 use super::*;
15553 use crate::terminal_journal::read_pause;
15554 use cortexkit_test_support::ScratchDir;
15555 use std::sync::mpsc as std_mpsc;
15556
15557 fn journaled_ring(
15558 journal: &Arc<crate::terminal_journal::TerminalJournal>,
15559 ) -> Arc<Mutex<TerminalRing>> {
15560 Arc::new(Mutex::new(
15561 TerminalRing::new(TerminalRingConfig::default(), 1)
15562 .with_journal(Some(Arc::clone(journal))),
15563 ))
15564 }
15565
15566 fn crash(at_ms: u64) -> ExitReport {
15567 ExitReport {
15568 kind: ExitKind::Crash,
15569 code: Some(1),
15570 signal: None,
15571 at_ms,
15572 }
15573 }
15574
15575 fn record_within(
15578 module_id: &'static str,
15579 ring: &Arc<Mutex<TerminalRing>>,
15580 at_ms: u64,
15581 bound: Duration,
15582 ) -> bool {
15583 let ring = Arc::clone(ring);
15584 let (done, done_rx) = std_mpsc::channel();
15585 std::thread::spawn(move || {
15586 record_terminal(
15587 module_id,
15588 &ring,
15589 &SpawnEventFeed::default(),
15590 &crash(at_ms),
15591 TerminalDisposition::Restarting,
15592 );
15593 let _ = done.send(());
15594 });
15595 done_rx.recv_timeout(bound).is_ok()
15596 }
15597
15598 #[test]
15603 fn exits_recorded_during_a_paused_history_read_are_not_blocked_or_half_merged() {
15604 let dir = ScratchDir::new("terminal-history-concurrent-read");
15605 let path = dir.join("terminals.jsonl");
15606 let journal = Arc::new(crate::terminal_journal::TerminalJournal::open(
15607 path.clone(),
15608 "daemon".into(),
15609 ));
15610 let reader_ring = journaled_ring(&journal);
15611 let other_ring = journaled_ring(&journal);
15612 assert!(record_within(
15613 "reader-module",
15614 &reader_ring,
15615 10,
15616 Duration::from_secs(5)
15617 ));
15618
15619 let (started, release) = read_pause::install(&path);
15620 let reading = {
15621 let ring = Arc::clone(&reader_ring);
15622 std::thread::spawn(move || durable_terminal_history_of(&ring, "reader-module"))
15623 };
15624 started
15625 .recv_timeout(Duration::from_secs(5))
15626 .expect("the history read reached its pause");
15627
15628 let bound = Duration::from_secs(1);
15629 assert!(
15630 record_within("other-module", &other_ring, 20, bound),
15631 "another module's exit waited on a history read (journal writer held)"
15632 );
15633 assert!(
15634 record_within("reader-module", &reader_ring, 30, bound),
15635 "the read module's own exit waited on its history read (ring held)"
15636 );
15637
15638 drop(release);
15639 let paused = reading.join().unwrap();
15640 assert_eq!(
15641 paused.entries.iter().map(|e| e.at_ms).collect::<Vec<_>>(),
15642 vec![10],
15643 "an exit recorded after the read began lands in neither half of it"
15644 );
15645 assert_eq!(paused.journal_skipped_lines, 0);
15646 assert_eq!(paused.journal_read_errors, 0);
15647
15648 let after = durable_terminal_history_of(&reader_ring, "reader-module");
15649 assert_eq!(
15650 after.entries.iter().map(|e| e.at_ms).collect::<Vec<_>>(),
15651 vec![10, 30],
15652 "the next read merges ring and journal with no duplicate"
15653 );
15654 assert_eq!(after.journal_skipped_lines, 0);
15655 }
15656}
15657
15658#[cfg(test)]
15663mod stderr_settle_tests {
15664 use std::{
15665 future::Future,
15666 io,
15667 pin::Pin,
15668 sync::{Arc, Mutex},
15669 task::{Context, Poll},
15670 time::Duration,
15671 };
15672
15673 use tokio::{
15674 io::{AsyncRead, ReadBuf},
15675 sync::oneshot,
15676 time::Instant,
15677 };
15678
15679 use super::{settle_stderr_pump, settle_stdout_pump, StderrPump};
15680 use crate::stderr_tail::{
15681 pump_stderr_to, pump_stdout_to, split_capture_stamp, untimed, CaptureState,
15682 ChildOutputSink, OutputSink, StderrRing, StderrTailConfig, TailEntry,
15683 };
15684
15685 const BOUND: Duration = Duration::from_millis(250);
15686
15687 struct HeldReader {
15691 before: Option<Vec<u8>>,
15692 gate: Option<oneshot::Receiver<()>>,
15693 after: io::Cursor<Vec<u8>>,
15694 }
15695
15696 impl AsyncRead for HeldReader {
15697 fn poll_read(
15698 mut self: Pin<&mut Self>,
15699 cx: &mut Context<'_>,
15700 buf: &mut ReadBuf<'_>,
15701 ) -> Poll<io::Result<()>> {
15702 if let Some(bytes) = self.before.take() {
15703 buf.put_slice(&bytes);
15704 return Poll::Ready(Ok(()));
15705 }
15706 if let Some(gate) = self.gate.as_mut() {
15707 match Pin::new(gate).poll(cx) {
15708 Poll::Pending => return Poll::Pending,
15709 Poll::Ready(_) => self.gate = None,
15710 }
15711 }
15712 Pin::new(&mut self.after).poll_read(cx, buf)
15713 }
15714 }
15715
15716 struct DiscardSink;
15717
15718 impl OutputSink for DiscardSink {
15719 fn write_line(&mut self, _line: &[u8]) {}
15720 }
15721
15722 fn line(text: &str) -> TailEntry {
15723 TailEntry::Line {
15724 text: text.to_string(),
15725 truncated: false,
15726 at_ms: None,
15727 }
15728 }
15729
15730 fn lock(ring: &Arc<Mutex<StderrRing>>) -> std::sync::MutexGuard<'_, StderrRing> {
15731 ring.lock().unwrap()
15732 }
15733
15734 fn held_pump(
15738 ring: &Arc<Mutex<StderrRing>>,
15739 before: &str,
15740 after: &str,
15741 ) -> (StderrPump, oneshot::Sender<()>) {
15742 let generation = lock(ring).begin_process();
15743 let (release, gate) = oneshot::channel();
15744 let reader = HeldReader {
15745 before: Some(before.as_bytes().to_vec()),
15746 gate: Some(gate),
15747 after: io::Cursor::new(after.as_bytes().to_vec()),
15748 };
15749 let task = tokio::spawn(pump_stderr_to(
15750 reader,
15751 Arc::clone(ring),
15752 generation,
15753 DiscardSink,
15754 ));
15755 (StderrPump { task, generation }, release)
15756 }
15757
15758 async fn wait_until(ring: &Arc<Mutex<StderrRing>>, done: impl Fn(&StderrRing) -> bool) {
15759 for _ in 0..1000 {
15760 if done(&lock(ring)) {
15761 return;
15762 }
15763 tokio::time::sleep(Duration::from_millis(1)).await;
15764 }
15765 panic!(
15766 "ring never reached the expected state: {:?}",
15767 lock(ring).snapshot(None, None)
15768 );
15769 }
15770
15771 #[tokio::test(start_paused = true)]
15772 async fn a_crash_line_the_reader_had_not_reached_by_the_bound_is_kept_before_the_restart() {
15773 let ring = Arc::new(Mutex::new(StderrRing::new(StderrTailConfig::default())));
15774 let (pump, release) = held_pump(&ring, "booting\n", "config error: missing storage\n");
15775
15776 settle_stderr_pump("crasher", &ring, pump, BOUND).await;
15777 let before_release = lock(&ring).snapshot(None, None);
15778 assert!(
15779 matches!(before_release.capture, CaptureState::Incomplete { .. }),
15780 "a reader that has not reached EOF cannot claim a whole tail: {before_release:?}"
15781 );
15782
15783 let next = lock(&ring).begin_process();
15786 lock(&ring).push_line_from(next, "next process booting");
15787 release.send(()).unwrap();
15788 wait_until(&ring, |ring| {
15789 ring.snapshot(None, None).capture == CaptureState::Captured
15790 })
15791 .await;
15792
15793 assert_eq!(
15794 untimed(lock(&ring).snapshot(None, None).entries),
15795 vec![
15796 line("booting"),
15797 line("config error: missing storage"),
15798 TailEntry::ProcessStart,
15799 line("next process booting"),
15800 ],
15801 "the crash's last line must survive a slow reader and stay in the crashed process's section"
15802 );
15803 }
15804
15805 #[tokio::test(start_paused = true)]
15806 async fn a_pipe_held_open_by_a_descendant_reads_incomplete_without_delaying_the_restart_past_the_bound(
15807 ) {
15808 let ring = Arc::new(Mutex::new(StderrRing::new(StderrTailConfig::default())));
15809 let (pump, _held) = held_pump(&ring, "parent exiting\n", "");
15812
15813 let started = Instant::now();
15814 settle_stderr_pump("orphaning", &ring, pump, BOUND).await;
15815 assert_eq!(
15816 started.elapsed(),
15817 BOUND,
15818 "the restart must wait exactly the bound for a pipe that stays open, no longer"
15819 );
15820
15821 let next = lock(&ring).begin_process();
15822 lock(&ring).push_line_from(next, "next process booting");
15823 tokio::time::sleep(Duration::from_secs(60)).await;
15824
15825 let snapshot = lock(&ring).snapshot(None, None);
15826 match &snapshot.capture {
15827 CaptureState::Incomplete { reason } => assert!(
15828 reason.contains("had not reached EOF") && reason.contains("250ms"),
15829 "the reason must say what is missing and after how long: {reason}"
15830 ),
15831 other => panic!("expected Incomplete while the pipe is held open, got {other:?}"),
15832 }
15833 assert_eq!(
15834 untimed(snapshot.entries),
15835 vec![
15836 line("parent exiting"),
15837 TailEntry::ProcessStart,
15838 line("next process booting"),
15839 ]
15840 );
15841 }
15842
15843 #[tokio::test(start_paused = true)]
15844 async fn a_reader_that_reaches_eof_within_the_bound_leaves_the_tail_captured() {
15845 let ring = Arc::new(Mutex::new(StderrRing::new(StderrTailConfig::default())));
15846 let (pump, release) = held_pump(&ring, "one\n", "two\n");
15847 release.send(()).unwrap();
15848
15849 settle_stderr_pump("clean", &ring, pump, BOUND).await;
15850
15851 let snapshot = lock(&ring).snapshot(None, None);
15852 assert_eq!(snapshot.capture, CaptureState::Captured);
15853 assert_eq!(untimed(snapshot.entries), vec![line("one"), line("two")]);
15854 }
15855
15856 #[tokio::test(start_paused = true)]
15857 async fn stdout_buffered_past_the_exit_drain_bound_reaches_the_capture_file() {
15858 const BURST: usize = 150;
15859 let capture = cortexkit_test_support::ScratchDir::new("stdout-exit-drain");
15860 let path = capture.join("two-pipe.stderr.log");
15861 let sink = ChildOutputSink::open(&path, cortexkit_log::Retention::default()).unwrap();
15862 let ring = Arc::new(Mutex::new(StderrRing::new(StderrTailConfig::default())));
15863 let generation = lock(&ring).begin_process();
15864 let stdout = (0..BURST)
15865 .map(|i| format!("out-{i:04}-{}\n", "o".repeat(64)))
15866 .collect::<String>();
15867 let stderr = (0..BURST)
15868 .map(|i| format!("err-{i:04}-{}\n", "e".repeat(64)))
15869 .collect::<String>();
15870 let first_chunk = stdout.find('\n').unwrap() + 1 + "out-0001-".len();
15875 let (release, gate) = oneshot::channel();
15876 let reader = HeldReader {
15877 before: Some(stdout.as_bytes()[..first_chunk].to_vec()),
15878 gate: Some(gate),
15879 after: io::Cursor::new(stdout.as_bytes()[first_chunk..stdout.len() - 1].to_vec()),
15880 };
15881 let (finished, completion) = oneshot::channel();
15882 let stdout_sink = sink.clone();
15883 let pump = tokio::spawn(async move {
15884 pump_stdout_to(reader, stdout_sink).await;
15885 finished.send(()).unwrap();
15886 });
15887 pump_stderr_to(io::Cursor::new(stderr.into_bytes()), ring, generation, sink).await;
15888
15889 let started = Instant::now();
15890 settle_stdout_pump("two-pipe", pump, BOUND).await;
15891 assert_eq!(
15892 started.elapsed(),
15893 BOUND,
15894 "a late reader must not hold up restart"
15895 );
15896 assert_eq!(
15897 std::fs::read_to_string(&path).unwrap().lines().count(),
15898 BURST + 1
15899 );
15900
15901 release
15902 .send(())
15903 .expect("stdout reader must survive the exit drain bound");
15904 tokio::time::timeout(Duration::from_secs(1), completion)
15905 .await
15906 .expect("released stdout reader must reach EOF")
15907 .expect("stdout capture must finish at EOF, not be cancelled");
15908 let contents = std::fs::read_to_string(&path).unwrap();
15909 let mut lines = contents
15910 .lines()
15911 .map(|line| split_capture_stamp(line).expect("capture stamp").1)
15912 .collect::<Vec<_>>();
15913 let mut expected = (0..BURST)
15914 .flat_map(|i| {
15915 [
15916 format!("out-{i:04}-{}", "o".repeat(64)),
15917 format!("err-{i:04}-{}", "e".repeat(64)),
15918 ]
15919 })
15920 .collect::<Vec<_>>();
15921 lines.sort_unstable();
15922 expected.sort_unstable();
15923 assert_eq!(
15924 lines, expected,
15925 "both pipes must retain every complete line exactly once"
15926 );
15927 }
15928}
15929
15930#[cfg(all(test, windows))]
15944mod job_containment_tests {
15945 use super::*;
15946 use cortexkit_test_support::ScratchDir;
15947 use std::{
15948 path::{Path, PathBuf},
15949 sync::{Arc, Mutex},
15950 time::{Duration, Instant},
15951 };
15952
15953 fn stub_path() -> PathBuf {
15959 let mut path = std::env::current_exe().expect("current_exe available in tests");
15960 path.pop();
15961 path.pop();
15962 path.push("fake-aft-stub.exe");
15963 assert!(
15964 path.exists(),
15965 "fake-aft-stub not built at {}: run `cargo test -p subc-core` (which builds \
15966 [[bin]] targets) rather than `cargo test -p subc-core --lib` (which does not)",
15967 path.display()
15968 );
15969 path
15970 }
15971
15972 fn read_grandchild_pid(path: &Path) -> u32 {
15974 let deadline = Instant::now() + Duration::from_secs(10);
15975 loop {
15976 if let Ok(contents) = std::fs::read_to_string(path) {
15977 if let Ok(pid) = contents.trim().parse() {
15978 return pid;
15979 }
15980 }
15981 assert!(
15982 Instant::now() < deadline,
15983 "the stub never recorded a grandchild pid at {}",
15984 path.display()
15985 );
15986 std::thread::sleep(Duration::from_millis(10));
15987 }
15988 }
15989
15990 struct Fixture {
15993 _dir: ScratchDir,
15994 module_id: String,
15995 grandchild: u32,
15996 child: Option<SupervisedChild>,
15997 registry: Arc<Registry>,
15998 snapshot: Arc<Mutex<SupervisorSnapshot>>,
15999 terminal_ring: Arc<Mutex<TerminalRing>>,
16000 spawn_events: SpawnEventFeed,
16001 }
16002
16003 fn fixture(label: &str, module_id: &str) -> Fixture {
16004 let dir = ScratchDir::new(label);
16005 let pid_file = dir.join("grandchild.pid");
16006 let supervisor = Supervisor::new_for_test(
16007 Arc::new(Registry::default()),
16008 RestartPolicy::new(3, Duration::ZERO),
16009 );
16010 let runtime = supervisor.runtime_config();
16011 let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::starting()));
16012 let spec = ModuleSpec {
16013 module_id: module_id.to_string(),
16014 program: stub_path(),
16015 args: Vec::new(),
16019 env: vec![
16020 ("FAKE_AFT_NEVER_CONNECT".to_string(), "1".to_string()),
16021 (
16022 "FAKE_AFT_GRANDCHILD_PID_FILE".to_string(),
16023 pid_file.display().to_string(),
16024 ),
16025 ],
16026 reserved: false,
16027 reserved_prefixes: Vec::new(),
16028 protocol: ModuleProtocol::Subc,
16029 overlap: Default::default(),
16030 };
16031 let child = spawn_and_mark_running(&spec, &runtime, &snapshot)
16032 .expect("spawn the supervised fixture");
16033 let grandchild = read_grandchild_pid(&pid_file);
16034 Fixture {
16035 _dir: dir,
16036 module_id: module_id.to_string(),
16037 grandchild,
16038 child: Some(child),
16039 registry: Arc::new(Registry::default()),
16040 snapshot,
16041 terminal_ring: Arc::clone(&runtime.terminal_ring),
16042 spawn_events: SpawnEventFeed::default(),
16043 }
16044 }
16045
16046 impl Fixture {
16047 async fn drain(&mut self) {
16049 let child = self
16050 .child
16051 .take()
16052 .expect("the fixture child is still present");
16053 drain_child_to_state(
16054 &self.module_id,
16055 ModuleProtocol::Subc,
16056 StopNotice::NotSent,
16059 &self.registry,
16060 None,
16061 &self.snapshot,
16062 &self.terminal_ring,
16063 &self.spawn_events,
16064 child,
16065 Duration::from_millis(500),
16066 ModuleState::Stopped,
16067 Some(false),
16068 )
16069 .await
16070 .expect("drain the supervised fixture");
16071 }
16072 }
16073
16074 #[tokio::test]
16080 async fn teardown_reaps_the_grandchild() {
16081 let mut fixture = fixture("teardown-grandchild", "tree-teardown");
16082 let grandchild = fixture.grandchild;
16083
16084 assert!(
16085 subc_jobobject::process_exists(grandchild),
16086 "grandchild {grandchild} must be alive before teardown, or this proves nothing"
16087 );
16088
16089 fixture.drain().await;
16090
16091 assert!(
16092 subc_jobobject::wait_for_process_exit(grandchild, Duration::from_secs(10)),
16093 "grandchild {grandchild} outlived module teardown: the tree was not contained"
16094 );
16095 }
16096
16097 #[test]
16110 fn an_uncontained_grandchild_survives_a_direct_child_kill() {
16111 let dir = ScratchDir::new("teardown-uncontained");
16112 let pid_file = dir.join("grandchild.pid");
16113 let mut child = std::process::Command::new(stub_path())
16114 .env("FAKE_AFT_NEVER_CONNECT", "1")
16115 .env(
16116 "FAKE_AFT_GRANDCHILD_PID_FILE",
16117 pid_file.display().to_string(),
16118 )
16119 .stdin(std::process::Stdio::null())
16120 .stdout(std::process::Stdio::null())
16121 .stderr(std::process::Stdio::null())
16122 .spawn()
16123 .expect("spawn the uncontained fixture");
16124 let grandchild = read_grandchild_pid(&pid_file);
16125
16126 child.kill().expect("kill the direct child");
16128 let _ = child.wait();
16129
16130 assert!(
16131 subc_jobobject::process_exists(grandchild),
16132 "grandchild {grandchild} died with the direct child, so this control no longer \
16133 distinguishes contained from uncontained teardown and the regression test is \
16134 passing vacuously"
16135 );
16136
16137 kill_tree(grandchild);
16140 }
16141
16142 #[tokio::test]
16151 async fn dropping_containment_reaps_the_grandchild() {
16152 let mut fixture = fixture("drop-containment", "tree-drop");
16153 let grandchild = fixture.grandchild;
16154
16155 assert!(subc_jobobject::process_exists(grandchild));
16156
16157 fixture.child.as_mut().expect("child present").job = None;
16159
16160 assert!(
16161 subc_jobobject::wait_for_process_exit(grandchild, Duration::from_secs(10)),
16162 "grandchild {grandchild} survived the containment handle closing, so a daemon \
16163 crash would leave the tree behind"
16164 );
16165 }
16166
16167 fn kill_tree(pid: u32) {
16169 let _ = std::process::Command::new("taskkill.exe")
16170 .args(["/PID", &pid.to_string(), "/T", "/F"])
16171 .stdin(std::process::Stdio::null())
16172 .stdout(std::process::Stdio::null())
16173 .stderr(std::process::Stdio::null())
16174 .status();
16175 assert!(
16176 subc_jobobject::wait_for_process_exit(pid, Duration::from_secs(10)),
16177 "could not clean up grandchild {pid}"
16178 );
16179 }
16180}
16181
16182#[cfg(test)]
16183mod privacy_trampoline_configuration_tests {
16184 #[tokio::test]
16185 #[cfg_attr(not(target_os = "macos"), ignore = "requires macOS privacy trampoline")]
16186 async fn macos_spawn_without_a_configured_trampoline_refuses_by_name() {
16187 #[cfg(target_os = "macos")]
16188 {
16189 let supervisor = super::Supervisor::new(
16190 std::sync::Arc::new(crate::Registry::default()),
16191 super::RestartPolicy::default(),
16192 );
16193 let error = supervisor.spawn(spec()).unwrap_err();
16194 assert!(
16195 error
16196 .to_string()
16197 .contains("no privacy trampoline configured"),
16198 "{error}"
16199 );
16200 }
16201 }
16202
16203 #[tokio::test]
16204 #[cfg_attr(not(target_os = "macos"), ignore = "requires macOS privacy trampoline")]
16205 async fn macos_wrong_trampoline_is_rejected_by_the_startup_probe() {
16206 #[cfg(target_os = "macos")]
16207 {
16208 let supervisor = super::Supervisor::new(
16209 std::sync::Arc::new(crate::Registry::default()),
16210 super::RestartPolicy::default(),
16211 )
16212 .with_privacy_trampoline(std::env::current_exe().unwrap());
16213 let error = supervisor.spawn(spec()).unwrap_err();
16214 assert!(
16215 error
16216 .to_string()
16217 .contains("binary does not implement the privacy trampoline protocol"),
16218 "{error}"
16219 );
16220 }
16221 }
16222
16223 #[cfg(target_os = "macos")]
16224 fn spec() -> super::ModuleSpec {
16225 super::ModuleSpec {
16226 module_id: "privacy-configuration".into(),
16227 program: "/bin/sleep".into(),
16228 args: vec!["30".into()],
16229 env: vec![],
16230 reserved: false,
16231 reserved_prefixes: vec![],
16232 protocol: subc_control::ModuleProtocol::None,
16233 overlap: super::ModuleOverlap::Exclusive,
16234 }
16235 }
16236}
16237
16238#[cfg(test)]
16239mod privacy_exec_boundary_tests {
16240 #[cfg(target_os = "macos")]
16241 use super::*;
16242 #[cfg(target_os = "macos")]
16243 use std::{
16244 io::{Read, Write},
16245 net::{TcpListener, TcpStream},
16246 };
16247
16248 #[cfg(target_os = "macos")]
16251 pub(super) fn before_image_sample(spec: &ModuleSpec, pid: u32) {
16252 if let Some((_, path)) = spec
16253 .env
16254 .iter()
16255 .find(|(key, _)| key == "SUBC_TEST_PRIVACY_SAMPLE_BARRIER")
16256 {
16257 let mut barrier = TcpStream::connect(path).unwrap();
16258 barrier
16259 .set_read_timeout(Some(Duration::from_secs(30)))
16260 .unwrap();
16261 barrier.write_all(&pid.to_ne_bytes()).unwrap();
16262 let mut release = [0];
16263 barrier.read_exact(&mut release).unwrap();
16264 assert_eq!(&release, b"X");
16265 }
16266 }
16267
16268 #[cfg(target_os = "macos")]
16269 fn spec(program: &str, args: &[&str]) -> ModuleSpec {
16270 ModuleSpec {
16271 module_id: "privacy-boundary".into(),
16272 program: program.into(),
16273 args: args.iter().map(|arg| (*arg).into()).collect(),
16274 env: vec![],
16275 reserved: false,
16276 reserved_prefixes: vec![],
16277 protocol: ModuleProtocol::None,
16278 overlap: ModuleOverlap::Exclusive,
16279 }
16280 }
16281
16282 #[cfg(target_os = "macos")]
16283 async fn accept(listener: TcpListener) -> TcpStream {
16284 let listener = tokio::net::TcpListener::from_std({
16286 listener.set_nonblocking(true).unwrap();
16287 listener
16288 })
16289 .unwrap();
16290 let (stream, _) = tokio::time::timeout(Duration::from_secs(30), listener.accept())
16291 .await
16292 .unwrap()
16293 .unwrap();
16294 let stream = stream.into_std().unwrap();
16295 stream.set_nonblocking(false).unwrap();
16296 stream
16297 .set_read_timeout(Some(Duration::from_secs(30)))
16298 .unwrap();
16299 stream
16300 }
16301
16302 #[tokio::test]
16303 #[cfg_attr(not(target_os = "macos"), ignore = "requires macOS privacy trampoline")]
16304 async fn macos_roster_withholds_a_nonnull_trampoline_image_until_exec_confirmation() {
16305 #[cfg(target_os = "macos")]
16306 {
16307 let root = cortexkit_test_support::ScratchDir::new("privacy-roster-barrier");
16308 let exec_listener = TcpListener::bind("127.0.0.1:0").unwrap();
16311 let sample_listener = TcpListener::bind("127.0.0.1:0").unwrap();
16312 let record = root.join("live-children.json");
16313 let supervisor =
16314 Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default())
16315 .with_live_children_record(&record);
16316 let runtime = supervisor.runtime_config();
16317 let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::starting()));
16318 let mut spec = spec("/bin/sleep", &["30"]);
16319 spec.env = vec![
16320 (
16321 "SUBC_TEST_PRIVACY_EXEC_BARRIER".into(),
16322 exec_listener.local_addr().unwrap().to_string(),
16323 ),
16324 (
16325 "SUBC_TEST_PRIVACY_SAMPLE_BARRIER".into(),
16326 sample_listener.local_addr().unwrap().to_string(),
16327 ),
16328 ];
16329 let spawn = tokio::task::spawn_blocking(move || {
16330 spawn_and_mark_running(&spec, &runtime, &snapshot).unwrap()
16331 });
16332 let mut sample = accept(sample_listener).await;
16333 let mut pid = [0; 4];
16334 sample.read_exact(&mut pid).unwrap();
16335 let pid = u32::from_ne_bytes(pid);
16336 let mut exec = accept(exec_listener).await;
16337 let mut ready = [0];
16338 exec.read_exact(&mut ready).unwrap();
16339 assert_eq!(&ready, b"R");
16340 let trampoline = subc_os::file_identity(&test_privacy_trampoline()).unwrap();
16343 assert_eq!(
16344 observe_spawned_image(pid).unwrap().executable,
16345 Some(trampoline)
16346 );
16347 sample.write_all(b"X").unwrap();
16348 let mut child = spawn.await.unwrap();
16349 let early = crate::live_children::read_record(&record).unwrap();
16350 assert_eq!(early.len(), 1);
16351 assert_eq!(early[0].pid, pid);
16352 assert_eq!(
16353 early[0].executable, None,
16354 "unconfirmed trampoline image entered the roster"
16355 );
16356 assert!(child.report_ready.get().is_none());
16357 child.privacy_exec.as_mut().unwrap().deadline =
16360 tokio::time::Instant::now() + Duration::from_secs(30);
16361 exec.write_all(b"X").unwrap();
16362 child.confirm_privacy_exec().await;
16363 assert_eq!(child.spawn_failure, None);
16364 assert!(child.report_ready.get().is_some());
16365 let confirmed = crate::live_children::read_record(&record).unwrap();
16366 let module = subc_os::file_identity(std::path::Path::new("/bin/sleep")).unwrap();
16367 assert_ne!(module, trampoline);
16368 assert_eq!(confirmed[0].executable, Some(module.into()));
16369 child.start_kill().unwrap();
16370 child.wait().await.unwrap();
16371 child.release_roster();
16372 }
16373 }
16374
16375 #[tokio::test]
16376 #[cfg_attr(not(target_os = "macos"), ignore = "requires macOS privacy trampoline")]
16377 async fn macos_already_exited_121_in_try_wait_is_a_module_exit_not_a_trampoline_refusal() {
16378 #[cfg(target_os = "macos")]
16379 {
16380 let registry = Arc::new(Registry::default());
16381 let policy = RestartPolicy::new(0, Duration::ZERO);
16382 let supervisor = Supervisor::new_for_test(Arc::clone(®istry), policy);
16383 let runtime = supervisor.runtime_config();
16384 let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::starting()));
16385 let spec = spec("/bin/sh", &["-c", "exit 121"]);
16386 let mut child = spawn_and_mark_running(&spec, &runtime, &snapshot).unwrap();
16390 let pid = child.pid;
16391 tokio::task::spawn_blocking(move || {
16392 subc_os::fork_exec_test::wait_for_child_exit_without_reaping(pid)
16393 })
16394 .await
16395 .unwrap()
16396 .unwrap();
16397 child.privacy_exec.as_mut().unwrap().deadline =
16398 tokio::time::Instant::now() + Duration::from_secs(30);
16399 let status = child.wait().await.unwrap();
16400 assert_eq!(status.code(), Some(121));
16401 assert!(child.privacy_exec.is_none());
16402 assert!(
16403 child.report_ready.get().is_none(),
16404 "an exited module must not publish a live pid"
16405 );
16406 let report = classify_reaped_child_exit(&snapshot, &child, &status);
16407 on_child_exit(
16408 &spec,
16409 policy,
16410 ®istry,
16411 &snapshot,
16412 &runtime.terminal_ring,
16413 &runtime.spawn_events,
16414 &runtime.child_roster,
16415 report,
16416 )
16417 .await;
16418 let state = lock_snapshot(&snapshot).unwrap();
16419 assert_eq!(state.state, ModuleState::Failed);
16420 assert_eq!(state.last_exit.as_ref().unwrap().code, Some(121));
16421 assert_eq!(state.reported_pid(), None);
16422 drop(state);
16423 let history = runtime.terminal_ring.lock().unwrap().snapshot();
16424 assert_eq!(history.entries.len(), 1);
16425 let terminal = &history.entries[0];
16426 assert_eq!(terminal.exit_code, Some(121));
16427 assert_eq!(terminal.exit_kind, subc_control::TerminalExitKind::Crash);
16428 assert_eq!(terminal.disposition, TerminalDisposition::Failed);
16429 assert_eq!(
16430 terminal.disposition_detail.as_deref(),
16431 Some(policy.budget_exhausted_detail().as_str()),
16432 "module exit 121 was classified as a trampoline refusal: {terminal:?}"
16433 );
16434 assert_eq!(child.spawn_failure, None);
16435 child.release_roster();
16436 }
16437 }
16438}
16439
16440#[cfg(all(test, unix))]
16444mod launch_nonce_descriptor_tests {
16445 use super::{ChildRoster, ModuleProtocol, ModuleSpec, SupervisorHandle};
16446 use crate::stderr_tail::{StderrRing, StderrTailConfig};
16447 use cortexkit_test_support::ScratchDir;
16448 use std::{
16449 path::PathBuf,
16450 sync::{Arc, Mutex},
16451 time::{Duration, Instant},
16452 };
16453
16454 async fn probe(role: super::SpawnRole) {
16455 let scratch = ScratchDir::new("launch-nonce-descriptor");
16456 let fd_copy = scratch.join("from-descriptor");
16457 let env_copy = scratch.join("environment");
16458 let script = format!(
16459 "env > '{env}.tmp' && mv '{env}.tmp' '{env}'; cat <&3 > '{fd}.tmp' && mv '{fd}.tmp' '{fd}'; sleep 30",
16460 fd = fd_copy.display(), env = env_copy.display(),
16461 );
16462 let xdg = |name: &str| (name.to_string(), scratch.join(name).display().to_string());
16463 let spec = ModuleSpec {
16464 module_id: "nonce-descriptor-probe".to_string(),
16465 program: PathBuf::from("/bin/sh"),
16466 args: vec!["-c".to_string(), script],
16467 env: vec![
16468 xdg("XDG_DATA_HOME"),
16469 xdg("XDG_RUNTIME_DIR"),
16470 xdg("XDG_CONFIG_HOME"),
16471 ("SUBC_LAUNCH_NONCE".to_string(), "must-not-leak".to_string()),
16472 ],
16473 reserved: true,
16474 reserved_prefixes: Vec::new(),
16475 protocol: ModuleProtocol::Subc,
16476 overlap: Default::default(),
16477 };
16478 let handle = SupervisorHandle::new();
16479 let ring = Arc::new(Mutex::new(StderrRing::new(StderrTailConfig::default())));
16480 let roster = ChildRoster::default();
16481 #[cfg(target_os = "macos")]
16482 {
16483 let path = super::test_privacy_trampoline();
16484 roster.set_privacy_trampoline(super::probe_privacy_trampoline(&path).map(|()| path));
16485 }
16486 let child = super::spawn_child_in_slot(
16487 &spec,
16488 None,
16489 Some(&handle),
16490 &ring,
16491 None,
16492 &roster,
16493 #[cfg(target_os = "linux")]
16494 None,
16495 role,
16496 matches!(role, super::SpawnRole::SwapCandidate),
16497 )
16498 .expect("spawn probe");
16499 let deadline = Instant::now() + Duration::from_secs(10);
16500 while !(fd_copy.exists() && env_copy.exists()) {
16501 assert!(Instant::now() < deadline, "probe never wrote its copies");
16502 tokio::time::sleep(Duration::from_millis(20)).await;
16503 }
16504 let nonce = std::fs::read_to_string(fd_copy).unwrap();
16505 assert!(!nonce.is_empty());
16506 let environment = std::fs::read_to_string(env_copy).unwrap();
16507 assert!(environment
16508 .lines()
16509 .any(|line| line.starts_with("SUBC_LAUNCH_NONCE_FD=3:")));
16510 let copy = environment
16511 .lines()
16512 .find_map(|line| line.strip_prefix("SUBC_LAUNCH_NONCE="));
16513 assert_eq!(
16514 copy, None,
16515 "Unix children must never receive the environment nonce"
16516 );
16517 if matches!(role, super::SpawnRole::Plain) {
16518 assert_eq!(
16519 handle.spawn_nonce(&spec.module_id).as_deref(),
16520 Some(nonce.as_str())
16521 );
16522 }
16523 drop(child);
16524 }
16525
16526 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
16527 async fn a_spawned_module_receives_its_nonce_only_on_descriptor_3() {
16528 probe(super::SpawnRole::Plain).await;
16529 }
16530
16531 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
16532 async fn swap_candidate_receives_its_nonce_only_on_descriptor_3() {
16533 probe(super::SpawnRole::SwapCandidate).await;
16534 }
16535}
16536
16537#[cfg(all(test, target_os = "linux"))]
16538mod cgroup_containment_tests {
16539 use super::*;
16540 use cortexkit_test_support::ScratchDir;
16541
16542 fn running(pid: u32) -> bool {
16543 std::fs::read_to_string(format!("/proc/{pid}/stat"))
16545 .ok()
16546 .and_then(|stat| {
16547 stat.rsplit_once(") ")
16548 .map(|(_, rest)| rest.starts_with('Z'))
16549 })
16550 .is_some_and(|zombie| !zombie)
16551 }
16552
16553 #[tokio::test]
16554 async fn linux_teardown_reaps_the_grandchild() {
16555 teardown_tree("linux_teardown_reaps_the_grandchild", false).await;
16556 }
16557
16558 #[tokio::test]
16559 async fn linux_shutdown_straggler_reaps_the_grandchild() {
16560 teardown_tree("linux_shutdown_straggler_reaps_the_grandchild", true).await;
16561 }
16562
16563 async fn teardown_tree(test_name: &str, shutdown: bool) {
16564 let dir = ScratchDir::new(test_name);
16565 let root = PathBuf::from(format!(
16566 "/sys/fs/cgroup/subc-tree-test-{test_name}-{}-{}",
16567 std::process::id(),
16568 unix_ms_now()
16569 ));
16570 if let Err(error) = std::fs::create_dir(&root) {
16571 assert!(
16572 std::env::var_os("SUBC_REQUIRE_CGROUP_TEST").is_none(),
16573 "required cgroup test cannot execute: {error}"
16574 );
16575 eprintln!(
16576 "SKIP {test_name}: no writable delegated cgroup at {}: {error}",
16577 root.display()
16578 );
16579 return;
16580 }
16581 let placement = subc_cgroup::prepare_at(&root)
16582 .expect("prepare isolated kernel cgroup")
16583 .expect("isolated cgroup is delegated");
16584 let module_id = "tree-teardown";
16585 let module = placement
16586 .module_path(module_id)
16587 .expect("create isolated module cgroup");
16588 if !module.join("cgroup.kill").exists() {
16589 std::fs::remove_dir(&module).unwrap();
16590 std::fs::remove_dir(root.join("subc-modules")).unwrap();
16591 std::fs::remove_dir(&root).unwrap();
16592 assert!(
16593 std::env::var_os("SUBC_REQUIRE_CGROUP_TEST").is_none(),
16594 "required cgroup.kill interface unavailable"
16595 );
16596 eprintln!("SKIP {test_name}: cgroup.kill unavailable (kernel < 5.14)");
16597 return;
16598 }
16599 let supervisor = Supervisor::new_for_test(
16600 Arc::new(Registry::default()),
16601 RestartPolicy::new(3, Duration::ZERO),
16602 )
16603 .with_cgroup_placement(Some(placement));
16604 let mut runtime = supervisor.runtime_config();
16605 runtime.child_roster = runtime
16606 .child_roster
16607 .for_module(Arc::new(Mutex::new(Duration::from_millis(100))));
16608 let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::starting()));
16609 let pid_file = dir.join("grandchild.pid");
16610 let spec = ModuleSpec {
16611 module_id: module_id.to_string(),
16612 program: PathBuf::from("/bin/sh"),
16613 args: vec![
16614 "-c".into(),
16615 "trap '' TERM; sleep 600 & echo $! > \"$1\"; wait".into(),
16616 "fixture".into(),
16617 pid_file.display().to_string(),
16618 ],
16619 env: ["XDG_DATA_HOME", "XDG_RUNTIME_DIR", "XDG_CONFIG_HOME"]
16620 .into_iter()
16621 .map(|key| (key.to_string(), dir.display().to_string()))
16622 .collect(),
16623 reserved: false,
16624 reserved_prefixes: Vec::new(),
16625 protocol: ModuleProtocol::None,
16626 overlap: Default::default(),
16627 };
16628 let child =
16629 spawn_and_mark_running(&spec, &runtime, &snapshot).expect("spawn supervised tree");
16630 let deadline = tokio::time::Instant::now() + Duration::from_secs(10);
16631 let grandchild: u32 = loop {
16632 if let Ok(contents) = std::fs::read_to_string(&pid_file) {
16633 if let Ok(pid) = contents.trim().parse() {
16634 break pid;
16635 }
16636 }
16637 assert!(
16638 tokio::time::Instant::now() < deadline,
16639 "grandchild pid was not recorded"
16640 );
16641 tokio::time::sleep(Duration::from_millis(10)).await;
16642 };
16643 assert!(
16644 running(grandchild),
16645 "grandchild must be alive before teardown"
16646 );
16647 if shutdown {
16648 let mut child = child;
16649 crate::child_roster::end_children_for_daemon_shutdown(
16650 &runtime.child_roster,
16651 false,
16652 std::future::pending(),
16653 )
16654 .await;
16655 child.wait().await.expect("reap shutdown straggler");
16656 } else {
16657 drain_child_to_state(
16658 module_id,
16659 ModuleProtocol::None,
16660 StopNotice::NotSent,
16661 &Registry::default(),
16662 None,
16663 &snapshot,
16664 &runtime.terminal_ring,
16665 &SpawnEventFeed::default(),
16666 child,
16667 Duration::from_millis(100),
16668 ModuleState::Stopped,
16669 Some(false),
16670 )
16671 .await
16672 .expect("real supervisor teardown");
16673 }
16674 let deadline = tokio::time::Instant::now() + Duration::from_secs(5);
16675 while running(grandchild) && tokio::time::Instant::now() < deadline {
16676 tokio::time::sleep(Duration::from_millis(10)).await;
16677 }
16678 let survived = running(grandchild);
16679 if survived {
16681 let pid = rustix::process::Pid::from_raw(grandchild as i32).unwrap();
16682 let _ = rustix::process::kill_process(pid, rustix::process::Signal::KILL);
16683 tokio::time::sleep(Duration::from_millis(100)).await;
16684 }
16685 if module.exists() {
16686 std::fs::remove_dir(&module).expect("remove empty module cgroup");
16687 }
16688 std::fs::remove_dir(root.join("subc-modules")).unwrap();
16689 std::fs::remove_dir(&root).unwrap();
16690 assert!(
16691 !survived,
16692 "grandchild {grandchild} outlived module teardown"
16693 );
16694 eprintln!("EXECUTED {test_name}: grandchild {grandchild} killed in isolated cgroup");
16695 }
16696}
16697
16698#[cfg(all(test, unix))]
16701mod record_write_stall_tests {
16702 use super::*;
16703 use std::sync::atomic::{AtomicBool, Ordering};
16704 use subc_transport::{authenticate_client, ConnectionInfo, Endpoint, SCHEMA_VERSION};
16705
16706 const MODULE_AUTH_DEADLINE: Duration = Duration::from_secs(2);
16709 const HELD_RECORD_WRITE: Duration = Duration::from_secs(3);
16713
16714 fn sleeping_spec() -> ModuleSpec {
16715 ModuleSpec {
16716 module_id: "record-write-stall".into(),
16717 program: "/bin/sh".into(),
16718 args: vec!["-c".into(), "exec sleep 600".into()],
16719 env: vec![],
16720 reserved: false,
16721 reserved_prefixes: vec![],
16722 protocol: ModuleProtocol::None,
16723 overlap: Default::default(),
16724 }
16725 }
16726
16727 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
16728 async fn module_auth_completes_while_the_restart_waits_on_the_record_write() {
16729 let home = cortexkit_test_support::ScratchDir::new("record-write-stall");
16730 let supervisor =
16731 Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default())
16732 .with_live_children_record(home.join("live-children.json"));
16733 let module = supervisor.spawn(sleeping_spec()).unwrap();
16734 let old_pid = lock_snapshot(&module.inner.snapshot)
16735 .unwrap()
16736 .pid
16737 .expect("the first process is running");
16738
16739 let key = vec![0x42; 32];
16742 let daemon_id = [0x24; 16];
16743 let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
16744 let port = listener.local_addr().unwrap().port();
16745 let server = tokio::spawn(crate::server::serve_listener(
16746 listener,
16747 Arc::new(crate::router::Router::with_default_self_handler()),
16748 crate::server::ServerAuth::with_limits(
16749 key.clone(),
16750 daemon_id,
16751 "test",
16752 crate::server::DEFAULT_AUTH_DEADLINE,
16753 4,
16754 ),
16755 ));
16756
16757 let (held_tx, held_rx) = std::sync::mpsc::channel();
16760 let armed = AtomicBool::new(true);
16761 supervisor
16762 .child_roster
16763 .set_record_write_hook_for_test(move |live| {
16764 let admitting_replacement = live.values().any(|entry| entry.pid != old_pid);
16765 if admitting_replacement && armed.swap(false, Ordering::SeqCst) {
16766 let _ = held_tx.send(());
16767 std::thread::sleep(HELD_RECORD_WRITE);
16768 }
16769 });
16770
16771 let module_process = std::thread::spawn(move || {
16774 held_rx
16775 .recv_timeout(Duration::from_secs(30))
16776 .expect("the restart reached the replacement's record write");
16777 let conn = ConnectionInfo {
16778 schema: SCHEMA_VERSION,
16779 wire_version: None,
16780 endpoints: vec![Endpoint {
16781 host: "127.0.0.1".to_owned(),
16782 port,
16783 }],
16784 key,
16785 daemon_id,
16786 pid: std::process::id(),
16787 daemon_ver: "test".to_owned(),
16788 };
16789 tokio::runtime::Builder::new_current_thread()
16790 .enable_all()
16791 .build()
16792 .unwrap()
16793 .block_on(async {
16794 let started = Instant::now();
16795 let mut stream = tokio::net::TcpStream::connect(("127.0.0.1", port))
16796 .await
16797 .unwrap();
16798 let result =
16799 authenticate_client(&mut stream, &conn, MODULE_AUTH_DEADLINE).await;
16800 (result, started.elapsed())
16801 })
16802 });
16803
16804 module.restart(Some(0)).await.unwrap();
16805 let (auth, elapsed) = tokio::task::spawn_blocking(move || module_process.join().unwrap())
16806 .await
16807 .unwrap();
16808 assert!(
16809 auth.is_ok(),
16810 "the daemon must answer a module's handshake while a restart waits on the \
16811 live-children record write; got {auth:?} after {elapsed:?}"
16812 );
16813
16814 tokio::time::timeout(Duration::from_secs(20), async {
16815 while lock_snapshot(&module.inner.snapshot)
16816 .unwrap()
16817 .pid
16818 .is_none_or(|pid| pid == old_pid)
16819 {
16820 tokio::time::sleep(Duration::from_millis(10)).await;
16821 }
16822 })
16823 .await
16824 .expect("the restart published its replacement");
16825 module.stop().await.unwrap();
16826 server.abort();
16827 }
16828}