1use std::{
2 collections::{HashMap, HashSet, VecDeque},
3 error::Error,
4 fmt, io,
5 path::PathBuf,
6 process::{ExitStatus, Stdio},
7 sync::{Arc, Mutex, OnceLock},
8 time::{Duration, SystemTime, UNIX_EPOCH},
9};
10
11use cortexkit_log::Retention;
12use serde_json::Value;
13use sha2::{Digest, Sha256};
14use subc_control::{
15 ClientControlPush, LiveSpawn, ModuleProtocol, RouteCloseReason, SpawnCursor, SpawnEvent,
16 SpawnEventKind, SpawnSnapshot, SupervisorHealthStatus, TerminalDisposition, TerminalExitKind,
17};
18use subc_protocol::{
19 manifest::{SelfSignalKind, SignalAnchor},
20 session::{
21 HealthReport, HealthStatus, ModuleControlCommand, ModuleControlRequest,
22 MODULE_CONTROL_OP_HEALTH_CHECK,
23 },
24 Flags, FrameType, Principal, Priority, SUBC_LAUNCH_NONCE_ENV, SUBC_MODULE_ID_ENV,
25};
26use tokio::{
27 process::{Child, Command},
28 sync::{mpsc, oneshot, watch, Mutex as AsyncMutex},
29 task::JoinHandle,
30 time::{sleep, sleep_until, timeout, timeout_at, Instant},
31};
32use tracing::{debug, error, info, warn};
33
34use crate::{
35 child_roster::ChildRoster,
36 daemon_config::{
37 CAPTURE_KEEP_ENV, CAPTURE_MAX_AGE_DAYS_ENV, CAPTURE_MAX_FILE_MB_ENV, CK_LOG_ENV,
38 },
39 forwarding::{
40 CloseReason, ForwardingError, ForwardingTable, GoodbyeTarget, ModuleControlRpcOutcome,
41 ModuleDrainTarget, PendingModuleControlRpc,
42 },
43 provenance::{ExecutableIdentityProbe, SpawnedFileIdentity},
44 registry::{ConnectionId, RegistrationEndReason, RegistryError},
45 stderr_tail::{
46 pump_stderr_to, pump_stdout_to, ChildOutputSink, StderrRing, StderrTailConfig,
47 StderrTailSnapshot,
48 },
49 terminal_ring::{TerminalHistorySnapshot, TerminalRecord, TerminalRing, TerminalRingConfig},
50 Frame, FrameSink, Registry,
51};
52
53#[cfg(not(windows))]
54use crate::provenance::spawned_file_identity;
55
56#[path = "supervise_swap.rs"]
57mod swap;
58
59#[path = "supervise_activity.rs"]
60mod activity;
61
62pub const SUBC_ARG: &str = "--subc";
68
69const DEFAULT_MAX_RESTARTS: u32 = 3;
70const DEFAULT_BACKOFF: Duration = Duration::from_millis(100);
71const DEFAULT_MAX_BACKOFF: Duration = Duration::from_secs(30);
72const DEFAULT_RESTART_WINDOW: Duration = Duration::from_secs(600);
76pub const DEFAULT_DRAIN_TIMEOUT: Duration = Duration::from_secs(30);
87const REGISTRY_RELEASE_TIMEOUT: Duration = Duration::from_secs(1);
88const REGISTRY_RELEASE_POLL: Duration = Duration::from_millis(10);
89const DRAIN_BUSY_REPROBE_INTERVAL: Duration = Duration::from_millis(250);
101const STDERR_PUMP_DRAIN_TIMEOUT: Duration = Duration::from_millis(250);
124pub const SPAWN_EVENT_RING_CAPACITY: usize = 4096;
126const SPAWN_SUBSCRIBER_BUFFER: usize = SPAWN_EVENT_RING_CAPACITY + 1;
127pub(crate) const SPAWN_SUBSCRIBER_LAGGED_CODE: &str = "spawn_subscriber_lagged";
132
133#[cfg(target_os = "macos")]
134struct PrivacyExec {
135 reader: tokio::io::unix::AsyncFd<std::io::PipeReader>,
136 deadline: tokio::time::Instant,
137 expected: Option<subc_os::FileIdentity>,
138 trampoline: Option<subc_os::FileIdentity>,
139 script: bool,
140 module_id: String,
141 #[cfg(test)]
142 confirmation_pause: Option<(String, Option<tokio::net::TcpStream>)>,
143}
144
145#[cfg(all(test, target_os = "macos"))]
146pub(crate) fn test_privacy_trampoline() -> PathBuf {
147 let path = std::env::current_exe()
150 .unwrap()
151 .parent()
152 .unwrap()
153 .parent()
154 .unwrap()
155 .join("privacy-trampoline-fixture");
156 assert!(
160 path.exists(),
161 "privacy-trampoline-fixture not built at {}: run `cargo build -p subc-daemon \
162 --bins --features test-support` or `cargo test -p subc-daemon` first",
163 path.display()
164 );
165 path
166}
167
168#[cfg(target_os = "macos")]
169fn probe_privacy_trampoline(path: &std::path::Path) -> Result<(), String> {
170 use std::io::Read;
171 let mut probe = std::process::Command::new(path)
172 .args(["__disclaim-exec", "--probe"])
173 .stdin(Stdio::null())
174 .stdout(Stdio::piped())
175 .stderr(Stdio::piped())
176 .spawn()
177 .map_err(|error| {
178 format!(
179 "privacy trampoline probe failed for {}: {error}",
180 path.display()
181 )
182 })?;
183 let deadline = std::time::Instant::now() + Duration::from_secs(5);
184 let status = loop {
185 match probe.try_wait() {
186 Ok(Some(status)) => break status,
187 Ok(None) if std::time::Instant::now() < deadline => {
188 std::thread::sleep(Duration::from_millis(5))
189 }
190 result => {
191 let _ = probe.kill();
192 let _ = probe.wait();
193 return Err(format!(
194 "privacy trampoline probe failed or timed out for {}: {result:?}",
195 path.display()
196 ));
197 }
198 }
199 };
200 let mut answer = String::new();
201 if let Some(stdout) = probe.stdout.take() {
202 let _ = stdout.take(256).read_to_string(&mut answer);
203 }
204 if status.success() && answer.trim() == subc_os::privacy_identity::TRAMPOLINE_PROBE {
205 return Ok(());
206 }
207 let mut diagnostic = String::new();
208 if let Some(stderr) = probe.stderr.take() {
209 let _ = stderr.take(1024).read_to_string(&mut diagnostic);
210 }
211 let cause = diagnostic
212 .trim()
213 .strip_prefix("ck-subc: own privacy identity refused: ")
214 .unwrap_or("binary does not implement the privacy trampoline protocol");
215 Err(format!(
216 "{cause}: probe of {} exited {status}",
217 path.display()
218 ))
219}
220
221#[cfg(target_os = "macos")]
222fn privacy_command(
223 spec: &ModuleSpec,
224 roster: &ChildRoster,
225) -> Result<
226 (
227 Command,
228 Option<PrivacyExec>,
229 subc_os::privacy_identity::ExecAcknowledgement,
230 ),
231 SuperviseError,
232> {
233 let failure = |cause: String| {
234 warn!(module_id = %spec.module_id, %cause, "privacy identity trampoline refused module spawn");
235 SuperviseError::Spawn {
236 program: spec.program.clone(),
237 source: io::Error::other(cause),
238 cgroup_path: None,
239 }
240 };
241 let trampoline = roster.privacy_trampoline().map_err(failure)?;
242 let program = if spec.program.components().count() == 1 && !spec.program.is_absolute() {
247 let path = spec
248 .env
249 .iter()
250 .find(|(key, _)| key == "PATH")
251 .map(|(_, value)| std::ffi::OsString::from(value))
252 .or_else(|| std::env::var_os("PATH"))
253 .unwrap_or_else(|| "/usr/bin:/bin".into());
254 std::env::split_paths(&path)
255 .map(|dir| dir.join(&spec.program))
256 .find(|path| path.is_file())
257 .unwrap_or_else(|| spec.program.clone())
258 } else {
259 spec.program.clone()
260 };
261 let expected = subc_os::file_identity(&program);
262 let trampoline_image = subc_os::file_identity(&trampoline);
263 let script = {
264 use std::io::Read;
265 let mut prefix = [0u8; 2];
266 std::fs::File::open(&program)
267 .is_ok_and(|mut file| file.read_exact(&mut prefix).is_ok() && &prefix == b"#!")
268 };
269 if expected.is_none() || expected == subc_os::file_identity(&trampoline) {
270 return Err(failure(
271 "privacy identity module executable is missing or is the trampoline itself".to_string(),
272 ));
273 }
274 let (reader, ack) = subc_os::privacy_identity::ExecAcknowledgement::pipe()
275 .map_err(|error| failure(error.to_string()))?;
276 let reader =
277 tokio::io::unix::AsyncFd::new(reader).map_err(|error| failure(error.to_string()))?;
278 let mut command = Command::new(&trampoline);
279 command
280 .arg("__disclaim-exec")
281 .arg(ack.fd().to_string())
282 .arg(&program);
283 ack.install(command.as_std_mut());
284 Ok((
285 command,
286 Some(PrivacyExec {
287 reader,
288 deadline: tokio::time::Instant::now() + Duration::from_secs(5),
289 expected,
290 trampoline: trampoline_image,
291 script,
292 module_id: spec.module_id.clone(),
293 #[cfg(test)]
294 confirmation_pause: spec
295 .env
296 .iter()
297 .find(|(key, _)| key == "SUBC_TEST_BEFORE_EXEC_CONFIRM")
298 .map(|(_, address)| (address.clone(), None)),
299 }),
300 ack,
301 ))
302}
303
304struct SupervisedChild {
305 child: Child,
306 exec: crate::activity::ExecOwner,
307 #[cfg(windows)]
308 nonce_delivery: Option<subc_os::LaunchNoncePipeDelivery>,
309 #[cfg(target_os = "macos")]
310 privacy_exec: Option<PrivacyExec>,
311 #[cfg(target_os = "macos")]
318 report_ready: Arc<OnceLock<()>>,
319 spawn_failure: Option<String>,
321 protocol: ModuleProtocol,
326 #[cfg(target_os = "linux")]
332 module_id: String,
333 #[cfg(target_os = "linux")]
334 cgroup_placement: Option<subc_cgroup::Placement>,
335 #[cfg(windows)]
357 job: Option<subc_jobobject::JobObject>,
358 stdout_pump: Option<JoinHandle<()>>,
359 stderr_pump: Option<StderrPump>,
360 stderr_ring: Arc<Mutex<StderrRing>>,
361 spawned_at: Instant,
364 spawned_at_ms: u64,
365 spawned_from: PathBuf,
366 launch_spec_digest: LaunchSpecDigest,
367 spawned_file_identity: Option<SpawnedFileIdentity>,
368 process_start_time: Option<u64>,
369 process_identity: Option<ProcessIdentity>,
370 pid: u32,
371 roster_guard: Option<crate::child_roster::RosterGuard>,
374}
375
376impl SupervisedChild {
377 fn id(&self) -> Option<u32> {
378 Some(self.pid)
379 }
380
381 fn process_identity(&self) -> Option<ProcessIdentity> {
382 self.process_identity
383 }
384
385 async fn wait(&mut self) -> io::Result<ExitStatus> {
386 #[cfg(target_os = "macos")]
387 self.confirm_privacy_exec().await;
388 let result = self.child.wait().await;
396 #[cfg(target_os = "linux")]
397 if result.is_ok() {
398 if let Some(placement) = self.cgroup_placement.as_ref() {
399 cleanup_reaped_cgroup(placement, &self.module_id).await;
400 self.cgroup_placement = None;
403 }
404 }
405 result
406 }
407
408 #[cfg(target_os = "macos")]
409 async fn confirm_privacy_exec(&mut self) {
410 let Some(pending) = &mut self.privacy_exec else {
411 return;
412 };
413 let result = tokio::time::timeout_at(pending.deadline, async {
414 let mut record = Vec::new();
415 loop {
416 let mut ready = pending.reader.readable().await?;
417 let read = ready.try_io(|reader| {
418 use std::io::Read;
419 let mut reader = reader.get_ref();
420 let mut buffer = [0u8; 256];
421 reader.read(&mut buffer).map(|count| (count, buffer))
422 });
423 match read {
424 Ok(Ok((0, _))) => return Ok::<_, io::Error>(record),
425 Ok(Ok((count, buffer))) => {
426 if record.len() + count > 1024 {
427 return Err(io::Error::other(
428 "privacy exec refusal record is too long",
429 ));
430 }
431 record.extend_from_slice(&buffer[..count]);
432 }
433 Ok(Err(error)) => return Err(error),
434 Err(_) => continue,
435 }
436 }
437 })
438 .await;
439 #[cfg(test)]
440 let result = if let Some((address, stream)) = &mut pending.confirmation_pause {
441 use tokio::io::{AsyncReadExt, AsyncWriteExt};
442 if stream.is_none() {
443 *stream = Some(
444 tokio::net::TcpStream::connect(address.as_str())
445 .await
446 .unwrap(),
447 );
448 stream.as_mut().unwrap().write_all(b"R").await.unwrap();
449 stream
450 .as_mut()
451 .unwrap()
452 .write_all(&self.pid.to_ne_bytes())
453 .await
454 .unwrap();
455 }
456 let mut reply = [0];
457 stream
458 .as_mut()
459 .unwrap()
460 .read_exact(&mut reply)
461 .await
462 .unwrap();
463 pending.confirmation_pause = None;
464 if &reply == b"F" {
465 Ok(Err(io::Error::other("injected late exec refusal")))
466 } else {
467 assert_eq!(&reply, b"X");
468 result
469 }
470 } else {
471 result
472 };
473 let pending = self.privacy_exec.as_ref().expect("pending exec");
476 let cause = match result {
477 Err(_) => Some("privacy identity trampoline did not exec within 5s".to_string()),
478 Ok(Err(error)) => Some(format!(
479 "privacy identity exec acknowledgement failed: {error}"
480 )),
481 Ok(Ok(record)) if !record.is_empty() => Some(
482 std::str::from_utf8(&record)
483 .ok()
484 .and_then(|record| {
485 record
486 .trim()
487 .strip_prefix(subc_os::privacy_identity::EXEC_REFUSAL_TAG)
488 })
489 .filter(|cause| !cause.is_empty())
490 .unwrap_or("invalid privacy exec refusal record")
491 .to_string(),
492 ),
493 Ok(Ok(_)) => match self.child.try_wait() {
494 Ok(Some(_status)) => None,
498 Err(error) => Some(format!("privacy identity trampoline wait failed: {error}")),
499 Ok(None) => {
500 let image = observe_spawned_image(self.pid);
501 if let Some(image) = image.filter(|image| {
502 image.executable.is_some()
503 && image.executable != pending.trampoline
504 && (image.executable == pending.expected || pending.script)
505 }) {
506 if let Err(error) = self
507 .roster_guard
508 .as_ref()
509 .map(|guard| guard.confirm_executable(image))
510 .transpose()
511 {
512 Some(format!(
513 "could not durably record confirmed module image: {error}"
514 ))
515 } else {
516 let _ = self.report_ready.set(());
517 info!(module_id = %pending.module_id, pid = self.pid,
518 "module spawned with own privacy identity (responsibility disclaimed)");
519 None
520 }
521 } else if image.is_none()
522 || image.is_some_and(|image| image.executable.is_none())
523 {
524 match tokio::time::timeout_at(pending.deadline, self.child.wait()).await {
531 Ok(Ok(_status)) => None,
532 Ok(Err(error)) => Some(format!("privacy identity trampoline wait failed: {error}")),
533 Err(_) => Some("privacy identity module executable remained unreadable until the 5s exec deadline".to_string()),
534 }
535 } else {
536 Some("privacy identity trampoline executable mismatch: expected the module program, not ck-subc".to_string())
537 }
538 }
539 },
540 };
541 let pending = self.privacy_exec.take().expect("pending exec");
542 self.exec.0.finish(if cause.is_some() {
545 crate::activity::ExecState::Refused
546 } else {
547 crate::activity::ExecState::CommittedExec
548 });
549 if let Some(cause) = cause {
550 warn!(module_id = %pending.module_id, pid = self.pid, %cause, "privacy identity trampoline refused module spawn");
551 self.spawn_failure = Some(cause);
552 if let Some(pid) = rustix::process::Pid::from_raw(self.pid as i32) {
555 let _ = rustix::process::kill_process_group(pid, rustix::process::Signal::KILL);
556 }
557 let _ = self.child.start_kill();
558 }
559 }
560
561 fn release_roster(&mut self) {
565 self.roster_guard = None;
566 }
567
568 fn start_kill(&mut self) -> io::Result<()> {
581 #[cfg(windows)]
582 if let Some(job) = &self.job {
583 if let Err(error) = job.terminate() {
584 debug!(
585 error = %error,
586 "job termination failed; the direct-child kill still owns the outcome"
587 );
588 }
589 }
590 #[cfg(target_os = "linux")]
591 kill_module_cgroup(self.cgroup_placement.as_ref(), &self.module_id);
592 self.child.start_kill()
593 }
594
595 async fn drain_stderr(&mut self, module_id: &str) {
596 if let Some(pump) = self.stdout_pump.take() {
597 settle_stdout_pump(module_id, pump, STDERR_PUMP_DRAIN_TIMEOUT).await;
598 }
599
600 let Some(pump) = self.stderr_pump.take() else {
601 return;
602 };
603 settle_stderr_pump(
604 module_id,
605 &self.stderr_ring,
606 pump,
607 STDERR_PUMP_DRAIN_TIMEOUT,
608 )
609 .await;
610 }
611}
612
613async fn settle_stdout_pump(module_id: &str, mut pump: JoinHandle<()>, bound: Duration) {
617 match timeout(bound, &mut pump).await {
618 Ok(Ok(())) => {}
619 Ok(Err(error)) => {
620 warn!(module_id, error = %error, "stdout pump ended unexpectedly");
621 }
622 Err(_) => {
623 drop(pump);
624 warn!(
625 module_id,
626 waited = ?bound,
627 "stdout pipe of the exited process has not drained; its reader keeps running without delaying the restart"
628 );
629 }
630 }
631}
632
633struct StderrPump {
636 task: JoinHandle<()>,
637 generation: u64,
638}
639
640async fn settle_stderr_pump(
646 module_id: &str,
647 ring: &Arc<Mutex<StderrRing>>,
648 pump: StderrPump,
649 bound: Duration,
650) {
651 let lock = || ring.lock().unwrap_or_else(|poisoned| poisoned.into_inner());
652 let StderrPump {
653 mut task,
654 generation,
655 } = pump;
656 lock().retire_pump(generation);
657 match timeout(bound, &mut task).await {
658 Ok(Ok(())) => {}
659 Ok(Err(err)) => {
660 let mut ring = lock();
661 ring.mark_incomplete_from(generation, format!("stderr pump ended unexpectedly: {err}"));
662 ring.finish_pump(generation);
663 warn!(module_id, error = %err, "stderr pump ended before clean EOF");
664 }
665 Err(_) => {
666 drop(task);
668 lock().mark_pump_late(
669 generation,
670 format!(
671 "stderr of the exited process had not reached EOF {bound:?} after it was \
672 retired (a descendant may still hold the pipe open); lines it still \
673 writes are kept in that process's section"
674 ),
675 );
676 warn!(
677 module_id,
678 waited = ?bound,
679 "stderr pipe of the exited process is still open; its reader keeps running without delaying the restart"
680 );
681 }
682 }
683}
684
685fn registration_release_events() -> &'static watch::Sender<u64> {
686 static EVENTS: OnceLock<watch::Sender<u64>> = OnceLock::new();
687 EVENTS.get_or_init(|| {
688 let (sender, _receiver) = watch::channel(0);
689 sender
690 })
691}
692
693pub(crate) fn notify_registration_release() {
694 let events = registration_release_events();
695 let next_generation = (*events.borrow()).wrapping_add(1);
696 events.send_replace(next_generation);
697}
698
699#[derive(Debug, Clone, PartialEq, Eq)]
701pub struct ModuleSpec {
702 pub module_id: String,
703 pub program: PathBuf,
704 pub args: Vec<String>,
705 pub env: Vec<(String, String)>,
706 pub reserved: bool,
711 pub reserved_prefixes: Vec<String>,
716 pub protocol: ModuleProtocol,
735 pub overlap: ModuleOverlap,
740}
741
742#[derive(Clone, PartialEq, Eq)]
745pub(crate) struct LaunchSpecDigest([u8; 32]);
746
747impl fmt::Debug for LaunchSpecDigest {
748 fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
749 f.write_str("LaunchSpecDigest([redacted])")
750 }
751}
752
753impl LaunchSpecDigest {
754 pub(crate) fn from_spec(spec: &ModuleSpec) -> Self {
755 fn field(hash: &mut Sha256, bytes: &[u8]) {
756 hash.update((bytes.len() as u64).to_le_bytes());
757 hash.update(bytes);
758 }
759 let mut hash = Sha256::new();
760 if let Some(program) = spec.program.to_str() {
761 field(&mut hash, b"utf8-program");
762 field(&mut hash, program.as_bytes());
763 } else {
764 #[cfg(unix)]
767 {
768 use std::os::unix::ffi::OsStrExt;
769 field(&mut hash, b"unix-program");
770 field(&mut hash, spec.program.as_os_str().as_bytes());
771 }
772 #[cfg(windows)]
773 {
774 use std::os::windows::ffi::OsStrExt;
775 field(&mut hash, b"windows-program");
776 let bytes = spec
777 .program
778 .as_os_str()
779 .encode_wide()
780 .flat_map(u16::to_le_bytes)
781 .collect::<Vec<_>>();
782 field(&mut hash, &bytes);
783 }
784 #[cfg(not(any(unix, windows)))]
785 field(&mut hash, spec.program.to_string_lossy().as_bytes());
786 }
787 hash.update((spec.args.len() as u64).to_le_bytes());
788 for arg in &spec.args {
789 field(&mut hash, arg.as_bytes());
790 }
791 let mut env = spec.env.iter().collect::<Vec<_>>();
792 env.sort_unstable();
793 hash.update((env.len() as u64).to_le_bytes());
794 for (key, value) in env {
795 field(&mut hash, key.as_bytes());
796 field(&mut hash, value.as_bytes());
797 }
798 field(
799 &mut hash,
800 match spec.protocol {
801 ModuleProtocol::Subc => b"subc",
802 ModuleProtocol::None => b"none",
803 },
804 );
805 Self(hash.finalize().into())
806 }
807}
808
809#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)]
816pub enum ModuleOverlap {
817 #[default]
819 Exclusive,
820 Safe,
832}
833
834impl ModuleOverlap {
835 pub fn as_str(self) -> &'static str {
836 match self {
837 Self::Exclusive => "exclusive",
838 Self::Safe => "safe",
839 }
840 }
841}
842
843pub const SUBC_SPAWN_ROLE_ENV: &str = "SUBC_SPAWN_ROLE";
853pub const SPAWN_ROLE_SWAP_CANDIDATE: &str = "swap_candidate";
855pub const DEFAULT_SWAP_READY_TIMEOUT: Duration = Duration::from_secs(100);
860
861#[derive(Debug, Clone, Copy, PartialEq, Eq)]
879pub struct RestartPolicy {
880 pub max_restarts: u32,
881 pub backoff: Duration,
884 pub max_backoff: Duration,
886 pub window: Duration,
890}
891
892impl RestartPolicy {
893 pub fn new(max_restarts: u32, backoff: Duration) -> Self {
897 Self {
898 max_restarts,
899 backoff,
900 max_backoff: DEFAULT_MAX_BACKOFF,
901 window: DEFAULT_RESTART_WINDOW,
902 }
903 }
904
905 pub fn with_max_backoff(mut self, max_backoff: Duration) -> Self {
906 self.max_backoff = max_backoff;
907 self
908 }
909
910 pub fn with_window(mut self, window: Duration) -> Self {
911 self.window = window;
912 self
913 }
914
915 fn delay_for_restart(&self, restart_in_window: u32) -> Duration {
920 if self.backoff.is_zero() || self.max_backoff.is_zero() {
921 return Duration::ZERO;
922 }
923
924 let mut delay = self.backoff;
925 for _ in 0..restart_in_window {
926 if delay >= self.max_backoff {
927 return self.max_backoff;
928 }
929 delay = delay
930 .checked_mul(10)
931 .unwrap_or(self.max_backoff)
932 .min(self.max_backoff);
933 }
934 delay.min(self.max_backoff)
935 }
936
937 fn budget_exhausted_detail(&self) -> String {
942 format!(
943 "crash budget exhausted: max_restarts={} within window_secs={}",
944 self.max_restarts,
945 self.window.as_secs()
946 )
947 }
948}
949
950impl Default for RestartPolicy {
951 fn default() -> Self {
952 Self {
953 max_restarts: DEFAULT_MAX_RESTARTS,
954 backoff: DEFAULT_BACKOFF,
955 max_backoff: DEFAULT_MAX_BACKOFF,
956 window: DEFAULT_RESTART_WINDOW,
957 }
958 }
959}
960
961#[derive(Debug, Clone, Copy, PartialEq, Eq)]
962struct CrashRestartSchedule {
963 restart_in_window: u32,
964 delay: Duration,
965}
966
967fn daemon_will_restart(
974 state: &mut SupervisorSnapshot,
975 policy: &RestartPolicy,
976 now: Instant,
977) -> bool {
978 state.enabled && state.crash_restarts_in_window(policy.window, now) < policy.max_restarts
979}
980
981const DEFAULT_HEALTH_CADENCE: Duration = Duration::from_secs(30);
982const DEFAULT_HEALTH_DEADLINE: Duration = Duration::from_secs(5);
983const DEFAULT_HEALTH_FAILURE_THRESHOLD: u32 = 3;
984const MAX_HEALTH_METRICS_BYTES: usize = 16 * 1024;
985
986#[derive(Debug, Clone, Copy, PartialEq, Eq)]
987pub enum HealthAction {
988 Report,
989 Restart,
990 Alert,
991}
992
993impl fmt::Display for HealthAction {
994 fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
995 f.write_str(match self {
996 Self::Report => "report",
997 Self::Restart => "restart",
998 Self::Alert => "alert",
999 })
1000 }
1001}
1002
1003#[derive(Debug, Clone, PartialEq, Eq)]
1004pub struct HealthConfig {
1005 pub http: Option<String>,
1009 pub cadence: Duration,
1010 pub deadline: Duration,
1011 pub failure_threshold: u32,
1012 pub on_degraded: HealthAction,
1013 pub on_failing: HealthAction,
1014 pub critical: bool,
1015}
1016
1017impl Default for HealthConfig {
1018 fn default() -> Self {
1019 Self {
1020 http: None,
1021 cadence: DEFAULT_HEALTH_CADENCE,
1022 deadline: DEFAULT_HEALTH_DEADLINE,
1023 failure_threshold: DEFAULT_HEALTH_FAILURE_THRESHOLD,
1024 on_degraded: HealthAction::Report,
1025 on_failing: HealthAction::Report,
1026 critical: false,
1027 }
1028 }
1029}
1030
1031#[derive(Debug, Clone, PartialEq)]
1049pub struct ModuleHealthStatus {
1050 pub status: SupervisorHealthStatus,
1051 pub last_probe_ms: Option<u64>,
1052 pub detail: Option<String>,
1053 pub metrics: Option<Value>,
1054 pub consecutive_failures: u32,
1055 pub late_answer_count: u64,
1058 pub last_late_answer_latency_ms: Option<u64>,
1060 pub last_action: Option<String>,
1061 pub last_action_ms: Option<u64>,
1065}
1066
1067impl Default for ModuleHealthStatus {
1068 fn default() -> Self {
1069 Self {
1070 status: SupervisorHealthStatus::Unknown,
1071 last_probe_ms: None,
1072 detail: None,
1073 metrics: None,
1074 consecutive_failures: 0,
1075 late_answer_count: 0,
1076 last_late_answer_latency_ms: None,
1077 last_action: None,
1078 last_action_ms: None,
1079 }
1080 }
1081}
1082
1083#[derive(Debug, Clone, Copy, PartialEq, Eq)]
1085pub enum ModuleState {
1086 Starting,
1087 Running,
1088 Unresponsive,
1089 Restarting,
1090 Draining,
1091 Stopped,
1092 Failed,
1093 Disabled,
1094}
1095
1096impl fmt::Display for ModuleState {
1097 fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
1098 f.write_str(match self {
1099 Self::Starting => "starting",
1100 Self::Running => "running",
1101 Self::Unresponsive => "unresponsive",
1102 Self::Restarting => "restarting",
1103 Self::Draining => "draining",
1104 Self::Stopped => "stopped",
1105 Self::Failed => "failed",
1106 Self::Disabled => "disabled",
1107 })
1108 }
1109}
1110
1111#[derive(Debug, Clone, Copy, PartialEq, Eq)]
1113pub enum ExitKind {
1114 Clean,
1115 Crash,
1116 DeliberateSeverance,
1117}
1118
1119impl From<ExitKind> for TerminalExitKind {
1120 fn from(kind: ExitKind) -> Self {
1121 match kind {
1122 ExitKind::Clean => Self::Clean,
1123 ExitKind::Crash => Self::Crash,
1124 ExitKind::DeliberateSeverance => Self::DeliberateSeverance,
1125 }
1126 }
1127}
1128
1129#[derive(Debug, Clone, Copy, PartialEq, Eq)]
1132pub(crate) struct ProcessIdentity {
1133 pub(crate) pid: u32,
1134 pub(crate) start_time: u64,
1135}
1136
1137#[derive(Debug, Clone, PartialEq, Eq)]
1139pub struct ExitReport {
1140 pub kind: ExitKind,
1141 pub code: Option<i32>,
1142 pub signal: Option<i32>,
1143 pub at_ms: u64,
1144}
1145
1146#[derive(Debug, Clone, PartialEq)]
1149pub struct ModuleStatus {
1150 pub module_id: String,
1151 pub waiting_for_previous_pid: Option<u32>,
1153 pub waiting_for_previous_detail: Option<String>,
1155 pub launch_nonce_source: Option<String>,
1160 pub state: ModuleState,
1161 pub enabled: bool,
1162 pub process_alive: bool,
1163 pub registration_active: bool,
1164 pub protocol: ModuleProtocol,
1169 pub live: bool,
1180 pub restart_count: u32,
1184 pub lifetime_restarts: u32,
1188 pub spawn_generation: u64,
1189 pub max_restarts: u32,
1194 pub restart_window: Duration,
1198 pub drain_timeout: Duration,
1202 pub restart_backoff: Duration,
1203 pub restart_max_backoff: Duration,
1204 pub pid: Option<u32>,
1209 pub spawned_at_ms: Option<u64>,
1210 pub spawned_from: Option<PathBuf>,
1211 pub(crate) launch_spec_digest: Option<LaunchSpecDigest>,
1212 pub process_start_time: Option<u64>,
1213 pub last_exit: Option<ExitReport>,
1214 pub health: ModuleHealthStatus,
1215}
1216
1217#[cfg(test)]
1218#[derive(Debug, Clone, Copy, PartialEq)]
1219struct ActorSelectCheckpoint {
1220 generation: u64,
1221 turn: u64,
1222 registered_connection: Option<ConnectionId>,
1223 next_probe_at: Option<Instant>,
1224 wake_after: Option<Duration>,
1225}
1226
1227#[derive(Debug, Clone, PartialEq)]
1228struct SupervisorSnapshot {
1229 exec_outcome: Option<Arc<crate::activity::ExecOutcome>>,
1230 #[cfg(windows)]
1231 pipe_consumed: Option<Arc<OnceLock<()>>>,
1232 #[cfg(test)]
1234 actor_turns: u64,
1235 #[cfg(test)]
1238 actor_select: Option<ActorSelectCheckpoint>,
1239 state: ModuleState,
1240 enabled: bool,
1241 process_alive: bool,
1242 spawned_protocol: Option<ModuleProtocol>,
1243 spawn_failure: Option<String>,
1244 waiting_for_previous_pid: Option<u32>,
1245 waiting_for_previous_detail: Option<String>,
1246 crash_restarts: VecDeque<Instant>,
1252 lifetime_restarts: u32,
1253 spawn_generation: u64,
1262 pid: Option<u32>,
1263 #[cfg(target_os = "macos")]
1264 report_ready: Option<Arc<OnceLock<()>>>,
1265 reaped_pid: Option<u32>,
1267 respawn_pending: bool,
1269 coalesced_restart_pending: bool,
1271 spawned_at_ms: Option<u64>,
1272 spawned_from: Option<PathBuf>,
1273 launch_spec_digest: Option<LaunchSpecDigest>,
1274 spawned_file_identity: Option<SpawnedFileIdentity>,
1275 process_start_time: Option<u64>,
1276 deliberate_severance: Option<ProcessIdentity>,
1277 last_exit: Option<ExitReport>,
1278 drain_disposition_detail: Option<String>,
1280 health: ModuleHealthStatus,
1281 in_alternate_slot: bool,
1286 draining_to_replace: bool,
1293 configuration_updated_since_spawn: bool,
1299}
1300
1301impl SupervisorSnapshot {
1302 fn reported_pid(&self) -> Option<u32> {
1307 #[cfg(target_os = "macos")]
1308 if self
1309 .report_ready
1310 .as_ref()
1311 .is_some_and(|ready| ready.get().is_none())
1312 {
1313 return None;
1314 }
1315 self.pid
1316 }
1317
1318 fn starting() -> Self {
1319 Self::new(ModuleState::Starting, true)
1320 }
1321
1322 fn disabled() -> Self {
1323 Self::new(ModuleState::Disabled, false)
1324 }
1325
1326 fn crash_restarts_in_window(&mut self, window: Duration, now: Instant) -> u32 {
1330 while let Some(oldest) = self.crash_restarts.front() {
1331 if now.duration_since(*oldest) > window {
1332 self.crash_restarts.pop_front();
1333 } else {
1334 break;
1335 }
1336 }
1337 u32::try_from(self.crash_restarts.len()).unwrap_or(u32::MAX)
1338 }
1339
1340 fn record_crash_restart(&mut self, policy: &RestartPolicy, now: Instant) {
1346 self.crash_restarts.push_back(now);
1347 while self.crash_restarts.len() > policy.max_restarts as usize {
1348 self.crash_restarts.pop_front();
1349 }
1350 self.lifetime_restarts += 1;
1351 }
1352
1353 fn next_crash_restart(
1357 &mut self,
1358 policy: &RestartPolicy,
1359 now: Instant,
1360 ) -> Option<CrashRestartSchedule> {
1361 let restart_in_window = self.crash_restarts_in_window(policy.window, now);
1362 if restart_in_window >= policy.max_restarts {
1363 return None;
1364 }
1365 self.record_crash_restart(policy, now);
1366 Some(CrashRestartSchedule {
1367 restart_in_window,
1368 delay: policy.delay_for_restart(restart_in_window),
1369 })
1370 }
1371
1372 fn clear_crash_restarts(&mut self) {
1377 self.crash_restarts.clear();
1378 }
1379
1380 fn new(state: ModuleState, enabled: bool) -> Self {
1381 Self {
1382 exec_outcome: None,
1383 #[cfg(test)]
1384 actor_turns: 0,
1385 #[cfg(test)]
1386 actor_select: None,
1387 state,
1388 enabled,
1389 process_alive: false,
1390 #[cfg(windows)]
1391 pipe_consumed: None,
1392 spawned_protocol: None,
1393 spawn_failure: None,
1394 waiting_for_previous_pid: None,
1395 waiting_for_previous_detail: None,
1396 crash_restarts: VecDeque::new(),
1397 lifetime_restarts: 0,
1398 spawn_generation: 0,
1399 pid: None,
1400 #[cfg(target_os = "macos")]
1401 report_ready: None,
1402 reaped_pid: None,
1403 respawn_pending: false,
1404 coalesced_restart_pending: false,
1405 spawned_at_ms: None,
1406 spawned_from: None,
1407 launch_spec_digest: None,
1408 spawned_file_identity: None,
1409 process_start_time: None,
1410 deliberate_severance: None,
1411 last_exit: None,
1412 drain_disposition_detail: None,
1413 health: ModuleHealthStatus::default(),
1414 in_alternate_slot: false,
1415 draining_to_replace: false,
1416 configuration_updated_since_spawn: false,
1417 }
1418 }
1419}
1420
1421type SharedSnapshot = Arc<Mutex<SupervisorSnapshot>>;
1422
1423type SpawnSubscriberKey = (ConnectionId, u64);
1424
1425#[derive(Debug)]
1426struct SpawnSubscriber {
1427 version: u8,
1428 frames: mpsc::Sender<Frame>,
1429 lagged: Option<oneshot::Sender<SpawnCursor>>,
1433}
1434
1435#[derive(Debug)]
1436struct SpawnEventState {
1437 daemon_incarnation: String,
1438 seq: u64,
1439 capacity: usize,
1440 live: HashMap<String, LiveSpawn>,
1441 generations: HashMap<String, u64>,
1442 events: VecDeque<SpawnEvent>,
1443 subscribers: HashMap<SpawnSubscriberKey, SpawnSubscriber>,
1444}
1445
1446impl Default for SpawnEventState {
1447 fn default() -> Self {
1448 Self {
1449 daemon_incarnation: "unconfigured".to_string(),
1450 seq: 0,
1451 capacity: SPAWN_EVENT_RING_CAPACITY,
1452 live: HashMap::new(),
1453 generations: HashMap::new(),
1454 events: VecDeque::new(),
1455 subscribers: HashMap::new(),
1456 }
1457 }
1458}
1459
1460#[derive(Debug, Clone, Default)]
1461struct SpawnEventFeed(
1462 Arc<Mutex<SpawnEventState>>,
1463 #[cfg(test)] Arc<Mutex<Option<std::net::TcpStream>>>,
1464);
1465
1466#[derive(Debug, Clone, PartialEq, Eq)]
1467pub(crate) enum SpawnSubscribeRefusal {
1468 ForeignIncarnation { current: String },
1469 TooOld { oldest: SpawnCursor },
1470 Frame(String),
1471}
1472
1473impl SpawnEventFeed {
1474 fn configure_incarnation(&self, daemon_incarnation: String) {
1475 let mut state = self.0.lock().unwrap_or_else(|p| p.into_inner());
1476 state.daemon_incarnation = daemon_incarnation;
1477 state.seq = 0;
1478 state.live.clear();
1479 state.generations.clear();
1480 state.events.clear();
1481 state.subscribers.clear();
1482 }
1483
1484 fn cursor(state: &SpawnEventState) -> SpawnCursor {
1485 SpawnCursor {
1486 daemon_incarnation: state.daemon_incarnation.clone(),
1487 seq: state.seq,
1488 }
1489 }
1490
1491 fn snapshot(&self) -> SpawnSnapshot {
1492 let state = self.0.lock().unwrap_or_else(|p| p.into_inner());
1493 let mut live = state.live.values().cloned().collect::<Vec<_>>();
1494 live.sort_by(|left, right| left.module_id.cmp(&right.module_id));
1495 SpawnSnapshot {
1496 cursor: Self::cursor(&state),
1497 ring_bound: state.capacity as u64,
1498 live,
1499 }
1500 }
1501
1502 fn emit_spawned(&self, module_id: &str, pid: u32, spawned_at_ms: u64) -> u64 {
1503 let mut state = self.0.lock().unwrap_or_else(|p| p.into_inner());
1504 let generation = state
1505 .generations
1506 .get(module_id)
1507 .copied()
1508 .unwrap_or(0)
1509 .checked_add(1)
1510 .expect("spawn generation exhausted");
1511 state.generations.insert(module_id.to_string(), generation);
1512 let live = LiveSpawn {
1513 module_id: module_id.to_string(),
1514 spawn_generation: generation,
1515 pid,
1516 spawned_at_ms,
1517 };
1518 state.live.insert(module_id.to_string(), live);
1519 Self::emit_locked(
1520 &mut state,
1521 SpawnEventKind::Spawned,
1522 module_id.to_string(),
1523 generation,
1524 pid,
1525 None,
1526 None,
1527 );
1528 generation
1529 }
1530
1531 fn emit_exited(&self, module_id: &str, exit_code: Option<i32>, exit_signal: Option<i32>) {
1532 let mut state = self.0.lock().unwrap_or_else(|p| p.into_inner());
1533 let Some(live) = state.live.remove(module_id) else {
1534 warn!(
1535 module_id,
1536 "terminal record had no live spawn event identity"
1537 );
1538 return;
1539 };
1540 Self::emit_locked(
1541 &mut state,
1542 SpawnEventKind::Exited,
1543 module_id.to_string(),
1544 live.spawn_generation,
1545 live.pid,
1546 exit_code,
1547 exit_signal,
1548 );
1549 }
1550
1551 fn emit_superseded_exited(
1558 &self,
1559 module_id: &str,
1560 spawn_generation: u64,
1561 pid: u32,
1562 exit_code: Option<i32>,
1563 exit_signal: Option<i32>,
1564 ) {
1565 let mut state = self.0.lock().unwrap_or_else(|p| p.into_inner());
1566 if state
1567 .live
1568 .get(module_id)
1569 .is_some_and(|live| live.spawn_generation == spawn_generation)
1570 {
1571 state.live.remove(module_id);
1572 }
1573 Self::emit_locked(
1574 &mut state,
1575 SpawnEventKind::Exited,
1576 module_id.to_string(),
1577 spawn_generation,
1578 pid,
1579 exit_code,
1580 exit_signal,
1581 );
1582 }
1583
1584 #[allow(clippy::too_many_arguments)]
1585 fn emit_locked(
1586 state: &mut SpawnEventState,
1587 kind: SpawnEventKind,
1588 module_id: String,
1589 spawn_generation: u64,
1590 pid: u32,
1591 exit_code: Option<i32>,
1592 exit_signal: Option<i32>,
1593 ) {
1594 state.seq = state
1595 .seq
1596 .checked_add(1)
1597 .expect("spawn event sequence exhausted");
1598 let event = SpawnEvent {
1599 cursor: Self::cursor(state),
1600 kind,
1601 module_id,
1602 spawn_generation,
1603 pid,
1604 exit_code,
1605 exit_signal,
1606 };
1607 state.events.push_back(event.clone());
1608 while state.events.len() > state.capacity {
1609 state.events.pop_front();
1610 }
1611 let body = match serde_json::to_vec(&event) {
1612 Ok(body) => body,
1613 Err(error) => {
1614 error!(%error, "failed to serialize supervisor spawn event");
1615 return;
1616 }
1617 };
1618 state.subscribers.retain(|(connection_id, corr), subscriber| {
1619 let frame = Frame::build_with_version(
1620 subscriber.version,
1621 FrameType::StreamData,
1622 control_flags(),
1623 0,
1624 0,
1625 *corr,
1626 body.clone(),
1627 );
1628 match frame {
1629 Ok(frame) => {
1630 if subscriber.frames.try_send(frame).is_ok() {
1631 true
1632 } else {
1633 warn!(connection_id = connection_id.get(), corr, "dropping lagged supervisor spawn subscriber");
1634 if let Some(lagged) = subscriber.lagged.take() {
1635 let _ = lagged.send(event.cursor.clone());
1636 }
1637 false
1638 }
1639 }
1640 Err(error) => {
1641 warn!(connection_id = connection_id.get(), corr, %error, "dropping supervisor spawn subscriber after frame build failure");
1642 false
1643 }
1644 }
1645 });
1646 }
1647
1648 fn subscribe(
1649 &self,
1650 connection_id: ConnectionId,
1651 corr: u64,
1652 version: u8,
1653 since: Option<SpawnCursor>,
1654 sink: FrameSink,
1655 ) -> Result<(), SpawnSubscribeRefusal> {
1656 let (frames, mut receiver) = mpsc::channel(SPAWN_SUBSCRIBER_BUFFER);
1657 let (lagged, mut lagged_rx) = oneshot::channel::<SpawnCursor>();
1658 {
1659 let mut state = self.0.lock().unwrap_or_else(|p| p.into_inner());
1660 let replay = if let Some(since) = since {
1661 if since.daemon_incarnation != state.daemon_incarnation {
1662 return Err(SpawnSubscribeRefusal::ForeignIncarnation {
1663 current: state.daemon_incarnation.clone(),
1664 });
1665 }
1666 if let Some(oldest) = state.events.front().map(|event| event.cursor.clone()) {
1667 if since.seq < oldest.seq.saturating_sub(1) {
1668 return Err(SpawnSubscribeRefusal::TooOld { oldest });
1669 }
1670 }
1671 state
1672 .events
1673 .iter()
1674 .filter(|event| event.cursor.seq > since.seq)
1675 .cloned()
1676 .collect::<Vec<_>>()
1677 } else {
1678 Vec::new()
1679 };
1680 for event in replay {
1681 let body = serde_json::to_vec(&event)
1682 .map_err(|error| SpawnSubscribeRefusal::Frame(error.to_string()))?;
1683 let frame = Frame::build_with_version(
1684 version,
1685 FrameType::StreamData,
1686 control_flags(),
1687 0,
1688 0,
1689 corr,
1690 body,
1691 )
1692 .map_err(|error| SpawnSubscribeRefusal::Frame(error.to_string()))?;
1693 frames
1694 .try_send(frame)
1695 .map_err(|error| SpawnSubscribeRefusal::Frame(error.to_string()))?;
1696 }
1697 state.subscribers.insert(
1698 (connection_id, corr),
1699 SpawnSubscriber {
1700 version,
1701 frames,
1702 lagged: Some(lagged),
1703 },
1704 );
1705 }
1706 tokio::spawn(async move {
1717 while let Some(frame) = receiver.recv().await {
1718 if sink.send(frame).await.is_err() {
1719 return;
1720 }
1721 }
1722 let Ok(first_undelivered) = lagged_rx.try_recv() else {
1723 return;
1724 };
1725 match spawn_subscriber_lagged_frame(version, corr, first_undelivered) {
1726 Ok(frame) => {
1727 let _ = sink.send(frame).await;
1728 }
1729 Err(error) => {
1730 error!(%error, corr, "failed to build lagged spawn subscriber terminal frame");
1731 }
1732 }
1733 });
1734 Ok(())
1735 }
1736
1737 fn cancel(&self, connection_id: ConnectionId, corr: u64) -> bool {
1738 let Some(subscriber) = self
1739 .0
1740 .lock()
1741 .unwrap_or_else(|p| p.into_inner())
1742 .subscribers
1743 .remove(&(connection_id, corr))
1744 else {
1745 return false;
1746 };
1747 if let Ok(frame) = Frame::build_with_version(
1748 subscriber.version,
1749 FrameType::StreamEnd,
1750 control_flags(),
1751 0,
1752 0,
1753 corr,
1754 Vec::new(),
1755 ) {
1756 tokio::spawn(async move {
1757 let _ = subscriber.frames.send(frame).await;
1758 });
1759 }
1760 true
1761 }
1762
1763 fn remove_connection(&self, connection_id: ConnectionId) {
1764 self.0
1765 .lock()
1766 .unwrap_or_else(|p| p.into_inner())
1767 .subscribers
1768 .retain(|(subscriber_connection, _), _| *subscriber_connection != connection_id);
1769 }
1770
1771 #[cfg(any(test, feature = "test-support"))]
1772 fn set_capacity(&self, capacity: usize) {
1773 self.0.lock().unwrap_or_else(|p| p.into_inner()).capacity = capacity;
1774 }
1775
1776 #[cfg(any(test, feature = "test-support"))]
1777 fn subscriber_count(&self) -> usize {
1778 self.0
1779 .lock()
1780 .unwrap_or_else(|p| p.into_inner())
1781 .subscribers
1782 .len()
1783 }
1784}
1785
1786fn spawn_subscriber_lagged_frame(
1789 version: u8,
1790 corr: u64,
1791 first_undelivered: SpawnCursor,
1792) -> Result<Frame, String> {
1793 let body = serde_json::to_vec(&subc_protocol::ErrorBody {
1794 code: SPAWN_SUBSCRIBER_LAGGED_CODE.to_string(),
1795 message: "spawn subscriber fell behind and was dropped; resubscribe from the last cursor received"
1796 .to_string(),
1797 detail: Some(serde_json::json!({
1798 "first_undelivered_cursor": first_undelivered
1799 })),
1800 })
1801 .map_err(|error| error.to_string())?;
1802 Frame::build_with_version(version, FrameType::Error, control_flags(), 0, 0, corr, body)
1803 .map_err(|error| error.to_string())
1804}
1805
1806pub trait ModuleProcessLiveness: Send + Sync {
1807 fn process_live(&self, module_id: &str) -> Option<bool>;
1808
1809 fn process_replacing(&self, _module_id: &str) -> bool {
1815 false
1816 }
1817}
1818
1819#[derive(Debug, Clone, Default)]
1821pub struct SupervisorProcessLiveness {
1822 snapshots: Arc<Mutex<HashMap<String, SharedSnapshot>>>,
1823}
1824
1825impl SupervisorProcessLiveness {
1826 pub fn new() -> Self {
1827 Self::default()
1828 }
1829
1830 fn track(&self, module_id: String, snapshot: SharedSnapshot) {
1831 let mut snapshots = self
1832 .snapshots
1833 .lock()
1834 .unwrap_or_else(|poisoned| poisoned.into_inner());
1835 snapshots.insert(module_id, snapshot);
1836 }
1837
1838 fn untrack_if_current(&self, module_id: &str, snapshot: &SharedSnapshot) {
1839 let mut snapshots = self
1840 .snapshots
1841 .lock()
1842 .unwrap_or_else(|poisoned| poisoned.into_inner());
1843 let is_current = snapshots
1844 .get(module_id)
1845 .map(|tracked| Arc::ptr_eq(tracked, snapshot))
1846 .unwrap_or(false);
1847 if is_current {
1848 snapshots.remove(module_id);
1849 }
1850 }
1851}
1852
1853impl ModuleProcessLiveness for SupervisorProcessLiveness {
1854 fn process_live(&self, module_id: &str) -> Option<bool> {
1855 let snapshot = {
1856 let snapshots = self
1857 .snapshots
1858 .lock()
1859 .unwrap_or_else(|poisoned| poisoned.into_inner());
1860 snapshots.get(module_id).cloned()
1861 }?;
1862 let snapshot = snapshot
1863 .lock()
1864 .unwrap_or_else(|poisoned| poisoned.into_inner());
1865 Some(snapshot.state == ModuleState::Running && snapshot.process_alive)
1866 }
1867
1868 fn process_replacing(&self, module_id: &str) -> bool {
1869 let Some(snapshot) = self
1870 .snapshots
1871 .lock()
1872 .unwrap_or_else(|poisoned| poisoned.into_inner())
1873 .get(module_id)
1874 .cloned()
1875 else {
1876 return false;
1877 };
1878 let snapshot = snapshot
1879 .lock()
1880 .unwrap_or_else(|poisoned| poisoned.into_inner());
1881 snapshot.enabled
1882 && match snapshot.state {
1883 ModuleState::Restarting => true,
1884 ModuleState::Draining => snapshot.draining_to_replace,
1885 ModuleState::Starting
1886 | ModuleState::Running
1887 | ModuleState::Unresponsive
1888 | ModuleState::Stopped
1889 | ModuleState::Failed
1890 | ModuleState::Disabled => false,
1891 }
1892 }
1893}
1894
1895#[cfg(test)]
1896#[derive(Debug, Default)]
1897struct ReloadExitRecordGate {
1898 reached: tokio::sync::Notify,
1899 resume: tokio::sync::Notify,
1900}
1901
1902#[derive(Debug, Clone, Copy)]
1903enum RespawnKind {
1904 Spawn,
1905 Reload,
1906}
1907
1908#[derive(Debug, Clone, Copy)]
1909struct PendingRespawn {
1910 deadline: Instant,
1911 kind: RespawnKind,
1912}
1913
1914type ReloadReply = oneshot::Sender<Result<(), SuperviseError>>;
1915
1916#[derive(Debug, Clone)]
1917struct SupervisorRuntimeConfig {
1918 start_requester: Option<Principal>,
1919 pending_activity_restart: bool,
1920 provenance_probe: ExecutableIdentityProbe,
1921 scheduled_respawn: Arc<Mutex<Option<PendingRespawn>>>,
1923 deferred_reload_reply: Arc<Mutex<Option<ReloadReply>>>,
1925 restart_policy: RestartPolicy,
1926 drain_timeout: Duration,
1929 effective_drain_timeout: Arc<Mutex<Duration>>,
1932 default_drain_timeout: Duration,
1935 health: HealthConfig,
1936 connection_file_path: Option<PathBuf>,
1937 capture_logs_dir: Option<PathBuf>,
1938 forwarding: Option<Arc<ForwardingTable>>,
1939 supervisor_handle: Option<SupervisorHandle>,
1942 stderr_ring: Arc<Mutex<StderrRing>>,
1949 terminal_ring: Arc<Mutex<TerminalRing>>,
1950 spawn_events: SpawnEventFeed,
1951 child_roster: ChildRoster,
1952 #[cfg(target_os = "linux")]
1953 cgroup_placement: Option<subc_cgroup::Placement>,
1954 #[cfg(test)]
1955 test_seed_stale_facts_before_enable_spawn: bool,
1956 #[cfg(test)]
1957 test_reload_exit_record_gate: Option<Arc<ReloadExitRecordGate>>,
1958}
1959
1960#[derive(Debug, Clone, PartialEq, Eq)]
1961struct SupervisedConfiguration {
1962 spec: ModuleSpec,
1963 health: HealthConfig,
1964}
1965
1966#[derive(Debug, Clone, Default)]
1972pub struct SupervisorHandle {
1973 activity: Arc<Mutex<Option<crate::activity::ActivityLedger>>>,
1974 activity_images: Arc<Mutex<HashMap<String, activity::ActivityImage>>>,
1975 modules: Arc<Mutex<HashMap<String, SupervisedModule>>>,
1976 configured_ids: Arc<Mutex<HashSet<String>>>,
1988 spawn_events: SpawnEventFeed,
1989 reserved_nonces: Arc<Mutex<HashMap<String, Option<String>>>>,
2000 removal_tombstones: Arc<Mutex<HashMap<String, u64>>>,
2006 spawn_nonces: Arc<Mutex<HashMap<String, String>>>,
2010 reserved_prefix_owners: Arc<Mutex<HashMap<String, String>>>,
2018 swaps: Arc<Mutex<HashMap<String, OpenSwap>>>,
2024 promotion_observer: PromotionObserverSlot,
2026 operation_lock: Arc<AsyncMutex<()>>,
2030}
2031
2032pub(crate) trait SwapPromotionObserver: Send + Sync {
2041 fn swap_promoted(&self, registration: &crate::registry::ModuleRegistration);
2042}
2043
2044#[derive(Clone, Default)]
2048struct PromotionObserverSlot(Arc<Mutex<Option<std::sync::Weak<dyn SwapPromotionObserver>>>>);
2049
2050impl fmt::Debug for PromotionObserverSlot {
2051 fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
2052 f.write_str("PromotionObserverSlot")
2053 }
2054}
2055
2056#[derive(Debug, Clone)]
2058struct OpenSwap {
2059 candidate_nonce: String,
2062 incumbent_nonce: Option<String>,
2067 candidate_admitted: bool,
2071}
2072
2073#[derive(Debug, Clone, Copy, PartialEq, Eq)]
2076pub(crate) enum SwapHelloAdmission {
2077 NotSwapping,
2080 Candidate,
2082 Refused,
2085}
2086
2087#[derive(Debug, Clone, PartialEq, Eq)]
2088pub(crate) enum ReservedHelloRejection {
2089 Exact {
2090 module_id: String,
2091 },
2092 Prefix {
2093 prefix: String,
2094 owner_module_id: String,
2095 },
2096}
2097
2098impl SupervisorHandle {
2099 pub fn new() -> Self {
2100 Self::default()
2101 }
2102
2103 pub(crate) fn spawn_snapshot(&self) -> SpawnSnapshot {
2104 self.spawn_events.snapshot()
2105 }
2106
2107 pub(crate) fn subscribe_spawns(
2108 &self,
2109 connection_id: ConnectionId,
2110 corr: u64,
2111 version: u8,
2112 since: Option<SpawnCursor>,
2113 sink: FrameSink,
2114 ) -> Result<(), SpawnSubscribeRefusal> {
2115 self.spawn_events
2116 .subscribe(connection_id, corr, version, since, sink)
2117 }
2118
2119 pub(crate) fn cancel_spawn_subscription(&self, connection_id: ConnectionId, corr: u64) -> bool {
2120 self.spawn_events.cancel(connection_id, corr)
2121 }
2122
2123 pub(crate) fn remove_spawn_subscribers(&self, connection_id: ConnectionId) {
2124 self.spawn_events.remove_connection(connection_id);
2125 }
2126
2127 #[cfg(any(test, feature = "test-support"))]
2128 pub fn set_spawn_event_capacity_for_test(&self, capacity: usize) {
2129 assert!(capacity > 0, "spawn event capacity must be non-zero");
2130 self.spawn_events.set_capacity(capacity);
2131 }
2132
2133 #[cfg(any(test, feature = "test-support"))]
2134 pub fn spawn_subscriber_count_for_test(&self) -> usize {
2135 self.spawn_events.subscriber_count()
2136 }
2137
2138 pub fn set_spawn_nonce(&self, module_id: &str, nonce: String) {
2141 self.spawn_nonces
2142 .lock()
2143 .unwrap_or_else(|poisoned| poisoned.into_inner())
2144 .insert(module_id.to_string(), nonce);
2145 }
2146
2147 pub fn set_reserved_nonce(&self, module_id: &str, nonce: String) {
2150 self.reserved_nonces
2151 .lock()
2152 .unwrap_or_else(|poisoned| poisoned.into_inner())
2153 .insert(module_id.to_string(), Some(nonce));
2154 }
2155
2156 pub fn set_reserved_prefixes(&self, owner_module_id: &str, prefixes: &[String]) {
2158 let mut owners = self
2159 .reserved_prefix_owners
2160 .lock()
2161 .unwrap_or_else(|poisoned| poisoned.into_inner());
2162 owners.retain(|_, owner| owner != owner_module_id);
2163 for prefix in prefixes {
2164 owners.insert(prefix.clone(), owner_module_id.to_string());
2165 }
2166 }
2167
2168 #[cfg(test)]
2170 pub(crate) fn spawn_nonce(&self, module_id: &str) -> Option<String> {
2171 self.spawn_nonces
2172 .lock()
2173 .unwrap_or_else(|poisoned| poisoned.into_inner())
2174 .get(module_id)
2175 .cloned()
2176 }
2177
2178 fn apply_identity_configuration(&self, spec: &ModuleSpec) {
2179 self.set_reserved_prefixes(&spec.module_id, &spec.reserved_prefixes);
2180 let spawn_nonce = self
2181 .spawn_nonces
2182 .lock()
2183 .unwrap_or_else(|poisoned| poisoned.into_inner())
2184 .get(&spec.module_id)
2185 .cloned();
2186 let mut reserved_nonces = self
2187 .reserved_nonces
2188 .lock()
2189 .unwrap_or_else(|poisoned| poisoned.into_inner());
2190 if spec.reserved {
2191 reserved_nonces.insert(spec.module_id.clone(), spawn_nonce);
2196 }
2197 drop(reserved_nonces);
2198 self.removal_tombstones
2202 .lock()
2203 .unwrap_or_else(|poisoned| poisoned.into_inner())
2204 .remove(&spec.module_id);
2205 }
2206
2207 pub fn reserved_hello_authorized(&self, module_id: &str, presented: Option<&str>) -> bool {
2212 self.reserved_hello_rejection(module_id, presented)
2213 .is_none()
2214 }
2215
2216 pub(crate) fn reserved_hello_rejection(
2217 &self,
2218 module_id: &str,
2219 presented: Option<&str>,
2220 ) -> Option<ReservedHelloRejection> {
2221 let nonces = self
2222 .reserved_nonces
2223 .lock()
2224 .unwrap_or_else(|poisoned| poisoned.into_inner());
2225 if let Some(expected) = nonces.get(module_id) {
2226 let authorized = match expected {
2230 Some(expected) => {
2231 presented.is_some_and(|p| constant_time_eq(expected.as_bytes(), p.as_bytes()))
2232 }
2233 None => false,
2234 };
2235 if authorized {
2236 return None;
2237 }
2238 return Some(ReservedHelloRejection::Exact {
2239 module_id: module_id.to_string(),
2240 });
2241 }
2242 drop(nonces);
2243
2244 let matched_prefix = self
2245 .reserved_prefix_owners
2246 .lock()
2247 .unwrap_or_else(|poisoned| poisoned.into_inner())
2248 .iter()
2249 .filter(|(prefix, _)| module_id.starts_with(prefix.as_str()))
2250 .max_by_key(|(prefix, _)| prefix.len())
2251 .map(|(prefix, owner)| (prefix.clone(), owner.clone()));
2252 let (prefix, owner_module_id) = matched_prefix?;
2253
2254 let authorized = presented.is_some_and(|presented| {
2255 self.spawn_nonces
2256 .lock()
2257 .unwrap_or_else(|poisoned| poisoned.into_inner())
2258 .get(&owner_module_id)
2259 .is_some_and(|expected| constant_time_eq(expected.as_bytes(), presented.as_bytes()))
2260 || self.swap_nonce_matches(&owner_module_id, presented)
2263 });
2264 if authorized {
2265 None
2266 } else {
2267 Some(ReservedHelloRejection::Prefix {
2268 prefix,
2269 owner_module_id,
2270 })
2271 }
2272 }
2273
2274 pub fn spawned_consumer_authorized(&self, module_id: &str, presented: &str) -> bool {
2279 if presented.is_empty() {
2280 return false;
2281 }
2282 let nonces = self
2283 .spawn_nonces
2284 .lock()
2285 .unwrap_or_else(|poisoned| poisoned.into_inner());
2286 let current = nonces
2287 .get(module_id)
2288 .is_some_and(|expected| constant_time_eq(expected.as_bytes(), presented.as_bytes()));
2289 drop(nonces);
2290 current || self.swap_nonce_matches(module_id, presented)
2295 }
2296
2297 fn swap_nonce_matches(&self, module_id: &str, presented: &str) -> bool {
2299 let swaps = self
2300 .swaps
2301 .lock()
2302 .unwrap_or_else(|poisoned| poisoned.into_inner());
2303 swaps.get(module_id).is_some_and(|swap| {
2304 constant_time_eq(swap.candidate_nonce.as_bytes(), presented.as_bytes())
2305 || swap.incumbent_nonce.as_deref().is_some_and(|incumbent| {
2306 constant_time_eq(incumbent.as_bytes(), presented.as_bytes())
2307 })
2308 })
2309 }
2310
2311 pub(crate) fn open_swap(&self, module_id: &str, candidate_nonce: String) {
2314 let incumbent_nonce = self
2315 .spawn_nonces
2316 .lock()
2317 .unwrap_or_else(|poisoned| poisoned.into_inner())
2318 .get(module_id)
2319 .cloned();
2320 self.swaps
2321 .lock()
2322 .unwrap_or_else(|poisoned| poisoned.into_inner())
2323 .insert(
2324 module_id.to_string(),
2325 OpenSwap {
2326 candidate_nonce,
2327 incumbent_nonce,
2328 candidate_admitted: false,
2329 },
2330 );
2331 }
2332
2333 pub(crate) fn close_swap(&self, module_id: &str) {
2336 self.swaps
2337 .lock()
2338 .unwrap_or_else(|poisoned| poisoned.into_inner())
2339 .remove(module_id);
2340 }
2341
2342 pub(crate) fn set_swap_promotion_observer(
2345 &self,
2346 observer: std::sync::Weak<dyn SwapPromotionObserver>,
2347 ) {
2348 *self
2349 .promotion_observer
2350 .0
2351 .lock()
2352 .unwrap_or_else(|poisoned| poisoned.into_inner()) = Some(observer);
2353 }
2354
2355 fn notify_swap_promoted(&self, registration: &crate::registry::ModuleRegistration) {
2358 let observer = self
2359 .promotion_observer
2360 .0
2361 .lock()
2362 .unwrap_or_else(|poisoned| poisoned.into_inner())
2363 .as_ref()
2364 .and_then(std::sync::Weak::upgrade);
2365 if let Some(observer) = observer {
2366 observer.swap_promoted(registration);
2367 }
2368 }
2369
2370 pub(crate) fn swap_open(&self, module_id: &str) -> bool {
2372 self.swaps
2373 .lock()
2374 .unwrap_or_else(|poisoned| poisoned.into_inner())
2375 .contains_key(module_id)
2376 }
2377
2378 fn promote_swap_nonce(&self, module_id: &str, reserved: bool) {
2383 let candidate_nonce = self
2384 .swaps
2385 .lock()
2386 .unwrap_or_else(|poisoned| poisoned.into_inner())
2387 .get(module_id)
2388 .map(|swap| swap.candidate_nonce.clone());
2389 let Some(nonce) = candidate_nonce else {
2390 return;
2391 };
2392 self.set_spawn_nonce(module_id, nonce.clone());
2393 if reserved {
2394 self.set_reserved_nonce(module_id, nonce);
2395 }
2396 }
2397
2398 pub(crate) fn swap_hello_admission(
2413 &self,
2414 module_id: &str,
2415 presented: Option<&str>,
2416 ) -> SwapHelloAdmission {
2417 let swaps = self
2418 .swaps
2419 .lock()
2420 .unwrap_or_else(|poisoned| poisoned.into_inner());
2421 let Some(swap) = swaps.get(module_id) else {
2422 return SwapHelloAdmission::NotSwapping;
2423 };
2424 let Some(presented) = presented else {
2425 return SwapHelloAdmission::Refused;
2426 };
2427 if constant_time_eq(swap.candidate_nonce.as_bytes(), presented.as_bytes()) {
2428 return if swap.candidate_admitted {
2429 SwapHelloAdmission::Refused
2430 } else {
2431 SwapHelloAdmission::Candidate
2432 };
2433 }
2434 if swap
2435 .incumbent_nonce
2436 .as_deref()
2437 .is_some_and(|incumbent| constant_time_eq(incumbent.as_bytes(), presented.as_bytes()))
2438 {
2439 return SwapHelloAdmission::NotSwapping;
2440 }
2441 SwapHelloAdmission::Refused
2442 }
2443
2444 pub(crate) fn mark_swap_candidate_admitted(&self, module_id: &str) {
2447 if let Some(swap) = self
2448 .swaps
2449 .lock()
2450 .unwrap_or_else(|poisoned| poisoned.into_inner())
2451 .get_mut(module_id)
2452 {
2453 swap.candidate_admitted = true;
2454 }
2455 }
2456
2457 pub fn spawn_launch_nonce_for(&self, module_id: &str) -> Option<String> {
2459 self.spawn_nonces
2460 .lock()
2461 .unwrap_or_else(|poisoned| poisoned.into_inner())
2462 .get(module_id)
2463 .cloned()
2464 }
2465
2466 pub fn reserved_launch_nonce_for(&self, module_id: &str) -> Option<String> {
2468 self.reserved_nonces
2469 .lock()
2470 .unwrap_or_else(|poisoned| poisoned.into_inner())
2471 .get(module_id)
2472 .cloned()
2473 .flatten()
2474 }
2475
2476 pub fn insert(&self, module: SupervisedModule) -> Option<SupervisedModule> {
2477 self.mark_configured(module.module_id());
2481 let mut modules = self
2482 .modules
2483 .lock()
2484 .unwrap_or_else(|poisoned| poisoned.into_inner());
2485 modules.insert(module.module_id().to_string(), module)
2486 }
2487
2488 fn mark_configured(&self, module_id: &str) {
2492 self.configured_ids
2493 .lock()
2494 .unwrap_or_else(|poisoned| poisoned.into_inner())
2495 .insert(module_id.to_string());
2496 }
2497
2498 fn unmark_configured_unless_rostered(&self, module_id: &str) {
2502 let modules = self
2503 .modules
2504 .lock()
2505 .unwrap_or_else(|poisoned| poisoned.into_inner());
2506 if !modules.contains_key(module_id) {
2507 self.configured_ids
2508 .lock()
2509 .unwrap_or_else(|poisoned| poisoned.into_inner())
2510 .remove(module_id);
2511 }
2512 }
2513
2514 pub(crate) fn is_configured(&self, module_id: &str) -> bool {
2522 self.configured_ids
2523 .lock()
2524 .unwrap_or_else(|poisoned| poisoned.into_inner())
2525 .contains(module_id)
2526 }
2527
2528 pub fn get(&self, module_id: &str) -> Option<SupervisedModule> {
2529 let modules = self
2530 .modules
2531 .lock()
2532 .unwrap_or_else(|poisoned| poisoned.into_inner());
2533 modules.get(module_id).cloned()
2534 }
2535
2536 pub(crate) fn record_late_health_answer(
2537 &self,
2538 module_id: &str,
2539 latency_ms: u64,
2540 ) -> Result<bool, SuperviseError> {
2541 let Some(module) = self.get(module_id) else {
2542 return Ok(false);
2543 };
2544 update_snapshot(&module.inner.snapshot, Some(module_id), |state| {
2545 state.health.late_answer_count = state.health.late_answer_count.saturating_add(1);
2546 state.health.last_late_answer_latency_ms = Some(latency_ms);
2547 state.health.consecutive_failures = 0;
2555 })?;
2556 Ok(true)
2557 }
2558
2559 pub fn record_deliberate_severance(&self, module_id: &str) -> Result<bool, SuperviseError> {
2565 let Some(module) = self.get(module_id) else {
2566 return Ok(false);
2567 };
2568 let snapshot = lock_snapshot(&module.inner.snapshot)?;
2569 let Some((pid, start_time)) = snapshot.pid.zip(snapshot.process_start_time) else {
2570 return Ok(false);
2571 };
2572 drop(snapshot);
2573 module.record_deliberate_severance(ProcessIdentity { pid, start_time })
2574 }
2575
2576 pub fn list(&self) -> Vec<SupervisedModule> {
2577 let modules = self
2578 .modules
2579 .lock()
2580 .unwrap_or_else(|poisoned| poisoned.into_inner());
2581 let mut modules = modules.values().cloned().collect::<Vec<_>>();
2582 modules.sort_by(|left, right| left.module_id().cmp(right.module_id()));
2583 modules
2584 }
2585
2586 pub(crate) fn retire(&self, module_id: &str) -> Option<SupervisedModule> {
2587 self.activity_images
2588 .lock()
2589 .unwrap_or_else(|p| p.into_inner())
2590 .remove(module_id);
2591 self.spawn_nonces
2592 .lock()
2593 .unwrap_or_else(|poisoned| poisoned.into_inner())
2594 .remove(module_id);
2595 self.close_swap(module_id);
2596 let mut reserved_nonces = self
2597 .reserved_nonces
2598 .lock()
2599 .unwrap_or_else(|poisoned| poisoned.into_inner());
2600 if reserved_nonces.contains_key(module_id) {
2601 reserved_nonces.insert(module_id.to_string(), None);
2604 }
2605 drop(reserved_nonces);
2606 self.reserved_prefix_owners
2607 .lock()
2608 .unwrap_or_else(|poisoned| poisoned.into_inner())
2609 .retain(|_, owner| owner != module_id);
2610 let removed = self
2611 .modules
2612 .lock()
2613 .unwrap_or_else(|poisoned| poisoned.into_inner())
2614 .remove(module_id);
2615 self.configured_ids
2616 .lock()
2617 .unwrap_or_else(|poisoned| poisoned.into_inner())
2618 .remove(module_id);
2619 removed
2620 }
2621
2622 pub(crate) fn record_rescan_removal(&self, module_id: &str) {
2625 self.removal_tombstones
2626 .lock()
2627 .unwrap_or_else(|poisoned| poisoned.into_inner())
2628 .insert(module_id.to_string(), unix_ms_now());
2629 }
2630
2631 pub(crate) fn removal_tombstone_age_ms(&self, module_id: &str) -> Option<u64> {
2633 self.removal_tombstones
2634 .lock()
2635 .unwrap_or_else(|poisoned| poisoned.into_inner())
2636 .get(module_id)
2637 .copied()
2638 .map(|removed_at_ms| unix_ms_now().saturating_sub(removed_at_ms))
2639 }
2640
2641 pub(crate) fn release_retained_reserved_gate(&self, module_id: &str) -> bool {
2646 if self.get(module_id).is_some() {
2647 return false;
2648 }
2649 let mut reserved_nonces = self
2650 .reserved_nonces
2651 .lock()
2652 .unwrap_or_else(|poisoned| poisoned.into_inner());
2653 if !matches!(reserved_nonces.get(module_id), Some(None)) {
2654 return false;
2655 }
2656 reserved_nonces.remove(module_id);
2657 true
2658 }
2659
2660 pub(crate) fn operation_lock(&self) -> Arc<AsyncMutex<()>> {
2661 Arc::clone(&self.operation_lock)
2662 }
2663}
2664
2665#[derive(Debug, Clone)]
2667pub struct Supervisor {
2668 registry: Arc<Registry>,
2669 restart_policy: RestartPolicy,
2670 drain_timeout: Duration,
2671 connection_file_path: Option<PathBuf>,
2672 capture_logs_dir: Option<PathBuf>,
2673 forwarding: Option<Arc<ForwardingTable>>,
2674 process_liveness: Arc<SupervisorProcessLiveness>,
2675 supervisor_handle: Option<SupervisorHandle>,
2676 health: HealthConfig,
2677 daemon_start_clock: crate::clock::StartClock,
2678 terminal_journal: Option<Arc<crate::terminal_journal::TerminalJournal>>,
2679 spawn_events: SpawnEventFeed,
2680 provenance_probe: ExecutableIdentityProbe,
2681 child_roster: ChildRoster,
2684 #[cfg(target_os = "linux")]
2685 cgroup_placement: Option<subc_cgroup::Placement>,
2686 #[cfg(test)]
2687 test_after_first_spawn: AfterFirstSpawnHook,
2688}
2689
2690pub(crate) struct RegisteredConfiguredModule {
2694 spec: ModuleSpec,
2695 runtime: SupervisorRuntimeConfig,
2696 module: SupervisedModule,
2697 commands: mpsc::Receiver<SupervisorCommand>,
2698}
2699
2700#[cfg(test)]
2706#[derive(Clone, Default)]
2707struct AfterFirstSpawnHook(Option<AfterFirstSpawnFn>);
2708
2709#[cfg(test)]
2710type AfterFirstSpawnFn = Arc<dyn Fn(&str) + Send + Sync>;
2711
2712#[cfg(test)]
2713impl fmt::Debug for AfterFirstSpawnHook {
2714 fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
2715 f.write_str("AfterFirstSpawnHook")
2716 }
2717}
2718
2719#[cfg(test)]
2720impl AfterFirstSpawnHook {
2721 fn run(&self, module_id: &str) {
2722 if let Some(hook) = &self.0 {
2723 hook(module_id);
2724 }
2725 }
2726}
2727
2728impl Supervisor {
2729 #[cfg(test)]
2730 pub(crate) fn new_for_test(registry: Arc<Registry>, policy: RestartPolicy) -> Self {
2731 let supervisor = Self::new(registry, policy);
2732 #[cfg(target_os = "macos")]
2733 let supervisor = supervisor.with_privacy_trampoline(test_privacy_trampoline());
2734 supervisor
2735 }
2736 pub fn with_privacy_trampoline(self, path: impl Into<PathBuf>) -> Self {
2741 let path = path.into();
2742 #[cfg(target_os = "macos")]
2743 {
2744 let result = probe_privacy_trampoline(&path).map(|()| path);
2745 if let Err(cause) = &result {
2746 error!(%cause, "macOS privacy identity unavailable; supervised launches will refuse");
2747 }
2748 self.child_roster.set_privacy_trampoline(result);
2749 }
2750 #[cfg(not(target_os = "macos"))]
2751 let _ = path;
2752 self
2753 }
2754 pub(crate) fn daemon_shutdown_flag(&self) -> crate::child_roster::DaemonShutdownFlag {
2757 self.child_roster.shutdown_flag()
2758 }
2759
2760 #[cfg(unix)]
2771 pub(crate) fn begin_daemon_shutdown(&self) {
2772 self.child_roster.close();
2773 if let Some(journal) = &self.terminal_journal {
2774 journal.stamp_shutdown();
2775 }
2776 }
2777
2778 #[cfg(unix)]
2782 pub(crate) async fn drain_for_daemon_shutdown(&self) -> Result<(), SuperviseError> {
2783 const NOTICE_BUDGET: Duration = Duration::from_millis(500);
2784 const DRAIN_BUDGET: Duration = Duration::from_secs(2);
2785 let Some(forwarding) = &self.forwarding else {
2786 return Ok(());
2787 };
2788 let modules = forwarding
2789 .begin_daemon_drain()
2790 .map_err(SuperviseError::Forwarding)?;
2791 let deadline_ms =
2792 unix_ms_now().saturating_add((NOTICE_BUDGET + DRAIN_BUDGET).as_millis() as u64);
2793 let mut notices = tokio::task::JoinSet::new();
2794 let mut drains = Vec::new();
2795 for (module_id, endpoint) in modules {
2796 let Some(target) = forwarding
2797 .begin_endpoint_drain(endpoint, RouteCloseReason::Restart)
2798 .map_err(SuperviseError::Forwarding)?
2799 else {
2800 continue;
2801 };
2802 let routes = forwarding
2803 .endpoint_routes(target.endpoint)
2804 .map_err(SuperviseError::Forwarding)?;
2805 let command = serde_json::to_vec(&ModuleControlCommand::Draining {
2811 reason: RouteCloseReason::Restart,
2812 deadline_ms,
2813 })
2814 .expect("module draining serializes");
2815 let mut recipients = vec![(target.sink.clone(), target.negotiated_ver, command)];
2816 let mut clients: Vec<(crate::forwarding::GoodbyeTarget, Vec<u16>)> = Vec::new();
2817 for route in routes {
2818 let client = route.goodbye_target;
2819 if let Some((_, channels)) = clients
2820 .iter_mut()
2821 .find(|(existing, _)| existing.connection_id == client.connection_id)
2822 {
2823 channels.push(client.channel);
2824 } else {
2825 let channel = client.channel;
2826 clients.push((client, vec![channel]));
2827 }
2828 }
2829 for (client, mut channels) in clients {
2830 channels.sort_unstable();
2831 channels.dedup();
2832 let closing = serde_json::to_vec(&ClientControlPush::RouteClosing {
2833 module_id: module_id.clone(),
2834 channels,
2835 reason: RouteCloseReason::Restart,
2836 })
2837 .expect("route closing serializes");
2838 recipients.push((client.sink, client.negotiated_ver, closing));
2839 }
2840 for (sink, version, body) in recipients {
2841 notices.spawn(async move {
2842 let frame = Frame::build_with_version(
2843 version,
2844 FrameType::Push,
2845 control_flags(),
2846 0,
2847 0,
2848 0,
2849 body,
2850 )
2851 .expect("bounded lifecycle notice frame builds");
2852 sink.send_flushed(frame).await
2853 });
2854 }
2855 let gauges = declared_busy_gauges(&self.registry, &module_id)?;
2856 drains.push((module_id, target.endpoint, gauges));
2857 }
2858 let notice_deadline = Instant::now() + NOTICE_BUDGET;
2861 while let Ok(Some(result)) = timeout_at(notice_deadline, notices.join_next()).await {
2862 if !matches!(result, Ok(Ok(()))) {
2863 warn!(?result, "daemon shutdown notice delivery failed");
2864 }
2865 }
2866 notices.abort_all();
2867 let deadline = Instant::now() + DRAIN_BUDGET;
2868 let mut waits = tokio::task::JoinSet::new();
2869 for (module_id, endpoint, gauges) in drains {
2870 let forwarding = Arc::clone(forwarding);
2871 let mut runtime = self.runtime_config();
2872 runtime.health.cadence = Duration::from_millis(100);
2873 waits.spawn(async move {
2874 let wait_result = wait_for_forwarding_quiescence(
2875 &forwarding,
2876 &module_id,
2877 &runtime,
2878 endpoint,
2879 deadline,
2880 &gauges,
2881 DrainScope::Endpoint(endpoint),
2882 &mut DrainGaugeTrace::new(&gauges),
2883 )
2884 .await;
2885 forwarding
2889 .record_daemon_drain_result(
2890 endpoint,
2891 drained_after_quiescence_wait(&wait_result),
2892 )
2893 .map_err(SuperviseError::Forwarding)?;
2894 wait_result
2895 });
2896 }
2897 while let Ok(Some(result)) = timeout_at(deadline, waits.join_next()).await {
2898 if !matches!(result, Ok(Ok(true))) {
2899 warn!(?result, "daemon shutdown drain did not reach quiescence");
2900 }
2901 }
2902 Ok(())
2903 }
2904
2905 #[cfg(unix)]
2917 pub(crate) async fn end_children_for_daemon_shutdown(
2918 &self,
2919 already_escalated: bool,
2920 escalate: impl std::future::Future<Output = ()>,
2921 ) {
2922 tokio::pin!(escalate);
2923 let mut escalated = already_escalated;
2924 if let Some(forwarding) = &self.forwarding {
2925 let reason = CloseReason::new(
2926 "daemon_shutdown",
2927 "the daemon is exiting after its shutdown notice and drain",
2928 );
2929 if escalated {
2930 send_module_goodbyes_for_daemon_shutdown(forwarding, &reason, false).await;
2933 } else {
2934 tokio::select! {
2935 biased;
2936 _ = escalate.as_mut() => {
2937 info!("second SIGTERM: abandoning module GOODBYE delivery");
2938 escalated = true;
2939 }
2940 _ = send_module_goodbyes_for_daemon_shutdown(forwarding, &reason, true) => {}
2941 }
2942 }
2943 let closed = forwarding.close_all_connections(&reason);
2944 debug!(closed, "closed established connections for daemon shutdown");
2945 }
2946 let escalated_here = escalated && !already_escalated;
2950 let remaining_escalate = async move {
2951 if escalated_here {
2952 std::future::pending::<()>().await;
2953 } else {
2954 escalate.await;
2955 }
2956 };
2957 crate::child_roster::end_children_for_daemon_shutdown(
2958 &self.child_roster,
2959 escalated,
2960 remaining_escalate,
2961 )
2962 .await;
2963 }
2964
2965 pub fn new(registry: Arc<Registry>, restart_policy: RestartPolicy) -> Self {
2966 Self {
2967 registry,
2968 restart_policy,
2969 drain_timeout: DEFAULT_DRAIN_TIMEOUT,
2970 connection_file_path: None,
2971 capture_logs_dir: None,
2972 forwarding: None,
2973 process_liveness: Arc::new(SupervisorProcessLiveness::default()),
2974 supervisor_handle: None,
2975 health: HealthConfig::default(),
2976 daemon_start_clock: crate::clock::StartClock::capture(),
2977 terminal_journal: None,
2978 spawn_events: SpawnEventFeed::default(),
2979 provenance_probe: ExecutableIdentityProbe::default(),
2980 child_roster: ChildRoster::default(),
2981 #[cfg(target_os = "linux")]
2982 cgroup_placement: None,
2983 #[cfg(test)]
2984 test_after_first_spawn: AfterFirstSpawnHook::default(),
2985 }
2986 }
2987
2988 pub fn with_drain_timeout(mut self, drain_timeout: Duration) -> Self {
2989 self.drain_timeout = drain_timeout;
2990 self
2991 }
2992
2993 pub(crate) fn default_drain_timeout(&self) -> Duration {
2995 self.drain_timeout
2996 }
2997
2998 pub fn with_process_liveness(
2999 mut self,
3000 process_liveness: Arc<SupervisorProcessLiveness>,
3001 ) -> Self {
3002 self.process_liveness = process_liveness;
3003 self
3004 }
3005
3006 pub fn with_connection_file_path(mut self, connection_file_path: impl Into<PathBuf>) -> Self {
3007 self.connection_file_path = Some(connection_file_path.into());
3008 self
3009 }
3010
3011 pub fn with_capture_logs_dir(mut self, logs_dir: impl Into<PathBuf>) -> Self {
3013 self.capture_logs_dir = Some(logs_dir.into());
3014 self
3015 }
3016
3017 pub fn with_daemon_incarnation(self, daemon_incarnation: String) -> Self {
3020 self.spawn_events.configure_incarnation(daemon_incarnation);
3024 self
3025 }
3026
3027 pub fn with_terminal_journal(self, path: PathBuf, daemon_incarnation: String) -> Self {
3030 let mut this = self.with_daemon_incarnation(daemon_incarnation.clone());
3031 this.terminal_journal = Some(Arc::new(crate::terminal_journal::TerminalJournal::open(
3032 path,
3033 daemon_incarnation,
3034 )));
3035 this
3036 }
3037
3038 pub fn with_forwarding(mut self, forwarding: Arc<ForwardingTable>) -> Self {
3039 self.forwarding = Some(forwarding);
3040 self
3041 }
3042
3043 pub fn with_handle(mut self, supervisor_handle: SupervisorHandle) -> Self {
3044 self.spawn_events = supervisor_handle.spawn_events.clone();
3045 self.supervisor_handle = Some(supervisor_handle);
3046 self
3047 }
3048
3049 pub fn with_health_config(mut self, health: HealthConfig) -> Self {
3050 self.health = health;
3051 self
3052 }
3053
3054 pub fn with_live_children_record(self, path: impl Into<PathBuf>) -> Self {
3058 self.child_roster.record_to(path.into());
3059 self
3060 }
3061
3062 #[cfg(target_os = "linux")]
3063 pub fn with_cgroup_placement(
3064 mut self,
3065 cgroup_placement: Option<subc_cgroup::Placement>,
3066 ) -> Self {
3067 self.cgroup_placement = cgroup_placement;
3068 self
3069 }
3070
3071 pub fn spawn(&self, spec: ModuleSpec) -> Result<SupervisedModule, SuperviseError> {
3077 validate_spec(&spec)?;
3078 self.establish_identity(&spec);
3079
3080 let runtime = self.runtime_config();
3081 let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::starting()));
3082 let activity_spawn = runtime.begin_activity_spawn(&spec.module_id);
3083 let spawned = spawn_child(
3084 &spec,
3085 runtime.connection_file_path.as_deref(),
3086 self.supervisor_handle.as_ref(),
3087 &runtime.stderr_ring,
3088 runtime.capture_logs_dir.as_deref(),
3089 &runtime.child_roster,
3090 #[cfg(target_os = "linux")]
3091 runtime.cgroup_placement.as_ref(),
3092 );
3093 #[cfg(test)]
3094 self.test_after_first_spawn.run(&spec.module_id);
3095 let child = match spawned {
3096 Ok(child) => child,
3097 Err(err) => {
3098 self.abandon_unrostered(&spec.module_id);
3101 return Err(err);
3102 }
3103 };
3104 self.mark_first_process_running(&spec, &runtime, &snapshot, &child)?;
3105 if let Some(spawn) = activity_spawn {
3106 spawn.complete(&snapshot);
3107 }
3108
3109 Ok(self.supervised_module(spec, runtime, snapshot, Some(child)))
3110 }
3111
3112 fn establish_identity(&self, spec: &ModuleSpec) {
3125 if let Some(supervisor_handle) = &self.supervisor_handle {
3126 supervisor_handle.apply_identity_configuration(spec);
3127 supervisor_handle.mark_configured(&spec.module_id);
3128 }
3129 }
3130
3131 fn abandon_unrostered(&self, module_id: &str) {
3134 if let Some(supervisor_handle) = &self.supervisor_handle {
3135 supervisor_handle.unmark_configured_unless_rostered(module_id);
3136 }
3137 }
3138
3139 fn mark_first_process_running(
3142 &self,
3143 spec: &ModuleSpec,
3144 runtime: &SupervisorRuntimeConfig,
3145 snapshot: &SharedSnapshot,
3146 child: &SupervisedChild,
3147 ) -> Result<(), SuperviseError> {
3148 if let Err(err) = set_running(snapshot, child, &spec.module_id, &runtime.spawn_events) {
3149 self.abandon_unrostered(&spec.module_id);
3150 return Err(err);
3151 }
3152 self.process_liveness
3153 .track(spec.module_id.clone(), Arc::clone(snapshot));
3154 Ok(())
3155 }
3156
3157 pub fn supervise_configured(
3163 &self,
3164 spec: ModuleSpec,
3165 enabled: bool,
3166 ) -> Result<SupervisedModule, SuperviseError> {
3167 self.supervise_configured_requested(
3168 spec,
3169 enabled,
3170 self.health.clone(),
3171 None,
3172 self.restart_policy,
3173 None,
3174 )
3175 }
3176
3177 pub fn supervise_configured_with_health(
3183 &self,
3184 spec: ModuleSpec,
3185 enabled: bool,
3186 health: HealthConfig,
3187 drain_timeout_ms: Option<u64>,
3188 restart_policy: RestartPolicy,
3189 ) -> Result<SupervisedModule, SuperviseError> {
3190 self.supervise_configured_requested(
3191 spec,
3192 enabled,
3193 health,
3194 drain_timeout_ms,
3195 restart_policy,
3196 None,
3197 )
3198 }
3199
3200 pub(crate) fn supervise_configured_requested(
3201 &self,
3202 spec: ModuleSpec,
3203 enabled: bool,
3204 health: HealthConfig,
3205 drain_timeout_ms: Option<u64>,
3206 restart_policy: RestartPolicy,
3207 requester: Option<Principal>,
3208 ) -> Result<SupervisedModule, SuperviseError> {
3209 let registered = self.register_configured_with_health(
3210 spec,
3211 enabled,
3212 health,
3213 drain_timeout_ms,
3214 restart_policy,
3215 )?;
3216 self.start_registered_configured(registered, requester)
3217 }
3218
3219 pub(crate) fn register_configured_with_health(
3224 &self,
3225 spec: ModuleSpec,
3226 enabled: bool,
3227 health: HealthConfig,
3228 drain_timeout_ms: Option<u64>,
3229 restart_policy: RestartPolicy,
3230 ) -> Result<RegisteredConfiguredModule, SuperviseError> {
3231 validate_spec(&spec)?;
3232 self.establish_identity(&spec);
3233
3234 let mut runtime = self.runtime_config();
3235 runtime.health = health;
3236 runtime.restart_policy = restart_policy;
3237 if let Some(ms) = drain_timeout_ms {
3238 runtime.drain_timeout = Duration::from_millis(ms);
3239 *runtime
3240 .effective_drain_timeout
3241 .lock()
3242 .unwrap_or_else(|poisoned| poisoned.into_inner()) = runtime.drain_timeout;
3243 }
3244 let snapshot = if enabled {
3245 SupervisorSnapshot::starting()
3246 } else {
3247 SupervisorSnapshot::disabled()
3248 };
3249 let snapshot = {
3250 let mut snapshot = snapshot;
3251 if enabled {
3252 if let Some((pid, reason)) = runtime.child_roster.previous_instance(&spec.module_id)
3253 {
3254 snapshot.waiting_for_previous_pid = Some(pid);
3255 snapshot.waiting_for_previous_detail = Some(reason.waiting_message(pid));
3256 }
3257 }
3258 snapshot
3259 };
3260 let (module, commands) =
3261 self.register_module(&spec, &runtime, Arc::new(Mutex::new(snapshot)));
3262 Ok(RegisteredConfiguredModule {
3263 spec,
3264 runtime,
3265 module,
3266 commands,
3267 })
3268 }
3269
3270 pub(crate) fn start_registered_configured(
3273 &self,
3274 registered: RegisteredConfiguredModule,
3275 requester: Option<Principal>,
3276 ) -> Result<SupervisedModule, SuperviseError> {
3277 let RegisteredConfiguredModule {
3278 spec,
3279 mut runtime,
3280 module,
3281 commands,
3282 } = registered;
3283 runtime.start_requester = requester;
3284 let snapshot = &module.inner.snapshot;
3285 let should_spawn = {
3286 let state = lock_snapshot(snapshot)?;
3287 state.enabled && state.waiting_for_previous_pid.is_none()
3288 };
3289 let child = if should_spawn {
3290 let activity_spawn = runtime.begin_activity_spawn(&spec.module_id);
3291 let spawned = spawn_child(
3292 &spec,
3293 runtime.connection_file_path.as_deref(),
3294 self.supervisor_handle.as_ref(),
3295 &runtime.stderr_ring,
3296 runtime.capture_logs_dir.as_deref(),
3297 &runtime.child_roster,
3298 #[cfg(target_os = "linux")]
3299 runtime.cgroup_placement.as_ref(),
3300 );
3301 #[cfg(test)]
3302 self.test_after_first_spawn.run(&spec.module_id);
3303 match spawned {
3304 Ok(child) => {
3305 self.mark_first_process_running(&spec, &runtime, snapshot, &child)?;
3306 if let Some(spawn) = activity_spawn {
3307 spawn.complete(snapshot);
3308 }
3309 Some(child)
3310 }
3311 Err(err) => {
3312 if runtime.health.critical {
3313 error!(
3314 module_id = %spec.module_id,
3315 program = %spec.program.display(),
3316 error = %err,
3317 "critical configured module failed to spawn; marking failed and alerting"
3318 );
3319 } else {
3320 error!(
3321 module_id = %spec.module_id,
3322 program = %spec.program.display(),
3323 error = %err,
3324 "configured module failed to spawn; marking failed and continuing"
3325 );
3326 }
3327 update_snapshot(snapshot, Some(&spec.module_id), |state| {
3328 state.state = ModuleState::Failed;
3329 clear_current_process_facts(state);
3330 })?;
3331 None
3332 }
3333 }
3334 } else {
3335 None
3336 };
3337 self.start_monitor(spec, runtime, &module, child, commands);
3338 Ok(module)
3339 }
3340
3341 fn runtime_config(&self) -> SupervisorRuntimeConfig {
3342 let effective_drain_timeout = Arc::new(Mutex::new(self.drain_timeout));
3343 SupervisorRuntimeConfig {
3344 start_requester: None,
3345 pending_activity_restart: false,
3346 provenance_probe: self.provenance_probe.clone(),
3347 scheduled_respawn: Arc::default(),
3348 deferred_reload_reply: Arc::default(),
3349 restart_policy: self.restart_policy,
3350 drain_timeout: self.drain_timeout,
3351 child_roster: self
3354 .child_roster
3355 .for_module(Arc::clone(&effective_drain_timeout)),
3356 effective_drain_timeout,
3357 default_drain_timeout: self.drain_timeout,
3358 health: self.health.clone(),
3359 connection_file_path: self.connection_file_path.clone(),
3360 capture_logs_dir: self.capture_logs_dir.clone(),
3361 forwarding: self.forwarding.clone(),
3362 supervisor_handle: self.supervisor_handle.clone(),
3363 stderr_ring: Arc::new(Mutex::new(StderrRing::new(StderrTailConfig::default()))),
3364 terminal_ring: Arc::new(Mutex::new(
3365 TerminalRing::new(
3366 TerminalRingConfig::default(),
3367 self.daemon_start_clock.started_at_ms(),
3368 )
3369 .with_start_clock(self.daemon_start_clock)
3370 .with_journal(self.terminal_journal.clone())
3371 .with_daemon_shutdown(self.child_roster.shutdown_flag()),
3372 )),
3373 spawn_events: self.spawn_events.clone(),
3374 #[cfg(target_os = "linux")]
3375 cgroup_placement: self.cgroup_placement.clone(),
3376 #[cfg(test)]
3377 test_seed_stale_facts_before_enable_spawn: false,
3378 #[cfg(test)]
3379 test_reload_exit_record_gate: None,
3380 }
3381 }
3382
3383 fn supervised_module(
3384 &self,
3385 spec: ModuleSpec,
3386 runtime: SupervisorRuntimeConfig,
3387 snapshot: SharedSnapshot,
3388 child: Option<SupervisedChild>,
3389 ) -> SupervisedModule {
3390 let (module, commands) = self.register_module(&spec, &runtime, snapshot);
3391 self.start_monitor(spec, runtime, &module, child, commands);
3392 module
3393 }
3394
3395 fn register_module(
3396 &self,
3397 spec: &ModuleSpec,
3398 runtime: &SupervisorRuntimeConfig,
3399 snapshot: SharedSnapshot,
3400 ) -> (SupervisedModule, mpsc::Receiver<SupervisorCommand>) {
3401 let configuration = Arc::new(Mutex::new(SupervisedConfiguration {
3402 spec: spec.clone(),
3403 health: runtime.health.clone(),
3404 }));
3405 let stderr_ring = Arc::clone(&runtime.stderr_ring);
3406 let terminal_ring = Arc::clone(&runtime.terminal_ring);
3407 let restart_policy = runtime.restart_policy;
3411 let effective_drain_timeout = Arc::clone(&runtime.effective_drain_timeout);
3412 let (tx, rx) = mpsc::channel(4);
3413 let module_id = spec.module_id.clone();
3414 let module = SupervisedModule {
3415 inner: Arc::new(SupervisedModuleInner {
3416 module_id: module_id.clone(),
3417 registry: Arc::clone(&self.registry),
3418 snapshot,
3419 configuration,
3420 stderr_ring,
3421 terminal_ring,
3422 commands: tx,
3423 monitor: Mutex::new(None),
3424 restart_policy,
3425 effective_drain_timeout,
3426 provenance_probe: self.provenance_probe.clone(),
3427 }),
3428 };
3429 if let Some(supervisor_handle) = &self.supervisor_handle {
3432 supervisor_handle.insert(module.clone());
3433 }
3434 (module, rx)
3435 }
3436
3437 fn start_monitor(
3438 &self,
3439 spec: ModuleSpec,
3440 runtime: SupervisorRuntimeConfig,
3441 module: &SupervisedModule,
3442 child: Option<SupervisedChild>,
3443 commands: mpsc::Receiver<SupervisorCommand>,
3444 ) {
3445 let monitor = tokio::spawn(supervise_loop(
3446 spec,
3447 runtime,
3448 Arc::clone(&self.registry),
3449 Arc::clone(&self.process_liveness),
3450 Arc::clone(&module.inner.snapshot),
3451 child,
3452 commands,
3453 ));
3454 *module
3455 .inner
3456 .monitor
3457 .lock()
3458 .unwrap_or_else(|p| p.into_inner()) = Some(monitor);
3459 }
3460}
3461
3462impl Default for Supervisor {
3463 fn default() -> Self {
3464 Self::new(Arc::new(Registry::default()), RestartPolicy::default())
3465 }
3466}
3467
3468#[derive(Clone)]
3470pub struct SupervisedModule {
3471 inner: Arc<SupervisedModuleInner>,
3472}
3473
3474struct SupervisedModuleInner {
3475 module_id: String,
3476 registry: Arc<Registry>,
3477 snapshot: SharedSnapshot,
3478 configuration: Arc<Mutex<SupervisedConfiguration>>,
3479 stderr_ring: Arc<Mutex<StderrRing>>,
3480 terminal_ring: Arc<Mutex<TerminalRing>>,
3481 commands: mpsc::Sender<SupervisorCommand>,
3482 monitor: Mutex<Option<JoinHandle<()>>>,
3483 restart_policy: RestartPolicy,
3487 effective_drain_timeout: Arc<Mutex<Duration>>,
3488 provenance_probe: ExecutableIdentityProbe,
3489}
3490
3491impl fmt::Debug for SupervisedModule {
3492 fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
3493 f.debug_struct("SupervisedModule")
3494 .field("module_id", &self.inner.module_id)
3495 .field("status", &self.status())
3496 .finish_non_exhaustive()
3497 }
3498}
3499
3500impl SupervisedModule {
3501 pub fn module_id(&self) -> &str {
3502 &self.inner.module_id
3503 }
3504
3505 #[cfg(test)]
3509 pub(crate) fn record_health_probe_failure_for_test(
3510 &self,
3511 detail: &str,
3512 ) -> Result<(), SuperviseError> {
3513 update_snapshot(&self.inner.snapshot, Some(&self.inner.module_id), |state| {
3514 state.health.consecutive_failures = state.health.consecutive_failures.saturating_add(1);
3515 state.health.detail = Some(detail.to_string());
3516 })
3517 }
3518
3519 pub fn state(&self) -> Result<ModuleState, SuperviseError> {
3520 Ok(lock_snapshot(&self.inner.snapshot)?.state)
3521 }
3522
3523 pub(crate) fn registration_end_reason(
3524 &self,
3525 ) -> Result<Option<RegistrationEndReason>, SuperviseError> {
3526 let snapshot = lock_snapshot(&self.inner.snapshot)?;
3527 Ok(match snapshot.state {
3528 ModuleState::Draining if snapshot.draining_to_replace => {
3529 Some(RegistrationEndReason::SupervisorRestart)
3530 }
3531 ModuleState::Draining => Some(RegistrationEndReason::SupervisorStop),
3532 ModuleState::Restarting if snapshot.draining_to_replace => {
3533 Some(RegistrationEndReason::SupervisorRestart)
3534 }
3535 _ => None,
3536 })
3537 }
3538
3539 pub fn stderr_tail(
3546 &self,
3547 max_lines: Option<usize>,
3548 max_bytes: Option<usize>,
3549 ) -> StderrTailSnapshot {
3550 self.inner
3551 .stderr_ring
3552 .lock()
3553 .unwrap_or_else(|poisoned| poisoned.into_inner())
3554 .snapshot(max_lines, max_bytes)
3555 }
3556
3557 pub fn terminal_history(&self) -> TerminalHistorySnapshot {
3562 self.inner
3563 .terminal_ring
3564 .lock()
3565 .unwrap_or_else(|poisoned| poisoned.into_inner())
3566 .snapshot()
3567 }
3568
3569 pub fn durable_terminal_history(&self) -> subc_control::TerminalHistory {
3574 durable_terminal_history_of(&self.inner.terminal_ring, &self.inner.module_id)
3575 }
3576
3577 pub(crate) async fn read_durable_terminal_history(
3582 &self,
3583 ) -> Result<subc_control::TerminalHistory, tokio::task::JoinError> {
3584 let terminal_ring = Arc::clone(&self.inner.terminal_ring);
3585 let module_id = self.inner.module_id.clone();
3586 tokio::task::spawn_blocking(move || durable_terminal_history_of(&terminal_ring, &module_id))
3587 .await
3588 }
3589
3590 pub fn status(&self) -> Result<ModuleStatus, SuperviseError> {
3591 self.status_with_snapshot_lock(&self.inner.snapshot, None)
3592 .map(|(status, _)| status)
3593 }
3594
3595 pub(crate) fn record_deliberate_severance(
3596 &self,
3597 identity: ProcessIdentity,
3598 ) -> Result<bool, SuperviseError> {
3599 let mut snapshot = lock_snapshot(&self.inner.snapshot)?;
3600 if snapshot.pid != Some(identity.pid)
3601 || snapshot.process_start_time != Some(identity.start_time)
3602 {
3603 return Ok(false);
3604 }
3605 snapshot.deliberate_severance = Some(identity);
3606 Ok(true)
3607 }
3608
3609 pub(crate) fn status_for_control(
3614 &self,
3615 caller: &'static str,
3616 ) -> Result<ModuleStatus, SuperviseError> {
3617 self.status_with_snapshot_lock(&self.inner.snapshot, Some(caller))
3618 .map(|(status, _)| status)
3619 }
3620
3621 fn status_with_snapshot_lock(
3622 &self,
3623 snapshot: &SharedSnapshot,
3624 caller: Option<&'static str>,
3625 ) -> Result<(ModuleStatus, Option<SpawnedFileIdentity>), SuperviseError> {
3626 let mut guard = match caller {
3627 Some(caller) => lock_snapshot_for_control(snapshot, &self.inner.module_id, caller)?,
3628 None => lock_snapshot(snapshot)?,
3629 };
3630 let restart_count =
3633 guard.crash_restarts_in_window(self.inner.restart_policy.window, Instant::now());
3634 let snapshot = guard.clone();
3635 drop(guard);
3636 let drain_timeout = *self.inner.effective_drain_timeout.lock().map_err(|_| {
3637 SuperviseError::StatePoisoned {
3638 module_id: Some(self.inner.module_id.clone()),
3639 }
3640 })?;
3641 let registration_active = self
3642 .inner
3643 .registry
3644 .get_module(&self.inner.module_id)
3645 .map_err(SuperviseError::Registry)?
3646 .is_some();
3647 let protocol = snapshot
3648 .spawned_protocol
3649 .unwrap_or(self.declared_protocol()?);
3650 let running_process =
3651 snapshot.enabled && snapshot.state == ModuleState::Running && snapshot.process_alive;
3652 let live = match protocol {
3658 ModuleProtocol::Subc => running_process && registration_active,
3659 ModuleProtocol::None => running_process,
3660 };
3661
3662 Ok((
3663 ModuleStatus {
3664 waiting_for_previous_pid: snapshot.waiting_for_previous_pid,
3665 waiting_for_previous_detail: snapshot.waiting_for_previous_detail.clone(),
3666 module_id: self.inner.module_id.clone(),
3667 launch_nonce_source: observed_launch_nonce_source(&snapshot),
3668 state: snapshot.state,
3669 enabled: snapshot.enabled,
3670 process_alive: snapshot.process_alive,
3671 registration_active,
3672 protocol,
3673 live,
3674 restart_count,
3675 lifetime_restarts: snapshot.lifetime_restarts,
3676 spawn_generation: snapshot.spawn_generation,
3677 max_restarts: self.inner.restart_policy.max_restarts,
3678 restart_window: self.inner.restart_policy.window,
3679 drain_timeout,
3680 restart_backoff: self.inner.restart_policy.backoff,
3681 restart_max_backoff: self.inner.restart_policy.max_backoff,
3682 pid: snapshot.reported_pid(),
3683 spawned_at_ms: snapshot.spawned_at_ms,
3684 spawned_from: snapshot.spawned_from,
3685 launch_spec_digest: snapshot.launch_spec_digest,
3686 process_start_time: snapshot.process_start_time,
3687 last_exit: snapshot.last_exit,
3688 health: snapshot.health,
3689 },
3690 snapshot.spawned_file_identity.clone(),
3691 ))
3692 }
3693
3694 #[cfg(test)]
3695 pub(crate) fn hold_snapshot_for_test(
3696 &self,
3697 acquired: std::sync::mpsc::Sender<()>,
3698 hold: Duration,
3699 ) -> std::thread::JoinHandle<()> {
3700 let snapshot = Arc::clone(&self.inner.snapshot);
3701 std::thread::spawn(move || {
3702 let _guard = snapshot.lock().expect("test snapshot lock is not poisoned");
3703 acquired
3704 .send(())
3705 .expect("test receiver waits for snapshot lock");
3706 std::thread::sleep(hold);
3707 })
3708 }
3709
3710 pub(crate) async fn status_and_running_image_agreement(
3716 &self,
3717 ) -> Result<(ModuleStatus, subc_control::RunningImageAgreement), SuperviseError> {
3718 let (status, identity) = self.status_with_snapshot_lock(&self.inner.snapshot, None)?;
3719 let image = self
3720 .inner
3721 .provenance_probe
3722 .observe(
3723 status.pid,
3724 status.spawned_from.as_deref(),
3725 identity,
3726 status.process_start_time,
3727 )
3728 .await;
3729 Ok((status, image))
3730 }
3731
3732 pub(crate) async fn running_image_agreement(&self) -> subc_control::RunningImageAgreement {
3733 let snapshot = match lock_snapshot(&self.inner.snapshot) {
3734 Ok(snapshot) => snapshot.clone(),
3735 Err(_) => {
3736 return subc_control::RunningImageAgreement::Unavailable {
3737 reason: subc_control::RunningImageUnavailableReason::NotRunning,
3738 };
3739 }
3740 };
3741 self.inner
3742 .provenance_probe
3743 .observe(
3744 snapshot.reported_pid(),
3745 snapshot.spawned_from.as_deref(),
3746 snapshot.spawned_file_identity.clone(),
3747 snapshot.process_start_time,
3748 )
3749 .await
3750 }
3751
3752 pub(crate) fn child_resource_usage(&self) -> subc_control::ChildResourceUsage {
3755 #[cfg(windows)]
3756 {
3757 let image = lock_snapshot(&self.inner.snapshot)
3758 .ok()
3759 .and_then(|snapshot| {
3760 snapshot
3761 .spawned_file_identity
3762 .as_ref()
3763 .and_then(|identity| identity.windows_image.clone())
3764 });
3765 match image {
3766 Some(image) => {
3767 crate::child_resources::read_windows_handle(image.process(), image.start_time())
3768 }
3769 None => subc_control::ChildResourceUsage::Unavailable {
3770 reason: subc_control::ChildResourceUnavailableReason::NotRunning,
3771 },
3772 }
3773 }
3774 #[cfg(not(windows))]
3775 {
3776 let (pid, start_time) = match lock_snapshot(&self.inner.snapshot) {
3777 Ok(snapshot) => (snapshot.reported_pid(), snapshot.process_start_time),
3778 Err(_) => {
3779 return subc_control::ChildResourceUsage::Unavailable {
3780 reason: subc_control::ChildResourceUnavailableReason::Unreadable,
3781 }
3782 }
3783 };
3784 crate::child_resources::read(pid, start_time)
3785 }
3786 }
3787
3788 pub(crate) fn will_recover_after_connection_loss(&self) -> Result<bool, SuperviseError> {
3789 let mut snapshot = lock_snapshot(&self.inner.snapshot)?;
3790 Ok(match snapshot.state {
3791 ModuleState::Restarting => true,
3792 ModuleState::Failed | ModuleState::Disabled => false,
3793 _ if snapshot.enabled && snapshot.respawn_pending => true,
3794 _ => daemon_will_restart(&mut snapshot, &self.inner.restart_policy, Instant::now()),
3795 })
3796 }
3797
3798 #[cfg(test)]
3799 pub(crate) fn is_warming(&self) -> Result<bool, SuperviseError> {
3800 self.is_warming_with_snapshot_lock(None)
3801 }
3802
3803 pub(crate) fn is_warming_for_control(
3804 &self,
3805 caller: &'static str,
3806 ) -> Result<bool, SuperviseError> {
3807 self.is_warming_with_snapshot_lock(Some(caller))
3808 }
3809
3810 fn is_warming_with_snapshot_lock(
3811 &self,
3812 caller: Option<&'static str>,
3813 ) -> Result<bool, SuperviseError> {
3814 let snapshot = match caller {
3815 Some(caller) => {
3816 lock_snapshot_for_control(&self.inner.snapshot, &self.inner.module_id, caller)?
3817 }
3818 None => lock_snapshot(&self.inner.snapshot)?,
3819 }
3820 .clone();
3821 Ok(matches!(
3822 snapshot.state,
3823 ModuleState::Starting | ModuleState::Running | ModuleState::Restarting
3824 ))
3825 }
3826
3827 pub async fn drain(&self) -> Result<(), SuperviseError> {
3829 self.stop().await
3830 }
3831
3832 pub(crate) async fn retire_requested(
3833 &self,
3834 requester: Option<Principal>,
3835 ) -> Result<(), SuperviseError> {
3836 match self.state()? {
3837 ModuleState::Stopped | ModuleState::Failed => return Ok(()),
3838 ModuleState::Starting
3839 | ModuleState::Running
3840 | ModuleState::Unresponsive
3841 | ModuleState::Restarting
3842 | ModuleState::Draining
3843 | ModuleState::Disabled => {}
3844 }
3845
3846 let (reply_tx, reply_rx) = oneshot::channel();
3847 self.inner
3848 .commands
3849 .send(SupervisorCommand::Retire {
3850 requester,
3851 reply: reply_tx,
3852 })
3853 .await
3854 .map_err(|_| SuperviseError::CommandClosed {
3855 module_id: self.inner.module_id.clone(),
3856 })?;
3857 reply_rx.await.map_err(|_| SuperviseError::CommandClosed {
3858 module_id: self.inner.module_id.clone(),
3859 })?
3860 }
3861
3862 pub async fn stop(&self) -> Result<(), SuperviseError> {
3863 self.stop_requested(None).await
3864 }
3865
3866 pub(crate) async fn stop_requested(
3867 &self,
3868 requester: Option<Principal>,
3869 ) -> Result<(), SuperviseError> {
3870 match self.state()? {
3871 ModuleState::Stopped | ModuleState::Failed => return Ok(()),
3872 ModuleState::Starting
3873 | ModuleState::Running
3874 | ModuleState::Unresponsive
3875 | ModuleState::Restarting
3876 | ModuleState::Draining
3877 | ModuleState::Disabled => {}
3878 }
3879
3880 let (reply_tx, reply_rx) = oneshot::channel();
3881 self.inner
3882 .commands
3883 .send(SupervisorCommand::Drain {
3884 requester,
3885 reply: reply_tx,
3886 })
3887 .await
3888 .map_err(|_| SuperviseError::CommandClosed {
3889 module_id: self.inner.module_id.clone(),
3890 })?;
3891 reply_rx.await.map_err(|_| SuperviseError::CommandClosed {
3892 module_id: self.inner.module_id.clone(),
3893 })?
3894 }
3895
3896 pub async fn restart(&self, drain_timeout_ms: Option<u64>) -> Result<(), SuperviseError> {
3897 self.restart_requested(drain_timeout_ms, None).await
3898 }
3899
3900 pub(crate) async fn restart_requested(
3901 &self,
3902 drain_timeout_ms: Option<u64>,
3903 requester: Option<Principal>,
3904 ) -> Result<(), SuperviseError> {
3905 let queued_at = Instant::now();
3906 let received_at_generation = lock_snapshot(&self.inner.snapshot)?.spawn_generation;
3907 let (reply_tx, reply_rx) = oneshot::channel();
3908 self.inner
3909 .commands
3910 .send(SupervisorCommand::Restart {
3911 requester,
3912 drain_timeout_ms,
3913 received_at_generation,
3914 queued_at,
3915 reply: reply_tx,
3916 })
3917 .await
3918 .map_err(|_| SuperviseError::CommandClosed {
3919 module_id: self.inner.module_id.clone(),
3920 })?;
3921 reply_rx.await.map_err(|_| SuperviseError::CommandClosed {
3922 module_id: self.inner.module_id.clone(),
3923 })?
3924 }
3925
3926 pub async fn swap(&self, ready_timeout: Option<Duration>) -> Result<(), SuperviseError> {
3931 self.swap_requested(ready_timeout, None).await
3932 }
3933
3934 pub(crate) async fn swap_requested(
3935 &self,
3936 ready_timeout: Option<Duration>,
3937 requester: Option<Principal>,
3938 ) -> Result<(), SuperviseError> {
3939 let (reply_tx, reply_rx) = oneshot::channel();
3940 self.inner
3941 .commands
3942 .send(SupervisorCommand::Swap {
3943 requester,
3944 ready_timeout,
3945 reply: reply_tx,
3946 })
3947 .await
3948 .map_err(|_| SuperviseError::CommandClosed {
3949 module_id: self.inner.module_id.clone(),
3950 })?;
3951 reply_rx.await.map_err(|_| SuperviseError::CommandClosed {
3952 module_id: self.inner.module_id.clone(),
3953 })?
3954 }
3955
3956 pub async fn reload(&self) -> Result<(), SuperviseError> {
3957 self.reload_requested(None).await
3958 }
3959
3960 pub(crate) async fn reload_requested(
3961 &self,
3962 requester: Option<Principal>,
3963 ) -> Result<(), SuperviseError> {
3964 let (reply_tx, reply_rx) = oneshot::channel();
3965 self.inner
3966 .commands
3967 .send(SupervisorCommand::Reload {
3968 requester,
3969 reply: reply_tx,
3970 })
3971 .await
3972 .map_err(|_| SuperviseError::CommandClosed {
3973 module_id: self.inner.module_id.clone(),
3974 })?;
3975 reply_rx.await.map_err(|_| SuperviseError::CommandClosed {
3976 module_id: self.inner.module_id.clone(),
3977 })?
3978 }
3979
3980 pub async fn set_enabled(&self, enabled: bool) -> Result<bool, SuperviseError> {
3981 self.set_enabled_requested(enabled, None).await
3982 }
3983
3984 pub(crate) async fn set_enabled_requested(
3985 &self,
3986 enabled: bool,
3987 requester: Option<Principal>,
3988 ) -> Result<bool, SuperviseError> {
3989 let (reply_tx, reply_rx) = oneshot::channel();
3990 self.inner
3991 .commands
3992 .send(SupervisorCommand::SetEnabled {
3993 requester,
3994 enabled,
3995 reply: reply_tx,
3996 })
3997 .await
3998 .map_err(|_| SuperviseError::CommandClosed {
3999 module_id: self.inner.module_id.clone(),
4000 })?;
4001 reply_rx.await.map_err(|_| SuperviseError::CommandClosed {
4002 module_id: self.inner.module_id.clone(),
4003 })?
4004 }
4005
4006 pub(crate) fn declared_protocol(&self) -> Result<ModuleProtocol, SuperviseError> {
4010 let configured = self
4011 .inner
4012 .configuration
4013 .lock()
4014 .map_err(|_| SuperviseError::StatePoisoned {
4015 module_id: Some(self.inner.module_id.clone()),
4016 })?
4017 .spec
4018 .protocol;
4019 let state = lock_snapshot(&self.inner.snapshot)?;
4020 Ok(state.spawned_protocol.unwrap_or(configured))
4021 }
4022
4023 pub(crate) fn configuration(&self) -> Result<(ModuleSpec, HealthConfig), SuperviseError> {
4024 let configuration =
4025 self.inner
4026 .configuration
4027 .lock()
4028 .map_err(|_| SuperviseError::StatePoisoned {
4029 module_id: Some(self.inner.module_id.clone()),
4030 })?;
4031 Ok((configuration.spec.clone(), configuration.health.clone()))
4032 }
4033
4034 #[cfg(any(test, feature = "test-support"))]
4038 pub async fn update_spec_for_test(&self, spec: ModuleSpec) -> Result<(), SuperviseError> {
4039 let (_, health) = self.configuration()?;
4040 let drain_timeout_ms = u64::try_from(
4041 self.inner
4042 .effective_drain_timeout
4043 .lock()
4044 .unwrap_or_else(|poisoned| poisoned.into_inner())
4045 .as_millis(),
4046 )
4047 .ok();
4048 self.update_configuration(spec, health, drain_timeout_ms)
4049 .await
4050 }
4051
4052 pub(crate) async fn update_configuration(
4053 &self,
4054 spec: ModuleSpec,
4055 health: HealthConfig,
4056 drain_timeout_ms: Option<u64>,
4057 ) -> Result<(), SuperviseError> {
4058 if spec.module_id != self.inner.module_id {
4059 return Err(SuperviseError::InvalidSpec {
4060 reason: "a supervised module's module_id cannot be changed".to_string(),
4061 });
4062 }
4063 validate_spec(&spec)?;
4064 let (reply_tx, reply_rx) = oneshot::channel();
4065 self.inner
4066 .commands
4067 .send(SupervisorCommand::UpdateConfiguration {
4068 spec: spec.clone(),
4069 health: health.clone(),
4070 drain_timeout_ms,
4071 reply: reply_tx,
4072 })
4073 .await
4074 .map_err(|_| SuperviseError::CommandClosed {
4075 module_id: self.inner.module_id.clone(),
4076 })?;
4077 reply_rx.await.map_err(|_| SuperviseError::CommandClosed {
4078 module_id: self.inner.module_id.clone(),
4079 })?;
4080 let mut configuration =
4081 self.inner
4082 .configuration
4083 .lock()
4084 .map_err(|_| SuperviseError::StatePoisoned {
4085 module_id: Some(self.inner.module_id.clone()),
4086 })?;
4087 configuration.spec = spec;
4088 configuration.health = health;
4089 Ok(())
4090 }
4091}
4092
4093impl Drop for SupervisedModuleInner {
4094 fn drop(&mut self) {
4095 let Ok(mut monitor) = self.monitor.lock() else {
4096 return;
4097 };
4098 if let Some(monitor) = monitor.as_ref().filter(|monitor| !monitor.is_finished()) {
4099 let _ = update_snapshot(&self.snapshot, Some(&self.module_id), |state| {
4100 state.state = ModuleState::Stopped;
4101 clear_current_process_facts(state);
4102 });
4103 monitor.abort();
4104 }
4105 let _ = monitor.take();
4106 }
4107}
4108
4109#[derive(Debug)]
4110enum SupervisorCommand {
4111 Drain {
4112 requester: Option<Principal>,
4113 reply: oneshot::Sender<Result<(), SuperviseError>>,
4114 },
4115 Retire {
4116 requester: Option<Principal>,
4117 reply: oneshot::Sender<Result<(), SuperviseError>>,
4118 },
4119 Restart {
4120 requester: Option<Principal>,
4121 drain_timeout_ms: Option<u64>,
4126 received_at_generation: u64,
4130 queued_at: Instant,
4133 reply: oneshot::Sender<Result<(), SuperviseError>>,
4134 },
4135 Reload {
4136 requester: Option<Principal>,
4137 reply: oneshot::Sender<Result<(), SuperviseError>>,
4138 },
4139 SetEnabled {
4140 requester: Option<Principal>,
4141 enabled: bool,
4142 reply: oneshot::Sender<Result<bool, SuperviseError>>,
4143 },
4144 UpdateConfiguration {
4145 spec: ModuleSpec,
4146 health: HealthConfig,
4147 drain_timeout_ms: Option<u64>,
4150 reply: oneshot::Sender<()>,
4151 },
4152 Swap {
4153 requester: Option<Principal>,
4154 ready_timeout: Option<Duration>,
4157 reply: oneshot::Sender<Result<(), SuperviseError>>,
4159 },
4160}
4161
4162#[derive(Debug)]
4163pub enum SuperviseError {
4164 InvalidSpec {
4165 reason: String,
4166 },
4167 Spawn {
4168 program: PathBuf,
4169 source: io::Error,
4170 cgroup_path: Option<PathBuf>,
4171 },
4172 Cgroup {
4173 module_id: String,
4174 source: io::Error,
4175 },
4176 LaunchNonce {
4179 reason: String,
4180 },
4181 Wait {
4182 module_id: String,
4183 source: io::Error,
4184 },
4185 Kill {
4186 module_id: String,
4187 source: io::Error,
4188 },
4189 Forwarding(ForwardingError),
4190 Registry(RegistryError),
4191 ReloadUnavailable {
4192 module_id: String,
4193 reason: String,
4194 },
4195 Disabled {
4200 module_id: String,
4201 },
4202 ReloadFailed {
4203 module_id: String,
4204 reason: String,
4205 },
4206 RegistrationStillActive {
4207 module_id: String,
4208 waited: Duration,
4209 },
4210 StatePoisoned {
4211 module_id: Option<String>,
4212 },
4213 CommandClosed {
4214 module_id: String,
4215 },
4216 SwapInProgress {
4220 module_id: String,
4221 },
4222 SwapRefused {
4224 module_id: String,
4225 reason: SwapRefusal,
4226 },
4227 SwapFailed {
4231 module_id: String,
4232 arm: SwapFailureArm,
4233 detail: String,
4234 candidate_exit: Option<ExitReport>,
4237 },
4238}
4239
4240#[derive(Debug, Clone, Copy, PartialEq, Eq)]
4242pub enum SwapRefusal {
4243 OverlapExclusive,
4245 NotRegistered,
4248 ProtocolNone,
4251 NotConfigured,
4254 AlreadySwapping,
4256}
4257
4258impl SwapRefusal {
4259 pub fn as_str(self) -> &'static str {
4260 match self {
4261 Self::OverlapExclusive => "overlap_exclusive",
4262 Self::NotRegistered => "not_registered",
4263 Self::ProtocolNone => "protocol_none",
4264 Self::NotConfigured => "not_configured",
4265 Self::AlreadySwapping => "already_swapping",
4266 }
4267 }
4268}
4269
4270#[derive(Debug, Clone, Copy, PartialEq, Eq)]
4273pub enum SwapFailureArm {
4274 SpawnFailed,
4276 NeverRegistered,
4278 NeverReady,
4280 CandidateExited,
4282 CandidateUnhealthy,
4284 Interrupted,
4288 CutoverLost,
4293}
4294
4295impl SwapFailureArm {
4296 pub fn as_str(self) -> &'static str {
4297 match self {
4298 Self::SpawnFailed => "spawn_failed",
4299 Self::NeverRegistered => "never_registered",
4300 Self::NeverReady => "never_ready",
4301 Self::CandidateExited => "candidate_exited",
4302 Self::CandidateUnhealthy => "candidate_unhealthy",
4303 Self::Interrupted => "interrupted",
4304 Self::CutoverLost => "cutover_lost",
4305 }
4306 }
4307}
4308
4309impl fmt::Display for SuperviseError {
4310 fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
4311 match self {
4312 Self::InvalidSpec { reason } => write!(f, "invalid module spec: {reason}"),
4313 Self::Spawn {
4314 program,
4315 source,
4316 cgroup_path: Some(cgroup_path),
4317 } => write!(
4318 f,
4319 "failed to place module in cgroup '{}' while spawning '{}': {source}",
4320 cgroup_path.display(),
4321 program.display()
4322 ),
4323 Self::Spawn {
4324 program,
4325 source,
4326 cgroup_path: None,
4327 } => write!(
4328 f,
4329 "failed to spawn module '{}': {source}",
4330 program.display()
4331 ),
4332 Self::Cgroup { module_id, source } => {
4333 write!(
4334 f,
4335 "failed to prepare cgroup for module '{module_id}': {source}"
4336 )
4337 }
4338 Self::LaunchNonce { reason } => {
4339 write!(
4340 f,
4341 "failed to generate reserved-module launch nonce: {reason}"
4342 )
4343 }
4344 Self::Wait { module_id, source } => {
4345 write!(f, "failed to wait for module '{module_id}': {source}")
4346 }
4347 Self::Kill { module_id, source } => {
4348 write!(f, "failed to kill module '{module_id}': {source}")
4349 }
4350 Self::Forwarding(err) => write!(f, "forwarding error: {err}"),
4351 Self::Registry(err) => write!(f, "registry error: {err}"),
4352 Self::ReloadUnavailable { module_id, reason } => {
4353 write!(f, "reload unavailable for module '{module_id}': {reason}")
4354 }
4355 Self::Disabled { module_id } => {
4356 write!(
4357 f,
4358 "module '{module_id}' is disabled; enable it before restart or reload"
4359 )
4360 }
4361 Self::ReloadFailed { module_id, reason } => {
4362 write!(f, "reload failed for module '{module_id}': {reason}")
4363 }
4364 Self::RegistrationStillActive { module_id, waited } => write!(
4365 f,
4366 "module '{module_id}' registration remained active after waiting {waited:?}"
4367 ),
4368 Self::StatePoisoned { module_id } => match module_id {
4369 Some(module_id) => {
4370 write!(f, "supervisor state for module '{module_id}' was poisoned")
4371 }
4372 None => write!(f, "supervisor state was poisoned"),
4373 },
4374 Self::CommandClosed { module_id } => {
4375 write!(
4376 f,
4377 "supervisor command channel for module '{module_id}' is closed"
4378 )
4379 }
4380 Self::SwapInProgress { module_id } => write!(
4381 f,
4382 "module '{module_id}' is being swapped; retry once the swap has cut over or failed, or stop the module to abort the swap"
4383 ),
4384 Self::SwapRefused { module_id, reason } => match reason {
4385 SwapRefusal::OverlapExclusive => write!(
4386 f,
4387 "module '{module_id}' is declared overlap: \"exclusive\" (the default): two processes of it must not run at once, so it cannot be swapped; use a plain restart, or declare overlap: \"safe\" in its config if it really tolerates a second process"
4388 ),
4389 SwapRefusal::NotRegistered => write!(
4390 f,
4391 "module '{module_id}' is not registered, so there is no serving process to keep while a replacement warms; use a plain restart"
4392 ),
4393 SwapRefusal::ProtocolNone => write!(
4394 f,
4395 "module '{module_id}' is protocol: \"none\" and never registers, so a swap could never see its replacement become ready; use a plain restart"
4396 ),
4397 SwapRefusal::NotConfigured => write!(
4398 f,
4399 "module '{module_id}' cannot be swapped: the supervisor was built without the forwarding table or shared handle a swap needs"
4400 ),
4401 SwapRefusal::AlreadySwapping => {
4402 write!(f, "module '{module_id}' is already being swapped")
4403 }
4404 },
4405 Self::SwapFailed {
4406 module_id,
4407 arm,
4408 detail,
4409 ..
4410 } => write!(
4411 f,
4412 "swap of module '{module_id}' failed ({}): {detail}; the running process was left serving",
4413 arm.as_str()
4414 ),
4415 }
4416 }
4417}
4418
4419impl Error for SuperviseError {
4420 fn source(&self) -> Option<&(dyn Error + 'static)> {
4421 match self {
4422 Self::Spawn { source, .. }
4423 | Self::Cgroup { source, .. }
4424 | Self::Wait { source, .. }
4425 | Self::Kill { source, .. } => Some(source),
4426 Self::Forwarding(err) => Some(err),
4427 Self::Registry(err) => Some(err),
4428 Self::LaunchNonce { .. }
4429 | Self::InvalidSpec { .. }
4430 | Self::ReloadUnavailable { .. }
4431 | Self::Disabled { .. }
4432 | Self::ReloadFailed { .. }
4433 | Self::RegistrationStillActive { .. }
4434 | Self::StatePoisoned { .. }
4435 | Self::CommandClosed { .. }
4436 | Self::SwapInProgress { .. }
4437 | Self::SwapRefused { .. }
4438 | Self::SwapFailed { .. } => None,
4439 }
4440 }
4441}
4442
4443pub(crate) fn validate_spec(spec: &ModuleSpec) -> Result<(), SuperviseError> {
4444 if spec.module_id.trim().is_empty() {
4445 return Err(SuperviseError::InvalidSpec {
4446 reason: "module_id must not be empty".to_string(),
4447 });
4448 }
4449
4450 Ok(())
4451}
4452
4453#[derive(Debug, Default)]
4454struct HealthProbeRuntime {
4455 configured_health: Option<HealthConfig>,
4456 registered_connection: Option<crate::ConnectionId>,
4457 advertised: bool,
4458 next_probe_at: Option<Instant>,
4459 probe_index: u64,
4460}
4461
4462fn running_protocol(spec: &ModuleSpec, snapshot: &SharedSnapshot) -> ModuleProtocol {
4463 lock_snapshot(snapshot)
4464 .ok()
4465 .and_then(|state| state.spawned_protocol)
4466 .unwrap_or(spec.protocol)
4467}
4468
4469impl HealthProbeRuntime {
4470 fn refresh_registration(
4471 &mut self,
4472 spec: &ModuleSpec,
4473 runtime: &SupervisorRuntimeConfig,
4474 registry: &Registry,
4475 snapshot: &SharedSnapshot,
4476 ) {
4477 if self.configured_health.as_ref() != Some(&runtime.health) {
4478 self.configured_health = Some(runtime.health.clone());
4479 self.next_probe_at = None;
4480 self.registered_connection = None;
4481 self.probe_index = 0;
4482 }
4483 if running_protocol(spec, snapshot) == ModuleProtocol::None {
4487 self.registered_connection = None;
4488 self.advertised = runtime.health.http.is_some();
4489 if !self.advertised {
4490 self.next_probe_at = None;
4491 let _ = update_snapshot(snapshot, Some(&spec.module_id), |state| {
4492 let unknown = ModuleHealthStatus::default();
4493 if state.health != unknown {
4494 state.health = unknown;
4495 }
4496 });
4497 } else if self.next_probe_at.is_none() {
4498 self.next_probe_at = Some(
4499 Instant::now()
4500 + jittered_health_delay(&spec.module_id, 0, runtime.health.cadence),
4501 );
4502 }
4503 return;
4504 }
4505
4506 let registration = match registry.get_module(&spec.module_id) {
4507 Ok(registration) => registration,
4508 Err(err) => {
4509 warn!(module_id = %spec.module_id, error = %err, "health prober could not read registry");
4510 self.advertised = false;
4511 self.next_probe_at = None;
4512 return;
4513 }
4514 };
4515
4516 let Some(registration) = registration else {
4517 self.registered_connection = None;
4518 self.advertised = false;
4519 self.next_probe_at = None;
4520 return;
4521 };
4522
4523 let advertised = registration
4524 .control_ops
4525 .iter()
4526 .any(|op| op == MODULE_CONTROL_OP_HEALTH_CHECK);
4527 if !advertised {
4528 self.registered_connection = Some(registration.connection_id);
4529 self.advertised = false;
4530 self.next_probe_at = None;
4531 let _ = update_snapshot(snapshot, Some(&spec.module_id), |state| {
4532 if state.health.status != SupervisorHealthStatus::Unknown
4533 || state.health.consecutive_failures != 0
4534 || state.health.last_probe_ms.is_some()
4535 || state.health.detail.is_some()
4536 || state.health.metrics.is_some()
4537 {
4538 state.health.status = SupervisorHealthStatus::Unknown;
4539 state.health.consecutive_failures = 0;
4540 state.health.last_probe_ms = None;
4541 state.health.detail = None;
4542 state.health.metrics = None;
4543 }
4544 });
4545 return;
4546 }
4547
4548 let reregistered = self.registered_connection != Some(registration.connection_id);
4549 self.registered_connection = Some(registration.connection_id);
4550 self.advertised = true;
4551 if reregistered || self.next_probe_at.is_none() {
4552 self.probe_index = 0;
4553 self.next_probe_at = Some(
4554 Instant::now() + jittered_health_delay(&spec.module_id, 0, runtime.health.cadence),
4555 );
4556 let _ = update_snapshot(snapshot, Some(&spec.module_id), |state| {
4557 state.health.status = SupervisorHealthStatus::Unknown;
4558 state.health.consecutive_failures = 0;
4559 state.health.detail = None;
4560 state.health.metrics = None;
4561 });
4562 }
4563 }
4564
4565 fn wake_after(&self) -> Option<Duration> {
4566 if !self.advertised {
4567 return None;
4568 }
4569 self.next_probe_at
4570 .map(|next| next.saturating_duration_since(Instant::now()))
4571 }
4572
4573 fn due(&self) -> bool {
4574 self.advertised
4575 && self
4576 .next_probe_at
4577 .is_some_and(|next| Instant::now() >= next)
4578 }
4579
4580 fn schedule_next(&mut self, spec: &ModuleSpec, cadence: Duration) {
4581 self.probe_index = self.probe_index.wrapping_add(1);
4582 self.next_probe_at = Some(
4583 Instant::now() + jittered_health_delay(&spec.module_id, self.probe_index, cadence),
4584 );
4585 }
4586}
4587
4588#[derive(Debug)]
4623enum HealthProbeEvidence {
4624 LaneDead,
4626 NoAnswer,
4628 BadAnswer,
4630 Misconfigured,
4632}
4633
4634#[derive(Debug)]
4635struct HealthProbeError {
4636 evidence: HealthProbeEvidence,
4637 message: String,
4638}
4639
4640impl HealthProbeError {
4641 fn lane_dead(message: impl Into<String>) -> Self {
4642 Self::with(HealthProbeEvidence::LaneDead, message)
4643 }
4644
4645 fn no_answer(message: impl Into<String>) -> Self {
4646 Self::with(HealthProbeEvidence::NoAnswer, message)
4647 }
4648
4649 fn bad_answer(message: impl Into<String>) -> Self {
4650 Self::with(HealthProbeEvidence::BadAnswer, message)
4651 }
4652
4653 fn misconfigured(message: impl Into<String>) -> Self {
4654 Self::with(HealthProbeEvidence::Misconfigured, message)
4655 }
4656
4657 fn with(evidence: HealthProbeEvidence, message: impl Into<String>) -> Self {
4658 Self {
4659 evidence,
4660 message: message.into(),
4661 }
4662 }
4663
4664 #[allow(dead_code)]
4678 fn is_proof_of_death(&self) -> bool {
4679 matches!(self.evidence, HealthProbeEvidence::LaneDead)
4680 }
4681
4682 fn label(&self) -> &'static str {
4690 match self.evidence {
4691 HealthProbeEvidence::LaneDead => "lane-dead",
4692 HealthProbeEvidence::NoAnswer => "no-answer",
4693 HealthProbeEvidence::BadAnswer => "bad-answer",
4694 HealthProbeEvidence::Misconfigured => "daemon-misconfigured",
4695 }
4696 }
4697}
4698
4699impl fmt::Display for HealthProbeError {
4700 fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
4701 f.write_str(&self.message)
4702 }
4703}
4704
4705async fn run_health_probe_cycle(
4706 spec: &ModuleSpec,
4707 runtime: &SupervisorRuntimeConfig,
4708 registry: &Registry,
4709 process_liveness: &SupervisorProcessLiveness,
4710 snapshot: &SharedSnapshot,
4711 child: &mut Option<SupervisedChild>,
4712) {
4713 let now_ms = unix_ms_now();
4714 let http = (running_protocol(spec, snapshot) == ModuleProtocol::None)
4715 .then_some(runtime.health.http.as_deref())
4716 .flatten();
4717 let result = match http {
4718 Some(url) => probe_http_health(url, runtime.health.deadline).await,
4719 None => probe_module_health(&spec.module_id, runtime, None).await,
4720 };
4721 match result {
4722 Ok(report) => {
4723 handle_health_report(
4724 spec,
4725 runtime,
4726 registry,
4727 process_liveness,
4728 snapshot,
4729 child,
4730 report,
4731 now_ms,
4732 )
4733 .await;
4734 }
4735 Err(err) => {
4736 if http.is_some() {
4737 let _ = update_snapshot(snapshot, Some(&spec.module_id), |state| {
4738 state.health.status = SupervisorHealthStatus::Failing;
4739 });
4740 }
4741 handle_health_probe_failure(
4742 spec,
4743 runtime,
4744 registry,
4745 process_liveness,
4746 snapshot,
4747 child,
4748 err,
4749 now_ms,
4750 )
4751 .await;
4752 }
4753 }
4754}
4755
4756pub(crate) struct HttpProbeTarget<'a> {
4757 address: std::net::SocketAddr,
4758 localhost: bool,
4759 authority: &'a str,
4760 path: String,
4761}
4762
4763pub(crate) fn parse_http_probe_url(url: &str) -> Result<HttpProbeTarget<'_>, String> {
4766 if url.bytes().any(|b| b <= b' ' || b == 127) || url.contains('#') {
4767 return Err("must not contain whitespace, controls, or a fragment".into());
4768 }
4769 let rest = url
4770 .strip_prefix("http://")
4771 .ok_or("must use plain http://")?;
4772 let split = rest.find(['/', '?']).unwrap_or(rest.len());
4773 let (authority, suffix) = rest.split_at(split);
4774 let (host, port) = if let Some(rest) = authority.strip_prefix("[::1]") {
4775 ("::1", rest)
4776 } else {
4777 let split = authority.find(':').unwrap_or(authority.len());
4778 authority.split_at(split)
4779 };
4780 let ip: std::net::IpAddr = match host {
4781 "127.0.0.1" | "localhost" => std::net::Ipv4Addr::LOCALHOST.into(),
4782 "::1" => std::net::Ipv6Addr::LOCALHOST.into(),
4783 _ => return Err("host must be 127.0.0.1, [::1], or localhost".into()),
4784 };
4785 let port = if port.is_empty() {
4786 80
4787 } else {
4788 port.strip_prefix(':')
4789 .and_then(|p| p.parse::<u16>().ok())
4790 .filter(|p| *p > 0)
4791 .ok_or("must have a valid nonzero TCP port")?
4792 };
4793 let path = if suffix.is_empty() {
4794 "/".into()
4795 } else if suffix.starts_with('?') {
4796 format!("/{suffix}")
4797 } else {
4798 suffix.into()
4799 };
4800 Ok(HttpProbeTarget {
4801 address: std::net::SocketAddr::new(ip, port),
4802 localhost: host == "localhost",
4803 authority,
4804 path,
4805 })
4806}
4807
4808async fn probe_http_health(
4809 url: &str,
4810 deadline: Duration,
4811) -> Result<HealthReport, HealthProbeError> {
4812 use tokio::io::{AsyncReadExt, AsyncWriteExt, BufReader};
4813 let target = parse_http_probe_url(url).map_err(HealthProbeError::misconfigured)?;
4814 let mut response_status = String::new();
4817 let mut body = Vec::new();
4818 let probe = async {
4819 let connection = match tokio::net::TcpStream::connect(target.address).await {
4822 Err(_) if target.localhost => {
4823 tokio::net::TcpStream::connect((
4824 std::net::Ipv6Addr::LOCALHOST,
4825 target.address.port(),
4826 ))
4827 .await
4828 }
4829 result => result,
4830 };
4831 let mut stream = connection.map_err(|error| {
4832 HealthProbeError::no_answer(format!("HTTP connect failed: {error}"))
4833 })?;
4834 stream.write_all(format!("GET {} HTTP/1.1\r\nHost: {}\r\nAccept-Encoding: identity\r\nConnection: close\r\n\r\n", target.path, target.authority).as_bytes()).await
4835 .map_err(|error| HealthProbeError::no_answer(format!("HTTP write failed: {error}")))?;
4836 let mut reader = BufReader::new(stream);
4837 let mut budget = 16 * 1024;
4838 let status = http_line(&mut reader, &mut budget).await?;
4839 let mut words = status.split_ascii_whitespace();
4840 let version = words.next();
4841 let code = words
4842 .next()
4843 .filter(|word| word.len() == 3)
4844 .and_then(|word| word.parse::<u16>().ok());
4845 if !matches!(version, Some("HTTP/1.1" | "HTTP/1.0"))
4846 || !code.is_some_and(|code| (100..600).contains(&code))
4847 {
4848 return Err(HealthProbeError::bad_answer(format!(
4849 "invalid HTTP status: {status}"
4850 )));
4851 }
4852 let code = code.expect("validated status code");
4853 response_status = status.clone();
4854 let mut length = None;
4855 let mut chunked = false;
4856 loop {
4857 let line = http_line(&mut reader, &mut budget).await?;
4858 if line.is_empty() {
4859 break;
4860 }
4861 if let Some((name, value)) = line.split_once(':') {
4862 if name.eq_ignore_ascii_case("content-length") {
4863 length = Some(value.trim().parse::<u64>().map_err(|_| {
4864 HealthProbeError::bad_answer("invalid HTTP Content-Length")
4865 })?);
4866 } else if name.eq_ignore_ascii_case("transfer-encoding") {
4867 chunked = value.trim().eq_ignore_ascii_case("chunked");
4868 }
4869 }
4870 }
4871 if chunked {
4872 while body.len() < 200 {
4873 let line = http_line(&mut reader, &mut budget).await?;
4874 let size = u64::from_str_radix(line.split(';').next().unwrap_or("").trim(), 16)
4875 .map_err(|_| HealthProbeError::bad_answer("invalid HTTP chunk size"))?;
4876 if size == 0 {
4877 break;
4878 }
4879 let count = size.min((200 - body.len()) as u64) as usize;
4880 let start = body.len();
4881 (&mut reader)
4882 .take(count as u64)
4883 .read_to_end(&mut body)
4884 .await
4885 .map_err(|error| {
4886 HealthProbeError::no_answer(format!("HTTP body read failed: {error}"))
4887 })?;
4888 if body.len() - start != count {
4889 return Err(HealthProbeError::bad_answer("truncated HTTP chunk"));
4890 }
4891 if size > count as u64 || body.len() == 200 {
4892 break;
4893 }
4894 if !http_line(&mut reader, &mut budget).await?.is_empty() {
4895 return Err(HealthProbeError::bad_answer("invalid HTTP chunk delimiter"));
4896 }
4897 }
4898 } else {
4899 reader
4900 .take(length.unwrap_or(200).min(200))
4901 .read_to_end(&mut body)
4902 .await
4903 .map_err(|error| {
4904 HealthProbeError::no_answer(format!("HTTP body read failed: {error}"))
4905 })?;
4906 }
4907 if (200..300).contains(&code) {
4908 Ok(HealthReport::ok())
4909 } else {
4910 Err(HealthProbeError::bad_answer(
4911 "HTTP health endpoint returned non-2xx",
4912 ))
4913 }
4914 };
4915 let mut result = timeout(deadline, probe).await.unwrap_or_else(|_| {
4916 Err(HealthProbeError::no_answer(format!(
4917 "HTTP probe timed out after {deadline:?}"
4918 )))
4919 });
4920 if let Err(error) = &mut result {
4921 if !response_status.is_empty() {
4922 error.message = format!(
4923 "{}; {response_status}: {}",
4924 error.message,
4925 String::from_utf8_lossy(&body)
4926 );
4927 }
4928 }
4929 result
4930}
4931
4932async fn http_line(
4933 reader: &mut tokio::io::BufReader<tokio::net::TcpStream>,
4934 remaining: &mut usize,
4935) -> Result<String, HealthProbeError> {
4936 use tokio::io::{AsyncBufReadExt, AsyncReadExt};
4937 let mut line = Vec::new();
4938 (&mut *reader)
4939 .take(*remaining as u64)
4940 .read_until(b'\n', &mut line)
4941 .await
4942 .map_err(|error| {
4943 HealthProbeError::no_answer(format!("HTTP header read failed: {error}"))
4944 })?;
4945 *remaining -= line.len();
4946 if !line.ends_with(b"\r\n") {
4947 return Err(HealthProbeError::bad_answer(
4948 "HTTP headers are incomplete or exceed 16 KiB",
4949 ));
4950 }
4951 line.truncate(line.len() - 2);
4952 String::from_utf8(line).map_err(|_| HealthProbeError::bad_answer("HTTP header is not UTF-8"))
4953}
4954
4955async fn probe_module_health(
4956 module_id: &str,
4957 runtime: &SupervisorRuntimeConfig,
4958 drain_deadline: Option<Instant>,
4959) -> Result<HealthReport, HealthProbeError> {
4960 let Some(forwarding) = runtime.forwarding.as_ref() else {
4961 return Err(HealthProbeError::misconfigured(
4962 "supervisor was not configured with a forwarding table",
4963 ));
4964 };
4965 let probe_started_at = Instant::now();
4966 let mut deadline = probe_started_at + runtime.health.deadline;
4967 if let Some(drain_deadline) = drain_deadline {
4968 deadline = deadline.min(drain_deadline);
4969 }
4970 let pending = if drain_deadline.is_some() {
4971 forwarding.begin_drain_health_probe_rpc_for(
4972 module_id,
4973 MODULE_CONTROL_OP_HEALTH_CHECK,
4974 probe_started_at,
4975 deadline,
4976 )
4977 } else {
4978 forwarding.begin_health_probe_rpc_for(
4979 module_id,
4980 MODULE_CONTROL_OP_HEALTH_CHECK,
4981 probe_started_at,
4982 deadline,
4983 )
4984 }
4985 .map_err(|err| {
4986 HealthProbeError::lane_dead(format!("failed to begin health.check RPC: {err}"))
4989 })?;
4990 await_health_probe(forwarding, pending, deadline, runtime.health.deadline).await
4991}
4992
4993async fn probe_endpoint_health(
5000 endpoint: crate::ModuleEndpointId,
5001 runtime: &SupervisorRuntimeConfig,
5002 deadline_cap: Option<Instant>,
5003) -> Result<HealthReport, HealthProbeError> {
5004 let Some(forwarding) = runtime.forwarding.as_ref() else {
5005 return Err(HealthProbeError::misconfigured(
5006 "supervisor was not configured with a forwarding table",
5007 ));
5008 };
5009 let probe_started_at = Instant::now();
5010 let mut deadline = probe_started_at + runtime.health.deadline;
5011 if let Some(cap) = deadline_cap {
5012 deadline = deadline.min(cap);
5013 }
5014 let pending = forwarding
5015 .begin_endpoint_health_probe_rpc_for(
5016 endpoint,
5017 MODULE_CONTROL_OP_HEALTH_CHECK,
5018 probe_started_at,
5019 deadline,
5020 )
5021 .map_err(|err| {
5022 HealthProbeError::lane_dead(format!("failed to begin health.check RPC: {err}"))
5023 })?;
5024 await_health_probe(forwarding, pending, deadline, runtime.health.deadline).await
5025}
5026
5027async fn await_health_probe(
5029 forwarding: &ForwardingTable,
5030 pending: PendingModuleControlRpc,
5031 deadline: Instant,
5032 probe_budget: Duration,
5033) -> Result<HealthReport, HealthProbeError> {
5034 let PendingModuleControlRpc {
5035 endpoint,
5036 module_sink,
5037 negotiated_ver,
5038 corr,
5039 receiver,
5040 } = pending;
5041 let body = serde_json::to_vec(&ModuleControlRequest::HealthCheck {}).map_err(|err| {
5042 HealthProbeError::misconfigured(format!("failed to encode health.check: {err}"))
5043 })?;
5044 let frame = Frame::build_with_version(
5045 negotiated_ver,
5046 FrameType::Request,
5047 control_flags(),
5048 0,
5049 0,
5050 corr,
5051 body,
5052 )
5053 .map_err(|err| {
5054 HealthProbeError::misconfigured(format!("failed to build health.check frame: {err}"))
5055 })?;
5056
5057 match timeout_at(deadline, module_sink.send(frame)).await {
5063 Ok(Ok(())) => {}
5064 Ok(Err(err)) => {
5065 let _ = forwarding.cancel_module_control_rpc(endpoint, corr);
5066 return Err(HealthProbeError::lane_dead(format!(
5069 "failed to send health.check: {err}"
5070 )));
5071 }
5072 Err(_elapsed) => {
5073 let _ = forwarding.cancel_module_control_rpc(endpoint, corr);
5074 return Err(HealthProbeError::no_answer(
5078 "health.check send timed out before enqueue (module egress full)",
5079 ));
5080 }
5081 }
5082
5083 match timeout_at(deadline, receiver).await {
5084 Ok(Ok(ModuleControlRpcOutcome::Response(response))) => {
5088 response.health_report().ok_or_else(|| {
5089 HealthProbeError::bad_answer("health.check RPC returned a non-health response")
5090 })
5091 }
5092 Ok(Ok(ModuleControlRpcOutcome::Rejected(body))) => Err(HealthProbeError::bad_answer(
5093 format!("health.check rejected: {}", body.message),
5094 )),
5095 Ok(Ok(ModuleControlRpcOutcome::ModuleGone(message))) => {
5096 Err(HealthProbeError::lane_dead(message))
5097 }
5098 Ok(Ok(ModuleControlRpcOutcome::MalformedResponse(message))) => {
5099 Err(HealthProbeError::bad_answer(message))
5100 }
5101 Ok(Ok(ModuleControlRpcOutcome::UnexpectedOp { expected, actual })) => {
5102 Err(HealthProbeError::bad_answer(format!(
5103 "expected module-control op '{expected}', got '{actual}'"
5104 )))
5105 }
5106 Ok(Ok(ModuleControlRpcOutcome::DeadlineElapsed)) => Err(HealthProbeError::bad_answer(
5110 "module answered health.check after its daemon deadline",
5111 )),
5112 Ok(Err(_)) => Err(HealthProbeError::misconfigured(
5113 "health.check waiter was canceled before the module responded",
5114 )),
5115 Err(_) => {
5116 let _ = forwarding.tombstone_health_probe_rpc(endpoint, corr);
5117 Err(HealthProbeError::no_answer(format!(
5118 "module did not answer health.check within {probe_budget:?}"
5119 )))
5120 }
5121 }
5122}
5123
5124#[allow(clippy::too_many_arguments)]
5125async fn handle_health_report(
5126 spec: &ModuleSpec,
5127 runtime: &SupervisorRuntimeConfig,
5128 registry: &Registry,
5129 process_liveness: &SupervisorProcessLiveness,
5130 snapshot: &SharedSnapshot,
5131 child: &mut Option<SupervisedChild>,
5132 report: HealthReport,
5133 now_ms: u64,
5134) {
5135 let status = supervisor_health_status(report.status);
5136 let detail = report.detail.clone();
5137 let metrics = truncate_health_metrics(report.metrics);
5138 let _ = update_snapshot(snapshot, Some(&spec.module_id), |state| {
5139 state.health.status = status;
5140 state.health.last_probe_ms = Some(now_ms);
5141 state.health.detail = detail.clone();
5142 state.health.metrics = metrics.clone();
5143 state.health.consecutive_failures = 0;
5144 });
5145
5146 let action = match report.status {
5147 HealthStatus::Ok => return,
5148 HealthStatus::Degraded => runtime.health.on_degraded,
5149 HealthStatus::Failing => runtime.health.on_failing,
5150 };
5151 apply_l3_health_action(
5152 spec,
5153 runtime,
5154 registry,
5155 process_liveness,
5156 snapshot,
5157 child,
5158 status,
5159 detail.as_deref(),
5160 action,
5161 now_ms,
5162 )
5163 .await;
5164}
5165
5166#[allow(clippy::too_many_arguments)]
5167async fn handle_health_probe_failure(
5168 spec: &ModuleSpec,
5169 runtime: &SupervisorRuntimeConfig,
5170 registry: &Registry,
5171 process_liveness: &SupervisorProcessLiveness,
5172 snapshot: &SharedSnapshot,
5173 child: &mut Option<SupervisedChild>,
5174 err: HealthProbeError,
5175 now_ms: u64,
5176) {
5177 let threshold = runtime.health.failure_threshold.max(1);
5178 let mut failures = 0;
5179 let detail = format!("[{}] {err}", err.label());
5184 let _ = update_snapshot(snapshot, Some(&spec.module_id), |state| {
5185 if state.spawned_protocol.unwrap_or(spec.protocol) == ModuleProtocol::Subc {
5188 state.health.status = SupervisorHealthStatus::Unknown;
5189 }
5190 state.health.last_probe_ms = Some(now_ms);
5191 state.health.consecutive_failures = state.health.consecutive_failures.saturating_add(1);
5192 state.health.detail = Some(detail.clone());
5193 state.health.metrics = None;
5194 failures = state.health.consecutive_failures;
5195 });
5196
5197 if failures < threshold {
5198 warn!(
5199 module_id = %spec.module_id,
5200 consecutive_failures = failures,
5201 threshold,
5202 evidence = err.label(),
5203 detail = %detail,
5204 "health.check probe failed"
5205 );
5206 return;
5207 }
5208
5209 let _ = update_snapshot(snapshot, Some(&spec.module_id), |state| {
5210 state.state = ModuleState::Unresponsive;
5211 state.health.status = SupervisorHealthStatus::Unresponsive;
5212 });
5213 if runtime.health.critical {
5217 error!(
5218 module_id = %spec.module_id,
5219 status = "unresponsive",
5220 evidence = err.label(),
5221 detail = %detail,
5222 "critical module health alert"
5223 );
5224 } else {
5225 warn!(
5226 module_id = %spec.module_id,
5227 status = "unresponsive",
5228 evidence = err.label(),
5229 detail = %detail,
5230 "module health threshold breached"
5231 );
5232 }
5233 if let Err(err) = health_restart_child(
5234 spec,
5235 runtime,
5236 registry,
5237 process_liveness,
5238 snapshot,
5239 child,
5240 SupervisorHealthStatus::Unresponsive,
5241 Some(&detail),
5242 now_ms,
5243 )
5244 .await
5245 {
5246 error!(module_id = %spec.module_id, error = %err, "health-triggered restart failed");
5247 }
5248}
5249
5250#[allow(clippy::too_many_arguments)]
5251async fn apply_l3_health_action(
5252 spec: &ModuleSpec,
5253 runtime: &SupervisorRuntimeConfig,
5254 registry: &Registry,
5255 process_liveness: &SupervisorProcessLiveness,
5256 snapshot: &SharedSnapshot,
5257 child: &mut Option<SupervisedChild>,
5258 status: SupervisorHealthStatus,
5259 detail: Option<&str>,
5260 action: HealthAction,
5261 now_ms: u64,
5262) {
5263 record_health_action(snapshot, &spec.module_id, action.to_string(), now_ms);
5264 match action {
5265 HealthAction::Report => {
5266 info!(
5267 module_id = %spec.module_id,
5268 status = ?status,
5269 detail,
5270 "module reported non-ok health"
5271 );
5272 }
5273 HealthAction::Alert => {
5274 error!(
5275 module_id = %spec.module_id,
5276 status = ?status,
5277 detail,
5278 "module health alert"
5279 );
5280 }
5281 HealthAction::Restart => {
5282 if let Err(err) = health_restart_child(
5283 spec,
5284 runtime,
5285 registry,
5286 process_liveness,
5287 snapshot,
5288 child,
5289 status,
5290 detail,
5291 now_ms,
5292 )
5293 .await
5294 {
5295 error!(module_id = %spec.module_id, error = %err, "health-triggered restart failed");
5296 }
5297 }
5298 }
5299}
5300
5301#[allow(clippy::too_many_arguments)]
5302async fn health_restart_child(
5303 spec: &ModuleSpec,
5304 runtime: &SupervisorRuntimeConfig,
5305 registry: &Registry,
5306 process_liveness: &SupervisorProcessLiveness,
5307 snapshot: &SharedSnapshot,
5308 child: &mut Option<SupervisedChild>,
5309 status: SupervisorHealthStatus,
5310 detail: Option<&str>,
5311 now_ms: u64,
5312) -> Result<(), SuperviseError> {
5313 let (enabled, schedule) = {
5314 let mut state = lock_snapshot(snapshot)?;
5315 let enabled = state.enabled;
5316 let schedule = if enabled {
5317 state.next_crash_restart(&runtime.restart_policy, Instant::now())
5318 } else {
5319 None
5320 };
5321 (enabled, schedule)
5322 };
5323
5324 if !enabled {
5325 return Err(SuperviseError::Disabled {
5326 module_id: spec.module_id.clone(),
5327 });
5328 }
5329
5330 if schedule.is_none() {
5331 record_health_action(snapshot, &spec.module_id, "failed".to_string(), now_ms);
5332 error!(
5333 module_id = %spec.module_id,
5334 status = ?status,
5335 detail,
5336 max_restarts = runtime.restart_policy.max_restarts,
5337 window_secs = runtime.restart_policy.window.as_secs(),
5338 reason = %runtime.restart_policy.budget_exhausted_detail(),
5339 "health restart budget exhausted; marking module failed"
5340 );
5341 let stop_notice = begin_forwarding_drain_if_configured(
5342 spec,
5343 runtime,
5344 registry,
5345 snapshot,
5346 Some(true),
5347 RouteCloseReason::Disable,
5348 )
5349 .await?;
5350 update_snapshot(snapshot, Some(&spec.module_id), |state| {
5351 state.drain_disposition_detail = Some(runtime.restart_policy.budget_exhausted_detail());
5352 })?;
5353 drain_optional_child(
5354 &spec.module_id,
5355 spec.protocol,
5356 stop_notice,
5357 registry,
5358 runtime.forwarding.as_deref(),
5359 snapshot,
5360 &runtime.terminal_ring,
5361 &runtime.spawn_events,
5362 child,
5363 runtime.drain_timeout,
5364 ModuleState::Failed,
5365 Some(true),
5366 )
5367 .await?;
5368 process_liveness.untrack_if_current(&spec.module_id, snapshot);
5369 return Ok(());
5370 }
5371
5372 let schedule = schedule.expect("a health restart must have a crash-restart schedule");
5373 let mut restart_count = 0;
5374 update_snapshot(snapshot, Some(&spec.module_id), |state| {
5375 restart_count = state.crash_restarts.len();
5376 state.state = ModuleState::Unresponsive;
5377 state.health.status = status;
5378 state.health.last_action = Some(HealthAction::Restart.to_string());
5379 state.health.last_action_ms = Some(now_ms);
5380 })?;
5381 warn!(
5382 module_id = %spec.module_id,
5383 status = ?status,
5384 detail,
5385 restart_count,
5386 restart_in_window = schedule.restart_in_window,
5387 delay_ms = schedule.delay.as_millis() as u64,
5388 "health-triggered module restart"
5389 );
5390
5391 let stop_notice = begin_forwarding_drain_if_configured(
5392 spec,
5393 runtime,
5394 registry,
5395 snapshot,
5396 Some(true),
5397 RouteCloseReason::Restart,
5398 )
5399 .await?;
5400 drain_optional_child(
5401 &spec.module_id,
5402 spec.protocol,
5403 stop_notice,
5404 registry,
5405 runtime.forwarding.as_deref(),
5406 snapshot,
5407 &runtime.terminal_ring,
5408 &runtime.spawn_events,
5409 child,
5410 runtime.drain_timeout,
5411 ModuleState::Restarting,
5412 Some(true),
5413 )
5414 .await?;
5415 schedule_respawn(
5416 runtime,
5417 snapshot,
5418 &spec.module_id,
5419 schedule.delay,
5420 RespawnKind::Spawn,
5421 )
5422}
5423
5424fn cancel_deferred_reload(runtime: &SupervisorRuntimeConfig, module_id: &str, reason: &str) {
5425 if let Some(reply) = runtime
5426 .deferred_reload_reply
5427 .lock()
5428 .unwrap_or_else(|p| p.into_inner())
5429 .take()
5430 {
5431 let _ = reply.send(Err(SuperviseError::ReloadFailed {
5432 module_id: module_id.to_string(),
5433 reason: reason.to_string(),
5434 }));
5435 }
5436}
5437
5438fn schedule_respawn(
5439 runtime: &SupervisorRuntimeConfig,
5440 snapshot: &SharedSnapshot,
5441 module_id: &str,
5442 delay: Duration,
5443 kind: RespawnKind,
5444) -> Result<(), SuperviseError> {
5445 cancel_deferred_reload(runtime, module_id, "respawn superseded by another restart");
5446 update_snapshot(snapshot, Some(module_id), |state| {
5447 state.respawn_pending = true
5448 })?;
5449 *runtime
5450 .scheduled_respawn
5451 .lock()
5452 .unwrap_or_else(|p| p.into_inner()) = Some(PendingRespawn {
5453 deadline: Instant::now() + delay,
5454 kind,
5455 });
5456 Ok(())
5457}
5458
5459fn record_health_action(snapshot: &SharedSnapshot, module_id: &str, action: String, now_ms: u64) {
5460 let _ = update_snapshot(snapshot, Some(module_id), |state| {
5461 state.health.last_action = Some(action);
5462 state.health.last_action_ms = Some(now_ms);
5463 });
5464}
5465
5466fn supervisor_health_status(status: HealthStatus) -> SupervisorHealthStatus {
5467 match status {
5468 HealthStatus::Ok => SupervisorHealthStatus::Ok,
5469 HealthStatus::Degraded => SupervisorHealthStatus::Degraded,
5470 HealthStatus::Failing => SupervisorHealthStatus::Failing,
5471 }
5472}
5473
5474fn truncate_health_metrics(metrics: Option<Value>) -> Option<Value> {
5486 let metrics = metrics?;
5487 match serde_json::to_vec(&metrics) {
5488 Ok(encoded) if encoded.len() > MAX_HEALTH_METRICS_BYTES => Some(serde_json::json!({
5489 "truncated": true,
5490 "original_bytes": encoded.len(),
5491 })),
5492 Ok(_) | Err(_) => Some(metrics),
5493 }
5494}
5495
5496fn jittered_health_delay(module_id: &str, probe_index: u64, cadence: Duration) -> Duration {
5502 if cadence.is_zero() {
5503 return Duration::ZERO;
5504 }
5505 let cadence_ms = cadence.as_millis() as u64;
5506 if cadence_ms == 0 {
5522 return cadence;
5523 }
5524 let jitter_span = (cadence_ms / 10).max(1);
5539 let hash = module_id.as_bytes().iter().fold(
5540 probe_index.wrapping_mul(0x9E37_79B9_7F4A_7C15),
5541 |acc, byte| {
5542 acc.wrapping_mul(1099511628211)
5543 .wrapping_add(u64::from(*byte))
5544 },
5545 );
5546 cadence + Duration::from_millis(hash % jitter_span)
5547}
5548
5549#[cfg(test)]
5550mod tests {
5551 use super::*;
5552
5553 #[test]
5554 fn readding_a_module_clears_its_rescan_removal_tombstone() {
5555 let handle = SupervisorHandle::new();
5556 let module_id = "readded-tombstone";
5557 handle.record_rescan_removal(module_id);
5558 assert!(handle.removal_tombstone_age_ms(module_id).is_some());
5559
5560 handle.apply_identity_configuration(&ModuleSpec {
5561 module_id: module_id.to_string(),
5562 program: PathBuf::from("/test/module"),
5563 args: Vec::new(),
5564 env: Vec::new(),
5565 reserved: false,
5566 reserved_prefixes: Vec::new(),
5567 protocol: ModuleProtocol::Subc,
5568 overlap: Default::default(),
5569 });
5570
5571 assert!(
5572 handle.removal_tombstone_age_ms(module_id).is_none(),
5573 "a re-added module must not retain a stale removal tombstone"
5574 );
5575 }
5576
5577 #[derive(Debug, PartialEq, Eq)]
5580 struct OwnerInSpawnWindow {
5581 module_id: String,
5582 configured: bool,
5583 on_roster: bool,
5584 admission_refusal: Option<&'static str>,
5585 }
5586
5587 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
5598 async fn a_new_module_is_configured_before_its_first_process_can_register() {
5599 use crate::scopes::ScopeTable;
5600 use subc_protocol::{error_codes, scope::ScopeSelector, Principal};
5601
5602 let dir = cortexkit_test_support::ScratchDir::new("configured-before-spawn");
5603 let stub = |module_id: &str, program: PathBuf| ModuleSpec {
5604 module_id: module_id.to_string(),
5605 program,
5606 args: Vec::new(),
5607 env: ["XDG_DATA_HOME", "XDG_RUNTIME_DIR", "XDG_CONFIG_HOME"]
5608 .into_iter()
5609 .map(|key| (key.to_string(), dir.path().display().to_string()))
5610 .chain([("FAKE_AFT_NEVER_CONNECT".to_string(), "1".to_string())])
5611 .collect(),
5612 reserved: false,
5613 reserved_prefixes: Vec::new(),
5614 protocol: ModuleProtocol::Subc,
5615 overlap: Default::default(),
5616 };
5617 let live = super::terminal_history_tests::fake_aft_stub_path();
5618 let missing = dir.path().join("definitely-missing-module");
5619
5620 let handle = SupervisorHandle::new();
5621 let mut supervisor =
5622 Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default())
5623 .with_handle(handle.clone());
5624 let observed = Arc::new(Mutex::new(Vec::<OwnerInSpawnWindow>::new()));
5625 let hook_handle = handle.clone();
5626 let hook_observed = Arc::clone(&observed);
5627 supervisor.test_after_first_spawn = AfterFirstSpawnHook(Some(Arc::new(move |module_id| {
5628 let configured = hook_handle.is_configured(module_id);
5631 let selector = ScopeSelector::new(
5632 Principal::Reserved {
5633 module_id: module_id.to_string(),
5634 },
5635 "s".to_string(),
5636 Some(1),
5637 );
5638 let carrier = Principal::Reserved {
5639 module_id: "carrier".to_string(),
5640 };
5641 let admission_refusal = match ScopeTable::new(Vec::<String>::new())
5642 .admit(&carrier, module_id, &selector, configured)
5643 {
5644 Ok(_) => None,
5645 Err(refusal) => Some(refusal.code),
5646 };
5647 hook_observed.lock().unwrap().push(OwnerInSpawnWindow {
5648 module_id: module_id.to_string(),
5649 configured,
5650 on_roster: hook_handle.get(module_id).is_some(),
5651 admission_refusal,
5652 });
5653 })));
5654
5655 let plain = supervisor.spawn(stub("plain", live.clone())).unwrap();
5656 let configured = supervisor
5657 .supervise_configured(stub("configured", live.clone()), true)
5658 .unwrap();
5659 let with_health = supervisor
5660 .supervise_configured_with_health(
5661 stub("with-health", live.clone()),
5662 true,
5663 HealthConfig::default(),
5664 None,
5665 RestartPolicy::default(),
5666 )
5667 .unwrap();
5668 let failed = supervisor
5671 .supervise_configured_with_health(
5672 stub("failed-spawn", missing.clone()),
5673 true,
5674 HealthConfig::default(),
5675 None,
5676 RestartPolicy::default(),
5677 )
5678 .unwrap();
5679 assert!(supervisor.spawn(stub("spawn-error", missing)).is_err());
5682
5683 let expected = [
5684 "plain",
5685 "configured",
5686 "with-health",
5687 "failed-spawn",
5688 "spawn-error",
5689 ]
5690 .into_iter()
5691 .map(|module_id| OwnerInSpawnWindow {
5692 module_id: module_id.to_string(),
5693 configured: true,
5694 on_roster: !matches!(module_id, "plain" | "spawn-error"),
5695 admission_refusal: Some(error_codes::SCOPE_NOT_SYNCED),
5696 })
5697 .collect::<Vec<_>>();
5698 assert_eq!(*observed.lock().unwrap(), expected);
5699
5700 for module_id in ["plain", "configured", "with-health", "failed-spawn"] {
5701 assert!(
5702 handle.get(module_id).is_some(),
5703 "{module_id} is on the roster"
5704 );
5705 assert!(
5706 handle.is_configured(module_id),
5707 "{module_id} stays configured"
5708 );
5709 }
5710 assert!(handle.get("spawn-error").is_none());
5711 assert!(
5712 !handle.is_configured("spawn-error"),
5713 "a plain spawn that failed must not leave its module marked configured"
5714 );
5715
5716 handle.retire("failed-spawn");
5718 assert!(!handle.is_configured("failed-spawn"));
5719
5720 for module in [plain, configured, with_health] {
5721 module.stop().await.unwrap();
5722 }
5723 drop(failed);
5724 }
5725
5726 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
5733 async fn a_configured_module_reads_starting_while_its_first_process_already_runs() {
5734 let dir = cortexkit_test_support::ScratchDir::new("starting-while-spawned");
5735 let spec = ModuleSpec {
5736 module_id: "spawned-not-admitted".to_string(),
5737 program: super::terminal_history_tests::fake_aft_stub_path(),
5738 args: Vec::new(),
5739 env: ["XDG_DATA_HOME", "XDG_RUNTIME_DIR", "XDG_CONFIG_HOME"]
5740 .into_iter()
5741 .map(|key| (key.to_string(), dir.path().display().to_string()))
5742 .chain([("FAKE_AFT_NEVER_CONNECT".to_string(), "1".to_string())])
5743 .collect(),
5744 reserved: false,
5745 reserved_prefixes: Vec::new(),
5746 protocol: ModuleProtocol::Subc,
5747 overlap: Default::default(),
5748 };
5749 let handle = SupervisorHandle::new();
5750 let mut supervisor =
5751 Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default())
5752 .with_handle(handle.clone());
5753 let observed = Arc::new(Mutex::new(None));
5754 let hook_handle = handle.clone();
5755 let hook_observed = Arc::clone(&observed);
5756 supervisor.test_after_first_spawn = AfterFirstSpawnHook(Some(Arc::new(move |module_id| {
5757 let state = hook_handle
5758 .get(module_id)
5759 .expect("a configured module is on the roster before it is spawned")
5760 .state()
5761 .unwrap();
5762 *hook_observed.lock().unwrap() = Some(state);
5763 })));
5764
5765 let module = supervisor.supervise_configured(spec, true).unwrap();
5766 assert_eq!(
5767 *observed.lock().unwrap(),
5768 Some(ModuleState::Starting),
5769 "between the spawn and its admission the supervisor still reports starting"
5770 );
5771 assert_eq!(module.state().unwrap(), ModuleState::Running);
5772 module.stop().await.unwrap();
5773 }
5774
5775 fn stale_process_snapshot(state: ModuleState, enabled: bool) -> SharedSnapshot {
5776 let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::new(state, enabled)));
5777 update_snapshot(&snapshot, Some("stale-process-facts"), |snapshot| {
5778 snapshot.process_alive = true;
5779 snapshot.pid = Some(41);
5780 snapshot.spawned_at_ms = Some(42);
5781 snapshot.spawned_from = Some(PathBuf::from("/spawned/module"));
5782 snapshot.launch_spec_digest = Some(LaunchSpecDigest([0xA5; 32]));
5783 snapshot.spawned_file_identity = Some(SpawnedFileIdentity {
5784 device: 43,
5785 inode: 44,
5786 #[cfg(windows)]
5787 windows_image: None,
5788 });
5789 })
5790 .unwrap();
5791 snapshot
5792 }
5793
5794 fn assert_snapshot_process_facts_cleared(snapshot: &SharedSnapshot) {
5795 let snapshot = lock_snapshot(snapshot).unwrap();
5796 assert!(!snapshot.process_alive);
5797 assert_eq!(snapshot.pid, None);
5798 assert_eq!(snapshot.spawned_at_ms, None);
5799 assert_eq!(snapshot.spawned_from, None);
5800 assert_eq!(snapshot.launch_spec_digest, None);
5801 assert_eq!(snapshot.spawned_file_identity, None);
5802 }
5803
5804 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
5805 async fn failed_enable_spawn_clears_preexisting_current_process_facts() {
5806 let supervisor =
5807 Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default());
5808 let mut runtime = supervisor.runtime_config();
5809 runtime.test_seed_stale_facts_before_enable_spawn = true;
5810 let snapshot = stale_process_snapshot(ModuleState::Disabled, false);
5811 let mut child = None;
5812 let spec = ModuleSpec {
5813 module_id: "failed-enable-clears-facts".to_string(),
5814 program: PathBuf::from("/definitely/missing/failed-enable-module"),
5815 args: Vec::new(),
5816 env: Vec::new(),
5817 reserved: false,
5818 reserved_prefixes: Vec::new(),
5819 protocol: ModuleProtocol::Subc,
5820 overlap: Default::default(),
5821 };
5822
5823 let result = set_child_enabled(
5824 &spec,
5825 &runtime,
5826 &supervisor.registry,
5827 &supervisor.process_liveness,
5828 &snapshot,
5829 &mut child,
5830 true,
5831 )
5832 .await;
5833
5834 assert!(matches!(result, Err(SuperviseError::Spawn { .. })));
5835 assert_eq!(lock_snapshot(&snapshot).unwrap().state, ModuleState::Failed);
5836 assert_snapshot_process_facts_cleared(&snapshot);
5837 }
5838
5839 #[tokio::test]
5840 async fn start_revives_stranded_restarting_but_not_pending_backoff() {
5841 let supervisor =
5842 Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default());
5843 let runtime = supervisor.runtime_config();
5844 let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::new(
5845 ModuleState::Restarting,
5846 true,
5847 )));
5848 let spec = ModuleSpec {
5849 module_id: "start-stranded-restarting".to_string(),
5850 program: super::terminal_history_tests::fake_aft_stub_path(),
5851 args: Vec::new(),
5852 env: vec![("FAKE_AFT_NEVER_CONNECT".to_string(), "1".to_string())],
5853 reserved: false,
5854 reserved_prefixes: Vec::new(),
5855 protocol: ModuleProtocol::None,
5856 overlap: Default::default(),
5857 };
5858 let mut child = None;
5859 lock_snapshot(&snapshot).unwrap().respawn_pending = true;
5860 assert!(!super::set_child_enabled(
5861 &spec,
5862 &runtime,
5863 &Registry::default(),
5864 &supervisor.process_liveness,
5865 &snapshot,
5866 &mut child,
5867 true
5868 )
5869 .await
5870 .unwrap());
5871 assert!(child.is_none());
5872 lock_snapshot(&snapshot).unwrap().respawn_pending = false;
5873 assert!(super::set_child_enabled(
5874 &spec,
5875 &runtime,
5876 &Registry::default(),
5877 &supervisor.process_liveness,
5878 &snapshot,
5879 &mut child,
5880 true
5881 )
5882 .await
5883 .unwrap());
5884 assert_eq!(
5885 lock_snapshot(&snapshot).unwrap().state,
5886 ModuleState::Running
5887 );
5888 let mut child = child.unwrap();
5889 child.start_kill().unwrap();
5890 child.wait().await.unwrap();
5891 }
5892
5893 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
5894 async fn failed_reload_spawn_clears_current_process_facts() {
5895 let supervisor =
5896 Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default());
5897 let mut runtime = supervisor.runtime_config();
5898 runtime.restart_policy = RestartPolicy::new(0, Duration::ZERO);
5899 let snapshot = stale_process_snapshot(ModuleState::Running, true);
5900 let mut child = None;
5901 let spec = ModuleSpec {
5902 module_id: "failed-reload-clears-facts".to_string(),
5903 program: PathBuf::from("/unused/failed-reload-module"),
5904 args: Vec::new(),
5905 env: Vec::new(),
5906 reserved: false,
5907 reserved_prefixes: Vec::new(),
5908 protocol: ModuleProtocol::Subc,
5909 overlap: Default::default(),
5910 };
5911
5912 let result = handle_reload_spawn_failure(
5913 &spec,
5914 &runtime,
5915 &supervisor.process_liveness,
5916 &snapshot,
5917 &mut child,
5918 "forced reload spawn failure".to_string(),
5919 )
5920 .await;
5921
5922 assert!(matches!(result, Err(SuperviseError::ReloadFailed { .. })));
5923 assert_eq!(lock_snapshot(&snapshot).unwrap().state, ModuleState::Failed);
5924 assert_snapshot_process_facts_cleared(&snapshot);
5925 }
5926
5927 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
5928 async fn dropping_a_module_with_an_active_monitor_clears_current_process_facts() {
5929 let supervisor =
5930 Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default());
5931 let snapshot = stale_process_snapshot(ModuleState::Running, true);
5932 let module = supervisor.supervised_module(
5933 ModuleSpec {
5934 module_id: "drop-clears-facts".to_string(),
5935 program: PathBuf::from("/unused/drop-module"),
5936 args: Vec::new(),
5937 env: Vec::new(),
5938 reserved: false,
5939 reserved_prefixes: Vec::new(),
5940 protocol: ModuleProtocol::Subc,
5941 overlap: Default::default(),
5942 },
5943 supervisor.runtime_config(),
5944 Arc::clone(&snapshot),
5945 None,
5946 );
5947 assert!(!module
5948 .inner
5949 .monitor
5950 .lock()
5951 .unwrap()
5952 .as_ref()
5953 .unwrap()
5954 .is_finished());
5955
5956 drop(module);
5957
5958 assert_eq!(
5959 lock_snapshot(&snapshot).unwrap().state,
5960 ModuleState::Stopped
5961 );
5962 assert_snapshot_process_facts_cleared(&snapshot);
5963 }
5964
5965 #[cfg(unix)]
5966 #[tokio::test]
5967 async fn rescan_preserves_running_protocol_until_respawn() {
5968 let dir = cortexkit_test_support::ScratchDir::new("rescan-protocol");
5969 let initial = ModuleSpec {
5970 module_id: "rescan-protocol".into(),
5971 program: PathBuf::from("/bin/sleep"),
5972 args: vec!["60".into()],
5973 env: ["XDG_DATA_HOME", "XDG_RUNTIME_DIR", "XDG_CONFIG_HOME"]
5974 .into_iter()
5975 .map(|key| (key.into(), dir.path().to_string_lossy().into_owned()))
5976 .collect(),
5977 reserved: false,
5978 reserved_prefixes: vec![],
5979 protocol: ModuleProtocol::None,
5980 overlap: Default::default(),
5981 };
5982 let supervisor =
5983 Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default());
5984 let module = supervisor.spawn(initial.clone()).unwrap();
5985 assert!(module.status().unwrap().live);
5986 let mut next = initial;
5987 next.protocol = ModuleProtocol::Subc;
5988 module
5989 .update_configuration(next.clone(), HealthConfig::default(), None)
5990 .await
5991 .unwrap();
5992 assert!(
5993 module.status().unwrap().live,
5994 "rescan must not require HELLO from the old non-wire process"
5995 );
5996 let runtime = supervisor.runtime_config();
5997 let action = on_child_exit(
5998 &next,
5999 RestartPolicy::default(),
6000 &supervisor.registry,
6001 &module.inner.snapshot,
6002 &runtime.terminal_ring,
6003 &runtime.spawn_events,
6004 &runtime.child_roster,
6005 ExitReport {
6006 kind: ExitKind::Clean,
6007 code: Some(0),
6008 signal: None,
6009 at_ms: unix_ms_now(),
6010 },
6011 )
6012 .await;
6013 assert!(
6014 matches!(action, NextAction::Restart { .. }),
6015 "the old non-wire process's clean exit must restart"
6016 );
6017 module.drain().await.unwrap();
6018 }
6019
6020 #[cfg(unix)]
6021 #[tokio::test]
6022 async fn orphan_identity_matches_path_names_and_shebang_interpreters() {
6023 use std::os::unix::fs::PermissionsExt;
6024 let dir = cortexkit_test_support::ScratchDir::new("spawn-image-identity");
6025 let script = dir.join("module.sh");
6026 std::fs::write(&script, "#!/bin/sh\nwhile :; do sleep 0.1; done\n").unwrap();
6027 std::fs::set_permissions(&script, std::fs::Permissions::from_mode(0o700)).unwrap();
6028 let record_path = dir.join("live-children.json");
6029 let supervisor =
6030 Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default())
6031 .with_live_children_record(&record_path);
6032 for (program, args) in [
6033 (PathBuf::from("sleep"), vec!["60".into()]),
6034 (script, vec![]),
6035 ] {
6036 let spec = ModuleSpec {
6037 module_id: "image-identity".into(),
6038 program: program.clone(),
6039 args,
6040 env: ["XDG_DATA_HOME", "XDG_RUNTIME_DIR", "XDG_CONFIG_HOME"]
6041 .into_iter()
6042 .map(|key| (key.into(), dir.to_string_lossy().into_owned()))
6043 .collect(),
6044 reserved: false,
6045 reserved_prefixes: vec![],
6046 protocol: ModuleProtocol::None,
6047 overlap: Default::default(),
6048 };
6049 let module = supervisor.spawn(spec).unwrap();
6050 #[cfg(target_os = "macos")]
6051 {
6052 let deadline = Instant::now() + Duration::from_secs(5);
6055 while crate::live_children::read_record(&record_path)
6056 .unwrap()
6057 .iter()
6058 .all(|entry| entry.executable.is_none())
6059 {
6060 assert!(Instant::now() < deadline, "module image was not confirmed");
6061 tokio::time::sleep(Duration::from_millis(5)).await;
6062 }
6063 }
6064 let entry = crate::live_children::read_record(&record_path)
6065 .unwrap()
6066 .pop()
6067 .unwrap();
6068 let observed = subc_os::Process::open(entry.pid)
6069 .unwrap()
6070 .unwrap()
6071 .observe()
6072 .unwrap();
6073 let verdict = crate::live_children::identity_verdict(&entry, entry.pid, &observed);
6074 module.drain().await.unwrap();
6075 assert_eq!(
6076 verdict,
6077 crate::live_children::IdentityVerdict::Matches,
6078 "program {program:?}: recorded {entry:?}, observed {observed:?}"
6079 );
6080 }
6081 }
6082
6083 #[cfg(unix)]
6084 fn http_fixture(
6085 dir: &std::path::Path,
6086 url: &str,
6087 threshold: u32,
6088 ) -> crate::daemon_config::ConfiguredModule {
6089 let path = dir.join("subc.jsonc");
6090 std::fs::write(&path, serde_json::json!({"version": 1, "modules": {"http-process": {
6091 "program": "/bin/sleep", "args": ["60"], "protocol": "none",
6092 "env": {"XDG_DATA_HOME":dir,"XDG_RUNTIME_DIR":dir,"XDG_CONFIG_HOME":dir},
6093 "health": {"http":url,"cadence_ms":20,"deadline_ms":200,"failure_threshold":threshold},
6094 "restart": {"backoff_ms":1,"max_backoff_ms":1}, "drain_timeout_ms":10
6095 }}}).to_string()).unwrap();
6096 crate::daemon_config::load(&path)
6097 .unwrap()
6098 .unwrap()
6099 .modules
6100 .pop()
6101 .unwrap()
6102 }
6103
6104 #[cfg(unix)]
6105 async fn wait_http_health(module: &SupervisedModule, status: SupervisorHealthStatus) {
6106 timeout(Duration::from_secs(5), async {
6107 loop {
6108 if module.status().unwrap().health.status == status {
6109 break;
6110 }
6111 sleep(Duration::from_millis(5)).await;
6112 }
6113 })
6114 .await
6115 .unwrap_or_else(|_| {
6116 panic!(
6117 "expected {status:?}, got {:?}",
6118 module.status().unwrap().health
6119 )
6120 });
6121 }
6122
6123 #[cfg(unix)]
6124 #[tokio::test]
6125 async fn http_health_status_flips_ok_failing_ok() {
6126 use tokio::io::{AsyncReadExt, AsyncWriteExt};
6127 let dir = cortexkit_test_support::ScratchDir::new("http-status-flips");
6128 let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
6129 let status = Arc::new(std::sync::atomic::AtomicU16::new(200));
6130 let serving_status = status.clone();
6131 let url = format!("http://{}/healthz", listener.local_addr().unwrap());
6132 let server = tokio::spawn(async move {
6133 loop {
6134 let (mut stream, _) = listener.accept().await.unwrap();
6135 let mut request = [0u8; 2048];
6136 let count = stream.read(&mut request).await.unwrap();
6137 assert!(count > 0, "a probe must send an HTTP request");
6138 let code = serving_status.load(std::sync::atomic::Ordering::SeqCst);
6139 let body = if code == 200 {
6140 "ready"
6141 } else {
6142 "{\"status\":\"unavailable\",\"error\":\"scratch failure\"}"
6143 };
6144 let response = format!("HTTP/1.1 {code} scratch\r\nContent-Length: {}\r\nConnection: close\r\n\r\n{body}", body.len());
6145 let _ = stream.write_all(response.as_bytes()).await;
6146 }
6147 });
6148 let configured = http_fixture(&dir, &url, 1000);
6149 let module =
6150 Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default())
6151 .supervise_configured_with_health(
6152 configured.module_spec(),
6153 true,
6154 configured.health,
6155 configured.drain_timeout_ms,
6156 configured.restart,
6157 )
6158 .unwrap();
6159 wait_http_health(&module, SupervisorHealthStatus::Ok).await;
6160 status.store(503, std::sync::atomic::Ordering::SeqCst);
6161 wait_http_health(&module, SupervisorHealthStatus::Failing).await;
6162 assert!(module
6163 .status()
6164 .unwrap()
6165 .health
6166 .detail
6167 .unwrap()
6168 .contains("scratch failure"));
6169 status.store(200, std::sync::atomic::Ordering::SeqCst);
6170 wait_http_health(&module, SupervisorHealthStatus::Ok).await;
6171 assert_eq!(module.status().unwrap().health.consecutive_failures, 0);
6172 let before = module.status().unwrap();
6173 let (spec, mut health) = module.configuration().unwrap();
6174 health.http = None;
6175 module
6176 .update_configuration(spec.clone(), health.clone(), Some(10))
6177 .await
6178 .unwrap();
6179 wait_http_health(&module, SupervisorHealthStatus::Unknown).await;
6180 health.http = Some(url);
6181 module
6182 .update_configuration(spec, health, Some(10))
6183 .await
6184 .unwrap();
6185 wait_http_health(&module, SupervisorHealthStatus::Ok).await;
6186 assert_eq!(
6187 module.status().unwrap().pid,
6188 before.pid,
6189 "changing a probe must apply live, not restart its process"
6190 );
6191 let (spec, mut health) = module.configuration().unwrap();
6192 health.failure_threshold = 2;
6193 module
6194 .update_configuration(spec, health, Some(10))
6195 .await
6196 .unwrap();
6197 status.store(503, std::sync::atomic::Ordering::SeqCst);
6198 timeout(Duration::from_secs(5), async {
6199 while module.status().unwrap().spawn_generation == before.spawn_generation {
6200 sleep(Duration::from_millis(5)).await;
6201 }
6202 })
6203 .await
6204 .expect("sustained HTTP 503 responses must trigger the health restart policy");
6205 module.drain().await.unwrap();
6206 server.abort();
6207 }
6208
6209 #[cfg(unix)]
6210 #[tokio::test]
6211 async fn http_health_no_listener_restarts_after_consecutive_failures() {
6212 let dir = cortexkit_test_support::ScratchDir::new("http-refused");
6213 let unused = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
6214 let url = format!("http://{}/healthz", unused.local_addr().unwrap());
6215 drop(unused);
6216 let configured = http_fixture(&dir, &url, 2);
6217 let module =
6218 Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default())
6219 .supervise_configured_with_health(
6220 configured.module_spec(),
6221 true,
6222 configured.health,
6223 configured.drain_timeout_ms,
6224 configured.restart,
6225 )
6226 .unwrap();
6227 let before = module.status().unwrap().spawn_generation;
6228 timeout(Duration::from_secs(5), async {
6229 loop {
6230 let status = module.status().unwrap();
6231 if status.spawn_generation > before {
6232 assert!(status.lifetime_restarts > 0);
6233 break;
6234 }
6235 sleep(Duration::from_millis(5)).await;
6236 }
6237 })
6238 .await
6239 .expect("sustained HTTP refusal must trigger the health restart policy");
6240 module.drain().await.unwrap();
6241 }
6242
6243 #[cfg(unix)]
6244 #[tokio::test]
6245 async fn http_health_timeout_honours_deadline() {
6246 let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
6247 let url = format!("http://{}/healthz", listener.local_addr().unwrap());
6248 let server = tokio::spawn(async move {
6249 let _held = listener.accept().await.unwrap();
6250 std::future::pending::<()>().await;
6251 });
6252 let error = timeout(
6253 Duration::from_secs(1),
6254 probe_http_health(&url, Duration::from_millis(10)),
6255 )
6256 .await
6257 .expect("the probe must enforce its own deadline")
6258 .unwrap_err();
6259 server.abort();
6260 assert!(error.to_string().contains("timed out"));
6261 }
6262
6263 #[cfg(unix)]
6264 #[tokio::test]
6265 async fn http_health_localhost_reaches_an_ipv6_only_listener() {
6266 use tokio::io::{AsyncReadExt, AsyncWriteExt};
6267 let listener = tokio::net::TcpListener::bind("[::1]:0").await.unwrap();
6268 let url = format!(
6269 "http://localhost:{}/healthz",
6270 listener.local_addr().unwrap().port()
6271 );
6272 let server = tokio::spawn(async move {
6273 let (mut stream, _) = listener.accept().await.unwrap();
6274 let mut request = [0u8; 2048];
6275 assert!(stream.read(&mut request).await.unwrap() > 0);
6276 stream
6277 .write_all(b"HTTP/1.1 200 OK\r\nContent-Length: 2\r\n\r\nok")
6278 .await
6279 .unwrap();
6280 });
6281 assert_eq!(
6286 probe_http_health(&url, Duration::from_secs(10))
6287 .await
6288 .unwrap()
6289 .status,
6290 HealthStatus::Ok
6291 );
6292 server.await.unwrap();
6293 }
6294
6295 #[cfg(unix)]
6296 #[tokio::test]
6297 async fn http_health_timeout_keeps_partial_status_and_body() {
6298 use tokio::io::{AsyncReadExt, AsyncWriteExt};
6299 let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
6300 let url = format!("http://{}/healthz", listener.local_addr().unwrap());
6301 let server = tokio::spawn(async move {
6302 let (mut stream, _) = listener.accept().await.unwrap();
6303 let mut request = [0u8; 2048];
6304 assert!(stream.read(&mut request).await.unwrap() > 0);
6305 stream
6306 .write_all(
6307 b"HTTP/1.1 503 Unavailable\r\nContent-Length: 100\r\n\r\npartial diagnostic",
6308 )
6309 .await
6310 .unwrap();
6311 std::future::pending::<()>().await;
6312 });
6313 let error = probe_http_health(&url, Duration::from_secs(1))
6314 .await
6315 .unwrap_err()
6316 .to_string();
6317 server.abort();
6318 assert!(
6319 error.contains("timed out")
6320 && error.contains("503 Unavailable")
6321 && error.contains("partial diagnostic"),
6322 "{error}"
6323 );
6324 }
6325
6326 #[cfg(unix)]
6327 #[tokio::test]
6328 async fn http_health_diagnostic_body_is_bounded_to_200_bytes() {
6329 use tokio::io::{AsyncReadExt, AsyncWriteExt};
6330 let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
6331 let url = format!("http://{}/healthz", listener.local_addr().unwrap());
6332 let server = tokio::spawn(async move {
6333 let (mut stream, _) = listener.accept().await.unwrap();
6334 let mut request = [0u8; 2048];
6335 assert!(stream.read(&mut request).await.unwrap() > 0);
6336 let body = format!("{}not-in-diagnostic", "x".repeat(400));
6337 let response = format!(
6338 "HTTP/1.1 503 Unavailable\r\nContent-Length: {}\r\n\r\n{body}",
6339 body.len()
6340 );
6341 stream.write_all(response.as_bytes()).await.unwrap();
6342 });
6343 let error = probe_http_health(&url, Duration::from_secs(1))
6344 .await
6345 .unwrap_err()
6346 .to_string();
6347 server.await.unwrap();
6348 assert!(error.ends_with(&"x".repeat(200)), "{error}");
6349 assert_eq!(error.split(": ").last().unwrap().len(), 200);
6350 assert!(!error.contains("not-in-diagnostic"));
6351 }
6352
6353 #[cfg(unix)]
6354 #[tokio::test]
6355 async fn http_health_real_nats_server_monitoring() {
6356 if std::process::Command::new("nats-server")
6357 .arg("--version")
6358 .env("XDG_DATA_HOME", std::env::temp_dir())
6359 .env("XDG_RUNTIME_DIR", std::env::temp_dir())
6360 .env("XDG_CONFIG_HOME", std::env::temp_dir())
6361 .output()
6362 .is_err()
6363 {
6364 eprintln!(
6365 "skipped: http_health_real_nats_server_monitoring (nats-server is not on PATH)"
6366 );
6367 return;
6368 }
6369 let dir = cortexkit_test_support::ScratchDir::new("nats-http-monitoring");
6370 let monitor = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
6371 let port = monitor.local_addr().unwrap().port();
6372 let client = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
6373 let client_port = client.local_addr().unwrap().port();
6374 let config = dir.join("server.conf");
6375 std::fs::write(&config, format!("listen: 127.0.0.1:{client_port}\nhttp: 127.0.0.1:{port}\njetstream {{ store_dir: \"{}\" }}\n", dir.join("jetstream").display())).unwrap();
6376 let mut configured = http_fixture(&dir, &format!("http://127.0.0.1:{port}/healthz"), 1000);
6377 configured.program = PathBuf::from("nats-server");
6378 configured.args = vec!["-c".into(), config.to_string_lossy().into_owned()];
6379 drop(monitor);
6380 drop(client);
6381 let module =
6382 Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default())
6383 .supervise_configured_with_health(
6384 configured.module_spec(),
6385 true,
6386 configured.health,
6387 configured.drain_timeout_ms,
6388 configured.restart,
6389 )
6390 .unwrap();
6391 wait_http_health(&module, SupervisorHealthStatus::Ok).await;
6392 module.drain().await.unwrap();
6393 }
6394
6395 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
6396 async fn configuration_update_does_not_replace_captured_running_process_facts() {
6397 let supervisor =
6398 Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default());
6399 let snapshot = stale_process_snapshot(ModuleState::Running, true);
6400 let initial = ModuleSpec {
6401 module_id: "rescan-preserves-spawn-facts".to_string(),
6402 program: PathBuf::from("/spawned/module"),
6403 args: Vec::new(),
6404 env: Vec::new(),
6405 reserved: false,
6406 reserved_prefixes: Vec::new(),
6407 protocol: ModuleProtocol::Subc,
6408 overlap: Default::default(),
6409 };
6410 let module = supervisor.supervised_module(
6411 initial.clone(),
6412 supervisor.runtime_config(),
6413 snapshot,
6414 None,
6415 );
6416 let before = module.status().unwrap();
6417 let mut replacement = initial;
6418 replacement.program = PathBuf::from("/rescanned/replacement-module");
6419
6420 module
6421 .update_configuration(replacement, HealthConfig::default(), None)
6422 .await
6423 .unwrap();
6424
6425 let after = module.status().unwrap();
6426 assert_eq!(after.pid, before.pid);
6427 assert_eq!(after.spawned_at_ms, before.spawned_at_ms);
6428 assert_eq!(after.spawned_from, before.spawned_from);
6429 drop(module);
6430 }
6431}
6432
6433fn unix_ms_now() -> u64 {
6434 SystemTime::now()
6435 .duration_since(UNIX_EPOCH)
6436 .map(|duration| duration.as_millis().min(u128::from(u64::MAX)) as u64)
6437 .unwrap_or(0)
6438}
6439
6440async fn supervise_loop(
6441 mut spec: ModuleSpec,
6442 mut runtime: SupervisorRuntimeConfig,
6443 registry: Arc<Registry>,
6444 process_liveness: Arc<SupervisorProcessLiveness>,
6445 snapshot: SharedSnapshot,
6446 mut child: Option<SupervisedChild>,
6447 mut commands: mpsc::Receiver<SupervisorCommand>,
6448) {
6449 let mut health_probe = HealthProbeRuntime::default();
6450 let mut registration_changes = match registry.subscribe_module_changes(&spec.module_id) {
6454 Ok(changes) => Some(changes),
6455 Err(err) => {
6456 warn!(module_id = %spec.module_id, error = %err, "health prober could not subscribe to registry");
6459 None
6460 }
6461 };
6462 let mut pending_respawn: Option<PendingRespawn> = None;
6466 let mut requeued: VecDeque<SupervisorCommand> = VecDeque::new();
6469 let mut previous_instance_recheck = tokio::time::interval_at(
6470 Instant::now() + Duration::from_secs(5),
6471 Duration::from_secs(5),
6472 );
6473 previous_instance_recheck.set_missed_tick_behavior(tokio::time::MissedTickBehavior::Skip);
6474 loop {
6475 #[cfg(test)]
6476 {
6477 lock_snapshot(&snapshot).unwrap().actor_select = None;
6478 }
6479 #[cfg(target_os = "macos")]
6480 if let Some(active) = child.as_mut() {
6481 active.confirm_privacy_exec().await;
6482 }
6483 if let Some(scheduled) = runtime
6484 .scheduled_respawn
6485 .lock()
6486 .unwrap_or_else(|p| p.into_inner())
6487 .take()
6488 {
6489 pending_respawn = Some(scheduled);
6490 }
6491 if pending_respawn.is_some() && (child.is_some() || !respawn_still_pending(&snapshot)) {
6492 pending_respawn = None;
6493 cancel_deferred_reload(
6494 &runtime,
6495 &spec.module_id,
6496 "respawn cancelled by a supervisor command",
6497 );
6498 }
6499 if child.is_none()
6500 && pending_respawn.is_none()
6501 && lock_snapshot(&snapshot).is_ok_and(|state| state.waiting_for_previous_pid.is_none())
6502 {
6503 cancel_deferred_reload(
6504 &runtime,
6505 &spec.module_id,
6506 "respawn cancelled before a replacement was spawned",
6507 );
6508 let _ = update_snapshot(&snapshot, Some(&spec.module_id), |state| {
6509 state.respawn_pending = false;
6510 state.coalesced_restart_pending = false;
6511 if matches!(
6512 state.state,
6513 ModuleState::Restarting
6514 | ModuleState::Starting
6515 | ModuleState::Draining
6516 | ModuleState::Unresponsive
6517 ) {
6518 error!(module_id = %spec.module_id, state = ?state.state,
6519 "supervision operation ended without a child or pending respawn; marking failed so start can retry");
6520 state.state = ModuleState::Failed;
6521 clear_current_process_facts(state);
6522 }
6523 });
6524 }
6525 if let Some(command) = requeued.pop_front() {
6526 if !handle_supervisor_command(
6527 command,
6528 &mut spec,
6529 &mut runtime,
6530 ®istry,
6531 &process_liveness,
6532 &snapshot,
6533 &mut child,
6534 &mut commands,
6535 &mut requeued,
6536 )
6537 .await
6538 {
6539 return;
6540 }
6541 if child.is_some() || !respawn_still_pending(&snapshot) {
6542 pending_respawn = None;
6543 let _ = update_snapshot(&snapshot, Some(&spec.module_id), |state| {
6544 state.respawn_pending = false
6545 });
6546 }
6547 continue;
6548 }
6549 if child.is_some() {
6550 #[cfg(test)]
6551 {
6552 lock_snapshot(&snapshot).unwrap().actor_turns += 1;
6553 }
6554 health_probe.refresh_registration(&spec, &runtime, ®istry, &snapshot);
6555 let wake_after = health_probe.wake_after();
6556 let probe_sleep = async move {
6557 match wake_after {
6558 Some(delay) => sleep(delay).await,
6559 None => std::future::pending().await,
6560 }
6561 };
6562 tokio::pin!(probe_sleep);
6563 let wire_child = running_protocol(&spec, &snapshot) == ModuleProtocol::Subc;
6568 #[cfg(test)]
6569 {
6570 let registration_pending = wire_child
6571 && registration_changes
6572 .as_ref()
6573 .is_some_and(|changes| changes.has_changed().unwrap());
6574 let mut state = lock_snapshot(&snapshot).unwrap();
6575 state.actor_select = (!registration_pending).then_some(ActorSelectCheckpoint {
6576 generation: state.spawn_generation,
6577 turn: state.actor_turns,
6578 registered_connection: health_probe.registered_connection,
6579 next_probe_at: health_probe.next_probe_at,
6580 wake_after,
6581 });
6582 }
6583 let active_child = child.as_mut().expect("child checked above");
6584 tokio::select! {
6585 wait_result = active_child.wait() => {
6586 #[cfg(test)]
6587 { lock_snapshot(&snapshot).unwrap().actor_select = None; }
6588 let exit_report = match wait_result {
6597 Ok(status) => classify_reaped_child_exit(&snapshot, active_child, &status),
6598 Err(err) => {
6599 active_child.drain_stderr(&spec.module_id).await;
6600 fail_after_wait_error(
6601 &spec.module_id,
6602 &snapshot,
6603 &runtime.terminal_ring,
6604 &runtime.spawn_events,
6605 format!("failed to wait for supervised module: {err}"),
6606 );
6607 untrack_if_registration_released(
6608 &process_liveness,
6609 ®istry,
6610 &spec.module_id,
6611 &snapshot,
6612 );
6613 error!(module_id = %spec.module_id, error = %err, "failed to wait for supervised module");
6614 child = None;
6615 continue;
6616 }
6617 };
6618 active_child.drain_stderr(&spec.module_id).await;
6619
6620 let next = on_child_exit(
6621 &spec,
6622 runtime.restart_policy,
6623 ®istry,
6624 &snapshot,
6625 &runtime.terminal_ring,
6626 &runtime.spawn_events,
6627 &runtime.child_roster,
6628 exit_report,
6629 ).await;
6630 runtime.start_requester = None;
6631 runtime.pending_activity_restart = false;
6632 active_child.release_roster();
6635 match next {
6636 NextAction::Stop { registration_released } => {
6637 if registration_released {
6638 process_liveness.untrack_if_current(&spec.module_id, &snapshot);
6639 }
6640 child = None;
6641 }
6642 NextAction::Restart { schedule } => {
6643 let delay = schedule.map_or(
6644 runtime.restart_policy.delay_for_restart(0),
6645 |schedule| schedule.delay,
6646 );
6647 if let Some(schedule) = schedule {
6648 log_crash_respawn(&spec.module_id, schedule);
6649 }
6650 child = None;
6658 pending_respawn = Some(PendingRespawn { deadline: Instant::now() + delay, kind: RespawnKind::Spawn });
6659 let _ = update_snapshot(&snapshot, Some(&spec.module_id), |state| state.respawn_pending = true);
6660 }
6661 }
6662 }
6663 command = commands.recv() => {
6664 #[cfg(test)]
6665 { lock_snapshot(&snapshot).unwrap().actor_select = None; }
6666 let Some(command) = command else {
6667 return;
6668 };
6669 if !handle_supervisor_command(
6670 command,
6671 &mut spec,
6672 &mut runtime,
6673 ®istry,
6674 &process_liveness,
6675 &snapshot,
6676 &mut child,
6677 &mut commands,
6678 &mut requeued,
6679 ).await {
6680 return;
6681 }
6682 }
6683 _ = async {
6684 match registration_changes.as_mut() {
6685 Some(changes) => { let _ = changes.changed().await; }
6686 None => std::future::pending().await,
6687 }
6688 }, if wire_child => {
6689 #[cfg(test)]
6690 { lock_snapshot(&snapshot).unwrap().actor_select = None; }
6691 }
6692 _ = &mut probe_sleep => {
6693 #[cfg(test)]
6694 { lock_snapshot(&snapshot).unwrap().actor_select = None; }
6695 if health_probe.due() {
6696 run_health_probe_cycle(
6697 &spec,
6698 &runtime,
6699 ®istry,
6700 &process_liveness,
6701 &snapshot,
6702 &mut child,
6703 ).await;
6704 if child.is_none() {
6705 runtime.start_requester = None;
6706 runtime.pending_activity_restart = false;
6707 }
6708 if child.is_some() {
6709 health_probe.schedule_next(&spec, runtime.health.cadence);
6710 }
6711 }
6712 }
6713 }
6714 } else if let Some(pending) = pending_respawn {
6715 tokio::select! {
6716 _ = sleep_until(pending.deadline) => {
6717 pending_respawn = None;
6718 let _ = update_snapshot(&snapshot, Some(&spec.module_id), |state| state.respawn_pending = false);
6719 if !respawn_still_pending(&snapshot) {
6723 continue;
6724 }
6725 if runtime.child_roster.is_closed() {
6730 let _ = update_snapshot(&snapshot, Some(&spec.module_id), |state| {
6731 state.state = ModuleState::Stopped;
6732 });
6733 debug!(module_id = %spec.module_id, "crash respawn cancelled by daemon shutdown");
6734 continue;
6735 }
6736 if let Err(err) = release_dead_registration(
6737 ®istry,
6738 runtime.forwarding.as_deref(),
6739 &snapshot,
6740 &spec.module_id,
6741 ).await {
6742 fail_snapshot(&snapshot, Some(&spec.module_id), None);
6743 error!(module_id = %spec.module_id, error = %err, "registration did not release before restart");
6744 continue;
6745 }
6746
6747 if matches!(pending.kind, RespawnKind::Reload) {
6748 let reply = runtime.deferred_reload_reply.lock().unwrap_or_else(|p| p.into_inner()).take();
6749 let result = finish_reload_child(&spec, &runtime, ®istry, &process_liveness, &snapshot, &mut child).await;
6750 if result.is_ok() && runtime.pending_activity_restart {
6751 runtime.record_activity_transition("subc.module_restarted", &spec.module_id, runtime.start_requester.clone());
6752 }
6753 runtime.pending_activity_restart = false;
6754 if let Some(reply) = reply { let _ = reply.send(result); }
6755 continue;
6756 }
6757 process_liveness.track(spec.module_id.clone(), Arc::clone(&snapshot));
6758 match spawn_and_mark_running(&spec, &runtime, &snapshot) {
6759 Ok(next_child) => {
6760 child = Some(next_child);
6761 if runtime.pending_activity_restart {
6762 runtime.record_activity_transition("subc.module_restarted", &spec.module_id, runtime.start_requester.clone());
6763 runtime.pending_activity_restart = false;
6764 }
6765 debug!(module_id = %spec.module_id, "supervised module restarted after crash");
6766 }
6767 Err(err) => {
6768 fail_snapshot(&snapshot, Some(&spec.module_id), None);
6769 process_liveness.untrack_if_current(&spec.module_id, &snapshot);
6770 error!(module_id = %spec.module_id, error = %err, "failed to restart supervised module");
6771 }
6772 }
6773 }
6774 command = commands.recv() => {
6775 let Some(command) = command else {
6776 return;
6777 };
6778 if !handle_supervisor_command(
6779 command,
6780 &mut spec,
6781 &mut runtime,
6782 ®istry,
6783 &process_liveness,
6784 &snapshot,
6785 &mut child,
6786 &mut commands,
6787 &mut requeued,
6788 ).await {
6789 return;
6790 }
6791 if child.is_some() || !respawn_still_pending(&snapshot) {
6796 pending_respawn = None;
6797 let _ = update_snapshot(&snapshot, Some(&spec.module_id), |state| state.respawn_pending = false);
6798 }
6799 }
6800 }
6801 } else {
6802 let command = if lock_snapshot(&snapshot)
6803 .is_ok_and(|state| state.waiting_for_previous_pid.is_some())
6804 {
6805 tokio::select! {
6806 command = commands.recv() => command,
6807 _ = previous_instance_recheck.tick() => {
6808 if runtime.child_roster.previous_instance_pid(&spec.module_id).is_none() {
6809 let enabled = lock_snapshot(&snapshot).is_ok_and(|state| state.enabled);
6810 if enabled && !runtime.child_roster.is_closed() {
6811 process_liveness.track(spec.module_id.clone(), Arc::clone(&snapshot));
6812 match spawn_and_mark_running(&spec, &runtime, &snapshot) {
6813 Ok(next_child) => child = Some(next_child),
6814 Err(error) => {
6815 fail_snapshot(&snapshot, Some(&spec.module_id), None);
6816 error!(module_id = %spec.module_id, %error, "failed to start module after its previous instance exited");
6817 }
6818 }
6819 } else {
6820 let _ = update_snapshot(&snapshot, Some(&spec.module_id), |state| {
6821 state.waiting_for_previous_pid = None;
6822 state.waiting_for_previous_detail = None;
6823 });
6824 }
6825 }
6826 continue;
6827 }
6828 }
6829 } else {
6830 commands.recv().await
6831 };
6832 let Some(command) = command else {
6833 return;
6834 };
6835 if !handle_supervisor_command(
6836 command,
6837 &mut spec,
6838 &mut runtime,
6839 ®istry,
6840 &process_liveness,
6841 &snapshot,
6842 &mut child,
6843 &mut commands,
6844 &mut requeued,
6845 )
6846 .await
6847 {
6848 return;
6849 }
6850 }
6851 }
6852}
6853
6854fn log_crash_respawn(module_id: &str, schedule: CrashRestartSchedule) {
6855 info!(
6856 module_id,
6857 restart_in_window = schedule.restart_in_window,
6858 delay_ms = schedule.delay.as_millis() as u64,
6859 "respawning after crash"
6860 );
6861}
6862
6863fn respawn_still_pending(snapshot: &SharedSnapshot) -> bool {
6869 matches!(
6870 lock_snapshot(snapshot),
6871 Ok(state) if state.enabled && state.state == ModuleState::Restarting
6872 )
6873}
6874
6875enum NextAction {
6876 Stop {
6877 registration_released: bool,
6878 },
6879 Restart {
6880 schedule: Option<CrashRestartSchedule>,
6881 },
6882}
6883
6884#[allow(clippy::too_many_arguments)]
6885async fn handle_supervisor_command(
6886 command: SupervisorCommand,
6887 spec: &mut ModuleSpec,
6888 runtime: &mut SupervisorRuntimeConfig,
6889 registry: &Arc<Registry>,
6890 process_liveness: &SupervisorProcessLiveness,
6891 snapshot: &SharedSnapshot,
6892 child: &mut Option<SupervisedChild>,
6893 commands: &mut mpsc::Receiver<SupervisorCommand>,
6894 requeued: &mut VecDeque<SupervisorCommand>,
6895) -> bool {
6896 let command =
6897 if let Some((pid, reason)) = runtime.child_roster.previous_instance(&spec.module_id) {
6898 match command {
6899 SupervisorCommand::Restart { reply, .. }
6900 | SupervisorCommand::Reload { reply, .. }
6901 | SupervisorCommand::Swap { reply, .. } => {
6902 let _ = reply.send(Err(previous_instance_error(spec, pid, reason)));
6903 return true;
6904 }
6905 SupervisorCommand::SetEnabled {
6906 enabled: true,
6907 reply,
6908 ..
6909 } => {
6910 let _ = reply.send(Err(previous_instance_error(spec, pid, reason)));
6911 return true;
6912 }
6913 other => other,
6914 }
6915 } else {
6916 command
6917 };
6918 match command {
6919 SupervisorCommand::Drain { requester, reply } => {
6920 let result = drain_optional_child(
6923 &spec.module_id,
6924 spec.protocol,
6925 StopNotice::NotSent,
6926 registry,
6927 runtime.forwarding.as_deref(),
6928 snapshot,
6929 &runtime.terminal_ring,
6930 &runtime.spawn_events,
6931 child,
6932 runtime.drain_timeout,
6933 ModuleState::Stopped,
6934 None,
6935 )
6936 .await;
6937 let registration_released = result.is_ok();
6938 if registration_released {
6939 runtime.record_activity_transition(
6940 "subc.module_stopped",
6941 &spec.module_id,
6942 requester,
6943 );
6944 }
6945 let _ = reply.send(result);
6946 if registration_released {
6947 process_liveness.untrack_if_current(&spec.module_id, snapshot);
6948 }
6949 false
6950 }
6951 SupervisorCommand::Retire { requester, reply } => {
6952 let result = async {
6953 let stop_notice = begin_forwarding_drain_if_configured(
6954 spec,
6955 runtime,
6956 registry,
6957 snapshot,
6958 None,
6959 RouteCloseReason::Disable,
6960 )
6961 .await?;
6962 drain_optional_child(
6963 &spec.module_id,
6964 spec.protocol,
6965 stop_notice,
6966 registry,
6967 runtime.forwarding.as_deref(),
6968 snapshot,
6969 &runtime.terminal_ring,
6970 &runtime.spawn_events,
6971 child,
6972 runtime.drain_timeout,
6973 ModuleState::Stopped,
6974 None,
6975 )
6976 .await
6977 }
6978 .await;
6979 let registration_released = result.is_ok();
6980 if registration_released {
6981 runtime.record_activity_transition(
6982 "subc.module_stopped",
6983 &spec.module_id,
6984 requester,
6985 );
6986 }
6987 let _ = reply.send(result);
6988 if registration_released {
6989 process_liveness.untrack_if_current(&spec.module_id, snapshot);
6990 }
6991 false
6992 }
6993 SupervisorCommand::Restart {
6994 requester,
6995 drain_timeout_ms,
6996 received_at_generation,
6997 queued_at,
6998 reply,
6999 } => {
7000 info!(
7004 module_id = %spec.module_id,
7005 queued_ms = u64::try_from(queued_at.elapsed().as_millis()).unwrap_or(u64::MAX),
7006 "restart command dequeued"
7007 );
7008 let validation = match lock_snapshot(snapshot) {
7020 Ok(state) if !state.enabled => Err(SuperviseError::Disabled {
7021 module_id: spec.module_id.clone(),
7022 }),
7023 Ok(_) => Ok(()),
7024 Err(err) => Err(err),
7025 };
7026 let initiated = validation.is_ok();
7027 let _ = reply.send(validation);
7028 let satisfied_by_generation = if initiated
7042 && child
7043 .as_ref()
7044 .is_some_and(|child| child.spawned_at > queued_at)
7045 {
7046 lock_snapshot(snapshot).ok().and_then(|state| {
7047 (!state.configuration_updated_since_spawn).then_some(state.spawn_generation)
7048 })
7049 } else {
7050 None
7051 };
7052 let satisfied_by_pending = initiated
7053 && child.is_none()
7054 && lock_snapshot(snapshot).ok().is_some_and(|mut state| {
7055 let pending = state.respawn_pending && !state.configuration_updated_since_spawn;
7059 if pending {
7060 state.coalesced_restart_pending = true;
7061 }
7062 pending
7063 });
7064 if satisfied_by_pending {
7065 debug!(module_id = %spec.module_id, "restart coalesced into the pending replacement");
7066 } else if let Some(generation) = satisfied_by_generation {
7067 info!(
7068 module_id = %spec.module_id,
7069 received_at_generation,
7070 "restart already satisfied by generation {generation}; not restarting again"
7071 );
7072 } else if initiated {
7073 let drain_timeout = drain_timeout_ms
7076 .map(Duration::from_millis)
7077 .unwrap_or(runtime.drain_timeout);
7078 if let Err(err) = restart_child(
7079 spec,
7080 runtime,
7081 registry,
7082 process_liveness,
7083 snapshot,
7084 child,
7085 drain_timeout,
7086 )
7087 .await
7088 {
7089 warn!(
7090 module_id = %spec.module_id,
7091 error = %err,
7092 "operator restart failed after initiation ack; module state carries the outcome"
7093 );
7094 let _ = update_snapshot(snapshot, Some(&spec.module_id), |state| {
7095 state.state = ModuleState::Failed;
7096 clear_current_process_facts(state);
7097 });
7098 } else {
7099 runtime.start_requester = requester;
7100 runtime.pending_activity_restart = true;
7101 }
7102 }
7103 true
7104 }
7105 SupervisorCommand::Reload { requester, reply } => {
7106 let result =
7107 reload_child(spec, runtime, registry, process_liveness, snapshot, child).await;
7108 if result.is_ok() {
7109 runtime.start_requester = requester;
7110 runtime.pending_activity_restart = true;
7111 }
7112 if result.is_ok()
7113 && runtime
7114 .scheduled_respawn
7115 .lock()
7116 .unwrap_or_else(|p| p.into_inner())
7117 .as_ref()
7118 .is_some_and(|pending| matches!(pending.kind, RespawnKind::Reload))
7119 {
7120 *runtime
7121 .deferred_reload_reply
7122 .lock()
7123 .unwrap_or_else(|p| p.into_inner()) = Some(reply);
7124 } else {
7125 let _ = reply.send(result);
7126 }
7127 true
7128 }
7129 SupervisorCommand::SetEnabled {
7130 enabled,
7131 requester,
7132 reply,
7133 } => {
7134 let previous_requester = runtime.start_requester.clone();
7135 if enabled {
7136 runtime.start_requester = requester.clone();
7137 }
7138 let result = set_child_enabled(
7139 spec,
7140 runtime,
7141 registry,
7142 process_liveness,
7143 snapshot,
7144 child,
7145 enabled,
7146 )
7147 .await;
7148 if matches!(result, Ok(true)) {
7149 runtime.pending_activity_restart = false;
7150 if enabled {
7151 runtime.start_requester = requester;
7152 } else {
7153 runtime.start_requester = None;
7154 runtime.record_activity_transition(
7155 "subc.module_disabled",
7156 &spec.module_id,
7157 requester,
7158 );
7159 }
7160 } else {
7161 runtime.start_requester = previous_requester;
7162 }
7163 let _ = reply.send(result);
7164 true
7165 }
7166 SupervisorCommand::UpdateConfiguration {
7167 spec: next_spec,
7168 health,
7169 drain_timeout_ms,
7170 reply,
7171 } => {
7172 if let Some(handle) = &runtime.supervisor_handle {
7173 handle.apply_identity_configuration(&next_spec);
7174 }
7175 *spec = next_spec;
7176 let _ = update_snapshot(snapshot, Some(&spec.module_id), |state| {
7177 state.configuration_updated_since_spawn = true;
7178 });
7179 let health_changed = runtime.health != health;
7180 runtime.health = health;
7181 if health_changed {
7184 let _ = update_snapshot(snapshot, Some(&spec.module_id), |state| {
7185 state.health = ModuleHealthStatus::default();
7186 });
7187 }
7188 runtime.drain_timeout = drain_timeout_ms
7189 .map(Duration::from_millis)
7190 .unwrap_or(runtime.default_drain_timeout);
7191 *runtime
7192 .effective_drain_timeout
7193 .lock()
7194 .unwrap_or_else(|poisoned| poisoned.into_inner()) = runtime.drain_timeout;
7195 let _ = reply.send(());
7196 true
7197 }
7198 SupervisorCommand::Swap {
7199 requester,
7200 ready_timeout,
7201 reply,
7202 } => {
7203 let mut swap_runtime = runtime.clone();
7204 swap_runtime.start_requester = requester;
7205 let before = lock_snapshot(snapshot)
7206 .ok()
7207 .map(|state| state.spawn_generation);
7208 let end = swap::run_swap(
7209 spec,
7210 &swap_runtime,
7211 registry,
7212 process_liveness,
7213 snapshot,
7214 child,
7215 commands,
7216 ready_timeout.unwrap_or(DEFAULT_SWAP_READY_TIMEOUT),
7217 reply,
7218 )
7219 .await;
7220 if before
7221 != lock_snapshot(snapshot)
7222 .ok()
7223 .map(|state| state.spawn_generation)
7224 {
7225 runtime.start_requester = swap_runtime.start_requester;
7226 }
7227 requeued.extend(end.requeue);
7228 true
7229 }
7230 }
7231}
7232
7233async fn restart_child(
7234 spec: &ModuleSpec,
7235 runtime: &SupervisorRuntimeConfig,
7236 registry: &Registry,
7237 process_liveness: &SupervisorProcessLiveness,
7238 snapshot: &SharedSnapshot,
7239 child: &mut Option<SupervisedChild>,
7240 drain_timeout: Duration,
7241) -> Result<(), SuperviseError> {
7242 if !lock_snapshot(snapshot)?.enabled {
7244 return Err(SuperviseError::Disabled {
7245 module_id: spec.module_id.clone(),
7246 });
7247 }
7248 let stop_notice = begin_forwarding_drain_with_timeout(
7249 spec,
7250 runtime,
7251 registry,
7252 snapshot,
7253 None,
7254 RouteCloseReason::Restart,
7255 drain_timeout,
7256 )
7257 .await?;
7258
7259 if child.is_some() {
7260 drain_optional_child(
7261 &spec.module_id,
7262 spec.protocol,
7263 stop_notice,
7264 registry,
7265 runtime.forwarding.as_deref(),
7266 snapshot,
7267 &runtime.terminal_ring,
7268 &runtime.spawn_events,
7269 child,
7270 drain_timeout,
7271 ModuleState::Restarting,
7272 Some(true),
7273 )
7274 .await?;
7275 } else {
7276 update_snapshot(snapshot, Some(&spec.module_id), |state| {
7277 state.enabled = true;
7278 state.state = ModuleState::Restarting;
7279 clear_current_process_facts(state);
7280 })?;
7281 release_dead_registration(
7282 registry,
7283 runtime.forwarding.as_deref(),
7284 snapshot,
7285 &spec.module_id,
7286 )
7287 .await?;
7288 }
7289
7290 reset_restart_count(snapshot, &spec.module_id)?;
7291 process_liveness.track(spec.module_id.clone(), Arc::clone(snapshot));
7292 schedule_respawn(
7293 runtime,
7294 snapshot,
7295 &spec.module_id,
7296 runtime.restart_policy.backoff,
7297 RespawnKind::Spawn,
7298 )
7299}
7300
7301async fn reload_child(
7302 spec: &ModuleSpec,
7303 runtime: &SupervisorRuntimeConfig,
7304 registry: &Registry,
7305 process_liveness: &SupervisorProcessLiveness,
7306 snapshot: &SharedSnapshot,
7307 child: &mut Option<SupervisedChild>,
7308) -> Result<(), SuperviseError> {
7309 if !lock_snapshot(snapshot)?.enabled {
7311 return Err(SuperviseError::Disabled {
7312 module_id: spec.module_id.clone(),
7313 });
7314 }
7315 let stop_notice = begin_forwarding_drain(
7316 spec,
7317 runtime,
7318 registry,
7319 snapshot,
7320 Some(true),
7321 RouteCloseReason::Reload,
7322 )
7323 .await?;
7324
7325 if child.is_some() {
7326 drain_optional_child(
7327 &spec.module_id,
7328 spec.protocol,
7329 stop_notice,
7330 registry,
7331 runtime.forwarding.as_deref(),
7332 snapshot,
7333 &runtime.terminal_ring,
7334 &runtime.spawn_events,
7335 child,
7336 runtime.drain_timeout,
7337 ModuleState::Restarting,
7338 Some(true),
7339 )
7340 .await?;
7341 } else {
7342 update_snapshot(snapshot, Some(&spec.module_id), |state| {
7343 state.enabled = true;
7344 state.state = ModuleState::Restarting;
7345 clear_current_process_facts(state);
7346 })?;
7347 release_dead_registration(
7348 registry,
7349 runtime.forwarding.as_deref(),
7350 snapshot,
7351 &spec.module_id,
7352 )
7353 .await?;
7354 }
7355
7356 reset_restart_count(snapshot, &spec.module_id)?;
7357 process_liveness.track(spec.module_id.clone(), Arc::clone(snapshot));
7358 schedule_respawn(
7359 runtime,
7360 snapshot,
7361 &spec.module_id,
7362 runtime.restart_policy.backoff,
7363 RespawnKind::Reload,
7364 )
7365}
7366
7367async fn finish_reload_child(
7368 spec: &ModuleSpec,
7369 runtime: &SupervisorRuntimeConfig,
7370 registry: &Registry,
7371 process_liveness: &SupervisorProcessLiveness,
7372 snapshot: &SharedSnapshot,
7373 child: &mut Option<SupervisedChild>,
7374) -> Result<(), SuperviseError> {
7375 process_liveness.track(spec.module_id.clone(), Arc::clone(snapshot));
7376 let next_child = match spawn_and_mark_running(spec, runtime, snapshot) {
7377 Ok(next_child) => next_child,
7378 Err(err) => {
7379 return handle_reload_spawn_failure(
7380 spec,
7381 runtime,
7382 process_liveness,
7383 snapshot,
7384 child,
7385 format!("new child failed to spawn: {err}"),
7386 )
7387 .await;
7388 }
7389 };
7390 *child = Some(next_child);
7391
7392 let wait_outcome = {
7393 let active_child = child.as_mut().expect("new reload child was just stored");
7394 wait_for_registration_after_reload(
7395 registry,
7396 &spec.module_id,
7397 snapshot,
7398 active_child,
7399 REGISTRY_RELEASE_TIMEOUT,
7400 )
7401 .await?
7402 };
7403
7404 match wait_outcome {
7405 RegistrationWaitOutcome::Registered => {
7406 debug!(module_id = %spec.module_id, "supervised module reloaded and registered");
7407 Ok(())
7408 }
7409 RegistrationWaitOutcome::Exited(exit_report) => {
7410 if let Some(active_child) = child.as_mut() {
7411 active_child.drain_stderr(&spec.module_id).await;
7412 }
7413 let mut exited_child = child.take().expect("exited reload child is still stored");
7416 #[cfg(test)]
7417 if let Some(gate) = &runtime.test_reload_exit_record_gate {
7418 gate.reached.notify_one();
7419 gate.resume.notified().await;
7420 }
7421 let result = handle_reload_child_registration_failure(
7422 spec,
7423 runtime,
7424 registry,
7425 process_liveness,
7426 snapshot,
7427 child,
7428 ReloadRegistrationFailure {
7429 exit_report: registration_failure_exit_report(exit_report),
7430 reason: exited_child
7431 .spawn_failure
7432 .clone()
7433 .unwrap_or_else(|| "new child exited before registering".to_string()),
7434 },
7435 )
7436 .await;
7437 exited_child.release_roster();
7438 result
7439 }
7440 RegistrationWaitOutcome::TimedOut => {
7441 let mut timed_out_child = child
7442 .take()
7443 .expect("timed-out reload child is still running");
7444 timed_out_child
7445 .start_kill()
7446 .map_err(|source| SuperviseError::Kill {
7447 module_id: spec.module_id.clone(),
7448 source,
7449 })?;
7450 let status = timed_out_child
7451 .wait()
7452 .await
7453 .map_err(|source| SuperviseError::Wait {
7454 module_id: spec.module_id.clone(),
7455 source,
7456 })?;
7457 timed_out_child.drain_stderr(&spec.module_id).await;
7458 handle_reload_child_registration_failure(
7459 spec,
7460 runtime,
7461 registry,
7462 process_liveness,
7463 snapshot,
7464 child,
7465 ReloadRegistrationFailure {
7466 exit_report: registration_failure_exit_report(classify_reaped_child_exit(
7467 snapshot,
7468 &timed_out_child,
7469 &status,
7470 )),
7471 reason: format!(
7472 "new child did not register within {:?}",
7473 REGISTRY_RELEASE_TIMEOUT
7474 ),
7475 },
7476 )
7477 .await
7478 }
7479 }
7480}
7481
7482async fn set_child_enabled(
7483 spec: &ModuleSpec,
7484 runtime: &SupervisorRuntimeConfig,
7485 registry: &Registry,
7486 process_liveness: &SupervisorProcessLiveness,
7487 snapshot: &SharedSnapshot,
7488 child: &mut Option<SupervisedChild>,
7489 enabled: bool,
7490) -> Result<bool, SuperviseError> {
7491 let (current_enabled, current_state, respawn_pending) = {
7492 let state = lock_snapshot(snapshot)?;
7493 (state.enabled, state.state, state.respawn_pending)
7494 };
7495 let revive_terminal = enabled
7503 && current_enabled
7504 && child.is_none()
7505 && (matches!(current_state, ModuleState::Failed | ModuleState::Stopped)
7506 || (current_state == ModuleState::Restarting && !respawn_pending));
7507 if current_enabled == enabled && !revive_terminal {
7508 return Ok(false);
7509 }
7510
7511 if enabled {
7512 update_snapshot(snapshot, Some(&spec.module_id), |state| {
7513 state.enabled = true;
7514 state.state = ModuleState::Starting;
7515 clear_current_process_facts(state);
7516 })?;
7517 #[cfg(test)]
7518 if runtime.test_seed_stale_facts_before_enable_spawn {
7519 update_snapshot(snapshot, Some(&spec.module_id), |state| {
7520 state.process_alive = true;
7521 state.pid = Some(41);
7522 state.spawned_at_ms = Some(42);
7523 state.spawned_from = Some(PathBuf::from("/spawned/module"));
7524 state.spawned_file_identity = Some(SpawnedFileIdentity {
7525 device: 43,
7526 inode: 44,
7527 #[cfg(windows)]
7528 windows_image: None,
7529 });
7530 })?;
7531 }
7532 release_dead_registration(
7533 registry,
7534 runtime.forwarding.as_deref(),
7535 snapshot,
7536 &spec.module_id,
7537 )
7538 .await?;
7539 reset_restart_count(snapshot, &spec.module_id)?;
7540 process_liveness.track(spec.module_id.clone(), Arc::clone(snapshot));
7541 let next_child = match spawn_and_mark_running(spec, runtime, snapshot) {
7542 Ok(next_child) => next_child,
7543 Err(err) => {
7544 if let Err(state_err) = update_snapshot(snapshot, Some(&spec.module_id), |state| {
7545 state.state = ModuleState::Failed;
7546 clear_current_process_facts(state);
7547 }) {
7548 error!(module_id = %spec.module_id, error = %state_err, "failed to record enable spawn failure");
7549 }
7550 process_liveness.untrack_if_current(&spec.module_id, snapshot);
7551 return Err(err);
7552 }
7553 };
7554 *child = Some(next_child);
7555 debug!(module_id = %spec.module_id, "supervised module enabled");
7556 Ok(true)
7557 } else {
7558 let stop_notice = begin_forwarding_drain_if_configured(
7559 spec,
7560 runtime,
7561 registry,
7562 snapshot,
7563 Some(false),
7564 RouteCloseReason::Disable,
7565 )
7566 .await?;
7567 drain_optional_child(
7568 &spec.module_id,
7569 spec.protocol,
7570 stop_notice,
7571 registry,
7572 runtime.forwarding.as_deref(),
7573 snapshot,
7574 &runtime.terminal_ring,
7575 &runtime.spawn_events,
7576 child,
7577 runtime.drain_timeout,
7578 ModuleState::Disabled,
7579 Some(false),
7580 )
7581 .await?;
7582 debug!(module_id = %spec.module_id, "supervised module disabled");
7583 Ok(true)
7584 }
7585}
7586
7587#[allow(clippy::too_many_arguments)]
7588async fn on_child_exit(
7589 spec: &ModuleSpec,
7590 policy: RestartPolicy,
7591 registry: &Registry,
7592 snapshot: &SharedSnapshot,
7593 terminal_ring: &Arc<Mutex<TerminalRing>>,
7594 spawn_events: &SpawnEventFeed,
7595 roster: &ChildRoster,
7596 exit_report: ExitReport,
7597) -> NextAction {
7598 if roster.is_closed() {
7604 return on_child_exit_during_daemon_shutdown(
7605 spec,
7606 registry,
7607 snapshot,
7608 terminal_ring,
7609 spawn_events,
7610 exit_report,
7611 )
7612 .await;
7613 }
7614 let unrequested_clean_exit_of_protocol_none = exit_report.kind == ExitKind::Clean
7629 && running_protocol(spec, snapshot) == ModuleProtocol::None;
7630 match exit_report.kind {
7631 ExitKind::Clean if !unrequested_clean_exit_of_protocol_none => {
7632 info!(
7633 module_id = %spec.module_id,
7634 exit_code = ?exit_report.code,
7635 exit_signal = ?exit_report.signal,
7636 "supervised module exited cleanly"
7637 );
7638 if let Err(err) = publish_after_terminal_record(
7639 &spec.module_id,
7640 || {
7641 record_terminal(
7642 &spec.module_id,
7643 terminal_ring,
7644 spawn_events,
7645 &exit_report,
7646 TerminalDisposition::Stopped,
7647 )
7648 },
7649 || {
7650 update_snapshot(snapshot, Some(&spec.module_id), |state| {
7651 state.state = ModuleState::Stopped;
7652 clear_current_process_facts(state);
7653 state.last_exit = Some(exit_report.clone());
7654 })
7655 },
7656 ) {
7657 error!(module_id = %spec.module_id, error = %err, "failed to publish clean module exit");
7658 }
7659 let registration_released = match wait_for_registration_release(
7660 registry,
7661 &spec.module_id,
7662 REGISTRY_RELEASE_TIMEOUT,
7663 )
7664 .await
7665 {
7666 Ok(()) => true,
7667 Err(err) => {
7668 warn!(module_id = %spec.module_id, error = %err, "registration still active after clean exit");
7669 false
7670 }
7671 };
7672 NextAction::Stop {
7673 registration_released,
7674 }
7675 }
7676 ExitKind::Clean | ExitKind::Crash => {
7677 if unrequested_clean_exit_of_protocol_none {
7678 warn!(
7679 module_id = %spec.module_id,
7680 exit_code = ?exit_report.code,
7681 exit_signal = ?exit_report.signal,
7682 "protocol-none module exited cleanly without a stop request; handling it as a crash"
7683 );
7684 } else {
7685 warn!(
7686 module_id = %spec.module_id,
7687 exit_code = ?exit_report.code,
7688 exit_signal = ?exit_report.signal,
7689 "supervised module exited abnormally (crash)"
7690 );
7691 }
7692 let mut restart_schedule = None;
7693 let mut disposition = TerminalDisposition::Disabled;
7694 let mut final_state = ModuleState::Disabled;
7695 let mut disposition_detail = lock_snapshot(snapshot)
7699 .ok()
7700 .and_then(|mut state| state.spawn_failure.take());
7701 let now = Instant::now();
7702 if let Err(err) = update_snapshot(snapshot, Some(&spec.module_id), |state| {
7703 if state.enabled {
7704 if let Some(schedule) = state.next_crash_restart(&policy, now) {
7705 state.respawn_pending = true;
7711 final_state = ModuleState::Restarting;
7712 restart_schedule = Some(schedule);
7713 disposition = TerminalDisposition::Restarting;
7714 } else {
7715 final_state = ModuleState::Failed;
7716 disposition = TerminalDisposition::Failed;
7717 let budget = policy.budget_exhausted_detail();
7718 disposition_detail =
7719 Some(disposition_detail.take().map_or_else(
7720 || budget.clone(),
7721 |cause| format!("{cause}; {budget}"),
7722 ));
7723 }
7724 } else {
7725 disposition = TerminalDisposition::Disabled;
7726 }
7727 }) {
7728 error!(module_id = %spec.module_id, error = %err, "failed to record crashed module exit");
7729 return NextAction::Stop {
7730 registration_released: false,
7731 };
7732 }
7733 if disposition == TerminalDisposition::Failed {
7734 error!(
7739 module_id = %spec.module_id,
7740 max_restarts = policy.max_restarts,
7741 window_secs = policy.window.as_secs(),
7742 "module stopped: {}",
7743 policy.budget_exhausted_detail()
7744 );
7745 }
7746 if let Err(err) = publish_after_terminal_record(
7747 &spec.module_id,
7748 || {
7749 record_terminal_with_detail(
7750 &spec.module_id,
7751 terminal_ring,
7752 spawn_events,
7753 &exit_report,
7754 disposition,
7755 disposition_detail,
7756 )
7757 },
7758 || {
7759 update_snapshot(snapshot, Some(&spec.module_id), |state| {
7760 state.state = final_state;
7761 clear_current_process_facts(state);
7762 state.last_exit = Some(exit_report.clone());
7763 })
7764 },
7765 ) {
7766 error!(module_id = %spec.module_id, error = %err, "failed to publish crashed module exit");
7767 }
7768
7769 if let Some(schedule) = restart_schedule {
7770 NextAction::Restart {
7771 schedule: Some(schedule),
7772 }
7773 } else {
7774 let registration_released = match wait_for_registration_release(
7775 registry,
7776 &spec.module_id,
7777 REGISTRY_RELEASE_TIMEOUT,
7778 )
7779 .await
7780 {
7781 Ok(()) => true,
7782 Err(err) => {
7783 warn!(module_id = %spec.module_id, error = %err, "registration still active after failed module");
7784 false
7785 }
7786 };
7787 NextAction::Stop {
7788 registration_released,
7789 }
7790 }
7791 }
7792 ExitKind::DeliberateSeverance => {
7793 warn!(
7794 module_id = %spec.module_id,
7795 exit_code = ?exit_report.code,
7796 exit_signal = ?exit_report.signal,
7797 "supervised module exited after deliberate connection severance"
7798 );
7799 let should_restart = match lock_snapshot(snapshot) {
7800 Ok(state) => state.enabled,
7801 Err(err) => {
7802 error!(module_id = %spec.module_id, error = %err, "failed to read deliberately severed module state");
7803 return NextAction::Stop {
7804 registration_released: false,
7805 };
7806 }
7807 };
7808 let final_state = if should_restart {
7809 ModuleState::Restarting
7810 } else {
7811 ModuleState::Disabled
7812 };
7813 if let Err(err) = publish_after_terminal_record(
7814 &spec.module_id,
7815 || {
7816 record_terminal(
7817 &spec.module_id,
7818 terminal_ring,
7819 spawn_events,
7820 &exit_report,
7821 terminal_disposition(final_state),
7822 )
7823 },
7824 || {
7825 update_snapshot(snapshot, Some(&spec.module_id), |state| {
7826 state.state = final_state;
7827 clear_current_process_facts(state);
7828 state.last_exit = Some(exit_report.clone());
7829 state.lifetime_restarts += 1;
7830 })
7831 },
7832 ) {
7833 error!(module_id = %spec.module_id, error = %err, "failed to publish deliberately severed module exit");
7834 return NextAction::Stop {
7835 registration_released: false,
7836 };
7837 }
7838
7839 if should_restart {
7840 NextAction::Restart { schedule: None }
7841 } else {
7842 let registration_released = match wait_for_registration_release(
7843 registry,
7844 &spec.module_id,
7845 REGISTRY_RELEASE_TIMEOUT,
7846 )
7847 .await
7848 {
7849 Ok(()) => true,
7850 Err(err) => {
7851 warn!(module_id = %spec.module_id, error = %err, "registration still active after deliberately severed module exit");
7852 false
7853 }
7854 };
7855 NextAction::Stop {
7856 registration_released,
7857 }
7858 }
7859 }
7860 }
7861}
7862
7863async fn on_child_exit_during_daemon_shutdown(
7864 spec: &ModuleSpec,
7865 registry: &Registry,
7866 snapshot: &SharedSnapshot,
7867 terminal_ring: &Arc<Mutex<TerminalRing>>,
7868 spawn_events: &SpawnEventFeed,
7869 exit_report: ExitReport,
7870) -> NextAction {
7871 info!(
7872 module_id = %spec.module_id,
7873 exit_code = ?exit_report.code,
7874 exit_signal = ?exit_report.signal,
7875 exit_kind = ?exit_report.kind,
7876 "supervised module exited during daemon shutdown; not restarting it"
7877 );
7878 if let Err(err) = publish_after_terminal_record(
7879 &spec.module_id,
7880 || {
7881 record_terminal(
7882 &spec.module_id,
7883 terminal_ring,
7884 spawn_events,
7885 &exit_report,
7886 TerminalDisposition::DaemonShutdown,
7887 )
7888 },
7889 || {
7890 update_snapshot(snapshot, Some(&spec.module_id), |state| {
7891 state.state = ModuleState::Stopped;
7892 clear_current_process_facts(state);
7893 state.last_exit = Some(exit_report.clone());
7894 })
7895 },
7896 ) {
7897 error!(module_id = %spec.module_id, error = %err, "failed to publish module exit during daemon shutdown");
7898 }
7899 let registration_released =
7900 wait_for_registration_release(registry, &spec.module_id, REGISTRY_RELEASE_TIMEOUT)
7901 .await
7902 .is_ok();
7903 NextAction::Stop {
7904 registration_released,
7905 }
7906}
7907
7908fn publish_after_terminal_record<T>(
7913 module_id: &str,
7914 record: impl FnOnce(),
7915 publish: impl FnOnce() -> T,
7916) -> T {
7917 let recorded = std::panic::catch_unwind(std::panic::AssertUnwindSafe(record));
7918 let published = publish();
7919 if let Err(panic) = recorded {
7920 error!(
7921 module_id,
7922 "terminal recording panicked; published final state before resuming unwind"
7923 );
7924 std::panic::resume_unwind(panic);
7925 }
7926 published
7927}
7928
7929fn fail_after_wait_error(
7930 module_id: &str,
7931 snapshot: &SharedSnapshot,
7932 terminal_ring: &Arc<Mutex<TerminalRing>>,
7933 spawn_events: &SpawnEventFeed,
7934 detail: String,
7935) {
7936 let report = wait_error_exit_report();
7937 publish_after_terminal_record(
7938 module_id,
7939 || {
7940 record_terminal_with_detail(
7941 module_id,
7942 terminal_ring,
7943 spawn_events,
7944 &report,
7945 TerminalDisposition::Failed,
7946 Some(detail),
7947 )
7948 },
7949 || fail_snapshot(snapshot, Some(module_id), Some(report.clone())),
7950 );
7951}
7952
7953#[cfg(test)]
7954fn record_wait_error_terminal(
7955 module_id: &str,
7956 terminal_ring: &Arc<Mutex<TerminalRing>>,
7957 spawn_events: &SpawnEventFeed,
7958) {
7959 record_terminal(
7960 module_id,
7961 terminal_ring,
7962 spawn_events,
7963 &wait_error_exit_report(),
7964 TerminalDisposition::Failed,
7965 );
7966}
7967
7968fn record_terminal(
7969 module_id: &str,
7970 terminal_ring: &Arc<Mutex<TerminalRing>>,
7971 spawn_events: &SpawnEventFeed,
7972 exit_report: &ExitReport,
7973 disposition: TerminalDisposition,
7974) {
7975 record_terminal_with_detail(
7976 module_id,
7977 terminal_ring,
7978 spawn_events,
7979 exit_report,
7980 disposition,
7981 None,
7982 );
7983}
7984
7985fn durable_terminal_history_of(
7989 terminal_ring: &Mutex<TerminalRing>,
7990 module_id: &str,
7991) -> subc_control::TerminalHistory {
7992 let read = terminal_ring
7993 .lock()
7994 .unwrap_or_else(|p| p.into_inner())
7995 .capture_durable_history();
7996 read.read(module_id)
7997}
7998
7999fn record_terminal_with_detail(
8000 module_id: &str,
8001 terminal_ring: &Arc<Mutex<TerminalRing>>,
8002 spawn_events: &SpawnEventFeed,
8003 exit_report: &ExitReport,
8004 disposition: TerminalDisposition,
8005 disposition_detail: Option<String>,
8006) {
8007 #[cfg(test)]
8008 terminal_history_tests::before_terminal_record(spawn_events);
8009 spawn_events.emit_exited(module_id, exit_report.code, exit_report.signal);
8010 let record = TerminalRecord {
8011 exit_code: exit_report.code,
8012 exit_signal: exit_report.signal,
8013 at_ms: exit_report.at_ms,
8014 disposition,
8015 exit_kind: exit_report.kind.into(),
8016 disposition_detail,
8017 };
8018 terminal_ring
8019 .lock()
8020 .unwrap_or_else(|poisoned| poisoned.into_inner())
8021 .record_exit(module_id, record);
8022}
8023
8024fn untrack_if_registration_released(
8025 process_liveness: &SupervisorProcessLiveness,
8026 registry: &Registry,
8027 module_id: &str,
8028 snapshot: &SharedSnapshot,
8029) {
8030 match registry.get_module(module_id) {
8031 Ok(None) => process_liveness.untrack_if_current(module_id, snapshot),
8032 Ok(Some(_)) => {}
8033 Err(err) => {
8034 warn!(module_id, error = %err, "could not determine whether supervisor liveness can be untracked");
8035 }
8036 }
8037}
8038
8039#[cfg(test)]
8053fn apply_wire_spawn_args(
8054 command: &mut Command,
8055 spec: &ModuleSpec,
8056 connection_file_path: Option<&std::path::Path>,
8057 handle: Option<&SupervisorHandle>,
8058) -> Result<Option<NonceHandoff>, SuperviseError> {
8059 apply_wire_spawn_args_for_role(
8060 command,
8061 spec,
8062 connection_file_path,
8063 handle,
8064 SpawnRole::Plain,
8065 )
8066}
8067
8068#[cfg(unix)]
8073type NonceHandoff = subc_os::LaunchNonceHandoff;
8074#[cfg(windows)]
8075type NonceHandoff = subc_os::LaunchNoncePipeHandoff;
8076#[cfg(not(any(unix, windows)))]
8077type NonceHandoff = std::convert::Infallible;
8078
8079fn apply_wire_spawn_args_for_role(
8100 command: &mut Command,
8101 spec: &ModuleSpec,
8102 connection_file_path: Option<&std::path::Path>,
8103 handle: Option<&SupervisorHandle>,
8104 role: SpawnRole,
8105) -> Result<Option<NonceHandoff>, SuperviseError> {
8106 command.env(SUBC_MODULE_ID_ENV, &spec.module_id);
8107 command.env_remove(subc_os::LAUNCH_NONCE_FD_ENV);
8112 #[cfg(windows)]
8113 {
8114 command.env_remove(subc_os::LAUNCH_NONCE_PIPE_ENV);
8115 command.env_remove(subc_os::LAUNCH_NONCE_PIPE_FALLBACK_ENV);
8116 }
8117 command.env_remove(SUBC_LAUNCH_NONCE_ENV);
8119 if spec.protocol == ModuleProtocol::None {
8120 return Ok(None);
8121 }
8122 if let Some(connection_file_path) = connection_file_path {
8123 command.arg(SUBC_ARG).arg(connection_file_path);
8124 }
8125
8126 let nonce = generate_launch_nonce()?;
8130 if let Some(handle) = handle {
8131 match role {
8132 SpawnRole::Plain => {
8133 handle.set_spawn_nonce(&spec.module_id, nonce.clone());
8134 if spec.reserved {
8135 handle.set_reserved_nonce(&spec.module_id, nonce.clone());
8136 }
8137 }
8138 SpawnRole::SwapCandidate => handle.open_swap(&spec.module_id, nonce.clone()),
8139 }
8140 }
8141 #[cfg(unix)]
8142 let handoff = {
8143 let handoff =
8144 subc_os::LaunchNonceHandoff::new(&nonce).map_err(|source| SuperviseError::Spawn {
8145 program: spec.program.clone(),
8146 source,
8147 cgroup_path: None,
8148 })?;
8149 command.env(subc_os::LAUNCH_NONCE_FD_ENV, handoff.fd_env_value());
8150 Some(handoff)
8151 };
8152 #[cfg(windows)]
8153 let handoff = prepare_windows_nonce_pipe(command, spec, &nonce);
8154 #[cfg(not(any(unix, windows)))]
8155 let handoff = None;
8156 #[cfg(not(unix))]
8160 command.env(SUBC_LAUNCH_NONCE_ENV, nonce);
8161 Ok(handoff)
8162}
8163
8164fn apply_child_env(command: &mut Command, spec: &ModuleSpec) {
8165 command.env_remove(CK_LOG_ENV);
8166 command.env_remove(SUBC_SPAWN_ROLE_ENV);
8173 for (key, value) in &spec.env {
8174 if matches!(
8178 key.as_str(),
8179 CAPTURE_MAX_FILE_MB_ENV | CAPTURE_KEEP_ENV | CAPTURE_MAX_AGE_DAYS_ENV
8180 ) || key == SUBC_SPAWN_ROLE_ENV
8181 {
8182 continue;
8183 }
8184 command.env(key, value);
8185 }
8186}
8187
8188#[cfg(windows)]
8189fn prepare_windows_nonce_pipe(
8190 command: &mut Command,
8191 spec: &ModuleSpec,
8192 nonce: &str,
8193) -> Option<NonceHandoff> {
8194 match subc_os::LaunchNoncePipeHandoff::new(nonce) {
8195 Ok(handoff) => {
8196 command.env(subc_os::LAUNCH_NONCE_PIPE_ENV, handoff.name());
8197 command.env(subc_os::LAUNCH_NONCE_PIPE_FALLBACK_ENV, "env");
8202 Some(handoff)
8203 }
8204 Err(error) => {
8205 warn!(module_id = %spec.module_id, %error, "could not prepare launch nonce pipe; retaining the environment handoff");
8206 None
8207 }
8208 }
8209}
8210
8211#[cfg(windows)]
8212fn start_windows_nonce_pipe(
8213 handoff: Option<NonceHandoff>,
8214 pid: u32,
8215 spec: &ModuleSpec,
8216) -> Option<subc_os::LaunchNoncePipeDelivery> {
8217 handoff.and_then(|handoff| match handoff.serve(pid, REGISTRY_RELEASE_TIMEOUT) {
8221 Ok(delivery) => Some(delivery),
8222 Err(error) => {
8223 warn!(module_id = %spec.module_id, %error, "could not serve launch nonce pipe; retaining the environment handoff");
8224 None
8225 }
8226 })
8227}
8228
8229fn observed_launch_nonce_source(snapshot: &SupervisorSnapshot) -> Option<String> {
8230 #[cfg(windows)]
8231 if snapshot.process_alive && snapshot.spawned_protocol == Some(ModuleProtocol::Subc) {
8232 return Some(
8233 if snapshot
8234 .pipe_consumed
8235 .as_ref()
8236 .is_some_and(|consumed| consumed.get().is_some())
8237 {
8238 "pipe"
8239 } else {
8240 "env"
8241 }
8242 .to_owned(),
8243 );
8244 }
8245 let _ = snapshot;
8246 None
8247}
8248
8249#[derive(Debug, Clone, Copy, PartialEq, Eq)]
8252enum SpawnRole {
8253 Plain,
8254 SwapCandidate,
8255}
8256
8257fn apply_spawn_role(command: &mut Command, role: SpawnRole) {
8260 if role == SpawnRole::SwapCandidate {
8261 command.env(SUBC_SPAWN_ROLE_ENV, SPAWN_ROLE_SWAP_CANDIDATE);
8262 }
8263}
8264
8265fn previous_instance_error(
8266 spec: &ModuleSpec,
8267 pid: u32,
8268 reason: crate::live_children::RetainedReason,
8269) -> SuperviseError {
8270 SuperviseError::Spawn {
8271 program: spec.program.clone(),
8272 source: io::Error::new(io::ErrorKind::WouldBlock, reason.waiting_message(pid)),
8273 cgroup_path: None,
8274 }
8275}
8276
8277fn spawn_child(
8278 spec: &ModuleSpec,
8279 connection_file_path: Option<&std::path::Path>,
8280 handle: Option<&SupervisorHandle>,
8281 ring: &Arc<Mutex<StderrRing>>,
8282 capture_logs_dir: Option<&std::path::Path>,
8283 roster: &ChildRoster,
8284 #[cfg(target_os = "linux")] cgroup_placement: Option<&subc_cgroup::Placement>,
8285) -> Result<SupervisedChild, SuperviseError> {
8286 spawn_child_in_slot(
8287 spec,
8288 connection_file_path,
8289 handle,
8290 ring,
8291 capture_logs_dir,
8292 roster,
8293 #[cfg(target_os = "linux")]
8294 cgroup_placement,
8295 SpawnRole::Plain,
8296 false,
8297 )
8298}
8299
8300#[allow(clippy::too_many_arguments)]
8313fn spawn_child_in_slot(
8314 spec: &ModuleSpec,
8315 connection_file_path: Option<&std::path::Path>,
8316 handle: Option<&SupervisorHandle>,
8317 ring: &Arc<Mutex<StderrRing>>,
8318 capture_logs_dir: Option<&std::path::Path>,
8319 roster: &ChildRoster,
8320 #[cfg(target_os = "linux")] cgroup_placement: Option<&subc_cgroup::Placement>,
8321 role: SpawnRole,
8322 alternate_slot: bool,
8323) -> Result<SupervisedChild, SuperviseError> {
8324 if let Some((pid, reason)) = roster.previous_instance(&spec.module_id) {
8325 return Err(previous_instance_error(spec, pid, reason));
8326 }
8327 if roster.is_closed() {
8328 return Err(SuperviseError::Spawn {
8329 program: spec.program.clone(),
8330 source: io::Error::other("the daemon is shutting down; not starting a new process"),
8331 cgroup_path: None,
8332 });
8333 }
8334 #[cfg(target_os = "linux")]
8335 let cgroup_name = {
8336 if cgroup_placement.is_none() {
8341 swap::cgroup_name(&spec.module_id, alternate_slot)
8342 } else {
8343 let nonce = generate_launch_nonce()?;
8344 let suffix = u128::from_str_radix(&nonce[..32], 16).expect("hex launch nonce");
8345 let mut end = spec.module_id.len().min(64);
8348 while !spec.module_id.is_char_boundary(end) {
8349 end -= 1;
8350 }
8351 format!(
8352 "{}_{suffix}",
8353 swap::cgroup_name(&spec.module_id[..end], alternate_slot)
8354 )
8355 }
8356 };
8357 #[cfg(not(target_os = "linux"))]
8358 let _ = alternate_slot;
8359 #[cfg(target_os = "macos")]
8360 let (mut command, privacy_exec, exec_ack) = privacy_command(spec, roster)?;
8361 #[cfg(windows)]
8362 let image_capture = windows_spawn_capture(spec).map_err(|source| SuperviseError::Spawn {
8363 program: spec.program.clone(),
8364 source,
8365 cgroup_path: None,
8366 })?;
8367 #[cfg(windows)]
8368 let mut command = Command::new(image_capture.path());
8369 #[cfg(not(any(target_os = "macos", windows)))]
8370 let mut command = Command::new(&spec.program);
8371 command.args(&spec.args);
8372 apply_child_env(&mut command, spec);
8402 apply_spawn_role(&mut command, role);
8403 let nonce_handoff =
8404 apply_wire_spawn_args_for_role(&mut command, spec, connection_file_path, handle, role)?;
8405
8406 #[cfg(target_os = "linux")]
8407 let cgroup_path = cgroup_placement
8408 .map(|placement| placement.module_path(&cgroup_name))
8409 .transpose()
8410 .map_err(|source| SuperviseError::Cgroup {
8411 module_id: spec.module_id.clone(),
8412 source,
8413 })?;
8414 #[cfg(not(target_os = "linux"))]
8415 let cgroup_path: Option<PathBuf> = None;
8416 #[cfg(target_os = "linux")]
8417 if let Some(path) = &cgroup_path {
8418 if let Err(error) = apply_cgroup_placement(&mut command, spec, path) {
8419 if let Some(placement) = cgroup_placement {
8420 remove_module_cgroup(placement, &cgroup_name);
8421 }
8422 return Err(error);
8423 }
8424 }
8425
8426 let output_sink = if let Some(logs_dir) = capture_logs_dir {
8427 let path = logs_dir.join(format!("{}.stderr.log", spec.module_id));
8428 match ChildOutputSink::open(&path, capture_retention(spec)) {
8429 Ok(sink) => sink,
8430 Err(error) => {
8431 warn!(
8432 module_id = %spec.module_id,
8433 path = %path.display(),
8434 error = %error,
8435 "could not open child output capture file; forwarding to stderr"
8436 );
8437 ChildOutputSink::Stderr
8438 }
8439 }
8440 } else {
8441 ChildOutputSink::Stderr
8442 };
8443
8444 command.stdout(Stdio::piped());
8445 command.stderr(Stdio::piped());
8446 command.kill_on_drop(true);
8447 #[cfg(unix)]
8464 command.process_group(0);
8465 command.stdin(Stdio::null());
8466 #[cfg(unix)]
8470 if let Some(handoff) = nonce_handoff {
8471 handoff.install_last(command.as_std_mut());
8472 }
8473 #[cfg(not(any(unix, windows)))]
8474 let _ = nonce_handoff;
8475
8476 #[cfg(windows)]
8481 {
8482 subc_jobobject::ensure_root_job().map_err(|source| SuperviseError::Spawn {
8483 program: spec.program.clone(),
8484 source,
8485 cgroup_path: None,
8486 })?;
8487 subc_jobobject::suspend_on_create_async(&mut command);
8488 }
8489 let spawned_at = Instant::now();
8493 let mut child = match command.spawn() {
8494 Ok(child) => child,
8495 Err(source) => {
8496 #[cfg(target_os = "linux")]
8497 if let Some(placement) = cgroup_placement {
8498 remove_module_cgroup(placement, &cgroup_name);
8499 }
8500 return Err(SuperviseError::Spawn {
8501 program: spec.program.clone(),
8502 source,
8503 cgroup_path,
8504 });
8505 }
8506 };
8507 #[cfg(test)]
8508 spawn_record_tests::after_create(&child, spec);
8509 #[cfg(target_os = "macos")]
8514 drop(exec_ack);
8515
8516 #[cfg(windows)]
8521 let windows_image = match image_capture.bind_tokio(&child) {
8522 Ok(image) => Arc::new(image),
8523 Err(source) => {
8524 kill_and_reap_rejected_child(&mut child);
8525 return Err(SuperviseError::Spawn {
8526 program: spec.program.clone(),
8527 source,
8528 cgroup_path: None,
8529 });
8530 }
8531 };
8532 #[cfg(windows)]
8535 let job = contain_spawned_child(&mut child, spec)?;
8536 let spawned_at_ms = unix_ms_now();
8537 #[cfg(not(windows))]
8538 let spawned_from = spec.program.clone();
8539 #[cfg(windows)]
8540 let spawned_from = windows_image.path().to_path_buf();
8541 #[cfg(not(windows))]
8542 let spawned_file_identity = spawned_file_identity(&spawned_from);
8543 #[cfg(windows)]
8544 let spawned_file_identity = Some(SpawnedFileIdentity {
8545 device: 0,
8546 inode: 0,
8547 windows_image: Some(Arc::clone(&windows_image)),
8548 });
8549 let pid = child.id().ok_or_else(|| SuperviseError::Spawn {
8550 program: spec.program.clone(),
8551 source: io::Error::other("spawned child exposed no live pid"),
8552 cgroup_path: cgroup_path.clone(),
8553 })?;
8554 #[cfg(windows)]
8555 let nonce_delivery = start_windows_nonce_pipe(nonce_handoff, pid, spec);
8556 #[cfg(not(windows))]
8557 let process_start_time = crate::provenance::process_start_time(pid);
8558 #[cfg(windows)]
8559 let process_start_time = Some(windows_image.start_time());
8560 let process_identity = process_start_time.map(|start_time| ProcessIdentity { pid, start_time });
8561 #[cfg(all(test, target_os = "macos"))]
8562 privacy_exec_boundary_tests::before_image_sample(spec, pid);
8563 #[cfg(not(windows))]
8568 let recorded_image = observe_spawned_image(pid);
8569 #[cfg(target_os = "macos")]
8573 let recorded_image = if privacy_exec.is_some() {
8574 None
8575 } else {
8576 recorded_image
8577 };
8578 #[cfg(target_os = "linux")]
8579 let recorded_cgroup_name = cgroup_path.as_ref().map(|_| cgroup_name.clone());
8580 #[cfg(not(target_os = "linux"))]
8581 let recorded_cgroup_name = None;
8582 let roster_guard = roster.admit(
8583 spec.module_id.clone(),
8584 pid,
8585 spec.protocol,
8586 process_start_time,
8587 crate::child_roster::RecordedIdentity {
8588 #[cfg(not(windows))]
8589 start_time: recorded_image.map(|image| image.start_time),
8590 #[cfg(windows)]
8591 start_time: process_start_time,
8592 #[cfg(not(windows))]
8593 executable: recorded_image
8594 .and_then(|image| image.executable)
8595 .map(crate::live_children::ExecutableIdentity::from),
8596 #[cfg(windows)]
8597 executable: Some(crate::live_children::ExecutableIdentity::from(
8598 windows_image.identity(),
8599 )),
8600 cgroup_name: recorded_cgroup_name,
8601 #[cfg(target_os = "linux")]
8602 cgroup_placement: cgroup_placement.cloned(),
8603 },
8604 );
8605 let roster_guard = match roster_guard {
8606 Ok(guard) => guard,
8607 Err(source) => {
8608 #[cfg(windows)]
8609 if let Some(job) = &job {
8610 let _ = job.terminate();
8611 }
8612 #[cfg(target_os = "linux")]
8613 kill_module_cgroup(cgroup_placement, &cgroup_name);
8614 kill_and_reap_rejected_child(&mut child);
8615 #[cfg(target_os = "linux")]
8616 if let Some(placement) = cgroup_placement {
8617 remove_module_cgroup(placement, &cgroup_name);
8618 }
8619 return Err(SuperviseError::Spawn {
8620 program: spec.program.clone(),
8621 source: io::Error::other(format!(
8622 "could not durably record module child: {source}"
8623 )),
8624 cgroup_path,
8625 });
8626 }
8627 };
8628 if roster.is_closed() {
8637 #[cfg(target_os = "linux")]
8639 kill_module_cgroup(cgroup_placement, &cgroup_name);
8640 if let Err(error) = child.start_kill() {
8641 debug!(module_id = %spec.module_id, pid, %error, "kill of a process spawned during daemon shutdown failed; it may already have exited");
8642 }
8643 #[cfg(target_os = "linux")]
8644 if let Some(placement) = cgroup_placement {
8645 while matches!(child.try_wait(), Ok(None)) {
8649 std::thread::yield_now();
8650 }
8651 if matches!(
8652 subc_cgroup::kill_module(Some(placement), &cgroup_name),
8653 subc_cgroup::KillOutcome::Killed
8654 ) {
8655 if let Ok(path) = placement.module_path(&cgroup_name) {
8656 while std::fs::read_to_string(path.join("cgroup.events"))
8657 .ok()
8658 .is_some_and(|events| events.lines().any(|line| line == "populated 1"))
8659 {
8660 std::thread::yield_now();
8661 }
8662 }
8663 }
8664 remove_module_cgroup(placement, &cgroup_name);
8665 }
8666 drop(roster_guard);
8667 return Err(SuperviseError::Spawn {
8668 program: spec.program.clone(),
8669 source: io::Error::other(
8670 "the daemon began shutting down while this process was starting; ended it",
8671 ),
8672 cgroup_path,
8673 });
8674 }
8675
8676 #[cfg(windows)]
8677 if let Err(error) = resume_suspended_child(pid, spec) {
8678 kill_and_reap_rejected_child(&mut child);
8679 return Err(error);
8680 }
8681
8682 let stdout_pump = match child.stdout.take() {
8683 Some(stdout) => Some(tokio::spawn(pump_stdout_to(stdout, output_sink.clone()))),
8684 None => {
8685 warn!(
8686 module_id = %spec.module_id,
8687 "spawned child exposed no stdout pipe; file capture will be incomplete"
8688 );
8689 None
8690 }
8691 };
8692 let stderr_pump = match child.stderr.take() {
8693 Some(stderr) => {
8694 let generation = ring
8695 .lock()
8696 .unwrap_or_else(|poisoned| poisoned.into_inner())
8697 .begin_process();
8698 Some(StderrPump {
8699 task: tokio::spawn(pump_stderr_to(
8700 stderr,
8701 Arc::clone(ring),
8702 generation,
8703 output_sink,
8704 )),
8705 generation,
8706 })
8707 }
8708 None => {
8709 ring.lock()
8713 .unwrap_or_else(|poisoned| poisoned.into_inner())
8714 .mark_not_captured("stderr pipe was not available on spawn");
8715 warn!(
8716 module_id = %spec.module_id,
8717 "spawned child exposed no stderr pipe; tail will be unavailable"
8718 );
8719 None
8720 }
8721 };
8722
8723 let exec = Arc::new(crate::activity::ExecOutcome::new());
8724 #[cfg(target_os = "macos")]
8725 let pending_exec = privacy_exec.is_some();
8726 #[cfg(not(target_os = "macos"))]
8727 let pending_exec = false;
8728 if !pending_exec {
8729 exec.finish(crate::activity::ExecState::CommittedExec);
8730 }
8731 Ok(SupervisedChild {
8732 child,
8733 exec: crate::activity::ExecOwner(exec),
8734 #[cfg(windows)]
8735 nonce_delivery,
8736 protocol: spec.protocol,
8737 #[cfg(target_os = "linux")]
8738 module_id: cgroup_name,
8739 #[cfg(target_os = "linux")]
8740 cgroup_placement: cgroup_placement.cloned(),
8741 #[cfg(windows)]
8742 job,
8743 stdout_pump,
8744 stderr_pump,
8745 stderr_ring: Arc::clone(ring),
8746 spawned_at,
8747 spawned_at_ms,
8748 spawned_from,
8749 launch_spec_digest: LaunchSpecDigest::from_spec(spec),
8750 spawned_file_identity,
8751 process_start_time,
8752 process_identity,
8753 pid,
8754 roster_guard: Some(roster_guard),
8755 #[cfg(target_os = "macos")]
8756 privacy_exec,
8757 #[cfg(target_os = "macos")]
8758 report_ready: Arc::new(OnceLock::new()),
8759 spawn_failure: None,
8760 })
8761}
8762
8763#[cfg(test)]
8764mod spawn_record_tests {
8765 use super::*;
8766
8767 thread_local! {
8768 static LAST_PID: std::cell::Cell<Option<u32>> = const { std::cell::Cell::new(None) };
8769 }
8770
8771 pub(super) fn after_create(child: &Child, _spec: &ModuleSpec) {
8772 LAST_PID.set(child.id());
8773 #[cfg(windows)]
8774 if let Some((_, address)) = _spec
8775 .env
8776 .iter()
8777 .find(|(key, _)| key == "SUBC_TEST_AFTER_CREATE")
8778 {
8779 use std::io::{Read, Write};
8780 let mut stream = std::net::TcpStream::connect(address).unwrap();
8781 stream
8782 .write_all(&child.id().unwrap().to_ne_bytes())
8783 .unwrap();
8784 let mut release = [0];
8785 stream.read_exact(&mut release).unwrap();
8786 }
8787 }
8788
8789 #[cfg(windows)]
8790 pub(super) fn assignment_failpoint(spec: &ModuleSpec) -> io::Result<()> {
8791 if spec
8792 .env
8793 .iter()
8794 .any(|(key, _)| key == "SUBC_TEST_ASSIGNMENT_FAILURE")
8795 {
8796 Err(io::Error::other("injected module job assignment failure"))
8797 } else {
8798 Ok(())
8799 }
8800 }
8801
8802 fn spec() -> ModuleSpec {
8803 ModuleSpec {
8804 module_id: "spawn-record-worker".into(),
8805 #[cfg(unix)]
8806 program: "/bin/sleep".into(),
8807 #[cfg(windows)]
8808 program: std::env::current_exe().unwrap(),
8809 #[cfg(unix)]
8810 args: vec!["60".into()],
8811 #[cfg(windows)]
8812 args: vec![
8813 "--exact".into(),
8814 "supervise::spawn_record_tests::windows_store_touch_fixture".into(),
8815 "--nocapture".into(),
8816 ],
8817 env: vec![],
8818 reserved: false,
8819 reserved_prefixes: vec![],
8820 protocol: ModuleProtocol::None,
8821 overlap: Default::default(),
8822 }
8823 }
8824
8825 #[tokio::test]
8826 async fn record_sync_failure_fails_spawn_and_reaps_child() {
8827 let home = cortexkit_test_support::ScratchDir::new("record-sync-failure");
8828 let supervisor =
8829 Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default())
8830 .with_live_children_record(home.join("live-children.json"));
8831 crate::live_children::fail_next_file_sync();
8832 let result = supervisor.spawn(spec());
8833 let failed = result.is_err();
8834 if let Ok(module) = result {
8835 module.stop().await.unwrap();
8836 }
8837 assert!(failed, "record failure must fail spawn");
8838 let pid = LAST_PID.get().expect("the real OS child was created");
8839 assert!(
8840 subc_os::Process::open(pid).unwrap().is_none(),
8841 "child {pid} survived rejected admission or was not reaped"
8842 );
8843 }
8844
8845 #[cfg(target_os = "macos")]
8846 #[test]
8847 #[ignore = "manual on-device latency measurement"]
8848 fn durable_record_write_latency_50() {
8849 let home = cortexkit_test_support::ScratchDir::new("record-latency");
8850 let path = home.join("live-children.json");
8851 let children = vec![crate::live_children::LiveChild {
8852 module_id: "latency-worker".into(),
8853 pid: std::process::id(),
8854 protocol: ModuleProtocol::None,
8855 start_time: Some(1),
8856 executable: None,
8857 cgroup_name: None,
8858 }];
8859 let mut samples = Vec::new();
8860 for _ in 0..50 {
8861 let start = std::time::Instant::now();
8862 crate::live_children::write_record(&path, &children).unwrap();
8863 samples.push(start.elapsed());
8864 }
8865 samples.sort();
8866 eprintln!(
8867 "durable_record_write_latency_50: n=50 median={:?} p95={:?}",
8868 (samples[24] + samples[25]) / 2,
8869 samples[47]
8870 );
8871 }
8872
8873 #[cfg(windows)]
8874 #[test]
8875 fn windows_store_touch_fixture() {
8876 if let Some(path) = std::env::var_os("SUBC_TEST_STORE_TOUCH") {
8877 std::fs::write(path, b"module ran").unwrap();
8878 }
8879 }
8880
8881 #[cfg(windows)]
8882 #[tokio::test]
8883 async fn windows_daemon_fixture() {
8884 let Ok(address) = std::env::var("SUBC_TEST_DAEMON_BARRIER") else {
8885 return;
8886 };
8887 let mut spec = spec();
8888 spec.env.push(("SUBC_TEST_AFTER_CREATE".into(), address));
8889 let supervisor =
8890 Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default());
8891 let _module = supervisor.spawn(spec).unwrap();
8892 }
8893
8894 #[cfg(windows)]
8895 #[test]
8896 fn windows_daemon_death_before_module_job_assignment_leaves_no_child() {
8897 use std::io::Read;
8898 let listener = std::net::TcpListener::bind("127.0.0.1:0").unwrap();
8899 let mut command = std::process::Command::new(std::env::current_exe().unwrap());
8902 command
8903 .args([
8904 "--exact",
8905 "supervise::spawn_record_tests::windows_daemon_fixture",
8906 "--nocapture",
8907 ])
8908 .env(
8909 "SUBC_TEST_DAEMON_BARRIER",
8910 listener.local_addr().unwrap().to_string(),
8911 );
8912 let mut daemon = subc_jobobject::spawn_contained(&mut command).unwrap();
8913 listener.set_nonblocking(true).unwrap();
8914 let deadline = std::time::Instant::now() + Duration::from_secs(30);
8915 let mut stream = loop {
8916 if let Ok((stream, _)) = listener.accept() {
8917 break stream;
8918 }
8919 assert!(
8920 daemon.child.try_wait().unwrap().is_none(),
8921 "daemon failed before rendezvous"
8922 );
8923 assert!(
8924 std::time::Instant::now() < deadline,
8925 "daemon did not reach CreateProcess failpoint"
8926 );
8927 std::thread::yield_now();
8928 };
8929 stream
8930 .set_read_timeout(Some(Duration::from_secs(30)))
8931 .unwrap();
8932 let mut pid = [0; 4];
8933 stream.read_exact(&mut pid).unwrap();
8934 let pid = u32::from_ne_bytes(pid);
8935 assert!(subc_jobobject::process_exists(pid));
8936 daemon.child.kill().unwrap();
8939 daemon.child.wait().unwrap();
8940 let exited = subc_jobobject::wait_for_process_exit(pid, Duration::from_secs(10));
8941 if !exited {
8942 daemon.job.terminate().unwrap();
8943 }
8944 assert!(exited, "suspended child {pid} outlived hard-killed daemon");
8945 }
8946
8947 #[cfg(windows)]
8948 #[tokio::test]
8949 async fn windows_containment_failure_never_resumes() {
8950 let home = cortexkit_test_support::ScratchDir::new("assignment-refusal");
8951 let sentinel = home.join("store-touched");
8952 let mut spec = spec();
8953 spec.env
8954 .push(("SUBC_TEST_ASSIGNMENT_FAILURE".into(), "1".into()));
8955 spec.env.push((
8956 "SUBC_TEST_STORE_TOUCH".into(),
8957 sentinel.display().to_string(),
8958 ));
8959 let supervisor =
8960 Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default());
8961 let result = supervisor.spawn(spec);
8962 if let Ok(module) = result {
8965 let _ = tokio::time::timeout(Duration::from_secs(10), async {
8966 while !sentinel.exists() && module.state().unwrap() == ModuleState::Running {
8967 tokio::task::yield_now().await;
8968 }
8969 })
8970 .await;
8971 let _ = module.stop().await;
8972 panic!(
8973 "assignment failure resumed a module; store touched={}",
8974 sentinel.exists()
8975 );
8976 }
8977 assert!(
8978 !sentinel.exists(),
8979 "refused module executed store-touch sentinel"
8980 );
8981 assert!(!subc_jobobject::process_exists(LAST_PID.get().unwrap()));
8982 }
8983
8984 #[cfg(windows)]
8985 #[tokio::test]
8986 async fn windows_nested_daemon_fixture() {
8987 let Some(sentinel) = std::env::var_os("SUBC_TEST_NESTED_STORE") else {
8988 return;
8989 };
8990 let mut spec = spec();
8991 spec.env.push((
8992 "SUBC_TEST_STORE_TOUCH".into(),
8993 sentinel.to_str().unwrap().into(),
8994 ));
8995 let supervisor =
8996 Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default());
8997 let module = supervisor
8998 .spawn(spec)
8999 .expect("per-module job must nest below daemon and host jobs");
9000 tokio::time::timeout(Duration::from_secs(10), async {
9001 while !PathBuf::from(&sentinel).exists() {
9002 tokio::task::yield_now().await;
9003 }
9004 })
9005 .await
9006 .unwrap();
9007 module.stop().await.unwrap();
9008 }
9009
9010 #[cfg(windows)]
9011 #[test]
9012 fn windows_module_job_nests_under_host_job() {
9013 let home = cortexkit_test_support::ScratchDir::new("nested-jobs");
9014 let sentinel = home.join("nested-store-touched");
9015 let mut command = std::process::Command::new(std::env::current_exe().unwrap());
9016 command
9017 .args([
9018 "--exact",
9019 "supervise::spawn_record_tests::windows_nested_daemon_fixture",
9020 "--nocapture",
9021 ])
9022 .env("SUBC_TEST_NESTED_STORE", &sentinel);
9023 let mut daemon = subc_jobobject::spawn_contained(&mut command).unwrap();
9024 assert!(daemon.child.wait().unwrap().success());
9025 assert!(sentinel.exists(), "nested protocol:none module never ran");
9026 }
9027}
9028
9029#[cfg(target_os = "linux")]
9030pub(crate) fn kill_module_cgroup(placement: Option<&subc_cgroup::Placement>, module_id: &str) {
9031 use subc_cgroup::KillOutcome;
9032 match subc_cgroup::kill_module(placement, module_id) {
9033 KillOutcome::Killed => {}
9034 KillOutcome::NotPlaced | KillOutcome::Unsupported => {
9035 debug!(
9036 module_id,
9037 "cgroup tree kill unavailable; using direct-child kill"
9038 );
9039 }
9040 KillOutcome::IoError { path, error } => {
9041 warn!(module_id, path = %path.display(), %error, "cgroup tree kill failed; using direct-child kill");
9042 }
9043 }
9044}
9045
9046#[cfg(windows)]
9054fn contain_spawned_child(
9055 child: &mut Child,
9056 spec: &ModuleSpec,
9057) -> Result<Option<subc_jobobject::JobObject>, SuperviseError> {
9058 let assignment = || {
9059 let job = subc_jobobject::JobObject::new()?;
9060 #[cfg(test)]
9061 spawn_record_tests::assignment_failpoint(spec)?;
9062 job.assign(child)?;
9063 Ok::<_, io::Error>(job)
9064 };
9065 match assignment() {
9066 Ok(job) => Ok(Some(job)),
9067 Err(source) => {
9068 kill_and_reap_rejected_child(child);
9069 Err(SuperviseError::Spawn {
9070 program: spec.program.clone(),
9071 source: io::Error::other(format!(
9072 "Windows module job containment failed: {source}"
9073 )),
9074 cgroup_path: None,
9075 })
9076 }
9077 }
9078}
9079
9080fn kill_and_reap_rejected_child(child: &mut Child) {
9083 #[cfg(unix)]
9084 if let Some(pid) = child
9085 .id()
9086 .and_then(|pid| rustix::process::Pid::from_raw(pid as i32))
9087 {
9088 let _ = rustix::process::kill_process_group(pid, rustix::process::Signal::KILL);
9089 }
9090 let _ = child.start_kill();
9091 while matches!(child.try_wait(), Ok(None)) {
9092 std::thread::yield_now();
9093 }
9094}
9095
9096#[cfg(windows)]
9101fn resume_suspended_child(pid: u32, spec: &ModuleSpec) -> Result<(), SuperviseError> {
9102 if let Err(source) = subc_jobobject::resume_main_thread(pid) {
9103 let _ = std::process::Command::new("taskkill.exe")
9107 .args(["/PID", &pid.to_string(), "/T", "/F"])
9108 .stdin(Stdio::null())
9109 .stdout(Stdio::null())
9110 .stderr(Stdio::null())
9111 .status();
9112 return Err(SuperviseError::Spawn {
9113 program: spec.program.clone(),
9114 source,
9115 cgroup_path: None,
9116 });
9117 }
9118 Ok(())
9119}
9120
9121#[cfg(target_os = "linux")]
9122fn remove_module_cgroup(placement: &subc_cgroup::Placement, module_id: &str) {
9123 match placement.remove_module(module_id) {
9124 Ok(()) => debug!(module_id, "removed module cgroup after process exit"),
9125 Err(error) => warn!(
9126 module_id,
9127 error = %error,
9128 "could not remove module cgroup after process exit; continuing teardown"
9129 ),
9130 }
9131}
9132
9133#[cfg(target_os = "linux")]
9134async fn cleanup_reaped_cgroup(placement: &subc_cgroup::Placement, module_id: &str) {
9135 if matches!(
9139 subc_cgroup::kill_module(Some(placement), module_id),
9140 subc_cgroup::KillOutcome::Killed
9141 ) {
9142 if let Ok(path) = placement.module_path(module_id) {
9143 while std::fs::read_to_string(path.join("cgroup.events"))
9144 .ok()
9145 .is_some_and(|events| events.lines().any(|line| line == "populated 1"))
9146 {
9147 sleep(Duration::from_millis(1)).await;
9148 }
9149 }
9150 }
9151 remove_module_cgroup(placement, module_id);
9152}
9153
9154#[cfg(target_os = "linux")]
9155fn apply_cgroup_placement(
9156 command: &mut Command,
9157 spec: &ModuleSpec,
9158 path: &std::path::Path,
9159) -> Result<(), SuperviseError> {
9160 subc_cgroup::apply(command, path).map_err(|source| SuperviseError::Cgroup {
9161 module_id: spec.module_id.clone(),
9162 source,
9163 })
9164}
9165
9166fn capture_retention(spec: &ModuleSpec) -> Retention {
9167 let defaults = Retention::default();
9168 let value = |name: &str| {
9169 spec.env
9170 .iter()
9171 .rev()
9172 .find_map(|(key, value)| (key == name).then_some(value.as_str()))
9173 };
9174 Retention {
9175 max_file_mb: value(CAPTURE_MAX_FILE_MB_ENV)
9176 .and_then(|value| value.parse().ok())
9177 .unwrap_or(defaults.max_file_mb),
9178 keep: value(CAPTURE_KEEP_ENV)
9179 .and_then(|value| value.parse().ok())
9180 .unwrap_or(defaults.keep),
9181 max_age_days: value(CAPTURE_MAX_AGE_DAYS_ENV)
9182 .and_then(|value| value.parse().ok())
9183 .unwrap_or(defaults.max_age_days),
9184 }
9185}
9186
9187fn generate_launch_nonce() -> Result<String, SuperviseError> {
9190 let mut bytes = [0u8; 32];
9191 getrandom::getrandom(&mut bytes).map_err(|source| SuperviseError::LaunchNonce {
9192 reason: source.to_string(),
9193 })?;
9194 let mut hex = String::with_capacity(64);
9195 for b in bytes {
9196 use std::fmt::Write;
9197 let _ = write!(hex, "{b:02x}");
9198 }
9199 Ok(hex)
9200}
9201
9202fn constant_time_eq(a: &[u8], b: &[u8]) -> bool {
9205 if a.len() != b.len() {
9206 return false;
9207 }
9208 let mut diff = 0u8;
9209 for (x, y) in a.iter().zip(b.iter()) {
9210 diff |= x ^ y;
9211 }
9212 diff == 0
9213}
9214
9215#[cfg(not(windows))]
9219fn observe_spawned_image(pid: u32) -> Option<subc_os::Observation> {
9220 let process = subc_os::Process::open(pid).ok().flatten()?;
9221 #[cfg(target_os = "linux")]
9222 return sample_after_linux_exec(
9223 || {
9224 let stat = std::fs::read_to_string(format!("/proc/{pid}/stat")).ok()?;
9225 let committed = linux_exec_committed(&stat);
9226 #[cfg(test)]
9227 spawn_image_tests::report_exec_state(committed);
9228 committed
9229 },
9230 || process.observe(),
9231 );
9232 #[cfg(not(target_os = "linux"))]
9233 process.observe()
9234}
9235
9236#[cfg(target_os = "linux")]
9240fn linux_exec_committed(stat: &str) -> Option<bool> {
9241 const PF_FORKNOEXEC: u64 = 0x40;
9242 let flags = stat.rsplit_once(')')?.1.split_whitespace().nth(6)?;
9243 Some(flags.parse::<u64>().ok()? & PF_FORKNOEXEC == 0)
9244}
9245
9246#[cfg(target_os = "linux")]
9247fn sample_after_linux_exec(
9248 mut exec_committed: impl FnMut() -> Option<bool>,
9249 observe: impl FnOnce() -> Option<subc_os::Observation>,
9250) -> Option<subc_os::Observation> {
9251 let deadline = std::time::Instant::now() + Duration::from_secs(1);
9252 while !exec_committed()? {
9253 if std::time::Instant::now() >= deadline {
9257 return None;
9258 }
9259 std::thread::sleep(Duration::from_micros(50));
9260 }
9261 observe()
9262}
9263
9264#[cfg(all(test, target_os = "linux"))]
9265mod spawn_image_tests {
9266 use super::*;
9267 use std::cell::{Cell, RefCell};
9268
9269 enum ExecEvent {
9270 Uncommitted,
9271 Image(Option<subc_os::Observation>),
9272 }
9273
9274 thread_local! {
9275 static EXEC_EVENTS: RefCell<Option<std::sync::mpsc::Sender<ExecEvent>>> = const { RefCell::new(None) };
9276 }
9277
9278 pub(super) fn report_exec_state(committed: Option<bool>) {
9279 if committed == Some(false) {
9280 EXEC_EVENTS.with(|events| {
9281 if let Some(events) = events.borrow_mut().take() {
9282 let _ = events.send(ExecEvent::Uncommitted);
9283 }
9284 });
9285 }
9286 }
9287
9288 #[test]
9289 fn linux_spawn_observation_waits_for_a_paused_fork_to_exec() {
9290 use std::io::{Read, Write};
9291 use std::os::fd::AsRawFd;
9292
9293 let descriptor = std::fs::File::open("/dev/null").unwrap();
9294 let mut command = std::process::Command::new("/bin/sleep");
9295 command.arg("30");
9296 let (mut ready, mut release) =
9297 subc_os::fork_exec_test::pause_before_exec(&mut command, descriptor.as_raw_fd())
9298 .unwrap();
9299 let spawn = std::thread::spawn(move || command.spawn());
9300 let mut message = [0u8; 8];
9301 ready.read_exact(&mut message).unwrap();
9302 let pid = i32::from_ne_bytes(message[..4].try_into().unwrap()) as u32;
9303 let before = std::fs::read_to_string(format!("/proc/{pid}/stat"));
9304 let inherited = subc_os::Process::open(pid).unwrap().unwrap().observe();
9305 let (events, received) = std::sync::mpsc::channel();
9306 let observer = std::thread::spawn(move || {
9307 EXEC_EVENTS.with(|slot| *slot.borrow_mut() = Some(events.clone()));
9308 let image = observe_spawned_image(pid);
9309 let _ = events.send(ExecEvent::Image(image));
9310 });
9311 let first = received.recv_timeout(Duration::from_secs(5));
9315 release.write_all(&[1]).unwrap();
9316 let mut child = spawn.join().unwrap().unwrap();
9317 let after = match first {
9318 Ok(ExecEvent::Uncommitted) => match received.recv_timeout(Duration::from_secs(5)) {
9319 Ok(ExecEvent::Image(image)) => image,
9320 _ => None,
9321 },
9322 Ok(ExecEvent::Image(image)) => image,
9323 Err(_) => None,
9324 };
9325 observer.join().unwrap();
9326 child.kill().unwrap();
9327 child.wait().unwrap();
9328
9329 assert_eq!(linux_exec_committed(&before.unwrap()), Some(false));
9330 assert_eq!(
9331 inherited.unwrap().executable,
9332 subc_os::file_identity(&std::env::current_exe().unwrap())
9333 );
9334 assert_eq!(
9335 after.unwrap().executable,
9336 subc_os::file_identity(std::path::Path::new("/bin/sleep"))
9337 );
9338 }
9339
9340 #[test]
9341 fn spawned_image_is_not_sampled_until_linux_exec_commits() {
9342 let committed = Cell::new(false);
9343 let inherited = subc_os::Observation {
9344 start_time: 23,
9345 executable: Some(subc_os::FileIdentity {
9346 device: 29,
9347 inode: 31,
9348 }),
9349 };
9350 let module = subc_os::Observation {
9351 start_time: 23,
9352 executable: Some(subc_os::FileIdentity {
9353 device: 29,
9354 inode: 37,
9355 }),
9356 };
9357 let polls = Cell::new(0);
9358 let image = sample_after_linux_exec(
9359 || {
9360 polls.set(polls.get() + 1);
9361 committed.set(polls.get() == 2);
9364 Some(committed.get())
9365 },
9366 || Some(if committed.get() { module } else { inherited }),
9367 );
9368 assert_eq!(image, Some(module));
9369 assert_eq!(polls.get(), 2);
9370 }
9371
9372 #[test]
9373 fn linux_exec_flag_is_read_from_flags_after_the_last_parenthesis() {
9374 assert_eq!(
9375 linux_exec_committed("17 (a ) b) R 1 2 3 4 5 4194368"),
9376 Some(false)
9377 );
9378 assert_eq!(
9379 linux_exec_committed("17 (a ) b) R 1 2 3 4 5 4194304"),
9380 Some(true)
9381 );
9382 assert_eq!(linux_exec_committed("17 (a) R 1 2 3 4 5 invalid"), None);
9383 assert_eq!(linux_exec_committed("17 (a) R"), None);
9384 }
9385
9386 #[test]
9387 fn unreadable_exec_state_never_samples_an_inherited_image() {
9388 assert_eq!(
9389 sample_after_linux_exec(
9390 || None,
9391 || panic!("must not sample before exec confirmation")
9392 ),
9393 None
9394 );
9395 }
9396
9397 #[test]
9398 fn uncommitted_exec_times_out_without_sampling_an_inherited_image() {
9399 assert_eq!(
9400 sample_after_linux_exec(
9401 || Some(false),
9402 || panic!("must not sample after an exec timeout")
9403 ),
9404 None
9405 );
9406 }
9407}
9408
9409#[cfg(windows)]
9412fn windows_spawn_capture(spec: &ModuleSpec) -> io::Result<subc_os::ExecutableCapture> {
9413 let mut program = spec.program.clone();
9414 if program.is_absolute() || program.components().count() > 1 {
9415 match subc_os::ExecutableCapture::open(&program) {
9418 Ok(capture) => return Ok(capture),
9419 Err(error)
9420 if error.kind() == io::ErrorKind::NotFound && program.extension().is_none() => {}
9421 Err(error) => return Err(error),
9422 }
9423 program.set_extension("exe");
9424 return subc_os::ExecutableCapture::open(&program);
9425 }
9426 if program.extension().is_none() {
9427 program.set_extension("exe");
9428 }
9429 let custom_path = spec
9430 .env
9431 .iter()
9432 .rev()
9433 .find(|(key, _)| key.eq_ignore_ascii_case("PATH"));
9434 let mut directories = Vec::new();
9435 if let Some(parent) = std::env::current_exe()?.parent() {
9438 directories.push(parent.to_path_buf());
9439 }
9440 directories.push(std::env::current_dir()?);
9441 if let Some(windows) = std::env::var_os("SystemRoot") {
9442 let windows = PathBuf::from(windows);
9443 directories.push(windows.join("System32"));
9444 directories.push(windows.clone());
9445 }
9446 let path = custom_path
9447 .map(|(_, value)| std::ffi::OsString::from(value))
9448 .or_else(|| std::env::var_os("PATH"));
9449 if let Some(path) = path {
9450 directories.extend(std::env::split_paths(&path));
9451 }
9452 for directory in directories {
9453 match subc_os::ExecutableCapture::open(&directory.join(&program)) {
9454 Ok(capture) => return Ok(capture),
9455 Err(error) if error.kind() == io::ErrorKind::NotFound => continue,
9456 Err(error) => return Err(error),
9457 }
9458 }
9459 Err(io::Error::new(
9460 io::ErrorKind::NotFound,
9461 "resolved executable not found",
9462 ))
9463}
9464
9465fn spawn_and_mark_running(
9466 spec: &ModuleSpec,
9467 runtime: &SupervisorRuntimeConfig,
9468 snapshot: &SharedSnapshot,
9469) -> Result<SupervisedChild, SuperviseError> {
9470 let activity_spawn = runtime.begin_activity_spawn(&spec.module_id);
9471 let child = spawn_child(
9472 spec,
9473 runtime.connection_file_path.as_deref(),
9474 runtime.supervisor_handle.as_ref(),
9475 &runtime.stderr_ring,
9476 runtime.capture_logs_dir.as_deref(),
9477 &runtime.child_roster,
9478 #[cfg(target_os = "linux")]
9479 runtime.cgroup_placement.as_ref(),
9480 )?;
9481 set_running(snapshot, &child, &spec.module_id, &runtime.spawn_events)?;
9482 if let Some(spawn) = activity_spawn {
9483 spawn.complete(snapshot);
9484 }
9485 Ok(child)
9486}
9487
9488enum RegistrationWaitOutcome {
9489 Registered,
9490 Exited(ExitReport),
9491 TimedOut,
9492}
9493
9494struct ReloadRegistrationFailure {
9495 exit_report: ExitReport,
9496 reason: String,
9497}
9498
9499#[derive(Debug, Clone, Copy, PartialEq, Eq)]
9500enum BusyGaugeObservation {
9501 Quiescent,
9502 Busy,
9503 Omitted,
9504}
9505
9506#[derive(Debug, Clone, PartialEq, Eq)]
9508enum GaugeVerdict {
9509 Zero,
9511 NonZero(u64),
9513 NotACount(String),
9517 Missing,
9519 ProbeFailed(&'static str),
9522}
9523
9524impl GaugeVerdict {
9525 fn reads_quiet(&self) -> bool {
9526 matches!(self, Self::Zero)
9527 }
9528}
9529
9530impl fmt::Display for GaugeVerdict {
9531 fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
9532 match self {
9533 Self::Zero => f.write_str("zero"),
9534 Self::NonZero(value) => write!(f, "non_zero({value})"),
9535 Self::NotACount(value) => write!(f, "not_a_count({value})"),
9536 Self::Missing => f.write_str("missing_from_health_body"),
9537 Self::ProbeFailed(kind) => write!(f, "probe_failed({kind})"),
9538 }
9539 }
9540}
9541
9542const GAUGE_VALUE_TEXT_LIMIT: usize = 32;
9545
9546fn gauge_verdicts(metrics: Option<&Value>, gauges: &[String]) -> Vec<GaugeVerdict> {
9548 let metrics = metrics.and_then(Value::as_object);
9549 gauges
9550 .iter()
9551 .map(
9552 |gauge| match metrics.and_then(|metrics| metrics.get(gauge)) {
9553 None => GaugeVerdict::Missing,
9554 Some(value) => match value.as_u64() {
9555 Some(0) => GaugeVerdict::Zero,
9556 Some(count) => GaugeVerdict::NonZero(count),
9557 None => {
9558 let mut text = value.to_string();
9559 if text.len() > GAUGE_VALUE_TEXT_LIMIT {
9560 let mut end = GAUGE_VALUE_TEXT_LIMIT;
9561 while !text.is_char_boundary(end) {
9562 end -= 1;
9563 }
9564 text.truncate(end);
9565 text.push_str("...");
9566 }
9567 GaugeVerdict::NotACount(text)
9568 }
9569 },
9570 },
9571 )
9572 .collect()
9573}
9574
9575fn busy_gauge_observation(verdicts: &[GaugeVerdict]) -> BusyGaugeObservation {
9578 if verdicts
9579 .iter()
9580 .any(|verdict| matches!(verdict, GaugeVerdict::Missing))
9581 {
9582 BusyGaugeObservation::Omitted
9583 } else if verdicts.iter().all(GaugeVerdict::reads_quiet) {
9584 BusyGaugeObservation::Quiescent
9585 } else {
9586 BusyGaugeObservation::Busy
9587 }
9588}
9589
9590#[derive(Debug, Clone, Default, PartialEq, Eq)]
9595struct GaugeTrace {
9596 latest: Option<(GaugeVerdict, Duration)>,
9597 last_value: Option<(GaugeVerdict, Duration)>,
9598}
9599
9600#[derive(Debug, Clone, Default, PartialEq, Eq)]
9606struct DrainGaugeTrace {
9607 probes: u32,
9609 gauges: Vec<(String, GaugeTrace)>,
9611}
9612
9613impl DrainGaugeTrace {
9614 fn new(gauges: &[String]) -> Self {
9615 Self {
9616 probes: 0,
9617 gauges: gauges
9618 .iter()
9619 .map(|gauge| (gauge.clone(), GaugeTrace::default()))
9620 .collect(),
9621 }
9622 }
9623
9624 fn record(&mut self, verdicts: &[GaugeVerdict], at: Duration) {
9626 self.probes = self.probes.saturating_add(1);
9627 for ((_, trace), verdict) in self.gauges.iter_mut().zip(verdicts) {
9628 trace.latest = Some((verdict.clone(), at));
9629 if matches!(
9630 verdict,
9631 GaugeVerdict::Zero | GaugeVerdict::NonZero(_) | GaugeVerdict::NotACount(_)
9632 ) {
9633 trace.last_value = Some((verdict.clone(), at));
9634 }
9635 }
9636 }
9637}
9638
9639impl fmt::Display for DrainGaugeTrace {
9644 fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
9645 if self.gauges.is_empty() {
9646 return f.write_str("none declared");
9647 }
9648 for (index, (name, trace)) in self.gauges.iter().enumerate() {
9649 if index > 0 {
9650 f.write_str(" | ")?;
9651 }
9652 write!(f, "{name}: ")?;
9653 match &trace.latest {
9654 Some((verdict, at)) => write!(f, "{verdict} @{}ms", at.as_millis())?,
9655 None => f.write_str("never probed")?,
9656 }
9657 match &trace.last_value {
9658 Some((value, at)) => write!(f, "; last value {value} @{}ms", at.as_millis())?,
9659 None => f.write_str("; no value ever read")?,
9660 }
9661 }
9662 Ok(())
9663 }
9664}
9665
9666pub(crate) fn declared_busy_gauges(
9667 registry: &Registry,
9668 module_id: &str,
9669) -> Result<Vec<String>, SuperviseError> {
9670 busy_gauges_of(
9671 registry
9672 .get_module(module_id)
9673 .map_err(SuperviseError::Registry)?,
9674 )
9675}
9676
9677fn declared_busy_gauges_for_connection(
9681 registry: &Registry,
9682 connection_id: ConnectionId,
9683) -> Result<Vec<String>, SuperviseError> {
9684 busy_gauges_of(
9685 registry
9686 .get_module_by_connection(connection_id)
9687 .map_err(SuperviseError::Registry)?,
9688 )
9689}
9690
9691fn busy_gauges_of(
9692 registration: Option<crate::registry::ModuleRegistration>,
9693) -> Result<Vec<String>, SuperviseError> {
9694 let Some(registration) = registration else {
9695 return Ok(Vec::new());
9696 };
9697 let Some(self_signals) = registration.manifest.self_signals else {
9698 return Ok(Vec::new());
9699 };
9700
9701 let mut gauges = Vec::new();
9702 for declaration in self_signals {
9703 if declaration.kind != SelfSignalKind::Busy {
9704 continue;
9705 }
9706 match declaration.anchored_to {
9707 SignalAnchor::HealthGauges { gauges: declared } if !declared.is_empty() => {
9708 gauges.extend(declared)
9709 }
9710 _ => {
9711 gauges.push(String::new());
9714 }
9715 }
9716 }
9717 Ok(gauges)
9718}
9719
9720#[allow(clippy::too_many_arguments)]
9729async fn wait_for_forwarding_quiescence(
9730 forwarding: &ForwardingTable,
9731 module_id: &str,
9732 runtime: &SupervisorRuntimeConfig,
9733 endpoint: crate::ModuleEndpointId,
9734 deadline: Instant,
9735 busy_gauges: &[String],
9736 scope: DrainScope,
9737 trace: &mut DrainGaugeTrace,
9738) -> Result<bool, SuperviseError> {
9739 let wait_started = Instant::now();
9740 let mut gauges_quiescent = busy_gauges.is_empty();
9741 let mut next_probe_at = wait_started;
9742 let mut omission_reported = false;
9743
9744 loop {
9745 let now = Instant::now();
9746 if !busy_gauges.is_empty() && now >= next_probe_at && now < deadline {
9747 let report = match scope {
9748 DrainScope::Active => probe_module_health(module_id, runtime, Some(deadline)).await,
9749 DrainScope::Endpoint(endpoint) => {
9750 probe_endpoint_health(endpoint, runtime, Some(deadline)).await
9751 }
9752 };
9753 let verdicts = match &report {
9754 Ok(report) => gauge_verdicts(report.metrics.as_ref(), busy_gauges),
9755 Err(err) => vec![GaugeVerdict::ProbeFailed(err.label()); busy_gauges.len()],
9756 };
9757 trace.record(&verdicts, wait_started.elapsed());
9758 gauges_quiescent = match report {
9759 Ok(_) => match busy_gauge_observation(&verdicts) {
9760 BusyGaugeObservation::Quiescent => true,
9761 BusyGaugeObservation::Busy => false,
9762 BusyGaugeObservation::Omitted => {
9763 if !omission_reported {
9766 forwarding
9767 .counters()
9768 .increment_drains_with_undeclared_gauge();
9769 let missing = busy_gauges
9770 .iter()
9771 .zip(&verdicts)
9772 .filter(|(_, verdict)| matches!(verdict, GaugeVerdict::Missing))
9773 .map(|(gauge, _)| gauge.as_str())
9774 .collect::<Vec<_>>()
9775 .join(",");
9776 warn!(
9777 module_id,
9778 missing_gauges = %missing,
9779 "drain health.check answer lacks a declared busy gauge; the drain treats the module as busy until it reports it"
9780 );
9781 omission_reported = true;
9782 }
9783 false
9784 }
9785 },
9786 Err(err) => {
9787 warn!(
9788 module_id,
9789 error = %err,
9790 "drain health.check did not produce declared busy gauges; treating module as busy"
9791 );
9792 false
9793 }
9794 };
9795 next_probe_at = Instant::now() + drain_busy_reprobe_interval();
9798 }
9799
9800 let in_flight = forwarding
9801 .endpoint_in_flight_count(endpoint)
9802 .map_err(SuperviseError::Forwarding)?;
9803 if in_flight == 0 && gauges_quiescent {
9804 return Ok(true);
9805 }
9806
9807 let now = Instant::now();
9808 if now >= deadline {
9809 return Ok(false);
9810 }
9811 let mut wait = deadline
9812 .saturating_duration_since(now)
9813 .min(REGISTRY_RELEASE_POLL);
9814 if !busy_gauges.is_empty() {
9815 wait = wait.min(next_probe_at.saturating_duration_since(now));
9816 }
9817 sleep(wait).await;
9818 }
9819}
9820
9821fn drain_busy_reprobe_interval() -> Duration {
9825 DRAIN_BUSY_REPROBE_INTERVAL.max(REGISTRY_RELEASE_POLL)
9826}
9827
9828fn warn_drain_timed_out(
9843 module_id: &str,
9844 drain_timeout: Duration,
9845 reason: RouteCloseReason,
9846 holdouts: &crate::forwarding::DrainHoldouts,
9847 gauges: &DrainGaugeTrace,
9848) {
9849 warn!(
9850 module_id = %module_id,
9851 waited = ?drain_timeout,
9852 ?reason,
9853 held_requests = holdouts.requests,
9854 held_routes = holdouts.routes,
9855 total_routes = holdouts.total_routes,
9856 top_connections = ?holdouts.top_connections,
9857 held = %holdouts
9860 .held
9861 .iter()
9862 .map(|(channel, corr)| format!("{channel}:{corr}"))
9863 .collect::<Vec<_>>()
9864 .join(","),
9865 gauge_probes = gauges.probes,
9866 busy_gauges = %gauges,
9867 "route drain timed out before request quiescence; forcing teardown"
9868 );
9869}
9870
9871fn drained_after_quiescence_wait(wait_result: &Result<bool, SuperviseError>) -> bool {
9879 match wait_result {
9880 Ok(drained) => *drained,
9881 Err(_) => false,
9882 }
9883}
9884
9885fn send_route_goodbyes(forwarding: &ForwardingTable, released_routes: Vec<GoodbyeTarget>) {
9886 for released in released_routes {
9887 let frame = match Frame::build_with_version(
9888 released.negotiated_ver,
9889 FrameType::Goodbye,
9890 control_flags(),
9891 released.channel,
9892 released.epoch,
9893 0,
9894 Vec::new(),
9895 ) {
9896 Ok(frame) => frame,
9897 Err(err) => {
9898 warn!(
9899 route_channel = released.channel,
9900 error = %err,
9901 "failed to build supervisor drain route GOODBYE frame"
9902 );
9903 continue;
9904 }
9905 };
9906 if !released.close_on_delivery_failure() {
9907 crate::forwarding::send_module_route_goodbye(
9908 &forwarding.counters(),
9909 &released.sink,
9910 frame,
9911 released.module_id.as_deref(),
9912 "supervisor drain",
9913 );
9914 continue;
9915 }
9916 if let Err(err) = released.sink.try_send(frame) {
9917 warn!(
9918 target_connection_id = released.connection_id.get(),
9919 route_channel = released.channel,
9920 error = %err,
9921 "supervisor drain route GOODBYE was not delivered to client; closing target connection"
9922 );
9923 let _ = forwarding.escalate_client_delivery_failure(
9924 released.connection_id,
9925 released.channel,
9926 released.epoch,
9927 CloseReason::new(
9928 "route_goodbye_delivery_failed",
9929 format!(
9930 "failed to enqueue supervisor drain route GOODBYE for channel {}: {err}",
9931 released.channel
9932 ),
9933 ),
9934 crate::forwarding::UndeliveredFrame {
9935 module_id: released.module_id.as_deref(),
9936 sink: &released.sink,
9937 },
9938 );
9939 }
9940 }
9941}
9942
9943fn send_module_draining(
9944 module_id: &str,
9945 reason: RouteCloseReason,
9946 deadline_ms: u64,
9947 target: &ModuleDrainTarget,
9948) {
9949 let body = match serde_json::to_vec(&ModuleControlCommand::Draining {
9950 reason,
9951 deadline_ms,
9952 }) {
9953 Ok(body) => body,
9954 Err(err) => {
9955 warn!(
9956 module_id,
9957 error = %err,
9958 "failed to encode module draining command"
9959 );
9960 return;
9961 }
9962 };
9963 let frame = match Frame::build_with_version(
9964 target.negotiated_ver,
9965 FrameType::Push,
9966 control_flags(),
9967 0,
9968 0,
9969 0,
9970 body,
9971 ) {
9972 Ok(frame) => frame,
9973 Err(err) => {
9974 warn!(
9975 module_id,
9976 error = %err,
9977 "failed to build module draining command frame"
9978 );
9979 return;
9980 }
9981 };
9982 if let Err(err) = target.sink.try_send(frame) {
9983 warn!(
9984 module_id,
9985 target_connection_id = target.endpoint.connection_id.get(),
9986 error = %err,
9987 "module draining command was not delivered to peer"
9988 );
9989 }
9990}
9991
9992fn module_goodbye_frame(module_id: &str, negotiated_ver: u8) -> Option<Frame> {
9994 match Frame::build_with_version(
9995 negotiated_ver,
9996 FrameType::Goodbye,
9997 control_flags(),
9998 0,
9999 0,
10000 0,
10001 Vec::new(),
10002 ) {
10003 Ok(frame) => Some(frame),
10004 Err(err) => {
10005 warn!(
10006 module_id,
10007 error = %err,
10008 "failed to build module GOODBYE frame"
10009 );
10010 None
10011 }
10012 }
10013}
10014
10015#[cfg(unix)]
10029async fn send_module_goodbyes_for_daemon_shutdown(
10030 forwarding: &Arc<ForwardingTable>,
10031 reason: &CloseReason,
10032 wait_for_flush: bool,
10033) {
10034 const GOODBYE_BUDGET: Duration = Duration::from_millis(500);
10035 let targets = match forwarding.module_connections() {
10036 Ok(targets) => targets,
10037 Err(err) => {
10038 warn!(error = %err, "could not list module connections for shutdown GOODBYE");
10039 return;
10040 }
10041 };
10042 let deadline = Instant::now() + GOODBYE_BUDGET;
10043 let mut sends = tokio::task::JoinSet::new();
10044 for target in targets {
10045 let Some(frame) = module_goodbye_frame(&target.module_id, target.negotiated_ver) else {
10046 continue;
10047 };
10048 if !wait_for_flush {
10049 if let Err(err) = target.sink.try_send(frame) {
10050 debug!(
10051 module_id = %target.module_id,
10052 error = %err,
10053 "shutdown module GOODBYE was not queued"
10054 );
10055 }
10056 continue;
10057 }
10058 let forwarding = Arc::clone(forwarding);
10059 let reason = reason.clone();
10060 sends.spawn(async move {
10061 match timeout_at(deadline, target.sink.send_flushed(frame)).await {
10062 Ok(Ok(())) => {}
10063 Ok(Err(err)) => debug!(
10064 module_id = %target.module_id,
10065 error = %err,
10066 "module connection closed before its shutdown GOODBYE was written"
10067 ),
10068 Err(_) => warn!(
10069 module_id = %target.module_id,
10070 budget = ?GOODBYE_BUDGET,
10071 "shutdown module GOODBYE was not written within its budget; closing anyway"
10072 ),
10073 }
10074 forwarding.request_connection_close(target.endpoint.connection_id, reason);
10075 });
10076 }
10077 while sends.join_next().await.is_some() {}
10079}
10080
10081fn send_module_goodbye(module_id: &str, forwarding: &ForwardingTable, target: &ModuleDrainTarget) {
10082 let Some(frame) = module_goodbye_frame(module_id, target.negotiated_ver) else {
10083 return;
10084 };
10085 if let Err(err) = target.sink.try_send(frame) {
10086 warn!(
10087 module_id,
10088 target_connection_id = target.endpoint.connection_id.get(),
10089 error = %err,
10090 "supervisor drain module GOODBYE was not delivered to peer; closing module connection"
10091 );
10092 forwarding.request_connection_close(
10093 target.endpoint.connection_id,
10094 CloseReason::new(
10095 "module_goodbye_delivery_failed",
10096 format!("failed to enqueue supervisor drain module GOODBYE for module '{module_id}': {err}"),
10097 ),
10098 );
10099 }
10100}
10101
10102#[derive(Clone, Copy)]
10103struct ForwardingDrainContext<'a> {
10104 spec: &'a ModuleSpec,
10105 runtime: &'a SupervisorRuntimeConfig,
10106 registry: &'a Registry,
10107 scope: DrainScope,
10108}
10109
10110#[derive(Debug, Clone, Copy, PartialEq, Eq)]
10112enum DrainScope {
10113 Active,
10116 Endpoint(crate::ModuleEndpointId),
10121}
10122
10123#[derive(Debug, Clone, Copy, PartialEq, Eq)]
10131enum StopNotice {
10132 SentOverConnection,
10135 NoConnection,
10139 NotSent,
10143}
10144
10145async fn begin_forwarding_drain(
10146 spec: &ModuleSpec,
10147 runtime: &SupervisorRuntimeConfig,
10148 registry: &Registry,
10149 snapshot: &SharedSnapshot,
10150 enabled: Option<bool>,
10151 reason: RouteCloseReason,
10152) -> Result<StopNotice, SuperviseError> {
10153 let Some(forwarding) = runtime.forwarding.as_ref() else {
10154 return Err(SuperviseError::ReloadUnavailable {
10155 module_id: spec.module_id.clone(),
10156 reason: "supervisor was not configured with a forwarding table".to_string(),
10157 });
10158 };
10159
10160 begin_forwarding_drain_with(
10161 forwarding,
10162 ForwardingDrainContext {
10163 spec,
10164 runtime,
10165 registry,
10166 scope: DrainScope::Active,
10167 },
10168 snapshot,
10169 enabled,
10170 reason,
10171 runtime.drain_timeout,
10172 )
10173 .await
10174}
10175
10176async fn begin_forwarding_drain_if_configured(
10177 spec: &ModuleSpec,
10178 runtime: &SupervisorRuntimeConfig,
10179 registry: &Registry,
10180 snapshot: &SharedSnapshot,
10181 enabled: Option<bool>,
10182 reason: RouteCloseReason,
10183) -> Result<StopNotice, SuperviseError> {
10184 begin_forwarding_drain_with_timeout(
10185 spec,
10186 runtime,
10187 registry,
10188 snapshot,
10189 enabled,
10190 reason,
10191 runtime.drain_timeout,
10192 )
10193 .await
10194}
10195
10196async fn begin_forwarding_drain_with_timeout(
10200 spec: &ModuleSpec,
10201 runtime: &SupervisorRuntimeConfig,
10202 registry: &Registry,
10203 snapshot: &SharedSnapshot,
10204 enabled: Option<bool>,
10205 reason: RouteCloseReason,
10206 drain_timeout: Duration,
10207) -> Result<StopNotice, SuperviseError> {
10208 let Some(forwarding) = runtime.forwarding.as_ref() else {
10209 return Ok(StopNotice::NotSent);
10210 };
10211
10212 begin_forwarding_drain_with(
10213 forwarding,
10214 ForwardingDrainContext {
10215 spec,
10216 runtime,
10217 registry,
10218 scope: DrainScope::Active,
10219 },
10220 snapshot,
10221 enabled,
10222 reason,
10223 drain_timeout,
10224 )
10225 .await
10226}
10227
10228async fn begin_forwarding_drain_with(
10229 forwarding: &ForwardingTable,
10230 context: ForwardingDrainContext<'_>,
10231 snapshot: &SharedSnapshot,
10232 enabled: Option<bool>,
10233 reason: RouteCloseReason,
10234 drain_timeout: Duration,
10235) -> Result<StopNotice, SuperviseError> {
10236 let ForwardingDrainContext {
10237 spec,
10238 runtime,
10239 registry,
10240 scope,
10241 } = context;
10242 debug_assert_ne!(reason, RouteCloseReason::Crash);
10243 let terminal = matches!(reason, RouteCloseReason::Disable);
10244 let drain_started_at = Instant::now();
10245 let drain_deadline = drain_started_at + drain_timeout;
10246 let deadline_ms =
10247 unix_ms_now().saturating_add(u64::try_from(drain_timeout.as_millis()).unwrap_or(u64::MAX));
10248 let busy_gauges = match scope {
10249 DrainScope::Active => declared_busy_gauges(registry, &spec.module_id)?,
10250 DrainScope::Endpoint(endpoint) => {
10251 declared_busy_gauges_for_connection(registry, endpoint.connection_id)?
10252 }
10253 };
10254
10255 let gate_started = Instant::now();
10258 let drain_target = match scope {
10259 DrainScope::Active => forwarding.begin_module_drain(&spec.module_id, reason),
10260 DrainScope::Endpoint(endpoint) => forwarding.begin_endpoint_drain(endpoint, reason),
10261 }
10262 .map_err(SuperviseError::Forwarding)?;
10263 info!(
10268 module_id = %spec.module_id,
10269 ?reason,
10270 gate_ms = u64::try_from(gate_started.elapsed().as_millis()).unwrap_or(u64::MAX),
10271 connected = drain_target.is_some(),
10272 "module drain began; route admission closed"
10273 );
10274 if scope == DrainScope::Active {
10275 update_snapshot(snapshot, Some(&spec.module_id), |state| {
10276 state.state = ModuleState::Draining;
10277 state.draining_to_replace =
10278 matches!(reason, RouteCloseReason::Restart | RouteCloseReason::Reload);
10279 if let Some(enabled) = enabled {
10280 state.enabled = enabled;
10281 }
10282 })?;
10283 }
10284
10285 let Some(target) = drain_target.as_ref() else {
10286 return Ok(StopNotice::NoConnection);
10290 };
10291 {
10292 send_module_draining(&spec.module_id, reason, deadline_ms, target);
10293 #[cfg(feature = "test-support")]
10294 forwarding.wait_before_route_closing_for_test().await;
10295 let routes = forwarding
10296 .endpoint_routes(target.endpoint)
10297 .map_err(SuperviseError::Forwarding)?;
10298 let routes_notified = routes.len();
10299 crate::control::send_route_control_pushes(
10300 forwarding,
10301 routes.clone(),
10302 ClientControlPush::RouteClosing {
10303 module_id: spec.module_id.clone(),
10304 channels: Vec::new(),
10305 reason,
10306 },
10307 );
10308 send_route_goodbyes(forwarding, target.abandoned_bindings.clone());
10309
10310 let mut gauge_trace = DrainGaugeTrace::new(&busy_gauges);
10316 let wait_result = wait_for_forwarding_quiescence(
10317 forwarding,
10318 &spec.module_id,
10319 runtime,
10320 target.endpoint,
10321 drain_deadline,
10322 &busy_gauges,
10323 scope,
10324 &mut gauge_trace,
10325 )
10326 .await;
10327 let drained = drained_after_quiescence_wait(&wait_result);
10328 if let Err(err) = &wait_result {
10329 error!(
10330 module_id = %spec.module_id,
10331 ?reason,
10332 error = %err,
10333 "forwarding quiescence wait failed after route.closing; forcing route.closed(drained: false) so the client is not left waiting on an unfulfilled promise"
10334 );
10335 } else if !drained {
10336 let holdouts = forwarding
10338 .endpoint_drain_holdouts(target.endpoint)
10339 .unwrap_or_default();
10340 warn_drain_timed_out(
10341 &spec.module_id,
10342 drain_timeout,
10343 reason,
10344 &holdouts,
10345 &gauge_trace,
10346 );
10347 }
10348 crate::control::send_route_control_pushes(
10349 forwarding,
10350 routes,
10351 ClientControlPush::RouteClosed {
10352 module_id: spec.module_id.clone(),
10353 channels: Vec::new(),
10354 reason,
10355 drained,
10356 abandoned: target.abandoned_bindings.len() as u32,
10357 excluded_subscriptions: target.excluded_subscriptions,
10358 terminal: Some(terminal),
10359 },
10360 );
10361 wait_result?;
10362
10363 let released_routes = match forwarding.release_module_endpoint_routes(target.endpoint) {
10369 Ok(routes) => routes,
10370 Err(err) => {
10371 warn!(
10372 module_id = %spec.module_id,
10373 ?reason,
10374 error = %err,
10375 "failed to release module endpoint routes after route.closed; module GOODBYE will still be sent"
10376 );
10377 send_module_goodbye(&spec.module_id, forwarding, target);
10378 return Err(SuperviseError::Forwarding(err));
10379 }
10380 };
10381 let route_goodbye_count = released_routes.len();
10382 send_route_goodbyes(forwarding, released_routes);
10383 send_module_goodbye(&spec.module_id, forwarding, target);
10384
10385 info!(
10391 module_id = %spec.module_id,
10392 ?reason,
10393 routes_notified,
10394 route_goodbyes = route_goodbye_count,
10395 abandoned_reservations = target.abandoned_bindings.len(),
10396 excluded_subscriptions = target.excluded_subscriptions,
10397 drained,
10398 "module drain complete; consumers notified via route.closing/route.closed pushes and per-route GOODBYE frames"
10399 );
10400 }
10401
10402 Ok(StopNotice::SentOverConnection)
10403}
10404
10405async fn wait_for_registration_after_reload(
10408 registry: &Registry,
10409 module_id: &str,
10410 snapshot: &SharedSnapshot,
10411 child: &mut SupervisedChild,
10412 wait: Duration,
10413) -> Result<RegistrationWaitOutcome, SuperviseError> {
10414 wait_for_slot_registration(
10415 registry,
10416 crate::registry::RegistrationSlot::Active(module_id),
10417 module_id,
10418 snapshot,
10419 child,
10420 wait,
10421 )
10422 .await
10423}
10424
10425async fn wait_for_slot_registration(
10433 registry: &Registry,
10434 slot: crate::registry::RegistrationSlot<'_>,
10435 module_id: &str,
10436 snapshot: &SharedSnapshot,
10437 child: &mut SupervisedChild,
10438 wait: Duration,
10439) -> Result<RegistrationWaitOutcome, SuperviseError> {
10440 let deadline = Instant::now() + wait;
10441 loop {
10442 if registry
10443 .registration(slot)
10444 .map_err(SuperviseError::Registry)?
10445 .is_some()
10446 {
10447 return Ok(RegistrationWaitOutcome::Registered);
10448 }
10449
10450 let now = Instant::now();
10451 if now >= deadline {
10452 return Ok(RegistrationWaitOutcome::TimedOut);
10453 }
10454 let remaining = deadline.saturating_duration_since(now);
10455 let poll = remaining.min(REGISTRY_RELEASE_POLL);
10456
10457 tokio::select! {
10458 wait_result = child.wait() => {
10459 let status = wait_result.map_err(|source| SuperviseError::Wait {
10460 module_id: module_id.to_string(),
10461 source,
10462 })?;
10463 return Ok(RegistrationWaitOutcome::Exited(classify_reaped_child_exit(
10464 snapshot,
10465 child,
10466 &status,
10467 )));
10468 }
10469 _ = sleep(poll) => {}
10470 }
10471 }
10472}
10473
10474fn registration_failure_exit_report(mut exit_report: ExitReport) -> ExitReport {
10475 if exit_report.kind != ExitKind::DeliberateSeverance {
10478 exit_report.kind = ExitKind::Crash;
10479 }
10480 exit_report
10481}
10482
10483async fn handle_reload_child_registration_failure(
10484 spec: &ModuleSpec,
10485 runtime: &SupervisorRuntimeConfig,
10486 registry: &Registry,
10487 process_liveness: &SupervisorProcessLiveness,
10488 snapshot: &SharedSnapshot,
10489 _child: &mut Option<SupervisedChild>,
10490 failure: ReloadRegistrationFailure,
10491) -> Result<(), SuperviseError> {
10492 let ReloadRegistrationFailure {
10493 exit_report,
10494 reason,
10495 } = failure;
10496 match on_child_exit(
10497 spec,
10498 runtime.restart_policy,
10499 registry,
10500 snapshot,
10501 &runtime.terminal_ring,
10502 &runtime.spawn_events,
10503 &runtime.child_roster,
10504 exit_report,
10505 )
10506 .await
10507 {
10508 NextAction::Stop {
10509 registration_released,
10510 } => {
10511 if registration_released {
10512 process_liveness.untrack_if_current(&spec.module_id, snapshot);
10513 }
10514 }
10515 NextAction::Restart { schedule } => {
10516 let delay = schedule.map_or(runtime.restart_policy.delay_for_restart(0), |schedule| {
10517 schedule.delay
10518 });
10519 if let Some(schedule) = schedule {
10520 log_crash_respawn(&spec.module_id, schedule);
10521 }
10522 schedule_respawn(
10523 runtime,
10524 snapshot,
10525 &spec.module_id,
10526 delay,
10527 RespawnKind::Spawn,
10528 )?;
10529 }
10530 }
10531 Err(SuperviseError::ReloadFailed {
10532 module_id: spec.module_id.clone(),
10533 reason,
10534 })
10535}
10536
10537async fn handle_reload_spawn_failure(
10538 spec: &ModuleSpec,
10539 runtime: &SupervisorRuntimeConfig,
10540 process_liveness: &SupervisorProcessLiveness,
10541 snapshot: &SharedSnapshot,
10542 _child: &mut Option<SupervisedChild>,
10543 reason: String,
10544) -> Result<(), SuperviseError> {
10545 let now = Instant::now();
10546 let mut schedule = None;
10547 update_snapshot(snapshot, Some(&spec.module_id), |state| {
10548 clear_current_process_facts(state);
10549 if state.enabled {
10550 schedule = state.next_crash_restart(&runtime.restart_policy, now);
10551 state.state = if schedule.is_some() {
10552 ModuleState::Restarting
10553 } else {
10554 ModuleState::Failed
10555 };
10556 } else {
10557 state.state = ModuleState::Disabled;
10558 }
10559 })?;
10560 if let Some(schedule) = schedule {
10561 schedule_respawn(
10562 runtime,
10563 snapshot,
10564 &spec.module_id,
10565 schedule.delay,
10566 RespawnKind::Spawn,
10567 )?;
10568 } else {
10569 process_liveness.untrack_if_current(&spec.module_id, snapshot);
10570 }
10571 Err(SuperviseError::ReloadFailed {
10572 module_id: spec.module_id.clone(),
10573 reason,
10574 })
10575}
10576
10577fn control_flags() -> Flags {
10578 Flags::new(false, Priority::Passive, false)
10579}
10580
10581#[allow(clippy::too_many_arguments)]
10582async fn drain_optional_child(
10583 module_id: &str,
10584 protocol: ModuleProtocol,
10585 stop_notice: StopNotice,
10586 registry: &Registry,
10587 forwarding: Option<&ForwardingTable>,
10588 snapshot: &SharedSnapshot,
10589 terminal_ring: &Arc<Mutex<TerminalRing>>,
10590 spawn_events: &SpawnEventFeed,
10591 child: &mut Option<SupervisedChild>,
10592 drain_timeout: Duration,
10593 final_state: ModuleState,
10594 enabled: Option<bool>,
10595) -> Result<(), SuperviseError> {
10596 if let Some(child) = child.take() {
10597 drain_child_to_state(
10598 module_id,
10599 protocol,
10600 stop_notice,
10601 registry,
10602 forwarding,
10603 snapshot,
10604 terminal_ring,
10605 spawn_events,
10606 child,
10607 drain_timeout,
10608 final_state,
10609 enabled,
10610 )
10611 .await
10612 } else {
10613 update_snapshot(snapshot, Some(module_id), |state| {
10614 state.state = final_state;
10615 if let Some(enabled) = enabled {
10616 state.enabled = enabled;
10617 }
10618 clear_current_process_facts(state);
10619 })?;
10620 release_dead_registration(registry, forwarding, snapshot, module_id).await
10621 }
10622}
10623
10624#[allow(clippy::too_many_arguments)]
10625async fn drain_child_to_state(
10626 module_id: &str,
10627 _protocol: ModuleProtocol,
10628 stop_notice: StopNotice,
10629 registry: &Registry,
10630 forwarding: Option<&ForwardingTable>,
10631 snapshot: &SharedSnapshot,
10632 terminal_ring: &Arc<Mutex<TerminalRing>>,
10633 spawn_events: &SpawnEventFeed,
10634 mut child: SupervisedChild,
10635 drain_timeout: Duration,
10636 final_state: ModuleState,
10637 enabled: Option<bool>,
10638) -> Result<(), SuperviseError> {
10639 let protocol = child.protocol;
10640 update_snapshot(snapshot, Some(module_id), |state| {
10641 state.state = ModuleState::Draining;
10642 state.draining_to_replace = final_state == ModuleState::Restarting;
10643 if let Some(enabled) = enabled {
10644 state.enabled = enabled;
10645 }
10646 })?;
10647
10648 if stop_notice != StopNotice::SentOverConnection {
10659 if protocol == ModuleProtocol::Subc && stop_notice == StopNotice::NoConnection {
10660 info!(
10661 module_id,
10662 pid = child.pid,
10663 budget_ms = u64::try_from(drain_timeout.as_millis()).unwrap_or(u64::MAX),
10664 "module has no connection yet; requesting stop by signal"
10665 );
10666 }
10667 request_graceful_stop(module_id, &child);
10668 }
10669
10670 let exit_report = match timeout(drain_timeout, child.wait()).await {
10671 Ok(Ok(status)) => classify_reaped_child_exit(snapshot, &child, &status),
10672 Ok(Err(source)) => {
10673 fail_after_wait_error(
10674 module_id,
10675 snapshot,
10676 terminal_ring,
10677 spawn_events,
10678 format!("failed to wait during drain: {source}"),
10679 );
10680 return Err(SuperviseError::Wait {
10681 module_id: module_id.to_string(),
10682 source,
10683 });
10684 }
10685 Err(_) => {
10686 warn!(
10699 module_id,
10700 pid = child.pid,
10701 budget_ms = u64::try_from(drain_timeout.as_millis()).unwrap_or(u64::MAX),
10702 reason = ?final_state,
10703 ?stop_notice,
10704 "drain budget expired before the module exited; killing it"
10705 );
10706 child.start_kill().map_err(|source| {
10707 fail_after_wait_error(
10708 module_id,
10709 snapshot,
10710 terminal_ring,
10711 spawn_events,
10712 format!("failed to kill during drain: {source}"),
10713 );
10714 SuperviseError::Kill {
10715 module_id: module_id.to_string(),
10716 source,
10717 }
10718 })?;
10719 let status = child.wait().await.map_err(|source| {
10720 fail_after_wait_error(
10721 module_id,
10722 snapshot,
10723 terminal_ring,
10724 spawn_events,
10725 format!("failed to wait after drain kill: {source}"),
10726 );
10727 SuperviseError::Wait {
10728 module_id: module_id.to_string(),
10729 source,
10730 }
10731 })?;
10732 classify_reaped_child_exit(snapshot, &child, &status)
10733 }
10734 };
10735
10736 let detail = lock_snapshot(snapshot)?.drain_disposition_detail.take();
10737 publish_after_terminal_record(
10738 module_id,
10739 || {
10740 record_terminal_with_detail(
10741 module_id,
10742 terminal_ring,
10743 spawn_events,
10744 &exit_report,
10745 terminal_disposition(final_state),
10746 detail,
10747 )
10748 },
10749 || {
10750 update_snapshot(snapshot, Some(module_id), |state| {
10751 state.state = final_state;
10752 if let Some(enabled) = enabled {
10753 state.enabled = enabled;
10754 }
10755 clear_current_process_facts(state);
10756 state.last_exit = Some(exit_report.clone());
10757 if exit_report.kind == ExitKind::DeliberateSeverance {
10758 state.lifetime_restarts += 1;
10759 }
10760 })
10761 },
10762 )?;
10763 child.drain_stderr(module_id).await;
10764
10765 release_dead_registration(registry, forwarding, snapshot, module_id).await
10766}
10767
10768#[cfg(unix)]
10788fn request_graceful_stop(module_id: &str, child: &SupervisedChild) {
10789 let Some(pid) = child
10790 .id()
10791 .and_then(|pid| i32::try_from(pid).ok())
10792 .and_then(rustix::process::Pid::from_raw)
10793 else {
10794 debug!(
10795 module_id,
10796 "no pid to signal for teardown; falling through to the drain wait"
10797 );
10798 return;
10799 };
10800 match rustix::process::kill_process(pid, rustix::process::Signal::TERM) {
10801 Ok(()) => debug!(
10802 module_id,
10803 "sent SIGTERM to a module nothing else asked to stop"
10804 ),
10805 Err(err) => debug!(
10806 module_id,
10807 error = %err,
10808 "SIGTERM to module failed; the drain wait and kill still apply"
10809 ),
10810 }
10811}
10812
10813#[cfg(not(unix))]
10821fn request_graceful_stop(module_id: &str, _child: &SupervisedChild) {
10822 debug!(
10823 module_id,
10824 "no graceful stop signal exists on this platform; teardown of a module nothing asked to stop waits, then kills"
10825 );
10826}
10827
10828fn terminal_disposition(final_state: ModuleState) -> TerminalDisposition {
10829 match final_state {
10830 ModuleState::Stopped => TerminalDisposition::Stopped,
10831 ModuleState::Disabled => TerminalDisposition::Disabled,
10832 ModuleState::Restarting => TerminalDisposition::Restarting,
10833 ModuleState::Failed => TerminalDisposition::Failed,
10834 ModuleState::Starting
10835 | ModuleState::Running
10836 | ModuleState::Unresponsive
10837 | ModuleState::Draining => {
10838 unreachable!("terminal exits only finish in terminal or restarting states")
10839 }
10840 }
10841}
10842
10843async fn release_dead_registration(
10851 registry: &Registry,
10852 forwarding: Option<&ForwardingTable>,
10853 snapshot: &SharedSnapshot,
10854 module_id: &str,
10855) -> Result<(), SuperviseError> {
10856 let result = async {
10857 let registration = registry
10858 .get_module(module_id)
10859 .map_err(SuperviseError::Registry)?;
10860 match wait_for_registration_release(registry, module_id, REGISTRY_RELEASE_TIMEOUT).await {
10861 Ok(()) => return Ok(()),
10862 Err(SuperviseError::RegistrationStillActive { .. }) => {}
10863 Err(err) => return Err(err),
10864 }
10865 let pid = lock_snapshot(snapshot)?.reaped_pid;
10866 if let (Some(registration), Some(forwarding), Some(pid)) = (registration, forwarding, pid) {
10867 warn!(
10868 module_id,
10869 pid,
10870 connection_id = registration.connection_id.get(),
10871 "reaped module registration outlived release grace; closing dead connection"
10872 );
10873 forwarding.request_connection_close(
10874 registration.connection_id,
10875 CloseReason::new(
10876 "supervised_process_reaped",
10877 format!("module '{module_id}' pid {pid} exited"),
10878 ),
10879 );
10880 wait_for_slot_registration_release(
10881 registry,
10882 crate::registry::RegistrationSlot::Connection(registration.connection_id),
10883 REGISTRY_RELEASE_TIMEOUT,
10884 )
10885 .await?;
10886 }
10887 wait_for_registration_release(registry, module_id, Duration::ZERO).await
10888 }
10889 .await;
10890 if let Err(err) = &result {
10891 fail_snapshot(snapshot, Some(module_id), None);
10892 error!(module_id, error = %err, "registration release failed after child exit; module is failed and start can retry");
10893 }
10894 result
10895}
10896
10897async fn wait_for_registration_release(
10900 registry: &Registry,
10901 module_id: &str,
10902 wait: Duration,
10903) -> Result<(), SuperviseError> {
10904 wait_for_slot_registration_release(
10905 registry,
10906 crate::registry::RegistrationSlot::Active(module_id),
10907 wait,
10908 )
10909 .await
10910}
10911
10912async fn wait_for_slot_registration_release(
10920 registry: &Registry,
10921 slot: crate::registry::RegistrationSlot<'_>,
10922 wait: Duration,
10923) -> Result<(), SuperviseError> {
10924 let deadline = Instant::now() + wait;
10925 let mut release_events = registration_release_events().subscribe();
10926 let still_active = |registration: &crate::registry::ModuleRegistration| {
10927 SuperviseError::RegistrationStillActive {
10928 module_id: registration.manifest.module_id.clone(),
10929 waited: wait,
10930 }
10931 };
10932 loop {
10933 let _observed_generation = *release_events.borrow_and_update();
10934 let Some(registration) = registry
10935 .registration(slot)
10936 .map_err(SuperviseError::Registry)?
10937 else {
10938 return Ok(());
10939 };
10940
10941 let now = Instant::now();
10942 if now >= deadline {
10943 return Err(still_active(®istration));
10944 }
10945
10946 let remaining = deadline.saturating_duration_since(now);
10947 match timeout(remaining, release_events.changed()).await {
10948 Ok(Ok(())) | Ok(Err(_)) => {}
10949 Err(_) => return Err(still_active(®istration)),
10950 }
10951 }
10952}
10953
10954#[cfg(test)]
10955mod slot_registration_wait_tests {
10956 use super::*;
10957 use crate::registry::{ConnectionId, RegistrationSlot};
10958 use subc_protocol::manifest::ModuleManifest;
10959
10960 #[tokio::test]
10961 async fn enable_release_failure_is_failed_and_a_second_enable_retries() {
10962 let registry = Arc::new(Registry::default());
10963 let supervisor = Supervisor::new_for_test(Arc::clone(®istry), RestartPolicy::default());
10964 let runtime = supervisor.runtime_config();
10965 let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::disabled()));
10966 let spec = ModuleSpec {
10967 module_id: "enable-stale-registration".to_string(),
10968 program: PathBuf::from("/missing/enable-retry-test"),
10969 args: Vec::new(),
10970 env: Vec::new(),
10971 reserved: false,
10972 reserved_prefixes: Vec::new(),
10973 protocol: ModuleProtocol::Subc,
10974 overlap: Default::default(),
10975 };
10976 let connection = ConnectionId::new(90);
10977 registry
10978 .register_with_control_ops(
10979 ModuleManifest::builder(&spec.module_id, "0.1.0").build(),
10980 1,
10981 connection,
10982 Vec::new(),
10983 )
10984 .unwrap();
10985 let mut child = None;
10986 let err = set_child_enabled(
10987 &spec,
10988 &runtime,
10989 ®istry,
10990 &supervisor.process_liveness,
10991 &snapshot,
10992 &mut child,
10993 true,
10994 )
10995 .await
10996 .unwrap_err();
10997 assert!(matches!(
10998 err,
10999 SuperviseError::RegistrationStillActive { .. }
11000 ));
11001 assert_eq!(lock_snapshot(&snapshot).unwrap().state, ModuleState::Failed);
11002 assert!(child.is_none());
11003 registry.deregister_connection(connection).unwrap();
11004 let err = set_child_enabled(
11005 &spec,
11006 &runtime,
11007 ®istry,
11008 &supervisor.process_liveness,
11009 &snapshot,
11010 &mut child,
11011 true,
11012 )
11013 .await
11014 .unwrap_err();
11015 assert!(
11016 matches!(err, SuperviseError::Spawn { .. }),
11017 "second enable must attempt a spawn: {err}"
11018 );
11019 assert_eq!(lock_snapshot(&snapshot).unwrap().state, ModuleState::Failed);
11020 }
11021
11022 const INCUMBENT: u64 = 1;
11023 const CANDIDATE: u64 = 2;
11024
11025 fn swapped_registry() -> Arc<Registry> {
11026 let registry = Arc::new(Registry::default());
11027 let manifest = ModuleManifest::builder("m", "0.1.0").build();
11028 registry
11029 .register_with_control_ops(
11030 manifest.clone(),
11031 1,
11032 ConnectionId::new(INCUMBENT),
11033 Vec::new(),
11034 )
11035 .unwrap();
11036 registry
11037 .register_candidate_with_control_ops(
11038 manifest,
11039 1,
11040 ConnectionId::new(CANDIDATE),
11041 Vec::new(),
11042 )
11043 .unwrap();
11044 registry
11045 }
11046
11047 #[tokio::test]
11051 async fn incumbent_release_is_awaited_by_connection_not_by_module_id() {
11052 let registry = swapped_registry();
11053 registry.promote_candidate("m").unwrap().unwrap();
11054
11055 assert!(matches!(
11056 wait_for_registration_release(®istry, "m", Duration::from_millis(50)).await,
11057 Err(SuperviseError::RegistrationStillActive { .. })
11058 ));
11059
11060 assert!(matches!(
11062 wait_for_slot_registration_release(
11063 ®istry,
11064 RegistrationSlot::Connection(ConnectionId::new(INCUMBENT)),
11065 Duration::from_millis(50),
11066 )
11067 .await,
11068 Err(SuperviseError::RegistrationStillActive { .. })
11069 ));
11070
11071 let releaser = Arc::clone(®istry);
11072 let release = tokio::spawn(async move {
11073 sleep(Duration::from_millis(20)).await;
11074 releaser
11075 .deregister_connection(ConnectionId::new(INCUMBENT))
11076 .unwrap();
11077 notify_registration_release();
11078 });
11079 wait_for_slot_registration_release(
11080 ®istry,
11081 RegistrationSlot::Connection(ConnectionId::new(INCUMBENT)),
11082 Duration::from_secs(5),
11083 )
11084 .await
11085 .expect("the incumbent's own registration is released");
11086 release.await.unwrap();
11087 assert!(registry.get_module("m").unwrap().is_some());
11088 }
11089
11090 #[tokio::test]
11093 async fn candidate_slot_wait_ignores_the_incumbents_registration() {
11094 let registry = swapped_registry();
11095 assert!(matches!(
11096 wait_for_slot_registration_release(
11097 ®istry,
11098 RegistrationSlot::Candidate("m"),
11099 Duration::from_millis(50),
11100 )
11101 .await,
11102 Err(SuperviseError::RegistrationStillActive { .. })
11103 ));
11104 registry
11105 .deregister_connection(ConnectionId::new(CANDIDATE))
11106 .unwrap();
11107 wait_for_slot_registration_release(
11108 ®istry,
11109 RegistrationSlot::Candidate("m"),
11110 Duration::from_millis(50),
11111 )
11112 .await
11113 .expect("a candidate slot with no candidate is released");
11114 assert!(registry
11115 .registration(RegistrationSlot::Active("m"))
11116 .unwrap()
11117 .is_some());
11118 }
11119}
11120
11121fn classify_exit(status: &ExitStatus) -> ExitReport {
11122 ExitReport {
11123 kind: if status.success() {
11124 ExitKind::Clean
11125 } else {
11126 ExitKind::Crash
11127 },
11128 code: status.code(),
11129 signal: exit_signal(status),
11130 at_ms: unix_ms_now(),
11131 }
11132}
11133
11134fn wait_error_exit_report() -> ExitReport {
11140 ExitReport {
11141 kind: ExitKind::Crash,
11142 code: None,
11143 signal: None,
11144 at_ms: unix_ms_now(),
11145 }
11146}
11147
11148#[cfg(unix)]
11149fn exit_signal(status: &ExitStatus) -> Option<i32> {
11150 use std::os::unix::process::ExitStatusExt;
11151
11152 status.signal()
11153}
11154
11155#[cfg(not(unix))]
11156fn exit_signal(_status: &ExitStatus) -> Option<i32> {
11157 None
11158}
11159
11160fn reset_restart_count(snapshot: &SharedSnapshot, module_id: &str) -> Result<(), SuperviseError> {
11166 update_snapshot(snapshot, Some(module_id), |state| {
11167 state.clear_crash_restarts();
11168 })
11169}
11170
11171fn set_running(
11172 snapshot: &SharedSnapshot,
11173 child: &SupervisedChild,
11174 module_id: &str,
11175 spawn_events: &SpawnEventFeed,
11176) -> Result<(), SuperviseError> {
11177 let mut state = snapshot.lock().map_err(|_| SuperviseError::StatePoisoned {
11178 module_id: Some(module_id.to_string()),
11179 })?;
11180 state.spawn_generation = spawn_events.emit_spawned(module_id, child.pid, child.spawned_at_ms);
11181 if std::mem::take(&mut state.coalesced_restart_pending) {
11182 let generation = state.spawn_generation;
11183 info!(module_id, "restart already satisfied by generation {generation}; coalesced pending request completed");
11184 }
11185 state.drain_disposition_detail = None;
11186 state.spawn_failure = None;
11187 state.waiting_for_previous_pid = None;
11188 state.waiting_for_previous_detail = None;
11189 state.in_alternate_slot = false;
11192 state.configuration_updated_since_spawn = false;
11193 state.spawned_protocol = Some(child.protocol);
11194 state.state = ModuleState::Running;
11195 state.enabled = true;
11196 state.process_alive = true;
11197 state.pid = child.id();
11198 #[cfg(windows)]
11199 {
11200 state.pipe_consumed = child
11201 .nonce_delivery
11202 .as_ref()
11203 .map(|delivery| delivery.consumed());
11204 }
11205 #[cfg(target_os = "macos")]
11206 {
11207 state.report_ready = Some(Arc::clone(&child.report_ready));
11208 }
11209 state.spawned_at_ms = Some(child.spawned_at_ms);
11210 state.exec_outcome = Some(Arc::clone(&child.exec.0));
11211 state.spawned_from = Some(child.spawned_from.clone());
11212 state.launch_spec_digest = Some(child.launch_spec_digest.clone());
11213 state.spawned_file_identity = child.spawned_file_identity.clone();
11214 state.process_start_time = child.process_start_time;
11215 Ok(())
11216}
11217
11218fn clear_current_process_facts(state: &mut SupervisorSnapshot) {
11219 state.waiting_for_previous_pid = None;
11220 state.waiting_for_previous_detail = None;
11221 state.process_alive = false;
11222 state.spawned_protocol = None;
11223 state.pid = None;
11224 #[cfg(windows)]
11225 {
11226 state.pipe_consumed = None;
11227 }
11228 #[cfg(target_os = "macos")]
11229 {
11230 state.report_ready = None;
11231 }
11232 state.spawned_at_ms = None;
11233 state.exec_outcome = None;
11234 state.spawned_from = None;
11235 state.launch_spec_digest = None;
11236 state.spawned_file_identity = None;
11237 state.process_start_time = None;
11238 state.deliberate_severance = None;
11239}
11240
11241#[cfg(test)]
11242fn record_deliberate_severance(
11243 snapshot: &SharedSnapshot,
11244 identity: ProcessIdentity,
11245) -> Result<(), SuperviseError> {
11246 update_snapshot(snapshot, None, |state| {
11247 state.deliberate_severance = Some(identity);
11248 })
11249}
11250
11251fn apply_deliberate_severance_marker(
11252 snapshot: &SharedSnapshot,
11253 exited_identity: Option<ProcessIdentity>,
11254 mut exit_report: ExitReport,
11255) -> ExitReport {
11256 let marker = lock_snapshot(snapshot)
11257 .ok()
11258 .and_then(|mut state| state.deliberate_severance.take());
11259 if marker.is_some() && marker == exited_identity {
11260 exit_report.kind = ExitKind::DeliberateSeverance;
11261 }
11262 exit_report
11263}
11264
11265fn classify_reaped_child_exit(
11266 snapshot: &SharedSnapshot,
11267 child: &SupervisedChild,
11268 status: &ExitStatus,
11269) -> ExitReport {
11270 let _ = update_snapshot(snapshot, None, |state| {
11271 state.reaped_pid = Some(child.pid);
11272 state.spawn_failure = child.spawn_failure.clone();
11273 });
11274 apply_deliberate_severance_marker(snapshot, child.process_identity(), classify_exit(status))
11275}
11276
11277fn fail_snapshot(
11278 snapshot: &SharedSnapshot,
11279 module_id: Option<&str>,
11280 last_exit: Option<ExitReport>,
11281) {
11282 if let Err(err) = update_snapshot(snapshot, module_id, |state| {
11283 state.state = ModuleState::Failed;
11284 clear_current_process_facts(state);
11285 if let Some(last_exit) = last_exit {
11286 state.last_exit = Some(last_exit);
11287 }
11288 }) {
11289 error!(error = %err, "failed to mark supervisor state failed");
11290 }
11291}
11292
11293fn update_snapshot(
11294 snapshot: &SharedSnapshot,
11295 module_id: Option<&str>,
11296 update: impl FnOnce(&mut SupervisorSnapshot),
11297) -> Result<(), SuperviseError> {
11298 let mut state = snapshot.lock().map_err(|_| SuperviseError::StatePoisoned {
11299 module_id: module_id.map(ToOwned::to_owned),
11300 })?;
11301 update(&mut state);
11302 Ok(())
11303}
11304
11305const SLOW_SNAPSHOT_LOCK_THRESHOLD: Duration = Duration::from_millis(250);
11306
11307fn lock_snapshot_for_control<'a>(
11308 snapshot: &'a SharedSnapshot,
11309 module_id: &str,
11310 caller: &'static str,
11311) -> Result<std::sync::MutexGuard<'a, SupervisorSnapshot>, SuperviseError> {
11312 let started_at = Instant::now();
11313 let guard = lock_snapshot(snapshot)?;
11314 let waited = started_at.elapsed();
11315 if waited >= SLOW_SNAPSHOT_LOCK_THRESHOLD {
11316 warn!(
11317 module_id = %module_id,
11318 waited_ms = waited.as_millis() as u64,
11319 caller = %caller,
11320 "slow snapshot lock"
11321 );
11322 }
11323 Ok(guard)
11324}
11325
11326fn lock_snapshot(
11327 snapshot: &SharedSnapshot,
11328) -> Result<std::sync::MutexGuard<'_, SupervisorSnapshot>, SuperviseError> {
11329 snapshot
11330 .lock()
11331 .map_err(|_| SuperviseError::StatePoisoned { module_id: None })
11332}
11333
11334#[cfg(test)]
11335mod terminal_history_tests {
11336 use std::{
11337 path::PathBuf,
11338 sync::Arc,
11339 time::{Duration, Instant},
11340 };
11341
11342 use tokio::{sync::mpsc, time::sleep};
11343
11344 use super::{
11345 apply_deliberate_severance_marker, daemon_will_restart, drain_child_to_state,
11346 drained_after_quiescence_wait, handle_reload_spawn_failure, health_restart_child,
11347 lock_snapshot, on_child_exit, record_deliberate_severance, record_wait_error_terminal,
11348 reset_restart_count, spawn_and_mark_running, update_snapshot, wait_error_exit_report,
11349 ExitKind, ExitReport, ModuleProtocol, ModuleSpec, ModuleState, NextAction, ProcessIdentity,
11350 RestartPolicy, SpawnEventKind, StopNotice, SuperviseError, SupervisedConfiguration,
11351 SupervisedModule, SupervisedModuleInner, Supervisor, SupervisorHandle,
11352 SupervisorHealthStatus, SupervisorSnapshot,
11353 };
11354 use super::Instant as ClockInstant;
11359 use crate::{
11360 registry::Registry,
11361 terminal_ring::{TerminalRing, TerminalRingConfig},
11362 };
11363 use std::sync::Mutex;
11364 use subc_control::TerminalDisposition;
11365
11366 pub(super) fn fake_aft_stub_path() -> PathBuf {
11371 let mut path = std::env::current_exe().expect("current_exe available in tests");
11372 path.pop();
11373 path.pop();
11374 path.push(if cfg!(windows) {
11375 "fake-aft-stub.exe"
11376 } else {
11377 "fake-aft-stub"
11378 });
11379 assert!(
11380 path.exists(),
11381 "fake-aft-stub not built at {}: run `cargo test -p subc-core` (which builds \
11382 [[bin]] targets) rather than `cargo test -p subc-core --lib` (which does not)",
11383 path.display()
11384 );
11385 path
11386 }
11387
11388 #[test]
11389 fn reserved_never_spawned_refuses_every_hello() {
11390 let supervisor = SupervisorHandle::default();
11395 supervisor.apply_identity_configuration(&ModuleSpec {
11396 module_id: "never-spawned".to_string(),
11397 program: PathBuf::from("/usr/bin/false"),
11398 args: Vec::new(),
11399 env: Vec::new(),
11400 reserved: true,
11401 reserved_prefixes: Vec::new(),
11402 protocol: ModuleProtocol::Subc,
11403 overlap: Default::default(),
11404 });
11405 assert!(
11406 supervisor
11407 .reserved_hello_rejection("never-spawned", Some("any-forged-nonce"))
11408 .is_some(),
11409 "forged nonce must refuse on a reserved never-spawned id"
11410 );
11411 assert!(
11412 supervisor
11413 .reserved_hello_rejection("never-spawned", None)
11414 .is_some(),
11415 "absent nonce must refuse on a reserved never-spawned id"
11416 );
11417 supervisor.set_spawn_nonce("never-spawned", "minted".to_string());
11419 supervisor.apply_identity_configuration(&ModuleSpec {
11420 module_id: "never-spawned".to_string(),
11421 program: PathBuf::from("/usr/bin/false"),
11422 args: Vec::new(),
11423 env: Vec::new(),
11424 reserved: true,
11425 reserved_prefixes: Vec::new(),
11426 protocol: ModuleProtocol::Subc,
11427 overlap: Default::default(),
11428 });
11429 assert!(supervisor
11430 .reserved_hello_rejection("never-spawned", Some("minted"))
11431 .is_none());
11432 assert!(supervisor
11433 .reserved_hello_rejection("never-spawned", Some("forged"))
11434 .is_some());
11435 }
11436
11437 fn seed_crash_restarts(state: &mut SupervisorSnapshot, count: u32) {
11440 let now = ClockInstant::now();
11441 for _ in 0..count {
11442 state.crash_restarts.push_back(now);
11443 }
11444 }
11445
11446 fn age_oldest_crash_restart_out_of_window(state: &mut SupervisorSnapshot, window: Duration) {
11450 let aged = state
11451 .crash_restarts
11452 .front()
11453 .expect("a crash restart must be recorded before it can be aged")
11454 .checked_sub(window + Duration::from_secs(1))
11455 .expect("the test clock is far enough from its origin to age an instant");
11456 state.crash_restarts[0] = aged;
11457 }
11458
11459 fn snapshot_with_restarts(enabled: bool, count: u32) -> SupervisorSnapshot {
11460 let mut state = SupervisorSnapshot::new(ModuleState::Running, enabled);
11461 seed_crash_restarts(&mut state, count);
11462 state
11463 }
11464
11465 #[test]
11466 fn daemon_owned_recovery_predicate_uses_the_pre_increment_budget() {
11467 let policy = RestartPolicy::new(3, Duration::ZERO);
11468 let now = ClockInstant::now();
11469 assert!(daemon_will_restart(
11470 &mut snapshot_with_restarts(true, 2),
11471 &policy,
11472 now
11473 ));
11474 assert!(!daemon_will_restart(
11475 &mut snapshot_with_restarts(true, 3),
11476 &policy,
11477 now
11478 ));
11479 assert!(!daemon_will_restart(
11480 &mut snapshot_with_restarts(false, 0),
11481 &policy,
11482 now
11483 ));
11484 }
11485
11486 #[test]
11487 fn crash_restart_backoff_escalates_with_in_window_count() {
11488 let policy = RestartPolicy::new(4, Duration::from_millis(100))
11489 .with_max_backoff(Duration::from_secs(30));
11490 let now = ClockInstant::now();
11491 let mut state = SupervisorSnapshot::new(ModuleState::Running, true);
11492 let schedules = (0..4)
11493 .map(|_| {
11494 state
11495 .next_crash_restart(&policy, now)
11496 .expect("the test policy allows four crash restarts")
11497 })
11498 .collect::<Vec<_>>();
11499
11500 assert_eq!(
11501 schedules
11502 .iter()
11503 .map(|schedule| schedule.restart_in_window)
11504 .collect::<Vec<_>>(),
11505 vec![0, 1, 2, 3]
11506 );
11507 assert_eq!(
11508 schedules
11509 .iter()
11510 .map(|schedule| schedule.delay)
11511 .collect::<Vec<_>>(),
11512 vec![
11513 Duration::from_millis(100),
11514 Duration::from_secs(1),
11515 Duration::from_secs(10),
11516 Duration::from_secs(30),
11517 ]
11518 );
11519 }
11520
11521 #[test]
11522 fn crash_restart_backoff_resets_after_ring_clear() {
11523 let policy = RestartPolicy::new(3, Duration::from_millis(100));
11524 let now = ClockInstant::now();
11525 let mut state = SupervisorSnapshot::new(ModuleState::Running, true);
11526 assert_eq!(
11527 state.next_crash_restart(&policy, now).unwrap().delay,
11528 Duration::from_millis(100)
11529 );
11530 assert_eq!(
11531 state.next_crash_restart(&policy, now).unwrap().delay,
11532 Duration::from_secs(1)
11533 );
11534
11535 state.clear_crash_restarts();
11536 let schedule = state
11537 .next_crash_restart(&policy, now)
11538 .expect("a cleared ring must allow another restart");
11539 assert_eq!(schedule.restart_in_window, 0);
11540 assert_eq!(schedule.delay, Duration::from_millis(100));
11541 }
11542
11543 #[test]
11544 fn crash_restart_backoff_ignores_aged_restarts() {
11545 let policy = RestartPolicy::new(3, Duration::from_millis(100));
11546 let now = ClockInstant::now();
11547 let mut state = SupervisorSnapshot::new(ModuleState::Running, true);
11548 state
11549 .next_crash_restart(&policy, now)
11550 .expect("the first restart is allowed");
11551 state
11552 .next_crash_restart(&policy, now)
11553 .expect("the second restart is allowed");
11554 state.crash_restarts[0] = now
11555 .checked_sub(policy.window + Duration::from_secs(1))
11556 .expect("the fake clock can age a restart past the window");
11557
11558 let schedule = state
11559 .next_crash_restart(&policy, now)
11560 .expect("an aged restart must release its slot");
11561 assert_eq!(schedule.restart_in_window, 1);
11562 assert_eq!(schedule.delay, Duration::from_secs(1));
11563 assert_eq!(state.crash_restarts.len(), 2);
11564 }
11565
11566 #[test]
11570 fn a_budget_spent_before_the_window_no_longer_refuses() {
11571 let policy = RestartPolicy::new(3, Duration::ZERO);
11572 let mut state = snapshot_with_restarts(true, 3);
11573 let now = ClockInstant::now();
11574 assert!(!daemon_will_restart(&mut state, &policy, now));
11575
11576 assert!(daemon_will_restart(
11577 &mut state,
11578 &policy,
11579 now + policy.window + Duration::from_secs(1)
11580 ));
11581 assert!(
11582 state.crash_restarts.is_empty(),
11583 "reading the budget must drop the instants that left the window"
11584 );
11585 }
11586
11587 fn module_with_recovery_snapshot(
11588 state: ModuleState,
11589 enabled: bool,
11590 restart_count: u32,
11591 ) -> SupervisedModule {
11592 let registry = Arc::new(Registry::default());
11593 let supervisor =
11594 Supervisor::new_for_test(Arc::clone(®istry), RestartPolicy::new(3, Duration::ZERO));
11595 let runtime = supervisor.runtime_config();
11596 let spec = ModuleSpec {
11597 module_id: "recovery-snapshot".to_string(),
11598 program: fake_aft_stub_path(),
11599 args: Vec::new(),
11600 env: Vec::new(),
11601 reserved: false,
11602 reserved_prefixes: Vec::new(),
11603 protocol: ModuleProtocol::Subc,
11604 overlap: Default::default(),
11605 };
11606 let mut snapshot = SupervisorSnapshot::new(state, enabled);
11607 seed_crash_restarts(&mut snapshot, restart_count);
11608 let (commands, _rx) = mpsc::channel(4);
11611 SupervisedModule {
11612 inner: Arc::new(SupervisedModuleInner {
11613 module_id: spec.module_id.clone(),
11614 registry,
11615 snapshot: Arc::new(Mutex::new(snapshot)),
11616 configuration: Arc::new(Mutex::new(SupervisedConfiguration {
11617 spec,
11618 health: runtime.health,
11619 })),
11620 stderr_ring: runtime.stderr_ring,
11621 terminal_ring: runtime.terminal_ring,
11622 commands,
11623 monitor: Mutex::new(None),
11624 restart_policy: runtime.restart_policy,
11625 effective_drain_timeout: runtime.effective_drain_timeout,
11626 provenance_probe: supervisor.provenance_probe.clone(),
11627 }),
11628 }
11629 }
11630
11631 #[cfg(target_os = "linux")]
11632 #[tokio::test]
11633 async fn no_cgroup_placement_does_not_block_fake_aft_stub_spawn() {
11634 let supervisor =
11635 Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default())
11636 .with_cgroup_placement(None);
11637 let result = supervisor.spawn(ModuleSpec {
11638 module_id: "no-cgroup-placement".to_string(),
11639 program: fake_aft_stub_path(),
11640 args: Vec::new(),
11641 env: Vec::new(),
11642 reserved: false,
11643 reserved_prefixes: Vec::new(),
11644 protocol: ModuleProtocol::Subc,
11645 overlap: Default::default(),
11646 });
11647
11648 assert!(
11649 result.is_ok(),
11650 "no delegation must not turn an otherwise valid spawn into a failure: {result:?}"
11651 );
11652 }
11653
11654 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
11655 async fn undecided_snapshot_uses_shared_restart_predicate() {
11656 assert!(module_with_recovery_snapshot(ModuleState::Running, true, 2)
11657 .will_recover_after_connection_loss()
11658 .unwrap());
11659 assert!(
11660 !module_with_recovery_snapshot(ModuleState::Running, true, 3)
11661 .will_recover_after_connection_loss()
11662 .unwrap()
11663 );
11664 }
11665
11666 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
11667 async fn restarting_snapshot_at_exhausted_budget_is_non_terminal() {
11668 assert!(
11669 module_with_recovery_snapshot(ModuleState::Restarting, true, 3)
11670 .will_recover_after_connection_loss()
11671 .unwrap()
11672 );
11673 }
11674
11675 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
11676 async fn terminal_phase_snapshots_are_terminal_before_budget_exhaustion() {
11677 assert!(!module_with_recovery_snapshot(ModuleState::Failed, true, 0)
11678 .will_recover_after_connection_loss()
11679 .unwrap());
11680 assert!(
11681 !module_with_recovery_snapshot(ModuleState::Disabled, true, 0)
11682 .will_recover_after_connection_loss()
11683 .unwrap()
11684 );
11685 }
11686
11687 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
11688 async fn warming_snapshot_is_limited_to_startup_phases() {
11689 for state in [
11690 ModuleState::Starting,
11691 ModuleState::Running,
11692 ModuleState::Restarting,
11693 ] {
11694 assert!(
11695 module_with_recovery_snapshot(state, true, 0)
11696 .is_warming()
11697 .unwrap(),
11698 "{state:?} should be warming"
11699 );
11700 }
11701 for state in [
11702 ModuleState::Unresponsive,
11703 ModuleState::Draining,
11704 ModuleState::Stopped,
11705 ModuleState::Failed,
11706 ModuleState::Disabled,
11707 ] {
11708 assert!(
11709 !module_with_recovery_snapshot(state, true, 0)
11710 .is_warming()
11711 .unwrap(),
11712 "{state:?} should not be warming"
11713 );
11714 }
11715 }
11716
11717 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
11718 async fn terminal_history_survives_respawn_and_keeps_both_crashes_in_order() {
11719 let registry = Arc::new(Registry::default());
11720 let supervisor =
11721 Supervisor::new_for_test(Arc::clone(®istry), RestartPolicy::new(1, Duration::ZERO));
11722 let module = supervisor
11723 .spawn(ModuleSpec {
11724 module_id: "terminal-history".to_string(),
11725 program: fake_aft_stub_path(),
11726 args: Vec::new(),
11727 env: vec![("FAKE_AFT_EXIT_CODE".to_string(), "23".to_string())],
11728 reserved: false,
11729 reserved_prefixes: Vec::new(),
11730 protocol: ModuleProtocol::Subc,
11731 overlap: Default::default(),
11732 })
11733 .unwrap();
11734
11735 let deadline = Instant::now() + Duration::from_secs(5);
11736 loop {
11737 let history = module.terminal_history();
11738 if history.entries.len() == 2 {
11739 assert_eq!(module.status().unwrap().state, ModuleState::Failed);
11740 assert_eq!(history.dropped, 0);
11741 assert_eq!(
11742 history
11743 .entries
11744 .iter()
11745 .map(|entry| entry.exit_code)
11746 .collect::<Vec<_>>(),
11747 vec![Some(23), Some(23)]
11748 );
11749 assert!(history.entries[0].at_ms <= history.entries[1].at_ms);
11750 return;
11751 }
11752 assert!(
11753 Instant::now() < deadline,
11754 "module did not retain two terminal exits: {history:?}"
11755 );
11756 sleep(Duration::from_millis(10)).await;
11757 }
11758 }
11759
11760 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
11764 async fn disable_during_crash_backoff_cancels_pending_respawn() {
11765 let backoff = Duration::from_secs(2);
11766 let supervisor = Supervisor::new_for_test(
11767 Arc::new(Registry::default()),
11768 RestartPolicy::new(10, backoff),
11769 );
11770 let module = supervisor
11771 .spawn(ModuleSpec {
11772 module_id: "disable-during-backoff".to_string(),
11773 program: fake_aft_stub_path(),
11774 args: Vec::new(),
11775 env: vec![("FAKE_AFT_EXIT_CODE".to_string(), "23".to_string())],
11776 reserved: false,
11777 reserved_prefixes: Vec::new(),
11778 protocol: ModuleProtocol::Subc,
11779 overlap: Default::default(),
11780 })
11781 .unwrap();
11782
11783 let deadline = Instant::now() + Duration::from_secs(5);
11785 loop {
11786 if module.status().unwrap().state == ModuleState::Restarting {
11787 break;
11788 }
11789 assert!(
11790 Instant::now() < deadline,
11791 "module never entered the crash backoff"
11792 );
11793 sleep(Duration::from_millis(10)).await;
11794 }
11795
11796 let started = Instant::now();
11797 module.set_enabled(false).await.unwrap();
11798 let waited = started.elapsed();
11799
11800 assert!(
11801 waited < backoff / 2,
11802 "disable waited {waited:?} behind the {backoff:?} crash backoff; the operator command must preempt the pending respawn"
11803 );
11804 assert_eq!(module.status().unwrap().state, ModuleState::Disabled);
11805
11806 sleep(backoff + Duration::from_millis(500)).await;
11808 let status = module.status().unwrap();
11809 assert_eq!(status.state, ModuleState::Disabled);
11810 assert_eq!(
11811 status.spawn_generation, 1,
11812 "module respawned after the operator disabled it"
11813 );
11814 }
11815
11816 #[cfg(unix)]
11819 fn protocol_none_sigterm_exits_clean_spec(
11820 module_id: &str,
11821 dir: &std::path::Path,
11822 ) -> (ModuleSpec, PathBuf, PathBuf) {
11823 let ready = dir.join("ready");
11824 let marker = dir.join("sigterm");
11825 let spec = ModuleSpec {
11826 module_id: module_id.to_string(),
11827 program: fake_aft_stub_path(),
11828 args: Vec::new(),
11829 env: vec![
11830 ("FAKE_AFT_NEVER_CONNECT".to_string(), "1".to_string()),
11831 (
11832 "FAKE_AFT_SIGTERM_MARKER_PATH".to_string(),
11833 marker.display().to_string(),
11834 ),
11835 (
11836 "FAKE_AFT_NEVER_CONNECT_READY_PATH".to_string(),
11837 ready.display().to_string(),
11838 ),
11839 ],
11840 reserved: false,
11841 reserved_prefixes: Vec::new(),
11842 protocol: ModuleProtocol::None,
11843 overlap: Default::default(),
11844 };
11845 (spec, ready, marker)
11846 }
11847
11848 #[cfg(unix)]
11852 async fn wait_for_file(path: &std::path::Path) {
11853 let deadline = Instant::now() + Duration::from_secs(10);
11854 while !path.exists() {
11855 assert!(
11856 Instant::now() < deadline,
11857 "{} never appeared",
11858 path.display()
11859 );
11860 sleep(Duration::from_millis(10)).await;
11861 }
11862 }
11863
11864 #[cfg(unix)]
11868 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
11869 async fn protocol_none_unrequested_clean_exit_restarts_as_a_crash() {
11870 let dir = cortexkit_test_support::ScratchDir::new("none-unrequested-clean-exit");
11871 let (spec, ready, marker) =
11872 protocol_none_sigterm_exits_clean_spec("none-unrequested-clean-exit", dir.path());
11873 let supervisor = Supervisor::new_for_test(
11874 Arc::new(Registry::default()),
11875 RestartPolicy::new(3, Duration::ZERO),
11876 );
11877 let module = supervisor.spawn(spec).unwrap();
11878 wait_for_file(&ready).await;
11879 let deadline = Instant::now() + Duration::from_secs(10);
11883 let first_pid = loop {
11884 let status = module.status().unwrap();
11885 if status.state == ModuleState::Running {
11886 if let Some(pid) = status.pid {
11887 break pid;
11888 }
11889 }
11890 assert!(
11891 Instant::now() < deadline,
11892 "a running module must report its pid after exec confirmation: {status:?}"
11893 );
11894 sleep(Duration::from_millis(10)).await;
11895 };
11896
11897 rustix::process::kill_process(
11898 rustix::process::Pid::from_raw(i32::try_from(first_pid).unwrap()).unwrap(),
11899 rustix::process::Signal::TERM,
11900 )
11901 .unwrap();
11902
11903 let deadline = Instant::now() + Duration::from_secs(10);
11904 let respawned = loop {
11905 let status = module.status().unwrap();
11906 if status.state == ModuleState::Running
11907 && status.pid.is_some_and(|pid| pid != first_pid)
11908 {
11909 break status;
11910 }
11911 assert!(
11912 Instant::now() < deadline,
11913 "protocol-none module was not respawned after an unrequested clean exit: {status:?}"
11914 );
11915 sleep(Duration::from_millis(10)).await;
11916 };
11917 assert_eq!(respawned.spawn_generation, 2);
11918 assert!(
11919 marker.exists(),
11920 "the child must have exited through its SIGTERM handler (exit 0), or this proves nothing about clean exits"
11921 );
11922
11923 let history = module.terminal_history();
11924 assert_eq!(history.entries.len(), 1, "{history:?}");
11925 let entry = &history.entries[0];
11926 assert_eq!(entry.exit_code, Some(0));
11927 assert_eq!(entry.exit_kind, subc_control::TerminalExitKind::Clean);
11928 assert_eq!(entry.disposition, TerminalDisposition::Restarting);
11929
11930 module.stop().await.unwrap();
11931 }
11932
11933 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
11937 async fn protocol_none_repeated_clean_exits_exhaust_the_restart_budget() {
11938 let supervisor = Supervisor::new_for_test(
11939 Arc::new(Registry::default()),
11940 RestartPolicy::new(1, Duration::ZERO),
11941 );
11942 let module = supervisor
11943 .spawn(ModuleSpec {
11944 module_id: "none-clean-exit-budget".to_string(),
11945 program: fake_aft_stub_path(),
11946 args: Vec::new(),
11947 env: vec![("FAKE_AFT_EXIT_CODE".to_string(), "0".to_string())],
11948 reserved: false,
11949 reserved_prefixes: Vec::new(),
11950 protocol: ModuleProtocol::None,
11951 overlap: Default::default(),
11952 })
11953 .unwrap();
11954
11955 let deadline = Instant::now() + Duration::from_secs(10);
11958 loop {
11959 let status = module.status().unwrap();
11960 if status.state == ModuleState::Failed {
11961 break;
11962 }
11963 assert!(
11964 Instant::now() < deadline,
11965 "module never exhausted its budget: {status:?} {:?}",
11966 module.terminal_history()
11967 );
11968 sleep(Duration::from_millis(10)).await;
11969 }
11970 let history = module.terminal_history();
11971 assert_eq!(
11972 history
11973 .entries
11974 .iter()
11975 .map(|entry| (entry.exit_code, entry.disposition.clone()))
11976 .collect::<Vec<_>>(),
11977 vec![
11978 (Some(0), TerminalDisposition::Restarting),
11979 (Some(0), TerminalDisposition::Failed),
11980 ]
11981 );
11982 let detail = history.entries[1]
11983 .disposition_detail
11984 .as_deref()
11985 .expect("a budget failure names the budget");
11986 assert!(detail.contains("max_restarts=1"), "{detail}");
11987 assert_eq!(module.status().unwrap().spawn_generation, 2);
11988 }
11989
11990 #[test]
11991 fn restart_budget_failure_is_published_after_its_terminal_record() {
11992 use std::io::{Read, Write};
11993 let supervisor = Supervisor::new_for_test(
11994 Arc::new(Registry::default()),
11995 RestartPolicy::new(0, Duration::ZERO),
11996 );
11997 let runtime = supervisor.runtime_config();
11998 let spec = ModuleSpec {
11999 protocol: ModuleProtocol::None,
12000 ..windowed_crash_spec("budget-publication-order")
12001 };
12002 let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::new(
12003 ModuleState::Running,
12004 true,
12005 )));
12006 lock_snapshot(&snapshot).unwrap().process_alive = true;
12007 let reaped_snapshot = snapshot.clone();
12008 let mut barrier = terminal_record_barrier(&runtime.spawn_events);
12009 let history = runtime.terminal_ring.clone();
12010 let reap = std::thread::spawn(move || {
12011 tokio::runtime::Builder::new_current_thread()
12012 .enable_all()
12013 .build()
12014 .unwrap()
12015 .block_on(on_child_exit(
12016 &spec,
12017 runtime.restart_policy,
12018 &supervisor.registry,
12019 &reaped_snapshot,
12020 &runtime.terminal_ring,
12021 &runtime.spawn_events,
12022 &runtime.child_roster,
12023 ExitReport {
12024 kind: ExitKind::Clean,
12025 code: Some(0),
12026 signal: None,
12027 at_ms: 1,
12028 },
12029 ))
12030 });
12031 let mut reached = [0];
12032 barrier.read_exact(&mut reached).unwrap();
12033 assert_eq!(&reached, b"R");
12034 let (before_state, before_alive) = {
12035 let state = lock_snapshot(&snapshot).unwrap();
12036 (state.state, state.process_alive)
12037 };
12038 let before_history = history.lock().unwrap().snapshot();
12039 barrier.write_all(b"X").unwrap();
12040 assert!(matches!(reap.join().unwrap(), NextAction::Stop { .. }));
12041 assert!(before_history.entries.is_empty());
12042 assert_ne!(
12043 before_state,
12044 ModuleState::Failed,
12045 "Failed was visible before its terminal record could be written"
12046 );
12047 assert!(before_alive);
12048 assert_eq!(lock_snapshot(&snapshot).unwrap().state, ModuleState::Failed);
12049 let history = history.lock().unwrap().snapshot();
12050 assert_eq!(history.entries.len(), 1);
12051 assert_eq!(history.entries[0].exit_code, Some(0));
12052 assert_eq!(history.entries[0].disposition, TerminalDisposition::Failed);
12053 }
12054
12055 #[test]
12056 fn crash_restart_at_budget_limit_remains_recoverable_while_recording() {
12057 use std::io::{Read, Write};
12058
12059 let supervisor = Supervisor::new_for_test(
12060 Arc::new(Registry::default()),
12061 RestartPolicy::new(3, Duration::ZERO),
12062 );
12063 let mut runtime = supervisor.runtime_config();
12064 let module = module_with_recovery_snapshot(ModuleState::Running, true, 2);
12065 let snapshot = module.inner.snapshot.clone();
12066 lock_snapshot(&snapshot).unwrap().process_alive = true;
12067 runtime.terminal_ring = module.inner.terminal_ring.clone();
12068 let spec = windowed_crash_spec(module.module_id());
12069 let mut barrier = terminal_record_barrier(&runtime.spawn_events);
12070 let reaped_snapshot = snapshot.clone();
12071 let reap = std::thread::spawn(move || {
12072 tokio::runtime::Builder::new_current_thread()
12073 .enable_all()
12074 .build()
12075 .unwrap()
12076 .block_on(on_child_exit(
12077 &spec,
12078 runtime.restart_policy,
12079 &supervisor.registry,
12080 &reaped_snapshot,
12081 &runtime.terminal_ring,
12082 &runtime.spawn_events,
12083 &runtime.child_roster,
12084 ExitReport {
12085 kind: ExitKind::Crash,
12086 code: Some(2),
12087 signal: None,
12088 at_ms: 1,
12089 },
12090 ))
12091 });
12092
12093 let mut reached = [0];
12097 barrier.read_exact(&mut reached).unwrap();
12098 assert_eq!(&reached, b"R");
12099 let before_status = module.status().unwrap();
12100 let before_history = module.terminal_history();
12101 let before_recovers = module.will_recover_after_connection_loss().unwrap();
12102 barrier.write_all(b"X").unwrap();
12103 assert!(matches!(reap.join().unwrap(), NextAction::Restart { .. }));
12104
12105 assert_eq!(before_status.state, ModuleState::Running);
12106 assert!(before_status.process_alive);
12107 assert_eq!(before_status.restart_count, 3);
12108 assert!(before_history.entries.is_empty());
12109 assert!(
12110 before_recovers,
12111 "connection cleanup would report a terminal close for a scheduled crash restart"
12112 );
12113 let restarting = module.status().unwrap();
12114 assert_eq!(restarting.state, ModuleState::Restarting);
12115 assert_eq!(restarting.restart_count, 3);
12116 assert!(module.will_recover_after_connection_loss().unwrap());
12117 let history = module.terminal_history();
12118 assert_eq!(history.entries.len(), 1);
12119 assert_eq!(
12120 history.entries[0].disposition,
12121 TerminalDisposition::Restarting
12122 );
12123
12124 update_snapshot(&snapshot, Some(module.module_id()), |state| {
12127 state.state = ModuleState::Running;
12128 state.respawn_pending = false;
12129 })
12130 .unwrap();
12131 assert!(!module.will_recover_after_connection_loss().unwrap());
12132 update_snapshot(&snapshot, Some(module.module_id()), |state| {
12133 state.enabled = false;
12134 state.respawn_pending = true;
12135 })
12136 .unwrap();
12137 assert!(!module.will_recover_after_connection_loss().unwrap());
12138 }
12139
12140 fn terminal_record_barrier(events: &super::SpawnEventFeed) -> std::net::TcpStream {
12141 use std::net::{TcpListener, TcpStream};
12142 let listener = TcpListener::bind("127.0.0.1:0").unwrap();
12143 let writer = TcpStream::connect(listener.local_addr().unwrap()).unwrap();
12144 let (barrier, _) = listener.accept().unwrap();
12145 for stream in [&writer, &barrier] {
12146 stream
12147 .set_read_timeout(Some(Duration::from_secs(10)))
12148 .unwrap();
12149 stream
12150 .set_write_timeout(Some(Duration::from_secs(10)))
12151 .unwrap();
12152 }
12153 *events.1.lock().unwrap() = Some(writer);
12154 barrier
12155 }
12156
12157 pub(super) fn before_terminal_record(events: &super::SpawnEventFeed) {
12158 use std::io::{Read, Write};
12159 let barrier = events.1.lock().unwrap().take();
12160 if let Some(mut barrier) = barrier {
12161 barrier.write_all(b"R").unwrap();
12162 let mut release = [0];
12163 barrier.read_exact(&mut release).unwrap();
12164 assert_eq!(&release, b"X");
12165 }
12166 }
12167
12168 #[test]
12169 fn health_restart_budget_failure_is_published_after_its_terminal_record() {
12170 use std::io::{Read, Write};
12171
12172 let supervisor = Supervisor::new_for_test(
12173 Arc::new(Registry::default()),
12174 RestartPolicy::new(0, Duration::ZERO),
12175 );
12176 let mut runtime = supervisor.runtime_config();
12177 let module = module_with_recovery_snapshot(ModuleState::Running, true, 0);
12178 let snapshot = module.inner.snapshot.clone();
12179 runtime.terminal_ring = module.inner.terminal_ring.clone();
12180 let spec = ModuleSpec {
12181 module_id: module.module_id().into(),
12182 program: fake_aft_stub_path(),
12183 protocol: ModuleProtocol::None,
12184 env: vec![("FAKE_AFT_EXIT_CODE".into(), "17".into())],
12185 ..windowed_crash_spec(module.module_id())
12186 };
12187 let mut barrier = terminal_record_barrier(&runtime.spawn_events);
12188 let reap = std::thread::spawn(move || {
12189 tokio::runtime::Builder::new_current_thread()
12190 .enable_all()
12191 .build()
12192 .unwrap()
12193 .block_on(async {
12194 let mut child =
12195 Some(spawn_and_mark_running(&spec, &runtime, &snapshot).unwrap());
12196 health_restart_child(
12197 &spec,
12198 &runtime,
12199 &supervisor.registry,
12200 &supervisor.process_liveness,
12201 &snapshot,
12202 &mut child,
12203 SupervisorHealthStatus::Failing,
12204 Some("health probe did not answer"),
12205 1,
12206 )
12207 .await
12208 .unwrap();
12209 assert!(child.is_none());
12210 });
12211 });
12212 let mut reached = [0];
12217 barrier.read_exact(&mut reached).unwrap();
12218 assert_eq!(&reached, b"R");
12219 let before_status = module.status().unwrap();
12220 let before_history = module.terminal_history();
12221 barrier.write_all(b"X").unwrap();
12222 reap.join().unwrap();
12223 assert!(before_history.entries.is_empty());
12224 assert_ne!(
12225 before_status.state,
12226 ModuleState::Failed,
12227 "Failed was visible before its terminal record could be written"
12228 );
12229 assert!(
12230 before_status.process_alive,
12231 "process death was visible before its terminal record could be written"
12232 );
12233 let failed = module.status().unwrap();
12234 assert_eq!(failed.state, ModuleState::Failed);
12235 assert!(!failed.process_alive);
12236 let history = module.terminal_history();
12237 assert_eq!(history.entries.len(), 1);
12238 assert_eq!(history.entries[0].disposition, TerminalDisposition::Failed);
12239 assert_eq!(
12240 history.entries[0].disposition_detail.as_deref(),
12241 Some("crash budget exhausted: max_restarts=0 within window_secs=600")
12242 );
12243 }
12244
12245 #[test]
12246 fn observed_exits_publish_process_death_after_terminal_history() {
12247 use std::io::{Read, Write};
12248 for (kind, enabled, shutdown, wait_error, expected_state) in [
12249 (ExitKind::Clean, true, false, false, ModuleState::Stopped),
12250 (ExitKind::Crash, true, false, false, ModuleState::Restarting),
12251 (ExitKind::Crash, false, false, false, ModuleState::Disabled),
12252 (
12253 ExitKind::DeliberateSeverance,
12254 true,
12255 false,
12256 false,
12257 ModuleState::Restarting,
12258 ),
12259 #[cfg(unix)]
12260 (ExitKind::Crash, true, true, false, ModuleState::Stopped),
12261 (ExitKind::Crash, true, false, true, ModuleState::Failed),
12262 ] {
12263 let supervisor = Supervisor::new_for_test(
12264 Arc::new(Registry::default()),
12265 RestartPolicy::new(1, Duration::ZERO),
12266 );
12267 let runtime = supervisor.runtime_config();
12268 #[cfg(unix)]
12269 if shutdown {
12270 runtime.child_roster.close();
12271 }
12272 #[cfg(not(unix))]
12273 let _ = shutdown;
12274 let spec = windowed_crash_spec("exit-publication-order");
12275 let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::new(
12276 ModuleState::Running,
12277 enabled,
12278 )));
12279 lock_snapshot(&snapshot).unwrap().process_alive = true;
12280 let reaped_snapshot = snapshot.clone();
12281 let history = runtime.terminal_ring.clone();
12282 let mut barrier = terminal_record_barrier(&runtime.spawn_events);
12283 let reap = std::thread::spawn(move || {
12284 if wait_error {
12285 super::fail_after_wait_error(
12286 &spec.module_id,
12287 &reaped_snapshot,
12288 &runtime.terminal_ring,
12289 &runtime.spawn_events,
12290 "wait failed".into(),
12291 );
12292 } else {
12293 tokio::runtime::Builder::new_current_thread()
12294 .enable_all()
12295 .build()
12296 .unwrap()
12297 .block_on(on_child_exit(
12298 &spec,
12299 runtime.restart_policy,
12300 &supervisor.registry,
12301 &reaped_snapshot,
12302 &runtime.terminal_ring,
12303 &runtime.spawn_events,
12304 &runtime.child_roster,
12305 ExitReport {
12306 kind,
12307 code: Some(0),
12308 signal: None,
12309 at_ms: 1,
12310 },
12311 ));
12312 }
12313 });
12314 let mut reached = [0];
12315 barrier.read_exact(&mut reached).unwrap();
12316 assert_eq!(&reached, b"R");
12317 let before_alive = lock_snapshot(&snapshot).unwrap().process_alive;
12318 let before_history = history.lock().unwrap().snapshot();
12319 barrier.write_all(b"X").unwrap();
12320 reap.join().unwrap();
12321 assert!(
12322 before_alive,
12323 "{kind:?}: death was published before recording"
12324 );
12325 assert!(before_history.entries.is_empty());
12326 assert_eq!(lock_snapshot(&snapshot).unwrap().state, expected_state);
12327 assert!(!lock_snapshot(&snapshot).unwrap().process_alive);
12328 assert_eq!(history.lock().unwrap().snapshot().entries.len(), 1);
12329 }
12330 }
12331
12332 #[tokio::test]
12333 async fn health_restart_budget_failure_remains_failed_when_journal_append_fails() {
12334 let dir = cortexkit_test_support::ScratchDir::new("health-budget-journal-failure");
12335 let path = dir.join("terminals.jsonl");
12336 std::fs::create_dir(&path).unwrap();
12337 let supervisor = Supervisor::new_for_test(
12338 Arc::new(Registry::default()),
12339 RestartPolicy::new(0, Duration::ZERO),
12340 )
12341 .with_terminal_journal(path, "health-budget-journal-failure".into());
12342 let runtime = supervisor.runtime_config();
12343 let spec = ModuleSpec {
12344 program: fake_aft_stub_path(),
12345 protocol: ModuleProtocol::None,
12346 env: vec![("FAKE_AFT_EXIT_CODE".into(), "17".into())],
12347 ..windowed_crash_spec("health-budget-journal-failure")
12348 };
12349 let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::starting()));
12350 let mut child = Some(spawn_and_mark_running(&spec, &runtime, &snapshot).unwrap());
12351 health_restart_child(
12352 &spec,
12353 &runtime,
12354 &supervisor.registry,
12355 &supervisor.process_liveness,
12356 &snapshot,
12357 &mut child,
12358 SupervisorHealthStatus::Failing,
12359 None,
12360 1,
12361 )
12362 .await
12363 .unwrap();
12364 assert_eq!(lock_snapshot(&snapshot).unwrap().state, ModuleState::Failed);
12365 assert!(!lock_snapshot(&snapshot).unwrap().process_alive);
12366 let history = runtime
12367 .terminal_ring
12368 .lock()
12369 .unwrap()
12370 .durable_history(&spec.module_id);
12371 assert!(history.journal_write_failures > 0);
12372 assert_eq!(history.entries.len(), 1);
12373 assert_eq!(history.entries[0].disposition, TerminalDisposition::Failed);
12374 }
12375
12376 #[test]
12377 fn health_restart_budget_failure_remains_failed_when_exit_recording_panics() {
12378 let supervisor = Supervisor::new_for_test(
12379 Arc::new(Registry::default()),
12380 RestartPolicy::new(0, Duration::ZERO),
12381 );
12382 let runtime = supervisor.runtime_config();
12383 let spec = ModuleSpec {
12384 program: fake_aft_stub_path(),
12385 protocol: ModuleProtocol::None,
12386 env: vec![("FAKE_AFT_EXIT_CODE".into(), "17".into())],
12387 ..windowed_crash_spec("health-budget-recording-panic")
12388 };
12389 let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::starting()));
12390 let result = std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| {
12391 tokio::runtime::Builder::new_current_thread()
12392 .enable_all()
12393 .build()
12394 .unwrap()
12395 .block_on(async {
12396 let mut child =
12397 Some(spawn_and_mark_running(&spec, &runtime, &snapshot).unwrap());
12398 runtime.spawn_events.0.lock().unwrap().seq = u64::MAX;
12399 health_restart_child(
12400 &spec,
12401 &runtime,
12402 &supervisor.registry,
12403 &supervisor.process_liveness,
12404 &snapshot,
12405 &mut child,
12406 SupervisorHealthStatus::Failing,
12407 None,
12408 1,
12409 )
12410 .await
12411 .unwrap();
12412 });
12413 }));
12414 let panic = result.expect_err("recording must still unwind");
12415 assert_eq!(
12416 panic.downcast_ref::<String>().map(String::as_str),
12417 Some("spawn event sequence exhausted")
12418 );
12419 let state = lock_snapshot(&snapshot).unwrap();
12420 assert_eq!(state.state, ModuleState::Failed);
12421 assert!(!state.process_alive);
12422 assert!(state.last_exit.is_some());
12423 }
12424
12425 #[tokio::test]
12426 async fn restart_budget_failure_remains_failed_when_journal_append_fails() {
12427 let dir = cortexkit_test_support::ScratchDir::new("budget-journal-failure");
12428 let path = dir.join("terminals.jsonl");
12429 std::fs::create_dir(&path).unwrap();
12430 let supervisor = Supervisor::new_for_test(
12431 Arc::new(Registry::default()),
12432 RestartPolicy::new(0, Duration::ZERO),
12433 )
12434 .with_terminal_journal(path, "budget-journal-failure".into());
12435 let runtime = supervisor.runtime_config();
12436 let spec = windowed_crash_spec("budget-journal-failure");
12437 let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::starting()));
12438 assert!(matches!(
12439 on_child_exit(
12440 &spec,
12441 runtime.restart_policy,
12442 &supervisor.registry,
12443 &snapshot,
12444 &runtime.terminal_ring,
12445 &runtime.spawn_events,
12446 &runtime.child_roster,
12447 crash_exit_report(1),
12448 )
12449 .await,
12450 NextAction::Stop { .. }
12451 ));
12452 assert_eq!(lock_snapshot(&snapshot).unwrap().state, ModuleState::Failed);
12453 let history = runtime
12454 .terminal_ring
12455 .lock()
12456 .unwrap()
12457 .durable_history(&spec.module_id);
12458 assert!(history.journal_write_failures > 0);
12459 assert_eq!(history.entries.len(), 1);
12460 assert_eq!(history.entries[0].disposition, TerminalDisposition::Failed);
12461 }
12462
12463 #[test]
12464 fn restart_budget_failure_remains_failed_when_exit_recording_panics() {
12465 let supervisor = Supervisor::new_for_test(
12466 Arc::new(Registry::default()),
12467 RestartPolicy::new(0, Duration::ZERO),
12468 );
12469 let runtime = supervisor.runtime_config();
12470 let spec = windowed_crash_spec("budget-recording-panic");
12471 let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::starting()));
12472 runtime.spawn_events.emit_spawned(&spec.module_id, 1, 1);
12473 runtime.spawn_events.0.lock().unwrap().seq = u64::MAX;
12476 let result = std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| {
12477 tokio::runtime::Builder::new_current_thread()
12478 .enable_all()
12479 .build()
12480 .unwrap()
12481 .block_on(on_child_exit(
12482 &spec,
12483 runtime.restart_policy,
12484 &supervisor.registry,
12485 &snapshot,
12486 &runtime.terminal_ring,
12487 &runtime.spawn_events,
12488 &runtime.child_roster,
12489 crash_exit_report(1),
12490 ))
12491 }));
12492 let panic = result.err().expect("recording must still unwind");
12493 assert_eq!(
12494 panic.downcast_ref::<String>().map(String::as_str),
12495 Some("spawn event sequence exhausted")
12496 );
12497 assert_eq!(lock_snapshot(&snapshot).unwrap().state, ModuleState::Failed);
12498 }
12499
12500 #[cfg(unix)]
12503 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
12504 async fn protocol_none_requested_stop_and_disable_do_not_respawn() {
12505 for disable in [false, true] {
12506 let label = if disable {
12507 "none-requested-disable"
12508 } else {
12509 "none-requested-stop"
12510 };
12511 let dir = cortexkit_test_support::ScratchDir::new(label);
12512 let (spec, ready, marker) = protocol_none_sigterm_exits_clean_spec(label, dir.path());
12513 let supervisor = Supervisor::new_for_test(
12514 Arc::new(Registry::default()),
12515 RestartPolicy::new(3, Duration::ZERO),
12516 );
12517 let module = supervisor.spawn(spec).unwrap();
12518 wait_for_file(&ready).await;
12519
12520 if disable {
12521 module.set_enabled(false).await.unwrap();
12522 } else {
12523 module.stop().await.unwrap();
12524 }
12525 assert!(
12526 marker.exists(),
12527 "{label}: the child must have left through its SIGTERM handler with exit 0"
12528 );
12529
12530 sleep(Duration::from_millis(500)).await;
12533 let status = module.status().unwrap();
12534 let expected = if disable {
12535 ModuleState::Disabled
12536 } else {
12537 ModuleState::Stopped
12538 };
12539 assert_eq!(status.state, expected, "{label}");
12540 assert_eq!(
12541 status.spawn_generation, 1,
12542 "{label}: respawned after a requested stop"
12543 );
12544 let history = module.terminal_history();
12545 assert_eq!(history.entries.len(), 1, "{label}: {history:?}");
12546 assert_eq!(history.entries[0].exit_code, Some(0), "{label}");
12547 assert_ne!(
12548 history.entries[0].disposition,
12549 TerminalDisposition::Restarting,
12550 "{label}"
12551 );
12552 }
12553 }
12554
12555 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
12558 async fn subc_wire_clean_exit_is_still_a_stop() {
12559 let supervisor = Supervisor::new_for_test(
12560 Arc::new(Registry::default()),
12561 RestartPolicy::new(3, Duration::ZERO),
12562 );
12563 let module = supervisor
12564 .spawn(ModuleSpec {
12565 module_id: "wire-clean-exit".to_string(),
12566 program: fake_aft_stub_path(),
12567 args: Vec::new(),
12568 env: vec![("FAKE_AFT_EXIT_CODE".to_string(), "0".to_string())],
12569 reserved: false,
12570 reserved_prefixes: Vec::new(),
12571 protocol: ModuleProtocol::Subc,
12572 overlap: Default::default(),
12573 })
12574 .unwrap();
12575
12576 let deadline = Instant::now() + Duration::from_secs(10);
12577 while module.terminal_history().entries.is_empty() {
12578 assert!(Instant::now() < deadline, "module never exited");
12579 sleep(Duration::from_millis(10)).await;
12580 }
12581 sleep(Duration::from_millis(500)).await;
12583 let status = module.status().unwrap();
12584 assert_eq!(status.state, ModuleState::Stopped);
12585 assert_eq!(status.spawn_generation, 1);
12586 let history = module.terminal_history();
12587 assert_eq!(history.entries.len(), 1, "{history:?}");
12588 assert_eq!(history.entries[0].exit_code, Some(0));
12589 assert_eq!(history.entries[0].disposition, TerminalDisposition::Stopped);
12590 }
12591
12592 #[cfg(unix)]
12593 #[tokio::test]
12594 async fn reload_exit_keeps_roster_until_terminal_is_recorded_during_shutdown() {
12595 let dir = cortexkit_test_support::ScratchDir::new("reload-roster-terminal-order");
12596 let record = dir.join("live-children.json");
12597 let supervisor = Supervisor::new_for_test(
12598 Arc::new(Registry::default()),
12599 RestartPolicy::new(0, Duration::ZERO),
12600 );
12601 let mut runtime = supervisor.runtime_config();
12602 runtime.child_roster.record_to(record.clone());
12603 let gate = Arc::new(super::ReloadExitRecordGate::default());
12604 runtime.test_reload_exit_record_gate = Some(Arc::clone(&gate));
12605 let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::starting()));
12606 let spec = ModuleSpec {
12607 module_id: "reload-exit-roster".into(),
12608 program: fake_aft_stub_path(),
12609 args: Vec::new(),
12610 env: vec![("FAKE_AFT_EXIT_CODE".into(), "23".into())],
12611 reserved: false,
12612 reserved_prefixes: Vec::new(),
12613 protocol: ModuleProtocol::Subc,
12614 overlap: Default::default(),
12615 };
12616 let mut child = None;
12617 let reload = super::finish_reload_child(
12618 &spec,
12619 &runtime,
12620 &supervisor.registry,
12621 &supervisor.process_liveness,
12622 &snapshot,
12623 &mut child,
12624 );
12625 tokio::pin!(reload);
12626 tokio::select! {
12627 result = &mut reload => panic!("reload missed the before-record gate: {result:?}"),
12628 _ = gate.reached.notified() => {}
12629 }
12630 assert!(runtime
12631 .terminal_ring
12632 .lock()
12633 .unwrap()
12634 .snapshot()
12635 .entries
12636 .is_empty());
12637 assert_eq!(
12638 crate::live_children::read_record(&record).unwrap().len(),
12639 1,
12640 "shutdown must still wait for the reaped child until its terminal record exists"
12641 );
12642 runtime.child_roster.close();
12643 gate.resume.notify_one();
12644 assert!(reload.await.is_err());
12645 assert!(crate::live_children::read_record(&record)
12646 .unwrap()
12647 .is_empty());
12648 let history = runtime.terminal_ring.lock().unwrap().snapshot();
12649 assert_eq!(history.entries.len(), 1);
12650 assert_eq!(
12651 history.entries[0].disposition,
12652 TerminalDisposition::DaemonShutdown
12653 );
12654 }
12655
12656 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
12660 async fn every_restart_increment_path_advances_lifetime_count() {
12661 let supervisor = Supervisor::new_for_test(
12662 Arc::new(Registry::default()),
12663 RestartPolicy::new(1, Duration::ZERO),
12664 );
12665 let runtime = supervisor.runtime_config();
12666 let spec = ModuleSpec {
12667 module_id: "lifetime-increment-path".to_string(),
12668 program: PathBuf::from("/unused/lifetime-increment-path"),
12669 args: Vec::new(),
12670 env: Vec::new(),
12671 reserved: false,
12672 reserved_prefixes: Vec::new(),
12673 protocol: ModuleProtocol::Subc,
12674 overlap: Default::default(),
12675 };
12676
12677 let crash_snapshot = Arc::new(Mutex::new(SupervisorSnapshot::starting()));
12678 assert!(matches!(
12679 on_child_exit(
12680 &spec,
12681 runtime.restart_policy,
12682 &supervisor.registry,
12683 &crash_snapshot,
12684 &runtime.terminal_ring,
12685 &runtime.spawn_events,
12686 &runtime.child_roster,
12687 ExitReport {
12688 kind: ExitKind::Crash,
12689 code: Some(1),
12690 signal: None,
12691 at_ms: 1,
12692 },
12693 )
12694 .await,
12695 NextAction::Restart { schedule: _ }
12696 ));
12697 let (crash_restarts, crash_lifetime) = {
12698 let state = lock_snapshot(&crash_snapshot).unwrap();
12699 (state.crash_restarts.len(), state.lifetime_restarts)
12700 };
12701 assert_eq!(crash_restarts, 1);
12702 assert_eq!(crash_lifetime, 1);
12703
12704 let health_snapshot = Arc::new(Mutex::new(SupervisorSnapshot::starting()));
12705 let mut health_child = None;
12706 assert!(matches!(
12707 health_restart_child(
12708 &spec,
12709 &runtime,
12710 &supervisor.registry,
12711 &supervisor.process_liveness,
12712 &health_snapshot,
12713 &mut health_child,
12714 SupervisorHealthStatus::Failing,
12715 None,
12716 2,
12717 )
12718 .await,
12719 Ok(())
12720 ));
12721 assert!(health_child.is_none());
12722 assert!(lock_snapshot(&health_snapshot).unwrap().respawn_pending);
12723 assert!(runtime.scheduled_respawn.lock().unwrap().take().is_some());
12724 let (health_restarts, health_lifetime) = {
12725 let state = lock_snapshot(&health_snapshot).unwrap();
12726 (state.crash_restarts.len(), state.lifetime_restarts)
12727 };
12728 assert_eq!(health_restarts, 1);
12729 assert_eq!(health_lifetime, 1);
12730
12731 let reload_snapshot = Arc::new(Mutex::new(SupervisorSnapshot::starting()));
12732 let mut reload_child = None;
12733 assert!(matches!(
12734 handle_reload_spawn_failure(
12735 &spec,
12736 &runtime,
12737 &supervisor.process_liveness,
12738 &reload_snapshot,
12739 &mut reload_child,
12740 "forced reload spawn failure".to_string(),
12741 )
12742 .await,
12743 Err(SuperviseError::ReloadFailed { .. })
12744 ));
12745 let (reload_restarts, reload_lifetime) = {
12746 let state = lock_snapshot(&reload_snapshot).unwrap();
12747 (state.crash_restarts.len(), state.lifetime_restarts)
12748 };
12749 assert_eq!(reload_restarts, 1);
12750 assert_eq!(reload_lifetime, 1);
12751 }
12752
12753 #[tokio::test]
12754 async fn deliberately_severed_live_child_records_lifetime_without_spending_restart_budget() {
12755 let supervisor = Supervisor::new_for_test(
12756 Arc::new(Registry::default()),
12757 RestartPolicy::new(3, Duration::ZERO),
12758 );
12759 let runtime = supervisor.runtime_config();
12760 let spec = ModuleSpec {
12761 module_id: "deliberately-severed".to_string(),
12762 program: PathBuf::from("/unused/deliberately-severed"),
12763 args: Vec::new(),
12764 env: Vec::new(),
12765 reserved: false,
12766 reserved_prefixes: Vec::new(),
12767 protocol: ModuleProtocol::Subc,
12768 overlap: Default::default(),
12769 };
12770 let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::starting()));
12771 let process = ProcessIdentity {
12772 pid: 41,
12773 start_time: 101,
12774 };
12775 record_deliberate_severance(&snapshot, process).unwrap();
12776 let exit_report = apply_deliberate_severance_marker(
12777 &snapshot,
12778 Some(process),
12779 ExitReport {
12780 kind: ExitKind::Crash,
12781 code: Some(1),
12782 signal: None,
12783 at_ms: 1,
12784 },
12785 );
12786 assert_eq!(exit_report.kind, ExitKind::DeliberateSeverance);
12787
12788 assert!(matches!(
12789 on_child_exit(
12790 &spec,
12791 runtime.restart_policy,
12792 &supervisor.registry,
12793 &snapshot,
12794 &runtime.terminal_ring,
12795 &runtime.spawn_events,
12796 &runtime.child_roster,
12797 exit_report,
12798 )
12799 .await,
12800 NextAction::Restart { schedule: _ }
12801 ));
12802 let state = lock_snapshot(&snapshot).unwrap();
12803 assert_eq!(state.lifetime_restarts, 1);
12804 assert_eq!(state.crash_restarts.len(), 0);
12805 }
12806
12807 #[tokio::test]
12808 async fn genuine_crash_spends_restart_budget_and_records_lifetime() {
12809 let supervisor = Supervisor::new_for_test(
12810 Arc::new(Registry::default()),
12811 RestartPolicy::new(3, Duration::ZERO),
12812 );
12813 let runtime = supervisor.runtime_config();
12814 let spec = ModuleSpec {
12815 module_id: "genuine-crash".to_string(),
12816 program: PathBuf::from("/unused/genuine-crash"),
12817 args: Vec::new(),
12818 env: Vec::new(),
12819 reserved: false,
12820 reserved_prefixes: Vec::new(),
12821 protocol: ModuleProtocol::Subc,
12822 overlap: Default::default(),
12823 };
12824 let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::starting()));
12825
12826 assert!(matches!(
12827 on_child_exit(
12828 &spec,
12829 runtime.restart_policy,
12830 &supervisor.registry,
12831 &snapshot,
12832 &runtime.terminal_ring,
12833 &runtime.spawn_events,
12834 &runtime.child_roster,
12835 ExitReport {
12836 kind: ExitKind::Crash,
12837 code: Some(1),
12838 signal: None,
12839 at_ms: 1,
12840 },
12841 )
12842 .await,
12843 NextAction::Restart { schedule: _ }
12844 ));
12845 let state = lock_snapshot(&snapshot).unwrap();
12846 assert_eq!(state.lifetime_restarts, 1);
12847 assert_eq!(state.crash_restarts.len(), 1);
12848 }
12849
12850 fn crash_exit_report(at_ms: u64) -> ExitReport {
12851 ExitReport {
12852 kind: ExitKind::Crash,
12853 code: Some(1),
12854 signal: None,
12855 at_ms,
12856 }
12857 }
12858
12859 fn windowed_crash_spec(module_id: &str) -> ModuleSpec {
12860 ModuleSpec {
12861 module_id: module_id.to_string(),
12862 program: PathBuf::from("/unused").join(module_id),
12863 args: Vec::new(),
12864 env: Vec::new(),
12865 reserved: false,
12866 reserved_prefixes: Vec::new(),
12867 protocol: ModuleProtocol::Subc,
12868 overlap: Default::default(),
12869 }
12870 }
12871
12872 #[tokio::test]
12878 async fn three_crashes_inside_the_window_stop_the_module_and_name_the_window() {
12879 let (logs, _guard) = crate::router::test_log::log_capture(tracing::Level::ERROR);
12880 let supervisor = Supervisor::new_for_test(
12881 Arc::new(Registry::default()),
12882 RestartPolicy::new(2, Duration::ZERO),
12883 );
12884 let runtime = supervisor.runtime_config();
12885 let spec = windowed_crash_spec("crash-loop-in-window");
12886 let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::starting()));
12887
12888 for attempt in 1..=2 {
12889 assert!(
12890 matches!(
12891 on_child_exit(
12892 &spec,
12893 runtime.restart_policy,
12894 &supervisor.registry,
12895 &snapshot,
12896 &runtime.terminal_ring,
12897 &runtime.spawn_events,
12898 &runtime.child_roster,
12899 crash_exit_report(attempt),
12900 )
12901 .await,
12902 NextAction::Restart { schedule: _ }
12903 ),
12904 "crash {attempt} is inside the budget and must respawn"
12905 );
12906 }
12907
12908 assert!(matches!(
12909 on_child_exit(
12910 &spec,
12911 runtime.restart_policy,
12912 &supervisor.registry,
12913 &snapshot,
12914 &runtime.terminal_ring,
12915 &runtime.spawn_events,
12916 &runtime.child_roster,
12917 crash_exit_report(3),
12918 )
12919 .await,
12920 NextAction::Stop { .. }
12921 ));
12922
12923 {
12924 let state = lock_snapshot(&snapshot).unwrap();
12925 assert_eq!(state.state, ModuleState::Failed);
12926 assert_eq!(state.crash_restarts.len(), 2);
12927 assert_eq!(state.lifetime_restarts, 2);
12928 }
12929
12930 let history = runtime
12931 .terminal_ring
12932 .lock()
12933 .expect("terminal ring is not poisoned")
12934 .snapshot();
12935 let last = history
12936 .entries
12937 .last()
12938 .expect("the refused crash is retained");
12939 assert_eq!(last.disposition, TerminalDisposition::Failed);
12940 assert_eq!(
12941 last.disposition_detail.as_deref(),
12942 Some("crash budget exhausted: max_restarts=2 within window_secs=600")
12943 );
12944
12945 let captured = crate::router::test_log::captured_logs(&logs);
12946 assert!(
12947 captured.contains("crash budget exhausted: max_restarts=2 within window_secs=600"),
12948 "the stop must be logged with its window: {captured}"
12949 );
12950 }
12951
12952 #[tokio::test]
12960 async fn a_crash_older_than_the_window_frees_its_slot_for_a_later_crash() {
12961 let supervisor = Supervisor::new_for_test(
12962 Arc::new(Registry::default()),
12963 RestartPolicy::new(2, Duration::ZERO),
12964 );
12965 let runtime = supervisor.runtime_config();
12966 let spec = windowed_crash_spec("crash-across-windows");
12967 let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::starting()));
12968
12969 for attempt in 1..=2 {
12970 assert!(matches!(
12971 on_child_exit(
12972 &spec,
12973 runtime.restart_policy,
12974 &supervisor.registry,
12975 &snapshot,
12976 &runtime.terminal_ring,
12977 &runtime.spawn_events,
12978 &runtime.child_roster,
12979 crash_exit_report(attempt),
12980 )
12981 .await,
12982 NextAction::Restart { schedule: _ }
12983 ));
12984 }
12985
12986 update_snapshot(&snapshot, Some(&spec.module_id), |state| {
12989 age_oldest_crash_restart_out_of_window(state, runtime.restart_policy.window);
12990 })
12991 .unwrap();
12992
12993 assert!(
12994 matches!(
12995 on_child_exit(
12996 &spec,
12997 runtime.restart_policy,
12998 &supervisor.registry,
12999 &snapshot,
13000 &runtime.terminal_ring,
13001 &runtime.spawn_events,
13002 &runtime.child_roster,
13003 crash_exit_report(3),
13004 )
13005 .await,
13006 NextAction::Restart { schedule: _ }
13007 ),
13008 "a crash older than the window must not hold a budget slot"
13009 );
13010
13011 let state = lock_snapshot(&snapshot).unwrap();
13012 assert_eq!(state.state, ModuleState::Restarting);
13013 assert_eq!(
13014 state.crash_restarts.len(),
13015 2,
13016 "the aged instant is dropped and the new one takes its place"
13017 );
13018 assert_eq!(
13019 state.lifetime_restarts, 3,
13020 "the ledger counts every restart, including the ones the window forgot"
13021 );
13022 }
13023
13024 #[tokio::test]
13029 async fn an_operator_restart_clears_the_ring_and_leaves_the_ledger_alone() {
13030 let supervisor = Supervisor::new_for_test(
13031 Arc::new(Registry::default()),
13032 RestartPolicy::new(2, Duration::ZERO),
13033 );
13034 let runtime = supervisor.runtime_config();
13035 let spec = windowed_crash_spec("operator-cleared-budget");
13036 let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::starting()));
13037
13038 for attempt in 1..=2 {
13039 assert!(matches!(
13040 on_child_exit(
13041 &spec,
13042 runtime.restart_policy,
13043 &supervisor.registry,
13044 &snapshot,
13045 &runtime.terminal_ring,
13046 &runtime.spawn_events,
13047 &runtime.child_roster,
13048 crash_exit_report(attempt),
13049 )
13050 .await,
13051 NextAction::Restart { schedule: _ }
13052 ));
13053 }
13054
13055 reset_restart_count(&snapshot, &spec.module_id).unwrap();
13056 {
13057 let state = lock_snapshot(&snapshot).unwrap();
13058 assert!(
13059 state.crash_restarts.is_empty(),
13060 "an operator restart returns the full budget"
13061 );
13062 assert_eq!(
13063 state.lifetime_restarts, 2,
13064 "clearing the budget must not unmake the crashes"
13065 );
13066 }
13067
13068 assert!(
13069 matches!(
13070 on_child_exit(
13071 &spec,
13072 runtime.restart_policy,
13073 &supervisor.registry,
13074 &snapshot,
13075 &runtime.terminal_ring,
13076 &runtime.spawn_events,
13077 &runtime.child_roster,
13078 crash_exit_report(3),
13079 )
13080 .await,
13081 NextAction::Restart { schedule: _ }
13082 ),
13083 "the cleared budget must be spendable again"
13084 );
13085 let state = lock_snapshot(&snapshot).unwrap();
13086 assert_eq!(state.crash_restarts.len(), 1);
13087 assert_eq!(state.lifetime_restarts, 3);
13088 }
13089
13090 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
13091 async fn severance_marker_for_a_dead_child_does_not_label_its_successor() {
13092 let severed = ProcessIdentity {
13093 pid: 41,
13094 start_time: 101,
13095 };
13096 let successor = ProcessIdentity {
13097 pid: 41,
13098 start_time: 202,
13099 };
13100 let module = module_with_recovery_snapshot(ModuleState::Running, true, 0);
13101 update_snapshot(&module.inner.snapshot, Some("recovery-snapshot"), |state| {
13102 state.pid = Some(successor.pid);
13103 state.process_start_time = Some(successor.start_time);
13104 })
13105 .unwrap();
13106 assert!(!module.record_deliberate_severance(severed).unwrap());
13107
13108 let exit_report = apply_deliberate_severance_marker(
13109 &module.inner.snapshot,
13110 Some(successor),
13111 ExitReport {
13112 kind: ExitKind::Crash,
13113 code: Some(1),
13114 signal: None,
13115 at_ms: 1,
13116 },
13117 );
13118
13119 assert_eq!(exit_report.kind, ExitKind::Crash);
13120 }
13121
13122 #[tokio::test]
13123 async fn drain_reap_marks_deliberate_severance_and_records_lifetime_without_budget() {
13124 let registry = Registry::default();
13125 let supervisor = Supervisor::new_for_test(
13126 Arc::new(Registry::default()),
13127 RestartPolicy::new(3, Duration::ZERO),
13128 );
13129 let runtime = supervisor.runtime_config();
13130 let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::starting()));
13131 let spec = ModuleSpec {
13132 module_id: "drain-deliberate-severance".to_string(),
13133 program: fake_aft_stub_path(),
13134 args: Vec::new(),
13135 env: vec![("FAKE_AFT_EXIT_CODE".to_string(), "23".to_string())],
13136 reserved: false,
13137 reserved_prefixes: Vec::new(),
13138 protocol: ModuleProtocol::Subc,
13139 overlap: Default::default(),
13140 };
13141 let mut child = spawn_and_mark_running(&spec, &runtime, &snapshot).unwrap();
13142 let process = ProcessIdentity {
13143 pid: 41,
13144 start_time: 101,
13145 };
13146 child.process_identity = Some(process);
13147 update_snapshot(&snapshot, Some(&spec.module_id), |state| {
13148 state.pid = Some(process.pid);
13149 state.process_start_time = Some(process.start_time);
13150 })
13151 .unwrap();
13152 record_deliberate_severance(&snapshot, process).unwrap();
13153
13154 drain_child_to_state(
13155 &spec.module_id,
13156 spec.protocol,
13157 StopNotice::SentOverConnection,
13160 ®istry,
13161 None,
13162 &snapshot,
13163 &runtime.terminal_ring,
13164 &runtime.spawn_events,
13165 child,
13166 Duration::from_secs(1),
13167 ModuleState::Stopped,
13168 Some(false),
13169 )
13170 .await
13171 .unwrap();
13172
13173 let state = lock_snapshot(&snapshot).unwrap();
13174 assert_eq!(
13175 state.last_exit.as_ref().map(|exit| exit.kind),
13176 Some(ExitKind::DeliberateSeverance)
13177 );
13178 assert_eq!(state.lifetime_restarts, 1);
13179 assert_eq!(state.crash_restarts.len(), 0);
13180 drop(state);
13181 let history = runtime.terminal_ring.lock().unwrap().snapshot();
13182 assert_eq!(
13183 history.entries[0].exit_kind,
13184 subc_control::TerminalExitKind::DeliberateSeverance
13185 );
13186 }
13187
13188 #[tokio::test]
13189 async fn ordinary_drain_reap_does_not_record_a_lifetime_restart() {
13190 let registry = Registry::default();
13191 let supervisor = Supervisor::new_for_test(
13192 Arc::new(Registry::default()),
13193 RestartPolicy::new(3, Duration::ZERO),
13194 );
13195 let runtime = supervisor.runtime_config();
13196 let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::starting()));
13197 let spec = ModuleSpec {
13198 module_id: "ordinary-drain".to_string(),
13199 program: fake_aft_stub_path(),
13200 args: Vec::new(),
13201 env: vec![("FAKE_AFT_EXIT_CODE".to_string(), "23".to_string())],
13202 reserved: false,
13203 reserved_prefixes: Vec::new(),
13204 protocol: ModuleProtocol::Subc,
13205 overlap: Default::default(),
13206 };
13207 let child = spawn_and_mark_running(&spec, &runtime, &snapshot).unwrap();
13208
13209 drain_child_to_state(
13210 &spec.module_id,
13211 spec.protocol,
13212 StopNotice::SentOverConnection,
13215 ®istry,
13216 None,
13217 &snapshot,
13218 &runtime.terminal_ring,
13219 &runtime.spawn_events,
13220 child,
13221 Duration::from_secs(1),
13222 ModuleState::Stopped,
13223 Some(false),
13224 )
13225 .await
13226 .unwrap();
13227
13228 let state = lock_snapshot(&snapshot).unwrap();
13229 assert_eq!(
13230 state.last_exit.as_ref().map(|exit| exit.kind),
13231 Some(ExitKind::Crash)
13232 );
13233 assert_eq!(state.lifetime_restarts, 0);
13234 assert_eq!(state.crash_restarts.len(), 0);
13235 }
13236
13237 #[test]
13238 fn fatal_connection_teardown_cannot_arm_a_marker_for_a_surviving_process() {
13239 assert!(!include_str!("server.rs")
13245 .contains("router.record_deliberate_connection_severance(ctx.connection_id)"));
13246 }
13247
13248 #[test]
13255 fn drained_after_quiescence_wait_passes_ok_through_and_forces_false_on_err() {
13256 assert!(drained_after_quiescence_wait(&Ok(true)));
13257 assert!(!drained_after_quiescence_wait(&Ok(false)));
13258 assert!(!drained_after_quiescence_wait(&Err(
13259 SuperviseError::StatePoisoned { module_id: None }
13260 )));
13261 }
13262
13263 #[test]
13272 fn wait_error_exit_report_records_a_failed_terminal_with_no_code_or_signal() {
13273 let ring = Arc::new(Mutex::new(TerminalRing::new(
13274 TerminalRingConfig::default(),
13275 0,
13276 )));
13277 record_wait_error_terminal("wait-error", &ring, &super::SpawnEventFeed::default());
13278
13279 let snapshot = ring.lock().unwrap().snapshot();
13280 assert_eq!(snapshot.entries.len(), 1);
13281 let entry = &snapshot.entries[0];
13282 assert_eq!(entry.exit_code, None);
13283 assert_eq!(entry.exit_signal, None);
13284 assert_eq!(entry.disposition, TerminalDisposition::Failed);
13285 }
13286
13287 #[test]
13288 fn wait_error_exit_path_preserves_spawn_event_density() {
13289 let feed = super::SpawnEventFeed::default();
13290 feed.configure_incarnation("wait-error-density".to_string());
13291 feed.emit_spawned("wait-error", 41, 1);
13292 let ring = Arc::new(Mutex::new(TerminalRing::new(
13293 TerminalRingConfig::default(),
13294 0,
13295 )));
13296
13297 record_wait_error_terminal("wait-error", &ring, &feed);
13298 feed.emit_spawned("after-wait-error", 42, 2);
13299
13300 let state = feed.0.lock().unwrap();
13301 let sequences = state
13302 .events
13303 .iter()
13304 .map(|event| event.cursor.seq)
13305 .collect::<Vec<_>>();
13306 assert_eq!(sequences, vec![1, 2, 3]);
13307 assert_eq!(state.events[1].kind, SpawnEventKind::Exited);
13308 assert_eq!(state.events[1].exit_code, None);
13309 assert_eq!(state.events[1].exit_signal, None);
13310 }
13311
13312 #[test]
13316 fn wait_error_exit_report_is_classified_as_a_crash() {
13317 assert_eq!(wait_error_exit_report().kind, ExitKind::Crash);
13318 }
13319}
13320
13321#[cfg(all(test, unix))]
13322mod restart_ordering_tests {
13323 use super::*;
13324 use std::io::Write;
13325 use tokio::io::AsyncReadExt;
13326
13327 #[test]
13328 #[ignore = "child process fixture"]
13329 fn serving_child() {
13330 let Ok(address_file) = std::env::var("RESTART_TEST_ADDRESS_FILE") else {
13331 return;
13332 };
13333 let listener = std::net::TcpListener::bind("127.0.0.1:0").unwrap();
13334 std::fs::write(address_file, listener.local_addr().unwrap().to_string()).unwrap();
13335 for stream in listener.incoming() {
13336 writeln!(stream.unwrap(), "{}", std::process::id()).unwrap();
13337 }
13338 }
13339
13340 async fn restart_before_publication(requests: usize) {
13341 let home = cortexkit_test_support::ScratchDir::new("restart-before-publication");
13342 let address_file = home.path().join("address");
13343 let spec = ModuleSpec {
13344 module_id: "restart-serving-child".into(),
13345 program: std::env::current_exe().unwrap(),
13346 args: vec![
13347 "--exact".into(),
13348 "supervise::restart_ordering_tests::serving_child".into(),
13349 "--ignored".into(),
13350 "--nocapture".into(),
13351 ],
13352 env: vec![(
13353 "RESTART_TEST_ADDRESS_FILE".into(),
13354 address_file.display().to_string(),
13355 )],
13356 reserved: false,
13357 reserved_prefixes: vec![],
13358 protocol: ModuleProtocol::None,
13359 overlap: Default::default(),
13360 };
13361 let mut supervisor =
13362 Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default());
13363 let (spawned_tx, spawned_rx) = oneshot::channel();
13364 let spawned_tx = Mutex::new(Some(spawned_tx));
13365 let (release_tx, release_rx) = std::sync::mpsc::channel();
13366 let release_rx = Mutex::new(release_rx);
13367 supervisor.test_after_first_spawn = AfterFirstSpawnHook(Some(Arc::new(move |_| {
13370 spawned_tx.lock().unwrap().take().unwrap().send(()).unwrap();
13371 release_rx
13372 .lock()
13373 .unwrap()
13374 .recv_timeout(Duration::from_secs(20))
13375 .unwrap();
13376 })));
13377 let registered = supervisor
13378 .register_configured_with_health(
13379 spec.clone(),
13380 true,
13381 HealthConfig::default(),
13382 None,
13383 RestartPolicy::new(3, Duration::ZERO),
13384 )
13385 .unwrap();
13386 let module = registered.module.clone();
13387 let startup = tokio::task::spawn_blocking(move || {
13388 supervisor.start_registered_configured(registered, None)
13389 });
13390 spawned_rx.await.unwrap();
13391 let old_pid = tokio::time::timeout(Duration::from_secs(10), async {
13392 let address = loop {
13393 if let Ok(address) = std::fs::read_to_string(&address_file) {
13394 if !address.is_empty() {
13395 break address;
13396 }
13397 }
13398 tokio::time::sleep(Duration::from_millis(1)).await;
13399 };
13400 let mut stream = tokio::net::TcpStream::connect(address).await.unwrap();
13401 let mut reply = String::new();
13402 stream.read_to_string(&mut reply).await.unwrap();
13403 reply.trim().parse::<u32>().unwrap()
13404 })
13405 .await
13406 .expect("child must serve before generation publication");
13407 assert_eq!(
13408 lock_snapshot(&module.inner.snapshot)
13409 .unwrap()
13410 .spawn_generation,
13411 0
13412 );
13413 let mut replies = Vec::new();
13414 for _ in 0..requests {
13415 let mut restart = Box::pin(module.restart(Some(0)));
13416 std::future::poll_fn(|cx| {
13419 assert!(std::future::Future::poll(restart.as_mut(), cx).is_pending());
13420 std::task::Poll::Ready(())
13421 })
13422 .await;
13423 replies.push(restart);
13424 }
13425 release_tx.send(()).unwrap();
13426 startup.await.unwrap().unwrap();
13427 for reply in replies {
13428 reply.await.unwrap();
13429 }
13430 let replacement = tokio::time::timeout(Duration::from_secs(10), async {
13433 loop {
13434 if lock_snapshot(&module.inner.snapshot)
13435 .unwrap()
13436 .spawn_generation
13437 > 1
13438 {
13439 break;
13440 }
13441 tokio::time::sleep(Duration::from_millis(1)).await;
13442 }
13443 })
13444 .await;
13445 module.update_spec_for_test(spec).await.unwrap();
13449 let (generation, pid) = {
13450 let state = lock_snapshot(&module.inner.snapshot).unwrap();
13451 (state.spawn_generation, state.pid)
13452 };
13453 module.stop().await.unwrap();
13454 replacement.expect("restart must publish a replacement for the already-serving child");
13455 assert_ne!(
13456 pid,
13457 Some(old_pid),
13458 "a serving child predating the restart must be replaced"
13459 );
13460 assert_eq!(generation, 2, "queued restarts must share one replacement");
13461 }
13462
13463 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
13464 async fn restart_received_after_serving_before_generation_publication_replaces_child() {
13465 restart_before_publication(1).await;
13466 }
13467
13468 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
13469 async fn restart_queued_behind_restart_does_not_kill_its_replacement() {
13470 restart_before_publication(2).await;
13471 }
13472
13473 #[tokio::test]
13474 async fn restart_queued_before_replacement_spawn_keeps_the_new_child() {
13475 let supervisor =
13476 Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default());
13477 let registered = supervisor
13478 .register_configured_with_health(
13479 ModuleSpec {
13480 module_id: "queued-before-replacement".into(),
13481 program: "/bin/sleep".into(),
13486 args: vec!["600".into()],
13487 env: vec![],
13488 reserved: false,
13489 reserved_prefixes: vec![],
13490 protocol: ModuleProtocol::None,
13491 overlap: Default::default(),
13492 },
13493 true,
13494 HealthConfig::default(),
13495 None,
13496 RestartPolicy::default(),
13497 )
13498 .unwrap();
13499 let RegisteredConfiguredModule {
13500 mut spec,
13501 mut runtime,
13502 module,
13503 mut commands,
13504 } = registered;
13505 let snapshot = &module.inner.snapshot;
13506 let mut restart = Box::pin(module.restart(Some(0)));
13507 std::future::poll_fn(|cx| {
13508 assert!(std::future::Future::poll(restart.as_mut(), cx).is_pending());
13509 std::task::Poll::Ready(())
13510 })
13511 .await;
13512 let mut child = Some(spawn_and_mark_running(&spec, &runtime, snapshot).unwrap());
13515 let replacement_pid = child.as_ref().unwrap().id();
13516 let command = commands.recv().await.unwrap();
13517 assert!(
13518 handle_supervisor_command(
13519 command,
13520 &mut spec,
13521 &mut runtime,
13522 &supervisor.registry,
13523 &supervisor.process_liveness,
13524 snapshot,
13525 &mut child,
13526 &mut commands,
13527 &mut VecDeque::new(),
13528 )
13529 .await
13530 );
13531 restart.await.unwrap();
13532 let kept_pid = child.as_ref().and_then(SupervisedChild::id);
13533 let pending = lock_snapshot(snapshot).unwrap().respawn_pending;
13534 if let Some(mut child) = child {
13535 child.start_kill().unwrap();
13536 child.wait().await.unwrap();
13537 }
13538 assert_eq!(
13539 kept_pid, replacement_pid,
13540 "a newer replacement must survive"
13541 );
13542 assert!(
13543 !pending,
13544 "a satisfied restart must not schedule another spawn"
13545 );
13546 }
13547}
13548
13549#[cfg(all(test, unix))]
13550mod health_event_tests {
13551 use super::*;
13552 use cortexkit_test_support::ScratchDir;
13553 use subc_protocol::{manifest::Concurrency, session::ModuleControlResponse};
13554
13555 struct Actor {
13556 module: SupervisedModule,
13557 registry: Arc<Registry>,
13558 forwarding: Arc<ForwardingTable>,
13559 spec: ModuleSpec,
13560 health: HealthConfig,
13561 rx: mpsc::Receiver<crate::router::OutboundFrame>,
13562 _home: ScratchDir,
13563 }
13564
13565 async fn wait_for<T>(reason: &str, mut observe: impl FnMut() -> Option<T>) -> T {
13566 let deadline = std::time::Instant::now() + Duration::from_secs(10);
13570 let tick = Instant::now();
13571 loop {
13572 if let Some(value) = observe() {
13573 assert_eq!(Instant::now(), tick, "{reason} must not wait for a timer");
13574 return value;
13575 }
13576 assert!(
13577 std::time::Instant::now() < deadline,
13578 "timed out waiting for {reason}"
13579 );
13580 tokio::task::yield_now().await;
13581 }
13582 }
13583
13584 impl Actor {
13585 async fn start(protocol: ModuleProtocol, cadence: Duration) -> Self {
13586 let home = ScratchDir::new("health-events");
13587 let registry = Arc::new(Registry::default());
13588 let forwarding = Arc::new(ForwardingTable::default());
13589 let health = HealthConfig {
13590 cadence,
13591 deadline: Duration::from_secs(3600),
13592 ..HealthConfig::default()
13593 };
13594 let supervisor =
13595 Supervisor::new_for_test(registry.clone(), RestartPolicy::new(3, Duration::ZERO))
13596 .with_forwarding(forwarding.clone())
13597 .with_health_config(health.clone());
13598 let spec = ModuleSpec {
13599 module_id: "event-health-child".into(),
13600 program: PathBuf::from("/bin/sleep"),
13608 args: vec!["600".into()],
13611 env: ["XDG_DATA_HOME", "XDG_RUNTIME_DIR", "XDG_CONFIG_HOME"]
13612 .into_iter()
13613 .map(|key| (key.into(), home.path().display().to_string()))
13614 .collect(),
13615 reserved: false,
13616 reserved_prefixes: vec![],
13617 protocol,
13618 overlap: Default::default(),
13619 };
13620 let module = supervisor.spawn(spec.clone()).unwrap();
13621 let (_, rx) = mpsc::channel(8);
13622 let actor = Self {
13623 module,
13624 registry,
13625 forwarding,
13626 spec,
13627 health,
13628 rx,
13629 _home: home,
13630 };
13631 actor
13632 .wait_for_select(|checkpoint| checkpoint.generation > 0)
13633 .await;
13634 actor
13635 }
13636
13637 fn turns(&self) -> u64 {
13638 lock_snapshot(&self.module.inner.snapshot)
13639 .unwrap()
13640 .actor_turns
13641 }
13642
13643 async fn wait_for_select(
13644 &self,
13645 expected: impl Fn(&ActorSelectCheckpoint) -> bool,
13646 ) -> ActorSelectCheckpoint {
13647 wait_for("actor select checkpoint", || {
13648 let state = lock_snapshot(&self.module.inner.snapshot).unwrap();
13649 state.actor_select.filter(|checkpoint| {
13650 checkpoint.generation == state.spawn_generation
13651 && state.state == ModuleState::Running
13652 && state.process_alive
13653 && state.reported_pid().is_some()
13654 && expected(checkpoint)
13655 })
13656 })
13657 .await
13658 }
13659
13660 fn hello(&mut self, connection: u64, health: bool) {
13661 let manifest =
13662 subc_protocol::manifest::ModuleManifest::builder(&self.spec.module_id, "1.0.0")
13663 .protocol_ver(subc_protocol::PROTOCOL_VERSION)
13664 .build();
13665 let connection = ConnectionId::new(connection);
13666 let (tx, rx) = mpsc::channel(8);
13667 self.rx = rx;
13668 self.forwarding
13669 .register_module_connection(
13670 connection,
13671 self.spec.module_id.clone(),
13672 subc_protocol::PROTOCOL_VERSION,
13673 Concurrency::ModuleManaged,
13674 FrameSink::new(tx),
13675 )
13676 .unwrap();
13677 self.registry
13678 .register_with_control_ops(
13679 manifest,
13680 subc_protocol::PROTOCOL_VERSION,
13681 connection,
13682 if health {
13683 vec![MODULE_CONTROL_OP_HEALTH_CHECK.into()]
13684 } else {
13685 vec![]
13686 },
13687 )
13688 .unwrap();
13689 }
13690
13691 async fn probe(&mut self) -> crate::router::OutboundFrame {
13692 let frame = wait_for("outbound health probe", || match self.rx.try_recv() {
13693 Ok(frame) => Some(frame),
13694 Err(mpsc::error::TryRecvError::Empty) => None,
13695 Err(mpsc::error::TryRecvError::Disconnected) => panic!("health peer disconnected"),
13696 })
13697 .await;
13698 let body: Value = serde_json::from_slice(&frame.body).unwrap();
13699 assert_eq!(body["op"], MODULE_CONTROL_OP_HEALTH_CHECK);
13700 frame
13701 }
13702
13703 fn no_probe(&mut self) {
13704 assert!(matches!(
13705 self.rx.try_recv(),
13706 Err(mpsc::error::TryRecvError::Empty)
13707 ));
13708 }
13709
13710 fn answer(&self, connection: u64, frame: crate::router::OutboundFrame) {
13711 self.forwarding
13712 .complete_module_control_rpc(
13713 ConnectionId::new(connection),
13714 frame.header.corr,
13715 Some(MODULE_CONTROL_OP_HEALTH_CHECK),
13716 ModuleControlRpcOutcome::Response(ModuleControlResponse::HealthCheck {
13717 status: HealthStatus::Ok,
13718 detail: None,
13719 metrics: None,
13720 }),
13721 )
13722 .unwrap();
13723 }
13724 }
13725
13726 #[tokio::test(start_paused = true)]
13727 async fn no_health_children_park_for_sixty_seconds() {
13728 let none = Actor::start(ModuleProtocol::None, Duration::from_secs(30)).await;
13729 let unregistered = Actor::start(ModuleProtocol::Subc, Duration::from_secs(30)).await;
13730 let mut unadvertised = Actor::start(ModuleProtocol::Subc, Duration::from_secs(30)).await;
13731 unadvertised.hello(1, false);
13732 for _ in 0..6000 {
13737 tokio::time::advance(Duration::from_millis(10)).await;
13738 tokio::task::yield_now().await;
13739 }
13740 for actor in [&none, &unregistered, &unadvertised] {
13741 assert!(
13742 actor.turns() <= 5,
13743 "no-health actor woke {} times",
13744 actor.turns()
13745 );
13746 assert_eq!(actor.module.state().unwrap(), ModuleState::Running);
13747 }
13748 }
13749
13750 #[tokio::test(start_paused = true)]
13751 async fn late_hello_starts_a_due_probe_in_the_notification_tick() {
13752 let mut actor = Actor::start(ModuleProtocol::Subc, Duration::ZERO).await;
13756 let tick = Instant::now();
13757 actor.hello(1, true);
13758 actor.probe().await;
13759 assert_eq!(Instant::now(), tick, "HELLO must not wait for a poll");
13760 }
13761
13762 #[tokio::test(start_paused = true)]
13763 async fn hello_keeps_the_jittered_cadence_and_catalog_updates_do_not_reset_it() {
13764 let mut actor = Actor::start(ModuleProtocol::Subc, Duration::from_secs(30)).await;
13765 let hello_tick = Instant::now();
13766 actor.hello(1, true);
13767 let armed = actor
13768 .wait_for_select(|checkpoint| {
13769 checkpoint.registered_connection == Some(ConnectionId::new(1))
13770 && checkpoint.next_probe_at.is_some()
13771 })
13772 .await;
13773 let delay = jittered_health_delay(&actor.spec.module_id, 0, actor.health.cadence);
13774 assert!((Duration::from_secs(30)..Duration::from_secs(33)).contains(&delay));
13775 let first_deadline = hello_tick + delay;
13776 assert_eq!(armed.next_probe_at, Some(first_deadline));
13777 tokio::time::advance(Duration::from_secs(29)).await;
13778 let turns = actor.turns();
13779 actor
13780 .registry
13781 .replace_catalog_for_connection(ConnectionId::new(1), vec![], None, Some(true))
13782 .unwrap();
13783 let updated = actor
13784 .wait_for_select(|checkpoint| checkpoint.turn > turns)
13785 .await;
13786 assert_eq!(
13787 updated.next_probe_at,
13788 Some(first_deadline),
13789 "catalog update must not reset cadence"
13790 );
13791 actor.no_probe();
13792 tokio::time::advance(first_deadline - Instant::now()).await;
13793 let frame = actor.probe().await;
13794 actor.answer(1, frame);
13795 let next = jittered_health_delay(&actor.spec.module_id, 1, actor.health.cadence);
13796 let next_deadline = Instant::now() + next;
13797 let rearmed = actor
13798 .wait_for_select(|checkpoint| {
13799 checkpoint
13800 .next_probe_at
13801 .is_some_and(|deadline| deadline > first_deadline)
13802 })
13803 .await;
13804 assert_eq!(rearmed.next_probe_at, Some(next_deadline));
13805 tokio::time::advance(next - Duration::from_nanos(1)).await;
13806 actor.no_probe();
13807 tokio::time::advance(Duration::from_nanos(1)).await;
13808 actor.probe().await;
13809 }
13810
13811 #[tokio::test(start_paused = true)]
13812 async fn disconnect_disarms_and_reconnect_rearms_health() {
13813 let mut actor = Actor::start(ModuleProtocol::Subc, Duration::from_secs(30)).await;
13814 actor.hello(1, true);
13815 actor
13816 .wait_for_select(|checkpoint| {
13817 checkpoint.registered_connection == Some(ConnectionId::new(1))
13818 && checkpoint.next_probe_at.is_some()
13819 })
13820 .await;
13821 let turns = actor.turns();
13822 let tick = Instant::now();
13823 actor
13824 .registry
13825 .deregister_connection(ConnectionId::new(1))
13826 .unwrap();
13827 let disarmed = actor
13828 .wait_for_select(|checkpoint| {
13829 checkpoint.turn > turns && checkpoint.registered_connection.is_none()
13830 })
13831 .await;
13832 assert_eq!(disarmed.next_probe_at, None);
13833 assert_eq!(disarmed.wake_after, None);
13834 assert_eq!(
13835 Instant::now(),
13836 tick,
13837 "disconnect must disarm in the notification tick"
13838 );
13839 tokio::time::advance(Duration::from_secs(60)).await;
13840 actor.no_probe();
13841 let reconnect_tick = Instant::now();
13842 actor.hello(2, true);
13843 let rearmed = actor
13844 .wait_for_select(|checkpoint| {
13845 checkpoint.registered_connection == Some(ConnectionId::new(2))
13846 && checkpoint.next_probe_at.is_some()
13847 })
13848 .await;
13849 let delay = jittered_health_delay(&actor.spec.module_id, 0, actor.health.cadence);
13850 assert_eq!(rearmed.next_probe_at, Some(reconnect_tick + delay));
13851 tokio::time::advance(delay).await;
13852 actor.probe().await;
13853 }
13854
13855 #[tokio::test(start_paused = true)]
13856 async fn reregistration_without_health_disarms_the_old_schedule() {
13857 let mut actor = Actor::start(ModuleProtocol::Subc, Duration::from_secs(30)).await;
13858 actor.hello(1, true);
13859 actor
13860 .wait_for_select(|checkpoint| {
13861 checkpoint.registered_connection == Some(ConnectionId::new(1))
13862 && checkpoint.next_probe_at.is_some()
13863 })
13864 .await;
13865 actor
13866 .registry
13867 .deregister_connection(ConnectionId::new(1))
13868 .unwrap();
13869 actor.hello(2, false);
13870 let disarmed = actor
13871 .wait_for_select(|checkpoint| {
13872 checkpoint.registered_connection == Some(ConnectionId::new(2))
13873 })
13874 .await;
13875 assert_eq!(disarmed.next_probe_at, None);
13876 assert_eq!(disarmed.wake_after, None);
13877 tokio::time::advance(Duration::from_secs(60)).await;
13878 actor.no_probe();
13879 }
13880
13881 #[tokio::test(start_paused = true)]
13882 async fn restart_waits_for_its_own_hello_before_rearming_health() {
13883 let mut actor = Actor::start(ModuleProtocol::Subc, Duration::from_secs(30)).await;
13884 actor.hello(1, true);
13885 let armed = actor
13886 .wait_for_select(|checkpoint| {
13887 checkpoint.registered_connection == Some(ConnectionId::new(1))
13888 && checkpoint.next_probe_at.is_some()
13889 })
13890 .await;
13891 actor
13895 .registry
13896 .deregister_connection(ConnectionId::new(1))
13897 .unwrap();
13898 actor.module.restart(Some(0)).await.unwrap();
13899 let parked = actor
13900 .wait_for_select(|checkpoint| {
13901 checkpoint.generation > armed.generation
13902 && checkpoint.registered_connection.is_none()
13903 && checkpoint.next_probe_at.is_none()
13904 })
13905 .await;
13906 assert_eq!(parked.wake_after, None);
13907 actor.rx.close();
13908 while actor.rx.try_recv().is_ok() {}
13911 let turns = actor.turns();
13912 tokio::time::advance(Duration::from_secs(60)).await;
13913 let after = actor.turns();
13914 eprintln!(
13915 "restart park turns: before={turns} after={after} delta={}",
13916 after - turns
13917 );
13918 assert_eq!(after, turns, "replacement must park before HELLO");
13919 let hello_tick = Instant::now();
13920 actor.hello(2, true);
13921 let rearmed = actor
13922 .wait_for_select(|checkpoint| {
13923 checkpoint.registered_connection == Some(ConnectionId::new(2))
13924 && checkpoint.next_probe_at.is_some()
13925 })
13926 .await;
13927 let delay = jittered_health_delay(&actor.spec.module_id, 0, actor.health.cadence);
13928 assert_eq!(rearmed.next_probe_at, Some(hello_tick + delay));
13929 tokio::time::advance(delay).await;
13930 actor.probe().await;
13931 }
13932
13933 #[tokio::test(start_paused = true)]
13934 async fn rescan_adds_and_removes_http_health_without_polling() {
13935 use tokio::{
13936 io::{AsyncReadExt, AsyncWriteExt},
13937 net::TcpListener,
13938 };
13939 let mut actor = Actor::start(ModuleProtocol::None, Duration::from_secs(30)).await;
13940 let listener = TcpListener::bind("127.0.0.1:0").await.unwrap();
13941 actor.health.http = Some(format!("http://{}/health", listener.local_addr().unwrap()));
13942 actor.health.cadence = Duration::from_millis(1);
13944 let (started_tx, mut started_rx) = mpsc::channel(8);
13945 let peer = tokio::spawn(async move {
13946 loop {
13947 let (mut socket, _) = listener.accept().await.unwrap();
13948 let mut request = [0; 1024];
13949 assert!(socket.read(&mut request).await.unwrap() > 0);
13950 started_tx.send(()).await.unwrap();
13951 socket
13952 .write_all(b"HTTP/1.1 200 OK\r\nContent-Length: 0\r\n\r\n")
13953 .await
13954 .unwrap();
13955 }
13956 });
13957 let tick = Instant::now();
13958 let turns = actor.turns();
13959 actor
13960 .module
13961 .update_configuration(actor.spec.clone(), actor.health.clone(), None)
13962 .await
13963 .unwrap();
13964 let armed = actor
13965 .wait_for_select(|checkpoint| {
13966 checkpoint.turn > turns && checkpoint.next_probe_at.is_some()
13967 })
13968 .await;
13969 assert_eq!(armed.next_probe_at, Some(tick + Duration::from_millis(1)));
13970 assert_eq!(Instant::now(), tick);
13971 tokio::time::advance(Duration::from_millis(1)).await;
13972 let mut started = false;
13973 wait_for("successful HTTP probe", || {
13974 started |= started_rx.try_recv().is_ok();
13975 (started && actor.module.status().unwrap().health.status == SupervisorHealthStatus::Ok)
13976 .then_some(())
13977 })
13978 .await;
13979 assert!(started, "added HTTP check must start at its first deadline");
13980 assert_eq!(
13981 actor.module.status().unwrap().health.status,
13982 SupervisorHealthStatus::Ok
13983 );
13984 assert_eq!(Instant::now(), tick + Duration::from_millis(1));
13985 actor.health.http = None;
13986 let turns = actor.turns();
13987 actor
13988 .module
13989 .update_configuration(actor.spec.clone(), actor.health.clone(), None)
13990 .await
13991 .unwrap();
13992 let parked = actor
13993 .wait_for_select(|checkpoint| {
13994 checkpoint.turn > turns && checkpoint.next_probe_at.is_none()
13995 })
13996 .await;
13997 assert_eq!(parked.wake_after, None);
13998 let turns = actor.turns();
13999 tokio::time::advance(Duration::from_secs(60)).await;
14000 assert_eq!(
14001 actor.turns(),
14002 turns,
14003 "removed HTTP check must park the actor"
14004 );
14005 assert!(
14006 started_rx.try_recv().is_err(),
14007 "removed HTTP check must stay disarmed"
14008 );
14009 assert_eq!(
14010 actor.module.status().unwrap().health,
14011 ModuleHealthStatus::default()
14012 );
14013 peer.abort();
14014 }
14015}
14016
14017#[cfg(test)]
14018mod drain_busy_reprobe_tests {
14019 use super::*;
14023 use std::collections::BTreeMap;
14024 use std::sync::Mutex as StdMutex;
14025 use subc_protocol::{manifest::Concurrency, session::ModuleControlResponse};
14026
14027 const MODULE_ID: &str = "drain-reprobe-module";
14028 const CONNECTION: u64 = 7;
14029 const GAUGE: &str = "runs_in_flight";
14030 const CADENCE: Duration = Duration::from_secs(30);
14033 const CEILING: Duration = Duration::from_secs(30);
14034
14035 #[derive(Clone, Copy)]
14037 enum Answer {
14038 BusyUntil { until: Instant, latency: Duration },
14041 OmitsGauge,
14043 Silent,
14045 }
14046
14047 struct Drain {
14048 forwarding: Arc<ForwardingTable>,
14049 runtime: SupervisorRuntimeConfig,
14050 endpoint: crate::ModuleEndpointId,
14051 probes: Arc<StdMutex<Vec<Instant>>>,
14053 responder: tokio::task::JoinHandle<()>,
14054 }
14055
14056 impl Drain {
14057 fn start(answer: Answer) -> Self {
14058 let registry = Arc::new(Registry::default());
14059 let forwarding = Arc::new(ForwardingTable::default());
14060 let health = HealthConfig {
14061 cadence: CADENCE,
14062 ..HealthConfig::default()
14063 };
14064 let runtime = Supervisor::new_for_test(registry, RestartPolicy::new(3, Duration::ZERO))
14065 .with_forwarding(forwarding.clone())
14066 .with_health_config(health)
14067 .runtime_config();
14068 let (tx, mut rx) = mpsc::channel::<crate::router::OutboundFrame>(8);
14069 let endpoint = forwarding
14070 .register_module_connection(
14071 ConnectionId::new(CONNECTION),
14072 MODULE_ID.to_string(),
14073 subc_protocol::PROTOCOL_VERSION,
14074 Concurrency::ModuleManaged,
14075 FrameSink::new(tx),
14076 )
14077 .unwrap();
14078 let probes = Arc::new(StdMutex::new(Vec::new()));
14079 let responder = tokio::spawn({
14080 let forwarding = forwarding.clone();
14081 let probes = probes.clone();
14082 async move {
14083 while let Some(frame) = rx.recv().await {
14084 let body: Value = serde_json::from_slice(&frame.body).unwrap();
14085 assert_eq!(body["op"], MODULE_CONTROL_OP_HEALTH_CHECK);
14086 probes.lock().unwrap().push(Instant::now());
14087 let metrics = match answer {
14088 Answer::BusyUntil { until, latency } => {
14089 sleep(latency).await;
14090 let busy = u64::from(Instant::now() < until);
14091 serde_json::json!({ GAUGE: busy })
14092 }
14093 Answer::OmitsGauge => serde_json::json!({ "queue_depth": 0 }),
14094 Answer::Silent => continue,
14095 };
14096 let _ = forwarding.complete_module_control_rpc(
14099 ConnectionId::new(CONNECTION),
14100 frame.header.corr,
14101 Some(MODULE_CONTROL_OP_HEALTH_CHECK),
14102 ModuleControlRpcOutcome::Response(ModuleControlResponse::HealthCheck {
14103 status: HealthStatus::Ok,
14104 detail: None,
14105 metrics: Some(metrics),
14106 }),
14107 );
14108 }
14109 }
14110 });
14111 Self {
14112 forwarding,
14113 runtime,
14114 endpoint,
14115 probes,
14116 responder,
14117 }
14118 }
14119
14120 async fn run(&self, ceiling: Duration) -> (bool, Duration) {
14123 let (drained, elapsed, _) = self.run_traced(ceiling).await;
14124 (drained, elapsed)
14125 }
14126
14127 async fn run_traced(&self, ceiling: Duration) -> (bool, Duration, DrainGaugeTrace) {
14129 let started = Instant::now();
14130 let mut trace = DrainGaugeTrace::new(&[GAUGE.to_string()]);
14131 let drained = wait_for_forwarding_quiescence(
14132 &self.forwarding,
14133 MODULE_ID,
14134 &self.runtime,
14135 self.endpoint,
14136 started + ceiling,
14137 &[GAUGE.to_string()],
14138 DrainScope::Active,
14139 &mut trace,
14140 )
14141 .await
14142 .unwrap();
14143 (drained, started.elapsed(), trace)
14144 }
14145
14146 fn probes(&self) -> Vec<Instant> {
14147 self.probes.lock().unwrap().clone()
14148 }
14149 }
14150
14151 impl Drop for Drain {
14152 fn drop(&mut self) {
14153 self.responder.abort();
14154 }
14155 }
14156
14157 #[tokio::test(start_paused = true)]
14158 async fn busy_at_first_probe_then_quiet_drains_long_before_the_ceiling() {
14159 let drain = Drain::start(Answer::BusyUntil {
14160 until: Instant::now() + Duration::from_secs(1),
14161 latency: Duration::ZERO,
14162 });
14163 let (drained, elapsed) = drain.run(CEILING).await;
14164 assert!(
14165 drained,
14166 "gauges that read zero during the drain must let it finish drained, \
14167 not wait out the {CADENCE:?} health cadence (elapsed {elapsed:?})"
14168 );
14169 assert!(
14170 elapsed < Duration::from_secs(3),
14171 "drain took {elapsed:?} after the gauges went quiet at 1s"
14172 );
14173 let probes = drain.probes();
14174 assert!(
14175 probes.len() >= 2,
14176 "the module was busy at the first probe, so a quiet verdict needs a second: {probes:?}"
14177 );
14178 }
14179
14180 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
14185 async fn an_instant_busy_first_answer_does_not_stop_later_probes_in_real_time() {
14186 let drain = Drain::start(Answer::BusyUntil {
14187 until: Instant::now() + Duration::from_millis(600),
14188 latency: Duration::ZERO,
14189 });
14190 let (drained, elapsed, trace) = drain.run_traced(CEILING).await;
14191 assert!(drained, "the gauge fell at 600ms: {trace}");
14192 assert!(
14193 elapsed < Duration::from_secs(3),
14194 "drain took {elapsed:?} after the gauge fell at 600ms: {trace}"
14195 );
14196 let probes = drain.probes();
14197 assert!(
14198 probes.len() >= 2 && trace.probes as usize == probes.len(),
14199 "the drain must re-probe after the first busy answer: {probes:?}, {trace}"
14200 );
14201 }
14202
14203 #[tokio::test(start_paused = true)]
14204 async fn busy_throughout_ends_undrained_at_the_ceiling_with_bounded_probes() {
14205 let started = Instant::now();
14206 let drain = Drain::start(Answer::BusyUntil {
14207 until: started + Duration::from_secs(3600),
14208 latency: Duration::ZERO,
14209 });
14210 let (drained, elapsed) = drain.run(CEILING).await;
14211 assert!(!drained, "a module busy throughout must not drain");
14212 assert!(
14213 elapsed >= CEILING && elapsed < CEILING + drain_busy_reprobe_interval(),
14214 "a busy module is held until the ceiling and no longer: {elapsed:?}"
14215 );
14216 let probes = drain.probes();
14219 let max_probes = (CEILING.as_millis() / drain_busy_reprobe_interval().as_millis()) + 1;
14220 assert!(
14221 probes.len() as u128 <= max_probes,
14222 "{} probes in a {CEILING:?} drain exceeds the {max_probes} the interval allows",
14223 probes.len()
14224 );
14225 for pair in probes.windows(2) {
14226 assert!(
14227 pair[1] - pair[0] >= drain_busy_reprobe_interval(),
14228 "consecutive probes {:?} apart",
14229 pair[1] - pair[0]
14230 );
14231 }
14232 }
14233
14234 #[tokio::test(start_paused = true)]
14235 async fn a_slow_probe_is_never_overlapped_by_the_next() {
14236 let latency = Duration::from_secs(1);
14239 let drain = Drain::start(Answer::BusyUntil {
14240 until: Instant::now() + Duration::from_secs(3600),
14241 latency,
14242 });
14243 let (drained, _) = drain.run(Duration::from_secs(5)).await;
14244 assert!(!drained);
14245 let probes = drain.probes();
14246 assert!(probes.len() >= 2, "the drain must re-probe: {probes:?}");
14247 for pair in probes.windows(2) {
14248 assert!(
14249 pair[1] - pair[0] >= latency + drain_busy_reprobe_interval(),
14250 "a probe started {:?} after the previous one, before it could finish",
14251 pair[1] - pair[0]
14252 );
14253 }
14254 }
14255
14256 #[tokio::test(start_paused = true)]
14257 async fn an_unanswered_probe_cannot_hold_the_drain_past_its_ceiling() {
14258 let drain = Drain::start(Answer::Silent);
14259 let ceiling = Duration::from_secs(2);
14260 let (drained, elapsed) = drain.run(ceiling).await;
14261 assert!(!drained, "an unanswered probe counts as busy");
14262 assert!(
14263 elapsed >= ceiling && elapsed < ceiling + drain_busy_reprobe_interval(),
14264 "the probe deadline is capped at the drain ceiling: {elapsed:?}"
14265 );
14266 assert_eq!(
14267 drain.probes().len(),
14268 1,
14269 "the unanswered probe holds the only slot until the ceiling"
14270 );
14271 }
14272
14273 #[derive(Clone, Default)]
14275 struct LogCapture {
14276 events: Arc<StdMutex<Vec<BTreeMap<String, String>>>>,
14277 }
14278
14279 impl<S: tracing::Subscriber> tracing_subscriber::Layer<S> for LogCapture {
14280 fn on_event(
14281 &self,
14282 event: &tracing::Event<'_>,
14283 _context: tracing_subscriber::layer::Context<'_, S>,
14284 ) {
14285 struct Fields(BTreeMap<String, String>);
14286 impl tracing::field::Visit for Fields {
14287 fn record_debug(&mut self, field: &tracing::field::Field, value: &dyn fmt::Debug) {
14288 self.0
14289 .insert(field.name().to_string(), format!("{value:?}"));
14290 }
14291 }
14292 let mut fields = Fields(BTreeMap::new());
14293 event.record(&mut fields);
14294 self.events.lock().unwrap().push(fields.0);
14295 }
14296 }
14297
14298 impl LogCapture {
14299 fn install(&self) -> tracing::subscriber::DefaultGuard {
14300 use tracing_subscriber::layer::SubscriberExt;
14301 tracing::subscriber::set_default(tracing_subscriber::registry().with(self.clone()))
14302 }
14303
14304 fn with_message(&self, needle: &str) -> Vec<BTreeMap<String, String>> {
14305 self.events
14306 .lock()
14307 .unwrap()
14308 .iter()
14309 .filter(|fields| fields.get("message").is_some_and(|m| m.contains(needle)))
14310 .cloned()
14311 .collect()
14312 }
14313 }
14314
14315 #[tokio::test(start_paused = true)]
14320 async fn a_missing_declared_gauge_is_named_on_the_drain_timeout_line() {
14321 let capture = LogCapture::default();
14322 let _guard = capture.install();
14323 let drain = Drain::start(Answer::OmitsGauge);
14324 let (drained, _, trace) = drain.run_traced(Duration::from_secs(2)).await;
14325 assert!(!drained, "a gauge never reported cannot read quiet");
14326 assert!(trace.probes >= 2, "the drain must re-probe: {trace:?}");
14327
14328 let missing = capture.with_message("lacks a declared busy gauge");
14329 assert_eq!(missing.len(), 1, "one warning per drain: {missing:?}");
14330 assert_eq!(
14331 missing[0].get("missing_gauges").map(String::as_str),
14332 Some(GAUGE)
14333 );
14334
14335 warn_drain_timed_out(
14336 MODULE_ID,
14337 Duration::from_secs(2),
14338 RouteCloseReason::Restart,
14339 &Default::default(),
14340 &trace,
14341 );
14342 let timeouts = capture.with_message("route drain timed out");
14343 assert_eq!(timeouts.len(), 1, "{timeouts:?}");
14344 let gauges = &timeouts[0]["busy_gauges"];
14345 assert!(
14346 gauges.starts_with(&format!("{GAUGE}: missing_from_health_body @")),
14347 "the timeout line must name the gauge and why it read busy: {gauges}"
14348 );
14349 assert!(
14350 gauges.ends_with("; no value ever read"),
14351 "no value was ever read for the gauge: {gauges}"
14352 );
14353 assert_eq!(timeouts[0]["gauge_probes"], trace.probes.to_string());
14354 }
14355
14356 #[tokio::test(start_paused = true)]
14360 async fn the_drain_trace_names_a_busy_value_and_an_unanswered_probe() {
14361 let busy = Drain::start(Answer::BusyUntil {
14362 until: Instant::now() + Duration::from_secs(3600),
14363 latency: Duration::ZERO,
14364 });
14365 let (_, _, trace) = busy.run_traced(Duration::from_secs(1)).await;
14366 let rendered = trace.to_string();
14367 assert!(
14368 rendered.starts_with(&format!("{GAUGE}: non_zero(1) @"))
14369 && rendered.contains("; last value non_zero(1) @"),
14370 "{rendered}"
14371 );
14372
14373 let silent = Drain::start(Answer::Silent);
14374 let (_, _, trace) = silent.run_traced(Duration::from_secs(1)).await;
14375 assert_eq!(
14378 trace.to_string(),
14379 format!("{GAUGE}: probe_failed(no-answer) @1000ms; no value ever read")
14380 );
14381 assert_eq!(trace.probes, 1);
14382 }
14383
14384 #[test]
14386 fn a_gauge_that_is_not_a_count_is_busy_and_shown_as_sent() {
14387 let gauges = [GAUGE.to_string()];
14388 let verdicts = gauge_verdicts(Some(&serde_json::json!({ GAUGE: 0.0 })), &gauges);
14389 assert_eq!(verdicts, vec![GaugeVerdict::NotACount("0.0".to_string())]);
14390 assert_eq!(
14391 busy_gauge_observation(&verdicts),
14392 BusyGaugeObservation::Busy
14393 );
14394 let zero = gauge_verdicts(Some(&serde_json::json!({ GAUGE: 0 })), &gauges);
14395 assert_eq!(
14396 busy_gauge_observation(&zero),
14397 BusyGaugeObservation::Quiescent
14398 );
14399 let none = gauge_verdicts(None, &gauges);
14400 assert_eq!(busy_gauge_observation(&none), BusyGaugeObservation::Omitted);
14401 }
14402}
14403
14404#[cfg(test)]
14405mod health_evidence_tests {
14406 use super::{HealthProbeError, HealthProbeEvidence};
14407 use std::collections::HashSet;
14408
14409 #[test]
14417 fn only_a_dead_lane_is_proof_of_death() {
14418 assert!(HealthProbeError::lane_dead("gone").is_proof_of_death());
14419 assert!(!HealthProbeError::no_answer("timed out").is_proof_of_death());
14423 assert!(!HealthProbeError::bad_answer("garbage").is_proof_of_death());
14424 assert!(!HealthProbeError::misconfigured("no table").is_proof_of_death());
14425 }
14426
14427 #[test]
14433 fn every_evidence_class_has_a_distinct_label() {
14434 let labels = [
14435 HealthProbeError::lane_dead("").label(),
14436 HealthProbeError::no_answer("").label(),
14437 HealthProbeError::bad_answer("").label(),
14438 HealthProbeError::misconfigured("").label(),
14439 ];
14440 let unique: HashSet<_> = labels.iter().collect();
14441 assert_eq!(unique.len(), labels.len(), "labels collided: {labels:?}");
14442 }
14443
14444 #[test]
14450 fn classification_preserves_the_original_message() {
14451 let err = HealthProbeError::no_answer("module did not answer within 5s");
14452 assert_eq!(err.to_string(), "module did not answer within 5s");
14453 assert!(matches!(err.evidence, HealthProbeEvidence::NoAnswer));
14454 }
14455}
14456
14457#[cfg(test)]
14458mod health_tombstone_tests {
14459 use std::{path::PathBuf, sync::Arc, time::Duration};
14460
14461 use subc_protocol::{
14462 manifest::Concurrency,
14463 session::{HealthStatus, ModuleControlResponse},
14464 };
14465 use tokio::sync::mpsc;
14466
14467 use super::{
14468 probe_module_health, HealthAction, HealthConfig, HealthProbeEvidence, ModuleProtocol,
14469 ModuleSpec, RestartPolicy, Supervisor, SupervisorRuntimeConfig,
14470 };
14471 use crate::{
14472 control::ControlHandler,
14473 forwarding::{ForwardingTable, ModuleControlRpcCompletion, ModuleControlRpcOutcome},
14474 registry::{ConnectionId, Registry},
14475 router::FrameSink,
14476 };
14477
14478 struct ProbeHarness {
14479 spec: ModuleSpec,
14480 runtime: SupervisorRuntimeConfig,
14481 forwarding: Arc<ForwardingTable>,
14482 module_connection: ConnectionId,
14483 module_rx: mpsc::Receiver<crate::router::OutboundFrame>,
14484 handler: ControlHandler,
14485 module: super::SupervisedModule,
14486 }
14487
14488 fn probe_harness() -> ProbeHarness {
14489 let registry = Arc::new(Registry::default());
14490 let forwarding = Arc::new(ForwardingTable::default());
14491 let supervisor_handle = super::SupervisorHandle::new();
14492 let health = HealthConfig {
14493 http: None,
14494 cadence: Duration::from_secs(30),
14495 deadline: Duration::from_secs(5),
14496 failure_threshold: 3,
14497 on_degraded: HealthAction::Report,
14498 on_failing: HealthAction::Report,
14499 critical: false,
14500 };
14501 let supervisor = Supervisor::new_for_test(Arc::clone(®istry), RestartPolicy::default())
14502 .with_forwarding(Arc::clone(&forwarding))
14503 .with_handle(supervisor_handle.clone())
14504 .with_health_config(health);
14505 let spec = ModuleSpec {
14506 module_id: "late-health-module".to_string(),
14507 program: PathBuf::from("disabled-module"),
14508 args: Vec::new(),
14509 env: Vec::new(),
14510 reserved: false,
14511 reserved_prefixes: Vec::new(),
14512 protocol: ModuleProtocol::Subc,
14513 overlap: Default::default(),
14514 };
14515 let module = supervisor
14516 .supervise_configured(spec.clone(), false)
14517 .unwrap();
14518 let runtime = supervisor.runtime_config();
14519 let handler = ControlHandler::with_forwarding(registry, Arc::clone(&forwarding))
14520 .with_supervisor(supervisor_handle);
14521 let module_connection = ConnectionId::new(700);
14522 let (module_tx, module_rx) = mpsc::channel(8);
14523 forwarding
14524 .register_module_connection(
14525 module_connection,
14526 spec.module_id.clone(),
14527 subc_protocol::PROTOCOL_VERSION,
14528 Concurrency::ModuleManaged,
14529 FrameSink::new(module_tx),
14530 )
14531 .unwrap();
14532
14533 ProbeHarness {
14534 spec,
14535 runtime,
14536 forwarding,
14537 module_connection,
14538 module_rx,
14539 handler,
14540 module,
14541 }
14542 }
14543
14544 async fn finish_after(
14545 harness: &mut ProbeHarness,
14546 stall: Duration,
14547 ) -> ModuleControlRpcCompletion {
14548 assert!(stall > harness.runtime.health.deadline);
14549 let deadline = harness.runtime.health.deadline;
14550 let probe = probe_module_health(&harness.spec.module_id, &harness.runtime, None);
14551 let answer = async {
14552 let frame = harness.module_rx.recv().await.expect("health.check frame");
14553 tokio::time::advance(deadline).await;
14554 tokio::task::yield_now().await;
14555 tokio::time::advance(stall - deadline).await;
14556 harness
14557 .forwarding
14558 .complete_module_control_rpc(
14559 harness.module_connection,
14560 frame.header.corr,
14561 Some("health.check"),
14562 ModuleControlRpcOutcome::Response(ModuleControlResponse::HealthCheck {
14563 status: HealthStatus::Ok,
14564 detail: None,
14565 metrics: None,
14566 }),
14567 )
14568 .unwrap()
14569 };
14570 let (probe_result, completion) = tokio::join!(probe, answer);
14571 let err = probe_result.expect_err("probe must miss its deadline");
14572 assert!(matches!(err.evidence, HealthProbeEvidence::NoAnswer));
14573 completion
14574 }
14575
14576 async fn time_out_without_answer(harness: &mut ProbeHarness) {
14577 let deadline = harness.runtime.health.deadline;
14578 let probe = probe_module_health(&harness.spec.module_id, &harness.runtime, None);
14579 let exhaust_deadline = async {
14580 let _frame = harness.module_rx.recv().await.expect("health.check frame");
14581 tokio::time::advance(deadline).await;
14582 tokio::task::yield_now().await;
14583 };
14584 let (probe_result, ()) = tokio::join!(probe, exhaust_deadline);
14585 let err = probe_result.expect_err("probe must miss its deadline");
14586 assert!(matches!(err.evidence, HealthProbeEvidence::NoAnswer));
14587 }
14588
14589 async fn run_probe_cycle(harness: &mut ProbeHarness, answer: bool) {
14590 let registry = Arc::clone(&harness.module.inner.registry);
14591 let snapshot = Arc::clone(&harness.module.inner.snapshot);
14592 let process_liveness = super::SupervisorProcessLiveness::default();
14593 let mut child = None;
14594 let cycle = super::run_health_probe_cycle(
14595 &harness.spec,
14596 &harness.runtime,
14597 ®istry,
14598 &process_liveness,
14599 &snapshot,
14600 &mut child,
14601 );
14602 let peer = async {
14603 let frame = harness.module_rx.recv().await.expect("health.check frame");
14604 if answer {
14605 harness
14606 .forwarding
14607 .complete_module_control_rpc(
14608 harness.module_connection,
14609 frame.header.corr,
14610 Some("health.check"),
14611 ModuleControlRpcOutcome::Response(ModuleControlResponse::HealthCheck {
14612 status: HealthStatus::Ok,
14613 detail: None,
14614 metrics: Some(serde_json::json!({"ready": true})),
14615 }),
14616 )
14617 .unwrap();
14618 } else {
14619 tokio::time::advance(harness.runtime.health.deadline).await;
14620 tokio::task::yield_now().await;
14621 }
14622 };
14623 tokio::join!(cycle, peer);
14624 }
14625
14626 #[tokio::test(start_paused = true)]
14627 async fn unanswered_probe_is_unknown_until_threshold_and_ok_report_recovers() {
14628 let mut harness = probe_harness();
14629 let monitor = harness.module.inner.monitor.lock().unwrap().take().unwrap();
14633 monitor.abort();
14634 let _ = monitor.await;
14635 super::update_snapshot(&harness.module.inner.snapshot, None, |state| {
14636 state.enabled = true;
14637 state.state = super::ModuleState::Running;
14638 state.process_alive = true;
14639 })
14640 .unwrap();
14641
14642 run_probe_cycle(&mut harness, true).await;
14643 assert_eq!(
14644 harness.module.status().unwrap().health.status,
14645 super::SupervisorHealthStatus::Ok
14646 );
14647
14648 for failures in 1..harness.runtime.health.failure_threshold {
14649 run_probe_cycle(&mut harness, false).await;
14650 let status = harness.module.status().unwrap();
14651 assert_eq!(status.health.status, super::SupervisorHealthStatus::Unknown);
14652 assert_eq!(status.health.consecutive_failures, failures);
14653 assert!(status.health.last_probe_ms.is_some());
14654 assert!(status.health.detail.unwrap().starts_with("[no-answer]"));
14655 assert!(status.health.metrics.is_none());
14656 assert_eq!(status.state, super::ModuleState::Running);
14657 assert!(status.process_alive);
14658 assert_eq!(status.restart_count, 0);
14659 assert_eq!(status.lifetime_restarts, 0);
14660 assert!(status.health.last_action.is_none());
14661 assert!(harness.runtime.scheduled_respawn.lock().unwrap().is_none());
14662 }
14663
14664 run_probe_cycle(&mut harness, true).await;
14665 let recovered = harness.module.status().unwrap();
14666 assert_eq!(recovered.health.status, super::SupervisorHealthStatus::Ok);
14667 assert_eq!(recovered.health.consecutive_failures, 0);
14668 assert!(recovered.health.detail.is_none());
14669 assert_eq!(
14670 recovered.health.metrics,
14671 Some(serde_json::json!({"ready": true}))
14672 );
14673 assert_eq!(recovered.lifetime_restarts, 0);
14674
14675 for failures in 1..=harness.runtime.health.failure_threshold {
14676 run_probe_cycle(&mut harness, false).await;
14677 let status = harness.module.status().unwrap();
14678 assert_eq!(status.health.consecutive_failures, failures);
14679 if failures < harness.runtime.health.failure_threshold {
14680 assert_eq!(status.health.status, super::SupervisorHealthStatus::Unknown);
14681 assert_eq!(status.state, super::ModuleState::Running);
14682 assert_eq!(status.lifetime_restarts, 0);
14683 assert!(harness.runtime.scheduled_respawn.lock().unwrap().is_none());
14684 } else {
14685 assert_eq!(
14686 status.health.status,
14687 super::SupervisorHealthStatus::Unresponsive
14688 );
14689 assert_eq!(status.state, super::ModuleState::Restarting);
14690 assert_eq!(status.restart_count, 1);
14691 assert_eq!(status.lifetime_restarts, 1);
14692 assert_eq!(status.health.last_action.as_deref(), Some("restart"));
14693 assert!(harness.runtime.scheduled_respawn.lock().unwrap().is_some());
14694 }
14695 }
14696 }
14697
14698 #[tokio::test(start_paused = true)]
14699 async fn late_health_answers_record_start_anchored_latency_for_two_stalls() {
14700 let mut harness = probe_harness();
14701
14702 let first = finish_after(&mut harness, Duration::from_secs(8)).await;
14703 let first_latency = match &first {
14704 ModuleControlRpcCompletion::LateHealthAnswer { latency, .. } => *latency,
14705 other => panic!("late answer was not retained: {other:?}"),
14706 };
14707 assert!(harness.handler.observe_module_control_completion(first));
14708
14709 let second = finish_after(&mut harness, Duration::from_secs(11)).await;
14710 let second_latency = match &second {
14711 ModuleControlRpcCompletion::LateHealthAnswer { latency, .. } => *latency,
14712 other => panic!("late answer was not retained: {other:?}"),
14713 };
14714 assert!(harness.handler.observe_module_control_completion(second));
14715
14716 assert_eq!(first_latency, Duration::from_secs(8));
14717 assert_eq!(
14718 second_latency - first_latency,
14719 Duration::from_secs(3),
14720 "latency must grow linearly with the additional stall"
14721 );
14722 let health = harness.module.status().unwrap().health;
14723 assert_eq!(health.late_answer_count, 2);
14724 assert_eq!(health.last_late_answer_latency_ms, Some(11_000));
14725 }
14726
14727 #[tokio::test(start_paused = true)]
14735 async fn late_answer_clears_the_consecutive_failure_streak() {
14736 let mut harness = probe_harness();
14737
14738 time_out_without_answer(&mut harness).await;
14740 harness
14741 .module
14742 .record_health_probe_failure_for_test("[no-answer] test miss")
14743 .unwrap();
14744 assert_eq!(
14745 harness.module.status().unwrap().health.consecutive_failures,
14746 1,
14747 "precondition: the miss must be on the streak before the late answer"
14748 );
14749
14750 let late = finish_after(&mut harness, Duration::from_secs(9)).await;
14752 assert!(matches!(
14753 late,
14754 ModuleControlRpcCompletion::LateHealthAnswer { .. }
14755 ));
14756 assert!(harness.handler.observe_module_control_completion(late));
14757
14758 let health = harness.module.status().unwrap().health;
14759 assert_eq!(
14760 health.consecutive_failures, 0,
14761 "a late answer is an answer: the streak must reset"
14762 );
14763 assert_eq!(health.late_answer_count, 1);
14764 }
14765
14766 #[tokio::test(start_paused = true)]
14767 async fn repeated_serial_probe_cycles_keep_one_tombstone_per_endpoint() {
14768 let mut harness = probe_harness();
14769
14770 for _ in 0..20 {
14771 time_out_without_answer(&mut harness).await;
14772 assert_eq!(
14773 harness.forwarding.health_probe_tombstone_count().unwrap(),
14774 1
14775 );
14776 }
14777 }
14778}
14779
14780#[cfg(test)]
14781mod child_env_tests {
14782 #[cfg(windows)]
14783 use super::{
14784 clear_current_process_facts, observed_launch_nonce_source, ModuleState, SupervisorSnapshot,
14785 };
14786 #[cfg(windows)]
14787 use std::sync::{Arc, OnceLock};
14788 #[cfg(windows)]
14789 #[test]
14790 fn windows_wire_spawn_prepares_pipe_and_keeps_phase_one_env() {
14791 let spec = spec(Vec::new());
14792 let mut command = Command::new("unused");
14793 let handoff = apply_wire_spawn_args(&mut command, &spec, None, None)
14794 .unwrap()
14795 .unwrap();
14796 let name = command
14797 .as_std()
14798 .get_envs()
14799 .find(|(key, _)| *key == OsStr::new(subc_os::launch_nonce::LAUNCH_NONCE_PIPE_ENV))
14800 .and_then(|(_, value)| value)
14801 .unwrap();
14802 assert_eq!(name, OsStr::new(handoff.name()));
14803 assert!(handoff.name().starts_with(r"\\.\pipe\subc-launch-"));
14804 assert!(command.as_std().get_envs().any(|(key, value)| key
14805 == OsStr::new(subc_os::LAUNCH_NONCE_PIPE_FALLBACK_ENV)
14806 && value == Some(OsStr::new("env"))));
14807 assert!(command
14808 .as_std()
14809 .get_envs()
14810 .any(|(key, value)| key == OsStr::new(SUBC_LAUNCH_NONCE_ENV) && value.is_some()));
14811 }
14812
14813 #[cfg(windows)]
14814 #[test]
14815 fn windows_source_reporting_tracks_consumption_and_resets_per_spawn() {
14816 let mut snapshot = SupervisorSnapshot::new(ModuleState::Running, true);
14817 snapshot.process_alive = true;
14818 snapshot.spawned_protocol = Some(ModuleProtocol::Subc);
14819 assert_eq!(
14820 observed_launch_nonce_source(&snapshot).as_deref(),
14821 Some("env")
14822 );
14823 let consumed = Arc::new(OnceLock::new());
14824 snapshot.pipe_consumed = Some(consumed.clone());
14825 assert_eq!(
14826 observed_launch_nonce_source(&snapshot).as_deref(),
14827 Some("env")
14828 );
14829 consumed.set(()).unwrap();
14830 assert_eq!(
14831 observed_launch_nonce_source(&snapshot).as_deref(),
14832 Some("pipe")
14833 );
14834 snapshot.pipe_consumed = Some(Arc::new(OnceLock::new()));
14835 assert_eq!(
14836 observed_launch_nonce_source(&snapshot).as_deref(),
14837 Some("env")
14838 );
14839 clear_current_process_facts(&mut snapshot);
14840 assert_eq!(observed_launch_nonce_source(&snapshot), None);
14841 }
14842 use super::{
14843 apply_child_env, apply_spawn_role, apply_wire_spawn_args, LaunchSpecDigest, ModuleProtocol,
14844 ModuleSpec, SpawnRole, SupervisorHandle, SPAWN_ROLE_SWAP_CANDIDATE, SUBC_ARG,
14845 SUBC_LAUNCH_NONCE_ENV, SUBC_MODULE_ID_ENV, SUBC_SPAWN_ROLE_ENV,
14846 };
14847 use std::{ffi::OsStr, path::PathBuf};
14848 use tokio::process::Command;
14849
14850 fn spec(env: Vec<(String, String)>) -> ModuleSpec {
14851 ModuleSpec {
14852 module_id: "env-plan".to_string(),
14853 program: PathBuf::from("/nonexistent"),
14854 args: Vec::new(),
14855 env,
14856 reserved: false,
14857 reserved_prefixes: Vec::new(),
14858 protocol: ModuleProtocol::Subc,
14859 overlap: Default::default(),
14860 }
14861 }
14862
14863 #[test]
14864 fn configured_launch_spec_digest_is_canonical_and_excludes_injected_inputs() {
14865 let base = spec(vec![("A".into(), "B=C".into()), ("Z".into(), "v".into())]);
14866 let digest = LaunchSpecDigest::from_spec(&base);
14867 let mut reordered = base.clone();
14868 reordered.env.reverse();
14869 reordered.module_id = "different-id".into();
14870 reordered.reserved = true;
14871 reordered.overlap = Default::default();
14872 assert_eq!(digest, LaunchSpecDigest::from_spec(&reordered));
14873 let mut delimiter_collision = base.clone();
14874 delimiter_collision.env[0] = ("A=B".into(), "C".into());
14875 assert_ne!(digest, LaunchSpecDigest::from_spec(&delimiter_collision));
14876 let mut changed = base.clone();
14877 changed.protocol = ModuleProtocol::None;
14878 assert_ne!(digest, LaunchSpecDigest::from_spec(&changed));
14879 changed = base.clone();
14880 changed.args.push("--flag".into());
14881 assert_ne!(digest, LaunchSpecDigest::from_spec(&changed));
14882 changed = base.clone();
14883 changed.program = PathBuf::from("/other");
14884 assert_ne!(digest, LaunchSpecDigest::from_spec(&changed));
14885 changed = base.clone();
14886 changed.env[1].1 = "other".into();
14887 assert_ne!(digest, LaunchSpecDigest::from_spec(&changed));
14888 assert!(!format!("{digest:?}").contains("B=C"));
14889 assert!(!format!("{digest:?}").contains("[u8"));
14890 }
14891
14892 #[cfg(unix)]
14893 #[test]
14894 fn native_program_bytes_do_not_collapse_through_lossy_utf8() {
14895 use std::os::unix::ffi::OsStringExt;
14896 let mut left = spec(Vec::new());
14897 left.program = std::ffi::OsString::from_vec(vec![b'/', 0xff]).into();
14898 let mut right = left.clone();
14899 right.program = std::ffi::OsString::from_vec(vec![b'/', 0xfe]).into();
14900 assert_eq!(
14901 left.program.to_string_lossy(),
14902 right.program.to_string_lossy()
14903 );
14904 assert_ne!(
14905 LaunchSpecDigest::from_spec(&left),
14906 LaunchSpecDigest::from_spec(&right)
14907 );
14908 }
14909
14910 #[test]
14924 fn ambient_ck_log_is_removed_and_a_configured_one_survives() {
14925 let mut command = Command::new("/nonexistent");
14926 apply_child_env(&mut command, &spec(Vec::new()));
14927 let removed = command
14928 .as_std()
14929 .get_envs()
14930 .any(|(key, value)| key == OsStr::new("CK_LOG") && value.is_none());
14931 assert!(
14932 removed,
14933 "ambient CK_LOG must be explicitly removed for an unconfigured module"
14934 );
14935
14936 let mut configured = Command::new("/nonexistent");
14937 apply_child_env(
14938 &mut configured,
14939 &spec(vec![("CK_LOG".to_string(), "debug".to_string())]),
14940 );
14941 let effective = configured
14942 .as_std()
14943 .get_envs()
14944 .filter(|(key, _)| *key == OsStr::new("CK_LOG"))
14945 .last()
14946 .map(|(_, value)| value.map(|v| v.to_string_lossy().into_owned()));
14947 assert_eq!(
14948 effective,
14949 Some(Some("debug".to_string())),
14950 "a module's configured CK_LOG must survive the ambient removal"
14951 );
14952 }
14953
14954 #[test]
14963 fn protocol_none_spawn_carries_no_subc_argument_and_no_nonce() {
14964 let connection_file = std::path::Path::new("/run/subc-connection.json");
14965 let handle = SupervisorHandle::new();
14966
14967 let mut none_spec = spec(Vec::new());
14968 none_spec.protocol = ModuleProtocol::None;
14969 let mut none = Command::new("/nonexistent");
14970 let none_handoff =
14971 apply_wire_spawn_args(&mut none, &none_spec, Some(connection_file), Some(&handle))
14972 .expect("protocol-none spawn args apply");
14973 assert!(
14974 none_handoff.is_none(),
14975 "protocol:none spawn must not receive a nonce descriptor"
14976 );
14977 #[cfg(windows)]
14978 assert!(!none.as_std().get_envs().any(|(key, value)| key
14979 == OsStr::new(subc_os::LAUNCH_NONCE_PIPE_FALLBACK_ENV)
14980 && value.is_some()));
14981 assert!(
14982 !none.as_std().get_envs().any(|(key, value)| key
14983 == OsStr::new(subc_os::LAUNCH_NONCE_FD_ENV)
14984 && value.is_some()),
14985 "protocol:none spawn must not name a nonce descriptor"
14986 );
14987 let none_args: Vec<String> = none
14988 .as_std()
14989 .get_args()
14990 .map(|a| a.to_string_lossy().into_owned())
14991 .collect();
14992 assert!(
14993 !none_args.iter().any(|a| a == SUBC_ARG),
14994 "protocol:none argv must not carry --subc; got {none_args:?}"
14995 );
14996 let none_has_nonce = none
14997 .as_std()
14998 .get_envs()
14999 .any(|(key, value)| key == OsStr::new(SUBC_LAUNCH_NONCE_ENV) && value.is_some());
15000 assert!(
15001 !none_has_nonce,
15002 "protocol:none spawn must not receive a launch nonce"
15003 );
15004 let none_has_module_id = none
15005 .as_std()
15006 .get_envs()
15007 .any(|(key, value)| key == OsStr::new(SUBC_MODULE_ID_ENV) && value.is_some());
15008 assert!(
15009 none_has_module_id,
15010 "SUBC_MODULE_ID is inert and stays on every path"
15011 );
15012 assert!(
15013 handle.spawn_nonce(&none_spec.module_id).is_none(),
15014 "no nonce record for a process that will never present one"
15015 );
15016
15017 let wire_spec = spec(Vec::new());
15019 let mut wire = Command::new("/nonexistent");
15020 let wire_handoff =
15021 apply_wire_spawn_args(&mut wire, &wire_spec, Some(connection_file), Some(&handle))
15022 .expect("subc-wire spawn args apply");
15023 let wire_fd_env = wire
15024 .as_std()
15025 .get_envs()
15026 .find(|(key, _)| *key == OsStr::new(subc_os::LAUNCH_NONCE_FD_ENV))
15027 .and_then(|(_, value)| value.map(|v| v.to_string_lossy().into_owned()));
15028 #[cfg(unix)]
15029 assert_eq!(
15030 wire_fd_env,
15031 Some(wire_handoff.expect("a descriptor handoff").fd_env_value()),
15032 "a subc-wire spawn names the pipe it will receive at descriptor 3"
15033 );
15034 #[cfg(windows)]
15035 assert!(wire_handoff.is_some() && wire_fd_env.is_none());
15036 #[cfg(not(any(unix, windows)))]
15037 assert!(wire_handoff.is_none() && wire_fd_env.is_none());
15038 let wire_args: Vec<String> = wire
15039 .as_std()
15040 .get_args()
15041 .map(|a| a.to_string_lossy().into_owned())
15042 .collect();
15043 assert_eq!(
15044 wire_args,
15045 vec![
15046 SUBC_ARG.to_string(),
15047 connection_file.to_string_lossy().into_owned()
15048 ],
15049 "a subc-wire spawn still carries --subc <path>"
15050 );
15051 assert_eq!(
15052 wire.as_std()
15053 .get_envs()
15054 .any(|(key, value)| key == OsStr::new(SUBC_LAUNCH_NONCE_ENV) && value.is_some()),
15055 !cfg!(unix),
15056 "only Windows supplies the environment nonce"
15057 );
15058 assert!(handle.spawn_nonce(&wire_spec.module_id).is_some());
15059 }
15060
15061 #[test]
15071 fn plain_spawn_removes_the_spawn_role_even_when_the_spec_sets_it() {
15072 let role = |command: &Command| {
15073 command
15074 .as_std()
15075 .get_envs()
15076 .filter(|(key, _)| *key == OsStr::new(SUBC_SPAWN_ROLE_ENV))
15077 .last()
15078 .map(|(_, value)| value.map(|v| v.to_string_lossy().into_owned()))
15079 };
15080 let forged = spec(vec![(
15081 SUBC_SPAWN_ROLE_ENV.to_string(),
15082 SPAWN_ROLE_SWAP_CANDIDATE.to_string(),
15083 )]);
15084
15085 let mut plain = Command::new("/nonexistent");
15086 apply_child_env(&mut plain, &forged);
15087 apply_spawn_role(&mut plain, SpawnRole::Plain);
15088 assert_eq!(
15089 role(&plain),
15090 Some(None),
15091 "a plain spawn must remove SUBC_SPAWN_ROLE, whatever the spec says"
15092 );
15093
15094 let mut candidate = Command::new("/nonexistent");
15095 apply_child_env(&mut candidate, &spec(Vec::new()));
15096 apply_spawn_role(&mut candidate, SpawnRole::SwapCandidate);
15097 assert_eq!(
15098 role(&candidate),
15099 Some(Some(SPAWN_ROLE_SWAP_CANDIDATE.to_string()))
15100 );
15101 }
15102
15103 #[test]
15109 fn daemon_private_capture_keys_are_not_passed_to_the_child() {
15110 let mut command = Command::new("/nonexistent");
15111 apply_child_env(
15112 &mut command,
15113 &spec(vec![
15114 (super::CAPTURE_KEEP_ENV.to_string(), "5".to_string()),
15115 ("KEPT".to_string(), "yes".to_string()),
15116 ]),
15117 );
15118 let keys: Vec<String> = command
15119 .as_std()
15120 .get_envs()
15121 .filter(|(_, value)| value.is_some())
15122 .map(|(key, _)| key.to_string_lossy().into_owned())
15123 .collect();
15124 assert!(keys.contains(&"KEPT".to_string()), "got {keys:?}");
15125 assert!(
15126 !keys.contains(&super::CAPTURE_KEEP_ENV.to_string()),
15127 "daemon-private capture key leaked to the child: {keys:?}"
15128 );
15129 }
15130}
15131
15132#[cfg(test)]
15133mod jitter_tests {
15134 use super::jittered_health_delay;
15135 use std::{collections::HashSet, time::Duration};
15136
15137 const FLEET: [&str; 14] = [
15146 "aft",
15147 "alfonso-core",
15148 "magic-context",
15149 "broca",
15150 "thalamus",
15151 "quota",
15152 "engram",
15153 "plexus",
15154 "cerebellum",
15155 "astrocyte",
15156 "synapse",
15157 "subc-mcp",
15158 "cortexkit-credentials",
15159 "subc-federation",
15160 ];
15161
15162 #[test]
15170 fn probe_delays_disperse_across_the_fleet() {
15171 let cadence = Duration::from_secs(30);
15172 let delays: HashSet<Duration> = FLEET
15173 .iter()
15174 .map(|id| jittered_health_delay(id, 0, cadence))
15175 .collect();
15176 assert_eq!(
15177 delays.len(),
15178 FLEET.len(),
15179 "every supervised module must land on its own probe offset"
15180 );
15181 }
15182
15183 #[test]
15189 fn jitter_only_delays_and_stays_within_one_tenth_of_cadence() {
15190 let cadence = Duration::from_secs(30);
15191 let span = cadence / 10;
15192 for id in FLEET {
15193 for probe_index in 0..8 {
15194 let delay = jittered_health_delay(id, probe_index, cadence);
15195 assert!(
15196 delay >= cadence,
15197 "{id}#{probe_index}: jitter must not shorten the cadence"
15198 );
15199 assert!(
15200 delay < cadence + span,
15201 "{id}#{probe_index}: jitter must stay inside one tenth of the cadence"
15202 );
15203 }
15204 }
15205 }
15206
15207 #[test]
15213 fn a_module_offset_is_stable_across_restarts() {
15214 let cadence = Duration::from_secs(30);
15215 for id in FLEET {
15216 assert_eq!(
15217 jittered_health_delay(id, 0, cadence),
15218 jittered_health_delay(id, 0, cadence),
15219 "{id}: the same module and probe index must produce the same offset"
15220 );
15221 }
15222 }
15223
15224 #[test]
15226 fn zero_cadence_yields_zero_delay() {
15227 assert_eq!(
15228 jittered_health_delay("aft", 0, Duration::ZERO),
15229 Duration::ZERO
15230 );
15231 }
15232}
15233
15234#[cfg(all(test, target_os = "linux"))]
15235mod cgroup_placement_tests {
15236 use super::{
15237 apply_cgroup_placement, remove_module_cgroup, LaunchSpecDigest, ModuleProtocol, ModuleSpec,
15238 SuperviseError, SupervisedChild,
15239 };
15240 use crate::stderr_tail::{StderrRing, StderrTailConfig};
15241 use cortexkit_test_support::ScratchDir;
15242 use std::{
15243 fs, io,
15244 path::{Path, PathBuf},
15245 sync::{Arc, Mutex},
15246 };
15247 use tokio::process::Command;
15248
15249 #[tokio::test]
15250 async fn unique_spawn_cgroups_clean_up_across_restarts_and_shutdown() {
15251 use super::*;
15252 let dir = ScratchDir::new("unique-spawn-cgroups");
15253 let root = PathBuf::from(format!(
15254 "/sys/fs/cgroup/subc-unique-test-{}-{}",
15255 std::process::id(),
15256 unix_ms_now()
15257 ));
15258 if let Err(error) = fs::create_dir(&root) {
15259 assert!(
15260 std::env::var_os("SUBC_REQUIRE_CGROUP_TEST").is_none(),
15261 "required cgroup test cannot execute: {error}"
15262 );
15263 eprintln!("SKIP unique_spawn_cgroups_clean_up_across_restarts_and_shutdown: {error}");
15264 return;
15265 }
15266 let placement = subc_cgroup::prepare_at(&root).unwrap().unwrap();
15267 let group_count = || {
15268 fs::read_dir(root.join("subc-modules"))
15269 .unwrap()
15270 .map(|entry| entry.unwrap().file_type().unwrap())
15271 .filter(|kind| kind.is_dir())
15272 .count()
15273 };
15274 let supervisor =
15275 Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default())
15276 .with_cgroup_placement(Some(placement.clone()));
15277 let runtime = supervisor.runtime_config();
15278 let mut spec = ModuleSpec {
15279 module_id: "unique-spawn".into(),
15280 program: PathBuf::from("/bin/sleep"),
15281 args: vec!["60".into()],
15282 env: ["XDG_DATA_HOME", "XDG_RUNTIME_DIR", "XDG_CONFIG_HOME"]
15283 .into_iter()
15284 .map(|key| (key.into(), dir.to_string_lossy().into_owned()))
15285 .collect(),
15286 reserved: false,
15287 reserved_prefixes: vec![],
15288 protocol: ModuleProtocol::None,
15289 overlap: Default::default(),
15290 };
15291 let spawn = |spec: &ModuleSpec| {
15292 spawn_child(
15293 spec,
15294 None,
15295 None,
15296 &runtime.stderr_ring,
15297 None,
15298 &runtime.child_roster,
15299 Some(&placement),
15300 )
15301 .unwrap()
15302 };
15303 let mut live = spawn(&spec);
15304 for _ in 0..3 {
15305 let next = spawn(&spec);
15307 assert_ne!(live.module_id, next.module_id);
15308 live.start_kill().unwrap();
15309 live.wait().await.unwrap();
15310 live = next;
15311 assert!(
15312 live.child.try_wait().unwrap().is_none(),
15313 "retiring the old slot must not kill the replacement"
15314 );
15315 assert_eq!(
15316 group_count(),
15317 1,
15318 "only the live spawn's cgroup should remain"
15319 );
15320 }
15321 supervisor.begin_daemon_shutdown();
15322 let reap = tokio::spawn(async move {
15323 live.wait().await.unwrap();
15324 });
15325 supervisor
15326 .end_children_for_daemon_shutdown(false, std::future::pending())
15327 .await;
15328 reap.await.unwrap();
15329 assert_eq!(group_count(), 0);
15330 spec.program = PathBuf::from("/bin/true");
15332 spec.args.clear();
15333 let fresh_roster = ChildRoster::default();
15334 let mut short = spawn_child(
15335 &spec,
15336 None,
15337 None,
15338 &runtime.stderr_ring,
15339 None,
15340 &fresh_roster,
15341 Some(&placement),
15342 )
15343 .unwrap();
15344 short.wait().await.unwrap();
15345 assert_eq!(group_count(), 0);
15346 spec.module_id = "_".repeat(255);
15347 let mut long_id = spawn_child(
15348 &spec,
15349 None,
15350 None,
15351 &runtime.stderr_ring,
15352 None,
15353 &fresh_roster,
15354 Some(&placement),
15355 )
15356 .unwrap();
15357 long_id.wait().await.unwrap();
15358 assert_eq!(
15359 group_count(),
15360 0,
15361 "valid long module IDs must not exceed cgroup NAME_MAX"
15362 );
15363 fs::remove_dir(root.join("subc-modules")).unwrap();
15364 fs::remove_dir(root).unwrap();
15365 eprintln!("EXECUTED unique_spawn_cgroups_clean_up_across_restarts_and_shutdown");
15366 }
15367
15368 #[test]
15369 fn failed_parent_cgroup_open_is_a_cgroup_supervision_error() {
15370 let path = Path::new("/definitely-missing-subc-cgroup");
15371 let mut command = Command::new("true");
15372 let error = apply_cgroup_placement(
15373 &mut command,
15374 &ModuleSpec {
15375 module_id: "broken-cgroup".to_string(),
15376 program: PathBuf::from("true"),
15377 args: Vec::new(),
15378 env: Vec::new(),
15379 reserved: false,
15380 reserved_prefixes: Vec::new(),
15381 protocol: ModuleProtocol::Subc,
15382 overlap: Default::default(),
15383 },
15384 path,
15385 )
15386 .expect_err("a parent cgroup open failure must reject the supervised spawn");
15387 let reason = error.to_string();
15388
15389 assert!(
15390 matches!(error, SuperviseError::Cgroup { .. }),
15391 "parent cgroup open must be reported as a cgroup supervision error: {reason}"
15392 );
15393 assert!(
15394 reason.contains("/definitely-missing-subc-cgroup/cgroup.procs"),
15395 "parent cgroup open failure must name cgroup.procs: {reason}"
15396 );
15397 }
15398
15399 #[tokio::test]
15400 async fn reaping_a_child_removes_its_empty_module_cgroup() {
15401 let root = ScratchDir::new("supervisor-reap-cgroup");
15402 fs::write(root.join("cgroup.procs"), b"").expect("write scratch cgroup marker");
15403 let placement = subc_cgroup::prepare_at(&root)
15404 .expect("prepare scratch cgroup root")
15405 .expect("scratch root has a cgroup.procs marker");
15406 let module_id = "reaped-module";
15407 let module = placement
15408 .module_path(module_id)
15409 .expect("create scratch module cgroup");
15410 let child = Command::new("true")
15411 .env("XDG_DATA_HOME", root.path())
15412 .env("XDG_RUNTIME_DIR", root.path())
15413 .env("XDG_CONFIG_HOME", root.path())
15414 .spawn()
15415 .expect("spawn short-lived child");
15416 let pid = child.id().expect("spawned child has pid");
15417 let mut child = SupervisedChild {
15418 child,
15419 exec: crate::activity::ExecOwner(Arc::new(crate::activity::ExecOutcome::new())),
15420 protocol: ModuleProtocol::Subc,
15421 module_id: module_id.to_string(),
15422 cgroup_placement: Some(placement),
15423 stdout_pump: None,
15424 stderr_pump: None,
15425 stderr_ring: Arc::new(Mutex::new(StderrRing::new(StderrTailConfig::default()))),
15426 spawned_at: tokio::time::Instant::now(),
15427 spawned_at_ms: 0,
15428 spawned_from: PathBuf::from("true"),
15429 launch_spec_digest: LaunchSpecDigest([0; 32]),
15430 spawned_file_identity: None,
15431 process_start_time: None,
15432 process_identity: None,
15433 pid,
15434 roster_guard: None,
15435 #[cfg(target_os = "macos")]
15436 privacy_exec: None,
15437 spawn_failure: None,
15438 };
15439
15440 child.wait().await.expect("reap short-lived child");
15441
15442 assert!(
15443 !module.exists(),
15444 "reaping the supervised child must remove its empty cgroup"
15445 );
15446 }
15447
15448 #[test]
15449 fn non_empty_cgroup_removal_is_reported_without_blocking_teardown() {
15450 let root = ScratchDir::new("supervisor-non-empty-cgroup");
15451 fs::write(root.join("cgroup.procs"), b"").expect("write scratch cgroup marker");
15452 let placement = subc_cgroup::prepare_at(&root)
15453 .expect("prepare scratch cgroup root")
15454 .expect("scratch root has a cgroup.procs marker");
15455 let module = placement
15456 .module_path("surviving-module")
15457 .expect("create scratch module cgroup");
15458 fs::write(module.join("surviving-process"), b"still present")
15459 .expect("make scratch cgroup non-empty");
15460 let (logs, _guard) = crate::router::test_log::log_capture(tracing::Level::WARN);
15461
15462 remove_module_cgroup(&placement, "surviving-module");
15463
15464 let logs = crate::router::test_log::captured_logs(&logs);
15465 assert!(
15466 module.exists(),
15467 "failed removal must leave the cgroup intact"
15468 );
15469 assert!(
15470 logs.contains("could not remove module cgroup after process exit; continuing teardown")
15471 && logs.contains("surviving-module"),
15472 "best-effort removal must report the failure without returning it: {logs}"
15473 );
15474 }
15475
15476 #[test]
15477 fn cgroup_pre_exec_spawn_failure_names_the_cgroup_path() {
15478 let cgroup_path = PathBuf::from("/sys/fs/cgroup/subc-modules/broken-module");
15479 let reason = SuperviseError::Spawn {
15480 program: PathBuf::from("/bin/true"),
15481 source: io::Error::from_raw_os_error(13),
15482 cgroup_path: Some(cgroup_path.clone()),
15483 }
15484 .to_string();
15485
15486 assert!(
15487 reason.contains(&cgroup_path.display().to_string()),
15488 "a pre_exec spawn failure must name the cgroup path: {reason}"
15489 );
15490 }
15491}
15492
15493#[cfg(test)]
15494mod spawn_subscriber_lag_tests {
15495 use super::*;
15496
15497 #[tokio::test]
15502 async fn lagged_spawn_subscriber_receives_a_terminal_lagged_error_after_its_queued_frames() {
15503 let feed = SpawnEventFeed::default();
15504 feed.configure_incarnation("lag-incarnation".to_string());
15505 let (tx, mut rx) = mpsc::channel(1);
15508 feed.subscribe(ConnectionId::new(1), 7, 1, None, FrameSink::new(tx))
15509 .expect("subscribe");
15510 let emitted = SPAWN_SUBSCRIBER_BUFFER + 16;
15511 for index in 0..emitted {
15512 feed.emit_spawned(&format!("lag-module-{index}"), 1000, 0);
15513 tokio::task::yield_now().await;
15516 }
15517 assert_eq!(
15518 feed.subscriber_count(),
15519 0,
15520 "the lagged subscriber must be removed"
15521 );
15522
15523 let mut data = Vec::new();
15524 let mut last = None;
15525 loop {
15526 let next = tokio::time::timeout(Duration::from_secs(5), rx.recv())
15527 .await
15528 .expect("the forwarder must finish once the subscriber is dropped");
15529 let Some(outbound) = next else { break };
15530 let frame = outbound.frame;
15531 if frame.header.ty == FrameType::StreamData {
15532 assert!(last.is_none(), "no data may follow the terminal frame");
15533 let event: SpawnEvent = serde_json::from_slice(&frame.body).unwrap();
15534 data.push(event.cursor.seq);
15535 } else {
15536 assert!(last.is_none(), "exactly one terminal frame");
15537 last = Some(frame);
15538 }
15539 }
15540 assert!(!data.is_empty(), "queued frames drain before the terminal");
15541 for pair in data.windows(2) {
15542 assert_eq!(
15543 pair[1],
15544 pair[0] + 1,
15545 "queued frames arrive dense and in order"
15546 );
15547 }
15548 let terminal = last.expect("a lagged subscriber must receive a terminal frame");
15549 assert_eq!(terminal.header.ty, FrameType::Error);
15550 assert_eq!(terminal.header.corr, 7);
15551 let body: subc_protocol::ErrorBody = serde_json::from_slice(&terminal.body).unwrap();
15552 assert_eq!(body.code, SPAWN_SUBSCRIBER_LAGGED_CODE);
15553 let detail = body.detail.expect("lagged error carries detail");
15554 assert_eq!(
15555 detail["first_undelivered_cursor"]["seq"],
15556 data.last().unwrap() + 1,
15557 "the named cursor is the first event the subscriber did not receive"
15558 );
15559 assert_eq!(
15560 detail["first_undelivered_cursor"]["daemon_incarnation"],
15561 "lag-incarnation"
15562 );
15563 }
15564}
15565
15566#[cfg(test)]
15567mod terminal_history_read_concurrency_tests {
15568 use super::*;
15569 use crate::terminal_journal::read_pause;
15570 use cortexkit_test_support::ScratchDir;
15571 use std::sync::mpsc as std_mpsc;
15572
15573 fn journaled_ring(
15574 journal: &Arc<crate::terminal_journal::TerminalJournal>,
15575 ) -> Arc<Mutex<TerminalRing>> {
15576 Arc::new(Mutex::new(
15577 TerminalRing::new(TerminalRingConfig::default(), 1)
15578 .with_journal(Some(Arc::clone(journal))),
15579 ))
15580 }
15581
15582 fn crash(at_ms: u64) -> ExitReport {
15583 ExitReport {
15584 kind: ExitKind::Crash,
15585 code: Some(1),
15586 signal: None,
15587 at_ms,
15588 }
15589 }
15590
15591 fn record_within(
15594 module_id: &'static str,
15595 ring: &Arc<Mutex<TerminalRing>>,
15596 at_ms: u64,
15597 bound: Duration,
15598 ) -> bool {
15599 let ring = Arc::clone(ring);
15600 let (done, done_rx) = std_mpsc::channel();
15601 std::thread::spawn(move || {
15602 record_terminal(
15603 module_id,
15604 &ring,
15605 &SpawnEventFeed::default(),
15606 &crash(at_ms),
15607 TerminalDisposition::Restarting,
15608 );
15609 let _ = done.send(());
15610 });
15611 done_rx.recv_timeout(bound).is_ok()
15612 }
15613
15614 #[test]
15619 fn exits_recorded_during_a_paused_history_read_are_not_blocked_or_half_merged() {
15620 let dir = ScratchDir::new("terminal-history-concurrent-read");
15621 let path = dir.join("terminals.jsonl");
15622 let journal = Arc::new(crate::terminal_journal::TerminalJournal::open(
15623 path.clone(),
15624 "daemon".into(),
15625 ));
15626 let reader_ring = journaled_ring(&journal);
15627 let other_ring = journaled_ring(&journal);
15628 assert!(record_within(
15629 "reader-module",
15630 &reader_ring,
15631 10,
15632 Duration::from_secs(5)
15633 ));
15634
15635 let (started, release) = read_pause::install(&path);
15636 let reading = {
15637 let ring = Arc::clone(&reader_ring);
15638 std::thread::spawn(move || durable_terminal_history_of(&ring, "reader-module"))
15639 };
15640 started
15641 .recv_timeout(Duration::from_secs(5))
15642 .expect("the history read reached its pause");
15643
15644 let bound = Duration::from_secs(1);
15645 assert!(
15646 record_within("other-module", &other_ring, 20, bound),
15647 "another module's exit waited on a history read (journal writer held)"
15648 );
15649 assert!(
15650 record_within("reader-module", &reader_ring, 30, bound),
15651 "the read module's own exit waited on its history read (ring held)"
15652 );
15653
15654 drop(release);
15655 let paused = reading.join().unwrap();
15656 assert_eq!(
15657 paused.entries.iter().map(|e| e.at_ms).collect::<Vec<_>>(),
15658 vec![10],
15659 "an exit recorded after the read began lands in neither half of it"
15660 );
15661 assert_eq!(paused.journal_skipped_lines, 0);
15662 assert_eq!(paused.journal_read_errors, 0);
15663
15664 let after = durable_terminal_history_of(&reader_ring, "reader-module");
15665 assert_eq!(
15666 after.entries.iter().map(|e| e.at_ms).collect::<Vec<_>>(),
15667 vec![10, 30],
15668 "the next read merges ring and journal with no duplicate"
15669 );
15670 assert_eq!(after.journal_skipped_lines, 0);
15671 }
15672}
15673
15674#[cfg(test)]
15679mod stderr_settle_tests {
15680 use std::{
15681 future::Future,
15682 io,
15683 pin::Pin,
15684 sync::{Arc, Mutex},
15685 task::{Context, Poll},
15686 time::Duration,
15687 };
15688
15689 use tokio::{
15690 io::{AsyncRead, ReadBuf},
15691 sync::oneshot,
15692 time::Instant,
15693 };
15694
15695 use super::{settle_stderr_pump, settle_stdout_pump, StderrPump};
15696 use crate::stderr_tail::{
15697 pump_stderr_to, pump_stdout_to, split_capture_stamp, untimed, CaptureState,
15698 ChildOutputSink, OutputSink, StderrRing, StderrTailConfig, TailEntry,
15699 };
15700
15701 const BOUND: Duration = Duration::from_millis(250);
15702
15703 struct HeldReader {
15707 before: Option<Vec<u8>>,
15708 gate: Option<oneshot::Receiver<()>>,
15709 after: io::Cursor<Vec<u8>>,
15710 }
15711
15712 impl AsyncRead for HeldReader {
15713 fn poll_read(
15714 mut self: Pin<&mut Self>,
15715 cx: &mut Context<'_>,
15716 buf: &mut ReadBuf<'_>,
15717 ) -> Poll<io::Result<()>> {
15718 if let Some(bytes) = self.before.take() {
15719 buf.put_slice(&bytes);
15720 return Poll::Ready(Ok(()));
15721 }
15722 if let Some(gate) = self.gate.as_mut() {
15723 match Pin::new(gate).poll(cx) {
15724 Poll::Pending => return Poll::Pending,
15725 Poll::Ready(_) => self.gate = None,
15726 }
15727 }
15728 Pin::new(&mut self.after).poll_read(cx, buf)
15729 }
15730 }
15731
15732 struct DiscardSink;
15733
15734 impl OutputSink for DiscardSink {
15735 fn write_line(&mut self, _line: &[u8]) {}
15736 }
15737
15738 fn line(text: &str) -> TailEntry {
15739 TailEntry::Line {
15740 text: text.to_string(),
15741 truncated: false,
15742 at_ms: None,
15743 }
15744 }
15745
15746 fn lock(ring: &Arc<Mutex<StderrRing>>) -> std::sync::MutexGuard<'_, StderrRing> {
15747 ring.lock().unwrap()
15748 }
15749
15750 fn held_pump(
15754 ring: &Arc<Mutex<StderrRing>>,
15755 before: &str,
15756 after: &str,
15757 ) -> (StderrPump, oneshot::Sender<()>) {
15758 let generation = lock(ring).begin_process();
15759 let (release, gate) = oneshot::channel();
15760 let reader = HeldReader {
15761 before: Some(before.as_bytes().to_vec()),
15762 gate: Some(gate),
15763 after: io::Cursor::new(after.as_bytes().to_vec()),
15764 };
15765 let task = tokio::spawn(pump_stderr_to(
15766 reader,
15767 Arc::clone(ring),
15768 generation,
15769 DiscardSink,
15770 ));
15771 (StderrPump { task, generation }, release)
15772 }
15773
15774 async fn wait_until(ring: &Arc<Mutex<StderrRing>>, done: impl Fn(&StderrRing) -> bool) {
15775 for _ in 0..1000 {
15776 if done(&lock(ring)) {
15777 return;
15778 }
15779 tokio::time::sleep(Duration::from_millis(1)).await;
15780 }
15781 panic!(
15782 "ring never reached the expected state: {:?}",
15783 lock(ring).snapshot(None, None)
15784 );
15785 }
15786
15787 #[tokio::test(start_paused = true)]
15788 async fn a_crash_line_the_reader_had_not_reached_by_the_bound_is_kept_before_the_restart() {
15789 let ring = Arc::new(Mutex::new(StderrRing::new(StderrTailConfig::default())));
15790 let (pump, release) = held_pump(&ring, "booting\n", "config error: missing storage\n");
15791
15792 settle_stderr_pump("crasher", &ring, pump, BOUND).await;
15793 let before_release = lock(&ring).snapshot(None, None);
15794 assert!(
15795 matches!(before_release.capture, CaptureState::Incomplete { .. }),
15796 "a reader that has not reached EOF cannot claim a whole tail: {before_release:?}"
15797 );
15798
15799 let next = lock(&ring).begin_process();
15802 lock(&ring).push_line_from(next, "next process booting");
15803 release.send(()).unwrap();
15804 wait_until(&ring, |ring| {
15805 ring.snapshot(None, None).capture == CaptureState::Captured
15806 })
15807 .await;
15808
15809 assert_eq!(
15810 untimed(lock(&ring).snapshot(None, None).entries),
15811 vec![
15812 line("booting"),
15813 line("config error: missing storage"),
15814 TailEntry::ProcessStart,
15815 line("next process booting"),
15816 ],
15817 "the crash's last line must survive a slow reader and stay in the crashed process's section"
15818 );
15819 }
15820
15821 #[tokio::test(start_paused = true)]
15822 async fn a_pipe_held_open_by_a_descendant_reads_incomplete_without_delaying_the_restart_past_the_bound(
15823 ) {
15824 let ring = Arc::new(Mutex::new(StderrRing::new(StderrTailConfig::default())));
15825 let (pump, _held) = held_pump(&ring, "parent exiting\n", "");
15828
15829 let started = Instant::now();
15830 settle_stderr_pump("orphaning", &ring, pump, BOUND).await;
15831 assert_eq!(
15832 started.elapsed(),
15833 BOUND,
15834 "the restart must wait exactly the bound for a pipe that stays open, no longer"
15835 );
15836
15837 let next = lock(&ring).begin_process();
15838 lock(&ring).push_line_from(next, "next process booting");
15839 tokio::time::sleep(Duration::from_secs(60)).await;
15840
15841 let snapshot = lock(&ring).snapshot(None, None);
15842 match &snapshot.capture {
15843 CaptureState::Incomplete { reason } => assert!(
15844 reason.contains("had not reached EOF") && reason.contains("250ms"),
15845 "the reason must say what is missing and after how long: {reason}"
15846 ),
15847 other => panic!("expected Incomplete while the pipe is held open, got {other:?}"),
15848 }
15849 assert_eq!(
15850 untimed(snapshot.entries),
15851 vec![
15852 line("parent exiting"),
15853 TailEntry::ProcessStart,
15854 line("next process booting"),
15855 ]
15856 );
15857 }
15858
15859 #[tokio::test(start_paused = true)]
15860 async fn a_reader_that_reaches_eof_within_the_bound_leaves_the_tail_captured() {
15861 let ring = Arc::new(Mutex::new(StderrRing::new(StderrTailConfig::default())));
15862 let (pump, release) = held_pump(&ring, "one\n", "two\n");
15863 release.send(()).unwrap();
15864
15865 settle_stderr_pump("clean", &ring, pump, BOUND).await;
15866
15867 let snapshot = lock(&ring).snapshot(None, None);
15868 assert_eq!(snapshot.capture, CaptureState::Captured);
15869 assert_eq!(untimed(snapshot.entries), vec![line("one"), line("two")]);
15870 }
15871
15872 #[tokio::test(start_paused = true)]
15873 async fn stdout_buffered_past_the_exit_drain_bound_reaches_the_capture_file() {
15874 const BURST: usize = 150;
15875 let capture = cortexkit_test_support::ScratchDir::new("stdout-exit-drain");
15876 let path = capture.join("two-pipe.stderr.log");
15877 let sink = ChildOutputSink::open(&path, cortexkit_log::Retention::default()).unwrap();
15878 let ring = Arc::new(Mutex::new(StderrRing::new(StderrTailConfig::default())));
15879 let generation = lock(&ring).begin_process();
15880 let stdout = (0..BURST)
15881 .map(|i| format!("out-{i:04}-{}\n", "o".repeat(64)))
15882 .collect::<String>();
15883 let stderr = (0..BURST)
15884 .map(|i| format!("err-{i:04}-{}\n", "e".repeat(64)))
15885 .collect::<String>();
15886 let first_chunk = stdout.find('\n').unwrap() + 1 + "out-0001-".len();
15891 let (release, gate) = oneshot::channel();
15892 let reader = HeldReader {
15893 before: Some(stdout.as_bytes()[..first_chunk].to_vec()),
15894 gate: Some(gate),
15895 after: io::Cursor::new(stdout.as_bytes()[first_chunk..stdout.len() - 1].to_vec()),
15896 };
15897 let (finished, completion) = oneshot::channel();
15898 let stdout_sink = sink.clone();
15899 let pump = tokio::spawn(async move {
15900 pump_stdout_to(reader, stdout_sink).await;
15901 finished.send(()).unwrap();
15902 });
15903 pump_stderr_to(io::Cursor::new(stderr.into_bytes()), ring, generation, sink).await;
15904
15905 let started = Instant::now();
15906 settle_stdout_pump("two-pipe", pump, BOUND).await;
15907 assert_eq!(
15908 started.elapsed(),
15909 BOUND,
15910 "a late reader must not hold up restart"
15911 );
15912 assert_eq!(
15913 std::fs::read_to_string(&path).unwrap().lines().count(),
15914 BURST + 1
15915 );
15916
15917 release
15918 .send(())
15919 .expect("stdout reader must survive the exit drain bound");
15920 tokio::time::timeout(Duration::from_secs(1), completion)
15921 .await
15922 .expect("released stdout reader must reach EOF")
15923 .expect("stdout capture must finish at EOF, not be cancelled");
15924 let contents = std::fs::read_to_string(&path).unwrap();
15925 let mut lines = contents
15926 .lines()
15927 .map(|line| split_capture_stamp(line).expect("capture stamp").1)
15928 .collect::<Vec<_>>();
15929 let mut expected = (0..BURST)
15930 .flat_map(|i| {
15931 [
15932 format!("out-{i:04}-{}", "o".repeat(64)),
15933 format!("err-{i:04}-{}", "e".repeat(64)),
15934 ]
15935 })
15936 .collect::<Vec<_>>();
15937 lines.sort_unstable();
15938 expected.sort_unstable();
15939 assert_eq!(
15940 lines, expected,
15941 "both pipes must retain every complete line exactly once"
15942 );
15943 }
15944}
15945
15946#[cfg(all(test, windows))]
15960mod job_containment_tests {
15961 use super::*;
15962 use cortexkit_test_support::ScratchDir;
15963 use std::{
15964 path::{Path, PathBuf},
15965 sync::{Arc, Mutex},
15966 time::{Duration, Instant},
15967 };
15968
15969 fn stub_path() -> PathBuf {
15975 let mut path = std::env::current_exe().expect("current_exe available in tests");
15976 path.pop();
15977 path.pop();
15978 path.push("fake-aft-stub.exe");
15979 assert!(
15980 path.exists(),
15981 "fake-aft-stub not built at {}: run `cargo test -p subc-core` (which builds \
15982 [[bin]] targets) rather than `cargo test -p subc-core --lib` (which does not)",
15983 path.display()
15984 );
15985 path
15986 }
15987
15988 fn read_grandchild_pid(path: &Path) -> u32 {
15990 let deadline = Instant::now() + Duration::from_secs(10);
15991 loop {
15992 if let Ok(contents) = std::fs::read_to_string(path) {
15993 if let Ok(pid) = contents.trim().parse() {
15994 return pid;
15995 }
15996 }
15997 assert!(
15998 Instant::now() < deadline,
15999 "the stub never recorded a grandchild pid at {}",
16000 path.display()
16001 );
16002 std::thread::sleep(Duration::from_millis(10));
16003 }
16004 }
16005
16006 struct Fixture {
16009 _dir: ScratchDir,
16010 module_id: String,
16011 grandchild: u32,
16012 child: Option<SupervisedChild>,
16013 registry: Arc<Registry>,
16014 snapshot: Arc<Mutex<SupervisorSnapshot>>,
16015 terminal_ring: Arc<Mutex<TerminalRing>>,
16016 spawn_events: SpawnEventFeed,
16017 }
16018
16019 fn fixture(label: &str, module_id: &str) -> Fixture {
16020 let dir = ScratchDir::new(label);
16021 let pid_file = dir.join("grandchild.pid");
16022 let supervisor = Supervisor::new_for_test(
16023 Arc::new(Registry::default()),
16024 RestartPolicy::new(3, Duration::ZERO),
16025 );
16026 let runtime = supervisor.runtime_config();
16027 let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::starting()));
16028 let spec = ModuleSpec {
16029 module_id: module_id.to_string(),
16030 program: stub_path(),
16031 args: Vec::new(),
16035 env: vec![
16036 ("FAKE_AFT_NEVER_CONNECT".to_string(), "1".to_string()),
16037 (
16038 "FAKE_AFT_GRANDCHILD_PID_FILE".to_string(),
16039 pid_file.display().to_string(),
16040 ),
16041 ],
16042 reserved: false,
16043 reserved_prefixes: Vec::new(),
16044 protocol: ModuleProtocol::Subc,
16045 overlap: Default::default(),
16046 };
16047 let child = spawn_and_mark_running(&spec, &runtime, &snapshot)
16048 .expect("spawn the supervised fixture");
16049 let grandchild = read_grandchild_pid(&pid_file);
16050 Fixture {
16051 _dir: dir,
16052 module_id: module_id.to_string(),
16053 grandchild,
16054 child: Some(child),
16055 registry: Arc::new(Registry::default()),
16056 snapshot,
16057 terminal_ring: Arc::clone(&runtime.terminal_ring),
16058 spawn_events: SpawnEventFeed::default(),
16059 }
16060 }
16061
16062 impl Fixture {
16063 async fn drain(&mut self) {
16065 let child = self
16066 .child
16067 .take()
16068 .expect("the fixture child is still present");
16069 drain_child_to_state(
16070 &self.module_id,
16071 ModuleProtocol::Subc,
16072 StopNotice::NotSent,
16075 &self.registry,
16076 None,
16077 &self.snapshot,
16078 &self.terminal_ring,
16079 &self.spawn_events,
16080 child,
16081 Duration::from_millis(500),
16082 ModuleState::Stopped,
16083 Some(false),
16084 )
16085 .await
16086 .expect("drain the supervised fixture");
16087 }
16088 }
16089
16090 #[tokio::test]
16096 async fn teardown_reaps_the_grandchild() {
16097 let mut fixture = fixture("teardown-grandchild", "tree-teardown");
16098 let grandchild = fixture.grandchild;
16099
16100 assert!(
16101 subc_jobobject::process_exists(grandchild),
16102 "grandchild {grandchild} must be alive before teardown, or this proves nothing"
16103 );
16104
16105 fixture.drain().await;
16106
16107 assert!(
16108 subc_jobobject::wait_for_process_exit(grandchild, Duration::from_secs(10)),
16109 "grandchild {grandchild} outlived module teardown: the tree was not contained"
16110 );
16111 }
16112
16113 #[test]
16126 fn an_uncontained_grandchild_survives_a_direct_child_kill() {
16127 let dir = ScratchDir::new("teardown-uncontained");
16128 let pid_file = dir.join("grandchild.pid");
16129 let mut child = std::process::Command::new(stub_path())
16130 .env("FAKE_AFT_NEVER_CONNECT", "1")
16131 .env(
16132 "FAKE_AFT_GRANDCHILD_PID_FILE",
16133 pid_file.display().to_string(),
16134 )
16135 .stdin(std::process::Stdio::null())
16136 .stdout(std::process::Stdio::null())
16137 .stderr(std::process::Stdio::null())
16138 .spawn()
16139 .expect("spawn the uncontained fixture");
16140 let grandchild = read_grandchild_pid(&pid_file);
16141
16142 child.kill().expect("kill the direct child");
16144 let _ = child.wait();
16145
16146 assert!(
16147 subc_jobobject::process_exists(grandchild),
16148 "grandchild {grandchild} died with the direct child, so this control no longer \
16149 distinguishes contained from uncontained teardown and the regression test is \
16150 passing vacuously"
16151 );
16152
16153 kill_tree(grandchild);
16156 }
16157
16158 #[tokio::test]
16167 async fn dropping_containment_reaps_the_grandchild() {
16168 let mut fixture = fixture("drop-containment", "tree-drop");
16169 let grandchild = fixture.grandchild;
16170
16171 assert!(subc_jobobject::process_exists(grandchild));
16172
16173 fixture.child.as_mut().expect("child present").job = None;
16175
16176 assert!(
16177 subc_jobobject::wait_for_process_exit(grandchild, Duration::from_secs(10)),
16178 "grandchild {grandchild} survived the containment handle closing, so a daemon \
16179 crash would leave the tree behind"
16180 );
16181 }
16182
16183 fn kill_tree(pid: u32) {
16185 let _ = std::process::Command::new("taskkill.exe")
16186 .args(["/PID", &pid.to_string(), "/T", "/F"])
16187 .stdin(std::process::Stdio::null())
16188 .stdout(std::process::Stdio::null())
16189 .stderr(std::process::Stdio::null())
16190 .status();
16191 assert!(
16192 subc_jobobject::wait_for_process_exit(pid, Duration::from_secs(10)),
16193 "could not clean up grandchild {pid}"
16194 );
16195 }
16196}
16197
16198#[cfg(test)]
16199mod privacy_trampoline_configuration_tests {
16200 #[tokio::test]
16201 #[cfg_attr(not(target_os = "macos"), ignore = "requires macOS privacy trampoline")]
16202 async fn macos_spawn_without_a_configured_trampoline_refuses_by_name() {
16203 #[cfg(target_os = "macos")]
16204 {
16205 let supervisor = super::Supervisor::new(
16206 std::sync::Arc::new(crate::Registry::default()),
16207 super::RestartPolicy::default(),
16208 );
16209 let error = supervisor.spawn(spec()).unwrap_err();
16210 assert!(
16211 error
16212 .to_string()
16213 .contains("no privacy trampoline configured"),
16214 "{error}"
16215 );
16216 }
16217 }
16218
16219 #[tokio::test]
16220 #[cfg_attr(not(target_os = "macos"), ignore = "requires macOS privacy trampoline")]
16221 async fn macos_wrong_trampoline_is_rejected_by_the_startup_probe() {
16222 #[cfg(target_os = "macos")]
16223 {
16224 let supervisor = super::Supervisor::new(
16225 std::sync::Arc::new(crate::Registry::default()),
16226 super::RestartPolicy::default(),
16227 )
16228 .with_privacy_trampoline(std::env::current_exe().unwrap());
16229 let error = supervisor.spawn(spec()).unwrap_err();
16230 assert!(
16231 error
16232 .to_string()
16233 .contains("binary does not implement the privacy trampoline protocol"),
16234 "{error}"
16235 );
16236 }
16237 }
16238
16239 #[cfg(target_os = "macos")]
16240 fn spec() -> super::ModuleSpec {
16241 super::ModuleSpec {
16242 module_id: "privacy-configuration".into(),
16243 program: "/bin/sleep".into(),
16244 args: vec!["30".into()],
16245 env: vec![],
16246 reserved: false,
16247 reserved_prefixes: vec![],
16248 protocol: subc_control::ModuleProtocol::None,
16249 overlap: super::ModuleOverlap::Exclusive,
16250 }
16251 }
16252}
16253
16254#[cfg(test)]
16255mod privacy_exec_boundary_tests {
16256 #[cfg(target_os = "macos")]
16257 use super::*;
16258 #[cfg(target_os = "macos")]
16259 use std::{
16260 io::{Read, Write},
16261 net::{TcpListener, TcpStream},
16262 };
16263
16264 #[cfg(target_os = "macos")]
16267 pub(super) fn before_image_sample(spec: &ModuleSpec, pid: u32) {
16268 if let Some((_, path)) = spec
16269 .env
16270 .iter()
16271 .find(|(key, _)| key == "SUBC_TEST_PRIVACY_SAMPLE_BARRIER")
16272 {
16273 let mut barrier = TcpStream::connect(path).unwrap();
16274 barrier
16275 .set_read_timeout(Some(Duration::from_secs(30)))
16276 .unwrap();
16277 barrier.write_all(&pid.to_ne_bytes()).unwrap();
16278 let mut release = [0];
16279 barrier.read_exact(&mut release).unwrap();
16280 assert_eq!(&release, b"X");
16281 }
16282 }
16283
16284 #[cfg(target_os = "macos")]
16285 fn spec(program: &str, args: &[&str]) -> ModuleSpec {
16286 ModuleSpec {
16287 module_id: "privacy-boundary".into(),
16288 program: program.into(),
16289 args: args.iter().map(|arg| (*arg).into()).collect(),
16290 env: vec![],
16291 reserved: false,
16292 reserved_prefixes: vec![],
16293 protocol: ModuleProtocol::None,
16294 overlap: ModuleOverlap::Exclusive,
16295 }
16296 }
16297
16298 #[cfg(target_os = "macos")]
16299 async fn accept(listener: TcpListener) -> TcpStream {
16300 let listener = tokio::net::TcpListener::from_std({
16302 listener.set_nonblocking(true).unwrap();
16303 listener
16304 })
16305 .unwrap();
16306 let (stream, _) = tokio::time::timeout(Duration::from_secs(30), listener.accept())
16307 .await
16308 .unwrap()
16309 .unwrap();
16310 let stream = stream.into_std().unwrap();
16311 stream.set_nonblocking(false).unwrap();
16312 stream
16313 .set_read_timeout(Some(Duration::from_secs(30)))
16314 .unwrap();
16315 stream
16316 }
16317
16318 #[tokio::test]
16319 #[cfg_attr(not(target_os = "macos"), ignore = "requires macOS privacy trampoline")]
16320 async fn macos_roster_withholds_a_nonnull_trampoline_image_until_exec_confirmation() {
16321 #[cfg(target_os = "macos")]
16322 {
16323 let root = cortexkit_test_support::ScratchDir::new("privacy-roster-barrier");
16324 let exec_listener = TcpListener::bind("127.0.0.1:0").unwrap();
16327 let sample_listener = TcpListener::bind("127.0.0.1:0").unwrap();
16328 let record = root.join("live-children.json");
16329 let supervisor =
16330 Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default())
16331 .with_live_children_record(&record);
16332 let runtime = supervisor.runtime_config();
16333 let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::starting()));
16334 let mut spec = spec("/bin/sleep", &["30"]);
16335 spec.env = vec![
16336 (
16337 "SUBC_TEST_PRIVACY_EXEC_BARRIER".into(),
16338 exec_listener.local_addr().unwrap().to_string(),
16339 ),
16340 (
16341 "SUBC_TEST_PRIVACY_SAMPLE_BARRIER".into(),
16342 sample_listener.local_addr().unwrap().to_string(),
16343 ),
16344 ];
16345 let spawn = tokio::task::spawn_blocking(move || {
16346 spawn_and_mark_running(&spec, &runtime, &snapshot).unwrap()
16347 });
16348 let mut sample = accept(sample_listener).await;
16349 let mut pid = [0; 4];
16350 sample.read_exact(&mut pid).unwrap();
16351 let pid = u32::from_ne_bytes(pid);
16352 let mut exec = accept(exec_listener).await;
16353 let mut ready = [0];
16354 exec.read_exact(&mut ready).unwrap();
16355 assert_eq!(&ready, b"R");
16356 let trampoline = subc_os::file_identity(&test_privacy_trampoline()).unwrap();
16359 assert_eq!(
16360 observe_spawned_image(pid).unwrap().executable,
16361 Some(trampoline)
16362 );
16363 sample.write_all(b"X").unwrap();
16364 let mut child = spawn.await.unwrap();
16365 let early = crate::live_children::read_record(&record).unwrap();
16366 assert_eq!(early.len(), 1);
16367 assert_eq!(early[0].pid, pid);
16368 assert_eq!(
16369 early[0].executable, None,
16370 "unconfirmed trampoline image entered the roster"
16371 );
16372 assert!(child.report_ready.get().is_none());
16373 child.privacy_exec.as_mut().unwrap().deadline =
16376 tokio::time::Instant::now() + Duration::from_secs(30);
16377 exec.write_all(b"X").unwrap();
16378 child.confirm_privacy_exec().await;
16379 assert_eq!(child.spawn_failure, None);
16380 assert!(child.report_ready.get().is_some());
16381 let confirmed = crate::live_children::read_record(&record).unwrap();
16382 let module = subc_os::file_identity(std::path::Path::new("/bin/sleep")).unwrap();
16383 assert_ne!(module, trampoline);
16384 assert_eq!(confirmed[0].executable, Some(module.into()));
16385 child.start_kill().unwrap();
16386 child.wait().await.unwrap();
16387 child.release_roster();
16388 }
16389 }
16390
16391 #[tokio::test]
16392 #[cfg_attr(not(target_os = "macos"), ignore = "requires macOS privacy trampoline")]
16393 async fn macos_already_exited_121_in_try_wait_is_a_module_exit_not_a_trampoline_refusal() {
16394 #[cfg(target_os = "macos")]
16395 {
16396 let registry = Arc::new(Registry::default());
16397 let policy = RestartPolicy::new(0, Duration::ZERO);
16398 let supervisor = Supervisor::new_for_test(Arc::clone(®istry), policy);
16399 let runtime = supervisor.runtime_config();
16400 let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::starting()));
16401 let spec = spec("/bin/sh", &["-c", "exit 121"]);
16402 let mut child = spawn_and_mark_running(&spec, &runtime, &snapshot).unwrap();
16406 let pid = child.pid;
16407 tokio::task::spawn_blocking(move || {
16408 subc_os::fork_exec_test::wait_for_child_exit_without_reaping(pid)
16409 })
16410 .await
16411 .unwrap()
16412 .unwrap();
16413 child.privacy_exec.as_mut().unwrap().deadline =
16414 tokio::time::Instant::now() + Duration::from_secs(30);
16415 let status = child.wait().await.unwrap();
16416 assert_eq!(status.code(), Some(121));
16417 assert!(child.privacy_exec.is_none());
16418 assert!(
16419 child.report_ready.get().is_none(),
16420 "an exited module must not publish a live pid"
16421 );
16422 let report = classify_reaped_child_exit(&snapshot, &child, &status);
16423 on_child_exit(
16424 &spec,
16425 policy,
16426 ®istry,
16427 &snapshot,
16428 &runtime.terminal_ring,
16429 &runtime.spawn_events,
16430 &runtime.child_roster,
16431 report,
16432 )
16433 .await;
16434 let state = lock_snapshot(&snapshot).unwrap();
16435 assert_eq!(state.state, ModuleState::Failed);
16436 assert_eq!(state.last_exit.as_ref().unwrap().code, Some(121));
16437 assert_eq!(state.reported_pid(), None);
16438 drop(state);
16439 let history = runtime.terminal_ring.lock().unwrap().snapshot();
16440 assert_eq!(history.entries.len(), 1);
16441 let terminal = &history.entries[0];
16442 assert_eq!(terminal.exit_code, Some(121));
16443 assert_eq!(terminal.exit_kind, subc_control::TerminalExitKind::Crash);
16444 assert_eq!(terminal.disposition, TerminalDisposition::Failed);
16445 assert_eq!(
16446 terminal.disposition_detail.as_deref(),
16447 Some(policy.budget_exhausted_detail().as_str()),
16448 "module exit 121 was classified as a trampoline refusal: {terminal:?}"
16449 );
16450 assert_eq!(child.spawn_failure, None);
16451 child.release_roster();
16452 }
16453 }
16454}
16455
16456#[cfg(all(test, unix))]
16460mod launch_nonce_descriptor_tests {
16461 use super::{ChildRoster, ModuleProtocol, ModuleSpec, SupervisorHandle};
16462 use crate::stderr_tail::{StderrRing, StderrTailConfig};
16463 use cortexkit_test_support::ScratchDir;
16464 use std::{
16465 path::PathBuf,
16466 sync::{Arc, Mutex},
16467 time::{Duration, Instant},
16468 };
16469
16470 async fn probe(role: super::SpawnRole) {
16471 let scratch = ScratchDir::new("launch-nonce-descriptor");
16472 let fd_copy = scratch.join("from-descriptor");
16473 let env_copy = scratch.join("environment");
16474 let script = format!(
16475 "env > '{env}.tmp' && mv '{env}.tmp' '{env}'; cat <&3 > '{fd}.tmp' && mv '{fd}.tmp' '{fd}'; sleep 30",
16476 fd = fd_copy.display(), env = env_copy.display(),
16477 );
16478 let xdg = |name: &str| (name.to_string(), scratch.join(name).display().to_string());
16479 let spec = ModuleSpec {
16480 module_id: "nonce-descriptor-probe".to_string(),
16481 program: PathBuf::from("/bin/sh"),
16482 args: vec!["-c".to_string(), script],
16483 env: vec![
16484 xdg("XDG_DATA_HOME"),
16485 xdg("XDG_RUNTIME_DIR"),
16486 xdg("XDG_CONFIG_HOME"),
16487 ("SUBC_LAUNCH_NONCE".to_string(), "must-not-leak".to_string()),
16488 ],
16489 reserved: true,
16490 reserved_prefixes: Vec::new(),
16491 protocol: ModuleProtocol::Subc,
16492 overlap: Default::default(),
16493 };
16494 let handle = SupervisorHandle::new();
16495 let ring = Arc::new(Mutex::new(StderrRing::new(StderrTailConfig::default())));
16496 let roster = ChildRoster::default();
16497 #[cfg(target_os = "macos")]
16498 {
16499 let path = super::test_privacy_trampoline();
16500 roster.set_privacy_trampoline(super::probe_privacy_trampoline(&path).map(|()| path));
16501 }
16502 let child = super::spawn_child_in_slot(
16503 &spec,
16504 None,
16505 Some(&handle),
16506 &ring,
16507 None,
16508 &roster,
16509 #[cfg(target_os = "linux")]
16510 None,
16511 role,
16512 matches!(role, super::SpawnRole::SwapCandidate),
16513 )
16514 .expect("spawn probe");
16515 let deadline = Instant::now() + Duration::from_secs(10);
16516 while !(fd_copy.exists() && env_copy.exists()) {
16517 assert!(Instant::now() < deadline, "probe never wrote its copies");
16518 tokio::time::sleep(Duration::from_millis(20)).await;
16519 }
16520 let nonce = std::fs::read_to_string(fd_copy).unwrap();
16521 assert!(!nonce.is_empty());
16522 let environment = std::fs::read_to_string(env_copy).unwrap();
16523 assert!(environment
16524 .lines()
16525 .any(|line| line.starts_with("SUBC_LAUNCH_NONCE_FD=3:")));
16526 let copy = environment
16527 .lines()
16528 .find_map(|line| line.strip_prefix("SUBC_LAUNCH_NONCE="));
16529 assert_eq!(
16530 copy, None,
16531 "Unix children must never receive the environment nonce"
16532 );
16533 if matches!(role, super::SpawnRole::Plain) {
16534 assert_eq!(
16535 handle.spawn_nonce(&spec.module_id).as_deref(),
16536 Some(nonce.as_str())
16537 );
16538 }
16539 drop(child);
16540 }
16541
16542 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
16543 async fn a_spawned_module_receives_its_nonce_only_on_descriptor_3() {
16544 probe(super::SpawnRole::Plain).await;
16545 }
16546
16547 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
16548 async fn swap_candidate_receives_its_nonce_only_on_descriptor_3() {
16549 probe(super::SpawnRole::SwapCandidate).await;
16550 }
16551}
16552
16553#[cfg(all(test, target_os = "linux"))]
16554mod cgroup_containment_tests {
16555 use super::*;
16556 use cortexkit_test_support::ScratchDir;
16557
16558 fn running(pid: u32) -> bool {
16559 std::fs::read_to_string(format!("/proc/{pid}/stat"))
16561 .ok()
16562 .and_then(|stat| {
16563 stat.rsplit_once(") ")
16564 .map(|(_, rest)| rest.starts_with('Z'))
16565 })
16566 .is_some_and(|zombie| !zombie)
16567 }
16568
16569 #[tokio::test]
16570 async fn linux_teardown_reaps_the_grandchild() {
16571 teardown_tree("linux_teardown_reaps_the_grandchild", false).await;
16572 }
16573
16574 #[tokio::test]
16575 async fn linux_shutdown_straggler_reaps_the_grandchild() {
16576 teardown_tree("linux_shutdown_straggler_reaps_the_grandchild", true).await;
16577 }
16578
16579 async fn teardown_tree(test_name: &str, shutdown: bool) {
16580 let dir = ScratchDir::new(test_name);
16581 let root = PathBuf::from(format!(
16582 "/sys/fs/cgroup/subc-tree-test-{test_name}-{}-{}",
16583 std::process::id(),
16584 unix_ms_now()
16585 ));
16586 if let Err(error) = std::fs::create_dir(&root) {
16587 assert!(
16588 std::env::var_os("SUBC_REQUIRE_CGROUP_TEST").is_none(),
16589 "required cgroup test cannot execute: {error}"
16590 );
16591 eprintln!(
16592 "SKIP {test_name}: no writable delegated cgroup at {}: {error}",
16593 root.display()
16594 );
16595 return;
16596 }
16597 let placement = subc_cgroup::prepare_at(&root)
16598 .expect("prepare isolated kernel cgroup")
16599 .expect("isolated cgroup is delegated");
16600 let module_id = "tree-teardown";
16601 let module = placement
16602 .module_path(module_id)
16603 .expect("create isolated module cgroup");
16604 if !module.join("cgroup.kill").exists() {
16605 std::fs::remove_dir(&module).unwrap();
16606 std::fs::remove_dir(root.join("subc-modules")).unwrap();
16607 std::fs::remove_dir(&root).unwrap();
16608 assert!(
16609 std::env::var_os("SUBC_REQUIRE_CGROUP_TEST").is_none(),
16610 "required cgroup.kill interface unavailable"
16611 );
16612 eprintln!("SKIP {test_name}: cgroup.kill unavailable (kernel < 5.14)");
16613 return;
16614 }
16615 let supervisor = Supervisor::new_for_test(
16616 Arc::new(Registry::default()),
16617 RestartPolicy::new(3, Duration::ZERO),
16618 )
16619 .with_cgroup_placement(Some(placement));
16620 let mut runtime = supervisor.runtime_config();
16621 runtime.child_roster = runtime
16622 .child_roster
16623 .for_module(Arc::new(Mutex::new(Duration::from_millis(100))));
16624 let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::starting()));
16625 let pid_file = dir.join("grandchild.pid");
16626 let spec = ModuleSpec {
16627 module_id: module_id.to_string(),
16628 program: PathBuf::from("/bin/sh"),
16629 args: vec![
16630 "-c".into(),
16631 "trap '' TERM; sleep 600 & echo $! > \"$1\"; wait".into(),
16632 "fixture".into(),
16633 pid_file.display().to_string(),
16634 ],
16635 env: ["XDG_DATA_HOME", "XDG_RUNTIME_DIR", "XDG_CONFIG_HOME"]
16636 .into_iter()
16637 .map(|key| (key.to_string(), dir.display().to_string()))
16638 .collect(),
16639 reserved: false,
16640 reserved_prefixes: Vec::new(),
16641 protocol: ModuleProtocol::None,
16642 overlap: Default::default(),
16643 };
16644 let child =
16645 spawn_and_mark_running(&spec, &runtime, &snapshot).expect("spawn supervised tree");
16646 let deadline = tokio::time::Instant::now() + Duration::from_secs(10);
16647 let grandchild: u32 = loop {
16648 if let Ok(contents) = std::fs::read_to_string(&pid_file) {
16649 if let Ok(pid) = contents.trim().parse() {
16650 break pid;
16651 }
16652 }
16653 assert!(
16654 tokio::time::Instant::now() < deadline,
16655 "grandchild pid was not recorded"
16656 );
16657 tokio::time::sleep(Duration::from_millis(10)).await;
16658 };
16659 assert!(
16660 running(grandchild),
16661 "grandchild must be alive before teardown"
16662 );
16663 if shutdown {
16664 let mut child = child;
16665 crate::child_roster::end_children_for_daemon_shutdown(
16666 &runtime.child_roster,
16667 false,
16668 std::future::pending(),
16669 )
16670 .await;
16671 child.wait().await.expect("reap shutdown straggler");
16672 } else {
16673 drain_child_to_state(
16674 module_id,
16675 ModuleProtocol::None,
16676 StopNotice::NotSent,
16677 &Registry::default(),
16678 None,
16679 &snapshot,
16680 &runtime.terminal_ring,
16681 &SpawnEventFeed::default(),
16682 child,
16683 Duration::from_millis(100),
16684 ModuleState::Stopped,
16685 Some(false),
16686 )
16687 .await
16688 .expect("real supervisor teardown");
16689 }
16690 let deadline = tokio::time::Instant::now() + Duration::from_secs(5);
16691 while running(grandchild) && tokio::time::Instant::now() < deadline {
16692 tokio::time::sleep(Duration::from_millis(10)).await;
16693 }
16694 let survived = running(grandchild);
16695 if survived {
16697 let pid = rustix::process::Pid::from_raw(grandchild as i32).unwrap();
16698 let _ = rustix::process::kill_process(pid, rustix::process::Signal::KILL);
16699 tokio::time::sleep(Duration::from_millis(100)).await;
16700 }
16701 if module.exists() {
16702 std::fs::remove_dir(&module).expect("remove empty module cgroup");
16703 }
16704 std::fs::remove_dir(root.join("subc-modules")).unwrap();
16705 std::fs::remove_dir(&root).unwrap();
16706 assert!(
16707 !survived,
16708 "grandchild {grandchild} outlived module teardown"
16709 );
16710 eprintln!("EXECUTED {test_name}: grandchild {grandchild} killed in isolated cgroup");
16711 }
16712}
16713
16714#[cfg(all(test, unix))]
16717mod record_write_stall_tests {
16718 use super::*;
16719 use std::sync::atomic::{AtomicBool, Ordering};
16720 use subc_transport::{authenticate_client, ConnectionInfo, Endpoint, SCHEMA_VERSION};
16721
16722 const MODULE_AUTH_DEADLINE: Duration = Duration::from_secs(2);
16725 const HELD_RECORD_WRITE: Duration = Duration::from_secs(3);
16729
16730 fn sleeping_spec() -> ModuleSpec {
16742 ModuleSpec {
16743 module_id: "record-write-stall".into(),
16744 program: "/bin/sleep".into(),
16745 args: vec!["600".into()],
16746 env: vec![],
16747 reserved: false,
16748 reserved_prefixes: vec![],
16749 protocol: ModuleProtocol::None,
16750 overlap: Default::default(),
16751 }
16752 }
16753
16754 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
16755 async fn module_auth_completes_while_the_restart_waits_on_the_record_write() {
16756 let home = cortexkit_test_support::ScratchDir::new("record-write-stall");
16757 let supervisor =
16758 Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default())
16759 .with_live_children_record(home.join("live-children.json"));
16760 let module = supervisor.spawn(sleeping_spec()).unwrap();
16761 let old_pid = tokio::time::timeout(Duration::from_secs(10), async {
16766 loop {
16767 if let Some(pid) = lock_snapshot(&module.inner.snapshot)
16768 .unwrap()
16769 .reported_pid()
16770 {
16771 return pid;
16772 }
16773 tokio::time::sleep(Duration::from_millis(10)).await;
16774 }
16775 })
16776 .await
16777 .expect("the first process was admitted as the module");
16778
16779 let key = vec![0x42; 32];
16782 let daemon_id = [0x24; 16];
16783 let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
16784 let port = listener.local_addr().unwrap().port();
16785 let server = tokio::spawn(crate::server::serve_listener(
16786 listener,
16787 Arc::new(crate::router::Router::with_default_self_handler()),
16788 crate::server::ServerAuth::with_limits(
16789 key.clone(),
16790 daemon_id,
16791 "test",
16792 crate::server::DEFAULT_AUTH_DEADLINE,
16793 4,
16794 ),
16795 ));
16796
16797 let (held_tx, held_rx) = std::sync::mpsc::channel();
16800 let armed = AtomicBool::new(true);
16801 supervisor
16802 .child_roster
16803 .set_record_write_hook_for_test(move |live| {
16804 let admitting_replacement = live.values().any(|entry| entry.pid != old_pid);
16805 if admitting_replacement && armed.swap(false, Ordering::SeqCst) {
16806 let _ = held_tx.send(());
16807 std::thread::sleep(HELD_RECORD_WRITE);
16808 }
16809 });
16810
16811 let module_process = std::thread::spawn(move || {
16814 held_rx
16815 .recv_timeout(Duration::from_secs(30))
16816 .expect("the restart reached the replacement's record write");
16817 let conn = ConnectionInfo {
16818 schema: SCHEMA_VERSION,
16819 wire_version: None,
16820 endpoints: vec![Endpoint {
16821 host: "127.0.0.1".to_owned(),
16822 port,
16823 }],
16824 key,
16825 daemon_id,
16826 pid: std::process::id(),
16827 daemon_ver: "test".to_owned(),
16828 };
16829 tokio::runtime::Builder::new_current_thread()
16830 .enable_all()
16831 .build()
16832 .unwrap()
16833 .block_on(async {
16834 let started = Instant::now();
16835 let mut stream = tokio::net::TcpStream::connect(("127.0.0.1", port))
16836 .await
16837 .unwrap();
16838 let result =
16839 authenticate_client(&mut stream, &conn, MODULE_AUTH_DEADLINE).await;
16840 (result, started.elapsed())
16841 })
16842 });
16843
16844 module.restart(Some(0)).await.unwrap();
16845 let (auth, elapsed) = tokio::task::spawn_blocking(move || module_process.join().unwrap())
16846 .await
16847 .unwrap();
16848 assert!(
16849 auth.is_ok(),
16850 "the daemon must answer a module's handshake while a restart waits on the \
16851 live-children record write; got {auth:?} after {elapsed:?}"
16852 );
16853
16854 tokio::time::timeout(Duration::from_secs(20), async {
16855 while lock_snapshot(&module.inner.snapshot)
16856 .unwrap()
16857 .reported_pid()
16858 .is_none_or(|pid| pid == old_pid)
16859 {
16860 tokio::time::sleep(Duration::from_millis(10)).await;
16861 }
16862 })
16863 .await
16864 .expect("the restart published its replacement");
16865 module.stop().await.unwrap();
16866 server.abort();
16867 }
16868}