1use std::{
2 collections::{HashMap, HashSet, VecDeque},
3 error::Error,
4 fmt, io,
5 path::PathBuf,
6 process::{ExitStatus, Stdio},
7 sync::{Arc, Mutex, OnceLock},
8 time::{Duration, SystemTime, UNIX_EPOCH},
9};
10
11use cortexkit_log::Retention;
12use serde_json::Value;
13use sha2::{Digest, Sha256};
14use subc_control::{
15 ClientControlPush, LiveSpawn, ModuleProtocol, RouteCloseReason, SpawnCursor, SpawnEvent,
16 SpawnEventKind, SpawnSnapshot, SupervisorHealthStatus, TerminalDisposition, TerminalExitKind,
17};
18use subc_protocol::{
19 manifest::{SelfSignalKind, SignalAnchor},
20 session::{
21 HealthReport, HealthStatus, ModuleControlCommand, ModuleControlRequest,
22 MODULE_CONTROL_OP_HEALTH_CHECK,
23 },
24 Flags, FrameType, Principal, Priority, SUBC_LAUNCH_NONCE_ENV, SUBC_MODULE_ID_ENV,
25};
26use tokio::{
27 process::{Child, Command},
28 sync::{mpsc, oneshot, watch, Mutex as AsyncMutex},
29 task::JoinHandle,
30 time::{sleep, sleep_until, timeout, timeout_at, Instant},
31};
32use tracing::{debug, error, info, warn};
33
34use crate::{
35 child_roster::ChildRoster,
36 daemon_config::{
37 CAPTURE_KEEP_ENV, CAPTURE_MAX_AGE_DAYS_ENV, CAPTURE_MAX_FILE_MB_ENV, CK_LOG_ENV,
38 },
39 forwarding::{
40 CloseReason, ForwardingError, ForwardingTable, GoodbyeTarget, ModuleControlRpcOutcome,
41 ModuleDrainTarget, PendingModuleControlRpc,
42 },
43 provenance::{ExecutableIdentityProbe, SpawnedFileIdentity},
44 registry::{ConnectionId, RegistrationEndReason, RegistryError},
45 stderr_tail::{
46 pump_stderr_to, pump_stdout_to, ChildOutputSink, StderrRing, StderrTailConfig,
47 StderrTailSnapshot,
48 },
49 terminal_ring::{TerminalHistorySnapshot, TerminalRecord, TerminalRing, TerminalRingConfig},
50 Frame, FrameSink, Registry,
51};
52
53#[cfg(not(windows))]
54use crate::provenance::spawned_file_identity;
55
56#[path = "supervise_swap.rs"]
57mod swap;
58
59#[path = "supervise_activity.rs"]
60mod activity;
61
62pub const SUBC_ARG: &str = "--subc";
68
69const DEFAULT_MAX_RESTARTS: u32 = 3;
70const DEFAULT_BACKOFF: Duration = Duration::from_millis(100);
71const DEFAULT_MAX_BACKOFF: Duration = Duration::from_secs(30);
72const DEFAULT_RESTART_WINDOW: Duration = Duration::from_secs(600);
76pub const DEFAULT_DRAIN_TIMEOUT: Duration = Duration::from_secs(30);
87const REGISTRY_RELEASE_TIMEOUT: Duration = Duration::from_secs(1);
88const REGISTRY_RELEASE_POLL: Duration = Duration::from_millis(10);
89const DRAIN_BUSY_REPROBE_INTERVAL: Duration = Duration::from_millis(250);
101const STDERR_PUMP_DRAIN_TIMEOUT: Duration = Duration::from_millis(250);
124pub const SPAWN_EVENT_RING_CAPACITY: usize = 4096;
126const SPAWN_SUBSCRIBER_BUFFER: usize = SPAWN_EVENT_RING_CAPACITY + 1;
127pub(crate) const SPAWN_SUBSCRIBER_LAGGED_CODE: &str = "spawn_subscriber_lagged";
132
133#[cfg(target_os = "macos")]
134struct PrivacyExec {
135 reader: tokio::io::unix::AsyncFd<std::io::PipeReader>,
136 deadline: tokio::time::Instant,
137 expected: Option<subc_os::FileIdentity>,
138 trampoline: Option<subc_os::FileIdentity>,
139 script: bool,
140 module_id: String,
141 #[cfg(test)]
142 confirmation_pause: Option<(String, Option<tokio::net::TcpStream>)>,
143}
144
145#[cfg(all(test, target_os = "macos"))]
146pub(crate) fn test_privacy_trampoline() -> PathBuf {
147 let path = std::env::current_exe()
150 .unwrap()
151 .parent()
152 .unwrap()
153 .parent()
154 .unwrap()
155 .join("privacy-trampoline-fixture");
156 assert!(
160 path.exists(),
161 "privacy-trampoline-fixture not built at {}: run `cargo build -p subc-daemon \
162 --bins --features test-support` or `cargo test -p subc-daemon` first",
163 path.display()
164 );
165 path
166}
167
168#[cfg(target_os = "macos")]
169fn probe_privacy_trampoline(path: &std::path::Path) -> Result<(), String> {
170 use std::io::Read;
171 let mut probe = std::process::Command::new(path)
172 .args(["__disclaim-exec", "--probe"])
173 .stdin(Stdio::null())
174 .stdout(Stdio::piped())
175 .stderr(Stdio::piped())
176 .spawn()
177 .map_err(|error| {
178 format!(
179 "privacy trampoline probe failed for {}: {error}",
180 path.display()
181 )
182 })?;
183 let deadline = std::time::Instant::now() + Duration::from_secs(5);
184 let status = loop {
185 match probe.try_wait() {
186 Ok(Some(status)) => break status,
187 Ok(None) if std::time::Instant::now() < deadline => {
188 std::thread::sleep(Duration::from_millis(5))
189 }
190 result => {
191 let _ = probe.kill();
192 let _ = probe.wait();
193 return Err(format!(
194 "privacy trampoline probe failed or timed out for {}: {result:?}",
195 path.display()
196 ));
197 }
198 }
199 };
200 let mut answer = String::new();
201 if let Some(stdout) = probe.stdout.take() {
202 let _ = stdout.take(256).read_to_string(&mut answer);
203 }
204 if status.success() && answer.trim() == subc_os::privacy_identity::TRAMPOLINE_PROBE {
205 return Ok(());
206 }
207 let mut diagnostic = String::new();
208 if let Some(stderr) = probe.stderr.take() {
209 let _ = stderr.take(1024).read_to_string(&mut diagnostic);
210 }
211 let cause = diagnostic
212 .trim()
213 .strip_prefix("ck-subc: own privacy identity refused: ")
214 .unwrap_or("binary does not implement the privacy trampoline protocol");
215 Err(format!(
216 "{cause}: probe of {} exited {status}",
217 path.display()
218 ))
219}
220
221#[cfg(target_os = "macos")]
222fn privacy_command(
223 spec: &ModuleSpec,
224 roster: &ChildRoster,
225) -> Result<
226 (
227 Command,
228 Option<PrivacyExec>,
229 subc_os::privacy_identity::ExecAcknowledgement,
230 ),
231 SuperviseError,
232> {
233 let failure = |cause: String| {
234 warn!(module_id = %spec.module_id, %cause, "privacy identity trampoline refused module spawn");
235 SuperviseError::Spawn {
236 program: spec.program.clone(),
237 source: io::Error::other(cause),
238 cgroup_path: None,
239 }
240 };
241 let trampoline = roster.privacy_trampoline().map_err(failure)?;
242 let program = if spec.program.components().count() == 1 && !spec.program.is_absolute() {
247 let path = spec
248 .env
249 .iter()
250 .find(|(key, _)| key == "PATH")
251 .map(|(_, value)| std::ffi::OsString::from(value))
252 .or_else(|| std::env::var_os("PATH"))
253 .unwrap_or_else(|| "/usr/bin:/bin".into());
254 std::env::split_paths(&path)
255 .map(|dir| dir.join(&spec.program))
256 .find(|path| path.is_file())
257 .unwrap_or_else(|| spec.program.clone())
258 } else {
259 spec.program.clone()
260 };
261 let expected = subc_os::file_identity(&program);
262 let trampoline_image = subc_os::file_identity(&trampoline);
263 let script = {
264 use std::io::Read;
265 let mut prefix = [0u8; 2];
266 std::fs::File::open(&program)
267 .is_ok_and(|mut file| file.read_exact(&mut prefix).is_ok() && &prefix == b"#!")
268 };
269 if expected.is_none() || expected == subc_os::file_identity(&trampoline) {
270 return Err(failure(
271 "privacy identity module executable is missing or is the trampoline itself".to_string(),
272 ));
273 }
274 let (reader, ack) = subc_os::privacy_identity::ExecAcknowledgement::pipe()
275 .map_err(|error| failure(error.to_string()))?;
276 let reader =
277 tokio::io::unix::AsyncFd::new(reader).map_err(|error| failure(error.to_string()))?;
278 let mut command = Command::new(&trampoline);
279 command
280 .arg("__disclaim-exec")
281 .arg(ack.fd().to_string())
282 .arg(&program);
283 ack.install(command.as_std_mut());
284 Ok((
285 command,
286 Some(PrivacyExec {
287 reader,
288 deadline: tokio::time::Instant::now() + Duration::from_secs(5),
289 expected,
290 trampoline: trampoline_image,
291 script,
292 module_id: spec.module_id.clone(),
293 #[cfg(test)]
294 confirmation_pause: spec
295 .env
296 .iter()
297 .find(|(key, _)| key == "SUBC_TEST_BEFORE_EXEC_CONFIRM")
298 .map(|(_, address)| (address.clone(), None)),
299 }),
300 ack,
301 ))
302}
303
304struct SupervisedChild {
305 child: Child,
306 exec: crate::activity::ExecOwner,
307 #[cfg(windows)]
308 nonce_delivery: Option<subc_os::LaunchNoncePipeDelivery>,
309 #[cfg(target_os = "macos")]
310 privacy_exec: Option<PrivacyExec>,
311 #[cfg(target_os = "macos")]
318 report_ready: Arc<OnceLock<()>>,
319 spawn_failure: Option<String>,
321 protocol: ModuleProtocol,
326 #[cfg(target_os = "linux")]
332 module_id: String,
333 #[cfg(target_os = "linux")]
334 cgroup_placement: Option<subc_cgroup::Placement>,
335 #[cfg(windows)]
357 job: Option<subc_jobobject::JobObject>,
358 stdout_pump: Option<JoinHandle<()>>,
359 stderr_pump: Option<StderrPump>,
360 stderr_ring: Arc<Mutex<StderrRing>>,
361 spawned_at: Instant,
364 spawned_at_ms: u64,
365 spawned_from: PathBuf,
366 launch_spec_digest: LaunchSpecDigest,
367 spawned_file_identity: Option<SpawnedFileIdentity>,
368 process_start_time: Option<u64>,
369 process_identity: Option<ProcessIdentity>,
370 pid: u32,
371 roster_guard: Option<crate::child_roster::RosterGuard>,
374}
375
376impl SupervisedChild {
377 fn id(&self) -> Option<u32> {
378 Some(self.pid)
379 }
380
381 fn process_identity(&self) -> Option<ProcessIdentity> {
382 self.process_identity
383 }
384
385 async fn wait(&mut self) -> io::Result<ExitStatus> {
386 #[cfg(target_os = "macos")]
387 self.confirm_privacy_exec().await;
388 let result = self.child.wait().await;
396 #[cfg(target_os = "linux")]
397 if result.is_ok() {
398 if let Some(placement) = self.cgroup_placement.as_ref() {
399 cleanup_reaped_cgroup(placement, &self.module_id).await;
400 self.cgroup_placement = None;
403 }
404 }
405 result
406 }
407
408 #[cfg(target_os = "macos")]
409 async fn confirm_privacy_exec(&mut self) {
410 let Some(pending) = &mut self.privacy_exec else {
411 return;
412 };
413 let result = tokio::time::timeout_at(pending.deadline, async {
414 let mut record = Vec::new();
415 loop {
416 let mut ready = pending.reader.readable().await?;
417 let read = ready.try_io(|reader| {
418 use std::io::Read;
419 let mut reader = reader.get_ref();
420 let mut buffer = [0u8; 256];
421 reader.read(&mut buffer).map(|count| (count, buffer))
422 });
423 match read {
424 Ok(Ok((0, _))) => return Ok::<_, io::Error>(record),
425 Ok(Ok((count, buffer))) => {
426 if record.len() + count > 1024 {
427 return Err(io::Error::other(
428 "privacy exec refusal record is too long",
429 ));
430 }
431 record.extend_from_slice(&buffer[..count]);
432 }
433 Ok(Err(error)) => return Err(error),
434 Err(_) => continue,
435 }
436 }
437 })
438 .await;
439 #[cfg(test)]
440 let result = if let Some((address, stream)) = &mut pending.confirmation_pause {
441 use tokio::io::{AsyncReadExt, AsyncWriteExt};
442 if stream.is_none() {
443 *stream = Some(
444 tokio::net::TcpStream::connect(address.as_str())
445 .await
446 .unwrap(),
447 );
448 stream.as_mut().unwrap().write_all(b"R").await.unwrap();
449 stream
450 .as_mut()
451 .unwrap()
452 .write_all(&self.pid.to_ne_bytes())
453 .await
454 .unwrap();
455 }
456 let mut reply = [0];
457 stream
458 .as_mut()
459 .unwrap()
460 .read_exact(&mut reply)
461 .await
462 .unwrap();
463 pending.confirmation_pause = None;
464 if &reply == b"F" {
465 Ok(Err(io::Error::other("injected late exec refusal")))
466 } else {
467 assert_eq!(&reply, b"X");
468 result
469 }
470 } else {
471 result
472 };
473 let pending = self.privacy_exec.as_ref().expect("pending exec");
476 let cause = match result {
477 Err(_) => Some("privacy identity trampoline did not exec within 5s".to_string()),
478 Ok(Err(error)) => Some(format!(
479 "privacy identity exec acknowledgement failed: {error}"
480 )),
481 Ok(Ok(record)) if !record.is_empty() => Some(
482 std::str::from_utf8(&record)
483 .ok()
484 .and_then(|record| {
485 record
486 .trim()
487 .strip_prefix(subc_os::privacy_identity::EXEC_REFUSAL_TAG)
488 })
489 .filter(|cause| !cause.is_empty())
490 .unwrap_or("invalid privacy exec refusal record")
491 .to_string(),
492 ),
493 Ok(Ok(_)) => match self.child.try_wait() {
494 Ok(Some(_status)) => None,
498 Err(error) => Some(format!("privacy identity trampoline wait failed: {error}")),
499 Ok(None) => {
500 let image = observe_spawned_image(self.pid);
501 if let Some(image) = image.filter(|image| {
502 image.executable.is_some()
503 && image.executable != pending.trampoline
504 && (image.executable == pending.expected || pending.script)
505 }) {
506 if let Err(error) = self
507 .roster_guard
508 .as_ref()
509 .map(|guard| guard.confirm_executable(image))
510 .transpose()
511 {
512 Some(format!(
513 "could not durably record confirmed module image: {error}"
514 ))
515 } else {
516 let _ = self.report_ready.set(());
517 info!(module_id = %pending.module_id, pid = self.pid,
518 "module spawned with own privacy identity (responsibility disclaimed)");
519 None
520 }
521 } else if image.is_none()
522 || image.is_some_and(|image| image.executable.is_none())
523 {
524 match tokio::time::timeout_at(pending.deadline, self.child.wait()).await {
531 Ok(Ok(_status)) => None,
532 Ok(Err(error)) => Some(format!("privacy identity trampoline wait failed: {error}")),
533 Err(_) => Some("privacy identity module executable remained unreadable until the 5s exec deadline".to_string()),
534 }
535 } else {
536 Some("privacy identity trampoline executable mismatch: expected the module program, not ck-subc".to_string())
537 }
538 }
539 },
540 };
541 let pending = self.privacy_exec.take().expect("pending exec");
542 self.exec.0.finish(if cause.is_some() {
545 crate::activity::ExecState::Refused
546 } else {
547 crate::activity::ExecState::CommittedExec
548 });
549 if let Some(cause) = cause {
550 warn!(module_id = %pending.module_id, pid = self.pid, %cause, "privacy identity trampoline refused module spawn");
551 self.spawn_failure = Some(cause);
552 if let Some(pid) = rustix::process::Pid::from_raw(self.pid as i32) {
555 let _ = rustix::process::kill_process_group(pid, rustix::process::Signal::KILL);
556 }
557 let _ = self.child.start_kill();
558 }
559 }
560
561 fn release_roster(&mut self) {
565 self.roster_guard = None;
566 }
567
568 fn start_kill(&mut self) -> io::Result<()> {
581 #[cfg(windows)]
582 if let Some(job) = &self.job {
583 if let Err(error) = job.terminate() {
584 debug!(
585 error = %error,
586 "job termination failed; the direct-child kill still owns the outcome"
587 );
588 }
589 }
590 #[cfg(target_os = "linux")]
591 kill_module_cgroup(self.cgroup_placement.as_ref(), &self.module_id);
592 self.child.start_kill()
593 }
594
595 async fn drain_stderr(&mut self, module_id: &str) {
596 if let Some(pump) = self.stdout_pump.take() {
597 settle_stdout_pump(module_id, pump, STDERR_PUMP_DRAIN_TIMEOUT).await;
598 }
599
600 let Some(pump) = self.stderr_pump.take() else {
601 return;
602 };
603 settle_stderr_pump(
604 module_id,
605 &self.stderr_ring,
606 pump,
607 STDERR_PUMP_DRAIN_TIMEOUT,
608 )
609 .await;
610 }
611}
612
613async fn settle_stdout_pump(module_id: &str, mut pump: JoinHandle<()>, bound: Duration) {
617 match timeout(bound, &mut pump).await {
618 Ok(Ok(())) => {}
619 Ok(Err(error)) => {
620 warn!(module_id, error = %error, "stdout pump ended unexpectedly");
621 }
622 Err(_) => {
623 drop(pump);
624 warn!(
625 module_id,
626 waited = ?bound,
627 "stdout pipe of the exited process has not drained; its reader keeps running without delaying the restart"
628 );
629 }
630 }
631}
632
633struct StderrPump {
636 task: JoinHandle<()>,
637 generation: u64,
638}
639
640async fn settle_stderr_pump(
646 module_id: &str,
647 ring: &Arc<Mutex<StderrRing>>,
648 pump: StderrPump,
649 bound: Duration,
650) {
651 let lock = || ring.lock().unwrap_or_else(|poisoned| poisoned.into_inner());
652 let StderrPump {
653 mut task,
654 generation,
655 } = pump;
656 lock().retire_pump(generation);
657 match timeout(bound, &mut task).await {
658 Ok(Ok(())) => {}
659 Ok(Err(err)) => {
660 let mut ring = lock();
661 ring.mark_incomplete_from(generation, format!("stderr pump ended unexpectedly: {err}"));
662 ring.finish_pump(generation);
663 warn!(module_id, error = %err, "stderr pump ended before clean EOF");
664 }
665 Err(_) => {
666 drop(task);
668 lock().mark_pump_late(
669 generation,
670 format!(
671 "stderr of the exited process had not reached EOF {bound:?} after it was \
672 retired (a descendant may still hold the pipe open); lines it still \
673 writes are kept in that process's section"
674 ),
675 );
676 warn!(
677 module_id,
678 waited = ?bound,
679 "stderr pipe of the exited process is still open; its reader keeps running without delaying the restart"
680 );
681 }
682 }
683}
684
685fn registration_release_events() -> &'static watch::Sender<u64> {
686 static EVENTS: OnceLock<watch::Sender<u64>> = OnceLock::new();
687 EVENTS.get_or_init(|| {
688 let (sender, _receiver) = watch::channel(0);
689 sender
690 })
691}
692
693pub(crate) fn notify_registration_release() {
694 let events = registration_release_events();
695 let next_generation = (*events.borrow()).wrapping_add(1);
696 events.send_replace(next_generation);
697}
698
699#[derive(Debug, Clone, PartialEq, Eq)]
701pub struct ModuleSpec {
702 pub module_id: String,
703 pub program: PathBuf,
704 pub args: Vec<String>,
705 pub env: Vec<(String, String)>,
706 pub reserved: bool,
711 pub reserved_prefixes: Vec<String>,
716 pub protocol: ModuleProtocol,
735 pub overlap: ModuleOverlap,
740}
741
742#[derive(Clone, PartialEq, Eq)]
745pub(crate) struct LaunchSpecDigest([u8; 32]);
746
747impl fmt::Debug for LaunchSpecDigest {
748 fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
749 f.write_str("LaunchSpecDigest([redacted])")
750 }
751}
752
753impl LaunchSpecDigest {
754 pub(crate) fn from_spec(spec: &ModuleSpec) -> Self {
755 fn field(hash: &mut Sha256, bytes: &[u8]) {
756 hash.update((bytes.len() as u64).to_le_bytes());
757 hash.update(bytes);
758 }
759 let mut hash = Sha256::new();
760 if let Some(program) = spec.program.to_str() {
761 field(&mut hash, b"utf8-program");
762 field(&mut hash, program.as_bytes());
763 } else {
764 #[cfg(unix)]
767 {
768 use std::os::unix::ffi::OsStrExt;
769 field(&mut hash, b"unix-program");
770 field(&mut hash, spec.program.as_os_str().as_bytes());
771 }
772 #[cfg(windows)]
773 {
774 use std::os::windows::ffi::OsStrExt;
775 field(&mut hash, b"windows-program");
776 let bytes = spec
777 .program
778 .as_os_str()
779 .encode_wide()
780 .flat_map(u16::to_le_bytes)
781 .collect::<Vec<_>>();
782 field(&mut hash, &bytes);
783 }
784 #[cfg(not(any(unix, windows)))]
785 field(&mut hash, spec.program.to_string_lossy().as_bytes());
786 }
787 hash.update((spec.args.len() as u64).to_le_bytes());
788 for arg in &spec.args {
789 field(&mut hash, arg.as_bytes());
790 }
791 let mut env = spec.env.iter().collect::<Vec<_>>();
792 env.sort_unstable();
793 hash.update((env.len() as u64).to_le_bytes());
794 for (key, value) in env {
795 field(&mut hash, key.as_bytes());
796 field(&mut hash, value.as_bytes());
797 }
798 field(
799 &mut hash,
800 match spec.protocol {
801 ModuleProtocol::Subc => b"subc",
802 ModuleProtocol::None => b"none",
803 },
804 );
805 Self(hash.finalize().into())
806 }
807}
808
809#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)]
816pub enum ModuleOverlap {
817 #[default]
819 Exclusive,
820 Safe,
832}
833
834impl ModuleOverlap {
835 pub fn as_str(self) -> &'static str {
836 match self {
837 Self::Exclusive => "exclusive",
838 Self::Safe => "safe",
839 }
840 }
841}
842
843pub const SUBC_SPAWN_ROLE_ENV: &str = "SUBC_SPAWN_ROLE";
853pub const SPAWN_ROLE_SWAP_CANDIDATE: &str = "swap_candidate";
855pub const DEFAULT_SWAP_READY_TIMEOUT: Duration = Duration::from_secs(100);
860
861#[derive(Debug, Clone, Copy, PartialEq, Eq)]
879pub struct RestartPolicy {
880 pub max_restarts: u32,
881 pub backoff: Duration,
884 pub max_backoff: Duration,
886 pub window: Duration,
890}
891
892impl RestartPolicy {
893 pub fn new(max_restarts: u32, backoff: Duration) -> Self {
897 Self {
898 max_restarts,
899 backoff,
900 max_backoff: DEFAULT_MAX_BACKOFF,
901 window: DEFAULT_RESTART_WINDOW,
902 }
903 }
904
905 pub fn with_max_backoff(mut self, max_backoff: Duration) -> Self {
906 self.max_backoff = max_backoff;
907 self
908 }
909
910 pub fn with_window(mut self, window: Duration) -> Self {
911 self.window = window;
912 self
913 }
914
915 fn delay_for_restart(&self, restart_in_window: u32) -> Duration {
920 if self.backoff.is_zero() || self.max_backoff.is_zero() {
921 return Duration::ZERO;
922 }
923
924 let mut delay = self.backoff;
925 for _ in 0..restart_in_window {
926 if delay >= self.max_backoff {
927 return self.max_backoff;
928 }
929 delay = delay
930 .checked_mul(10)
931 .unwrap_or(self.max_backoff)
932 .min(self.max_backoff);
933 }
934 delay.min(self.max_backoff)
935 }
936
937 fn budget_exhausted_detail(&self) -> String {
942 format!(
943 "crash budget exhausted: max_restarts={} within window_secs={}",
944 self.max_restarts,
945 self.window.as_secs()
946 )
947 }
948}
949
950impl Default for RestartPolicy {
951 fn default() -> Self {
952 Self {
953 max_restarts: DEFAULT_MAX_RESTARTS,
954 backoff: DEFAULT_BACKOFF,
955 max_backoff: DEFAULT_MAX_BACKOFF,
956 window: DEFAULT_RESTART_WINDOW,
957 }
958 }
959}
960
961#[derive(Debug, Clone, Copy, PartialEq, Eq)]
962struct CrashRestartSchedule {
963 restart_in_window: u32,
964 delay: Duration,
965}
966
967fn daemon_will_restart(
974 state: &mut SupervisorSnapshot,
975 policy: &RestartPolicy,
976 now: Instant,
977) -> bool {
978 state.enabled && state.crash_restarts_in_window(policy.window, now) < policy.max_restarts
979}
980
981const DEFAULT_HEALTH_CADENCE: Duration = Duration::from_secs(30);
982const DEFAULT_HEALTH_DEADLINE: Duration = Duration::from_secs(5);
983const DEFAULT_HEALTH_FAILURE_THRESHOLD: u32 = 3;
984const MAX_HEALTH_METRICS_BYTES: usize = 16 * 1024;
985
986#[derive(Debug, Clone, Copy, PartialEq, Eq)]
987pub enum HealthAction {
988 Report,
989 Restart,
990 Alert,
991}
992
993impl fmt::Display for HealthAction {
994 fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
995 f.write_str(match self {
996 Self::Report => "report",
997 Self::Restart => "restart",
998 Self::Alert => "alert",
999 })
1000 }
1001}
1002
1003#[derive(Debug, Clone, PartialEq, Eq)]
1004pub struct HealthConfig {
1005 pub http: Option<String>,
1009 pub cadence: Duration,
1010 pub deadline: Duration,
1011 pub failure_threshold: u32,
1012 pub on_degraded: HealthAction,
1013 pub on_failing: HealthAction,
1014 pub critical: bool,
1015}
1016
1017impl Default for HealthConfig {
1018 fn default() -> Self {
1019 Self {
1020 http: None,
1021 cadence: DEFAULT_HEALTH_CADENCE,
1022 deadline: DEFAULT_HEALTH_DEADLINE,
1023 failure_threshold: DEFAULT_HEALTH_FAILURE_THRESHOLD,
1024 on_degraded: HealthAction::Report,
1025 on_failing: HealthAction::Report,
1026 critical: false,
1027 }
1028 }
1029}
1030
1031#[derive(Debug, Clone, PartialEq)]
1049pub struct ModuleHealthStatus {
1050 pub status: SupervisorHealthStatus,
1051 pub last_probe_ms: Option<u64>,
1052 pub detail: Option<String>,
1053 pub metrics: Option<Value>,
1054 pub consecutive_failures: u32,
1055 pub late_answer_count: u64,
1058 pub last_late_answer_latency_ms: Option<u64>,
1060 pub last_action: Option<String>,
1061 pub last_action_ms: Option<u64>,
1065}
1066
1067impl Default for ModuleHealthStatus {
1068 fn default() -> Self {
1069 Self {
1070 status: SupervisorHealthStatus::Unknown,
1071 last_probe_ms: None,
1072 detail: None,
1073 metrics: None,
1074 consecutive_failures: 0,
1075 late_answer_count: 0,
1076 last_late_answer_latency_ms: None,
1077 last_action: None,
1078 last_action_ms: None,
1079 }
1080 }
1081}
1082
1083#[derive(Debug, Clone, Copy, PartialEq, Eq)]
1085pub enum ModuleState {
1086 Starting,
1087 Running,
1088 Unresponsive,
1089 Restarting,
1090 Draining,
1091 Stopped,
1092 Failed,
1093 Disabled,
1094}
1095
1096impl fmt::Display for ModuleState {
1097 fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
1098 f.write_str(match self {
1099 Self::Starting => "starting",
1100 Self::Running => "running",
1101 Self::Unresponsive => "unresponsive",
1102 Self::Restarting => "restarting",
1103 Self::Draining => "draining",
1104 Self::Stopped => "stopped",
1105 Self::Failed => "failed",
1106 Self::Disabled => "disabled",
1107 })
1108 }
1109}
1110
1111#[derive(Debug, Clone, Copy, PartialEq, Eq)]
1113pub enum ExitKind {
1114 Clean,
1115 Crash,
1116 DeliberateSeverance,
1117}
1118
1119impl From<ExitKind> for TerminalExitKind {
1120 fn from(kind: ExitKind) -> Self {
1121 match kind {
1122 ExitKind::Clean => Self::Clean,
1123 ExitKind::Crash => Self::Crash,
1124 ExitKind::DeliberateSeverance => Self::DeliberateSeverance,
1125 }
1126 }
1127}
1128
1129#[derive(Debug, Clone, Copy, PartialEq, Eq)]
1132pub(crate) struct ProcessIdentity {
1133 pub(crate) pid: u32,
1134 pub(crate) start_time: u64,
1135}
1136
1137#[derive(Debug, Clone, PartialEq, Eq)]
1139pub struct ExitReport {
1140 pub kind: ExitKind,
1141 pub code: Option<i32>,
1142 pub signal: Option<i32>,
1143 pub at_ms: u64,
1144}
1145
1146#[derive(Debug, Clone, PartialEq)]
1149pub struct ModuleStatus {
1150 pub module_id: String,
1151 pub waiting_for_previous_pid: Option<u32>,
1153 pub waiting_for_previous_detail: Option<String>,
1155 pub launch_nonce_source: Option<String>,
1160 pub state: ModuleState,
1161 pub enabled: bool,
1162 pub process_alive: bool,
1163 pub registration_active: bool,
1164 pub protocol: ModuleProtocol,
1169 pub live: bool,
1180 pub restart_count: u32,
1184 pub lifetime_restarts: u32,
1188 pub spawn_generation: u64,
1189 pub max_restarts: u32,
1194 pub restart_window: Duration,
1198 pub drain_timeout: Duration,
1202 pub restart_backoff: Duration,
1203 pub restart_max_backoff: Duration,
1204 pub pid: Option<u32>,
1209 pub spawned_at_ms: Option<u64>,
1210 pub spawned_from: Option<PathBuf>,
1211 pub(crate) launch_spec_digest: Option<LaunchSpecDigest>,
1212 pub process_start_time: Option<u64>,
1213 pub last_exit: Option<ExitReport>,
1214 pub health: ModuleHealthStatus,
1215}
1216
1217#[cfg(test)]
1218#[derive(Debug, Clone, Copy, PartialEq)]
1219struct ActorSelectCheckpoint {
1220 generation: u64,
1221 turn: u64,
1222 registered_connection: Option<ConnectionId>,
1223 next_probe_at: Option<Instant>,
1224 wake_after: Option<Duration>,
1225}
1226
1227#[derive(Debug, Clone, PartialEq)]
1228struct SupervisorSnapshot {
1229 exec_outcome: Option<Arc<crate::activity::ExecOutcome>>,
1230 #[cfg(windows)]
1231 pipe_consumed: Option<Arc<OnceLock<()>>>,
1232 #[cfg(test)]
1234 actor_turns: u64,
1235 #[cfg(test)]
1238 actor_select: Option<ActorSelectCheckpoint>,
1239 state: ModuleState,
1240 enabled: bool,
1241 process_alive: bool,
1242 spawned_protocol: Option<ModuleProtocol>,
1243 spawn_failure: Option<String>,
1244 waiting_for_previous_pid: Option<u32>,
1245 waiting_for_previous_detail: Option<String>,
1246 crash_restarts: VecDeque<Instant>,
1252 lifetime_restarts: u32,
1253 spawn_generation: u64,
1262 pid: Option<u32>,
1263 #[cfg(target_os = "macos")]
1264 report_ready: Option<Arc<OnceLock<()>>>,
1265 reaped_pid: Option<u32>,
1267 respawn_pending: bool,
1269 coalesced_restart_pending: bool,
1271 spawned_at_ms: Option<u64>,
1272 spawned_from: Option<PathBuf>,
1273 launch_spec_digest: Option<LaunchSpecDigest>,
1274 spawned_file_identity: Option<SpawnedFileIdentity>,
1275 process_start_time: Option<u64>,
1276 deliberate_severance: Option<ProcessIdentity>,
1277 last_exit: Option<ExitReport>,
1278 drain_disposition_detail: Option<String>,
1280 health: ModuleHealthStatus,
1281 in_alternate_slot: bool,
1286 draining_to_replace: bool,
1293 configuration_updated_since_spawn: bool,
1299}
1300
1301impl SupervisorSnapshot {
1302 fn reported_pid(&self) -> Option<u32> {
1307 #[cfg(target_os = "macos")]
1308 if self
1309 .report_ready
1310 .as_ref()
1311 .is_some_and(|ready| ready.get().is_none())
1312 {
1313 return None;
1314 }
1315 self.pid
1316 }
1317
1318 fn starting() -> Self {
1319 Self::new(ModuleState::Starting, true)
1320 }
1321
1322 fn disabled() -> Self {
1323 Self::new(ModuleState::Disabled, false)
1324 }
1325
1326 fn crash_restarts_in_window(&mut self, window: Duration, now: Instant) -> u32 {
1330 while let Some(oldest) = self.crash_restarts.front() {
1331 if now.duration_since(*oldest) > window {
1332 self.crash_restarts.pop_front();
1333 } else {
1334 break;
1335 }
1336 }
1337 u32::try_from(self.crash_restarts.len()).unwrap_or(u32::MAX)
1338 }
1339
1340 fn record_crash_restart(&mut self, policy: &RestartPolicy, now: Instant) {
1346 self.crash_restarts.push_back(now);
1347 while self.crash_restarts.len() > policy.max_restarts as usize {
1348 self.crash_restarts.pop_front();
1349 }
1350 self.lifetime_restarts += 1;
1351 }
1352
1353 fn next_crash_restart(
1357 &mut self,
1358 policy: &RestartPolicy,
1359 now: Instant,
1360 ) -> Option<CrashRestartSchedule> {
1361 let restart_in_window = self.crash_restarts_in_window(policy.window, now);
1362 if restart_in_window >= policy.max_restarts {
1363 return None;
1364 }
1365 self.record_crash_restart(policy, now);
1366 Some(CrashRestartSchedule {
1367 restart_in_window,
1368 delay: policy.delay_for_restart(restart_in_window),
1369 })
1370 }
1371
1372 fn clear_crash_restarts(&mut self) {
1377 self.crash_restarts.clear();
1378 }
1379
1380 fn new(state: ModuleState, enabled: bool) -> Self {
1381 Self {
1382 exec_outcome: None,
1383 #[cfg(test)]
1384 actor_turns: 0,
1385 #[cfg(test)]
1386 actor_select: None,
1387 state,
1388 enabled,
1389 process_alive: false,
1390 #[cfg(windows)]
1391 pipe_consumed: None,
1392 spawned_protocol: None,
1393 spawn_failure: None,
1394 waiting_for_previous_pid: None,
1395 waiting_for_previous_detail: None,
1396 crash_restarts: VecDeque::new(),
1397 lifetime_restarts: 0,
1398 spawn_generation: 0,
1399 pid: None,
1400 #[cfg(target_os = "macos")]
1401 report_ready: None,
1402 reaped_pid: None,
1403 respawn_pending: false,
1404 coalesced_restart_pending: false,
1405 spawned_at_ms: None,
1406 spawned_from: None,
1407 launch_spec_digest: None,
1408 spawned_file_identity: None,
1409 process_start_time: None,
1410 deliberate_severance: None,
1411 last_exit: None,
1412 drain_disposition_detail: None,
1413 health: ModuleHealthStatus::default(),
1414 in_alternate_slot: false,
1415 draining_to_replace: false,
1416 configuration_updated_since_spawn: false,
1417 }
1418 }
1419}
1420
1421type SharedSnapshot = Arc<Mutex<SupervisorSnapshot>>;
1422
1423type SpawnSubscriberKey = (ConnectionId, u64);
1424
1425#[derive(Debug)]
1426struct SpawnSubscriber {
1427 version: u8,
1428 frames: mpsc::Sender<Frame>,
1429 lagged: Option<oneshot::Sender<SpawnCursor>>,
1433}
1434
1435#[derive(Debug)]
1436struct SpawnEventState {
1437 daemon_incarnation: String,
1438 seq: u64,
1439 capacity: usize,
1440 live: HashMap<String, LiveSpawn>,
1441 generations: HashMap<String, u64>,
1442 events: VecDeque<SpawnEvent>,
1443 subscribers: HashMap<SpawnSubscriberKey, SpawnSubscriber>,
1444}
1445
1446impl Default for SpawnEventState {
1447 fn default() -> Self {
1448 Self {
1449 daemon_incarnation: "unconfigured".to_string(),
1450 seq: 0,
1451 capacity: SPAWN_EVENT_RING_CAPACITY,
1452 live: HashMap::new(),
1453 generations: HashMap::new(),
1454 events: VecDeque::new(),
1455 subscribers: HashMap::new(),
1456 }
1457 }
1458}
1459
1460#[derive(Debug, Clone, Default)]
1461struct SpawnEventFeed(
1462 Arc<Mutex<SpawnEventState>>,
1463 #[cfg(test)] Arc<Mutex<Option<std::net::TcpStream>>>,
1464);
1465
1466#[derive(Debug, Clone, PartialEq, Eq)]
1467pub(crate) enum SpawnSubscribeRefusal {
1468 ForeignIncarnation { current: String },
1469 TooOld { oldest: SpawnCursor },
1470 Frame(String),
1471}
1472
1473impl SpawnEventFeed {
1474 fn configure_incarnation(&self, daemon_incarnation: String) {
1475 let mut state = self.0.lock().unwrap_or_else(|p| p.into_inner());
1476 state.daemon_incarnation = daemon_incarnation;
1477 state.seq = 0;
1478 state.live.clear();
1479 state.generations.clear();
1480 state.events.clear();
1481 state.subscribers.clear();
1482 }
1483
1484 fn cursor(state: &SpawnEventState) -> SpawnCursor {
1485 SpawnCursor {
1486 daemon_incarnation: state.daemon_incarnation.clone(),
1487 seq: state.seq,
1488 }
1489 }
1490
1491 fn snapshot(&self) -> SpawnSnapshot {
1492 let state = self.0.lock().unwrap_or_else(|p| p.into_inner());
1493 let mut live = state.live.values().cloned().collect::<Vec<_>>();
1494 live.sort_by(|left, right| left.module_id.cmp(&right.module_id));
1495 SpawnSnapshot {
1496 cursor: Self::cursor(&state),
1497 ring_bound: state.capacity as u64,
1498 live,
1499 }
1500 }
1501
1502 fn emit_spawned(&self, module_id: &str, pid: u32, spawned_at_ms: u64) -> u64 {
1503 let mut state = self.0.lock().unwrap_or_else(|p| p.into_inner());
1504 let generation = state
1505 .generations
1506 .get(module_id)
1507 .copied()
1508 .unwrap_or(0)
1509 .checked_add(1)
1510 .expect("spawn generation exhausted");
1511 state.generations.insert(module_id.to_string(), generation);
1512 let live = LiveSpawn {
1513 module_id: module_id.to_string(),
1514 spawn_generation: generation,
1515 pid,
1516 spawned_at_ms,
1517 };
1518 state.live.insert(module_id.to_string(), live);
1519 Self::emit_locked(
1520 &mut state,
1521 SpawnEventKind::Spawned,
1522 module_id.to_string(),
1523 generation,
1524 pid,
1525 None,
1526 None,
1527 );
1528 generation
1529 }
1530
1531 fn emit_exited(&self, module_id: &str, exit_code: Option<i32>, exit_signal: Option<i32>) {
1532 let mut state = self.0.lock().unwrap_or_else(|p| p.into_inner());
1533 let Some(live) = state.live.remove(module_id) else {
1534 warn!(
1535 module_id,
1536 "terminal record had no live spawn event identity"
1537 );
1538 return;
1539 };
1540 Self::emit_locked(
1541 &mut state,
1542 SpawnEventKind::Exited,
1543 module_id.to_string(),
1544 live.spawn_generation,
1545 live.pid,
1546 exit_code,
1547 exit_signal,
1548 );
1549 }
1550
1551 fn emit_superseded_exited(
1558 &self,
1559 module_id: &str,
1560 spawn_generation: u64,
1561 pid: u32,
1562 exit_code: Option<i32>,
1563 exit_signal: Option<i32>,
1564 ) {
1565 let mut state = self.0.lock().unwrap_or_else(|p| p.into_inner());
1566 if state
1567 .live
1568 .get(module_id)
1569 .is_some_and(|live| live.spawn_generation == spawn_generation)
1570 {
1571 state.live.remove(module_id);
1572 }
1573 Self::emit_locked(
1574 &mut state,
1575 SpawnEventKind::Exited,
1576 module_id.to_string(),
1577 spawn_generation,
1578 pid,
1579 exit_code,
1580 exit_signal,
1581 );
1582 }
1583
1584 #[allow(clippy::too_many_arguments)]
1585 fn emit_locked(
1586 state: &mut SpawnEventState,
1587 kind: SpawnEventKind,
1588 module_id: String,
1589 spawn_generation: u64,
1590 pid: u32,
1591 exit_code: Option<i32>,
1592 exit_signal: Option<i32>,
1593 ) {
1594 state.seq = state
1595 .seq
1596 .checked_add(1)
1597 .expect("spawn event sequence exhausted");
1598 let event = SpawnEvent {
1599 cursor: Self::cursor(state),
1600 kind,
1601 module_id,
1602 spawn_generation,
1603 pid,
1604 exit_code,
1605 exit_signal,
1606 };
1607 state.events.push_back(event.clone());
1608 while state.events.len() > state.capacity {
1609 state.events.pop_front();
1610 }
1611 let body = match serde_json::to_vec(&event) {
1612 Ok(body) => body,
1613 Err(error) => {
1614 error!(%error, "failed to serialize supervisor spawn event");
1615 return;
1616 }
1617 };
1618 state.subscribers.retain(|(connection_id, corr), subscriber| {
1619 let frame = Frame::build_with_version(
1620 subscriber.version,
1621 FrameType::StreamData,
1622 control_flags(),
1623 0,
1624 0,
1625 *corr,
1626 body.clone(),
1627 );
1628 match frame {
1629 Ok(frame) => {
1630 if subscriber.frames.try_send(frame).is_ok() {
1631 true
1632 } else {
1633 warn!(connection_id = connection_id.get(), corr, "dropping lagged supervisor spawn subscriber");
1634 if let Some(lagged) = subscriber.lagged.take() {
1635 let _ = lagged.send(event.cursor.clone());
1636 }
1637 false
1638 }
1639 }
1640 Err(error) => {
1641 warn!(connection_id = connection_id.get(), corr, %error, "dropping supervisor spawn subscriber after frame build failure");
1642 false
1643 }
1644 }
1645 });
1646 }
1647
1648 fn subscribe(
1649 &self,
1650 connection_id: ConnectionId,
1651 corr: u64,
1652 version: u8,
1653 since: Option<SpawnCursor>,
1654 sink: FrameSink,
1655 ) -> Result<(), SpawnSubscribeRefusal> {
1656 let (frames, mut receiver) = mpsc::channel(SPAWN_SUBSCRIBER_BUFFER);
1657 let (lagged, mut lagged_rx) = oneshot::channel::<SpawnCursor>();
1658 {
1659 let mut state = self.0.lock().unwrap_or_else(|p| p.into_inner());
1660 let replay = if let Some(since) = since {
1661 if since.daemon_incarnation != state.daemon_incarnation {
1662 return Err(SpawnSubscribeRefusal::ForeignIncarnation {
1663 current: state.daemon_incarnation.clone(),
1664 });
1665 }
1666 if let Some(oldest) = state.events.front().map(|event| event.cursor.clone()) {
1667 if since.seq < oldest.seq.saturating_sub(1) {
1668 return Err(SpawnSubscribeRefusal::TooOld { oldest });
1669 }
1670 }
1671 state
1672 .events
1673 .iter()
1674 .filter(|event| event.cursor.seq > since.seq)
1675 .cloned()
1676 .collect::<Vec<_>>()
1677 } else {
1678 Vec::new()
1679 };
1680 for event in replay {
1681 let body = serde_json::to_vec(&event)
1682 .map_err(|error| SpawnSubscribeRefusal::Frame(error.to_string()))?;
1683 let frame = Frame::build_with_version(
1684 version,
1685 FrameType::StreamData,
1686 control_flags(),
1687 0,
1688 0,
1689 corr,
1690 body,
1691 )
1692 .map_err(|error| SpawnSubscribeRefusal::Frame(error.to_string()))?;
1693 frames
1694 .try_send(frame)
1695 .map_err(|error| SpawnSubscribeRefusal::Frame(error.to_string()))?;
1696 }
1697 state.subscribers.insert(
1698 (connection_id, corr),
1699 SpawnSubscriber {
1700 version,
1701 frames,
1702 lagged: Some(lagged),
1703 },
1704 );
1705 }
1706 tokio::spawn(async move {
1717 while let Some(frame) = receiver.recv().await {
1718 if sink.send(frame).await.is_err() {
1719 return;
1720 }
1721 }
1722 let Ok(first_undelivered) = lagged_rx.try_recv() else {
1723 return;
1724 };
1725 match spawn_subscriber_lagged_frame(version, corr, first_undelivered) {
1726 Ok(frame) => {
1727 let _ = sink.send(frame).await;
1728 }
1729 Err(error) => {
1730 error!(%error, corr, "failed to build lagged spawn subscriber terminal frame");
1731 }
1732 }
1733 });
1734 Ok(())
1735 }
1736
1737 fn cancel(&self, connection_id: ConnectionId, corr: u64) -> bool {
1738 let Some(subscriber) = self
1739 .0
1740 .lock()
1741 .unwrap_or_else(|p| p.into_inner())
1742 .subscribers
1743 .remove(&(connection_id, corr))
1744 else {
1745 return false;
1746 };
1747 if let Ok(frame) = Frame::build_with_version(
1748 subscriber.version,
1749 FrameType::StreamEnd,
1750 control_flags(),
1751 0,
1752 0,
1753 corr,
1754 Vec::new(),
1755 ) {
1756 tokio::spawn(async move {
1757 let _ = subscriber.frames.send(frame).await;
1758 });
1759 }
1760 true
1761 }
1762
1763 fn remove_connection(&self, connection_id: ConnectionId) {
1764 self.0
1765 .lock()
1766 .unwrap_or_else(|p| p.into_inner())
1767 .subscribers
1768 .retain(|(subscriber_connection, _), _| *subscriber_connection != connection_id);
1769 }
1770
1771 #[cfg(any(test, feature = "test-support"))]
1772 fn set_capacity(&self, capacity: usize) {
1773 self.0.lock().unwrap_or_else(|p| p.into_inner()).capacity = capacity;
1774 }
1775
1776 #[cfg(any(test, feature = "test-support"))]
1777 fn subscriber_count(&self) -> usize {
1778 self.0
1779 .lock()
1780 .unwrap_or_else(|p| p.into_inner())
1781 .subscribers
1782 .len()
1783 }
1784}
1785
1786fn spawn_subscriber_lagged_frame(
1789 version: u8,
1790 corr: u64,
1791 first_undelivered: SpawnCursor,
1792) -> Result<Frame, String> {
1793 let body = serde_json::to_vec(&subc_protocol::ErrorBody {
1794 code: SPAWN_SUBSCRIBER_LAGGED_CODE.to_string(),
1795 message: "spawn subscriber fell behind and was dropped; resubscribe from the last cursor received"
1796 .to_string(),
1797 detail: Some(serde_json::json!({
1798 "first_undelivered_cursor": first_undelivered
1799 })),
1800 })
1801 .map_err(|error| error.to_string())?;
1802 Frame::build_with_version(version, FrameType::Error, control_flags(), 0, 0, corr, body)
1803 .map_err(|error| error.to_string())
1804}
1805
1806pub trait ModuleProcessLiveness: Send + Sync {
1807 fn process_live(&self, module_id: &str) -> Option<bool>;
1808
1809 fn process_replacing(&self, _module_id: &str) -> bool {
1815 false
1816 }
1817}
1818
1819#[derive(Debug, Clone, Default)]
1821pub struct SupervisorProcessLiveness {
1822 snapshots: Arc<Mutex<HashMap<String, SharedSnapshot>>>,
1823}
1824
1825impl SupervisorProcessLiveness {
1826 pub fn new() -> Self {
1827 Self::default()
1828 }
1829
1830 fn track(&self, module_id: String, snapshot: SharedSnapshot) {
1831 let mut snapshots = self
1832 .snapshots
1833 .lock()
1834 .unwrap_or_else(|poisoned| poisoned.into_inner());
1835 snapshots.insert(module_id, snapshot);
1836 }
1837
1838 fn untrack_if_current(&self, module_id: &str, snapshot: &SharedSnapshot) {
1839 let mut snapshots = self
1840 .snapshots
1841 .lock()
1842 .unwrap_or_else(|poisoned| poisoned.into_inner());
1843 let is_current = snapshots
1844 .get(module_id)
1845 .map(|tracked| Arc::ptr_eq(tracked, snapshot))
1846 .unwrap_or(false);
1847 if is_current {
1848 snapshots.remove(module_id);
1849 }
1850 }
1851}
1852
1853impl ModuleProcessLiveness for SupervisorProcessLiveness {
1854 fn process_live(&self, module_id: &str) -> Option<bool> {
1855 let snapshot = {
1856 let snapshots = self
1857 .snapshots
1858 .lock()
1859 .unwrap_or_else(|poisoned| poisoned.into_inner());
1860 snapshots.get(module_id).cloned()
1861 }?;
1862 let snapshot = snapshot
1863 .lock()
1864 .unwrap_or_else(|poisoned| poisoned.into_inner());
1865 Some(snapshot.state == ModuleState::Running && snapshot.process_alive)
1866 }
1867
1868 fn process_replacing(&self, module_id: &str) -> bool {
1869 let Some(snapshot) = self
1870 .snapshots
1871 .lock()
1872 .unwrap_or_else(|poisoned| poisoned.into_inner())
1873 .get(module_id)
1874 .cloned()
1875 else {
1876 return false;
1877 };
1878 let snapshot = snapshot
1879 .lock()
1880 .unwrap_or_else(|poisoned| poisoned.into_inner());
1881 snapshot.enabled
1882 && match snapshot.state {
1883 ModuleState::Restarting => true,
1884 ModuleState::Draining => snapshot.draining_to_replace,
1885 ModuleState::Starting
1886 | ModuleState::Running
1887 | ModuleState::Unresponsive
1888 | ModuleState::Stopped
1889 | ModuleState::Failed
1890 | ModuleState::Disabled => false,
1891 }
1892 }
1893}
1894
1895#[cfg(test)]
1896#[derive(Debug, Default)]
1897struct ReloadExitRecordGate {
1898 reached: tokio::sync::Notify,
1899 resume: tokio::sync::Notify,
1900}
1901
1902#[derive(Debug, Clone, Copy)]
1903enum RespawnKind {
1904 Spawn,
1905 Reload,
1906}
1907
1908#[derive(Debug, Clone, Copy)]
1909struct PendingRespawn {
1910 deadline: Instant,
1911 kind: RespawnKind,
1912}
1913
1914type ReloadReply = oneshot::Sender<Result<(), SuperviseError>>;
1915
1916#[derive(Debug, Clone)]
1917struct SupervisorRuntimeConfig {
1918 start_requester: Option<Principal>,
1919 pending_activity_restart: bool,
1920 provenance_probe: ExecutableIdentityProbe,
1921 scheduled_respawn: Arc<Mutex<Option<PendingRespawn>>>,
1923 deferred_reload_reply: Arc<Mutex<Option<ReloadReply>>>,
1925 restart_policy: RestartPolicy,
1926 drain_timeout: Duration,
1929 effective_drain_timeout: Arc<Mutex<Duration>>,
1932 default_drain_timeout: Duration,
1935 health: HealthConfig,
1936 connection_file_path: Option<PathBuf>,
1937 capture_logs_dir: Option<PathBuf>,
1938 forwarding: Option<Arc<ForwardingTable>>,
1939 supervisor_handle: Option<SupervisorHandle>,
1942 stderr_ring: Arc<Mutex<StderrRing>>,
1949 terminal_ring: Arc<Mutex<TerminalRing>>,
1950 spawn_events: SpawnEventFeed,
1951 child_roster: ChildRoster,
1952 #[cfg(target_os = "linux")]
1953 cgroup_placement: Option<subc_cgroup::Placement>,
1954 #[cfg(test)]
1955 test_seed_stale_facts_before_enable_spawn: bool,
1956 #[cfg(test)]
1957 test_reload_exit_record_gate: Option<Arc<ReloadExitRecordGate>>,
1958}
1959
1960#[derive(Debug, Clone, PartialEq, Eq)]
1961struct SupervisedConfiguration {
1962 spec: ModuleSpec,
1963 health: HealthConfig,
1964}
1965
1966#[derive(Debug, Clone, Default)]
1972pub struct SupervisorHandle {
1973 activity: Arc<Mutex<Option<crate::activity::ActivityLedger>>>,
1974 activity_images: Arc<Mutex<HashMap<String, activity::ActivityImage>>>,
1975 modules: Arc<Mutex<HashMap<String, SupervisedModule>>>,
1976 configured_ids: Arc<Mutex<HashSet<String>>>,
1988 spawn_events: SpawnEventFeed,
1989 reserved_nonces: Arc<Mutex<HashMap<String, Option<String>>>>,
2000 removal_tombstones: Arc<Mutex<HashMap<String, u64>>>,
2006 spawn_nonces: Arc<Mutex<HashMap<String, String>>>,
2010 reserved_prefix_owners: Arc<Mutex<HashMap<String, String>>>,
2018 swaps: Arc<Mutex<HashMap<String, OpenSwap>>>,
2024 promotion_observer: PromotionObserverSlot,
2026 operation_lock: Arc<AsyncMutex<()>>,
2030}
2031
2032pub(crate) trait SwapPromotionObserver: Send + Sync {
2041 fn swap_promoted(&self, registration: &crate::registry::ModuleRegistration);
2042}
2043
2044#[derive(Clone, Default)]
2048struct PromotionObserverSlot(Arc<Mutex<Option<std::sync::Weak<dyn SwapPromotionObserver>>>>);
2049
2050impl fmt::Debug for PromotionObserverSlot {
2051 fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
2052 f.write_str("PromotionObserverSlot")
2053 }
2054}
2055
2056#[derive(Debug, Clone)]
2058struct OpenSwap {
2059 candidate_nonce: String,
2062 incumbent_nonce: Option<String>,
2067 candidate_admitted: bool,
2071}
2072
2073#[derive(Debug, Clone, Copy, PartialEq, Eq)]
2076pub(crate) enum SwapHelloAdmission {
2077 NotSwapping,
2080 Candidate,
2082 Refused,
2085}
2086
2087#[derive(Debug, Clone, PartialEq, Eq)]
2088pub(crate) enum ReservedHelloRejection {
2089 Exact {
2090 module_id: String,
2091 },
2092 Prefix {
2093 prefix: String,
2094 owner_module_id: String,
2095 },
2096}
2097
2098impl SupervisorHandle {
2099 pub fn new() -> Self {
2100 Self::default()
2101 }
2102
2103 pub(crate) fn spawn_snapshot(&self) -> SpawnSnapshot {
2104 self.spawn_events.snapshot()
2105 }
2106
2107 pub(crate) fn subscribe_spawns(
2108 &self,
2109 connection_id: ConnectionId,
2110 corr: u64,
2111 version: u8,
2112 since: Option<SpawnCursor>,
2113 sink: FrameSink,
2114 ) -> Result<(), SpawnSubscribeRefusal> {
2115 self.spawn_events
2116 .subscribe(connection_id, corr, version, since, sink)
2117 }
2118
2119 pub(crate) fn cancel_spawn_subscription(&self, connection_id: ConnectionId, corr: u64) -> bool {
2120 self.spawn_events.cancel(connection_id, corr)
2121 }
2122
2123 pub(crate) fn remove_spawn_subscribers(&self, connection_id: ConnectionId) {
2124 self.spawn_events.remove_connection(connection_id);
2125 }
2126
2127 #[cfg(any(test, feature = "test-support"))]
2128 pub fn set_spawn_event_capacity_for_test(&self, capacity: usize) {
2129 assert!(capacity > 0, "spawn event capacity must be non-zero");
2130 self.spawn_events.set_capacity(capacity);
2131 }
2132
2133 #[cfg(any(test, feature = "test-support"))]
2134 pub fn spawn_subscriber_count_for_test(&self) -> usize {
2135 self.spawn_events.subscriber_count()
2136 }
2137
2138 pub fn set_spawn_nonce(&self, module_id: &str, nonce: String) {
2141 self.spawn_nonces
2142 .lock()
2143 .unwrap_or_else(|poisoned| poisoned.into_inner())
2144 .insert(module_id.to_string(), nonce);
2145 }
2146
2147 pub fn set_reserved_nonce(&self, module_id: &str, nonce: String) {
2150 self.reserved_nonces
2151 .lock()
2152 .unwrap_or_else(|poisoned| poisoned.into_inner())
2153 .insert(module_id.to_string(), Some(nonce));
2154 }
2155
2156 pub fn set_reserved_prefixes(&self, owner_module_id: &str, prefixes: &[String]) {
2158 let mut owners = self
2159 .reserved_prefix_owners
2160 .lock()
2161 .unwrap_or_else(|poisoned| poisoned.into_inner());
2162 owners.retain(|_, owner| owner != owner_module_id);
2163 for prefix in prefixes {
2164 owners.insert(prefix.clone(), owner_module_id.to_string());
2165 }
2166 }
2167
2168 #[cfg(test)]
2170 pub(crate) fn spawn_nonce(&self, module_id: &str) -> Option<String> {
2171 self.spawn_nonces
2172 .lock()
2173 .unwrap_or_else(|poisoned| poisoned.into_inner())
2174 .get(module_id)
2175 .cloned()
2176 }
2177
2178 fn apply_identity_configuration(&self, spec: &ModuleSpec) {
2179 self.set_reserved_prefixes(&spec.module_id, &spec.reserved_prefixes);
2180 let spawn_nonce = self
2181 .spawn_nonces
2182 .lock()
2183 .unwrap_or_else(|poisoned| poisoned.into_inner())
2184 .get(&spec.module_id)
2185 .cloned();
2186 let mut reserved_nonces = self
2187 .reserved_nonces
2188 .lock()
2189 .unwrap_or_else(|poisoned| poisoned.into_inner());
2190 if spec.reserved {
2191 reserved_nonces.insert(spec.module_id.clone(), spawn_nonce);
2196 }
2197 drop(reserved_nonces);
2198 self.removal_tombstones
2202 .lock()
2203 .unwrap_or_else(|poisoned| poisoned.into_inner())
2204 .remove(&spec.module_id);
2205 }
2206
2207 pub fn reserved_hello_authorized(&self, module_id: &str, presented: Option<&str>) -> bool {
2212 self.reserved_hello_verification(module_id, presented)
2213 .is_ok()
2214 }
2215
2216 pub(crate) fn reserved_hello_verification(
2219 &self,
2220 module_id: &str,
2221 presented: Option<&str>,
2222 ) -> Result<bool, ReservedHelloRejection> {
2223 let nonces = self
2224 .reserved_nonces
2225 .lock()
2226 .unwrap_or_else(|poisoned| poisoned.into_inner());
2227 if let Some(expected) = nonces.get(module_id) {
2228 let authorized = match expected {
2232 Some(expected) => {
2233 presented.is_some_and(|p| constant_time_eq(expected.as_bytes(), p.as_bytes()))
2234 }
2235 None => false,
2236 };
2237 if authorized {
2238 return Ok(true);
2239 }
2240 return Err(ReservedHelloRejection::Exact {
2241 module_id: module_id.to_string(),
2242 });
2243 }
2244 drop(nonces);
2245
2246 let matched_prefix = self
2247 .reserved_prefix_owners
2248 .lock()
2249 .unwrap_or_else(|poisoned| poisoned.into_inner())
2250 .iter()
2251 .filter(|(prefix, _)| module_id.starts_with(prefix.as_str()))
2252 .max_by_key(|(prefix, _)| prefix.len())
2253 .map(|(prefix, owner)| (prefix.clone(), owner.clone()));
2254 let Some((prefix, owner_module_id)) = matched_prefix else {
2255 return Ok(false);
2256 };
2257
2258 let authorized = presented.is_some_and(|presented| {
2259 self.spawn_nonces
2260 .lock()
2261 .unwrap_or_else(|poisoned| poisoned.into_inner())
2262 .get(&owner_module_id)
2263 .is_some_and(|expected| constant_time_eq(expected.as_bytes(), presented.as_bytes()))
2264 || self.swap_nonce_matches(&owner_module_id, presented)
2267 });
2268 if authorized {
2269 Ok(true)
2270 } else {
2271 Err(ReservedHelloRejection::Prefix {
2272 prefix,
2273 owner_module_id,
2274 })
2275 }
2276 }
2277
2278 pub fn spawned_consumer_authorized(&self, module_id: &str, presented: &str) -> bool {
2283 if presented.is_empty() {
2284 return false;
2285 }
2286 let nonces = self
2287 .spawn_nonces
2288 .lock()
2289 .unwrap_or_else(|poisoned| poisoned.into_inner());
2290 let current = nonces
2291 .get(module_id)
2292 .is_some_and(|expected| constant_time_eq(expected.as_bytes(), presented.as_bytes()));
2293 drop(nonces);
2294 current || self.swap_nonce_matches(module_id, presented)
2299 }
2300
2301 fn swap_nonce_matches(&self, module_id: &str, presented: &str) -> bool {
2303 let swaps = self
2304 .swaps
2305 .lock()
2306 .unwrap_or_else(|poisoned| poisoned.into_inner());
2307 swaps.get(module_id).is_some_and(|swap| {
2308 constant_time_eq(swap.candidate_nonce.as_bytes(), presented.as_bytes())
2309 || swap.incumbent_nonce.as_deref().is_some_and(|incumbent| {
2310 constant_time_eq(incumbent.as_bytes(), presented.as_bytes())
2311 })
2312 })
2313 }
2314
2315 pub(crate) fn open_swap(&self, module_id: &str, candidate_nonce: String) {
2318 let incumbent_nonce = self
2319 .spawn_nonces
2320 .lock()
2321 .unwrap_or_else(|poisoned| poisoned.into_inner())
2322 .get(module_id)
2323 .cloned();
2324 self.swaps
2325 .lock()
2326 .unwrap_or_else(|poisoned| poisoned.into_inner())
2327 .insert(
2328 module_id.to_string(),
2329 OpenSwap {
2330 candidate_nonce,
2331 incumbent_nonce,
2332 candidate_admitted: false,
2333 },
2334 );
2335 }
2336
2337 pub(crate) fn close_swap(&self, module_id: &str) {
2340 self.swaps
2341 .lock()
2342 .unwrap_or_else(|poisoned| poisoned.into_inner())
2343 .remove(module_id);
2344 }
2345
2346 pub(crate) fn set_swap_promotion_observer(
2349 &self,
2350 observer: std::sync::Weak<dyn SwapPromotionObserver>,
2351 ) {
2352 *self
2353 .promotion_observer
2354 .0
2355 .lock()
2356 .unwrap_or_else(|poisoned| poisoned.into_inner()) = Some(observer);
2357 }
2358
2359 fn notify_swap_promoted(&self, registration: &crate::registry::ModuleRegistration) {
2362 let observer = self
2363 .promotion_observer
2364 .0
2365 .lock()
2366 .unwrap_or_else(|poisoned| poisoned.into_inner())
2367 .as_ref()
2368 .and_then(std::sync::Weak::upgrade);
2369 if let Some(observer) = observer {
2370 observer.swap_promoted(registration);
2371 }
2372 }
2373
2374 pub(crate) fn swap_open(&self, module_id: &str) -> bool {
2376 self.swaps
2377 .lock()
2378 .unwrap_or_else(|poisoned| poisoned.into_inner())
2379 .contains_key(module_id)
2380 }
2381
2382 fn promote_swap_nonce(&self, module_id: &str, reserved: bool) {
2387 let candidate_nonce = self
2388 .swaps
2389 .lock()
2390 .unwrap_or_else(|poisoned| poisoned.into_inner())
2391 .get(module_id)
2392 .map(|swap| swap.candidate_nonce.clone());
2393 let Some(nonce) = candidate_nonce else {
2394 return;
2395 };
2396 self.set_spawn_nonce(module_id, nonce.clone());
2397 if reserved {
2398 self.set_reserved_nonce(module_id, nonce);
2399 }
2400 }
2401
2402 pub(crate) fn swap_hello_admission(
2417 &self,
2418 module_id: &str,
2419 presented: Option<&str>,
2420 ) -> SwapHelloAdmission {
2421 let swaps = self
2422 .swaps
2423 .lock()
2424 .unwrap_or_else(|poisoned| poisoned.into_inner());
2425 let Some(swap) = swaps.get(module_id) else {
2426 return SwapHelloAdmission::NotSwapping;
2427 };
2428 let Some(presented) = presented else {
2429 return SwapHelloAdmission::Refused;
2430 };
2431 if constant_time_eq(swap.candidate_nonce.as_bytes(), presented.as_bytes()) {
2432 return if swap.candidate_admitted {
2433 SwapHelloAdmission::Refused
2434 } else {
2435 SwapHelloAdmission::Candidate
2436 };
2437 }
2438 if swap
2439 .incumbent_nonce
2440 .as_deref()
2441 .is_some_and(|incumbent| constant_time_eq(incumbent.as_bytes(), presented.as_bytes()))
2442 {
2443 return SwapHelloAdmission::NotSwapping;
2444 }
2445 SwapHelloAdmission::Refused
2446 }
2447
2448 pub(crate) fn mark_swap_candidate_admitted(&self, module_id: &str) {
2451 if let Some(swap) = self
2452 .swaps
2453 .lock()
2454 .unwrap_or_else(|poisoned| poisoned.into_inner())
2455 .get_mut(module_id)
2456 {
2457 swap.candidate_admitted = true;
2458 }
2459 }
2460
2461 pub fn spawn_launch_nonce_for(&self, module_id: &str) -> Option<String> {
2463 self.spawn_nonces
2464 .lock()
2465 .unwrap_or_else(|poisoned| poisoned.into_inner())
2466 .get(module_id)
2467 .cloned()
2468 }
2469
2470 pub fn reserved_launch_nonce_for(&self, module_id: &str) -> Option<String> {
2472 self.reserved_nonces
2473 .lock()
2474 .unwrap_or_else(|poisoned| poisoned.into_inner())
2475 .get(module_id)
2476 .cloned()
2477 .flatten()
2478 }
2479
2480 pub fn insert(&self, module: SupervisedModule) -> Option<SupervisedModule> {
2481 self.mark_configured(module.module_id());
2485 let mut modules = self
2486 .modules
2487 .lock()
2488 .unwrap_or_else(|poisoned| poisoned.into_inner());
2489 modules.insert(module.module_id().to_string(), module)
2490 }
2491
2492 fn mark_configured(&self, module_id: &str) {
2496 self.configured_ids
2497 .lock()
2498 .unwrap_or_else(|poisoned| poisoned.into_inner())
2499 .insert(module_id.to_string());
2500 }
2501
2502 fn unmark_configured_unless_rostered(&self, module_id: &str) {
2506 let modules = self
2507 .modules
2508 .lock()
2509 .unwrap_or_else(|poisoned| poisoned.into_inner());
2510 if !modules.contains_key(module_id) {
2511 self.configured_ids
2512 .lock()
2513 .unwrap_or_else(|poisoned| poisoned.into_inner())
2514 .remove(module_id);
2515 }
2516 }
2517
2518 pub(crate) fn is_configured(&self, module_id: &str) -> bool {
2526 self.configured_ids
2527 .lock()
2528 .unwrap_or_else(|poisoned| poisoned.into_inner())
2529 .contains(module_id)
2530 }
2531
2532 pub fn get(&self, module_id: &str) -> Option<SupervisedModule> {
2533 let modules = self
2534 .modules
2535 .lock()
2536 .unwrap_or_else(|poisoned| poisoned.into_inner());
2537 modules.get(module_id).cloned()
2538 }
2539
2540 pub(crate) fn record_late_health_answer(
2541 &self,
2542 module_id: &str,
2543 latency_ms: u64,
2544 ) -> Result<bool, SuperviseError> {
2545 let Some(module) = self.get(module_id) else {
2546 return Ok(false);
2547 };
2548 update_snapshot(&module.inner.snapshot, Some(module_id), |state| {
2549 state.health.late_answer_count = state.health.late_answer_count.saturating_add(1);
2550 state.health.last_late_answer_latency_ms = Some(latency_ms);
2551 state.health.consecutive_failures = 0;
2559 })?;
2560 Ok(true)
2561 }
2562
2563 pub fn record_deliberate_severance(&self, module_id: &str) -> Result<bool, SuperviseError> {
2569 let Some(module) = self.get(module_id) else {
2570 return Ok(false);
2571 };
2572 let snapshot = lock_snapshot(&module.inner.snapshot)?;
2573 let Some((pid, start_time)) = snapshot.pid.zip(snapshot.process_start_time) else {
2574 return Ok(false);
2575 };
2576 drop(snapshot);
2577 module.record_deliberate_severance(ProcessIdentity { pid, start_time })
2578 }
2579
2580 pub fn list(&self) -> Vec<SupervisedModule> {
2581 let modules = self
2582 .modules
2583 .lock()
2584 .unwrap_or_else(|poisoned| poisoned.into_inner());
2585 let mut modules = modules.values().cloned().collect::<Vec<_>>();
2586 modules.sort_by(|left, right| left.module_id().cmp(right.module_id()));
2587 modules
2588 }
2589
2590 pub(crate) fn retire(&self, module_id: &str) -> Option<SupervisedModule> {
2591 self.activity_images
2592 .lock()
2593 .unwrap_or_else(|p| p.into_inner())
2594 .remove(module_id);
2595 self.spawn_nonces
2596 .lock()
2597 .unwrap_or_else(|poisoned| poisoned.into_inner())
2598 .remove(module_id);
2599 self.close_swap(module_id);
2600 let mut reserved_nonces = self
2601 .reserved_nonces
2602 .lock()
2603 .unwrap_or_else(|poisoned| poisoned.into_inner());
2604 if reserved_nonces.contains_key(module_id) {
2605 reserved_nonces.insert(module_id.to_string(), None);
2608 }
2609 drop(reserved_nonces);
2610 self.reserved_prefix_owners
2611 .lock()
2612 .unwrap_or_else(|poisoned| poisoned.into_inner())
2613 .retain(|_, owner| owner != module_id);
2614 let removed = self
2615 .modules
2616 .lock()
2617 .unwrap_or_else(|poisoned| poisoned.into_inner())
2618 .remove(module_id);
2619 self.configured_ids
2620 .lock()
2621 .unwrap_or_else(|poisoned| poisoned.into_inner())
2622 .remove(module_id);
2623 removed
2624 }
2625
2626 pub(crate) fn record_rescan_removal(&self, module_id: &str) {
2629 self.removal_tombstones
2630 .lock()
2631 .unwrap_or_else(|poisoned| poisoned.into_inner())
2632 .insert(module_id.to_string(), unix_ms_now());
2633 }
2634
2635 pub(crate) fn removal_tombstone_age_ms(&self, module_id: &str) -> Option<u64> {
2637 self.removal_tombstones
2638 .lock()
2639 .unwrap_or_else(|poisoned| poisoned.into_inner())
2640 .get(module_id)
2641 .copied()
2642 .map(|removed_at_ms| unix_ms_now().saturating_sub(removed_at_ms))
2643 }
2644
2645 pub(crate) fn release_retained_reserved_gate(&self, module_id: &str) -> bool {
2650 if self.get(module_id).is_some() {
2651 return false;
2652 }
2653 let mut reserved_nonces = self
2654 .reserved_nonces
2655 .lock()
2656 .unwrap_or_else(|poisoned| poisoned.into_inner());
2657 if !matches!(reserved_nonces.get(module_id), Some(None)) {
2658 return false;
2659 }
2660 reserved_nonces.remove(module_id);
2661 true
2662 }
2663
2664 pub(crate) fn operation_lock(&self) -> Arc<AsyncMutex<()>> {
2665 Arc::clone(&self.operation_lock)
2666 }
2667}
2668
2669#[derive(Debug, Clone)]
2671pub struct Supervisor {
2672 registry: Arc<Registry>,
2673 restart_policy: RestartPolicy,
2674 drain_timeout: Duration,
2675 connection_file_path: Option<PathBuf>,
2676 capture_logs_dir: Option<PathBuf>,
2677 forwarding: Option<Arc<ForwardingTable>>,
2678 process_liveness: Arc<SupervisorProcessLiveness>,
2679 supervisor_handle: Option<SupervisorHandle>,
2680 health: HealthConfig,
2681 daemon_start_clock: crate::clock::StartClock,
2682 terminal_journal: Option<Arc<crate::terminal_journal::TerminalJournal>>,
2683 spawn_events: SpawnEventFeed,
2684 provenance_probe: ExecutableIdentityProbe,
2685 child_roster: ChildRoster,
2688 #[cfg(target_os = "linux")]
2689 cgroup_placement: Option<subc_cgroup::Placement>,
2690 #[cfg(test)]
2691 test_after_first_spawn: AfterFirstSpawnHook,
2692}
2693
2694pub(crate) struct RegisteredConfiguredModule {
2698 spec: ModuleSpec,
2699 runtime: SupervisorRuntimeConfig,
2700 module: SupervisedModule,
2701 commands: mpsc::Receiver<SupervisorCommand>,
2702}
2703
2704#[cfg(test)]
2710#[derive(Clone, Default)]
2711struct AfterFirstSpawnHook(Option<AfterFirstSpawnFn>);
2712
2713#[cfg(test)]
2714type AfterFirstSpawnFn = Arc<dyn Fn(&str) + Send + Sync>;
2715
2716#[cfg(test)]
2717impl fmt::Debug for AfterFirstSpawnHook {
2718 fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
2719 f.write_str("AfterFirstSpawnHook")
2720 }
2721}
2722
2723#[cfg(test)]
2724impl AfterFirstSpawnHook {
2725 fn run(&self, module_id: &str) {
2726 if let Some(hook) = &self.0 {
2727 hook(module_id);
2728 }
2729 }
2730}
2731
2732impl Supervisor {
2733 #[cfg(test)]
2734 pub(crate) fn new_for_test(registry: Arc<Registry>, policy: RestartPolicy) -> Self {
2735 let supervisor = Self::new(registry, policy);
2736 #[cfg(target_os = "macos")]
2737 let supervisor = supervisor.with_privacy_trampoline(test_privacy_trampoline());
2738 supervisor
2739 }
2740 pub fn with_privacy_trampoline(self, path: impl Into<PathBuf>) -> Self {
2745 let path = path.into();
2746 #[cfg(target_os = "macos")]
2747 {
2748 let result = probe_privacy_trampoline(&path).map(|()| path);
2749 if let Err(cause) = &result {
2750 error!(%cause, "macOS privacy identity unavailable; supervised launches will refuse");
2751 }
2752 self.child_roster.set_privacy_trampoline(result);
2753 }
2754 #[cfg(not(target_os = "macos"))]
2755 let _ = path;
2756 self
2757 }
2758 pub(crate) fn daemon_shutdown_flag(&self) -> crate::child_roster::DaemonShutdownFlag {
2761 self.child_roster.shutdown_flag()
2762 }
2763
2764 #[cfg(unix)]
2775 pub(crate) fn begin_daemon_shutdown(&self) {
2776 self.child_roster.close();
2777 if let Some(journal) = &self.terminal_journal {
2778 journal.stamp_shutdown();
2779 }
2780 }
2781
2782 #[cfg(unix)]
2786 pub(crate) async fn drain_for_daemon_shutdown(&self) -> Result<(), SuperviseError> {
2787 const NOTICE_BUDGET: Duration = Duration::from_millis(500);
2788 const DRAIN_BUDGET: Duration = Duration::from_secs(2);
2789 let Some(forwarding) = &self.forwarding else {
2790 return Ok(());
2791 };
2792 let notice_started = Instant::now();
2793 let modules = forwarding
2794 .begin_daemon_drain()
2795 .map_err(SuperviseError::Forwarding)?;
2796 let deadline_ms =
2797 unix_ms_now().saturating_add((NOTICE_BUDGET + DRAIN_BUDGET).as_millis() as u64);
2798 let mut notices = tokio::task::JoinSet::new();
2799 let mut drains = Vec::new();
2800 for (module_id, endpoint) in modules {
2801 let Some(target) = forwarding
2802 .begin_endpoint_drain(endpoint, RouteCloseReason::Restart)
2803 .map_err(SuperviseError::Forwarding)?
2804 else {
2805 continue;
2806 };
2807 let routes = forwarding
2808 .endpoint_routes(target.endpoint)
2809 .map_err(SuperviseError::Forwarding)?;
2810 let command = serde_json::to_vec(&ModuleControlCommand::Draining {
2816 reason: RouteCloseReason::Restart,
2817 deadline_ms,
2818 })
2819 .expect("module draining serializes");
2820 let mut recipients = vec![(target.sink.clone(), target.negotiated_ver, command)];
2821 let mut clients: Vec<(crate::forwarding::GoodbyeTarget, Vec<u16>)> = Vec::new();
2822 for route in routes {
2823 let client = route.goodbye_target;
2824 if let Some((_, channels)) = clients
2825 .iter_mut()
2826 .find(|(existing, _)| existing.connection_id == client.connection_id)
2827 {
2828 channels.push(client.channel);
2829 } else {
2830 let channel = client.channel;
2831 clients.push((client, vec![channel]));
2832 }
2833 }
2834 for (client, mut channels) in clients {
2835 channels.sort_unstable();
2836 channels.dedup();
2837 let closing = serde_json::to_vec(&ClientControlPush::RouteClosing {
2838 module_id: module_id.clone(),
2839 channels,
2840 reason: RouteCloseReason::Restart,
2841 })
2842 .expect("route closing serializes");
2843 recipients.push((client.sink, client.negotiated_ver, closing));
2844 }
2845 for (sink, version, body) in recipients {
2846 notices.spawn(async move {
2847 let frame = Frame::build_with_version(
2848 version,
2849 FrameType::Push,
2850 control_flags(),
2851 0,
2852 0,
2853 0,
2854 body,
2855 )
2856 .expect("bounded lifecycle notice frame builds");
2857 sink.send_flushed(frame).await
2858 });
2859 }
2860 let gauges = declared_busy_gauges(&self.registry, &module_id)?;
2861 drains.push((module_id, target.endpoint, gauges));
2862 }
2863 let notice_deadline = Instant::now() + NOTICE_BUDGET;
2866 while let Ok(Some(result)) = timeout_at(notice_deadline, notices.join_next()).await {
2867 if !matches!(result, Ok(Ok(()))) {
2868 warn!(?result, "daemon shutdown notice delivery failed");
2869 }
2870 }
2871 notices.abort_all();
2872 let notice_elapsed = notice_started.elapsed();
2873 let drain_started = Instant::now();
2874 let deadline = drain_started + DRAIN_BUDGET;
2875 let mut waits = tokio::task::JoinSet::new();
2876 for (module_id, endpoint, gauges) in drains {
2877 let forwarding = Arc::clone(forwarding);
2878 let mut runtime = self.runtime_config();
2879 runtime.health.cadence = Duration::from_millis(100);
2880 waits.spawn(async move {
2881 let wait_result = wait_for_forwarding_quiescence(
2882 &forwarding,
2883 &module_id,
2884 &runtime,
2885 endpoint,
2886 deadline,
2887 &gauges,
2888 DrainScope::Endpoint(endpoint),
2889 &mut DrainGaugeTrace::new(&gauges),
2890 )
2891 .await;
2892 forwarding
2896 .record_daemon_drain_result(
2897 endpoint,
2898 drained_after_quiescence_wait(&wait_result),
2899 )
2900 .map_err(SuperviseError::Forwarding)?;
2901 wait_result
2902 });
2903 }
2904 while let Ok(Some(result)) = timeout_at(deadline, waits.join_next()).await {
2905 if !matches!(result, Ok(Ok(true))) {
2906 warn!(?result, "daemon shutdown drain did not reach quiescence");
2907 }
2908 }
2909 info!(
2912 notice_elapsed_ms = notice_elapsed.as_millis() as u64,
2913 drain_elapsed_ms = drain_started.elapsed().as_millis() as u64,
2914 "daemon shutdown forwarding drain complete"
2915 );
2916 Ok(())
2917 }
2918
2919 #[cfg(unix)]
2931 pub(crate) async fn end_children_for_daemon_shutdown(
2932 &self,
2933 already_escalated: bool,
2934 escalate: impl std::future::Future<Output = ()>,
2935 ) {
2936 tokio::pin!(escalate);
2937 let mut escalated = already_escalated;
2938 if let Some(forwarding) = &self.forwarding {
2939 let reason = CloseReason::new(
2940 "daemon_shutdown",
2941 "the daemon is exiting after its shutdown notice and drain",
2942 );
2943 if escalated {
2944 send_module_goodbyes_for_daemon_shutdown(forwarding, &reason, false).await;
2947 } else {
2948 tokio::select! {
2949 biased;
2950 _ = escalate.as_mut() => {
2951 info!("second SIGTERM: abandoning module GOODBYE delivery");
2952 escalated = true;
2953 }
2954 _ = send_module_goodbyes_for_daemon_shutdown(forwarding, &reason, true) => {}
2955 }
2956 }
2957 let closed = forwarding.close_all_connections(&reason);
2958 debug!(closed, "closed established connections for daemon shutdown");
2959 }
2960 let escalated_here = escalated && !already_escalated;
2964 let remaining_escalate = async move {
2965 if escalated_here {
2966 std::future::pending::<()>().await;
2967 } else {
2968 escalate.await;
2969 }
2970 };
2971 crate::child_roster::end_children_for_daemon_shutdown(
2972 &self.child_roster,
2973 escalated,
2974 remaining_escalate,
2975 )
2976 .await;
2977 }
2978
2979 pub fn new(registry: Arc<Registry>, restart_policy: RestartPolicy) -> Self {
2980 Self {
2981 registry,
2982 restart_policy,
2983 drain_timeout: DEFAULT_DRAIN_TIMEOUT,
2984 connection_file_path: None,
2985 capture_logs_dir: None,
2986 forwarding: None,
2987 process_liveness: Arc::new(SupervisorProcessLiveness::default()),
2988 supervisor_handle: None,
2989 health: HealthConfig::default(),
2990 daemon_start_clock: crate::clock::StartClock::capture(),
2991 terminal_journal: None,
2992 spawn_events: SpawnEventFeed::default(),
2993 provenance_probe: ExecutableIdentityProbe::default(),
2994 child_roster: ChildRoster::default(),
2995 #[cfg(target_os = "linux")]
2996 cgroup_placement: None,
2997 #[cfg(test)]
2998 test_after_first_spawn: AfterFirstSpawnHook::default(),
2999 }
3000 }
3001
3002 pub fn with_drain_timeout(mut self, drain_timeout: Duration) -> Self {
3003 self.drain_timeout = drain_timeout;
3004 self
3005 }
3006
3007 pub(crate) fn default_drain_timeout(&self) -> Duration {
3009 self.drain_timeout
3010 }
3011
3012 pub fn with_process_liveness(
3013 mut self,
3014 process_liveness: Arc<SupervisorProcessLiveness>,
3015 ) -> Self {
3016 self.process_liveness = process_liveness;
3017 self
3018 }
3019
3020 pub fn with_connection_file_path(mut self, connection_file_path: impl Into<PathBuf>) -> Self {
3021 self.connection_file_path = Some(connection_file_path.into());
3022 self
3023 }
3024
3025 pub fn with_capture_logs_dir(mut self, logs_dir: impl Into<PathBuf>) -> Self {
3027 self.capture_logs_dir = Some(logs_dir.into());
3028 self
3029 }
3030
3031 pub fn with_daemon_incarnation(self, daemon_incarnation: String) -> Self {
3034 self.spawn_events.configure_incarnation(daemon_incarnation);
3038 self
3039 }
3040
3041 pub fn with_terminal_journal(self, path: PathBuf, daemon_incarnation: String) -> Self {
3044 let mut this = self.with_daemon_incarnation(daemon_incarnation.clone());
3045 this.terminal_journal = Some(Arc::new(crate::terminal_journal::TerminalJournal::open(
3046 path,
3047 daemon_incarnation,
3048 )));
3049 this
3050 }
3051
3052 pub fn with_forwarding(mut self, forwarding: Arc<ForwardingTable>) -> Self {
3053 self.forwarding = Some(forwarding);
3054 self
3055 }
3056
3057 pub fn with_handle(mut self, supervisor_handle: SupervisorHandle) -> Self {
3058 self.spawn_events = supervisor_handle.spawn_events.clone();
3059 self.supervisor_handle = Some(supervisor_handle);
3060 self
3061 }
3062
3063 pub fn with_health_config(mut self, health: HealthConfig) -> Self {
3064 self.health = health;
3065 self
3066 }
3067
3068 pub fn with_live_children_record(self, path: impl Into<PathBuf>) -> Self {
3072 self.child_roster.record_to(path.into());
3073 self
3074 }
3075
3076 #[cfg(target_os = "linux")]
3077 pub fn with_cgroup_placement(
3078 mut self,
3079 cgroup_placement: Option<subc_cgroup::Placement>,
3080 ) -> Self {
3081 self.cgroup_placement = cgroup_placement;
3082 self
3083 }
3084
3085 pub fn spawn(&self, spec: ModuleSpec) -> Result<SupervisedModule, SuperviseError> {
3091 validate_spec(&spec)?;
3092 self.establish_identity(&spec);
3093
3094 let runtime = self.runtime_config();
3095 let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::starting()));
3096 let activity_spawn = runtime.begin_activity_spawn(&spec.module_id);
3097 let spawned = spawn_child(
3098 &spec,
3099 runtime.connection_file_path.as_deref(),
3100 self.supervisor_handle.as_ref(),
3101 &runtime.stderr_ring,
3102 runtime.capture_logs_dir.as_deref(),
3103 &runtime.child_roster,
3104 #[cfg(target_os = "linux")]
3105 runtime.cgroup_placement.as_ref(),
3106 );
3107 #[cfg(test)]
3108 self.test_after_first_spawn.run(&spec.module_id);
3109 let child = match spawned {
3110 Ok(child) => child,
3111 Err(err) => {
3112 self.abandon_unrostered(&spec.module_id);
3115 return Err(err);
3116 }
3117 };
3118 self.mark_first_process_running(&spec, &runtime, &snapshot, &child)?;
3119 if let Some(spawn) = activity_spawn {
3120 spawn.complete(&snapshot);
3121 }
3122
3123 Ok(self.supervised_module(spec, runtime, snapshot, Some(child)))
3124 }
3125
3126 fn establish_identity(&self, spec: &ModuleSpec) {
3139 if let Some(supervisor_handle) = &self.supervisor_handle {
3140 supervisor_handle.apply_identity_configuration(spec);
3141 supervisor_handle.mark_configured(&spec.module_id);
3142 }
3143 }
3144
3145 fn abandon_unrostered(&self, module_id: &str) {
3148 if let Some(supervisor_handle) = &self.supervisor_handle {
3149 supervisor_handle.unmark_configured_unless_rostered(module_id);
3150 }
3151 }
3152
3153 fn mark_first_process_running(
3156 &self,
3157 spec: &ModuleSpec,
3158 runtime: &SupervisorRuntimeConfig,
3159 snapshot: &SharedSnapshot,
3160 child: &SupervisedChild,
3161 ) -> Result<(), SuperviseError> {
3162 if let Err(err) = set_running(snapshot, child, &spec.module_id, &runtime.spawn_events) {
3163 self.abandon_unrostered(&spec.module_id);
3164 return Err(err);
3165 }
3166 self.process_liveness
3167 .track(spec.module_id.clone(), Arc::clone(snapshot));
3168 Ok(())
3169 }
3170
3171 pub fn supervise_configured(
3177 &self,
3178 spec: ModuleSpec,
3179 enabled: bool,
3180 ) -> Result<SupervisedModule, SuperviseError> {
3181 self.supervise_configured_requested(
3182 spec,
3183 enabled,
3184 self.health.clone(),
3185 None,
3186 self.restart_policy,
3187 None,
3188 )
3189 }
3190
3191 pub fn supervise_configured_with_health(
3197 &self,
3198 spec: ModuleSpec,
3199 enabled: bool,
3200 health: HealthConfig,
3201 drain_timeout_ms: Option<u64>,
3202 restart_policy: RestartPolicy,
3203 ) -> Result<SupervisedModule, SuperviseError> {
3204 self.supervise_configured_requested(
3205 spec,
3206 enabled,
3207 health,
3208 drain_timeout_ms,
3209 restart_policy,
3210 None,
3211 )
3212 }
3213
3214 pub(crate) fn supervise_configured_requested(
3215 &self,
3216 spec: ModuleSpec,
3217 enabled: bool,
3218 health: HealthConfig,
3219 drain_timeout_ms: Option<u64>,
3220 restart_policy: RestartPolicy,
3221 requester: Option<Principal>,
3222 ) -> Result<SupervisedModule, SuperviseError> {
3223 let registered = self.register_configured_with_health(
3224 spec,
3225 enabled,
3226 health,
3227 drain_timeout_ms,
3228 restart_policy,
3229 )?;
3230 self.start_registered_configured(registered, requester)
3231 }
3232
3233 pub(crate) fn register_configured_with_health(
3238 &self,
3239 spec: ModuleSpec,
3240 enabled: bool,
3241 health: HealthConfig,
3242 drain_timeout_ms: Option<u64>,
3243 restart_policy: RestartPolicy,
3244 ) -> Result<RegisteredConfiguredModule, SuperviseError> {
3245 validate_spec(&spec)?;
3246 self.establish_identity(&spec);
3247
3248 let mut runtime = self.runtime_config();
3249 runtime.health = health;
3250 runtime.restart_policy = restart_policy;
3251 if let Some(ms) = drain_timeout_ms {
3252 runtime.drain_timeout = Duration::from_millis(ms);
3253 *runtime
3254 .effective_drain_timeout
3255 .lock()
3256 .unwrap_or_else(|poisoned| poisoned.into_inner()) = runtime.drain_timeout;
3257 }
3258 let snapshot = if enabled {
3259 SupervisorSnapshot::starting()
3260 } else {
3261 SupervisorSnapshot::disabled()
3262 };
3263 let snapshot = {
3264 let mut snapshot = snapshot;
3265 if enabled {
3266 if let Some((pid, reason)) = runtime.child_roster.previous_instance(&spec.module_id)
3267 {
3268 snapshot.waiting_for_previous_pid = Some(pid);
3269 snapshot.waiting_for_previous_detail = Some(reason.waiting_message(pid));
3270 }
3271 }
3272 snapshot
3273 };
3274 let (module, commands) =
3275 self.register_module(&spec, &runtime, Arc::new(Mutex::new(snapshot)));
3276 Ok(RegisteredConfiguredModule {
3277 spec,
3278 runtime,
3279 module,
3280 commands,
3281 })
3282 }
3283
3284 pub(crate) fn start_registered_configured(
3287 &self,
3288 registered: RegisteredConfiguredModule,
3289 requester: Option<Principal>,
3290 ) -> Result<SupervisedModule, SuperviseError> {
3291 let RegisteredConfiguredModule {
3292 spec,
3293 mut runtime,
3294 module,
3295 commands,
3296 } = registered;
3297 runtime.start_requester = requester;
3298 let snapshot = &module.inner.snapshot;
3299 let should_spawn = {
3300 let state = lock_snapshot(snapshot)?;
3301 state.enabled && state.waiting_for_previous_pid.is_none()
3302 };
3303 let child = if should_spawn {
3304 let activity_spawn = runtime.begin_activity_spawn(&spec.module_id);
3305 let spawned = spawn_child(
3306 &spec,
3307 runtime.connection_file_path.as_deref(),
3308 self.supervisor_handle.as_ref(),
3309 &runtime.stderr_ring,
3310 runtime.capture_logs_dir.as_deref(),
3311 &runtime.child_roster,
3312 #[cfg(target_os = "linux")]
3313 runtime.cgroup_placement.as_ref(),
3314 );
3315 #[cfg(test)]
3316 self.test_after_first_spawn.run(&spec.module_id);
3317 match spawned {
3318 Ok(child) => {
3319 self.mark_first_process_running(&spec, &runtime, snapshot, &child)?;
3320 if let Some(spawn) = activity_spawn {
3321 spawn.complete(snapshot);
3322 }
3323 Some(child)
3324 }
3325 Err(err) => {
3326 if runtime.health.critical {
3327 error!(
3328 module_id = %spec.module_id,
3329 program = %spec.program.display(),
3330 error = %err,
3331 "critical configured module failed to spawn; marking failed and alerting"
3332 );
3333 } else {
3334 error!(
3335 module_id = %spec.module_id,
3336 program = %spec.program.display(),
3337 error = %err,
3338 "configured module failed to spawn; marking failed and continuing"
3339 );
3340 }
3341 update_snapshot(snapshot, Some(&spec.module_id), |state| {
3342 state.state = ModuleState::Failed;
3343 clear_current_process_facts(state);
3344 })?;
3345 None
3346 }
3347 }
3348 } else {
3349 None
3350 };
3351 self.start_monitor(spec, runtime, &module, child, commands);
3352 Ok(module)
3353 }
3354
3355 fn runtime_config(&self) -> SupervisorRuntimeConfig {
3356 let effective_drain_timeout = Arc::new(Mutex::new(self.drain_timeout));
3357 SupervisorRuntimeConfig {
3358 start_requester: None,
3359 pending_activity_restart: false,
3360 provenance_probe: self.provenance_probe.clone(),
3361 scheduled_respawn: Arc::default(),
3362 deferred_reload_reply: Arc::default(),
3363 restart_policy: self.restart_policy,
3364 drain_timeout: self.drain_timeout,
3365 child_roster: self
3368 .child_roster
3369 .for_module(Arc::clone(&effective_drain_timeout)),
3370 effective_drain_timeout,
3371 default_drain_timeout: self.drain_timeout,
3372 health: self.health.clone(),
3373 connection_file_path: self.connection_file_path.clone(),
3374 capture_logs_dir: self.capture_logs_dir.clone(),
3375 forwarding: self.forwarding.clone(),
3376 supervisor_handle: self.supervisor_handle.clone(),
3377 stderr_ring: Arc::new(Mutex::new(StderrRing::new(StderrTailConfig::default()))),
3378 terminal_ring: Arc::new(Mutex::new(
3379 TerminalRing::new(
3380 TerminalRingConfig::default(),
3381 self.daemon_start_clock.started_at_ms(),
3382 )
3383 .with_start_clock(self.daemon_start_clock)
3384 .with_journal(self.terminal_journal.clone())
3385 .with_daemon_shutdown(self.child_roster.shutdown_flag()),
3386 )),
3387 spawn_events: self.spawn_events.clone(),
3388 #[cfg(target_os = "linux")]
3389 cgroup_placement: self.cgroup_placement.clone(),
3390 #[cfg(test)]
3391 test_seed_stale_facts_before_enable_spawn: false,
3392 #[cfg(test)]
3393 test_reload_exit_record_gate: None,
3394 }
3395 }
3396
3397 fn supervised_module(
3398 &self,
3399 spec: ModuleSpec,
3400 runtime: SupervisorRuntimeConfig,
3401 snapshot: SharedSnapshot,
3402 child: Option<SupervisedChild>,
3403 ) -> SupervisedModule {
3404 let (module, commands) = self.register_module(&spec, &runtime, snapshot);
3405 self.start_monitor(spec, runtime, &module, child, commands);
3406 module
3407 }
3408
3409 fn register_module(
3410 &self,
3411 spec: &ModuleSpec,
3412 runtime: &SupervisorRuntimeConfig,
3413 snapshot: SharedSnapshot,
3414 ) -> (SupervisedModule, mpsc::Receiver<SupervisorCommand>) {
3415 let configuration = Arc::new(Mutex::new(SupervisedConfiguration {
3416 spec: spec.clone(),
3417 health: runtime.health.clone(),
3418 }));
3419 let stderr_ring = Arc::clone(&runtime.stderr_ring);
3420 let terminal_ring = Arc::clone(&runtime.terminal_ring);
3421 let restart_policy = runtime.restart_policy;
3425 let effective_drain_timeout = Arc::clone(&runtime.effective_drain_timeout);
3426 let (tx, rx) = mpsc::channel(4);
3427 let module_id = spec.module_id.clone();
3428 let module = SupervisedModule {
3429 inner: Arc::new(SupervisedModuleInner {
3430 module_id: module_id.clone(),
3431 registry: Arc::clone(&self.registry),
3432 snapshot,
3433 configuration,
3434 stderr_ring,
3435 terminal_ring,
3436 commands: tx,
3437 monitor: Mutex::new(None),
3438 restart_policy,
3439 effective_drain_timeout,
3440 provenance_probe: self.provenance_probe.clone(),
3441 }),
3442 };
3443 if let Some(supervisor_handle) = &self.supervisor_handle {
3446 supervisor_handle.insert(module.clone());
3447 }
3448 (module, rx)
3449 }
3450
3451 fn start_monitor(
3452 &self,
3453 spec: ModuleSpec,
3454 runtime: SupervisorRuntimeConfig,
3455 module: &SupervisedModule,
3456 child: Option<SupervisedChild>,
3457 commands: mpsc::Receiver<SupervisorCommand>,
3458 ) {
3459 let monitor = tokio::spawn(supervise_loop(
3460 spec,
3461 runtime,
3462 Arc::clone(&self.registry),
3463 Arc::clone(&self.process_liveness),
3464 Arc::clone(&module.inner.snapshot),
3465 child,
3466 commands,
3467 ));
3468 *module
3469 .inner
3470 .monitor
3471 .lock()
3472 .unwrap_or_else(|p| p.into_inner()) = Some(monitor);
3473 }
3474}
3475
3476impl Default for Supervisor {
3477 fn default() -> Self {
3478 Self::new(Arc::new(Registry::default()), RestartPolicy::default())
3479 }
3480}
3481
3482#[derive(Clone)]
3484pub struct SupervisedModule {
3485 inner: Arc<SupervisedModuleInner>,
3486}
3487
3488struct SupervisedModuleInner {
3489 module_id: String,
3490 registry: Arc<Registry>,
3491 snapshot: SharedSnapshot,
3492 configuration: Arc<Mutex<SupervisedConfiguration>>,
3493 stderr_ring: Arc<Mutex<StderrRing>>,
3494 terminal_ring: Arc<Mutex<TerminalRing>>,
3495 commands: mpsc::Sender<SupervisorCommand>,
3496 monitor: Mutex<Option<JoinHandle<()>>>,
3497 restart_policy: RestartPolicy,
3501 effective_drain_timeout: Arc<Mutex<Duration>>,
3502 provenance_probe: ExecutableIdentityProbe,
3503}
3504
3505impl fmt::Debug for SupervisedModule {
3506 fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
3507 f.debug_struct("SupervisedModule")
3508 .field("module_id", &self.inner.module_id)
3509 .field("status", &self.status())
3510 .finish_non_exhaustive()
3511 }
3512}
3513
3514impl SupervisedModule {
3515 pub fn module_id(&self) -> &str {
3516 &self.inner.module_id
3517 }
3518
3519 #[cfg(test)]
3523 pub(crate) fn record_health_probe_failure_for_test(
3524 &self,
3525 detail: &str,
3526 ) -> Result<(), SuperviseError> {
3527 update_snapshot(&self.inner.snapshot, Some(&self.inner.module_id), |state| {
3528 state.health.consecutive_failures = state.health.consecutive_failures.saturating_add(1);
3529 state.health.detail = Some(detail.to_string());
3530 })
3531 }
3532
3533 pub fn state(&self) -> Result<ModuleState, SuperviseError> {
3534 Ok(lock_snapshot(&self.inner.snapshot)?.state)
3535 }
3536
3537 pub(crate) fn registration_end_reason(
3538 &self,
3539 ) -> Result<Option<RegistrationEndReason>, SuperviseError> {
3540 let snapshot = lock_snapshot(&self.inner.snapshot)?;
3541 Ok(match snapshot.state {
3542 ModuleState::Draining if snapshot.draining_to_replace => {
3543 Some(RegistrationEndReason::SupervisorRestart)
3544 }
3545 ModuleState::Draining => Some(RegistrationEndReason::SupervisorStop),
3546 ModuleState::Restarting if snapshot.draining_to_replace => {
3547 Some(RegistrationEndReason::SupervisorRestart)
3548 }
3549 _ => None,
3550 })
3551 }
3552
3553 pub fn stderr_tail(
3560 &self,
3561 max_lines: Option<usize>,
3562 max_bytes: Option<usize>,
3563 ) -> StderrTailSnapshot {
3564 self.inner
3565 .stderr_ring
3566 .lock()
3567 .unwrap_or_else(|poisoned| poisoned.into_inner())
3568 .snapshot(max_lines, max_bytes)
3569 }
3570
3571 pub fn terminal_history(&self) -> TerminalHistorySnapshot {
3576 self.inner
3577 .terminal_ring
3578 .lock()
3579 .unwrap_or_else(|poisoned| poisoned.into_inner())
3580 .snapshot()
3581 }
3582
3583 pub fn durable_terminal_history(&self) -> subc_control::TerminalHistory {
3588 durable_terminal_history_of(&self.inner.terminal_ring, &self.inner.module_id)
3589 }
3590
3591 pub(crate) async fn read_durable_terminal_history(
3596 &self,
3597 ) -> Result<subc_control::TerminalHistory, tokio::task::JoinError> {
3598 let terminal_ring = Arc::clone(&self.inner.terminal_ring);
3599 let module_id = self.inner.module_id.clone();
3600 tokio::task::spawn_blocking(move || durable_terminal_history_of(&terminal_ring, &module_id))
3601 .await
3602 }
3603
3604 pub fn status(&self) -> Result<ModuleStatus, SuperviseError> {
3605 self.status_with_snapshot_lock(&self.inner.snapshot, None)
3606 .map(|(status, _)| status)
3607 }
3608
3609 pub(crate) fn record_deliberate_severance(
3610 &self,
3611 identity: ProcessIdentity,
3612 ) -> Result<bool, SuperviseError> {
3613 let mut snapshot = lock_snapshot(&self.inner.snapshot)?;
3614 if snapshot.pid != Some(identity.pid)
3615 || snapshot.process_start_time != Some(identity.start_time)
3616 {
3617 return Ok(false);
3618 }
3619 snapshot.deliberate_severance = Some(identity);
3620 Ok(true)
3621 }
3622
3623 pub(crate) fn status_for_control(
3628 &self,
3629 caller: &'static str,
3630 ) -> Result<ModuleStatus, SuperviseError> {
3631 self.status_with_snapshot_lock(&self.inner.snapshot, Some(caller))
3632 .map(|(status, _)| status)
3633 }
3634
3635 fn status_with_snapshot_lock(
3636 &self,
3637 snapshot: &SharedSnapshot,
3638 caller: Option<&'static str>,
3639 ) -> Result<(ModuleStatus, Option<SpawnedFileIdentity>), SuperviseError> {
3640 let mut guard = match caller {
3641 Some(caller) => lock_snapshot_for_control(snapshot, &self.inner.module_id, caller)?,
3642 None => lock_snapshot(snapshot)?,
3643 };
3644 let restart_count =
3647 guard.crash_restarts_in_window(self.inner.restart_policy.window, Instant::now());
3648 let snapshot = guard.clone();
3649 drop(guard);
3650 let drain_timeout = *self.inner.effective_drain_timeout.lock().map_err(|_| {
3651 SuperviseError::StatePoisoned {
3652 module_id: Some(self.inner.module_id.clone()),
3653 }
3654 })?;
3655 let registration_active = self
3656 .inner
3657 .registry
3658 .get_module(&self.inner.module_id)
3659 .map_err(SuperviseError::Registry)?
3660 .is_some();
3661 let protocol = snapshot
3662 .spawned_protocol
3663 .unwrap_or(self.declared_protocol()?);
3664 let running_process =
3665 snapshot.enabled && snapshot.state == ModuleState::Running && snapshot.process_alive;
3666 let live = match protocol {
3672 ModuleProtocol::Subc => running_process && registration_active,
3673 ModuleProtocol::None => running_process,
3674 };
3675
3676 Ok((
3677 ModuleStatus {
3678 waiting_for_previous_pid: snapshot.waiting_for_previous_pid,
3679 waiting_for_previous_detail: snapshot.waiting_for_previous_detail.clone(),
3680 module_id: self.inner.module_id.clone(),
3681 launch_nonce_source: observed_launch_nonce_source(&snapshot),
3682 state: snapshot.state,
3683 enabled: snapshot.enabled,
3684 process_alive: snapshot.process_alive,
3685 registration_active,
3686 protocol,
3687 live,
3688 restart_count,
3689 lifetime_restarts: snapshot.lifetime_restarts,
3690 spawn_generation: snapshot.spawn_generation,
3691 max_restarts: self.inner.restart_policy.max_restarts,
3692 restart_window: self.inner.restart_policy.window,
3693 drain_timeout,
3694 restart_backoff: self.inner.restart_policy.backoff,
3695 restart_max_backoff: self.inner.restart_policy.max_backoff,
3696 pid: snapshot.reported_pid(),
3697 spawned_at_ms: snapshot.spawned_at_ms,
3698 spawned_from: snapshot.spawned_from,
3699 launch_spec_digest: snapshot.launch_spec_digest,
3700 process_start_time: snapshot.process_start_time,
3701 last_exit: snapshot.last_exit,
3702 health: snapshot.health,
3703 },
3704 snapshot.spawned_file_identity.clone(),
3705 ))
3706 }
3707
3708 #[cfg(test)]
3709 pub(crate) fn hold_snapshot_for_test(
3710 &self,
3711 acquired: std::sync::mpsc::Sender<()>,
3712 hold: Duration,
3713 ) -> std::thread::JoinHandle<()> {
3714 let snapshot = Arc::clone(&self.inner.snapshot);
3715 std::thread::spawn(move || {
3716 let _guard = snapshot.lock().expect("test snapshot lock is not poisoned");
3717 acquired
3718 .send(())
3719 .expect("test receiver waits for snapshot lock");
3720 std::thread::sleep(hold);
3721 })
3722 }
3723
3724 pub(crate) async fn status_and_running_image_agreement(
3730 &self,
3731 ) -> Result<(ModuleStatus, subc_control::RunningImageAgreement), SuperviseError> {
3732 let (status, identity) = self.status_with_snapshot_lock(&self.inner.snapshot, None)?;
3733 let image = self
3734 .inner
3735 .provenance_probe
3736 .observe(
3737 status.pid,
3738 status.spawned_from.as_deref(),
3739 identity,
3740 status.process_start_time,
3741 )
3742 .await;
3743 Ok((status, image))
3744 }
3745
3746 pub(crate) async fn running_image_agreement(&self) -> subc_control::RunningImageAgreement {
3747 let snapshot = match lock_snapshot(&self.inner.snapshot) {
3748 Ok(snapshot) => snapshot.clone(),
3749 Err(_) => {
3750 return subc_control::RunningImageAgreement::Unavailable {
3751 reason: subc_control::RunningImageUnavailableReason::NotRunning,
3752 };
3753 }
3754 };
3755 self.inner
3756 .provenance_probe
3757 .observe(
3758 snapshot.reported_pid(),
3759 snapshot.spawned_from.as_deref(),
3760 snapshot.spawned_file_identity.clone(),
3761 snapshot.process_start_time,
3762 )
3763 .await
3764 }
3765
3766 pub(crate) fn child_resource_usage(&self) -> subc_control::ChildResourceUsage {
3769 #[cfg(windows)]
3770 {
3771 let image = lock_snapshot(&self.inner.snapshot)
3772 .ok()
3773 .and_then(|snapshot| {
3774 snapshot
3775 .spawned_file_identity
3776 .as_ref()
3777 .and_then(|identity| identity.windows_image.clone())
3778 });
3779 match image {
3780 Some(image) => {
3781 crate::child_resources::read_windows_handle(image.process(), image.start_time())
3782 }
3783 None => subc_control::ChildResourceUsage::Unavailable {
3784 reason: subc_control::ChildResourceUnavailableReason::NotRunning,
3785 },
3786 }
3787 }
3788 #[cfg(not(windows))]
3789 {
3790 let (pid, start_time) = match lock_snapshot(&self.inner.snapshot) {
3791 Ok(snapshot) => (snapshot.reported_pid(), snapshot.process_start_time),
3792 Err(_) => {
3793 return subc_control::ChildResourceUsage::Unavailable {
3794 reason: subc_control::ChildResourceUnavailableReason::Unreadable,
3795 }
3796 }
3797 };
3798 crate::child_resources::read(pid, start_time)
3799 }
3800 }
3801
3802 pub(crate) fn will_recover_after_connection_loss(&self) -> Result<bool, SuperviseError> {
3803 let mut snapshot = lock_snapshot(&self.inner.snapshot)?;
3804 Ok(match snapshot.state {
3805 ModuleState::Restarting => true,
3806 ModuleState::Failed | ModuleState::Disabled => false,
3807 _ if snapshot.enabled && snapshot.respawn_pending => true,
3808 _ => daemon_will_restart(&mut snapshot, &self.inner.restart_policy, Instant::now()),
3809 })
3810 }
3811
3812 #[cfg(test)]
3813 pub(crate) fn is_warming(&self) -> Result<bool, SuperviseError> {
3814 self.is_warming_with_snapshot_lock(None)
3815 }
3816
3817 pub(crate) fn is_warming_for_control(
3818 &self,
3819 caller: &'static str,
3820 ) -> Result<bool, SuperviseError> {
3821 self.is_warming_with_snapshot_lock(Some(caller))
3822 }
3823
3824 fn is_warming_with_snapshot_lock(
3825 &self,
3826 caller: Option<&'static str>,
3827 ) -> Result<bool, SuperviseError> {
3828 let snapshot = match caller {
3829 Some(caller) => {
3830 lock_snapshot_for_control(&self.inner.snapshot, &self.inner.module_id, caller)?
3831 }
3832 None => lock_snapshot(&self.inner.snapshot)?,
3833 }
3834 .clone();
3835 Ok(matches!(
3836 snapshot.state,
3837 ModuleState::Starting | ModuleState::Running | ModuleState::Restarting
3838 ))
3839 }
3840
3841 pub async fn drain(&self) -> Result<(), SuperviseError> {
3843 self.stop().await
3844 }
3845
3846 pub(crate) async fn retire_requested(
3847 &self,
3848 requester: Option<Principal>,
3849 ) -> Result<(), SuperviseError> {
3850 match self.state()? {
3851 ModuleState::Stopped | ModuleState::Failed => return Ok(()),
3852 ModuleState::Starting
3853 | ModuleState::Running
3854 | ModuleState::Unresponsive
3855 | ModuleState::Restarting
3856 | ModuleState::Draining
3857 | ModuleState::Disabled => {}
3858 }
3859
3860 let (reply_tx, reply_rx) = oneshot::channel();
3861 self.inner
3862 .commands
3863 .send(SupervisorCommand::Retire {
3864 requester,
3865 reply: reply_tx,
3866 })
3867 .await
3868 .map_err(|_| SuperviseError::CommandClosed {
3869 module_id: self.inner.module_id.clone(),
3870 })?;
3871 reply_rx.await.map_err(|_| SuperviseError::CommandClosed {
3872 module_id: self.inner.module_id.clone(),
3873 })?
3874 }
3875
3876 pub async fn stop(&self) -> Result<(), SuperviseError> {
3877 self.stop_requested(None).await
3878 }
3879
3880 pub(crate) async fn stop_requested(
3881 &self,
3882 requester: Option<Principal>,
3883 ) -> Result<(), SuperviseError> {
3884 match self.state()? {
3885 ModuleState::Stopped | ModuleState::Failed => return Ok(()),
3886 ModuleState::Starting
3887 | ModuleState::Running
3888 | ModuleState::Unresponsive
3889 | ModuleState::Restarting
3890 | ModuleState::Draining
3891 | ModuleState::Disabled => {}
3892 }
3893
3894 let (reply_tx, reply_rx) = oneshot::channel();
3895 self.inner
3896 .commands
3897 .send(SupervisorCommand::Drain {
3898 requester,
3899 reply: reply_tx,
3900 })
3901 .await
3902 .map_err(|_| SuperviseError::CommandClosed {
3903 module_id: self.inner.module_id.clone(),
3904 })?;
3905 reply_rx.await.map_err(|_| SuperviseError::CommandClosed {
3906 module_id: self.inner.module_id.clone(),
3907 })?
3908 }
3909
3910 pub async fn restart(&self, drain_timeout_ms: Option<u64>) -> Result<(), SuperviseError> {
3911 self.restart_requested(drain_timeout_ms, None).await
3912 }
3913
3914 pub(crate) async fn restart_requested(
3915 &self,
3916 drain_timeout_ms: Option<u64>,
3917 requester: Option<Principal>,
3918 ) -> Result<(), SuperviseError> {
3919 let queued_at = Instant::now();
3920 let received_at_generation = lock_snapshot(&self.inner.snapshot)?.spawn_generation;
3921 let (reply_tx, reply_rx) = oneshot::channel();
3922 self.inner
3923 .commands
3924 .send(SupervisorCommand::Restart {
3925 requester,
3926 drain_timeout_ms,
3927 received_at_generation,
3928 queued_at,
3929 reply: reply_tx,
3930 })
3931 .await
3932 .map_err(|_| SuperviseError::CommandClosed {
3933 module_id: self.inner.module_id.clone(),
3934 })?;
3935 reply_rx.await.map_err(|_| SuperviseError::CommandClosed {
3936 module_id: self.inner.module_id.clone(),
3937 })?
3938 }
3939
3940 pub async fn swap(&self, ready_timeout: Option<Duration>) -> Result<(), SuperviseError> {
3945 self.swap_requested(ready_timeout, None).await
3946 }
3947
3948 pub(crate) async fn swap_requested(
3949 &self,
3950 ready_timeout: Option<Duration>,
3951 requester: Option<Principal>,
3952 ) -> Result<(), SuperviseError> {
3953 let (reply_tx, reply_rx) = oneshot::channel();
3954 self.inner
3955 .commands
3956 .send(SupervisorCommand::Swap {
3957 requester,
3958 ready_timeout,
3959 reply: reply_tx,
3960 })
3961 .await
3962 .map_err(|_| SuperviseError::CommandClosed {
3963 module_id: self.inner.module_id.clone(),
3964 })?;
3965 reply_rx.await.map_err(|_| SuperviseError::CommandClosed {
3966 module_id: self.inner.module_id.clone(),
3967 })?
3968 }
3969
3970 pub async fn reload(&self) -> Result<(), SuperviseError> {
3971 self.reload_requested(None).await
3972 }
3973
3974 pub(crate) async fn reload_requested(
3975 &self,
3976 requester: Option<Principal>,
3977 ) -> Result<(), SuperviseError> {
3978 let (reply_tx, reply_rx) = oneshot::channel();
3979 self.inner
3980 .commands
3981 .send(SupervisorCommand::Reload {
3982 requester,
3983 reply: reply_tx,
3984 })
3985 .await
3986 .map_err(|_| SuperviseError::CommandClosed {
3987 module_id: self.inner.module_id.clone(),
3988 })?;
3989 reply_rx.await.map_err(|_| SuperviseError::CommandClosed {
3990 module_id: self.inner.module_id.clone(),
3991 })?
3992 }
3993
3994 pub async fn set_enabled(&self, enabled: bool) -> Result<bool, SuperviseError> {
3995 self.set_enabled_requested(enabled, None).await
3996 }
3997
3998 pub(crate) async fn set_enabled_requested(
3999 &self,
4000 enabled: bool,
4001 requester: Option<Principal>,
4002 ) -> Result<bool, SuperviseError> {
4003 let (reply_tx, reply_rx) = oneshot::channel();
4004 self.inner
4005 .commands
4006 .send(SupervisorCommand::SetEnabled {
4007 requester,
4008 enabled,
4009 reply: reply_tx,
4010 })
4011 .await
4012 .map_err(|_| SuperviseError::CommandClosed {
4013 module_id: self.inner.module_id.clone(),
4014 })?;
4015 reply_rx.await.map_err(|_| SuperviseError::CommandClosed {
4016 module_id: self.inner.module_id.clone(),
4017 })?
4018 }
4019
4020 pub(crate) fn declared_protocol(&self) -> Result<ModuleProtocol, SuperviseError> {
4024 let configured = self
4025 .inner
4026 .configuration
4027 .lock()
4028 .map_err(|_| SuperviseError::StatePoisoned {
4029 module_id: Some(self.inner.module_id.clone()),
4030 })?
4031 .spec
4032 .protocol;
4033 let state = lock_snapshot(&self.inner.snapshot)?;
4034 Ok(state.spawned_protocol.unwrap_or(configured))
4035 }
4036
4037 pub(crate) fn configuration(&self) -> Result<(ModuleSpec, HealthConfig), SuperviseError> {
4038 let configuration =
4039 self.inner
4040 .configuration
4041 .lock()
4042 .map_err(|_| SuperviseError::StatePoisoned {
4043 module_id: Some(self.inner.module_id.clone()),
4044 })?;
4045 Ok((configuration.spec.clone(), configuration.health.clone()))
4046 }
4047
4048 #[cfg(any(test, feature = "test-support"))]
4052 pub async fn update_spec_for_test(&self, spec: ModuleSpec) -> Result<(), SuperviseError> {
4053 let (_, health) = self.configuration()?;
4054 let drain_timeout_ms = u64::try_from(
4055 self.inner
4056 .effective_drain_timeout
4057 .lock()
4058 .unwrap_or_else(|poisoned| poisoned.into_inner())
4059 .as_millis(),
4060 )
4061 .ok();
4062 self.update_configuration(spec, health, drain_timeout_ms)
4063 .await
4064 }
4065
4066 pub(crate) async fn update_configuration(
4067 &self,
4068 spec: ModuleSpec,
4069 health: HealthConfig,
4070 drain_timeout_ms: Option<u64>,
4071 ) -> Result<(), SuperviseError> {
4072 if spec.module_id != self.inner.module_id {
4073 return Err(SuperviseError::InvalidSpec {
4074 reason: "a supervised module's module_id cannot be changed".to_string(),
4075 });
4076 }
4077 validate_spec(&spec)?;
4078 let (reply_tx, reply_rx) = oneshot::channel();
4079 self.inner
4080 .commands
4081 .send(SupervisorCommand::UpdateConfiguration {
4082 spec: spec.clone(),
4083 health: health.clone(),
4084 drain_timeout_ms,
4085 reply: reply_tx,
4086 })
4087 .await
4088 .map_err(|_| SuperviseError::CommandClosed {
4089 module_id: self.inner.module_id.clone(),
4090 })?;
4091 reply_rx.await.map_err(|_| SuperviseError::CommandClosed {
4092 module_id: self.inner.module_id.clone(),
4093 })?;
4094 let mut configuration =
4095 self.inner
4096 .configuration
4097 .lock()
4098 .map_err(|_| SuperviseError::StatePoisoned {
4099 module_id: Some(self.inner.module_id.clone()),
4100 })?;
4101 configuration.spec = spec;
4102 configuration.health = health;
4103 Ok(())
4104 }
4105}
4106
4107impl Drop for SupervisedModuleInner {
4108 fn drop(&mut self) {
4109 let Ok(mut monitor) = self.monitor.lock() else {
4110 return;
4111 };
4112 if let Some(monitor) = monitor.as_ref().filter(|monitor| !monitor.is_finished()) {
4113 let _ = update_snapshot(&self.snapshot, Some(&self.module_id), |state| {
4114 state.state = ModuleState::Stopped;
4115 clear_current_process_facts(state);
4116 });
4117 monitor.abort();
4118 }
4119 let _ = monitor.take();
4120 }
4121}
4122
4123#[derive(Debug)]
4124enum SupervisorCommand {
4125 Drain {
4126 requester: Option<Principal>,
4127 reply: oneshot::Sender<Result<(), SuperviseError>>,
4128 },
4129 Retire {
4130 requester: Option<Principal>,
4131 reply: oneshot::Sender<Result<(), SuperviseError>>,
4132 },
4133 Restart {
4134 requester: Option<Principal>,
4135 drain_timeout_ms: Option<u64>,
4140 received_at_generation: u64,
4144 queued_at: Instant,
4147 reply: oneshot::Sender<Result<(), SuperviseError>>,
4148 },
4149 Reload {
4150 requester: Option<Principal>,
4151 reply: oneshot::Sender<Result<(), SuperviseError>>,
4152 },
4153 SetEnabled {
4154 requester: Option<Principal>,
4155 enabled: bool,
4156 reply: oneshot::Sender<Result<bool, SuperviseError>>,
4157 },
4158 UpdateConfiguration {
4159 spec: ModuleSpec,
4160 health: HealthConfig,
4161 drain_timeout_ms: Option<u64>,
4164 reply: oneshot::Sender<()>,
4165 },
4166 Swap {
4167 requester: Option<Principal>,
4168 ready_timeout: Option<Duration>,
4171 reply: oneshot::Sender<Result<(), SuperviseError>>,
4173 },
4174}
4175
4176#[derive(Debug)]
4177pub enum SuperviseError {
4178 InvalidSpec {
4179 reason: String,
4180 },
4181 Spawn {
4182 program: PathBuf,
4183 source: io::Error,
4184 cgroup_path: Option<PathBuf>,
4185 },
4186 Cgroup {
4187 module_id: String,
4188 source: io::Error,
4189 },
4190 LaunchNonce {
4193 reason: String,
4194 },
4195 Wait {
4196 module_id: String,
4197 source: io::Error,
4198 },
4199 Kill {
4200 module_id: String,
4201 source: io::Error,
4202 },
4203 Forwarding(ForwardingError),
4204 Registry(RegistryError),
4205 ReloadUnavailable {
4206 module_id: String,
4207 reason: String,
4208 },
4209 Disabled {
4214 module_id: String,
4215 },
4216 ReloadFailed {
4217 module_id: String,
4218 reason: String,
4219 },
4220 RegistrationStillActive {
4221 module_id: String,
4222 waited: Duration,
4223 },
4224 StatePoisoned {
4225 module_id: Option<String>,
4226 },
4227 CommandClosed {
4228 module_id: String,
4229 },
4230 SwapInProgress {
4234 module_id: String,
4235 },
4236 SwapRefused {
4238 module_id: String,
4239 reason: SwapRefusal,
4240 },
4241 SwapFailed {
4245 module_id: String,
4246 arm: SwapFailureArm,
4247 detail: String,
4248 candidate_exit: Option<ExitReport>,
4251 },
4252}
4253
4254#[derive(Debug, Clone, Copy, PartialEq, Eq)]
4256pub enum SwapRefusal {
4257 OverlapExclusive,
4259 NotRegistered,
4262 ProtocolNone,
4265 NotConfigured,
4268 AlreadySwapping,
4270}
4271
4272impl SwapRefusal {
4273 pub fn as_str(self) -> &'static str {
4274 match self {
4275 Self::OverlapExclusive => "overlap_exclusive",
4276 Self::NotRegistered => "not_registered",
4277 Self::ProtocolNone => "protocol_none",
4278 Self::NotConfigured => "not_configured",
4279 Self::AlreadySwapping => "already_swapping",
4280 }
4281 }
4282}
4283
4284#[derive(Debug, Clone, Copy, PartialEq, Eq)]
4287pub enum SwapFailureArm {
4288 SpawnFailed,
4290 NeverRegistered,
4292 NeverReady,
4294 CandidateExited,
4296 CandidateUnhealthy,
4298 Interrupted,
4302 CutoverLost,
4307}
4308
4309impl SwapFailureArm {
4310 pub fn as_str(self) -> &'static str {
4311 match self {
4312 Self::SpawnFailed => "spawn_failed",
4313 Self::NeverRegistered => "never_registered",
4314 Self::NeverReady => "never_ready",
4315 Self::CandidateExited => "candidate_exited",
4316 Self::CandidateUnhealthy => "candidate_unhealthy",
4317 Self::Interrupted => "interrupted",
4318 Self::CutoverLost => "cutover_lost",
4319 }
4320 }
4321}
4322
4323impl fmt::Display for SuperviseError {
4324 fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
4325 match self {
4326 Self::InvalidSpec { reason } => write!(f, "invalid module spec: {reason}"),
4327 Self::Spawn {
4328 program,
4329 source,
4330 cgroup_path: Some(cgroup_path),
4331 } => write!(
4332 f,
4333 "failed to place module in cgroup '{}' while spawning '{}': {source}",
4334 cgroup_path.display(),
4335 program.display()
4336 ),
4337 Self::Spawn {
4338 program,
4339 source,
4340 cgroup_path: None,
4341 } => write!(
4342 f,
4343 "failed to spawn module '{}': {source}",
4344 program.display()
4345 ),
4346 Self::Cgroup { module_id, source } => {
4347 write!(
4348 f,
4349 "failed to prepare cgroup for module '{module_id}': {source}"
4350 )
4351 }
4352 Self::LaunchNonce { reason } => {
4353 write!(
4354 f,
4355 "failed to generate reserved-module launch nonce: {reason}"
4356 )
4357 }
4358 Self::Wait { module_id, source } => {
4359 write!(f, "failed to wait for module '{module_id}': {source}")
4360 }
4361 Self::Kill { module_id, source } => {
4362 write!(f, "failed to kill module '{module_id}': {source}")
4363 }
4364 Self::Forwarding(err) => write!(f, "forwarding error: {err}"),
4365 Self::Registry(err) => write!(f, "registry error: {err}"),
4366 Self::ReloadUnavailable { module_id, reason } => {
4367 write!(f, "reload unavailable for module '{module_id}': {reason}")
4368 }
4369 Self::Disabled { module_id } => {
4370 write!(
4371 f,
4372 "module '{module_id}' is disabled; enable it before restart or reload"
4373 )
4374 }
4375 Self::ReloadFailed { module_id, reason } => {
4376 write!(f, "reload failed for module '{module_id}': {reason}")
4377 }
4378 Self::RegistrationStillActive { module_id, waited } => write!(
4379 f,
4380 "module '{module_id}' registration remained active after waiting {waited:?}"
4381 ),
4382 Self::StatePoisoned { module_id } => match module_id {
4383 Some(module_id) => {
4384 write!(f, "supervisor state for module '{module_id}' was poisoned")
4385 }
4386 None => write!(f, "supervisor state was poisoned"),
4387 },
4388 Self::CommandClosed { module_id } => {
4389 write!(
4390 f,
4391 "supervisor command channel for module '{module_id}' is closed"
4392 )
4393 }
4394 Self::SwapInProgress { module_id } => write!(
4395 f,
4396 "module '{module_id}' is being swapped; retry once the swap has cut over or failed, or stop the module to abort the swap"
4397 ),
4398 Self::SwapRefused { module_id, reason } => match reason {
4399 SwapRefusal::OverlapExclusive => write!(
4400 f,
4401 "module '{module_id}' is declared overlap: \"exclusive\" (the default): two processes of it must not run at once, so it cannot be swapped; use a plain restart, or declare overlap: \"safe\" in its config if it really tolerates a second process"
4402 ),
4403 SwapRefusal::NotRegistered => write!(
4404 f,
4405 "module '{module_id}' is not registered, so there is no serving process to keep while a replacement warms; use a plain restart"
4406 ),
4407 SwapRefusal::ProtocolNone => write!(
4408 f,
4409 "module '{module_id}' is protocol: \"none\" and never registers, so a swap could never see its replacement become ready; use a plain restart"
4410 ),
4411 SwapRefusal::NotConfigured => write!(
4412 f,
4413 "module '{module_id}' cannot be swapped: the supervisor was built without the forwarding table or shared handle a swap needs"
4414 ),
4415 SwapRefusal::AlreadySwapping => {
4416 write!(f, "module '{module_id}' is already being swapped")
4417 }
4418 },
4419 Self::SwapFailed {
4420 module_id,
4421 arm,
4422 detail,
4423 ..
4424 } => write!(
4425 f,
4426 "swap of module '{module_id}' failed ({}): {detail}; the running process was left serving",
4427 arm.as_str()
4428 ),
4429 }
4430 }
4431}
4432
4433impl Error for SuperviseError {
4434 fn source(&self) -> Option<&(dyn Error + 'static)> {
4435 match self {
4436 Self::Spawn { source, .. }
4437 | Self::Cgroup { source, .. }
4438 | Self::Wait { source, .. }
4439 | Self::Kill { source, .. } => Some(source),
4440 Self::Forwarding(err) => Some(err),
4441 Self::Registry(err) => Some(err),
4442 Self::LaunchNonce { .. }
4443 | Self::InvalidSpec { .. }
4444 | Self::ReloadUnavailable { .. }
4445 | Self::Disabled { .. }
4446 | Self::ReloadFailed { .. }
4447 | Self::RegistrationStillActive { .. }
4448 | Self::StatePoisoned { .. }
4449 | Self::CommandClosed { .. }
4450 | Self::SwapInProgress { .. }
4451 | Self::SwapRefused { .. }
4452 | Self::SwapFailed { .. } => None,
4453 }
4454 }
4455}
4456
4457pub(crate) fn validate_spec(spec: &ModuleSpec) -> Result<(), SuperviseError> {
4458 if spec.module_id.trim().is_empty() {
4459 return Err(SuperviseError::InvalidSpec {
4460 reason: "module_id must not be empty".to_string(),
4461 });
4462 }
4463
4464 Ok(())
4465}
4466
4467#[derive(Debug, Default)]
4468struct HealthProbeRuntime {
4469 spawn_generation: Option<u64>,
4470 configured_health: Option<HealthConfig>,
4471 registered_connection: Option<crate::ConnectionId>,
4472 advertised: bool,
4473 next_probe_at: Option<Instant>,
4474 probe_index: u64,
4475}
4476
4477fn running_protocol(spec: &ModuleSpec, snapshot: &SharedSnapshot) -> ModuleProtocol {
4478 lock_snapshot(snapshot)
4479 .ok()
4480 .and_then(|state| state.spawned_protocol)
4481 .unwrap_or(spec.protocol)
4482}
4483
4484impl HealthProbeRuntime {
4485 fn refresh_registration(
4486 &mut self,
4487 spec: &ModuleSpec,
4488 runtime: &SupervisorRuntimeConfig,
4489 registry: &Registry,
4490 snapshot: &SharedSnapshot,
4491 ) {
4492 let generation = lock_snapshot(snapshot)
4493 .ok()
4494 .map(|state| state.spawn_generation);
4495 if self.spawn_generation != generation {
4496 *self = Self {
4500 spawn_generation: generation,
4501 ..Self::default()
4502 };
4503 }
4504 if self.configured_health.as_ref() != Some(&runtime.health) {
4505 self.configured_health = Some(runtime.health.clone());
4506 self.next_probe_at = None;
4507 self.registered_connection = None;
4508 self.probe_index = 0;
4509 }
4510 if running_protocol(spec, snapshot) == ModuleProtocol::None {
4514 self.registered_connection = None;
4515 self.advertised = runtime.health.http.is_some();
4516 if !self.advertised {
4517 self.next_probe_at = None;
4518 let _ = update_snapshot(snapshot, Some(&spec.module_id), |state| {
4519 let unknown = ModuleHealthStatus::default();
4520 if state.health != unknown {
4521 state.health = unknown;
4522 }
4523 });
4524 } else if self.next_probe_at.is_none() {
4525 self.next_probe_at = Some(
4526 Instant::now()
4527 + jittered_health_delay(&spec.module_id, 0, runtime.health.cadence),
4528 );
4529 }
4530 return;
4531 }
4532
4533 let registration = match registry.get_module(&spec.module_id) {
4534 Ok(registration) => registration,
4535 Err(err) => {
4536 warn!(module_id = %spec.module_id, error = %err, "health prober could not read registry");
4537 self.advertised = false;
4538 self.next_probe_at = None;
4539 return;
4540 }
4541 };
4542
4543 let Some(registration) = registration else {
4544 self.registered_connection = None;
4545 self.advertised = false;
4546 self.next_probe_at = None;
4547 return;
4548 };
4549
4550 let advertised = registration
4551 .control_ops
4552 .iter()
4553 .any(|op| op == MODULE_CONTROL_OP_HEALTH_CHECK);
4554 if !advertised {
4555 self.registered_connection = Some(registration.connection_id);
4556 self.advertised = false;
4557 self.next_probe_at = None;
4558 let _ = update_snapshot(snapshot, Some(&spec.module_id), |state| {
4559 if state.health.status != SupervisorHealthStatus::Unknown
4560 || state.health.consecutive_failures != 0
4561 || state.health.last_probe_ms.is_some()
4562 || state.health.detail.is_some()
4563 || state.health.metrics.is_some()
4564 {
4565 state.health.status = SupervisorHealthStatus::Unknown;
4566 state.health.consecutive_failures = 0;
4567 state.health.last_probe_ms = None;
4568 state.health.detail = None;
4569 state.health.metrics = None;
4570 }
4571 });
4572 return;
4573 }
4574
4575 let reregistered = self.registered_connection != Some(registration.connection_id);
4576 self.registered_connection = Some(registration.connection_id);
4577 self.advertised = true;
4578 if reregistered || self.next_probe_at.is_none() {
4579 self.probe_index = 0;
4580 self.next_probe_at = Some(
4581 Instant::now() + jittered_health_delay(&spec.module_id, 0, runtime.health.cadence),
4582 );
4583 let _ = update_snapshot(snapshot, Some(&spec.module_id), |state| {
4584 state.health.status = SupervisorHealthStatus::Unknown;
4585 state.health.consecutive_failures = 0;
4586 state.health.detail = None;
4587 state.health.metrics = None;
4588 });
4589 }
4590 }
4591
4592 fn wake_after(&self) -> Option<Duration> {
4593 if !self.advertised {
4594 return None;
4595 }
4596 self.next_probe_at
4597 .map(|next| next.saturating_duration_since(Instant::now()))
4598 }
4599
4600 fn due(&self) -> bool {
4601 self.advertised
4602 && self
4603 .next_probe_at
4604 .is_some_and(|next| Instant::now() >= next)
4605 }
4606
4607 fn schedule_next(&mut self, spec: &ModuleSpec, cadence: Duration) {
4608 self.probe_index = self.probe_index.wrapping_add(1);
4609 self.next_probe_at = Some(
4610 Instant::now() + jittered_health_delay(&spec.module_id, self.probe_index, cadence),
4611 );
4612 }
4613}
4614
4615#[derive(Debug)]
4650enum HealthProbeEvidence {
4651 LaneDead,
4653 NoAnswer,
4655 BadAnswer,
4657 Misconfigured,
4659}
4660
4661#[derive(Debug)]
4662struct HealthProbeError {
4663 evidence: HealthProbeEvidence,
4664 message: String,
4665}
4666
4667impl HealthProbeError {
4668 fn lane_dead(message: impl Into<String>) -> Self {
4669 Self::with(HealthProbeEvidence::LaneDead, message)
4670 }
4671
4672 fn no_answer(message: impl Into<String>) -> Self {
4673 Self::with(HealthProbeEvidence::NoAnswer, message)
4674 }
4675
4676 fn bad_answer(message: impl Into<String>) -> Self {
4677 Self::with(HealthProbeEvidence::BadAnswer, message)
4678 }
4679
4680 fn misconfigured(message: impl Into<String>) -> Self {
4681 Self::with(HealthProbeEvidence::Misconfigured, message)
4682 }
4683
4684 fn with(evidence: HealthProbeEvidence, message: impl Into<String>) -> Self {
4685 Self {
4686 evidence,
4687 message: message.into(),
4688 }
4689 }
4690
4691 #[allow(dead_code)]
4705 fn is_proof_of_death(&self) -> bool {
4706 matches!(self.evidence, HealthProbeEvidence::LaneDead)
4707 }
4708
4709 fn label(&self) -> &'static str {
4717 match self.evidence {
4718 HealthProbeEvidence::LaneDead => "lane-dead",
4719 HealthProbeEvidence::NoAnswer => "no-answer",
4720 HealthProbeEvidence::BadAnswer => "bad-answer",
4721 HealthProbeEvidence::Misconfigured => "daemon-misconfigured",
4722 }
4723 }
4724}
4725
4726impl fmt::Display for HealthProbeError {
4727 fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
4728 f.write_str(&self.message)
4729 }
4730}
4731
4732async fn run_health_probe_cycle(
4733 spec: &ModuleSpec,
4734 runtime: &SupervisorRuntimeConfig,
4735 registry: &Registry,
4736 process_liveness: &SupervisorProcessLiveness,
4737 snapshot: &SharedSnapshot,
4738 child: &mut Option<SupervisedChild>,
4739) {
4740 let now_ms = unix_ms_now();
4741 let http = (running_protocol(spec, snapshot) == ModuleProtocol::None)
4742 .then_some(runtime.health.http.as_deref())
4743 .flatten();
4744 let result = match http {
4745 Some(url) => probe_http_health(url, runtime.health.deadline).await,
4746 None => probe_module_health(&spec.module_id, runtime, None).await,
4747 };
4748 match result {
4749 Ok(report) => {
4750 handle_health_report(
4751 spec,
4752 runtime,
4753 registry,
4754 process_liveness,
4755 snapshot,
4756 child,
4757 report,
4758 now_ms,
4759 )
4760 .await;
4761 }
4762 Err(err) => {
4763 if http.is_some() {
4764 let _ = update_snapshot(snapshot, Some(&spec.module_id), |state| {
4765 state.health.status = SupervisorHealthStatus::Failing;
4766 });
4767 }
4768 handle_health_probe_failure(
4769 spec,
4770 runtime,
4771 registry,
4772 process_liveness,
4773 snapshot,
4774 child,
4775 err,
4776 now_ms,
4777 )
4778 .await;
4779 }
4780 }
4781}
4782
4783pub(crate) struct HttpProbeTarget<'a> {
4784 address: std::net::SocketAddr,
4785 localhost: bool,
4786 authority: &'a str,
4787 path: String,
4788}
4789
4790pub(crate) fn parse_http_probe_url(url: &str) -> Result<HttpProbeTarget<'_>, String> {
4793 if url.bytes().any(|b| b <= b' ' || b == 127) || url.contains('#') {
4794 return Err("must not contain whitespace, controls, or a fragment".into());
4795 }
4796 let rest = url
4797 .strip_prefix("http://")
4798 .ok_or("must use plain http://")?;
4799 let split = rest.find(['/', '?']).unwrap_or(rest.len());
4800 let (authority, suffix) = rest.split_at(split);
4801 let (host, port) = if let Some(rest) = authority.strip_prefix("[::1]") {
4802 ("::1", rest)
4803 } else {
4804 let split = authority.find(':').unwrap_or(authority.len());
4805 authority.split_at(split)
4806 };
4807 let ip: std::net::IpAddr = match host {
4808 "127.0.0.1" | "localhost" => std::net::Ipv4Addr::LOCALHOST.into(),
4809 "::1" => std::net::Ipv6Addr::LOCALHOST.into(),
4810 _ => return Err("host must be 127.0.0.1, [::1], or localhost".into()),
4811 };
4812 let port = if port.is_empty() {
4813 80
4814 } else {
4815 port.strip_prefix(':')
4816 .and_then(|p| p.parse::<u16>().ok())
4817 .filter(|p| *p > 0)
4818 .ok_or("must have a valid nonzero TCP port")?
4819 };
4820 let path = if suffix.is_empty() {
4821 "/".into()
4822 } else if suffix.starts_with('?') {
4823 format!("/{suffix}")
4824 } else {
4825 suffix.into()
4826 };
4827 Ok(HttpProbeTarget {
4828 address: std::net::SocketAddr::new(ip, port),
4829 localhost: host == "localhost",
4830 authority,
4831 path,
4832 })
4833}
4834
4835async fn probe_http_health(
4836 url: &str,
4837 deadline: Duration,
4838) -> Result<HealthReport, HealthProbeError> {
4839 use tokio::io::{AsyncReadExt, AsyncWriteExt, BufReader};
4840 let target = parse_http_probe_url(url).map_err(HealthProbeError::misconfigured)?;
4841 let mut response_status = String::new();
4844 let mut body = Vec::new();
4845 let probe = async {
4846 let connection = match tokio::net::TcpStream::connect(target.address).await {
4849 Err(_) if target.localhost => {
4850 tokio::net::TcpStream::connect((
4851 std::net::Ipv6Addr::LOCALHOST,
4852 target.address.port(),
4853 ))
4854 .await
4855 }
4856 result => result,
4857 };
4858 let mut stream = connection.map_err(|error| {
4859 HealthProbeError::no_answer(format!("HTTP connect failed: {error}"))
4860 })?;
4861 stream.write_all(format!("GET {} HTTP/1.1\r\nHost: {}\r\nAccept-Encoding: identity\r\nConnection: close\r\n\r\n", target.path, target.authority).as_bytes()).await
4862 .map_err(|error| HealthProbeError::no_answer(format!("HTTP write failed: {error}")))?;
4863 let mut reader = BufReader::new(stream);
4864 let mut budget = 16 * 1024;
4865 let status = http_line(&mut reader, &mut budget).await?;
4866 let mut words = status.split_ascii_whitespace();
4867 let version = words.next();
4868 let code = words
4869 .next()
4870 .filter(|word| word.len() == 3)
4871 .and_then(|word| word.parse::<u16>().ok());
4872 if !matches!(version, Some("HTTP/1.1" | "HTTP/1.0"))
4873 || !code.is_some_and(|code| (100..600).contains(&code))
4874 {
4875 return Err(HealthProbeError::bad_answer(format!(
4876 "invalid HTTP status: {status}"
4877 )));
4878 }
4879 let code = code.expect("validated status code");
4880 response_status = status.clone();
4881 let mut length = None;
4882 let mut chunked = false;
4883 loop {
4884 let line = http_line(&mut reader, &mut budget).await?;
4885 if line.is_empty() {
4886 break;
4887 }
4888 if let Some((name, value)) = line.split_once(':') {
4889 if name.eq_ignore_ascii_case("content-length") {
4890 length = Some(value.trim().parse::<u64>().map_err(|_| {
4891 HealthProbeError::bad_answer("invalid HTTP Content-Length")
4892 })?);
4893 } else if name.eq_ignore_ascii_case("transfer-encoding") {
4894 chunked = value.trim().eq_ignore_ascii_case("chunked");
4895 }
4896 }
4897 }
4898 if chunked {
4899 while body.len() < 200 {
4900 let line = http_line(&mut reader, &mut budget).await?;
4901 let size = u64::from_str_radix(line.split(';').next().unwrap_or("").trim(), 16)
4902 .map_err(|_| HealthProbeError::bad_answer("invalid HTTP chunk size"))?;
4903 if size == 0 {
4904 break;
4905 }
4906 let count = size.min((200 - body.len()) as u64) as usize;
4907 let start = body.len();
4908 (&mut reader)
4909 .take(count as u64)
4910 .read_to_end(&mut body)
4911 .await
4912 .map_err(|error| {
4913 HealthProbeError::no_answer(format!("HTTP body read failed: {error}"))
4914 })?;
4915 if body.len() - start != count {
4916 return Err(HealthProbeError::bad_answer("truncated HTTP chunk"));
4917 }
4918 if size > count as u64 || body.len() == 200 {
4919 break;
4920 }
4921 if !http_line(&mut reader, &mut budget).await?.is_empty() {
4922 return Err(HealthProbeError::bad_answer("invalid HTTP chunk delimiter"));
4923 }
4924 }
4925 } else {
4926 reader
4927 .take(length.unwrap_or(200).min(200))
4928 .read_to_end(&mut body)
4929 .await
4930 .map_err(|error| {
4931 HealthProbeError::no_answer(format!("HTTP body read failed: {error}"))
4932 })?;
4933 }
4934 if (200..300).contains(&code) {
4935 Ok(HealthReport::ok())
4936 } else {
4937 Err(HealthProbeError::bad_answer(
4938 "HTTP health endpoint returned non-2xx",
4939 ))
4940 }
4941 };
4942 let mut result = timeout(deadline, probe).await.unwrap_or_else(|_| {
4943 Err(HealthProbeError::no_answer(format!(
4944 "HTTP probe timed out after {deadline:?}"
4945 )))
4946 });
4947 if let Err(error) = &mut result {
4948 if !response_status.is_empty() {
4949 error.message = format!(
4950 "{}; {response_status}: {}",
4951 error.message,
4952 String::from_utf8_lossy(&body)
4953 );
4954 }
4955 }
4956 result
4957}
4958
4959async fn http_line(
4960 reader: &mut tokio::io::BufReader<tokio::net::TcpStream>,
4961 remaining: &mut usize,
4962) -> Result<String, HealthProbeError> {
4963 use tokio::io::{AsyncBufReadExt, AsyncReadExt};
4964 let mut line = Vec::new();
4965 (&mut *reader)
4966 .take(*remaining as u64)
4967 .read_until(b'\n', &mut line)
4968 .await
4969 .map_err(|error| {
4970 HealthProbeError::no_answer(format!("HTTP header read failed: {error}"))
4971 })?;
4972 *remaining -= line.len();
4973 if !line.ends_with(b"\r\n") {
4974 return Err(HealthProbeError::bad_answer(
4975 "HTTP headers are incomplete or exceed 16 KiB",
4976 ));
4977 }
4978 line.truncate(line.len() - 2);
4979 String::from_utf8(line).map_err(|_| HealthProbeError::bad_answer("HTTP header is not UTF-8"))
4980}
4981
4982async fn probe_module_health(
4983 module_id: &str,
4984 runtime: &SupervisorRuntimeConfig,
4985 drain_deadline: Option<Instant>,
4986) -> Result<HealthReport, HealthProbeError> {
4987 let Some(forwarding) = runtime.forwarding.as_ref() else {
4988 return Err(HealthProbeError::misconfigured(
4989 "supervisor was not configured with a forwarding table",
4990 ));
4991 };
4992 let probe_started_at = Instant::now();
4993 let mut deadline = probe_started_at + runtime.health.deadline;
4994 if let Some(drain_deadline) = drain_deadline {
4995 deadline = deadline.min(drain_deadline);
4996 }
4997 let pending = if drain_deadline.is_some() {
4998 forwarding.begin_drain_health_probe_rpc_for(
4999 module_id,
5000 MODULE_CONTROL_OP_HEALTH_CHECK,
5001 probe_started_at,
5002 deadline,
5003 )
5004 } else {
5005 forwarding.begin_health_probe_rpc_for(
5006 module_id,
5007 MODULE_CONTROL_OP_HEALTH_CHECK,
5008 probe_started_at,
5009 deadline,
5010 )
5011 }
5012 .map_err(|err| {
5013 HealthProbeError::lane_dead(format!("failed to begin health.check RPC: {err}"))
5016 })?;
5017 await_health_probe(forwarding, pending, deadline, runtime.health.deadline).await
5018}
5019
5020async fn probe_endpoint_health(
5027 endpoint: crate::ModuleEndpointId,
5028 runtime: &SupervisorRuntimeConfig,
5029 deadline_cap: Option<Instant>,
5030) -> Result<HealthReport, HealthProbeError> {
5031 let Some(forwarding) = runtime.forwarding.as_ref() else {
5032 return Err(HealthProbeError::misconfigured(
5033 "supervisor was not configured with a forwarding table",
5034 ));
5035 };
5036 let probe_started_at = Instant::now();
5037 let mut deadline = probe_started_at + runtime.health.deadline;
5038 if let Some(cap) = deadline_cap {
5039 deadline = deadline.min(cap);
5040 }
5041 let pending = forwarding
5042 .begin_endpoint_health_probe_rpc_for(
5043 endpoint,
5044 MODULE_CONTROL_OP_HEALTH_CHECK,
5045 probe_started_at,
5046 deadline,
5047 )
5048 .map_err(|err| {
5049 HealthProbeError::lane_dead(format!("failed to begin health.check RPC: {err}"))
5050 })?;
5051 await_health_probe(forwarding, pending, deadline, runtime.health.deadline).await
5052}
5053
5054async fn await_health_probe(
5056 forwarding: &ForwardingTable,
5057 pending: PendingModuleControlRpc,
5058 deadline: Instant,
5059 probe_budget: Duration,
5060) -> Result<HealthReport, HealthProbeError> {
5061 let PendingModuleControlRpc {
5062 endpoint,
5063 module_sink,
5064 negotiated_ver,
5065 corr,
5066 receiver,
5067 } = pending;
5068 let body = serde_json::to_vec(&ModuleControlRequest::HealthCheck {}).map_err(|err| {
5069 HealthProbeError::misconfigured(format!("failed to encode health.check: {err}"))
5070 })?;
5071 let frame = Frame::build_with_version(
5072 negotiated_ver,
5073 FrameType::Request,
5074 control_flags(),
5075 0,
5076 0,
5077 corr,
5078 body,
5079 )
5080 .map_err(|err| {
5081 HealthProbeError::misconfigured(format!("failed to build health.check frame: {err}"))
5082 })?;
5083
5084 match timeout_at(deadline, module_sink.send(frame)).await {
5090 Ok(Ok(())) => {}
5091 Ok(Err(err)) => {
5092 let _ = forwarding.cancel_module_control_rpc(endpoint, corr);
5093 return Err(HealthProbeError::lane_dead(format!(
5096 "failed to send health.check: {err}"
5097 )));
5098 }
5099 Err(_elapsed) => {
5100 let _ = forwarding.cancel_module_control_rpc(endpoint, corr);
5101 return Err(HealthProbeError::no_answer(
5105 "health.check send timed out before enqueue (module egress full)",
5106 ));
5107 }
5108 }
5109
5110 match timeout_at(deadline, receiver).await {
5111 Ok(Ok(ModuleControlRpcOutcome::Response(response))) => {
5115 response.health_report().ok_or_else(|| {
5116 HealthProbeError::bad_answer("health.check RPC returned a non-health response")
5117 })
5118 }
5119 Ok(Ok(ModuleControlRpcOutcome::Rejected(body))) => Err(HealthProbeError::bad_answer(
5120 format!("health.check rejected: {}", body.message),
5121 )),
5122 Ok(Ok(ModuleControlRpcOutcome::ModuleGone(message))) => {
5123 Err(HealthProbeError::lane_dead(message))
5124 }
5125 Ok(Ok(ModuleControlRpcOutcome::MalformedResponse(message))) => {
5126 Err(HealthProbeError::bad_answer(message))
5127 }
5128 Ok(Ok(ModuleControlRpcOutcome::UnexpectedOp { expected, actual })) => {
5129 Err(HealthProbeError::bad_answer(format!(
5130 "expected module-control op '{expected}', got '{actual}'"
5131 )))
5132 }
5133 Ok(Ok(ModuleControlRpcOutcome::DeadlineElapsed)) => Err(HealthProbeError::bad_answer(
5137 "module answered health.check after its daemon deadline",
5138 )),
5139 Ok(Err(_)) => Err(HealthProbeError::misconfigured(
5140 "health.check waiter was canceled before the module responded",
5141 )),
5142 Err(_) => {
5143 let _ = forwarding.tombstone_health_probe_rpc(endpoint, corr);
5144 Err(HealthProbeError::no_answer(format!(
5145 "module did not answer health.check within {probe_budget:?}"
5146 )))
5147 }
5148 }
5149}
5150
5151#[allow(clippy::too_many_arguments)]
5152async fn handle_health_report(
5153 spec: &ModuleSpec,
5154 runtime: &SupervisorRuntimeConfig,
5155 registry: &Registry,
5156 process_liveness: &SupervisorProcessLiveness,
5157 snapshot: &SharedSnapshot,
5158 child: &mut Option<SupervisedChild>,
5159 report: HealthReport,
5160 now_ms: u64,
5161) {
5162 let status = supervisor_health_status(report.status);
5163 let detail = report.detail.clone();
5164 let metrics = truncate_health_metrics(report.metrics);
5165 let _ = update_snapshot(snapshot, Some(&spec.module_id), |state| {
5166 state.health.status = status;
5167 state.health.last_probe_ms = Some(now_ms);
5168 state.health.detail = detail.clone();
5169 state.health.metrics = metrics.clone();
5170 state.health.consecutive_failures = 0;
5171 });
5172
5173 let action = match report.status {
5174 HealthStatus::Ok => return,
5175 HealthStatus::Degraded => runtime.health.on_degraded,
5176 HealthStatus::Failing => runtime.health.on_failing,
5177 };
5178 apply_l3_health_action(
5179 spec,
5180 runtime,
5181 registry,
5182 process_liveness,
5183 snapshot,
5184 child,
5185 status,
5186 detail.as_deref(),
5187 action,
5188 now_ms,
5189 )
5190 .await;
5191}
5192
5193#[allow(clippy::too_many_arguments)]
5194async fn handle_health_probe_failure(
5195 spec: &ModuleSpec,
5196 runtime: &SupervisorRuntimeConfig,
5197 registry: &Registry,
5198 process_liveness: &SupervisorProcessLiveness,
5199 snapshot: &SharedSnapshot,
5200 child: &mut Option<SupervisedChild>,
5201 err: HealthProbeError,
5202 now_ms: u64,
5203) {
5204 let threshold = runtime.health.failure_threshold.max(1);
5205 let mut failures = 0;
5206 let detail = format!("[{}] {err}", err.label());
5211 let _ = update_snapshot(snapshot, Some(&spec.module_id), |state| {
5212 if state.spawned_protocol.unwrap_or(spec.protocol) == ModuleProtocol::Subc {
5215 state.health.status = SupervisorHealthStatus::Unknown;
5216 }
5217 state.health.last_probe_ms = Some(now_ms);
5218 state.health.consecutive_failures = state.health.consecutive_failures.saturating_add(1);
5219 state.health.detail = Some(detail.clone());
5220 state.health.metrics = None;
5221 failures = state.health.consecutive_failures;
5222 });
5223
5224 if failures < threshold {
5225 warn!(
5226 module_id = %spec.module_id,
5227 consecutive_failures = failures,
5228 threshold,
5229 evidence = err.label(),
5230 detail = %detail,
5231 "health.check probe failed"
5232 );
5233 return;
5234 }
5235
5236 let _ = update_snapshot(snapshot, Some(&spec.module_id), |state| {
5237 state.state = ModuleState::Unresponsive;
5238 state.health.status = SupervisorHealthStatus::Unresponsive;
5239 });
5240 if runtime.health.critical {
5244 error!(
5245 module_id = %spec.module_id,
5246 status = "unresponsive",
5247 evidence = err.label(),
5248 detail = %detail,
5249 "critical module health alert"
5250 );
5251 } else {
5252 warn!(
5253 module_id = %spec.module_id,
5254 status = "unresponsive",
5255 evidence = err.label(),
5256 detail = %detail,
5257 "module health threshold breached"
5258 );
5259 }
5260 if let Err(err) = health_restart_child(
5261 spec,
5262 runtime,
5263 registry,
5264 process_liveness,
5265 snapshot,
5266 child,
5267 SupervisorHealthStatus::Unresponsive,
5268 Some(&detail),
5269 now_ms,
5270 )
5271 .await
5272 {
5273 error!(module_id = %spec.module_id, error = %err, "health-triggered restart failed");
5274 }
5275}
5276
5277#[allow(clippy::too_many_arguments)]
5278async fn apply_l3_health_action(
5279 spec: &ModuleSpec,
5280 runtime: &SupervisorRuntimeConfig,
5281 registry: &Registry,
5282 process_liveness: &SupervisorProcessLiveness,
5283 snapshot: &SharedSnapshot,
5284 child: &mut Option<SupervisedChild>,
5285 status: SupervisorHealthStatus,
5286 detail: Option<&str>,
5287 action: HealthAction,
5288 now_ms: u64,
5289) {
5290 record_health_action(snapshot, &spec.module_id, action.to_string(), now_ms);
5291 match action {
5292 HealthAction::Report => {
5293 info!(
5294 module_id = %spec.module_id,
5295 status = ?status,
5296 detail,
5297 "module reported non-ok health"
5298 );
5299 }
5300 HealthAction::Alert => {
5301 error!(
5302 module_id = %spec.module_id,
5303 status = ?status,
5304 detail,
5305 "module health alert"
5306 );
5307 }
5308 HealthAction::Restart => {
5309 if let Err(err) = health_restart_child(
5310 spec,
5311 runtime,
5312 registry,
5313 process_liveness,
5314 snapshot,
5315 child,
5316 status,
5317 detail,
5318 now_ms,
5319 )
5320 .await
5321 {
5322 error!(module_id = %spec.module_id, error = %err, "health-triggered restart failed");
5323 }
5324 }
5325 }
5326}
5327
5328#[allow(clippy::too_many_arguments)]
5329async fn health_restart_child(
5330 spec: &ModuleSpec,
5331 runtime: &SupervisorRuntimeConfig,
5332 registry: &Registry,
5333 process_liveness: &SupervisorProcessLiveness,
5334 snapshot: &SharedSnapshot,
5335 child: &mut Option<SupervisedChild>,
5336 status: SupervisorHealthStatus,
5337 detail: Option<&str>,
5338 now_ms: u64,
5339) -> Result<(), SuperviseError> {
5340 let (enabled, schedule) = {
5341 let mut state = lock_snapshot(snapshot)?;
5342 let enabled = state.enabled;
5343 let schedule = if enabled {
5344 state.next_crash_restart(&runtime.restart_policy, Instant::now())
5345 } else {
5346 None
5347 };
5348 (enabled, schedule)
5349 };
5350
5351 if !enabled {
5352 return Err(SuperviseError::Disabled {
5353 module_id: spec.module_id.clone(),
5354 });
5355 }
5356
5357 if schedule.is_none() {
5358 record_health_action(snapshot, &spec.module_id, "failed".to_string(), now_ms);
5359 error!(
5360 module_id = %spec.module_id,
5361 status = ?status,
5362 detail,
5363 max_restarts = runtime.restart_policy.max_restarts,
5364 window_secs = runtime.restart_policy.window.as_secs(),
5365 reason = %runtime.restart_policy.budget_exhausted_detail(),
5366 "health restart budget exhausted; marking module failed"
5367 );
5368 let stop_notice = begin_forwarding_drain_if_configured(
5369 spec,
5370 runtime,
5371 registry,
5372 snapshot,
5373 Some(true),
5374 RouteCloseReason::Disable,
5375 )
5376 .await?;
5377 update_snapshot(snapshot, Some(&spec.module_id), |state| {
5378 state.drain_disposition_detail = Some(runtime.restart_policy.budget_exhausted_detail());
5379 })?;
5380 drain_optional_child(
5381 &spec.module_id,
5382 spec.protocol,
5383 stop_notice,
5384 registry,
5385 runtime.forwarding.as_deref(),
5386 snapshot,
5387 &runtime.terminal_ring,
5388 &runtime.spawn_events,
5389 child,
5390 runtime.drain_timeout,
5391 ModuleState::Failed,
5392 Some(true),
5393 )
5394 .await?;
5395 process_liveness.untrack_if_current(&spec.module_id, snapshot);
5396 return Ok(());
5397 }
5398
5399 let schedule = schedule.expect("a health restart must have a crash-restart schedule");
5400 let mut restart_count = 0;
5401 update_snapshot(snapshot, Some(&spec.module_id), |state| {
5402 restart_count = state.crash_restarts.len();
5403 state.state = ModuleState::Unresponsive;
5404 state.health.status = status;
5405 state.health.last_action = Some(HealthAction::Restart.to_string());
5406 state.health.last_action_ms = Some(now_ms);
5407 })?;
5408 warn!(
5409 module_id = %spec.module_id,
5410 status = ?status,
5411 detail,
5412 restart_count,
5413 restart_in_window = schedule.restart_in_window,
5414 delay_ms = schedule.delay.as_millis() as u64,
5415 "health-triggered module restart"
5416 );
5417
5418 let stop_notice = begin_forwarding_drain_if_configured(
5419 spec,
5420 runtime,
5421 registry,
5422 snapshot,
5423 Some(true),
5424 RouteCloseReason::Restart,
5425 )
5426 .await?;
5427 drain_optional_child(
5428 &spec.module_id,
5429 spec.protocol,
5430 stop_notice,
5431 registry,
5432 runtime.forwarding.as_deref(),
5433 snapshot,
5434 &runtime.terminal_ring,
5435 &runtime.spawn_events,
5436 child,
5437 runtime.drain_timeout,
5438 ModuleState::Restarting,
5439 Some(true),
5440 )
5441 .await?;
5442 schedule_respawn(
5443 runtime,
5444 snapshot,
5445 &spec.module_id,
5446 schedule.delay,
5447 RespawnKind::Spawn,
5448 )
5449}
5450
5451fn cancel_deferred_reload(runtime: &SupervisorRuntimeConfig, module_id: &str, reason: &str) {
5452 if let Some(reply) = runtime
5453 .deferred_reload_reply
5454 .lock()
5455 .unwrap_or_else(|p| p.into_inner())
5456 .take()
5457 {
5458 let _ = reply.send(Err(SuperviseError::ReloadFailed {
5459 module_id: module_id.to_string(),
5460 reason: reason.to_string(),
5461 }));
5462 }
5463}
5464
5465fn schedule_respawn(
5466 runtime: &SupervisorRuntimeConfig,
5467 snapshot: &SharedSnapshot,
5468 module_id: &str,
5469 delay: Duration,
5470 kind: RespawnKind,
5471) -> Result<(), SuperviseError> {
5472 cancel_deferred_reload(runtime, module_id, "respawn superseded by another restart");
5473 update_snapshot(snapshot, Some(module_id), |state| {
5474 state.respawn_pending = true
5475 })?;
5476 *runtime
5477 .scheduled_respawn
5478 .lock()
5479 .unwrap_or_else(|p| p.into_inner()) = Some(PendingRespawn {
5480 deadline: Instant::now() + delay,
5481 kind,
5482 });
5483 Ok(())
5484}
5485
5486fn record_health_action(snapshot: &SharedSnapshot, module_id: &str, action: String, now_ms: u64) {
5487 let _ = update_snapshot(snapshot, Some(module_id), |state| {
5488 state.health.last_action = Some(action);
5489 state.health.last_action_ms = Some(now_ms);
5490 });
5491}
5492
5493fn supervisor_health_status(status: HealthStatus) -> SupervisorHealthStatus {
5494 match status {
5495 HealthStatus::Ok => SupervisorHealthStatus::Ok,
5496 HealthStatus::Degraded => SupervisorHealthStatus::Degraded,
5497 HealthStatus::Failing => SupervisorHealthStatus::Failing,
5498 }
5499}
5500
5501fn truncate_health_metrics(metrics: Option<Value>) -> Option<Value> {
5513 let metrics = metrics?;
5514 match serde_json::to_vec(&metrics) {
5515 Ok(encoded) if encoded.len() > MAX_HEALTH_METRICS_BYTES => Some(serde_json::json!({
5516 "truncated": true,
5517 "original_bytes": encoded.len(),
5518 })),
5519 Ok(_) | Err(_) => Some(metrics),
5520 }
5521}
5522
5523fn jittered_health_delay(module_id: &str, probe_index: u64, cadence: Duration) -> Duration {
5529 if cadence.is_zero() {
5530 return Duration::ZERO;
5531 }
5532 let cadence_ms = cadence.as_millis() as u64;
5533 if cadence_ms == 0 {
5549 return cadence;
5550 }
5551 let jitter_span = (cadence_ms / 10).max(1);
5566 let hash = module_id.as_bytes().iter().fold(
5567 probe_index.wrapping_mul(0x9E37_79B9_7F4A_7C15),
5568 |acc, byte| {
5569 acc.wrapping_mul(1099511628211)
5570 .wrapping_add(u64::from(*byte))
5571 },
5572 );
5573 cadence + Duration::from_millis(hash % jitter_span)
5574}
5575
5576#[cfg(test)]
5577mod tests {
5578 use super::*;
5579
5580 #[test]
5581 fn readding_a_module_clears_its_rescan_removal_tombstone() {
5582 let handle = SupervisorHandle::new();
5583 let module_id = "readded-tombstone";
5584 handle.record_rescan_removal(module_id);
5585 assert!(handle.removal_tombstone_age_ms(module_id).is_some());
5586
5587 handle.apply_identity_configuration(&ModuleSpec {
5588 module_id: module_id.to_string(),
5589 program: PathBuf::from("/test/module"),
5590 args: Vec::new(),
5591 env: Vec::new(),
5592 reserved: false,
5593 reserved_prefixes: Vec::new(),
5594 protocol: ModuleProtocol::Subc,
5595 overlap: Default::default(),
5596 });
5597
5598 assert!(
5599 handle.removal_tombstone_age_ms(module_id).is_none(),
5600 "a re-added module must not retain a stale removal tombstone"
5601 );
5602 }
5603
5604 #[derive(Debug, PartialEq, Eq)]
5607 struct OwnerInSpawnWindow {
5608 module_id: String,
5609 configured: bool,
5610 on_roster: bool,
5611 admission_refusal: Option<&'static str>,
5612 }
5613
5614 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
5625 async fn a_new_module_is_configured_before_its_first_process_can_register() {
5626 use crate::scopes::ScopeTable;
5627 use subc_protocol::{error_codes, scope::ScopeSelector, Principal};
5628
5629 let dir = cortexkit_test_support::ScratchDir::new("configured-before-spawn");
5630 let stub = |module_id: &str, program: PathBuf| ModuleSpec {
5631 module_id: module_id.to_string(),
5632 program,
5633 args: Vec::new(),
5634 env: ["XDG_DATA_HOME", "XDG_RUNTIME_DIR", "XDG_CONFIG_HOME"]
5635 .into_iter()
5636 .map(|key| (key.to_string(), dir.path().display().to_string()))
5637 .chain([("FAKE_AFT_NEVER_CONNECT".to_string(), "1".to_string())])
5638 .collect(),
5639 reserved: false,
5640 reserved_prefixes: Vec::new(),
5641 protocol: ModuleProtocol::Subc,
5642 overlap: Default::default(),
5643 };
5644 let live = super::terminal_history_tests::fake_aft_stub_path();
5645 let missing = dir.path().join("definitely-missing-module");
5646
5647 let handle = SupervisorHandle::new();
5648 let mut supervisor =
5649 Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default())
5650 .with_handle(handle.clone());
5651 let observed = Arc::new(Mutex::new(Vec::<OwnerInSpawnWindow>::new()));
5652 let hook_handle = handle.clone();
5653 let hook_observed = Arc::clone(&observed);
5654 supervisor.test_after_first_spawn = AfterFirstSpawnHook(Some(Arc::new(move |module_id| {
5655 let configured = hook_handle.is_configured(module_id);
5658 let selector = ScopeSelector::new(
5659 Principal::Reserved {
5660 module_id: module_id.to_string(),
5661 },
5662 "s".to_string(),
5663 Some(1),
5664 );
5665 let carrier = Principal::Reserved {
5666 module_id: "carrier".to_string(),
5667 };
5668 let admission_refusal = match ScopeTable::new(Vec::<String>::new())
5669 .admit(&carrier, module_id, &selector, configured)
5670 {
5671 Ok(_) => None,
5672 Err(refusal) => Some(refusal.code),
5673 };
5674 hook_observed.lock().unwrap().push(OwnerInSpawnWindow {
5675 module_id: module_id.to_string(),
5676 configured,
5677 on_roster: hook_handle.get(module_id).is_some(),
5678 admission_refusal,
5679 });
5680 })));
5681
5682 let plain = supervisor.spawn(stub("plain", live.clone())).unwrap();
5683 let configured = supervisor
5684 .supervise_configured(stub("configured", live.clone()), true)
5685 .unwrap();
5686 let with_health = supervisor
5687 .supervise_configured_with_health(
5688 stub("with-health", live.clone()),
5689 true,
5690 HealthConfig::default(),
5691 None,
5692 RestartPolicy::default(),
5693 )
5694 .unwrap();
5695 let failed = supervisor
5698 .supervise_configured_with_health(
5699 stub("failed-spawn", missing.clone()),
5700 true,
5701 HealthConfig::default(),
5702 None,
5703 RestartPolicy::default(),
5704 )
5705 .unwrap();
5706 assert!(supervisor.spawn(stub("spawn-error", missing)).is_err());
5709
5710 let expected = [
5711 "plain",
5712 "configured",
5713 "with-health",
5714 "failed-spawn",
5715 "spawn-error",
5716 ]
5717 .into_iter()
5718 .map(|module_id| OwnerInSpawnWindow {
5719 module_id: module_id.to_string(),
5720 configured: true,
5721 on_roster: !matches!(module_id, "plain" | "spawn-error"),
5722 admission_refusal: Some(error_codes::SCOPE_NOT_SYNCED),
5723 })
5724 .collect::<Vec<_>>();
5725 assert_eq!(*observed.lock().unwrap(), expected);
5726
5727 for module_id in ["plain", "configured", "with-health", "failed-spawn"] {
5728 assert!(
5729 handle.get(module_id).is_some(),
5730 "{module_id} is on the roster"
5731 );
5732 assert!(
5733 handle.is_configured(module_id),
5734 "{module_id} stays configured"
5735 );
5736 }
5737 assert!(handle.get("spawn-error").is_none());
5738 assert!(
5739 !handle.is_configured("spawn-error"),
5740 "a plain spawn that failed must not leave its module marked configured"
5741 );
5742
5743 handle.retire("failed-spawn");
5745 assert!(!handle.is_configured("failed-spawn"));
5746
5747 for module in [plain, configured, with_health] {
5748 module.stop().await.unwrap();
5749 }
5750 drop(failed);
5751 }
5752
5753 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
5760 async fn a_configured_module_reads_starting_while_its_first_process_already_runs() {
5761 let dir = cortexkit_test_support::ScratchDir::new("starting-while-spawned");
5762 let spec = ModuleSpec {
5763 module_id: "spawned-not-admitted".to_string(),
5764 program: super::terminal_history_tests::fake_aft_stub_path(),
5765 args: Vec::new(),
5766 env: ["XDG_DATA_HOME", "XDG_RUNTIME_DIR", "XDG_CONFIG_HOME"]
5767 .into_iter()
5768 .map(|key| (key.to_string(), dir.path().display().to_string()))
5769 .chain([("FAKE_AFT_NEVER_CONNECT".to_string(), "1".to_string())])
5770 .collect(),
5771 reserved: false,
5772 reserved_prefixes: Vec::new(),
5773 protocol: ModuleProtocol::Subc,
5774 overlap: Default::default(),
5775 };
5776 let handle = SupervisorHandle::new();
5777 let mut supervisor =
5778 Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default())
5779 .with_handle(handle.clone());
5780 let observed = Arc::new(Mutex::new(None));
5781 let hook_handle = handle.clone();
5782 let hook_observed = Arc::clone(&observed);
5783 supervisor.test_after_first_spawn = AfterFirstSpawnHook(Some(Arc::new(move |module_id| {
5784 let state = hook_handle
5785 .get(module_id)
5786 .expect("a configured module is on the roster before it is spawned")
5787 .state()
5788 .unwrap();
5789 *hook_observed.lock().unwrap() = Some(state);
5790 })));
5791
5792 let module = supervisor.supervise_configured(spec, true).unwrap();
5793 assert_eq!(
5794 *observed.lock().unwrap(),
5795 Some(ModuleState::Starting),
5796 "between the spawn and its admission the supervisor still reports starting"
5797 );
5798 assert_eq!(module.state().unwrap(), ModuleState::Running);
5799 module.stop().await.unwrap();
5800 }
5801
5802 fn stale_process_snapshot(state: ModuleState, enabled: bool) -> SharedSnapshot {
5803 let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::new(state, enabled)));
5804 update_snapshot(&snapshot, Some("stale-process-facts"), |snapshot| {
5805 snapshot.process_alive = true;
5806 snapshot.pid = Some(41);
5807 snapshot.spawned_at_ms = Some(42);
5808 snapshot.spawned_from = Some(PathBuf::from("/spawned/module"));
5809 snapshot.launch_spec_digest = Some(LaunchSpecDigest([0xA5; 32]));
5810 snapshot.spawned_file_identity = Some(SpawnedFileIdentity {
5811 device: 43,
5812 inode: 44,
5813 #[cfg(windows)]
5814 windows_image: None,
5815 });
5816 })
5817 .unwrap();
5818 snapshot
5819 }
5820
5821 fn assert_snapshot_process_facts_cleared(snapshot: &SharedSnapshot) {
5822 let snapshot = lock_snapshot(snapshot).unwrap();
5823 assert!(!snapshot.process_alive);
5824 assert_eq!(snapshot.pid, None);
5825 assert_eq!(snapshot.spawned_at_ms, None);
5826 assert_eq!(snapshot.spawned_from, None);
5827 assert_eq!(snapshot.launch_spec_digest, None);
5828 assert_eq!(snapshot.spawned_file_identity, None);
5829 }
5830
5831 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
5832 async fn failed_enable_spawn_clears_preexisting_current_process_facts() {
5833 let supervisor =
5834 Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default());
5835 let mut runtime = supervisor.runtime_config();
5836 runtime.test_seed_stale_facts_before_enable_spawn = true;
5837 let snapshot = stale_process_snapshot(ModuleState::Disabled, false);
5838 let mut child = None;
5839 let spec = ModuleSpec {
5840 module_id: "failed-enable-clears-facts".to_string(),
5841 program: PathBuf::from("/definitely/missing/failed-enable-module"),
5842 args: Vec::new(),
5843 env: Vec::new(),
5844 reserved: false,
5845 reserved_prefixes: Vec::new(),
5846 protocol: ModuleProtocol::Subc,
5847 overlap: Default::default(),
5848 };
5849
5850 let result = set_child_enabled(
5851 &spec,
5852 &runtime,
5853 &supervisor.registry,
5854 &supervisor.process_liveness,
5855 &snapshot,
5856 &mut child,
5857 true,
5858 )
5859 .await;
5860
5861 assert!(matches!(result, Err(SuperviseError::Spawn { .. })));
5862 assert_eq!(lock_snapshot(&snapshot).unwrap().state, ModuleState::Failed);
5863 assert_snapshot_process_facts_cleared(&snapshot);
5864 }
5865
5866 #[tokio::test]
5867 async fn start_revives_stranded_restarting_but_not_pending_backoff() {
5868 let supervisor =
5869 Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default());
5870 let runtime = supervisor.runtime_config();
5871 let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::new(
5872 ModuleState::Restarting,
5873 true,
5874 )));
5875 let spec = ModuleSpec {
5876 module_id: "start-stranded-restarting".to_string(),
5877 program: super::terminal_history_tests::fake_aft_stub_path(),
5878 args: Vec::new(),
5879 env: vec![("FAKE_AFT_NEVER_CONNECT".to_string(), "1".to_string())],
5880 reserved: false,
5881 reserved_prefixes: Vec::new(),
5882 protocol: ModuleProtocol::None,
5883 overlap: Default::default(),
5884 };
5885 let mut child = None;
5886 lock_snapshot(&snapshot).unwrap().respawn_pending = true;
5887 assert!(!super::set_child_enabled(
5888 &spec,
5889 &runtime,
5890 &Registry::default(),
5891 &supervisor.process_liveness,
5892 &snapshot,
5893 &mut child,
5894 true
5895 )
5896 .await
5897 .unwrap());
5898 assert!(child.is_none());
5899 lock_snapshot(&snapshot).unwrap().respawn_pending = false;
5900 assert!(super::set_child_enabled(
5901 &spec,
5902 &runtime,
5903 &Registry::default(),
5904 &supervisor.process_liveness,
5905 &snapshot,
5906 &mut child,
5907 true
5908 )
5909 .await
5910 .unwrap());
5911 assert_eq!(
5912 lock_snapshot(&snapshot).unwrap().state,
5913 ModuleState::Running
5914 );
5915 let mut child = child.unwrap();
5916 child.start_kill().unwrap();
5917 child.wait().await.unwrap();
5918 }
5919
5920 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
5921 async fn failed_reload_spawn_clears_current_process_facts() {
5922 let supervisor =
5923 Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default());
5924 let mut runtime = supervisor.runtime_config();
5925 runtime.restart_policy = RestartPolicy::new(0, Duration::ZERO);
5926 let snapshot = stale_process_snapshot(ModuleState::Running, true);
5927 let mut child = None;
5928 let spec = ModuleSpec {
5929 module_id: "failed-reload-clears-facts".to_string(),
5930 program: PathBuf::from("/unused/failed-reload-module"),
5931 args: Vec::new(),
5932 env: Vec::new(),
5933 reserved: false,
5934 reserved_prefixes: Vec::new(),
5935 protocol: ModuleProtocol::Subc,
5936 overlap: Default::default(),
5937 };
5938
5939 let result = handle_reload_spawn_failure(
5940 &spec,
5941 &runtime,
5942 &supervisor.process_liveness,
5943 &snapshot,
5944 &mut child,
5945 "forced reload spawn failure".to_string(),
5946 )
5947 .await;
5948
5949 assert!(matches!(result, Err(SuperviseError::ReloadFailed { .. })));
5950 assert_eq!(lock_snapshot(&snapshot).unwrap().state, ModuleState::Failed);
5951 assert_snapshot_process_facts_cleared(&snapshot);
5952 }
5953
5954 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
5955 async fn dropping_a_module_with_an_active_monitor_clears_current_process_facts() {
5956 let supervisor =
5957 Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default());
5958 let snapshot = stale_process_snapshot(ModuleState::Running, true);
5959 let module = supervisor.supervised_module(
5960 ModuleSpec {
5961 module_id: "drop-clears-facts".to_string(),
5962 program: PathBuf::from("/unused/drop-module"),
5963 args: Vec::new(),
5964 env: Vec::new(),
5965 reserved: false,
5966 reserved_prefixes: Vec::new(),
5967 protocol: ModuleProtocol::Subc,
5968 overlap: Default::default(),
5969 },
5970 supervisor.runtime_config(),
5971 Arc::clone(&snapshot),
5972 None,
5973 );
5974 assert!(!module
5975 .inner
5976 .monitor
5977 .lock()
5978 .unwrap()
5979 .as_ref()
5980 .unwrap()
5981 .is_finished());
5982
5983 drop(module);
5984
5985 assert_eq!(
5986 lock_snapshot(&snapshot).unwrap().state,
5987 ModuleState::Stopped
5988 );
5989 assert_snapshot_process_facts_cleared(&snapshot);
5990 }
5991
5992 #[cfg(unix)]
5993 #[tokio::test]
5994 async fn rescan_preserves_running_protocol_until_respawn() {
5995 let dir = cortexkit_test_support::ScratchDir::new("rescan-protocol");
5996 let initial = ModuleSpec {
5997 module_id: "rescan-protocol".into(),
5998 program: PathBuf::from("/bin/sleep"),
5999 args: vec!["60".into()],
6000 env: ["XDG_DATA_HOME", "XDG_RUNTIME_DIR", "XDG_CONFIG_HOME"]
6001 .into_iter()
6002 .map(|key| (key.into(), dir.path().to_string_lossy().into_owned()))
6003 .collect(),
6004 reserved: false,
6005 reserved_prefixes: vec![],
6006 protocol: ModuleProtocol::None,
6007 overlap: Default::default(),
6008 };
6009 let supervisor =
6010 Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default());
6011 let module = supervisor.spawn(initial.clone()).unwrap();
6012 assert!(module.status().unwrap().live);
6013 let mut next = initial;
6014 next.protocol = ModuleProtocol::Subc;
6015 module
6016 .update_configuration(next.clone(), HealthConfig::default(), None)
6017 .await
6018 .unwrap();
6019 assert!(
6020 module.status().unwrap().live,
6021 "rescan must not require HELLO from the old non-wire process"
6022 );
6023 let runtime = supervisor.runtime_config();
6024 let action = on_child_exit(
6025 &next,
6026 RestartPolicy::default(),
6027 &supervisor.registry,
6028 &module.inner.snapshot,
6029 &runtime.terminal_ring,
6030 &runtime.spawn_events,
6031 &runtime.child_roster,
6032 ExitReport {
6033 kind: ExitKind::Clean,
6034 code: Some(0),
6035 signal: None,
6036 at_ms: unix_ms_now(),
6037 },
6038 )
6039 .await;
6040 assert!(
6041 matches!(action, NextAction::Restart { .. }),
6042 "the old non-wire process's clean exit must restart"
6043 );
6044 module.drain().await.unwrap();
6045 }
6046
6047 #[cfg(unix)]
6048 #[tokio::test]
6049 async fn orphan_identity_matches_path_names_and_shebang_interpreters() {
6050 use std::os::unix::fs::PermissionsExt;
6051 let dir = cortexkit_test_support::ScratchDir::new("spawn-image-identity");
6052 let script = dir.join("module.sh");
6053 std::fs::write(&script, "#!/bin/sh\nwhile :; do sleep 0.1; done\n").unwrap();
6054 std::fs::set_permissions(&script, std::fs::Permissions::from_mode(0o700)).unwrap();
6055 let record_path = dir.join("live-children.json");
6056 let supervisor =
6057 Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default())
6058 .with_live_children_record(&record_path);
6059 for (program, args) in [
6060 (PathBuf::from("sleep"), vec!["60".into()]),
6061 (script, vec![]),
6062 ] {
6063 let spec = ModuleSpec {
6064 module_id: "image-identity".into(),
6065 program: program.clone(),
6066 args,
6067 env: ["XDG_DATA_HOME", "XDG_RUNTIME_DIR", "XDG_CONFIG_HOME"]
6068 .into_iter()
6069 .map(|key| (key.into(), dir.to_string_lossy().into_owned()))
6070 .collect(),
6071 reserved: false,
6072 reserved_prefixes: vec![],
6073 protocol: ModuleProtocol::None,
6074 overlap: Default::default(),
6075 };
6076 let module = supervisor.spawn(spec).unwrap();
6077 #[cfg(target_os = "macos")]
6078 {
6079 let deadline = Instant::now() + Duration::from_secs(5);
6082 while crate::live_children::read_record(&record_path)
6083 .unwrap()
6084 .iter()
6085 .all(|entry| entry.executable.is_none())
6086 {
6087 assert!(Instant::now() < deadline, "module image was not confirmed");
6088 tokio::time::sleep(Duration::from_millis(5)).await;
6089 }
6090 }
6091 let entry = crate::live_children::read_record(&record_path)
6092 .unwrap()
6093 .pop()
6094 .unwrap();
6095 let observed = subc_os::Process::open(entry.pid)
6096 .unwrap()
6097 .unwrap()
6098 .observe()
6099 .unwrap();
6100 let verdict = crate::live_children::identity_verdict(&entry, entry.pid, &observed);
6101 module.drain().await.unwrap();
6102 assert_eq!(
6103 verdict,
6104 crate::live_children::IdentityVerdict::Matches,
6105 "program {program:?}: recorded {entry:?}, observed {observed:?}"
6106 );
6107 }
6108 }
6109
6110 #[cfg(unix)]
6111 fn http_fixture(
6112 dir: &std::path::Path,
6113 url: &str,
6114 threshold: u32,
6115 ) -> crate::daemon_config::ConfiguredModule {
6116 let path = dir.join("subc.jsonc");
6117 std::fs::write(&path, serde_json::json!({"version": 1, "modules": {"http-process": {
6118 "program": "/bin/sleep", "args": ["60"], "protocol": "none",
6119 "env": {"XDG_DATA_HOME":dir,"XDG_RUNTIME_DIR":dir,"XDG_CONFIG_HOME":dir},
6120 "health": {"http":url,"cadence_ms":20,"deadline_ms":200,"failure_threshold":threshold},
6121 "restart": {"backoff_ms":1,"max_backoff_ms":1}, "drain_timeout_ms":10
6122 }}}).to_string()).unwrap();
6123 crate::daemon_config::load(&path)
6124 .unwrap()
6125 .unwrap()
6126 .modules
6127 .pop()
6128 .unwrap()
6129 }
6130
6131 #[cfg(unix)]
6132 async fn wait_http_health(module: &SupervisedModule, status: SupervisorHealthStatus) {
6133 timeout(Duration::from_secs(5), async {
6134 loop {
6135 if module.status().unwrap().health.status == status {
6136 break;
6137 }
6138 sleep(Duration::from_millis(5)).await;
6139 }
6140 })
6141 .await
6142 .unwrap_or_else(|_| {
6143 panic!(
6144 "expected {status:?}, got {:?}",
6145 module.status().unwrap().health
6146 )
6147 });
6148 }
6149
6150 #[cfg(unix)]
6151 #[tokio::test]
6152 async fn http_health_status_flips_ok_failing_ok() {
6153 use tokio::io::{AsyncReadExt, AsyncWriteExt};
6154 let dir = cortexkit_test_support::ScratchDir::new("http-status-flips");
6155 let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
6156 let status = Arc::new(std::sync::atomic::AtomicU16::new(200));
6157 let serving_status = status.clone();
6158 let url = format!("http://{}/healthz", listener.local_addr().unwrap());
6159 let server = tokio::spawn(async move {
6160 loop {
6161 let (mut stream, _) = listener.accept().await.unwrap();
6162 let mut request = [0u8; 2048];
6163 let count = stream.read(&mut request).await.unwrap();
6164 assert!(count > 0, "a probe must send an HTTP request");
6165 let code = serving_status.load(std::sync::atomic::Ordering::SeqCst);
6166 let body = if code == 200 {
6167 "ready"
6168 } else {
6169 "{\"status\":\"unavailable\",\"error\":\"scratch failure\"}"
6170 };
6171 let response = format!("HTTP/1.1 {code} scratch\r\nContent-Length: {}\r\nConnection: close\r\n\r\n{body}", body.len());
6172 let _ = stream.write_all(response.as_bytes()).await;
6173 }
6174 });
6175 let configured = http_fixture(&dir, &url, 1000);
6176 let module =
6177 Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default())
6178 .supervise_configured_with_health(
6179 configured.module_spec(),
6180 true,
6181 configured.health,
6182 configured.drain_timeout_ms,
6183 configured.restart,
6184 )
6185 .unwrap();
6186 wait_http_health(&module, SupervisorHealthStatus::Ok).await;
6187 status.store(503, std::sync::atomic::Ordering::SeqCst);
6188 wait_http_health(&module, SupervisorHealthStatus::Failing).await;
6189 assert!(module
6190 .status()
6191 .unwrap()
6192 .health
6193 .detail
6194 .unwrap()
6195 .contains("scratch failure"));
6196 status.store(200, std::sync::atomic::Ordering::SeqCst);
6197 wait_http_health(&module, SupervisorHealthStatus::Ok).await;
6198 assert_eq!(module.status().unwrap().health.consecutive_failures, 0);
6199 let before = module.status().unwrap();
6200 let (spec, mut health) = module.configuration().unwrap();
6201 health.http = None;
6202 module
6203 .update_configuration(spec.clone(), health.clone(), Some(10))
6204 .await
6205 .unwrap();
6206 wait_http_health(&module, SupervisorHealthStatus::Unknown).await;
6207 health.http = Some(url);
6208 module
6209 .update_configuration(spec, health, Some(10))
6210 .await
6211 .unwrap();
6212 wait_http_health(&module, SupervisorHealthStatus::Ok).await;
6213 assert_eq!(
6214 module.status().unwrap().pid,
6215 before.pid,
6216 "changing a probe must apply live, not restart its process"
6217 );
6218 let (spec, mut health) = module.configuration().unwrap();
6219 health.failure_threshold = 2;
6220 module
6221 .update_configuration(spec, health, Some(10))
6222 .await
6223 .unwrap();
6224 status.store(503, std::sync::atomic::Ordering::SeqCst);
6225 timeout(Duration::from_secs(5), async {
6226 while module.status().unwrap().spawn_generation == before.spawn_generation {
6227 sleep(Duration::from_millis(5)).await;
6228 }
6229 })
6230 .await
6231 .expect("sustained HTTP 503 responses must trigger the health restart policy");
6232 module.drain().await.unwrap();
6233 server.abort();
6234 }
6235
6236 #[cfg(unix)]
6237 #[tokio::test]
6238 async fn http_health_replacement_recovers_after_one_restart() {
6239 use tokio::io::{AsyncReadExt, AsyncWriteExt};
6240 let dir = cortexkit_test_support::ScratchDir::new("http-replacement-recovers");
6241 let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
6242 let url = format!("http://{}/healthz", listener.local_addr().unwrap());
6243 let mut configured = http_fixture(&dir, &url, 2);
6244 configured.restart.max_restarts = 2;
6245 let module =
6246 Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default())
6247 .supervise_configured_with_health(
6248 configured.module_spec(),
6249 true,
6250 configured.health,
6251 configured.drain_timeout_ms,
6252 configured.restart,
6253 )
6254 .unwrap();
6255 let first_generation = module.status().unwrap().spawn_generation;
6256 let observed = module.clone();
6257 let (healthy_tx, mut healthy_rx) = mpsc::channel(8);
6258 let server = tokio::spawn(async move {
6259 let mut generation = first_generation;
6260 let mut probes = 0;
6261 loop {
6262 let (mut stream, _) = listener.accept().await.unwrap();
6263 let mut request = [0; 2048];
6264 assert!(stream.read(&mut request).await.unwrap() > 0);
6265 let current = observed.status().unwrap().spawn_generation;
6266 if current != generation {
6267 generation = current;
6268 probes = 0;
6269 }
6270 probes += 1;
6271 let healthy = generation != first_generation && probes > 1;
6274 let code = if healthy { 200 } else { 503 };
6275 let response = format!(
6276 "HTTP/1.1 {code} scratch\r\nContent-Length: 0\r\nConnection: close\r\n\r\n"
6277 );
6278 let _ = stream.write_all(response.as_bytes()).await;
6279 if healthy {
6280 healthy_tx.send(()).await.unwrap();
6281 }
6282 }
6283 });
6284 timeout(Duration::from_secs(5), async {
6285 let mut healthy_probes = 0;
6286 loop {
6287 if module.state().unwrap() == ModuleState::Failed {
6288 break;
6289 }
6290 while healthy_rx.try_recv().is_ok() {
6291 healthy_probes += 1;
6292 }
6293 if healthy_probes >= 3
6294 && module.status().unwrap().health.status == SupervisorHealthStatus::Ok
6295 {
6296 break;
6297 }
6298 sleep(Duration::from_millis(5)).await;
6299 }
6300 })
6301 .await
6302 .expect("the replacement must settle healthy without exhausting its restart budget");
6303 let status = module.status().unwrap();
6304 module.drain().await.unwrap();
6305 server.abort();
6306 assert_eq!(status.state, ModuleState::Running);
6307 assert_eq!(status.health.status, SupervisorHealthStatus::Ok);
6308 assert_eq!(status.health.consecutive_failures, 0);
6309 assert_eq!(status.lifetime_restarts, 1);
6310 assert_eq!(status.restart_count, 1);
6311 }
6312
6313 #[cfg(unix)]
6314 #[tokio::test]
6315 async fn http_health_sustained_failure_exhausts_restart_budget() {
6316 use std::collections::BTreeMap;
6317 use tokio::io::{AsyncReadExt, AsyncWriteExt};
6318 let dir = cortexkit_test_support::ScratchDir::new("http-health-budget");
6319 let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
6320 let url = format!("http://{}/healthz", listener.local_addr().unwrap());
6321 let mut configured = http_fixture(&dir, &url, 2);
6322 configured.restart.max_restarts = 2;
6323 let module =
6324 Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default())
6325 .supervise_configured_with_health(
6326 configured.module_spec(),
6327 true,
6328 configured.health,
6329 configured.drain_timeout_ms,
6330 configured.restart,
6331 )
6332 .unwrap();
6333 let observed = module.clone();
6334 let probes = Arc::new(Mutex::new(BTreeMap::<u64, u32>::new()));
6335 let served = probes.clone();
6336 let server = tokio::spawn(async move {
6337 loop {
6338 let (mut stream, _) = listener.accept().await.unwrap();
6339 let mut request = [0; 2048];
6340 assert!(stream.read(&mut request).await.unwrap() > 0);
6341 *served
6342 .lock()
6343 .unwrap()
6344 .entry(observed.status().unwrap().spawn_generation)
6345 .or_default() += 1;
6346 let _ = stream
6347 .write_all(b"HTTP/1.1 503 Unavailable\r\nContent-Length: 0\r\nConnection: close\r\n\r\n")
6348 .await;
6349 }
6350 });
6351 timeout(Duration::from_secs(5), async {
6352 while module.state().unwrap() != ModuleState::Failed {
6353 sleep(Duration::from_millis(5)).await;
6354 }
6355 })
6356 .await
6357 .expect("sustained failures must still exhaust the health restart budget");
6358 let status = module.status().unwrap();
6359 server.abort();
6360 assert_eq!(status.restart_count, 2);
6361 assert_eq!(status.lifetime_restarts, 2);
6362 assert!(!status.process_alive);
6363 assert_eq!(
6364 probes.lock().unwrap().values().copied().collect::<Vec<_>>(),
6365 vec![2, 2, 2]
6366 );
6367 }
6368
6369 #[cfg(unix)]
6370 #[tokio::test]
6371 async fn spawn_clears_the_previous_process_health() {
6372 let dir = cortexkit_test_support::ScratchDir::new("spawn-fresh-health");
6373 let mut configured = http_fixture(&dir, "http://127.0.0.1:1/healthz", 2);
6374 configured.health.cadence = Duration::from_secs(3600);
6375 let module =
6376 Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default())
6377 .supervise_configured_with_health(
6378 configured.module_spec(),
6379 false,
6380 configured.health,
6381 configured.drain_timeout_ms,
6382 configured.restart,
6383 )
6384 .unwrap();
6385 update_snapshot(&module.inner.snapshot, Some(module.module_id()), |state| {
6386 state.health.status = SupervisorHealthStatus::Unresponsive;
6387 state.health.consecutive_failures = 3;
6388 state.health.detail = Some("old process refused connections".into());
6389 state.health.metrics = Some(serde_json::json!({"old": true}));
6390 state.health.last_probe_ms = Some(123);
6391 state.health.last_action = Some("restart".into());
6392 state.health.last_action_ms = Some(124);
6393 })
6394 .unwrap();
6395 module.set_enabled(true).await.unwrap();
6396 let health = module.status().unwrap().health;
6397 module.drain().await.unwrap();
6398 assert_eq!(health.consecutive_failures, 0);
6399 assert_eq!(health.status, SupervisorHealthStatus::Unknown);
6400 assert_eq!(health.detail, None);
6401 assert_eq!(health.metrics, None);
6402 assert_eq!(health.last_probe_ms, None);
6403 assert_eq!(health.last_action.as_deref(), Some("restart"));
6404 assert_eq!(health.last_action_ms, Some(124));
6405 }
6406
6407 #[cfg(unix)]
6408 #[tokio::test]
6409 async fn http_health_no_listener_restarts_after_consecutive_failures() {
6410 let dir = cortexkit_test_support::ScratchDir::new("http-refused");
6411 let unused = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
6412 let url = format!("http://{}/healthz", unused.local_addr().unwrap());
6413 drop(unused);
6414 let configured = http_fixture(&dir, &url, 2);
6415 let module =
6416 Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default())
6417 .supervise_configured_with_health(
6418 configured.module_spec(),
6419 true,
6420 configured.health,
6421 configured.drain_timeout_ms,
6422 configured.restart,
6423 )
6424 .unwrap();
6425 let before = module.status().unwrap().spawn_generation;
6426 timeout(Duration::from_secs(5), async {
6427 loop {
6428 let status = module.status().unwrap();
6429 if status.spawn_generation > before {
6430 assert!(status.lifetime_restarts > 0);
6431 break;
6432 }
6433 sleep(Duration::from_millis(5)).await;
6434 }
6435 })
6436 .await
6437 .expect("sustained HTTP refusal must trigger the health restart policy");
6438 module.drain().await.unwrap();
6439 }
6440
6441 #[cfg(unix)]
6442 #[tokio::test]
6443 async fn http_health_timeout_honours_deadline() {
6444 let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
6445 let url = format!("http://{}/healthz", listener.local_addr().unwrap());
6446 let server = tokio::spawn(async move {
6447 let _held = listener.accept().await.unwrap();
6448 std::future::pending::<()>().await;
6449 });
6450 let error = timeout(
6451 Duration::from_secs(1),
6452 probe_http_health(&url, Duration::from_millis(10)),
6453 )
6454 .await
6455 .expect("the probe must enforce its own deadline")
6456 .unwrap_err();
6457 server.abort();
6458 assert!(error.to_string().contains("timed out"));
6459 }
6460
6461 #[cfg(unix)]
6462 #[tokio::test]
6463 async fn http_health_localhost_reaches_an_ipv6_only_listener() {
6464 use tokio::io::{AsyncReadExt, AsyncWriteExt};
6465 let listener = tokio::net::TcpListener::bind("[::1]:0").await.unwrap();
6466 let url = format!(
6467 "http://localhost:{}/healthz",
6468 listener.local_addr().unwrap().port()
6469 );
6470 let server = tokio::spawn(async move {
6471 let (mut stream, _) = listener.accept().await.unwrap();
6472 let mut request = [0u8; 2048];
6473 assert!(stream.read(&mut request).await.unwrap() > 0);
6474 stream
6475 .write_all(b"HTTP/1.1 200 OK\r\nContent-Length: 2\r\n\r\nok")
6476 .await
6477 .unwrap();
6478 });
6479 assert_eq!(
6484 probe_http_health(&url, Duration::from_secs(10))
6485 .await
6486 .unwrap()
6487 .status,
6488 HealthStatus::Ok
6489 );
6490 server.await.unwrap();
6491 }
6492
6493 #[cfg(unix)]
6494 #[tokio::test]
6495 async fn http_health_timeout_keeps_partial_status_and_body() {
6496 use tokio::io::{AsyncReadExt, AsyncWriteExt};
6497 let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
6498 let url = format!("http://{}/healthz", listener.local_addr().unwrap());
6499 let server = tokio::spawn(async move {
6500 let (mut stream, _) = listener.accept().await.unwrap();
6501 let mut request = [0u8; 2048];
6502 assert!(stream.read(&mut request).await.unwrap() > 0);
6503 stream
6504 .write_all(
6505 b"HTTP/1.1 503 Unavailable\r\nContent-Length: 100\r\n\r\npartial diagnostic",
6506 )
6507 .await
6508 .unwrap();
6509 std::future::pending::<()>().await;
6510 });
6511 let error = probe_http_health(&url, Duration::from_secs(1))
6512 .await
6513 .unwrap_err()
6514 .to_string();
6515 server.abort();
6516 assert!(
6517 error.contains("timed out")
6518 && error.contains("503 Unavailable")
6519 && error.contains("partial diagnostic"),
6520 "{error}"
6521 );
6522 }
6523
6524 #[cfg(unix)]
6525 #[tokio::test]
6526 async fn http_health_diagnostic_body_is_bounded_to_200_bytes() {
6527 use tokio::io::{AsyncReadExt, AsyncWriteExt};
6528 let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
6529 let url = format!("http://{}/healthz", listener.local_addr().unwrap());
6530 let server = tokio::spawn(async move {
6531 let (mut stream, _) = listener.accept().await.unwrap();
6532 let mut request = [0u8; 2048];
6533 assert!(stream.read(&mut request).await.unwrap() > 0);
6534 let body = format!("{}not-in-diagnostic", "x".repeat(400));
6535 let response = format!(
6536 "HTTP/1.1 503 Unavailable\r\nContent-Length: {}\r\n\r\n{body}",
6537 body.len()
6538 );
6539 stream.write_all(response.as_bytes()).await.unwrap();
6540 });
6541 let error = probe_http_health(&url, Duration::from_secs(1))
6542 .await
6543 .unwrap_err()
6544 .to_string();
6545 server.await.unwrap();
6546 assert!(error.ends_with(&"x".repeat(200)), "{error}");
6547 assert_eq!(error.split(": ").last().unwrap().len(), 200);
6548 assert!(!error.contains("not-in-diagnostic"));
6549 }
6550
6551 #[cfg(unix)]
6552 #[tokio::test]
6553 async fn http_health_real_nats_server_monitoring() {
6554 if std::process::Command::new("nats-server")
6555 .arg("--version")
6556 .env("XDG_DATA_HOME", std::env::temp_dir())
6557 .env("XDG_RUNTIME_DIR", std::env::temp_dir())
6558 .env("XDG_CONFIG_HOME", std::env::temp_dir())
6559 .output()
6560 .is_err()
6561 {
6562 eprintln!(
6563 "skipped: http_health_real_nats_server_monitoring (nats-server is not on PATH)"
6564 );
6565 return;
6566 }
6567 let dir = cortexkit_test_support::ScratchDir::new("nats-http-monitoring");
6568 let monitor = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
6569 let port = monitor.local_addr().unwrap().port();
6570 let client = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
6571 let client_port = client.local_addr().unwrap().port();
6572 let config = dir.join("server.conf");
6573 std::fs::write(&config, format!("listen: 127.0.0.1:{client_port}\nhttp: 127.0.0.1:{port}\njetstream {{ store_dir: \"{}\" }}\n", dir.join("jetstream").display())).unwrap();
6574 let mut configured = http_fixture(&dir, &format!("http://127.0.0.1:{port}/healthz"), 1000);
6575 configured.program = PathBuf::from("nats-server");
6576 configured.args = vec!["-c".into(), config.to_string_lossy().into_owned()];
6577 drop(monitor);
6578 drop(client);
6579 let module =
6580 Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default())
6581 .supervise_configured_with_health(
6582 configured.module_spec(),
6583 true,
6584 configured.health,
6585 configured.drain_timeout_ms,
6586 configured.restart,
6587 )
6588 .unwrap();
6589 wait_http_health(&module, SupervisorHealthStatus::Ok).await;
6590 module.drain().await.unwrap();
6591 }
6592
6593 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
6594 async fn configuration_update_does_not_replace_captured_running_process_facts() {
6595 let supervisor =
6596 Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default());
6597 let snapshot = stale_process_snapshot(ModuleState::Running, true);
6598 let initial = ModuleSpec {
6599 module_id: "rescan-preserves-spawn-facts".to_string(),
6600 program: PathBuf::from("/spawned/module"),
6601 args: Vec::new(),
6602 env: Vec::new(),
6603 reserved: false,
6604 reserved_prefixes: Vec::new(),
6605 protocol: ModuleProtocol::Subc,
6606 overlap: Default::default(),
6607 };
6608 let module = supervisor.supervised_module(
6609 initial.clone(),
6610 supervisor.runtime_config(),
6611 snapshot,
6612 None,
6613 );
6614 let before = module.status().unwrap();
6615 let mut replacement = initial;
6616 replacement.program = PathBuf::from("/rescanned/replacement-module");
6617
6618 module
6619 .update_configuration(replacement, HealthConfig::default(), None)
6620 .await
6621 .unwrap();
6622
6623 let after = module.status().unwrap();
6624 assert_eq!(after.pid, before.pid);
6625 assert_eq!(after.spawned_at_ms, before.spawned_at_ms);
6626 assert_eq!(after.spawned_from, before.spawned_from);
6627 drop(module);
6628 }
6629}
6630
6631fn unix_ms_now() -> u64 {
6632 SystemTime::now()
6633 .duration_since(UNIX_EPOCH)
6634 .map(|duration| duration.as_millis().min(u128::from(u64::MAX)) as u64)
6635 .unwrap_or(0)
6636}
6637
6638async fn supervise_loop(
6639 mut spec: ModuleSpec,
6640 mut runtime: SupervisorRuntimeConfig,
6641 registry: Arc<Registry>,
6642 process_liveness: Arc<SupervisorProcessLiveness>,
6643 snapshot: SharedSnapshot,
6644 mut child: Option<SupervisedChild>,
6645 mut commands: mpsc::Receiver<SupervisorCommand>,
6646) {
6647 let mut health_probe = HealthProbeRuntime::default();
6648 let mut registration_changes = match registry.subscribe_module_changes(&spec.module_id) {
6652 Ok(changes) => Some(changes),
6653 Err(err) => {
6654 warn!(module_id = %spec.module_id, error = %err, "health prober could not subscribe to registry");
6657 None
6658 }
6659 };
6660 let mut pending_respawn: Option<PendingRespawn> = None;
6664 let mut requeued: VecDeque<SupervisorCommand> = VecDeque::new();
6667 let mut previous_instance_recheck = tokio::time::interval_at(
6668 Instant::now() + Duration::from_secs(5),
6669 Duration::from_secs(5),
6670 );
6671 previous_instance_recheck.set_missed_tick_behavior(tokio::time::MissedTickBehavior::Skip);
6672 loop {
6673 #[cfg(test)]
6674 {
6675 lock_snapshot(&snapshot).unwrap().actor_select = None;
6676 }
6677 #[cfg(target_os = "macos")]
6678 if let Some(active) = child.as_mut() {
6679 active.confirm_privacy_exec().await;
6680 }
6681 if let Some(scheduled) = runtime
6682 .scheduled_respawn
6683 .lock()
6684 .unwrap_or_else(|p| p.into_inner())
6685 .take()
6686 {
6687 pending_respawn = Some(scheduled);
6688 }
6689 if pending_respawn.is_some() && (child.is_some() || !respawn_still_pending(&snapshot)) {
6690 pending_respawn = None;
6691 cancel_deferred_reload(
6692 &runtime,
6693 &spec.module_id,
6694 "respawn cancelled by a supervisor command",
6695 );
6696 }
6697 if child.is_none()
6698 && pending_respawn.is_none()
6699 && lock_snapshot(&snapshot).is_ok_and(|state| state.waiting_for_previous_pid.is_none())
6700 {
6701 cancel_deferred_reload(
6702 &runtime,
6703 &spec.module_id,
6704 "respawn cancelled before a replacement was spawned",
6705 );
6706 let _ = update_snapshot(&snapshot, Some(&spec.module_id), |state| {
6707 state.respawn_pending = false;
6708 state.coalesced_restart_pending = false;
6709 if matches!(
6710 state.state,
6711 ModuleState::Restarting
6712 | ModuleState::Starting
6713 | ModuleState::Draining
6714 | ModuleState::Unresponsive
6715 ) {
6716 error!(module_id = %spec.module_id, state = ?state.state,
6717 "supervision operation ended without a child or pending respawn; marking failed so start can retry");
6718 state.state = ModuleState::Failed;
6719 clear_current_process_facts(state);
6720 }
6721 });
6722 }
6723 if let Some(command) = requeued.pop_front() {
6724 if !handle_supervisor_command(
6725 command,
6726 &mut spec,
6727 &mut runtime,
6728 ®istry,
6729 &process_liveness,
6730 &snapshot,
6731 &mut child,
6732 &mut commands,
6733 &mut requeued,
6734 )
6735 .await
6736 {
6737 return;
6738 }
6739 if child.is_some() || !respawn_still_pending(&snapshot) {
6740 pending_respawn = None;
6741 let _ = update_snapshot(&snapshot, Some(&spec.module_id), |state| {
6742 state.respawn_pending = false
6743 });
6744 }
6745 continue;
6746 }
6747 if child.is_some() {
6748 #[cfg(test)]
6749 {
6750 lock_snapshot(&snapshot).unwrap().actor_turns += 1;
6751 }
6752 health_probe.refresh_registration(&spec, &runtime, ®istry, &snapshot);
6753 let wake_after = health_probe.wake_after();
6754 let probe_sleep = async move {
6755 match wake_after {
6756 Some(delay) => sleep(delay).await,
6757 None => std::future::pending().await,
6758 }
6759 };
6760 tokio::pin!(probe_sleep);
6761 let wire_child = running_protocol(&spec, &snapshot) == ModuleProtocol::Subc;
6766 #[cfg(test)]
6767 {
6768 let registration_pending = wire_child
6769 && registration_changes
6770 .as_ref()
6771 .is_some_and(|changes| changes.has_changed().unwrap());
6772 let mut state = lock_snapshot(&snapshot).unwrap();
6773 state.actor_select = (!registration_pending).then_some(ActorSelectCheckpoint {
6774 generation: state.spawn_generation,
6775 turn: state.actor_turns,
6776 registered_connection: health_probe.registered_connection,
6777 next_probe_at: health_probe.next_probe_at,
6778 wake_after,
6779 });
6780 }
6781 let active_child = child.as_mut().expect("child checked above");
6782 tokio::select! {
6783 wait_result = active_child.wait() => {
6784 #[cfg(test)]
6785 { lock_snapshot(&snapshot).unwrap().actor_select = None; }
6786 let exit_report = match wait_result {
6795 Ok(status) => classify_reaped_child_exit(&snapshot, active_child, &status),
6796 Err(err) => {
6797 active_child.drain_stderr(&spec.module_id).await;
6798 fail_after_wait_error(
6799 &spec.module_id,
6800 &snapshot,
6801 &runtime.terminal_ring,
6802 &runtime.spawn_events,
6803 format!("failed to wait for supervised module: {err}"),
6804 );
6805 untrack_if_registration_released(
6806 &process_liveness,
6807 ®istry,
6808 &spec.module_id,
6809 &snapshot,
6810 );
6811 error!(module_id = %spec.module_id, error = %err, "failed to wait for supervised module");
6812 child = None;
6813 continue;
6814 }
6815 };
6816 active_child.drain_stderr(&spec.module_id).await;
6817
6818 let next = on_child_exit(
6819 &spec,
6820 runtime.restart_policy,
6821 ®istry,
6822 &snapshot,
6823 &runtime.terminal_ring,
6824 &runtime.spawn_events,
6825 &runtime.child_roster,
6826 exit_report,
6827 ).await;
6828 runtime.start_requester = None;
6829 runtime.pending_activity_restart = false;
6830 active_child.release_roster();
6833 match next {
6834 NextAction::Stop { registration_released } => {
6835 if registration_released {
6836 process_liveness.untrack_if_current(&spec.module_id, &snapshot);
6837 }
6838 child = None;
6839 }
6840 NextAction::Restart { schedule } => {
6841 let delay = schedule.map_or(
6842 runtime.restart_policy.delay_for_restart(0),
6843 |schedule| schedule.delay,
6844 );
6845 if let Some(schedule) = schedule {
6846 log_crash_respawn(&spec.module_id, schedule);
6847 }
6848 child = None;
6856 pending_respawn = Some(PendingRespawn { deadline: Instant::now() + delay, kind: RespawnKind::Spawn });
6857 let _ = update_snapshot(&snapshot, Some(&spec.module_id), |state| state.respawn_pending = true);
6858 }
6859 }
6860 }
6861 command = commands.recv() => {
6862 #[cfg(test)]
6863 { lock_snapshot(&snapshot).unwrap().actor_select = None; }
6864 let Some(command) = command else {
6865 return;
6866 };
6867 if !handle_supervisor_command(
6868 command,
6869 &mut spec,
6870 &mut runtime,
6871 ®istry,
6872 &process_liveness,
6873 &snapshot,
6874 &mut child,
6875 &mut commands,
6876 &mut requeued,
6877 ).await {
6878 return;
6879 }
6880 }
6881 _ = async {
6882 match registration_changes.as_mut() {
6883 Some(changes) => { let _ = changes.changed().await; }
6884 None => std::future::pending().await,
6885 }
6886 }, if wire_child => {
6887 #[cfg(test)]
6888 { lock_snapshot(&snapshot).unwrap().actor_select = None; }
6889 }
6890 _ = &mut probe_sleep => {
6891 #[cfg(test)]
6892 { lock_snapshot(&snapshot).unwrap().actor_select = None; }
6893 if health_probe.due() {
6894 run_health_probe_cycle(
6895 &spec,
6896 &runtime,
6897 ®istry,
6898 &process_liveness,
6899 &snapshot,
6900 &mut child,
6901 ).await;
6902 if child.is_none() {
6903 runtime.start_requester = None;
6904 runtime.pending_activity_restart = false;
6905 }
6906 if child.is_some() {
6907 health_probe.schedule_next(&spec, runtime.health.cadence);
6908 }
6909 }
6910 }
6911 }
6912 } else if let Some(pending) = pending_respawn {
6913 tokio::select! {
6914 _ = sleep_until(pending.deadline) => {
6915 pending_respawn = None;
6916 let _ = update_snapshot(&snapshot, Some(&spec.module_id), |state| state.respawn_pending = false);
6917 if !respawn_still_pending(&snapshot) {
6921 continue;
6922 }
6923 if runtime.child_roster.is_closed() {
6928 let _ = update_snapshot(&snapshot, Some(&spec.module_id), |state| {
6929 state.state = ModuleState::Stopped;
6930 });
6931 debug!(module_id = %spec.module_id, "crash respawn cancelled by daemon shutdown");
6932 continue;
6933 }
6934 if let Err(err) = release_dead_registration(
6935 ®istry,
6936 runtime.forwarding.as_deref(),
6937 &snapshot,
6938 &spec.module_id,
6939 ).await {
6940 fail_snapshot(&snapshot, Some(&spec.module_id), None);
6941 error!(module_id = %spec.module_id, error = %err, "registration did not release before restart");
6942 continue;
6943 }
6944
6945 if matches!(pending.kind, RespawnKind::Reload) {
6946 let reply = runtime.deferred_reload_reply.lock().unwrap_or_else(|p| p.into_inner()).take();
6947 let result = finish_reload_child(&spec, &runtime, ®istry, &process_liveness, &snapshot, &mut child).await;
6948 if result.is_ok() && runtime.pending_activity_restart {
6949 runtime.record_activity_transition("subc.module_restarted", &spec.module_id, runtime.start_requester.clone());
6950 }
6951 runtime.pending_activity_restart = false;
6952 if let Some(reply) = reply { let _ = reply.send(result); }
6953 continue;
6954 }
6955 process_liveness.track(spec.module_id.clone(), Arc::clone(&snapshot));
6956 match spawn_and_mark_running(&spec, &runtime, &snapshot) {
6957 Ok(next_child) => {
6958 child = Some(next_child);
6959 if runtime.pending_activity_restart {
6960 runtime.record_activity_transition("subc.module_restarted", &spec.module_id, runtime.start_requester.clone());
6961 runtime.pending_activity_restart = false;
6962 }
6963 debug!(module_id = %spec.module_id, "supervised module restarted after crash");
6964 }
6965 Err(err) => {
6966 fail_snapshot(&snapshot, Some(&spec.module_id), None);
6967 process_liveness.untrack_if_current(&spec.module_id, &snapshot);
6968 error!(module_id = %spec.module_id, error = %err, "failed to restart supervised module");
6969 }
6970 }
6971 }
6972 command = commands.recv() => {
6973 let Some(command) = command else {
6974 return;
6975 };
6976 if !handle_supervisor_command(
6977 command,
6978 &mut spec,
6979 &mut runtime,
6980 ®istry,
6981 &process_liveness,
6982 &snapshot,
6983 &mut child,
6984 &mut commands,
6985 &mut requeued,
6986 ).await {
6987 return;
6988 }
6989 if child.is_some() || !respawn_still_pending(&snapshot) {
6994 pending_respawn = None;
6995 let _ = update_snapshot(&snapshot, Some(&spec.module_id), |state| state.respawn_pending = false);
6996 }
6997 }
6998 }
6999 } else {
7000 let command = if lock_snapshot(&snapshot)
7001 .is_ok_and(|state| state.waiting_for_previous_pid.is_some())
7002 {
7003 tokio::select! {
7004 command = commands.recv() => command,
7005 _ = previous_instance_recheck.tick() => {
7006 if runtime.child_roster.previous_instance_pid(&spec.module_id).is_none() {
7007 let enabled = lock_snapshot(&snapshot).is_ok_and(|state| state.enabled);
7008 if enabled && !runtime.child_roster.is_closed() {
7009 process_liveness.track(spec.module_id.clone(), Arc::clone(&snapshot));
7010 match spawn_and_mark_running(&spec, &runtime, &snapshot) {
7011 Ok(next_child) => child = Some(next_child),
7012 Err(error) => {
7013 fail_snapshot(&snapshot, Some(&spec.module_id), None);
7014 error!(module_id = %spec.module_id, %error, "failed to start module after its previous instance exited");
7015 }
7016 }
7017 } else {
7018 let _ = update_snapshot(&snapshot, Some(&spec.module_id), |state| {
7019 state.waiting_for_previous_pid = None;
7020 state.waiting_for_previous_detail = None;
7021 });
7022 }
7023 }
7024 continue;
7025 }
7026 }
7027 } else {
7028 commands.recv().await
7029 };
7030 let Some(command) = command else {
7031 return;
7032 };
7033 if !handle_supervisor_command(
7034 command,
7035 &mut spec,
7036 &mut runtime,
7037 ®istry,
7038 &process_liveness,
7039 &snapshot,
7040 &mut child,
7041 &mut commands,
7042 &mut requeued,
7043 )
7044 .await
7045 {
7046 return;
7047 }
7048 }
7049 }
7050}
7051
7052fn log_crash_respawn(module_id: &str, schedule: CrashRestartSchedule) {
7053 info!(
7054 module_id,
7055 restart_in_window = schedule.restart_in_window,
7056 delay_ms = schedule.delay.as_millis() as u64,
7057 "respawning after crash"
7058 );
7059}
7060
7061fn respawn_still_pending(snapshot: &SharedSnapshot) -> bool {
7067 matches!(
7068 lock_snapshot(snapshot),
7069 Ok(state) if state.enabled && state.state == ModuleState::Restarting
7070 )
7071}
7072
7073enum NextAction {
7074 Stop {
7075 registration_released: bool,
7076 },
7077 Restart {
7078 schedule: Option<CrashRestartSchedule>,
7079 },
7080}
7081
7082#[allow(clippy::too_many_arguments)]
7083async fn handle_supervisor_command(
7084 command: SupervisorCommand,
7085 spec: &mut ModuleSpec,
7086 runtime: &mut SupervisorRuntimeConfig,
7087 registry: &Arc<Registry>,
7088 process_liveness: &SupervisorProcessLiveness,
7089 snapshot: &SharedSnapshot,
7090 child: &mut Option<SupervisedChild>,
7091 commands: &mut mpsc::Receiver<SupervisorCommand>,
7092 requeued: &mut VecDeque<SupervisorCommand>,
7093) -> bool {
7094 let command =
7095 if let Some((pid, reason)) = runtime.child_roster.previous_instance(&spec.module_id) {
7096 match command {
7097 SupervisorCommand::Restart { reply, .. }
7098 | SupervisorCommand::Reload { reply, .. }
7099 | SupervisorCommand::Swap { reply, .. } => {
7100 let _ = reply.send(Err(previous_instance_error(spec, pid, reason)));
7101 return true;
7102 }
7103 SupervisorCommand::SetEnabled {
7104 enabled: true,
7105 reply,
7106 ..
7107 } => {
7108 let _ = reply.send(Err(previous_instance_error(spec, pid, reason)));
7109 return true;
7110 }
7111 other => other,
7112 }
7113 } else {
7114 command
7115 };
7116 match command {
7117 SupervisorCommand::Drain { requester, reply } => {
7118 let result = drain_optional_child(
7121 &spec.module_id,
7122 spec.protocol,
7123 StopNotice::NotSent,
7124 registry,
7125 runtime.forwarding.as_deref(),
7126 snapshot,
7127 &runtime.terminal_ring,
7128 &runtime.spawn_events,
7129 child,
7130 runtime.drain_timeout,
7131 ModuleState::Stopped,
7132 None,
7133 )
7134 .await;
7135 let registration_released = result.is_ok();
7136 if registration_released {
7137 runtime.record_activity_transition(
7138 "subc.module_stopped",
7139 &spec.module_id,
7140 requester,
7141 );
7142 }
7143 let _ = reply.send(result);
7144 if registration_released {
7145 process_liveness.untrack_if_current(&spec.module_id, snapshot);
7146 }
7147 false
7148 }
7149 SupervisorCommand::Retire { requester, reply } => {
7150 let result = async {
7151 let stop_notice = begin_forwarding_drain_if_configured(
7152 spec,
7153 runtime,
7154 registry,
7155 snapshot,
7156 None,
7157 RouteCloseReason::Disable,
7158 )
7159 .await?;
7160 drain_optional_child(
7161 &spec.module_id,
7162 spec.protocol,
7163 stop_notice,
7164 registry,
7165 runtime.forwarding.as_deref(),
7166 snapshot,
7167 &runtime.terminal_ring,
7168 &runtime.spawn_events,
7169 child,
7170 runtime.drain_timeout,
7171 ModuleState::Stopped,
7172 None,
7173 )
7174 .await
7175 }
7176 .await;
7177 let registration_released = result.is_ok();
7178 if registration_released {
7179 runtime.record_activity_transition(
7180 "subc.module_stopped",
7181 &spec.module_id,
7182 requester,
7183 );
7184 }
7185 let _ = reply.send(result);
7186 if registration_released {
7187 process_liveness.untrack_if_current(&spec.module_id, snapshot);
7188 }
7189 false
7190 }
7191 SupervisorCommand::Restart {
7192 requester,
7193 drain_timeout_ms,
7194 received_at_generation,
7195 queued_at,
7196 reply,
7197 } => {
7198 info!(
7202 module_id = %spec.module_id,
7203 queued_ms = u64::try_from(queued_at.elapsed().as_millis()).unwrap_or(u64::MAX),
7204 "restart command dequeued"
7205 );
7206 let validation = match lock_snapshot(snapshot) {
7218 Ok(state) if !state.enabled => Err(SuperviseError::Disabled {
7219 module_id: spec.module_id.clone(),
7220 }),
7221 Ok(_) => Ok(()),
7222 Err(err) => Err(err),
7223 };
7224 let initiated = validation.is_ok();
7225 let _ = reply.send(validation);
7226 let satisfied_by_generation = if initiated
7240 && child
7241 .as_ref()
7242 .is_some_and(|child| child.spawned_at > queued_at)
7243 {
7244 lock_snapshot(snapshot).ok().and_then(|state| {
7245 (!state.configuration_updated_since_spawn).then_some(state.spawn_generation)
7246 })
7247 } else {
7248 None
7249 };
7250 let satisfied_by_pending = initiated
7251 && child.is_none()
7252 && lock_snapshot(snapshot).ok().is_some_and(|mut state| {
7253 let pending = state.respawn_pending && !state.configuration_updated_since_spawn;
7257 if pending {
7258 state.coalesced_restart_pending = true;
7259 }
7260 pending
7261 });
7262 if satisfied_by_pending {
7263 debug!(module_id = %spec.module_id, "restart coalesced into the pending replacement");
7264 } else if let Some(generation) = satisfied_by_generation {
7265 info!(
7266 module_id = %spec.module_id,
7267 received_at_generation,
7268 "restart already satisfied by generation {generation}; not restarting again"
7269 );
7270 } else if initiated {
7271 let drain_timeout = drain_timeout_ms
7274 .map(Duration::from_millis)
7275 .unwrap_or(runtime.drain_timeout);
7276 if let Err(err) = restart_child(
7277 spec,
7278 runtime,
7279 registry,
7280 process_liveness,
7281 snapshot,
7282 child,
7283 drain_timeout,
7284 )
7285 .await
7286 {
7287 warn!(
7288 module_id = %spec.module_id,
7289 error = %err,
7290 "operator restart failed after initiation ack; module state carries the outcome"
7291 );
7292 let _ = update_snapshot(snapshot, Some(&spec.module_id), |state| {
7293 state.state = ModuleState::Failed;
7294 clear_current_process_facts(state);
7295 });
7296 } else {
7297 runtime.start_requester = requester;
7298 runtime.pending_activity_restart = true;
7299 }
7300 }
7301 true
7302 }
7303 SupervisorCommand::Reload { requester, reply } => {
7304 let result =
7305 reload_child(spec, runtime, registry, process_liveness, snapshot, child).await;
7306 if result.is_ok() {
7307 runtime.start_requester = requester;
7308 runtime.pending_activity_restart = true;
7309 }
7310 if result.is_ok()
7311 && runtime
7312 .scheduled_respawn
7313 .lock()
7314 .unwrap_or_else(|p| p.into_inner())
7315 .as_ref()
7316 .is_some_and(|pending| matches!(pending.kind, RespawnKind::Reload))
7317 {
7318 *runtime
7319 .deferred_reload_reply
7320 .lock()
7321 .unwrap_or_else(|p| p.into_inner()) = Some(reply);
7322 } else {
7323 let _ = reply.send(result);
7324 }
7325 true
7326 }
7327 SupervisorCommand::SetEnabled {
7328 enabled,
7329 requester,
7330 reply,
7331 } => {
7332 let previous_requester = runtime.start_requester.clone();
7333 if enabled {
7334 runtime.start_requester = requester.clone();
7335 }
7336 let result = set_child_enabled(
7337 spec,
7338 runtime,
7339 registry,
7340 process_liveness,
7341 snapshot,
7342 child,
7343 enabled,
7344 )
7345 .await;
7346 if matches!(result, Ok(true)) {
7347 runtime.pending_activity_restart = false;
7348 if enabled {
7349 runtime.start_requester = requester;
7350 } else {
7351 runtime.start_requester = None;
7352 runtime.record_activity_transition(
7353 "subc.module_disabled",
7354 &spec.module_id,
7355 requester,
7356 );
7357 }
7358 } else {
7359 runtime.start_requester = previous_requester;
7360 }
7361 let _ = reply.send(result);
7362 true
7363 }
7364 SupervisorCommand::UpdateConfiguration {
7365 spec: next_spec,
7366 health,
7367 drain_timeout_ms,
7368 reply,
7369 } => {
7370 if let Some(handle) = &runtime.supervisor_handle {
7371 handle.apply_identity_configuration(&next_spec);
7372 }
7373 *spec = next_spec;
7374 let _ = update_snapshot(snapshot, Some(&spec.module_id), |state| {
7375 state.configuration_updated_since_spawn = true;
7376 });
7377 let health_changed = runtime.health != health;
7378 runtime.health = health;
7379 if health_changed {
7382 let _ = update_snapshot(snapshot, Some(&spec.module_id), |state| {
7383 state.health = ModuleHealthStatus::default();
7384 });
7385 }
7386 runtime.drain_timeout = drain_timeout_ms
7387 .map(Duration::from_millis)
7388 .unwrap_or(runtime.default_drain_timeout);
7389 *runtime
7390 .effective_drain_timeout
7391 .lock()
7392 .unwrap_or_else(|poisoned| poisoned.into_inner()) = runtime.drain_timeout;
7393 let _ = reply.send(());
7394 true
7395 }
7396 SupervisorCommand::Swap {
7397 requester,
7398 ready_timeout,
7399 reply,
7400 } => {
7401 let mut swap_runtime = runtime.clone();
7402 swap_runtime.start_requester = requester;
7403 let before = lock_snapshot(snapshot)
7404 .ok()
7405 .map(|state| state.spawn_generation);
7406 let end = swap::run_swap(
7407 spec,
7408 &swap_runtime,
7409 registry,
7410 process_liveness,
7411 snapshot,
7412 child,
7413 commands,
7414 ready_timeout.unwrap_or(DEFAULT_SWAP_READY_TIMEOUT),
7415 reply,
7416 )
7417 .await;
7418 if before
7419 != lock_snapshot(snapshot)
7420 .ok()
7421 .map(|state| state.spawn_generation)
7422 {
7423 runtime.start_requester = swap_runtime.start_requester;
7424 }
7425 requeued.extend(end.requeue);
7426 true
7427 }
7428 }
7429}
7430
7431async fn restart_child(
7432 spec: &ModuleSpec,
7433 runtime: &SupervisorRuntimeConfig,
7434 registry: &Registry,
7435 process_liveness: &SupervisorProcessLiveness,
7436 snapshot: &SharedSnapshot,
7437 child: &mut Option<SupervisedChild>,
7438 drain_timeout: Duration,
7439) -> Result<(), SuperviseError> {
7440 if !lock_snapshot(snapshot)?.enabled {
7442 return Err(SuperviseError::Disabled {
7443 module_id: spec.module_id.clone(),
7444 });
7445 }
7446 let stop_notice = begin_forwarding_drain_with_timeout(
7447 spec,
7448 runtime,
7449 registry,
7450 snapshot,
7451 None,
7452 RouteCloseReason::Restart,
7453 drain_timeout,
7454 )
7455 .await?;
7456
7457 if child.is_some() {
7458 drain_optional_child(
7459 &spec.module_id,
7460 spec.protocol,
7461 stop_notice,
7462 registry,
7463 runtime.forwarding.as_deref(),
7464 snapshot,
7465 &runtime.terminal_ring,
7466 &runtime.spawn_events,
7467 child,
7468 drain_timeout,
7469 ModuleState::Restarting,
7470 Some(true),
7471 )
7472 .await?;
7473 } else {
7474 update_snapshot(snapshot, Some(&spec.module_id), |state| {
7475 state.enabled = true;
7476 state.state = ModuleState::Restarting;
7477 clear_current_process_facts(state);
7478 })?;
7479 release_dead_registration(
7480 registry,
7481 runtime.forwarding.as_deref(),
7482 snapshot,
7483 &spec.module_id,
7484 )
7485 .await?;
7486 }
7487
7488 reset_restart_count(snapshot, &spec.module_id)?;
7489 process_liveness.track(spec.module_id.clone(), Arc::clone(snapshot));
7490 schedule_respawn(
7491 runtime,
7492 snapshot,
7493 &spec.module_id,
7494 runtime.restart_policy.backoff,
7495 RespawnKind::Spawn,
7496 )
7497}
7498
7499async fn reload_child(
7500 spec: &ModuleSpec,
7501 runtime: &SupervisorRuntimeConfig,
7502 registry: &Registry,
7503 process_liveness: &SupervisorProcessLiveness,
7504 snapshot: &SharedSnapshot,
7505 child: &mut Option<SupervisedChild>,
7506) -> Result<(), SuperviseError> {
7507 if !lock_snapshot(snapshot)?.enabled {
7509 return Err(SuperviseError::Disabled {
7510 module_id: spec.module_id.clone(),
7511 });
7512 }
7513 let stop_notice = begin_forwarding_drain(
7514 spec,
7515 runtime,
7516 registry,
7517 snapshot,
7518 Some(true),
7519 RouteCloseReason::Reload,
7520 )
7521 .await?;
7522
7523 if child.is_some() {
7524 drain_optional_child(
7525 &spec.module_id,
7526 spec.protocol,
7527 stop_notice,
7528 registry,
7529 runtime.forwarding.as_deref(),
7530 snapshot,
7531 &runtime.terminal_ring,
7532 &runtime.spawn_events,
7533 child,
7534 runtime.drain_timeout,
7535 ModuleState::Restarting,
7536 Some(true),
7537 )
7538 .await?;
7539 } else {
7540 update_snapshot(snapshot, Some(&spec.module_id), |state| {
7541 state.enabled = true;
7542 state.state = ModuleState::Restarting;
7543 clear_current_process_facts(state);
7544 })?;
7545 release_dead_registration(
7546 registry,
7547 runtime.forwarding.as_deref(),
7548 snapshot,
7549 &spec.module_id,
7550 )
7551 .await?;
7552 }
7553
7554 reset_restart_count(snapshot, &spec.module_id)?;
7555 process_liveness.track(spec.module_id.clone(), Arc::clone(snapshot));
7556 schedule_respawn(
7557 runtime,
7558 snapshot,
7559 &spec.module_id,
7560 runtime.restart_policy.backoff,
7561 RespawnKind::Reload,
7562 )
7563}
7564
7565async fn finish_reload_child(
7566 spec: &ModuleSpec,
7567 runtime: &SupervisorRuntimeConfig,
7568 registry: &Registry,
7569 process_liveness: &SupervisorProcessLiveness,
7570 snapshot: &SharedSnapshot,
7571 child: &mut Option<SupervisedChild>,
7572) -> Result<(), SuperviseError> {
7573 process_liveness.track(spec.module_id.clone(), Arc::clone(snapshot));
7574 let next_child = match spawn_and_mark_running(spec, runtime, snapshot) {
7575 Ok(next_child) => next_child,
7576 Err(err) => {
7577 return handle_reload_spawn_failure(
7578 spec,
7579 runtime,
7580 process_liveness,
7581 snapshot,
7582 child,
7583 format!("new child failed to spawn: {err}"),
7584 )
7585 .await;
7586 }
7587 };
7588 *child = Some(next_child);
7589
7590 let wait_outcome = {
7591 let active_child = child.as_mut().expect("new reload child was just stored");
7592 wait_for_registration_after_reload(
7593 registry,
7594 &spec.module_id,
7595 snapshot,
7596 active_child,
7597 REGISTRY_RELEASE_TIMEOUT,
7598 )
7599 .await?
7600 };
7601
7602 match wait_outcome {
7603 RegistrationWaitOutcome::Registered => {
7604 debug!(module_id = %spec.module_id, "supervised module reloaded and registered");
7605 Ok(())
7606 }
7607 RegistrationWaitOutcome::Exited(exit_report) => {
7608 if let Some(active_child) = child.as_mut() {
7609 active_child.drain_stderr(&spec.module_id).await;
7610 }
7611 let mut exited_child = child.take().expect("exited reload child is still stored");
7614 #[cfg(test)]
7615 if let Some(gate) = &runtime.test_reload_exit_record_gate {
7616 gate.reached.notify_one();
7617 gate.resume.notified().await;
7618 }
7619 let result = handle_reload_child_registration_failure(
7620 spec,
7621 runtime,
7622 registry,
7623 process_liveness,
7624 snapshot,
7625 child,
7626 ReloadRegistrationFailure {
7627 exit_report: registration_failure_exit_report(exit_report),
7628 reason: exited_child
7629 .spawn_failure
7630 .clone()
7631 .unwrap_or_else(|| "new child exited before registering".to_string()),
7632 },
7633 )
7634 .await;
7635 exited_child.release_roster();
7636 result
7637 }
7638 RegistrationWaitOutcome::TimedOut => {
7639 let mut timed_out_child = child
7640 .take()
7641 .expect("timed-out reload child is still running");
7642 timed_out_child
7643 .start_kill()
7644 .map_err(|source| SuperviseError::Kill {
7645 module_id: spec.module_id.clone(),
7646 source,
7647 })?;
7648 let status = timed_out_child
7649 .wait()
7650 .await
7651 .map_err(|source| SuperviseError::Wait {
7652 module_id: spec.module_id.clone(),
7653 source,
7654 })?;
7655 timed_out_child.drain_stderr(&spec.module_id).await;
7656 handle_reload_child_registration_failure(
7657 spec,
7658 runtime,
7659 registry,
7660 process_liveness,
7661 snapshot,
7662 child,
7663 ReloadRegistrationFailure {
7664 exit_report: registration_failure_exit_report(classify_reaped_child_exit(
7665 snapshot,
7666 &timed_out_child,
7667 &status,
7668 )),
7669 reason: format!(
7670 "new child did not register within {:?}",
7671 REGISTRY_RELEASE_TIMEOUT
7672 ),
7673 },
7674 )
7675 .await
7676 }
7677 }
7678}
7679
7680async fn set_child_enabled(
7681 spec: &ModuleSpec,
7682 runtime: &SupervisorRuntimeConfig,
7683 registry: &Registry,
7684 process_liveness: &SupervisorProcessLiveness,
7685 snapshot: &SharedSnapshot,
7686 child: &mut Option<SupervisedChild>,
7687 enabled: bool,
7688) -> Result<bool, SuperviseError> {
7689 let (current_enabled, current_state, respawn_pending) = {
7690 let state = lock_snapshot(snapshot)?;
7691 (state.enabled, state.state, state.respawn_pending)
7692 };
7693 let revive_terminal = enabled
7701 && current_enabled
7702 && child.is_none()
7703 && (matches!(current_state, ModuleState::Failed | ModuleState::Stopped)
7704 || (current_state == ModuleState::Restarting && !respawn_pending));
7705 if current_enabled == enabled && !revive_terminal {
7706 return Ok(false);
7707 }
7708
7709 if enabled {
7710 update_snapshot(snapshot, Some(&spec.module_id), |state| {
7711 state.enabled = true;
7712 state.state = ModuleState::Starting;
7713 clear_current_process_facts(state);
7714 })?;
7715 #[cfg(test)]
7716 if runtime.test_seed_stale_facts_before_enable_spawn {
7717 update_snapshot(snapshot, Some(&spec.module_id), |state| {
7718 state.process_alive = true;
7719 state.pid = Some(41);
7720 state.spawned_at_ms = Some(42);
7721 state.spawned_from = Some(PathBuf::from("/spawned/module"));
7722 state.spawned_file_identity = Some(SpawnedFileIdentity {
7723 device: 43,
7724 inode: 44,
7725 #[cfg(windows)]
7726 windows_image: None,
7727 });
7728 })?;
7729 }
7730 release_dead_registration(
7731 registry,
7732 runtime.forwarding.as_deref(),
7733 snapshot,
7734 &spec.module_id,
7735 )
7736 .await?;
7737 reset_restart_count(snapshot, &spec.module_id)?;
7738 process_liveness.track(spec.module_id.clone(), Arc::clone(snapshot));
7739 let next_child = match spawn_and_mark_running(spec, runtime, snapshot) {
7740 Ok(next_child) => next_child,
7741 Err(err) => {
7742 if let Err(state_err) = update_snapshot(snapshot, Some(&spec.module_id), |state| {
7743 state.state = ModuleState::Failed;
7744 clear_current_process_facts(state);
7745 }) {
7746 error!(module_id = %spec.module_id, error = %state_err, "failed to record enable spawn failure");
7747 }
7748 process_liveness.untrack_if_current(&spec.module_id, snapshot);
7749 return Err(err);
7750 }
7751 };
7752 *child = Some(next_child);
7753 debug!(module_id = %spec.module_id, "supervised module enabled");
7754 Ok(true)
7755 } else {
7756 let stop_notice = begin_forwarding_drain_if_configured(
7757 spec,
7758 runtime,
7759 registry,
7760 snapshot,
7761 Some(false),
7762 RouteCloseReason::Disable,
7763 )
7764 .await?;
7765 drain_optional_child(
7766 &spec.module_id,
7767 spec.protocol,
7768 stop_notice,
7769 registry,
7770 runtime.forwarding.as_deref(),
7771 snapshot,
7772 &runtime.terminal_ring,
7773 &runtime.spawn_events,
7774 child,
7775 runtime.drain_timeout,
7776 ModuleState::Disabled,
7777 Some(false),
7778 )
7779 .await?;
7780 debug!(module_id = %spec.module_id, "supervised module disabled");
7781 Ok(true)
7782 }
7783}
7784
7785#[allow(clippy::too_many_arguments)]
7786async fn on_child_exit(
7787 spec: &ModuleSpec,
7788 policy: RestartPolicy,
7789 registry: &Registry,
7790 snapshot: &SharedSnapshot,
7791 terminal_ring: &Arc<Mutex<TerminalRing>>,
7792 spawn_events: &SpawnEventFeed,
7793 roster: &ChildRoster,
7794 exit_report: ExitReport,
7795) -> NextAction {
7796 if roster.is_closed() {
7802 return on_child_exit_during_daemon_shutdown(
7803 spec,
7804 registry,
7805 snapshot,
7806 terminal_ring,
7807 spawn_events,
7808 exit_report,
7809 )
7810 .await;
7811 }
7812 let unrequested_clean_exit_of_protocol_none = exit_report.kind == ExitKind::Clean
7827 && running_protocol(spec, snapshot) == ModuleProtocol::None;
7828 match exit_report.kind {
7829 ExitKind::Clean if !unrequested_clean_exit_of_protocol_none => {
7830 info!(
7831 module_id = %spec.module_id,
7832 exit_code = ?exit_report.code,
7833 exit_signal = ?exit_report.signal,
7834 "supervised module exited cleanly"
7835 );
7836 if let Err(err) = publish_after_terminal_record(
7837 &spec.module_id,
7838 || {
7839 record_terminal(
7840 &spec.module_id,
7841 terminal_ring,
7842 spawn_events,
7843 &exit_report,
7844 TerminalDisposition::Stopped,
7845 )
7846 },
7847 || {
7848 update_snapshot(snapshot, Some(&spec.module_id), |state| {
7849 state.state = ModuleState::Stopped;
7850 clear_current_process_facts(state);
7851 state.last_exit = Some(exit_report.clone());
7852 })
7853 },
7854 ) {
7855 error!(module_id = %spec.module_id, error = %err, "failed to publish clean module exit");
7856 }
7857 let registration_released = match wait_for_registration_release(
7858 registry,
7859 &spec.module_id,
7860 REGISTRY_RELEASE_TIMEOUT,
7861 )
7862 .await
7863 {
7864 Ok(()) => true,
7865 Err(err) => {
7866 warn!(module_id = %spec.module_id, error = %err, "registration still active after clean exit");
7867 false
7868 }
7869 };
7870 NextAction::Stop {
7871 registration_released,
7872 }
7873 }
7874 ExitKind::Clean | ExitKind::Crash => {
7875 if unrequested_clean_exit_of_protocol_none {
7876 warn!(
7877 module_id = %spec.module_id,
7878 exit_code = ?exit_report.code,
7879 exit_signal = ?exit_report.signal,
7880 "protocol-none module exited cleanly without a stop request; handling it as a crash"
7881 );
7882 } else {
7883 warn!(
7884 module_id = %spec.module_id,
7885 exit_code = ?exit_report.code,
7886 exit_signal = ?exit_report.signal,
7887 "supervised module exited abnormally (crash)"
7888 );
7889 }
7890 let mut restart_schedule = None;
7891 let mut disposition = TerminalDisposition::Disabled;
7892 let mut final_state = ModuleState::Disabled;
7893 let mut disposition_detail = lock_snapshot(snapshot)
7897 .ok()
7898 .and_then(|mut state| state.spawn_failure.take());
7899 let now = Instant::now();
7900 if let Err(err) = update_snapshot(snapshot, Some(&spec.module_id), |state| {
7901 if state.enabled {
7902 if let Some(schedule) = state.next_crash_restart(&policy, now) {
7903 state.respawn_pending = true;
7909 final_state = ModuleState::Restarting;
7910 restart_schedule = Some(schedule);
7911 disposition = TerminalDisposition::Restarting;
7912 } else {
7913 final_state = ModuleState::Failed;
7914 disposition = TerminalDisposition::Failed;
7915 let budget = policy.budget_exhausted_detail();
7916 disposition_detail =
7917 Some(disposition_detail.take().map_or_else(
7918 || budget.clone(),
7919 |cause| format!("{cause}; {budget}"),
7920 ));
7921 }
7922 } else {
7923 disposition = TerminalDisposition::Disabled;
7924 }
7925 }) {
7926 error!(module_id = %spec.module_id, error = %err, "failed to record crashed module exit");
7927 return NextAction::Stop {
7928 registration_released: false,
7929 };
7930 }
7931 if disposition == TerminalDisposition::Failed {
7932 error!(
7937 module_id = %spec.module_id,
7938 max_restarts = policy.max_restarts,
7939 window_secs = policy.window.as_secs(),
7940 "module stopped: {}",
7941 policy.budget_exhausted_detail()
7942 );
7943 }
7944 if let Err(err) = publish_after_terminal_record(
7945 &spec.module_id,
7946 || {
7947 record_terminal_with_detail(
7948 &spec.module_id,
7949 terminal_ring,
7950 spawn_events,
7951 &exit_report,
7952 disposition,
7953 disposition_detail,
7954 )
7955 },
7956 || {
7957 update_snapshot(snapshot, Some(&spec.module_id), |state| {
7958 state.state = final_state;
7959 clear_current_process_facts(state);
7960 state.last_exit = Some(exit_report.clone());
7961 })
7962 },
7963 ) {
7964 error!(module_id = %spec.module_id, error = %err, "failed to publish crashed module exit");
7965 }
7966
7967 if let Some(schedule) = restart_schedule {
7968 NextAction::Restart {
7969 schedule: Some(schedule),
7970 }
7971 } else {
7972 let registration_released = match wait_for_registration_release(
7973 registry,
7974 &spec.module_id,
7975 REGISTRY_RELEASE_TIMEOUT,
7976 )
7977 .await
7978 {
7979 Ok(()) => true,
7980 Err(err) => {
7981 warn!(module_id = %spec.module_id, error = %err, "registration still active after failed module");
7982 false
7983 }
7984 };
7985 NextAction::Stop {
7986 registration_released,
7987 }
7988 }
7989 }
7990 ExitKind::DeliberateSeverance => {
7991 warn!(
7992 module_id = %spec.module_id,
7993 exit_code = ?exit_report.code,
7994 exit_signal = ?exit_report.signal,
7995 "supervised module exited after deliberate connection severance"
7996 );
7997 let should_restart = match lock_snapshot(snapshot) {
7998 Ok(state) => state.enabled,
7999 Err(err) => {
8000 error!(module_id = %spec.module_id, error = %err, "failed to read deliberately severed module state");
8001 return NextAction::Stop {
8002 registration_released: false,
8003 };
8004 }
8005 };
8006 let final_state = if should_restart {
8007 ModuleState::Restarting
8008 } else {
8009 ModuleState::Disabled
8010 };
8011 if let Err(err) = publish_after_terminal_record(
8012 &spec.module_id,
8013 || {
8014 record_terminal(
8015 &spec.module_id,
8016 terminal_ring,
8017 spawn_events,
8018 &exit_report,
8019 terminal_disposition(final_state),
8020 )
8021 },
8022 || {
8023 update_snapshot(snapshot, Some(&spec.module_id), |state| {
8024 state.state = final_state;
8025 clear_current_process_facts(state);
8026 state.last_exit = Some(exit_report.clone());
8027 state.lifetime_restarts += 1;
8028 })
8029 },
8030 ) {
8031 error!(module_id = %spec.module_id, error = %err, "failed to publish deliberately severed module exit");
8032 return NextAction::Stop {
8033 registration_released: false,
8034 };
8035 }
8036
8037 if should_restart {
8038 NextAction::Restart { schedule: None }
8039 } else {
8040 let registration_released = match wait_for_registration_release(
8041 registry,
8042 &spec.module_id,
8043 REGISTRY_RELEASE_TIMEOUT,
8044 )
8045 .await
8046 {
8047 Ok(()) => true,
8048 Err(err) => {
8049 warn!(module_id = %spec.module_id, error = %err, "registration still active after deliberately severed module exit");
8050 false
8051 }
8052 };
8053 NextAction::Stop {
8054 registration_released,
8055 }
8056 }
8057 }
8058 }
8059}
8060
8061async fn on_child_exit_during_daemon_shutdown(
8062 spec: &ModuleSpec,
8063 registry: &Registry,
8064 snapshot: &SharedSnapshot,
8065 terminal_ring: &Arc<Mutex<TerminalRing>>,
8066 spawn_events: &SpawnEventFeed,
8067 exit_report: ExitReport,
8068) -> NextAction {
8069 info!(
8070 module_id = %spec.module_id,
8071 exit_code = ?exit_report.code,
8072 exit_signal = ?exit_report.signal,
8073 exit_kind = ?exit_report.kind,
8074 "supervised module exited during daemon shutdown; not restarting it"
8075 );
8076 if let Err(err) = publish_after_terminal_record(
8077 &spec.module_id,
8078 || {
8079 record_terminal(
8080 &spec.module_id,
8081 terminal_ring,
8082 spawn_events,
8083 &exit_report,
8084 TerminalDisposition::DaemonShutdown,
8085 )
8086 },
8087 || {
8088 update_snapshot(snapshot, Some(&spec.module_id), |state| {
8089 state.state = ModuleState::Stopped;
8090 clear_current_process_facts(state);
8091 state.last_exit = Some(exit_report.clone());
8092 })
8093 },
8094 ) {
8095 error!(module_id = %spec.module_id, error = %err, "failed to publish module exit during daemon shutdown");
8096 }
8097 let registration_released =
8098 wait_for_registration_release(registry, &spec.module_id, REGISTRY_RELEASE_TIMEOUT)
8099 .await
8100 .is_ok();
8101 NextAction::Stop {
8102 registration_released,
8103 }
8104}
8105
8106fn publish_after_terminal_record<T>(
8111 module_id: &str,
8112 record: impl FnOnce(),
8113 publish: impl FnOnce() -> T,
8114) -> T {
8115 let recorded = std::panic::catch_unwind(std::panic::AssertUnwindSafe(record));
8116 let published = publish();
8117 if let Err(panic) = recorded {
8118 error!(
8119 module_id,
8120 "terminal recording panicked; published final state before resuming unwind"
8121 );
8122 std::panic::resume_unwind(panic);
8123 }
8124 published
8125}
8126
8127fn fail_after_wait_error(
8128 module_id: &str,
8129 snapshot: &SharedSnapshot,
8130 terminal_ring: &Arc<Mutex<TerminalRing>>,
8131 spawn_events: &SpawnEventFeed,
8132 detail: String,
8133) {
8134 let report = wait_error_exit_report();
8135 publish_after_terminal_record(
8136 module_id,
8137 || {
8138 record_terminal_with_detail(
8139 module_id,
8140 terminal_ring,
8141 spawn_events,
8142 &report,
8143 TerminalDisposition::Failed,
8144 Some(detail),
8145 )
8146 },
8147 || fail_snapshot(snapshot, Some(module_id), Some(report.clone())),
8148 );
8149}
8150
8151#[cfg(test)]
8152fn record_wait_error_terminal(
8153 module_id: &str,
8154 terminal_ring: &Arc<Mutex<TerminalRing>>,
8155 spawn_events: &SpawnEventFeed,
8156) {
8157 record_terminal(
8158 module_id,
8159 terminal_ring,
8160 spawn_events,
8161 &wait_error_exit_report(),
8162 TerminalDisposition::Failed,
8163 );
8164}
8165
8166fn record_terminal(
8167 module_id: &str,
8168 terminal_ring: &Arc<Mutex<TerminalRing>>,
8169 spawn_events: &SpawnEventFeed,
8170 exit_report: &ExitReport,
8171 disposition: TerminalDisposition,
8172) {
8173 record_terminal_with_detail(
8174 module_id,
8175 terminal_ring,
8176 spawn_events,
8177 exit_report,
8178 disposition,
8179 None,
8180 );
8181}
8182
8183fn durable_terminal_history_of(
8187 terminal_ring: &Mutex<TerminalRing>,
8188 module_id: &str,
8189) -> subc_control::TerminalHistory {
8190 let read = terminal_ring
8191 .lock()
8192 .unwrap_or_else(|p| p.into_inner())
8193 .capture_durable_history();
8194 read.read(module_id)
8195}
8196
8197fn record_terminal_with_detail(
8198 module_id: &str,
8199 terminal_ring: &Arc<Mutex<TerminalRing>>,
8200 spawn_events: &SpawnEventFeed,
8201 exit_report: &ExitReport,
8202 disposition: TerminalDisposition,
8203 disposition_detail: Option<String>,
8204) {
8205 #[cfg(test)]
8206 terminal_history_tests::before_terminal_record(spawn_events);
8207 spawn_events.emit_exited(module_id, exit_report.code, exit_report.signal);
8208 let record = TerminalRecord {
8209 exit_code: exit_report.code,
8210 exit_signal: exit_report.signal,
8211 at_ms: exit_report.at_ms,
8212 disposition,
8213 exit_kind: exit_report.kind.into(),
8214 disposition_detail,
8215 };
8216 terminal_ring
8217 .lock()
8218 .unwrap_or_else(|poisoned| poisoned.into_inner())
8219 .record_exit(module_id, record);
8220}
8221
8222fn untrack_if_registration_released(
8223 process_liveness: &SupervisorProcessLiveness,
8224 registry: &Registry,
8225 module_id: &str,
8226 snapshot: &SharedSnapshot,
8227) {
8228 match registry.get_module(module_id) {
8229 Ok(None) => process_liveness.untrack_if_current(module_id, snapshot),
8230 Ok(Some(_)) => {}
8231 Err(err) => {
8232 warn!(module_id, error = %err, "could not determine whether supervisor liveness can be untracked");
8233 }
8234 }
8235}
8236
8237#[cfg(test)]
8251fn apply_wire_spawn_args(
8252 command: &mut Command,
8253 spec: &ModuleSpec,
8254 connection_file_path: Option<&std::path::Path>,
8255 handle: Option<&SupervisorHandle>,
8256) -> Result<Option<NonceHandoff>, SuperviseError> {
8257 apply_wire_spawn_args_for_role(
8258 command,
8259 spec,
8260 connection_file_path,
8261 handle,
8262 SpawnRole::Plain,
8263 )
8264}
8265
8266#[cfg(unix)]
8271type NonceHandoff = subc_os::LaunchNonceHandoff;
8272#[cfg(windows)]
8273type NonceHandoff = subc_os::LaunchNoncePipeHandoff;
8274#[cfg(not(any(unix, windows)))]
8275type NonceHandoff = std::convert::Infallible;
8276
8277fn apply_wire_spawn_args_for_role(
8298 command: &mut Command,
8299 spec: &ModuleSpec,
8300 connection_file_path: Option<&std::path::Path>,
8301 handle: Option<&SupervisorHandle>,
8302 role: SpawnRole,
8303) -> Result<Option<NonceHandoff>, SuperviseError> {
8304 command.env(SUBC_MODULE_ID_ENV, &spec.module_id);
8305 command.env_remove(subc_os::LAUNCH_NONCE_FD_ENV);
8310 #[cfg(windows)]
8311 {
8312 command.env_remove(subc_os::LAUNCH_NONCE_PIPE_ENV);
8313 command.env_remove(subc_os::LAUNCH_NONCE_PIPE_FALLBACK_ENV);
8314 }
8315 command.env_remove(SUBC_LAUNCH_NONCE_ENV);
8317 if spec.protocol == ModuleProtocol::None {
8318 return Ok(None);
8319 }
8320 if let Some(connection_file_path) = connection_file_path {
8321 command.arg(SUBC_ARG).arg(connection_file_path);
8322 }
8323
8324 let nonce = generate_launch_nonce()?;
8328 if let Some(handle) = handle {
8329 match role {
8330 SpawnRole::Plain => {
8331 handle.set_spawn_nonce(&spec.module_id, nonce.clone());
8332 if spec.reserved {
8333 handle.set_reserved_nonce(&spec.module_id, nonce.clone());
8334 }
8335 }
8336 SpawnRole::SwapCandidate => handle.open_swap(&spec.module_id, nonce.clone()),
8337 }
8338 }
8339 #[cfg(unix)]
8340 let handoff = {
8341 let handoff =
8342 subc_os::LaunchNonceHandoff::new(&nonce).map_err(|source| SuperviseError::Spawn {
8343 program: spec.program.clone(),
8344 source,
8345 cgroup_path: None,
8346 })?;
8347 command.env(subc_os::LAUNCH_NONCE_FD_ENV, handoff.fd_env_value());
8348 Some(handoff)
8349 };
8350 #[cfg(windows)]
8351 let handoff = prepare_windows_nonce_pipe(command, spec, &nonce);
8352 #[cfg(not(any(unix, windows)))]
8353 let handoff = None;
8354 #[cfg(not(unix))]
8358 command.env(SUBC_LAUNCH_NONCE_ENV, nonce);
8359 Ok(handoff)
8360}
8361
8362fn apply_child_env(command: &mut Command, spec: &ModuleSpec) {
8363 command.env_remove(CK_LOG_ENV);
8364 command.env_remove(SUBC_SPAWN_ROLE_ENV);
8371 for (key, value) in &spec.env {
8372 if matches!(
8376 key.as_str(),
8377 CAPTURE_MAX_FILE_MB_ENV | CAPTURE_KEEP_ENV | CAPTURE_MAX_AGE_DAYS_ENV
8378 ) || key == SUBC_SPAWN_ROLE_ENV
8379 {
8380 continue;
8381 }
8382 command.env(key, value);
8383 }
8384}
8385
8386#[cfg(windows)]
8387fn prepare_windows_nonce_pipe(
8388 command: &mut Command,
8389 spec: &ModuleSpec,
8390 nonce: &str,
8391) -> Option<NonceHandoff> {
8392 match subc_os::LaunchNoncePipeHandoff::new(nonce) {
8393 Ok(handoff) => {
8394 command.env(subc_os::LAUNCH_NONCE_PIPE_ENV, handoff.name());
8395 command.env(subc_os::LAUNCH_NONCE_PIPE_FALLBACK_ENV, "env");
8400 Some(handoff)
8401 }
8402 Err(error) => {
8403 warn!(module_id = %spec.module_id, %error, "could not prepare launch nonce pipe; retaining the environment handoff");
8404 None
8405 }
8406 }
8407}
8408
8409#[cfg(windows)]
8410fn start_windows_nonce_pipe(
8411 handoff: Option<NonceHandoff>,
8412 pid: u32,
8413 spec: &ModuleSpec,
8414) -> Option<subc_os::LaunchNoncePipeDelivery> {
8415 handoff.and_then(|handoff| match handoff.serve(pid, REGISTRY_RELEASE_TIMEOUT) {
8419 Ok(delivery) => Some(delivery),
8420 Err(error) => {
8421 warn!(module_id = %spec.module_id, %error, "could not serve launch nonce pipe; retaining the environment handoff");
8422 None
8423 }
8424 })
8425}
8426
8427fn observed_launch_nonce_source(snapshot: &SupervisorSnapshot) -> Option<String> {
8428 #[cfg(windows)]
8429 if snapshot.process_alive && snapshot.spawned_protocol == Some(ModuleProtocol::Subc) {
8430 return Some(
8431 if snapshot
8432 .pipe_consumed
8433 .as_ref()
8434 .is_some_and(|consumed| consumed.get().is_some())
8435 {
8436 "pipe"
8437 } else {
8438 "env"
8439 }
8440 .to_owned(),
8441 );
8442 }
8443 let _ = snapshot;
8444 None
8445}
8446
8447#[derive(Debug, Clone, Copy, PartialEq, Eq)]
8450enum SpawnRole {
8451 Plain,
8452 SwapCandidate,
8453}
8454
8455fn apply_spawn_role(command: &mut Command, role: SpawnRole) {
8458 if role == SpawnRole::SwapCandidate {
8459 command.env(SUBC_SPAWN_ROLE_ENV, SPAWN_ROLE_SWAP_CANDIDATE);
8460 }
8461}
8462
8463fn previous_instance_error(
8464 spec: &ModuleSpec,
8465 pid: u32,
8466 reason: crate::live_children::RetainedReason,
8467) -> SuperviseError {
8468 SuperviseError::Spawn {
8469 program: spec.program.clone(),
8470 source: io::Error::new(io::ErrorKind::WouldBlock, reason.waiting_message(pid)),
8471 cgroup_path: None,
8472 }
8473}
8474
8475fn spawn_child(
8476 spec: &ModuleSpec,
8477 connection_file_path: Option<&std::path::Path>,
8478 handle: Option<&SupervisorHandle>,
8479 ring: &Arc<Mutex<StderrRing>>,
8480 capture_logs_dir: Option<&std::path::Path>,
8481 roster: &ChildRoster,
8482 #[cfg(target_os = "linux")] cgroup_placement: Option<&subc_cgroup::Placement>,
8483) -> Result<SupervisedChild, SuperviseError> {
8484 spawn_child_in_slot(
8485 spec,
8486 connection_file_path,
8487 handle,
8488 ring,
8489 capture_logs_dir,
8490 roster,
8491 #[cfg(target_os = "linux")]
8492 cgroup_placement,
8493 SpawnRole::Plain,
8494 false,
8495 )
8496}
8497
8498#[allow(clippy::too_many_arguments)]
8511fn spawn_child_in_slot(
8512 spec: &ModuleSpec,
8513 connection_file_path: Option<&std::path::Path>,
8514 handle: Option<&SupervisorHandle>,
8515 ring: &Arc<Mutex<StderrRing>>,
8516 capture_logs_dir: Option<&std::path::Path>,
8517 roster: &ChildRoster,
8518 #[cfg(target_os = "linux")] cgroup_placement: Option<&subc_cgroup::Placement>,
8519 role: SpawnRole,
8520 alternate_slot: bool,
8521) -> Result<SupervisedChild, SuperviseError> {
8522 if let Some((pid, reason)) = roster.previous_instance(&spec.module_id) {
8523 return Err(previous_instance_error(spec, pid, reason));
8524 }
8525 if roster.is_closed() {
8526 return Err(SuperviseError::Spawn {
8527 program: spec.program.clone(),
8528 source: io::Error::other("the daemon is shutting down; not starting a new process"),
8529 cgroup_path: None,
8530 });
8531 }
8532 #[cfg(target_os = "linux")]
8533 let cgroup_name = {
8534 if cgroup_placement.is_none() {
8539 swap::cgroup_name(&spec.module_id, alternate_slot)
8540 } else {
8541 let nonce = generate_launch_nonce()?;
8542 let suffix = u128::from_str_radix(&nonce[..32], 16).expect("hex launch nonce");
8543 let mut end = spec.module_id.len().min(64);
8546 while !spec.module_id.is_char_boundary(end) {
8547 end -= 1;
8548 }
8549 format!(
8550 "{}_{suffix}",
8551 swap::cgroup_name(&spec.module_id[..end], alternate_slot)
8552 )
8553 }
8554 };
8555 #[cfg(not(target_os = "linux"))]
8556 let _ = alternate_slot;
8557 #[cfg(target_os = "macos")]
8558 let (mut command, privacy_exec, exec_ack) = privacy_command(spec, roster)?;
8559 #[cfg(windows)]
8560 let image_capture = windows_spawn_capture(spec).map_err(|source| SuperviseError::Spawn {
8561 program: spec.program.clone(),
8562 source,
8563 cgroup_path: None,
8564 })?;
8565 #[cfg(windows)]
8566 let mut command = Command::new(image_capture.path());
8567 #[cfg(not(any(target_os = "macos", windows)))]
8568 let mut command = Command::new(&spec.program);
8569 command.args(&spec.args);
8570 apply_child_env(&mut command, spec);
8600 apply_spawn_role(&mut command, role);
8601 let nonce_handoff =
8602 apply_wire_spawn_args_for_role(&mut command, spec, connection_file_path, handle, role)?;
8603
8604 #[cfg(target_os = "linux")]
8605 let cgroup_path = cgroup_placement
8606 .map(|placement| placement.module_path(&cgroup_name))
8607 .transpose()
8608 .map_err(|source| SuperviseError::Cgroup {
8609 module_id: spec.module_id.clone(),
8610 source,
8611 })?;
8612 #[cfg(not(target_os = "linux"))]
8613 let cgroup_path: Option<PathBuf> = None;
8614 #[cfg(target_os = "linux")]
8615 if let Some(path) = &cgroup_path {
8616 if let Err(error) = apply_cgroup_placement(&mut command, spec, path) {
8617 if let Some(placement) = cgroup_placement {
8618 remove_module_cgroup(placement, &cgroup_name);
8619 }
8620 return Err(error);
8621 }
8622 }
8623
8624 let output_sink = if let Some(logs_dir) = capture_logs_dir {
8625 let path = logs_dir.join(format!("{}.stderr.log", spec.module_id));
8626 match ChildOutputSink::open(&path, capture_retention(spec)) {
8627 Ok(sink) => sink,
8628 Err(error) => {
8629 warn!(
8630 module_id = %spec.module_id,
8631 path = %path.display(),
8632 error = %error,
8633 "could not open child output capture file; forwarding to stderr"
8634 );
8635 ChildOutputSink::Stderr
8636 }
8637 }
8638 } else {
8639 ChildOutputSink::Stderr
8640 };
8641
8642 command.stdout(Stdio::piped());
8643 command.stderr(Stdio::piped());
8644 command.kill_on_drop(true);
8645 #[cfg(unix)]
8662 command.process_group(0);
8663 command.stdin(Stdio::null());
8664 #[cfg(unix)]
8668 if let Some(handoff) = nonce_handoff {
8669 handoff.install_last(command.as_std_mut());
8670 }
8671 #[cfg(not(any(unix, windows)))]
8672 let _ = nonce_handoff;
8673
8674 #[cfg(windows)]
8679 {
8680 subc_jobobject::ensure_root_job().map_err(|source| SuperviseError::Spawn {
8681 program: spec.program.clone(),
8682 source,
8683 cgroup_path: None,
8684 })?;
8685 subc_jobobject::suspend_on_create_async(&mut command);
8686 }
8687 let spawned_at = Instant::now();
8691 let mut child = match command.spawn() {
8692 Ok(child) => child,
8693 Err(source) => {
8694 #[cfg(target_os = "linux")]
8695 if let Some(placement) = cgroup_placement {
8696 remove_module_cgroup(placement, &cgroup_name);
8697 }
8698 return Err(SuperviseError::Spawn {
8699 program: spec.program.clone(),
8700 source,
8701 cgroup_path,
8702 });
8703 }
8704 };
8705 #[cfg(test)]
8706 spawn_record_tests::after_create(&child, spec);
8707 #[cfg(target_os = "macos")]
8712 drop(exec_ack);
8713
8714 #[cfg(windows)]
8719 let windows_image = match image_capture.bind_tokio(&child) {
8720 Ok(image) => Arc::new(image),
8721 Err(source) => {
8722 kill_and_reap_rejected_child(&mut child);
8723 return Err(SuperviseError::Spawn {
8724 program: spec.program.clone(),
8725 source,
8726 cgroup_path: None,
8727 });
8728 }
8729 };
8730 #[cfg(windows)]
8733 let job = contain_spawned_child(&mut child, spec)?;
8734 let spawned_at_ms = unix_ms_now();
8735 #[cfg(not(windows))]
8736 let spawned_from = spec.program.clone();
8737 #[cfg(windows)]
8738 let spawned_from = windows_image.path().to_path_buf();
8739 #[cfg(not(windows))]
8740 let spawned_file_identity = spawned_file_identity(&spawned_from);
8741 #[cfg(windows)]
8742 let spawned_file_identity = Some(SpawnedFileIdentity {
8743 device: 0,
8744 inode: 0,
8745 windows_image: Some(Arc::clone(&windows_image)),
8746 });
8747 let pid = child.id().ok_or_else(|| SuperviseError::Spawn {
8748 program: spec.program.clone(),
8749 source: io::Error::other("spawned child exposed no live pid"),
8750 cgroup_path: cgroup_path.clone(),
8751 })?;
8752 #[cfg(windows)]
8753 let nonce_delivery = start_windows_nonce_pipe(nonce_handoff, pid, spec);
8754 #[cfg(not(windows))]
8755 let process_start_time = crate::provenance::process_start_time(pid);
8756 #[cfg(windows)]
8757 let process_start_time = Some(windows_image.start_time());
8758 let process_identity = process_start_time.map(|start_time| ProcessIdentity { pid, start_time });
8759 #[cfg(all(test, target_os = "macos"))]
8760 privacy_exec_boundary_tests::before_image_sample(spec, pid);
8761 #[cfg(not(windows))]
8766 let recorded_image = observe_spawned_image(pid);
8767 #[cfg(target_os = "macos")]
8771 let recorded_image = if privacy_exec.is_some() {
8772 None
8773 } else {
8774 recorded_image
8775 };
8776 #[cfg(target_os = "linux")]
8777 let recorded_cgroup_name = cgroup_path.as_ref().map(|_| cgroup_name.clone());
8778 #[cfg(not(target_os = "linux"))]
8779 let recorded_cgroup_name = None;
8780 let roster_guard = roster.admit(
8781 spec.module_id.clone(),
8782 pid,
8783 spec.protocol,
8784 process_start_time,
8785 crate::child_roster::RecordedIdentity {
8786 #[cfg(not(windows))]
8787 start_time: recorded_image.map(|image| image.start_time),
8788 #[cfg(windows)]
8789 start_time: process_start_time,
8790 #[cfg(not(windows))]
8791 executable: recorded_image
8792 .and_then(|image| image.executable)
8793 .map(crate::live_children::ExecutableIdentity::from),
8794 #[cfg(windows)]
8795 executable: Some(crate::live_children::ExecutableIdentity::from(
8796 windows_image.identity(),
8797 )),
8798 cgroup_name: recorded_cgroup_name,
8799 #[cfg(target_os = "linux")]
8800 cgroup_placement: cgroup_placement.cloned(),
8801 },
8802 );
8803 let roster_guard = match roster_guard {
8804 Ok(guard) => guard,
8805 Err(failure) => {
8806 return failure.with_cleanup(|source| {
8807 #[cfg(windows)]
8808 if let Some(job) = &job {
8809 let _ = job.terminate();
8810 }
8811 #[cfg(target_os = "linux")]
8812 kill_module_cgroup(cgroup_placement, &cgroup_name);
8813 kill_and_reap_rejected_child(&mut child);
8814 #[cfg(target_os = "linux")]
8815 if let Some(placement) = cgroup_placement {
8816 remove_module_cgroup(placement, &cgroup_name);
8817 }
8818 Err(SuperviseError::Spawn {
8819 program: spec.program.clone(),
8820 source: io::Error::other(format!(
8821 "could not durably record module child: {source}"
8822 )),
8823 cgroup_path,
8824 })
8825 });
8826 }
8827 };
8828 if roster.is_closed() {
8833 #[cfg(target_os = "linux")]
8835 kill_module_cgroup(cgroup_placement, &cgroup_name);
8836 if let Err(error) = child.start_kill() {
8837 debug!(module_id = %spec.module_id, pid, %error, "kill of a process spawned during daemon shutdown failed; it may already have exited");
8838 }
8839 #[cfg(target_os = "linux")]
8840 if let Some(placement) = cgroup_placement {
8841 while matches!(child.try_wait(), Ok(None)) {
8845 std::thread::yield_now();
8846 }
8847 if matches!(
8848 subc_cgroup::kill_module(Some(placement), &cgroup_name),
8849 subc_cgroup::KillOutcome::Killed
8850 ) {
8851 if let Ok(path) = placement.module_path(&cgroup_name) {
8852 while std::fs::read_to_string(path.join("cgroup.events"))
8853 .ok()
8854 .is_some_and(|events| events.lines().any(|line| line == "populated 1"))
8855 {
8856 std::thread::yield_now();
8857 }
8858 }
8859 }
8860 remove_module_cgroup(placement, &cgroup_name);
8861 }
8862 drop(roster_guard);
8863 return Err(SuperviseError::Spawn {
8864 program: spec.program.clone(),
8865 source: io::Error::other(
8866 "the daemon began shutting down while this process was starting; ended it",
8867 ),
8868 cgroup_path,
8869 });
8870 }
8871
8872 #[cfg(windows)]
8873 if let Err(error) = resume_suspended_child(pid, spec) {
8874 kill_and_reap_rejected_child(&mut child);
8875 return Err(error);
8876 }
8877
8878 let stdout_pump = match child.stdout.take() {
8879 Some(stdout) => Some(tokio::spawn(pump_stdout_to(stdout, output_sink.clone()))),
8880 None => {
8881 warn!(
8882 module_id = %spec.module_id,
8883 "spawned child exposed no stdout pipe; file capture will be incomplete"
8884 );
8885 None
8886 }
8887 };
8888 let stderr_pump = match child.stderr.take() {
8889 Some(stderr) => {
8890 let generation = ring
8891 .lock()
8892 .unwrap_or_else(|poisoned| poisoned.into_inner())
8893 .begin_process();
8894 Some(StderrPump {
8895 task: tokio::spawn(pump_stderr_to(
8896 stderr,
8897 Arc::clone(ring),
8898 generation,
8899 output_sink,
8900 )),
8901 generation,
8902 })
8903 }
8904 None => {
8905 ring.lock()
8909 .unwrap_or_else(|poisoned| poisoned.into_inner())
8910 .mark_not_captured("stderr pipe was not available on spawn");
8911 warn!(
8912 module_id = %spec.module_id,
8913 "spawned child exposed no stderr pipe; tail will be unavailable"
8914 );
8915 None
8916 }
8917 };
8918
8919 let exec = Arc::new(crate::activity::ExecOutcome::new());
8920 #[cfg(target_os = "macos")]
8921 let pending_exec = privacy_exec.is_some();
8922 #[cfg(not(target_os = "macos"))]
8923 let pending_exec = false;
8924 if !pending_exec {
8925 exec.finish(crate::activity::ExecState::CommittedExec);
8926 }
8927 Ok(SupervisedChild {
8928 child,
8929 exec: crate::activity::ExecOwner(exec),
8930 #[cfg(windows)]
8931 nonce_delivery,
8932 protocol: spec.protocol,
8933 #[cfg(target_os = "linux")]
8934 module_id: cgroup_name,
8935 #[cfg(target_os = "linux")]
8936 cgroup_placement: cgroup_placement.cloned(),
8937 #[cfg(windows)]
8938 job,
8939 stdout_pump,
8940 stderr_pump,
8941 stderr_ring: Arc::clone(ring),
8942 spawned_at,
8943 spawned_at_ms,
8944 spawned_from,
8945 launch_spec_digest: LaunchSpecDigest::from_spec(spec),
8946 spawned_file_identity,
8947 process_start_time,
8948 process_identity,
8949 pid,
8950 roster_guard: Some(roster_guard),
8951 #[cfg(target_os = "macos")]
8952 privacy_exec,
8953 #[cfg(target_os = "macos")]
8954 report_ready: Arc::new(OnceLock::new()),
8955 spawn_failure: None,
8956 })
8957}
8958
8959#[cfg(test)]
8960mod spawn_record_tests {
8961 use super::*;
8962
8963 thread_local! {
8964 static LAST_PID: std::cell::Cell<Option<u32>> = const { std::cell::Cell::new(None) };
8965 }
8966
8967 pub(super) fn after_create(child: &Child, _spec: &ModuleSpec) {
8968 LAST_PID.set(child.id());
8969 #[cfg(windows)]
8970 if let Some((_, address)) = _spec
8971 .env
8972 .iter()
8973 .find(|(key, _)| key == "SUBC_TEST_AFTER_CREATE")
8974 {
8975 use std::io::{Read, Write};
8976 let mut stream = std::net::TcpStream::connect(address).unwrap();
8977 stream
8978 .write_all(&child.id().unwrap().to_ne_bytes())
8979 .unwrap();
8980 let mut release = [0];
8981 stream.read_exact(&mut release).unwrap();
8982 }
8983 }
8984
8985 #[cfg(windows)]
8986 pub(super) fn assignment_failpoint(spec: &ModuleSpec) -> io::Result<()> {
8987 if spec
8988 .env
8989 .iter()
8990 .any(|(key, _)| key == "SUBC_TEST_ASSIGNMENT_FAILURE")
8991 {
8992 Err(io::Error::other("injected module job assignment failure"))
8993 } else {
8994 Ok(())
8995 }
8996 }
8997
8998 fn spec() -> ModuleSpec {
8999 ModuleSpec {
9000 module_id: "spawn-record-worker".into(),
9001 #[cfg(unix)]
9002 program: "/bin/sleep".into(),
9003 #[cfg(windows)]
9004 program: std::env::current_exe().unwrap(),
9005 #[cfg(unix)]
9006 args: vec!["60".into()],
9007 #[cfg(windows)]
9008 args: vec![
9009 "--exact".into(),
9010 "supervise::spawn_record_tests::windows_store_touch_fixture".into(),
9011 "--nocapture".into(),
9012 ],
9013 env: vec![],
9014 reserved: false,
9015 reserved_prefixes: vec![],
9016 protocol: ModuleProtocol::None,
9017 overlap: Default::default(),
9018 }
9019 }
9020
9021 #[tokio::test]
9022 async fn record_sync_failure_fails_spawn_and_reaps_child() {
9023 let home = cortexkit_test_support::ScratchDir::new("record-sync-failure");
9024 let supervisor =
9025 Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default())
9026 .with_live_children_record(home.join("live-children.json"));
9027 crate::live_children::fail_next_file_sync();
9028 let result = supervisor.spawn(spec());
9029 let failed = result.is_err();
9030 if let Ok(module) = result {
9031 module.stop().await.unwrap();
9032 }
9033 assert!(failed, "record failure must fail spawn");
9034 let pid = LAST_PID.get().expect("the real OS child was created");
9035 assert!(
9036 subc_os::Process::open(pid).unwrap().is_none(),
9037 "child {pid} survived rejected admission or was not reaped"
9038 );
9039 }
9040
9041 #[cfg(target_os = "macos")]
9042 #[test]
9043 #[ignore = "manual on-device latency measurement"]
9044 fn durable_record_write_latency_50() {
9045 let home = cortexkit_test_support::ScratchDir::new("record-latency");
9046 let path = home.join("live-children.json");
9047 let children = vec![crate::live_children::LiveChild {
9048 module_id: "latency-worker".into(),
9049 pid: std::process::id(),
9050 protocol: ModuleProtocol::None,
9051 start_time: Some(1),
9052 executable: None,
9053 cgroup_name: None,
9054 }];
9055 let mut samples = Vec::new();
9056 for _ in 0..50 {
9057 let start = std::time::Instant::now();
9058 crate::live_children::write_record(&path, &children).unwrap();
9059 samples.push(start.elapsed());
9060 }
9061 samples.sort();
9062 eprintln!(
9063 "durable_record_write_latency_50: n=50 median={:?} p95={:?}",
9064 (samples[24] + samples[25]) / 2,
9065 samples[47]
9066 );
9067 }
9068
9069 #[cfg(windows)]
9070 #[test]
9071 fn windows_store_touch_fixture() {
9072 if let Some(path) = std::env::var_os("SUBC_TEST_STORE_TOUCH") {
9073 std::fs::write(path, b"module ran").unwrap();
9074 }
9075 }
9076
9077 #[cfg(windows)]
9078 #[tokio::test]
9079 async fn windows_daemon_fixture() {
9080 let Ok(address) = std::env::var("SUBC_TEST_DAEMON_BARRIER") else {
9081 return;
9082 };
9083 let mut spec = spec();
9084 spec.env.push(("SUBC_TEST_AFTER_CREATE".into(), address));
9085 let supervisor =
9086 Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default());
9087 let _module = supervisor.spawn(spec).unwrap();
9088 }
9089
9090 #[cfg(windows)]
9091 #[test]
9092 fn windows_daemon_death_before_module_job_assignment_leaves_no_child() {
9093 use std::io::Read;
9094 let listener = std::net::TcpListener::bind("127.0.0.1:0").unwrap();
9095 let mut command = std::process::Command::new(std::env::current_exe().unwrap());
9098 command
9099 .args([
9100 "--exact",
9101 "supervise::spawn_record_tests::windows_daemon_fixture",
9102 "--nocapture",
9103 ])
9104 .env(
9105 "SUBC_TEST_DAEMON_BARRIER",
9106 listener.local_addr().unwrap().to_string(),
9107 );
9108 let mut daemon = subc_jobobject::spawn_contained(&mut command).unwrap();
9109 listener.set_nonblocking(true).unwrap();
9110 let deadline = std::time::Instant::now() + Duration::from_secs(30);
9111 let mut stream = loop {
9112 if let Ok((stream, _)) = listener.accept() {
9113 break stream;
9114 }
9115 assert!(
9116 daemon.child.try_wait().unwrap().is_none(),
9117 "daemon failed before rendezvous"
9118 );
9119 assert!(
9120 std::time::Instant::now() < deadline,
9121 "daemon did not reach CreateProcess failpoint"
9122 );
9123 std::thread::yield_now();
9124 };
9125 stream
9126 .set_read_timeout(Some(Duration::from_secs(30)))
9127 .unwrap();
9128 let mut pid = [0; 4];
9129 stream.read_exact(&mut pid).unwrap();
9130 let pid = u32::from_ne_bytes(pid);
9131 assert!(subc_jobobject::process_exists(pid));
9132 daemon.child.kill().unwrap();
9135 daemon.child.wait().unwrap();
9136 let exited = subc_jobobject::wait_for_process_exit(pid, Duration::from_secs(10));
9137 if !exited {
9138 daemon.job.terminate().unwrap();
9139 }
9140 assert!(exited, "suspended child {pid} outlived hard-killed daemon");
9141 }
9142
9143 #[cfg(windows)]
9144 #[tokio::test]
9145 async fn windows_containment_failure_never_resumes() {
9146 let home = cortexkit_test_support::ScratchDir::new("assignment-refusal");
9147 let sentinel = home.join("store-touched");
9148 let mut spec = spec();
9149 spec.env
9150 .push(("SUBC_TEST_ASSIGNMENT_FAILURE".into(), "1".into()));
9151 spec.env.push((
9152 "SUBC_TEST_STORE_TOUCH".into(),
9153 sentinel.display().to_string(),
9154 ));
9155 let supervisor =
9156 Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default());
9157 let result = supervisor.spawn(spec);
9158 if let Ok(module) = result {
9161 let _ = tokio::time::timeout(Duration::from_secs(10), async {
9162 while !sentinel.exists() && module.state().unwrap() == ModuleState::Running {
9163 tokio::task::yield_now().await;
9164 }
9165 })
9166 .await;
9167 let _ = module.stop().await;
9168 panic!(
9169 "assignment failure resumed a module; store touched={}",
9170 sentinel.exists()
9171 );
9172 }
9173 assert!(
9174 !sentinel.exists(),
9175 "refused module executed store-touch sentinel"
9176 );
9177 assert!(!subc_jobobject::process_exists(LAST_PID.get().unwrap()));
9178 }
9179
9180 #[cfg(windows)]
9181 #[tokio::test]
9182 async fn windows_nested_daemon_fixture() {
9183 let Some(sentinel) = std::env::var_os("SUBC_TEST_NESTED_STORE") else {
9184 return;
9185 };
9186 let mut spec = spec();
9187 spec.env.push((
9188 "SUBC_TEST_STORE_TOUCH".into(),
9189 sentinel.to_str().unwrap().into(),
9190 ));
9191 let supervisor =
9192 Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default());
9193 let module = supervisor
9194 .spawn(spec)
9195 .expect("per-module job must nest below daemon and host jobs");
9196 tokio::time::timeout(Duration::from_secs(10), async {
9197 while !PathBuf::from(&sentinel).exists() {
9198 tokio::task::yield_now().await;
9199 }
9200 })
9201 .await
9202 .unwrap();
9203 module.stop().await.unwrap();
9204 }
9205
9206 #[cfg(windows)]
9207 #[test]
9208 fn windows_module_job_nests_under_host_job() {
9209 let home = cortexkit_test_support::ScratchDir::new("nested-jobs");
9210 let sentinel = home.join("nested-store-touched");
9211 let mut command = std::process::Command::new(std::env::current_exe().unwrap());
9212 command
9213 .args([
9214 "--exact",
9215 "supervise::spawn_record_tests::windows_nested_daemon_fixture",
9216 "--nocapture",
9217 ])
9218 .env("SUBC_TEST_NESTED_STORE", &sentinel);
9219 let mut daemon = subc_jobobject::spawn_contained(&mut command).unwrap();
9220 assert!(daemon.child.wait().unwrap().success());
9221 assert!(sentinel.exists(), "nested protocol:none module never ran");
9222 }
9223}
9224
9225#[cfg(target_os = "linux")]
9226pub(crate) fn kill_module_cgroup(placement: Option<&subc_cgroup::Placement>, module_id: &str) {
9227 use subc_cgroup::KillOutcome;
9228 match subc_cgroup::kill_module(placement, module_id) {
9229 KillOutcome::Killed => {}
9230 KillOutcome::NotPlaced | KillOutcome::Unsupported => {
9231 debug!(
9232 module_id,
9233 "cgroup tree kill unavailable; using direct-child kill"
9234 );
9235 }
9236 KillOutcome::IoError { path, error } => {
9237 warn!(module_id, path = %path.display(), %error, "cgroup tree kill failed; using direct-child kill");
9238 }
9239 }
9240}
9241
9242#[cfg(windows)]
9250fn contain_spawned_child(
9251 child: &mut Child,
9252 spec: &ModuleSpec,
9253) -> Result<Option<subc_jobobject::JobObject>, SuperviseError> {
9254 let assignment = || {
9255 let job = subc_jobobject::JobObject::new()?;
9256 #[cfg(test)]
9257 spawn_record_tests::assignment_failpoint(spec)?;
9258 job.assign(child)?;
9259 Ok::<_, io::Error>(job)
9260 };
9261 match assignment() {
9262 Ok(job) => Ok(Some(job)),
9263 Err(source) => {
9264 kill_and_reap_rejected_child(child);
9265 Err(SuperviseError::Spawn {
9266 program: spec.program.clone(),
9267 source: io::Error::other(format!(
9268 "Windows module job containment failed: {source}"
9269 )),
9270 cgroup_path: None,
9271 })
9272 }
9273 }
9274}
9275
9276fn kill_and_reap_rejected_child(child: &mut Child) {
9279 #[cfg(unix)]
9280 if let Some(pid) = child
9281 .id()
9282 .and_then(|pid| rustix::process::Pid::from_raw(pid as i32))
9283 {
9284 let _ = rustix::process::kill_process_group(pid, rustix::process::Signal::KILL);
9285 }
9286 let _ = child.start_kill();
9287 while matches!(child.try_wait(), Ok(None)) {
9288 std::thread::yield_now();
9289 }
9290}
9291
9292#[cfg(windows)]
9297fn resume_suspended_child(pid: u32, spec: &ModuleSpec) -> Result<(), SuperviseError> {
9298 if let Err(source) = subc_jobobject::resume_main_thread(pid) {
9299 let _ = std::process::Command::new("taskkill.exe")
9303 .args(["/PID", &pid.to_string(), "/T", "/F"])
9304 .stdin(Stdio::null())
9305 .stdout(Stdio::null())
9306 .stderr(Stdio::null())
9307 .status();
9308 return Err(SuperviseError::Spawn {
9309 program: spec.program.clone(),
9310 source,
9311 cgroup_path: None,
9312 });
9313 }
9314 Ok(())
9315}
9316
9317#[cfg(target_os = "linux")]
9318fn remove_module_cgroup(placement: &subc_cgroup::Placement, module_id: &str) {
9319 match placement.remove_module(module_id) {
9320 Ok(()) => debug!(module_id, "removed module cgroup after process exit"),
9321 Err(error) => warn!(
9322 module_id,
9323 error = %error,
9324 "could not remove module cgroup after process exit; continuing teardown"
9325 ),
9326 }
9327}
9328
9329#[cfg(target_os = "linux")]
9330async fn cleanup_reaped_cgroup(placement: &subc_cgroup::Placement, module_id: &str) {
9331 if matches!(
9335 subc_cgroup::kill_module(Some(placement), module_id),
9336 subc_cgroup::KillOutcome::Killed
9337 ) {
9338 if let Ok(path) = placement.module_path(module_id) {
9339 while std::fs::read_to_string(path.join("cgroup.events"))
9340 .ok()
9341 .is_some_and(|events| events.lines().any(|line| line == "populated 1"))
9342 {
9343 sleep(Duration::from_millis(1)).await;
9344 }
9345 }
9346 }
9347 remove_module_cgroup(placement, module_id);
9348}
9349
9350#[cfg(target_os = "linux")]
9351fn apply_cgroup_placement(
9352 command: &mut Command,
9353 spec: &ModuleSpec,
9354 path: &std::path::Path,
9355) -> Result<(), SuperviseError> {
9356 subc_cgroup::apply(command, path).map_err(|source| SuperviseError::Cgroup {
9357 module_id: spec.module_id.clone(),
9358 source,
9359 })
9360}
9361
9362fn capture_retention(spec: &ModuleSpec) -> Retention {
9363 let defaults = Retention::default();
9364 let value = |name: &str| {
9365 spec.env
9366 .iter()
9367 .rev()
9368 .find_map(|(key, value)| (key == name).then_some(value.as_str()))
9369 };
9370 Retention {
9371 max_file_mb: value(CAPTURE_MAX_FILE_MB_ENV)
9372 .and_then(|value| value.parse().ok())
9373 .unwrap_or(defaults.max_file_mb),
9374 keep: value(CAPTURE_KEEP_ENV)
9375 .and_then(|value| value.parse().ok())
9376 .unwrap_or(defaults.keep),
9377 max_age_days: value(CAPTURE_MAX_AGE_DAYS_ENV)
9378 .and_then(|value| value.parse().ok())
9379 .unwrap_or(defaults.max_age_days),
9380 }
9381}
9382
9383fn generate_launch_nonce() -> Result<String, SuperviseError> {
9386 let mut bytes = [0u8; 32];
9387 getrandom::getrandom(&mut bytes).map_err(|source| SuperviseError::LaunchNonce {
9388 reason: source.to_string(),
9389 })?;
9390 let mut hex = String::with_capacity(64);
9391 for b in bytes {
9392 use std::fmt::Write;
9393 let _ = write!(hex, "{b:02x}");
9394 }
9395 Ok(hex)
9396}
9397
9398fn constant_time_eq(a: &[u8], b: &[u8]) -> bool {
9401 if a.len() != b.len() {
9402 return false;
9403 }
9404 let mut diff = 0u8;
9405 for (x, y) in a.iter().zip(b.iter()) {
9406 diff |= x ^ y;
9407 }
9408 diff == 0
9409}
9410
9411#[cfg(not(windows))]
9415fn observe_spawned_image(pid: u32) -> Option<subc_os::Observation> {
9416 let process = subc_os::Process::open(pid).ok().flatten()?;
9417 #[cfg(target_os = "linux")]
9418 return sample_after_linux_exec(
9419 || {
9420 let stat = std::fs::read_to_string(format!("/proc/{pid}/stat")).ok()?;
9421 let committed = linux_exec_committed(&stat);
9422 #[cfg(test)]
9423 spawn_image_tests::report_exec_state(committed);
9424 committed
9425 },
9426 || process.observe(),
9427 );
9428 #[cfg(not(target_os = "linux"))]
9429 process.observe()
9430}
9431
9432#[cfg(target_os = "linux")]
9436fn linux_exec_committed(stat: &str) -> Option<bool> {
9437 const PF_FORKNOEXEC: u64 = 0x40;
9438 let flags = stat.rsplit_once(')')?.1.split_whitespace().nth(6)?;
9439 Some(flags.parse::<u64>().ok()? & PF_FORKNOEXEC == 0)
9440}
9441
9442#[cfg(target_os = "linux")]
9443fn sample_after_linux_exec(
9444 mut exec_committed: impl FnMut() -> Option<bool>,
9445 observe: impl FnOnce() -> Option<subc_os::Observation>,
9446) -> Option<subc_os::Observation> {
9447 let deadline = std::time::Instant::now() + Duration::from_secs(1);
9448 while !exec_committed()? {
9449 if std::time::Instant::now() >= deadline {
9453 return None;
9454 }
9455 std::thread::sleep(Duration::from_micros(50));
9456 }
9457 observe()
9458}
9459
9460#[cfg(all(test, target_os = "linux"))]
9461mod spawn_image_tests {
9462 use super::*;
9463 use std::cell::{Cell, RefCell};
9464
9465 enum ExecEvent {
9466 Uncommitted,
9467 Image(Option<subc_os::Observation>),
9468 }
9469
9470 thread_local! {
9471 static EXEC_EVENTS: RefCell<Option<std::sync::mpsc::Sender<ExecEvent>>> = const { RefCell::new(None) };
9472 }
9473
9474 pub(super) fn report_exec_state(committed: Option<bool>) {
9475 if committed == Some(false) {
9476 EXEC_EVENTS.with(|events| {
9477 if let Some(events) = events.borrow_mut().take() {
9478 let _ = events.send(ExecEvent::Uncommitted);
9479 }
9480 });
9481 }
9482 }
9483
9484 #[test]
9485 fn linux_spawn_observation_waits_for_a_paused_fork_to_exec() {
9486 use std::io::{Read, Write};
9487 use std::os::fd::AsRawFd;
9488
9489 let descriptor = std::fs::File::open("/dev/null").unwrap();
9490 let mut command = std::process::Command::new("/bin/sleep");
9491 command.arg("30");
9492 let (mut ready, mut release) =
9493 subc_os::fork_exec_test::pause_before_exec(&mut command, descriptor.as_raw_fd())
9494 .unwrap();
9495 let spawn = std::thread::spawn(move || command.spawn());
9496 let mut message = [0u8; 8];
9497 ready.read_exact(&mut message).unwrap();
9498 let pid = i32::from_ne_bytes(message[..4].try_into().unwrap()) as u32;
9499 let before = std::fs::read_to_string(format!("/proc/{pid}/stat"));
9500 let inherited = subc_os::Process::open(pid).unwrap().unwrap().observe();
9501 let (events, received) = std::sync::mpsc::channel();
9502 let observer = std::thread::spawn(move || {
9503 EXEC_EVENTS.with(|slot| *slot.borrow_mut() = Some(events.clone()));
9504 let image = observe_spawned_image(pid);
9505 let _ = events.send(ExecEvent::Image(image));
9506 });
9507 let first = received.recv_timeout(Duration::from_secs(5));
9511 release.write_all(&[1]).unwrap();
9512 let mut child = spawn.join().unwrap().unwrap();
9513 let after = match first {
9514 Ok(ExecEvent::Uncommitted) => match received.recv_timeout(Duration::from_secs(5)) {
9515 Ok(ExecEvent::Image(image)) => image,
9516 _ => None,
9517 },
9518 Ok(ExecEvent::Image(image)) => image,
9519 Err(_) => None,
9520 };
9521 observer.join().unwrap();
9522 child.kill().unwrap();
9523 child.wait().unwrap();
9524
9525 assert_eq!(linux_exec_committed(&before.unwrap()), Some(false));
9526 assert_eq!(
9527 inherited.unwrap().executable,
9528 subc_os::file_identity(&std::env::current_exe().unwrap())
9529 );
9530 assert_eq!(
9531 after.unwrap().executable,
9532 subc_os::file_identity(std::path::Path::new("/bin/sleep"))
9533 );
9534 }
9535
9536 #[test]
9537 fn spawned_image_is_not_sampled_until_linux_exec_commits() {
9538 let committed = Cell::new(false);
9539 let inherited = subc_os::Observation {
9540 start_time: 23,
9541 executable: Some(subc_os::FileIdentity {
9542 device: 29,
9543 inode: 31,
9544 }),
9545 };
9546 let module = subc_os::Observation {
9547 start_time: 23,
9548 executable: Some(subc_os::FileIdentity {
9549 device: 29,
9550 inode: 37,
9551 }),
9552 };
9553 let polls = Cell::new(0);
9554 let image = sample_after_linux_exec(
9555 || {
9556 polls.set(polls.get() + 1);
9557 committed.set(polls.get() == 2);
9560 Some(committed.get())
9561 },
9562 || Some(if committed.get() { module } else { inherited }),
9563 );
9564 assert_eq!(image, Some(module));
9565 assert_eq!(polls.get(), 2);
9566 }
9567
9568 #[test]
9569 fn linux_exec_flag_is_read_from_flags_after_the_last_parenthesis() {
9570 assert_eq!(
9571 linux_exec_committed("17 (a ) b) R 1 2 3 4 5 4194368"),
9572 Some(false)
9573 );
9574 assert_eq!(
9575 linux_exec_committed("17 (a ) b) R 1 2 3 4 5 4194304"),
9576 Some(true)
9577 );
9578 assert_eq!(linux_exec_committed("17 (a) R 1 2 3 4 5 invalid"), None);
9579 assert_eq!(linux_exec_committed("17 (a) R"), None);
9580 }
9581
9582 #[test]
9583 fn unreadable_exec_state_never_samples_an_inherited_image() {
9584 assert_eq!(
9585 sample_after_linux_exec(
9586 || None,
9587 || panic!("must not sample before exec confirmation")
9588 ),
9589 None
9590 );
9591 }
9592
9593 #[test]
9594 fn uncommitted_exec_times_out_without_sampling_an_inherited_image() {
9595 assert_eq!(
9596 sample_after_linux_exec(
9597 || Some(false),
9598 || panic!("must not sample after an exec timeout")
9599 ),
9600 None
9601 );
9602 }
9603}
9604
9605#[cfg(windows)]
9608fn windows_spawn_capture(spec: &ModuleSpec) -> io::Result<subc_os::ExecutableCapture> {
9609 let mut program = spec.program.clone();
9610 if program.is_absolute() || program.components().count() > 1 {
9611 match subc_os::ExecutableCapture::open(&program) {
9614 Ok(capture) => return Ok(capture),
9615 Err(error)
9616 if error.kind() == io::ErrorKind::NotFound && program.extension().is_none() => {}
9617 Err(error) => return Err(error),
9618 }
9619 program.set_extension("exe");
9620 return subc_os::ExecutableCapture::open(&program);
9621 }
9622 if program.extension().is_none() {
9623 program.set_extension("exe");
9624 }
9625 let custom_path = spec
9626 .env
9627 .iter()
9628 .rev()
9629 .find(|(key, _)| key.eq_ignore_ascii_case("PATH"));
9630 let mut directories = Vec::new();
9631 if let Some(parent) = std::env::current_exe()?.parent() {
9634 directories.push(parent.to_path_buf());
9635 }
9636 directories.push(std::env::current_dir()?);
9637 if let Some(windows) = std::env::var_os("SystemRoot") {
9638 let windows = PathBuf::from(windows);
9639 directories.push(windows.join("System32"));
9640 directories.push(windows.clone());
9641 }
9642 let path = custom_path
9643 .map(|(_, value)| std::ffi::OsString::from(value))
9644 .or_else(|| std::env::var_os("PATH"));
9645 if let Some(path) = path {
9646 directories.extend(std::env::split_paths(&path));
9647 }
9648 for directory in directories {
9649 match subc_os::ExecutableCapture::open(&directory.join(&program)) {
9650 Ok(capture) => return Ok(capture),
9651 Err(error) if error.kind() == io::ErrorKind::NotFound => continue,
9652 Err(error) => return Err(error),
9653 }
9654 }
9655 Err(io::Error::new(
9656 io::ErrorKind::NotFound,
9657 "resolved executable not found",
9658 ))
9659}
9660
9661fn spawn_and_mark_running(
9662 spec: &ModuleSpec,
9663 runtime: &SupervisorRuntimeConfig,
9664 snapshot: &SharedSnapshot,
9665) -> Result<SupervisedChild, SuperviseError> {
9666 let activity_spawn = runtime.begin_activity_spawn(&spec.module_id);
9667 let child = spawn_child(
9668 spec,
9669 runtime.connection_file_path.as_deref(),
9670 runtime.supervisor_handle.as_ref(),
9671 &runtime.stderr_ring,
9672 runtime.capture_logs_dir.as_deref(),
9673 &runtime.child_roster,
9674 #[cfg(target_os = "linux")]
9675 runtime.cgroup_placement.as_ref(),
9676 )?;
9677 set_running(snapshot, &child, &spec.module_id, &runtime.spawn_events)?;
9678 if let Some(spawn) = activity_spawn {
9679 spawn.complete(snapshot);
9680 }
9681 Ok(child)
9682}
9683
9684enum RegistrationWaitOutcome {
9685 Registered,
9686 Exited(ExitReport),
9687 TimedOut,
9688}
9689
9690struct ReloadRegistrationFailure {
9691 exit_report: ExitReport,
9692 reason: String,
9693}
9694
9695#[derive(Debug, Clone, Copy, PartialEq, Eq)]
9696enum BusyGaugeObservation {
9697 Quiescent,
9698 Busy,
9699 Omitted,
9700}
9701
9702#[derive(Debug, Clone, PartialEq, Eq)]
9704enum GaugeVerdict {
9705 Zero,
9707 NonZero(u64),
9709 NotACount(String),
9713 Missing,
9715 ProbeFailed(&'static str),
9718}
9719
9720impl GaugeVerdict {
9721 fn reads_quiet(&self) -> bool {
9722 matches!(self, Self::Zero)
9723 }
9724}
9725
9726impl fmt::Display for GaugeVerdict {
9727 fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
9728 match self {
9729 Self::Zero => f.write_str("zero"),
9730 Self::NonZero(value) => write!(f, "non_zero({value})"),
9731 Self::NotACount(value) => write!(f, "not_a_count({value})"),
9732 Self::Missing => f.write_str("missing_from_health_body"),
9733 Self::ProbeFailed(kind) => write!(f, "probe_failed({kind})"),
9734 }
9735 }
9736}
9737
9738const GAUGE_VALUE_TEXT_LIMIT: usize = 32;
9741
9742fn gauge_verdicts(metrics: Option<&Value>, gauges: &[String]) -> Vec<GaugeVerdict> {
9744 let metrics = metrics.and_then(Value::as_object);
9745 gauges
9746 .iter()
9747 .map(
9748 |gauge| match metrics.and_then(|metrics| metrics.get(gauge)) {
9749 None => GaugeVerdict::Missing,
9750 Some(value) => match value.as_u64() {
9751 Some(0) => GaugeVerdict::Zero,
9752 Some(count) => GaugeVerdict::NonZero(count),
9753 None => {
9754 let mut text = value.to_string();
9755 if text.len() > GAUGE_VALUE_TEXT_LIMIT {
9756 let mut end = GAUGE_VALUE_TEXT_LIMIT;
9757 while !text.is_char_boundary(end) {
9758 end -= 1;
9759 }
9760 text.truncate(end);
9761 text.push_str("...");
9762 }
9763 GaugeVerdict::NotACount(text)
9764 }
9765 },
9766 },
9767 )
9768 .collect()
9769}
9770
9771fn busy_gauge_observation(verdicts: &[GaugeVerdict]) -> BusyGaugeObservation {
9774 if verdicts
9775 .iter()
9776 .any(|verdict| matches!(verdict, GaugeVerdict::Missing))
9777 {
9778 BusyGaugeObservation::Omitted
9779 } else if verdicts.iter().all(GaugeVerdict::reads_quiet) {
9780 BusyGaugeObservation::Quiescent
9781 } else {
9782 BusyGaugeObservation::Busy
9783 }
9784}
9785
9786#[derive(Debug, Clone, Default, PartialEq, Eq)]
9791struct GaugeTrace {
9792 latest: Option<(GaugeVerdict, Duration)>,
9793 last_value: Option<(GaugeVerdict, Duration)>,
9794}
9795
9796#[derive(Debug, Clone, Default, PartialEq, Eq)]
9802struct DrainGaugeTrace {
9803 probes: u32,
9805 gauges: Vec<(String, GaugeTrace)>,
9807}
9808
9809impl DrainGaugeTrace {
9810 fn new(gauges: &[String]) -> Self {
9811 Self {
9812 probes: 0,
9813 gauges: gauges
9814 .iter()
9815 .map(|gauge| (gauge.clone(), GaugeTrace::default()))
9816 .collect(),
9817 }
9818 }
9819
9820 fn record(&mut self, verdicts: &[GaugeVerdict], at: Duration) {
9822 self.probes = self.probes.saturating_add(1);
9823 for ((_, trace), verdict) in self.gauges.iter_mut().zip(verdicts) {
9824 trace.latest = Some((verdict.clone(), at));
9825 if matches!(
9826 verdict,
9827 GaugeVerdict::Zero | GaugeVerdict::NonZero(_) | GaugeVerdict::NotACount(_)
9828 ) {
9829 trace.last_value = Some((verdict.clone(), at));
9830 }
9831 }
9832 }
9833}
9834
9835impl fmt::Display for DrainGaugeTrace {
9840 fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
9841 if self.gauges.is_empty() {
9842 return f.write_str("none declared");
9843 }
9844 for (index, (name, trace)) in self.gauges.iter().enumerate() {
9845 if index > 0 {
9846 f.write_str(" | ")?;
9847 }
9848 write!(f, "{name}: ")?;
9849 match &trace.latest {
9850 Some((verdict, at)) => write!(f, "{verdict} @{}ms", at.as_millis())?,
9851 None => f.write_str("never probed")?,
9852 }
9853 match &trace.last_value {
9854 Some((value, at)) => write!(f, "; last value {value} @{}ms", at.as_millis())?,
9855 None => f.write_str("; no value ever read")?,
9856 }
9857 }
9858 Ok(())
9859 }
9860}
9861
9862pub(crate) fn declared_busy_gauges(
9863 registry: &Registry,
9864 module_id: &str,
9865) -> Result<Vec<String>, SuperviseError> {
9866 busy_gauges_of(
9867 registry
9868 .get_module(module_id)
9869 .map_err(SuperviseError::Registry)?,
9870 )
9871}
9872
9873fn declared_busy_gauges_for_connection(
9877 registry: &Registry,
9878 connection_id: ConnectionId,
9879) -> Result<Vec<String>, SuperviseError> {
9880 busy_gauges_of(
9881 registry
9882 .get_module_by_connection(connection_id)
9883 .map_err(SuperviseError::Registry)?,
9884 )
9885}
9886
9887fn busy_gauges_of(
9888 registration: Option<crate::registry::ModuleRegistration>,
9889) -> Result<Vec<String>, SuperviseError> {
9890 let Some(registration) = registration else {
9891 return Ok(Vec::new());
9892 };
9893 let Some(self_signals) = registration.manifest.self_signals else {
9894 return Ok(Vec::new());
9895 };
9896
9897 let mut gauges = Vec::new();
9898 for declaration in self_signals {
9899 if declaration.kind != SelfSignalKind::Busy {
9900 continue;
9901 }
9902 match declaration.anchored_to {
9903 SignalAnchor::HealthGauges { gauges: declared } if !declared.is_empty() => {
9904 gauges.extend(declared)
9905 }
9906 _ => {
9907 gauges.push(String::new());
9910 }
9911 }
9912 }
9913 Ok(gauges)
9914}
9915
9916#[allow(clippy::too_many_arguments)]
9925async fn wait_for_forwarding_quiescence(
9926 forwarding: &ForwardingTable,
9927 module_id: &str,
9928 runtime: &SupervisorRuntimeConfig,
9929 endpoint: crate::ModuleEndpointId,
9930 deadline: Instant,
9931 busy_gauges: &[String],
9932 scope: DrainScope,
9933 trace: &mut DrainGaugeTrace,
9934) -> Result<bool, SuperviseError> {
9935 let wait_started = Instant::now();
9936 let mut gauges_quiescent = busy_gauges.is_empty();
9937 let mut next_probe_at = wait_started;
9938 let mut omission_reported = false;
9939
9940 loop {
9941 let now = Instant::now();
9942 if !busy_gauges.is_empty() && now >= next_probe_at && now < deadline {
9943 let report = match scope {
9944 DrainScope::Active => probe_module_health(module_id, runtime, Some(deadline)).await,
9945 DrainScope::Endpoint(endpoint) => {
9946 probe_endpoint_health(endpoint, runtime, Some(deadline)).await
9947 }
9948 };
9949 let verdicts = match &report {
9950 Ok(report) => gauge_verdicts(report.metrics.as_ref(), busy_gauges),
9951 Err(err) => vec![GaugeVerdict::ProbeFailed(err.label()); busy_gauges.len()],
9952 };
9953 trace.record(&verdicts, wait_started.elapsed());
9954 gauges_quiescent = match report {
9955 Ok(_) => match busy_gauge_observation(&verdicts) {
9956 BusyGaugeObservation::Quiescent => true,
9957 BusyGaugeObservation::Busy => false,
9958 BusyGaugeObservation::Omitted => {
9959 if !omission_reported {
9962 forwarding
9963 .counters()
9964 .increment_drains_with_undeclared_gauge();
9965 let missing = busy_gauges
9966 .iter()
9967 .zip(&verdicts)
9968 .filter(|(_, verdict)| matches!(verdict, GaugeVerdict::Missing))
9969 .map(|(gauge, _)| gauge.as_str())
9970 .collect::<Vec<_>>()
9971 .join(",");
9972 warn!(
9973 module_id,
9974 missing_gauges = %missing,
9975 "drain health.check answer lacks a declared busy gauge; the drain treats the module as busy until it reports it"
9976 );
9977 omission_reported = true;
9978 }
9979 false
9980 }
9981 },
9982 Err(err) => {
9983 warn!(
9984 module_id,
9985 error = %err,
9986 "drain health.check did not produce declared busy gauges; treating module as busy"
9987 );
9988 false
9989 }
9990 };
9991 next_probe_at = Instant::now() + drain_busy_reprobe_interval();
9994 }
9995
9996 let in_flight = forwarding
9997 .endpoint_in_flight_count(endpoint)
9998 .map_err(SuperviseError::Forwarding)?;
9999 if in_flight == 0 && gauges_quiescent {
10000 return Ok(true);
10001 }
10002
10003 let now = Instant::now();
10004 if now >= deadline {
10005 return Ok(false);
10006 }
10007 let mut wait = deadline
10008 .saturating_duration_since(now)
10009 .min(REGISTRY_RELEASE_POLL);
10010 if !busy_gauges.is_empty() {
10011 wait = wait.min(next_probe_at.saturating_duration_since(now));
10012 }
10013 sleep(wait).await;
10014 }
10015}
10016
10017fn drain_busy_reprobe_interval() -> Duration {
10021 DRAIN_BUSY_REPROBE_INTERVAL.max(REGISTRY_RELEASE_POLL)
10022}
10023
10024fn warn_drain_timed_out(
10039 module_id: &str,
10040 drain_timeout: Duration,
10041 reason: RouteCloseReason,
10042 holdouts: &crate::forwarding::DrainHoldouts,
10043 gauges: &DrainGaugeTrace,
10044) {
10045 warn!(
10046 module_id = %module_id,
10047 waited = ?drain_timeout,
10048 ?reason,
10049 held_requests = holdouts.requests,
10050 held_routes = holdouts.routes,
10051 total_routes = holdouts.total_routes,
10052 top_connections = ?holdouts.top_connections,
10053 held = %holdouts
10056 .held
10057 .iter()
10058 .map(|(channel, corr)| format!("{channel}:{corr}"))
10059 .collect::<Vec<_>>()
10060 .join(","),
10061 gauge_probes = gauges.probes,
10062 busy_gauges = %gauges,
10063 "route drain timed out before request quiescence; forcing teardown"
10064 );
10065}
10066
10067fn drained_after_quiescence_wait(wait_result: &Result<bool, SuperviseError>) -> bool {
10075 match wait_result {
10076 Ok(drained) => *drained,
10077 Err(_) => false,
10078 }
10079}
10080
10081fn send_route_goodbyes(forwarding: &ForwardingTable, released_routes: Vec<GoodbyeTarget>) {
10082 for released in released_routes {
10083 let frame = match Frame::build_with_version(
10084 released.negotiated_ver,
10085 FrameType::Goodbye,
10086 control_flags(),
10087 released.channel,
10088 released.epoch,
10089 0,
10090 Vec::new(),
10091 ) {
10092 Ok(frame) => frame,
10093 Err(err) => {
10094 warn!(
10095 route_channel = released.channel,
10096 error = %err,
10097 "failed to build supervisor drain route GOODBYE frame"
10098 );
10099 continue;
10100 }
10101 };
10102 if !released.close_on_delivery_failure() {
10103 crate::forwarding::send_module_route_goodbye(
10104 &forwarding.counters(),
10105 &released.sink,
10106 frame,
10107 released.module_id.as_deref(),
10108 "supervisor drain",
10109 );
10110 continue;
10111 }
10112 if let Err(err) = released.sink.try_send(frame) {
10113 warn!(
10114 target_connection_id = released.connection_id.get(),
10115 route_channel = released.channel,
10116 error = %err,
10117 "supervisor drain route GOODBYE was not delivered to client; closing target connection"
10118 );
10119 let _ = forwarding.escalate_client_delivery_failure(
10120 released.connection_id,
10121 released.channel,
10122 released.epoch,
10123 CloseReason::new(
10124 "route_goodbye_delivery_failed",
10125 format!(
10126 "failed to enqueue supervisor drain route GOODBYE for channel {}: {err}",
10127 released.channel
10128 ),
10129 ),
10130 crate::forwarding::UndeliveredFrame {
10131 module_id: released.module_id.as_deref(),
10132 sink: &released.sink,
10133 },
10134 );
10135 }
10136 }
10137}
10138
10139fn send_module_draining(
10140 module_id: &str,
10141 reason: RouteCloseReason,
10142 deadline_ms: u64,
10143 target: &ModuleDrainTarget,
10144) {
10145 let body = match serde_json::to_vec(&ModuleControlCommand::Draining {
10146 reason,
10147 deadline_ms,
10148 }) {
10149 Ok(body) => body,
10150 Err(err) => {
10151 warn!(
10152 module_id,
10153 error = %err,
10154 "failed to encode module draining command"
10155 );
10156 return;
10157 }
10158 };
10159 let frame = match Frame::build_with_version(
10160 target.negotiated_ver,
10161 FrameType::Push,
10162 control_flags(),
10163 0,
10164 0,
10165 0,
10166 body,
10167 ) {
10168 Ok(frame) => frame,
10169 Err(err) => {
10170 warn!(
10171 module_id,
10172 error = %err,
10173 "failed to build module draining command frame"
10174 );
10175 return;
10176 }
10177 };
10178 if let Err(err) = target.sink.try_send(frame) {
10179 warn!(
10180 module_id,
10181 target_connection_id = target.endpoint.connection_id.get(),
10182 error = %err,
10183 "module draining command was not delivered to peer"
10184 );
10185 }
10186}
10187
10188fn module_goodbye_frame(module_id: &str, negotiated_ver: u8) -> Option<Frame> {
10190 match Frame::build_with_version(
10191 negotiated_ver,
10192 FrameType::Goodbye,
10193 control_flags(),
10194 0,
10195 0,
10196 0,
10197 Vec::new(),
10198 ) {
10199 Ok(frame) => Some(frame),
10200 Err(err) => {
10201 warn!(
10202 module_id,
10203 error = %err,
10204 "failed to build module GOODBYE frame"
10205 );
10206 None
10207 }
10208 }
10209}
10210
10211#[cfg(unix)]
10225async fn send_module_goodbyes_for_daemon_shutdown(
10226 forwarding: &Arc<ForwardingTable>,
10227 reason: &CloseReason,
10228 wait_for_flush: bool,
10229) {
10230 const GOODBYE_BUDGET: Duration = Duration::from_millis(500);
10231 let targets = match forwarding.module_connections() {
10232 Ok(targets) => targets,
10233 Err(err) => {
10234 warn!(error = %err, "could not list module connections for shutdown GOODBYE");
10235 return;
10236 }
10237 };
10238 let deadline = Instant::now() + GOODBYE_BUDGET;
10239 let mut sends = tokio::task::JoinSet::new();
10240 for target in targets {
10241 let Some(frame) = module_goodbye_frame(&target.module_id, target.negotiated_ver) else {
10242 continue;
10243 };
10244 if !wait_for_flush {
10245 if let Err(err) = target.sink.try_send(frame) {
10246 debug!(
10247 module_id = %target.module_id,
10248 error = %err,
10249 "shutdown module GOODBYE was not queued"
10250 );
10251 }
10252 continue;
10253 }
10254 let forwarding = Arc::clone(forwarding);
10255 let reason = reason.clone();
10256 sends.spawn(async move {
10257 match timeout_at(deadline, target.sink.send_flushed(frame)).await {
10258 Ok(Ok(())) => {}
10259 Ok(Err(err)) => debug!(
10260 module_id = %target.module_id,
10261 error = %err,
10262 "module connection closed before its shutdown GOODBYE was written"
10263 ),
10264 Err(_) => warn!(
10265 module_id = %target.module_id,
10266 budget = ?GOODBYE_BUDGET,
10267 "shutdown module GOODBYE was not written within its budget; closing anyway"
10268 ),
10269 }
10270 forwarding.request_connection_close(target.endpoint.connection_id, reason);
10271 });
10272 }
10273 while sends.join_next().await.is_some() {}
10275}
10276
10277fn send_module_goodbye(module_id: &str, forwarding: &ForwardingTable, target: &ModuleDrainTarget) {
10278 let Some(frame) = module_goodbye_frame(module_id, target.negotiated_ver) else {
10279 return;
10280 };
10281 if let Err(err) = target.sink.try_send(frame) {
10282 warn!(
10283 module_id,
10284 target_connection_id = target.endpoint.connection_id.get(),
10285 error = %err,
10286 "supervisor drain module GOODBYE was not delivered to peer; closing module connection"
10287 );
10288 forwarding.request_connection_close(
10289 target.endpoint.connection_id,
10290 CloseReason::new(
10291 "module_goodbye_delivery_failed",
10292 format!("failed to enqueue supervisor drain module GOODBYE for module '{module_id}': {err}"),
10293 ),
10294 );
10295 }
10296}
10297
10298#[derive(Clone, Copy)]
10299struct ForwardingDrainContext<'a> {
10300 spec: &'a ModuleSpec,
10301 runtime: &'a SupervisorRuntimeConfig,
10302 registry: &'a Registry,
10303 scope: DrainScope,
10304}
10305
10306#[derive(Debug, Clone, Copy, PartialEq, Eq)]
10308enum DrainScope {
10309 Active,
10312 Endpoint(crate::ModuleEndpointId),
10317}
10318
10319#[derive(Debug, Clone, Copy, PartialEq, Eq)]
10327enum StopNotice {
10328 SentOverConnection,
10331 NoConnection,
10335 NotSent,
10339}
10340
10341async fn begin_forwarding_drain(
10342 spec: &ModuleSpec,
10343 runtime: &SupervisorRuntimeConfig,
10344 registry: &Registry,
10345 snapshot: &SharedSnapshot,
10346 enabled: Option<bool>,
10347 reason: RouteCloseReason,
10348) -> Result<StopNotice, SuperviseError> {
10349 let Some(forwarding) = runtime.forwarding.as_ref() else {
10350 return Err(SuperviseError::ReloadUnavailable {
10351 module_id: spec.module_id.clone(),
10352 reason: "supervisor was not configured with a forwarding table".to_string(),
10353 });
10354 };
10355
10356 begin_forwarding_drain_with(
10357 forwarding,
10358 ForwardingDrainContext {
10359 spec,
10360 runtime,
10361 registry,
10362 scope: DrainScope::Active,
10363 },
10364 snapshot,
10365 enabled,
10366 reason,
10367 runtime.drain_timeout,
10368 )
10369 .await
10370}
10371
10372async fn begin_forwarding_drain_if_configured(
10373 spec: &ModuleSpec,
10374 runtime: &SupervisorRuntimeConfig,
10375 registry: &Registry,
10376 snapshot: &SharedSnapshot,
10377 enabled: Option<bool>,
10378 reason: RouteCloseReason,
10379) -> Result<StopNotice, SuperviseError> {
10380 begin_forwarding_drain_with_timeout(
10381 spec,
10382 runtime,
10383 registry,
10384 snapshot,
10385 enabled,
10386 reason,
10387 runtime.drain_timeout,
10388 )
10389 .await
10390}
10391
10392async fn begin_forwarding_drain_with_timeout(
10396 spec: &ModuleSpec,
10397 runtime: &SupervisorRuntimeConfig,
10398 registry: &Registry,
10399 snapshot: &SharedSnapshot,
10400 enabled: Option<bool>,
10401 reason: RouteCloseReason,
10402 drain_timeout: Duration,
10403) -> Result<StopNotice, SuperviseError> {
10404 let Some(forwarding) = runtime.forwarding.as_ref() else {
10405 return Ok(StopNotice::NotSent);
10406 };
10407
10408 begin_forwarding_drain_with(
10409 forwarding,
10410 ForwardingDrainContext {
10411 spec,
10412 runtime,
10413 registry,
10414 scope: DrainScope::Active,
10415 },
10416 snapshot,
10417 enabled,
10418 reason,
10419 drain_timeout,
10420 )
10421 .await
10422}
10423
10424async fn begin_forwarding_drain_with(
10425 forwarding: &ForwardingTable,
10426 context: ForwardingDrainContext<'_>,
10427 snapshot: &SharedSnapshot,
10428 enabled: Option<bool>,
10429 reason: RouteCloseReason,
10430 drain_timeout: Duration,
10431) -> Result<StopNotice, SuperviseError> {
10432 let ForwardingDrainContext {
10433 spec,
10434 runtime,
10435 registry,
10436 scope,
10437 } = context;
10438 debug_assert_ne!(reason, RouteCloseReason::Crash);
10439 let terminal = matches!(reason, RouteCloseReason::Disable);
10440 let drain_started_at = Instant::now();
10441 let drain_deadline = drain_started_at + drain_timeout;
10442 let deadline_ms =
10443 unix_ms_now().saturating_add(u64::try_from(drain_timeout.as_millis()).unwrap_or(u64::MAX));
10444 let busy_gauges = match scope {
10445 DrainScope::Active => declared_busy_gauges(registry, &spec.module_id)?,
10446 DrainScope::Endpoint(endpoint) => {
10447 declared_busy_gauges_for_connection(registry, endpoint.connection_id)?
10448 }
10449 };
10450
10451 let gate_started = Instant::now();
10454 let drain_target = match scope {
10455 DrainScope::Active => forwarding.begin_module_drain(&spec.module_id, reason),
10456 DrainScope::Endpoint(endpoint) => forwarding.begin_endpoint_drain(endpoint, reason),
10457 }
10458 .map_err(SuperviseError::Forwarding)?;
10459 info!(
10464 module_id = %spec.module_id,
10465 ?reason,
10466 gate_ms = u64::try_from(gate_started.elapsed().as_millis()).unwrap_or(u64::MAX),
10467 connected = drain_target.is_some(),
10468 "module drain began; route admission closed"
10469 );
10470 if scope == DrainScope::Active {
10471 update_snapshot(snapshot, Some(&spec.module_id), |state| {
10472 state.state = ModuleState::Draining;
10473 state.draining_to_replace =
10474 matches!(reason, RouteCloseReason::Restart | RouteCloseReason::Reload);
10475 if let Some(enabled) = enabled {
10476 state.enabled = enabled;
10477 }
10478 })?;
10479 }
10480
10481 let Some(target) = drain_target.as_ref() else {
10482 return Ok(StopNotice::NoConnection);
10486 };
10487 {
10488 send_module_draining(&spec.module_id, reason, deadline_ms, target);
10489 #[cfg(feature = "test-support")]
10490 forwarding.wait_before_route_closing_for_test().await;
10491 let routes = forwarding
10492 .endpoint_routes(target.endpoint)
10493 .map_err(SuperviseError::Forwarding)?;
10494 let routes_notified = routes.len();
10495 crate::control::send_route_control_pushes(
10496 forwarding,
10497 routes.clone(),
10498 ClientControlPush::RouteClosing {
10499 module_id: spec.module_id.clone(),
10500 channels: Vec::new(),
10501 reason,
10502 },
10503 );
10504 send_route_goodbyes(forwarding, target.abandoned_bindings.clone());
10505
10506 let mut gauge_trace = DrainGaugeTrace::new(&busy_gauges);
10512 let wait_result = wait_for_forwarding_quiescence(
10513 forwarding,
10514 &spec.module_id,
10515 runtime,
10516 target.endpoint,
10517 drain_deadline,
10518 &busy_gauges,
10519 scope,
10520 &mut gauge_trace,
10521 )
10522 .await;
10523 let drained = drained_after_quiescence_wait(&wait_result);
10524 if let Err(err) = &wait_result {
10525 error!(
10526 module_id = %spec.module_id,
10527 ?reason,
10528 error = %err,
10529 "forwarding quiescence wait failed after route.closing; forcing route.closed(drained: false) so the client is not left waiting on an unfulfilled promise"
10530 );
10531 } else if !drained {
10532 let holdouts = forwarding
10534 .endpoint_drain_holdouts(target.endpoint)
10535 .unwrap_or_default();
10536 warn_drain_timed_out(
10537 &spec.module_id,
10538 drain_timeout,
10539 reason,
10540 &holdouts,
10541 &gauge_trace,
10542 );
10543 }
10544 crate::control::send_route_control_pushes(
10545 forwarding,
10546 routes,
10547 ClientControlPush::RouteClosed {
10548 module_id: spec.module_id.clone(),
10549 channels: Vec::new(),
10550 reason,
10551 drained,
10552 abandoned: target.abandoned_bindings.len() as u32,
10553 excluded_subscriptions: target.excluded_subscriptions,
10554 terminal: Some(terminal),
10555 },
10556 );
10557 wait_result?;
10558
10559 let released_routes = match forwarding.release_module_endpoint_routes(target.endpoint) {
10565 Ok(routes) => routes,
10566 Err(err) => {
10567 warn!(
10568 module_id = %spec.module_id,
10569 ?reason,
10570 error = %err,
10571 "failed to release module endpoint routes after route.closed; module GOODBYE will still be sent"
10572 );
10573 send_module_goodbye(&spec.module_id, forwarding, target);
10574 return Err(SuperviseError::Forwarding(err));
10575 }
10576 };
10577 let route_goodbye_count = released_routes.len();
10578 send_route_goodbyes(forwarding, released_routes);
10579 send_module_goodbye(&spec.module_id, forwarding, target);
10580
10581 info!(
10587 module_id = %spec.module_id,
10588 ?reason,
10589 routes_notified,
10590 route_goodbyes = route_goodbye_count,
10591 abandoned_reservations = target.abandoned_bindings.len(),
10592 excluded_subscriptions = target.excluded_subscriptions,
10593 drained,
10594 "module drain complete; consumers notified via route.closing/route.closed pushes and per-route GOODBYE frames"
10595 );
10596 }
10597
10598 Ok(StopNotice::SentOverConnection)
10599}
10600
10601async fn wait_for_registration_after_reload(
10604 registry: &Registry,
10605 module_id: &str,
10606 snapshot: &SharedSnapshot,
10607 child: &mut SupervisedChild,
10608 wait: Duration,
10609) -> Result<RegistrationWaitOutcome, SuperviseError> {
10610 wait_for_slot_registration(
10611 registry,
10612 crate::registry::RegistrationSlot::Active(module_id),
10613 module_id,
10614 snapshot,
10615 child,
10616 wait,
10617 )
10618 .await
10619}
10620
10621async fn wait_for_slot_registration(
10629 registry: &Registry,
10630 slot: crate::registry::RegistrationSlot<'_>,
10631 module_id: &str,
10632 snapshot: &SharedSnapshot,
10633 child: &mut SupervisedChild,
10634 wait: Duration,
10635) -> Result<RegistrationWaitOutcome, SuperviseError> {
10636 let deadline = Instant::now() + wait;
10637 loop {
10638 if registry
10639 .registration(slot)
10640 .map_err(SuperviseError::Registry)?
10641 .is_some()
10642 {
10643 return Ok(RegistrationWaitOutcome::Registered);
10644 }
10645
10646 let now = Instant::now();
10647 if now >= deadline {
10648 return Ok(RegistrationWaitOutcome::TimedOut);
10649 }
10650 let remaining = deadline.saturating_duration_since(now);
10651 let poll = remaining.min(REGISTRY_RELEASE_POLL);
10652
10653 tokio::select! {
10654 wait_result = child.wait() => {
10655 let status = wait_result.map_err(|source| SuperviseError::Wait {
10656 module_id: module_id.to_string(),
10657 source,
10658 })?;
10659 return Ok(RegistrationWaitOutcome::Exited(classify_reaped_child_exit(
10660 snapshot,
10661 child,
10662 &status,
10663 )));
10664 }
10665 _ = sleep(poll) => {}
10666 }
10667 }
10668}
10669
10670fn registration_failure_exit_report(mut exit_report: ExitReport) -> ExitReport {
10671 if exit_report.kind != ExitKind::DeliberateSeverance {
10674 exit_report.kind = ExitKind::Crash;
10675 }
10676 exit_report
10677}
10678
10679async fn handle_reload_child_registration_failure(
10680 spec: &ModuleSpec,
10681 runtime: &SupervisorRuntimeConfig,
10682 registry: &Registry,
10683 process_liveness: &SupervisorProcessLiveness,
10684 snapshot: &SharedSnapshot,
10685 _child: &mut Option<SupervisedChild>,
10686 failure: ReloadRegistrationFailure,
10687) -> Result<(), SuperviseError> {
10688 let ReloadRegistrationFailure {
10689 exit_report,
10690 reason,
10691 } = failure;
10692 match on_child_exit(
10693 spec,
10694 runtime.restart_policy,
10695 registry,
10696 snapshot,
10697 &runtime.terminal_ring,
10698 &runtime.spawn_events,
10699 &runtime.child_roster,
10700 exit_report,
10701 )
10702 .await
10703 {
10704 NextAction::Stop {
10705 registration_released,
10706 } => {
10707 if registration_released {
10708 process_liveness.untrack_if_current(&spec.module_id, snapshot);
10709 }
10710 }
10711 NextAction::Restart { schedule } => {
10712 let delay = schedule.map_or(runtime.restart_policy.delay_for_restart(0), |schedule| {
10713 schedule.delay
10714 });
10715 if let Some(schedule) = schedule {
10716 log_crash_respawn(&spec.module_id, schedule);
10717 }
10718 schedule_respawn(
10719 runtime,
10720 snapshot,
10721 &spec.module_id,
10722 delay,
10723 RespawnKind::Spawn,
10724 )?;
10725 }
10726 }
10727 Err(SuperviseError::ReloadFailed {
10728 module_id: spec.module_id.clone(),
10729 reason,
10730 })
10731}
10732
10733async fn handle_reload_spawn_failure(
10734 spec: &ModuleSpec,
10735 runtime: &SupervisorRuntimeConfig,
10736 process_liveness: &SupervisorProcessLiveness,
10737 snapshot: &SharedSnapshot,
10738 _child: &mut Option<SupervisedChild>,
10739 reason: String,
10740) -> Result<(), SuperviseError> {
10741 let now = Instant::now();
10742 let mut schedule = None;
10743 update_snapshot(snapshot, Some(&spec.module_id), |state| {
10744 clear_current_process_facts(state);
10745 if state.enabled {
10746 schedule = state.next_crash_restart(&runtime.restart_policy, now);
10747 state.state = if schedule.is_some() {
10748 ModuleState::Restarting
10749 } else {
10750 ModuleState::Failed
10751 };
10752 } else {
10753 state.state = ModuleState::Disabled;
10754 }
10755 })?;
10756 if let Some(schedule) = schedule {
10757 schedule_respawn(
10758 runtime,
10759 snapshot,
10760 &spec.module_id,
10761 schedule.delay,
10762 RespawnKind::Spawn,
10763 )?;
10764 } else {
10765 process_liveness.untrack_if_current(&spec.module_id, snapshot);
10766 }
10767 Err(SuperviseError::ReloadFailed {
10768 module_id: spec.module_id.clone(),
10769 reason,
10770 })
10771}
10772
10773fn control_flags() -> Flags {
10774 Flags::new(false, Priority::Passive, false)
10775}
10776
10777#[allow(clippy::too_many_arguments)]
10778async fn drain_optional_child(
10779 module_id: &str,
10780 protocol: ModuleProtocol,
10781 stop_notice: StopNotice,
10782 registry: &Registry,
10783 forwarding: Option<&ForwardingTable>,
10784 snapshot: &SharedSnapshot,
10785 terminal_ring: &Arc<Mutex<TerminalRing>>,
10786 spawn_events: &SpawnEventFeed,
10787 child: &mut Option<SupervisedChild>,
10788 drain_timeout: Duration,
10789 final_state: ModuleState,
10790 enabled: Option<bool>,
10791) -> Result<(), SuperviseError> {
10792 if let Some(child) = child.take() {
10793 drain_child_to_state(
10794 module_id,
10795 protocol,
10796 stop_notice,
10797 registry,
10798 forwarding,
10799 snapshot,
10800 terminal_ring,
10801 spawn_events,
10802 child,
10803 drain_timeout,
10804 final_state,
10805 enabled,
10806 )
10807 .await
10808 } else {
10809 update_snapshot(snapshot, Some(module_id), |state| {
10810 state.state = final_state;
10811 if let Some(enabled) = enabled {
10812 state.enabled = enabled;
10813 }
10814 clear_current_process_facts(state);
10815 })?;
10816 release_dead_registration(registry, forwarding, snapshot, module_id).await
10817 }
10818}
10819
10820#[allow(clippy::too_many_arguments)]
10821async fn drain_child_to_state(
10822 module_id: &str,
10823 _protocol: ModuleProtocol,
10824 stop_notice: StopNotice,
10825 registry: &Registry,
10826 forwarding: Option<&ForwardingTable>,
10827 snapshot: &SharedSnapshot,
10828 terminal_ring: &Arc<Mutex<TerminalRing>>,
10829 spawn_events: &SpawnEventFeed,
10830 mut child: SupervisedChild,
10831 drain_timeout: Duration,
10832 final_state: ModuleState,
10833 enabled: Option<bool>,
10834) -> Result<(), SuperviseError> {
10835 let protocol = child.protocol;
10836 update_snapshot(snapshot, Some(module_id), |state| {
10837 state.state = ModuleState::Draining;
10838 state.draining_to_replace = final_state == ModuleState::Restarting;
10839 if let Some(enabled) = enabled {
10840 state.enabled = enabled;
10841 }
10842 })?;
10843
10844 if stop_notice != StopNotice::SentOverConnection {
10855 if protocol == ModuleProtocol::Subc && stop_notice == StopNotice::NoConnection {
10856 info!(
10857 module_id,
10858 pid = child.pid,
10859 budget_ms = u64::try_from(drain_timeout.as_millis()).unwrap_or(u64::MAX),
10860 "module has no connection yet; requesting stop by signal"
10861 );
10862 }
10863 request_graceful_stop(module_id, &child);
10864 }
10865
10866 let exit_report = match timeout(drain_timeout, child.wait()).await {
10867 Ok(Ok(status)) => classify_reaped_child_exit(snapshot, &child, &status),
10868 Ok(Err(source)) => {
10869 fail_after_wait_error(
10870 module_id,
10871 snapshot,
10872 terminal_ring,
10873 spawn_events,
10874 format!("failed to wait during drain: {source}"),
10875 );
10876 return Err(SuperviseError::Wait {
10877 module_id: module_id.to_string(),
10878 source,
10879 });
10880 }
10881 Err(_) => {
10882 warn!(
10895 module_id,
10896 pid = child.pid,
10897 budget_ms = u64::try_from(drain_timeout.as_millis()).unwrap_or(u64::MAX),
10898 reason = ?final_state,
10899 ?stop_notice,
10900 "drain budget expired before the module exited; killing it"
10901 );
10902 child.start_kill().map_err(|source| {
10903 fail_after_wait_error(
10904 module_id,
10905 snapshot,
10906 terminal_ring,
10907 spawn_events,
10908 format!("failed to kill during drain: {source}"),
10909 );
10910 SuperviseError::Kill {
10911 module_id: module_id.to_string(),
10912 source,
10913 }
10914 })?;
10915 let status = child.wait().await.map_err(|source| {
10916 fail_after_wait_error(
10917 module_id,
10918 snapshot,
10919 terminal_ring,
10920 spawn_events,
10921 format!("failed to wait after drain kill: {source}"),
10922 );
10923 SuperviseError::Wait {
10924 module_id: module_id.to_string(),
10925 source,
10926 }
10927 })?;
10928 classify_reaped_child_exit(snapshot, &child, &status)
10929 }
10930 };
10931
10932 let detail = lock_snapshot(snapshot)?.drain_disposition_detail.take();
10933 publish_after_terminal_record(
10934 module_id,
10935 || {
10936 record_terminal_with_detail(
10937 module_id,
10938 terminal_ring,
10939 spawn_events,
10940 &exit_report,
10941 terminal_disposition(final_state),
10942 detail,
10943 )
10944 },
10945 || {
10946 update_snapshot(snapshot, Some(module_id), |state| {
10947 state.state = final_state;
10948 if let Some(enabled) = enabled {
10949 state.enabled = enabled;
10950 }
10951 clear_current_process_facts(state);
10952 state.last_exit = Some(exit_report.clone());
10953 if exit_report.kind == ExitKind::DeliberateSeverance {
10954 state.lifetime_restarts += 1;
10955 }
10956 })
10957 },
10958 )?;
10959 child.drain_stderr(module_id).await;
10960
10961 release_dead_registration(registry, forwarding, snapshot, module_id).await
10962}
10963
10964#[cfg(unix)]
10984fn request_graceful_stop(module_id: &str, child: &SupervisedChild) {
10985 let Some(pid) = child
10986 .id()
10987 .and_then(|pid| i32::try_from(pid).ok())
10988 .and_then(rustix::process::Pid::from_raw)
10989 else {
10990 debug!(
10991 module_id,
10992 "no pid to signal for teardown; falling through to the drain wait"
10993 );
10994 return;
10995 };
10996 match rustix::process::kill_process(pid, rustix::process::Signal::TERM) {
10997 Ok(()) => debug!(
10998 module_id,
10999 "sent SIGTERM to a module nothing else asked to stop"
11000 ),
11001 Err(err) => debug!(
11002 module_id,
11003 error = %err,
11004 "SIGTERM to module failed; the drain wait and kill still apply"
11005 ),
11006 }
11007}
11008
11009#[cfg(not(unix))]
11017fn request_graceful_stop(module_id: &str, _child: &SupervisedChild) {
11018 debug!(
11019 module_id,
11020 "no graceful stop signal exists on this platform; teardown of a module nothing asked to stop waits, then kills"
11021 );
11022}
11023
11024fn terminal_disposition(final_state: ModuleState) -> TerminalDisposition {
11025 match final_state {
11026 ModuleState::Stopped => TerminalDisposition::Stopped,
11027 ModuleState::Disabled => TerminalDisposition::Disabled,
11028 ModuleState::Restarting => TerminalDisposition::Restarting,
11029 ModuleState::Failed => TerminalDisposition::Failed,
11030 ModuleState::Starting
11031 | ModuleState::Running
11032 | ModuleState::Unresponsive
11033 | ModuleState::Draining => {
11034 unreachable!("terminal exits only finish in terminal or restarting states")
11035 }
11036 }
11037}
11038
11039async fn release_dead_registration(
11047 registry: &Registry,
11048 forwarding: Option<&ForwardingTable>,
11049 snapshot: &SharedSnapshot,
11050 module_id: &str,
11051) -> Result<(), SuperviseError> {
11052 let result = async {
11053 let registration = registry
11054 .get_module(module_id)
11055 .map_err(SuperviseError::Registry)?;
11056 match wait_for_registration_release(registry, module_id, REGISTRY_RELEASE_TIMEOUT).await {
11057 Ok(()) => return Ok(()),
11058 Err(SuperviseError::RegistrationStillActive { .. }) => {}
11059 Err(err) => return Err(err),
11060 }
11061 let pid = lock_snapshot(snapshot)?.reaped_pid;
11062 if let (Some(registration), Some(forwarding), Some(pid)) = (registration, forwarding, pid) {
11063 warn!(
11064 module_id,
11065 pid,
11066 connection_id = registration.connection_id.get(),
11067 "reaped module registration outlived release grace; closing dead connection"
11068 );
11069 forwarding.request_connection_close(
11070 registration.connection_id,
11071 CloseReason::new(
11072 "supervised_process_reaped",
11073 format!("module '{module_id}' pid {pid} exited"),
11074 ),
11075 );
11076 wait_for_slot_registration_release(
11077 registry,
11078 crate::registry::RegistrationSlot::Connection(registration.connection_id),
11079 REGISTRY_RELEASE_TIMEOUT,
11080 )
11081 .await?;
11082 }
11083 wait_for_registration_release(registry, module_id, Duration::ZERO).await
11084 }
11085 .await;
11086 if let Err(err) = &result {
11087 fail_snapshot(snapshot, Some(module_id), None);
11088 error!(module_id, error = %err, "registration release failed after child exit; module is failed and start can retry");
11089 }
11090 result
11091}
11092
11093async fn wait_for_registration_release(
11096 registry: &Registry,
11097 module_id: &str,
11098 wait: Duration,
11099) -> Result<(), SuperviseError> {
11100 wait_for_slot_registration_release(
11101 registry,
11102 crate::registry::RegistrationSlot::Active(module_id),
11103 wait,
11104 )
11105 .await
11106}
11107
11108async fn wait_for_slot_registration_release(
11116 registry: &Registry,
11117 slot: crate::registry::RegistrationSlot<'_>,
11118 wait: Duration,
11119) -> Result<(), SuperviseError> {
11120 let deadline = Instant::now() + wait;
11121 let mut release_events = registration_release_events().subscribe();
11122 let still_active = |registration: &crate::registry::ModuleRegistration| {
11123 SuperviseError::RegistrationStillActive {
11124 module_id: registration.manifest.module_id.clone(),
11125 waited: wait,
11126 }
11127 };
11128 loop {
11129 let _observed_generation = *release_events.borrow_and_update();
11130 let Some(registration) = registry
11131 .registration(slot)
11132 .map_err(SuperviseError::Registry)?
11133 else {
11134 return Ok(());
11135 };
11136
11137 let now = Instant::now();
11138 if now >= deadline {
11139 return Err(still_active(®istration));
11140 }
11141
11142 let remaining = deadline.saturating_duration_since(now);
11143 match timeout(remaining, release_events.changed()).await {
11144 Ok(Ok(())) | Ok(Err(_)) => {}
11145 Err(_) => return Err(still_active(®istration)),
11146 }
11147 }
11148}
11149
11150#[cfg(test)]
11151mod slot_registration_wait_tests {
11152 use super::*;
11153 use crate::registry::{ConnectionId, RegistrationSlot};
11154 use subc_protocol::manifest::ModuleManifest;
11155
11156 #[tokio::test]
11157 async fn enable_release_failure_is_failed_and_a_second_enable_retries() {
11158 let registry = Arc::new(Registry::default());
11159 let supervisor = Supervisor::new_for_test(Arc::clone(®istry), RestartPolicy::default());
11160 let runtime = supervisor.runtime_config();
11161 let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::disabled()));
11162 let spec = ModuleSpec {
11163 module_id: "enable-stale-registration".to_string(),
11164 program: PathBuf::from("/missing/enable-retry-test"),
11165 args: Vec::new(),
11166 env: Vec::new(),
11167 reserved: false,
11168 reserved_prefixes: Vec::new(),
11169 protocol: ModuleProtocol::Subc,
11170 overlap: Default::default(),
11171 };
11172 let connection = ConnectionId::new(90);
11173 registry
11174 .register_with_control_ops(
11175 ModuleManifest::builder(&spec.module_id, "0.1.0").build(),
11176 1,
11177 connection,
11178 Vec::new(),
11179 )
11180 .unwrap();
11181 let mut child = None;
11182 let err = set_child_enabled(
11183 &spec,
11184 &runtime,
11185 ®istry,
11186 &supervisor.process_liveness,
11187 &snapshot,
11188 &mut child,
11189 true,
11190 )
11191 .await
11192 .unwrap_err();
11193 assert!(matches!(
11194 err,
11195 SuperviseError::RegistrationStillActive { .. }
11196 ));
11197 assert_eq!(lock_snapshot(&snapshot).unwrap().state, ModuleState::Failed);
11198 assert!(child.is_none());
11199 registry.deregister_connection(connection).unwrap();
11200 let err = set_child_enabled(
11201 &spec,
11202 &runtime,
11203 ®istry,
11204 &supervisor.process_liveness,
11205 &snapshot,
11206 &mut child,
11207 true,
11208 )
11209 .await
11210 .unwrap_err();
11211 assert!(
11212 matches!(err, SuperviseError::Spawn { .. }),
11213 "second enable must attempt a spawn: {err}"
11214 );
11215 assert_eq!(lock_snapshot(&snapshot).unwrap().state, ModuleState::Failed);
11216 }
11217
11218 const INCUMBENT: u64 = 1;
11219 const CANDIDATE: u64 = 2;
11220
11221 fn swapped_registry() -> Arc<Registry> {
11222 let registry = Arc::new(Registry::default());
11223 let manifest = ModuleManifest::builder("m", "0.1.0").build();
11224 registry
11225 .register_with_control_ops(
11226 manifest.clone(),
11227 1,
11228 ConnectionId::new(INCUMBENT),
11229 Vec::new(),
11230 )
11231 .unwrap();
11232 registry
11233 .register_candidate_with_control_ops(
11234 manifest,
11235 1,
11236 ConnectionId::new(CANDIDATE),
11237 Vec::new(),
11238 )
11239 .unwrap();
11240 registry
11241 }
11242
11243 #[tokio::test]
11247 async fn incumbent_release_is_awaited_by_connection_not_by_module_id() {
11248 let registry = swapped_registry();
11249 registry.promote_candidate("m").unwrap().unwrap();
11250
11251 assert!(matches!(
11252 wait_for_registration_release(®istry, "m", Duration::from_millis(50)).await,
11253 Err(SuperviseError::RegistrationStillActive { .. })
11254 ));
11255
11256 assert!(matches!(
11258 wait_for_slot_registration_release(
11259 ®istry,
11260 RegistrationSlot::Connection(ConnectionId::new(INCUMBENT)),
11261 Duration::from_millis(50),
11262 )
11263 .await,
11264 Err(SuperviseError::RegistrationStillActive { .. })
11265 ));
11266
11267 let releaser = Arc::clone(®istry);
11268 let release = tokio::spawn(async move {
11269 sleep(Duration::from_millis(20)).await;
11270 releaser
11271 .deregister_connection(ConnectionId::new(INCUMBENT))
11272 .unwrap();
11273 notify_registration_release();
11274 });
11275 wait_for_slot_registration_release(
11276 ®istry,
11277 RegistrationSlot::Connection(ConnectionId::new(INCUMBENT)),
11278 Duration::from_secs(5),
11279 )
11280 .await
11281 .expect("the incumbent's own registration is released");
11282 release.await.unwrap();
11283 assert!(registry.get_module("m").unwrap().is_some());
11284 }
11285
11286 #[tokio::test]
11289 async fn candidate_slot_wait_ignores_the_incumbents_registration() {
11290 let registry = swapped_registry();
11291 assert!(matches!(
11292 wait_for_slot_registration_release(
11293 ®istry,
11294 RegistrationSlot::Candidate("m"),
11295 Duration::from_millis(50),
11296 )
11297 .await,
11298 Err(SuperviseError::RegistrationStillActive { .. })
11299 ));
11300 registry
11301 .deregister_connection(ConnectionId::new(CANDIDATE))
11302 .unwrap();
11303 wait_for_slot_registration_release(
11304 ®istry,
11305 RegistrationSlot::Candidate("m"),
11306 Duration::from_millis(50),
11307 )
11308 .await
11309 .expect("a candidate slot with no candidate is released");
11310 assert!(registry
11311 .registration(RegistrationSlot::Active("m"))
11312 .unwrap()
11313 .is_some());
11314 }
11315}
11316
11317fn classify_exit(status: &ExitStatus) -> ExitReport {
11318 ExitReport {
11319 kind: if status.success() {
11320 ExitKind::Clean
11321 } else {
11322 ExitKind::Crash
11323 },
11324 code: status.code(),
11325 signal: exit_signal(status),
11326 at_ms: unix_ms_now(),
11327 }
11328}
11329
11330fn wait_error_exit_report() -> ExitReport {
11336 ExitReport {
11337 kind: ExitKind::Crash,
11338 code: None,
11339 signal: None,
11340 at_ms: unix_ms_now(),
11341 }
11342}
11343
11344#[cfg(unix)]
11345fn exit_signal(status: &ExitStatus) -> Option<i32> {
11346 use std::os::unix::process::ExitStatusExt;
11347
11348 status.signal()
11349}
11350
11351#[cfg(not(unix))]
11352fn exit_signal(_status: &ExitStatus) -> Option<i32> {
11353 None
11354}
11355
11356fn reset_restart_count(snapshot: &SharedSnapshot, module_id: &str) -> Result<(), SuperviseError> {
11362 update_snapshot(snapshot, Some(module_id), |state| {
11363 state.clear_crash_restarts();
11364 })
11365}
11366
11367fn set_running(
11368 snapshot: &SharedSnapshot,
11369 child: &SupervisedChild,
11370 module_id: &str,
11371 spawn_events: &SpawnEventFeed,
11372) -> Result<(), SuperviseError> {
11373 let mut state = snapshot.lock().map_err(|_| SuperviseError::StatePoisoned {
11374 module_id: Some(module_id.to_string()),
11375 })?;
11376 state.spawn_generation = spawn_events.emit_spawned(module_id, child.pid, child.spawned_at_ms);
11377 if std::mem::take(&mut state.coalesced_restart_pending) {
11378 let generation = state.spawn_generation;
11379 info!(module_id, "restart already satisfied by generation {generation}; coalesced pending request completed");
11380 }
11381 state.drain_disposition_detail = None;
11382 state.spawn_failure = None;
11383 state.waiting_for_previous_pid = None;
11384 state.waiting_for_previous_detail = None;
11385 state.in_alternate_slot = false;
11388 state.configuration_updated_since_spawn = false;
11389 state.spawned_protocol = Some(child.protocol);
11390 state.state = ModuleState::Running;
11391 state.enabled = true;
11392 state.process_alive = true;
11393 state.pid = child.id();
11394 state.health.status = SupervisorHealthStatus::Unknown;
11397 state.health.consecutive_failures = 0;
11398 state.health.detail = None;
11399 state.health.metrics = None;
11400 state.health.last_probe_ms = None;
11401 #[cfg(windows)]
11402 {
11403 state.pipe_consumed = child
11404 .nonce_delivery
11405 .as_ref()
11406 .map(|delivery| delivery.consumed());
11407 }
11408 #[cfg(target_os = "macos")]
11409 {
11410 state.report_ready = Some(Arc::clone(&child.report_ready));
11411 }
11412 state.spawned_at_ms = Some(child.spawned_at_ms);
11413 state.exec_outcome = Some(Arc::clone(&child.exec.0));
11414 state.spawned_from = Some(child.spawned_from.clone());
11415 state.launch_spec_digest = Some(child.launch_spec_digest.clone());
11416 state.spawned_file_identity = child.spawned_file_identity.clone();
11417 state.process_start_time = child.process_start_time;
11418 Ok(())
11419}
11420
11421fn clear_current_process_facts(state: &mut SupervisorSnapshot) {
11422 state.waiting_for_previous_pid = None;
11423 state.waiting_for_previous_detail = None;
11424 state.process_alive = false;
11425 state.spawned_protocol = None;
11426 state.pid = None;
11427 #[cfg(windows)]
11428 {
11429 state.pipe_consumed = None;
11430 }
11431 #[cfg(target_os = "macos")]
11432 {
11433 state.report_ready = None;
11434 }
11435 state.spawned_at_ms = None;
11436 state.exec_outcome = None;
11437 state.spawned_from = None;
11438 state.launch_spec_digest = None;
11439 state.spawned_file_identity = None;
11440 state.process_start_time = None;
11441 state.deliberate_severance = None;
11442}
11443
11444#[cfg(test)]
11445fn record_deliberate_severance(
11446 snapshot: &SharedSnapshot,
11447 identity: ProcessIdentity,
11448) -> Result<(), SuperviseError> {
11449 update_snapshot(snapshot, None, |state| {
11450 state.deliberate_severance = Some(identity);
11451 })
11452}
11453
11454fn apply_deliberate_severance_marker(
11455 snapshot: &SharedSnapshot,
11456 exited_identity: Option<ProcessIdentity>,
11457 mut exit_report: ExitReport,
11458) -> ExitReport {
11459 let marker = lock_snapshot(snapshot)
11460 .ok()
11461 .and_then(|mut state| state.deliberate_severance.take());
11462 if marker.is_some() && marker == exited_identity {
11463 exit_report.kind = ExitKind::DeliberateSeverance;
11464 }
11465 exit_report
11466}
11467
11468fn classify_reaped_child_exit(
11469 snapshot: &SharedSnapshot,
11470 child: &SupervisedChild,
11471 status: &ExitStatus,
11472) -> ExitReport {
11473 let _ = update_snapshot(snapshot, None, |state| {
11474 state.reaped_pid = Some(child.pid);
11475 state.spawn_failure = child.spawn_failure.clone();
11476 });
11477 apply_deliberate_severance_marker(snapshot, child.process_identity(), classify_exit(status))
11478}
11479
11480fn fail_snapshot(
11481 snapshot: &SharedSnapshot,
11482 module_id: Option<&str>,
11483 last_exit: Option<ExitReport>,
11484) {
11485 if let Err(err) = update_snapshot(snapshot, module_id, |state| {
11486 state.state = ModuleState::Failed;
11487 clear_current_process_facts(state);
11488 if let Some(last_exit) = last_exit {
11489 state.last_exit = Some(last_exit);
11490 }
11491 }) {
11492 error!(error = %err, "failed to mark supervisor state failed");
11493 }
11494}
11495
11496fn update_snapshot(
11497 snapshot: &SharedSnapshot,
11498 module_id: Option<&str>,
11499 update: impl FnOnce(&mut SupervisorSnapshot),
11500) -> Result<(), SuperviseError> {
11501 let mut state = snapshot.lock().map_err(|_| SuperviseError::StatePoisoned {
11502 module_id: module_id.map(ToOwned::to_owned),
11503 })?;
11504 update(&mut state);
11505 Ok(())
11506}
11507
11508const SLOW_SNAPSHOT_LOCK_THRESHOLD: Duration = Duration::from_millis(250);
11509
11510fn lock_snapshot_for_control<'a>(
11511 snapshot: &'a SharedSnapshot,
11512 module_id: &str,
11513 caller: &'static str,
11514) -> Result<std::sync::MutexGuard<'a, SupervisorSnapshot>, SuperviseError> {
11515 let started_at = Instant::now();
11516 let guard = lock_snapshot(snapshot)?;
11517 let waited = started_at.elapsed();
11518 if waited >= SLOW_SNAPSHOT_LOCK_THRESHOLD {
11519 warn!(
11520 module_id = %module_id,
11521 waited_ms = waited.as_millis() as u64,
11522 caller = %caller,
11523 "slow snapshot lock"
11524 );
11525 }
11526 Ok(guard)
11527}
11528
11529fn lock_snapshot(
11530 snapshot: &SharedSnapshot,
11531) -> Result<std::sync::MutexGuard<'_, SupervisorSnapshot>, SuperviseError> {
11532 snapshot
11533 .lock()
11534 .map_err(|_| SuperviseError::StatePoisoned { module_id: None })
11535}
11536
11537#[cfg(test)]
11538mod terminal_history_tests {
11539 use std::{
11540 path::PathBuf,
11541 sync::Arc,
11542 time::{Duration, Instant},
11543 };
11544
11545 use tokio::{sync::mpsc, time::sleep};
11546
11547 use super::{
11548 apply_deliberate_severance_marker, daemon_will_restart, drain_child_to_state,
11549 drained_after_quiescence_wait, handle_reload_spawn_failure, health_restart_child,
11550 lock_snapshot, on_child_exit, record_deliberate_severance, record_wait_error_terminal,
11551 reset_restart_count, spawn_and_mark_running, update_snapshot, wait_error_exit_report,
11552 ExitKind, ExitReport, ModuleProtocol, ModuleSpec, ModuleState, NextAction, ProcessIdentity,
11553 RestartPolicy, SpawnEventKind, StopNotice, SuperviseError, SupervisedConfiguration,
11554 SupervisedModule, SupervisedModuleInner, Supervisor, SupervisorHandle,
11555 SupervisorHealthStatus, SupervisorSnapshot,
11556 };
11557 use super::Instant as ClockInstant;
11562 use crate::{
11563 registry::Registry,
11564 terminal_ring::{TerminalRing, TerminalRingConfig},
11565 };
11566 use std::sync::Mutex;
11567 use subc_control::TerminalDisposition;
11568
11569 pub(super) fn fake_aft_stub_path() -> PathBuf {
11574 let mut path = std::env::current_exe().expect("current_exe available in tests");
11575 path.pop();
11576 path.pop();
11577 path.push(if cfg!(windows) {
11578 "fake-aft-stub.exe"
11579 } else {
11580 "fake-aft-stub"
11581 });
11582 assert!(
11583 path.exists(),
11584 "fake-aft-stub not built at {}: run `cargo test -p subc-core` (which builds \
11585 [[bin]] targets) rather than `cargo test -p subc-core --lib` (which does not)",
11586 path.display()
11587 );
11588 path
11589 }
11590
11591 #[test]
11592 fn reserved_never_spawned_refuses_every_hello() {
11593 let supervisor = SupervisorHandle::default();
11598 supervisor.apply_identity_configuration(&ModuleSpec {
11599 module_id: "never-spawned".to_string(),
11600 program: PathBuf::from("/usr/bin/false"),
11601 args: Vec::new(),
11602 env: Vec::new(),
11603 reserved: true,
11604 reserved_prefixes: Vec::new(),
11605 protocol: ModuleProtocol::Subc,
11606 overlap: Default::default(),
11607 });
11608 assert!(
11609 supervisor
11610 .reserved_hello_verification("never-spawned", Some("any-forged-nonce"))
11611 .is_err(),
11612 "forged nonce must refuse on a reserved never-spawned id"
11613 );
11614 assert!(
11615 supervisor
11616 .reserved_hello_verification("never-spawned", None)
11617 .is_err(),
11618 "absent nonce must refuse on a reserved never-spawned id"
11619 );
11620 supervisor.set_spawn_nonce("never-spawned", "minted".to_string());
11622 supervisor.apply_identity_configuration(&ModuleSpec {
11623 module_id: "never-spawned".to_string(),
11624 program: PathBuf::from("/usr/bin/false"),
11625 args: Vec::new(),
11626 env: Vec::new(),
11627 reserved: true,
11628 reserved_prefixes: Vec::new(),
11629 protocol: ModuleProtocol::Subc,
11630 overlap: Default::default(),
11631 });
11632 assert!(supervisor
11633 .reserved_hello_verification("never-spawned", Some("minted"))
11634 .is_ok());
11635 assert!(supervisor
11636 .reserved_hello_verification("never-spawned", Some("forged"))
11637 .is_err());
11638 }
11639
11640 fn seed_crash_restarts(state: &mut SupervisorSnapshot, count: u32) {
11643 let now = ClockInstant::now();
11644 for _ in 0..count {
11645 state.crash_restarts.push_back(now);
11646 }
11647 }
11648
11649 fn age_oldest_crash_restart_out_of_window(state: &mut SupervisorSnapshot, window: Duration) {
11653 let aged = state
11654 .crash_restarts
11655 .front()
11656 .expect("a crash restart must be recorded before it can be aged")
11657 .checked_sub(window + Duration::from_secs(1))
11658 .expect("the test clock is far enough from its origin to age an instant");
11659 state.crash_restarts[0] = aged;
11660 }
11661
11662 fn snapshot_with_restarts(enabled: bool, count: u32) -> SupervisorSnapshot {
11663 let mut state = SupervisorSnapshot::new(ModuleState::Running, enabled);
11664 seed_crash_restarts(&mut state, count);
11665 state
11666 }
11667
11668 #[test]
11669 fn daemon_owned_recovery_predicate_uses_the_pre_increment_budget() {
11670 let policy = RestartPolicy::new(3, Duration::ZERO);
11671 let now = ClockInstant::now();
11672 assert!(daemon_will_restart(
11673 &mut snapshot_with_restarts(true, 2),
11674 &policy,
11675 now
11676 ));
11677 assert!(!daemon_will_restart(
11678 &mut snapshot_with_restarts(true, 3),
11679 &policy,
11680 now
11681 ));
11682 assert!(!daemon_will_restart(
11683 &mut snapshot_with_restarts(false, 0),
11684 &policy,
11685 now
11686 ));
11687 }
11688
11689 #[test]
11690 fn crash_restart_backoff_escalates_with_in_window_count() {
11691 let policy = RestartPolicy::new(4, Duration::from_millis(100))
11692 .with_max_backoff(Duration::from_secs(30));
11693 let now = ClockInstant::now();
11694 let mut state = SupervisorSnapshot::new(ModuleState::Running, true);
11695 let schedules = (0..4)
11696 .map(|_| {
11697 state
11698 .next_crash_restart(&policy, now)
11699 .expect("the test policy allows four crash restarts")
11700 })
11701 .collect::<Vec<_>>();
11702
11703 assert_eq!(
11704 schedules
11705 .iter()
11706 .map(|schedule| schedule.restart_in_window)
11707 .collect::<Vec<_>>(),
11708 vec![0, 1, 2, 3]
11709 );
11710 assert_eq!(
11711 schedules
11712 .iter()
11713 .map(|schedule| schedule.delay)
11714 .collect::<Vec<_>>(),
11715 vec![
11716 Duration::from_millis(100),
11717 Duration::from_secs(1),
11718 Duration::from_secs(10),
11719 Duration::from_secs(30),
11720 ]
11721 );
11722 }
11723
11724 #[test]
11725 fn crash_restart_backoff_resets_after_ring_clear() {
11726 let policy = RestartPolicy::new(3, Duration::from_millis(100));
11727 let now = ClockInstant::now();
11728 let mut state = SupervisorSnapshot::new(ModuleState::Running, true);
11729 assert_eq!(
11730 state.next_crash_restart(&policy, now).unwrap().delay,
11731 Duration::from_millis(100)
11732 );
11733 assert_eq!(
11734 state.next_crash_restart(&policy, now).unwrap().delay,
11735 Duration::from_secs(1)
11736 );
11737
11738 state.clear_crash_restarts();
11739 let schedule = state
11740 .next_crash_restart(&policy, now)
11741 .expect("a cleared ring must allow another restart");
11742 assert_eq!(schedule.restart_in_window, 0);
11743 assert_eq!(schedule.delay, Duration::from_millis(100));
11744 }
11745
11746 #[test]
11747 fn crash_restart_backoff_ignores_aged_restarts() {
11748 let policy = RestartPolicy::new(3, Duration::from_millis(100));
11749 let now = ClockInstant::now();
11750 let mut state = SupervisorSnapshot::new(ModuleState::Running, true);
11751 state
11752 .next_crash_restart(&policy, now)
11753 .expect("the first restart is allowed");
11754 state
11755 .next_crash_restart(&policy, now)
11756 .expect("the second restart is allowed");
11757 state.crash_restarts[0] = now
11758 .checked_sub(policy.window + Duration::from_secs(1))
11759 .expect("the fake clock can age a restart past the window");
11760
11761 let schedule = state
11762 .next_crash_restart(&policy, now)
11763 .expect("an aged restart must release its slot");
11764 assert_eq!(schedule.restart_in_window, 1);
11765 assert_eq!(schedule.delay, Duration::from_secs(1));
11766 assert_eq!(state.crash_restarts.len(), 2);
11767 }
11768
11769 #[test]
11773 fn a_budget_spent_before_the_window_no_longer_refuses() {
11774 let policy = RestartPolicy::new(3, Duration::ZERO);
11775 let mut state = snapshot_with_restarts(true, 3);
11776 let now = ClockInstant::now();
11777 assert!(!daemon_will_restart(&mut state, &policy, now));
11778
11779 assert!(daemon_will_restart(
11780 &mut state,
11781 &policy,
11782 now + policy.window + Duration::from_secs(1)
11783 ));
11784 assert!(
11785 state.crash_restarts.is_empty(),
11786 "reading the budget must drop the instants that left the window"
11787 );
11788 }
11789
11790 fn module_with_recovery_snapshot(
11791 state: ModuleState,
11792 enabled: bool,
11793 restart_count: u32,
11794 ) -> SupervisedModule {
11795 let registry = Arc::new(Registry::default());
11796 let supervisor =
11797 Supervisor::new_for_test(Arc::clone(®istry), RestartPolicy::new(3, Duration::ZERO));
11798 let runtime = supervisor.runtime_config();
11799 let spec = ModuleSpec {
11800 module_id: "recovery-snapshot".to_string(),
11801 program: fake_aft_stub_path(),
11802 args: Vec::new(),
11803 env: Vec::new(),
11804 reserved: false,
11805 reserved_prefixes: Vec::new(),
11806 protocol: ModuleProtocol::Subc,
11807 overlap: Default::default(),
11808 };
11809 let mut snapshot = SupervisorSnapshot::new(state, enabled);
11810 seed_crash_restarts(&mut snapshot, restart_count);
11811 let (commands, _rx) = mpsc::channel(4);
11814 SupervisedModule {
11815 inner: Arc::new(SupervisedModuleInner {
11816 module_id: spec.module_id.clone(),
11817 registry,
11818 snapshot: Arc::new(Mutex::new(snapshot)),
11819 configuration: Arc::new(Mutex::new(SupervisedConfiguration {
11820 spec,
11821 health: runtime.health,
11822 })),
11823 stderr_ring: runtime.stderr_ring,
11824 terminal_ring: runtime.terminal_ring,
11825 commands,
11826 monitor: Mutex::new(None),
11827 restart_policy: runtime.restart_policy,
11828 effective_drain_timeout: runtime.effective_drain_timeout,
11829 provenance_probe: supervisor.provenance_probe.clone(),
11830 }),
11831 }
11832 }
11833
11834 #[cfg(target_os = "linux")]
11835 #[tokio::test]
11836 async fn no_cgroup_placement_does_not_block_fake_aft_stub_spawn() {
11837 let supervisor =
11838 Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default())
11839 .with_cgroup_placement(None);
11840 let result = supervisor.spawn(ModuleSpec {
11841 module_id: "no-cgroup-placement".to_string(),
11842 program: fake_aft_stub_path(),
11843 args: Vec::new(),
11844 env: Vec::new(),
11845 reserved: false,
11846 reserved_prefixes: Vec::new(),
11847 protocol: ModuleProtocol::Subc,
11848 overlap: Default::default(),
11849 });
11850
11851 assert!(
11852 result.is_ok(),
11853 "no delegation must not turn an otherwise valid spawn into a failure: {result:?}"
11854 );
11855 }
11856
11857 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
11858 async fn undecided_snapshot_uses_shared_restart_predicate() {
11859 assert!(module_with_recovery_snapshot(ModuleState::Running, true, 2)
11860 .will_recover_after_connection_loss()
11861 .unwrap());
11862 assert!(
11863 !module_with_recovery_snapshot(ModuleState::Running, true, 3)
11864 .will_recover_after_connection_loss()
11865 .unwrap()
11866 );
11867 }
11868
11869 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
11870 async fn restarting_snapshot_at_exhausted_budget_is_non_terminal() {
11871 assert!(
11872 module_with_recovery_snapshot(ModuleState::Restarting, true, 3)
11873 .will_recover_after_connection_loss()
11874 .unwrap()
11875 );
11876 }
11877
11878 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
11879 async fn terminal_phase_snapshots_are_terminal_before_budget_exhaustion() {
11880 assert!(!module_with_recovery_snapshot(ModuleState::Failed, true, 0)
11881 .will_recover_after_connection_loss()
11882 .unwrap());
11883 assert!(
11884 !module_with_recovery_snapshot(ModuleState::Disabled, true, 0)
11885 .will_recover_after_connection_loss()
11886 .unwrap()
11887 );
11888 }
11889
11890 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
11891 async fn warming_snapshot_is_limited_to_startup_phases() {
11892 for state in [
11893 ModuleState::Starting,
11894 ModuleState::Running,
11895 ModuleState::Restarting,
11896 ] {
11897 assert!(
11898 module_with_recovery_snapshot(state, true, 0)
11899 .is_warming()
11900 .unwrap(),
11901 "{state:?} should be warming"
11902 );
11903 }
11904 for state in [
11905 ModuleState::Unresponsive,
11906 ModuleState::Draining,
11907 ModuleState::Stopped,
11908 ModuleState::Failed,
11909 ModuleState::Disabled,
11910 ] {
11911 assert!(
11912 !module_with_recovery_snapshot(state, true, 0)
11913 .is_warming()
11914 .unwrap(),
11915 "{state:?} should not be warming"
11916 );
11917 }
11918 }
11919
11920 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
11921 async fn terminal_history_survives_respawn_and_keeps_both_crashes_in_order() {
11922 let registry = Arc::new(Registry::default());
11923 let supervisor =
11924 Supervisor::new_for_test(Arc::clone(®istry), RestartPolicy::new(1, Duration::ZERO));
11925 let module = supervisor
11926 .spawn(ModuleSpec {
11927 module_id: "terminal-history".to_string(),
11928 program: fake_aft_stub_path(),
11929 args: Vec::new(),
11930 env: vec![("FAKE_AFT_EXIT_CODE".to_string(), "23".to_string())],
11931 reserved: false,
11932 reserved_prefixes: Vec::new(),
11933 protocol: ModuleProtocol::Subc,
11934 overlap: Default::default(),
11935 })
11936 .unwrap();
11937
11938 let deadline = Instant::now() + Duration::from_secs(5);
11939 loop {
11940 let history = module.terminal_history();
11941 if history.entries.len() == 2 {
11942 assert_eq!(module.status().unwrap().state, ModuleState::Failed);
11943 assert_eq!(history.dropped, 0);
11944 assert_eq!(
11945 history
11946 .entries
11947 .iter()
11948 .map(|entry| entry.exit_code)
11949 .collect::<Vec<_>>(),
11950 vec![Some(23), Some(23)]
11951 );
11952 assert!(history.entries[0].at_ms <= history.entries[1].at_ms);
11953 return;
11954 }
11955 assert!(
11956 Instant::now() < deadline,
11957 "module did not retain two terminal exits: {history:?}"
11958 );
11959 sleep(Duration::from_millis(10)).await;
11960 }
11961 }
11962
11963 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
11967 async fn disable_during_crash_backoff_cancels_pending_respawn() {
11968 let backoff = Duration::from_secs(2);
11969 let supervisor = Supervisor::new_for_test(
11970 Arc::new(Registry::default()),
11971 RestartPolicy::new(10, backoff),
11972 );
11973 let module = supervisor
11974 .spawn(ModuleSpec {
11975 module_id: "disable-during-backoff".to_string(),
11976 program: fake_aft_stub_path(),
11977 args: Vec::new(),
11978 env: vec![("FAKE_AFT_EXIT_CODE".to_string(), "23".to_string())],
11979 reserved: false,
11980 reserved_prefixes: Vec::new(),
11981 protocol: ModuleProtocol::Subc,
11982 overlap: Default::default(),
11983 })
11984 .unwrap();
11985
11986 let deadline = Instant::now() + Duration::from_secs(5);
11988 loop {
11989 if module.status().unwrap().state == ModuleState::Restarting {
11990 break;
11991 }
11992 assert!(
11993 Instant::now() < deadline,
11994 "module never entered the crash backoff"
11995 );
11996 sleep(Duration::from_millis(10)).await;
11997 }
11998
11999 let started = Instant::now();
12000 module.set_enabled(false).await.unwrap();
12001 let waited = started.elapsed();
12002
12003 assert!(
12004 waited < backoff / 2,
12005 "disable waited {waited:?} behind the {backoff:?} crash backoff; the operator command must preempt the pending respawn"
12006 );
12007 assert_eq!(module.status().unwrap().state, ModuleState::Disabled);
12008
12009 sleep(backoff + Duration::from_millis(500)).await;
12011 let status = module.status().unwrap();
12012 assert_eq!(status.state, ModuleState::Disabled);
12013 assert_eq!(
12014 status.spawn_generation, 1,
12015 "module respawned after the operator disabled it"
12016 );
12017 }
12018
12019 #[cfg(unix)]
12022 fn protocol_none_sigterm_exits_clean_spec(
12023 module_id: &str,
12024 dir: &std::path::Path,
12025 ) -> (ModuleSpec, PathBuf, PathBuf) {
12026 let ready = dir.join("ready");
12027 let marker = dir.join("sigterm");
12028 let spec = ModuleSpec {
12029 module_id: module_id.to_string(),
12030 program: fake_aft_stub_path(),
12031 args: Vec::new(),
12032 env: vec![
12033 ("FAKE_AFT_NEVER_CONNECT".to_string(), "1".to_string()),
12034 (
12035 "FAKE_AFT_SIGTERM_MARKER_PATH".to_string(),
12036 marker.display().to_string(),
12037 ),
12038 (
12039 "FAKE_AFT_NEVER_CONNECT_READY_PATH".to_string(),
12040 ready.display().to_string(),
12041 ),
12042 ],
12043 reserved: false,
12044 reserved_prefixes: Vec::new(),
12045 protocol: ModuleProtocol::None,
12046 overlap: Default::default(),
12047 };
12048 (spec, ready, marker)
12049 }
12050
12051 #[cfg(unix)]
12055 async fn wait_for_file(path: &std::path::Path) {
12056 let deadline = Instant::now() + Duration::from_secs(10);
12057 while !path.exists() {
12058 assert!(
12059 Instant::now() < deadline,
12060 "{} never appeared",
12061 path.display()
12062 );
12063 sleep(Duration::from_millis(10)).await;
12064 }
12065 }
12066
12067 #[cfg(unix)]
12071 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
12072 async fn protocol_none_unrequested_clean_exit_restarts_as_a_crash() {
12073 let dir = cortexkit_test_support::ScratchDir::new("none-unrequested-clean-exit");
12074 let (spec, ready, marker) =
12075 protocol_none_sigterm_exits_clean_spec("none-unrequested-clean-exit", dir.path());
12076 let supervisor = Supervisor::new_for_test(
12077 Arc::new(Registry::default()),
12078 RestartPolicy::new(3, Duration::ZERO),
12079 );
12080 let module = supervisor.spawn(spec).unwrap();
12081 wait_for_file(&ready).await;
12082 let deadline = Instant::now() + Duration::from_secs(10);
12086 let first_pid = loop {
12087 let status = module.status().unwrap();
12088 if status.state == ModuleState::Running {
12089 if let Some(pid) = status.pid {
12090 break pid;
12091 }
12092 }
12093 assert!(
12094 Instant::now() < deadline,
12095 "a running module must report its pid after exec confirmation: {status:?}"
12096 );
12097 sleep(Duration::from_millis(10)).await;
12098 };
12099
12100 rustix::process::kill_process(
12101 rustix::process::Pid::from_raw(i32::try_from(first_pid).unwrap()).unwrap(),
12102 rustix::process::Signal::TERM,
12103 )
12104 .unwrap();
12105
12106 let deadline = Instant::now() + Duration::from_secs(10);
12107 let respawned = loop {
12108 let status = module.status().unwrap();
12109 if status.state == ModuleState::Running
12110 && status.pid.is_some_and(|pid| pid != first_pid)
12111 {
12112 break status;
12113 }
12114 assert!(
12115 Instant::now() < deadline,
12116 "protocol-none module was not respawned after an unrequested clean exit: {status:?}"
12117 );
12118 sleep(Duration::from_millis(10)).await;
12119 };
12120 assert_eq!(respawned.spawn_generation, 2);
12121 assert!(
12122 marker.exists(),
12123 "the child must have exited through its SIGTERM handler (exit 0), or this proves nothing about clean exits"
12124 );
12125
12126 let history = module.terminal_history();
12127 assert_eq!(history.entries.len(), 1, "{history:?}");
12128 let entry = &history.entries[0];
12129 assert_eq!(entry.exit_code, Some(0));
12130 assert_eq!(entry.exit_kind, subc_control::TerminalExitKind::Clean);
12131 assert_eq!(entry.disposition, TerminalDisposition::Restarting);
12132
12133 module.stop().await.unwrap();
12134 }
12135
12136 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
12140 async fn protocol_none_repeated_clean_exits_exhaust_the_restart_budget() {
12141 let supervisor = Supervisor::new_for_test(
12142 Arc::new(Registry::default()),
12143 RestartPolicy::new(1, Duration::ZERO),
12144 );
12145 let module = supervisor
12146 .spawn(ModuleSpec {
12147 module_id: "none-clean-exit-budget".to_string(),
12148 program: fake_aft_stub_path(),
12149 args: Vec::new(),
12150 env: vec![("FAKE_AFT_EXIT_CODE".to_string(), "0".to_string())],
12151 reserved: false,
12152 reserved_prefixes: Vec::new(),
12153 protocol: ModuleProtocol::None,
12154 overlap: Default::default(),
12155 })
12156 .unwrap();
12157
12158 let deadline = Instant::now() + Duration::from_secs(10);
12161 loop {
12162 let status = module.status().unwrap();
12163 if status.state == ModuleState::Failed {
12164 break;
12165 }
12166 assert!(
12167 Instant::now() < deadline,
12168 "module never exhausted its budget: {status:?} {:?}",
12169 module.terminal_history()
12170 );
12171 sleep(Duration::from_millis(10)).await;
12172 }
12173 let history = module.terminal_history();
12174 assert_eq!(
12175 history
12176 .entries
12177 .iter()
12178 .map(|entry| (entry.exit_code, entry.disposition.clone()))
12179 .collect::<Vec<_>>(),
12180 vec![
12181 (Some(0), TerminalDisposition::Restarting),
12182 (Some(0), TerminalDisposition::Failed),
12183 ]
12184 );
12185 let detail = history.entries[1]
12186 .disposition_detail
12187 .as_deref()
12188 .expect("a budget failure names the budget");
12189 assert!(detail.contains("max_restarts=1"), "{detail}");
12190 assert_eq!(module.status().unwrap().spawn_generation, 2);
12191 }
12192
12193 #[test]
12194 fn restart_budget_failure_is_published_after_its_terminal_record() {
12195 use std::io::{Read, Write};
12196 let supervisor = Supervisor::new_for_test(
12197 Arc::new(Registry::default()),
12198 RestartPolicy::new(0, Duration::ZERO),
12199 );
12200 let runtime = supervisor.runtime_config();
12201 let spec = ModuleSpec {
12202 protocol: ModuleProtocol::None,
12203 ..windowed_crash_spec("budget-publication-order")
12204 };
12205 let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::new(
12206 ModuleState::Running,
12207 true,
12208 )));
12209 lock_snapshot(&snapshot).unwrap().process_alive = true;
12210 let reaped_snapshot = snapshot.clone();
12211 let mut barrier = terminal_record_barrier(&runtime.spawn_events);
12212 let history = runtime.terminal_ring.clone();
12213 let reap = std::thread::spawn(move || {
12214 tokio::runtime::Builder::new_current_thread()
12215 .enable_all()
12216 .build()
12217 .unwrap()
12218 .block_on(on_child_exit(
12219 &spec,
12220 runtime.restart_policy,
12221 &supervisor.registry,
12222 &reaped_snapshot,
12223 &runtime.terminal_ring,
12224 &runtime.spawn_events,
12225 &runtime.child_roster,
12226 ExitReport {
12227 kind: ExitKind::Clean,
12228 code: Some(0),
12229 signal: None,
12230 at_ms: 1,
12231 },
12232 ))
12233 });
12234 let mut reached = [0];
12235 barrier.read_exact(&mut reached).unwrap();
12236 assert_eq!(&reached, b"R");
12237 let (before_state, before_alive) = {
12238 let state = lock_snapshot(&snapshot).unwrap();
12239 (state.state, state.process_alive)
12240 };
12241 let before_history = history.lock().unwrap().snapshot();
12242 barrier.write_all(b"X").unwrap();
12243 assert!(matches!(reap.join().unwrap(), NextAction::Stop { .. }));
12244 assert!(before_history.entries.is_empty());
12245 assert_ne!(
12246 before_state,
12247 ModuleState::Failed,
12248 "Failed was visible before its terminal record could be written"
12249 );
12250 assert!(before_alive);
12251 assert_eq!(lock_snapshot(&snapshot).unwrap().state, ModuleState::Failed);
12252 let history = history.lock().unwrap().snapshot();
12253 assert_eq!(history.entries.len(), 1);
12254 assert_eq!(history.entries[0].exit_code, Some(0));
12255 assert_eq!(history.entries[0].disposition, TerminalDisposition::Failed);
12256 }
12257
12258 #[test]
12259 fn crash_restart_at_budget_limit_remains_recoverable_while_recording() {
12260 use std::io::{Read, Write};
12261
12262 let supervisor = Supervisor::new_for_test(
12263 Arc::new(Registry::default()),
12264 RestartPolicy::new(3, Duration::ZERO),
12265 );
12266 let mut runtime = supervisor.runtime_config();
12267 let module = module_with_recovery_snapshot(ModuleState::Running, true, 2);
12268 let snapshot = module.inner.snapshot.clone();
12269 lock_snapshot(&snapshot).unwrap().process_alive = true;
12270 runtime.terminal_ring = module.inner.terminal_ring.clone();
12271 let spec = windowed_crash_spec(module.module_id());
12272 let mut barrier = terminal_record_barrier(&runtime.spawn_events);
12273 let reaped_snapshot = snapshot.clone();
12274 let reap = std::thread::spawn(move || {
12275 tokio::runtime::Builder::new_current_thread()
12276 .enable_all()
12277 .build()
12278 .unwrap()
12279 .block_on(on_child_exit(
12280 &spec,
12281 runtime.restart_policy,
12282 &supervisor.registry,
12283 &reaped_snapshot,
12284 &runtime.terminal_ring,
12285 &runtime.spawn_events,
12286 &runtime.child_roster,
12287 ExitReport {
12288 kind: ExitKind::Crash,
12289 code: Some(2),
12290 signal: None,
12291 at_ms: 1,
12292 },
12293 ))
12294 });
12295
12296 let mut reached = [0];
12300 barrier.read_exact(&mut reached).unwrap();
12301 assert_eq!(&reached, b"R");
12302 let before_status = module.status().unwrap();
12303 let before_history = module.terminal_history();
12304 let before_recovers = module.will_recover_after_connection_loss().unwrap();
12305 barrier.write_all(b"X").unwrap();
12306 assert!(matches!(reap.join().unwrap(), NextAction::Restart { .. }));
12307
12308 assert_eq!(before_status.state, ModuleState::Running);
12309 assert!(before_status.process_alive);
12310 assert_eq!(before_status.restart_count, 3);
12311 assert!(before_history.entries.is_empty());
12312 assert!(
12313 before_recovers,
12314 "connection cleanup would report a terminal close for a scheduled crash restart"
12315 );
12316 let restarting = module.status().unwrap();
12317 assert_eq!(restarting.state, ModuleState::Restarting);
12318 assert_eq!(restarting.restart_count, 3);
12319 assert!(module.will_recover_after_connection_loss().unwrap());
12320 let history = module.terminal_history();
12321 assert_eq!(history.entries.len(), 1);
12322 assert_eq!(
12323 history.entries[0].disposition,
12324 TerminalDisposition::Restarting
12325 );
12326
12327 update_snapshot(&snapshot, Some(module.module_id()), |state| {
12330 state.state = ModuleState::Running;
12331 state.respawn_pending = false;
12332 })
12333 .unwrap();
12334 assert!(!module.will_recover_after_connection_loss().unwrap());
12335 update_snapshot(&snapshot, Some(module.module_id()), |state| {
12336 state.enabled = false;
12337 state.respawn_pending = true;
12338 })
12339 .unwrap();
12340 assert!(!module.will_recover_after_connection_loss().unwrap());
12341 }
12342
12343 fn terminal_record_barrier(events: &super::SpawnEventFeed) -> std::net::TcpStream {
12344 use std::net::{TcpListener, TcpStream};
12345 let listener = TcpListener::bind("127.0.0.1:0").unwrap();
12346 let writer = TcpStream::connect(listener.local_addr().unwrap()).unwrap();
12347 let (barrier, _) = listener.accept().unwrap();
12348 for stream in [&writer, &barrier] {
12349 stream
12350 .set_read_timeout(Some(Duration::from_secs(10)))
12351 .unwrap();
12352 stream
12353 .set_write_timeout(Some(Duration::from_secs(10)))
12354 .unwrap();
12355 }
12356 *events.1.lock().unwrap() = Some(writer);
12357 barrier
12358 }
12359
12360 pub(super) fn before_terminal_record(events: &super::SpawnEventFeed) {
12361 use std::io::{Read, Write};
12362 let barrier = events.1.lock().unwrap().take();
12363 if let Some(mut barrier) = barrier {
12364 barrier.write_all(b"R").unwrap();
12365 let mut release = [0];
12366 barrier.read_exact(&mut release).unwrap();
12367 assert_eq!(&release, b"X");
12368 }
12369 }
12370
12371 #[test]
12372 fn health_restart_budget_failure_is_published_after_its_terminal_record() {
12373 use std::io::{Read, Write};
12374
12375 let supervisor = Supervisor::new_for_test(
12376 Arc::new(Registry::default()),
12377 RestartPolicy::new(0, Duration::ZERO),
12378 );
12379 let mut runtime = supervisor.runtime_config();
12380 let module = module_with_recovery_snapshot(ModuleState::Running, true, 0);
12381 let snapshot = module.inner.snapshot.clone();
12382 runtime.terminal_ring = module.inner.terminal_ring.clone();
12383 let spec = ModuleSpec {
12384 module_id: module.module_id().into(),
12385 program: fake_aft_stub_path(),
12386 protocol: ModuleProtocol::None,
12387 env: vec![("FAKE_AFT_EXIT_CODE".into(), "17".into())],
12388 ..windowed_crash_spec(module.module_id())
12389 };
12390 let mut barrier = terminal_record_barrier(&runtime.spawn_events);
12391 let reap = std::thread::spawn(move || {
12392 tokio::runtime::Builder::new_current_thread()
12393 .enable_all()
12394 .build()
12395 .unwrap()
12396 .block_on(async {
12397 let mut child =
12398 Some(spawn_and_mark_running(&spec, &runtime, &snapshot).unwrap());
12399 health_restart_child(
12400 &spec,
12401 &runtime,
12402 &supervisor.registry,
12403 &supervisor.process_liveness,
12404 &snapshot,
12405 &mut child,
12406 SupervisorHealthStatus::Failing,
12407 Some("health probe did not answer"),
12408 1,
12409 )
12410 .await
12411 .unwrap();
12412 assert!(child.is_none());
12413 });
12414 });
12415 let mut reached = [0];
12420 barrier.read_exact(&mut reached).unwrap();
12421 assert_eq!(&reached, b"R");
12422 let before_status = module.status().unwrap();
12423 let before_history = module.terminal_history();
12424 barrier.write_all(b"X").unwrap();
12425 reap.join().unwrap();
12426 assert!(before_history.entries.is_empty());
12427 assert_ne!(
12428 before_status.state,
12429 ModuleState::Failed,
12430 "Failed was visible before its terminal record could be written"
12431 );
12432 assert!(
12433 before_status.process_alive,
12434 "process death was visible before its terminal record could be written"
12435 );
12436 let failed = module.status().unwrap();
12437 assert_eq!(failed.state, ModuleState::Failed);
12438 assert!(!failed.process_alive);
12439 let history = module.terminal_history();
12440 assert_eq!(history.entries.len(), 1);
12441 assert_eq!(history.entries[0].disposition, TerminalDisposition::Failed);
12442 assert_eq!(
12443 history.entries[0].disposition_detail.as_deref(),
12444 Some("crash budget exhausted: max_restarts=0 within window_secs=600")
12445 );
12446 }
12447
12448 #[test]
12449 fn observed_exits_publish_process_death_after_terminal_history() {
12450 use std::io::{Read, Write};
12451 for (kind, enabled, shutdown, wait_error, expected_state) in [
12452 (ExitKind::Clean, true, false, false, ModuleState::Stopped),
12453 (ExitKind::Crash, true, false, false, ModuleState::Restarting),
12454 (ExitKind::Crash, false, false, false, ModuleState::Disabled),
12455 (
12456 ExitKind::DeliberateSeverance,
12457 true,
12458 false,
12459 false,
12460 ModuleState::Restarting,
12461 ),
12462 #[cfg(unix)]
12463 (ExitKind::Crash, true, true, false, ModuleState::Stopped),
12464 (ExitKind::Crash, true, false, true, ModuleState::Failed),
12465 ] {
12466 let supervisor = Supervisor::new_for_test(
12467 Arc::new(Registry::default()),
12468 RestartPolicy::new(1, Duration::ZERO),
12469 );
12470 let runtime = supervisor.runtime_config();
12471 #[cfg(unix)]
12472 if shutdown {
12473 runtime.child_roster.close();
12474 }
12475 #[cfg(not(unix))]
12476 let _ = shutdown;
12477 let spec = windowed_crash_spec("exit-publication-order");
12478 let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::new(
12479 ModuleState::Running,
12480 enabled,
12481 )));
12482 lock_snapshot(&snapshot).unwrap().process_alive = true;
12483 let reaped_snapshot = snapshot.clone();
12484 let history = runtime.terminal_ring.clone();
12485 let mut barrier = terminal_record_barrier(&runtime.spawn_events);
12486 let reap = std::thread::spawn(move || {
12487 if wait_error {
12488 super::fail_after_wait_error(
12489 &spec.module_id,
12490 &reaped_snapshot,
12491 &runtime.terminal_ring,
12492 &runtime.spawn_events,
12493 "wait failed".into(),
12494 );
12495 } else {
12496 tokio::runtime::Builder::new_current_thread()
12497 .enable_all()
12498 .build()
12499 .unwrap()
12500 .block_on(on_child_exit(
12501 &spec,
12502 runtime.restart_policy,
12503 &supervisor.registry,
12504 &reaped_snapshot,
12505 &runtime.terminal_ring,
12506 &runtime.spawn_events,
12507 &runtime.child_roster,
12508 ExitReport {
12509 kind,
12510 code: Some(0),
12511 signal: None,
12512 at_ms: 1,
12513 },
12514 ));
12515 }
12516 });
12517 let mut reached = [0];
12518 barrier.read_exact(&mut reached).unwrap();
12519 assert_eq!(&reached, b"R");
12520 let before_alive = lock_snapshot(&snapshot).unwrap().process_alive;
12521 let before_history = history.lock().unwrap().snapshot();
12522 barrier.write_all(b"X").unwrap();
12523 reap.join().unwrap();
12524 assert!(
12525 before_alive,
12526 "{kind:?}: death was published before recording"
12527 );
12528 assert!(before_history.entries.is_empty());
12529 assert_eq!(lock_snapshot(&snapshot).unwrap().state, expected_state);
12530 assert!(!lock_snapshot(&snapshot).unwrap().process_alive);
12531 assert_eq!(history.lock().unwrap().snapshot().entries.len(), 1);
12532 }
12533 }
12534
12535 #[tokio::test]
12536 async fn health_restart_budget_failure_remains_failed_when_journal_append_fails() {
12537 let dir = cortexkit_test_support::ScratchDir::new("health-budget-journal-failure");
12538 let path = dir.join("terminals.jsonl");
12539 std::fs::create_dir(&path).unwrap();
12540 let supervisor = Supervisor::new_for_test(
12541 Arc::new(Registry::default()),
12542 RestartPolicy::new(0, Duration::ZERO),
12543 )
12544 .with_terminal_journal(path, "health-budget-journal-failure".into());
12545 let runtime = supervisor.runtime_config();
12546 let spec = ModuleSpec {
12547 program: fake_aft_stub_path(),
12548 protocol: ModuleProtocol::None,
12549 env: vec![("FAKE_AFT_EXIT_CODE".into(), "17".into())],
12550 ..windowed_crash_spec("health-budget-journal-failure")
12551 };
12552 let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::starting()));
12553 let mut child = Some(spawn_and_mark_running(&spec, &runtime, &snapshot).unwrap());
12554 health_restart_child(
12555 &spec,
12556 &runtime,
12557 &supervisor.registry,
12558 &supervisor.process_liveness,
12559 &snapshot,
12560 &mut child,
12561 SupervisorHealthStatus::Failing,
12562 None,
12563 1,
12564 )
12565 .await
12566 .unwrap();
12567 assert_eq!(lock_snapshot(&snapshot).unwrap().state, ModuleState::Failed);
12568 assert!(!lock_snapshot(&snapshot).unwrap().process_alive);
12569 let history = runtime
12570 .terminal_ring
12571 .lock()
12572 .unwrap()
12573 .durable_history(&spec.module_id);
12574 assert!(history.journal_write_failures > 0);
12575 assert_eq!(history.entries.len(), 1);
12576 assert_eq!(history.entries[0].disposition, TerminalDisposition::Failed);
12577 }
12578
12579 #[test]
12580 fn health_restart_budget_failure_remains_failed_when_exit_recording_panics() {
12581 let supervisor = Supervisor::new_for_test(
12582 Arc::new(Registry::default()),
12583 RestartPolicy::new(0, Duration::ZERO),
12584 );
12585 let runtime = supervisor.runtime_config();
12586 let spec = ModuleSpec {
12587 program: fake_aft_stub_path(),
12588 protocol: ModuleProtocol::None,
12589 env: vec![("FAKE_AFT_EXIT_CODE".into(), "17".into())],
12590 ..windowed_crash_spec("health-budget-recording-panic")
12591 };
12592 let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::starting()));
12593 let result = std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| {
12594 tokio::runtime::Builder::new_current_thread()
12595 .enable_all()
12596 .build()
12597 .unwrap()
12598 .block_on(async {
12599 let mut child =
12600 Some(spawn_and_mark_running(&spec, &runtime, &snapshot).unwrap());
12601 runtime.spawn_events.0.lock().unwrap().seq = u64::MAX;
12602 health_restart_child(
12603 &spec,
12604 &runtime,
12605 &supervisor.registry,
12606 &supervisor.process_liveness,
12607 &snapshot,
12608 &mut child,
12609 SupervisorHealthStatus::Failing,
12610 None,
12611 1,
12612 )
12613 .await
12614 .unwrap();
12615 });
12616 }));
12617 let panic = result.expect_err("recording must still unwind");
12618 assert_eq!(
12619 panic.downcast_ref::<String>().map(String::as_str),
12620 Some("spawn event sequence exhausted")
12621 );
12622 let state = lock_snapshot(&snapshot).unwrap();
12623 assert_eq!(state.state, ModuleState::Failed);
12624 assert!(!state.process_alive);
12625 assert!(state.last_exit.is_some());
12626 }
12627
12628 #[tokio::test]
12629 async fn restart_budget_failure_remains_failed_when_journal_append_fails() {
12630 let dir = cortexkit_test_support::ScratchDir::new("budget-journal-failure");
12631 let path = dir.join("terminals.jsonl");
12632 std::fs::create_dir(&path).unwrap();
12633 let supervisor = Supervisor::new_for_test(
12634 Arc::new(Registry::default()),
12635 RestartPolicy::new(0, Duration::ZERO),
12636 )
12637 .with_terminal_journal(path, "budget-journal-failure".into());
12638 let runtime = supervisor.runtime_config();
12639 let spec = windowed_crash_spec("budget-journal-failure");
12640 let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::starting()));
12641 assert!(matches!(
12642 on_child_exit(
12643 &spec,
12644 runtime.restart_policy,
12645 &supervisor.registry,
12646 &snapshot,
12647 &runtime.terminal_ring,
12648 &runtime.spawn_events,
12649 &runtime.child_roster,
12650 crash_exit_report(1),
12651 )
12652 .await,
12653 NextAction::Stop { .. }
12654 ));
12655 assert_eq!(lock_snapshot(&snapshot).unwrap().state, ModuleState::Failed);
12656 let history = runtime
12657 .terminal_ring
12658 .lock()
12659 .unwrap()
12660 .durable_history(&spec.module_id);
12661 assert!(history.journal_write_failures > 0);
12662 assert_eq!(history.entries.len(), 1);
12663 assert_eq!(history.entries[0].disposition, TerminalDisposition::Failed);
12664 }
12665
12666 #[test]
12667 fn restart_budget_failure_remains_failed_when_exit_recording_panics() {
12668 let supervisor = Supervisor::new_for_test(
12669 Arc::new(Registry::default()),
12670 RestartPolicy::new(0, Duration::ZERO),
12671 );
12672 let runtime = supervisor.runtime_config();
12673 let spec = windowed_crash_spec("budget-recording-panic");
12674 let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::starting()));
12675 runtime.spawn_events.emit_spawned(&spec.module_id, 1, 1);
12676 runtime.spawn_events.0.lock().unwrap().seq = u64::MAX;
12679 let result = std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| {
12680 tokio::runtime::Builder::new_current_thread()
12681 .enable_all()
12682 .build()
12683 .unwrap()
12684 .block_on(on_child_exit(
12685 &spec,
12686 runtime.restart_policy,
12687 &supervisor.registry,
12688 &snapshot,
12689 &runtime.terminal_ring,
12690 &runtime.spawn_events,
12691 &runtime.child_roster,
12692 crash_exit_report(1),
12693 ))
12694 }));
12695 let panic = result.err().expect("recording must still unwind");
12696 assert_eq!(
12697 panic.downcast_ref::<String>().map(String::as_str),
12698 Some("spawn event sequence exhausted")
12699 );
12700 assert_eq!(lock_snapshot(&snapshot).unwrap().state, ModuleState::Failed);
12701 }
12702
12703 #[cfg(unix)]
12706 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
12707 async fn protocol_none_requested_stop_and_disable_do_not_respawn() {
12708 for disable in [false, true] {
12709 let label = if disable {
12710 "none-requested-disable"
12711 } else {
12712 "none-requested-stop"
12713 };
12714 let dir = cortexkit_test_support::ScratchDir::new(label);
12715 let (spec, ready, marker) = protocol_none_sigterm_exits_clean_spec(label, dir.path());
12716 let supervisor = Supervisor::new_for_test(
12717 Arc::new(Registry::default()),
12718 RestartPolicy::new(3, Duration::ZERO),
12719 );
12720 let module = supervisor.spawn(spec).unwrap();
12721 wait_for_file(&ready).await;
12722
12723 if disable {
12724 module.set_enabled(false).await.unwrap();
12725 } else {
12726 module.stop().await.unwrap();
12727 }
12728 assert!(
12729 marker.exists(),
12730 "{label}: the child must have left through its SIGTERM handler with exit 0"
12731 );
12732
12733 sleep(Duration::from_millis(500)).await;
12736 let status = module.status().unwrap();
12737 let expected = if disable {
12738 ModuleState::Disabled
12739 } else {
12740 ModuleState::Stopped
12741 };
12742 assert_eq!(status.state, expected, "{label}");
12743 assert_eq!(
12744 status.spawn_generation, 1,
12745 "{label}: respawned after a requested stop"
12746 );
12747 let history = module.terminal_history();
12748 assert_eq!(history.entries.len(), 1, "{label}: {history:?}");
12749 assert_eq!(history.entries[0].exit_code, Some(0), "{label}");
12750 assert_ne!(
12751 history.entries[0].disposition,
12752 TerminalDisposition::Restarting,
12753 "{label}"
12754 );
12755 }
12756 }
12757
12758 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
12761 async fn subc_wire_clean_exit_is_still_a_stop() {
12762 let supervisor = Supervisor::new_for_test(
12763 Arc::new(Registry::default()),
12764 RestartPolicy::new(3, Duration::ZERO),
12765 );
12766 let module = supervisor
12767 .spawn(ModuleSpec {
12768 module_id: "wire-clean-exit".to_string(),
12769 program: fake_aft_stub_path(),
12770 args: Vec::new(),
12771 env: vec![("FAKE_AFT_EXIT_CODE".to_string(), "0".to_string())],
12772 reserved: false,
12773 reserved_prefixes: Vec::new(),
12774 protocol: ModuleProtocol::Subc,
12775 overlap: Default::default(),
12776 })
12777 .unwrap();
12778
12779 let deadline = Instant::now() + Duration::from_secs(10);
12780 while module.terminal_history().entries.is_empty() {
12781 assert!(Instant::now() < deadline, "module never exited");
12782 sleep(Duration::from_millis(10)).await;
12783 }
12784 sleep(Duration::from_millis(500)).await;
12786 let status = module.status().unwrap();
12787 assert_eq!(status.state, ModuleState::Stopped);
12788 assert_eq!(status.spawn_generation, 1);
12789 let history = module.terminal_history();
12790 assert_eq!(history.entries.len(), 1, "{history:?}");
12791 assert_eq!(history.entries[0].exit_code, Some(0));
12792 assert_eq!(history.entries[0].disposition, TerminalDisposition::Stopped);
12793 }
12794
12795 #[cfg(unix)]
12796 #[tokio::test]
12797 async fn reload_exit_keeps_roster_until_terminal_is_recorded_during_shutdown() {
12798 let dir = cortexkit_test_support::ScratchDir::new("reload-roster-terminal-order");
12799 let record = dir.join("live-children.json");
12800 let supervisor = Supervisor::new_for_test(
12801 Arc::new(Registry::default()),
12802 RestartPolicy::new(0, Duration::ZERO),
12803 );
12804 let mut runtime = supervisor.runtime_config();
12805 runtime.child_roster.record_to(record.clone());
12806 let gate = Arc::new(super::ReloadExitRecordGate::default());
12807 runtime.test_reload_exit_record_gate = Some(Arc::clone(&gate));
12808 let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::starting()));
12809 let spec = ModuleSpec {
12810 module_id: "reload-exit-roster".into(),
12811 program: fake_aft_stub_path(),
12812 args: Vec::new(),
12813 env: vec![("FAKE_AFT_EXIT_CODE".into(), "23".into())],
12814 reserved: false,
12815 reserved_prefixes: Vec::new(),
12816 protocol: ModuleProtocol::Subc,
12817 overlap: Default::default(),
12818 };
12819 let mut child = None;
12820 let reload = super::finish_reload_child(
12821 &spec,
12822 &runtime,
12823 &supervisor.registry,
12824 &supervisor.process_liveness,
12825 &snapshot,
12826 &mut child,
12827 );
12828 tokio::pin!(reload);
12829 tokio::select! {
12830 result = &mut reload => panic!("reload missed the before-record gate: {result:?}"),
12831 _ = gate.reached.notified() => {}
12832 }
12833 assert!(runtime
12834 .terminal_ring
12835 .lock()
12836 .unwrap()
12837 .snapshot()
12838 .entries
12839 .is_empty());
12840 assert_eq!(
12841 crate::live_children::read_record(&record).unwrap().len(),
12842 1,
12843 "shutdown must still wait for the reaped child until its terminal record exists"
12844 );
12845 runtime.child_roster.close();
12846 gate.resume.notify_one();
12847 assert!(reload.await.is_err());
12848 assert!(crate::live_children::read_record(&record)
12849 .unwrap()
12850 .is_empty());
12851 let history = runtime.terminal_ring.lock().unwrap().snapshot();
12852 assert_eq!(history.entries.len(), 1);
12853 assert_eq!(
12854 history.entries[0].disposition,
12855 TerminalDisposition::DaemonShutdown
12856 );
12857 }
12858
12859 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
12863 async fn every_restart_increment_path_advances_lifetime_count() {
12864 let supervisor = Supervisor::new_for_test(
12865 Arc::new(Registry::default()),
12866 RestartPolicy::new(1, Duration::ZERO),
12867 );
12868 let runtime = supervisor.runtime_config();
12869 let spec = ModuleSpec {
12870 module_id: "lifetime-increment-path".to_string(),
12871 program: PathBuf::from("/unused/lifetime-increment-path"),
12872 args: Vec::new(),
12873 env: Vec::new(),
12874 reserved: false,
12875 reserved_prefixes: Vec::new(),
12876 protocol: ModuleProtocol::Subc,
12877 overlap: Default::default(),
12878 };
12879
12880 let crash_snapshot = Arc::new(Mutex::new(SupervisorSnapshot::starting()));
12881 assert!(matches!(
12882 on_child_exit(
12883 &spec,
12884 runtime.restart_policy,
12885 &supervisor.registry,
12886 &crash_snapshot,
12887 &runtime.terminal_ring,
12888 &runtime.spawn_events,
12889 &runtime.child_roster,
12890 ExitReport {
12891 kind: ExitKind::Crash,
12892 code: Some(1),
12893 signal: None,
12894 at_ms: 1,
12895 },
12896 )
12897 .await,
12898 NextAction::Restart { schedule: _ }
12899 ));
12900 let (crash_restarts, crash_lifetime) = {
12901 let state = lock_snapshot(&crash_snapshot).unwrap();
12902 (state.crash_restarts.len(), state.lifetime_restarts)
12903 };
12904 assert_eq!(crash_restarts, 1);
12905 assert_eq!(crash_lifetime, 1);
12906
12907 let health_snapshot = Arc::new(Mutex::new(SupervisorSnapshot::starting()));
12908 let mut health_child = None;
12909 assert!(matches!(
12910 health_restart_child(
12911 &spec,
12912 &runtime,
12913 &supervisor.registry,
12914 &supervisor.process_liveness,
12915 &health_snapshot,
12916 &mut health_child,
12917 SupervisorHealthStatus::Failing,
12918 None,
12919 2,
12920 )
12921 .await,
12922 Ok(())
12923 ));
12924 assert!(health_child.is_none());
12925 assert!(lock_snapshot(&health_snapshot).unwrap().respawn_pending);
12926 assert!(runtime.scheduled_respawn.lock().unwrap().take().is_some());
12927 let (health_restarts, health_lifetime) = {
12928 let state = lock_snapshot(&health_snapshot).unwrap();
12929 (state.crash_restarts.len(), state.lifetime_restarts)
12930 };
12931 assert_eq!(health_restarts, 1);
12932 assert_eq!(health_lifetime, 1);
12933
12934 let reload_snapshot = Arc::new(Mutex::new(SupervisorSnapshot::starting()));
12935 let mut reload_child = None;
12936 assert!(matches!(
12937 handle_reload_spawn_failure(
12938 &spec,
12939 &runtime,
12940 &supervisor.process_liveness,
12941 &reload_snapshot,
12942 &mut reload_child,
12943 "forced reload spawn failure".to_string(),
12944 )
12945 .await,
12946 Err(SuperviseError::ReloadFailed { .. })
12947 ));
12948 let (reload_restarts, reload_lifetime) = {
12949 let state = lock_snapshot(&reload_snapshot).unwrap();
12950 (state.crash_restarts.len(), state.lifetime_restarts)
12951 };
12952 assert_eq!(reload_restarts, 1);
12953 assert_eq!(reload_lifetime, 1);
12954 }
12955
12956 #[tokio::test]
12957 async fn deliberately_severed_live_child_records_lifetime_without_spending_restart_budget() {
12958 let supervisor = Supervisor::new_for_test(
12959 Arc::new(Registry::default()),
12960 RestartPolicy::new(3, Duration::ZERO),
12961 );
12962 let runtime = supervisor.runtime_config();
12963 let spec = ModuleSpec {
12964 module_id: "deliberately-severed".to_string(),
12965 program: PathBuf::from("/unused/deliberately-severed"),
12966 args: Vec::new(),
12967 env: Vec::new(),
12968 reserved: false,
12969 reserved_prefixes: Vec::new(),
12970 protocol: ModuleProtocol::Subc,
12971 overlap: Default::default(),
12972 };
12973 let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::starting()));
12974 let process = ProcessIdentity {
12975 pid: 41,
12976 start_time: 101,
12977 };
12978 record_deliberate_severance(&snapshot, process).unwrap();
12979 let exit_report = apply_deliberate_severance_marker(
12980 &snapshot,
12981 Some(process),
12982 ExitReport {
12983 kind: ExitKind::Crash,
12984 code: Some(1),
12985 signal: None,
12986 at_ms: 1,
12987 },
12988 );
12989 assert_eq!(exit_report.kind, ExitKind::DeliberateSeverance);
12990
12991 assert!(matches!(
12992 on_child_exit(
12993 &spec,
12994 runtime.restart_policy,
12995 &supervisor.registry,
12996 &snapshot,
12997 &runtime.terminal_ring,
12998 &runtime.spawn_events,
12999 &runtime.child_roster,
13000 exit_report,
13001 )
13002 .await,
13003 NextAction::Restart { schedule: _ }
13004 ));
13005 let state = lock_snapshot(&snapshot).unwrap();
13006 assert_eq!(state.lifetime_restarts, 1);
13007 assert_eq!(state.crash_restarts.len(), 0);
13008 }
13009
13010 #[tokio::test]
13011 async fn genuine_crash_spends_restart_budget_and_records_lifetime() {
13012 let supervisor = Supervisor::new_for_test(
13013 Arc::new(Registry::default()),
13014 RestartPolicy::new(3, Duration::ZERO),
13015 );
13016 let runtime = supervisor.runtime_config();
13017 let spec = ModuleSpec {
13018 module_id: "genuine-crash".to_string(),
13019 program: PathBuf::from("/unused/genuine-crash"),
13020 args: Vec::new(),
13021 env: Vec::new(),
13022 reserved: false,
13023 reserved_prefixes: Vec::new(),
13024 protocol: ModuleProtocol::Subc,
13025 overlap: Default::default(),
13026 };
13027 let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::starting()));
13028
13029 assert!(matches!(
13030 on_child_exit(
13031 &spec,
13032 runtime.restart_policy,
13033 &supervisor.registry,
13034 &snapshot,
13035 &runtime.terminal_ring,
13036 &runtime.spawn_events,
13037 &runtime.child_roster,
13038 ExitReport {
13039 kind: ExitKind::Crash,
13040 code: Some(1),
13041 signal: None,
13042 at_ms: 1,
13043 },
13044 )
13045 .await,
13046 NextAction::Restart { schedule: _ }
13047 ));
13048 let state = lock_snapshot(&snapshot).unwrap();
13049 assert_eq!(state.lifetime_restarts, 1);
13050 assert_eq!(state.crash_restarts.len(), 1);
13051 }
13052
13053 fn crash_exit_report(at_ms: u64) -> ExitReport {
13054 ExitReport {
13055 kind: ExitKind::Crash,
13056 code: Some(1),
13057 signal: None,
13058 at_ms,
13059 }
13060 }
13061
13062 fn windowed_crash_spec(module_id: &str) -> ModuleSpec {
13063 ModuleSpec {
13064 module_id: module_id.to_string(),
13065 program: PathBuf::from("/unused").join(module_id),
13066 args: Vec::new(),
13067 env: Vec::new(),
13068 reserved: false,
13069 reserved_prefixes: Vec::new(),
13070 protocol: ModuleProtocol::Subc,
13071 overlap: Default::default(),
13072 }
13073 }
13074
13075 #[tokio::test]
13081 async fn three_crashes_inside_the_window_stop_the_module_and_name_the_window() {
13082 let (logs, _guard) = crate::router::test_log::log_capture(tracing::Level::ERROR);
13083 let supervisor = Supervisor::new_for_test(
13084 Arc::new(Registry::default()),
13085 RestartPolicy::new(2, Duration::ZERO),
13086 );
13087 let runtime = supervisor.runtime_config();
13088 let spec = windowed_crash_spec("crash-loop-in-window");
13089 let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::starting()));
13090
13091 for attempt in 1..=2 {
13092 assert!(
13093 matches!(
13094 on_child_exit(
13095 &spec,
13096 runtime.restart_policy,
13097 &supervisor.registry,
13098 &snapshot,
13099 &runtime.terminal_ring,
13100 &runtime.spawn_events,
13101 &runtime.child_roster,
13102 crash_exit_report(attempt),
13103 )
13104 .await,
13105 NextAction::Restart { schedule: _ }
13106 ),
13107 "crash {attempt} is inside the budget and must respawn"
13108 );
13109 }
13110
13111 assert!(matches!(
13112 on_child_exit(
13113 &spec,
13114 runtime.restart_policy,
13115 &supervisor.registry,
13116 &snapshot,
13117 &runtime.terminal_ring,
13118 &runtime.spawn_events,
13119 &runtime.child_roster,
13120 crash_exit_report(3),
13121 )
13122 .await,
13123 NextAction::Stop { .. }
13124 ));
13125
13126 {
13127 let state = lock_snapshot(&snapshot).unwrap();
13128 assert_eq!(state.state, ModuleState::Failed);
13129 assert_eq!(state.crash_restarts.len(), 2);
13130 assert_eq!(state.lifetime_restarts, 2);
13131 }
13132
13133 let history = runtime
13134 .terminal_ring
13135 .lock()
13136 .expect("terminal ring is not poisoned")
13137 .snapshot();
13138 let last = history
13139 .entries
13140 .last()
13141 .expect("the refused crash is retained");
13142 assert_eq!(last.disposition, TerminalDisposition::Failed);
13143 assert_eq!(
13144 last.disposition_detail.as_deref(),
13145 Some("crash budget exhausted: max_restarts=2 within window_secs=600")
13146 );
13147
13148 let captured = crate::router::test_log::captured_logs(&logs);
13149 assert!(
13150 captured.contains("crash budget exhausted: max_restarts=2 within window_secs=600"),
13151 "the stop must be logged with its window: {captured}"
13152 );
13153 }
13154
13155 #[tokio::test]
13163 async fn a_crash_older_than_the_window_frees_its_slot_for_a_later_crash() {
13164 let supervisor = Supervisor::new_for_test(
13165 Arc::new(Registry::default()),
13166 RestartPolicy::new(2, Duration::ZERO),
13167 );
13168 let runtime = supervisor.runtime_config();
13169 let spec = windowed_crash_spec("crash-across-windows");
13170 let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::starting()));
13171
13172 for attempt in 1..=2 {
13173 assert!(matches!(
13174 on_child_exit(
13175 &spec,
13176 runtime.restart_policy,
13177 &supervisor.registry,
13178 &snapshot,
13179 &runtime.terminal_ring,
13180 &runtime.spawn_events,
13181 &runtime.child_roster,
13182 crash_exit_report(attempt),
13183 )
13184 .await,
13185 NextAction::Restart { schedule: _ }
13186 ));
13187 }
13188
13189 update_snapshot(&snapshot, Some(&spec.module_id), |state| {
13192 age_oldest_crash_restart_out_of_window(state, runtime.restart_policy.window);
13193 })
13194 .unwrap();
13195
13196 assert!(
13197 matches!(
13198 on_child_exit(
13199 &spec,
13200 runtime.restart_policy,
13201 &supervisor.registry,
13202 &snapshot,
13203 &runtime.terminal_ring,
13204 &runtime.spawn_events,
13205 &runtime.child_roster,
13206 crash_exit_report(3),
13207 )
13208 .await,
13209 NextAction::Restart { schedule: _ }
13210 ),
13211 "a crash older than the window must not hold a budget slot"
13212 );
13213
13214 let state = lock_snapshot(&snapshot).unwrap();
13215 assert_eq!(state.state, ModuleState::Restarting);
13216 assert_eq!(
13217 state.crash_restarts.len(),
13218 2,
13219 "the aged instant is dropped and the new one takes its place"
13220 );
13221 assert_eq!(
13222 state.lifetime_restarts, 3,
13223 "the ledger counts every restart, including the ones the window forgot"
13224 );
13225 }
13226
13227 #[tokio::test]
13232 async fn an_operator_restart_clears_the_ring_and_leaves_the_ledger_alone() {
13233 let supervisor = Supervisor::new_for_test(
13234 Arc::new(Registry::default()),
13235 RestartPolicy::new(2, Duration::ZERO),
13236 );
13237 let runtime = supervisor.runtime_config();
13238 let spec = windowed_crash_spec("operator-cleared-budget");
13239 let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::starting()));
13240
13241 for attempt in 1..=2 {
13242 assert!(matches!(
13243 on_child_exit(
13244 &spec,
13245 runtime.restart_policy,
13246 &supervisor.registry,
13247 &snapshot,
13248 &runtime.terminal_ring,
13249 &runtime.spawn_events,
13250 &runtime.child_roster,
13251 crash_exit_report(attempt),
13252 )
13253 .await,
13254 NextAction::Restart { schedule: _ }
13255 ));
13256 }
13257
13258 reset_restart_count(&snapshot, &spec.module_id).unwrap();
13259 {
13260 let state = lock_snapshot(&snapshot).unwrap();
13261 assert!(
13262 state.crash_restarts.is_empty(),
13263 "an operator restart returns the full budget"
13264 );
13265 assert_eq!(
13266 state.lifetime_restarts, 2,
13267 "clearing the budget must not unmake the crashes"
13268 );
13269 }
13270
13271 assert!(
13272 matches!(
13273 on_child_exit(
13274 &spec,
13275 runtime.restart_policy,
13276 &supervisor.registry,
13277 &snapshot,
13278 &runtime.terminal_ring,
13279 &runtime.spawn_events,
13280 &runtime.child_roster,
13281 crash_exit_report(3),
13282 )
13283 .await,
13284 NextAction::Restart { schedule: _ }
13285 ),
13286 "the cleared budget must be spendable again"
13287 );
13288 let state = lock_snapshot(&snapshot).unwrap();
13289 assert_eq!(state.crash_restarts.len(), 1);
13290 assert_eq!(state.lifetime_restarts, 3);
13291 }
13292
13293 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
13294 async fn severance_marker_for_a_dead_child_does_not_label_its_successor() {
13295 let severed = ProcessIdentity {
13296 pid: 41,
13297 start_time: 101,
13298 };
13299 let successor = ProcessIdentity {
13300 pid: 41,
13301 start_time: 202,
13302 };
13303 let module = module_with_recovery_snapshot(ModuleState::Running, true, 0);
13304 update_snapshot(&module.inner.snapshot, Some("recovery-snapshot"), |state| {
13305 state.pid = Some(successor.pid);
13306 state.process_start_time = Some(successor.start_time);
13307 })
13308 .unwrap();
13309 assert!(!module.record_deliberate_severance(severed).unwrap());
13310
13311 let exit_report = apply_deliberate_severance_marker(
13312 &module.inner.snapshot,
13313 Some(successor),
13314 ExitReport {
13315 kind: ExitKind::Crash,
13316 code: Some(1),
13317 signal: None,
13318 at_ms: 1,
13319 },
13320 );
13321
13322 assert_eq!(exit_report.kind, ExitKind::Crash);
13323 }
13324
13325 #[tokio::test]
13326 async fn drain_reap_marks_deliberate_severance_and_records_lifetime_without_budget() {
13327 let registry = Registry::default();
13328 let supervisor = Supervisor::new_for_test(
13329 Arc::new(Registry::default()),
13330 RestartPolicy::new(3, Duration::ZERO),
13331 );
13332 let runtime = supervisor.runtime_config();
13333 let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::starting()));
13334 let spec = ModuleSpec {
13335 module_id: "drain-deliberate-severance".to_string(),
13336 program: fake_aft_stub_path(),
13337 args: Vec::new(),
13338 env: vec![("FAKE_AFT_EXIT_CODE".to_string(), "23".to_string())],
13339 reserved: false,
13340 reserved_prefixes: Vec::new(),
13341 protocol: ModuleProtocol::Subc,
13342 overlap: Default::default(),
13343 };
13344 let mut child = spawn_and_mark_running(&spec, &runtime, &snapshot).unwrap();
13345 let process = ProcessIdentity {
13346 pid: 41,
13347 start_time: 101,
13348 };
13349 child.process_identity = Some(process);
13350 update_snapshot(&snapshot, Some(&spec.module_id), |state| {
13351 state.pid = Some(process.pid);
13352 state.process_start_time = Some(process.start_time);
13353 })
13354 .unwrap();
13355 record_deliberate_severance(&snapshot, process).unwrap();
13356
13357 drain_child_to_state(
13358 &spec.module_id,
13359 spec.protocol,
13360 StopNotice::SentOverConnection,
13363 ®istry,
13364 None,
13365 &snapshot,
13366 &runtime.terminal_ring,
13367 &runtime.spawn_events,
13368 child,
13369 Duration::from_secs(1),
13370 ModuleState::Stopped,
13371 Some(false),
13372 )
13373 .await
13374 .unwrap();
13375
13376 let state = lock_snapshot(&snapshot).unwrap();
13377 assert_eq!(
13378 state.last_exit.as_ref().map(|exit| exit.kind),
13379 Some(ExitKind::DeliberateSeverance)
13380 );
13381 assert_eq!(state.lifetime_restarts, 1);
13382 assert_eq!(state.crash_restarts.len(), 0);
13383 drop(state);
13384 let history = runtime.terminal_ring.lock().unwrap().snapshot();
13385 assert_eq!(
13386 history.entries[0].exit_kind,
13387 subc_control::TerminalExitKind::DeliberateSeverance
13388 );
13389 }
13390
13391 #[tokio::test]
13392 async fn ordinary_drain_reap_does_not_record_a_lifetime_restart() {
13393 let registry = Registry::default();
13394 let supervisor = Supervisor::new_for_test(
13395 Arc::new(Registry::default()),
13396 RestartPolicy::new(3, Duration::ZERO),
13397 );
13398 let runtime = supervisor.runtime_config();
13399 let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::starting()));
13400 let spec = ModuleSpec {
13401 module_id: "ordinary-drain".to_string(),
13402 program: fake_aft_stub_path(),
13403 args: Vec::new(),
13404 env: vec![("FAKE_AFT_EXIT_CODE".to_string(), "23".to_string())],
13405 reserved: false,
13406 reserved_prefixes: Vec::new(),
13407 protocol: ModuleProtocol::Subc,
13408 overlap: Default::default(),
13409 };
13410 let child = spawn_and_mark_running(&spec, &runtime, &snapshot).unwrap();
13411
13412 drain_child_to_state(
13413 &spec.module_id,
13414 spec.protocol,
13415 StopNotice::SentOverConnection,
13418 ®istry,
13419 None,
13420 &snapshot,
13421 &runtime.terminal_ring,
13422 &runtime.spawn_events,
13423 child,
13424 Duration::from_secs(1),
13425 ModuleState::Stopped,
13426 Some(false),
13427 )
13428 .await
13429 .unwrap();
13430
13431 let state = lock_snapshot(&snapshot).unwrap();
13432 assert_eq!(
13433 state.last_exit.as_ref().map(|exit| exit.kind),
13434 Some(ExitKind::Crash)
13435 );
13436 assert_eq!(state.lifetime_restarts, 0);
13437 assert_eq!(state.crash_restarts.len(), 0);
13438 }
13439
13440 #[test]
13441 fn fatal_connection_teardown_cannot_arm_a_marker_for_a_surviving_process() {
13442 assert!(!include_str!("server.rs")
13448 .contains("router.record_deliberate_connection_severance(ctx.connection_id)"));
13449 }
13450
13451 #[test]
13458 fn drained_after_quiescence_wait_passes_ok_through_and_forces_false_on_err() {
13459 assert!(drained_after_quiescence_wait(&Ok(true)));
13460 assert!(!drained_after_quiescence_wait(&Ok(false)));
13461 assert!(!drained_after_quiescence_wait(&Err(
13462 SuperviseError::StatePoisoned { module_id: None }
13463 )));
13464 }
13465
13466 #[test]
13475 fn wait_error_exit_report_records_a_failed_terminal_with_no_code_or_signal() {
13476 let ring = Arc::new(Mutex::new(TerminalRing::new(
13477 TerminalRingConfig::default(),
13478 0,
13479 )));
13480 record_wait_error_terminal("wait-error", &ring, &super::SpawnEventFeed::default());
13481
13482 let snapshot = ring.lock().unwrap().snapshot();
13483 assert_eq!(snapshot.entries.len(), 1);
13484 let entry = &snapshot.entries[0];
13485 assert_eq!(entry.exit_code, None);
13486 assert_eq!(entry.exit_signal, None);
13487 assert_eq!(entry.disposition, TerminalDisposition::Failed);
13488 }
13489
13490 #[test]
13491 fn wait_error_exit_path_preserves_spawn_event_density() {
13492 let feed = super::SpawnEventFeed::default();
13493 feed.configure_incarnation("wait-error-density".to_string());
13494 feed.emit_spawned("wait-error", 41, 1);
13495 let ring = Arc::new(Mutex::new(TerminalRing::new(
13496 TerminalRingConfig::default(),
13497 0,
13498 )));
13499
13500 record_wait_error_terminal("wait-error", &ring, &feed);
13501 feed.emit_spawned("after-wait-error", 42, 2);
13502
13503 let state = feed.0.lock().unwrap();
13504 let sequences = state
13505 .events
13506 .iter()
13507 .map(|event| event.cursor.seq)
13508 .collect::<Vec<_>>();
13509 assert_eq!(sequences, vec![1, 2, 3]);
13510 assert_eq!(state.events[1].kind, SpawnEventKind::Exited);
13511 assert_eq!(state.events[1].exit_code, None);
13512 assert_eq!(state.events[1].exit_signal, None);
13513 }
13514
13515 #[test]
13519 fn wait_error_exit_report_is_classified_as_a_crash() {
13520 assert_eq!(wait_error_exit_report().kind, ExitKind::Crash);
13521 }
13522}
13523
13524#[cfg(all(test, unix))]
13525mod restart_ordering_tests {
13526 use super::*;
13527 use std::io::Write;
13528 use tokio::io::AsyncReadExt;
13529
13530 #[test]
13531 #[ignore = "child process fixture"]
13532 fn serving_child() {
13533 let Ok(address_file) = std::env::var("RESTART_TEST_ADDRESS_FILE") else {
13534 return;
13535 };
13536 let listener = std::net::TcpListener::bind("127.0.0.1:0").unwrap();
13537 std::fs::write(address_file, listener.local_addr().unwrap().to_string()).unwrap();
13538 for stream in listener.incoming() {
13539 writeln!(stream.unwrap(), "{}", std::process::id()).unwrap();
13540 }
13541 }
13542
13543 async fn restart_before_publication(requests: usize) {
13544 let home = cortexkit_test_support::ScratchDir::new("restart-before-publication");
13545 let address_file = home.path().join("address");
13546 let spec = ModuleSpec {
13547 module_id: "restart-serving-child".into(),
13548 program: std::env::current_exe().unwrap(),
13549 args: vec![
13550 "--exact".into(),
13551 "supervise::restart_ordering_tests::serving_child".into(),
13552 "--ignored".into(),
13553 "--nocapture".into(),
13554 ],
13555 env: vec![(
13556 "RESTART_TEST_ADDRESS_FILE".into(),
13557 address_file.display().to_string(),
13558 )],
13559 reserved: false,
13560 reserved_prefixes: vec![],
13561 protocol: ModuleProtocol::None,
13562 overlap: Default::default(),
13563 };
13564 let mut supervisor =
13565 Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default());
13566 let (spawned_tx, spawned_rx) = oneshot::channel();
13567 let spawned_tx = Mutex::new(Some(spawned_tx));
13568 let (release_tx, release_rx) = std::sync::mpsc::channel();
13569 let release_rx = Mutex::new(release_rx);
13570 supervisor.test_after_first_spawn = AfterFirstSpawnHook(Some(Arc::new(move |_| {
13573 spawned_tx.lock().unwrap().take().unwrap().send(()).unwrap();
13574 release_rx
13575 .lock()
13576 .unwrap()
13577 .recv_timeout(Duration::from_secs(20))
13578 .unwrap();
13579 })));
13580 let registered = supervisor
13581 .register_configured_with_health(
13582 spec.clone(),
13583 true,
13584 HealthConfig::default(),
13585 None,
13586 RestartPolicy::new(3, Duration::ZERO),
13587 )
13588 .unwrap();
13589 let module = registered.module.clone();
13590 let startup = tokio::task::spawn_blocking(move || {
13591 supervisor.start_registered_configured(registered, None)
13592 });
13593 spawned_rx.await.unwrap();
13594 let old_pid = tokio::time::timeout(Duration::from_secs(10), async {
13595 let address = loop {
13596 if let Ok(address) = std::fs::read_to_string(&address_file) {
13597 if !address.is_empty() {
13598 break address;
13599 }
13600 }
13601 tokio::time::sleep(Duration::from_millis(1)).await;
13602 };
13603 let mut stream = tokio::net::TcpStream::connect(address).await.unwrap();
13604 let mut reply = String::new();
13605 stream.read_to_string(&mut reply).await.unwrap();
13606 reply.trim().parse::<u32>().unwrap()
13607 })
13608 .await
13609 .expect("child must serve before generation publication");
13610 assert_eq!(
13611 lock_snapshot(&module.inner.snapshot)
13612 .unwrap()
13613 .spawn_generation,
13614 0
13615 );
13616 let mut replies = Vec::new();
13617 for _ in 0..requests {
13618 let mut restart = Box::pin(module.restart(Some(0)));
13619 std::future::poll_fn(|cx| {
13622 assert!(std::future::Future::poll(restart.as_mut(), cx).is_pending());
13623 std::task::Poll::Ready(())
13624 })
13625 .await;
13626 replies.push(restart);
13627 }
13628 release_tx.send(()).unwrap();
13629 startup.await.unwrap().unwrap();
13630 for reply in replies {
13631 reply.await.unwrap();
13632 }
13633 let replacement = tokio::time::timeout(Duration::from_secs(10), async {
13636 loop {
13637 if lock_snapshot(&module.inner.snapshot)
13638 .unwrap()
13639 .spawn_generation
13640 > 1
13641 {
13642 break;
13643 }
13644 tokio::time::sleep(Duration::from_millis(1)).await;
13645 }
13646 })
13647 .await;
13648 module.update_spec_for_test(spec).await.unwrap();
13652 let (generation, pid) = {
13653 let state = lock_snapshot(&module.inner.snapshot).unwrap();
13654 (state.spawn_generation, state.pid)
13655 };
13656 module.stop().await.unwrap();
13657 replacement.expect("restart must publish a replacement for the already-serving child");
13658 assert_ne!(
13659 pid,
13660 Some(old_pid),
13661 "a serving child predating the restart must be replaced"
13662 );
13663 assert_eq!(generation, 2, "queued restarts must share one replacement");
13664 }
13665
13666 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
13667 async fn restart_received_after_serving_before_generation_publication_replaces_child() {
13668 restart_before_publication(1).await;
13669 }
13670
13671 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
13672 async fn restart_queued_behind_restart_does_not_kill_its_replacement() {
13673 restart_before_publication(2).await;
13674 }
13675
13676 #[tokio::test]
13677 async fn restart_queued_before_replacement_spawn_keeps_the_new_child() {
13678 let supervisor =
13679 Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default());
13680 let registered = supervisor
13681 .register_configured_with_health(
13682 ModuleSpec {
13683 module_id: "queued-before-replacement".into(),
13684 program: "/bin/sleep".into(),
13689 args: vec!["600".into()],
13690 env: vec![],
13691 reserved: false,
13692 reserved_prefixes: vec![],
13693 protocol: ModuleProtocol::None,
13694 overlap: Default::default(),
13695 },
13696 true,
13697 HealthConfig::default(),
13698 None,
13699 RestartPolicy::default(),
13700 )
13701 .unwrap();
13702 let RegisteredConfiguredModule {
13703 mut spec,
13704 mut runtime,
13705 module,
13706 mut commands,
13707 } = registered;
13708 let snapshot = &module.inner.snapshot;
13709 let mut restart = Box::pin(module.restart(Some(0)));
13710 std::future::poll_fn(|cx| {
13711 assert!(std::future::Future::poll(restart.as_mut(), cx).is_pending());
13712 std::task::Poll::Ready(())
13713 })
13714 .await;
13715 let mut child = Some(spawn_and_mark_running(&spec, &runtime, snapshot).unwrap());
13718 let replacement_pid = child.as_ref().unwrap().id();
13719 let command = commands.recv().await.unwrap();
13720 assert!(
13721 handle_supervisor_command(
13722 command,
13723 &mut spec,
13724 &mut runtime,
13725 &supervisor.registry,
13726 &supervisor.process_liveness,
13727 snapshot,
13728 &mut child,
13729 &mut commands,
13730 &mut VecDeque::new(),
13731 )
13732 .await
13733 );
13734 restart.await.unwrap();
13735 let kept_pid = child.as_ref().and_then(SupervisedChild::id);
13736 let pending = lock_snapshot(snapshot).unwrap().respawn_pending;
13737 if let Some(mut child) = child {
13738 child.start_kill().unwrap();
13739 child.wait().await.unwrap();
13740 }
13741 assert_eq!(
13742 kept_pid, replacement_pid,
13743 "a newer replacement must survive"
13744 );
13745 assert!(
13746 !pending,
13747 "a satisfied restart must not schedule another spawn"
13748 );
13749 }
13750}
13751
13752#[cfg(all(test, unix))]
13753mod health_event_tests {
13754 use super::*;
13755 use cortexkit_test_support::ScratchDir;
13756 use subc_protocol::{manifest::Concurrency, session::ModuleControlResponse};
13757
13758 struct Actor {
13759 module: SupervisedModule,
13760 registry: Arc<Registry>,
13761 forwarding: Arc<ForwardingTable>,
13762 spec: ModuleSpec,
13763 health: HealthConfig,
13764 rx: mpsc::Receiver<crate::router::OutboundFrame>,
13765 _home: ScratchDir,
13766 }
13767
13768 async fn wait_for<T>(reason: &str, mut observe: impl FnMut() -> Option<T>) -> T {
13769 let deadline = std::time::Instant::now() + Duration::from_secs(10);
13773 let tick = Instant::now();
13774 loop {
13775 if let Some(value) = observe() {
13776 assert_eq!(Instant::now(), tick, "{reason} must not wait for a timer");
13777 return value;
13778 }
13779 assert!(
13780 std::time::Instant::now() < deadline,
13781 "timed out waiting for {reason}"
13782 );
13783 tokio::task::yield_now().await;
13784 }
13785 }
13786
13787 impl Actor {
13788 async fn start(protocol: ModuleProtocol, cadence: Duration) -> Self {
13789 let home = ScratchDir::new("health-events");
13790 let registry = Arc::new(Registry::default());
13791 let forwarding = Arc::new(ForwardingTable::default());
13792 let health = HealthConfig {
13793 cadence,
13794 deadline: Duration::from_secs(3600),
13795 ..HealthConfig::default()
13796 };
13797 let supervisor =
13798 Supervisor::new_for_test(registry.clone(), RestartPolicy::new(3, Duration::ZERO))
13799 .with_forwarding(forwarding.clone())
13800 .with_health_config(health.clone());
13801 let spec = ModuleSpec {
13802 module_id: "event-health-child".into(),
13803 program: PathBuf::from("/bin/sleep"),
13811 args: vec!["600".into()],
13814 env: ["XDG_DATA_HOME", "XDG_RUNTIME_DIR", "XDG_CONFIG_HOME"]
13815 .into_iter()
13816 .map(|key| (key.into(), home.path().display().to_string()))
13817 .collect(),
13818 reserved: false,
13819 reserved_prefixes: vec![],
13820 protocol,
13821 overlap: Default::default(),
13822 };
13823 let module = supervisor.spawn(spec.clone()).unwrap();
13824 let (_, rx) = mpsc::channel(8);
13825 let actor = Self {
13826 module,
13827 registry,
13828 forwarding,
13829 spec,
13830 health,
13831 rx,
13832 _home: home,
13833 };
13834 actor
13835 .wait_for_select(|checkpoint| checkpoint.generation > 0)
13836 .await;
13837 actor
13838 }
13839
13840 fn turns(&self) -> u64 {
13841 lock_snapshot(&self.module.inner.snapshot)
13842 .unwrap()
13843 .actor_turns
13844 }
13845
13846 async fn wait_for_select(
13847 &self,
13848 expected: impl Fn(&ActorSelectCheckpoint) -> bool,
13849 ) -> ActorSelectCheckpoint {
13850 wait_for("actor select checkpoint", || {
13851 let state = lock_snapshot(&self.module.inner.snapshot).unwrap();
13852 state.actor_select.filter(|checkpoint| {
13853 checkpoint.generation == state.spawn_generation
13854 && state.state == ModuleState::Running
13855 && state.process_alive
13856 && state.reported_pid().is_some()
13857 && expected(checkpoint)
13858 })
13859 })
13860 .await
13861 }
13862
13863 fn hello(&mut self, connection: u64, health: bool) {
13864 let manifest =
13865 subc_protocol::manifest::ModuleManifest::builder(&self.spec.module_id, "1.0.0")
13866 .protocol_ver(subc_protocol::PROTOCOL_VERSION)
13867 .build();
13868 let connection = ConnectionId::new(connection);
13869 let (tx, rx) = mpsc::channel(8);
13870 self.rx = rx;
13871 self.forwarding
13872 .register_module_connection(
13873 connection,
13874 self.spec.module_id.clone(),
13875 subc_protocol::PROTOCOL_VERSION,
13876 Concurrency::ModuleManaged,
13877 FrameSink::new(tx),
13878 )
13879 .unwrap();
13880 self.registry
13881 .register_with_control_ops(
13882 manifest,
13883 subc_protocol::PROTOCOL_VERSION,
13884 connection,
13885 if health {
13886 vec![MODULE_CONTROL_OP_HEALTH_CHECK.into()]
13887 } else {
13888 vec![]
13889 },
13890 )
13891 .unwrap();
13892 }
13893
13894 async fn probe(&mut self) -> crate::router::OutboundFrame {
13895 let frame = wait_for("outbound health probe", || match self.rx.try_recv() {
13896 Ok(frame) => Some(frame),
13897 Err(mpsc::error::TryRecvError::Empty) => None,
13898 Err(mpsc::error::TryRecvError::Disconnected) => panic!("health peer disconnected"),
13899 })
13900 .await;
13901 let body: Value = serde_json::from_slice(&frame.body).unwrap();
13902 assert_eq!(body["op"], MODULE_CONTROL_OP_HEALTH_CHECK);
13903 frame
13904 }
13905
13906 fn no_probe(&mut self) {
13907 assert!(matches!(
13908 self.rx.try_recv(),
13909 Err(mpsc::error::TryRecvError::Empty)
13910 ));
13911 }
13912
13913 fn answer(&self, connection: u64, frame: crate::router::OutboundFrame) {
13914 self.forwarding
13915 .complete_module_control_rpc(
13916 ConnectionId::new(connection),
13917 frame.header.corr,
13918 Some(MODULE_CONTROL_OP_HEALTH_CHECK),
13919 ModuleControlRpcOutcome::Response(ModuleControlResponse::HealthCheck {
13920 status: HealthStatus::Ok,
13921 detail: None,
13922 metrics: None,
13923 }),
13924 )
13925 .unwrap();
13926 }
13927 }
13928
13929 #[tokio::test(start_paused = true)]
13930 async fn no_health_children_park_for_sixty_seconds() {
13931 let none = Actor::start(ModuleProtocol::None, Duration::from_secs(30)).await;
13932 let unregistered = Actor::start(ModuleProtocol::Subc, Duration::from_secs(30)).await;
13933 let mut unadvertised = Actor::start(ModuleProtocol::Subc, Duration::from_secs(30)).await;
13934 unadvertised.hello(1, false);
13935 for _ in 0..6000 {
13940 tokio::time::advance(Duration::from_millis(10)).await;
13941 tokio::task::yield_now().await;
13942 }
13943 for actor in [&none, &unregistered, &unadvertised] {
13944 assert!(
13945 actor.turns() <= 5,
13946 "no-health actor woke {} times",
13947 actor.turns()
13948 );
13949 assert_eq!(actor.module.state().unwrap(), ModuleState::Running);
13950 }
13951 }
13952
13953 #[tokio::test(start_paused = true)]
13954 async fn late_hello_starts_a_due_probe_in_the_notification_tick() {
13955 let mut actor = Actor::start(ModuleProtocol::Subc, Duration::ZERO).await;
13959 let tick = Instant::now();
13960 actor.hello(1, true);
13961 actor.probe().await;
13962 assert_eq!(Instant::now(), tick, "HELLO must not wait for a poll");
13963 }
13964
13965 #[tokio::test(start_paused = true)]
13966 async fn hello_keeps_the_jittered_cadence_and_catalog_updates_do_not_reset_it() {
13967 let mut actor = Actor::start(ModuleProtocol::Subc, Duration::from_secs(30)).await;
13968 let hello_tick = Instant::now();
13969 actor.hello(1, true);
13970 let armed = actor
13971 .wait_for_select(|checkpoint| {
13972 checkpoint.registered_connection == Some(ConnectionId::new(1))
13973 && checkpoint.next_probe_at.is_some()
13974 })
13975 .await;
13976 let delay = jittered_health_delay(&actor.spec.module_id, 0, actor.health.cadence);
13977 assert!((Duration::from_secs(30)..Duration::from_secs(33)).contains(&delay));
13978 let first_deadline = hello_tick + delay;
13979 assert_eq!(armed.next_probe_at, Some(first_deadline));
13980 tokio::time::advance(Duration::from_secs(29)).await;
13981 let turns = actor.turns();
13982 actor
13983 .registry
13984 .replace_catalog_for_connection(ConnectionId::new(1), vec![], None, Some(true))
13985 .unwrap();
13986 let updated = actor
13987 .wait_for_select(|checkpoint| checkpoint.turn > turns)
13988 .await;
13989 assert_eq!(
13990 updated.next_probe_at,
13991 Some(first_deadline),
13992 "catalog update must not reset cadence"
13993 );
13994 actor.no_probe();
13995 tokio::time::advance(first_deadline - Instant::now()).await;
13996 let frame = actor.probe().await;
13997 actor.answer(1, frame);
13998 let next = jittered_health_delay(&actor.spec.module_id, 1, actor.health.cadence);
13999 let next_deadline = Instant::now() + next;
14000 let rearmed = actor
14001 .wait_for_select(|checkpoint| {
14002 checkpoint
14003 .next_probe_at
14004 .is_some_and(|deadline| deadline > first_deadline)
14005 })
14006 .await;
14007 assert_eq!(rearmed.next_probe_at, Some(next_deadline));
14008 tokio::time::advance(next - Duration::from_nanos(1)).await;
14009 actor.no_probe();
14010 tokio::time::advance(Duration::from_nanos(1)).await;
14011 actor.probe().await;
14012 }
14013
14014 #[tokio::test(start_paused = true)]
14015 async fn disconnect_disarms_and_reconnect_rearms_health() {
14016 let mut actor = Actor::start(ModuleProtocol::Subc, Duration::from_secs(30)).await;
14017 actor.hello(1, true);
14018 actor
14019 .wait_for_select(|checkpoint| {
14020 checkpoint.registered_connection == Some(ConnectionId::new(1))
14021 && checkpoint.next_probe_at.is_some()
14022 })
14023 .await;
14024 let turns = actor.turns();
14025 let tick = Instant::now();
14026 actor
14027 .registry
14028 .deregister_connection(ConnectionId::new(1))
14029 .unwrap();
14030 let disarmed = actor
14031 .wait_for_select(|checkpoint| {
14032 checkpoint.turn > turns && checkpoint.registered_connection.is_none()
14033 })
14034 .await;
14035 assert_eq!(disarmed.next_probe_at, None);
14036 assert_eq!(disarmed.wake_after, None);
14037 assert_eq!(
14038 Instant::now(),
14039 tick,
14040 "disconnect must disarm in the notification tick"
14041 );
14042 tokio::time::advance(Duration::from_secs(60)).await;
14043 actor.no_probe();
14044 let reconnect_tick = Instant::now();
14045 actor.hello(2, true);
14046 let rearmed = actor
14047 .wait_for_select(|checkpoint| {
14048 checkpoint.registered_connection == Some(ConnectionId::new(2))
14049 && checkpoint.next_probe_at.is_some()
14050 })
14051 .await;
14052 let delay = jittered_health_delay(&actor.spec.module_id, 0, actor.health.cadence);
14053 assert_eq!(rearmed.next_probe_at, Some(reconnect_tick + delay));
14054 tokio::time::advance(delay).await;
14055 actor.probe().await;
14056 }
14057
14058 #[tokio::test(start_paused = true)]
14059 async fn reregistration_without_health_disarms_the_old_schedule() {
14060 let mut actor = Actor::start(ModuleProtocol::Subc, Duration::from_secs(30)).await;
14061 actor.hello(1, true);
14062 actor
14063 .wait_for_select(|checkpoint| {
14064 checkpoint.registered_connection == Some(ConnectionId::new(1))
14065 && checkpoint.next_probe_at.is_some()
14066 })
14067 .await;
14068 actor
14069 .registry
14070 .deregister_connection(ConnectionId::new(1))
14071 .unwrap();
14072 actor.hello(2, false);
14073 let disarmed = actor
14074 .wait_for_select(|checkpoint| {
14075 checkpoint.registered_connection == Some(ConnectionId::new(2))
14076 })
14077 .await;
14078 assert_eq!(disarmed.next_probe_at, None);
14079 assert_eq!(disarmed.wake_after, None);
14080 tokio::time::advance(Duration::from_secs(60)).await;
14081 actor.no_probe();
14082 }
14083
14084 #[tokio::test(start_paused = true)]
14085 async fn restart_waits_for_its_own_hello_before_rearming_health() {
14086 let mut actor = Actor::start(ModuleProtocol::Subc, Duration::from_secs(30)).await;
14087 actor.hello(1, true);
14088 let armed = actor
14089 .wait_for_select(|checkpoint| {
14090 checkpoint.registered_connection == Some(ConnectionId::new(1))
14091 && checkpoint.next_probe_at.is_some()
14092 })
14093 .await;
14094 actor
14098 .registry
14099 .deregister_connection(ConnectionId::new(1))
14100 .unwrap();
14101 actor.module.restart(Some(0)).await.unwrap();
14102 let parked = actor
14103 .wait_for_select(|checkpoint| {
14104 checkpoint.generation > armed.generation
14105 && checkpoint.registered_connection.is_none()
14106 && checkpoint.next_probe_at.is_none()
14107 })
14108 .await;
14109 assert_eq!(parked.wake_after, None);
14110 actor.rx.close();
14111 while actor.rx.try_recv().is_ok() {}
14114 let turns = actor.turns();
14115 tokio::time::advance(Duration::from_secs(60)).await;
14116 let after = actor.turns();
14117 eprintln!(
14118 "restart park turns: before={turns} after={after} delta={}",
14119 after - turns
14120 );
14121 assert_eq!(after, turns, "replacement must park before HELLO");
14122 let hello_tick = Instant::now();
14123 actor.hello(2, true);
14124 let rearmed = actor
14125 .wait_for_select(|checkpoint| {
14126 checkpoint.registered_connection == Some(ConnectionId::new(2))
14127 && checkpoint.next_probe_at.is_some()
14128 })
14129 .await;
14130 let delay = jittered_health_delay(&actor.spec.module_id, 0, actor.health.cadence);
14131 assert_eq!(rearmed.next_probe_at, Some(hello_tick + delay));
14132 tokio::time::advance(delay).await;
14133 actor.probe().await;
14134 }
14135
14136 #[tokio::test(start_paused = true)]
14137 async fn http_probe_waits_a_full_cadence_for_each_spawn() {
14138 use tokio::{
14139 io::{AsyncReadExt, AsyncWriteExt},
14140 net::TcpListener,
14141 };
14142 let mut actor = Actor::start(ModuleProtocol::None, Duration::from_secs(30)).await;
14143 let listener = TcpListener::bind("127.0.0.1:0").await.unwrap();
14144 actor.health.http = Some(format!("http://{}/healthz", listener.local_addr().unwrap()));
14145 let (started_tx, mut started_rx) = mpsc::channel(8);
14146 let peer = tokio::spawn(async move {
14147 loop {
14148 let (mut socket, _) = listener.accept().await.unwrap();
14149 let mut request = [0; 1024];
14150 assert!(socket.read(&mut request).await.unwrap() > 0);
14151 started_tx.send(Instant::now()).await.unwrap();
14152 socket
14153 .write_all(b"HTTP/1.1 200 OK\r\nContent-Length: 0\r\n\r\n")
14154 .await
14155 .unwrap();
14156 }
14157 });
14158 actor
14159 .module
14160 .update_configuration(actor.spec.clone(), actor.health.clone(), None)
14161 .await
14162 .unwrap();
14163 let first_tick = Instant::now();
14164 let first = actor
14165 .wait_for_select(|checkpoint| checkpoint.next_probe_at.is_some())
14166 .await;
14167 assert!(first.next_probe_at.unwrap() >= first_tick + actor.health.cadence);
14168 assert!(started_rx.try_recv().is_err());
14169
14170 tokio::time::advance(
14173 first.next_probe_at.unwrap() - Instant::now() - Duration::from_secs(1),
14174 )
14175 .await;
14176 actor.module.restart(Some(0)).await.unwrap();
14177 let spawn_tick = Instant::now();
14178 let replacement = actor
14179 .wait_for_select(|checkpoint| checkpoint.generation > first.generation)
14180 .await;
14181 let deadline = replacement.next_probe_at.unwrap();
14182 assert!(
14183 deadline >= spawn_tick + actor.health.cadence,
14184 "a replacement must not inherit its predecessor's probe deadline"
14185 );
14186 tokio::time::advance(actor.health.cadence - Duration::from_nanos(1)).await;
14187 assert!(
14188 started_rx.try_recv().is_err(),
14189 "no HTTP probe during startup grace"
14190 );
14191 tokio::time::advance(deadline - Instant::now()).await;
14192 let landed = wait_for("first replacement HTTP probe", || {
14193 started_rx.try_recv().ok()
14194 })
14195 .await;
14196 assert!(landed >= spawn_tick + actor.health.cadence);
14197 wait_for("healthy replacement", || {
14198 (actor.module.status().unwrap().health.status == SupervisorHealthStatus::Ok)
14199 .then_some(())
14200 })
14201 .await;
14202 peer.abort();
14203 }
14204
14205 #[tokio::test(start_paused = true)]
14206 async fn rescan_adds_and_removes_http_health_without_polling() {
14207 use tokio::{
14208 io::{AsyncReadExt, AsyncWriteExt},
14209 net::TcpListener,
14210 };
14211 let mut actor = Actor::start(ModuleProtocol::None, Duration::from_secs(30)).await;
14212 let listener = TcpListener::bind("127.0.0.1:0").await.unwrap();
14213 actor.health.http = Some(format!("http://{}/health", listener.local_addr().unwrap()));
14214 actor.health.cadence = Duration::from_millis(1);
14216 let (started_tx, mut started_rx) = mpsc::channel(8);
14217 let peer = tokio::spawn(async move {
14218 loop {
14219 let (mut socket, _) = listener.accept().await.unwrap();
14220 let mut request = [0; 1024];
14221 assert!(socket.read(&mut request).await.unwrap() > 0);
14222 started_tx.send(()).await.unwrap();
14223 socket
14224 .write_all(b"HTTP/1.1 200 OK\r\nContent-Length: 0\r\n\r\n")
14225 .await
14226 .unwrap();
14227 }
14228 });
14229 let tick = Instant::now();
14230 let turns = actor.turns();
14231 actor
14232 .module
14233 .update_configuration(actor.spec.clone(), actor.health.clone(), None)
14234 .await
14235 .unwrap();
14236 let armed = actor
14237 .wait_for_select(|checkpoint| {
14238 checkpoint.turn > turns && checkpoint.next_probe_at.is_some()
14239 })
14240 .await;
14241 assert_eq!(armed.next_probe_at, Some(tick + Duration::from_millis(1)));
14242 assert_eq!(Instant::now(), tick);
14243 tokio::time::advance(Duration::from_millis(1)).await;
14244 let mut started = false;
14245 wait_for("successful HTTP probe", || {
14246 started |= started_rx.try_recv().is_ok();
14247 (started && actor.module.status().unwrap().health.status == SupervisorHealthStatus::Ok)
14248 .then_some(())
14249 })
14250 .await;
14251 assert!(started, "added HTTP check must start at its first deadline");
14252 assert_eq!(
14253 actor.module.status().unwrap().health.status,
14254 SupervisorHealthStatus::Ok
14255 );
14256 assert_eq!(Instant::now(), tick + Duration::from_millis(1));
14257 actor.health.http = None;
14258 let turns = actor.turns();
14259 actor
14260 .module
14261 .update_configuration(actor.spec.clone(), actor.health.clone(), None)
14262 .await
14263 .unwrap();
14264 let parked = actor
14265 .wait_for_select(|checkpoint| {
14266 checkpoint.turn > turns && checkpoint.next_probe_at.is_none()
14267 })
14268 .await;
14269 assert_eq!(parked.wake_after, None);
14270 let turns = actor.turns();
14271 tokio::time::advance(Duration::from_secs(60)).await;
14272 assert_eq!(
14273 actor.turns(),
14274 turns,
14275 "removed HTTP check must park the actor"
14276 );
14277 assert!(
14278 started_rx.try_recv().is_err(),
14279 "removed HTTP check must stay disarmed"
14280 );
14281 assert_eq!(
14282 actor.module.status().unwrap().health,
14283 ModuleHealthStatus::default()
14284 );
14285 peer.abort();
14286 }
14287}
14288
14289#[cfg(test)]
14290mod drain_busy_reprobe_tests {
14291 use super::*;
14295 use std::collections::BTreeMap;
14296 use std::sync::Mutex as StdMutex;
14297 use subc_protocol::{manifest::Concurrency, session::ModuleControlResponse};
14298
14299 const MODULE_ID: &str = "drain-reprobe-module";
14300 const CONNECTION: u64 = 7;
14301 const GAUGE: &str = "runs_in_flight";
14302 const CADENCE: Duration = Duration::from_secs(30);
14305 const CEILING: Duration = Duration::from_secs(30);
14306
14307 #[derive(Clone, Copy)]
14309 enum Answer {
14310 BusyUntil { until: Instant, latency: Duration },
14313 OmitsGauge,
14315 Silent,
14317 }
14318
14319 struct Drain {
14320 forwarding: Arc<ForwardingTable>,
14321 runtime: SupervisorRuntimeConfig,
14322 endpoint: crate::ModuleEndpointId,
14323 probes: Arc<StdMutex<Vec<Instant>>>,
14325 responder: tokio::task::JoinHandle<()>,
14326 }
14327
14328 impl Drain {
14329 fn start(answer: Answer) -> Self {
14330 let registry = Arc::new(Registry::default());
14331 let forwarding = Arc::new(ForwardingTable::default());
14332 let health = HealthConfig {
14333 cadence: CADENCE,
14334 ..HealthConfig::default()
14335 };
14336 let runtime = Supervisor::new_for_test(registry, RestartPolicy::new(3, Duration::ZERO))
14337 .with_forwarding(forwarding.clone())
14338 .with_health_config(health)
14339 .runtime_config();
14340 let (tx, mut rx) = mpsc::channel::<crate::router::OutboundFrame>(8);
14341 let endpoint = forwarding
14342 .register_module_connection(
14343 ConnectionId::new(CONNECTION),
14344 MODULE_ID.to_string(),
14345 subc_protocol::PROTOCOL_VERSION,
14346 Concurrency::ModuleManaged,
14347 FrameSink::new(tx),
14348 )
14349 .unwrap();
14350 let probes = Arc::new(StdMutex::new(Vec::new()));
14351 let responder = tokio::spawn({
14352 let forwarding = forwarding.clone();
14353 let probes = probes.clone();
14354 async move {
14355 while let Some(frame) = rx.recv().await {
14356 let body: Value = serde_json::from_slice(&frame.body).unwrap();
14357 assert_eq!(body["op"], MODULE_CONTROL_OP_HEALTH_CHECK);
14358 probes.lock().unwrap().push(Instant::now());
14359 let metrics = match answer {
14360 Answer::BusyUntil { until, latency } => {
14361 sleep(latency).await;
14362 let busy = u64::from(Instant::now() < until);
14363 serde_json::json!({ GAUGE: busy })
14364 }
14365 Answer::OmitsGauge => serde_json::json!({ "queue_depth": 0 }),
14366 Answer::Silent => continue,
14367 };
14368 let _ = forwarding.complete_module_control_rpc(
14371 ConnectionId::new(CONNECTION),
14372 frame.header.corr,
14373 Some(MODULE_CONTROL_OP_HEALTH_CHECK),
14374 ModuleControlRpcOutcome::Response(ModuleControlResponse::HealthCheck {
14375 status: HealthStatus::Ok,
14376 detail: None,
14377 metrics: Some(metrics),
14378 }),
14379 );
14380 }
14381 }
14382 });
14383 Self {
14384 forwarding,
14385 runtime,
14386 endpoint,
14387 probes,
14388 responder,
14389 }
14390 }
14391
14392 async fn run(&self, ceiling: Duration) -> (bool, Duration) {
14395 let (drained, elapsed, _) = self.run_traced(ceiling).await;
14396 (drained, elapsed)
14397 }
14398
14399 async fn run_traced(&self, ceiling: Duration) -> (bool, Duration, DrainGaugeTrace) {
14401 let started = Instant::now();
14402 let mut trace = DrainGaugeTrace::new(&[GAUGE.to_string()]);
14403 let drained = wait_for_forwarding_quiescence(
14404 &self.forwarding,
14405 MODULE_ID,
14406 &self.runtime,
14407 self.endpoint,
14408 started + ceiling,
14409 &[GAUGE.to_string()],
14410 DrainScope::Active,
14411 &mut trace,
14412 )
14413 .await
14414 .unwrap();
14415 (drained, started.elapsed(), trace)
14416 }
14417
14418 fn probes(&self) -> Vec<Instant> {
14419 self.probes.lock().unwrap().clone()
14420 }
14421 }
14422
14423 impl Drop for Drain {
14424 fn drop(&mut self) {
14425 self.responder.abort();
14426 }
14427 }
14428
14429 #[tokio::test(start_paused = true)]
14430 async fn busy_at_first_probe_then_quiet_drains_long_before_the_ceiling() {
14431 let drain = Drain::start(Answer::BusyUntil {
14432 until: Instant::now() + Duration::from_secs(1),
14433 latency: Duration::ZERO,
14434 });
14435 let (drained, elapsed) = drain.run(CEILING).await;
14436 assert!(
14437 drained,
14438 "gauges that read zero during the drain must let it finish drained, \
14439 not wait out the {CADENCE:?} health cadence (elapsed {elapsed:?})"
14440 );
14441 assert!(
14442 elapsed < Duration::from_secs(3),
14443 "drain took {elapsed:?} after the gauges went quiet at 1s"
14444 );
14445 let probes = drain.probes();
14446 assert!(
14447 probes.len() >= 2,
14448 "the module was busy at the first probe, so a quiet verdict needs a second: {probes:?}"
14449 );
14450 }
14451
14452 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
14457 async fn an_instant_busy_first_answer_does_not_stop_later_probes_in_real_time() {
14458 let drain = Drain::start(Answer::BusyUntil {
14459 until: Instant::now() + Duration::from_millis(600),
14460 latency: Duration::ZERO,
14461 });
14462 let (drained, elapsed, trace) = drain.run_traced(CEILING).await;
14463 assert!(drained, "the gauge fell at 600ms: {trace}");
14464 assert!(
14465 elapsed < Duration::from_secs(3),
14466 "drain took {elapsed:?} after the gauge fell at 600ms: {trace}"
14467 );
14468 let probes = drain.probes();
14469 assert!(
14470 probes.len() >= 2 && trace.probes as usize == probes.len(),
14471 "the drain must re-probe after the first busy answer: {probes:?}, {trace}"
14472 );
14473 }
14474
14475 #[tokio::test(start_paused = true)]
14476 async fn busy_throughout_ends_undrained_at_the_ceiling_with_bounded_probes() {
14477 let started = Instant::now();
14478 let drain = Drain::start(Answer::BusyUntil {
14479 until: started + Duration::from_secs(3600),
14480 latency: Duration::ZERO,
14481 });
14482 let (drained, elapsed) = drain.run(CEILING).await;
14483 assert!(!drained, "a module busy throughout must not drain");
14484 assert!(
14485 elapsed >= CEILING && elapsed < CEILING + drain_busy_reprobe_interval(),
14486 "a busy module is held until the ceiling and no longer: {elapsed:?}"
14487 );
14488 let probes = drain.probes();
14491 let max_probes = (CEILING.as_millis() / drain_busy_reprobe_interval().as_millis()) + 1;
14492 assert!(
14493 probes.len() as u128 <= max_probes,
14494 "{} probes in a {CEILING:?} drain exceeds the {max_probes} the interval allows",
14495 probes.len()
14496 );
14497 for pair in probes.windows(2) {
14498 assert!(
14499 pair[1] - pair[0] >= drain_busy_reprobe_interval(),
14500 "consecutive probes {:?} apart",
14501 pair[1] - pair[0]
14502 );
14503 }
14504 }
14505
14506 #[tokio::test(start_paused = true)]
14507 async fn a_slow_probe_is_never_overlapped_by_the_next() {
14508 let latency = Duration::from_secs(1);
14511 let drain = Drain::start(Answer::BusyUntil {
14512 until: Instant::now() + Duration::from_secs(3600),
14513 latency,
14514 });
14515 let (drained, _) = drain.run(Duration::from_secs(5)).await;
14516 assert!(!drained);
14517 let probes = drain.probes();
14518 assert!(probes.len() >= 2, "the drain must re-probe: {probes:?}");
14519 for pair in probes.windows(2) {
14520 assert!(
14521 pair[1] - pair[0] >= latency + drain_busy_reprobe_interval(),
14522 "a probe started {:?} after the previous one, before it could finish",
14523 pair[1] - pair[0]
14524 );
14525 }
14526 }
14527
14528 #[tokio::test(start_paused = true)]
14529 async fn an_unanswered_probe_cannot_hold_the_drain_past_its_ceiling() {
14530 let drain = Drain::start(Answer::Silent);
14531 let ceiling = Duration::from_secs(2);
14532 let (drained, elapsed) = drain.run(ceiling).await;
14533 assert!(!drained, "an unanswered probe counts as busy");
14534 assert!(
14535 elapsed >= ceiling && elapsed < ceiling + drain_busy_reprobe_interval(),
14536 "the probe deadline is capped at the drain ceiling: {elapsed:?}"
14537 );
14538 assert_eq!(
14539 drain.probes().len(),
14540 1,
14541 "the unanswered probe holds the only slot until the ceiling"
14542 );
14543 }
14544
14545 #[derive(Clone, Default)]
14547 struct LogCapture {
14548 events: Arc<StdMutex<Vec<BTreeMap<String, String>>>>,
14549 }
14550
14551 impl<S: tracing::Subscriber> tracing_subscriber::Layer<S> for LogCapture {
14552 fn on_event(
14553 &self,
14554 event: &tracing::Event<'_>,
14555 _context: tracing_subscriber::layer::Context<'_, S>,
14556 ) {
14557 struct Fields(BTreeMap<String, String>);
14558 impl tracing::field::Visit for Fields {
14559 fn record_debug(&mut self, field: &tracing::field::Field, value: &dyn fmt::Debug) {
14560 self.0
14561 .insert(field.name().to_string(), format!("{value:?}"));
14562 }
14563 }
14564 let mut fields = Fields(BTreeMap::new());
14565 event.record(&mut fields);
14566 self.events.lock().unwrap().push(fields.0);
14567 }
14568 }
14569
14570 impl LogCapture {
14571 fn install(&self) -> tracing::subscriber::DefaultGuard {
14572 use tracing_subscriber::layer::SubscriberExt;
14573 tracing::subscriber::set_default(tracing_subscriber::registry().with(self.clone()))
14574 }
14575
14576 fn with_message(&self, needle: &str) -> Vec<BTreeMap<String, String>> {
14577 self.events
14578 .lock()
14579 .unwrap()
14580 .iter()
14581 .filter(|fields| fields.get("message").is_some_and(|m| m.contains(needle)))
14582 .cloned()
14583 .collect()
14584 }
14585 }
14586
14587 #[tokio::test(start_paused = true)]
14592 async fn a_missing_declared_gauge_is_named_on_the_drain_timeout_line() {
14593 let capture = LogCapture::default();
14594 let _guard = capture.install();
14595 let drain = Drain::start(Answer::OmitsGauge);
14596 let (drained, _, trace) = drain.run_traced(Duration::from_secs(2)).await;
14597 assert!(!drained, "a gauge never reported cannot read quiet");
14598 assert!(trace.probes >= 2, "the drain must re-probe: {trace:?}");
14599
14600 let missing = capture.with_message("lacks a declared busy gauge");
14601 assert_eq!(missing.len(), 1, "one warning per drain: {missing:?}");
14602 assert_eq!(
14603 missing[0].get("missing_gauges").map(String::as_str),
14604 Some(GAUGE)
14605 );
14606
14607 warn_drain_timed_out(
14608 MODULE_ID,
14609 Duration::from_secs(2),
14610 RouteCloseReason::Restart,
14611 &Default::default(),
14612 &trace,
14613 );
14614 let timeouts = capture.with_message("route drain timed out");
14615 assert_eq!(timeouts.len(), 1, "{timeouts:?}");
14616 let gauges = &timeouts[0]["busy_gauges"];
14617 assert!(
14618 gauges.starts_with(&format!("{GAUGE}: missing_from_health_body @")),
14619 "the timeout line must name the gauge and why it read busy: {gauges}"
14620 );
14621 assert!(
14622 gauges.ends_with("; no value ever read"),
14623 "no value was ever read for the gauge: {gauges}"
14624 );
14625 assert_eq!(timeouts[0]["gauge_probes"], trace.probes.to_string());
14626 }
14627
14628 #[tokio::test(start_paused = true)]
14632 async fn the_drain_trace_names_a_busy_value_and_an_unanswered_probe() {
14633 let busy = Drain::start(Answer::BusyUntil {
14634 until: Instant::now() + Duration::from_secs(3600),
14635 latency: Duration::ZERO,
14636 });
14637 let (_, _, trace) = busy.run_traced(Duration::from_secs(1)).await;
14638 let rendered = trace.to_string();
14639 assert!(
14640 rendered.starts_with(&format!("{GAUGE}: non_zero(1) @"))
14641 && rendered.contains("; last value non_zero(1) @"),
14642 "{rendered}"
14643 );
14644
14645 let silent = Drain::start(Answer::Silent);
14646 let (_, _, trace) = silent.run_traced(Duration::from_secs(1)).await;
14647 assert_eq!(
14650 trace.to_string(),
14651 format!("{GAUGE}: probe_failed(no-answer) @1000ms; no value ever read")
14652 );
14653 assert_eq!(trace.probes, 1);
14654 }
14655
14656 #[test]
14658 fn a_gauge_that_is_not_a_count_is_busy_and_shown_as_sent() {
14659 let gauges = [GAUGE.to_string()];
14660 let verdicts = gauge_verdicts(Some(&serde_json::json!({ GAUGE: 0.0 })), &gauges);
14661 assert_eq!(verdicts, vec![GaugeVerdict::NotACount("0.0".to_string())]);
14662 assert_eq!(
14663 busy_gauge_observation(&verdicts),
14664 BusyGaugeObservation::Busy
14665 );
14666 let zero = gauge_verdicts(Some(&serde_json::json!({ GAUGE: 0 })), &gauges);
14667 assert_eq!(
14668 busy_gauge_observation(&zero),
14669 BusyGaugeObservation::Quiescent
14670 );
14671 let none = gauge_verdicts(None, &gauges);
14672 assert_eq!(busy_gauge_observation(&none), BusyGaugeObservation::Omitted);
14673 }
14674}
14675
14676#[cfg(test)]
14677mod health_evidence_tests {
14678 use super::{HealthProbeError, HealthProbeEvidence};
14679 use std::collections::HashSet;
14680
14681 #[test]
14689 fn only_a_dead_lane_is_proof_of_death() {
14690 assert!(HealthProbeError::lane_dead("gone").is_proof_of_death());
14691 assert!(!HealthProbeError::no_answer("timed out").is_proof_of_death());
14695 assert!(!HealthProbeError::bad_answer("garbage").is_proof_of_death());
14696 assert!(!HealthProbeError::misconfigured("no table").is_proof_of_death());
14697 }
14698
14699 #[test]
14705 fn every_evidence_class_has_a_distinct_label() {
14706 let labels = [
14707 HealthProbeError::lane_dead("").label(),
14708 HealthProbeError::no_answer("").label(),
14709 HealthProbeError::bad_answer("").label(),
14710 HealthProbeError::misconfigured("").label(),
14711 ];
14712 let unique: HashSet<_> = labels.iter().collect();
14713 assert_eq!(unique.len(), labels.len(), "labels collided: {labels:?}");
14714 }
14715
14716 #[test]
14722 fn classification_preserves_the_original_message() {
14723 let err = HealthProbeError::no_answer("module did not answer within 5s");
14724 assert_eq!(err.to_string(), "module did not answer within 5s");
14725 assert!(matches!(err.evidence, HealthProbeEvidence::NoAnswer));
14726 }
14727}
14728
14729#[cfg(test)]
14730mod health_tombstone_tests {
14731 use std::{path::PathBuf, sync::Arc, time::Duration};
14732
14733 use subc_protocol::{
14734 manifest::Concurrency,
14735 session::{HealthStatus, ModuleControlResponse},
14736 };
14737 use tokio::sync::mpsc;
14738
14739 use super::{
14740 probe_module_health, HealthAction, HealthConfig, HealthProbeEvidence, ModuleProtocol,
14741 ModuleSpec, RestartPolicy, Supervisor, SupervisorRuntimeConfig,
14742 };
14743 use crate::{
14744 control::ControlHandler,
14745 forwarding::{ForwardingTable, ModuleControlRpcCompletion, ModuleControlRpcOutcome},
14746 registry::{ConnectionId, Registry},
14747 router::FrameSink,
14748 };
14749
14750 struct ProbeHarness {
14751 spec: ModuleSpec,
14752 runtime: SupervisorRuntimeConfig,
14753 forwarding: Arc<ForwardingTable>,
14754 module_connection: ConnectionId,
14755 module_rx: mpsc::Receiver<crate::router::OutboundFrame>,
14756 handler: ControlHandler,
14757 module: super::SupervisedModule,
14758 }
14759
14760 fn probe_harness() -> ProbeHarness {
14761 let registry = Arc::new(Registry::default());
14762 let forwarding = Arc::new(ForwardingTable::default());
14763 let supervisor_handle = super::SupervisorHandle::new();
14764 let health = HealthConfig {
14765 http: None,
14766 cadence: Duration::from_secs(30),
14767 deadline: Duration::from_secs(5),
14768 failure_threshold: 3,
14769 on_degraded: HealthAction::Report,
14770 on_failing: HealthAction::Report,
14771 critical: false,
14772 };
14773 let supervisor = Supervisor::new_for_test(Arc::clone(®istry), RestartPolicy::default())
14774 .with_forwarding(Arc::clone(&forwarding))
14775 .with_handle(supervisor_handle.clone())
14776 .with_health_config(health);
14777 let spec = ModuleSpec {
14778 module_id: "late-health-module".to_string(),
14779 program: PathBuf::from("disabled-module"),
14780 args: Vec::new(),
14781 env: Vec::new(),
14782 reserved: false,
14783 reserved_prefixes: Vec::new(),
14784 protocol: ModuleProtocol::Subc,
14785 overlap: Default::default(),
14786 };
14787 let module = supervisor
14788 .supervise_configured(spec.clone(), false)
14789 .unwrap();
14790 let runtime = supervisor.runtime_config();
14791 let handler = ControlHandler::with_forwarding(registry, Arc::clone(&forwarding))
14792 .with_supervisor(supervisor_handle);
14793 let module_connection = ConnectionId::new(700);
14794 let (module_tx, module_rx) = mpsc::channel(8);
14795 forwarding
14796 .register_module_connection(
14797 module_connection,
14798 spec.module_id.clone(),
14799 subc_protocol::PROTOCOL_VERSION,
14800 Concurrency::ModuleManaged,
14801 FrameSink::new(module_tx),
14802 )
14803 .unwrap();
14804
14805 ProbeHarness {
14806 spec,
14807 runtime,
14808 forwarding,
14809 module_connection,
14810 module_rx,
14811 handler,
14812 module,
14813 }
14814 }
14815
14816 async fn finish_after(
14817 harness: &mut ProbeHarness,
14818 stall: Duration,
14819 ) -> ModuleControlRpcCompletion {
14820 assert!(stall > harness.runtime.health.deadline);
14821 let deadline = harness.runtime.health.deadline;
14822 let probe = probe_module_health(&harness.spec.module_id, &harness.runtime, None);
14823 let answer = async {
14824 let frame = harness.module_rx.recv().await.expect("health.check frame");
14825 tokio::time::advance(deadline).await;
14826 tokio::task::yield_now().await;
14827 tokio::time::advance(stall - deadline).await;
14828 harness
14829 .forwarding
14830 .complete_module_control_rpc(
14831 harness.module_connection,
14832 frame.header.corr,
14833 Some("health.check"),
14834 ModuleControlRpcOutcome::Response(ModuleControlResponse::HealthCheck {
14835 status: HealthStatus::Ok,
14836 detail: None,
14837 metrics: None,
14838 }),
14839 )
14840 .unwrap()
14841 };
14842 let (probe_result, completion) = tokio::join!(probe, answer);
14843 let err = probe_result.expect_err("probe must miss its deadline");
14844 assert!(matches!(err.evidence, HealthProbeEvidence::NoAnswer));
14845 completion
14846 }
14847
14848 async fn time_out_without_answer(harness: &mut ProbeHarness) {
14849 let deadline = harness.runtime.health.deadline;
14850 let probe = probe_module_health(&harness.spec.module_id, &harness.runtime, None);
14851 let exhaust_deadline = async {
14852 let _frame = harness.module_rx.recv().await.expect("health.check frame");
14853 tokio::time::advance(deadline).await;
14854 tokio::task::yield_now().await;
14855 };
14856 let (probe_result, ()) = tokio::join!(probe, exhaust_deadline);
14857 let err = probe_result.expect_err("probe must miss its deadline");
14858 assert!(matches!(err.evidence, HealthProbeEvidence::NoAnswer));
14859 }
14860
14861 async fn run_probe_cycle(harness: &mut ProbeHarness, answer: bool) {
14862 let registry = Arc::clone(&harness.module.inner.registry);
14863 let snapshot = Arc::clone(&harness.module.inner.snapshot);
14864 let process_liveness = super::SupervisorProcessLiveness::default();
14865 let mut child = None;
14866 let cycle = super::run_health_probe_cycle(
14867 &harness.spec,
14868 &harness.runtime,
14869 ®istry,
14870 &process_liveness,
14871 &snapshot,
14872 &mut child,
14873 );
14874 let peer = async {
14875 let frame = harness.module_rx.recv().await.expect("health.check frame");
14876 if answer {
14877 harness
14878 .forwarding
14879 .complete_module_control_rpc(
14880 harness.module_connection,
14881 frame.header.corr,
14882 Some("health.check"),
14883 ModuleControlRpcOutcome::Response(ModuleControlResponse::HealthCheck {
14884 status: HealthStatus::Ok,
14885 detail: None,
14886 metrics: Some(serde_json::json!({"ready": true})),
14887 }),
14888 )
14889 .unwrap();
14890 } else {
14891 tokio::time::advance(harness.runtime.health.deadline).await;
14892 tokio::task::yield_now().await;
14893 }
14894 };
14895 tokio::join!(cycle, peer);
14896 }
14897
14898 #[tokio::test(start_paused = true)]
14899 async fn unanswered_probe_is_unknown_until_threshold_and_ok_report_recovers() {
14900 let mut harness = probe_harness();
14901 let monitor = harness.module.inner.monitor.lock().unwrap().take().unwrap();
14905 monitor.abort();
14906 let _ = monitor.await;
14907 super::update_snapshot(&harness.module.inner.snapshot, None, |state| {
14908 state.enabled = true;
14909 state.state = super::ModuleState::Running;
14910 state.process_alive = true;
14911 })
14912 .unwrap();
14913
14914 run_probe_cycle(&mut harness, true).await;
14915 assert_eq!(
14916 harness.module.status().unwrap().health.status,
14917 super::SupervisorHealthStatus::Ok
14918 );
14919
14920 for failures in 1..harness.runtime.health.failure_threshold {
14921 run_probe_cycle(&mut harness, false).await;
14922 let status = harness.module.status().unwrap();
14923 assert_eq!(status.health.status, super::SupervisorHealthStatus::Unknown);
14924 assert_eq!(status.health.consecutive_failures, failures);
14925 assert!(status.health.last_probe_ms.is_some());
14926 assert!(status.health.detail.unwrap().starts_with("[no-answer]"));
14927 assert!(status.health.metrics.is_none());
14928 assert_eq!(status.state, super::ModuleState::Running);
14929 assert!(status.process_alive);
14930 assert_eq!(status.restart_count, 0);
14931 assert_eq!(status.lifetime_restarts, 0);
14932 assert!(status.health.last_action.is_none());
14933 assert!(harness.runtime.scheduled_respawn.lock().unwrap().is_none());
14934 }
14935
14936 run_probe_cycle(&mut harness, true).await;
14937 let recovered = harness.module.status().unwrap();
14938 assert_eq!(recovered.health.status, super::SupervisorHealthStatus::Ok);
14939 assert_eq!(recovered.health.consecutive_failures, 0);
14940 assert!(recovered.health.detail.is_none());
14941 assert_eq!(
14942 recovered.health.metrics,
14943 Some(serde_json::json!({"ready": true}))
14944 );
14945 assert_eq!(recovered.lifetime_restarts, 0);
14946
14947 for failures in 1..=harness.runtime.health.failure_threshold {
14948 run_probe_cycle(&mut harness, false).await;
14949 let status = harness.module.status().unwrap();
14950 assert_eq!(status.health.consecutive_failures, failures);
14951 if failures < harness.runtime.health.failure_threshold {
14952 assert_eq!(status.health.status, super::SupervisorHealthStatus::Unknown);
14953 assert_eq!(status.state, super::ModuleState::Running);
14954 assert_eq!(status.lifetime_restarts, 0);
14955 assert!(harness.runtime.scheduled_respawn.lock().unwrap().is_none());
14956 } else {
14957 assert_eq!(
14958 status.health.status,
14959 super::SupervisorHealthStatus::Unresponsive
14960 );
14961 assert_eq!(status.state, super::ModuleState::Restarting);
14962 assert_eq!(status.restart_count, 1);
14963 assert_eq!(status.lifetime_restarts, 1);
14964 assert_eq!(status.health.last_action.as_deref(), Some("restart"));
14965 assert!(harness.runtime.scheduled_respawn.lock().unwrap().is_some());
14966 }
14967 }
14968 }
14969
14970 #[tokio::test(start_paused = true)]
14971 async fn late_health_answers_record_start_anchored_latency_for_two_stalls() {
14972 let mut harness = probe_harness();
14973
14974 let first = finish_after(&mut harness, Duration::from_secs(8)).await;
14975 let first_latency = match &first {
14976 ModuleControlRpcCompletion::LateHealthAnswer { latency, .. } => *latency,
14977 other => panic!("late answer was not retained: {other:?}"),
14978 };
14979 assert!(harness.handler.observe_module_control_completion(first));
14980
14981 let second = finish_after(&mut harness, Duration::from_secs(11)).await;
14982 let second_latency = match &second {
14983 ModuleControlRpcCompletion::LateHealthAnswer { latency, .. } => *latency,
14984 other => panic!("late answer was not retained: {other:?}"),
14985 };
14986 assert!(harness.handler.observe_module_control_completion(second));
14987
14988 assert_eq!(first_latency, Duration::from_secs(8));
14989 assert_eq!(
14990 second_latency - first_latency,
14991 Duration::from_secs(3),
14992 "latency must grow linearly with the additional stall"
14993 );
14994 let health = harness.module.status().unwrap().health;
14995 assert_eq!(health.late_answer_count, 2);
14996 assert_eq!(health.last_late_answer_latency_ms, Some(11_000));
14997 }
14998
14999 #[tokio::test(start_paused = true)]
15007 async fn late_answer_clears_the_consecutive_failure_streak() {
15008 let mut harness = probe_harness();
15009
15010 time_out_without_answer(&mut harness).await;
15012 harness
15013 .module
15014 .record_health_probe_failure_for_test("[no-answer] test miss")
15015 .unwrap();
15016 assert_eq!(
15017 harness.module.status().unwrap().health.consecutive_failures,
15018 1,
15019 "precondition: the miss must be on the streak before the late answer"
15020 );
15021
15022 let late = finish_after(&mut harness, Duration::from_secs(9)).await;
15024 assert!(matches!(
15025 late,
15026 ModuleControlRpcCompletion::LateHealthAnswer { .. }
15027 ));
15028 assert!(harness.handler.observe_module_control_completion(late));
15029
15030 let health = harness.module.status().unwrap().health;
15031 assert_eq!(
15032 health.consecutive_failures, 0,
15033 "a late answer is an answer: the streak must reset"
15034 );
15035 assert_eq!(health.late_answer_count, 1);
15036 }
15037
15038 #[tokio::test(start_paused = true)]
15039 async fn repeated_serial_probe_cycles_keep_one_tombstone_per_endpoint() {
15040 let mut harness = probe_harness();
15041
15042 for _ in 0..20 {
15043 time_out_without_answer(&mut harness).await;
15044 assert_eq!(
15045 harness.forwarding.health_probe_tombstone_count().unwrap(),
15046 1
15047 );
15048 }
15049 }
15050}
15051
15052#[cfg(test)]
15053mod child_env_tests {
15054 #[cfg(windows)]
15055 use super::{
15056 clear_current_process_facts, observed_launch_nonce_source, ModuleState, SupervisorSnapshot,
15057 };
15058 #[cfg(windows)]
15059 use std::sync::{Arc, OnceLock};
15060 #[cfg(windows)]
15061 #[test]
15062 fn windows_wire_spawn_prepares_pipe_and_keeps_phase_one_env() {
15063 let spec = spec(Vec::new());
15064 let mut command = Command::new("unused");
15065 let handoff = apply_wire_spawn_args(&mut command, &spec, None, None)
15066 .unwrap()
15067 .unwrap();
15068 let name = command
15069 .as_std()
15070 .get_envs()
15071 .find(|(key, _)| *key == OsStr::new(subc_os::launch_nonce::LAUNCH_NONCE_PIPE_ENV))
15072 .and_then(|(_, value)| value)
15073 .unwrap();
15074 assert_eq!(name, OsStr::new(handoff.name()));
15075 assert!(handoff.name().starts_with(r"\\.\pipe\subc-launch-"));
15076 assert!(command.as_std().get_envs().any(|(key, value)| key
15077 == OsStr::new(subc_os::LAUNCH_NONCE_PIPE_FALLBACK_ENV)
15078 && value == Some(OsStr::new("env"))));
15079 assert!(command
15080 .as_std()
15081 .get_envs()
15082 .any(|(key, value)| key == OsStr::new(SUBC_LAUNCH_NONCE_ENV) && value.is_some()));
15083 }
15084
15085 #[cfg(windows)]
15086 #[test]
15087 fn windows_source_reporting_tracks_consumption_and_resets_per_spawn() {
15088 let mut snapshot = SupervisorSnapshot::new(ModuleState::Running, true);
15089 snapshot.process_alive = true;
15090 snapshot.spawned_protocol = Some(ModuleProtocol::Subc);
15091 assert_eq!(
15092 observed_launch_nonce_source(&snapshot).as_deref(),
15093 Some("env")
15094 );
15095 let consumed = Arc::new(OnceLock::new());
15096 snapshot.pipe_consumed = Some(consumed.clone());
15097 assert_eq!(
15098 observed_launch_nonce_source(&snapshot).as_deref(),
15099 Some("env")
15100 );
15101 consumed.set(()).unwrap();
15102 assert_eq!(
15103 observed_launch_nonce_source(&snapshot).as_deref(),
15104 Some("pipe")
15105 );
15106 snapshot.pipe_consumed = Some(Arc::new(OnceLock::new()));
15107 assert_eq!(
15108 observed_launch_nonce_source(&snapshot).as_deref(),
15109 Some("env")
15110 );
15111 clear_current_process_facts(&mut snapshot);
15112 assert_eq!(observed_launch_nonce_source(&snapshot), None);
15113 }
15114 use super::{
15115 apply_child_env, apply_spawn_role, apply_wire_spawn_args, LaunchSpecDigest, ModuleProtocol,
15116 ModuleSpec, SpawnRole, SupervisorHandle, SPAWN_ROLE_SWAP_CANDIDATE, SUBC_ARG,
15117 SUBC_LAUNCH_NONCE_ENV, SUBC_MODULE_ID_ENV, SUBC_SPAWN_ROLE_ENV,
15118 };
15119 use std::{ffi::OsStr, path::PathBuf};
15120 use tokio::process::Command;
15121
15122 fn spec(env: Vec<(String, String)>) -> ModuleSpec {
15123 ModuleSpec {
15124 module_id: "env-plan".to_string(),
15125 program: PathBuf::from("/nonexistent"),
15126 args: Vec::new(),
15127 env,
15128 reserved: false,
15129 reserved_prefixes: Vec::new(),
15130 protocol: ModuleProtocol::Subc,
15131 overlap: Default::default(),
15132 }
15133 }
15134
15135 #[test]
15136 fn configured_launch_spec_digest_is_canonical_and_excludes_injected_inputs() {
15137 let base = spec(vec![("A".into(), "B=C".into()), ("Z".into(), "v".into())]);
15138 let digest = LaunchSpecDigest::from_spec(&base);
15139 let mut reordered = base.clone();
15140 reordered.env.reverse();
15141 reordered.module_id = "different-id".into();
15142 reordered.reserved = true;
15143 reordered.overlap = Default::default();
15144 assert_eq!(digest, LaunchSpecDigest::from_spec(&reordered));
15145 let mut delimiter_collision = base.clone();
15146 delimiter_collision.env[0] = ("A=B".into(), "C".into());
15147 assert_ne!(digest, LaunchSpecDigest::from_spec(&delimiter_collision));
15148 let mut changed = base.clone();
15149 changed.protocol = ModuleProtocol::None;
15150 assert_ne!(digest, LaunchSpecDigest::from_spec(&changed));
15151 changed = base.clone();
15152 changed.args.push("--flag".into());
15153 assert_ne!(digest, LaunchSpecDigest::from_spec(&changed));
15154 changed = base.clone();
15155 changed.program = PathBuf::from("/other");
15156 assert_ne!(digest, LaunchSpecDigest::from_spec(&changed));
15157 changed = base.clone();
15158 changed.env[1].1 = "other".into();
15159 assert_ne!(digest, LaunchSpecDigest::from_spec(&changed));
15160 assert!(!format!("{digest:?}").contains("B=C"));
15161 assert!(!format!("{digest:?}").contains("[u8"));
15162 }
15163
15164 #[cfg(unix)]
15165 #[test]
15166 fn native_program_bytes_do_not_collapse_through_lossy_utf8() {
15167 use std::os::unix::ffi::OsStringExt;
15168 let mut left = spec(Vec::new());
15169 left.program = std::ffi::OsString::from_vec(vec![b'/', 0xff]).into();
15170 let mut right = left.clone();
15171 right.program = std::ffi::OsString::from_vec(vec![b'/', 0xfe]).into();
15172 assert_eq!(
15173 left.program.to_string_lossy(),
15174 right.program.to_string_lossy()
15175 );
15176 assert_ne!(
15177 LaunchSpecDigest::from_spec(&left),
15178 LaunchSpecDigest::from_spec(&right)
15179 );
15180 }
15181
15182 #[test]
15196 fn ambient_ck_log_is_removed_and_a_configured_one_survives() {
15197 let mut command = Command::new("/nonexistent");
15198 apply_child_env(&mut command, &spec(Vec::new()));
15199 let removed = command
15200 .as_std()
15201 .get_envs()
15202 .any(|(key, value)| key == OsStr::new("CK_LOG") && value.is_none());
15203 assert!(
15204 removed,
15205 "ambient CK_LOG must be explicitly removed for an unconfigured module"
15206 );
15207
15208 let mut configured = Command::new("/nonexistent");
15209 apply_child_env(
15210 &mut configured,
15211 &spec(vec![("CK_LOG".to_string(), "debug".to_string())]),
15212 );
15213 let effective = configured
15214 .as_std()
15215 .get_envs()
15216 .filter(|(key, _)| *key == OsStr::new("CK_LOG"))
15217 .last()
15218 .map(|(_, value)| value.map(|v| v.to_string_lossy().into_owned()));
15219 assert_eq!(
15220 effective,
15221 Some(Some("debug".to_string())),
15222 "a module's configured CK_LOG must survive the ambient removal"
15223 );
15224 }
15225
15226 #[test]
15235 fn protocol_none_spawn_carries_no_subc_argument_and_no_nonce() {
15236 let connection_file = std::path::Path::new("/run/subc-connection.json");
15237 let handle = SupervisorHandle::new();
15238
15239 let mut none_spec = spec(Vec::new());
15240 none_spec.protocol = ModuleProtocol::None;
15241 let mut none = Command::new("/nonexistent");
15242 let none_handoff =
15243 apply_wire_spawn_args(&mut none, &none_spec, Some(connection_file), Some(&handle))
15244 .expect("protocol-none spawn args apply");
15245 assert!(
15246 none_handoff.is_none(),
15247 "protocol:none spawn must not receive a nonce descriptor"
15248 );
15249 #[cfg(windows)]
15250 assert!(!none.as_std().get_envs().any(|(key, value)| key
15251 == OsStr::new(subc_os::LAUNCH_NONCE_PIPE_FALLBACK_ENV)
15252 && value.is_some()));
15253 assert!(
15254 !none.as_std().get_envs().any(|(key, value)| key
15255 == OsStr::new(subc_os::LAUNCH_NONCE_FD_ENV)
15256 && value.is_some()),
15257 "protocol:none spawn must not name a nonce descriptor"
15258 );
15259 let none_args: Vec<String> = none
15260 .as_std()
15261 .get_args()
15262 .map(|a| a.to_string_lossy().into_owned())
15263 .collect();
15264 assert!(
15265 !none_args.iter().any(|a| a == SUBC_ARG),
15266 "protocol:none argv must not carry --subc; got {none_args:?}"
15267 );
15268 let none_has_nonce = none
15269 .as_std()
15270 .get_envs()
15271 .any(|(key, value)| key == OsStr::new(SUBC_LAUNCH_NONCE_ENV) && value.is_some());
15272 assert!(
15273 !none_has_nonce,
15274 "protocol:none spawn must not receive a launch nonce"
15275 );
15276 let none_has_module_id = none
15277 .as_std()
15278 .get_envs()
15279 .any(|(key, value)| key == OsStr::new(SUBC_MODULE_ID_ENV) && value.is_some());
15280 assert!(
15281 none_has_module_id,
15282 "SUBC_MODULE_ID is inert and stays on every path"
15283 );
15284 assert!(
15285 handle.spawn_nonce(&none_spec.module_id).is_none(),
15286 "no nonce record for a process that will never present one"
15287 );
15288
15289 let wire_spec = spec(Vec::new());
15291 let mut wire = Command::new("/nonexistent");
15292 let wire_handoff =
15293 apply_wire_spawn_args(&mut wire, &wire_spec, Some(connection_file), Some(&handle))
15294 .expect("subc-wire spawn args apply");
15295 let wire_fd_env = wire
15296 .as_std()
15297 .get_envs()
15298 .find(|(key, _)| *key == OsStr::new(subc_os::LAUNCH_NONCE_FD_ENV))
15299 .and_then(|(_, value)| value.map(|v| v.to_string_lossy().into_owned()));
15300 #[cfg(unix)]
15301 assert_eq!(
15302 wire_fd_env,
15303 Some(wire_handoff.expect("a descriptor handoff").fd_env_value()),
15304 "a subc-wire spawn names the pipe it will receive at descriptor 3"
15305 );
15306 #[cfg(windows)]
15307 assert!(wire_handoff.is_some() && wire_fd_env.is_none());
15308 #[cfg(not(any(unix, windows)))]
15309 assert!(wire_handoff.is_none() && wire_fd_env.is_none());
15310 let wire_args: Vec<String> = wire
15311 .as_std()
15312 .get_args()
15313 .map(|a| a.to_string_lossy().into_owned())
15314 .collect();
15315 assert_eq!(
15316 wire_args,
15317 vec![
15318 SUBC_ARG.to_string(),
15319 connection_file.to_string_lossy().into_owned()
15320 ],
15321 "a subc-wire spawn still carries --subc <path>"
15322 );
15323 assert_eq!(
15324 wire.as_std()
15325 .get_envs()
15326 .any(|(key, value)| key == OsStr::new(SUBC_LAUNCH_NONCE_ENV) && value.is_some()),
15327 !cfg!(unix),
15328 "only Windows supplies the environment nonce"
15329 );
15330 assert!(handle.spawn_nonce(&wire_spec.module_id).is_some());
15331 }
15332
15333 #[test]
15343 fn plain_spawn_removes_the_spawn_role_even_when_the_spec_sets_it() {
15344 let role = |command: &Command| {
15345 command
15346 .as_std()
15347 .get_envs()
15348 .filter(|(key, _)| *key == OsStr::new(SUBC_SPAWN_ROLE_ENV))
15349 .last()
15350 .map(|(_, value)| value.map(|v| v.to_string_lossy().into_owned()))
15351 };
15352 let forged = spec(vec![(
15353 SUBC_SPAWN_ROLE_ENV.to_string(),
15354 SPAWN_ROLE_SWAP_CANDIDATE.to_string(),
15355 )]);
15356
15357 let mut plain = Command::new("/nonexistent");
15358 apply_child_env(&mut plain, &forged);
15359 apply_spawn_role(&mut plain, SpawnRole::Plain);
15360 assert_eq!(
15361 role(&plain),
15362 Some(None),
15363 "a plain spawn must remove SUBC_SPAWN_ROLE, whatever the spec says"
15364 );
15365
15366 let mut candidate = Command::new("/nonexistent");
15367 apply_child_env(&mut candidate, &spec(Vec::new()));
15368 apply_spawn_role(&mut candidate, SpawnRole::SwapCandidate);
15369 assert_eq!(
15370 role(&candidate),
15371 Some(Some(SPAWN_ROLE_SWAP_CANDIDATE.to_string()))
15372 );
15373 }
15374
15375 #[test]
15381 fn daemon_private_capture_keys_are_not_passed_to_the_child() {
15382 let mut command = Command::new("/nonexistent");
15383 apply_child_env(
15384 &mut command,
15385 &spec(vec![
15386 (super::CAPTURE_KEEP_ENV.to_string(), "5".to_string()),
15387 ("KEPT".to_string(), "yes".to_string()),
15388 ]),
15389 );
15390 let keys: Vec<String> = command
15391 .as_std()
15392 .get_envs()
15393 .filter(|(_, value)| value.is_some())
15394 .map(|(key, _)| key.to_string_lossy().into_owned())
15395 .collect();
15396 assert!(keys.contains(&"KEPT".to_string()), "got {keys:?}");
15397 assert!(
15398 !keys.contains(&super::CAPTURE_KEEP_ENV.to_string()),
15399 "daemon-private capture key leaked to the child: {keys:?}"
15400 );
15401 }
15402}
15403
15404#[cfg(test)]
15405mod jitter_tests {
15406 use super::jittered_health_delay;
15407 use std::{collections::HashSet, time::Duration};
15408
15409 const FLEET: [&str; 14] = [
15418 "aft",
15419 "alfonso-core",
15420 "magic-context",
15421 "broca",
15422 "thalamus",
15423 "quota",
15424 "engram",
15425 "plexus",
15426 "cerebellum",
15427 "astrocyte",
15428 "synapse",
15429 "subc-mcp",
15430 "cortexkit-credentials",
15431 "subc-federation",
15432 ];
15433
15434 #[test]
15442 fn probe_delays_disperse_across_the_fleet() {
15443 let cadence = Duration::from_secs(30);
15444 let delays: HashSet<Duration> = FLEET
15445 .iter()
15446 .map(|id| jittered_health_delay(id, 0, cadence))
15447 .collect();
15448 assert_eq!(
15449 delays.len(),
15450 FLEET.len(),
15451 "every supervised module must land on its own probe offset"
15452 );
15453 }
15454
15455 #[test]
15461 fn jitter_only_delays_and_stays_within_one_tenth_of_cadence() {
15462 let cadence = Duration::from_secs(30);
15463 let span = cadence / 10;
15464 for id in FLEET {
15465 for probe_index in 0..8 {
15466 let delay = jittered_health_delay(id, probe_index, cadence);
15467 assert!(
15468 delay >= cadence,
15469 "{id}#{probe_index}: jitter must not shorten the cadence"
15470 );
15471 assert!(
15472 delay < cadence + span,
15473 "{id}#{probe_index}: jitter must stay inside one tenth of the cadence"
15474 );
15475 }
15476 }
15477 }
15478
15479 #[test]
15485 fn a_module_offset_is_stable_across_restarts() {
15486 let cadence = Duration::from_secs(30);
15487 for id in FLEET {
15488 assert_eq!(
15489 jittered_health_delay(id, 0, cadence),
15490 jittered_health_delay(id, 0, cadence),
15491 "{id}: the same module and probe index must produce the same offset"
15492 );
15493 }
15494 }
15495
15496 #[test]
15498 fn zero_cadence_yields_zero_delay() {
15499 assert_eq!(
15500 jittered_health_delay("aft", 0, Duration::ZERO),
15501 Duration::ZERO
15502 );
15503 }
15504}
15505
15506#[cfg(all(test, target_os = "linux"))]
15507mod cgroup_placement_tests {
15508 use super::{
15509 apply_cgroup_placement, remove_module_cgroup, LaunchSpecDigest, ModuleProtocol, ModuleSpec,
15510 SuperviseError, SupervisedChild,
15511 };
15512 use crate::stderr_tail::{StderrRing, StderrTailConfig};
15513 use cortexkit_test_support::ScratchDir;
15514 use std::{
15515 fs, io,
15516 path::{Path, PathBuf},
15517 sync::{Arc, Mutex},
15518 };
15519 use tokio::process::Command;
15520
15521 #[tokio::test]
15522 async fn unique_spawn_cgroups_clean_up_across_restarts_and_shutdown() {
15523 use super::*;
15524 let dir = ScratchDir::new("unique-spawn-cgroups");
15525 let root = PathBuf::from(format!(
15526 "/sys/fs/cgroup/subc-unique-test-{}-{}",
15527 std::process::id(),
15528 unix_ms_now()
15529 ));
15530 if let Err(error) = fs::create_dir(&root) {
15531 assert!(
15532 std::env::var_os("SUBC_REQUIRE_CGROUP_TEST").is_none(),
15533 "required cgroup test cannot execute: {error}"
15534 );
15535 eprintln!("SKIP unique_spawn_cgroups_clean_up_across_restarts_and_shutdown: {error}");
15536 return;
15537 }
15538 let placement = subc_cgroup::prepare_at(&root).unwrap().unwrap();
15539 let group_count = || {
15540 fs::read_dir(root.join("subc-modules"))
15541 .unwrap()
15542 .map(|entry| entry.unwrap().file_type().unwrap())
15543 .filter(|kind| kind.is_dir())
15544 .count()
15545 };
15546 let supervisor =
15547 Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default())
15548 .with_cgroup_placement(Some(placement.clone()));
15549 let runtime = supervisor.runtime_config();
15550 let mut spec = ModuleSpec {
15551 module_id: "unique-spawn".into(),
15552 program: PathBuf::from("/bin/sleep"),
15553 args: vec!["60".into()],
15554 env: ["XDG_DATA_HOME", "XDG_RUNTIME_DIR", "XDG_CONFIG_HOME"]
15555 .into_iter()
15556 .map(|key| (key.into(), dir.to_string_lossy().into_owned()))
15557 .collect(),
15558 reserved: false,
15559 reserved_prefixes: vec![],
15560 protocol: ModuleProtocol::None,
15561 overlap: Default::default(),
15562 };
15563 let spawn = |spec: &ModuleSpec| {
15564 spawn_child(
15565 spec,
15566 None,
15567 None,
15568 &runtime.stderr_ring,
15569 None,
15570 &runtime.child_roster,
15571 Some(&placement),
15572 )
15573 .unwrap()
15574 };
15575 let mut live = spawn(&spec);
15576 for _ in 0..3 {
15577 let next = spawn(&spec);
15579 assert_ne!(live.module_id, next.module_id);
15580 live.start_kill().unwrap();
15581 live.wait().await.unwrap();
15582 live = next;
15583 assert!(
15584 live.child.try_wait().unwrap().is_none(),
15585 "retiring the old slot must not kill the replacement"
15586 );
15587 assert_eq!(
15588 group_count(),
15589 1,
15590 "only the live spawn's cgroup should remain"
15591 );
15592 }
15593 supervisor.begin_daemon_shutdown();
15594 let reap = tokio::spawn(async move {
15595 live.wait().await.unwrap();
15596 });
15597 supervisor
15598 .end_children_for_daemon_shutdown(false, std::future::pending())
15599 .await;
15600 reap.await.unwrap();
15601 assert_eq!(group_count(), 0);
15602 spec.program = PathBuf::from("/bin/true");
15604 spec.args.clear();
15605 let fresh_roster = ChildRoster::default();
15606 let mut short = spawn_child(
15607 &spec,
15608 None,
15609 None,
15610 &runtime.stderr_ring,
15611 None,
15612 &fresh_roster,
15613 Some(&placement),
15614 )
15615 .unwrap();
15616 short.wait().await.unwrap();
15617 assert_eq!(group_count(), 0);
15618 spec.module_id = "_".repeat(255);
15619 let mut long_id = spawn_child(
15620 &spec,
15621 None,
15622 None,
15623 &runtime.stderr_ring,
15624 None,
15625 &fresh_roster,
15626 Some(&placement),
15627 )
15628 .unwrap();
15629 long_id.wait().await.unwrap();
15630 assert_eq!(
15631 group_count(),
15632 0,
15633 "valid long module IDs must not exceed cgroup NAME_MAX"
15634 );
15635 fs::remove_dir(root.join("subc-modules")).unwrap();
15636 fs::remove_dir(root).unwrap();
15637 eprintln!("EXECUTED unique_spawn_cgroups_clean_up_across_restarts_and_shutdown");
15638 }
15639
15640 #[test]
15641 fn failed_parent_cgroup_open_is_a_cgroup_supervision_error() {
15642 let path = Path::new("/definitely-missing-subc-cgroup");
15643 let mut command = Command::new("true");
15644 let error = apply_cgroup_placement(
15645 &mut command,
15646 &ModuleSpec {
15647 module_id: "broken-cgroup".to_string(),
15648 program: PathBuf::from("true"),
15649 args: Vec::new(),
15650 env: Vec::new(),
15651 reserved: false,
15652 reserved_prefixes: Vec::new(),
15653 protocol: ModuleProtocol::Subc,
15654 overlap: Default::default(),
15655 },
15656 path,
15657 )
15658 .expect_err("a parent cgroup open failure must reject the supervised spawn");
15659 let reason = error.to_string();
15660
15661 assert!(
15662 matches!(error, SuperviseError::Cgroup { .. }),
15663 "parent cgroup open must be reported as a cgroup supervision error: {reason}"
15664 );
15665 assert!(
15666 reason.contains("/definitely-missing-subc-cgroup/cgroup.procs"),
15667 "parent cgroup open failure must name cgroup.procs: {reason}"
15668 );
15669 }
15670
15671 #[tokio::test]
15672 async fn reaping_a_child_removes_its_empty_module_cgroup() {
15673 let root = ScratchDir::new("supervisor-reap-cgroup");
15674 fs::write(root.join("cgroup.procs"), b"").expect("write scratch cgroup marker");
15675 let placement = subc_cgroup::prepare_at(&root)
15676 .expect("prepare scratch cgroup root")
15677 .expect("scratch root has a cgroup.procs marker");
15678 let module_id = "reaped-module";
15679 let module = placement
15680 .module_path(module_id)
15681 .expect("create scratch module cgroup");
15682 let child = Command::new("true")
15683 .env("XDG_DATA_HOME", root.path())
15684 .env("XDG_RUNTIME_DIR", root.path())
15685 .env("XDG_CONFIG_HOME", root.path())
15686 .spawn()
15687 .expect("spawn short-lived child");
15688 let pid = child.id().expect("spawned child has pid");
15689 let mut child = SupervisedChild {
15690 child,
15691 exec: crate::activity::ExecOwner(Arc::new(crate::activity::ExecOutcome::new())),
15692 protocol: ModuleProtocol::Subc,
15693 module_id: module_id.to_string(),
15694 cgroup_placement: Some(placement),
15695 stdout_pump: None,
15696 stderr_pump: None,
15697 stderr_ring: Arc::new(Mutex::new(StderrRing::new(StderrTailConfig::default()))),
15698 spawned_at: tokio::time::Instant::now(),
15699 spawned_at_ms: 0,
15700 spawned_from: PathBuf::from("true"),
15701 launch_spec_digest: LaunchSpecDigest([0; 32]),
15702 spawned_file_identity: None,
15703 process_start_time: None,
15704 process_identity: None,
15705 pid,
15706 roster_guard: None,
15707 #[cfg(target_os = "macos")]
15708 privacy_exec: None,
15709 spawn_failure: None,
15710 };
15711
15712 child.wait().await.expect("reap short-lived child");
15713
15714 assert!(
15715 !module.exists(),
15716 "reaping the supervised child must remove its empty cgroup"
15717 );
15718 }
15719
15720 #[test]
15721 fn non_empty_cgroup_removal_is_reported_without_blocking_teardown() {
15722 let root = ScratchDir::new("supervisor-non-empty-cgroup");
15723 fs::write(root.join("cgroup.procs"), b"").expect("write scratch cgroup marker");
15724 let placement = subc_cgroup::prepare_at(&root)
15725 .expect("prepare scratch cgroup root")
15726 .expect("scratch root has a cgroup.procs marker");
15727 let module = placement
15728 .module_path("surviving-module")
15729 .expect("create scratch module cgroup");
15730 fs::write(module.join("surviving-process"), b"still present")
15731 .expect("make scratch cgroup non-empty");
15732 let (logs, _guard) = crate::router::test_log::log_capture(tracing::Level::WARN);
15733
15734 remove_module_cgroup(&placement, "surviving-module");
15735
15736 let logs = crate::router::test_log::captured_logs(&logs);
15737 assert!(
15738 module.exists(),
15739 "failed removal must leave the cgroup intact"
15740 );
15741 assert!(
15742 logs.contains("could not remove module cgroup after process exit; continuing teardown")
15743 && logs.contains("surviving-module"),
15744 "best-effort removal must report the failure without returning it: {logs}"
15745 );
15746 }
15747
15748 #[test]
15749 fn cgroup_pre_exec_spawn_failure_names_the_cgroup_path() {
15750 let cgroup_path = PathBuf::from("/sys/fs/cgroup/subc-modules/broken-module");
15751 let reason = SuperviseError::Spawn {
15752 program: PathBuf::from("/bin/true"),
15753 source: io::Error::from_raw_os_error(13),
15754 cgroup_path: Some(cgroup_path.clone()),
15755 }
15756 .to_string();
15757
15758 assert!(
15759 reason.contains(&cgroup_path.display().to_string()),
15760 "a pre_exec spawn failure must name the cgroup path: {reason}"
15761 );
15762 }
15763}
15764
15765#[cfg(test)]
15766mod spawn_subscriber_lag_tests {
15767 use super::*;
15768
15769 #[tokio::test]
15774 async fn lagged_spawn_subscriber_receives_a_terminal_lagged_error_after_its_queued_frames() {
15775 let feed = SpawnEventFeed::default();
15776 feed.configure_incarnation("lag-incarnation".to_string());
15777 let (tx, mut rx) = mpsc::channel(1);
15780 feed.subscribe(ConnectionId::new(1), 7, 1, None, FrameSink::new(tx))
15781 .expect("subscribe");
15782 let emitted = SPAWN_SUBSCRIBER_BUFFER + 16;
15783 for index in 0..emitted {
15784 feed.emit_spawned(&format!("lag-module-{index}"), 1000, 0);
15785 tokio::task::yield_now().await;
15788 }
15789 assert_eq!(
15790 feed.subscriber_count(),
15791 0,
15792 "the lagged subscriber must be removed"
15793 );
15794
15795 let mut data = Vec::new();
15796 let mut last = None;
15797 loop {
15798 let next = tokio::time::timeout(Duration::from_secs(5), rx.recv())
15799 .await
15800 .expect("the forwarder must finish once the subscriber is dropped");
15801 let Some(outbound) = next else { break };
15802 let frame = outbound.frame;
15803 if frame.header.ty == FrameType::StreamData {
15804 assert!(last.is_none(), "no data may follow the terminal frame");
15805 let event: SpawnEvent = serde_json::from_slice(&frame.body).unwrap();
15806 data.push(event.cursor.seq);
15807 } else {
15808 assert!(last.is_none(), "exactly one terminal frame");
15809 last = Some(frame);
15810 }
15811 }
15812 assert!(!data.is_empty(), "queued frames drain before the terminal");
15813 for pair in data.windows(2) {
15814 assert_eq!(
15815 pair[1],
15816 pair[0] + 1,
15817 "queued frames arrive dense and in order"
15818 );
15819 }
15820 let terminal = last.expect("a lagged subscriber must receive a terminal frame");
15821 assert_eq!(terminal.header.ty, FrameType::Error);
15822 assert_eq!(terminal.header.corr, 7);
15823 let body: subc_protocol::ErrorBody = serde_json::from_slice(&terminal.body).unwrap();
15824 assert_eq!(body.code, SPAWN_SUBSCRIBER_LAGGED_CODE);
15825 let detail = body.detail.expect("lagged error carries detail");
15826 assert_eq!(
15827 detail["first_undelivered_cursor"]["seq"],
15828 data.last().unwrap() + 1,
15829 "the named cursor is the first event the subscriber did not receive"
15830 );
15831 assert_eq!(
15832 detail["first_undelivered_cursor"]["daemon_incarnation"],
15833 "lag-incarnation"
15834 );
15835 }
15836}
15837
15838#[cfg(test)]
15839mod terminal_history_read_concurrency_tests {
15840 use super::*;
15841 use crate::terminal_journal::read_pause;
15842 use cortexkit_test_support::ScratchDir;
15843 use std::sync::mpsc as std_mpsc;
15844
15845 fn journaled_ring(
15846 journal: &Arc<crate::terminal_journal::TerminalJournal>,
15847 ) -> Arc<Mutex<TerminalRing>> {
15848 Arc::new(Mutex::new(
15849 TerminalRing::new(TerminalRingConfig::default(), 1)
15850 .with_journal(Some(Arc::clone(journal))),
15851 ))
15852 }
15853
15854 fn crash(at_ms: u64) -> ExitReport {
15855 ExitReport {
15856 kind: ExitKind::Crash,
15857 code: Some(1),
15858 signal: None,
15859 at_ms,
15860 }
15861 }
15862
15863 fn record_within(
15866 module_id: &'static str,
15867 ring: &Arc<Mutex<TerminalRing>>,
15868 at_ms: u64,
15869 bound: Duration,
15870 ) -> bool {
15871 let ring = Arc::clone(ring);
15872 let (done, done_rx) = std_mpsc::channel();
15873 std::thread::spawn(move || {
15874 record_terminal(
15875 module_id,
15876 &ring,
15877 &SpawnEventFeed::default(),
15878 &crash(at_ms),
15879 TerminalDisposition::Restarting,
15880 );
15881 let _ = done.send(());
15882 });
15883 done_rx.recv_timeout(bound).is_ok()
15884 }
15885
15886 #[test]
15891 fn exits_recorded_during_a_paused_history_read_are_not_blocked_or_half_merged() {
15892 let dir = ScratchDir::new("terminal-history-concurrent-read");
15893 let path = dir.join("terminals.jsonl");
15894 let journal = Arc::new(crate::terminal_journal::TerminalJournal::open(
15895 path.clone(),
15896 "daemon".into(),
15897 ));
15898 let reader_ring = journaled_ring(&journal);
15899 let other_ring = journaled_ring(&journal);
15900 assert!(record_within(
15901 "reader-module",
15902 &reader_ring,
15903 10,
15904 Duration::from_secs(5)
15905 ));
15906
15907 let (started, release) = read_pause::install(&path);
15908 let reading = {
15909 let ring = Arc::clone(&reader_ring);
15910 std::thread::spawn(move || durable_terminal_history_of(&ring, "reader-module"))
15911 };
15912 started
15913 .recv_timeout(Duration::from_secs(5))
15914 .expect("the history read reached its pause");
15915
15916 let bound = Duration::from_secs(1);
15917 assert!(
15918 record_within("other-module", &other_ring, 20, bound),
15919 "another module's exit waited on a history read (journal writer held)"
15920 );
15921 assert!(
15922 record_within("reader-module", &reader_ring, 30, bound),
15923 "the read module's own exit waited on its history read (ring held)"
15924 );
15925
15926 drop(release);
15927 let paused = reading.join().unwrap();
15928 assert_eq!(
15929 paused.entries.iter().map(|e| e.at_ms).collect::<Vec<_>>(),
15930 vec![10],
15931 "an exit recorded after the read began lands in neither half of it"
15932 );
15933 assert_eq!(paused.journal_skipped_lines, 0);
15934 assert_eq!(paused.journal_read_errors, 0);
15935
15936 let after = durable_terminal_history_of(&reader_ring, "reader-module");
15937 assert_eq!(
15938 after.entries.iter().map(|e| e.at_ms).collect::<Vec<_>>(),
15939 vec![10, 30],
15940 "the next read merges ring and journal with no duplicate"
15941 );
15942 assert_eq!(after.journal_skipped_lines, 0);
15943 }
15944}
15945
15946#[cfg(test)]
15951mod stderr_settle_tests {
15952 use std::{
15953 future::Future,
15954 io,
15955 pin::Pin,
15956 sync::{Arc, Mutex},
15957 task::{Context, Poll},
15958 time::Duration,
15959 };
15960
15961 use tokio::{
15962 io::{AsyncRead, ReadBuf},
15963 sync::oneshot,
15964 time::Instant,
15965 };
15966
15967 use super::{settle_stderr_pump, settle_stdout_pump, StderrPump};
15968 use crate::stderr_tail::{
15969 pump_stderr_to, pump_stdout_to, split_capture_stamp, untimed, CaptureState,
15970 ChildOutputSink, OutputSink, StderrRing, StderrTailConfig, TailEntry,
15971 };
15972
15973 const BOUND: Duration = Duration::from_millis(250);
15974
15975 struct HeldReader {
15979 before: Option<Vec<u8>>,
15980 gate: Option<oneshot::Receiver<()>>,
15981 after: io::Cursor<Vec<u8>>,
15982 }
15983
15984 impl AsyncRead for HeldReader {
15985 fn poll_read(
15986 mut self: Pin<&mut Self>,
15987 cx: &mut Context<'_>,
15988 buf: &mut ReadBuf<'_>,
15989 ) -> Poll<io::Result<()>> {
15990 if let Some(bytes) = self.before.take() {
15991 buf.put_slice(&bytes);
15992 return Poll::Ready(Ok(()));
15993 }
15994 if let Some(gate) = self.gate.as_mut() {
15995 match Pin::new(gate).poll(cx) {
15996 Poll::Pending => return Poll::Pending,
15997 Poll::Ready(_) => self.gate = None,
15998 }
15999 }
16000 Pin::new(&mut self.after).poll_read(cx, buf)
16001 }
16002 }
16003
16004 struct DiscardSink;
16005
16006 impl OutputSink for DiscardSink {
16007 fn write_line(&mut self, _line: &[u8]) {}
16008 }
16009
16010 fn line(text: &str) -> TailEntry {
16011 TailEntry::Line {
16012 text: text.to_string(),
16013 truncated: false,
16014 at_ms: None,
16015 }
16016 }
16017
16018 fn lock(ring: &Arc<Mutex<StderrRing>>) -> std::sync::MutexGuard<'_, StderrRing> {
16019 ring.lock().unwrap()
16020 }
16021
16022 fn held_pump(
16026 ring: &Arc<Mutex<StderrRing>>,
16027 before: &str,
16028 after: &str,
16029 ) -> (StderrPump, oneshot::Sender<()>) {
16030 let generation = lock(ring).begin_process();
16031 let (release, gate) = oneshot::channel();
16032 let reader = HeldReader {
16033 before: Some(before.as_bytes().to_vec()),
16034 gate: Some(gate),
16035 after: io::Cursor::new(after.as_bytes().to_vec()),
16036 };
16037 let task = tokio::spawn(pump_stderr_to(
16038 reader,
16039 Arc::clone(ring),
16040 generation,
16041 DiscardSink,
16042 ));
16043 (StderrPump { task, generation }, release)
16044 }
16045
16046 async fn wait_until(ring: &Arc<Mutex<StderrRing>>, done: impl Fn(&StderrRing) -> bool) {
16047 for _ in 0..1000 {
16048 if done(&lock(ring)) {
16049 return;
16050 }
16051 tokio::time::sleep(Duration::from_millis(1)).await;
16052 }
16053 panic!(
16054 "ring never reached the expected state: {:?}",
16055 lock(ring).snapshot(None, None)
16056 );
16057 }
16058
16059 #[tokio::test(start_paused = true)]
16060 async fn a_crash_line_the_reader_had_not_reached_by_the_bound_is_kept_before_the_restart() {
16061 let ring = Arc::new(Mutex::new(StderrRing::new(StderrTailConfig::default())));
16062 let (pump, release) = held_pump(&ring, "booting\n", "config error: missing storage\n");
16063
16064 settle_stderr_pump("crasher", &ring, pump, BOUND).await;
16065 let before_release = lock(&ring).snapshot(None, None);
16066 assert!(
16067 matches!(before_release.capture, CaptureState::Incomplete { .. }),
16068 "a reader that has not reached EOF cannot claim a whole tail: {before_release:?}"
16069 );
16070
16071 let next = lock(&ring).begin_process();
16074 lock(&ring).push_line_from(next, "next process booting");
16075 release.send(()).unwrap();
16076 wait_until(&ring, |ring| {
16077 ring.snapshot(None, None).capture == CaptureState::Captured
16078 })
16079 .await;
16080
16081 assert_eq!(
16082 untimed(lock(&ring).snapshot(None, None).entries),
16083 vec![
16084 line("booting"),
16085 line("config error: missing storage"),
16086 TailEntry::ProcessStart,
16087 line("next process booting"),
16088 ],
16089 "the crash's last line must survive a slow reader and stay in the crashed process's section"
16090 );
16091 }
16092
16093 #[tokio::test(start_paused = true)]
16094 async fn a_pipe_held_open_by_a_descendant_reads_incomplete_without_delaying_the_restart_past_the_bound(
16095 ) {
16096 let ring = Arc::new(Mutex::new(StderrRing::new(StderrTailConfig::default())));
16097 let (pump, _held) = held_pump(&ring, "parent exiting\n", "");
16100
16101 let started = Instant::now();
16102 settle_stderr_pump("orphaning", &ring, pump, BOUND).await;
16103 assert_eq!(
16104 started.elapsed(),
16105 BOUND,
16106 "the restart must wait exactly the bound for a pipe that stays open, no longer"
16107 );
16108
16109 let next = lock(&ring).begin_process();
16110 lock(&ring).push_line_from(next, "next process booting");
16111 tokio::time::sleep(Duration::from_secs(60)).await;
16112
16113 let snapshot = lock(&ring).snapshot(None, None);
16114 match &snapshot.capture {
16115 CaptureState::Incomplete { reason } => assert!(
16116 reason.contains("had not reached EOF") && reason.contains("250ms"),
16117 "the reason must say what is missing and after how long: {reason}"
16118 ),
16119 other => panic!("expected Incomplete while the pipe is held open, got {other:?}"),
16120 }
16121 assert_eq!(
16122 untimed(snapshot.entries),
16123 vec![
16124 line("parent exiting"),
16125 TailEntry::ProcessStart,
16126 line("next process booting"),
16127 ]
16128 );
16129 }
16130
16131 #[tokio::test(start_paused = true)]
16132 async fn a_reader_that_reaches_eof_within_the_bound_leaves_the_tail_captured() {
16133 let ring = Arc::new(Mutex::new(StderrRing::new(StderrTailConfig::default())));
16134 let (pump, release) = held_pump(&ring, "one\n", "two\n");
16135 release.send(()).unwrap();
16136
16137 settle_stderr_pump("clean", &ring, pump, BOUND).await;
16138
16139 let snapshot = lock(&ring).snapshot(None, None);
16140 assert_eq!(snapshot.capture, CaptureState::Captured);
16141 assert_eq!(untimed(snapshot.entries), vec![line("one"), line("two")]);
16142 }
16143
16144 #[tokio::test(start_paused = true)]
16145 async fn stdout_buffered_past_the_exit_drain_bound_reaches_the_capture_file() {
16146 const BURST: usize = 150;
16147 let capture = cortexkit_test_support::ScratchDir::new("stdout-exit-drain");
16148 let path = capture.join("two-pipe.stderr.log");
16149 let sink = ChildOutputSink::open(&path, cortexkit_log::Retention::default()).unwrap();
16150 let ring = Arc::new(Mutex::new(StderrRing::new(StderrTailConfig::default())));
16151 let generation = lock(&ring).begin_process();
16152 let stdout = (0..BURST)
16153 .map(|i| format!("out-{i:04}-{}\n", "o".repeat(64)))
16154 .collect::<String>();
16155 let stderr = (0..BURST)
16156 .map(|i| format!("err-{i:04}-{}\n", "e".repeat(64)))
16157 .collect::<String>();
16158 let first_chunk = stdout.find('\n').unwrap() + 1 + "out-0001-".len();
16163 let (release, gate) = oneshot::channel();
16164 let reader = HeldReader {
16165 before: Some(stdout.as_bytes()[..first_chunk].to_vec()),
16166 gate: Some(gate),
16167 after: io::Cursor::new(stdout.as_bytes()[first_chunk..stdout.len() - 1].to_vec()),
16168 };
16169 let (finished, completion) = oneshot::channel();
16170 let stdout_sink = sink.clone();
16171 let pump = tokio::spawn(async move {
16172 pump_stdout_to(reader, stdout_sink).await;
16173 finished.send(()).unwrap();
16174 });
16175 pump_stderr_to(io::Cursor::new(stderr.into_bytes()), ring, generation, sink).await;
16176
16177 let started = Instant::now();
16178 settle_stdout_pump("two-pipe", pump, BOUND).await;
16179 assert_eq!(
16180 started.elapsed(),
16181 BOUND,
16182 "a late reader must not hold up restart"
16183 );
16184 assert_eq!(
16185 std::fs::read_to_string(&path).unwrap().lines().count(),
16186 BURST + 1
16187 );
16188
16189 release
16190 .send(())
16191 .expect("stdout reader must survive the exit drain bound");
16192 tokio::time::timeout(Duration::from_secs(1), completion)
16193 .await
16194 .expect("released stdout reader must reach EOF")
16195 .expect("stdout capture must finish at EOF, not be cancelled");
16196 let contents = std::fs::read_to_string(&path).unwrap();
16197 let mut lines = contents
16198 .lines()
16199 .map(|line| split_capture_stamp(line).expect("capture stamp").1)
16200 .collect::<Vec<_>>();
16201 let mut expected = (0..BURST)
16202 .flat_map(|i| {
16203 [
16204 format!("out-{i:04}-{}", "o".repeat(64)),
16205 format!("err-{i:04}-{}", "e".repeat(64)),
16206 ]
16207 })
16208 .collect::<Vec<_>>();
16209 lines.sort_unstable();
16210 expected.sort_unstable();
16211 assert_eq!(
16212 lines, expected,
16213 "both pipes must retain every complete line exactly once"
16214 );
16215 }
16216}
16217
16218#[cfg(all(test, windows))]
16232mod job_containment_tests {
16233 use super::*;
16234 use cortexkit_test_support::ScratchDir;
16235 use std::{
16236 path::{Path, PathBuf},
16237 sync::{Arc, Mutex},
16238 time::{Duration, Instant},
16239 };
16240
16241 fn stub_path() -> PathBuf {
16247 let mut path = std::env::current_exe().expect("current_exe available in tests");
16248 path.pop();
16249 path.pop();
16250 path.push("fake-aft-stub.exe");
16251 assert!(
16252 path.exists(),
16253 "fake-aft-stub not built at {}: run `cargo test -p subc-core` (which builds \
16254 [[bin]] targets) rather than `cargo test -p subc-core --lib` (which does not)",
16255 path.display()
16256 );
16257 path
16258 }
16259
16260 fn read_grandchild_pid(path: &Path) -> u32 {
16262 let deadline = Instant::now() + Duration::from_secs(10);
16263 loop {
16264 if let Ok(contents) = std::fs::read_to_string(path) {
16265 if let Ok(pid) = contents.trim().parse() {
16266 return pid;
16267 }
16268 }
16269 assert!(
16270 Instant::now() < deadline,
16271 "the stub never recorded a grandchild pid at {}",
16272 path.display()
16273 );
16274 std::thread::sleep(Duration::from_millis(10));
16275 }
16276 }
16277
16278 struct Fixture {
16281 _dir: ScratchDir,
16282 module_id: String,
16283 grandchild: u32,
16284 child: Option<SupervisedChild>,
16285 registry: Arc<Registry>,
16286 snapshot: Arc<Mutex<SupervisorSnapshot>>,
16287 terminal_ring: Arc<Mutex<TerminalRing>>,
16288 spawn_events: SpawnEventFeed,
16289 }
16290
16291 fn fixture(label: &str, module_id: &str) -> Fixture {
16292 let dir = ScratchDir::new(label);
16293 let pid_file = dir.join("grandchild.pid");
16294 let supervisor = Supervisor::new_for_test(
16295 Arc::new(Registry::default()),
16296 RestartPolicy::new(3, Duration::ZERO),
16297 );
16298 let runtime = supervisor.runtime_config();
16299 let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::starting()));
16300 let spec = ModuleSpec {
16301 module_id: module_id.to_string(),
16302 program: stub_path(),
16303 args: Vec::new(),
16307 env: vec![
16308 ("FAKE_AFT_NEVER_CONNECT".to_string(), "1".to_string()),
16309 (
16310 "FAKE_AFT_GRANDCHILD_PID_FILE".to_string(),
16311 pid_file.display().to_string(),
16312 ),
16313 ],
16314 reserved: false,
16315 reserved_prefixes: Vec::new(),
16316 protocol: ModuleProtocol::Subc,
16317 overlap: Default::default(),
16318 };
16319 let child = spawn_and_mark_running(&spec, &runtime, &snapshot)
16320 .expect("spawn the supervised fixture");
16321 let grandchild = read_grandchild_pid(&pid_file);
16322 Fixture {
16323 _dir: dir,
16324 module_id: module_id.to_string(),
16325 grandchild,
16326 child: Some(child),
16327 registry: Arc::new(Registry::default()),
16328 snapshot,
16329 terminal_ring: Arc::clone(&runtime.terminal_ring),
16330 spawn_events: SpawnEventFeed::default(),
16331 }
16332 }
16333
16334 impl Fixture {
16335 async fn drain(&mut self) {
16337 let child = self
16338 .child
16339 .take()
16340 .expect("the fixture child is still present");
16341 drain_child_to_state(
16342 &self.module_id,
16343 ModuleProtocol::Subc,
16344 StopNotice::NotSent,
16347 &self.registry,
16348 None,
16349 &self.snapshot,
16350 &self.terminal_ring,
16351 &self.spawn_events,
16352 child,
16353 Duration::from_millis(500),
16354 ModuleState::Stopped,
16355 Some(false),
16356 )
16357 .await
16358 .expect("drain the supervised fixture");
16359 }
16360 }
16361
16362 #[tokio::test]
16368 async fn teardown_reaps_the_grandchild() {
16369 let mut fixture = fixture("teardown-grandchild", "tree-teardown");
16370 let grandchild = fixture.grandchild;
16371
16372 assert!(
16373 subc_jobobject::process_exists(grandchild),
16374 "grandchild {grandchild} must be alive before teardown, or this proves nothing"
16375 );
16376
16377 fixture.drain().await;
16378
16379 assert!(
16380 subc_jobobject::wait_for_process_exit(grandchild, Duration::from_secs(10)),
16381 "grandchild {grandchild} outlived module teardown: the tree was not contained"
16382 );
16383 }
16384
16385 #[test]
16398 fn an_uncontained_grandchild_survives_a_direct_child_kill() {
16399 let dir = ScratchDir::new("teardown-uncontained");
16400 let pid_file = dir.join("grandchild.pid");
16401 let mut child = std::process::Command::new(stub_path())
16402 .env("FAKE_AFT_NEVER_CONNECT", "1")
16403 .env(
16404 "FAKE_AFT_GRANDCHILD_PID_FILE",
16405 pid_file.display().to_string(),
16406 )
16407 .stdin(std::process::Stdio::null())
16408 .stdout(std::process::Stdio::null())
16409 .stderr(std::process::Stdio::null())
16410 .spawn()
16411 .expect("spawn the uncontained fixture");
16412 let grandchild = read_grandchild_pid(&pid_file);
16413
16414 child.kill().expect("kill the direct child");
16416 let _ = child.wait();
16417
16418 assert!(
16419 subc_jobobject::process_exists(grandchild),
16420 "grandchild {grandchild} died with the direct child, so this control no longer \
16421 distinguishes contained from uncontained teardown and the regression test is \
16422 passing vacuously"
16423 );
16424
16425 kill_tree(grandchild);
16428 }
16429
16430 #[tokio::test]
16439 async fn dropping_containment_reaps_the_grandchild() {
16440 let mut fixture = fixture("drop-containment", "tree-drop");
16441 let grandchild = fixture.grandchild;
16442
16443 assert!(subc_jobobject::process_exists(grandchild));
16444
16445 fixture.child.as_mut().expect("child present").job = None;
16447
16448 assert!(
16449 subc_jobobject::wait_for_process_exit(grandchild, Duration::from_secs(10)),
16450 "grandchild {grandchild} survived the containment handle closing, so a daemon \
16451 crash would leave the tree behind"
16452 );
16453 }
16454
16455 fn kill_tree(pid: u32) {
16457 let _ = std::process::Command::new("taskkill.exe")
16458 .args(["/PID", &pid.to_string(), "/T", "/F"])
16459 .stdin(std::process::Stdio::null())
16460 .stdout(std::process::Stdio::null())
16461 .stderr(std::process::Stdio::null())
16462 .status();
16463 assert!(
16464 subc_jobobject::wait_for_process_exit(pid, Duration::from_secs(10)),
16465 "could not clean up grandchild {pid}"
16466 );
16467 }
16468}
16469
16470#[cfg(test)]
16471mod privacy_trampoline_configuration_tests {
16472 #[tokio::test]
16473 #[cfg_attr(not(target_os = "macos"), ignore = "requires macOS privacy trampoline")]
16474 async fn macos_spawn_without_a_configured_trampoline_refuses_by_name() {
16475 #[cfg(target_os = "macos")]
16476 {
16477 let supervisor = super::Supervisor::new(
16478 std::sync::Arc::new(crate::Registry::default()),
16479 super::RestartPolicy::default(),
16480 );
16481 let error = supervisor.spawn(spec()).unwrap_err();
16482 assert!(
16483 error
16484 .to_string()
16485 .contains("no privacy trampoline configured"),
16486 "{error}"
16487 );
16488 }
16489 }
16490
16491 #[tokio::test]
16492 #[cfg_attr(not(target_os = "macos"), ignore = "requires macOS privacy trampoline")]
16493 async fn macos_wrong_trampoline_is_rejected_by_the_startup_probe() {
16494 #[cfg(target_os = "macos")]
16495 {
16496 let supervisor = super::Supervisor::new(
16497 std::sync::Arc::new(crate::Registry::default()),
16498 super::RestartPolicy::default(),
16499 )
16500 .with_privacy_trampoline(std::env::current_exe().unwrap());
16501 let error = supervisor.spawn(spec()).unwrap_err();
16502 assert!(
16503 error
16504 .to_string()
16505 .contains("binary does not implement the privacy trampoline protocol"),
16506 "{error}"
16507 );
16508 }
16509 }
16510
16511 #[cfg(target_os = "macos")]
16512 fn spec() -> super::ModuleSpec {
16513 super::ModuleSpec {
16514 module_id: "privacy-configuration".into(),
16515 program: "/bin/sleep".into(),
16516 args: vec!["30".into()],
16517 env: vec![],
16518 reserved: false,
16519 reserved_prefixes: vec![],
16520 protocol: subc_control::ModuleProtocol::None,
16521 overlap: super::ModuleOverlap::Exclusive,
16522 }
16523 }
16524}
16525
16526#[cfg(test)]
16527mod privacy_exec_boundary_tests {
16528 #[cfg(target_os = "macos")]
16529 use super::*;
16530 #[cfg(target_os = "macos")]
16531 use std::{
16532 io::{Read, Write},
16533 net::{TcpListener, TcpStream},
16534 };
16535
16536 #[cfg(target_os = "macos")]
16539 pub(super) fn before_image_sample(spec: &ModuleSpec, pid: u32) {
16540 if let Some((_, path)) = spec
16541 .env
16542 .iter()
16543 .find(|(key, _)| key == "SUBC_TEST_PRIVACY_SAMPLE_BARRIER")
16544 {
16545 let mut barrier = TcpStream::connect(path).unwrap();
16546 barrier
16547 .set_read_timeout(Some(Duration::from_secs(30)))
16548 .unwrap();
16549 barrier.write_all(&pid.to_ne_bytes()).unwrap();
16550 let mut release = [0];
16551 barrier.read_exact(&mut release).unwrap();
16552 assert_eq!(&release, b"X");
16553 }
16554 }
16555
16556 #[cfg(target_os = "macos")]
16557 fn spec(program: &str, args: &[&str]) -> ModuleSpec {
16558 ModuleSpec {
16559 module_id: "privacy-boundary".into(),
16560 program: program.into(),
16561 args: args.iter().map(|arg| (*arg).into()).collect(),
16562 env: vec![],
16563 reserved: false,
16564 reserved_prefixes: vec![],
16565 protocol: ModuleProtocol::None,
16566 overlap: ModuleOverlap::Exclusive,
16567 }
16568 }
16569
16570 #[cfg(target_os = "macos")]
16571 async fn accept(listener: TcpListener) -> TcpStream {
16572 let listener = tokio::net::TcpListener::from_std({
16574 listener.set_nonblocking(true).unwrap();
16575 listener
16576 })
16577 .unwrap();
16578 let (stream, _) = tokio::time::timeout(Duration::from_secs(30), listener.accept())
16579 .await
16580 .unwrap()
16581 .unwrap();
16582 let stream = stream.into_std().unwrap();
16583 stream.set_nonblocking(false).unwrap();
16584 stream
16585 .set_read_timeout(Some(Duration::from_secs(30)))
16586 .unwrap();
16587 stream
16588 }
16589
16590 #[tokio::test]
16591 #[cfg_attr(not(target_os = "macos"), ignore = "requires macOS privacy trampoline")]
16592 async fn macos_roster_withholds_a_nonnull_trampoline_image_until_exec_confirmation() {
16593 #[cfg(target_os = "macos")]
16594 {
16595 let root = cortexkit_test_support::ScratchDir::new("privacy-roster-barrier");
16596 let exec_listener = TcpListener::bind("127.0.0.1:0").unwrap();
16599 let sample_listener = TcpListener::bind("127.0.0.1:0").unwrap();
16600 let record = root.join("live-children.json");
16601 let supervisor =
16602 Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default())
16603 .with_live_children_record(&record);
16604 let runtime = supervisor.runtime_config();
16605 let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::starting()));
16606 let mut spec = spec("/bin/sleep", &["30"]);
16607 spec.env = vec![
16608 (
16609 "SUBC_TEST_PRIVACY_EXEC_BARRIER".into(),
16610 exec_listener.local_addr().unwrap().to_string(),
16611 ),
16612 (
16613 "SUBC_TEST_PRIVACY_SAMPLE_BARRIER".into(),
16614 sample_listener.local_addr().unwrap().to_string(),
16615 ),
16616 ];
16617 let spawn = tokio::task::spawn_blocking(move || {
16618 spawn_and_mark_running(&spec, &runtime, &snapshot).unwrap()
16619 });
16620 let mut sample = accept(sample_listener).await;
16621 let mut pid = [0; 4];
16622 sample.read_exact(&mut pid).unwrap();
16623 let pid = u32::from_ne_bytes(pid);
16624 let mut exec = accept(exec_listener).await;
16625 let mut ready = [0];
16626 exec.read_exact(&mut ready).unwrap();
16627 assert_eq!(&ready, b"R");
16628 let trampoline = subc_os::file_identity(&test_privacy_trampoline()).unwrap();
16631 assert_eq!(
16632 observe_spawned_image(pid).unwrap().executable,
16633 Some(trampoline)
16634 );
16635 sample.write_all(b"X").unwrap();
16636 let mut child = spawn.await.unwrap();
16637 let early = crate::live_children::read_record(&record).unwrap();
16638 assert_eq!(early.len(), 1);
16639 assert_eq!(early[0].pid, pid);
16640 assert_eq!(
16641 early[0].executable, None,
16642 "unconfirmed trampoline image entered the roster"
16643 );
16644 assert!(child.report_ready.get().is_none());
16645 child.privacy_exec.as_mut().unwrap().deadline =
16648 tokio::time::Instant::now() + Duration::from_secs(30);
16649 exec.write_all(b"X").unwrap();
16650 child.confirm_privacy_exec().await;
16651 assert_eq!(child.spawn_failure, None);
16652 assert!(child.report_ready.get().is_some());
16653 let confirmed = crate::live_children::read_record(&record).unwrap();
16654 let module = subc_os::file_identity(std::path::Path::new("/bin/sleep")).unwrap();
16655 assert_ne!(module, trampoline);
16656 assert_eq!(confirmed[0].executable, Some(module.into()));
16657 child.start_kill().unwrap();
16658 child.wait().await.unwrap();
16659 child.release_roster();
16660 }
16661 }
16662
16663 #[tokio::test]
16664 #[cfg_attr(not(target_os = "macos"), ignore = "requires macOS privacy trampoline")]
16665 async fn macos_already_exited_121_in_try_wait_is_a_module_exit_not_a_trampoline_refusal() {
16666 #[cfg(target_os = "macos")]
16667 {
16668 let registry = Arc::new(Registry::default());
16669 let policy = RestartPolicy::new(0, Duration::ZERO);
16670 let supervisor = Supervisor::new_for_test(Arc::clone(®istry), policy);
16671 let runtime = supervisor.runtime_config();
16672 let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::starting()));
16673 let spec = spec("/bin/sh", &["-c", "exit 121"]);
16674 let mut child = spawn_and_mark_running(&spec, &runtime, &snapshot).unwrap();
16678 let pid = child.pid;
16679 tokio::task::spawn_blocking(move || {
16680 subc_os::fork_exec_test::wait_for_child_exit_without_reaping(pid)
16681 })
16682 .await
16683 .unwrap()
16684 .unwrap();
16685 child.privacy_exec.as_mut().unwrap().deadline =
16686 tokio::time::Instant::now() + Duration::from_secs(30);
16687 let status = child.wait().await.unwrap();
16688 assert_eq!(status.code(), Some(121));
16689 assert!(child.privacy_exec.is_none());
16690 assert!(
16691 child.report_ready.get().is_none(),
16692 "an exited module must not publish a live pid"
16693 );
16694 let report = classify_reaped_child_exit(&snapshot, &child, &status);
16695 on_child_exit(
16696 &spec,
16697 policy,
16698 ®istry,
16699 &snapshot,
16700 &runtime.terminal_ring,
16701 &runtime.spawn_events,
16702 &runtime.child_roster,
16703 report,
16704 )
16705 .await;
16706 let state = lock_snapshot(&snapshot).unwrap();
16707 assert_eq!(state.state, ModuleState::Failed);
16708 assert_eq!(state.last_exit.as_ref().unwrap().code, Some(121));
16709 assert_eq!(state.reported_pid(), None);
16710 drop(state);
16711 let history = runtime.terminal_ring.lock().unwrap().snapshot();
16712 assert_eq!(history.entries.len(), 1);
16713 let terminal = &history.entries[0];
16714 assert_eq!(terminal.exit_code, Some(121));
16715 assert_eq!(terminal.exit_kind, subc_control::TerminalExitKind::Crash);
16716 assert_eq!(terminal.disposition, TerminalDisposition::Failed);
16717 assert_eq!(
16718 terminal.disposition_detail.as_deref(),
16719 Some(policy.budget_exhausted_detail().as_str()),
16720 "module exit 121 was classified as a trampoline refusal: {terminal:?}"
16721 );
16722 assert_eq!(child.spawn_failure, None);
16723 child.release_roster();
16724 }
16725 }
16726}
16727
16728#[cfg(all(test, unix))]
16732mod launch_nonce_descriptor_tests {
16733 use super::{ChildRoster, ModuleProtocol, ModuleSpec, SupervisorHandle};
16734 use crate::stderr_tail::{StderrRing, StderrTailConfig};
16735 use cortexkit_test_support::ScratchDir;
16736 use std::{
16737 path::PathBuf,
16738 sync::{Arc, Mutex},
16739 time::{Duration, Instant},
16740 };
16741
16742 async fn probe(role: super::SpawnRole) {
16743 let scratch = ScratchDir::new("launch-nonce-descriptor");
16744 let fd_copy = scratch.join("from-descriptor");
16745 let env_copy = scratch.join("environment");
16746 let script = format!(
16747 "env > '{env}.tmp' && mv '{env}.tmp' '{env}'; cat <&3 > '{fd}.tmp' && mv '{fd}.tmp' '{fd}'; sleep 30",
16748 fd = fd_copy.display(), env = env_copy.display(),
16749 );
16750 let xdg = |name: &str| (name.to_string(), scratch.join(name).display().to_string());
16751 let spec = ModuleSpec {
16752 module_id: "nonce-descriptor-probe".to_string(),
16753 program: PathBuf::from("/bin/sh"),
16754 args: vec!["-c".to_string(), script],
16755 env: vec![
16756 xdg("XDG_DATA_HOME"),
16757 xdg("XDG_RUNTIME_DIR"),
16758 xdg("XDG_CONFIG_HOME"),
16759 ("SUBC_LAUNCH_NONCE".to_string(), "must-not-leak".to_string()),
16760 ],
16761 reserved: true,
16762 reserved_prefixes: Vec::new(),
16763 protocol: ModuleProtocol::Subc,
16764 overlap: Default::default(),
16765 };
16766 let handle = SupervisorHandle::new();
16767 let ring = Arc::new(Mutex::new(StderrRing::new(StderrTailConfig::default())));
16768 let roster = ChildRoster::default();
16769 #[cfg(target_os = "macos")]
16770 {
16771 let path = super::test_privacy_trampoline();
16772 roster.set_privacy_trampoline(super::probe_privacy_trampoline(&path).map(|()| path));
16773 }
16774 let child = super::spawn_child_in_slot(
16775 &spec,
16776 None,
16777 Some(&handle),
16778 &ring,
16779 None,
16780 &roster,
16781 #[cfg(target_os = "linux")]
16782 None,
16783 role,
16784 matches!(role, super::SpawnRole::SwapCandidate),
16785 )
16786 .expect("spawn probe");
16787 let deadline = Instant::now() + Duration::from_secs(10);
16788 while !(fd_copy.exists() && env_copy.exists()) {
16789 assert!(Instant::now() < deadline, "probe never wrote its copies");
16790 tokio::time::sleep(Duration::from_millis(20)).await;
16791 }
16792 let nonce = std::fs::read_to_string(fd_copy).unwrap();
16793 assert!(!nonce.is_empty());
16794 let environment = std::fs::read_to_string(env_copy).unwrap();
16795 assert!(environment
16796 .lines()
16797 .any(|line| line.starts_with("SUBC_LAUNCH_NONCE_FD=3:")));
16798 let copy = environment
16799 .lines()
16800 .find_map(|line| line.strip_prefix("SUBC_LAUNCH_NONCE="));
16801 assert_eq!(
16802 copy, None,
16803 "Unix children must never receive the environment nonce"
16804 );
16805 if matches!(role, super::SpawnRole::Plain) {
16806 assert_eq!(
16807 handle.spawn_nonce(&spec.module_id).as_deref(),
16808 Some(nonce.as_str())
16809 );
16810 }
16811 drop(child);
16812 }
16813
16814 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
16815 async fn a_spawned_module_receives_its_nonce_only_on_descriptor_3() {
16816 probe(super::SpawnRole::Plain).await;
16817 }
16818
16819 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
16820 async fn swap_candidate_receives_its_nonce_only_on_descriptor_3() {
16821 probe(super::SpawnRole::SwapCandidate).await;
16822 }
16823}
16824
16825#[cfg(all(test, target_os = "linux"))]
16826mod cgroup_containment_tests {
16827 use super::*;
16828 use cortexkit_test_support::ScratchDir;
16829
16830 fn running(pid: u32) -> bool {
16831 std::fs::read_to_string(format!("/proc/{pid}/stat"))
16833 .ok()
16834 .and_then(|stat| {
16835 stat.rsplit_once(") ")
16836 .map(|(_, rest)| rest.starts_with('Z'))
16837 })
16838 .is_some_and(|zombie| !zombie)
16839 }
16840
16841 #[tokio::test]
16842 async fn linux_teardown_reaps_the_grandchild() {
16843 teardown_tree("linux_teardown_reaps_the_grandchild", false).await;
16844 }
16845
16846 #[tokio::test]
16847 async fn linux_shutdown_straggler_reaps_the_grandchild() {
16848 teardown_tree("linux_shutdown_straggler_reaps_the_grandchild", true).await;
16849 }
16850
16851 async fn teardown_tree(test_name: &str, shutdown: bool) {
16852 let dir = ScratchDir::new(test_name);
16853 let root = PathBuf::from(format!(
16854 "/sys/fs/cgroup/subc-tree-test-{test_name}-{}-{}",
16855 std::process::id(),
16856 unix_ms_now()
16857 ));
16858 if let Err(error) = std::fs::create_dir(&root) {
16859 assert!(
16860 std::env::var_os("SUBC_REQUIRE_CGROUP_TEST").is_none(),
16861 "required cgroup test cannot execute: {error}"
16862 );
16863 eprintln!(
16864 "SKIP {test_name}: no writable delegated cgroup at {}: {error}",
16865 root.display()
16866 );
16867 return;
16868 }
16869 let placement = subc_cgroup::prepare_at(&root)
16870 .expect("prepare isolated kernel cgroup")
16871 .expect("isolated cgroup is delegated");
16872 let module_id = "tree-teardown";
16873 let module = placement
16874 .module_path(module_id)
16875 .expect("create isolated module cgroup");
16876 if !module.join("cgroup.kill").exists() {
16877 std::fs::remove_dir(&module).unwrap();
16878 std::fs::remove_dir(root.join("subc-modules")).unwrap();
16879 std::fs::remove_dir(&root).unwrap();
16880 assert!(
16881 std::env::var_os("SUBC_REQUIRE_CGROUP_TEST").is_none(),
16882 "required cgroup.kill interface unavailable"
16883 );
16884 eprintln!("SKIP {test_name}: cgroup.kill unavailable (kernel < 5.14)");
16885 return;
16886 }
16887 let supervisor = Supervisor::new_for_test(
16888 Arc::new(Registry::default()),
16889 RestartPolicy::new(3, Duration::ZERO),
16890 )
16891 .with_cgroup_placement(Some(placement));
16892 let mut runtime = supervisor.runtime_config();
16893 runtime.child_roster = runtime
16894 .child_roster
16895 .for_module(Arc::new(Mutex::new(Duration::from_millis(100))));
16896 let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::starting()));
16897 let pid_file = dir.join("grandchild.pid");
16898 let spec = ModuleSpec {
16899 module_id: module_id.to_string(),
16900 program: PathBuf::from("/bin/sh"),
16901 args: vec![
16902 "-c".into(),
16903 "trap '' TERM; sleep 600 & echo $! > \"$1\"; wait".into(),
16904 "fixture".into(),
16905 pid_file.display().to_string(),
16906 ],
16907 env: ["XDG_DATA_HOME", "XDG_RUNTIME_DIR", "XDG_CONFIG_HOME"]
16908 .into_iter()
16909 .map(|key| (key.to_string(), dir.display().to_string()))
16910 .collect(),
16911 reserved: false,
16912 reserved_prefixes: Vec::new(),
16913 protocol: ModuleProtocol::None,
16914 overlap: Default::default(),
16915 };
16916 let child =
16917 spawn_and_mark_running(&spec, &runtime, &snapshot).expect("spawn supervised tree");
16918 let deadline = tokio::time::Instant::now() + Duration::from_secs(10);
16919 let grandchild: u32 = loop {
16920 if let Ok(contents) = std::fs::read_to_string(&pid_file) {
16921 if let Ok(pid) = contents.trim().parse() {
16922 break pid;
16923 }
16924 }
16925 assert!(
16926 tokio::time::Instant::now() < deadline,
16927 "grandchild pid was not recorded"
16928 );
16929 tokio::time::sleep(Duration::from_millis(10)).await;
16930 };
16931 assert!(
16932 running(grandchild),
16933 "grandchild must be alive before teardown"
16934 );
16935 if shutdown {
16936 let mut child = child;
16937 crate::child_roster::end_children_for_daemon_shutdown(
16938 &runtime.child_roster,
16939 false,
16940 std::future::pending(),
16941 )
16942 .await;
16943 child.wait().await.expect("reap shutdown straggler");
16944 } else {
16945 drain_child_to_state(
16946 module_id,
16947 ModuleProtocol::None,
16948 StopNotice::NotSent,
16949 &Registry::default(),
16950 None,
16951 &snapshot,
16952 &runtime.terminal_ring,
16953 &SpawnEventFeed::default(),
16954 child,
16955 Duration::from_millis(100),
16956 ModuleState::Stopped,
16957 Some(false),
16958 )
16959 .await
16960 .expect("real supervisor teardown");
16961 }
16962 let deadline = tokio::time::Instant::now() + Duration::from_secs(5);
16963 while running(grandchild) && tokio::time::Instant::now() < deadline {
16964 tokio::time::sleep(Duration::from_millis(10)).await;
16965 }
16966 let survived = running(grandchild);
16967 if survived {
16969 let pid = rustix::process::Pid::from_raw(grandchild as i32).unwrap();
16970 let _ = rustix::process::kill_process(pid, rustix::process::Signal::KILL);
16971 tokio::time::sleep(Duration::from_millis(100)).await;
16972 }
16973 if module.exists() {
16974 std::fs::remove_dir(&module).expect("remove empty module cgroup");
16975 }
16976 std::fs::remove_dir(root.join("subc-modules")).unwrap();
16977 std::fs::remove_dir(&root).unwrap();
16978 assert!(
16979 !survived,
16980 "grandchild {grandchild} outlived module teardown"
16981 );
16982 eprintln!("EXECUTED {test_name}: grandchild {grandchild} killed in isolated cgroup");
16983 }
16984}
16985
16986#[cfg(all(test, unix))]
16989mod record_write_stall_tests {
16990 use super::*;
16991 use std::sync::atomic::{AtomicBool, Ordering};
16992 use subc_transport::{authenticate_client, ConnectionInfo, Endpoint, SCHEMA_VERSION};
16993
16994 const MODULE_AUTH_DEADLINE: Duration = Duration::from_secs(2);
16997 const HELD_RECORD_WRITE: Duration = Duration::from_secs(3);
17001
17002 fn sleeping_spec() -> ModuleSpec {
17014 ModuleSpec {
17015 module_id: "record-write-stall".into(),
17016 program: "/bin/sleep".into(),
17017 args: vec!["600".into()],
17018 env: vec![],
17019 reserved: false,
17020 reserved_prefixes: vec![],
17021 protocol: ModuleProtocol::None,
17022 overlap: Default::default(),
17023 }
17024 }
17025
17026 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
17027 async fn module_auth_completes_while_the_restart_waits_on_the_record_write() {
17028 let home = cortexkit_test_support::ScratchDir::new("record-write-stall");
17029 let supervisor =
17030 Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default())
17031 .with_live_children_record(home.join("live-children.json"));
17032 let module = supervisor.spawn(sleeping_spec()).unwrap();
17033 let old_pid = tokio::time::timeout(Duration::from_secs(10), async {
17038 loop {
17039 if let Some(pid) = lock_snapshot(&module.inner.snapshot)
17040 .unwrap()
17041 .reported_pid()
17042 {
17043 return pid;
17044 }
17045 tokio::time::sleep(Duration::from_millis(10)).await;
17046 }
17047 })
17048 .await
17049 .expect("the first process was admitted as the module");
17050
17051 let key = vec![0x42; 32];
17054 let daemon_id = [0x24; 16];
17055 let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
17056 let port = listener.local_addr().unwrap().port();
17057 let server = tokio::spawn(crate::server::serve_listener(
17058 listener,
17059 Arc::new(crate::router::Router::with_default_self_handler()),
17060 crate::server::ServerAuth::with_limits(
17061 key.clone(),
17062 daemon_id,
17063 "test",
17064 crate::server::DEFAULT_AUTH_DEADLINE,
17065 4,
17066 ),
17067 ));
17068
17069 let (held_tx, held_rx) = std::sync::mpsc::channel();
17072 let armed = AtomicBool::new(true);
17073 supervisor
17074 .child_roster
17075 .set_record_write_hook_for_test(move |live| {
17076 let admitting_replacement = live.values().any(|entry| entry.pid != old_pid);
17077 if admitting_replacement && armed.swap(false, Ordering::SeqCst) {
17078 let _ = held_tx.send(());
17079 std::thread::sleep(HELD_RECORD_WRITE);
17080 }
17081 });
17082
17083 let module_process = std::thread::spawn(move || {
17086 held_rx
17087 .recv_timeout(Duration::from_secs(30))
17088 .expect("the restart reached the replacement's record write");
17089 let conn = ConnectionInfo {
17090 schema: SCHEMA_VERSION,
17091 wire_version: None,
17092 endpoints: vec![Endpoint {
17093 host: "127.0.0.1".to_owned(),
17094 port,
17095 }],
17096 key,
17097 daemon_id,
17098 pid: std::process::id(),
17099 daemon_ver: "test".to_owned(),
17100 };
17101 tokio::runtime::Builder::new_current_thread()
17102 .enable_all()
17103 .build()
17104 .unwrap()
17105 .block_on(async {
17106 let started = Instant::now();
17107 let mut stream = tokio::net::TcpStream::connect(("127.0.0.1", port))
17108 .await
17109 .unwrap();
17110 let result =
17111 authenticate_client(&mut stream, &conn, MODULE_AUTH_DEADLINE).await;
17112 (result, started.elapsed())
17113 })
17114 });
17115
17116 module.restart(Some(0)).await.unwrap();
17117 let (auth, elapsed) = tokio::task::spawn_blocking(move || module_process.join().unwrap())
17118 .await
17119 .unwrap();
17120 assert!(
17121 auth.is_ok(),
17122 "the daemon must answer a module's handshake while a restart waits on the \
17123 live-children record write; got {auth:?} after {elapsed:?}"
17124 );
17125
17126 tokio::time::timeout(Duration::from_secs(20), async {
17127 while lock_snapshot(&module.inner.snapshot)
17128 .unwrap()
17129 .reported_pid()
17130 .is_none_or(|pid| pid == old_pid)
17131 {
17132 tokio::time::sleep(Duration::from_millis(10)).await;
17133 }
17134 })
17135 .await
17136 .expect("the restart published its replacement");
17137 module.stop().await.unwrap();
17138 server.abort();
17139 }
17140}