1use std::{
2 collections::{HashMap, HashSet, VecDeque},
3 error::Error,
4 fmt, io,
5 path::PathBuf,
6 process::{ExitStatus, Stdio},
7 sync::{Arc, Mutex, OnceLock},
8 time::{Duration, SystemTime, UNIX_EPOCH},
9};
10
11use cortexkit_log::Retention;
12use serde_json::Value;
13use sha2::{Digest, Sha256};
14use subc_control::{
15 ClientControlPush, LiveSpawn, ModuleProtocol, RouteCloseReason, SpawnCursor, SpawnEvent,
16 SpawnEventKind, SpawnSnapshot, SupervisorHealthStatus, TerminalDisposition, TerminalExitKind,
17};
18use subc_protocol::{
19 manifest::{SelfSignalKind, SignalAnchor},
20 session::{
21 HealthReport, HealthStatus, ModuleControlCommand, ModuleControlRequest,
22 MODULE_CONTROL_OP_HEALTH_CHECK,
23 },
24 Flags, FrameType, Principal, Priority, SUBC_LAUNCH_NONCE_ENV, SUBC_MODULE_ID_ENV,
25};
26use tokio::{
27 process::{Child, Command},
28 sync::{mpsc, oneshot, watch, Mutex as AsyncMutex},
29 task::JoinHandle,
30 time::{sleep, sleep_until, timeout, timeout_at, Instant},
31};
32use tracing::{debug, error, info, warn};
33
34use crate::{
35 child_roster::ChildRoster,
36 daemon_config::{
37 CAPTURE_KEEP_ENV, CAPTURE_MAX_AGE_DAYS_ENV, CAPTURE_MAX_FILE_MB_ENV, CK_LOG_ENV,
38 },
39 forwarding::{
40 CloseReason, ForwardingError, ForwardingTable, GoodbyeTarget, ModuleControlRpcOutcome,
41 ModuleDrainTarget, PendingModuleControlRpc,
42 },
43 provenance::{ExecutableIdentityProbe, SpawnedFileIdentity},
44 registry::{ConnectionId, RegistrationEndReason, RegistryError},
45 stderr_tail::{
46 pump_stderr_to, pump_stdout_to, ChildOutputSink, StderrRing, StderrTailConfig,
47 StderrTailSnapshot,
48 },
49 terminal_ring::{TerminalHistorySnapshot, TerminalRecord, TerminalRing, TerminalRingConfig},
50 Frame, FrameSink, Registry,
51};
52
53#[cfg(not(windows))]
54use crate::provenance::spawned_file_identity;
55
56#[path = "supervise_swap.rs"]
57mod swap;
58
59#[path = "supervise_activity.rs"]
60mod activity;
61
62pub const SUBC_ARG: &str = "--subc";
68
69const DEFAULT_MAX_RESTARTS: u32 = 3;
70const DEFAULT_BACKOFF: Duration = Duration::from_millis(100);
71const DEFAULT_MAX_BACKOFF: Duration = Duration::from_secs(30);
72const DEFAULT_RESTART_WINDOW: Duration = Duration::from_secs(600);
76pub const DEFAULT_DRAIN_TIMEOUT: Duration = Duration::from_secs(30);
87const REGISTRY_RELEASE_TIMEOUT: Duration = Duration::from_secs(1);
88const REGISTRY_RELEASE_POLL: Duration = Duration::from_millis(10);
89const DRAIN_BUSY_REPROBE_INTERVAL: Duration = Duration::from_millis(250);
101const STDERR_PUMP_DRAIN_TIMEOUT: Duration = Duration::from_millis(250);
124pub const SPAWN_EVENT_RING_CAPACITY: usize = 4096;
126const SPAWN_SUBSCRIBER_BUFFER: usize = SPAWN_EVENT_RING_CAPACITY + 1;
127pub(crate) const SPAWN_SUBSCRIBER_LAGGED_CODE: &str = "spawn_subscriber_lagged";
132
133#[cfg(target_os = "macos")]
134struct PrivacyExec {
135 reader: tokio::io::unix::AsyncFd<std::io::PipeReader>,
136 deadline: tokio::time::Instant,
137 expected: Option<subc_os::FileIdentity>,
138 trampoline: Option<subc_os::FileIdentity>,
139 script: bool,
140 module_id: String,
141 #[cfg(test)]
142 confirmation_pause: Option<(String, Option<tokio::net::TcpStream>)>,
143}
144
145#[cfg(all(test, target_os = "macos"))]
146pub(crate) fn test_privacy_trampoline() -> PathBuf {
147 let path = std::env::current_exe()
150 .unwrap()
151 .parent()
152 .unwrap()
153 .parent()
154 .unwrap()
155 .join("privacy-trampoline-fixture");
156 assert!(
160 path.exists(),
161 "privacy-trampoline-fixture not built at {}: run `cargo build -p subc-daemon \
162 --bins --features test-support` or `cargo test -p subc-daemon` first",
163 path.display()
164 );
165 path
166}
167
168#[cfg(target_os = "macos")]
169fn probe_privacy_trampoline(path: &std::path::Path) -> Result<(), String> {
170 use std::io::Read;
171 let mut probe = std::process::Command::new(path)
172 .args(["__disclaim-exec", "--probe"])
173 .stdin(Stdio::null())
174 .stdout(Stdio::piped())
175 .stderr(Stdio::piped())
176 .spawn()
177 .map_err(|error| {
178 format!(
179 "privacy trampoline probe failed for {}: {error}",
180 path.display()
181 )
182 })?;
183 let deadline = std::time::Instant::now() + Duration::from_secs(5);
184 let status = loop {
185 match probe.try_wait() {
186 Ok(Some(status)) => break status,
187 Ok(None) if std::time::Instant::now() < deadline => {
188 std::thread::sleep(Duration::from_millis(5))
189 }
190 result => {
191 let _ = probe.kill();
192 let _ = probe.wait();
193 return Err(format!(
194 "privacy trampoline probe failed or timed out for {}: {result:?}",
195 path.display()
196 ));
197 }
198 }
199 };
200 let mut answer = String::new();
201 if let Some(stdout) = probe.stdout.take() {
202 let _ = stdout.take(256).read_to_string(&mut answer);
203 }
204 if status.success() && answer.trim() == subc_os::privacy_identity::TRAMPOLINE_PROBE {
205 return Ok(());
206 }
207 let mut diagnostic = String::new();
208 if let Some(stderr) = probe.stderr.take() {
209 let _ = stderr.take(1024).read_to_string(&mut diagnostic);
210 }
211 let cause = diagnostic
212 .trim()
213 .strip_prefix("ck-subc: own privacy identity refused: ")
214 .unwrap_or("binary does not implement the privacy trampoline protocol");
215 Err(format!(
216 "{cause}: probe of {} exited {status}",
217 path.display()
218 ))
219}
220
221#[cfg(target_os = "macos")]
222fn privacy_command(
223 spec: &ModuleSpec,
224 roster: &ChildRoster,
225) -> Result<
226 (
227 Command,
228 Option<PrivacyExec>,
229 subc_os::privacy_identity::ExecAcknowledgement,
230 ),
231 SuperviseError,
232> {
233 let failure = |cause: String| {
234 warn!(module_id = %spec.module_id, %cause, "privacy identity trampoline refused module spawn");
235 SuperviseError::Spawn {
236 program: spec.program.clone(),
237 source: io::Error::other(cause),
238 cgroup_path: None,
239 }
240 };
241 let trampoline = roster.privacy_trampoline().map_err(failure)?;
242 let program = if spec.program.components().count() == 1 && !spec.program.is_absolute() {
247 let path = spec
248 .env
249 .iter()
250 .find(|(key, _)| key == "PATH")
251 .map(|(_, value)| std::ffi::OsString::from(value))
252 .or_else(|| std::env::var_os("PATH"))
253 .unwrap_or_else(|| "/usr/bin:/bin".into());
254 std::env::split_paths(&path)
255 .map(|dir| dir.join(&spec.program))
256 .find(|path| path.is_file())
257 .unwrap_or_else(|| spec.program.clone())
258 } else {
259 spec.program.clone()
260 };
261 let expected = subc_os::file_identity(&program);
262 let trampoline_image = subc_os::file_identity(&trampoline);
263 let script = {
264 use std::io::Read;
265 let mut prefix = [0u8; 2];
266 std::fs::File::open(&program)
267 .is_ok_and(|mut file| file.read_exact(&mut prefix).is_ok() && &prefix == b"#!")
268 };
269 if expected.is_none() || expected == subc_os::file_identity(&trampoline) {
270 return Err(failure(
271 "privacy identity module executable is missing or is the trampoline itself".to_string(),
272 ));
273 }
274 let (reader, ack) = subc_os::privacy_identity::ExecAcknowledgement::pipe()
275 .map_err(|error| failure(error.to_string()))?;
276 let reader =
277 tokio::io::unix::AsyncFd::new(reader).map_err(|error| failure(error.to_string()))?;
278 let mut command = Command::new(&trampoline);
279 command
280 .arg("__disclaim-exec")
281 .arg(ack.fd().to_string())
282 .arg(&program);
283 ack.install(command.as_std_mut());
284 Ok((
285 command,
286 Some(PrivacyExec {
287 reader,
288 deadline: tokio::time::Instant::now() + Duration::from_secs(5),
289 expected,
290 trampoline: trampoline_image,
291 script,
292 module_id: spec.module_id.clone(),
293 #[cfg(test)]
294 confirmation_pause: spec
295 .env
296 .iter()
297 .find(|(key, _)| key == "SUBC_TEST_BEFORE_EXEC_CONFIRM")
298 .map(|(_, address)| (address.clone(), None)),
299 }),
300 ack,
301 ))
302}
303
304struct SupervisedChild {
305 child: Child,
306 exec: crate::activity::ExecOwner,
307 #[cfg(windows)]
308 nonce_delivery: Option<subc_os::LaunchNoncePipeDelivery>,
309 #[cfg(target_os = "macos")]
310 privacy_exec: Option<PrivacyExec>,
311 #[cfg(target_os = "macos")]
318 report_ready: Arc<OnceLock<()>>,
319 spawn_failure: Option<String>,
321 protocol: ModuleProtocol,
326 #[cfg(target_os = "linux")]
332 module_id: String,
333 #[cfg(target_os = "linux")]
334 cgroup_placement: Option<subc_cgroup::Placement>,
335 #[cfg(windows)]
357 job: Option<subc_jobobject::JobObject>,
358 stdout_pump: Option<JoinHandle<()>>,
359 stderr_pump: Option<StderrPump>,
360 stderr_ring: Arc<Mutex<StderrRing>>,
361 spawned_at: Instant,
364 spawned_at_ms: u64,
365 spawned_from: PathBuf,
366 launch_spec_digest: LaunchSpecDigest,
367 spawned_file_identity: Option<SpawnedFileIdentity>,
368 process_start_time: Option<u64>,
369 process_identity: Option<ProcessIdentity>,
370 pid: u32,
371 roster_guard: Option<crate::child_roster::RosterGuard>,
374}
375
376impl SupervisedChild {
377 fn id(&self) -> Option<u32> {
378 Some(self.pid)
379 }
380
381 fn process_identity(&self) -> Option<ProcessIdentity> {
382 self.process_identity
383 }
384
385 async fn wait(&mut self) -> io::Result<ExitStatus> {
386 #[cfg(target_os = "macos")]
387 self.confirm_privacy_exec().await;
388 let result = self.child.wait().await;
396 #[cfg(target_os = "linux")]
397 if result.is_ok() {
398 if let Some(placement) = self.cgroup_placement.as_ref() {
399 cleanup_reaped_cgroup(placement, &self.module_id).await;
400 self.cgroup_placement = None;
403 }
404 }
405 result
406 }
407
408 #[cfg(target_os = "macos")]
409 async fn confirm_privacy_exec(&mut self) {
410 let Some(pending) = &mut self.privacy_exec else {
411 return;
412 };
413 let result = tokio::time::timeout_at(pending.deadline, async {
414 let mut record = Vec::new();
415 loop {
416 let mut ready = pending.reader.readable().await?;
417 let read = ready.try_io(|reader| {
418 use std::io::Read;
419 let mut reader = reader.get_ref();
420 let mut buffer = [0u8; 256];
421 reader.read(&mut buffer).map(|count| (count, buffer))
422 });
423 match read {
424 Ok(Ok((0, _))) => return Ok::<_, io::Error>(record),
425 Ok(Ok((count, buffer))) => {
426 if record.len() + count > 1024 {
427 return Err(io::Error::other(
428 "privacy exec refusal record is too long",
429 ));
430 }
431 record.extend_from_slice(&buffer[..count]);
432 }
433 Ok(Err(error)) => return Err(error),
434 Err(_) => continue,
435 }
436 }
437 })
438 .await;
439 #[cfg(test)]
440 let result = if let Some((address, stream)) = &mut pending.confirmation_pause {
441 use tokio::io::{AsyncReadExt, AsyncWriteExt};
442 if stream.is_none() {
443 *stream = Some(
444 tokio::net::TcpStream::connect(address.as_str())
445 .await
446 .unwrap(),
447 );
448 stream.as_mut().unwrap().write_all(b"R").await.unwrap();
449 stream
450 .as_mut()
451 .unwrap()
452 .write_all(&self.pid.to_ne_bytes())
453 .await
454 .unwrap();
455 }
456 let mut reply = [0];
457 stream
458 .as_mut()
459 .unwrap()
460 .read_exact(&mut reply)
461 .await
462 .unwrap();
463 pending.confirmation_pause = None;
464 if &reply == b"F" {
465 Ok(Err(io::Error::other("injected late exec refusal")))
466 } else {
467 assert_eq!(&reply, b"X");
468 result
469 }
470 } else {
471 result
472 };
473 let pending = self.privacy_exec.as_ref().expect("pending exec");
476 let cause = match result {
477 Err(_) => Some("privacy identity trampoline did not exec within 5s".to_string()),
478 Ok(Err(error)) => Some(format!(
479 "privacy identity exec acknowledgement failed: {error}"
480 )),
481 Ok(Ok(record)) if !record.is_empty() => Some(
482 std::str::from_utf8(&record)
483 .ok()
484 .and_then(|record| {
485 record
486 .trim()
487 .strip_prefix(subc_os::privacy_identity::EXEC_REFUSAL_TAG)
488 })
489 .filter(|cause| !cause.is_empty())
490 .unwrap_or("invalid privacy exec refusal record")
491 .to_string(),
492 ),
493 Ok(Ok(_)) => match self.child.try_wait() {
494 Ok(Some(_status)) => None,
498 Err(error) => Some(format!("privacy identity trampoline wait failed: {error}")),
499 Ok(None) => {
500 let image = observe_spawned_image(self.pid);
501 if let Some(image) = image.filter(|image| {
502 image.executable.is_some()
503 && image.executable != pending.trampoline
504 && (image.executable == pending.expected || pending.script)
505 }) {
506 if let Err(error) = self
507 .roster_guard
508 .as_ref()
509 .map(|guard| guard.confirm_executable(image))
510 .transpose()
511 {
512 Some(format!(
513 "could not durably record confirmed module image: {error}"
514 ))
515 } else {
516 let _ = self.report_ready.set(());
517 info!(module_id = %pending.module_id, pid = self.pid,
518 "module spawned with own privacy identity (responsibility disclaimed)");
519 None
520 }
521 } else if image.is_none()
522 || image.is_some_and(|image| image.executable.is_none())
523 {
524 match tokio::time::timeout_at(pending.deadline, self.child.wait()).await {
531 Ok(Ok(_status)) => None,
532 Ok(Err(error)) => Some(format!("privacy identity trampoline wait failed: {error}")),
533 Err(_) => Some("privacy identity module executable remained unreadable until the 5s exec deadline".to_string()),
534 }
535 } else {
536 Some("privacy identity trampoline executable mismatch: expected the module program, not ck-subc".to_string())
537 }
538 }
539 },
540 };
541 let pending = self.privacy_exec.take().expect("pending exec");
542 self.exec.0.finish(if cause.is_some() {
545 crate::activity::ExecState::Refused
546 } else {
547 crate::activity::ExecState::CommittedExec
548 });
549 if let Some(cause) = cause {
550 warn!(module_id = %pending.module_id, pid = self.pid, %cause, "privacy identity trampoline refused module spawn");
551 self.spawn_failure = Some(cause);
552 if let Some(pid) = rustix::process::Pid::from_raw(self.pid as i32) {
555 let _ = rustix::process::kill_process_group(pid, rustix::process::Signal::KILL);
556 }
557 let _ = self.child.start_kill();
558 }
559 }
560
561 fn release_roster(&mut self) {
565 self.roster_guard = None;
566 }
567
568 fn start_kill(&mut self) -> io::Result<()> {
581 #[cfg(windows)]
582 if let Some(job) = &self.job {
583 if let Err(error) = job.terminate() {
584 debug!(
585 error = %error,
586 "job termination failed; the direct-child kill still owns the outcome"
587 );
588 }
589 }
590 #[cfg(target_os = "linux")]
591 kill_module_cgroup(self.cgroup_placement.as_ref(), &self.module_id);
592 self.child.start_kill()
593 }
594
595 async fn drain_stderr(&mut self, module_id: &str) {
596 if let Some(pump) = self.stdout_pump.take() {
597 settle_stdout_pump(module_id, pump, STDERR_PUMP_DRAIN_TIMEOUT).await;
598 }
599
600 let Some(pump) = self.stderr_pump.take() else {
601 return;
602 };
603 settle_stderr_pump(
604 module_id,
605 &self.stderr_ring,
606 pump,
607 STDERR_PUMP_DRAIN_TIMEOUT,
608 )
609 .await;
610 }
611}
612
613async fn settle_stdout_pump(module_id: &str, mut pump: JoinHandle<()>, bound: Duration) {
617 match timeout(bound, &mut pump).await {
618 Ok(Ok(())) => {}
619 Ok(Err(error)) => {
620 warn!(module_id, error = %error, "stdout pump ended unexpectedly");
621 }
622 Err(_) => {
623 drop(pump);
624 warn!(
625 module_id,
626 waited = ?bound,
627 "stdout pipe of the exited process has not drained; its reader keeps running without delaying the restart"
628 );
629 }
630 }
631}
632
633struct StderrPump {
636 task: JoinHandle<()>,
637 generation: u64,
638}
639
640async fn settle_stderr_pump(
646 module_id: &str,
647 ring: &Arc<Mutex<StderrRing>>,
648 pump: StderrPump,
649 bound: Duration,
650) {
651 let lock = || ring.lock().unwrap_or_else(|poisoned| poisoned.into_inner());
652 let StderrPump {
653 mut task,
654 generation,
655 } = pump;
656 lock().retire_pump(generation);
657 match timeout(bound, &mut task).await {
658 Ok(Ok(())) => {}
659 Ok(Err(err)) => {
660 let mut ring = lock();
661 ring.mark_incomplete_from(generation, format!("stderr pump ended unexpectedly: {err}"));
662 ring.finish_pump(generation);
663 warn!(module_id, error = %err, "stderr pump ended before clean EOF");
664 }
665 Err(_) => {
666 drop(task);
668 lock().mark_pump_late(
669 generation,
670 format!(
671 "stderr of the exited process had not reached EOF {bound:?} after it was \
672 retired (a descendant may still hold the pipe open); lines it still \
673 writes are kept in that process's section"
674 ),
675 );
676 warn!(
677 module_id,
678 waited = ?bound,
679 "stderr pipe of the exited process is still open; its reader keeps running without delaying the restart"
680 );
681 }
682 }
683}
684
685fn registration_release_events() -> &'static watch::Sender<u64> {
686 static EVENTS: OnceLock<watch::Sender<u64>> = OnceLock::new();
687 EVENTS.get_or_init(|| {
688 let (sender, _receiver) = watch::channel(0);
689 sender
690 })
691}
692
693pub(crate) fn notify_registration_release() {
694 let events = registration_release_events();
695 let next_generation = (*events.borrow()).wrapping_add(1);
696 events.send_replace(next_generation);
697}
698
699#[derive(Debug, Clone, PartialEq, Eq)]
701pub struct ModuleSpec {
702 pub module_id: String,
703 pub program: PathBuf,
704 pub args: Vec<String>,
705 pub env: Vec<(String, String)>,
706 pub reserved: bool,
711 pub reserved_prefixes: Vec<String>,
716 pub protocol: ModuleProtocol,
735 pub overlap: ModuleOverlap,
740}
741
742#[derive(Clone, PartialEq, Eq)]
745pub(crate) struct LaunchSpecDigest([u8; 32]);
746
747impl fmt::Debug for LaunchSpecDigest {
748 fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
749 f.write_str("LaunchSpecDigest([redacted])")
750 }
751}
752
753impl LaunchSpecDigest {
754 pub(crate) fn from_spec(spec: &ModuleSpec) -> Self {
755 fn field(hash: &mut Sha256, bytes: &[u8]) {
756 hash.update((bytes.len() as u64).to_le_bytes());
757 hash.update(bytes);
758 }
759 let mut hash = Sha256::new();
760 if let Some(program) = spec.program.to_str() {
761 field(&mut hash, b"utf8-program");
762 field(&mut hash, program.as_bytes());
763 } else {
764 #[cfg(unix)]
767 {
768 use std::os::unix::ffi::OsStrExt;
769 field(&mut hash, b"unix-program");
770 field(&mut hash, spec.program.as_os_str().as_bytes());
771 }
772 #[cfg(windows)]
773 {
774 use std::os::windows::ffi::OsStrExt;
775 field(&mut hash, b"windows-program");
776 let bytes = spec
777 .program
778 .as_os_str()
779 .encode_wide()
780 .flat_map(u16::to_le_bytes)
781 .collect::<Vec<_>>();
782 field(&mut hash, &bytes);
783 }
784 #[cfg(not(any(unix, windows)))]
785 field(&mut hash, spec.program.to_string_lossy().as_bytes());
786 }
787 hash.update((spec.args.len() as u64).to_le_bytes());
788 for arg in &spec.args {
789 field(&mut hash, arg.as_bytes());
790 }
791 let mut env = spec.env.iter().collect::<Vec<_>>();
792 env.sort_unstable();
793 hash.update((env.len() as u64).to_le_bytes());
794 for (key, value) in env {
795 field(&mut hash, key.as_bytes());
796 field(&mut hash, value.as_bytes());
797 }
798 field(
799 &mut hash,
800 match spec.protocol {
801 ModuleProtocol::Subc => b"subc",
802 ModuleProtocol::None => b"none",
803 },
804 );
805 Self(hash.finalize().into())
806 }
807}
808
809#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)]
816pub enum ModuleOverlap {
817 #[default]
819 Exclusive,
820 Safe,
832}
833
834impl ModuleOverlap {
835 pub fn as_str(self) -> &'static str {
836 match self {
837 Self::Exclusive => "exclusive",
838 Self::Safe => "safe",
839 }
840 }
841}
842
843pub const SUBC_SPAWN_ROLE_ENV: &str = "SUBC_SPAWN_ROLE";
853pub const SPAWN_ROLE_SWAP_CANDIDATE: &str = "swap_candidate";
855pub const DEFAULT_SWAP_READY_TIMEOUT: Duration = Duration::from_secs(100);
860
861#[derive(Debug, Clone, Copy, PartialEq, Eq)]
879pub struct RestartPolicy {
880 pub max_restarts: u32,
881 pub backoff: Duration,
884 pub max_backoff: Duration,
886 pub window: Duration,
890}
891
892impl RestartPolicy {
893 pub fn new(max_restarts: u32, backoff: Duration) -> Self {
897 Self {
898 max_restarts,
899 backoff,
900 max_backoff: DEFAULT_MAX_BACKOFF,
901 window: DEFAULT_RESTART_WINDOW,
902 }
903 }
904
905 pub fn with_max_backoff(mut self, max_backoff: Duration) -> Self {
906 self.max_backoff = max_backoff;
907 self
908 }
909
910 pub fn with_window(mut self, window: Duration) -> Self {
911 self.window = window;
912 self
913 }
914
915 fn delay_for_restart(&self, restart_in_window: u32) -> Duration {
920 if self.backoff.is_zero() || self.max_backoff.is_zero() {
921 return Duration::ZERO;
922 }
923
924 let mut delay = self.backoff;
925 for _ in 0..restart_in_window {
926 if delay >= self.max_backoff {
927 return self.max_backoff;
928 }
929 delay = delay
930 .checked_mul(10)
931 .unwrap_or(self.max_backoff)
932 .min(self.max_backoff);
933 }
934 delay.min(self.max_backoff)
935 }
936
937 fn budget_exhausted_detail(&self) -> String {
942 format!(
943 "crash budget exhausted: max_restarts={} within window_secs={}",
944 self.max_restarts,
945 self.window.as_secs()
946 )
947 }
948}
949
950impl Default for RestartPolicy {
951 fn default() -> Self {
952 Self {
953 max_restarts: DEFAULT_MAX_RESTARTS,
954 backoff: DEFAULT_BACKOFF,
955 max_backoff: DEFAULT_MAX_BACKOFF,
956 window: DEFAULT_RESTART_WINDOW,
957 }
958 }
959}
960
961#[derive(Debug, Clone, Copy, PartialEq, Eq)]
962struct CrashRestartSchedule {
963 restart_in_window: u32,
964 delay: Duration,
965}
966
967fn daemon_will_restart(
974 state: &mut SupervisorSnapshot,
975 policy: &RestartPolicy,
976 now: Instant,
977) -> bool {
978 state.enabled && state.crash_restarts_in_window(policy.window, now) < policy.max_restarts
979}
980
981const DEFAULT_HEALTH_CADENCE: Duration = Duration::from_secs(30);
982const DEFAULT_HEALTH_DEADLINE: Duration = Duration::from_secs(5);
983const DEFAULT_HEALTH_FAILURE_THRESHOLD: u32 = 3;
984const MAX_HEALTH_METRICS_BYTES: usize = 16 * 1024;
985
986#[derive(Debug, Clone, Copy, PartialEq, Eq)]
987pub enum HealthAction {
988 Report,
989 Restart,
990 Alert,
991}
992
993impl fmt::Display for HealthAction {
994 fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
995 f.write_str(match self {
996 Self::Report => "report",
997 Self::Restart => "restart",
998 Self::Alert => "alert",
999 })
1000 }
1001}
1002
1003#[derive(Debug, Clone, PartialEq, Eq)]
1004pub struct HealthConfig {
1005 pub http: Option<String>,
1009 pub cadence: Duration,
1010 pub deadline: Duration,
1011 pub failure_threshold: u32,
1012 pub on_degraded: HealthAction,
1013 pub on_failing: HealthAction,
1014 pub critical: bool,
1015}
1016
1017impl Default for HealthConfig {
1018 fn default() -> Self {
1019 Self {
1020 http: None,
1021 cadence: DEFAULT_HEALTH_CADENCE,
1022 deadline: DEFAULT_HEALTH_DEADLINE,
1023 failure_threshold: DEFAULT_HEALTH_FAILURE_THRESHOLD,
1024 on_degraded: HealthAction::Report,
1025 on_failing: HealthAction::Report,
1026 critical: false,
1027 }
1028 }
1029}
1030
1031#[derive(Debug, Clone, PartialEq)]
1049pub struct ModuleHealthStatus {
1050 pub status: SupervisorHealthStatus,
1051 pub last_probe_ms: Option<u64>,
1052 pub detail: Option<String>,
1053 pub metrics: Option<Value>,
1054 pub consecutive_failures: u32,
1055 pub late_answer_count: u64,
1058 pub last_late_answer_latency_ms: Option<u64>,
1060 pub last_action: Option<String>,
1061 pub last_action_ms: Option<u64>,
1065}
1066
1067impl Default for ModuleHealthStatus {
1068 fn default() -> Self {
1069 Self {
1070 status: SupervisorHealthStatus::Unknown,
1071 last_probe_ms: None,
1072 detail: None,
1073 metrics: None,
1074 consecutive_failures: 0,
1075 late_answer_count: 0,
1076 last_late_answer_latency_ms: None,
1077 last_action: None,
1078 last_action_ms: None,
1079 }
1080 }
1081}
1082
1083#[derive(Debug, Clone, Copy, PartialEq, Eq)]
1085pub enum ModuleState {
1086 Starting,
1087 Running,
1088 Unresponsive,
1089 Restarting,
1090 Draining,
1091 Stopped,
1092 Failed,
1093 Disabled,
1094}
1095
1096impl fmt::Display for ModuleState {
1097 fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
1098 f.write_str(match self {
1099 Self::Starting => "starting",
1100 Self::Running => "running",
1101 Self::Unresponsive => "unresponsive",
1102 Self::Restarting => "restarting",
1103 Self::Draining => "draining",
1104 Self::Stopped => "stopped",
1105 Self::Failed => "failed",
1106 Self::Disabled => "disabled",
1107 })
1108 }
1109}
1110
1111#[derive(Debug, Clone, Copy, PartialEq, Eq)]
1113pub enum ExitKind {
1114 Clean,
1115 Crash,
1116 DeliberateSeverance,
1117}
1118
1119impl From<ExitKind> for TerminalExitKind {
1120 fn from(kind: ExitKind) -> Self {
1121 match kind {
1122 ExitKind::Clean => Self::Clean,
1123 ExitKind::Crash => Self::Crash,
1124 ExitKind::DeliberateSeverance => Self::DeliberateSeverance,
1125 }
1126 }
1127}
1128
1129#[derive(Debug, Clone, Copy, PartialEq, Eq)]
1132pub(crate) struct ProcessIdentity {
1133 pub(crate) pid: u32,
1134 pub(crate) start_time: u64,
1135}
1136
1137#[derive(Debug, Clone, PartialEq, Eq)]
1139pub struct ExitReport {
1140 pub kind: ExitKind,
1141 pub code: Option<i32>,
1142 pub signal: Option<i32>,
1143 pub at_ms: u64,
1144}
1145
1146#[derive(Debug, Clone, PartialEq)]
1149pub struct ModuleStatus {
1150 pub module_id: String,
1151 pub waiting_for_previous_pid: Option<u32>,
1153 pub waiting_for_previous_detail: Option<String>,
1155 pub launch_nonce_source: Option<String>,
1160 pub state: ModuleState,
1161 pub enabled: bool,
1162 pub process_alive: bool,
1163 pub registration_active: bool,
1164 pub protocol: ModuleProtocol,
1169 pub live: bool,
1180 pub restart_count: u32,
1184 pub lifetime_restarts: u32,
1188 pub spawn_generation: u64,
1189 pub max_restarts: u32,
1194 pub restart_window: Duration,
1198 pub drain_timeout: Duration,
1202 pub restart_backoff: Duration,
1203 pub restart_max_backoff: Duration,
1204 pub pid: Option<u32>,
1209 pub spawned_at_ms: Option<u64>,
1210 pub spawned_from: Option<PathBuf>,
1211 pub(crate) launch_spec_digest: Option<LaunchSpecDigest>,
1212 pub process_start_time: Option<u64>,
1213 pub last_exit: Option<ExitReport>,
1214 pub health: ModuleHealthStatus,
1215}
1216
1217#[cfg(test)]
1218#[derive(Debug, Clone, Copy, PartialEq)]
1219struct ActorSelectCheckpoint {
1220 generation: u64,
1221 turn: u64,
1222 registered_connection: Option<ConnectionId>,
1223 next_probe_at: Option<Instant>,
1224 wake_after: Option<Duration>,
1225}
1226
1227#[derive(Debug, Clone, PartialEq)]
1228struct SupervisorSnapshot {
1229 exec_outcome: Option<Arc<crate::activity::ExecOutcome>>,
1230 #[cfg(windows)]
1231 pipe_consumed: Option<Arc<OnceLock<()>>>,
1232 #[cfg(test)]
1234 actor_turns: u64,
1235 #[cfg(test)]
1238 actor_select: Option<ActorSelectCheckpoint>,
1239 state: ModuleState,
1240 enabled: bool,
1241 process_alive: bool,
1242 spawned_protocol: Option<ModuleProtocol>,
1243 spawn_failure: Option<String>,
1244 waiting_for_previous_pid: Option<u32>,
1245 waiting_for_previous_detail: Option<String>,
1246 crash_restarts: VecDeque<Instant>,
1252 lifetime_restarts: u32,
1253 spawn_generation: u64,
1262 pid: Option<u32>,
1263 #[cfg(target_os = "macos")]
1264 report_ready: Option<Arc<OnceLock<()>>>,
1265 reaped_pid: Option<u32>,
1267 respawn_pending: bool,
1269 coalesced_restart_pending: bool,
1271 spawned_at_ms: Option<u64>,
1272 spawned_from: Option<PathBuf>,
1273 launch_spec_digest: Option<LaunchSpecDigest>,
1274 spawned_file_identity: Option<SpawnedFileIdentity>,
1275 process_start_time: Option<u64>,
1276 deliberate_severance: Option<ProcessIdentity>,
1277 last_exit: Option<ExitReport>,
1278 drain_disposition_detail: Option<String>,
1280 health: ModuleHealthStatus,
1281 in_alternate_slot: bool,
1286 draining_to_replace: bool,
1293 configuration_updated_since_spawn: bool,
1299}
1300
1301impl SupervisorSnapshot {
1302 fn reported_pid(&self) -> Option<u32> {
1307 #[cfg(target_os = "macos")]
1308 if self
1309 .report_ready
1310 .as_ref()
1311 .is_some_and(|ready| ready.get().is_none())
1312 {
1313 return None;
1314 }
1315 self.pid
1316 }
1317
1318 fn starting() -> Self {
1319 Self::new(ModuleState::Starting, true)
1320 }
1321
1322 fn disabled() -> Self {
1323 Self::new(ModuleState::Disabled, false)
1324 }
1325
1326 fn crash_restarts_in_window(&mut self, window: Duration, now: Instant) -> u32 {
1330 while let Some(oldest) = self.crash_restarts.front() {
1331 if now.duration_since(*oldest) > window {
1332 self.crash_restarts.pop_front();
1333 } else {
1334 break;
1335 }
1336 }
1337 u32::try_from(self.crash_restarts.len()).unwrap_or(u32::MAX)
1338 }
1339
1340 fn record_crash_restart(&mut self, policy: &RestartPolicy, now: Instant) {
1346 self.crash_restarts.push_back(now);
1347 while self.crash_restarts.len() > policy.max_restarts as usize {
1348 self.crash_restarts.pop_front();
1349 }
1350 self.lifetime_restarts += 1;
1351 }
1352
1353 fn next_crash_restart(
1357 &mut self,
1358 policy: &RestartPolicy,
1359 now: Instant,
1360 ) -> Option<CrashRestartSchedule> {
1361 let restart_in_window = self.crash_restarts_in_window(policy.window, now);
1362 if restart_in_window >= policy.max_restarts {
1363 return None;
1364 }
1365 self.record_crash_restart(policy, now);
1366 Some(CrashRestartSchedule {
1367 restart_in_window,
1368 delay: policy.delay_for_restart(restart_in_window),
1369 })
1370 }
1371
1372 fn clear_crash_restarts(&mut self) {
1377 self.crash_restarts.clear();
1378 }
1379
1380 fn new(state: ModuleState, enabled: bool) -> Self {
1381 Self {
1382 exec_outcome: None,
1383 #[cfg(test)]
1384 actor_turns: 0,
1385 #[cfg(test)]
1386 actor_select: None,
1387 state,
1388 enabled,
1389 process_alive: false,
1390 #[cfg(windows)]
1391 pipe_consumed: None,
1392 spawned_protocol: None,
1393 spawn_failure: None,
1394 waiting_for_previous_pid: None,
1395 waiting_for_previous_detail: None,
1396 crash_restarts: VecDeque::new(),
1397 lifetime_restarts: 0,
1398 spawn_generation: 0,
1399 pid: None,
1400 #[cfg(target_os = "macos")]
1401 report_ready: None,
1402 reaped_pid: None,
1403 respawn_pending: false,
1404 coalesced_restart_pending: false,
1405 spawned_at_ms: None,
1406 spawned_from: None,
1407 launch_spec_digest: None,
1408 spawned_file_identity: None,
1409 process_start_time: None,
1410 deliberate_severance: None,
1411 last_exit: None,
1412 drain_disposition_detail: None,
1413 health: ModuleHealthStatus::default(),
1414 in_alternate_slot: false,
1415 draining_to_replace: false,
1416 configuration_updated_since_spawn: false,
1417 }
1418 }
1419}
1420
1421type SharedSnapshot = Arc<Mutex<SupervisorSnapshot>>;
1422
1423type SpawnSubscriberKey = (ConnectionId, u64);
1424
1425#[derive(Debug)]
1426struct SpawnSubscriber {
1427 version: u8,
1428 frames: mpsc::Sender<Frame>,
1429 lagged: Option<oneshot::Sender<SpawnCursor>>,
1433}
1434
1435#[derive(Debug)]
1436struct SpawnEventState {
1437 daemon_incarnation: String,
1438 seq: u64,
1439 capacity: usize,
1440 live: HashMap<String, LiveSpawn>,
1441 generations: HashMap<String, u64>,
1442 events: VecDeque<SpawnEvent>,
1443 subscribers: HashMap<SpawnSubscriberKey, SpawnSubscriber>,
1444}
1445
1446impl Default for SpawnEventState {
1447 fn default() -> Self {
1448 Self {
1449 daemon_incarnation: "unconfigured".to_string(),
1450 seq: 0,
1451 capacity: SPAWN_EVENT_RING_CAPACITY,
1452 live: HashMap::new(),
1453 generations: HashMap::new(),
1454 events: VecDeque::new(),
1455 subscribers: HashMap::new(),
1456 }
1457 }
1458}
1459
1460#[derive(Debug, Clone, Default)]
1461struct SpawnEventFeed(
1462 Arc<Mutex<SpawnEventState>>,
1463 #[cfg(test)] Arc<Mutex<Option<std::net::TcpStream>>>,
1464);
1465
1466#[derive(Debug, Clone, PartialEq, Eq)]
1467pub(crate) enum SpawnSubscribeRefusal {
1468 ForeignIncarnation { current: String },
1469 TooOld { oldest: SpawnCursor },
1470 Frame(String),
1471}
1472
1473impl SpawnEventFeed {
1474 fn configure_incarnation(&self, daemon_incarnation: String) {
1475 let mut state = self.0.lock().unwrap_or_else(|p| p.into_inner());
1476 state.daemon_incarnation = daemon_incarnation;
1477 state.seq = 0;
1478 state.live.clear();
1479 state.generations.clear();
1480 state.events.clear();
1481 state.subscribers.clear();
1482 }
1483
1484 fn cursor(state: &SpawnEventState) -> SpawnCursor {
1485 SpawnCursor {
1486 daemon_incarnation: state.daemon_incarnation.clone(),
1487 seq: state.seq,
1488 }
1489 }
1490
1491 fn snapshot(&self) -> SpawnSnapshot {
1492 let state = self.0.lock().unwrap_or_else(|p| p.into_inner());
1493 let mut live = state.live.values().cloned().collect::<Vec<_>>();
1494 live.sort_by(|left, right| left.module_id.cmp(&right.module_id));
1495 SpawnSnapshot {
1496 cursor: Self::cursor(&state),
1497 ring_bound: state.capacity as u64,
1498 live,
1499 }
1500 }
1501
1502 fn emit_spawned(&self, module_id: &str, pid: u32, spawned_at_ms: u64) -> u64 {
1503 let mut state = self.0.lock().unwrap_or_else(|p| p.into_inner());
1504 let generation = state
1505 .generations
1506 .get(module_id)
1507 .copied()
1508 .unwrap_or(0)
1509 .checked_add(1)
1510 .expect("spawn generation exhausted");
1511 state.generations.insert(module_id.to_string(), generation);
1512 let live = LiveSpawn {
1513 module_id: module_id.to_string(),
1514 spawn_generation: generation,
1515 pid,
1516 spawned_at_ms,
1517 };
1518 state.live.insert(module_id.to_string(), live);
1519 Self::emit_locked(
1520 &mut state,
1521 SpawnEventKind::Spawned,
1522 module_id.to_string(),
1523 generation,
1524 pid,
1525 None,
1526 None,
1527 );
1528 generation
1529 }
1530
1531 fn emit_exited(&self, module_id: &str, exit_code: Option<i32>, exit_signal: Option<i32>) {
1532 let mut state = self.0.lock().unwrap_or_else(|p| p.into_inner());
1533 let Some(live) = state.live.remove(module_id) else {
1534 warn!(
1535 module_id,
1536 "terminal record had no live spawn event identity"
1537 );
1538 return;
1539 };
1540 Self::emit_locked(
1541 &mut state,
1542 SpawnEventKind::Exited,
1543 module_id.to_string(),
1544 live.spawn_generation,
1545 live.pid,
1546 exit_code,
1547 exit_signal,
1548 );
1549 }
1550
1551 fn emit_superseded_exited(
1558 &self,
1559 module_id: &str,
1560 spawn_generation: u64,
1561 pid: u32,
1562 exit_code: Option<i32>,
1563 exit_signal: Option<i32>,
1564 ) {
1565 let mut state = self.0.lock().unwrap_or_else(|p| p.into_inner());
1566 if state
1567 .live
1568 .get(module_id)
1569 .is_some_and(|live| live.spawn_generation == spawn_generation)
1570 {
1571 state.live.remove(module_id);
1572 }
1573 Self::emit_locked(
1574 &mut state,
1575 SpawnEventKind::Exited,
1576 module_id.to_string(),
1577 spawn_generation,
1578 pid,
1579 exit_code,
1580 exit_signal,
1581 );
1582 }
1583
1584 #[allow(clippy::too_many_arguments)]
1585 fn emit_locked(
1586 state: &mut SpawnEventState,
1587 kind: SpawnEventKind,
1588 module_id: String,
1589 spawn_generation: u64,
1590 pid: u32,
1591 exit_code: Option<i32>,
1592 exit_signal: Option<i32>,
1593 ) {
1594 state.seq = state
1595 .seq
1596 .checked_add(1)
1597 .expect("spawn event sequence exhausted");
1598 let event = SpawnEvent {
1599 cursor: Self::cursor(state),
1600 kind,
1601 module_id,
1602 spawn_generation,
1603 pid,
1604 exit_code,
1605 exit_signal,
1606 };
1607 state.events.push_back(event.clone());
1608 while state.events.len() > state.capacity {
1609 state.events.pop_front();
1610 }
1611 let body = match serde_json::to_vec(&event) {
1612 Ok(body) => body,
1613 Err(error) => {
1614 error!(%error, "failed to serialize supervisor spawn event");
1615 return;
1616 }
1617 };
1618 state.subscribers.retain(|(connection_id, corr), subscriber| {
1619 let frame = Frame::build_with_version(
1620 subscriber.version,
1621 FrameType::StreamData,
1622 control_flags(),
1623 0,
1624 0,
1625 *corr,
1626 body.clone(),
1627 );
1628 match frame {
1629 Ok(frame) => {
1630 if subscriber.frames.try_send(frame).is_ok() {
1631 true
1632 } else {
1633 warn!(connection_id = connection_id.get(), corr, "dropping lagged supervisor spawn subscriber");
1634 if let Some(lagged) = subscriber.lagged.take() {
1635 let _ = lagged.send(event.cursor.clone());
1636 }
1637 false
1638 }
1639 }
1640 Err(error) => {
1641 warn!(connection_id = connection_id.get(), corr, %error, "dropping supervisor spawn subscriber after frame build failure");
1642 false
1643 }
1644 }
1645 });
1646 }
1647
1648 fn subscribe(
1649 &self,
1650 connection_id: ConnectionId,
1651 corr: u64,
1652 version: u8,
1653 since: Option<SpawnCursor>,
1654 sink: FrameSink,
1655 ) -> Result<(), SpawnSubscribeRefusal> {
1656 let (frames, mut receiver) = mpsc::channel(SPAWN_SUBSCRIBER_BUFFER);
1657 let (lagged, mut lagged_rx) = oneshot::channel::<SpawnCursor>();
1658 {
1659 let mut state = self.0.lock().unwrap_or_else(|p| p.into_inner());
1660 let replay = if let Some(since) = since {
1661 if since.daemon_incarnation != state.daemon_incarnation {
1662 return Err(SpawnSubscribeRefusal::ForeignIncarnation {
1663 current: state.daemon_incarnation.clone(),
1664 });
1665 }
1666 if let Some(oldest) = state.events.front().map(|event| event.cursor.clone()) {
1667 if since.seq < oldest.seq.saturating_sub(1) {
1668 return Err(SpawnSubscribeRefusal::TooOld { oldest });
1669 }
1670 }
1671 state
1672 .events
1673 .iter()
1674 .filter(|event| event.cursor.seq > since.seq)
1675 .cloned()
1676 .collect::<Vec<_>>()
1677 } else {
1678 Vec::new()
1679 };
1680 for event in replay {
1681 let body = serde_json::to_vec(&event)
1682 .map_err(|error| SpawnSubscribeRefusal::Frame(error.to_string()))?;
1683 let frame = Frame::build_with_version(
1684 version,
1685 FrameType::StreamData,
1686 control_flags(),
1687 0,
1688 0,
1689 corr,
1690 body,
1691 )
1692 .map_err(|error| SpawnSubscribeRefusal::Frame(error.to_string()))?;
1693 frames
1694 .try_send(frame)
1695 .map_err(|error| SpawnSubscribeRefusal::Frame(error.to_string()))?;
1696 }
1697 state.subscribers.insert(
1698 (connection_id, corr),
1699 SpawnSubscriber {
1700 version,
1701 frames,
1702 lagged: Some(lagged),
1703 },
1704 );
1705 }
1706 tokio::spawn(async move {
1717 while let Some(frame) = receiver.recv().await {
1718 if sink.send(frame).await.is_err() {
1719 return;
1720 }
1721 }
1722 let Ok(first_undelivered) = lagged_rx.try_recv() else {
1723 return;
1724 };
1725 match spawn_subscriber_lagged_frame(version, corr, first_undelivered) {
1726 Ok(frame) => {
1727 let _ = sink.send(frame).await;
1728 }
1729 Err(error) => {
1730 error!(%error, corr, "failed to build lagged spawn subscriber terminal frame");
1731 }
1732 }
1733 });
1734 Ok(())
1735 }
1736
1737 fn cancel(&self, connection_id: ConnectionId, corr: u64) -> bool {
1738 let Some(subscriber) = self
1739 .0
1740 .lock()
1741 .unwrap_or_else(|p| p.into_inner())
1742 .subscribers
1743 .remove(&(connection_id, corr))
1744 else {
1745 return false;
1746 };
1747 if let Ok(frame) = Frame::build_with_version(
1748 subscriber.version,
1749 FrameType::StreamEnd,
1750 control_flags(),
1751 0,
1752 0,
1753 corr,
1754 Vec::new(),
1755 ) {
1756 tokio::spawn(async move {
1757 let _ = subscriber.frames.send(frame).await;
1758 });
1759 }
1760 true
1761 }
1762
1763 fn remove_connection(&self, connection_id: ConnectionId) {
1764 self.0
1765 .lock()
1766 .unwrap_or_else(|p| p.into_inner())
1767 .subscribers
1768 .retain(|(subscriber_connection, _), _| *subscriber_connection != connection_id);
1769 }
1770
1771 #[cfg(any(test, feature = "test-support"))]
1772 fn set_capacity(&self, capacity: usize) {
1773 self.0.lock().unwrap_or_else(|p| p.into_inner()).capacity = capacity;
1774 }
1775
1776 #[cfg(any(test, feature = "test-support"))]
1777 fn subscriber_count(&self) -> usize {
1778 self.0
1779 .lock()
1780 .unwrap_or_else(|p| p.into_inner())
1781 .subscribers
1782 .len()
1783 }
1784}
1785
1786fn spawn_subscriber_lagged_frame(
1789 version: u8,
1790 corr: u64,
1791 first_undelivered: SpawnCursor,
1792) -> Result<Frame, String> {
1793 let body = serde_json::to_vec(&subc_protocol::ErrorBody {
1794 code: SPAWN_SUBSCRIBER_LAGGED_CODE.to_string(),
1795 message: "spawn subscriber fell behind and was dropped; resubscribe from the last cursor received"
1796 .to_string(),
1797 detail: Some(serde_json::json!({
1798 "first_undelivered_cursor": first_undelivered
1799 })),
1800 })
1801 .map_err(|error| error.to_string())?;
1802 Frame::build_with_version(version, FrameType::Error, control_flags(), 0, 0, corr, body)
1803 .map_err(|error| error.to_string())
1804}
1805
1806pub trait ModuleProcessLiveness: Send + Sync {
1807 fn process_live(&self, module_id: &str) -> Option<bool>;
1808
1809 fn process_replacing(&self, _module_id: &str) -> bool {
1815 false
1816 }
1817}
1818
1819#[derive(Debug, Clone, Default)]
1821pub struct SupervisorProcessLiveness {
1822 snapshots: Arc<Mutex<HashMap<String, SharedSnapshot>>>,
1823}
1824
1825impl SupervisorProcessLiveness {
1826 pub fn new() -> Self {
1827 Self::default()
1828 }
1829
1830 fn track(&self, module_id: String, snapshot: SharedSnapshot) {
1831 let mut snapshots = self
1832 .snapshots
1833 .lock()
1834 .unwrap_or_else(|poisoned| poisoned.into_inner());
1835 snapshots.insert(module_id, snapshot);
1836 }
1837
1838 fn untrack_if_current(&self, module_id: &str, snapshot: &SharedSnapshot) {
1839 let mut snapshots = self
1840 .snapshots
1841 .lock()
1842 .unwrap_or_else(|poisoned| poisoned.into_inner());
1843 let is_current = snapshots
1844 .get(module_id)
1845 .map(|tracked| Arc::ptr_eq(tracked, snapshot))
1846 .unwrap_or(false);
1847 if is_current {
1848 snapshots.remove(module_id);
1849 }
1850 }
1851}
1852
1853impl ModuleProcessLiveness for SupervisorProcessLiveness {
1854 fn process_live(&self, module_id: &str) -> Option<bool> {
1855 let snapshot = {
1856 let snapshots = self
1857 .snapshots
1858 .lock()
1859 .unwrap_or_else(|poisoned| poisoned.into_inner());
1860 snapshots.get(module_id).cloned()
1861 }?;
1862 let snapshot = snapshot
1863 .lock()
1864 .unwrap_or_else(|poisoned| poisoned.into_inner());
1865 Some(snapshot.state == ModuleState::Running && snapshot.process_alive)
1866 }
1867
1868 fn process_replacing(&self, module_id: &str) -> bool {
1869 let Some(snapshot) = self
1870 .snapshots
1871 .lock()
1872 .unwrap_or_else(|poisoned| poisoned.into_inner())
1873 .get(module_id)
1874 .cloned()
1875 else {
1876 return false;
1877 };
1878 let snapshot = snapshot
1879 .lock()
1880 .unwrap_or_else(|poisoned| poisoned.into_inner());
1881 snapshot.enabled
1882 && match snapshot.state {
1883 ModuleState::Restarting => true,
1884 ModuleState::Draining => snapshot.draining_to_replace,
1885 ModuleState::Starting
1886 | ModuleState::Running
1887 | ModuleState::Unresponsive
1888 | ModuleState::Stopped
1889 | ModuleState::Failed
1890 | ModuleState::Disabled => false,
1891 }
1892 }
1893}
1894
1895#[cfg(test)]
1896#[derive(Debug, Default)]
1897struct ReloadExitRecordGate {
1898 reached: tokio::sync::Notify,
1899 resume: tokio::sync::Notify,
1900}
1901
1902#[derive(Debug, Clone, Copy)]
1903enum RespawnKind {
1904 Spawn,
1905 Reload,
1906}
1907
1908#[derive(Debug, Clone, Copy)]
1909struct PendingRespawn {
1910 deadline: Instant,
1911 kind: RespawnKind,
1912}
1913
1914type ReloadReply = oneshot::Sender<Result<(), SuperviseError>>;
1915
1916#[derive(Debug, Clone)]
1917struct SupervisorRuntimeConfig {
1918 start_requester: Option<Principal>,
1919 pending_activity_restart: bool,
1920 provenance_probe: ExecutableIdentityProbe,
1921 scheduled_respawn: Arc<Mutex<Option<PendingRespawn>>>,
1923 deferred_reload_reply: Arc<Mutex<Option<ReloadReply>>>,
1925 restart_policy: RestartPolicy,
1926 drain_timeout: Duration,
1929 effective_drain_timeout: Arc<Mutex<Duration>>,
1932 default_drain_timeout: Duration,
1935 health: HealthConfig,
1936 connection_file_path: Option<PathBuf>,
1937 capture_logs_dir: Option<PathBuf>,
1938 forwarding: Option<Arc<ForwardingTable>>,
1939 supervisor_handle: Option<SupervisorHandle>,
1942 stderr_ring: Arc<Mutex<StderrRing>>,
1949 terminal_ring: Arc<Mutex<TerminalRing>>,
1950 spawn_events: SpawnEventFeed,
1951 child_roster: ChildRoster,
1952 #[cfg(target_os = "linux")]
1953 cgroup_placement: Option<subc_cgroup::Placement>,
1954 #[cfg(test)]
1955 test_seed_stale_facts_before_enable_spawn: bool,
1956 #[cfg(test)]
1957 test_reload_exit_record_gate: Option<Arc<ReloadExitRecordGate>>,
1958}
1959
1960#[derive(Debug, Clone, PartialEq, Eq)]
1961struct SupervisedConfiguration {
1962 spec: ModuleSpec,
1963 health: HealthConfig,
1964}
1965
1966#[derive(Debug, Clone, Default)]
1972pub struct SupervisorHandle {
1973 activity: Arc<Mutex<Option<crate::activity::ActivityLedger>>>,
1974 activity_images: Arc<Mutex<HashMap<String, activity::ActivityImage>>>,
1975 modules: Arc<Mutex<HashMap<String, SupervisedModule>>>,
1976 configured_ids: Arc<Mutex<HashSet<String>>>,
1988 spawn_events: SpawnEventFeed,
1989 reserved_nonces: Arc<Mutex<HashMap<String, Option<String>>>>,
2000 removal_tombstones: Arc<Mutex<HashMap<String, u64>>>,
2006 spawn_nonces: Arc<Mutex<HashMap<String, String>>>,
2010 reserved_prefix_owners: Arc<Mutex<HashMap<String, String>>>,
2018 swaps: Arc<Mutex<HashMap<String, OpenSwap>>>,
2024 promotion_observer: PromotionObserverSlot,
2026 operation_lock: Arc<AsyncMutex<()>>,
2030}
2031
2032pub(crate) trait SwapPromotionObserver: Send + Sync {
2041 fn swap_promoted(&self, registration: &crate::registry::ModuleRegistration);
2042}
2043
2044#[derive(Clone, Default)]
2048struct PromotionObserverSlot(Arc<Mutex<Option<std::sync::Weak<dyn SwapPromotionObserver>>>>);
2049
2050impl fmt::Debug for PromotionObserverSlot {
2051 fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
2052 f.write_str("PromotionObserverSlot")
2053 }
2054}
2055
2056#[derive(Debug, Clone)]
2058struct OpenSwap {
2059 candidate_nonce: String,
2062 incumbent_nonce: Option<String>,
2067 candidate_admitted: bool,
2071}
2072
2073#[derive(Debug, Clone, Copy, PartialEq, Eq)]
2076pub(crate) enum SwapHelloAdmission {
2077 NotSwapping,
2080 Candidate,
2082 Refused,
2085}
2086
2087#[derive(Debug, Clone, PartialEq, Eq)]
2088pub(crate) enum ReservedHelloRejection {
2089 Exact {
2090 module_id: String,
2091 },
2092 Prefix {
2093 prefix: String,
2094 owner_module_id: String,
2095 },
2096}
2097
2098impl SupervisorHandle {
2099 pub fn new() -> Self {
2100 Self::default()
2101 }
2102
2103 pub(crate) fn spawn_snapshot(&self) -> SpawnSnapshot {
2104 self.spawn_events.snapshot()
2105 }
2106
2107 pub(crate) fn subscribe_spawns(
2108 &self,
2109 connection_id: ConnectionId,
2110 corr: u64,
2111 version: u8,
2112 since: Option<SpawnCursor>,
2113 sink: FrameSink,
2114 ) -> Result<(), SpawnSubscribeRefusal> {
2115 self.spawn_events
2116 .subscribe(connection_id, corr, version, since, sink)
2117 }
2118
2119 pub(crate) fn cancel_spawn_subscription(&self, connection_id: ConnectionId, corr: u64) -> bool {
2120 self.spawn_events.cancel(connection_id, corr)
2121 }
2122
2123 pub(crate) fn remove_spawn_subscribers(&self, connection_id: ConnectionId) {
2124 self.spawn_events.remove_connection(connection_id);
2125 }
2126
2127 #[cfg(any(test, feature = "test-support"))]
2128 pub fn set_spawn_event_capacity_for_test(&self, capacity: usize) {
2129 assert!(capacity > 0, "spawn event capacity must be non-zero");
2130 self.spawn_events.set_capacity(capacity);
2131 }
2132
2133 #[cfg(any(test, feature = "test-support"))]
2134 pub fn spawn_subscriber_count_for_test(&self) -> usize {
2135 self.spawn_events.subscriber_count()
2136 }
2137
2138 pub fn set_spawn_nonce(&self, module_id: &str, nonce: String) {
2141 self.spawn_nonces
2142 .lock()
2143 .unwrap_or_else(|poisoned| poisoned.into_inner())
2144 .insert(module_id.to_string(), nonce);
2145 }
2146
2147 pub fn set_reserved_nonce(&self, module_id: &str, nonce: String) {
2150 self.reserved_nonces
2151 .lock()
2152 .unwrap_or_else(|poisoned| poisoned.into_inner())
2153 .insert(module_id.to_string(), Some(nonce));
2154 }
2155
2156 pub fn set_reserved_prefixes(&self, owner_module_id: &str, prefixes: &[String]) {
2158 let mut owners = self
2159 .reserved_prefix_owners
2160 .lock()
2161 .unwrap_or_else(|poisoned| poisoned.into_inner());
2162 owners.retain(|_, owner| owner != owner_module_id);
2163 for prefix in prefixes {
2164 owners.insert(prefix.clone(), owner_module_id.to_string());
2165 }
2166 }
2167
2168 #[cfg(test)]
2170 pub(crate) fn spawn_nonce(&self, module_id: &str) -> Option<String> {
2171 self.spawn_nonces
2172 .lock()
2173 .unwrap_or_else(|poisoned| poisoned.into_inner())
2174 .get(module_id)
2175 .cloned()
2176 }
2177
2178 fn apply_identity_configuration(&self, spec: &ModuleSpec) {
2179 self.set_reserved_prefixes(&spec.module_id, &spec.reserved_prefixes);
2180 let spawn_nonce = self
2181 .spawn_nonces
2182 .lock()
2183 .unwrap_or_else(|poisoned| poisoned.into_inner())
2184 .get(&spec.module_id)
2185 .cloned();
2186 let mut reserved_nonces = self
2187 .reserved_nonces
2188 .lock()
2189 .unwrap_or_else(|poisoned| poisoned.into_inner());
2190 if spec.reserved {
2191 reserved_nonces.insert(spec.module_id.clone(), spawn_nonce);
2196 }
2197 drop(reserved_nonces);
2198 self.removal_tombstones
2202 .lock()
2203 .unwrap_or_else(|poisoned| poisoned.into_inner())
2204 .remove(&spec.module_id);
2205 }
2206
2207 pub fn reserved_hello_authorized(&self, module_id: &str, presented: Option<&str>) -> bool {
2212 self.reserved_hello_verification(module_id, presented)
2213 .is_ok()
2214 }
2215
2216 pub(crate) fn reserved_hello_verification(
2219 &self,
2220 module_id: &str,
2221 presented: Option<&str>,
2222 ) -> Result<bool, ReservedHelloRejection> {
2223 let nonces = self
2224 .reserved_nonces
2225 .lock()
2226 .unwrap_or_else(|poisoned| poisoned.into_inner());
2227 if let Some(expected) = nonces.get(module_id) {
2228 let authorized = match expected {
2232 Some(expected) => {
2233 presented.is_some_and(|p| constant_time_eq(expected.as_bytes(), p.as_bytes()))
2234 }
2235 None => false,
2236 };
2237 if authorized {
2238 return Ok(true);
2239 }
2240 return Err(ReservedHelloRejection::Exact {
2241 module_id: module_id.to_string(),
2242 });
2243 }
2244 drop(nonces);
2245
2246 let matched_prefix = self
2247 .reserved_prefix_owners
2248 .lock()
2249 .unwrap_or_else(|poisoned| poisoned.into_inner())
2250 .iter()
2251 .filter(|(prefix, _)| module_id.starts_with(prefix.as_str()))
2252 .max_by_key(|(prefix, _)| prefix.len())
2253 .map(|(prefix, owner)| (prefix.clone(), owner.clone()));
2254 let Some((prefix, owner_module_id)) = matched_prefix else {
2255 return Ok(false);
2256 };
2257
2258 let authorized = presented.is_some_and(|presented| {
2259 self.spawn_nonces
2260 .lock()
2261 .unwrap_or_else(|poisoned| poisoned.into_inner())
2262 .get(&owner_module_id)
2263 .is_some_and(|expected| constant_time_eq(expected.as_bytes(), presented.as_bytes()))
2264 || self.swap_nonce_matches(&owner_module_id, presented)
2267 });
2268 if authorized {
2269 Ok(true)
2270 } else {
2271 Err(ReservedHelloRejection::Prefix {
2272 prefix,
2273 owner_module_id,
2274 })
2275 }
2276 }
2277
2278 pub fn spawned_consumer_authorized(&self, module_id: &str, presented: &str) -> bool {
2283 if presented.is_empty() {
2284 return false;
2285 }
2286 let nonces = self
2287 .spawn_nonces
2288 .lock()
2289 .unwrap_or_else(|poisoned| poisoned.into_inner());
2290 let current = nonces
2291 .get(module_id)
2292 .is_some_and(|expected| constant_time_eq(expected.as_bytes(), presented.as_bytes()));
2293 drop(nonces);
2294 current || self.swap_nonce_matches(module_id, presented)
2299 }
2300
2301 fn swap_nonce_matches(&self, module_id: &str, presented: &str) -> bool {
2303 let swaps = self
2304 .swaps
2305 .lock()
2306 .unwrap_or_else(|poisoned| poisoned.into_inner());
2307 swaps.get(module_id).is_some_and(|swap| {
2308 constant_time_eq(swap.candidate_nonce.as_bytes(), presented.as_bytes())
2309 || swap.incumbent_nonce.as_deref().is_some_and(|incumbent| {
2310 constant_time_eq(incumbent.as_bytes(), presented.as_bytes())
2311 })
2312 })
2313 }
2314
2315 pub(crate) fn open_swap(&self, module_id: &str, candidate_nonce: String) {
2318 let incumbent_nonce = self
2319 .spawn_nonces
2320 .lock()
2321 .unwrap_or_else(|poisoned| poisoned.into_inner())
2322 .get(module_id)
2323 .cloned();
2324 self.swaps
2325 .lock()
2326 .unwrap_or_else(|poisoned| poisoned.into_inner())
2327 .insert(
2328 module_id.to_string(),
2329 OpenSwap {
2330 candidate_nonce,
2331 incumbent_nonce,
2332 candidate_admitted: false,
2333 },
2334 );
2335 }
2336
2337 pub(crate) fn close_swap(&self, module_id: &str) {
2340 self.swaps
2341 .lock()
2342 .unwrap_or_else(|poisoned| poisoned.into_inner())
2343 .remove(module_id);
2344 }
2345
2346 pub(crate) fn set_swap_promotion_observer(
2349 &self,
2350 observer: std::sync::Weak<dyn SwapPromotionObserver>,
2351 ) {
2352 *self
2353 .promotion_observer
2354 .0
2355 .lock()
2356 .unwrap_or_else(|poisoned| poisoned.into_inner()) = Some(observer);
2357 }
2358
2359 fn notify_swap_promoted(&self, registration: &crate::registry::ModuleRegistration) {
2362 let observer = self
2363 .promotion_observer
2364 .0
2365 .lock()
2366 .unwrap_or_else(|poisoned| poisoned.into_inner())
2367 .as_ref()
2368 .and_then(std::sync::Weak::upgrade);
2369 if let Some(observer) = observer {
2370 observer.swap_promoted(registration);
2371 }
2372 }
2373
2374 pub(crate) fn swap_open(&self, module_id: &str) -> bool {
2376 self.swaps
2377 .lock()
2378 .unwrap_or_else(|poisoned| poisoned.into_inner())
2379 .contains_key(module_id)
2380 }
2381
2382 fn promote_swap_nonce(&self, module_id: &str, reserved: bool) {
2387 let candidate_nonce = self
2388 .swaps
2389 .lock()
2390 .unwrap_or_else(|poisoned| poisoned.into_inner())
2391 .get(module_id)
2392 .map(|swap| swap.candidate_nonce.clone());
2393 let Some(nonce) = candidate_nonce else {
2394 return;
2395 };
2396 self.set_spawn_nonce(module_id, nonce.clone());
2397 if reserved {
2398 self.set_reserved_nonce(module_id, nonce);
2399 }
2400 }
2401
2402 pub(crate) fn swap_hello_admission(
2417 &self,
2418 module_id: &str,
2419 presented: Option<&str>,
2420 ) -> SwapHelloAdmission {
2421 let swaps = self
2422 .swaps
2423 .lock()
2424 .unwrap_or_else(|poisoned| poisoned.into_inner());
2425 let Some(swap) = swaps.get(module_id) else {
2426 return SwapHelloAdmission::NotSwapping;
2427 };
2428 let Some(presented) = presented else {
2429 return SwapHelloAdmission::Refused;
2430 };
2431 if constant_time_eq(swap.candidate_nonce.as_bytes(), presented.as_bytes()) {
2432 return if swap.candidate_admitted {
2433 SwapHelloAdmission::Refused
2434 } else {
2435 SwapHelloAdmission::Candidate
2436 };
2437 }
2438 if swap
2439 .incumbent_nonce
2440 .as_deref()
2441 .is_some_and(|incumbent| constant_time_eq(incumbent.as_bytes(), presented.as_bytes()))
2442 {
2443 return SwapHelloAdmission::NotSwapping;
2444 }
2445 SwapHelloAdmission::Refused
2446 }
2447
2448 pub(crate) fn mark_swap_candidate_admitted(&self, module_id: &str) {
2451 if let Some(swap) = self
2452 .swaps
2453 .lock()
2454 .unwrap_or_else(|poisoned| poisoned.into_inner())
2455 .get_mut(module_id)
2456 {
2457 swap.candidate_admitted = true;
2458 }
2459 }
2460
2461 pub fn spawn_launch_nonce_for(&self, module_id: &str) -> Option<String> {
2463 self.spawn_nonces
2464 .lock()
2465 .unwrap_or_else(|poisoned| poisoned.into_inner())
2466 .get(module_id)
2467 .cloned()
2468 }
2469
2470 pub fn reserved_launch_nonce_for(&self, module_id: &str) -> Option<String> {
2472 self.reserved_nonces
2473 .lock()
2474 .unwrap_or_else(|poisoned| poisoned.into_inner())
2475 .get(module_id)
2476 .cloned()
2477 .flatten()
2478 }
2479
2480 pub fn insert(&self, module: SupervisedModule) -> Option<SupervisedModule> {
2481 self.mark_configured(module.module_id());
2485 let mut modules = self
2486 .modules
2487 .lock()
2488 .unwrap_or_else(|poisoned| poisoned.into_inner());
2489 modules.insert(module.module_id().to_string(), module)
2490 }
2491
2492 fn mark_configured(&self, module_id: &str) {
2496 self.configured_ids
2497 .lock()
2498 .unwrap_or_else(|poisoned| poisoned.into_inner())
2499 .insert(module_id.to_string());
2500 }
2501
2502 fn unmark_configured_unless_rostered(&self, module_id: &str) {
2506 let modules = self
2507 .modules
2508 .lock()
2509 .unwrap_or_else(|poisoned| poisoned.into_inner());
2510 if !modules.contains_key(module_id) {
2511 self.configured_ids
2512 .lock()
2513 .unwrap_or_else(|poisoned| poisoned.into_inner())
2514 .remove(module_id);
2515 }
2516 }
2517
2518 pub(crate) fn is_configured(&self, module_id: &str) -> bool {
2526 self.configured_ids
2527 .lock()
2528 .unwrap_or_else(|poisoned| poisoned.into_inner())
2529 .contains(module_id)
2530 }
2531
2532 pub fn get(&self, module_id: &str) -> Option<SupervisedModule> {
2533 let modules = self
2534 .modules
2535 .lock()
2536 .unwrap_or_else(|poisoned| poisoned.into_inner());
2537 modules.get(module_id).cloned()
2538 }
2539
2540 pub(crate) fn record_late_health_answer(
2541 &self,
2542 module_id: &str,
2543 latency_ms: u64,
2544 ) -> Result<bool, SuperviseError> {
2545 let Some(module) = self.get(module_id) else {
2546 return Ok(false);
2547 };
2548 update_snapshot(&module.inner.snapshot, Some(module_id), |state| {
2549 state.health.late_answer_count = state.health.late_answer_count.saturating_add(1);
2550 state.health.last_late_answer_latency_ms = Some(latency_ms);
2551 state.health.consecutive_failures = 0;
2559 })?;
2560 Ok(true)
2561 }
2562
2563 pub fn record_deliberate_severance(&self, module_id: &str) -> Result<bool, SuperviseError> {
2569 let Some(module) = self.get(module_id) else {
2570 return Ok(false);
2571 };
2572 let snapshot = lock_snapshot(&module.inner.snapshot)?;
2573 let Some((pid, start_time)) = snapshot.pid.zip(snapshot.process_start_time) else {
2574 return Ok(false);
2575 };
2576 drop(snapshot);
2577 module.record_deliberate_severance(ProcessIdentity { pid, start_time })
2578 }
2579
2580 pub fn list(&self) -> Vec<SupervisedModule> {
2581 let modules = self
2582 .modules
2583 .lock()
2584 .unwrap_or_else(|poisoned| poisoned.into_inner());
2585 let mut modules = modules.values().cloned().collect::<Vec<_>>();
2586 modules.sort_by(|left, right| left.module_id().cmp(right.module_id()));
2587 modules
2588 }
2589
2590 pub(crate) fn retire(&self, module_id: &str) -> Option<SupervisedModule> {
2591 self.activity_images
2592 .lock()
2593 .unwrap_or_else(|p| p.into_inner())
2594 .remove(module_id);
2595 self.spawn_nonces
2596 .lock()
2597 .unwrap_or_else(|poisoned| poisoned.into_inner())
2598 .remove(module_id);
2599 self.close_swap(module_id);
2600 let mut reserved_nonces = self
2601 .reserved_nonces
2602 .lock()
2603 .unwrap_or_else(|poisoned| poisoned.into_inner());
2604 if reserved_nonces.contains_key(module_id) {
2605 reserved_nonces.insert(module_id.to_string(), None);
2608 }
2609 drop(reserved_nonces);
2610 self.reserved_prefix_owners
2611 .lock()
2612 .unwrap_or_else(|poisoned| poisoned.into_inner())
2613 .retain(|_, owner| owner != module_id);
2614 let removed = self
2615 .modules
2616 .lock()
2617 .unwrap_or_else(|poisoned| poisoned.into_inner())
2618 .remove(module_id);
2619 self.configured_ids
2620 .lock()
2621 .unwrap_or_else(|poisoned| poisoned.into_inner())
2622 .remove(module_id);
2623 removed
2624 }
2625
2626 pub(crate) fn record_rescan_removal(&self, module_id: &str) {
2629 self.removal_tombstones
2630 .lock()
2631 .unwrap_or_else(|poisoned| poisoned.into_inner())
2632 .insert(module_id.to_string(), unix_ms_now());
2633 }
2634
2635 pub(crate) fn removal_tombstone_age_ms(&self, module_id: &str) -> Option<u64> {
2637 self.removal_tombstones
2638 .lock()
2639 .unwrap_or_else(|poisoned| poisoned.into_inner())
2640 .get(module_id)
2641 .copied()
2642 .map(|removed_at_ms| unix_ms_now().saturating_sub(removed_at_ms))
2643 }
2644
2645 pub(crate) fn release_retained_reserved_gate(&self, module_id: &str) -> bool {
2650 if self.get(module_id).is_some() {
2651 return false;
2652 }
2653 let mut reserved_nonces = self
2654 .reserved_nonces
2655 .lock()
2656 .unwrap_or_else(|poisoned| poisoned.into_inner());
2657 if !matches!(reserved_nonces.get(module_id), Some(None)) {
2658 return false;
2659 }
2660 reserved_nonces.remove(module_id);
2661 true
2662 }
2663
2664 pub(crate) fn operation_lock(&self) -> Arc<AsyncMutex<()>> {
2665 Arc::clone(&self.operation_lock)
2666 }
2667}
2668
2669#[derive(Debug, Clone)]
2671pub struct Supervisor {
2672 registry: Arc<Registry>,
2673 restart_policy: RestartPolicy,
2674 drain_timeout: Duration,
2675 connection_file_path: Option<PathBuf>,
2676 capture_logs_dir: Option<PathBuf>,
2677 forwarding: Option<Arc<ForwardingTable>>,
2678 process_liveness: Arc<SupervisorProcessLiveness>,
2679 supervisor_handle: Option<SupervisorHandle>,
2680 health: HealthConfig,
2681 daemon_start_clock: crate::clock::StartClock,
2682 terminal_journal: Option<Arc<crate::terminal_journal::TerminalJournal>>,
2683 spawn_events: SpawnEventFeed,
2684 provenance_probe: ExecutableIdentityProbe,
2685 child_roster: ChildRoster,
2688 #[cfg(target_os = "linux")]
2689 cgroup_placement: Option<subc_cgroup::Placement>,
2690 #[cfg(test)]
2691 test_after_first_spawn: AfterFirstSpawnHook,
2692}
2693
2694pub(crate) struct RegisteredConfiguredModule {
2698 spec: ModuleSpec,
2699 runtime: SupervisorRuntimeConfig,
2700 module: SupervisedModule,
2701 commands: mpsc::Receiver<SupervisorCommand>,
2702}
2703
2704#[cfg(test)]
2710#[derive(Clone, Default)]
2711struct AfterFirstSpawnHook(Option<AfterFirstSpawnFn>);
2712
2713#[cfg(test)]
2714type AfterFirstSpawnFn = Arc<dyn Fn(&str) + Send + Sync>;
2715
2716#[cfg(test)]
2717impl fmt::Debug for AfterFirstSpawnHook {
2718 fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
2719 f.write_str("AfterFirstSpawnHook")
2720 }
2721}
2722
2723#[cfg(test)]
2724impl AfterFirstSpawnHook {
2725 fn run(&self, module_id: &str) {
2726 if let Some(hook) = &self.0 {
2727 hook(module_id);
2728 }
2729 }
2730}
2731
2732impl Supervisor {
2733 #[cfg(test)]
2734 pub(crate) fn new_for_test(registry: Arc<Registry>, policy: RestartPolicy) -> Self {
2735 let supervisor = Self::new(registry, policy);
2736 #[cfg(target_os = "macos")]
2737 let supervisor = supervisor.with_privacy_trampoline(test_privacy_trampoline());
2738 supervisor
2739 }
2740 pub fn with_privacy_trampoline(self, path: impl Into<PathBuf>) -> Self {
2745 let path = path.into();
2746 #[cfg(target_os = "macos")]
2747 {
2748 let result = probe_privacy_trampoline(&path).map(|()| path);
2749 if let Err(cause) = &result {
2750 error!(%cause, "macOS privacy identity unavailable; supervised launches will refuse");
2751 }
2752 self.child_roster.set_privacy_trampoline(result);
2753 }
2754 #[cfg(not(target_os = "macos"))]
2755 let _ = path;
2756 self
2757 }
2758 pub(crate) fn daemon_shutdown_flag(&self) -> crate::child_roster::DaemonShutdownFlag {
2761 self.child_roster.shutdown_flag()
2762 }
2763
2764 #[cfg(unix)]
2775 pub(crate) fn begin_daemon_shutdown(&self) {
2776 self.child_roster.close();
2777 if let Some(journal) = &self.terminal_journal {
2778 journal.stamp_shutdown();
2779 }
2780 }
2781
2782 #[cfg(unix)]
2786 pub(crate) async fn drain_for_daemon_shutdown(&self) -> Result<(), SuperviseError> {
2787 const NOTICE_BUDGET: Duration = Duration::from_millis(500);
2788 const DRAIN_BUDGET: Duration = Duration::from_secs(2);
2789 let Some(forwarding) = &self.forwarding else {
2790 return Ok(());
2791 };
2792 let modules = forwarding
2793 .begin_daemon_drain()
2794 .map_err(SuperviseError::Forwarding)?;
2795 let deadline_ms =
2796 unix_ms_now().saturating_add((NOTICE_BUDGET + DRAIN_BUDGET).as_millis() as u64);
2797 let mut notices = tokio::task::JoinSet::new();
2798 let mut drains = Vec::new();
2799 for (module_id, endpoint) in modules {
2800 let Some(target) = forwarding
2801 .begin_endpoint_drain(endpoint, RouteCloseReason::Restart)
2802 .map_err(SuperviseError::Forwarding)?
2803 else {
2804 continue;
2805 };
2806 let routes = forwarding
2807 .endpoint_routes(target.endpoint)
2808 .map_err(SuperviseError::Forwarding)?;
2809 let command = serde_json::to_vec(&ModuleControlCommand::Draining {
2815 reason: RouteCloseReason::Restart,
2816 deadline_ms,
2817 })
2818 .expect("module draining serializes");
2819 let mut recipients = vec![(target.sink.clone(), target.negotiated_ver, command)];
2820 let mut clients: Vec<(crate::forwarding::GoodbyeTarget, Vec<u16>)> = Vec::new();
2821 for route in routes {
2822 let client = route.goodbye_target;
2823 if let Some((_, channels)) = clients
2824 .iter_mut()
2825 .find(|(existing, _)| existing.connection_id == client.connection_id)
2826 {
2827 channels.push(client.channel);
2828 } else {
2829 let channel = client.channel;
2830 clients.push((client, vec![channel]));
2831 }
2832 }
2833 for (client, mut channels) in clients {
2834 channels.sort_unstable();
2835 channels.dedup();
2836 let closing = serde_json::to_vec(&ClientControlPush::RouteClosing {
2837 module_id: module_id.clone(),
2838 channels,
2839 reason: RouteCloseReason::Restart,
2840 })
2841 .expect("route closing serializes");
2842 recipients.push((client.sink, client.negotiated_ver, closing));
2843 }
2844 for (sink, version, body) in recipients {
2845 notices.spawn(async move {
2846 let frame = Frame::build_with_version(
2847 version,
2848 FrameType::Push,
2849 control_flags(),
2850 0,
2851 0,
2852 0,
2853 body,
2854 )
2855 .expect("bounded lifecycle notice frame builds");
2856 sink.send_flushed(frame).await
2857 });
2858 }
2859 let gauges = declared_busy_gauges(&self.registry, &module_id)?;
2860 drains.push((module_id, target.endpoint, gauges));
2861 }
2862 let notice_deadline = Instant::now() + NOTICE_BUDGET;
2865 while let Ok(Some(result)) = timeout_at(notice_deadline, notices.join_next()).await {
2866 if !matches!(result, Ok(Ok(()))) {
2867 warn!(?result, "daemon shutdown notice delivery failed");
2868 }
2869 }
2870 notices.abort_all();
2871 let deadline = Instant::now() + DRAIN_BUDGET;
2872 let mut waits = tokio::task::JoinSet::new();
2873 for (module_id, endpoint, gauges) in drains {
2874 let forwarding = Arc::clone(forwarding);
2875 let mut runtime = self.runtime_config();
2876 runtime.health.cadence = Duration::from_millis(100);
2877 waits.spawn(async move {
2878 let wait_result = wait_for_forwarding_quiescence(
2879 &forwarding,
2880 &module_id,
2881 &runtime,
2882 endpoint,
2883 deadline,
2884 &gauges,
2885 DrainScope::Endpoint(endpoint),
2886 &mut DrainGaugeTrace::new(&gauges),
2887 )
2888 .await;
2889 forwarding
2893 .record_daemon_drain_result(
2894 endpoint,
2895 drained_after_quiescence_wait(&wait_result),
2896 )
2897 .map_err(SuperviseError::Forwarding)?;
2898 wait_result
2899 });
2900 }
2901 while let Ok(Some(result)) = timeout_at(deadline, waits.join_next()).await {
2902 if !matches!(result, Ok(Ok(true))) {
2903 warn!(?result, "daemon shutdown drain did not reach quiescence");
2904 }
2905 }
2906 Ok(())
2907 }
2908
2909 #[cfg(unix)]
2921 pub(crate) async fn end_children_for_daemon_shutdown(
2922 &self,
2923 already_escalated: bool,
2924 escalate: impl std::future::Future<Output = ()>,
2925 ) {
2926 tokio::pin!(escalate);
2927 let mut escalated = already_escalated;
2928 if let Some(forwarding) = &self.forwarding {
2929 let reason = CloseReason::new(
2930 "daemon_shutdown",
2931 "the daemon is exiting after its shutdown notice and drain",
2932 );
2933 if escalated {
2934 send_module_goodbyes_for_daemon_shutdown(forwarding, &reason, false).await;
2937 } else {
2938 tokio::select! {
2939 biased;
2940 _ = escalate.as_mut() => {
2941 info!("second SIGTERM: abandoning module GOODBYE delivery");
2942 escalated = true;
2943 }
2944 _ = send_module_goodbyes_for_daemon_shutdown(forwarding, &reason, true) => {}
2945 }
2946 }
2947 let closed = forwarding.close_all_connections(&reason);
2948 debug!(closed, "closed established connections for daemon shutdown");
2949 }
2950 let escalated_here = escalated && !already_escalated;
2954 let remaining_escalate = async move {
2955 if escalated_here {
2956 std::future::pending::<()>().await;
2957 } else {
2958 escalate.await;
2959 }
2960 };
2961 crate::child_roster::end_children_for_daemon_shutdown(
2962 &self.child_roster,
2963 escalated,
2964 remaining_escalate,
2965 )
2966 .await;
2967 }
2968
2969 pub fn new(registry: Arc<Registry>, restart_policy: RestartPolicy) -> Self {
2970 Self {
2971 registry,
2972 restart_policy,
2973 drain_timeout: DEFAULT_DRAIN_TIMEOUT,
2974 connection_file_path: None,
2975 capture_logs_dir: None,
2976 forwarding: None,
2977 process_liveness: Arc::new(SupervisorProcessLiveness::default()),
2978 supervisor_handle: None,
2979 health: HealthConfig::default(),
2980 daemon_start_clock: crate::clock::StartClock::capture(),
2981 terminal_journal: None,
2982 spawn_events: SpawnEventFeed::default(),
2983 provenance_probe: ExecutableIdentityProbe::default(),
2984 child_roster: ChildRoster::default(),
2985 #[cfg(target_os = "linux")]
2986 cgroup_placement: None,
2987 #[cfg(test)]
2988 test_after_first_spawn: AfterFirstSpawnHook::default(),
2989 }
2990 }
2991
2992 pub fn with_drain_timeout(mut self, drain_timeout: Duration) -> Self {
2993 self.drain_timeout = drain_timeout;
2994 self
2995 }
2996
2997 pub(crate) fn default_drain_timeout(&self) -> Duration {
2999 self.drain_timeout
3000 }
3001
3002 pub fn with_process_liveness(
3003 mut self,
3004 process_liveness: Arc<SupervisorProcessLiveness>,
3005 ) -> Self {
3006 self.process_liveness = process_liveness;
3007 self
3008 }
3009
3010 pub fn with_connection_file_path(mut self, connection_file_path: impl Into<PathBuf>) -> Self {
3011 self.connection_file_path = Some(connection_file_path.into());
3012 self
3013 }
3014
3015 pub fn with_capture_logs_dir(mut self, logs_dir: impl Into<PathBuf>) -> Self {
3017 self.capture_logs_dir = Some(logs_dir.into());
3018 self
3019 }
3020
3021 pub fn with_daemon_incarnation(self, daemon_incarnation: String) -> Self {
3024 self.spawn_events.configure_incarnation(daemon_incarnation);
3028 self
3029 }
3030
3031 pub fn with_terminal_journal(self, path: PathBuf, daemon_incarnation: String) -> Self {
3034 let mut this = self.with_daemon_incarnation(daemon_incarnation.clone());
3035 this.terminal_journal = Some(Arc::new(crate::terminal_journal::TerminalJournal::open(
3036 path,
3037 daemon_incarnation,
3038 )));
3039 this
3040 }
3041
3042 pub fn with_forwarding(mut self, forwarding: Arc<ForwardingTable>) -> Self {
3043 self.forwarding = Some(forwarding);
3044 self
3045 }
3046
3047 pub fn with_handle(mut self, supervisor_handle: SupervisorHandle) -> Self {
3048 self.spawn_events = supervisor_handle.spawn_events.clone();
3049 self.supervisor_handle = Some(supervisor_handle);
3050 self
3051 }
3052
3053 pub fn with_health_config(mut self, health: HealthConfig) -> Self {
3054 self.health = health;
3055 self
3056 }
3057
3058 pub fn with_live_children_record(self, path: impl Into<PathBuf>) -> Self {
3062 self.child_roster.record_to(path.into());
3063 self
3064 }
3065
3066 #[cfg(target_os = "linux")]
3067 pub fn with_cgroup_placement(
3068 mut self,
3069 cgroup_placement: Option<subc_cgroup::Placement>,
3070 ) -> Self {
3071 self.cgroup_placement = cgroup_placement;
3072 self
3073 }
3074
3075 pub fn spawn(&self, spec: ModuleSpec) -> Result<SupervisedModule, SuperviseError> {
3081 validate_spec(&spec)?;
3082 self.establish_identity(&spec);
3083
3084 let runtime = self.runtime_config();
3085 let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::starting()));
3086 let activity_spawn = runtime.begin_activity_spawn(&spec.module_id);
3087 let spawned = spawn_child(
3088 &spec,
3089 runtime.connection_file_path.as_deref(),
3090 self.supervisor_handle.as_ref(),
3091 &runtime.stderr_ring,
3092 runtime.capture_logs_dir.as_deref(),
3093 &runtime.child_roster,
3094 #[cfg(target_os = "linux")]
3095 runtime.cgroup_placement.as_ref(),
3096 );
3097 #[cfg(test)]
3098 self.test_after_first_spawn.run(&spec.module_id);
3099 let child = match spawned {
3100 Ok(child) => child,
3101 Err(err) => {
3102 self.abandon_unrostered(&spec.module_id);
3105 return Err(err);
3106 }
3107 };
3108 self.mark_first_process_running(&spec, &runtime, &snapshot, &child)?;
3109 if let Some(spawn) = activity_spawn {
3110 spawn.complete(&snapshot);
3111 }
3112
3113 Ok(self.supervised_module(spec, runtime, snapshot, Some(child)))
3114 }
3115
3116 fn establish_identity(&self, spec: &ModuleSpec) {
3129 if let Some(supervisor_handle) = &self.supervisor_handle {
3130 supervisor_handle.apply_identity_configuration(spec);
3131 supervisor_handle.mark_configured(&spec.module_id);
3132 }
3133 }
3134
3135 fn abandon_unrostered(&self, module_id: &str) {
3138 if let Some(supervisor_handle) = &self.supervisor_handle {
3139 supervisor_handle.unmark_configured_unless_rostered(module_id);
3140 }
3141 }
3142
3143 fn mark_first_process_running(
3146 &self,
3147 spec: &ModuleSpec,
3148 runtime: &SupervisorRuntimeConfig,
3149 snapshot: &SharedSnapshot,
3150 child: &SupervisedChild,
3151 ) -> Result<(), SuperviseError> {
3152 if let Err(err) = set_running(snapshot, child, &spec.module_id, &runtime.spawn_events) {
3153 self.abandon_unrostered(&spec.module_id);
3154 return Err(err);
3155 }
3156 self.process_liveness
3157 .track(spec.module_id.clone(), Arc::clone(snapshot));
3158 Ok(())
3159 }
3160
3161 pub fn supervise_configured(
3167 &self,
3168 spec: ModuleSpec,
3169 enabled: bool,
3170 ) -> Result<SupervisedModule, SuperviseError> {
3171 self.supervise_configured_requested(
3172 spec,
3173 enabled,
3174 self.health.clone(),
3175 None,
3176 self.restart_policy,
3177 None,
3178 )
3179 }
3180
3181 pub fn supervise_configured_with_health(
3187 &self,
3188 spec: ModuleSpec,
3189 enabled: bool,
3190 health: HealthConfig,
3191 drain_timeout_ms: Option<u64>,
3192 restart_policy: RestartPolicy,
3193 ) -> Result<SupervisedModule, SuperviseError> {
3194 self.supervise_configured_requested(
3195 spec,
3196 enabled,
3197 health,
3198 drain_timeout_ms,
3199 restart_policy,
3200 None,
3201 )
3202 }
3203
3204 pub(crate) fn supervise_configured_requested(
3205 &self,
3206 spec: ModuleSpec,
3207 enabled: bool,
3208 health: HealthConfig,
3209 drain_timeout_ms: Option<u64>,
3210 restart_policy: RestartPolicy,
3211 requester: Option<Principal>,
3212 ) -> Result<SupervisedModule, SuperviseError> {
3213 let registered = self.register_configured_with_health(
3214 spec,
3215 enabled,
3216 health,
3217 drain_timeout_ms,
3218 restart_policy,
3219 )?;
3220 self.start_registered_configured(registered, requester)
3221 }
3222
3223 pub(crate) fn register_configured_with_health(
3228 &self,
3229 spec: ModuleSpec,
3230 enabled: bool,
3231 health: HealthConfig,
3232 drain_timeout_ms: Option<u64>,
3233 restart_policy: RestartPolicy,
3234 ) -> Result<RegisteredConfiguredModule, SuperviseError> {
3235 validate_spec(&spec)?;
3236 self.establish_identity(&spec);
3237
3238 let mut runtime = self.runtime_config();
3239 runtime.health = health;
3240 runtime.restart_policy = restart_policy;
3241 if let Some(ms) = drain_timeout_ms {
3242 runtime.drain_timeout = Duration::from_millis(ms);
3243 *runtime
3244 .effective_drain_timeout
3245 .lock()
3246 .unwrap_or_else(|poisoned| poisoned.into_inner()) = runtime.drain_timeout;
3247 }
3248 let snapshot = if enabled {
3249 SupervisorSnapshot::starting()
3250 } else {
3251 SupervisorSnapshot::disabled()
3252 };
3253 let snapshot = {
3254 let mut snapshot = snapshot;
3255 if enabled {
3256 if let Some((pid, reason)) = runtime.child_roster.previous_instance(&spec.module_id)
3257 {
3258 snapshot.waiting_for_previous_pid = Some(pid);
3259 snapshot.waiting_for_previous_detail = Some(reason.waiting_message(pid));
3260 }
3261 }
3262 snapshot
3263 };
3264 let (module, commands) =
3265 self.register_module(&spec, &runtime, Arc::new(Mutex::new(snapshot)));
3266 Ok(RegisteredConfiguredModule {
3267 spec,
3268 runtime,
3269 module,
3270 commands,
3271 })
3272 }
3273
3274 pub(crate) fn start_registered_configured(
3277 &self,
3278 registered: RegisteredConfiguredModule,
3279 requester: Option<Principal>,
3280 ) -> Result<SupervisedModule, SuperviseError> {
3281 let RegisteredConfiguredModule {
3282 spec,
3283 mut runtime,
3284 module,
3285 commands,
3286 } = registered;
3287 runtime.start_requester = requester;
3288 let snapshot = &module.inner.snapshot;
3289 let should_spawn = {
3290 let state = lock_snapshot(snapshot)?;
3291 state.enabled && state.waiting_for_previous_pid.is_none()
3292 };
3293 let child = if should_spawn {
3294 let activity_spawn = runtime.begin_activity_spawn(&spec.module_id);
3295 let spawned = spawn_child(
3296 &spec,
3297 runtime.connection_file_path.as_deref(),
3298 self.supervisor_handle.as_ref(),
3299 &runtime.stderr_ring,
3300 runtime.capture_logs_dir.as_deref(),
3301 &runtime.child_roster,
3302 #[cfg(target_os = "linux")]
3303 runtime.cgroup_placement.as_ref(),
3304 );
3305 #[cfg(test)]
3306 self.test_after_first_spawn.run(&spec.module_id);
3307 match spawned {
3308 Ok(child) => {
3309 self.mark_first_process_running(&spec, &runtime, snapshot, &child)?;
3310 if let Some(spawn) = activity_spawn {
3311 spawn.complete(snapshot);
3312 }
3313 Some(child)
3314 }
3315 Err(err) => {
3316 if runtime.health.critical {
3317 error!(
3318 module_id = %spec.module_id,
3319 program = %spec.program.display(),
3320 error = %err,
3321 "critical configured module failed to spawn; marking failed and alerting"
3322 );
3323 } else {
3324 error!(
3325 module_id = %spec.module_id,
3326 program = %spec.program.display(),
3327 error = %err,
3328 "configured module failed to spawn; marking failed and continuing"
3329 );
3330 }
3331 update_snapshot(snapshot, Some(&spec.module_id), |state| {
3332 state.state = ModuleState::Failed;
3333 clear_current_process_facts(state);
3334 })?;
3335 None
3336 }
3337 }
3338 } else {
3339 None
3340 };
3341 self.start_monitor(spec, runtime, &module, child, commands);
3342 Ok(module)
3343 }
3344
3345 fn runtime_config(&self) -> SupervisorRuntimeConfig {
3346 let effective_drain_timeout = Arc::new(Mutex::new(self.drain_timeout));
3347 SupervisorRuntimeConfig {
3348 start_requester: None,
3349 pending_activity_restart: false,
3350 provenance_probe: self.provenance_probe.clone(),
3351 scheduled_respawn: Arc::default(),
3352 deferred_reload_reply: Arc::default(),
3353 restart_policy: self.restart_policy,
3354 drain_timeout: self.drain_timeout,
3355 child_roster: self
3358 .child_roster
3359 .for_module(Arc::clone(&effective_drain_timeout)),
3360 effective_drain_timeout,
3361 default_drain_timeout: self.drain_timeout,
3362 health: self.health.clone(),
3363 connection_file_path: self.connection_file_path.clone(),
3364 capture_logs_dir: self.capture_logs_dir.clone(),
3365 forwarding: self.forwarding.clone(),
3366 supervisor_handle: self.supervisor_handle.clone(),
3367 stderr_ring: Arc::new(Mutex::new(StderrRing::new(StderrTailConfig::default()))),
3368 terminal_ring: Arc::new(Mutex::new(
3369 TerminalRing::new(
3370 TerminalRingConfig::default(),
3371 self.daemon_start_clock.started_at_ms(),
3372 )
3373 .with_start_clock(self.daemon_start_clock)
3374 .with_journal(self.terminal_journal.clone())
3375 .with_daemon_shutdown(self.child_roster.shutdown_flag()),
3376 )),
3377 spawn_events: self.spawn_events.clone(),
3378 #[cfg(target_os = "linux")]
3379 cgroup_placement: self.cgroup_placement.clone(),
3380 #[cfg(test)]
3381 test_seed_stale_facts_before_enable_spawn: false,
3382 #[cfg(test)]
3383 test_reload_exit_record_gate: None,
3384 }
3385 }
3386
3387 fn supervised_module(
3388 &self,
3389 spec: ModuleSpec,
3390 runtime: SupervisorRuntimeConfig,
3391 snapshot: SharedSnapshot,
3392 child: Option<SupervisedChild>,
3393 ) -> SupervisedModule {
3394 let (module, commands) = self.register_module(&spec, &runtime, snapshot);
3395 self.start_monitor(spec, runtime, &module, child, commands);
3396 module
3397 }
3398
3399 fn register_module(
3400 &self,
3401 spec: &ModuleSpec,
3402 runtime: &SupervisorRuntimeConfig,
3403 snapshot: SharedSnapshot,
3404 ) -> (SupervisedModule, mpsc::Receiver<SupervisorCommand>) {
3405 let configuration = Arc::new(Mutex::new(SupervisedConfiguration {
3406 spec: spec.clone(),
3407 health: runtime.health.clone(),
3408 }));
3409 let stderr_ring = Arc::clone(&runtime.stderr_ring);
3410 let terminal_ring = Arc::clone(&runtime.terminal_ring);
3411 let restart_policy = runtime.restart_policy;
3415 let effective_drain_timeout = Arc::clone(&runtime.effective_drain_timeout);
3416 let (tx, rx) = mpsc::channel(4);
3417 let module_id = spec.module_id.clone();
3418 let module = SupervisedModule {
3419 inner: Arc::new(SupervisedModuleInner {
3420 module_id: module_id.clone(),
3421 registry: Arc::clone(&self.registry),
3422 snapshot,
3423 configuration,
3424 stderr_ring,
3425 terminal_ring,
3426 commands: tx,
3427 monitor: Mutex::new(None),
3428 restart_policy,
3429 effective_drain_timeout,
3430 provenance_probe: self.provenance_probe.clone(),
3431 }),
3432 };
3433 if let Some(supervisor_handle) = &self.supervisor_handle {
3436 supervisor_handle.insert(module.clone());
3437 }
3438 (module, rx)
3439 }
3440
3441 fn start_monitor(
3442 &self,
3443 spec: ModuleSpec,
3444 runtime: SupervisorRuntimeConfig,
3445 module: &SupervisedModule,
3446 child: Option<SupervisedChild>,
3447 commands: mpsc::Receiver<SupervisorCommand>,
3448 ) {
3449 let monitor = tokio::spawn(supervise_loop(
3450 spec,
3451 runtime,
3452 Arc::clone(&self.registry),
3453 Arc::clone(&self.process_liveness),
3454 Arc::clone(&module.inner.snapshot),
3455 child,
3456 commands,
3457 ));
3458 *module
3459 .inner
3460 .monitor
3461 .lock()
3462 .unwrap_or_else(|p| p.into_inner()) = Some(monitor);
3463 }
3464}
3465
3466impl Default for Supervisor {
3467 fn default() -> Self {
3468 Self::new(Arc::new(Registry::default()), RestartPolicy::default())
3469 }
3470}
3471
3472#[derive(Clone)]
3474pub struct SupervisedModule {
3475 inner: Arc<SupervisedModuleInner>,
3476}
3477
3478struct SupervisedModuleInner {
3479 module_id: String,
3480 registry: Arc<Registry>,
3481 snapshot: SharedSnapshot,
3482 configuration: Arc<Mutex<SupervisedConfiguration>>,
3483 stderr_ring: Arc<Mutex<StderrRing>>,
3484 terminal_ring: Arc<Mutex<TerminalRing>>,
3485 commands: mpsc::Sender<SupervisorCommand>,
3486 monitor: Mutex<Option<JoinHandle<()>>>,
3487 restart_policy: RestartPolicy,
3491 effective_drain_timeout: Arc<Mutex<Duration>>,
3492 provenance_probe: ExecutableIdentityProbe,
3493}
3494
3495impl fmt::Debug for SupervisedModule {
3496 fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
3497 f.debug_struct("SupervisedModule")
3498 .field("module_id", &self.inner.module_id)
3499 .field("status", &self.status())
3500 .finish_non_exhaustive()
3501 }
3502}
3503
3504impl SupervisedModule {
3505 pub fn module_id(&self) -> &str {
3506 &self.inner.module_id
3507 }
3508
3509 #[cfg(test)]
3513 pub(crate) fn record_health_probe_failure_for_test(
3514 &self,
3515 detail: &str,
3516 ) -> Result<(), SuperviseError> {
3517 update_snapshot(&self.inner.snapshot, Some(&self.inner.module_id), |state| {
3518 state.health.consecutive_failures = state.health.consecutive_failures.saturating_add(1);
3519 state.health.detail = Some(detail.to_string());
3520 })
3521 }
3522
3523 pub fn state(&self) -> Result<ModuleState, SuperviseError> {
3524 Ok(lock_snapshot(&self.inner.snapshot)?.state)
3525 }
3526
3527 pub(crate) fn registration_end_reason(
3528 &self,
3529 ) -> Result<Option<RegistrationEndReason>, SuperviseError> {
3530 let snapshot = lock_snapshot(&self.inner.snapshot)?;
3531 Ok(match snapshot.state {
3532 ModuleState::Draining if snapshot.draining_to_replace => {
3533 Some(RegistrationEndReason::SupervisorRestart)
3534 }
3535 ModuleState::Draining => Some(RegistrationEndReason::SupervisorStop),
3536 ModuleState::Restarting if snapshot.draining_to_replace => {
3537 Some(RegistrationEndReason::SupervisorRestart)
3538 }
3539 _ => None,
3540 })
3541 }
3542
3543 pub fn stderr_tail(
3550 &self,
3551 max_lines: Option<usize>,
3552 max_bytes: Option<usize>,
3553 ) -> StderrTailSnapshot {
3554 self.inner
3555 .stderr_ring
3556 .lock()
3557 .unwrap_or_else(|poisoned| poisoned.into_inner())
3558 .snapshot(max_lines, max_bytes)
3559 }
3560
3561 pub fn terminal_history(&self) -> TerminalHistorySnapshot {
3566 self.inner
3567 .terminal_ring
3568 .lock()
3569 .unwrap_or_else(|poisoned| poisoned.into_inner())
3570 .snapshot()
3571 }
3572
3573 pub fn durable_terminal_history(&self) -> subc_control::TerminalHistory {
3578 durable_terminal_history_of(&self.inner.terminal_ring, &self.inner.module_id)
3579 }
3580
3581 pub(crate) async fn read_durable_terminal_history(
3586 &self,
3587 ) -> Result<subc_control::TerminalHistory, tokio::task::JoinError> {
3588 let terminal_ring = Arc::clone(&self.inner.terminal_ring);
3589 let module_id = self.inner.module_id.clone();
3590 tokio::task::spawn_blocking(move || durable_terminal_history_of(&terminal_ring, &module_id))
3591 .await
3592 }
3593
3594 pub fn status(&self) -> Result<ModuleStatus, SuperviseError> {
3595 self.status_with_snapshot_lock(&self.inner.snapshot, None)
3596 .map(|(status, _)| status)
3597 }
3598
3599 pub(crate) fn record_deliberate_severance(
3600 &self,
3601 identity: ProcessIdentity,
3602 ) -> Result<bool, SuperviseError> {
3603 let mut snapshot = lock_snapshot(&self.inner.snapshot)?;
3604 if snapshot.pid != Some(identity.pid)
3605 || snapshot.process_start_time != Some(identity.start_time)
3606 {
3607 return Ok(false);
3608 }
3609 snapshot.deliberate_severance = Some(identity);
3610 Ok(true)
3611 }
3612
3613 pub(crate) fn status_for_control(
3618 &self,
3619 caller: &'static str,
3620 ) -> Result<ModuleStatus, SuperviseError> {
3621 self.status_with_snapshot_lock(&self.inner.snapshot, Some(caller))
3622 .map(|(status, _)| status)
3623 }
3624
3625 fn status_with_snapshot_lock(
3626 &self,
3627 snapshot: &SharedSnapshot,
3628 caller: Option<&'static str>,
3629 ) -> Result<(ModuleStatus, Option<SpawnedFileIdentity>), SuperviseError> {
3630 let mut guard = match caller {
3631 Some(caller) => lock_snapshot_for_control(snapshot, &self.inner.module_id, caller)?,
3632 None => lock_snapshot(snapshot)?,
3633 };
3634 let restart_count =
3637 guard.crash_restarts_in_window(self.inner.restart_policy.window, Instant::now());
3638 let snapshot = guard.clone();
3639 drop(guard);
3640 let drain_timeout = *self.inner.effective_drain_timeout.lock().map_err(|_| {
3641 SuperviseError::StatePoisoned {
3642 module_id: Some(self.inner.module_id.clone()),
3643 }
3644 })?;
3645 let registration_active = self
3646 .inner
3647 .registry
3648 .get_module(&self.inner.module_id)
3649 .map_err(SuperviseError::Registry)?
3650 .is_some();
3651 let protocol = snapshot
3652 .spawned_protocol
3653 .unwrap_or(self.declared_protocol()?);
3654 let running_process =
3655 snapshot.enabled && snapshot.state == ModuleState::Running && snapshot.process_alive;
3656 let live = match protocol {
3662 ModuleProtocol::Subc => running_process && registration_active,
3663 ModuleProtocol::None => running_process,
3664 };
3665
3666 Ok((
3667 ModuleStatus {
3668 waiting_for_previous_pid: snapshot.waiting_for_previous_pid,
3669 waiting_for_previous_detail: snapshot.waiting_for_previous_detail.clone(),
3670 module_id: self.inner.module_id.clone(),
3671 launch_nonce_source: observed_launch_nonce_source(&snapshot),
3672 state: snapshot.state,
3673 enabled: snapshot.enabled,
3674 process_alive: snapshot.process_alive,
3675 registration_active,
3676 protocol,
3677 live,
3678 restart_count,
3679 lifetime_restarts: snapshot.lifetime_restarts,
3680 spawn_generation: snapshot.spawn_generation,
3681 max_restarts: self.inner.restart_policy.max_restarts,
3682 restart_window: self.inner.restart_policy.window,
3683 drain_timeout,
3684 restart_backoff: self.inner.restart_policy.backoff,
3685 restart_max_backoff: self.inner.restart_policy.max_backoff,
3686 pid: snapshot.reported_pid(),
3687 spawned_at_ms: snapshot.spawned_at_ms,
3688 spawned_from: snapshot.spawned_from,
3689 launch_spec_digest: snapshot.launch_spec_digest,
3690 process_start_time: snapshot.process_start_time,
3691 last_exit: snapshot.last_exit,
3692 health: snapshot.health,
3693 },
3694 snapshot.spawned_file_identity.clone(),
3695 ))
3696 }
3697
3698 #[cfg(test)]
3699 pub(crate) fn hold_snapshot_for_test(
3700 &self,
3701 acquired: std::sync::mpsc::Sender<()>,
3702 hold: Duration,
3703 ) -> std::thread::JoinHandle<()> {
3704 let snapshot = Arc::clone(&self.inner.snapshot);
3705 std::thread::spawn(move || {
3706 let _guard = snapshot.lock().expect("test snapshot lock is not poisoned");
3707 acquired
3708 .send(())
3709 .expect("test receiver waits for snapshot lock");
3710 std::thread::sleep(hold);
3711 })
3712 }
3713
3714 pub(crate) async fn status_and_running_image_agreement(
3720 &self,
3721 ) -> Result<(ModuleStatus, subc_control::RunningImageAgreement), SuperviseError> {
3722 let (status, identity) = self.status_with_snapshot_lock(&self.inner.snapshot, None)?;
3723 let image = self
3724 .inner
3725 .provenance_probe
3726 .observe(
3727 status.pid,
3728 status.spawned_from.as_deref(),
3729 identity,
3730 status.process_start_time,
3731 )
3732 .await;
3733 Ok((status, image))
3734 }
3735
3736 pub(crate) async fn running_image_agreement(&self) -> subc_control::RunningImageAgreement {
3737 let snapshot = match lock_snapshot(&self.inner.snapshot) {
3738 Ok(snapshot) => snapshot.clone(),
3739 Err(_) => {
3740 return subc_control::RunningImageAgreement::Unavailable {
3741 reason: subc_control::RunningImageUnavailableReason::NotRunning,
3742 };
3743 }
3744 };
3745 self.inner
3746 .provenance_probe
3747 .observe(
3748 snapshot.reported_pid(),
3749 snapshot.spawned_from.as_deref(),
3750 snapshot.spawned_file_identity.clone(),
3751 snapshot.process_start_time,
3752 )
3753 .await
3754 }
3755
3756 pub(crate) fn child_resource_usage(&self) -> subc_control::ChildResourceUsage {
3759 #[cfg(windows)]
3760 {
3761 let image = lock_snapshot(&self.inner.snapshot)
3762 .ok()
3763 .and_then(|snapshot| {
3764 snapshot
3765 .spawned_file_identity
3766 .as_ref()
3767 .and_then(|identity| identity.windows_image.clone())
3768 });
3769 match image {
3770 Some(image) => {
3771 crate::child_resources::read_windows_handle(image.process(), image.start_time())
3772 }
3773 None => subc_control::ChildResourceUsage::Unavailable {
3774 reason: subc_control::ChildResourceUnavailableReason::NotRunning,
3775 },
3776 }
3777 }
3778 #[cfg(not(windows))]
3779 {
3780 let (pid, start_time) = match lock_snapshot(&self.inner.snapshot) {
3781 Ok(snapshot) => (snapshot.reported_pid(), snapshot.process_start_time),
3782 Err(_) => {
3783 return subc_control::ChildResourceUsage::Unavailable {
3784 reason: subc_control::ChildResourceUnavailableReason::Unreadable,
3785 }
3786 }
3787 };
3788 crate::child_resources::read(pid, start_time)
3789 }
3790 }
3791
3792 pub(crate) fn will_recover_after_connection_loss(&self) -> Result<bool, SuperviseError> {
3793 let mut snapshot = lock_snapshot(&self.inner.snapshot)?;
3794 Ok(match snapshot.state {
3795 ModuleState::Restarting => true,
3796 ModuleState::Failed | ModuleState::Disabled => false,
3797 _ if snapshot.enabled && snapshot.respawn_pending => true,
3798 _ => daemon_will_restart(&mut snapshot, &self.inner.restart_policy, Instant::now()),
3799 })
3800 }
3801
3802 #[cfg(test)]
3803 pub(crate) fn is_warming(&self) -> Result<bool, SuperviseError> {
3804 self.is_warming_with_snapshot_lock(None)
3805 }
3806
3807 pub(crate) fn is_warming_for_control(
3808 &self,
3809 caller: &'static str,
3810 ) -> Result<bool, SuperviseError> {
3811 self.is_warming_with_snapshot_lock(Some(caller))
3812 }
3813
3814 fn is_warming_with_snapshot_lock(
3815 &self,
3816 caller: Option<&'static str>,
3817 ) -> Result<bool, SuperviseError> {
3818 let snapshot = match caller {
3819 Some(caller) => {
3820 lock_snapshot_for_control(&self.inner.snapshot, &self.inner.module_id, caller)?
3821 }
3822 None => lock_snapshot(&self.inner.snapshot)?,
3823 }
3824 .clone();
3825 Ok(matches!(
3826 snapshot.state,
3827 ModuleState::Starting | ModuleState::Running | ModuleState::Restarting
3828 ))
3829 }
3830
3831 pub async fn drain(&self) -> Result<(), SuperviseError> {
3833 self.stop().await
3834 }
3835
3836 pub(crate) async fn retire_requested(
3837 &self,
3838 requester: Option<Principal>,
3839 ) -> Result<(), SuperviseError> {
3840 match self.state()? {
3841 ModuleState::Stopped | ModuleState::Failed => return Ok(()),
3842 ModuleState::Starting
3843 | ModuleState::Running
3844 | ModuleState::Unresponsive
3845 | ModuleState::Restarting
3846 | ModuleState::Draining
3847 | ModuleState::Disabled => {}
3848 }
3849
3850 let (reply_tx, reply_rx) = oneshot::channel();
3851 self.inner
3852 .commands
3853 .send(SupervisorCommand::Retire {
3854 requester,
3855 reply: reply_tx,
3856 })
3857 .await
3858 .map_err(|_| SuperviseError::CommandClosed {
3859 module_id: self.inner.module_id.clone(),
3860 })?;
3861 reply_rx.await.map_err(|_| SuperviseError::CommandClosed {
3862 module_id: self.inner.module_id.clone(),
3863 })?
3864 }
3865
3866 pub async fn stop(&self) -> Result<(), SuperviseError> {
3867 self.stop_requested(None).await
3868 }
3869
3870 pub(crate) async fn stop_requested(
3871 &self,
3872 requester: Option<Principal>,
3873 ) -> Result<(), SuperviseError> {
3874 match self.state()? {
3875 ModuleState::Stopped | ModuleState::Failed => return Ok(()),
3876 ModuleState::Starting
3877 | ModuleState::Running
3878 | ModuleState::Unresponsive
3879 | ModuleState::Restarting
3880 | ModuleState::Draining
3881 | ModuleState::Disabled => {}
3882 }
3883
3884 let (reply_tx, reply_rx) = oneshot::channel();
3885 self.inner
3886 .commands
3887 .send(SupervisorCommand::Drain {
3888 requester,
3889 reply: reply_tx,
3890 })
3891 .await
3892 .map_err(|_| SuperviseError::CommandClosed {
3893 module_id: self.inner.module_id.clone(),
3894 })?;
3895 reply_rx.await.map_err(|_| SuperviseError::CommandClosed {
3896 module_id: self.inner.module_id.clone(),
3897 })?
3898 }
3899
3900 pub async fn restart(&self, drain_timeout_ms: Option<u64>) -> Result<(), SuperviseError> {
3901 self.restart_requested(drain_timeout_ms, None).await
3902 }
3903
3904 pub(crate) async fn restart_requested(
3905 &self,
3906 drain_timeout_ms: Option<u64>,
3907 requester: Option<Principal>,
3908 ) -> Result<(), SuperviseError> {
3909 let queued_at = Instant::now();
3910 let received_at_generation = lock_snapshot(&self.inner.snapshot)?.spawn_generation;
3911 let (reply_tx, reply_rx) = oneshot::channel();
3912 self.inner
3913 .commands
3914 .send(SupervisorCommand::Restart {
3915 requester,
3916 drain_timeout_ms,
3917 received_at_generation,
3918 queued_at,
3919 reply: reply_tx,
3920 })
3921 .await
3922 .map_err(|_| SuperviseError::CommandClosed {
3923 module_id: self.inner.module_id.clone(),
3924 })?;
3925 reply_rx.await.map_err(|_| SuperviseError::CommandClosed {
3926 module_id: self.inner.module_id.clone(),
3927 })?
3928 }
3929
3930 pub async fn swap(&self, ready_timeout: Option<Duration>) -> Result<(), SuperviseError> {
3935 self.swap_requested(ready_timeout, None).await
3936 }
3937
3938 pub(crate) async fn swap_requested(
3939 &self,
3940 ready_timeout: Option<Duration>,
3941 requester: Option<Principal>,
3942 ) -> Result<(), SuperviseError> {
3943 let (reply_tx, reply_rx) = oneshot::channel();
3944 self.inner
3945 .commands
3946 .send(SupervisorCommand::Swap {
3947 requester,
3948 ready_timeout,
3949 reply: reply_tx,
3950 })
3951 .await
3952 .map_err(|_| SuperviseError::CommandClosed {
3953 module_id: self.inner.module_id.clone(),
3954 })?;
3955 reply_rx.await.map_err(|_| SuperviseError::CommandClosed {
3956 module_id: self.inner.module_id.clone(),
3957 })?
3958 }
3959
3960 pub async fn reload(&self) -> Result<(), SuperviseError> {
3961 self.reload_requested(None).await
3962 }
3963
3964 pub(crate) async fn reload_requested(
3965 &self,
3966 requester: Option<Principal>,
3967 ) -> Result<(), SuperviseError> {
3968 let (reply_tx, reply_rx) = oneshot::channel();
3969 self.inner
3970 .commands
3971 .send(SupervisorCommand::Reload {
3972 requester,
3973 reply: reply_tx,
3974 })
3975 .await
3976 .map_err(|_| SuperviseError::CommandClosed {
3977 module_id: self.inner.module_id.clone(),
3978 })?;
3979 reply_rx.await.map_err(|_| SuperviseError::CommandClosed {
3980 module_id: self.inner.module_id.clone(),
3981 })?
3982 }
3983
3984 pub async fn set_enabled(&self, enabled: bool) -> Result<bool, SuperviseError> {
3985 self.set_enabled_requested(enabled, None).await
3986 }
3987
3988 pub(crate) async fn set_enabled_requested(
3989 &self,
3990 enabled: bool,
3991 requester: Option<Principal>,
3992 ) -> Result<bool, SuperviseError> {
3993 let (reply_tx, reply_rx) = oneshot::channel();
3994 self.inner
3995 .commands
3996 .send(SupervisorCommand::SetEnabled {
3997 requester,
3998 enabled,
3999 reply: reply_tx,
4000 })
4001 .await
4002 .map_err(|_| SuperviseError::CommandClosed {
4003 module_id: self.inner.module_id.clone(),
4004 })?;
4005 reply_rx.await.map_err(|_| SuperviseError::CommandClosed {
4006 module_id: self.inner.module_id.clone(),
4007 })?
4008 }
4009
4010 pub(crate) fn declared_protocol(&self) -> Result<ModuleProtocol, SuperviseError> {
4014 let configured = self
4015 .inner
4016 .configuration
4017 .lock()
4018 .map_err(|_| SuperviseError::StatePoisoned {
4019 module_id: Some(self.inner.module_id.clone()),
4020 })?
4021 .spec
4022 .protocol;
4023 let state = lock_snapshot(&self.inner.snapshot)?;
4024 Ok(state.spawned_protocol.unwrap_or(configured))
4025 }
4026
4027 pub(crate) fn configuration(&self) -> Result<(ModuleSpec, HealthConfig), SuperviseError> {
4028 let configuration =
4029 self.inner
4030 .configuration
4031 .lock()
4032 .map_err(|_| SuperviseError::StatePoisoned {
4033 module_id: Some(self.inner.module_id.clone()),
4034 })?;
4035 Ok((configuration.spec.clone(), configuration.health.clone()))
4036 }
4037
4038 #[cfg(any(test, feature = "test-support"))]
4042 pub async fn update_spec_for_test(&self, spec: ModuleSpec) -> Result<(), SuperviseError> {
4043 let (_, health) = self.configuration()?;
4044 let drain_timeout_ms = u64::try_from(
4045 self.inner
4046 .effective_drain_timeout
4047 .lock()
4048 .unwrap_or_else(|poisoned| poisoned.into_inner())
4049 .as_millis(),
4050 )
4051 .ok();
4052 self.update_configuration(spec, health, drain_timeout_ms)
4053 .await
4054 }
4055
4056 pub(crate) async fn update_configuration(
4057 &self,
4058 spec: ModuleSpec,
4059 health: HealthConfig,
4060 drain_timeout_ms: Option<u64>,
4061 ) -> Result<(), SuperviseError> {
4062 if spec.module_id != self.inner.module_id {
4063 return Err(SuperviseError::InvalidSpec {
4064 reason: "a supervised module's module_id cannot be changed".to_string(),
4065 });
4066 }
4067 validate_spec(&spec)?;
4068 let (reply_tx, reply_rx) = oneshot::channel();
4069 self.inner
4070 .commands
4071 .send(SupervisorCommand::UpdateConfiguration {
4072 spec: spec.clone(),
4073 health: health.clone(),
4074 drain_timeout_ms,
4075 reply: reply_tx,
4076 })
4077 .await
4078 .map_err(|_| SuperviseError::CommandClosed {
4079 module_id: self.inner.module_id.clone(),
4080 })?;
4081 reply_rx.await.map_err(|_| SuperviseError::CommandClosed {
4082 module_id: self.inner.module_id.clone(),
4083 })?;
4084 let mut configuration =
4085 self.inner
4086 .configuration
4087 .lock()
4088 .map_err(|_| SuperviseError::StatePoisoned {
4089 module_id: Some(self.inner.module_id.clone()),
4090 })?;
4091 configuration.spec = spec;
4092 configuration.health = health;
4093 Ok(())
4094 }
4095}
4096
4097impl Drop for SupervisedModuleInner {
4098 fn drop(&mut self) {
4099 let Ok(mut monitor) = self.monitor.lock() else {
4100 return;
4101 };
4102 if let Some(monitor) = monitor.as_ref().filter(|monitor| !monitor.is_finished()) {
4103 let _ = update_snapshot(&self.snapshot, Some(&self.module_id), |state| {
4104 state.state = ModuleState::Stopped;
4105 clear_current_process_facts(state);
4106 });
4107 monitor.abort();
4108 }
4109 let _ = monitor.take();
4110 }
4111}
4112
4113#[derive(Debug)]
4114enum SupervisorCommand {
4115 Drain {
4116 requester: Option<Principal>,
4117 reply: oneshot::Sender<Result<(), SuperviseError>>,
4118 },
4119 Retire {
4120 requester: Option<Principal>,
4121 reply: oneshot::Sender<Result<(), SuperviseError>>,
4122 },
4123 Restart {
4124 requester: Option<Principal>,
4125 drain_timeout_ms: Option<u64>,
4130 received_at_generation: u64,
4134 queued_at: Instant,
4137 reply: oneshot::Sender<Result<(), SuperviseError>>,
4138 },
4139 Reload {
4140 requester: Option<Principal>,
4141 reply: oneshot::Sender<Result<(), SuperviseError>>,
4142 },
4143 SetEnabled {
4144 requester: Option<Principal>,
4145 enabled: bool,
4146 reply: oneshot::Sender<Result<bool, SuperviseError>>,
4147 },
4148 UpdateConfiguration {
4149 spec: ModuleSpec,
4150 health: HealthConfig,
4151 drain_timeout_ms: Option<u64>,
4154 reply: oneshot::Sender<()>,
4155 },
4156 Swap {
4157 requester: Option<Principal>,
4158 ready_timeout: Option<Duration>,
4161 reply: oneshot::Sender<Result<(), SuperviseError>>,
4163 },
4164}
4165
4166#[derive(Debug)]
4167pub enum SuperviseError {
4168 InvalidSpec {
4169 reason: String,
4170 },
4171 Spawn {
4172 program: PathBuf,
4173 source: io::Error,
4174 cgroup_path: Option<PathBuf>,
4175 },
4176 Cgroup {
4177 module_id: String,
4178 source: io::Error,
4179 },
4180 LaunchNonce {
4183 reason: String,
4184 },
4185 Wait {
4186 module_id: String,
4187 source: io::Error,
4188 },
4189 Kill {
4190 module_id: String,
4191 source: io::Error,
4192 },
4193 Forwarding(ForwardingError),
4194 Registry(RegistryError),
4195 ReloadUnavailable {
4196 module_id: String,
4197 reason: String,
4198 },
4199 Disabled {
4204 module_id: String,
4205 },
4206 ReloadFailed {
4207 module_id: String,
4208 reason: String,
4209 },
4210 RegistrationStillActive {
4211 module_id: String,
4212 waited: Duration,
4213 },
4214 StatePoisoned {
4215 module_id: Option<String>,
4216 },
4217 CommandClosed {
4218 module_id: String,
4219 },
4220 SwapInProgress {
4224 module_id: String,
4225 },
4226 SwapRefused {
4228 module_id: String,
4229 reason: SwapRefusal,
4230 },
4231 SwapFailed {
4235 module_id: String,
4236 arm: SwapFailureArm,
4237 detail: String,
4238 candidate_exit: Option<ExitReport>,
4241 },
4242}
4243
4244#[derive(Debug, Clone, Copy, PartialEq, Eq)]
4246pub enum SwapRefusal {
4247 OverlapExclusive,
4249 NotRegistered,
4252 ProtocolNone,
4255 NotConfigured,
4258 AlreadySwapping,
4260}
4261
4262impl SwapRefusal {
4263 pub fn as_str(self) -> &'static str {
4264 match self {
4265 Self::OverlapExclusive => "overlap_exclusive",
4266 Self::NotRegistered => "not_registered",
4267 Self::ProtocolNone => "protocol_none",
4268 Self::NotConfigured => "not_configured",
4269 Self::AlreadySwapping => "already_swapping",
4270 }
4271 }
4272}
4273
4274#[derive(Debug, Clone, Copy, PartialEq, Eq)]
4277pub enum SwapFailureArm {
4278 SpawnFailed,
4280 NeverRegistered,
4282 NeverReady,
4284 CandidateExited,
4286 CandidateUnhealthy,
4288 Interrupted,
4292 CutoverLost,
4297}
4298
4299impl SwapFailureArm {
4300 pub fn as_str(self) -> &'static str {
4301 match self {
4302 Self::SpawnFailed => "spawn_failed",
4303 Self::NeverRegistered => "never_registered",
4304 Self::NeverReady => "never_ready",
4305 Self::CandidateExited => "candidate_exited",
4306 Self::CandidateUnhealthy => "candidate_unhealthy",
4307 Self::Interrupted => "interrupted",
4308 Self::CutoverLost => "cutover_lost",
4309 }
4310 }
4311}
4312
4313impl fmt::Display for SuperviseError {
4314 fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
4315 match self {
4316 Self::InvalidSpec { reason } => write!(f, "invalid module spec: {reason}"),
4317 Self::Spawn {
4318 program,
4319 source,
4320 cgroup_path: Some(cgroup_path),
4321 } => write!(
4322 f,
4323 "failed to place module in cgroup '{}' while spawning '{}': {source}",
4324 cgroup_path.display(),
4325 program.display()
4326 ),
4327 Self::Spawn {
4328 program,
4329 source,
4330 cgroup_path: None,
4331 } => write!(
4332 f,
4333 "failed to spawn module '{}': {source}",
4334 program.display()
4335 ),
4336 Self::Cgroup { module_id, source } => {
4337 write!(
4338 f,
4339 "failed to prepare cgroup for module '{module_id}': {source}"
4340 )
4341 }
4342 Self::LaunchNonce { reason } => {
4343 write!(
4344 f,
4345 "failed to generate reserved-module launch nonce: {reason}"
4346 )
4347 }
4348 Self::Wait { module_id, source } => {
4349 write!(f, "failed to wait for module '{module_id}': {source}")
4350 }
4351 Self::Kill { module_id, source } => {
4352 write!(f, "failed to kill module '{module_id}': {source}")
4353 }
4354 Self::Forwarding(err) => write!(f, "forwarding error: {err}"),
4355 Self::Registry(err) => write!(f, "registry error: {err}"),
4356 Self::ReloadUnavailable { module_id, reason } => {
4357 write!(f, "reload unavailable for module '{module_id}': {reason}")
4358 }
4359 Self::Disabled { module_id } => {
4360 write!(
4361 f,
4362 "module '{module_id}' is disabled; enable it before restart or reload"
4363 )
4364 }
4365 Self::ReloadFailed { module_id, reason } => {
4366 write!(f, "reload failed for module '{module_id}': {reason}")
4367 }
4368 Self::RegistrationStillActive { module_id, waited } => write!(
4369 f,
4370 "module '{module_id}' registration remained active after waiting {waited:?}"
4371 ),
4372 Self::StatePoisoned { module_id } => match module_id {
4373 Some(module_id) => {
4374 write!(f, "supervisor state for module '{module_id}' was poisoned")
4375 }
4376 None => write!(f, "supervisor state was poisoned"),
4377 },
4378 Self::CommandClosed { module_id } => {
4379 write!(
4380 f,
4381 "supervisor command channel for module '{module_id}' is closed"
4382 )
4383 }
4384 Self::SwapInProgress { module_id } => write!(
4385 f,
4386 "module '{module_id}' is being swapped; retry once the swap has cut over or failed, or stop the module to abort the swap"
4387 ),
4388 Self::SwapRefused { module_id, reason } => match reason {
4389 SwapRefusal::OverlapExclusive => write!(
4390 f,
4391 "module '{module_id}' is declared overlap: \"exclusive\" (the default): two processes of it must not run at once, so it cannot be swapped; use a plain restart, or declare overlap: \"safe\" in its config if it really tolerates a second process"
4392 ),
4393 SwapRefusal::NotRegistered => write!(
4394 f,
4395 "module '{module_id}' is not registered, so there is no serving process to keep while a replacement warms; use a plain restart"
4396 ),
4397 SwapRefusal::ProtocolNone => write!(
4398 f,
4399 "module '{module_id}' is protocol: \"none\" and never registers, so a swap could never see its replacement become ready; use a plain restart"
4400 ),
4401 SwapRefusal::NotConfigured => write!(
4402 f,
4403 "module '{module_id}' cannot be swapped: the supervisor was built without the forwarding table or shared handle a swap needs"
4404 ),
4405 SwapRefusal::AlreadySwapping => {
4406 write!(f, "module '{module_id}' is already being swapped")
4407 }
4408 },
4409 Self::SwapFailed {
4410 module_id,
4411 arm,
4412 detail,
4413 ..
4414 } => write!(
4415 f,
4416 "swap of module '{module_id}' failed ({}): {detail}; the running process was left serving",
4417 arm.as_str()
4418 ),
4419 }
4420 }
4421}
4422
4423impl Error for SuperviseError {
4424 fn source(&self) -> Option<&(dyn Error + 'static)> {
4425 match self {
4426 Self::Spawn { source, .. }
4427 | Self::Cgroup { source, .. }
4428 | Self::Wait { source, .. }
4429 | Self::Kill { source, .. } => Some(source),
4430 Self::Forwarding(err) => Some(err),
4431 Self::Registry(err) => Some(err),
4432 Self::LaunchNonce { .. }
4433 | Self::InvalidSpec { .. }
4434 | Self::ReloadUnavailable { .. }
4435 | Self::Disabled { .. }
4436 | Self::ReloadFailed { .. }
4437 | Self::RegistrationStillActive { .. }
4438 | Self::StatePoisoned { .. }
4439 | Self::CommandClosed { .. }
4440 | Self::SwapInProgress { .. }
4441 | Self::SwapRefused { .. }
4442 | Self::SwapFailed { .. } => None,
4443 }
4444 }
4445}
4446
4447pub(crate) fn validate_spec(spec: &ModuleSpec) -> Result<(), SuperviseError> {
4448 if spec.module_id.trim().is_empty() {
4449 return Err(SuperviseError::InvalidSpec {
4450 reason: "module_id must not be empty".to_string(),
4451 });
4452 }
4453
4454 Ok(())
4455}
4456
4457#[derive(Debug, Default)]
4458struct HealthProbeRuntime {
4459 spawn_generation: Option<u64>,
4460 configured_health: Option<HealthConfig>,
4461 registered_connection: Option<crate::ConnectionId>,
4462 advertised: bool,
4463 next_probe_at: Option<Instant>,
4464 probe_index: u64,
4465}
4466
4467fn running_protocol(spec: &ModuleSpec, snapshot: &SharedSnapshot) -> ModuleProtocol {
4468 lock_snapshot(snapshot)
4469 .ok()
4470 .and_then(|state| state.spawned_protocol)
4471 .unwrap_or(spec.protocol)
4472}
4473
4474impl HealthProbeRuntime {
4475 fn refresh_registration(
4476 &mut self,
4477 spec: &ModuleSpec,
4478 runtime: &SupervisorRuntimeConfig,
4479 registry: &Registry,
4480 snapshot: &SharedSnapshot,
4481 ) {
4482 let generation = lock_snapshot(snapshot)
4483 .ok()
4484 .map(|state| state.spawn_generation);
4485 if self.spawn_generation != generation {
4486 *self = Self {
4490 spawn_generation: generation,
4491 ..Self::default()
4492 };
4493 }
4494 if self.configured_health.as_ref() != Some(&runtime.health) {
4495 self.configured_health = Some(runtime.health.clone());
4496 self.next_probe_at = None;
4497 self.registered_connection = None;
4498 self.probe_index = 0;
4499 }
4500 if running_protocol(spec, snapshot) == ModuleProtocol::None {
4504 self.registered_connection = None;
4505 self.advertised = runtime.health.http.is_some();
4506 if !self.advertised {
4507 self.next_probe_at = None;
4508 let _ = update_snapshot(snapshot, Some(&spec.module_id), |state| {
4509 let unknown = ModuleHealthStatus::default();
4510 if state.health != unknown {
4511 state.health = unknown;
4512 }
4513 });
4514 } else if self.next_probe_at.is_none() {
4515 self.next_probe_at = Some(
4516 Instant::now()
4517 + jittered_health_delay(&spec.module_id, 0, runtime.health.cadence),
4518 );
4519 }
4520 return;
4521 }
4522
4523 let registration = match registry.get_module(&spec.module_id) {
4524 Ok(registration) => registration,
4525 Err(err) => {
4526 warn!(module_id = %spec.module_id, error = %err, "health prober could not read registry");
4527 self.advertised = false;
4528 self.next_probe_at = None;
4529 return;
4530 }
4531 };
4532
4533 let Some(registration) = registration else {
4534 self.registered_connection = None;
4535 self.advertised = false;
4536 self.next_probe_at = None;
4537 return;
4538 };
4539
4540 let advertised = registration
4541 .control_ops
4542 .iter()
4543 .any(|op| op == MODULE_CONTROL_OP_HEALTH_CHECK);
4544 if !advertised {
4545 self.registered_connection = Some(registration.connection_id);
4546 self.advertised = false;
4547 self.next_probe_at = None;
4548 let _ = update_snapshot(snapshot, Some(&spec.module_id), |state| {
4549 if state.health.status != SupervisorHealthStatus::Unknown
4550 || state.health.consecutive_failures != 0
4551 || state.health.last_probe_ms.is_some()
4552 || state.health.detail.is_some()
4553 || state.health.metrics.is_some()
4554 {
4555 state.health.status = SupervisorHealthStatus::Unknown;
4556 state.health.consecutive_failures = 0;
4557 state.health.last_probe_ms = None;
4558 state.health.detail = None;
4559 state.health.metrics = None;
4560 }
4561 });
4562 return;
4563 }
4564
4565 let reregistered = self.registered_connection != Some(registration.connection_id);
4566 self.registered_connection = Some(registration.connection_id);
4567 self.advertised = true;
4568 if reregistered || self.next_probe_at.is_none() {
4569 self.probe_index = 0;
4570 self.next_probe_at = Some(
4571 Instant::now() + jittered_health_delay(&spec.module_id, 0, runtime.health.cadence),
4572 );
4573 let _ = update_snapshot(snapshot, Some(&spec.module_id), |state| {
4574 state.health.status = SupervisorHealthStatus::Unknown;
4575 state.health.consecutive_failures = 0;
4576 state.health.detail = None;
4577 state.health.metrics = None;
4578 });
4579 }
4580 }
4581
4582 fn wake_after(&self) -> Option<Duration> {
4583 if !self.advertised {
4584 return None;
4585 }
4586 self.next_probe_at
4587 .map(|next| next.saturating_duration_since(Instant::now()))
4588 }
4589
4590 fn due(&self) -> bool {
4591 self.advertised
4592 && self
4593 .next_probe_at
4594 .is_some_and(|next| Instant::now() >= next)
4595 }
4596
4597 fn schedule_next(&mut self, spec: &ModuleSpec, cadence: Duration) {
4598 self.probe_index = self.probe_index.wrapping_add(1);
4599 self.next_probe_at = Some(
4600 Instant::now() + jittered_health_delay(&spec.module_id, self.probe_index, cadence),
4601 );
4602 }
4603}
4604
4605#[derive(Debug)]
4640enum HealthProbeEvidence {
4641 LaneDead,
4643 NoAnswer,
4645 BadAnswer,
4647 Misconfigured,
4649}
4650
4651#[derive(Debug)]
4652struct HealthProbeError {
4653 evidence: HealthProbeEvidence,
4654 message: String,
4655}
4656
4657impl HealthProbeError {
4658 fn lane_dead(message: impl Into<String>) -> Self {
4659 Self::with(HealthProbeEvidence::LaneDead, message)
4660 }
4661
4662 fn no_answer(message: impl Into<String>) -> Self {
4663 Self::with(HealthProbeEvidence::NoAnswer, message)
4664 }
4665
4666 fn bad_answer(message: impl Into<String>) -> Self {
4667 Self::with(HealthProbeEvidence::BadAnswer, message)
4668 }
4669
4670 fn misconfigured(message: impl Into<String>) -> Self {
4671 Self::with(HealthProbeEvidence::Misconfigured, message)
4672 }
4673
4674 fn with(evidence: HealthProbeEvidence, message: impl Into<String>) -> Self {
4675 Self {
4676 evidence,
4677 message: message.into(),
4678 }
4679 }
4680
4681 #[allow(dead_code)]
4695 fn is_proof_of_death(&self) -> bool {
4696 matches!(self.evidence, HealthProbeEvidence::LaneDead)
4697 }
4698
4699 fn label(&self) -> &'static str {
4707 match self.evidence {
4708 HealthProbeEvidence::LaneDead => "lane-dead",
4709 HealthProbeEvidence::NoAnswer => "no-answer",
4710 HealthProbeEvidence::BadAnswer => "bad-answer",
4711 HealthProbeEvidence::Misconfigured => "daemon-misconfigured",
4712 }
4713 }
4714}
4715
4716impl fmt::Display for HealthProbeError {
4717 fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
4718 f.write_str(&self.message)
4719 }
4720}
4721
4722async fn run_health_probe_cycle(
4723 spec: &ModuleSpec,
4724 runtime: &SupervisorRuntimeConfig,
4725 registry: &Registry,
4726 process_liveness: &SupervisorProcessLiveness,
4727 snapshot: &SharedSnapshot,
4728 child: &mut Option<SupervisedChild>,
4729) {
4730 let now_ms = unix_ms_now();
4731 let http = (running_protocol(spec, snapshot) == ModuleProtocol::None)
4732 .then_some(runtime.health.http.as_deref())
4733 .flatten();
4734 let result = match http {
4735 Some(url) => probe_http_health(url, runtime.health.deadline).await,
4736 None => probe_module_health(&spec.module_id, runtime, None).await,
4737 };
4738 match result {
4739 Ok(report) => {
4740 handle_health_report(
4741 spec,
4742 runtime,
4743 registry,
4744 process_liveness,
4745 snapshot,
4746 child,
4747 report,
4748 now_ms,
4749 )
4750 .await;
4751 }
4752 Err(err) => {
4753 if http.is_some() {
4754 let _ = update_snapshot(snapshot, Some(&spec.module_id), |state| {
4755 state.health.status = SupervisorHealthStatus::Failing;
4756 });
4757 }
4758 handle_health_probe_failure(
4759 spec,
4760 runtime,
4761 registry,
4762 process_liveness,
4763 snapshot,
4764 child,
4765 err,
4766 now_ms,
4767 )
4768 .await;
4769 }
4770 }
4771}
4772
4773pub(crate) struct HttpProbeTarget<'a> {
4774 address: std::net::SocketAddr,
4775 localhost: bool,
4776 authority: &'a str,
4777 path: String,
4778}
4779
4780pub(crate) fn parse_http_probe_url(url: &str) -> Result<HttpProbeTarget<'_>, String> {
4783 if url.bytes().any(|b| b <= b' ' || b == 127) || url.contains('#') {
4784 return Err("must not contain whitespace, controls, or a fragment".into());
4785 }
4786 let rest = url
4787 .strip_prefix("http://")
4788 .ok_or("must use plain http://")?;
4789 let split = rest.find(['/', '?']).unwrap_or(rest.len());
4790 let (authority, suffix) = rest.split_at(split);
4791 let (host, port) = if let Some(rest) = authority.strip_prefix("[::1]") {
4792 ("::1", rest)
4793 } else {
4794 let split = authority.find(':').unwrap_or(authority.len());
4795 authority.split_at(split)
4796 };
4797 let ip: std::net::IpAddr = match host {
4798 "127.0.0.1" | "localhost" => std::net::Ipv4Addr::LOCALHOST.into(),
4799 "::1" => std::net::Ipv6Addr::LOCALHOST.into(),
4800 _ => return Err("host must be 127.0.0.1, [::1], or localhost".into()),
4801 };
4802 let port = if port.is_empty() {
4803 80
4804 } else {
4805 port.strip_prefix(':')
4806 .and_then(|p| p.parse::<u16>().ok())
4807 .filter(|p| *p > 0)
4808 .ok_or("must have a valid nonzero TCP port")?
4809 };
4810 let path = if suffix.is_empty() {
4811 "/".into()
4812 } else if suffix.starts_with('?') {
4813 format!("/{suffix}")
4814 } else {
4815 suffix.into()
4816 };
4817 Ok(HttpProbeTarget {
4818 address: std::net::SocketAddr::new(ip, port),
4819 localhost: host == "localhost",
4820 authority,
4821 path,
4822 })
4823}
4824
4825async fn probe_http_health(
4826 url: &str,
4827 deadline: Duration,
4828) -> Result<HealthReport, HealthProbeError> {
4829 use tokio::io::{AsyncReadExt, AsyncWriteExt, BufReader};
4830 let target = parse_http_probe_url(url).map_err(HealthProbeError::misconfigured)?;
4831 let mut response_status = String::new();
4834 let mut body = Vec::new();
4835 let probe = async {
4836 let connection = match tokio::net::TcpStream::connect(target.address).await {
4839 Err(_) if target.localhost => {
4840 tokio::net::TcpStream::connect((
4841 std::net::Ipv6Addr::LOCALHOST,
4842 target.address.port(),
4843 ))
4844 .await
4845 }
4846 result => result,
4847 };
4848 let mut stream = connection.map_err(|error| {
4849 HealthProbeError::no_answer(format!("HTTP connect failed: {error}"))
4850 })?;
4851 stream.write_all(format!("GET {} HTTP/1.1\r\nHost: {}\r\nAccept-Encoding: identity\r\nConnection: close\r\n\r\n", target.path, target.authority).as_bytes()).await
4852 .map_err(|error| HealthProbeError::no_answer(format!("HTTP write failed: {error}")))?;
4853 let mut reader = BufReader::new(stream);
4854 let mut budget = 16 * 1024;
4855 let status = http_line(&mut reader, &mut budget).await?;
4856 let mut words = status.split_ascii_whitespace();
4857 let version = words.next();
4858 let code = words
4859 .next()
4860 .filter(|word| word.len() == 3)
4861 .and_then(|word| word.parse::<u16>().ok());
4862 if !matches!(version, Some("HTTP/1.1" | "HTTP/1.0"))
4863 || !code.is_some_and(|code| (100..600).contains(&code))
4864 {
4865 return Err(HealthProbeError::bad_answer(format!(
4866 "invalid HTTP status: {status}"
4867 )));
4868 }
4869 let code = code.expect("validated status code");
4870 response_status = status.clone();
4871 let mut length = None;
4872 let mut chunked = false;
4873 loop {
4874 let line = http_line(&mut reader, &mut budget).await?;
4875 if line.is_empty() {
4876 break;
4877 }
4878 if let Some((name, value)) = line.split_once(':') {
4879 if name.eq_ignore_ascii_case("content-length") {
4880 length = Some(value.trim().parse::<u64>().map_err(|_| {
4881 HealthProbeError::bad_answer("invalid HTTP Content-Length")
4882 })?);
4883 } else if name.eq_ignore_ascii_case("transfer-encoding") {
4884 chunked = value.trim().eq_ignore_ascii_case("chunked");
4885 }
4886 }
4887 }
4888 if chunked {
4889 while body.len() < 200 {
4890 let line = http_line(&mut reader, &mut budget).await?;
4891 let size = u64::from_str_radix(line.split(';').next().unwrap_or("").trim(), 16)
4892 .map_err(|_| HealthProbeError::bad_answer("invalid HTTP chunk size"))?;
4893 if size == 0 {
4894 break;
4895 }
4896 let count = size.min((200 - body.len()) as u64) as usize;
4897 let start = body.len();
4898 (&mut reader)
4899 .take(count as u64)
4900 .read_to_end(&mut body)
4901 .await
4902 .map_err(|error| {
4903 HealthProbeError::no_answer(format!("HTTP body read failed: {error}"))
4904 })?;
4905 if body.len() - start != count {
4906 return Err(HealthProbeError::bad_answer("truncated HTTP chunk"));
4907 }
4908 if size > count as u64 || body.len() == 200 {
4909 break;
4910 }
4911 if !http_line(&mut reader, &mut budget).await?.is_empty() {
4912 return Err(HealthProbeError::bad_answer("invalid HTTP chunk delimiter"));
4913 }
4914 }
4915 } else {
4916 reader
4917 .take(length.unwrap_or(200).min(200))
4918 .read_to_end(&mut body)
4919 .await
4920 .map_err(|error| {
4921 HealthProbeError::no_answer(format!("HTTP body read failed: {error}"))
4922 })?;
4923 }
4924 if (200..300).contains(&code) {
4925 Ok(HealthReport::ok())
4926 } else {
4927 Err(HealthProbeError::bad_answer(
4928 "HTTP health endpoint returned non-2xx",
4929 ))
4930 }
4931 };
4932 let mut result = timeout(deadline, probe).await.unwrap_or_else(|_| {
4933 Err(HealthProbeError::no_answer(format!(
4934 "HTTP probe timed out after {deadline:?}"
4935 )))
4936 });
4937 if let Err(error) = &mut result {
4938 if !response_status.is_empty() {
4939 error.message = format!(
4940 "{}; {response_status}: {}",
4941 error.message,
4942 String::from_utf8_lossy(&body)
4943 );
4944 }
4945 }
4946 result
4947}
4948
4949async fn http_line(
4950 reader: &mut tokio::io::BufReader<tokio::net::TcpStream>,
4951 remaining: &mut usize,
4952) -> Result<String, HealthProbeError> {
4953 use tokio::io::{AsyncBufReadExt, AsyncReadExt};
4954 let mut line = Vec::new();
4955 (&mut *reader)
4956 .take(*remaining as u64)
4957 .read_until(b'\n', &mut line)
4958 .await
4959 .map_err(|error| {
4960 HealthProbeError::no_answer(format!("HTTP header read failed: {error}"))
4961 })?;
4962 *remaining -= line.len();
4963 if !line.ends_with(b"\r\n") {
4964 return Err(HealthProbeError::bad_answer(
4965 "HTTP headers are incomplete or exceed 16 KiB",
4966 ));
4967 }
4968 line.truncate(line.len() - 2);
4969 String::from_utf8(line).map_err(|_| HealthProbeError::bad_answer("HTTP header is not UTF-8"))
4970}
4971
4972async fn probe_module_health(
4973 module_id: &str,
4974 runtime: &SupervisorRuntimeConfig,
4975 drain_deadline: Option<Instant>,
4976) -> Result<HealthReport, HealthProbeError> {
4977 let Some(forwarding) = runtime.forwarding.as_ref() else {
4978 return Err(HealthProbeError::misconfigured(
4979 "supervisor was not configured with a forwarding table",
4980 ));
4981 };
4982 let probe_started_at = Instant::now();
4983 let mut deadline = probe_started_at + runtime.health.deadline;
4984 if let Some(drain_deadline) = drain_deadline {
4985 deadline = deadline.min(drain_deadline);
4986 }
4987 let pending = if drain_deadline.is_some() {
4988 forwarding.begin_drain_health_probe_rpc_for(
4989 module_id,
4990 MODULE_CONTROL_OP_HEALTH_CHECK,
4991 probe_started_at,
4992 deadline,
4993 )
4994 } else {
4995 forwarding.begin_health_probe_rpc_for(
4996 module_id,
4997 MODULE_CONTROL_OP_HEALTH_CHECK,
4998 probe_started_at,
4999 deadline,
5000 )
5001 }
5002 .map_err(|err| {
5003 HealthProbeError::lane_dead(format!("failed to begin health.check RPC: {err}"))
5006 })?;
5007 await_health_probe(forwarding, pending, deadline, runtime.health.deadline).await
5008}
5009
5010async fn probe_endpoint_health(
5017 endpoint: crate::ModuleEndpointId,
5018 runtime: &SupervisorRuntimeConfig,
5019 deadline_cap: Option<Instant>,
5020) -> Result<HealthReport, HealthProbeError> {
5021 let Some(forwarding) = runtime.forwarding.as_ref() else {
5022 return Err(HealthProbeError::misconfigured(
5023 "supervisor was not configured with a forwarding table",
5024 ));
5025 };
5026 let probe_started_at = Instant::now();
5027 let mut deadline = probe_started_at + runtime.health.deadline;
5028 if let Some(cap) = deadline_cap {
5029 deadline = deadline.min(cap);
5030 }
5031 let pending = forwarding
5032 .begin_endpoint_health_probe_rpc_for(
5033 endpoint,
5034 MODULE_CONTROL_OP_HEALTH_CHECK,
5035 probe_started_at,
5036 deadline,
5037 )
5038 .map_err(|err| {
5039 HealthProbeError::lane_dead(format!("failed to begin health.check RPC: {err}"))
5040 })?;
5041 await_health_probe(forwarding, pending, deadline, runtime.health.deadline).await
5042}
5043
5044async fn await_health_probe(
5046 forwarding: &ForwardingTable,
5047 pending: PendingModuleControlRpc,
5048 deadline: Instant,
5049 probe_budget: Duration,
5050) -> Result<HealthReport, HealthProbeError> {
5051 let PendingModuleControlRpc {
5052 endpoint,
5053 module_sink,
5054 negotiated_ver,
5055 corr,
5056 receiver,
5057 } = pending;
5058 let body = serde_json::to_vec(&ModuleControlRequest::HealthCheck {}).map_err(|err| {
5059 HealthProbeError::misconfigured(format!("failed to encode health.check: {err}"))
5060 })?;
5061 let frame = Frame::build_with_version(
5062 negotiated_ver,
5063 FrameType::Request,
5064 control_flags(),
5065 0,
5066 0,
5067 corr,
5068 body,
5069 )
5070 .map_err(|err| {
5071 HealthProbeError::misconfigured(format!("failed to build health.check frame: {err}"))
5072 })?;
5073
5074 match timeout_at(deadline, module_sink.send(frame)).await {
5080 Ok(Ok(())) => {}
5081 Ok(Err(err)) => {
5082 let _ = forwarding.cancel_module_control_rpc(endpoint, corr);
5083 return Err(HealthProbeError::lane_dead(format!(
5086 "failed to send health.check: {err}"
5087 )));
5088 }
5089 Err(_elapsed) => {
5090 let _ = forwarding.cancel_module_control_rpc(endpoint, corr);
5091 return Err(HealthProbeError::no_answer(
5095 "health.check send timed out before enqueue (module egress full)",
5096 ));
5097 }
5098 }
5099
5100 match timeout_at(deadline, receiver).await {
5101 Ok(Ok(ModuleControlRpcOutcome::Response(response))) => {
5105 response.health_report().ok_or_else(|| {
5106 HealthProbeError::bad_answer("health.check RPC returned a non-health response")
5107 })
5108 }
5109 Ok(Ok(ModuleControlRpcOutcome::Rejected(body))) => Err(HealthProbeError::bad_answer(
5110 format!("health.check rejected: {}", body.message),
5111 )),
5112 Ok(Ok(ModuleControlRpcOutcome::ModuleGone(message))) => {
5113 Err(HealthProbeError::lane_dead(message))
5114 }
5115 Ok(Ok(ModuleControlRpcOutcome::MalformedResponse(message))) => {
5116 Err(HealthProbeError::bad_answer(message))
5117 }
5118 Ok(Ok(ModuleControlRpcOutcome::UnexpectedOp { expected, actual })) => {
5119 Err(HealthProbeError::bad_answer(format!(
5120 "expected module-control op '{expected}', got '{actual}'"
5121 )))
5122 }
5123 Ok(Ok(ModuleControlRpcOutcome::DeadlineElapsed)) => Err(HealthProbeError::bad_answer(
5127 "module answered health.check after its daemon deadline",
5128 )),
5129 Ok(Err(_)) => Err(HealthProbeError::misconfigured(
5130 "health.check waiter was canceled before the module responded",
5131 )),
5132 Err(_) => {
5133 let _ = forwarding.tombstone_health_probe_rpc(endpoint, corr);
5134 Err(HealthProbeError::no_answer(format!(
5135 "module did not answer health.check within {probe_budget:?}"
5136 )))
5137 }
5138 }
5139}
5140
5141#[allow(clippy::too_many_arguments)]
5142async fn handle_health_report(
5143 spec: &ModuleSpec,
5144 runtime: &SupervisorRuntimeConfig,
5145 registry: &Registry,
5146 process_liveness: &SupervisorProcessLiveness,
5147 snapshot: &SharedSnapshot,
5148 child: &mut Option<SupervisedChild>,
5149 report: HealthReport,
5150 now_ms: u64,
5151) {
5152 let status = supervisor_health_status(report.status);
5153 let detail = report.detail.clone();
5154 let metrics = truncate_health_metrics(report.metrics);
5155 let _ = update_snapshot(snapshot, Some(&spec.module_id), |state| {
5156 state.health.status = status;
5157 state.health.last_probe_ms = Some(now_ms);
5158 state.health.detail = detail.clone();
5159 state.health.metrics = metrics.clone();
5160 state.health.consecutive_failures = 0;
5161 });
5162
5163 let action = match report.status {
5164 HealthStatus::Ok => return,
5165 HealthStatus::Degraded => runtime.health.on_degraded,
5166 HealthStatus::Failing => runtime.health.on_failing,
5167 };
5168 apply_l3_health_action(
5169 spec,
5170 runtime,
5171 registry,
5172 process_liveness,
5173 snapshot,
5174 child,
5175 status,
5176 detail.as_deref(),
5177 action,
5178 now_ms,
5179 )
5180 .await;
5181}
5182
5183#[allow(clippy::too_many_arguments)]
5184async fn handle_health_probe_failure(
5185 spec: &ModuleSpec,
5186 runtime: &SupervisorRuntimeConfig,
5187 registry: &Registry,
5188 process_liveness: &SupervisorProcessLiveness,
5189 snapshot: &SharedSnapshot,
5190 child: &mut Option<SupervisedChild>,
5191 err: HealthProbeError,
5192 now_ms: u64,
5193) {
5194 let threshold = runtime.health.failure_threshold.max(1);
5195 let mut failures = 0;
5196 let detail = format!("[{}] {err}", err.label());
5201 let _ = update_snapshot(snapshot, Some(&spec.module_id), |state| {
5202 if state.spawned_protocol.unwrap_or(spec.protocol) == ModuleProtocol::Subc {
5205 state.health.status = SupervisorHealthStatus::Unknown;
5206 }
5207 state.health.last_probe_ms = Some(now_ms);
5208 state.health.consecutive_failures = state.health.consecutive_failures.saturating_add(1);
5209 state.health.detail = Some(detail.clone());
5210 state.health.metrics = None;
5211 failures = state.health.consecutive_failures;
5212 });
5213
5214 if failures < threshold {
5215 warn!(
5216 module_id = %spec.module_id,
5217 consecutive_failures = failures,
5218 threshold,
5219 evidence = err.label(),
5220 detail = %detail,
5221 "health.check probe failed"
5222 );
5223 return;
5224 }
5225
5226 let _ = update_snapshot(snapshot, Some(&spec.module_id), |state| {
5227 state.state = ModuleState::Unresponsive;
5228 state.health.status = SupervisorHealthStatus::Unresponsive;
5229 });
5230 if runtime.health.critical {
5234 error!(
5235 module_id = %spec.module_id,
5236 status = "unresponsive",
5237 evidence = err.label(),
5238 detail = %detail,
5239 "critical module health alert"
5240 );
5241 } else {
5242 warn!(
5243 module_id = %spec.module_id,
5244 status = "unresponsive",
5245 evidence = err.label(),
5246 detail = %detail,
5247 "module health threshold breached"
5248 );
5249 }
5250 if let Err(err) = health_restart_child(
5251 spec,
5252 runtime,
5253 registry,
5254 process_liveness,
5255 snapshot,
5256 child,
5257 SupervisorHealthStatus::Unresponsive,
5258 Some(&detail),
5259 now_ms,
5260 )
5261 .await
5262 {
5263 error!(module_id = %spec.module_id, error = %err, "health-triggered restart failed");
5264 }
5265}
5266
5267#[allow(clippy::too_many_arguments)]
5268async fn apply_l3_health_action(
5269 spec: &ModuleSpec,
5270 runtime: &SupervisorRuntimeConfig,
5271 registry: &Registry,
5272 process_liveness: &SupervisorProcessLiveness,
5273 snapshot: &SharedSnapshot,
5274 child: &mut Option<SupervisedChild>,
5275 status: SupervisorHealthStatus,
5276 detail: Option<&str>,
5277 action: HealthAction,
5278 now_ms: u64,
5279) {
5280 record_health_action(snapshot, &spec.module_id, action.to_string(), now_ms);
5281 match action {
5282 HealthAction::Report => {
5283 info!(
5284 module_id = %spec.module_id,
5285 status = ?status,
5286 detail,
5287 "module reported non-ok health"
5288 );
5289 }
5290 HealthAction::Alert => {
5291 error!(
5292 module_id = %spec.module_id,
5293 status = ?status,
5294 detail,
5295 "module health alert"
5296 );
5297 }
5298 HealthAction::Restart => {
5299 if let Err(err) = health_restart_child(
5300 spec,
5301 runtime,
5302 registry,
5303 process_liveness,
5304 snapshot,
5305 child,
5306 status,
5307 detail,
5308 now_ms,
5309 )
5310 .await
5311 {
5312 error!(module_id = %spec.module_id, error = %err, "health-triggered restart failed");
5313 }
5314 }
5315 }
5316}
5317
5318#[allow(clippy::too_many_arguments)]
5319async fn health_restart_child(
5320 spec: &ModuleSpec,
5321 runtime: &SupervisorRuntimeConfig,
5322 registry: &Registry,
5323 process_liveness: &SupervisorProcessLiveness,
5324 snapshot: &SharedSnapshot,
5325 child: &mut Option<SupervisedChild>,
5326 status: SupervisorHealthStatus,
5327 detail: Option<&str>,
5328 now_ms: u64,
5329) -> Result<(), SuperviseError> {
5330 let (enabled, schedule) = {
5331 let mut state = lock_snapshot(snapshot)?;
5332 let enabled = state.enabled;
5333 let schedule = if enabled {
5334 state.next_crash_restart(&runtime.restart_policy, Instant::now())
5335 } else {
5336 None
5337 };
5338 (enabled, schedule)
5339 };
5340
5341 if !enabled {
5342 return Err(SuperviseError::Disabled {
5343 module_id: spec.module_id.clone(),
5344 });
5345 }
5346
5347 if schedule.is_none() {
5348 record_health_action(snapshot, &spec.module_id, "failed".to_string(), now_ms);
5349 error!(
5350 module_id = %spec.module_id,
5351 status = ?status,
5352 detail,
5353 max_restarts = runtime.restart_policy.max_restarts,
5354 window_secs = runtime.restart_policy.window.as_secs(),
5355 reason = %runtime.restart_policy.budget_exhausted_detail(),
5356 "health restart budget exhausted; marking module failed"
5357 );
5358 let stop_notice = begin_forwarding_drain_if_configured(
5359 spec,
5360 runtime,
5361 registry,
5362 snapshot,
5363 Some(true),
5364 RouteCloseReason::Disable,
5365 )
5366 .await?;
5367 update_snapshot(snapshot, Some(&spec.module_id), |state| {
5368 state.drain_disposition_detail = Some(runtime.restart_policy.budget_exhausted_detail());
5369 })?;
5370 drain_optional_child(
5371 &spec.module_id,
5372 spec.protocol,
5373 stop_notice,
5374 registry,
5375 runtime.forwarding.as_deref(),
5376 snapshot,
5377 &runtime.terminal_ring,
5378 &runtime.spawn_events,
5379 child,
5380 runtime.drain_timeout,
5381 ModuleState::Failed,
5382 Some(true),
5383 )
5384 .await?;
5385 process_liveness.untrack_if_current(&spec.module_id, snapshot);
5386 return Ok(());
5387 }
5388
5389 let schedule = schedule.expect("a health restart must have a crash-restart schedule");
5390 let mut restart_count = 0;
5391 update_snapshot(snapshot, Some(&spec.module_id), |state| {
5392 restart_count = state.crash_restarts.len();
5393 state.state = ModuleState::Unresponsive;
5394 state.health.status = status;
5395 state.health.last_action = Some(HealthAction::Restart.to_string());
5396 state.health.last_action_ms = Some(now_ms);
5397 })?;
5398 warn!(
5399 module_id = %spec.module_id,
5400 status = ?status,
5401 detail,
5402 restart_count,
5403 restart_in_window = schedule.restart_in_window,
5404 delay_ms = schedule.delay.as_millis() as u64,
5405 "health-triggered module restart"
5406 );
5407
5408 let stop_notice = begin_forwarding_drain_if_configured(
5409 spec,
5410 runtime,
5411 registry,
5412 snapshot,
5413 Some(true),
5414 RouteCloseReason::Restart,
5415 )
5416 .await?;
5417 drain_optional_child(
5418 &spec.module_id,
5419 spec.protocol,
5420 stop_notice,
5421 registry,
5422 runtime.forwarding.as_deref(),
5423 snapshot,
5424 &runtime.terminal_ring,
5425 &runtime.spawn_events,
5426 child,
5427 runtime.drain_timeout,
5428 ModuleState::Restarting,
5429 Some(true),
5430 )
5431 .await?;
5432 schedule_respawn(
5433 runtime,
5434 snapshot,
5435 &spec.module_id,
5436 schedule.delay,
5437 RespawnKind::Spawn,
5438 )
5439}
5440
5441fn cancel_deferred_reload(runtime: &SupervisorRuntimeConfig, module_id: &str, reason: &str) {
5442 if let Some(reply) = runtime
5443 .deferred_reload_reply
5444 .lock()
5445 .unwrap_or_else(|p| p.into_inner())
5446 .take()
5447 {
5448 let _ = reply.send(Err(SuperviseError::ReloadFailed {
5449 module_id: module_id.to_string(),
5450 reason: reason.to_string(),
5451 }));
5452 }
5453}
5454
5455fn schedule_respawn(
5456 runtime: &SupervisorRuntimeConfig,
5457 snapshot: &SharedSnapshot,
5458 module_id: &str,
5459 delay: Duration,
5460 kind: RespawnKind,
5461) -> Result<(), SuperviseError> {
5462 cancel_deferred_reload(runtime, module_id, "respawn superseded by another restart");
5463 update_snapshot(snapshot, Some(module_id), |state| {
5464 state.respawn_pending = true
5465 })?;
5466 *runtime
5467 .scheduled_respawn
5468 .lock()
5469 .unwrap_or_else(|p| p.into_inner()) = Some(PendingRespawn {
5470 deadline: Instant::now() + delay,
5471 kind,
5472 });
5473 Ok(())
5474}
5475
5476fn record_health_action(snapshot: &SharedSnapshot, module_id: &str, action: String, now_ms: u64) {
5477 let _ = update_snapshot(snapshot, Some(module_id), |state| {
5478 state.health.last_action = Some(action);
5479 state.health.last_action_ms = Some(now_ms);
5480 });
5481}
5482
5483fn supervisor_health_status(status: HealthStatus) -> SupervisorHealthStatus {
5484 match status {
5485 HealthStatus::Ok => SupervisorHealthStatus::Ok,
5486 HealthStatus::Degraded => SupervisorHealthStatus::Degraded,
5487 HealthStatus::Failing => SupervisorHealthStatus::Failing,
5488 }
5489}
5490
5491fn truncate_health_metrics(metrics: Option<Value>) -> Option<Value> {
5503 let metrics = metrics?;
5504 match serde_json::to_vec(&metrics) {
5505 Ok(encoded) if encoded.len() > MAX_HEALTH_METRICS_BYTES => Some(serde_json::json!({
5506 "truncated": true,
5507 "original_bytes": encoded.len(),
5508 })),
5509 Ok(_) | Err(_) => Some(metrics),
5510 }
5511}
5512
5513fn jittered_health_delay(module_id: &str, probe_index: u64, cadence: Duration) -> Duration {
5519 if cadence.is_zero() {
5520 return Duration::ZERO;
5521 }
5522 let cadence_ms = cadence.as_millis() as u64;
5523 if cadence_ms == 0 {
5539 return cadence;
5540 }
5541 let jitter_span = (cadence_ms / 10).max(1);
5556 let hash = module_id.as_bytes().iter().fold(
5557 probe_index.wrapping_mul(0x9E37_79B9_7F4A_7C15),
5558 |acc, byte| {
5559 acc.wrapping_mul(1099511628211)
5560 .wrapping_add(u64::from(*byte))
5561 },
5562 );
5563 cadence + Duration::from_millis(hash % jitter_span)
5564}
5565
5566#[cfg(test)]
5567mod tests {
5568 use super::*;
5569
5570 #[test]
5571 fn readding_a_module_clears_its_rescan_removal_tombstone() {
5572 let handle = SupervisorHandle::new();
5573 let module_id = "readded-tombstone";
5574 handle.record_rescan_removal(module_id);
5575 assert!(handle.removal_tombstone_age_ms(module_id).is_some());
5576
5577 handle.apply_identity_configuration(&ModuleSpec {
5578 module_id: module_id.to_string(),
5579 program: PathBuf::from("/test/module"),
5580 args: Vec::new(),
5581 env: Vec::new(),
5582 reserved: false,
5583 reserved_prefixes: Vec::new(),
5584 protocol: ModuleProtocol::Subc,
5585 overlap: Default::default(),
5586 });
5587
5588 assert!(
5589 handle.removal_tombstone_age_ms(module_id).is_none(),
5590 "a re-added module must not retain a stale removal tombstone"
5591 );
5592 }
5593
5594 #[derive(Debug, PartialEq, Eq)]
5597 struct OwnerInSpawnWindow {
5598 module_id: String,
5599 configured: bool,
5600 on_roster: bool,
5601 admission_refusal: Option<&'static str>,
5602 }
5603
5604 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
5615 async fn a_new_module_is_configured_before_its_first_process_can_register() {
5616 use crate::scopes::ScopeTable;
5617 use subc_protocol::{error_codes, scope::ScopeSelector, Principal};
5618
5619 let dir = cortexkit_test_support::ScratchDir::new("configured-before-spawn");
5620 let stub = |module_id: &str, program: PathBuf| ModuleSpec {
5621 module_id: module_id.to_string(),
5622 program,
5623 args: Vec::new(),
5624 env: ["XDG_DATA_HOME", "XDG_RUNTIME_DIR", "XDG_CONFIG_HOME"]
5625 .into_iter()
5626 .map(|key| (key.to_string(), dir.path().display().to_string()))
5627 .chain([("FAKE_AFT_NEVER_CONNECT".to_string(), "1".to_string())])
5628 .collect(),
5629 reserved: false,
5630 reserved_prefixes: Vec::new(),
5631 protocol: ModuleProtocol::Subc,
5632 overlap: Default::default(),
5633 };
5634 let live = super::terminal_history_tests::fake_aft_stub_path();
5635 let missing = dir.path().join("definitely-missing-module");
5636
5637 let handle = SupervisorHandle::new();
5638 let mut supervisor =
5639 Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default())
5640 .with_handle(handle.clone());
5641 let observed = Arc::new(Mutex::new(Vec::<OwnerInSpawnWindow>::new()));
5642 let hook_handle = handle.clone();
5643 let hook_observed = Arc::clone(&observed);
5644 supervisor.test_after_first_spawn = AfterFirstSpawnHook(Some(Arc::new(move |module_id| {
5645 let configured = hook_handle.is_configured(module_id);
5648 let selector = ScopeSelector::new(
5649 Principal::Reserved {
5650 module_id: module_id.to_string(),
5651 },
5652 "s".to_string(),
5653 Some(1),
5654 );
5655 let carrier = Principal::Reserved {
5656 module_id: "carrier".to_string(),
5657 };
5658 let admission_refusal = match ScopeTable::new(Vec::<String>::new())
5659 .admit(&carrier, module_id, &selector, configured)
5660 {
5661 Ok(_) => None,
5662 Err(refusal) => Some(refusal.code),
5663 };
5664 hook_observed.lock().unwrap().push(OwnerInSpawnWindow {
5665 module_id: module_id.to_string(),
5666 configured,
5667 on_roster: hook_handle.get(module_id).is_some(),
5668 admission_refusal,
5669 });
5670 })));
5671
5672 let plain = supervisor.spawn(stub("plain", live.clone())).unwrap();
5673 let configured = supervisor
5674 .supervise_configured(stub("configured", live.clone()), true)
5675 .unwrap();
5676 let with_health = supervisor
5677 .supervise_configured_with_health(
5678 stub("with-health", live.clone()),
5679 true,
5680 HealthConfig::default(),
5681 None,
5682 RestartPolicy::default(),
5683 )
5684 .unwrap();
5685 let failed = supervisor
5688 .supervise_configured_with_health(
5689 stub("failed-spawn", missing.clone()),
5690 true,
5691 HealthConfig::default(),
5692 None,
5693 RestartPolicy::default(),
5694 )
5695 .unwrap();
5696 assert!(supervisor.spawn(stub("spawn-error", missing)).is_err());
5699
5700 let expected = [
5701 "plain",
5702 "configured",
5703 "with-health",
5704 "failed-spawn",
5705 "spawn-error",
5706 ]
5707 .into_iter()
5708 .map(|module_id| OwnerInSpawnWindow {
5709 module_id: module_id.to_string(),
5710 configured: true,
5711 on_roster: !matches!(module_id, "plain" | "spawn-error"),
5712 admission_refusal: Some(error_codes::SCOPE_NOT_SYNCED),
5713 })
5714 .collect::<Vec<_>>();
5715 assert_eq!(*observed.lock().unwrap(), expected);
5716
5717 for module_id in ["plain", "configured", "with-health", "failed-spawn"] {
5718 assert!(
5719 handle.get(module_id).is_some(),
5720 "{module_id} is on the roster"
5721 );
5722 assert!(
5723 handle.is_configured(module_id),
5724 "{module_id} stays configured"
5725 );
5726 }
5727 assert!(handle.get("spawn-error").is_none());
5728 assert!(
5729 !handle.is_configured("spawn-error"),
5730 "a plain spawn that failed must not leave its module marked configured"
5731 );
5732
5733 handle.retire("failed-spawn");
5735 assert!(!handle.is_configured("failed-spawn"));
5736
5737 for module in [plain, configured, with_health] {
5738 module.stop().await.unwrap();
5739 }
5740 drop(failed);
5741 }
5742
5743 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
5750 async fn a_configured_module_reads_starting_while_its_first_process_already_runs() {
5751 let dir = cortexkit_test_support::ScratchDir::new("starting-while-spawned");
5752 let spec = ModuleSpec {
5753 module_id: "spawned-not-admitted".to_string(),
5754 program: super::terminal_history_tests::fake_aft_stub_path(),
5755 args: Vec::new(),
5756 env: ["XDG_DATA_HOME", "XDG_RUNTIME_DIR", "XDG_CONFIG_HOME"]
5757 .into_iter()
5758 .map(|key| (key.to_string(), dir.path().display().to_string()))
5759 .chain([("FAKE_AFT_NEVER_CONNECT".to_string(), "1".to_string())])
5760 .collect(),
5761 reserved: false,
5762 reserved_prefixes: Vec::new(),
5763 protocol: ModuleProtocol::Subc,
5764 overlap: Default::default(),
5765 };
5766 let handle = SupervisorHandle::new();
5767 let mut supervisor =
5768 Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default())
5769 .with_handle(handle.clone());
5770 let observed = Arc::new(Mutex::new(None));
5771 let hook_handle = handle.clone();
5772 let hook_observed = Arc::clone(&observed);
5773 supervisor.test_after_first_spawn = AfterFirstSpawnHook(Some(Arc::new(move |module_id| {
5774 let state = hook_handle
5775 .get(module_id)
5776 .expect("a configured module is on the roster before it is spawned")
5777 .state()
5778 .unwrap();
5779 *hook_observed.lock().unwrap() = Some(state);
5780 })));
5781
5782 let module = supervisor.supervise_configured(spec, true).unwrap();
5783 assert_eq!(
5784 *observed.lock().unwrap(),
5785 Some(ModuleState::Starting),
5786 "between the spawn and its admission the supervisor still reports starting"
5787 );
5788 assert_eq!(module.state().unwrap(), ModuleState::Running);
5789 module.stop().await.unwrap();
5790 }
5791
5792 fn stale_process_snapshot(state: ModuleState, enabled: bool) -> SharedSnapshot {
5793 let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::new(state, enabled)));
5794 update_snapshot(&snapshot, Some("stale-process-facts"), |snapshot| {
5795 snapshot.process_alive = true;
5796 snapshot.pid = Some(41);
5797 snapshot.spawned_at_ms = Some(42);
5798 snapshot.spawned_from = Some(PathBuf::from("/spawned/module"));
5799 snapshot.launch_spec_digest = Some(LaunchSpecDigest([0xA5; 32]));
5800 snapshot.spawned_file_identity = Some(SpawnedFileIdentity {
5801 device: 43,
5802 inode: 44,
5803 #[cfg(windows)]
5804 windows_image: None,
5805 });
5806 })
5807 .unwrap();
5808 snapshot
5809 }
5810
5811 fn assert_snapshot_process_facts_cleared(snapshot: &SharedSnapshot) {
5812 let snapshot = lock_snapshot(snapshot).unwrap();
5813 assert!(!snapshot.process_alive);
5814 assert_eq!(snapshot.pid, None);
5815 assert_eq!(snapshot.spawned_at_ms, None);
5816 assert_eq!(snapshot.spawned_from, None);
5817 assert_eq!(snapshot.launch_spec_digest, None);
5818 assert_eq!(snapshot.spawned_file_identity, None);
5819 }
5820
5821 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
5822 async fn failed_enable_spawn_clears_preexisting_current_process_facts() {
5823 let supervisor =
5824 Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default());
5825 let mut runtime = supervisor.runtime_config();
5826 runtime.test_seed_stale_facts_before_enable_spawn = true;
5827 let snapshot = stale_process_snapshot(ModuleState::Disabled, false);
5828 let mut child = None;
5829 let spec = ModuleSpec {
5830 module_id: "failed-enable-clears-facts".to_string(),
5831 program: PathBuf::from("/definitely/missing/failed-enable-module"),
5832 args: Vec::new(),
5833 env: Vec::new(),
5834 reserved: false,
5835 reserved_prefixes: Vec::new(),
5836 protocol: ModuleProtocol::Subc,
5837 overlap: Default::default(),
5838 };
5839
5840 let result = set_child_enabled(
5841 &spec,
5842 &runtime,
5843 &supervisor.registry,
5844 &supervisor.process_liveness,
5845 &snapshot,
5846 &mut child,
5847 true,
5848 )
5849 .await;
5850
5851 assert!(matches!(result, Err(SuperviseError::Spawn { .. })));
5852 assert_eq!(lock_snapshot(&snapshot).unwrap().state, ModuleState::Failed);
5853 assert_snapshot_process_facts_cleared(&snapshot);
5854 }
5855
5856 #[tokio::test]
5857 async fn start_revives_stranded_restarting_but_not_pending_backoff() {
5858 let supervisor =
5859 Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default());
5860 let runtime = supervisor.runtime_config();
5861 let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::new(
5862 ModuleState::Restarting,
5863 true,
5864 )));
5865 let spec = ModuleSpec {
5866 module_id: "start-stranded-restarting".to_string(),
5867 program: super::terminal_history_tests::fake_aft_stub_path(),
5868 args: Vec::new(),
5869 env: vec![("FAKE_AFT_NEVER_CONNECT".to_string(), "1".to_string())],
5870 reserved: false,
5871 reserved_prefixes: Vec::new(),
5872 protocol: ModuleProtocol::None,
5873 overlap: Default::default(),
5874 };
5875 let mut child = None;
5876 lock_snapshot(&snapshot).unwrap().respawn_pending = true;
5877 assert!(!super::set_child_enabled(
5878 &spec,
5879 &runtime,
5880 &Registry::default(),
5881 &supervisor.process_liveness,
5882 &snapshot,
5883 &mut child,
5884 true
5885 )
5886 .await
5887 .unwrap());
5888 assert!(child.is_none());
5889 lock_snapshot(&snapshot).unwrap().respawn_pending = false;
5890 assert!(super::set_child_enabled(
5891 &spec,
5892 &runtime,
5893 &Registry::default(),
5894 &supervisor.process_liveness,
5895 &snapshot,
5896 &mut child,
5897 true
5898 )
5899 .await
5900 .unwrap());
5901 assert_eq!(
5902 lock_snapshot(&snapshot).unwrap().state,
5903 ModuleState::Running
5904 );
5905 let mut child = child.unwrap();
5906 child.start_kill().unwrap();
5907 child.wait().await.unwrap();
5908 }
5909
5910 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
5911 async fn failed_reload_spawn_clears_current_process_facts() {
5912 let supervisor =
5913 Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default());
5914 let mut runtime = supervisor.runtime_config();
5915 runtime.restart_policy = RestartPolicy::new(0, Duration::ZERO);
5916 let snapshot = stale_process_snapshot(ModuleState::Running, true);
5917 let mut child = None;
5918 let spec = ModuleSpec {
5919 module_id: "failed-reload-clears-facts".to_string(),
5920 program: PathBuf::from("/unused/failed-reload-module"),
5921 args: Vec::new(),
5922 env: Vec::new(),
5923 reserved: false,
5924 reserved_prefixes: Vec::new(),
5925 protocol: ModuleProtocol::Subc,
5926 overlap: Default::default(),
5927 };
5928
5929 let result = handle_reload_spawn_failure(
5930 &spec,
5931 &runtime,
5932 &supervisor.process_liveness,
5933 &snapshot,
5934 &mut child,
5935 "forced reload spawn failure".to_string(),
5936 )
5937 .await;
5938
5939 assert!(matches!(result, Err(SuperviseError::ReloadFailed { .. })));
5940 assert_eq!(lock_snapshot(&snapshot).unwrap().state, ModuleState::Failed);
5941 assert_snapshot_process_facts_cleared(&snapshot);
5942 }
5943
5944 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
5945 async fn dropping_a_module_with_an_active_monitor_clears_current_process_facts() {
5946 let supervisor =
5947 Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default());
5948 let snapshot = stale_process_snapshot(ModuleState::Running, true);
5949 let module = supervisor.supervised_module(
5950 ModuleSpec {
5951 module_id: "drop-clears-facts".to_string(),
5952 program: PathBuf::from("/unused/drop-module"),
5953 args: Vec::new(),
5954 env: Vec::new(),
5955 reserved: false,
5956 reserved_prefixes: Vec::new(),
5957 protocol: ModuleProtocol::Subc,
5958 overlap: Default::default(),
5959 },
5960 supervisor.runtime_config(),
5961 Arc::clone(&snapshot),
5962 None,
5963 );
5964 assert!(!module
5965 .inner
5966 .monitor
5967 .lock()
5968 .unwrap()
5969 .as_ref()
5970 .unwrap()
5971 .is_finished());
5972
5973 drop(module);
5974
5975 assert_eq!(
5976 lock_snapshot(&snapshot).unwrap().state,
5977 ModuleState::Stopped
5978 );
5979 assert_snapshot_process_facts_cleared(&snapshot);
5980 }
5981
5982 #[cfg(unix)]
5983 #[tokio::test]
5984 async fn rescan_preserves_running_protocol_until_respawn() {
5985 let dir = cortexkit_test_support::ScratchDir::new("rescan-protocol");
5986 let initial = ModuleSpec {
5987 module_id: "rescan-protocol".into(),
5988 program: PathBuf::from("/bin/sleep"),
5989 args: vec!["60".into()],
5990 env: ["XDG_DATA_HOME", "XDG_RUNTIME_DIR", "XDG_CONFIG_HOME"]
5991 .into_iter()
5992 .map(|key| (key.into(), dir.path().to_string_lossy().into_owned()))
5993 .collect(),
5994 reserved: false,
5995 reserved_prefixes: vec![],
5996 protocol: ModuleProtocol::None,
5997 overlap: Default::default(),
5998 };
5999 let supervisor =
6000 Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default());
6001 let module = supervisor.spawn(initial.clone()).unwrap();
6002 assert!(module.status().unwrap().live);
6003 let mut next = initial;
6004 next.protocol = ModuleProtocol::Subc;
6005 module
6006 .update_configuration(next.clone(), HealthConfig::default(), None)
6007 .await
6008 .unwrap();
6009 assert!(
6010 module.status().unwrap().live,
6011 "rescan must not require HELLO from the old non-wire process"
6012 );
6013 let runtime = supervisor.runtime_config();
6014 let action = on_child_exit(
6015 &next,
6016 RestartPolicy::default(),
6017 &supervisor.registry,
6018 &module.inner.snapshot,
6019 &runtime.terminal_ring,
6020 &runtime.spawn_events,
6021 &runtime.child_roster,
6022 ExitReport {
6023 kind: ExitKind::Clean,
6024 code: Some(0),
6025 signal: None,
6026 at_ms: unix_ms_now(),
6027 },
6028 )
6029 .await;
6030 assert!(
6031 matches!(action, NextAction::Restart { .. }),
6032 "the old non-wire process's clean exit must restart"
6033 );
6034 module.drain().await.unwrap();
6035 }
6036
6037 #[cfg(unix)]
6038 #[tokio::test]
6039 async fn orphan_identity_matches_path_names_and_shebang_interpreters() {
6040 use std::os::unix::fs::PermissionsExt;
6041 let dir = cortexkit_test_support::ScratchDir::new("spawn-image-identity");
6042 let script = dir.join("module.sh");
6043 std::fs::write(&script, "#!/bin/sh\nwhile :; do sleep 0.1; done\n").unwrap();
6044 std::fs::set_permissions(&script, std::fs::Permissions::from_mode(0o700)).unwrap();
6045 let record_path = dir.join("live-children.json");
6046 let supervisor =
6047 Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default())
6048 .with_live_children_record(&record_path);
6049 for (program, args) in [
6050 (PathBuf::from("sleep"), vec!["60".into()]),
6051 (script, vec![]),
6052 ] {
6053 let spec = ModuleSpec {
6054 module_id: "image-identity".into(),
6055 program: program.clone(),
6056 args,
6057 env: ["XDG_DATA_HOME", "XDG_RUNTIME_DIR", "XDG_CONFIG_HOME"]
6058 .into_iter()
6059 .map(|key| (key.into(), dir.to_string_lossy().into_owned()))
6060 .collect(),
6061 reserved: false,
6062 reserved_prefixes: vec![],
6063 protocol: ModuleProtocol::None,
6064 overlap: Default::default(),
6065 };
6066 let module = supervisor.spawn(spec).unwrap();
6067 #[cfg(target_os = "macos")]
6068 {
6069 let deadline = Instant::now() + Duration::from_secs(5);
6072 while crate::live_children::read_record(&record_path)
6073 .unwrap()
6074 .iter()
6075 .all(|entry| entry.executable.is_none())
6076 {
6077 assert!(Instant::now() < deadline, "module image was not confirmed");
6078 tokio::time::sleep(Duration::from_millis(5)).await;
6079 }
6080 }
6081 let entry = crate::live_children::read_record(&record_path)
6082 .unwrap()
6083 .pop()
6084 .unwrap();
6085 let observed = subc_os::Process::open(entry.pid)
6086 .unwrap()
6087 .unwrap()
6088 .observe()
6089 .unwrap();
6090 let verdict = crate::live_children::identity_verdict(&entry, entry.pid, &observed);
6091 module.drain().await.unwrap();
6092 assert_eq!(
6093 verdict,
6094 crate::live_children::IdentityVerdict::Matches,
6095 "program {program:?}: recorded {entry:?}, observed {observed:?}"
6096 );
6097 }
6098 }
6099
6100 #[cfg(unix)]
6101 fn http_fixture(
6102 dir: &std::path::Path,
6103 url: &str,
6104 threshold: u32,
6105 ) -> crate::daemon_config::ConfiguredModule {
6106 let path = dir.join("subc.jsonc");
6107 std::fs::write(&path, serde_json::json!({"version": 1, "modules": {"http-process": {
6108 "program": "/bin/sleep", "args": ["60"], "protocol": "none",
6109 "env": {"XDG_DATA_HOME":dir,"XDG_RUNTIME_DIR":dir,"XDG_CONFIG_HOME":dir},
6110 "health": {"http":url,"cadence_ms":20,"deadline_ms":200,"failure_threshold":threshold},
6111 "restart": {"backoff_ms":1,"max_backoff_ms":1}, "drain_timeout_ms":10
6112 }}}).to_string()).unwrap();
6113 crate::daemon_config::load(&path)
6114 .unwrap()
6115 .unwrap()
6116 .modules
6117 .pop()
6118 .unwrap()
6119 }
6120
6121 #[cfg(unix)]
6122 async fn wait_http_health(module: &SupervisedModule, status: SupervisorHealthStatus) {
6123 timeout(Duration::from_secs(5), async {
6124 loop {
6125 if module.status().unwrap().health.status == status {
6126 break;
6127 }
6128 sleep(Duration::from_millis(5)).await;
6129 }
6130 })
6131 .await
6132 .unwrap_or_else(|_| {
6133 panic!(
6134 "expected {status:?}, got {:?}",
6135 module.status().unwrap().health
6136 )
6137 });
6138 }
6139
6140 #[cfg(unix)]
6141 #[tokio::test]
6142 async fn http_health_status_flips_ok_failing_ok() {
6143 use tokio::io::{AsyncReadExt, AsyncWriteExt};
6144 let dir = cortexkit_test_support::ScratchDir::new("http-status-flips");
6145 let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
6146 let status = Arc::new(std::sync::atomic::AtomicU16::new(200));
6147 let serving_status = status.clone();
6148 let url = format!("http://{}/healthz", listener.local_addr().unwrap());
6149 let server = tokio::spawn(async move {
6150 loop {
6151 let (mut stream, _) = listener.accept().await.unwrap();
6152 let mut request = [0u8; 2048];
6153 let count = stream.read(&mut request).await.unwrap();
6154 assert!(count > 0, "a probe must send an HTTP request");
6155 let code = serving_status.load(std::sync::atomic::Ordering::SeqCst);
6156 let body = if code == 200 {
6157 "ready"
6158 } else {
6159 "{\"status\":\"unavailable\",\"error\":\"scratch failure\"}"
6160 };
6161 let response = format!("HTTP/1.1 {code} scratch\r\nContent-Length: {}\r\nConnection: close\r\n\r\n{body}", body.len());
6162 let _ = stream.write_all(response.as_bytes()).await;
6163 }
6164 });
6165 let configured = http_fixture(&dir, &url, 1000);
6166 let module =
6167 Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default())
6168 .supervise_configured_with_health(
6169 configured.module_spec(),
6170 true,
6171 configured.health,
6172 configured.drain_timeout_ms,
6173 configured.restart,
6174 )
6175 .unwrap();
6176 wait_http_health(&module, SupervisorHealthStatus::Ok).await;
6177 status.store(503, std::sync::atomic::Ordering::SeqCst);
6178 wait_http_health(&module, SupervisorHealthStatus::Failing).await;
6179 assert!(module
6180 .status()
6181 .unwrap()
6182 .health
6183 .detail
6184 .unwrap()
6185 .contains("scratch failure"));
6186 status.store(200, std::sync::atomic::Ordering::SeqCst);
6187 wait_http_health(&module, SupervisorHealthStatus::Ok).await;
6188 assert_eq!(module.status().unwrap().health.consecutive_failures, 0);
6189 let before = module.status().unwrap();
6190 let (spec, mut health) = module.configuration().unwrap();
6191 health.http = None;
6192 module
6193 .update_configuration(spec.clone(), health.clone(), Some(10))
6194 .await
6195 .unwrap();
6196 wait_http_health(&module, SupervisorHealthStatus::Unknown).await;
6197 health.http = Some(url);
6198 module
6199 .update_configuration(spec, health, Some(10))
6200 .await
6201 .unwrap();
6202 wait_http_health(&module, SupervisorHealthStatus::Ok).await;
6203 assert_eq!(
6204 module.status().unwrap().pid,
6205 before.pid,
6206 "changing a probe must apply live, not restart its process"
6207 );
6208 let (spec, mut health) = module.configuration().unwrap();
6209 health.failure_threshold = 2;
6210 module
6211 .update_configuration(spec, health, Some(10))
6212 .await
6213 .unwrap();
6214 status.store(503, std::sync::atomic::Ordering::SeqCst);
6215 timeout(Duration::from_secs(5), async {
6216 while module.status().unwrap().spawn_generation == before.spawn_generation {
6217 sleep(Duration::from_millis(5)).await;
6218 }
6219 })
6220 .await
6221 .expect("sustained HTTP 503 responses must trigger the health restart policy");
6222 module.drain().await.unwrap();
6223 server.abort();
6224 }
6225
6226 #[cfg(unix)]
6227 #[tokio::test]
6228 async fn http_health_replacement_recovers_after_one_restart() {
6229 use tokio::io::{AsyncReadExt, AsyncWriteExt};
6230 let dir = cortexkit_test_support::ScratchDir::new("http-replacement-recovers");
6231 let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
6232 let url = format!("http://{}/healthz", listener.local_addr().unwrap());
6233 let mut configured = http_fixture(&dir, &url, 2);
6234 configured.restart.max_restarts = 2;
6235 let module =
6236 Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default())
6237 .supervise_configured_with_health(
6238 configured.module_spec(),
6239 true,
6240 configured.health,
6241 configured.drain_timeout_ms,
6242 configured.restart,
6243 )
6244 .unwrap();
6245 let first_generation = module.status().unwrap().spawn_generation;
6246 let observed = module.clone();
6247 let (healthy_tx, mut healthy_rx) = mpsc::channel(8);
6248 let server = tokio::spawn(async move {
6249 let mut generation = first_generation;
6250 let mut probes = 0;
6251 loop {
6252 let (mut stream, _) = listener.accept().await.unwrap();
6253 let mut request = [0; 2048];
6254 assert!(stream.read(&mut request).await.unwrap() > 0);
6255 let current = observed.status().unwrap().spawn_generation;
6256 if current != generation {
6257 generation = current;
6258 probes = 0;
6259 }
6260 probes += 1;
6261 let healthy = generation != first_generation && probes > 1;
6264 let code = if healthy { 200 } else { 503 };
6265 let response = format!(
6266 "HTTP/1.1 {code} scratch\r\nContent-Length: 0\r\nConnection: close\r\n\r\n"
6267 );
6268 let _ = stream.write_all(response.as_bytes()).await;
6269 if healthy {
6270 healthy_tx.send(()).await.unwrap();
6271 }
6272 }
6273 });
6274 timeout(Duration::from_secs(5), async {
6275 let mut healthy_probes = 0;
6276 loop {
6277 if module.state().unwrap() == ModuleState::Failed {
6278 break;
6279 }
6280 while healthy_rx.try_recv().is_ok() {
6281 healthy_probes += 1;
6282 }
6283 if healthy_probes >= 3
6284 && module.status().unwrap().health.status == SupervisorHealthStatus::Ok
6285 {
6286 break;
6287 }
6288 sleep(Duration::from_millis(5)).await;
6289 }
6290 })
6291 .await
6292 .expect("the replacement must settle healthy without exhausting its restart budget");
6293 let status = module.status().unwrap();
6294 module.drain().await.unwrap();
6295 server.abort();
6296 assert_eq!(status.state, ModuleState::Running);
6297 assert_eq!(status.health.status, SupervisorHealthStatus::Ok);
6298 assert_eq!(status.health.consecutive_failures, 0);
6299 assert_eq!(status.lifetime_restarts, 1);
6300 assert_eq!(status.restart_count, 1);
6301 }
6302
6303 #[cfg(unix)]
6304 #[tokio::test]
6305 async fn http_health_sustained_failure_exhausts_restart_budget() {
6306 use std::collections::BTreeMap;
6307 use tokio::io::{AsyncReadExt, AsyncWriteExt};
6308 let dir = cortexkit_test_support::ScratchDir::new("http-health-budget");
6309 let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
6310 let url = format!("http://{}/healthz", listener.local_addr().unwrap());
6311 let mut configured = http_fixture(&dir, &url, 2);
6312 configured.restart.max_restarts = 2;
6313 let module =
6314 Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default())
6315 .supervise_configured_with_health(
6316 configured.module_spec(),
6317 true,
6318 configured.health,
6319 configured.drain_timeout_ms,
6320 configured.restart,
6321 )
6322 .unwrap();
6323 let observed = module.clone();
6324 let probes = Arc::new(Mutex::new(BTreeMap::<u64, u32>::new()));
6325 let served = probes.clone();
6326 let server = tokio::spawn(async move {
6327 loop {
6328 let (mut stream, _) = listener.accept().await.unwrap();
6329 let mut request = [0; 2048];
6330 assert!(stream.read(&mut request).await.unwrap() > 0);
6331 *served
6332 .lock()
6333 .unwrap()
6334 .entry(observed.status().unwrap().spawn_generation)
6335 .or_default() += 1;
6336 let _ = stream
6337 .write_all(b"HTTP/1.1 503 Unavailable\r\nContent-Length: 0\r\nConnection: close\r\n\r\n")
6338 .await;
6339 }
6340 });
6341 timeout(Duration::from_secs(5), async {
6342 while module.state().unwrap() != ModuleState::Failed {
6343 sleep(Duration::from_millis(5)).await;
6344 }
6345 })
6346 .await
6347 .expect("sustained failures must still exhaust the health restart budget");
6348 let status = module.status().unwrap();
6349 server.abort();
6350 assert_eq!(status.restart_count, 2);
6351 assert_eq!(status.lifetime_restarts, 2);
6352 assert!(!status.process_alive);
6353 assert_eq!(
6354 probes.lock().unwrap().values().copied().collect::<Vec<_>>(),
6355 vec![2, 2, 2]
6356 );
6357 }
6358
6359 #[cfg(unix)]
6360 #[tokio::test]
6361 async fn spawn_clears_the_previous_process_health() {
6362 let dir = cortexkit_test_support::ScratchDir::new("spawn-fresh-health");
6363 let mut configured = http_fixture(&dir, "http://127.0.0.1:1/healthz", 2);
6364 configured.health.cadence = Duration::from_secs(3600);
6365 let module =
6366 Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default())
6367 .supervise_configured_with_health(
6368 configured.module_spec(),
6369 false,
6370 configured.health,
6371 configured.drain_timeout_ms,
6372 configured.restart,
6373 )
6374 .unwrap();
6375 update_snapshot(&module.inner.snapshot, Some(module.module_id()), |state| {
6376 state.health.status = SupervisorHealthStatus::Unresponsive;
6377 state.health.consecutive_failures = 3;
6378 state.health.detail = Some("old process refused connections".into());
6379 state.health.metrics = Some(serde_json::json!({"old": true}));
6380 state.health.last_probe_ms = Some(123);
6381 state.health.last_action = Some("restart".into());
6382 state.health.last_action_ms = Some(124);
6383 })
6384 .unwrap();
6385 module.set_enabled(true).await.unwrap();
6386 let health = module.status().unwrap().health;
6387 module.drain().await.unwrap();
6388 assert_eq!(health.consecutive_failures, 0);
6389 assert_eq!(health.status, SupervisorHealthStatus::Unknown);
6390 assert_eq!(health.detail, None);
6391 assert_eq!(health.metrics, None);
6392 assert_eq!(health.last_probe_ms, None);
6393 assert_eq!(health.last_action.as_deref(), Some("restart"));
6394 assert_eq!(health.last_action_ms, Some(124));
6395 }
6396
6397 #[cfg(unix)]
6398 #[tokio::test]
6399 async fn http_health_no_listener_restarts_after_consecutive_failures() {
6400 let dir = cortexkit_test_support::ScratchDir::new("http-refused");
6401 let unused = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
6402 let url = format!("http://{}/healthz", unused.local_addr().unwrap());
6403 drop(unused);
6404 let configured = http_fixture(&dir, &url, 2);
6405 let module =
6406 Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default())
6407 .supervise_configured_with_health(
6408 configured.module_spec(),
6409 true,
6410 configured.health,
6411 configured.drain_timeout_ms,
6412 configured.restart,
6413 )
6414 .unwrap();
6415 let before = module.status().unwrap().spawn_generation;
6416 timeout(Duration::from_secs(5), async {
6417 loop {
6418 let status = module.status().unwrap();
6419 if status.spawn_generation > before {
6420 assert!(status.lifetime_restarts > 0);
6421 break;
6422 }
6423 sleep(Duration::from_millis(5)).await;
6424 }
6425 })
6426 .await
6427 .expect("sustained HTTP refusal must trigger the health restart policy");
6428 module.drain().await.unwrap();
6429 }
6430
6431 #[cfg(unix)]
6432 #[tokio::test]
6433 async fn http_health_timeout_honours_deadline() {
6434 let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
6435 let url = format!("http://{}/healthz", listener.local_addr().unwrap());
6436 let server = tokio::spawn(async move {
6437 let _held = listener.accept().await.unwrap();
6438 std::future::pending::<()>().await;
6439 });
6440 let error = timeout(
6441 Duration::from_secs(1),
6442 probe_http_health(&url, Duration::from_millis(10)),
6443 )
6444 .await
6445 .expect("the probe must enforce its own deadline")
6446 .unwrap_err();
6447 server.abort();
6448 assert!(error.to_string().contains("timed out"));
6449 }
6450
6451 #[cfg(unix)]
6452 #[tokio::test]
6453 async fn http_health_localhost_reaches_an_ipv6_only_listener() {
6454 use tokio::io::{AsyncReadExt, AsyncWriteExt};
6455 let listener = tokio::net::TcpListener::bind("[::1]:0").await.unwrap();
6456 let url = format!(
6457 "http://localhost:{}/healthz",
6458 listener.local_addr().unwrap().port()
6459 );
6460 let server = tokio::spawn(async move {
6461 let (mut stream, _) = listener.accept().await.unwrap();
6462 let mut request = [0u8; 2048];
6463 assert!(stream.read(&mut request).await.unwrap() > 0);
6464 stream
6465 .write_all(b"HTTP/1.1 200 OK\r\nContent-Length: 2\r\n\r\nok")
6466 .await
6467 .unwrap();
6468 });
6469 assert_eq!(
6474 probe_http_health(&url, Duration::from_secs(10))
6475 .await
6476 .unwrap()
6477 .status,
6478 HealthStatus::Ok
6479 );
6480 server.await.unwrap();
6481 }
6482
6483 #[cfg(unix)]
6484 #[tokio::test]
6485 async fn http_health_timeout_keeps_partial_status_and_body() {
6486 use tokio::io::{AsyncReadExt, AsyncWriteExt};
6487 let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
6488 let url = format!("http://{}/healthz", listener.local_addr().unwrap());
6489 let server = tokio::spawn(async move {
6490 let (mut stream, _) = listener.accept().await.unwrap();
6491 let mut request = [0u8; 2048];
6492 assert!(stream.read(&mut request).await.unwrap() > 0);
6493 stream
6494 .write_all(
6495 b"HTTP/1.1 503 Unavailable\r\nContent-Length: 100\r\n\r\npartial diagnostic",
6496 )
6497 .await
6498 .unwrap();
6499 std::future::pending::<()>().await;
6500 });
6501 let error = probe_http_health(&url, Duration::from_secs(1))
6502 .await
6503 .unwrap_err()
6504 .to_string();
6505 server.abort();
6506 assert!(
6507 error.contains("timed out")
6508 && error.contains("503 Unavailable")
6509 && error.contains("partial diagnostic"),
6510 "{error}"
6511 );
6512 }
6513
6514 #[cfg(unix)]
6515 #[tokio::test]
6516 async fn http_health_diagnostic_body_is_bounded_to_200_bytes() {
6517 use tokio::io::{AsyncReadExt, AsyncWriteExt};
6518 let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
6519 let url = format!("http://{}/healthz", listener.local_addr().unwrap());
6520 let server = tokio::spawn(async move {
6521 let (mut stream, _) = listener.accept().await.unwrap();
6522 let mut request = [0u8; 2048];
6523 assert!(stream.read(&mut request).await.unwrap() > 0);
6524 let body = format!("{}not-in-diagnostic", "x".repeat(400));
6525 let response = format!(
6526 "HTTP/1.1 503 Unavailable\r\nContent-Length: {}\r\n\r\n{body}",
6527 body.len()
6528 );
6529 stream.write_all(response.as_bytes()).await.unwrap();
6530 });
6531 let error = probe_http_health(&url, Duration::from_secs(1))
6532 .await
6533 .unwrap_err()
6534 .to_string();
6535 server.await.unwrap();
6536 assert!(error.ends_with(&"x".repeat(200)), "{error}");
6537 assert_eq!(error.split(": ").last().unwrap().len(), 200);
6538 assert!(!error.contains("not-in-diagnostic"));
6539 }
6540
6541 #[cfg(unix)]
6542 #[tokio::test]
6543 async fn http_health_real_nats_server_monitoring() {
6544 if std::process::Command::new("nats-server")
6545 .arg("--version")
6546 .env("XDG_DATA_HOME", std::env::temp_dir())
6547 .env("XDG_RUNTIME_DIR", std::env::temp_dir())
6548 .env("XDG_CONFIG_HOME", std::env::temp_dir())
6549 .output()
6550 .is_err()
6551 {
6552 eprintln!(
6553 "skipped: http_health_real_nats_server_monitoring (nats-server is not on PATH)"
6554 );
6555 return;
6556 }
6557 let dir = cortexkit_test_support::ScratchDir::new("nats-http-monitoring");
6558 let monitor = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
6559 let port = monitor.local_addr().unwrap().port();
6560 let client = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
6561 let client_port = client.local_addr().unwrap().port();
6562 let config = dir.join("server.conf");
6563 std::fs::write(&config, format!("listen: 127.0.0.1:{client_port}\nhttp: 127.0.0.1:{port}\njetstream {{ store_dir: \"{}\" }}\n", dir.join("jetstream").display())).unwrap();
6564 let mut configured = http_fixture(&dir, &format!("http://127.0.0.1:{port}/healthz"), 1000);
6565 configured.program = PathBuf::from("nats-server");
6566 configured.args = vec!["-c".into(), config.to_string_lossy().into_owned()];
6567 drop(monitor);
6568 drop(client);
6569 let module =
6570 Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default())
6571 .supervise_configured_with_health(
6572 configured.module_spec(),
6573 true,
6574 configured.health,
6575 configured.drain_timeout_ms,
6576 configured.restart,
6577 )
6578 .unwrap();
6579 wait_http_health(&module, SupervisorHealthStatus::Ok).await;
6580 module.drain().await.unwrap();
6581 }
6582
6583 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
6584 async fn configuration_update_does_not_replace_captured_running_process_facts() {
6585 let supervisor =
6586 Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default());
6587 let snapshot = stale_process_snapshot(ModuleState::Running, true);
6588 let initial = ModuleSpec {
6589 module_id: "rescan-preserves-spawn-facts".to_string(),
6590 program: PathBuf::from("/spawned/module"),
6591 args: Vec::new(),
6592 env: Vec::new(),
6593 reserved: false,
6594 reserved_prefixes: Vec::new(),
6595 protocol: ModuleProtocol::Subc,
6596 overlap: Default::default(),
6597 };
6598 let module = supervisor.supervised_module(
6599 initial.clone(),
6600 supervisor.runtime_config(),
6601 snapshot,
6602 None,
6603 );
6604 let before = module.status().unwrap();
6605 let mut replacement = initial;
6606 replacement.program = PathBuf::from("/rescanned/replacement-module");
6607
6608 module
6609 .update_configuration(replacement, HealthConfig::default(), None)
6610 .await
6611 .unwrap();
6612
6613 let after = module.status().unwrap();
6614 assert_eq!(after.pid, before.pid);
6615 assert_eq!(after.spawned_at_ms, before.spawned_at_ms);
6616 assert_eq!(after.spawned_from, before.spawned_from);
6617 drop(module);
6618 }
6619}
6620
6621fn unix_ms_now() -> u64 {
6622 SystemTime::now()
6623 .duration_since(UNIX_EPOCH)
6624 .map(|duration| duration.as_millis().min(u128::from(u64::MAX)) as u64)
6625 .unwrap_or(0)
6626}
6627
6628async fn supervise_loop(
6629 mut spec: ModuleSpec,
6630 mut runtime: SupervisorRuntimeConfig,
6631 registry: Arc<Registry>,
6632 process_liveness: Arc<SupervisorProcessLiveness>,
6633 snapshot: SharedSnapshot,
6634 mut child: Option<SupervisedChild>,
6635 mut commands: mpsc::Receiver<SupervisorCommand>,
6636) {
6637 let mut health_probe = HealthProbeRuntime::default();
6638 let mut registration_changes = match registry.subscribe_module_changes(&spec.module_id) {
6642 Ok(changes) => Some(changes),
6643 Err(err) => {
6644 warn!(module_id = %spec.module_id, error = %err, "health prober could not subscribe to registry");
6647 None
6648 }
6649 };
6650 let mut pending_respawn: Option<PendingRespawn> = None;
6654 let mut requeued: VecDeque<SupervisorCommand> = VecDeque::new();
6657 let mut previous_instance_recheck = tokio::time::interval_at(
6658 Instant::now() + Duration::from_secs(5),
6659 Duration::from_secs(5),
6660 );
6661 previous_instance_recheck.set_missed_tick_behavior(tokio::time::MissedTickBehavior::Skip);
6662 loop {
6663 #[cfg(test)]
6664 {
6665 lock_snapshot(&snapshot).unwrap().actor_select = None;
6666 }
6667 #[cfg(target_os = "macos")]
6668 if let Some(active) = child.as_mut() {
6669 active.confirm_privacy_exec().await;
6670 }
6671 if let Some(scheduled) = runtime
6672 .scheduled_respawn
6673 .lock()
6674 .unwrap_or_else(|p| p.into_inner())
6675 .take()
6676 {
6677 pending_respawn = Some(scheduled);
6678 }
6679 if pending_respawn.is_some() && (child.is_some() || !respawn_still_pending(&snapshot)) {
6680 pending_respawn = None;
6681 cancel_deferred_reload(
6682 &runtime,
6683 &spec.module_id,
6684 "respawn cancelled by a supervisor command",
6685 );
6686 }
6687 if child.is_none()
6688 && pending_respawn.is_none()
6689 && lock_snapshot(&snapshot).is_ok_and(|state| state.waiting_for_previous_pid.is_none())
6690 {
6691 cancel_deferred_reload(
6692 &runtime,
6693 &spec.module_id,
6694 "respawn cancelled before a replacement was spawned",
6695 );
6696 let _ = update_snapshot(&snapshot, Some(&spec.module_id), |state| {
6697 state.respawn_pending = false;
6698 state.coalesced_restart_pending = false;
6699 if matches!(
6700 state.state,
6701 ModuleState::Restarting
6702 | ModuleState::Starting
6703 | ModuleState::Draining
6704 | ModuleState::Unresponsive
6705 ) {
6706 error!(module_id = %spec.module_id, state = ?state.state,
6707 "supervision operation ended without a child or pending respawn; marking failed so start can retry");
6708 state.state = ModuleState::Failed;
6709 clear_current_process_facts(state);
6710 }
6711 });
6712 }
6713 if let Some(command) = requeued.pop_front() {
6714 if !handle_supervisor_command(
6715 command,
6716 &mut spec,
6717 &mut runtime,
6718 ®istry,
6719 &process_liveness,
6720 &snapshot,
6721 &mut child,
6722 &mut commands,
6723 &mut requeued,
6724 )
6725 .await
6726 {
6727 return;
6728 }
6729 if child.is_some() || !respawn_still_pending(&snapshot) {
6730 pending_respawn = None;
6731 let _ = update_snapshot(&snapshot, Some(&spec.module_id), |state| {
6732 state.respawn_pending = false
6733 });
6734 }
6735 continue;
6736 }
6737 if child.is_some() {
6738 #[cfg(test)]
6739 {
6740 lock_snapshot(&snapshot).unwrap().actor_turns += 1;
6741 }
6742 health_probe.refresh_registration(&spec, &runtime, ®istry, &snapshot);
6743 let wake_after = health_probe.wake_after();
6744 let probe_sleep = async move {
6745 match wake_after {
6746 Some(delay) => sleep(delay).await,
6747 None => std::future::pending().await,
6748 }
6749 };
6750 tokio::pin!(probe_sleep);
6751 let wire_child = running_protocol(&spec, &snapshot) == ModuleProtocol::Subc;
6756 #[cfg(test)]
6757 {
6758 let registration_pending = wire_child
6759 && registration_changes
6760 .as_ref()
6761 .is_some_and(|changes| changes.has_changed().unwrap());
6762 let mut state = lock_snapshot(&snapshot).unwrap();
6763 state.actor_select = (!registration_pending).then_some(ActorSelectCheckpoint {
6764 generation: state.spawn_generation,
6765 turn: state.actor_turns,
6766 registered_connection: health_probe.registered_connection,
6767 next_probe_at: health_probe.next_probe_at,
6768 wake_after,
6769 });
6770 }
6771 let active_child = child.as_mut().expect("child checked above");
6772 tokio::select! {
6773 wait_result = active_child.wait() => {
6774 #[cfg(test)]
6775 { lock_snapshot(&snapshot).unwrap().actor_select = None; }
6776 let exit_report = match wait_result {
6785 Ok(status) => classify_reaped_child_exit(&snapshot, active_child, &status),
6786 Err(err) => {
6787 active_child.drain_stderr(&spec.module_id).await;
6788 fail_after_wait_error(
6789 &spec.module_id,
6790 &snapshot,
6791 &runtime.terminal_ring,
6792 &runtime.spawn_events,
6793 format!("failed to wait for supervised module: {err}"),
6794 );
6795 untrack_if_registration_released(
6796 &process_liveness,
6797 ®istry,
6798 &spec.module_id,
6799 &snapshot,
6800 );
6801 error!(module_id = %spec.module_id, error = %err, "failed to wait for supervised module");
6802 child = None;
6803 continue;
6804 }
6805 };
6806 active_child.drain_stderr(&spec.module_id).await;
6807
6808 let next = on_child_exit(
6809 &spec,
6810 runtime.restart_policy,
6811 ®istry,
6812 &snapshot,
6813 &runtime.terminal_ring,
6814 &runtime.spawn_events,
6815 &runtime.child_roster,
6816 exit_report,
6817 ).await;
6818 runtime.start_requester = None;
6819 runtime.pending_activity_restart = false;
6820 active_child.release_roster();
6823 match next {
6824 NextAction::Stop { registration_released } => {
6825 if registration_released {
6826 process_liveness.untrack_if_current(&spec.module_id, &snapshot);
6827 }
6828 child = None;
6829 }
6830 NextAction::Restart { schedule } => {
6831 let delay = schedule.map_or(
6832 runtime.restart_policy.delay_for_restart(0),
6833 |schedule| schedule.delay,
6834 );
6835 if let Some(schedule) = schedule {
6836 log_crash_respawn(&spec.module_id, schedule);
6837 }
6838 child = None;
6846 pending_respawn = Some(PendingRespawn { deadline: Instant::now() + delay, kind: RespawnKind::Spawn });
6847 let _ = update_snapshot(&snapshot, Some(&spec.module_id), |state| state.respawn_pending = true);
6848 }
6849 }
6850 }
6851 command = commands.recv() => {
6852 #[cfg(test)]
6853 { lock_snapshot(&snapshot).unwrap().actor_select = None; }
6854 let Some(command) = command else {
6855 return;
6856 };
6857 if !handle_supervisor_command(
6858 command,
6859 &mut spec,
6860 &mut runtime,
6861 ®istry,
6862 &process_liveness,
6863 &snapshot,
6864 &mut child,
6865 &mut commands,
6866 &mut requeued,
6867 ).await {
6868 return;
6869 }
6870 }
6871 _ = async {
6872 match registration_changes.as_mut() {
6873 Some(changes) => { let _ = changes.changed().await; }
6874 None => std::future::pending().await,
6875 }
6876 }, if wire_child => {
6877 #[cfg(test)]
6878 { lock_snapshot(&snapshot).unwrap().actor_select = None; }
6879 }
6880 _ = &mut probe_sleep => {
6881 #[cfg(test)]
6882 { lock_snapshot(&snapshot).unwrap().actor_select = None; }
6883 if health_probe.due() {
6884 run_health_probe_cycle(
6885 &spec,
6886 &runtime,
6887 ®istry,
6888 &process_liveness,
6889 &snapshot,
6890 &mut child,
6891 ).await;
6892 if child.is_none() {
6893 runtime.start_requester = None;
6894 runtime.pending_activity_restart = false;
6895 }
6896 if child.is_some() {
6897 health_probe.schedule_next(&spec, runtime.health.cadence);
6898 }
6899 }
6900 }
6901 }
6902 } else if let Some(pending) = pending_respawn {
6903 tokio::select! {
6904 _ = sleep_until(pending.deadline) => {
6905 pending_respawn = None;
6906 let _ = update_snapshot(&snapshot, Some(&spec.module_id), |state| state.respawn_pending = false);
6907 if !respawn_still_pending(&snapshot) {
6911 continue;
6912 }
6913 if runtime.child_roster.is_closed() {
6918 let _ = update_snapshot(&snapshot, Some(&spec.module_id), |state| {
6919 state.state = ModuleState::Stopped;
6920 });
6921 debug!(module_id = %spec.module_id, "crash respawn cancelled by daemon shutdown");
6922 continue;
6923 }
6924 if let Err(err) = release_dead_registration(
6925 ®istry,
6926 runtime.forwarding.as_deref(),
6927 &snapshot,
6928 &spec.module_id,
6929 ).await {
6930 fail_snapshot(&snapshot, Some(&spec.module_id), None);
6931 error!(module_id = %spec.module_id, error = %err, "registration did not release before restart");
6932 continue;
6933 }
6934
6935 if matches!(pending.kind, RespawnKind::Reload) {
6936 let reply = runtime.deferred_reload_reply.lock().unwrap_or_else(|p| p.into_inner()).take();
6937 let result = finish_reload_child(&spec, &runtime, ®istry, &process_liveness, &snapshot, &mut child).await;
6938 if result.is_ok() && runtime.pending_activity_restart {
6939 runtime.record_activity_transition("subc.module_restarted", &spec.module_id, runtime.start_requester.clone());
6940 }
6941 runtime.pending_activity_restart = false;
6942 if let Some(reply) = reply { let _ = reply.send(result); }
6943 continue;
6944 }
6945 process_liveness.track(spec.module_id.clone(), Arc::clone(&snapshot));
6946 match spawn_and_mark_running(&spec, &runtime, &snapshot) {
6947 Ok(next_child) => {
6948 child = Some(next_child);
6949 if runtime.pending_activity_restart {
6950 runtime.record_activity_transition("subc.module_restarted", &spec.module_id, runtime.start_requester.clone());
6951 runtime.pending_activity_restart = false;
6952 }
6953 debug!(module_id = %spec.module_id, "supervised module restarted after crash");
6954 }
6955 Err(err) => {
6956 fail_snapshot(&snapshot, Some(&spec.module_id), None);
6957 process_liveness.untrack_if_current(&spec.module_id, &snapshot);
6958 error!(module_id = %spec.module_id, error = %err, "failed to restart supervised module");
6959 }
6960 }
6961 }
6962 command = commands.recv() => {
6963 let Some(command) = command else {
6964 return;
6965 };
6966 if !handle_supervisor_command(
6967 command,
6968 &mut spec,
6969 &mut runtime,
6970 ®istry,
6971 &process_liveness,
6972 &snapshot,
6973 &mut child,
6974 &mut commands,
6975 &mut requeued,
6976 ).await {
6977 return;
6978 }
6979 if child.is_some() || !respawn_still_pending(&snapshot) {
6984 pending_respawn = None;
6985 let _ = update_snapshot(&snapshot, Some(&spec.module_id), |state| state.respawn_pending = false);
6986 }
6987 }
6988 }
6989 } else {
6990 let command = if lock_snapshot(&snapshot)
6991 .is_ok_and(|state| state.waiting_for_previous_pid.is_some())
6992 {
6993 tokio::select! {
6994 command = commands.recv() => command,
6995 _ = previous_instance_recheck.tick() => {
6996 if runtime.child_roster.previous_instance_pid(&spec.module_id).is_none() {
6997 let enabled = lock_snapshot(&snapshot).is_ok_and(|state| state.enabled);
6998 if enabled && !runtime.child_roster.is_closed() {
6999 process_liveness.track(spec.module_id.clone(), Arc::clone(&snapshot));
7000 match spawn_and_mark_running(&spec, &runtime, &snapshot) {
7001 Ok(next_child) => child = Some(next_child),
7002 Err(error) => {
7003 fail_snapshot(&snapshot, Some(&spec.module_id), None);
7004 error!(module_id = %spec.module_id, %error, "failed to start module after its previous instance exited");
7005 }
7006 }
7007 } else {
7008 let _ = update_snapshot(&snapshot, Some(&spec.module_id), |state| {
7009 state.waiting_for_previous_pid = None;
7010 state.waiting_for_previous_detail = None;
7011 });
7012 }
7013 }
7014 continue;
7015 }
7016 }
7017 } else {
7018 commands.recv().await
7019 };
7020 let Some(command) = command else {
7021 return;
7022 };
7023 if !handle_supervisor_command(
7024 command,
7025 &mut spec,
7026 &mut runtime,
7027 ®istry,
7028 &process_liveness,
7029 &snapshot,
7030 &mut child,
7031 &mut commands,
7032 &mut requeued,
7033 )
7034 .await
7035 {
7036 return;
7037 }
7038 }
7039 }
7040}
7041
7042fn log_crash_respawn(module_id: &str, schedule: CrashRestartSchedule) {
7043 info!(
7044 module_id,
7045 restart_in_window = schedule.restart_in_window,
7046 delay_ms = schedule.delay.as_millis() as u64,
7047 "respawning after crash"
7048 );
7049}
7050
7051fn respawn_still_pending(snapshot: &SharedSnapshot) -> bool {
7057 matches!(
7058 lock_snapshot(snapshot),
7059 Ok(state) if state.enabled && state.state == ModuleState::Restarting
7060 )
7061}
7062
7063enum NextAction {
7064 Stop {
7065 registration_released: bool,
7066 },
7067 Restart {
7068 schedule: Option<CrashRestartSchedule>,
7069 },
7070}
7071
7072#[allow(clippy::too_many_arguments)]
7073async fn handle_supervisor_command(
7074 command: SupervisorCommand,
7075 spec: &mut ModuleSpec,
7076 runtime: &mut SupervisorRuntimeConfig,
7077 registry: &Arc<Registry>,
7078 process_liveness: &SupervisorProcessLiveness,
7079 snapshot: &SharedSnapshot,
7080 child: &mut Option<SupervisedChild>,
7081 commands: &mut mpsc::Receiver<SupervisorCommand>,
7082 requeued: &mut VecDeque<SupervisorCommand>,
7083) -> bool {
7084 let command =
7085 if let Some((pid, reason)) = runtime.child_roster.previous_instance(&spec.module_id) {
7086 match command {
7087 SupervisorCommand::Restart { reply, .. }
7088 | SupervisorCommand::Reload { reply, .. }
7089 | SupervisorCommand::Swap { reply, .. } => {
7090 let _ = reply.send(Err(previous_instance_error(spec, pid, reason)));
7091 return true;
7092 }
7093 SupervisorCommand::SetEnabled {
7094 enabled: true,
7095 reply,
7096 ..
7097 } => {
7098 let _ = reply.send(Err(previous_instance_error(spec, pid, reason)));
7099 return true;
7100 }
7101 other => other,
7102 }
7103 } else {
7104 command
7105 };
7106 match command {
7107 SupervisorCommand::Drain { requester, reply } => {
7108 let result = drain_optional_child(
7111 &spec.module_id,
7112 spec.protocol,
7113 StopNotice::NotSent,
7114 registry,
7115 runtime.forwarding.as_deref(),
7116 snapshot,
7117 &runtime.terminal_ring,
7118 &runtime.spawn_events,
7119 child,
7120 runtime.drain_timeout,
7121 ModuleState::Stopped,
7122 None,
7123 )
7124 .await;
7125 let registration_released = result.is_ok();
7126 if registration_released {
7127 runtime.record_activity_transition(
7128 "subc.module_stopped",
7129 &spec.module_id,
7130 requester,
7131 );
7132 }
7133 let _ = reply.send(result);
7134 if registration_released {
7135 process_liveness.untrack_if_current(&spec.module_id, snapshot);
7136 }
7137 false
7138 }
7139 SupervisorCommand::Retire { requester, reply } => {
7140 let result = async {
7141 let stop_notice = begin_forwarding_drain_if_configured(
7142 spec,
7143 runtime,
7144 registry,
7145 snapshot,
7146 None,
7147 RouteCloseReason::Disable,
7148 )
7149 .await?;
7150 drain_optional_child(
7151 &spec.module_id,
7152 spec.protocol,
7153 stop_notice,
7154 registry,
7155 runtime.forwarding.as_deref(),
7156 snapshot,
7157 &runtime.terminal_ring,
7158 &runtime.spawn_events,
7159 child,
7160 runtime.drain_timeout,
7161 ModuleState::Stopped,
7162 None,
7163 )
7164 .await
7165 }
7166 .await;
7167 let registration_released = result.is_ok();
7168 if registration_released {
7169 runtime.record_activity_transition(
7170 "subc.module_stopped",
7171 &spec.module_id,
7172 requester,
7173 );
7174 }
7175 let _ = reply.send(result);
7176 if registration_released {
7177 process_liveness.untrack_if_current(&spec.module_id, snapshot);
7178 }
7179 false
7180 }
7181 SupervisorCommand::Restart {
7182 requester,
7183 drain_timeout_ms,
7184 received_at_generation,
7185 queued_at,
7186 reply,
7187 } => {
7188 info!(
7192 module_id = %spec.module_id,
7193 queued_ms = u64::try_from(queued_at.elapsed().as_millis()).unwrap_or(u64::MAX),
7194 "restart command dequeued"
7195 );
7196 let validation = match lock_snapshot(snapshot) {
7208 Ok(state) if !state.enabled => Err(SuperviseError::Disabled {
7209 module_id: spec.module_id.clone(),
7210 }),
7211 Ok(_) => Ok(()),
7212 Err(err) => Err(err),
7213 };
7214 let initiated = validation.is_ok();
7215 let _ = reply.send(validation);
7216 let satisfied_by_generation = if initiated
7230 && child
7231 .as_ref()
7232 .is_some_and(|child| child.spawned_at > queued_at)
7233 {
7234 lock_snapshot(snapshot).ok().and_then(|state| {
7235 (!state.configuration_updated_since_spawn).then_some(state.spawn_generation)
7236 })
7237 } else {
7238 None
7239 };
7240 let satisfied_by_pending = initiated
7241 && child.is_none()
7242 && lock_snapshot(snapshot).ok().is_some_and(|mut state| {
7243 let pending = state.respawn_pending && !state.configuration_updated_since_spawn;
7247 if pending {
7248 state.coalesced_restart_pending = true;
7249 }
7250 pending
7251 });
7252 if satisfied_by_pending {
7253 debug!(module_id = %spec.module_id, "restart coalesced into the pending replacement");
7254 } else if let Some(generation) = satisfied_by_generation {
7255 info!(
7256 module_id = %spec.module_id,
7257 received_at_generation,
7258 "restart already satisfied by generation {generation}; not restarting again"
7259 );
7260 } else if initiated {
7261 let drain_timeout = drain_timeout_ms
7264 .map(Duration::from_millis)
7265 .unwrap_or(runtime.drain_timeout);
7266 if let Err(err) = restart_child(
7267 spec,
7268 runtime,
7269 registry,
7270 process_liveness,
7271 snapshot,
7272 child,
7273 drain_timeout,
7274 )
7275 .await
7276 {
7277 warn!(
7278 module_id = %spec.module_id,
7279 error = %err,
7280 "operator restart failed after initiation ack; module state carries the outcome"
7281 );
7282 let _ = update_snapshot(snapshot, Some(&spec.module_id), |state| {
7283 state.state = ModuleState::Failed;
7284 clear_current_process_facts(state);
7285 });
7286 } else {
7287 runtime.start_requester = requester;
7288 runtime.pending_activity_restart = true;
7289 }
7290 }
7291 true
7292 }
7293 SupervisorCommand::Reload { requester, reply } => {
7294 let result =
7295 reload_child(spec, runtime, registry, process_liveness, snapshot, child).await;
7296 if result.is_ok() {
7297 runtime.start_requester = requester;
7298 runtime.pending_activity_restart = true;
7299 }
7300 if result.is_ok()
7301 && runtime
7302 .scheduled_respawn
7303 .lock()
7304 .unwrap_or_else(|p| p.into_inner())
7305 .as_ref()
7306 .is_some_and(|pending| matches!(pending.kind, RespawnKind::Reload))
7307 {
7308 *runtime
7309 .deferred_reload_reply
7310 .lock()
7311 .unwrap_or_else(|p| p.into_inner()) = Some(reply);
7312 } else {
7313 let _ = reply.send(result);
7314 }
7315 true
7316 }
7317 SupervisorCommand::SetEnabled {
7318 enabled,
7319 requester,
7320 reply,
7321 } => {
7322 let previous_requester = runtime.start_requester.clone();
7323 if enabled {
7324 runtime.start_requester = requester.clone();
7325 }
7326 let result = set_child_enabled(
7327 spec,
7328 runtime,
7329 registry,
7330 process_liveness,
7331 snapshot,
7332 child,
7333 enabled,
7334 )
7335 .await;
7336 if matches!(result, Ok(true)) {
7337 runtime.pending_activity_restart = false;
7338 if enabled {
7339 runtime.start_requester = requester;
7340 } else {
7341 runtime.start_requester = None;
7342 runtime.record_activity_transition(
7343 "subc.module_disabled",
7344 &spec.module_id,
7345 requester,
7346 );
7347 }
7348 } else {
7349 runtime.start_requester = previous_requester;
7350 }
7351 let _ = reply.send(result);
7352 true
7353 }
7354 SupervisorCommand::UpdateConfiguration {
7355 spec: next_spec,
7356 health,
7357 drain_timeout_ms,
7358 reply,
7359 } => {
7360 if let Some(handle) = &runtime.supervisor_handle {
7361 handle.apply_identity_configuration(&next_spec);
7362 }
7363 *spec = next_spec;
7364 let _ = update_snapshot(snapshot, Some(&spec.module_id), |state| {
7365 state.configuration_updated_since_spawn = true;
7366 });
7367 let health_changed = runtime.health != health;
7368 runtime.health = health;
7369 if health_changed {
7372 let _ = update_snapshot(snapshot, Some(&spec.module_id), |state| {
7373 state.health = ModuleHealthStatus::default();
7374 });
7375 }
7376 runtime.drain_timeout = drain_timeout_ms
7377 .map(Duration::from_millis)
7378 .unwrap_or(runtime.default_drain_timeout);
7379 *runtime
7380 .effective_drain_timeout
7381 .lock()
7382 .unwrap_or_else(|poisoned| poisoned.into_inner()) = runtime.drain_timeout;
7383 let _ = reply.send(());
7384 true
7385 }
7386 SupervisorCommand::Swap {
7387 requester,
7388 ready_timeout,
7389 reply,
7390 } => {
7391 let mut swap_runtime = runtime.clone();
7392 swap_runtime.start_requester = requester;
7393 let before = lock_snapshot(snapshot)
7394 .ok()
7395 .map(|state| state.spawn_generation);
7396 let end = swap::run_swap(
7397 spec,
7398 &swap_runtime,
7399 registry,
7400 process_liveness,
7401 snapshot,
7402 child,
7403 commands,
7404 ready_timeout.unwrap_or(DEFAULT_SWAP_READY_TIMEOUT),
7405 reply,
7406 )
7407 .await;
7408 if before
7409 != lock_snapshot(snapshot)
7410 .ok()
7411 .map(|state| state.spawn_generation)
7412 {
7413 runtime.start_requester = swap_runtime.start_requester;
7414 }
7415 requeued.extend(end.requeue);
7416 true
7417 }
7418 }
7419}
7420
7421async fn restart_child(
7422 spec: &ModuleSpec,
7423 runtime: &SupervisorRuntimeConfig,
7424 registry: &Registry,
7425 process_liveness: &SupervisorProcessLiveness,
7426 snapshot: &SharedSnapshot,
7427 child: &mut Option<SupervisedChild>,
7428 drain_timeout: Duration,
7429) -> Result<(), SuperviseError> {
7430 if !lock_snapshot(snapshot)?.enabled {
7432 return Err(SuperviseError::Disabled {
7433 module_id: spec.module_id.clone(),
7434 });
7435 }
7436 let stop_notice = begin_forwarding_drain_with_timeout(
7437 spec,
7438 runtime,
7439 registry,
7440 snapshot,
7441 None,
7442 RouteCloseReason::Restart,
7443 drain_timeout,
7444 )
7445 .await?;
7446
7447 if child.is_some() {
7448 drain_optional_child(
7449 &spec.module_id,
7450 spec.protocol,
7451 stop_notice,
7452 registry,
7453 runtime.forwarding.as_deref(),
7454 snapshot,
7455 &runtime.terminal_ring,
7456 &runtime.spawn_events,
7457 child,
7458 drain_timeout,
7459 ModuleState::Restarting,
7460 Some(true),
7461 )
7462 .await?;
7463 } else {
7464 update_snapshot(snapshot, Some(&spec.module_id), |state| {
7465 state.enabled = true;
7466 state.state = ModuleState::Restarting;
7467 clear_current_process_facts(state);
7468 })?;
7469 release_dead_registration(
7470 registry,
7471 runtime.forwarding.as_deref(),
7472 snapshot,
7473 &spec.module_id,
7474 )
7475 .await?;
7476 }
7477
7478 reset_restart_count(snapshot, &spec.module_id)?;
7479 process_liveness.track(spec.module_id.clone(), Arc::clone(snapshot));
7480 schedule_respawn(
7481 runtime,
7482 snapshot,
7483 &spec.module_id,
7484 runtime.restart_policy.backoff,
7485 RespawnKind::Spawn,
7486 )
7487}
7488
7489async fn reload_child(
7490 spec: &ModuleSpec,
7491 runtime: &SupervisorRuntimeConfig,
7492 registry: &Registry,
7493 process_liveness: &SupervisorProcessLiveness,
7494 snapshot: &SharedSnapshot,
7495 child: &mut Option<SupervisedChild>,
7496) -> Result<(), SuperviseError> {
7497 if !lock_snapshot(snapshot)?.enabled {
7499 return Err(SuperviseError::Disabled {
7500 module_id: spec.module_id.clone(),
7501 });
7502 }
7503 let stop_notice = begin_forwarding_drain(
7504 spec,
7505 runtime,
7506 registry,
7507 snapshot,
7508 Some(true),
7509 RouteCloseReason::Reload,
7510 )
7511 .await?;
7512
7513 if child.is_some() {
7514 drain_optional_child(
7515 &spec.module_id,
7516 spec.protocol,
7517 stop_notice,
7518 registry,
7519 runtime.forwarding.as_deref(),
7520 snapshot,
7521 &runtime.terminal_ring,
7522 &runtime.spawn_events,
7523 child,
7524 runtime.drain_timeout,
7525 ModuleState::Restarting,
7526 Some(true),
7527 )
7528 .await?;
7529 } else {
7530 update_snapshot(snapshot, Some(&spec.module_id), |state| {
7531 state.enabled = true;
7532 state.state = ModuleState::Restarting;
7533 clear_current_process_facts(state);
7534 })?;
7535 release_dead_registration(
7536 registry,
7537 runtime.forwarding.as_deref(),
7538 snapshot,
7539 &spec.module_id,
7540 )
7541 .await?;
7542 }
7543
7544 reset_restart_count(snapshot, &spec.module_id)?;
7545 process_liveness.track(spec.module_id.clone(), Arc::clone(snapshot));
7546 schedule_respawn(
7547 runtime,
7548 snapshot,
7549 &spec.module_id,
7550 runtime.restart_policy.backoff,
7551 RespawnKind::Reload,
7552 )
7553}
7554
7555async fn finish_reload_child(
7556 spec: &ModuleSpec,
7557 runtime: &SupervisorRuntimeConfig,
7558 registry: &Registry,
7559 process_liveness: &SupervisorProcessLiveness,
7560 snapshot: &SharedSnapshot,
7561 child: &mut Option<SupervisedChild>,
7562) -> Result<(), SuperviseError> {
7563 process_liveness.track(spec.module_id.clone(), Arc::clone(snapshot));
7564 let next_child = match spawn_and_mark_running(spec, runtime, snapshot) {
7565 Ok(next_child) => next_child,
7566 Err(err) => {
7567 return handle_reload_spawn_failure(
7568 spec,
7569 runtime,
7570 process_liveness,
7571 snapshot,
7572 child,
7573 format!("new child failed to spawn: {err}"),
7574 )
7575 .await;
7576 }
7577 };
7578 *child = Some(next_child);
7579
7580 let wait_outcome = {
7581 let active_child = child.as_mut().expect("new reload child was just stored");
7582 wait_for_registration_after_reload(
7583 registry,
7584 &spec.module_id,
7585 snapshot,
7586 active_child,
7587 REGISTRY_RELEASE_TIMEOUT,
7588 )
7589 .await?
7590 };
7591
7592 match wait_outcome {
7593 RegistrationWaitOutcome::Registered => {
7594 debug!(module_id = %spec.module_id, "supervised module reloaded and registered");
7595 Ok(())
7596 }
7597 RegistrationWaitOutcome::Exited(exit_report) => {
7598 if let Some(active_child) = child.as_mut() {
7599 active_child.drain_stderr(&spec.module_id).await;
7600 }
7601 let mut exited_child = child.take().expect("exited reload child is still stored");
7604 #[cfg(test)]
7605 if let Some(gate) = &runtime.test_reload_exit_record_gate {
7606 gate.reached.notify_one();
7607 gate.resume.notified().await;
7608 }
7609 let result = handle_reload_child_registration_failure(
7610 spec,
7611 runtime,
7612 registry,
7613 process_liveness,
7614 snapshot,
7615 child,
7616 ReloadRegistrationFailure {
7617 exit_report: registration_failure_exit_report(exit_report),
7618 reason: exited_child
7619 .spawn_failure
7620 .clone()
7621 .unwrap_or_else(|| "new child exited before registering".to_string()),
7622 },
7623 )
7624 .await;
7625 exited_child.release_roster();
7626 result
7627 }
7628 RegistrationWaitOutcome::TimedOut => {
7629 let mut timed_out_child = child
7630 .take()
7631 .expect("timed-out reload child is still running");
7632 timed_out_child
7633 .start_kill()
7634 .map_err(|source| SuperviseError::Kill {
7635 module_id: spec.module_id.clone(),
7636 source,
7637 })?;
7638 let status = timed_out_child
7639 .wait()
7640 .await
7641 .map_err(|source| SuperviseError::Wait {
7642 module_id: spec.module_id.clone(),
7643 source,
7644 })?;
7645 timed_out_child.drain_stderr(&spec.module_id).await;
7646 handle_reload_child_registration_failure(
7647 spec,
7648 runtime,
7649 registry,
7650 process_liveness,
7651 snapshot,
7652 child,
7653 ReloadRegistrationFailure {
7654 exit_report: registration_failure_exit_report(classify_reaped_child_exit(
7655 snapshot,
7656 &timed_out_child,
7657 &status,
7658 )),
7659 reason: format!(
7660 "new child did not register within {:?}",
7661 REGISTRY_RELEASE_TIMEOUT
7662 ),
7663 },
7664 )
7665 .await
7666 }
7667 }
7668}
7669
7670async fn set_child_enabled(
7671 spec: &ModuleSpec,
7672 runtime: &SupervisorRuntimeConfig,
7673 registry: &Registry,
7674 process_liveness: &SupervisorProcessLiveness,
7675 snapshot: &SharedSnapshot,
7676 child: &mut Option<SupervisedChild>,
7677 enabled: bool,
7678) -> Result<bool, SuperviseError> {
7679 let (current_enabled, current_state, respawn_pending) = {
7680 let state = lock_snapshot(snapshot)?;
7681 (state.enabled, state.state, state.respawn_pending)
7682 };
7683 let revive_terminal = enabled
7691 && current_enabled
7692 && child.is_none()
7693 && (matches!(current_state, ModuleState::Failed | ModuleState::Stopped)
7694 || (current_state == ModuleState::Restarting && !respawn_pending));
7695 if current_enabled == enabled && !revive_terminal {
7696 return Ok(false);
7697 }
7698
7699 if enabled {
7700 update_snapshot(snapshot, Some(&spec.module_id), |state| {
7701 state.enabled = true;
7702 state.state = ModuleState::Starting;
7703 clear_current_process_facts(state);
7704 })?;
7705 #[cfg(test)]
7706 if runtime.test_seed_stale_facts_before_enable_spawn {
7707 update_snapshot(snapshot, Some(&spec.module_id), |state| {
7708 state.process_alive = true;
7709 state.pid = Some(41);
7710 state.spawned_at_ms = Some(42);
7711 state.spawned_from = Some(PathBuf::from("/spawned/module"));
7712 state.spawned_file_identity = Some(SpawnedFileIdentity {
7713 device: 43,
7714 inode: 44,
7715 #[cfg(windows)]
7716 windows_image: None,
7717 });
7718 })?;
7719 }
7720 release_dead_registration(
7721 registry,
7722 runtime.forwarding.as_deref(),
7723 snapshot,
7724 &spec.module_id,
7725 )
7726 .await?;
7727 reset_restart_count(snapshot, &spec.module_id)?;
7728 process_liveness.track(spec.module_id.clone(), Arc::clone(snapshot));
7729 let next_child = match spawn_and_mark_running(spec, runtime, snapshot) {
7730 Ok(next_child) => next_child,
7731 Err(err) => {
7732 if let Err(state_err) = update_snapshot(snapshot, Some(&spec.module_id), |state| {
7733 state.state = ModuleState::Failed;
7734 clear_current_process_facts(state);
7735 }) {
7736 error!(module_id = %spec.module_id, error = %state_err, "failed to record enable spawn failure");
7737 }
7738 process_liveness.untrack_if_current(&spec.module_id, snapshot);
7739 return Err(err);
7740 }
7741 };
7742 *child = Some(next_child);
7743 debug!(module_id = %spec.module_id, "supervised module enabled");
7744 Ok(true)
7745 } else {
7746 let stop_notice = begin_forwarding_drain_if_configured(
7747 spec,
7748 runtime,
7749 registry,
7750 snapshot,
7751 Some(false),
7752 RouteCloseReason::Disable,
7753 )
7754 .await?;
7755 drain_optional_child(
7756 &spec.module_id,
7757 spec.protocol,
7758 stop_notice,
7759 registry,
7760 runtime.forwarding.as_deref(),
7761 snapshot,
7762 &runtime.terminal_ring,
7763 &runtime.spawn_events,
7764 child,
7765 runtime.drain_timeout,
7766 ModuleState::Disabled,
7767 Some(false),
7768 )
7769 .await?;
7770 debug!(module_id = %spec.module_id, "supervised module disabled");
7771 Ok(true)
7772 }
7773}
7774
7775#[allow(clippy::too_many_arguments)]
7776async fn on_child_exit(
7777 spec: &ModuleSpec,
7778 policy: RestartPolicy,
7779 registry: &Registry,
7780 snapshot: &SharedSnapshot,
7781 terminal_ring: &Arc<Mutex<TerminalRing>>,
7782 spawn_events: &SpawnEventFeed,
7783 roster: &ChildRoster,
7784 exit_report: ExitReport,
7785) -> NextAction {
7786 if roster.is_closed() {
7792 return on_child_exit_during_daemon_shutdown(
7793 spec,
7794 registry,
7795 snapshot,
7796 terminal_ring,
7797 spawn_events,
7798 exit_report,
7799 )
7800 .await;
7801 }
7802 let unrequested_clean_exit_of_protocol_none = exit_report.kind == ExitKind::Clean
7817 && running_protocol(spec, snapshot) == ModuleProtocol::None;
7818 match exit_report.kind {
7819 ExitKind::Clean if !unrequested_clean_exit_of_protocol_none => {
7820 info!(
7821 module_id = %spec.module_id,
7822 exit_code = ?exit_report.code,
7823 exit_signal = ?exit_report.signal,
7824 "supervised module exited cleanly"
7825 );
7826 if let Err(err) = publish_after_terminal_record(
7827 &spec.module_id,
7828 || {
7829 record_terminal(
7830 &spec.module_id,
7831 terminal_ring,
7832 spawn_events,
7833 &exit_report,
7834 TerminalDisposition::Stopped,
7835 )
7836 },
7837 || {
7838 update_snapshot(snapshot, Some(&spec.module_id), |state| {
7839 state.state = ModuleState::Stopped;
7840 clear_current_process_facts(state);
7841 state.last_exit = Some(exit_report.clone());
7842 })
7843 },
7844 ) {
7845 error!(module_id = %spec.module_id, error = %err, "failed to publish clean module exit");
7846 }
7847 let registration_released = match wait_for_registration_release(
7848 registry,
7849 &spec.module_id,
7850 REGISTRY_RELEASE_TIMEOUT,
7851 )
7852 .await
7853 {
7854 Ok(()) => true,
7855 Err(err) => {
7856 warn!(module_id = %spec.module_id, error = %err, "registration still active after clean exit");
7857 false
7858 }
7859 };
7860 NextAction::Stop {
7861 registration_released,
7862 }
7863 }
7864 ExitKind::Clean | ExitKind::Crash => {
7865 if unrequested_clean_exit_of_protocol_none {
7866 warn!(
7867 module_id = %spec.module_id,
7868 exit_code = ?exit_report.code,
7869 exit_signal = ?exit_report.signal,
7870 "protocol-none module exited cleanly without a stop request; handling it as a crash"
7871 );
7872 } else {
7873 warn!(
7874 module_id = %spec.module_id,
7875 exit_code = ?exit_report.code,
7876 exit_signal = ?exit_report.signal,
7877 "supervised module exited abnormally (crash)"
7878 );
7879 }
7880 let mut restart_schedule = None;
7881 let mut disposition = TerminalDisposition::Disabled;
7882 let mut final_state = ModuleState::Disabled;
7883 let mut disposition_detail = lock_snapshot(snapshot)
7887 .ok()
7888 .and_then(|mut state| state.spawn_failure.take());
7889 let now = Instant::now();
7890 if let Err(err) = update_snapshot(snapshot, Some(&spec.module_id), |state| {
7891 if state.enabled {
7892 if let Some(schedule) = state.next_crash_restart(&policy, now) {
7893 state.respawn_pending = true;
7899 final_state = ModuleState::Restarting;
7900 restart_schedule = Some(schedule);
7901 disposition = TerminalDisposition::Restarting;
7902 } else {
7903 final_state = ModuleState::Failed;
7904 disposition = TerminalDisposition::Failed;
7905 let budget = policy.budget_exhausted_detail();
7906 disposition_detail =
7907 Some(disposition_detail.take().map_or_else(
7908 || budget.clone(),
7909 |cause| format!("{cause}; {budget}"),
7910 ));
7911 }
7912 } else {
7913 disposition = TerminalDisposition::Disabled;
7914 }
7915 }) {
7916 error!(module_id = %spec.module_id, error = %err, "failed to record crashed module exit");
7917 return NextAction::Stop {
7918 registration_released: false,
7919 };
7920 }
7921 if disposition == TerminalDisposition::Failed {
7922 error!(
7927 module_id = %spec.module_id,
7928 max_restarts = policy.max_restarts,
7929 window_secs = policy.window.as_secs(),
7930 "module stopped: {}",
7931 policy.budget_exhausted_detail()
7932 );
7933 }
7934 if let Err(err) = publish_after_terminal_record(
7935 &spec.module_id,
7936 || {
7937 record_terminal_with_detail(
7938 &spec.module_id,
7939 terminal_ring,
7940 spawn_events,
7941 &exit_report,
7942 disposition,
7943 disposition_detail,
7944 )
7945 },
7946 || {
7947 update_snapshot(snapshot, Some(&spec.module_id), |state| {
7948 state.state = final_state;
7949 clear_current_process_facts(state);
7950 state.last_exit = Some(exit_report.clone());
7951 })
7952 },
7953 ) {
7954 error!(module_id = %spec.module_id, error = %err, "failed to publish crashed module exit");
7955 }
7956
7957 if let Some(schedule) = restart_schedule {
7958 NextAction::Restart {
7959 schedule: Some(schedule),
7960 }
7961 } else {
7962 let registration_released = match wait_for_registration_release(
7963 registry,
7964 &spec.module_id,
7965 REGISTRY_RELEASE_TIMEOUT,
7966 )
7967 .await
7968 {
7969 Ok(()) => true,
7970 Err(err) => {
7971 warn!(module_id = %spec.module_id, error = %err, "registration still active after failed module");
7972 false
7973 }
7974 };
7975 NextAction::Stop {
7976 registration_released,
7977 }
7978 }
7979 }
7980 ExitKind::DeliberateSeverance => {
7981 warn!(
7982 module_id = %spec.module_id,
7983 exit_code = ?exit_report.code,
7984 exit_signal = ?exit_report.signal,
7985 "supervised module exited after deliberate connection severance"
7986 );
7987 let should_restart = match lock_snapshot(snapshot) {
7988 Ok(state) => state.enabled,
7989 Err(err) => {
7990 error!(module_id = %spec.module_id, error = %err, "failed to read deliberately severed module state");
7991 return NextAction::Stop {
7992 registration_released: false,
7993 };
7994 }
7995 };
7996 let final_state = if should_restart {
7997 ModuleState::Restarting
7998 } else {
7999 ModuleState::Disabled
8000 };
8001 if let Err(err) = publish_after_terminal_record(
8002 &spec.module_id,
8003 || {
8004 record_terminal(
8005 &spec.module_id,
8006 terminal_ring,
8007 spawn_events,
8008 &exit_report,
8009 terminal_disposition(final_state),
8010 )
8011 },
8012 || {
8013 update_snapshot(snapshot, Some(&spec.module_id), |state| {
8014 state.state = final_state;
8015 clear_current_process_facts(state);
8016 state.last_exit = Some(exit_report.clone());
8017 state.lifetime_restarts += 1;
8018 })
8019 },
8020 ) {
8021 error!(module_id = %spec.module_id, error = %err, "failed to publish deliberately severed module exit");
8022 return NextAction::Stop {
8023 registration_released: false,
8024 };
8025 }
8026
8027 if should_restart {
8028 NextAction::Restart { schedule: None }
8029 } else {
8030 let registration_released = match wait_for_registration_release(
8031 registry,
8032 &spec.module_id,
8033 REGISTRY_RELEASE_TIMEOUT,
8034 )
8035 .await
8036 {
8037 Ok(()) => true,
8038 Err(err) => {
8039 warn!(module_id = %spec.module_id, error = %err, "registration still active after deliberately severed module exit");
8040 false
8041 }
8042 };
8043 NextAction::Stop {
8044 registration_released,
8045 }
8046 }
8047 }
8048 }
8049}
8050
8051async fn on_child_exit_during_daemon_shutdown(
8052 spec: &ModuleSpec,
8053 registry: &Registry,
8054 snapshot: &SharedSnapshot,
8055 terminal_ring: &Arc<Mutex<TerminalRing>>,
8056 spawn_events: &SpawnEventFeed,
8057 exit_report: ExitReport,
8058) -> NextAction {
8059 info!(
8060 module_id = %spec.module_id,
8061 exit_code = ?exit_report.code,
8062 exit_signal = ?exit_report.signal,
8063 exit_kind = ?exit_report.kind,
8064 "supervised module exited during daemon shutdown; not restarting it"
8065 );
8066 if let Err(err) = publish_after_terminal_record(
8067 &spec.module_id,
8068 || {
8069 record_terminal(
8070 &spec.module_id,
8071 terminal_ring,
8072 spawn_events,
8073 &exit_report,
8074 TerminalDisposition::DaemonShutdown,
8075 )
8076 },
8077 || {
8078 update_snapshot(snapshot, Some(&spec.module_id), |state| {
8079 state.state = ModuleState::Stopped;
8080 clear_current_process_facts(state);
8081 state.last_exit = Some(exit_report.clone());
8082 })
8083 },
8084 ) {
8085 error!(module_id = %spec.module_id, error = %err, "failed to publish module exit during daemon shutdown");
8086 }
8087 let registration_released =
8088 wait_for_registration_release(registry, &spec.module_id, REGISTRY_RELEASE_TIMEOUT)
8089 .await
8090 .is_ok();
8091 NextAction::Stop {
8092 registration_released,
8093 }
8094}
8095
8096fn publish_after_terminal_record<T>(
8101 module_id: &str,
8102 record: impl FnOnce(),
8103 publish: impl FnOnce() -> T,
8104) -> T {
8105 let recorded = std::panic::catch_unwind(std::panic::AssertUnwindSafe(record));
8106 let published = publish();
8107 if let Err(panic) = recorded {
8108 error!(
8109 module_id,
8110 "terminal recording panicked; published final state before resuming unwind"
8111 );
8112 std::panic::resume_unwind(panic);
8113 }
8114 published
8115}
8116
8117fn fail_after_wait_error(
8118 module_id: &str,
8119 snapshot: &SharedSnapshot,
8120 terminal_ring: &Arc<Mutex<TerminalRing>>,
8121 spawn_events: &SpawnEventFeed,
8122 detail: String,
8123) {
8124 let report = wait_error_exit_report();
8125 publish_after_terminal_record(
8126 module_id,
8127 || {
8128 record_terminal_with_detail(
8129 module_id,
8130 terminal_ring,
8131 spawn_events,
8132 &report,
8133 TerminalDisposition::Failed,
8134 Some(detail),
8135 )
8136 },
8137 || fail_snapshot(snapshot, Some(module_id), Some(report.clone())),
8138 );
8139}
8140
8141#[cfg(test)]
8142fn record_wait_error_terminal(
8143 module_id: &str,
8144 terminal_ring: &Arc<Mutex<TerminalRing>>,
8145 spawn_events: &SpawnEventFeed,
8146) {
8147 record_terminal(
8148 module_id,
8149 terminal_ring,
8150 spawn_events,
8151 &wait_error_exit_report(),
8152 TerminalDisposition::Failed,
8153 );
8154}
8155
8156fn record_terminal(
8157 module_id: &str,
8158 terminal_ring: &Arc<Mutex<TerminalRing>>,
8159 spawn_events: &SpawnEventFeed,
8160 exit_report: &ExitReport,
8161 disposition: TerminalDisposition,
8162) {
8163 record_terminal_with_detail(
8164 module_id,
8165 terminal_ring,
8166 spawn_events,
8167 exit_report,
8168 disposition,
8169 None,
8170 );
8171}
8172
8173fn durable_terminal_history_of(
8177 terminal_ring: &Mutex<TerminalRing>,
8178 module_id: &str,
8179) -> subc_control::TerminalHistory {
8180 let read = terminal_ring
8181 .lock()
8182 .unwrap_or_else(|p| p.into_inner())
8183 .capture_durable_history();
8184 read.read(module_id)
8185}
8186
8187fn record_terminal_with_detail(
8188 module_id: &str,
8189 terminal_ring: &Arc<Mutex<TerminalRing>>,
8190 spawn_events: &SpawnEventFeed,
8191 exit_report: &ExitReport,
8192 disposition: TerminalDisposition,
8193 disposition_detail: Option<String>,
8194) {
8195 #[cfg(test)]
8196 terminal_history_tests::before_terminal_record(spawn_events);
8197 spawn_events.emit_exited(module_id, exit_report.code, exit_report.signal);
8198 let record = TerminalRecord {
8199 exit_code: exit_report.code,
8200 exit_signal: exit_report.signal,
8201 at_ms: exit_report.at_ms,
8202 disposition,
8203 exit_kind: exit_report.kind.into(),
8204 disposition_detail,
8205 };
8206 terminal_ring
8207 .lock()
8208 .unwrap_or_else(|poisoned| poisoned.into_inner())
8209 .record_exit(module_id, record);
8210}
8211
8212fn untrack_if_registration_released(
8213 process_liveness: &SupervisorProcessLiveness,
8214 registry: &Registry,
8215 module_id: &str,
8216 snapshot: &SharedSnapshot,
8217) {
8218 match registry.get_module(module_id) {
8219 Ok(None) => process_liveness.untrack_if_current(module_id, snapshot),
8220 Ok(Some(_)) => {}
8221 Err(err) => {
8222 warn!(module_id, error = %err, "could not determine whether supervisor liveness can be untracked");
8223 }
8224 }
8225}
8226
8227#[cfg(test)]
8241fn apply_wire_spawn_args(
8242 command: &mut Command,
8243 spec: &ModuleSpec,
8244 connection_file_path: Option<&std::path::Path>,
8245 handle: Option<&SupervisorHandle>,
8246) -> Result<Option<NonceHandoff>, SuperviseError> {
8247 apply_wire_spawn_args_for_role(
8248 command,
8249 spec,
8250 connection_file_path,
8251 handle,
8252 SpawnRole::Plain,
8253 )
8254}
8255
8256#[cfg(unix)]
8261type NonceHandoff = subc_os::LaunchNonceHandoff;
8262#[cfg(windows)]
8263type NonceHandoff = subc_os::LaunchNoncePipeHandoff;
8264#[cfg(not(any(unix, windows)))]
8265type NonceHandoff = std::convert::Infallible;
8266
8267fn apply_wire_spawn_args_for_role(
8288 command: &mut Command,
8289 spec: &ModuleSpec,
8290 connection_file_path: Option<&std::path::Path>,
8291 handle: Option<&SupervisorHandle>,
8292 role: SpawnRole,
8293) -> Result<Option<NonceHandoff>, SuperviseError> {
8294 command.env(SUBC_MODULE_ID_ENV, &spec.module_id);
8295 command.env_remove(subc_os::LAUNCH_NONCE_FD_ENV);
8300 #[cfg(windows)]
8301 {
8302 command.env_remove(subc_os::LAUNCH_NONCE_PIPE_ENV);
8303 command.env_remove(subc_os::LAUNCH_NONCE_PIPE_FALLBACK_ENV);
8304 }
8305 command.env_remove(SUBC_LAUNCH_NONCE_ENV);
8307 if spec.protocol == ModuleProtocol::None {
8308 return Ok(None);
8309 }
8310 if let Some(connection_file_path) = connection_file_path {
8311 command.arg(SUBC_ARG).arg(connection_file_path);
8312 }
8313
8314 let nonce = generate_launch_nonce()?;
8318 if let Some(handle) = handle {
8319 match role {
8320 SpawnRole::Plain => {
8321 handle.set_spawn_nonce(&spec.module_id, nonce.clone());
8322 if spec.reserved {
8323 handle.set_reserved_nonce(&spec.module_id, nonce.clone());
8324 }
8325 }
8326 SpawnRole::SwapCandidate => handle.open_swap(&spec.module_id, nonce.clone()),
8327 }
8328 }
8329 #[cfg(unix)]
8330 let handoff = {
8331 let handoff =
8332 subc_os::LaunchNonceHandoff::new(&nonce).map_err(|source| SuperviseError::Spawn {
8333 program: spec.program.clone(),
8334 source,
8335 cgroup_path: None,
8336 })?;
8337 command.env(subc_os::LAUNCH_NONCE_FD_ENV, handoff.fd_env_value());
8338 Some(handoff)
8339 };
8340 #[cfg(windows)]
8341 let handoff = prepare_windows_nonce_pipe(command, spec, &nonce);
8342 #[cfg(not(any(unix, windows)))]
8343 let handoff = None;
8344 #[cfg(not(unix))]
8348 command.env(SUBC_LAUNCH_NONCE_ENV, nonce);
8349 Ok(handoff)
8350}
8351
8352fn apply_child_env(command: &mut Command, spec: &ModuleSpec) {
8353 command.env_remove(CK_LOG_ENV);
8354 command.env_remove(SUBC_SPAWN_ROLE_ENV);
8361 for (key, value) in &spec.env {
8362 if matches!(
8366 key.as_str(),
8367 CAPTURE_MAX_FILE_MB_ENV | CAPTURE_KEEP_ENV | CAPTURE_MAX_AGE_DAYS_ENV
8368 ) || key == SUBC_SPAWN_ROLE_ENV
8369 {
8370 continue;
8371 }
8372 command.env(key, value);
8373 }
8374}
8375
8376#[cfg(windows)]
8377fn prepare_windows_nonce_pipe(
8378 command: &mut Command,
8379 spec: &ModuleSpec,
8380 nonce: &str,
8381) -> Option<NonceHandoff> {
8382 match subc_os::LaunchNoncePipeHandoff::new(nonce) {
8383 Ok(handoff) => {
8384 command.env(subc_os::LAUNCH_NONCE_PIPE_ENV, handoff.name());
8385 command.env(subc_os::LAUNCH_NONCE_PIPE_FALLBACK_ENV, "env");
8390 Some(handoff)
8391 }
8392 Err(error) => {
8393 warn!(module_id = %spec.module_id, %error, "could not prepare launch nonce pipe; retaining the environment handoff");
8394 None
8395 }
8396 }
8397}
8398
8399#[cfg(windows)]
8400fn start_windows_nonce_pipe(
8401 handoff: Option<NonceHandoff>,
8402 pid: u32,
8403 spec: &ModuleSpec,
8404) -> Option<subc_os::LaunchNoncePipeDelivery> {
8405 handoff.and_then(|handoff| match handoff.serve(pid, REGISTRY_RELEASE_TIMEOUT) {
8409 Ok(delivery) => Some(delivery),
8410 Err(error) => {
8411 warn!(module_id = %spec.module_id, %error, "could not serve launch nonce pipe; retaining the environment handoff");
8412 None
8413 }
8414 })
8415}
8416
8417fn observed_launch_nonce_source(snapshot: &SupervisorSnapshot) -> Option<String> {
8418 #[cfg(windows)]
8419 if snapshot.process_alive && snapshot.spawned_protocol == Some(ModuleProtocol::Subc) {
8420 return Some(
8421 if snapshot
8422 .pipe_consumed
8423 .as_ref()
8424 .is_some_and(|consumed| consumed.get().is_some())
8425 {
8426 "pipe"
8427 } else {
8428 "env"
8429 }
8430 .to_owned(),
8431 );
8432 }
8433 let _ = snapshot;
8434 None
8435}
8436
8437#[derive(Debug, Clone, Copy, PartialEq, Eq)]
8440enum SpawnRole {
8441 Plain,
8442 SwapCandidate,
8443}
8444
8445fn apply_spawn_role(command: &mut Command, role: SpawnRole) {
8448 if role == SpawnRole::SwapCandidate {
8449 command.env(SUBC_SPAWN_ROLE_ENV, SPAWN_ROLE_SWAP_CANDIDATE);
8450 }
8451}
8452
8453fn previous_instance_error(
8454 spec: &ModuleSpec,
8455 pid: u32,
8456 reason: crate::live_children::RetainedReason,
8457) -> SuperviseError {
8458 SuperviseError::Spawn {
8459 program: spec.program.clone(),
8460 source: io::Error::new(io::ErrorKind::WouldBlock, reason.waiting_message(pid)),
8461 cgroup_path: None,
8462 }
8463}
8464
8465fn spawn_child(
8466 spec: &ModuleSpec,
8467 connection_file_path: Option<&std::path::Path>,
8468 handle: Option<&SupervisorHandle>,
8469 ring: &Arc<Mutex<StderrRing>>,
8470 capture_logs_dir: Option<&std::path::Path>,
8471 roster: &ChildRoster,
8472 #[cfg(target_os = "linux")] cgroup_placement: Option<&subc_cgroup::Placement>,
8473) -> Result<SupervisedChild, SuperviseError> {
8474 spawn_child_in_slot(
8475 spec,
8476 connection_file_path,
8477 handle,
8478 ring,
8479 capture_logs_dir,
8480 roster,
8481 #[cfg(target_os = "linux")]
8482 cgroup_placement,
8483 SpawnRole::Plain,
8484 false,
8485 )
8486}
8487
8488#[allow(clippy::too_many_arguments)]
8501fn spawn_child_in_slot(
8502 spec: &ModuleSpec,
8503 connection_file_path: Option<&std::path::Path>,
8504 handle: Option<&SupervisorHandle>,
8505 ring: &Arc<Mutex<StderrRing>>,
8506 capture_logs_dir: Option<&std::path::Path>,
8507 roster: &ChildRoster,
8508 #[cfg(target_os = "linux")] cgroup_placement: Option<&subc_cgroup::Placement>,
8509 role: SpawnRole,
8510 alternate_slot: bool,
8511) -> Result<SupervisedChild, SuperviseError> {
8512 if let Some((pid, reason)) = roster.previous_instance(&spec.module_id) {
8513 return Err(previous_instance_error(spec, pid, reason));
8514 }
8515 if roster.is_closed() {
8516 return Err(SuperviseError::Spawn {
8517 program: spec.program.clone(),
8518 source: io::Error::other("the daemon is shutting down; not starting a new process"),
8519 cgroup_path: None,
8520 });
8521 }
8522 #[cfg(target_os = "linux")]
8523 let cgroup_name = {
8524 if cgroup_placement.is_none() {
8529 swap::cgroup_name(&spec.module_id, alternate_slot)
8530 } else {
8531 let nonce = generate_launch_nonce()?;
8532 let suffix = u128::from_str_radix(&nonce[..32], 16).expect("hex launch nonce");
8533 let mut end = spec.module_id.len().min(64);
8536 while !spec.module_id.is_char_boundary(end) {
8537 end -= 1;
8538 }
8539 format!(
8540 "{}_{suffix}",
8541 swap::cgroup_name(&spec.module_id[..end], alternate_slot)
8542 )
8543 }
8544 };
8545 #[cfg(not(target_os = "linux"))]
8546 let _ = alternate_slot;
8547 #[cfg(target_os = "macos")]
8548 let (mut command, privacy_exec, exec_ack) = privacy_command(spec, roster)?;
8549 #[cfg(windows)]
8550 let image_capture = windows_spawn_capture(spec).map_err(|source| SuperviseError::Spawn {
8551 program: spec.program.clone(),
8552 source,
8553 cgroup_path: None,
8554 })?;
8555 #[cfg(windows)]
8556 let mut command = Command::new(image_capture.path());
8557 #[cfg(not(any(target_os = "macos", windows)))]
8558 let mut command = Command::new(&spec.program);
8559 command.args(&spec.args);
8560 apply_child_env(&mut command, spec);
8590 apply_spawn_role(&mut command, role);
8591 let nonce_handoff =
8592 apply_wire_spawn_args_for_role(&mut command, spec, connection_file_path, handle, role)?;
8593
8594 #[cfg(target_os = "linux")]
8595 let cgroup_path = cgroup_placement
8596 .map(|placement| placement.module_path(&cgroup_name))
8597 .transpose()
8598 .map_err(|source| SuperviseError::Cgroup {
8599 module_id: spec.module_id.clone(),
8600 source,
8601 })?;
8602 #[cfg(not(target_os = "linux"))]
8603 let cgroup_path: Option<PathBuf> = None;
8604 #[cfg(target_os = "linux")]
8605 if let Some(path) = &cgroup_path {
8606 if let Err(error) = apply_cgroup_placement(&mut command, spec, path) {
8607 if let Some(placement) = cgroup_placement {
8608 remove_module_cgroup(placement, &cgroup_name);
8609 }
8610 return Err(error);
8611 }
8612 }
8613
8614 let output_sink = if let Some(logs_dir) = capture_logs_dir {
8615 let path = logs_dir.join(format!("{}.stderr.log", spec.module_id));
8616 match ChildOutputSink::open(&path, capture_retention(spec)) {
8617 Ok(sink) => sink,
8618 Err(error) => {
8619 warn!(
8620 module_id = %spec.module_id,
8621 path = %path.display(),
8622 error = %error,
8623 "could not open child output capture file; forwarding to stderr"
8624 );
8625 ChildOutputSink::Stderr
8626 }
8627 }
8628 } else {
8629 ChildOutputSink::Stderr
8630 };
8631
8632 command.stdout(Stdio::piped());
8633 command.stderr(Stdio::piped());
8634 command.kill_on_drop(true);
8635 #[cfg(unix)]
8652 command.process_group(0);
8653 command.stdin(Stdio::null());
8654 #[cfg(unix)]
8658 if let Some(handoff) = nonce_handoff {
8659 handoff.install_last(command.as_std_mut());
8660 }
8661 #[cfg(not(any(unix, windows)))]
8662 let _ = nonce_handoff;
8663
8664 #[cfg(windows)]
8669 {
8670 subc_jobobject::ensure_root_job().map_err(|source| SuperviseError::Spawn {
8671 program: spec.program.clone(),
8672 source,
8673 cgroup_path: None,
8674 })?;
8675 subc_jobobject::suspend_on_create_async(&mut command);
8676 }
8677 let spawned_at = Instant::now();
8681 let mut child = match command.spawn() {
8682 Ok(child) => child,
8683 Err(source) => {
8684 #[cfg(target_os = "linux")]
8685 if let Some(placement) = cgroup_placement {
8686 remove_module_cgroup(placement, &cgroup_name);
8687 }
8688 return Err(SuperviseError::Spawn {
8689 program: spec.program.clone(),
8690 source,
8691 cgroup_path,
8692 });
8693 }
8694 };
8695 #[cfg(test)]
8696 spawn_record_tests::after_create(&child, spec);
8697 #[cfg(target_os = "macos")]
8702 drop(exec_ack);
8703
8704 #[cfg(windows)]
8709 let windows_image = match image_capture.bind_tokio(&child) {
8710 Ok(image) => Arc::new(image),
8711 Err(source) => {
8712 kill_and_reap_rejected_child(&mut child);
8713 return Err(SuperviseError::Spawn {
8714 program: spec.program.clone(),
8715 source,
8716 cgroup_path: None,
8717 });
8718 }
8719 };
8720 #[cfg(windows)]
8723 let job = contain_spawned_child(&mut child, spec)?;
8724 let spawned_at_ms = unix_ms_now();
8725 #[cfg(not(windows))]
8726 let spawned_from = spec.program.clone();
8727 #[cfg(windows)]
8728 let spawned_from = windows_image.path().to_path_buf();
8729 #[cfg(not(windows))]
8730 let spawned_file_identity = spawned_file_identity(&spawned_from);
8731 #[cfg(windows)]
8732 let spawned_file_identity = Some(SpawnedFileIdentity {
8733 device: 0,
8734 inode: 0,
8735 windows_image: Some(Arc::clone(&windows_image)),
8736 });
8737 let pid = child.id().ok_or_else(|| SuperviseError::Spawn {
8738 program: spec.program.clone(),
8739 source: io::Error::other("spawned child exposed no live pid"),
8740 cgroup_path: cgroup_path.clone(),
8741 })?;
8742 #[cfg(windows)]
8743 let nonce_delivery = start_windows_nonce_pipe(nonce_handoff, pid, spec);
8744 #[cfg(not(windows))]
8745 let process_start_time = crate::provenance::process_start_time(pid);
8746 #[cfg(windows)]
8747 let process_start_time = Some(windows_image.start_time());
8748 let process_identity = process_start_time.map(|start_time| ProcessIdentity { pid, start_time });
8749 #[cfg(all(test, target_os = "macos"))]
8750 privacy_exec_boundary_tests::before_image_sample(spec, pid);
8751 #[cfg(not(windows))]
8756 let recorded_image = observe_spawned_image(pid);
8757 #[cfg(target_os = "macos")]
8761 let recorded_image = if privacy_exec.is_some() {
8762 None
8763 } else {
8764 recorded_image
8765 };
8766 #[cfg(target_os = "linux")]
8767 let recorded_cgroup_name = cgroup_path.as_ref().map(|_| cgroup_name.clone());
8768 #[cfg(not(target_os = "linux"))]
8769 let recorded_cgroup_name = None;
8770 let roster_guard = roster.admit(
8771 spec.module_id.clone(),
8772 pid,
8773 spec.protocol,
8774 process_start_time,
8775 crate::child_roster::RecordedIdentity {
8776 #[cfg(not(windows))]
8777 start_time: recorded_image.map(|image| image.start_time),
8778 #[cfg(windows)]
8779 start_time: process_start_time,
8780 #[cfg(not(windows))]
8781 executable: recorded_image
8782 .and_then(|image| image.executable)
8783 .map(crate::live_children::ExecutableIdentity::from),
8784 #[cfg(windows)]
8785 executable: Some(crate::live_children::ExecutableIdentity::from(
8786 windows_image.identity(),
8787 )),
8788 cgroup_name: recorded_cgroup_name,
8789 #[cfg(target_os = "linux")]
8790 cgroup_placement: cgroup_placement.cloned(),
8791 },
8792 );
8793 let roster_guard = match roster_guard {
8794 Ok(guard) => guard,
8795 Err(source) => {
8796 #[cfg(windows)]
8797 if let Some(job) = &job {
8798 let _ = job.terminate();
8799 }
8800 #[cfg(target_os = "linux")]
8801 kill_module_cgroup(cgroup_placement, &cgroup_name);
8802 kill_and_reap_rejected_child(&mut child);
8803 #[cfg(target_os = "linux")]
8804 if let Some(placement) = cgroup_placement {
8805 remove_module_cgroup(placement, &cgroup_name);
8806 }
8807 return Err(SuperviseError::Spawn {
8808 program: spec.program.clone(),
8809 source: io::Error::other(format!(
8810 "could not durably record module child: {source}"
8811 )),
8812 cgroup_path,
8813 });
8814 }
8815 };
8816 if roster.is_closed() {
8825 #[cfg(target_os = "linux")]
8827 kill_module_cgroup(cgroup_placement, &cgroup_name);
8828 if let Err(error) = child.start_kill() {
8829 debug!(module_id = %spec.module_id, pid, %error, "kill of a process spawned during daemon shutdown failed; it may already have exited");
8830 }
8831 #[cfg(target_os = "linux")]
8832 if let Some(placement) = cgroup_placement {
8833 while matches!(child.try_wait(), Ok(None)) {
8837 std::thread::yield_now();
8838 }
8839 if matches!(
8840 subc_cgroup::kill_module(Some(placement), &cgroup_name),
8841 subc_cgroup::KillOutcome::Killed
8842 ) {
8843 if let Ok(path) = placement.module_path(&cgroup_name) {
8844 while std::fs::read_to_string(path.join("cgroup.events"))
8845 .ok()
8846 .is_some_and(|events| events.lines().any(|line| line == "populated 1"))
8847 {
8848 std::thread::yield_now();
8849 }
8850 }
8851 }
8852 remove_module_cgroup(placement, &cgroup_name);
8853 }
8854 drop(roster_guard);
8855 return Err(SuperviseError::Spawn {
8856 program: spec.program.clone(),
8857 source: io::Error::other(
8858 "the daemon began shutting down while this process was starting; ended it",
8859 ),
8860 cgroup_path,
8861 });
8862 }
8863
8864 #[cfg(windows)]
8865 if let Err(error) = resume_suspended_child(pid, spec) {
8866 kill_and_reap_rejected_child(&mut child);
8867 return Err(error);
8868 }
8869
8870 let stdout_pump = match child.stdout.take() {
8871 Some(stdout) => Some(tokio::spawn(pump_stdout_to(stdout, output_sink.clone()))),
8872 None => {
8873 warn!(
8874 module_id = %spec.module_id,
8875 "spawned child exposed no stdout pipe; file capture will be incomplete"
8876 );
8877 None
8878 }
8879 };
8880 let stderr_pump = match child.stderr.take() {
8881 Some(stderr) => {
8882 let generation = ring
8883 .lock()
8884 .unwrap_or_else(|poisoned| poisoned.into_inner())
8885 .begin_process();
8886 Some(StderrPump {
8887 task: tokio::spawn(pump_stderr_to(
8888 stderr,
8889 Arc::clone(ring),
8890 generation,
8891 output_sink,
8892 )),
8893 generation,
8894 })
8895 }
8896 None => {
8897 ring.lock()
8901 .unwrap_or_else(|poisoned| poisoned.into_inner())
8902 .mark_not_captured("stderr pipe was not available on spawn");
8903 warn!(
8904 module_id = %spec.module_id,
8905 "spawned child exposed no stderr pipe; tail will be unavailable"
8906 );
8907 None
8908 }
8909 };
8910
8911 let exec = Arc::new(crate::activity::ExecOutcome::new());
8912 #[cfg(target_os = "macos")]
8913 let pending_exec = privacy_exec.is_some();
8914 #[cfg(not(target_os = "macos"))]
8915 let pending_exec = false;
8916 if !pending_exec {
8917 exec.finish(crate::activity::ExecState::CommittedExec);
8918 }
8919 Ok(SupervisedChild {
8920 child,
8921 exec: crate::activity::ExecOwner(exec),
8922 #[cfg(windows)]
8923 nonce_delivery,
8924 protocol: spec.protocol,
8925 #[cfg(target_os = "linux")]
8926 module_id: cgroup_name,
8927 #[cfg(target_os = "linux")]
8928 cgroup_placement: cgroup_placement.cloned(),
8929 #[cfg(windows)]
8930 job,
8931 stdout_pump,
8932 stderr_pump,
8933 stderr_ring: Arc::clone(ring),
8934 spawned_at,
8935 spawned_at_ms,
8936 spawned_from,
8937 launch_spec_digest: LaunchSpecDigest::from_spec(spec),
8938 spawned_file_identity,
8939 process_start_time,
8940 process_identity,
8941 pid,
8942 roster_guard: Some(roster_guard),
8943 #[cfg(target_os = "macos")]
8944 privacy_exec,
8945 #[cfg(target_os = "macos")]
8946 report_ready: Arc::new(OnceLock::new()),
8947 spawn_failure: None,
8948 })
8949}
8950
8951#[cfg(test)]
8952mod spawn_record_tests {
8953 use super::*;
8954
8955 thread_local! {
8956 static LAST_PID: std::cell::Cell<Option<u32>> = const { std::cell::Cell::new(None) };
8957 }
8958
8959 pub(super) fn after_create(child: &Child, _spec: &ModuleSpec) {
8960 LAST_PID.set(child.id());
8961 #[cfg(windows)]
8962 if let Some((_, address)) = _spec
8963 .env
8964 .iter()
8965 .find(|(key, _)| key == "SUBC_TEST_AFTER_CREATE")
8966 {
8967 use std::io::{Read, Write};
8968 let mut stream = std::net::TcpStream::connect(address).unwrap();
8969 stream
8970 .write_all(&child.id().unwrap().to_ne_bytes())
8971 .unwrap();
8972 let mut release = [0];
8973 stream.read_exact(&mut release).unwrap();
8974 }
8975 }
8976
8977 #[cfg(windows)]
8978 pub(super) fn assignment_failpoint(spec: &ModuleSpec) -> io::Result<()> {
8979 if spec
8980 .env
8981 .iter()
8982 .any(|(key, _)| key == "SUBC_TEST_ASSIGNMENT_FAILURE")
8983 {
8984 Err(io::Error::other("injected module job assignment failure"))
8985 } else {
8986 Ok(())
8987 }
8988 }
8989
8990 fn spec() -> ModuleSpec {
8991 ModuleSpec {
8992 module_id: "spawn-record-worker".into(),
8993 #[cfg(unix)]
8994 program: "/bin/sleep".into(),
8995 #[cfg(windows)]
8996 program: std::env::current_exe().unwrap(),
8997 #[cfg(unix)]
8998 args: vec!["60".into()],
8999 #[cfg(windows)]
9000 args: vec![
9001 "--exact".into(),
9002 "supervise::spawn_record_tests::windows_store_touch_fixture".into(),
9003 "--nocapture".into(),
9004 ],
9005 env: vec![],
9006 reserved: false,
9007 reserved_prefixes: vec![],
9008 protocol: ModuleProtocol::None,
9009 overlap: Default::default(),
9010 }
9011 }
9012
9013 #[tokio::test]
9014 async fn record_sync_failure_fails_spawn_and_reaps_child() {
9015 let home = cortexkit_test_support::ScratchDir::new("record-sync-failure");
9016 let supervisor =
9017 Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default())
9018 .with_live_children_record(home.join("live-children.json"));
9019 crate::live_children::fail_next_file_sync();
9020 let result = supervisor.spawn(spec());
9021 let failed = result.is_err();
9022 if let Ok(module) = result {
9023 module.stop().await.unwrap();
9024 }
9025 assert!(failed, "record failure must fail spawn");
9026 let pid = LAST_PID.get().expect("the real OS child was created");
9027 assert!(
9028 subc_os::Process::open(pid).unwrap().is_none(),
9029 "child {pid} survived rejected admission or was not reaped"
9030 );
9031 }
9032
9033 #[cfg(target_os = "macos")]
9034 #[test]
9035 #[ignore = "manual on-device latency measurement"]
9036 fn durable_record_write_latency_50() {
9037 let home = cortexkit_test_support::ScratchDir::new("record-latency");
9038 let path = home.join("live-children.json");
9039 let children = vec![crate::live_children::LiveChild {
9040 module_id: "latency-worker".into(),
9041 pid: std::process::id(),
9042 protocol: ModuleProtocol::None,
9043 start_time: Some(1),
9044 executable: None,
9045 cgroup_name: None,
9046 }];
9047 let mut samples = Vec::new();
9048 for _ in 0..50 {
9049 let start = std::time::Instant::now();
9050 crate::live_children::write_record(&path, &children).unwrap();
9051 samples.push(start.elapsed());
9052 }
9053 samples.sort();
9054 eprintln!(
9055 "durable_record_write_latency_50: n=50 median={:?} p95={:?}",
9056 (samples[24] + samples[25]) / 2,
9057 samples[47]
9058 );
9059 }
9060
9061 #[cfg(windows)]
9062 #[test]
9063 fn windows_store_touch_fixture() {
9064 if let Some(path) = std::env::var_os("SUBC_TEST_STORE_TOUCH") {
9065 std::fs::write(path, b"module ran").unwrap();
9066 }
9067 }
9068
9069 #[cfg(windows)]
9070 #[tokio::test]
9071 async fn windows_daemon_fixture() {
9072 let Ok(address) = std::env::var("SUBC_TEST_DAEMON_BARRIER") else {
9073 return;
9074 };
9075 let mut spec = spec();
9076 spec.env.push(("SUBC_TEST_AFTER_CREATE".into(), address));
9077 let supervisor =
9078 Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default());
9079 let _module = supervisor.spawn(spec).unwrap();
9080 }
9081
9082 #[cfg(windows)]
9083 #[test]
9084 fn windows_daemon_death_before_module_job_assignment_leaves_no_child() {
9085 use std::io::Read;
9086 let listener = std::net::TcpListener::bind("127.0.0.1:0").unwrap();
9087 let mut command = std::process::Command::new(std::env::current_exe().unwrap());
9090 command
9091 .args([
9092 "--exact",
9093 "supervise::spawn_record_tests::windows_daemon_fixture",
9094 "--nocapture",
9095 ])
9096 .env(
9097 "SUBC_TEST_DAEMON_BARRIER",
9098 listener.local_addr().unwrap().to_string(),
9099 );
9100 let mut daemon = subc_jobobject::spawn_contained(&mut command).unwrap();
9101 listener.set_nonblocking(true).unwrap();
9102 let deadline = std::time::Instant::now() + Duration::from_secs(30);
9103 let mut stream = loop {
9104 if let Ok((stream, _)) = listener.accept() {
9105 break stream;
9106 }
9107 assert!(
9108 daemon.child.try_wait().unwrap().is_none(),
9109 "daemon failed before rendezvous"
9110 );
9111 assert!(
9112 std::time::Instant::now() < deadline,
9113 "daemon did not reach CreateProcess failpoint"
9114 );
9115 std::thread::yield_now();
9116 };
9117 stream
9118 .set_read_timeout(Some(Duration::from_secs(30)))
9119 .unwrap();
9120 let mut pid = [0; 4];
9121 stream.read_exact(&mut pid).unwrap();
9122 let pid = u32::from_ne_bytes(pid);
9123 assert!(subc_jobobject::process_exists(pid));
9124 daemon.child.kill().unwrap();
9127 daemon.child.wait().unwrap();
9128 let exited = subc_jobobject::wait_for_process_exit(pid, Duration::from_secs(10));
9129 if !exited {
9130 daemon.job.terminate().unwrap();
9131 }
9132 assert!(exited, "suspended child {pid} outlived hard-killed daemon");
9133 }
9134
9135 #[cfg(windows)]
9136 #[tokio::test]
9137 async fn windows_containment_failure_never_resumes() {
9138 let home = cortexkit_test_support::ScratchDir::new("assignment-refusal");
9139 let sentinel = home.join("store-touched");
9140 let mut spec = spec();
9141 spec.env
9142 .push(("SUBC_TEST_ASSIGNMENT_FAILURE".into(), "1".into()));
9143 spec.env.push((
9144 "SUBC_TEST_STORE_TOUCH".into(),
9145 sentinel.display().to_string(),
9146 ));
9147 let supervisor =
9148 Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default());
9149 let result = supervisor.spawn(spec);
9150 if let Ok(module) = result {
9153 let _ = tokio::time::timeout(Duration::from_secs(10), async {
9154 while !sentinel.exists() && module.state().unwrap() == ModuleState::Running {
9155 tokio::task::yield_now().await;
9156 }
9157 })
9158 .await;
9159 let _ = module.stop().await;
9160 panic!(
9161 "assignment failure resumed a module; store touched={}",
9162 sentinel.exists()
9163 );
9164 }
9165 assert!(
9166 !sentinel.exists(),
9167 "refused module executed store-touch sentinel"
9168 );
9169 assert!(!subc_jobobject::process_exists(LAST_PID.get().unwrap()));
9170 }
9171
9172 #[cfg(windows)]
9173 #[tokio::test]
9174 async fn windows_nested_daemon_fixture() {
9175 let Some(sentinel) = std::env::var_os("SUBC_TEST_NESTED_STORE") else {
9176 return;
9177 };
9178 let mut spec = spec();
9179 spec.env.push((
9180 "SUBC_TEST_STORE_TOUCH".into(),
9181 sentinel.to_str().unwrap().into(),
9182 ));
9183 let supervisor =
9184 Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default());
9185 let module = supervisor
9186 .spawn(spec)
9187 .expect("per-module job must nest below daemon and host jobs");
9188 tokio::time::timeout(Duration::from_secs(10), async {
9189 while !PathBuf::from(&sentinel).exists() {
9190 tokio::task::yield_now().await;
9191 }
9192 })
9193 .await
9194 .unwrap();
9195 module.stop().await.unwrap();
9196 }
9197
9198 #[cfg(windows)]
9199 #[test]
9200 fn windows_module_job_nests_under_host_job() {
9201 let home = cortexkit_test_support::ScratchDir::new("nested-jobs");
9202 let sentinel = home.join("nested-store-touched");
9203 let mut command = std::process::Command::new(std::env::current_exe().unwrap());
9204 command
9205 .args([
9206 "--exact",
9207 "supervise::spawn_record_tests::windows_nested_daemon_fixture",
9208 "--nocapture",
9209 ])
9210 .env("SUBC_TEST_NESTED_STORE", &sentinel);
9211 let mut daemon = subc_jobobject::spawn_contained(&mut command).unwrap();
9212 assert!(daemon.child.wait().unwrap().success());
9213 assert!(sentinel.exists(), "nested protocol:none module never ran");
9214 }
9215}
9216
9217#[cfg(target_os = "linux")]
9218pub(crate) fn kill_module_cgroup(placement: Option<&subc_cgroup::Placement>, module_id: &str) {
9219 use subc_cgroup::KillOutcome;
9220 match subc_cgroup::kill_module(placement, module_id) {
9221 KillOutcome::Killed => {}
9222 KillOutcome::NotPlaced | KillOutcome::Unsupported => {
9223 debug!(
9224 module_id,
9225 "cgroup tree kill unavailable; using direct-child kill"
9226 );
9227 }
9228 KillOutcome::IoError { path, error } => {
9229 warn!(module_id, path = %path.display(), %error, "cgroup tree kill failed; using direct-child kill");
9230 }
9231 }
9232}
9233
9234#[cfg(windows)]
9242fn contain_spawned_child(
9243 child: &mut Child,
9244 spec: &ModuleSpec,
9245) -> Result<Option<subc_jobobject::JobObject>, SuperviseError> {
9246 let assignment = || {
9247 let job = subc_jobobject::JobObject::new()?;
9248 #[cfg(test)]
9249 spawn_record_tests::assignment_failpoint(spec)?;
9250 job.assign(child)?;
9251 Ok::<_, io::Error>(job)
9252 };
9253 match assignment() {
9254 Ok(job) => Ok(Some(job)),
9255 Err(source) => {
9256 kill_and_reap_rejected_child(child);
9257 Err(SuperviseError::Spawn {
9258 program: spec.program.clone(),
9259 source: io::Error::other(format!(
9260 "Windows module job containment failed: {source}"
9261 )),
9262 cgroup_path: None,
9263 })
9264 }
9265 }
9266}
9267
9268fn kill_and_reap_rejected_child(child: &mut Child) {
9271 #[cfg(unix)]
9272 if let Some(pid) = child
9273 .id()
9274 .and_then(|pid| rustix::process::Pid::from_raw(pid as i32))
9275 {
9276 let _ = rustix::process::kill_process_group(pid, rustix::process::Signal::KILL);
9277 }
9278 let _ = child.start_kill();
9279 while matches!(child.try_wait(), Ok(None)) {
9280 std::thread::yield_now();
9281 }
9282}
9283
9284#[cfg(windows)]
9289fn resume_suspended_child(pid: u32, spec: &ModuleSpec) -> Result<(), SuperviseError> {
9290 if let Err(source) = subc_jobobject::resume_main_thread(pid) {
9291 let _ = std::process::Command::new("taskkill.exe")
9295 .args(["/PID", &pid.to_string(), "/T", "/F"])
9296 .stdin(Stdio::null())
9297 .stdout(Stdio::null())
9298 .stderr(Stdio::null())
9299 .status();
9300 return Err(SuperviseError::Spawn {
9301 program: spec.program.clone(),
9302 source,
9303 cgroup_path: None,
9304 });
9305 }
9306 Ok(())
9307}
9308
9309#[cfg(target_os = "linux")]
9310fn remove_module_cgroup(placement: &subc_cgroup::Placement, module_id: &str) {
9311 match placement.remove_module(module_id) {
9312 Ok(()) => debug!(module_id, "removed module cgroup after process exit"),
9313 Err(error) => warn!(
9314 module_id,
9315 error = %error,
9316 "could not remove module cgroup after process exit; continuing teardown"
9317 ),
9318 }
9319}
9320
9321#[cfg(target_os = "linux")]
9322async fn cleanup_reaped_cgroup(placement: &subc_cgroup::Placement, module_id: &str) {
9323 if matches!(
9327 subc_cgroup::kill_module(Some(placement), module_id),
9328 subc_cgroup::KillOutcome::Killed
9329 ) {
9330 if let Ok(path) = placement.module_path(module_id) {
9331 while std::fs::read_to_string(path.join("cgroup.events"))
9332 .ok()
9333 .is_some_and(|events| events.lines().any(|line| line == "populated 1"))
9334 {
9335 sleep(Duration::from_millis(1)).await;
9336 }
9337 }
9338 }
9339 remove_module_cgroup(placement, module_id);
9340}
9341
9342#[cfg(target_os = "linux")]
9343fn apply_cgroup_placement(
9344 command: &mut Command,
9345 spec: &ModuleSpec,
9346 path: &std::path::Path,
9347) -> Result<(), SuperviseError> {
9348 subc_cgroup::apply(command, path).map_err(|source| SuperviseError::Cgroup {
9349 module_id: spec.module_id.clone(),
9350 source,
9351 })
9352}
9353
9354fn capture_retention(spec: &ModuleSpec) -> Retention {
9355 let defaults = Retention::default();
9356 let value = |name: &str| {
9357 spec.env
9358 .iter()
9359 .rev()
9360 .find_map(|(key, value)| (key == name).then_some(value.as_str()))
9361 };
9362 Retention {
9363 max_file_mb: value(CAPTURE_MAX_FILE_MB_ENV)
9364 .and_then(|value| value.parse().ok())
9365 .unwrap_or(defaults.max_file_mb),
9366 keep: value(CAPTURE_KEEP_ENV)
9367 .and_then(|value| value.parse().ok())
9368 .unwrap_or(defaults.keep),
9369 max_age_days: value(CAPTURE_MAX_AGE_DAYS_ENV)
9370 .and_then(|value| value.parse().ok())
9371 .unwrap_or(defaults.max_age_days),
9372 }
9373}
9374
9375fn generate_launch_nonce() -> Result<String, SuperviseError> {
9378 let mut bytes = [0u8; 32];
9379 getrandom::getrandom(&mut bytes).map_err(|source| SuperviseError::LaunchNonce {
9380 reason: source.to_string(),
9381 })?;
9382 let mut hex = String::with_capacity(64);
9383 for b in bytes {
9384 use std::fmt::Write;
9385 let _ = write!(hex, "{b:02x}");
9386 }
9387 Ok(hex)
9388}
9389
9390fn constant_time_eq(a: &[u8], b: &[u8]) -> bool {
9393 if a.len() != b.len() {
9394 return false;
9395 }
9396 let mut diff = 0u8;
9397 for (x, y) in a.iter().zip(b.iter()) {
9398 diff |= x ^ y;
9399 }
9400 diff == 0
9401}
9402
9403#[cfg(not(windows))]
9407fn observe_spawned_image(pid: u32) -> Option<subc_os::Observation> {
9408 let process = subc_os::Process::open(pid).ok().flatten()?;
9409 #[cfg(target_os = "linux")]
9410 return sample_after_linux_exec(
9411 || {
9412 let stat = std::fs::read_to_string(format!("/proc/{pid}/stat")).ok()?;
9413 let committed = linux_exec_committed(&stat);
9414 #[cfg(test)]
9415 spawn_image_tests::report_exec_state(committed);
9416 committed
9417 },
9418 || process.observe(),
9419 );
9420 #[cfg(not(target_os = "linux"))]
9421 process.observe()
9422}
9423
9424#[cfg(target_os = "linux")]
9428fn linux_exec_committed(stat: &str) -> Option<bool> {
9429 const PF_FORKNOEXEC: u64 = 0x40;
9430 let flags = stat.rsplit_once(')')?.1.split_whitespace().nth(6)?;
9431 Some(flags.parse::<u64>().ok()? & PF_FORKNOEXEC == 0)
9432}
9433
9434#[cfg(target_os = "linux")]
9435fn sample_after_linux_exec(
9436 mut exec_committed: impl FnMut() -> Option<bool>,
9437 observe: impl FnOnce() -> Option<subc_os::Observation>,
9438) -> Option<subc_os::Observation> {
9439 let deadline = std::time::Instant::now() + Duration::from_secs(1);
9440 while !exec_committed()? {
9441 if std::time::Instant::now() >= deadline {
9445 return None;
9446 }
9447 std::thread::sleep(Duration::from_micros(50));
9448 }
9449 observe()
9450}
9451
9452#[cfg(all(test, target_os = "linux"))]
9453mod spawn_image_tests {
9454 use super::*;
9455 use std::cell::{Cell, RefCell};
9456
9457 enum ExecEvent {
9458 Uncommitted,
9459 Image(Option<subc_os::Observation>),
9460 }
9461
9462 thread_local! {
9463 static EXEC_EVENTS: RefCell<Option<std::sync::mpsc::Sender<ExecEvent>>> = const { RefCell::new(None) };
9464 }
9465
9466 pub(super) fn report_exec_state(committed: Option<bool>) {
9467 if committed == Some(false) {
9468 EXEC_EVENTS.with(|events| {
9469 if let Some(events) = events.borrow_mut().take() {
9470 let _ = events.send(ExecEvent::Uncommitted);
9471 }
9472 });
9473 }
9474 }
9475
9476 #[test]
9477 fn linux_spawn_observation_waits_for_a_paused_fork_to_exec() {
9478 use std::io::{Read, Write};
9479 use std::os::fd::AsRawFd;
9480
9481 let descriptor = std::fs::File::open("/dev/null").unwrap();
9482 let mut command = std::process::Command::new("/bin/sleep");
9483 command.arg("30");
9484 let (mut ready, mut release) =
9485 subc_os::fork_exec_test::pause_before_exec(&mut command, descriptor.as_raw_fd())
9486 .unwrap();
9487 let spawn = std::thread::spawn(move || command.spawn());
9488 let mut message = [0u8; 8];
9489 ready.read_exact(&mut message).unwrap();
9490 let pid = i32::from_ne_bytes(message[..4].try_into().unwrap()) as u32;
9491 let before = std::fs::read_to_string(format!("/proc/{pid}/stat"));
9492 let inherited = subc_os::Process::open(pid).unwrap().unwrap().observe();
9493 let (events, received) = std::sync::mpsc::channel();
9494 let observer = std::thread::spawn(move || {
9495 EXEC_EVENTS.with(|slot| *slot.borrow_mut() = Some(events.clone()));
9496 let image = observe_spawned_image(pid);
9497 let _ = events.send(ExecEvent::Image(image));
9498 });
9499 let first = received.recv_timeout(Duration::from_secs(5));
9503 release.write_all(&[1]).unwrap();
9504 let mut child = spawn.join().unwrap().unwrap();
9505 let after = match first {
9506 Ok(ExecEvent::Uncommitted) => match received.recv_timeout(Duration::from_secs(5)) {
9507 Ok(ExecEvent::Image(image)) => image,
9508 _ => None,
9509 },
9510 Ok(ExecEvent::Image(image)) => image,
9511 Err(_) => None,
9512 };
9513 observer.join().unwrap();
9514 child.kill().unwrap();
9515 child.wait().unwrap();
9516
9517 assert_eq!(linux_exec_committed(&before.unwrap()), Some(false));
9518 assert_eq!(
9519 inherited.unwrap().executable,
9520 subc_os::file_identity(&std::env::current_exe().unwrap())
9521 );
9522 assert_eq!(
9523 after.unwrap().executable,
9524 subc_os::file_identity(std::path::Path::new("/bin/sleep"))
9525 );
9526 }
9527
9528 #[test]
9529 fn spawned_image_is_not_sampled_until_linux_exec_commits() {
9530 let committed = Cell::new(false);
9531 let inherited = subc_os::Observation {
9532 start_time: 23,
9533 executable: Some(subc_os::FileIdentity {
9534 device: 29,
9535 inode: 31,
9536 }),
9537 };
9538 let module = subc_os::Observation {
9539 start_time: 23,
9540 executable: Some(subc_os::FileIdentity {
9541 device: 29,
9542 inode: 37,
9543 }),
9544 };
9545 let polls = Cell::new(0);
9546 let image = sample_after_linux_exec(
9547 || {
9548 polls.set(polls.get() + 1);
9549 committed.set(polls.get() == 2);
9552 Some(committed.get())
9553 },
9554 || Some(if committed.get() { module } else { inherited }),
9555 );
9556 assert_eq!(image, Some(module));
9557 assert_eq!(polls.get(), 2);
9558 }
9559
9560 #[test]
9561 fn linux_exec_flag_is_read_from_flags_after_the_last_parenthesis() {
9562 assert_eq!(
9563 linux_exec_committed("17 (a ) b) R 1 2 3 4 5 4194368"),
9564 Some(false)
9565 );
9566 assert_eq!(
9567 linux_exec_committed("17 (a ) b) R 1 2 3 4 5 4194304"),
9568 Some(true)
9569 );
9570 assert_eq!(linux_exec_committed("17 (a) R 1 2 3 4 5 invalid"), None);
9571 assert_eq!(linux_exec_committed("17 (a) R"), None);
9572 }
9573
9574 #[test]
9575 fn unreadable_exec_state_never_samples_an_inherited_image() {
9576 assert_eq!(
9577 sample_after_linux_exec(
9578 || None,
9579 || panic!("must not sample before exec confirmation")
9580 ),
9581 None
9582 );
9583 }
9584
9585 #[test]
9586 fn uncommitted_exec_times_out_without_sampling_an_inherited_image() {
9587 assert_eq!(
9588 sample_after_linux_exec(
9589 || Some(false),
9590 || panic!("must not sample after an exec timeout")
9591 ),
9592 None
9593 );
9594 }
9595}
9596
9597#[cfg(windows)]
9600fn windows_spawn_capture(spec: &ModuleSpec) -> io::Result<subc_os::ExecutableCapture> {
9601 let mut program = spec.program.clone();
9602 if program.is_absolute() || program.components().count() > 1 {
9603 match subc_os::ExecutableCapture::open(&program) {
9606 Ok(capture) => return Ok(capture),
9607 Err(error)
9608 if error.kind() == io::ErrorKind::NotFound && program.extension().is_none() => {}
9609 Err(error) => return Err(error),
9610 }
9611 program.set_extension("exe");
9612 return subc_os::ExecutableCapture::open(&program);
9613 }
9614 if program.extension().is_none() {
9615 program.set_extension("exe");
9616 }
9617 let custom_path = spec
9618 .env
9619 .iter()
9620 .rev()
9621 .find(|(key, _)| key.eq_ignore_ascii_case("PATH"));
9622 let mut directories = Vec::new();
9623 if let Some(parent) = std::env::current_exe()?.parent() {
9626 directories.push(parent.to_path_buf());
9627 }
9628 directories.push(std::env::current_dir()?);
9629 if let Some(windows) = std::env::var_os("SystemRoot") {
9630 let windows = PathBuf::from(windows);
9631 directories.push(windows.join("System32"));
9632 directories.push(windows.clone());
9633 }
9634 let path = custom_path
9635 .map(|(_, value)| std::ffi::OsString::from(value))
9636 .or_else(|| std::env::var_os("PATH"));
9637 if let Some(path) = path {
9638 directories.extend(std::env::split_paths(&path));
9639 }
9640 for directory in directories {
9641 match subc_os::ExecutableCapture::open(&directory.join(&program)) {
9642 Ok(capture) => return Ok(capture),
9643 Err(error) if error.kind() == io::ErrorKind::NotFound => continue,
9644 Err(error) => return Err(error),
9645 }
9646 }
9647 Err(io::Error::new(
9648 io::ErrorKind::NotFound,
9649 "resolved executable not found",
9650 ))
9651}
9652
9653fn spawn_and_mark_running(
9654 spec: &ModuleSpec,
9655 runtime: &SupervisorRuntimeConfig,
9656 snapshot: &SharedSnapshot,
9657) -> Result<SupervisedChild, SuperviseError> {
9658 let activity_spawn = runtime.begin_activity_spawn(&spec.module_id);
9659 let child = spawn_child(
9660 spec,
9661 runtime.connection_file_path.as_deref(),
9662 runtime.supervisor_handle.as_ref(),
9663 &runtime.stderr_ring,
9664 runtime.capture_logs_dir.as_deref(),
9665 &runtime.child_roster,
9666 #[cfg(target_os = "linux")]
9667 runtime.cgroup_placement.as_ref(),
9668 )?;
9669 set_running(snapshot, &child, &spec.module_id, &runtime.spawn_events)?;
9670 if let Some(spawn) = activity_spawn {
9671 spawn.complete(snapshot);
9672 }
9673 Ok(child)
9674}
9675
9676enum RegistrationWaitOutcome {
9677 Registered,
9678 Exited(ExitReport),
9679 TimedOut,
9680}
9681
9682struct ReloadRegistrationFailure {
9683 exit_report: ExitReport,
9684 reason: String,
9685}
9686
9687#[derive(Debug, Clone, Copy, PartialEq, Eq)]
9688enum BusyGaugeObservation {
9689 Quiescent,
9690 Busy,
9691 Omitted,
9692}
9693
9694#[derive(Debug, Clone, PartialEq, Eq)]
9696enum GaugeVerdict {
9697 Zero,
9699 NonZero(u64),
9701 NotACount(String),
9705 Missing,
9707 ProbeFailed(&'static str),
9710}
9711
9712impl GaugeVerdict {
9713 fn reads_quiet(&self) -> bool {
9714 matches!(self, Self::Zero)
9715 }
9716}
9717
9718impl fmt::Display for GaugeVerdict {
9719 fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
9720 match self {
9721 Self::Zero => f.write_str("zero"),
9722 Self::NonZero(value) => write!(f, "non_zero({value})"),
9723 Self::NotACount(value) => write!(f, "not_a_count({value})"),
9724 Self::Missing => f.write_str("missing_from_health_body"),
9725 Self::ProbeFailed(kind) => write!(f, "probe_failed({kind})"),
9726 }
9727 }
9728}
9729
9730const GAUGE_VALUE_TEXT_LIMIT: usize = 32;
9733
9734fn gauge_verdicts(metrics: Option<&Value>, gauges: &[String]) -> Vec<GaugeVerdict> {
9736 let metrics = metrics.and_then(Value::as_object);
9737 gauges
9738 .iter()
9739 .map(
9740 |gauge| match metrics.and_then(|metrics| metrics.get(gauge)) {
9741 None => GaugeVerdict::Missing,
9742 Some(value) => match value.as_u64() {
9743 Some(0) => GaugeVerdict::Zero,
9744 Some(count) => GaugeVerdict::NonZero(count),
9745 None => {
9746 let mut text = value.to_string();
9747 if text.len() > GAUGE_VALUE_TEXT_LIMIT {
9748 let mut end = GAUGE_VALUE_TEXT_LIMIT;
9749 while !text.is_char_boundary(end) {
9750 end -= 1;
9751 }
9752 text.truncate(end);
9753 text.push_str("...");
9754 }
9755 GaugeVerdict::NotACount(text)
9756 }
9757 },
9758 },
9759 )
9760 .collect()
9761}
9762
9763fn busy_gauge_observation(verdicts: &[GaugeVerdict]) -> BusyGaugeObservation {
9766 if verdicts
9767 .iter()
9768 .any(|verdict| matches!(verdict, GaugeVerdict::Missing))
9769 {
9770 BusyGaugeObservation::Omitted
9771 } else if verdicts.iter().all(GaugeVerdict::reads_quiet) {
9772 BusyGaugeObservation::Quiescent
9773 } else {
9774 BusyGaugeObservation::Busy
9775 }
9776}
9777
9778#[derive(Debug, Clone, Default, PartialEq, Eq)]
9783struct GaugeTrace {
9784 latest: Option<(GaugeVerdict, Duration)>,
9785 last_value: Option<(GaugeVerdict, Duration)>,
9786}
9787
9788#[derive(Debug, Clone, Default, PartialEq, Eq)]
9794struct DrainGaugeTrace {
9795 probes: u32,
9797 gauges: Vec<(String, GaugeTrace)>,
9799}
9800
9801impl DrainGaugeTrace {
9802 fn new(gauges: &[String]) -> Self {
9803 Self {
9804 probes: 0,
9805 gauges: gauges
9806 .iter()
9807 .map(|gauge| (gauge.clone(), GaugeTrace::default()))
9808 .collect(),
9809 }
9810 }
9811
9812 fn record(&mut self, verdicts: &[GaugeVerdict], at: Duration) {
9814 self.probes = self.probes.saturating_add(1);
9815 for ((_, trace), verdict) in self.gauges.iter_mut().zip(verdicts) {
9816 trace.latest = Some((verdict.clone(), at));
9817 if matches!(
9818 verdict,
9819 GaugeVerdict::Zero | GaugeVerdict::NonZero(_) | GaugeVerdict::NotACount(_)
9820 ) {
9821 trace.last_value = Some((verdict.clone(), at));
9822 }
9823 }
9824 }
9825}
9826
9827impl fmt::Display for DrainGaugeTrace {
9832 fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
9833 if self.gauges.is_empty() {
9834 return f.write_str("none declared");
9835 }
9836 for (index, (name, trace)) in self.gauges.iter().enumerate() {
9837 if index > 0 {
9838 f.write_str(" | ")?;
9839 }
9840 write!(f, "{name}: ")?;
9841 match &trace.latest {
9842 Some((verdict, at)) => write!(f, "{verdict} @{}ms", at.as_millis())?,
9843 None => f.write_str("never probed")?,
9844 }
9845 match &trace.last_value {
9846 Some((value, at)) => write!(f, "; last value {value} @{}ms", at.as_millis())?,
9847 None => f.write_str("; no value ever read")?,
9848 }
9849 }
9850 Ok(())
9851 }
9852}
9853
9854pub(crate) fn declared_busy_gauges(
9855 registry: &Registry,
9856 module_id: &str,
9857) -> Result<Vec<String>, SuperviseError> {
9858 busy_gauges_of(
9859 registry
9860 .get_module(module_id)
9861 .map_err(SuperviseError::Registry)?,
9862 )
9863}
9864
9865fn declared_busy_gauges_for_connection(
9869 registry: &Registry,
9870 connection_id: ConnectionId,
9871) -> Result<Vec<String>, SuperviseError> {
9872 busy_gauges_of(
9873 registry
9874 .get_module_by_connection(connection_id)
9875 .map_err(SuperviseError::Registry)?,
9876 )
9877}
9878
9879fn busy_gauges_of(
9880 registration: Option<crate::registry::ModuleRegistration>,
9881) -> Result<Vec<String>, SuperviseError> {
9882 let Some(registration) = registration else {
9883 return Ok(Vec::new());
9884 };
9885 let Some(self_signals) = registration.manifest.self_signals else {
9886 return Ok(Vec::new());
9887 };
9888
9889 let mut gauges = Vec::new();
9890 for declaration in self_signals {
9891 if declaration.kind != SelfSignalKind::Busy {
9892 continue;
9893 }
9894 match declaration.anchored_to {
9895 SignalAnchor::HealthGauges { gauges: declared } if !declared.is_empty() => {
9896 gauges.extend(declared)
9897 }
9898 _ => {
9899 gauges.push(String::new());
9902 }
9903 }
9904 }
9905 Ok(gauges)
9906}
9907
9908#[allow(clippy::too_many_arguments)]
9917async fn wait_for_forwarding_quiescence(
9918 forwarding: &ForwardingTable,
9919 module_id: &str,
9920 runtime: &SupervisorRuntimeConfig,
9921 endpoint: crate::ModuleEndpointId,
9922 deadline: Instant,
9923 busy_gauges: &[String],
9924 scope: DrainScope,
9925 trace: &mut DrainGaugeTrace,
9926) -> Result<bool, SuperviseError> {
9927 let wait_started = Instant::now();
9928 let mut gauges_quiescent = busy_gauges.is_empty();
9929 let mut next_probe_at = wait_started;
9930 let mut omission_reported = false;
9931
9932 loop {
9933 let now = Instant::now();
9934 if !busy_gauges.is_empty() && now >= next_probe_at && now < deadline {
9935 let report = match scope {
9936 DrainScope::Active => probe_module_health(module_id, runtime, Some(deadline)).await,
9937 DrainScope::Endpoint(endpoint) => {
9938 probe_endpoint_health(endpoint, runtime, Some(deadline)).await
9939 }
9940 };
9941 let verdicts = match &report {
9942 Ok(report) => gauge_verdicts(report.metrics.as_ref(), busy_gauges),
9943 Err(err) => vec![GaugeVerdict::ProbeFailed(err.label()); busy_gauges.len()],
9944 };
9945 trace.record(&verdicts, wait_started.elapsed());
9946 gauges_quiescent = match report {
9947 Ok(_) => match busy_gauge_observation(&verdicts) {
9948 BusyGaugeObservation::Quiescent => true,
9949 BusyGaugeObservation::Busy => false,
9950 BusyGaugeObservation::Omitted => {
9951 if !omission_reported {
9954 forwarding
9955 .counters()
9956 .increment_drains_with_undeclared_gauge();
9957 let missing = busy_gauges
9958 .iter()
9959 .zip(&verdicts)
9960 .filter(|(_, verdict)| matches!(verdict, GaugeVerdict::Missing))
9961 .map(|(gauge, _)| gauge.as_str())
9962 .collect::<Vec<_>>()
9963 .join(",");
9964 warn!(
9965 module_id,
9966 missing_gauges = %missing,
9967 "drain health.check answer lacks a declared busy gauge; the drain treats the module as busy until it reports it"
9968 );
9969 omission_reported = true;
9970 }
9971 false
9972 }
9973 },
9974 Err(err) => {
9975 warn!(
9976 module_id,
9977 error = %err,
9978 "drain health.check did not produce declared busy gauges; treating module as busy"
9979 );
9980 false
9981 }
9982 };
9983 next_probe_at = Instant::now() + drain_busy_reprobe_interval();
9986 }
9987
9988 let in_flight = forwarding
9989 .endpoint_in_flight_count(endpoint)
9990 .map_err(SuperviseError::Forwarding)?;
9991 if in_flight == 0 && gauges_quiescent {
9992 return Ok(true);
9993 }
9994
9995 let now = Instant::now();
9996 if now >= deadline {
9997 return Ok(false);
9998 }
9999 let mut wait = deadline
10000 .saturating_duration_since(now)
10001 .min(REGISTRY_RELEASE_POLL);
10002 if !busy_gauges.is_empty() {
10003 wait = wait.min(next_probe_at.saturating_duration_since(now));
10004 }
10005 sleep(wait).await;
10006 }
10007}
10008
10009fn drain_busy_reprobe_interval() -> Duration {
10013 DRAIN_BUSY_REPROBE_INTERVAL.max(REGISTRY_RELEASE_POLL)
10014}
10015
10016fn warn_drain_timed_out(
10031 module_id: &str,
10032 drain_timeout: Duration,
10033 reason: RouteCloseReason,
10034 holdouts: &crate::forwarding::DrainHoldouts,
10035 gauges: &DrainGaugeTrace,
10036) {
10037 warn!(
10038 module_id = %module_id,
10039 waited = ?drain_timeout,
10040 ?reason,
10041 held_requests = holdouts.requests,
10042 held_routes = holdouts.routes,
10043 total_routes = holdouts.total_routes,
10044 top_connections = ?holdouts.top_connections,
10045 held = %holdouts
10048 .held
10049 .iter()
10050 .map(|(channel, corr)| format!("{channel}:{corr}"))
10051 .collect::<Vec<_>>()
10052 .join(","),
10053 gauge_probes = gauges.probes,
10054 busy_gauges = %gauges,
10055 "route drain timed out before request quiescence; forcing teardown"
10056 );
10057}
10058
10059fn drained_after_quiescence_wait(wait_result: &Result<bool, SuperviseError>) -> bool {
10067 match wait_result {
10068 Ok(drained) => *drained,
10069 Err(_) => false,
10070 }
10071}
10072
10073fn send_route_goodbyes(forwarding: &ForwardingTable, released_routes: Vec<GoodbyeTarget>) {
10074 for released in released_routes {
10075 let frame = match Frame::build_with_version(
10076 released.negotiated_ver,
10077 FrameType::Goodbye,
10078 control_flags(),
10079 released.channel,
10080 released.epoch,
10081 0,
10082 Vec::new(),
10083 ) {
10084 Ok(frame) => frame,
10085 Err(err) => {
10086 warn!(
10087 route_channel = released.channel,
10088 error = %err,
10089 "failed to build supervisor drain route GOODBYE frame"
10090 );
10091 continue;
10092 }
10093 };
10094 if !released.close_on_delivery_failure() {
10095 crate::forwarding::send_module_route_goodbye(
10096 &forwarding.counters(),
10097 &released.sink,
10098 frame,
10099 released.module_id.as_deref(),
10100 "supervisor drain",
10101 );
10102 continue;
10103 }
10104 if let Err(err) = released.sink.try_send(frame) {
10105 warn!(
10106 target_connection_id = released.connection_id.get(),
10107 route_channel = released.channel,
10108 error = %err,
10109 "supervisor drain route GOODBYE was not delivered to client; closing target connection"
10110 );
10111 let _ = forwarding.escalate_client_delivery_failure(
10112 released.connection_id,
10113 released.channel,
10114 released.epoch,
10115 CloseReason::new(
10116 "route_goodbye_delivery_failed",
10117 format!(
10118 "failed to enqueue supervisor drain route GOODBYE for channel {}: {err}",
10119 released.channel
10120 ),
10121 ),
10122 crate::forwarding::UndeliveredFrame {
10123 module_id: released.module_id.as_deref(),
10124 sink: &released.sink,
10125 },
10126 );
10127 }
10128 }
10129}
10130
10131fn send_module_draining(
10132 module_id: &str,
10133 reason: RouteCloseReason,
10134 deadline_ms: u64,
10135 target: &ModuleDrainTarget,
10136) {
10137 let body = match serde_json::to_vec(&ModuleControlCommand::Draining {
10138 reason,
10139 deadline_ms,
10140 }) {
10141 Ok(body) => body,
10142 Err(err) => {
10143 warn!(
10144 module_id,
10145 error = %err,
10146 "failed to encode module draining command"
10147 );
10148 return;
10149 }
10150 };
10151 let frame = match Frame::build_with_version(
10152 target.negotiated_ver,
10153 FrameType::Push,
10154 control_flags(),
10155 0,
10156 0,
10157 0,
10158 body,
10159 ) {
10160 Ok(frame) => frame,
10161 Err(err) => {
10162 warn!(
10163 module_id,
10164 error = %err,
10165 "failed to build module draining command frame"
10166 );
10167 return;
10168 }
10169 };
10170 if let Err(err) = target.sink.try_send(frame) {
10171 warn!(
10172 module_id,
10173 target_connection_id = target.endpoint.connection_id.get(),
10174 error = %err,
10175 "module draining command was not delivered to peer"
10176 );
10177 }
10178}
10179
10180fn module_goodbye_frame(module_id: &str, negotiated_ver: u8) -> Option<Frame> {
10182 match Frame::build_with_version(
10183 negotiated_ver,
10184 FrameType::Goodbye,
10185 control_flags(),
10186 0,
10187 0,
10188 0,
10189 Vec::new(),
10190 ) {
10191 Ok(frame) => Some(frame),
10192 Err(err) => {
10193 warn!(
10194 module_id,
10195 error = %err,
10196 "failed to build module GOODBYE frame"
10197 );
10198 None
10199 }
10200 }
10201}
10202
10203#[cfg(unix)]
10217async fn send_module_goodbyes_for_daemon_shutdown(
10218 forwarding: &Arc<ForwardingTable>,
10219 reason: &CloseReason,
10220 wait_for_flush: bool,
10221) {
10222 const GOODBYE_BUDGET: Duration = Duration::from_millis(500);
10223 let targets = match forwarding.module_connections() {
10224 Ok(targets) => targets,
10225 Err(err) => {
10226 warn!(error = %err, "could not list module connections for shutdown GOODBYE");
10227 return;
10228 }
10229 };
10230 let deadline = Instant::now() + GOODBYE_BUDGET;
10231 let mut sends = tokio::task::JoinSet::new();
10232 for target in targets {
10233 let Some(frame) = module_goodbye_frame(&target.module_id, target.negotiated_ver) else {
10234 continue;
10235 };
10236 if !wait_for_flush {
10237 if let Err(err) = target.sink.try_send(frame) {
10238 debug!(
10239 module_id = %target.module_id,
10240 error = %err,
10241 "shutdown module GOODBYE was not queued"
10242 );
10243 }
10244 continue;
10245 }
10246 let forwarding = Arc::clone(forwarding);
10247 let reason = reason.clone();
10248 sends.spawn(async move {
10249 match timeout_at(deadline, target.sink.send_flushed(frame)).await {
10250 Ok(Ok(())) => {}
10251 Ok(Err(err)) => debug!(
10252 module_id = %target.module_id,
10253 error = %err,
10254 "module connection closed before its shutdown GOODBYE was written"
10255 ),
10256 Err(_) => warn!(
10257 module_id = %target.module_id,
10258 budget = ?GOODBYE_BUDGET,
10259 "shutdown module GOODBYE was not written within its budget; closing anyway"
10260 ),
10261 }
10262 forwarding.request_connection_close(target.endpoint.connection_id, reason);
10263 });
10264 }
10265 while sends.join_next().await.is_some() {}
10267}
10268
10269fn send_module_goodbye(module_id: &str, forwarding: &ForwardingTable, target: &ModuleDrainTarget) {
10270 let Some(frame) = module_goodbye_frame(module_id, target.negotiated_ver) else {
10271 return;
10272 };
10273 if let Err(err) = target.sink.try_send(frame) {
10274 warn!(
10275 module_id,
10276 target_connection_id = target.endpoint.connection_id.get(),
10277 error = %err,
10278 "supervisor drain module GOODBYE was not delivered to peer; closing module connection"
10279 );
10280 forwarding.request_connection_close(
10281 target.endpoint.connection_id,
10282 CloseReason::new(
10283 "module_goodbye_delivery_failed",
10284 format!("failed to enqueue supervisor drain module GOODBYE for module '{module_id}': {err}"),
10285 ),
10286 );
10287 }
10288}
10289
10290#[derive(Clone, Copy)]
10291struct ForwardingDrainContext<'a> {
10292 spec: &'a ModuleSpec,
10293 runtime: &'a SupervisorRuntimeConfig,
10294 registry: &'a Registry,
10295 scope: DrainScope,
10296}
10297
10298#[derive(Debug, Clone, Copy, PartialEq, Eq)]
10300enum DrainScope {
10301 Active,
10304 Endpoint(crate::ModuleEndpointId),
10309}
10310
10311#[derive(Debug, Clone, Copy, PartialEq, Eq)]
10319enum StopNotice {
10320 SentOverConnection,
10323 NoConnection,
10327 NotSent,
10331}
10332
10333async fn begin_forwarding_drain(
10334 spec: &ModuleSpec,
10335 runtime: &SupervisorRuntimeConfig,
10336 registry: &Registry,
10337 snapshot: &SharedSnapshot,
10338 enabled: Option<bool>,
10339 reason: RouteCloseReason,
10340) -> Result<StopNotice, SuperviseError> {
10341 let Some(forwarding) = runtime.forwarding.as_ref() else {
10342 return Err(SuperviseError::ReloadUnavailable {
10343 module_id: spec.module_id.clone(),
10344 reason: "supervisor was not configured with a forwarding table".to_string(),
10345 });
10346 };
10347
10348 begin_forwarding_drain_with(
10349 forwarding,
10350 ForwardingDrainContext {
10351 spec,
10352 runtime,
10353 registry,
10354 scope: DrainScope::Active,
10355 },
10356 snapshot,
10357 enabled,
10358 reason,
10359 runtime.drain_timeout,
10360 )
10361 .await
10362}
10363
10364async fn begin_forwarding_drain_if_configured(
10365 spec: &ModuleSpec,
10366 runtime: &SupervisorRuntimeConfig,
10367 registry: &Registry,
10368 snapshot: &SharedSnapshot,
10369 enabled: Option<bool>,
10370 reason: RouteCloseReason,
10371) -> Result<StopNotice, SuperviseError> {
10372 begin_forwarding_drain_with_timeout(
10373 spec,
10374 runtime,
10375 registry,
10376 snapshot,
10377 enabled,
10378 reason,
10379 runtime.drain_timeout,
10380 )
10381 .await
10382}
10383
10384async fn begin_forwarding_drain_with_timeout(
10388 spec: &ModuleSpec,
10389 runtime: &SupervisorRuntimeConfig,
10390 registry: &Registry,
10391 snapshot: &SharedSnapshot,
10392 enabled: Option<bool>,
10393 reason: RouteCloseReason,
10394 drain_timeout: Duration,
10395) -> Result<StopNotice, SuperviseError> {
10396 let Some(forwarding) = runtime.forwarding.as_ref() else {
10397 return Ok(StopNotice::NotSent);
10398 };
10399
10400 begin_forwarding_drain_with(
10401 forwarding,
10402 ForwardingDrainContext {
10403 spec,
10404 runtime,
10405 registry,
10406 scope: DrainScope::Active,
10407 },
10408 snapshot,
10409 enabled,
10410 reason,
10411 drain_timeout,
10412 )
10413 .await
10414}
10415
10416async fn begin_forwarding_drain_with(
10417 forwarding: &ForwardingTable,
10418 context: ForwardingDrainContext<'_>,
10419 snapshot: &SharedSnapshot,
10420 enabled: Option<bool>,
10421 reason: RouteCloseReason,
10422 drain_timeout: Duration,
10423) -> Result<StopNotice, SuperviseError> {
10424 let ForwardingDrainContext {
10425 spec,
10426 runtime,
10427 registry,
10428 scope,
10429 } = context;
10430 debug_assert_ne!(reason, RouteCloseReason::Crash);
10431 let terminal = matches!(reason, RouteCloseReason::Disable);
10432 let drain_started_at = Instant::now();
10433 let drain_deadline = drain_started_at + drain_timeout;
10434 let deadline_ms =
10435 unix_ms_now().saturating_add(u64::try_from(drain_timeout.as_millis()).unwrap_or(u64::MAX));
10436 let busy_gauges = match scope {
10437 DrainScope::Active => declared_busy_gauges(registry, &spec.module_id)?,
10438 DrainScope::Endpoint(endpoint) => {
10439 declared_busy_gauges_for_connection(registry, endpoint.connection_id)?
10440 }
10441 };
10442
10443 let gate_started = Instant::now();
10446 let drain_target = match scope {
10447 DrainScope::Active => forwarding.begin_module_drain(&spec.module_id, reason),
10448 DrainScope::Endpoint(endpoint) => forwarding.begin_endpoint_drain(endpoint, reason),
10449 }
10450 .map_err(SuperviseError::Forwarding)?;
10451 info!(
10456 module_id = %spec.module_id,
10457 ?reason,
10458 gate_ms = u64::try_from(gate_started.elapsed().as_millis()).unwrap_or(u64::MAX),
10459 connected = drain_target.is_some(),
10460 "module drain began; route admission closed"
10461 );
10462 if scope == DrainScope::Active {
10463 update_snapshot(snapshot, Some(&spec.module_id), |state| {
10464 state.state = ModuleState::Draining;
10465 state.draining_to_replace =
10466 matches!(reason, RouteCloseReason::Restart | RouteCloseReason::Reload);
10467 if let Some(enabled) = enabled {
10468 state.enabled = enabled;
10469 }
10470 })?;
10471 }
10472
10473 let Some(target) = drain_target.as_ref() else {
10474 return Ok(StopNotice::NoConnection);
10478 };
10479 {
10480 send_module_draining(&spec.module_id, reason, deadline_ms, target);
10481 #[cfg(feature = "test-support")]
10482 forwarding.wait_before_route_closing_for_test().await;
10483 let routes = forwarding
10484 .endpoint_routes(target.endpoint)
10485 .map_err(SuperviseError::Forwarding)?;
10486 let routes_notified = routes.len();
10487 crate::control::send_route_control_pushes(
10488 forwarding,
10489 routes.clone(),
10490 ClientControlPush::RouteClosing {
10491 module_id: spec.module_id.clone(),
10492 channels: Vec::new(),
10493 reason,
10494 },
10495 );
10496 send_route_goodbyes(forwarding, target.abandoned_bindings.clone());
10497
10498 let mut gauge_trace = DrainGaugeTrace::new(&busy_gauges);
10504 let wait_result = wait_for_forwarding_quiescence(
10505 forwarding,
10506 &spec.module_id,
10507 runtime,
10508 target.endpoint,
10509 drain_deadline,
10510 &busy_gauges,
10511 scope,
10512 &mut gauge_trace,
10513 )
10514 .await;
10515 let drained = drained_after_quiescence_wait(&wait_result);
10516 if let Err(err) = &wait_result {
10517 error!(
10518 module_id = %spec.module_id,
10519 ?reason,
10520 error = %err,
10521 "forwarding quiescence wait failed after route.closing; forcing route.closed(drained: false) so the client is not left waiting on an unfulfilled promise"
10522 );
10523 } else if !drained {
10524 let holdouts = forwarding
10526 .endpoint_drain_holdouts(target.endpoint)
10527 .unwrap_or_default();
10528 warn_drain_timed_out(
10529 &spec.module_id,
10530 drain_timeout,
10531 reason,
10532 &holdouts,
10533 &gauge_trace,
10534 );
10535 }
10536 crate::control::send_route_control_pushes(
10537 forwarding,
10538 routes,
10539 ClientControlPush::RouteClosed {
10540 module_id: spec.module_id.clone(),
10541 channels: Vec::new(),
10542 reason,
10543 drained,
10544 abandoned: target.abandoned_bindings.len() as u32,
10545 excluded_subscriptions: target.excluded_subscriptions,
10546 terminal: Some(terminal),
10547 },
10548 );
10549 wait_result?;
10550
10551 let released_routes = match forwarding.release_module_endpoint_routes(target.endpoint) {
10557 Ok(routes) => routes,
10558 Err(err) => {
10559 warn!(
10560 module_id = %spec.module_id,
10561 ?reason,
10562 error = %err,
10563 "failed to release module endpoint routes after route.closed; module GOODBYE will still be sent"
10564 );
10565 send_module_goodbye(&spec.module_id, forwarding, target);
10566 return Err(SuperviseError::Forwarding(err));
10567 }
10568 };
10569 let route_goodbye_count = released_routes.len();
10570 send_route_goodbyes(forwarding, released_routes);
10571 send_module_goodbye(&spec.module_id, forwarding, target);
10572
10573 info!(
10579 module_id = %spec.module_id,
10580 ?reason,
10581 routes_notified,
10582 route_goodbyes = route_goodbye_count,
10583 abandoned_reservations = target.abandoned_bindings.len(),
10584 excluded_subscriptions = target.excluded_subscriptions,
10585 drained,
10586 "module drain complete; consumers notified via route.closing/route.closed pushes and per-route GOODBYE frames"
10587 );
10588 }
10589
10590 Ok(StopNotice::SentOverConnection)
10591}
10592
10593async fn wait_for_registration_after_reload(
10596 registry: &Registry,
10597 module_id: &str,
10598 snapshot: &SharedSnapshot,
10599 child: &mut SupervisedChild,
10600 wait: Duration,
10601) -> Result<RegistrationWaitOutcome, SuperviseError> {
10602 wait_for_slot_registration(
10603 registry,
10604 crate::registry::RegistrationSlot::Active(module_id),
10605 module_id,
10606 snapshot,
10607 child,
10608 wait,
10609 )
10610 .await
10611}
10612
10613async fn wait_for_slot_registration(
10621 registry: &Registry,
10622 slot: crate::registry::RegistrationSlot<'_>,
10623 module_id: &str,
10624 snapshot: &SharedSnapshot,
10625 child: &mut SupervisedChild,
10626 wait: Duration,
10627) -> Result<RegistrationWaitOutcome, SuperviseError> {
10628 let deadline = Instant::now() + wait;
10629 loop {
10630 if registry
10631 .registration(slot)
10632 .map_err(SuperviseError::Registry)?
10633 .is_some()
10634 {
10635 return Ok(RegistrationWaitOutcome::Registered);
10636 }
10637
10638 let now = Instant::now();
10639 if now >= deadline {
10640 return Ok(RegistrationWaitOutcome::TimedOut);
10641 }
10642 let remaining = deadline.saturating_duration_since(now);
10643 let poll = remaining.min(REGISTRY_RELEASE_POLL);
10644
10645 tokio::select! {
10646 wait_result = child.wait() => {
10647 let status = wait_result.map_err(|source| SuperviseError::Wait {
10648 module_id: module_id.to_string(),
10649 source,
10650 })?;
10651 return Ok(RegistrationWaitOutcome::Exited(classify_reaped_child_exit(
10652 snapshot,
10653 child,
10654 &status,
10655 )));
10656 }
10657 _ = sleep(poll) => {}
10658 }
10659 }
10660}
10661
10662fn registration_failure_exit_report(mut exit_report: ExitReport) -> ExitReport {
10663 if exit_report.kind != ExitKind::DeliberateSeverance {
10666 exit_report.kind = ExitKind::Crash;
10667 }
10668 exit_report
10669}
10670
10671async fn handle_reload_child_registration_failure(
10672 spec: &ModuleSpec,
10673 runtime: &SupervisorRuntimeConfig,
10674 registry: &Registry,
10675 process_liveness: &SupervisorProcessLiveness,
10676 snapshot: &SharedSnapshot,
10677 _child: &mut Option<SupervisedChild>,
10678 failure: ReloadRegistrationFailure,
10679) -> Result<(), SuperviseError> {
10680 let ReloadRegistrationFailure {
10681 exit_report,
10682 reason,
10683 } = failure;
10684 match on_child_exit(
10685 spec,
10686 runtime.restart_policy,
10687 registry,
10688 snapshot,
10689 &runtime.terminal_ring,
10690 &runtime.spawn_events,
10691 &runtime.child_roster,
10692 exit_report,
10693 )
10694 .await
10695 {
10696 NextAction::Stop {
10697 registration_released,
10698 } => {
10699 if registration_released {
10700 process_liveness.untrack_if_current(&spec.module_id, snapshot);
10701 }
10702 }
10703 NextAction::Restart { schedule } => {
10704 let delay = schedule.map_or(runtime.restart_policy.delay_for_restart(0), |schedule| {
10705 schedule.delay
10706 });
10707 if let Some(schedule) = schedule {
10708 log_crash_respawn(&spec.module_id, schedule);
10709 }
10710 schedule_respawn(
10711 runtime,
10712 snapshot,
10713 &spec.module_id,
10714 delay,
10715 RespawnKind::Spawn,
10716 )?;
10717 }
10718 }
10719 Err(SuperviseError::ReloadFailed {
10720 module_id: spec.module_id.clone(),
10721 reason,
10722 })
10723}
10724
10725async fn handle_reload_spawn_failure(
10726 spec: &ModuleSpec,
10727 runtime: &SupervisorRuntimeConfig,
10728 process_liveness: &SupervisorProcessLiveness,
10729 snapshot: &SharedSnapshot,
10730 _child: &mut Option<SupervisedChild>,
10731 reason: String,
10732) -> Result<(), SuperviseError> {
10733 let now = Instant::now();
10734 let mut schedule = None;
10735 update_snapshot(snapshot, Some(&spec.module_id), |state| {
10736 clear_current_process_facts(state);
10737 if state.enabled {
10738 schedule = state.next_crash_restart(&runtime.restart_policy, now);
10739 state.state = if schedule.is_some() {
10740 ModuleState::Restarting
10741 } else {
10742 ModuleState::Failed
10743 };
10744 } else {
10745 state.state = ModuleState::Disabled;
10746 }
10747 })?;
10748 if let Some(schedule) = schedule {
10749 schedule_respawn(
10750 runtime,
10751 snapshot,
10752 &spec.module_id,
10753 schedule.delay,
10754 RespawnKind::Spawn,
10755 )?;
10756 } else {
10757 process_liveness.untrack_if_current(&spec.module_id, snapshot);
10758 }
10759 Err(SuperviseError::ReloadFailed {
10760 module_id: spec.module_id.clone(),
10761 reason,
10762 })
10763}
10764
10765fn control_flags() -> Flags {
10766 Flags::new(false, Priority::Passive, false)
10767}
10768
10769#[allow(clippy::too_many_arguments)]
10770async fn drain_optional_child(
10771 module_id: &str,
10772 protocol: ModuleProtocol,
10773 stop_notice: StopNotice,
10774 registry: &Registry,
10775 forwarding: Option<&ForwardingTable>,
10776 snapshot: &SharedSnapshot,
10777 terminal_ring: &Arc<Mutex<TerminalRing>>,
10778 spawn_events: &SpawnEventFeed,
10779 child: &mut Option<SupervisedChild>,
10780 drain_timeout: Duration,
10781 final_state: ModuleState,
10782 enabled: Option<bool>,
10783) -> Result<(), SuperviseError> {
10784 if let Some(child) = child.take() {
10785 drain_child_to_state(
10786 module_id,
10787 protocol,
10788 stop_notice,
10789 registry,
10790 forwarding,
10791 snapshot,
10792 terminal_ring,
10793 spawn_events,
10794 child,
10795 drain_timeout,
10796 final_state,
10797 enabled,
10798 )
10799 .await
10800 } else {
10801 update_snapshot(snapshot, Some(module_id), |state| {
10802 state.state = final_state;
10803 if let Some(enabled) = enabled {
10804 state.enabled = enabled;
10805 }
10806 clear_current_process_facts(state);
10807 })?;
10808 release_dead_registration(registry, forwarding, snapshot, module_id).await
10809 }
10810}
10811
10812#[allow(clippy::too_many_arguments)]
10813async fn drain_child_to_state(
10814 module_id: &str,
10815 _protocol: ModuleProtocol,
10816 stop_notice: StopNotice,
10817 registry: &Registry,
10818 forwarding: Option<&ForwardingTable>,
10819 snapshot: &SharedSnapshot,
10820 terminal_ring: &Arc<Mutex<TerminalRing>>,
10821 spawn_events: &SpawnEventFeed,
10822 mut child: SupervisedChild,
10823 drain_timeout: Duration,
10824 final_state: ModuleState,
10825 enabled: Option<bool>,
10826) -> Result<(), SuperviseError> {
10827 let protocol = child.protocol;
10828 update_snapshot(snapshot, Some(module_id), |state| {
10829 state.state = ModuleState::Draining;
10830 state.draining_to_replace = final_state == ModuleState::Restarting;
10831 if let Some(enabled) = enabled {
10832 state.enabled = enabled;
10833 }
10834 })?;
10835
10836 if stop_notice != StopNotice::SentOverConnection {
10847 if protocol == ModuleProtocol::Subc && stop_notice == StopNotice::NoConnection {
10848 info!(
10849 module_id,
10850 pid = child.pid,
10851 budget_ms = u64::try_from(drain_timeout.as_millis()).unwrap_or(u64::MAX),
10852 "module has no connection yet; requesting stop by signal"
10853 );
10854 }
10855 request_graceful_stop(module_id, &child);
10856 }
10857
10858 let exit_report = match timeout(drain_timeout, child.wait()).await {
10859 Ok(Ok(status)) => classify_reaped_child_exit(snapshot, &child, &status),
10860 Ok(Err(source)) => {
10861 fail_after_wait_error(
10862 module_id,
10863 snapshot,
10864 terminal_ring,
10865 spawn_events,
10866 format!("failed to wait during drain: {source}"),
10867 );
10868 return Err(SuperviseError::Wait {
10869 module_id: module_id.to_string(),
10870 source,
10871 });
10872 }
10873 Err(_) => {
10874 warn!(
10887 module_id,
10888 pid = child.pid,
10889 budget_ms = u64::try_from(drain_timeout.as_millis()).unwrap_or(u64::MAX),
10890 reason = ?final_state,
10891 ?stop_notice,
10892 "drain budget expired before the module exited; killing it"
10893 );
10894 child.start_kill().map_err(|source| {
10895 fail_after_wait_error(
10896 module_id,
10897 snapshot,
10898 terminal_ring,
10899 spawn_events,
10900 format!("failed to kill during drain: {source}"),
10901 );
10902 SuperviseError::Kill {
10903 module_id: module_id.to_string(),
10904 source,
10905 }
10906 })?;
10907 let status = child.wait().await.map_err(|source| {
10908 fail_after_wait_error(
10909 module_id,
10910 snapshot,
10911 terminal_ring,
10912 spawn_events,
10913 format!("failed to wait after drain kill: {source}"),
10914 );
10915 SuperviseError::Wait {
10916 module_id: module_id.to_string(),
10917 source,
10918 }
10919 })?;
10920 classify_reaped_child_exit(snapshot, &child, &status)
10921 }
10922 };
10923
10924 let detail = lock_snapshot(snapshot)?.drain_disposition_detail.take();
10925 publish_after_terminal_record(
10926 module_id,
10927 || {
10928 record_terminal_with_detail(
10929 module_id,
10930 terminal_ring,
10931 spawn_events,
10932 &exit_report,
10933 terminal_disposition(final_state),
10934 detail,
10935 )
10936 },
10937 || {
10938 update_snapshot(snapshot, Some(module_id), |state| {
10939 state.state = final_state;
10940 if let Some(enabled) = enabled {
10941 state.enabled = enabled;
10942 }
10943 clear_current_process_facts(state);
10944 state.last_exit = Some(exit_report.clone());
10945 if exit_report.kind == ExitKind::DeliberateSeverance {
10946 state.lifetime_restarts += 1;
10947 }
10948 })
10949 },
10950 )?;
10951 child.drain_stderr(module_id).await;
10952
10953 release_dead_registration(registry, forwarding, snapshot, module_id).await
10954}
10955
10956#[cfg(unix)]
10976fn request_graceful_stop(module_id: &str, child: &SupervisedChild) {
10977 let Some(pid) = child
10978 .id()
10979 .and_then(|pid| i32::try_from(pid).ok())
10980 .and_then(rustix::process::Pid::from_raw)
10981 else {
10982 debug!(
10983 module_id,
10984 "no pid to signal for teardown; falling through to the drain wait"
10985 );
10986 return;
10987 };
10988 match rustix::process::kill_process(pid, rustix::process::Signal::TERM) {
10989 Ok(()) => debug!(
10990 module_id,
10991 "sent SIGTERM to a module nothing else asked to stop"
10992 ),
10993 Err(err) => debug!(
10994 module_id,
10995 error = %err,
10996 "SIGTERM to module failed; the drain wait and kill still apply"
10997 ),
10998 }
10999}
11000
11001#[cfg(not(unix))]
11009fn request_graceful_stop(module_id: &str, _child: &SupervisedChild) {
11010 debug!(
11011 module_id,
11012 "no graceful stop signal exists on this platform; teardown of a module nothing asked to stop waits, then kills"
11013 );
11014}
11015
11016fn terminal_disposition(final_state: ModuleState) -> TerminalDisposition {
11017 match final_state {
11018 ModuleState::Stopped => TerminalDisposition::Stopped,
11019 ModuleState::Disabled => TerminalDisposition::Disabled,
11020 ModuleState::Restarting => TerminalDisposition::Restarting,
11021 ModuleState::Failed => TerminalDisposition::Failed,
11022 ModuleState::Starting
11023 | ModuleState::Running
11024 | ModuleState::Unresponsive
11025 | ModuleState::Draining => {
11026 unreachable!("terminal exits only finish in terminal or restarting states")
11027 }
11028 }
11029}
11030
11031async fn release_dead_registration(
11039 registry: &Registry,
11040 forwarding: Option<&ForwardingTable>,
11041 snapshot: &SharedSnapshot,
11042 module_id: &str,
11043) -> Result<(), SuperviseError> {
11044 let result = async {
11045 let registration = registry
11046 .get_module(module_id)
11047 .map_err(SuperviseError::Registry)?;
11048 match wait_for_registration_release(registry, module_id, REGISTRY_RELEASE_TIMEOUT).await {
11049 Ok(()) => return Ok(()),
11050 Err(SuperviseError::RegistrationStillActive { .. }) => {}
11051 Err(err) => return Err(err),
11052 }
11053 let pid = lock_snapshot(snapshot)?.reaped_pid;
11054 if let (Some(registration), Some(forwarding), Some(pid)) = (registration, forwarding, pid) {
11055 warn!(
11056 module_id,
11057 pid,
11058 connection_id = registration.connection_id.get(),
11059 "reaped module registration outlived release grace; closing dead connection"
11060 );
11061 forwarding.request_connection_close(
11062 registration.connection_id,
11063 CloseReason::new(
11064 "supervised_process_reaped",
11065 format!("module '{module_id}' pid {pid} exited"),
11066 ),
11067 );
11068 wait_for_slot_registration_release(
11069 registry,
11070 crate::registry::RegistrationSlot::Connection(registration.connection_id),
11071 REGISTRY_RELEASE_TIMEOUT,
11072 )
11073 .await?;
11074 }
11075 wait_for_registration_release(registry, module_id, Duration::ZERO).await
11076 }
11077 .await;
11078 if let Err(err) = &result {
11079 fail_snapshot(snapshot, Some(module_id), None);
11080 error!(module_id, error = %err, "registration release failed after child exit; module is failed and start can retry");
11081 }
11082 result
11083}
11084
11085async fn wait_for_registration_release(
11088 registry: &Registry,
11089 module_id: &str,
11090 wait: Duration,
11091) -> Result<(), SuperviseError> {
11092 wait_for_slot_registration_release(
11093 registry,
11094 crate::registry::RegistrationSlot::Active(module_id),
11095 wait,
11096 )
11097 .await
11098}
11099
11100async fn wait_for_slot_registration_release(
11108 registry: &Registry,
11109 slot: crate::registry::RegistrationSlot<'_>,
11110 wait: Duration,
11111) -> Result<(), SuperviseError> {
11112 let deadline = Instant::now() + wait;
11113 let mut release_events = registration_release_events().subscribe();
11114 let still_active = |registration: &crate::registry::ModuleRegistration| {
11115 SuperviseError::RegistrationStillActive {
11116 module_id: registration.manifest.module_id.clone(),
11117 waited: wait,
11118 }
11119 };
11120 loop {
11121 let _observed_generation = *release_events.borrow_and_update();
11122 let Some(registration) = registry
11123 .registration(slot)
11124 .map_err(SuperviseError::Registry)?
11125 else {
11126 return Ok(());
11127 };
11128
11129 let now = Instant::now();
11130 if now >= deadline {
11131 return Err(still_active(®istration));
11132 }
11133
11134 let remaining = deadline.saturating_duration_since(now);
11135 match timeout(remaining, release_events.changed()).await {
11136 Ok(Ok(())) | Ok(Err(_)) => {}
11137 Err(_) => return Err(still_active(®istration)),
11138 }
11139 }
11140}
11141
11142#[cfg(test)]
11143mod slot_registration_wait_tests {
11144 use super::*;
11145 use crate::registry::{ConnectionId, RegistrationSlot};
11146 use subc_protocol::manifest::ModuleManifest;
11147
11148 #[tokio::test]
11149 async fn enable_release_failure_is_failed_and_a_second_enable_retries() {
11150 let registry = Arc::new(Registry::default());
11151 let supervisor = Supervisor::new_for_test(Arc::clone(®istry), RestartPolicy::default());
11152 let runtime = supervisor.runtime_config();
11153 let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::disabled()));
11154 let spec = ModuleSpec {
11155 module_id: "enable-stale-registration".to_string(),
11156 program: PathBuf::from("/missing/enable-retry-test"),
11157 args: Vec::new(),
11158 env: Vec::new(),
11159 reserved: false,
11160 reserved_prefixes: Vec::new(),
11161 protocol: ModuleProtocol::Subc,
11162 overlap: Default::default(),
11163 };
11164 let connection = ConnectionId::new(90);
11165 registry
11166 .register_with_control_ops(
11167 ModuleManifest::builder(&spec.module_id, "0.1.0").build(),
11168 1,
11169 connection,
11170 Vec::new(),
11171 )
11172 .unwrap();
11173 let mut child = None;
11174 let err = set_child_enabled(
11175 &spec,
11176 &runtime,
11177 ®istry,
11178 &supervisor.process_liveness,
11179 &snapshot,
11180 &mut child,
11181 true,
11182 )
11183 .await
11184 .unwrap_err();
11185 assert!(matches!(
11186 err,
11187 SuperviseError::RegistrationStillActive { .. }
11188 ));
11189 assert_eq!(lock_snapshot(&snapshot).unwrap().state, ModuleState::Failed);
11190 assert!(child.is_none());
11191 registry.deregister_connection(connection).unwrap();
11192 let err = set_child_enabled(
11193 &spec,
11194 &runtime,
11195 ®istry,
11196 &supervisor.process_liveness,
11197 &snapshot,
11198 &mut child,
11199 true,
11200 )
11201 .await
11202 .unwrap_err();
11203 assert!(
11204 matches!(err, SuperviseError::Spawn { .. }),
11205 "second enable must attempt a spawn: {err}"
11206 );
11207 assert_eq!(lock_snapshot(&snapshot).unwrap().state, ModuleState::Failed);
11208 }
11209
11210 const INCUMBENT: u64 = 1;
11211 const CANDIDATE: u64 = 2;
11212
11213 fn swapped_registry() -> Arc<Registry> {
11214 let registry = Arc::new(Registry::default());
11215 let manifest = ModuleManifest::builder("m", "0.1.0").build();
11216 registry
11217 .register_with_control_ops(
11218 manifest.clone(),
11219 1,
11220 ConnectionId::new(INCUMBENT),
11221 Vec::new(),
11222 )
11223 .unwrap();
11224 registry
11225 .register_candidate_with_control_ops(
11226 manifest,
11227 1,
11228 ConnectionId::new(CANDIDATE),
11229 Vec::new(),
11230 )
11231 .unwrap();
11232 registry
11233 }
11234
11235 #[tokio::test]
11239 async fn incumbent_release_is_awaited_by_connection_not_by_module_id() {
11240 let registry = swapped_registry();
11241 registry.promote_candidate("m").unwrap().unwrap();
11242
11243 assert!(matches!(
11244 wait_for_registration_release(®istry, "m", Duration::from_millis(50)).await,
11245 Err(SuperviseError::RegistrationStillActive { .. })
11246 ));
11247
11248 assert!(matches!(
11250 wait_for_slot_registration_release(
11251 ®istry,
11252 RegistrationSlot::Connection(ConnectionId::new(INCUMBENT)),
11253 Duration::from_millis(50),
11254 )
11255 .await,
11256 Err(SuperviseError::RegistrationStillActive { .. })
11257 ));
11258
11259 let releaser = Arc::clone(®istry);
11260 let release = tokio::spawn(async move {
11261 sleep(Duration::from_millis(20)).await;
11262 releaser
11263 .deregister_connection(ConnectionId::new(INCUMBENT))
11264 .unwrap();
11265 notify_registration_release();
11266 });
11267 wait_for_slot_registration_release(
11268 ®istry,
11269 RegistrationSlot::Connection(ConnectionId::new(INCUMBENT)),
11270 Duration::from_secs(5),
11271 )
11272 .await
11273 .expect("the incumbent's own registration is released");
11274 release.await.unwrap();
11275 assert!(registry.get_module("m").unwrap().is_some());
11276 }
11277
11278 #[tokio::test]
11281 async fn candidate_slot_wait_ignores_the_incumbents_registration() {
11282 let registry = swapped_registry();
11283 assert!(matches!(
11284 wait_for_slot_registration_release(
11285 ®istry,
11286 RegistrationSlot::Candidate("m"),
11287 Duration::from_millis(50),
11288 )
11289 .await,
11290 Err(SuperviseError::RegistrationStillActive { .. })
11291 ));
11292 registry
11293 .deregister_connection(ConnectionId::new(CANDIDATE))
11294 .unwrap();
11295 wait_for_slot_registration_release(
11296 ®istry,
11297 RegistrationSlot::Candidate("m"),
11298 Duration::from_millis(50),
11299 )
11300 .await
11301 .expect("a candidate slot with no candidate is released");
11302 assert!(registry
11303 .registration(RegistrationSlot::Active("m"))
11304 .unwrap()
11305 .is_some());
11306 }
11307}
11308
11309fn classify_exit(status: &ExitStatus) -> ExitReport {
11310 ExitReport {
11311 kind: if status.success() {
11312 ExitKind::Clean
11313 } else {
11314 ExitKind::Crash
11315 },
11316 code: status.code(),
11317 signal: exit_signal(status),
11318 at_ms: unix_ms_now(),
11319 }
11320}
11321
11322fn wait_error_exit_report() -> ExitReport {
11328 ExitReport {
11329 kind: ExitKind::Crash,
11330 code: None,
11331 signal: None,
11332 at_ms: unix_ms_now(),
11333 }
11334}
11335
11336#[cfg(unix)]
11337fn exit_signal(status: &ExitStatus) -> Option<i32> {
11338 use std::os::unix::process::ExitStatusExt;
11339
11340 status.signal()
11341}
11342
11343#[cfg(not(unix))]
11344fn exit_signal(_status: &ExitStatus) -> Option<i32> {
11345 None
11346}
11347
11348fn reset_restart_count(snapshot: &SharedSnapshot, module_id: &str) -> Result<(), SuperviseError> {
11354 update_snapshot(snapshot, Some(module_id), |state| {
11355 state.clear_crash_restarts();
11356 })
11357}
11358
11359fn set_running(
11360 snapshot: &SharedSnapshot,
11361 child: &SupervisedChild,
11362 module_id: &str,
11363 spawn_events: &SpawnEventFeed,
11364) -> Result<(), SuperviseError> {
11365 let mut state = snapshot.lock().map_err(|_| SuperviseError::StatePoisoned {
11366 module_id: Some(module_id.to_string()),
11367 })?;
11368 state.spawn_generation = spawn_events.emit_spawned(module_id, child.pid, child.spawned_at_ms);
11369 if std::mem::take(&mut state.coalesced_restart_pending) {
11370 let generation = state.spawn_generation;
11371 info!(module_id, "restart already satisfied by generation {generation}; coalesced pending request completed");
11372 }
11373 state.drain_disposition_detail = None;
11374 state.spawn_failure = None;
11375 state.waiting_for_previous_pid = None;
11376 state.waiting_for_previous_detail = None;
11377 state.in_alternate_slot = false;
11380 state.configuration_updated_since_spawn = false;
11381 state.spawned_protocol = Some(child.protocol);
11382 state.state = ModuleState::Running;
11383 state.enabled = true;
11384 state.process_alive = true;
11385 state.pid = child.id();
11386 state.health.status = SupervisorHealthStatus::Unknown;
11389 state.health.consecutive_failures = 0;
11390 state.health.detail = None;
11391 state.health.metrics = None;
11392 state.health.last_probe_ms = None;
11393 #[cfg(windows)]
11394 {
11395 state.pipe_consumed = child
11396 .nonce_delivery
11397 .as_ref()
11398 .map(|delivery| delivery.consumed());
11399 }
11400 #[cfg(target_os = "macos")]
11401 {
11402 state.report_ready = Some(Arc::clone(&child.report_ready));
11403 }
11404 state.spawned_at_ms = Some(child.spawned_at_ms);
11405 state.exec_outcome = Some(Arc::clone(&child.exec.0));
11406 state.spawned_from = Some(child.spawned_from.clone());
11407 state.launch_spec_digest = Some(child.launch_spec_digest.clone());
11408 state.spawned_file_identity = child.spawned_file_identity.clone();
11409 state.process_start_time = child.process_start_time;
11410 Ok(())
11411}
11412
11413fn clear_current_process_facts(state: &mut SupervisorSnapshot) {
11414 state.waiting_for_previous_pid = None;
11415 state.waiting_for_previous_detail = None;
11416 state.process_alive = false;
11417 state.spawned_protocol = None;
11418 state.pid = None;
11419 #[cfg(windows)]
11420 {
11421 state.pipe_consumed = None;
11422 }
11423 #[cfg(target_os = "macos")]
11424 {
11425 state.report_ready = None;
11426 }
11427 state.spawned_at_ms = None;
11428 state.exec_outcome = None;
11429 state.spawned_from = None;
11430 state.launch_spec_digest = None;
11431 state.spawned_file_identity = None;
11432 state.process_start_time = None;
11433 state.deliberate_severance = None;
11434}
11435
11436#[cfg(test)]
11437fn record_deliberate_severance(
11438 snapshot: &SharedSnapshot,
11439 identity: ProcessIdentity,
11440) -> Result<(), SuperviseError> {
11441 update_snapshot(snapshot, None, |state| {
11442 state.deliberate_severance = Some(identity);
11443 })
11444}
11445
11446fn apply_deliberate_severance_marker(
11447 snapshot: &SharedSnapshot,
11448 exited_identity: Option<ProcessIdentity>,
11449 mut exit_report: ExitReport,
11450) -> ExitReport {
11451 let marker = lock_snapshot(snapshot)
11452 .ok()
11453 .and_then(|mut state| state.deliberate_severance.take());
11454 if marker.is_some() && marker == exited_identity {
11455 exit_report.kind = ExitKind::DeliberateSeverance;
11456 }
11457 exit_report
11458}
11459
11460fn classify_reaped_child_exit(
11461 snapshot: &SharedSnapshot,
11462 child: &SupervisedChild,
11463 status: &ExitStatus,
11464) -> ExitReport {
11465 let _ = update_snapshot(snapshot, None, |state| {
11466 state.reaped_pid = Some(child.pid);
11467 state.spawn_failure = child.spawn_failure.clone();
11468 });
11469 apply_deliberate_severance_marker(snapshot, child.process_identity(), classify_exit(status))
11470}
11471
11472fn fail_snapshot(
11473 snapshot: &SharedSnapshot,
11474 module_id: Option<&str>,
11475 last_exit: Option<ExitReport>,
11476) {
11477 if let Err(err) = update_snapshot(snapshot, module_id, |state| {
11478 state.state = ModuleState::Failed;
11479 clear_current_process_facts(state);
11480 if let Some(last_exit) = last_exit {
11481 state.last_exit = Some(last_exit);
11482 }
11483 }) {
11484 error!(error = %err, "failed to mark supervisor state failed");
11485 }
11486}
11487
11488fn update_snapshot(
11489 snapshot: &SharedSnapshot,
11490 module_id: Option<&str>,
11491 update: impl FnOnce(&mut SupervisorSnapshot),
11492) -> Result<(), SuperviseError> {
11493 let mut state = snapshot.lock().map_err(|_| SuperviseError::StatePoisoned {
11494 module_id: module_id.map(ToOwned::to_owned),
11495 })?;
11496 update(&mut state);
11497 Ok(())
11498}
11499
11500const SLOW_SNAPSHOT_LOCK_THRESHOLD: Duration = Duration::from_millis(250);
11501
11502fn lock_snapshot_for_control<'a>(
11503 snapshot: &'a SharedSnapshot,
11504 module_id: &str,
11505 caller: &'static str,
11506) -> Result<std::sync::MutexGuard<'a, SupervisorSnapshot>, SuperviseError> {
11507 let started_at = Instant::now();
11508 let guard = lock_snapshot(snapshot)?;
11509 let waited = started_at.elapsed();
11510 if waited >= SLOW_SNAPSHOT_LOCK_THRESHOLD {
11511 warn!(
11512 module_id = %module_id,
11513 waited_ms = waited.as_millis() as u64,
11514 caller = %caller,
11515 "slow snapshot lock"
11516 );
11517 }
11518 Ok(guard)
11519}
11520
11521fn lock_snapshot(
11522 snapshot: &SharedSnapshot,
11523) -> Result<std::sync::MutexGuard<'_, SupervisorSnapshot>, SuperviseError> {
11524 snapshot
11525 .lock()
11526 .map_err(|_| SuperviseError::StatePoisoned { module_id: None })
11527}
11528
11529#[cfg(test)]
11530mod terminal_history_tests {
11531 use std::{
11532 path::PathBuf,
11533 sync::Arc,
11534 time::{Duration, Instant},
11535 };
11536
11537 use tokio::{sync::mpsc, time::sleep};
11538
11539 use super::{
11540 apply_deliberate_severance_marker, daemon_will_restart, drain_child_to_state,
11541 drained_after_quiescence_wait, handle_reload_spawn_failure, health_restart_child,
11542 lock_snapshot, on_child_exit, record_deliberate_severance, record_wait_error_terminal,
11543 reset_restart_count, spawn_and_mark_running, update_snapshot, wait_error_exit_report,
11544 ExitKind, ExitReport, ModuleProtocol, ModuleSpec, ModuleState, NextAction, ProcessIdentity,
11545 RestartPolicy, SpawnEventKind, StopNotice, SuperviseError, SupervisedConfiguration,
11546 SupervisedModule, SupervisedModuleInner, Supervisor, SupervisorHandle,
11547 SupervisorHealthStatus, SupervisorSnapshot,
11548 };
11549 use super::Instant as ClockInstant;
11554 use crate::{
11555 registry::Registry,
11556 terminal_ring::{TerminalRing, TerminalRingConfig},
11557 };
11558 use std::sync::Mutex;
11559 use subc_control::TerminalDisposition;
11560
11561 pub(super) fn fake_aft_stub_path() -> PathBuf {
11566 let mut path = std::env::current_exe().expect("current_exe available in tests");
11567 path.pop();
11568 path.pop();
11569 path.push(if cfg!(windows) {
11570 "fake-aft-stub.exe"
11571 } else {
11572 "fake-aft-stub"
11573 });
11574 assert!(
11575 path.exists(),
11576 "fake-aft-stub not built at {}: run `cargo test -p subc-core` (which builds \
11577 [[bin]] targets) rather than `cargo test -p subc-core --lib` (which does not)",
11578 path.display()
11579 );
11580 path
11581 }
11582
11583 #[test]
11584 fn reserved_never_spawned_refuses_every_hello() {
11585 let supervisor = SupervisorHandle::default();
11590 supervisor.apply_identity_configuration(&ModuleSpec {
11591 module_id: "never-spawned".to_string(),
11592 program: PathBuf::from("/usr/bin/false"),
11593 args: Vec::new(),
11594 env: Vec::new(),
11595 reserved: true,
11596 reserved_prefixes: Vec::new(),
11597 protocol: ModuleProtocol::Subc,
11598 overlap: Default::default(),
11599 });
11600 assert!(
11601 supervisor
11602 .reserved_hello_verification("never-spawned", Some("any-forged-nonce"))
11603 .is_err(),
11604 "forged nonce must refuse on a reserved never-spawned id"
11605 );
11606 assert!(
11607 supervisor
11608 .reserved_hello_verification("never-spawned", None)
11609 .is_err(),
11610 "absent nonce must refuse on a reserved never-spawned id"
11611 );
11612 supervisor.set_spawn_nonce("never-spawned", "minted".to_string());
11614 supervisor.apply_identity_configuration(&ModuleSpec {
11615 module_id: "never-spawned".to_string(),
11616 program: PathBuf::from("/usr/bin/false"),
11617 args: Vec::new(),
11618 env: Vec::new(),
11619 reserved: true,
11620 reserved_prefixes: Vec::new(),
11621 protocol: ModuleProtocol::Subc,
11622 overlap: Default::default(),
11623 });
11624 assert!(supervisor
11625 .reserved_hello_verification("never-spawned", Some("minted"))
11626 .is_ok());
11627 assert!(supervisor
11628 .reserved_hello_verification("never-spawned", Some("forged"))
11629 .is_err());
11630 }
11631
11632 fn seed_crash_restarts(state: &mut SupervisorSnapshot, count: u32) {
11635 let now = ClockInstant::now();
11636 for _ in 0..count {
11637 state.crash_restarts.push_back(now);
11638 }
11639 }
11640
11641 fn age_oldest_crash_restart_out_of_window(state: &mut SupervisorSnapshot, window: Duration) {
11645 let aged = state
11646 .crash_restarts
11647 .front()
11648 .expect("a crash restart must be recorded before it can be aged")
11649 .checked_sub(window + Duration::from_secs(1))
11650 .expect("the test clock is far enough from its origin to age an instant");
11651 state.crash_restarts[0] = aged;
11652 }
11653
11654 fn snapshot_with_restarts(enabled: bool, count: u32) -> SupervisorSnapshot {
11655 let mut state = SupervisorSnapshot::new(ModuleState::Running, enabled);
11656 seed_crash_restarts(&mut state, count);
11657 state
11658 }
11659
11660 #[test]
11661 fn daemon_owned_recovery_predicate_uses_the_pre_increment_budget() {
11662 let policy = RestartPolicy::new(3, Duration::ZERO);
11663 let now = ClockInstant::now();
11664 assert!(daemon_will_restart(
11665 &mut snapshot_with_restarts(true, 2),
11666 &policy,
11667 now
11668 ));
11669 assert!(!daemon_will_restart(
11670 &mut snapshot_with_restarts(true, 3),
11671 &policy,
11672 now
11673 ));
11674 assert!(!daemon_will_restart(
11675 &mut snapshot_with_restarts(false, 0),
11676 &policy,
11677 now
11678 ));
11679 }
11680
11681 #[test]
11682 fn crash_restart_backoff_escalates_with_in_window_count() {
11683 let policy = RestartPolicy::new(4, Duration::from_millis(100))
11684 .with_max_backoff(Duration::from_secs(30));
11685 let now = ClockInstant::now();
11686 let mut state = SupervisorSnapshot::new(ModuleState::Running, true);
11687 let schedules = (0..4)
11688 .map(|_| {
11689 state
11690 .next_crash_restart(&policy, now)
11691 .expect("the test policy allows four crash restarts")
11692 })
11693 .collect::<Vec<_>>();
11694
11695 assert_eq!(
11696 schedules
11697 .iter()
11698 .map(|schedule| schedule.restart_in_window)
11699 .collect::<Vec<_>>(),
11700 vec![0, 1, 2, 3]
11701 );
11702 assert_eq!(
11703 schedules
11704 .iter()
11705 .map(|schedule| schedule.delay)
11706 .collect::<Vec<_>>(),
11707 vec![
11708 Duration::from_millis(100),
11709 Duration::from_secs(1),
11710 Duration::from_secs(10),
11711 Duration::from_secs(30),
11712 ]
11713 );
11714 }
11715
11716 #[test]
11717 fn crash_restart_backoff_resets_after_ring_clear() {
11718 let policy = RestartPolicy::new(3, Duration::from_millis(100));
11719 let now = ClockInstant::now();
11720 let mut state = SupervisorSnapshot::new(ModuleState::Running, true);
11721 assert_eq!(
11722 state.next_crash_restart(&policy, now).unwrap().delay,
11723 Duration::from_millis(100)
11724 );
11725 assert_eq!(
11726 state.next_crash_restart(&policy, now).unwrap().delay,
11727 Duration::from_secs(1)
11728 );
11729
11730 state.clear_crash_restarts();
11731 let schedule = state
11732 .next_crash_restart(&policy, now)
11733 .expect("a cleared ring must allow another restart");
11734 assert_eq!(schedule.restart_in_window, 0);
11735 assert_eq!(schedule.delay, Duration::from_millis(100));
11736 }
11737
11738 #[test]
11739 fn crash_restart_backoff_ignores_aged_restarts() {
11740 let policy = RestartPolicy::new(3, Duration::from_millis(100));
11741 let now = ClockInstant::now();
11742 let mut state = SupervisorSnapshot::new(ModuleState::Running, true);
11743 state
11744 .next_crash_restart(&policy, now)
11745 .expect("the first restart is allowed");
11746 state
11747 .next_crash_restart(&policy, now)
11748 .expect("the second restart is allowed");
11749 state.crash_restarts[0] = now
11750 .checked_sub(policy.window + Duration::from_secs(1))
11751 .expect("the fake clock can age a restart past the window");
11752
11753 let schedule = state
11754 .next_crash_restart(&policy, now)
11755 .expect("an aged restart must release its slot");
11756 assert_eq!(schedule.restart_in_window, 1);
11757 assert_eq!(schedule.delay, Duration::from_secs(1));
11758 assert_eq!(state.crash_restarts.len(), 2);
11759 }
11760
11761 #[test]
11765 fn a_budget_spent_before_the_window_no_longer_refuses() {
11766 let policy = RestartPolicy::new(3, Duration::ZERO);
11767 let mut state = snapshot_with_restarts(true, 3);
11768 let now = ClockInstant::now();
11769 assert!(!daemon_will_restart(&mut state, &policy, now));
11770
11771 assert!(daemon_will_restart(
11772 &mut state,
11773 &policy,
11774 now + policy.window + Duration::from_secs(1)
11775 ));
11776 assert!(
11777 state.crash_restarts.is_empty(),
11778 "reading the budget must drop the instants that left the window"
11779 );
11780 }
11781
11782 fn module_with_recovery_snapshot(
11783 state: ModuleState,
11784 enabled: bool,
11785 restart_count: u32,
11786 ) -> SupervisedModule {
11787 let registry = Arc::new(Registry::default());
11788 let supervisor =
11789 Supervisor::new_for_test(Arc::clone(®istry), RestartPolicy::new(3, Duration::ZERO));
11790 let runtime = supervisor.runtime_config();
11791 let spec = ModuleSpec {
11792 module_id: "recovery-snapshot".to_string(),
11793 program: fake_aft_stub_path(),
11794 args: Vec::new(),
11795 env: Vec::new(),
11796 reserved: false,
11797 reserved_prefixes: Vec::new(),
11798 protocol: ModuleProtocol::Subc,
11799 overlap: Default::default(),
11800 };
11801 let mut snapshot = SupervisorSnapshot::new(state, enabled);
11802 seed_crash_restarts(&mut snapshot, restart_count);
11803 let (commands, _rx) = mpsc::channel(4);
11806 SupervisedModule {
11807 inner: Arc::new(SupervisedModuleInner {
11808 module_id: spec.module_id.clone(),
11809 registry,
11810 snapshot: Arc::new(Mutex::new(snapshot)),
11811 configuration: Arc::new(Mutex::new(SupervisedConfiguration {
11812 spec,
11813 health: runtime.health,
11814 })),
11815 stderr_ring: runtime.stderr_ring,
11816 terminal_ring: runtime.terminal_ring,
11817 commands,
11818 monitor: Mutex::new(None),
11819 restart_policy: runtime.restart_policy,
11820 effective_drain_timeout: runtime.effective_drain_timeout,
11821 provenance_probe: supervisor.provenance_probe.clone(),
11822 }),
11823 }
11824 }
11825
11826 #[cfg(target_os = "linux")]
11827 #[tokio::test]
11828 async fn no_cgroup_placement_does_not_block_fake_aft_stub_spawn() {
11829 let supervisor =
11830 Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default())
11831 .with_cgroup_placement(None);
11832 let result = supervisor.spawn(ModuleSpec {
11833 module_id: "no-cgroup-placement".to_string(),
11834 program: fake_aft_stub_path(),
11835 args: Vec::new(),
11836 env: Vec::new(),
11837 reserved: false,
11838 reserved_prefixes: Vec::new(),
11839 protocol: ModuleProtocol::Subc,
11840 overlap: Default::default(),
11841 });
11842
11843 assert!(
11844 result.is_ok(),
11845 "no delegation must not turn an otherwise valid spawn into a failure: {result:?}"
11846 );
11847 }
11848
11849 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
11850 async fn undecided_snapshot_uses_shared_restart_predicate() {
11851 assert!(module_with_recovery_snapshot(ModuleState::Running, true, 2)
11852 .will_recover_after_connection_loss()
11853 .unwrap());
11854 assert!(
11855 !module_with_recovery_snapshot(ModuleState::Running, true, 3)
11856 .will_recover_after_connection_loss()
11857 .unwrap()
11858 );
11859 }
11860
11861 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
11862 async fn restarting_snapshot_at_exhausted_budget_is_non_terminal() {
11863 assert!(
11864 module_with_recovery_snapshot(ModuleState::Restarting, true, 3)
11865 .will_recover_after_connection_loss()
11866 .unwrap()
11867 );
11868 }
11869
11870 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
11871 async fn terminal_phase_snapshots_are_terminal_before_budget_exhaustion() {
11872 assert!(!module_with_recovery_snapshot(ModuleState::Failed, true, 0)
11873 .will_recover_after_connection_loss()
11874 .unwrap());
11875 assert!(
11876 !module_with_recovery_snapshot(ModuleState::Disabled, true, 0)
11877 .will_recover_after_connection_loss()
11878 .unwrap()
11879 );
11880 }
11881
11882 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
11883 async fn warming_snapshot_is_limited_to_startup_phases() {
11884 for state in [
11885 ModuleState::Starting,
11886 ModuleState::Running,
11887 ModuleState::Restarting,
11888 ] {
11889 assert!(
11890 module_with_recovery_snapshot(state, true, 0)
11891 .is_warming()
11892 .unwrap(),
11893 "{state:?} should be warming"
11894 );
11895 }
11896 for state in [
11897 ModuleState::Unresponsive,
11898 ModuleState::Draining,
11899 ModuleState::Stopped,
11900 ModuleState::Failed,
11901 ModuleState::Disabled,
11902 ] {
11903 assert!(
11904 !module_with_recovery_snapshot(state, true, 0)
11905 .is_warming()
11906 .unwrap(),
11907 "{state:?} should not be warming"
11908 );
11909 }
11910 }
11911
11912 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
11913 async fn terminal_history_survives_respawn_and_keeps_both_crashes_in_order() {
11914 let registry = Arc::new(Registry::default());
11915 let supervisor =
11916 Supervisor::new_for_test(Arc::clone(®istry), RestartPolicy::new(1, Duration::ZERO));
11917 let module = supervisor
11918 .spawn(ModuleSpec {
11919 module_id: "terminal-history".to_string(),
11920 program: fake_aft_stub_path(),
11921 args: Vec::new(),
11922 env: vec![("FAKE_AFT_EXIT_CODE".to_string(), "23".to_string())],
11923 reserved: false,
11924 reserved_prefixes: Vec::new(),
11925 protocol: ModuleProtocol::Subc,
11926 overlap: Default::default(),
11927 })
11928 .unwrap();
11929
11930 let deadline = Instant::now() + Duration::from_secs(5);
11931 loop {
11932 let history = module.terminal_history();
11933 if history.entries.len() == 2 {
11934 assert_eq!(module.status().unwrap().state, ModuleState::Failed);
11935 assert_eq!(history.dropped, 0);
11936 assert_eq!(
11937 history
11938 .entries
11939 .iter()
11940 .map(|entry| entry.exit_code)
11941 .collect::<Vec<_>>(),
11942 vec![Some(23), Some(23)]
11943 );
11944 assert!(history.entries[0].at_ms <= history.entries[1].at_ms);
11945 return;
11946 }
11947 assert!(
11948 Instant::now() < deadline,
11949 "module did not retain two terminal exits: {history:?}"
11950 );
11951 sleep(Duration::from_millis(10)).await;
11952 }
11953 }
11954
11955 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
11959 async fn disable_during_crash_backoff_cancels_pending_respawn() {
11960 let backoff = Duration::from_secs(2);
11961 let supervisor = Supervisor::new_for_test(
11962 Arc::new(Registry::default()),
11963 RestartPolicy::new(10, backoff),
11964 );
11965 let module = supervisor
11966 .spawn(ModuleSpec {
11967 module_id: "disable-during-backoff".to_string(),
11968 program: fake_aft_stub_path(),
11969 args: Vec::new(),
11970 env: vec![("FAKE_AFT_EXIT_CODE".to_string(), "23".to_string())],
11971 reserved: false,
11972 reserved_prefixes: Vec::new(),
11973 protocol: ModuleProtocol::Subc,
11974 overlap: Default::default(),
11975 })
11976 .unwrap();
11977
11978 let deadline = Instant::now() + Duration::from_secs(5);
11980 loop {
11981 if module.status().unwrap().state == ModuleState::Restarting {
11982 break;
11983 }
11984 assert!(
11985 Instant::now() < deadline,
11986 "module never entered the crash backoff"
11987 );
11988 sleep(Duration::from_millis(10)).await;
11989 }
11990
11991 let started = Instant::now();
11992 module.set_enabled(false).await.unwrap();
11993 let waited = started.elapsed();
11994
11995 assert!(
11996 waited < backoff / 2,
11997 "disable waited {waited:?} behind the {backoff:?} crash backoff; the operator command must preempt the pending respawn"
11998 );
11999 assert_eq!(module.status().unwrap().state, ModuleState::Disabled);
12000
12001 sleep(backoff + Duration::from_millis(500)).await;
12003 let status = module.status().unwrap();
12004 assert_eq!(status.state, ModuleState::Disabled);
12005 assert_eq!(
12006 status.spawn_generation, 1,
12007 "module respawned after the operator disabled it"
12008 );
12009 }
12010
12011 #[cfg(unix)]
12014 fn protocol_none_sigterm_exits_clean_spec(
12015 module_id: &str,
12016 dir: &std::path::Path,
12017 ) -> (ModuleSpec, PathBuf, PathBuf) {
12018 let ready = dir.join("ready");
12019 let marker = dir.join("sigterm");
12020 let spec = ModuleSpec {
12021 module_id: module_id.to_string(),
12022 program: fake_aft_stub_path(),
12023 args: Vec::new(),
12024 env: vec![
12025 ("FAKE_AFT_NEVER_CONNECT".to_string(), "1".to_string()),
12026 (
12027 "FAKE_AFT_SIGTERM_MARKER_PATH".to_string(),
12028 marker.display().to_string(),
12029 ),
12030 (
12031 "FAKE_AFT_NEVER_CONNECT_READY_PATH".to_string(),
12032 ready.display().to_string(),
12033 ),
12034 ],
12035 reserved: false,
12036 reserved_prefixes: Vec::new(),
12037 protocol: ModuleProtocol::None,
12038 overlap: Default::default(),
12039 };
12040 (spec, ready, marker)
12041 }
12042
12043 #[cfg(unix)]
12047 async fn wait_for_file(path: &std::path::Path) {
12048 let deadline = Instant::now() + Duration::from_secs(10);
12049 while !path.exists() {
12050 assert!(
12051 Instant::now() < deadline,
12052 "{} never appeared",
12053 path.display()
12054 );
12055 sleep(Duration::from_millis(10)).await;
12056 }
12057 }
12058
12059 #[cfg(unix)]
12063 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
12064 async fn protocol_none_unrequested_clean_exit_restarts_as_a_crash() {
12065 let dir = cortexkit_test_support::ScratchDir::new("none-unrequested-clean-exit");
12066 let (spec, ready, marker) =
12067 protocol_none_sigterm_exits_clean_spec("none-unrequested-clean-exit", dir.path());
12068 let supervisor = Supervisor::new_for_test(
12069 Arc::new(Registry::default()),
12070 RestartPolicy::new(3, Duration::ZERO),
12071 );
12072 let module = supervisor.spawn(spec).unwrap();
12073 wait_for_file(&ready).await;
12074 let deadline = Instant::now() + Duration::from_secs(10);
12078 let first_pid = loop {
12079 let status = module.status().unwrap();
12080 if status.state == ModuleState::Running {
12081 if let Some(pid) = status.pid {
12082 break pid;
12083 }
12084 }
12085 assert!(
12086 Instant::now() < deadline,
12087 "a running module must report its pid after exec confirmation: {status:?}"
12088 );
12089 sleep(Duration::from_millis(10)).await;
12090 };
12091
12092 rustix::process::kill_process(
12093 rustix::process::Pid::from_raw(i32::try_from(first_pid).unwrap()).unwrap(),
12094 rustix::process::Signal::TERM,
12095 )
12096 .unwrap();
12097
12098 let deadline = Instant::now() + Duration::from_secs(10);
12099 let respawned = loop {
12100 let status = module.status().unwrap();
12101 if status.state == ModuleState::Running
12102 && status.pid.is_some_and(|pid| pid != first_pid)
12103 {
12104 break status;
12105 }
12106 assert!(
12107 Instant::now() < deadline,
12108 "protocol-none module was not respawned after an unrequested clean exit: {status:?}"
12109 );
12110 sleep(Duration::from_millis(10)).await;
12111 };
12112 assert_eq!(respawned.spawn_generation, 2);
12113 assert!(
12114 marker.exists(),
12115 "the child must have exited through its SIGTERM handler (exit 0), or this proves nothing about clean exits"
12116 );
12117
12118 let history = module.terminal_history();
12119 assert_eq!(history.entries.len(), 1, "{history:?}");
12120 let entry = &history.entries[0];
12121 assert_eq!(entry.exit_code, Some(0));
12122 assert_eq!(entry.exit_kind, subc_control::TerminalExitKind::Clean);
12123 assert_eq!(entry.disposition, TerminalDisposition::Restarting);
12124
12125 module.stop().await.unwrap();
12126 }
12127
12128 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
12132 async fn protocol_none_repeated_clean_exits_exhaust_the_restart_budget() {
12133 let supervisor = Supervisor::new_for_test(
12134 Arc::new(Registry::default()),
12135 RestartPolicy::new(1, Duration::ZERO),
12136 );
12137 let module = supervisor
12138 .spawn(ModuleSpec {
12139 module_id: "none-clean-exit-budget".to_string(),
12140 program: fake_aft_stub_path(),
12141 args: Vec::new(),
12142 env: vec![("FAKE_AFT_EXIT_CODE".to_string(), "0".to_string())],
12143 reserved: false,
12144 reserved_prefixes: Vec::new(),
12145 protocol: ModuleProtocol::None,
12146 overlap: Default::default(),
12147 })
12148 .unwrap();
12149
12150 let deadline = Instant::now() + Duration::from_secs(10);
12153 loop {
12154 let status = module.status().unwrap();
12155 if status.state == ModuleState::Failed {
12156 break;
12157 }
12158 assert!(
12159 Instant::now() < deadline,
12160 "module never exhausted its budget: {status:?} {:?}",
12161 module.terminal_history()
12162 );
12163 sleep(Duration::from_millis(10)).await;
12164 }
12165 let history = module.terminal_history();
12166 assert_eq!(
12167 history
12168 .entries
12169 .iter()
12170 .map(|entry| (entry.exit_code, entry.disposition.clone()))
12171 .collect::<Vec<_>>(),
12172 vec![
12173 (Some(0), TerminalDisposition::Restarting),
12174 (Some(0), TerminalDisposition::Failed),
12175 ]
12176 );
12177 let detail = history.entries[1]
12178 .disposition_detail
12179 .as_deref()
12180 .expect("a budget failure names the budget");
12181 assert!(detail.contains("max_restarts=1"), "{detail}");
12182 assert_eq!(module.status().unwrap().spawn_generation, 2);
12183 }
12184
12185 #[test]
12186 fn restart_budget_failure_is_published_after_its_terminal_record() {
12187 use std::io::{Read, Write};
12188 let supervisor = Supervisor::new_for_test(
12189 Arc::new(Registry::default()),
12190 RestartPolicy::new(0, Duration::ZERO),
12191 );
12192 let runtime = supervisor.runtime_config();
12193 let spec = ModuleSpec {
12194 protocol: ModuleProtocol::None,
12195 ..windowed_crash_spec("budget-publication-order")
12196 };
12197 let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::new(
12198 ModuleState::Running,
12199 true,
12200 )));
12201 lock_snapshot(&snapshot).unwrap().process_alive = true;
12202 let reaped_snapshot = snapshot.clone();
12203 let mut barrier = terminal_record_barrier(&runtime.spawn_events);
12204 let history = runtime.terminal_ring.clone();
12205 let reap = std::thread::spawn(move || {
12206 tokio::runtime::Builder::new_current_thread()
12207 .enable_all()
12208 .build()
12209 .unwrap()
12210 .block_on(on_child_exit(
12211 &spec,
12212 runtime.restart_policy,
12213 &supervisor.registry,
12214 &reaped_snapshot,
12215 &runtime.terminal_ring,
12216 &runtime.spawn_events,
12217 &runtime.child_roster,
12218 ExitReport {
12219 kind: ExitKind::Clean,
12220 code: Some(0),
12221 signal: None,
12222 at_ms: 1,
12223 },
12224 ))
12225 });
12226 let mut reached = [0];
12227 barrier.read_exact(&mut reached).unwrap();
12228 assert_eq!(&reached, b"R");
12229 let (before_state, before_alive) = {
12230 let state = lock_snapshot(&snapshot).unwrap();
12231 (state.state, state.process_alive)
12232 };
12233 let before_history = history.lock().unwrap().snapshot();
12234 barrier.write_all(b"X").unwrap();
12235 assert!(matches!(reap.join().unwrap(), NextAction::Stop { .. }));
12236 assert!(before_history.entries.is_empty());
12237 assert_ne!(
12238 before_state,
12239 ModuleState::Failed,
12240 "Failed was visible before its terminal record could be written"
12241 );
12242 assert!(before_alive);
12243 assert_eq!(lock_snapshot(&snapshot).unwrap().state, ModuleState::Failed);
12244 let history = history.lock().unwrap().snapshot();
12245 assert_eq!(history.entries.len(), 1);
12246 assert_eq!(history.entries[0].exit_code, Some(0));
12247 assert_eq!(history.entries[0].disposition, TerminalDisposition::Failed);
12248 }
12249
12250 #[test]
12251 fn crash_restart_at_budget_limit_remains_recoverable_while_recording() {
12252 use std::io::{Read, Write};
12253
12254 let supervisor = Supervisor::new_for_test(
12255 Arc::new(Registry::default()),
12256 RestartPolicy::new(3, Duration::ZERO),
12257 );
12258 let mut runtime = supervisor.runtime_config();
12259 let module = module_with_recovery_snapshot(ModuleState::Running, true, 2);
12260 let snapshot = module.inner.snapshot.clone();
12261 lock_snapshot(&snapshot).unwrap().process_alive = true;
12262 runtime.terminal_ring = module.inner.terminal_ring.clone();
12263 let spec = windowed_crash_spec(module.module_id());
12264 let mut barrier = terminal_record_barrier(&runtime.spawn_events);
12265 let reaped_snapshot = snapshot.clone();
12266 let reap = std::thread::spawn(move || {
12267 tokio::runtime::Builder::new_current_thread()
12268 .enable_all()
12269 .build()
12270 .unwrap()
12271 .block_on(on_child_exit(
12272 &spec,
12273 runtime.restart_policy,
12274 &supervisor.registry,
12275 &reaped_snapshot,
12276 &runtime.terminal_ring,
12277 &runtime.spawn_events,
12278 &runtime.child_roster,
12279 ExitReport {
12280 kind: ExitKind::Crash,
12281 code: Some(2),
12282 signal: None,
12283 at_ms: 1,
12284 },
12285 ))
12286 });
12287
12288 let mut reached = [0];
12292 barrier.read_exact(&mut reached).unwrap();
12293 assert_eq!(&reached, b"R");
12294 let before_status = module.status().unwrap();
12295 let before_history = module.terminal_history();
12296 let before_recovers = module.will_recover_after_connection_loss().unwrap();
12297 barrier.write_all(b"X").unwrap();
12298 assert!(matches!(reap.join().unwrap(), NextAction::Restart { .. }));
12299
12300 assert_eq!(before_status.state, ModuleState::Running);
12301 assert!(before_status.process_alive);
12302 assert_eq!(before_status.restart_count, 3);
12303 assert!(before_history.entries.is_empty());
12304 assert!(
12305 before_recovers,
12306 "connection cleanup would report a terminal close for a scheduled crash restart"
12307 );
12308 let restarting = module.status().unwrap();
12309 assert_eq!(restarting.state, ModuleState::Restarting);
12310 assert_eq!(restarting.restart_count, 3);
12311 assert!(module.will_recover_after_connection_loss().unwrap());
12312 let history = module.terminal_history();
12313 assert_eq!(history.entries.len(), 1);
12314 assert_eq!(
12315 history.entries[0].disposition,
12316 TerminalDisposition::Restarting
12317 );
12318
12319 update_snapshot(&snapshot, Some(module.module_id()), |state| {
12322 state.state = ModuleState::Running;
12323 state.respawn_pending = false;
12324 })
12325 .unwrap();
12326 assert!(!module.will_recover_after_connection_loss().unwrap());
12327 update_snapshot(&snapshot, Some(module.module_id()), |state| {
12328 state.enabled = false;
12329 state.respawn_pending = true;
12330 })
12331 .unwrap();
12332 assert!(!module.will_recover_after_connection_loss().unwrap());
12333 }
12334
12335 fn terminal_record_barrier(events: &super::SpawnEventFeed) -> std::net::TcpStream {
12336 use std::net::{TcpListener, TcpStream};
12337 let listener = TcpListener::bind("127.0.0.1:0").unwrap();
12338 let writer = TcpStream::connect(listener.local_addr().unwrap()).unwrap();
12339 let (barrier, _) = listener.accept().unwrap();
12340 for stream in [&writer, &barrier] {
12341 stream
12342 .set_read_timeout(Some(Duration::from_secs(10)))
12343 .unwrap();
12344 stream
12345 .set_write_timeout(Some(Duration::from_secs(10)))
12346 .unwrap();
12347 }
12348 *events.1.lock().unwrap() = Some(writer);
12349 barrier
12350 }
12351
12352 pub(super) fn before_terminal_record(events: &super::SpawnEventFeed) {
12353 use std::io::{Read, Write};
12354 let barrier = events.1.lock().unwrap().take();
12355 if let Some(mut barrier) = barrier {
12356 barrier.write_all(b"R").unwrap();
12357 let mut release = [0];
12358 barrier.read_exact(&mut release).unwrap();
12359 assert_eq!(&release, b"X");
12360 }
12361 }
12362
12363 #[test]
12364 fn health_restart_budget_failure_is_published_after_its_terminal_record() {
12365 use std::io::{Read, Write};
12366
12367 let supervisor = Supervisor::new_for_test(
12368 Arc::new(Registry::default()),
12369 RestartPolicy::new(0, Duration::ZERO),
12370 );
12371 let mut runtime = supervisor.runtime_config();
12372 let module = module_with_recovery_snapshot(ModuleState::Running, true, 0);
12373 let snapshot = module.inner.snapshot.clone();
12374 runtime.terminal_ring = module.inner.terminal_ring.clone();
12375 let spec = ModuleSpec {
12376 module_id: module.module_id().into(),
12377 program: fake_aft_stub_path(),
12378 protocol: ModuleProtocol::None,
12379 env: vec![("FAKE_AFT_EXIT_CODE".into(), "17".into())],
12380 ..windowed_crash_spec(module.module_id())
12381 };
12382 let mut barrier = terminal_record_barrier(&runtime.spawn_events);
12383 let reap = std::thread::spawn(move || {
12384 tokio::runtime::Builder::new_current_thread()
12385 .enable_all()
12386 .build()
12387 .unwrap()
12388 .block_on(async {
12389 let mut child =
12390 Some(spawn_and_mark_running(&spec, &runtime, &snapshot).unwrap());
12391 health_restart_child(
12392 &spec,
12393 &runtime,
12394 &supervisor.registry,
12395 &supervisor.process_liveness,
12396 &snapshot,
12397 &mut child,
12398 SupervisorHealthStatus::Failing,
12399 Some("health probe did not answer"),
12400 1,
12401 )
12402 .await
12403 .unwrap();
12404 assert!(child.is_none());
12405 });
12406 });
12407 let mut reached = [0];
12412 barrier.read_exact(&mut reached).unwrap();
12413 assert_eq!(&reached, b"R");
12414 let before_status = module.status().unwrap();
12415 let before_history = module.terminal_history();
12416 barrier.write_all(b"X").unwrap();
12417 reap.join().unwrap();
12418 assert!(before_history.entries.is_empty());
12419 assert_ne!(
12420 before_status.state,
12421 ModuleState::Failed,
12422 "Failed was visible before its terminal record could be written"
12423 );
12424 assert!(
12425 before_status.process_alive,
12426 "process death was visible before its terminal record could be written"
12427 );
12428 let failed = module.status().unwrap();
12429 assert_eq!(failed.state, ModuleState::Failed);
12430 assert!(!failed.process_alive);
12431 let history = module.terminal_history();
12432 assert_eq!(history.entries.len(), 1);
12433 assert_eq!(history.entries[0].disposition, TerminalDisposition::Failed);
12434 assert_eq!(
12435 history.entries[0].disposition_detail.as_deref(),
12436 Some("crash budget exhausted: max_restarts=0 within window_secs=600")
12437 );
12438 }
12439
12440 #[test]
12441 fn observed_exits_publish_process_death_after_terminal_history() {
12442 use std::io::{Read, Write};
12443 for (kind, enabled, shutdown, wait_error, expected_state) in [
12444 (ExitKind::Clean, true, false, false, ModuleState::Stopped),
12445 (ExitKind::Crash, true, false, false, ModuleState::Restarting),
12446 (ExitKind::Crash, false, false, false, ModuleState::Disabled),
12447 (
12448 ExitKind::DeliberateSeverance,
12449 true,
12450 false,
12451 false,
12452 ModuleState::Restarting,
12453 ),
12454 #[cfg(unix)]
12455 (ExitKind::Crash, true, true, false, ModuleState::Stopped),
12456 (ExitKind::Crash, true, false, true, ModuleState::Failed),
12457 ] {
12458 let supervisor = Supervisor::new_for_test(
12459 Arc::new(Registry::default()),
12460 RestartPolicy::new(1, Duration::ZERO),
12461 );
12462 let runtime = supervisor.runtime_config();
12463 #[cfg(unix)]
12464 if shutdown {
12465 runtime.child_roster.close();
12466 }
12467 #[cfg(not(unix))]
12468 let _ = shutdown;
12469 let spec = windowed_crash_spec("exit-publication-order");
12470 let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::new(
12471 ModuleState::Running,
12472 enabled,
12473 )));
12474 lock_snapshot(&snapshot).unwrap().process_alive = true;
12475 let reaped_snapshot = snapshot.clone();
12476 let history = runtime.terminal_ring.clone();
12477 let mut barrier = terminal_record_barrier(&runtime.spawn_events);
12478 let reap = std::thread::spawn(move || {
12479 if wait_error {
12480 super::fail_after_wait_error(
12481 &spec.module_id,
12482 &reaped_snapshot,
12483 &runtime.terminal_ring,
12484 &runtime.spawn_events,
12485 "wait failed".into(),
12486 );
12487 } else {
12488 tokio::runtime::Builder::new_current_thread()
12489 .enable_all()
12490 .build()
12491 .unwrap()
12492 .block_on(on_child_exit(
12493 &spec,
12494 runtime.restart_policy,
12495 &supervisor.registry,
12496 &reaped_snapshot,
12497 &runtime.terminal_ring,
12498 &runtime.spawn_events,
12499 &runtime.child_roster,
12500 ExitReport {
12501 kind,
12502 code: Some(0),
12503 signal: None,
12504 at_ms: 1,
12505 },
12506 ));
12507 }
12508 });
12509 let mut reached = [0];
12510 barrier.read_exact(&mut reached).unwrap();
12511 assert_eq!(&reached, b"R");
12512 let before_alive = lock_snapshot(&snapshot).unwrap().process_alive;
12513 let before_history = history.lock().unwrap().snapshot();
12514 barrier.write_all(b"X").unwrap();
12515 reap.join().unwrap();
12516 assert!(
12517 before_alive,
12518 "{kind:?}: death was published before recording"
12519 );
12520 assert!(before_history.entries.is_empty());
12521 assert_eq!(lock_snapshot(&snapshot).unwrap().state, expected_state);
12522 assert!(!lock_snapshot(&snapshot).unwrap().process_alive);
12523 assert_eq!(history.lock().unwrap().snapshot().entries.len(), 1);
12524 }
12525 }
12526
12527 #[tokio::test]
12528 async fn health_restart_budget_failure_remains_failed_when_journal_append_fails() {
12529 let dir = cortexkit_test_support::ScratchDir::new("health-budget-journal-failure");
12530 let path = dir.join("terminals.jsonl");
12531 std::fs::create_dir(&path).unwrap();
12532 let supervisor = Supervisor::new_for_test(
12533 Arc::new(Registry::default()),
12534 RestartPolicy::new(0, Duration::ZERO),
12535 )
12536 .with_terminal_journal(path, "health-budget-journal-failure".into());
12537 let runtime = supervisor.runtime_config();
12538 let spec = ModuleSpec {
12539 program: fake_aft_stub_path(),
12540 protocol: ModuleProtocol::None,
12541 env: vec![("FAKE_AFT_EXIT_CODE".into(), "17".into())],
12542 ..windowed_crash_spec("health-budget-journal-failure")
12543 };
12544 let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::starting()));
12545 let mut child = Some(spawn_and_mark_running(&spec, &runtime, &snapshot).unwrap());
12546 health_restart_child(
12547 &spec,
12548 &runtime,
12549 &supervisor.registry,
12550 &supervisor.process_liveness,
12551 &snapshot,
12552 &mut child,
12553 SupervisorHealthStatus::Failing,
12554 None,
12555 1,
12556 )
12557 .await
12558 .unwrap();
12559 assert_eq!(lock_snapshot(&snapshot).unwrap().state, ModuleState::Failed);
12560 assert!(!lock_snapshot(&snapshot).unwrap().process_alive);
12561 let history = runtime
12562 .terminal_ring
12563 .lock()
12564 .unwrap()
12565 .durable_history(&spec.module_id);
12566 assert!(history.journal_write_failures > 0);
12567 assert_eq!(history.entries.len(), 1);
12568 assert_eq!(history.entries[0].disposition, TerminalDisposition::Failed);
12569 }
12570
12571 #[test]
12572 fn health_restart_budget_failure_remains_failed_when_exit_recording_panics() {
12573 let supervisor = Supervisor::new_for_test(
12574 Arc::new(Registry::default()),
12575 RestartPolicy::new(0, Duration::ZERO),
12576 );
12577 let runtime = supervisor.runtime_config();
12578 let spec = ModuleSpec {
12579 program: fake_aft_stub_path(),
12580 protocol: ModuleProtocol::None,
12581 env: vec![("FAKE_AFT_EXIT_CODE".into(), "17".into())],
12582 ..windowed_crash_spec("health-budget-recording-panic")
12583 };
12584 let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::starting()));
12585 let result = std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| {
12586 tokio::runtime::Builder::new_current_thread()
12587 .enable_all()
12588 .build()
12589 .unwrap()
12590 .block_on(async {
12591 let mut child =
12592 Some(spawn_and_mark_running(&spec, &runtime, &snapshot).unwrap());
12593 runtime.spawn_events.0.lock().unwrap().seq = u64::MAX;
12594 health_restart_child(
12595 &spec,
12596 &runtime,
12597 &supervisor.registry,
12598 &supervisor.process_liveness,
12599 &snapshot,
12600 &mut child,
12601 SupervisorHealthStatus::Failing,
12602 None,
12603 1,
12604 )
12605 .await
12606 .unwrap();
12607 });
12608 }));
12609 let panic = result.expect_err("recording must still unwind");
12610 assert_eq!(
12611 panic.downcast_ref::<String>().map(String::as_str),
12612 Some("spawn event sequence exhausted")
12613 );
12614 let state = lock_snapshot(&snapshot).unwrap();
12615 assert_eq!(state.state, ModuleState::Failed);
12616 assert!(!state.process_alive);
12617 assert!(state.last_exit.is_some());
12618 }
12619
12620 #[tokio::test]
12621 async fn restart_budget_failure_remains_failed_when_journal_append_fails() {
12622 let dir = cortexkit_test_support::ScratchDir::new("budget-journal-failure");
12623 let path = dir.join("terminals.jsonl");
12624 std::fs::create_dir(&path).unwrap();
12625 let supervisor = Supervisor::new_for_test(
12626 Arc::new(Registry::default()),
12627 RestartPolicy::new(0, Duration::ZERO),
12628 )
12629 .with_terminal_journal(path, "budget-journal-failure".into());
12630 let runtime = supervisor.runtime_config();
12631 let spec = windowed_crash_spec("budget-journal-failure");
12632 let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::starting()));
12633 assert!(matches!(
12634 on_child_exit(
12635 &spec,
12636 runtime.restart_policy,
12637 &supervisor.registry,
12638 &snapshot,
12639 &runtime.terminal_ring,
12640 &runtime.spawn_events,
12641 &runtime.child_roster,
12642 crash_exit_report(1),
12643 )
12644 .await,
12645 NextAction::Stop { .. }
12646 ));
12647 assert_eq!(lock_snapshot(&snapshot).unwrap().state, ModuleState::Failed);
12648 let history = runtime
12649 .terminal_ring
12650 .lock()
12651 .unwrap()
12652 .durable_history(&spec.module_id);
12653 assert!(history.journal_write_failures > 0);
12654 assert_eq!(history.entries.len(), 1);
12655 assert_eq!(history.entries[0].disposition, TerminalDisposition::Failed);
12656 }
12657
12658 #[test]
12659 fn restart_budget_failure_remains_failed_when_exit_recording_panics() {
12660 let supervisor = Supervisor::new_for_test(
12661 Arc::new(Registry::default()),
12662 RestartPolicy::new(0, Duration::ZERO),
12663 );
12664 let runtime = supervisor.runtime_config();
12665 let spec = windowed_crash_spec("budget-recording-panic");
12666 let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::starting()));
12667 runtime.spawn_events.emit_spawned(&spec.module_id, 1, 1);
12668 runtime.spawn_events.0.lock().unwrap().seq = u64::MAX;
12671 let result = std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| {
12672 tokio::runtime::Builder::new_current_thread()
12673 .enable_all()
12674 .build()
12675 .unwrap()
12676 .block_on(on_child_exit(
12677 &spec,
12678 runtime.restart_policy,
12679 &supervisor.registry,
12680 &snapshot,
12681 &runtime.terminal_ring,
12682 &runtime.spawn_events,
12683 &runtime.child_roster,
12684 crash_exit_report(1),
12685 ))
12686 }));
12687 let panic = result.err().expect("recording must still unwind");
12688 assert_eq!(
12689 panic.downcast_ref::<String>().map(String::as_str),
12690 Some("spawn event sequence exhausted")
12691 );
12692 assert_eq!(lock_snapshot(&snapshot).unwrap().state, ModuleState::Failed);
12693 }
12694
12695 #[cfg(unix)]
12698 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
12699 async fn protocol_none_requested_stop_and_disable_do_not_respawn() {
12700 for disable in [false, true] {
12701 let label = if disable {
12702 "none-requested-disable"
12703 } else {
12704 "none-requested-stop"
12705 };
12706 let dir = cortexkit_test_support::ScratchDir::new(label);
12707 let (spec, ready, marker) = protocol_none_sigterm_exits_clean_spec(label, dir.path());
12708 let supervisor = Supervisor::new_for_test(
12709 Arc::new(Registry::default()),
12710 RestartPolicy::new(3, Duration::ZERO),
12711 );
12712 let module = supervisor.spawn(spec).unwrap();
12713 wait_for_file(&ready).await;
12714
12715 if disable {
12716 module.set_enabled(false).await.unwrap();
12717 } else {
12718 module.stop().await.unwrap();
12719 }
12720 assert!(
12721 marker.exists(),
12722 "{label}: the child must have left through its SIGTERM handler with exit 0"
12723 );
12724
12725 sleep(Duration::from_millis(500)).await;
12728 let status = module.status().unwrap();
12729 let expected = if disable {
12730 ModuleState::Disabled
12731 } else {
12732 ModuleState::Stopped
12733 };
12734 assert_eq!(status.state, expected, "{label}");
12735 assert_eq!(
12736 status.spawn_generation, 1,
12737 "{label}: respawned after a requested stop"
12738 );
12739 let history = module.terminal_history();
12740 assert_eq!(history.entries.len(), 1, "{label}: {history:?}");
12741 assert_eq!(history.entries[0].exit_code, Some(0), "{label}");
12742 assert_ne!(
12743 history.entries[0].disposition,
12744 TerminalDisposition::Restarting,
12745 "{label}"
12746 );
12747 }
12748 }
12749
12750 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
12753 async fn subc_wire_clean_exit_is_still_a_stop() {
12754 let supervisor = Supervisor::new_for_test(
12755 Arc::new(Registry::default()),
12756 RestartPolicy::new(3, Duration::ZERO),
12757 );
12758 let module = supervisor
12759 .spawn(ModuleSpec {
12760 module_id: "wire-clean-exit".to_string(),
12761 program: fake_aft_stub_path(),
12762 args: Vec::new(),
12763 env: vec![("FAKE_AFT_EXIT_CODE".to_string(), "0".to_string())],
12764 reserved: false,
12765 reserved_prefixes: Vec::new(),
12766 protocol: ModuleProtocol::Subc,
12767 overlap: Default::default(),
12768 })
12769 .unwrap();
12770
12771 let deadline = Instant::now() + Duration::from_secs(10);
12772 while module.terminal_history().entries.is_empty() {
12773 assert!(Instant::now() < deadline, "module never exited");
12774 sleep(Duration::from_millis(10)).await;
12775 }
12776 sleep(Duration::from_millis(500)).await;
12778 let status = module.status().unwrap();
12779 assert_eq!(status.state, ModuleState::Stopped);
12780 assert_eq!(status.spawn_generation, 1);
12781 let history = module.terminal_history();
12782 assert_eq!(history.entries.len(), 1, "{history:?}");
12783 assert_eq!(history.entries[0].exit_code, Some(0));
12784 assert_eq!(history.entries[0].disposition, TerminalDisposition::Stopped);
12785 }
12786
12787 #[cfg(unix)]
12788 #[tokio::test]
12789 async fn reload_exit_keeps_roster_until_terminal_is_recorded_during_shutdown() {
12790 let dir = cortexkit_test_support::ScratchDir::new("reload-roster-terminal-order");
12791 let record = dir.join("live-children.json");
12792 let supervisor = Supervisor::new_for_test(
12793 Arc::new(Registry::default()),
12794 RestartPolicy::new(0, Duration::ZERO),
12795 );
12796 let mut runtime = supervisor.runtime_config();
12797 runtime.child_roster.record_to(record.clone());
12798 let gate = Arc::new(super::ReloadExitRecordGate::default());
12799 runtime.test_reload_exit_record_gate = Some(Arc::clone(&gate));
12800 let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::starting()));
12801 let spec = ModuleSpec {
12802 module_id: "reload-exit-roster".into(),
12803 program: fake_aft_stub_path(),
12804 args: Vec::new(),
12805 env: vec![("FAKE_AFT_EXIT_CODE".into(), "23".into())],
12806 reserved: false,
12807 reserved_prefixes: Vec::new(),
12808 protocol: ModuleProtocol::Subc,
12809 overlap: Default::default(),
12810 };
12811 let mut child = None;
12812 let reload = super::finish_reload_child(
12813 &spec,
12814 &runtime,
12815 &supervisor.registry,
12816 &supervisor.process_liveness,
12817 &snapshot,
12818 &mut child,
12819 );
12820 tokio::pin!(reload);
12821 tokio::select! {
12822 result = &mut reload => panic!("reload missed the before-record gate: {result:?}"),
12823 _ = gate.reached.notified() => {}
12824 }
12825 assert!(runtime
12826 .terminal_ring
12827 .lock()
12828 .unwrap()
12829 .snapshot()
12830 .entries
12831 .is_empty());
12832 assert_eq!(
12833 crate::live_children::read_record(&record).unwrap().len(),
12834 1,
12835 "shutdown must still wait for the reaped child until its terminal record exists"
12836 );
12837 runtime.child_roster.close();
12838 gate.resume.notify_one();
12839 assert!(reload.await.is_err());
12840 assert!(crate::live_children::read_record(&record)
12841 .unwrap()
12842 .is_empty());
12843 let history = runtime.terminal_ring.lock().unwrap().snapshot();
12844 assert_eq!(history.entries.len(), 1);
12845 assert_eq!(
12846 history.entries[0].disposition,
12847 TerminalDisposition::DaemonShutdown
12848 );
12849 }
12850
12851 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
12855 async fn every_restart_increment_path_advances_lifetime_count() {
12856 let supervisor = Supervisor::new_for_test(
12857 Arc::new(Registry::default()),
12858 RestartPolicy::new(1, Duration::ZERO),
12859 );
12860 let runtime = supervisor.runtime_config();
12861 let spec = ModuleSpec {
12862 module_id: "lifetime-increment-path".to_string(),
12863 program: PathBuf::from("/unused/lifetime-increment-path"),
12864 args: Vec::new(),
12865 env: Vec::new(),
12866 reserved: false,
12867 reserved_prefixes: Vec::new(),
12868 protocol: ModuleProtocol::Subc,
12869 overlap: Default::default(),
12870 };
12871
12872 let crash_snapshot = Arc::new(Mutex::new(SupervisorSnapshot::starting()));
12873 assert!(matches!(
12874 on_child_exit(
12875 &spec,
12876 runtime.restart_policy,
12877 &supervisor.registry,
12878 &crash_snapshot,
12879 &runtime.terminal_ring,
12880 &runtime.spawn_events,
12881 &runtime.child_roster,
12882 ExitReport {
12883 kind: ExitKind::Crash,
12884 code: Some(1),
12885 signal: None,
12886 at_ms: 1,
12887 },
12888 )
12889 .await,
12890 NextAction::Restart { schedule: _ }
12891 ));
12892 let (crash_restarts, crash_lifetime) = {
12893 let state = lock_snapshot(&crash_snapshot).unwrap();
12894 (state.crash_restarts.len(), state.lifetime_restarts)
12895 };
12896 assert_eq!(crash_restarts, 1);
12897 assert_eq!(crash_lifetime, 1);
12898
12899 let health_snapshot = Arc::new(Mutex::new(SupervisorSnapshot::starting()));
12900 let mut health_child = None;
12901 assert!(matches!(
12902 health_restart_child(
12903 &spec,
12904 &runtime,
12905 &supervisor.registry,
12906 &supervisor.process_liveness,
12907 &health_snapshot,
12908 &mut health_child,
12909 SupervisorHealthStatus::Failing,
12910 None,
12911 2,
12912 )
12913 .await,
12914 Ok(())
12915 ));
12916 assert!(health_child.is_none());
12917 assert!(lock_snapshot(&health_snapshot).unwrap().respawn_pending);
12918 assert!(runtime.scheduled_respawn.lock().unwrap().take().is_some());
12919 let (health_restarts, health_lifetime) = {
12920 let state = lock_snapshot(&health_snapshot).unwrap();
12921 (state.crash_restarts.len(), state.lifetime_restarts)
12922 };
12923 assert_eq!(health_restarts, 1);
12924 assert_eq!(health_lifetime, 1);
12925
12926 let reload_snapshot = Arc::new(Mutex::new(SupervisorSnapshot::starting()));
12927 let mut reload_child = None;
12928 assert!(matches!(
12929 handle_reload_spawn_failure(
12930 &spec,
12931 &runtime,
12932 &supervisor.process_liveness,
12933 &reload_snapshot,
12934 &mut reload_child,
12935 "forced reload spawn failure".to_string(),
12936 )
12937 .await,
12938 Err(SuperviseError::ReloadFailed { .. })
12939 ));
12940 let (reload_restarts, reload_lifetime) = {
12941 let state = lock_snapshot(&reload_snapshot).unwrap();
12942 (state.crash_restarts.len(), state.lifetime_restarts)
12943 };
12944 assert_eq!(reload_restarts, 1);
12945 assert_eq!(reload_lifetime, 1);
12946 }
12947
12948 #[tokio::test]
12949 async fn deliberately_severed_live_child_records_lifetime_without_spending_restart_budget() {
12950 let supervisor = Supervisor::new_for_test(
12951 Arc::new(Registry::default()),
12952 RestartPolicy::new(3, Duration::ZERO),
12953 );
12954 let runtime = supervisor.runtime_config();
12955 let spec = ModuleSpec {
12956 module_id: "deliberately-severed".to_string(),
12957 program: PathBuf::from("/unused/deliberately-severed"),
12958 args: Vec::new(),
12959 env: Vec::new(),
12960 reserved: false,
12961 reserved_prefixes: Vec::new(),
12962 protocol: ModuleProtocol::Subc,
12963 overlap: Default::default(),
12964 };
12965 let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::starting()));
12966 let process = ProcessIdentity {
12967 pid: 41,
12968 start_time: 101,
12969 };
12970 record_deliberate_severance(&snapshot, process).unwrap();
12971 let exit_report = apply_deliberate_severance_marker(
12972 &snapshot,
12973 Some(process),
12974 ExitReport {
12975 kind: ExitKind::Crash,
12976 code: Some(1),
12977 signal: None,
12978 at_ms: 1,
12979 },
12980 );
12981 assert_eq!(exit_report.kind, ExitKind::DeliberateSeverance);
12982
12983 assert!(matches!(
12984 on_child_exit(
12985 &spec,
12986 runtime.restart_policy,
12987 &supervisor.registry,
12988 &snapshot,
12989 &runtime.terminal_ring,
12990 &runtime.spawn_events,
12991 &runtime.child_roster,
12992 exit_report,
12993 )
12994 .await,
12995 NextAction::Restart { schedule: _ }
12996 ));
12997 let state = lock_snapshot(&snapshot).unwrap();
12998 assert_eq!(state.lifetime_restarts, 1);
12999 assert_eq!(state.crash_restarts.len(), 0);
13000 }
13001
13002 #[tokio::test]
13003 async fn genuine_crash_spends_restart_budget_and_records_lifetime() {
13004 let supervisor = Supervisor::new_for_test(
13005 Arc::new(Registry::default()),
13006 RestartPolicy::new(3, Duration::ZERO),
13007 );
13008 let runtime = supervisor.runtime_config();
13009 let spec = ModuleSpec {
13010 module_id: "genuine-crash".to_string(),
13011 program: PathBuf::from("/unused/genuine-crash"),
13012 args: Vec::new(),
13013 env: Vec::new(),
13014 reserved: false,
13015 reserved_prefixes: Vec::new(),
13016 protocol: ModuleProtocol::Subc,
13017 overlap: Default::default(),
13018 };
13019 let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::starting()));
13020
13021 assert!(matches!(
13022 on_child_exit(
13023 &spec,
13024 runtime.restart_policy,
13025 &supervisor.registry,
13026 &snapshot,
13027 &runtime.terminal_ring,
13028 &runtime.spawn_events,
13029 &runtime.child_roster,
13030 ExitReport {
13031 kind: ExitKind::Crash,
13032 code: Some(1),
13033 signal: None,
13034 at_ms: 1,
13035 },
13036 )
13037 .await,
13038 NextAction::Restart { schedule: _ }
13039 ));
13040 let state = lock_snapshot(&snapshot).unwrap();
13041 assert_eq!(state.lifetime_restarts, 1);
13042 assert_eq!(state.crash_restarts.len(), 1);
13043 }
13044
13045 fn crash_exit_report(at_ms: u64) -> ExitReport {
13046 ExitReport {
13047 kind: ExitKind::Crash,
13048 code: Some(1),
13049 signal: None,
13050 at_ms,
13051 }
13052 }
13053
13054 fn windowed_crash_spec(module_id: &str) -> ModuleSpec {
13055 ModuleSpec {
13056 module_id: module_id.to_string(),
13057 program: PathBuf::from("/unused").join(module_id),
13058 args: Vec::new(),
13059 env: Vec::new(),
13060 reserved: false,
13061 reserved_prefixes: Vec::new(),
13062 protocol: ModuleProtocol::Subc,
13063 overlap: Default::default(),
13064 }
13065 }
13066
13067 #[tokio::test]
13073 async fn three_crashes_inside_the_window_stop_the_module_and_name_the_window() {
13074 let (logs, _guard) = crate::router::test_log::log_capture(tracing::Level::ERROR);
13075 let supervisor = Supervisor::new_for_test(
13076 Arc::new(Registry::default()),
13077 RestartPolicy::new(2, Duration::ZERO),
13078 );
13079 let runtime = supervisor.runtime_config();
13080 let spec = windowed_crash_spec("crash-loop-in-window");
13081 let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::starting()));
13082
13083 for attempt in 1..=2 {
13084 assert!(
13085 matches!(
13086 on_child_exit(
13087 &spec,
13088 runtime.restart_policy,
13089 &supervisor.registry,
13090 &snapshot,
13091 &runtime.terminal_ring,
13092 &runtime.spawn_events,
13093 &runtime.child_roster,
13094 crash_exit_report(attempt),
13095 )
13096 .await,
13097 NextAction::Restart { schedule: _ }
13098 ),
13099 "crash {attempt} is inside the budget and must respawn"
13100 );
13101 }
13102
13103 assert!(matches!(
13104 on_child_exit(
13105 &spec,
13106 runtime.restart_policy,
13107 &supervisor.registry,
13108 &snapshot,
13109 &runtime.terminal_ring,
13110 &runtime.spawn_events,
13111 &runtime.child_roster,
13112 crash_exit_report(3),
13113 )
13114 .await,
13115 NextAction::Stop { .. }
13116 ));
13117
13118 {
13119 let state = lock_snapshot(&snapshot).unwrap();
13120 assert_eq!(state.state, ModuleState::Failed);
13121 assert_eq!(state.crash_restarts.len(), 2);
13122 assert_eq!(state.lifetime_restarts, 2);
13123 }
13124
13125 let history = runtime
13126 .terminal_ring
13127 .lock()
13128 .expect("terminal ring is not poisoned")
13129 .snapshot();
13130 let last = history
13131 .entries
13132 .last()
13133 .expect("the refused crash is retained");
13134 assert_eq!(last.disposition, TerminalDisposition::Failed);
13135 assert_eq!(
13136 last.disposition_detail.as_deref(),
13137 Some("crash budget exhausted: max_restarts=2 within window_secs=600")
13138 );
13139
13140 let captured = crate::router::test_log::captured_logs(&logs);
13141 assert!(
13142 captured.contains("crash budget exhausted: max_restarts=2 within window_secs=600"),
13143 "the stop must be logged with its window: {captured}"
13144 );
13145 }
13146
13147 #[tokio::test]
13155 async fn a_crash_older_than_the_window_frees_its_slot_for_a_later_crash() {
13156 let supervisor = Supervisor::new_for_test(
13157 Arc::new(Registry::default()),
13158 RestartPolicy::new(2, Duration::ZERO),
13159 );
13160 let runtime = supervisor.runtime_config();
13161 let spec = windowed_crash_spec("crash-across-windows");
13162 let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::starting()));
13163
13164 for attempt in 1..=2 {
13165 assert!(matches!(
13166 on_child_exit(
13167 &spec,
13168 runtime.restart_policy,
13169 &supervisor.registry,
13170 &snapshot,
13171 &runtime.terminal_ring,
13172 &runtime.spawn_events,
13173 &runtime.child_roster,
13174 crash_exit_report(attempt),
13175 )
13176 .await,
13177 NextAction::Restart { schedule: _ }
13178 ));
13179 }
13180
13181 update_snapshot(&snapshot, Some(&spec.module_id), |state| {
13184 age_oldest_crash_restart_out_of_window(state, runtime.restart_policy.window);
13185 })
13186 .unwrap();
13187
13188 assert!(
13189 matches!(
13190 on_child_exit(
13191 &spec,
13192 runtime.restart_policy,
13193 &supervisor.registry,
13194 &snapshot,
13195 &runtime.terminal_ring,
13196 &runtime.spawn_events,
13197 &runtime.child_roster,
13198 crash_exit_report(3),
13199 )
13200 .await,
13201 NextAction::Restart { schedule: _ }
13202 ),
13203 "a crash older than the window must not hold a budget slot"
13204 );
13205
13206 let state = lock_snapshot(&snapshot).unwrap();
13207 assert_eq!(state.state, ModuleState::Restarting);
13208 assert_eq!(
13209 state.crash_restarts.len(),
13210 2,
13211 "the aged instant is dropped and the new one takes its place"
13212 );
13213 assert_eq!(
13214 state.lifetime_restarts, 3,
13215 "the ledger counts every restart, including the ones the window forgot"
13216 );
13217 }
13218
13219 #[tokio::test]
13224 async fn an_operator_restart_clears_the_ring_and_leaves_the_ledger_alone() {
13225 let supervisor = Supervisor::new_for_test(
13226 Arc::new(Registry::default()),
13227 RestartPolicy::new(2, Duration::ZERO),
13228 );
13229 let runtime = supervisor.runtime_config();
13230 let spec = windowed_crash_spec("operator-cleared-budget");
13231 let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::starting()));
13232
13233 for attempt in 1..=2 {
13234 assert!(matches!(
13235 on_child_exit(
13236 &spec,
13237 runtime.restart_policy,
13238 &supervisor.registry,
13239 &snapshot,
13240 &runtime.terminal_ring,
13241 &runtime.spawn_events,
13242 &runtime.child_roster,
13243 crash_exit_report(attempt),
13244 )
13245 .await,
13246 NextAction::Restart { schedule: _ }
13247 ));
13248 }
13249
13250 reset_restart_count(&snapshot, &spec.module_id).unwrap();
13251 {
13252 let state = lock_snapshot(&snapshot).unwrap();
13253 assert!(
13254 state.crash_restarts.is_empty(),
13255 "an operator restart returns the full budget"
13256 );
13257 assert_eq!(
13258 state.lifetime_restarts, 2,
13259 "clearing the budget must not unmake the crashes"
13260 );
13261 }
13262
13263 assert!(
13264 matches!(
13265 on_child_exit(
13266 &spec,
13267 runtime.restart_policy,
13268 &supervisor.registry,
13269 &snapshot,
13270 &runtime.terminal_ring,
13271 &runtime.spawn_events,
13272 &runtime.child_roster,
13273 crash_exit_report(3),
13274 )
13275 .await,
13276 NextAction::Restart { schedule: _ }
13277 ),
13278 "the cleared budget must be spendable again"
13279 );
13280 let state = lock_snapshot(&snapshot).unwrap();
13281 assert_eq!(state.crash_restarts.len(), 1);
13282 assert_eq!(state.lifetime_restarts, 3);
13283 }
13284
13285 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
13286 async fn severance_marker_for_a_dead_child_does_not_label_its_successor() {
13287 let severed = ProcessIdentity {
13288 pid: 41,
13289 start_time: 101,
13290 };
13291 let successor = ProcessIdentity {
13292 pid: 41,
13293 start_time: 202,
13294 };
13295 let module = module_with_recovery_snapshot(ModuleState::Running, true, 0);
13296 update_snapshot(&module.inner.snapshot, Some("recovery-snapshot"), |state| {
13297 state.pid = Some(successor.pid);
13298 state.process_start_time = Some(successor.start_time);
13299 })
13300 .unwrap();
13301 assert!(!module.record_deliberate_severance(severed).unwrap());
13302
13303 let exit_report = apply_deliberate_severance_marker(
13304 &module.inner.snapshot,
13305 Some(successor),
13306 ExitReport {
13307 kind: ExitKind::Crash,
13308 code: Some(1),
13309 signal: None,
13310 at_ms: 1,
13311 },
13312 );
13313
13314 assert_eq!(exit_report.kind, ExitKind::Crash);
13315 }
13316
13317 #[tokio::test]
13318 async fn drain_reap_marks_deliberate_severance_and_records_lifetime_without_budget() {
13319 let registry = Registry::default();
13320 let supervisor = Supervisor::new_for_test(
13321 Arc::new(Registry::default()),
13322 RestartPolicy::new(3, Duration::ZERO),
13323 );
13324 let runtime = supervisor.runtime_config();
13325 let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::starting()));
13326 let spec = ModuleSpec {
13327 module_id: "drain-deliberate-severance".to_string(),
13328 program: fake_aft_stub_path(),
13329 args: Vec::new(),
13330 env: vec![("FAKE_AFT_EXIT_CODE".to_string(), "23".to_string())],
13331 reserved: false,
13332 reserved_prefixes: Vec::new(),
13333 protocol: ModuleProtocol::Subc,
13334 overlap: Default::default(),
13335 };
13336 let mut child = spawn_and_mark_running(&spec, &runtime, &snapshot).unwrap();
13337 let process = ProcessIdentity {
13338 pid: 41,
13339 start_time: 101,
13340 };
13341 child.process_identity = Some(process);
13342 update_snapshot(&snapshot, Some(&spec.module_id), |state| {
13343 state.pid = Some(process.pid);
13344 state.process_start_time = Some(process.start_time);
13345 })
13346 .unwrap();
13347 record_deliberate_severance(&snapshot, process).unwrap();
13348
13349 drain_child_to_state(
13350 &spec.module_id,
13351 spec.protocol,
13352 StopNotice::SentOverConnection,
13355 ®istry,
13356 None,
13357 &snapshot,
13358 &runtime.terminal_ring,
13359 &runtime.spawn_events,
13360 child,
13361 Duration::from_secs(1),
13362 ModuleState::Stopped,
13363 Some(false),
13364 )
13365 .await
13366 .unwrap();
13367
13368 let state = lock_snapshot(&snapshot).unwrap();
13369 assert_eq!(
13370 state.last_exit.as_ref().map(|exit| exit.kind),
13371 Some(ExitKind::DeliberateSeverance)
13372 );
13373 assert_eq!(state.lifetime_restarts, 1);
13374 assert_eq!(state.crash_restarts.len(), 0);
13375 drop(state);
13376 let history = runtime.terminal_ring.lock().unwrap().snapshot();
13377 assert_eq!(
13378 history.entries[0].exit_kind,
13379 subc_control::TerminalExitKind::DeliberateSeverance
13380 );
13381 }
13382
13383 #[tokio::test]
13384 async fn ordinary_drain_reap_does_not_record_a_lifetime_restart() {
13385 let registry = Registry::default();
13386 let supervisor = Supervisor::new_for_test(
13387 Arc::new(Registry::default()),
13388 RestartPolicy::new(3, Duration::ZERO),
13389 );
13390 let runtime = supervisor.runtime_config();
13391 let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::starting()));
13392 let spec = ModuleSpec {
13393 module_id: "ordinary-drain".to_string(),
13394 program: fake_aft_stub_path(),
13395 args: Vec::new(),
13396 env: vec![("FAKE_AFT_EXIT_CODE".to_string(), "23".to_string())],
13397 reserved: false,
13398 reserved_prefixes: Vec::new(),
13399 protocol: ModuleProtocol::Subc,
13400 overlap: Default::default(),
13401 };
13402 let child = spawn_and_mark_running(&spec, &runtime, &snapshot).unwrap();
13403
13404 drain_child_to_state(
13405 &spec.module_id,
13406 spec.protocol,
13407 StopNotice::SentOverConnection,
13410 ®istry,
13411 None,
13412 &snapshot,
13413 &runtime.terminal_ring,
13414 &runtime.spawn_events,
13415 child,
13416 Duration::from_secs(1),
13417 ModuleState::Stopped,
13418 Some(false),
13419 )
13420 .await
13421 .unwrap();
13422
13423 let state = lock_snapshot(&snapshot).unwrap();
13424 assert_eq!(
13425 state.last_exit.as_ref().map(|exit| exit.kind),
13426 Some(ExitKind::Crash)
13427 );
13428 assert_eq!(state.lifetime_restarts, 0);
13429 assert_eq!(state.crash_restarts.len(), 0);
13430 }
13431
13432 #[test]
13433 fn fatal_connection_teardown_cannot_arm_a_marker_for_a_surviving_process() {
13434 assert!(!include_str!("server.rs")
13440 .contains("router.record_deliberate_connection_severance(ctx.connection_id)"));
13441 }
13442
13443 #[test]
13450 fn drained_after_quiescence_wait_passes_ok_through_and_forces_false_on_err() {
13451 assert!(drained_after_quiescence_wait(&Ok(true)));
13452 assert!(!drained_after_quiescence_wait(&Ok(false)));
13453 assert!(!drained_after_quiescence_wait(&Err(
13454 SuperviseError::StatePoisoned { module_id: None }
13455 )));
13456 }
13457
13458 #[test]
13467 fn wait_error_exit_report_records_a_failed_terminal_with_no_code_or_signal() {
13468 let ring = Arc::new(Mutex::new(TerminalRing::new(
13469 TerminalRingConfig::default(),
13470 0,
13471 )));
13472 record_wait_error_terminal("wait-error", &ring, &super::SpawnEventFeed::default());
13473
13474 let snapshot = ring.lock().unwrap().snapshot();
13475 assert_eq!(snapshot.entries.len(), 1);
13476 let entry = &snapshot.entries[0];
13477 assert_eq!(entry.exit_code, None);
13478 assert_eq!(entry.exit_signal, None);
13479 assert_eq!(entry.disposition, TerminalDisposition::Failed);
13480 }
13481
13482 #[test]
13483 fn wait_error_exit_path_preserves_spawn_event_density() {
13484 let feed = super::SpawnEventFeed::default();
13485 feed.configure_incarnation("wait-error-density".to_string());
13486 feed.emit_spawned("wait-error", 41, 1);
13487 let ring = Arc::new(Mutex::new(TerminalRing::new(
13488 TerminalRingConfig::default(),
13489 0,
13490 )));
13491
13492 record_wait_error_terminal("wait-error", &ring, &feed);
13493 feed.emit_spawned("after-wait-error", 42, 2);
13494
13495 let state = feed.0.lock().unwrap();
13496 let sequences = state
13497 .events
13498 .iter()
13499 .map(|event| event.cursor.seq)
13500 .collect::<Vec<_>>();
13501 assert_eq!(sequences, vec![1, 2, 3]);
13502 assert_eq!(state.events[1].kind, SpawnEventKind::Exited);
13503 assert_eq!(state.events[1].exit_code, None);
13504 assert_eq!(state.events[1].exit_signal, None);
13505 }
13506
13507 #[test]
13511 fn wait_error_exit_report_is_classified_as_a_crash() {
13512 assert_eq!(wait_error_exit_report().kind, ExitKind::Crash);
13513 }
13514}
13515
13516#[cfg(all(test, unix))]
13517mod restart_ordering_tests {
13518 use super::*;
13519 use std::io::Write;
13520 use tokio::io::AsyncReadExt;
13521
13522 #[test]
13523 #[ignore = "child process fixture"]
13524 fn serving_child() {
13525 let Ok(address_file) = std::env::var("RESTART_TEST_ADDRESS_FILE") else {
13526 return;
13527 };
13528 let listener = std::net::TcpListener::bind("127.0.0.1:0").unwrap();
13529 std::fs::write(address_file, listener.local_addr().unwrap().to_string()).unwrap();
13530 for stream in listener.incoming() {
13531 writeln!(stream.unwrap(), "{}", std::process::id()).unwrap();
13532 }
13533 }
13534
13535 async fn restart_before_publication(requests: usize) {
13536 let home = cortexkit_test_support::ScratchDir::new("restart-before-publication");
13537 let address_file = home.path().join("address");
13538 let spec = ModuleSpec {
13539 module_id: "restart-serving-child".into(),
13540 program: std::env::current_exe().unwrap(),
13541 args: vec![
13542 "--exact".into(),
13543 "supervise::restart_ordering_tests::serving_child".into(),
13544 "--ignored".into(),
13545 "--nocapture".into(),
13546 ],
13547 env: vec![(
13548 "RESTART_TEST_ADDRESS_FILE".into(),
13549 address_file.display().to_string(),
13550 )],
13551 reserved: false,
13552 reserved_prefixes: vec![],
13553 protocol: ModuleProtocol::None,
13554 overlap: Default::default(),
13555 };
13556 let mut supervisor =
13557 Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default());
13558 let (spawned_tx, spawned_rx) = oneshot::channel();
13559 let spawned_tx = Mutex::new(Some(spawned_tx));
13560 let (release_tx, release_rx) = std::sync::mpsc::channel();
13561 let release_rx = Mutex::new(release_rx);
13562 supervisor.test_after_first_spawn = AfterFirstSpawnHook(Some(Arc::new(move |_| {
13565 spawned_tx.lock().unwrap().take().unwrap().send(()).unwrap();
13566 release_rx
13567 .lock()
13568 .unwrap()
13569 .recv_timeout(Duration::from_secs(20))
13570 .unwrap();
13571 })));
13572 let registered = supervisor
13573 .register_configured_with_health(
13574 spec.clone(),
13575 true,
13576 HealthConfig::default(),
13577 None,
13578 RestartPolicy::new(3, Duration::ZERO),
13579 )
13580 .unwrap();
13581 let module = registered.module.clone();
13582 let startup = tokio::task::spawn_blocking(move || {
13583 supervisor.start_registered_configured(registered, None)
13584 });
13585 spawned_rx.await.unwrap();
13586 let old_pid = tokio::time::timeout(Duration::from_secs(10), async {
13587 let address = loop {
13588 if let Ok(address) = std::fs::read_to_string(&address_file) {
13589 if !address.is_empty() {
13590 break address;
13591 }
13592 }
13593 tokio::time::sleep(Duration::from_millis(1)).await;
13594 };
13595 let mut stream = tokio::net::TcpStream::connect(address).await.unwrap();
13596 let mut reply = String::new();
13597 stream.read_to_string(&mut reply).await.unwrap();
13598 reply.trim().parse::<u32>().unwrap()
13599 })
13600 .await
13601 .expect("child must serve before generation publication");
13602 assert_eq!(
13603 lock_snapshot(&module.inner.snapshot)
13604 .unwrap()
13605 .spawn_generation,
13606 0
13607 );
13608 let mut replies = Vec::new();
13609 for _ in 0..requests {
13610 let mut restart = Box::pin(module.restart(Some(0)));
13611 std::future::poll_fn(|cx| {
13614 assert!(std::future::Future::poll(restart.as_mut(), cx).is_pending());
13615 std::task::Poll::Ready(())
13616 })
13617 .await;
13618 replies.push(restart);
13619 }
13620 release_tx.send(()).unwrap();
13621 startup.await.unwrap().unwrap();
13622 for reply in replies {
13623 reply.await.unwrap();
13624 }
13625 let replacement = tokio::time::timeout(Duration::from_secs(10), async {
13628 loop {
13629 if lock_snapshot(&module.inner.snapshot)
13630 .unwrap()
13631 .spawn_generation
13632 > 1
13633 {
13634 break;
13635 }
13636 tokio::time::sleep(Duration::from_millis(1)).await;
13637 }
13638 })
13639 .await;
13640 module.update_spec_for_test(spec).await.unwrap();
13644 let (generation, pid) = {
13645 let state = lock_snapshot(&module.inner.snapshot).unwrap();
13646 (state.spawn_generation, state.pid)
13647 };
13648 module.stop().await.unwrap();
13649 replacement.expect("restart must publish a replacement for the already-serving child");
13650 assert_ne!(
13651 pid,
13652 Some(old_pid),
13653 "a serving child predating the restart must be replaced"
13654 );
13655 assert_eq!(generation, 2, "queued restarts must share one replacement");
13656 }
13657
13658 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
13659 async fn restart_received_after_serving_before_generation_publication_replaces_child() {
13660 restart_before_publication(1).await;
13661 }
13662
13663 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
13664 async fn restart_queued_behind_restart_does_not_kill_its_replacement() {
13665 restart_before_publication(2).await;
13666 }
13667
13668 #[tokio::test]
13669 async fn restart_queued_before_replacement_spawn_keeps_the_new_child() {
13670 let supervisor =
13671 Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default());
13672 let registered = supervisor
13673 .register_configured_with_health(
13674 ModuleSpec {
13675 module_id: "queued-before-replacement".into(),
13676 program: "/bin/sleep".into(),
13681 args: vec!["600".into()],
13682 env: vec![],
13683 reserved: false,
13684 reserved_prefixes: vec![],
13685 protocol: ModuleProtocol::None,
13686 overlap: Default::default(),
13687 },
13688 true,
13689 HealthConfig::default(),
13690 None,
13691 RestartPolicy::default(),
13692 )
13693 .unwrap();
13694 let RegisteredConfiguredModule {
13695 mut spec,
13696 mut runtime,
13697 module,
13698 mut commands,
13699 } = registered;
13700 let snapshot = &module.inner.snapshot;
13701 let mut restart = Box::pin(module.restart(Some(0)));
13702 std::future::poll_fn(|cx| {
13703 assert!(std::future::Future::poll(restart.as_mut(), cx).is_pending());
13704 std::task::Poll::Ready(())
13705 })
13706 .await;
13707 let mut child = Some(spawn_and_mark_running(&spec, &runtime, snapshot).unwrap());
13710 let replacement_pid = child.as_ref().unwrap().id();
13711 let command = commands.recv().await.unwrap();
13712 assert!(
13713 handle_supervisor_command(
13714 command,
13715 &mut spec,
13716 &mut runtime,
13717 &supervisor.registry,
13718 &supervisor.process_liveness,
13719 snapshot,
13720 &mut child,
13721 &mut commands,
13722 &mut VecDeque::new(),
13723 )
13724 .await
13725 );
13726 restart.await.unwrap();
13727 let kept_pid = child.as_ref().and_then(SupervisedChild::id);
13728 let pending = lock_snapshot(snapshot).unwrap().respawn_pending;
13729 if let Some(mut child) = child {
13730 child.start_kill().unwrap();
13731 child.wait().await.unwrap();
13732 }
13733 assert_eq!(
13734 kept_pid, replacement_pid,
13735 "a newer replacement must survive"
13736 );
13737 assert!(
13738 !pending,
13739 "a satisfied restart must not schedule another spawn"
13740 );
13741 }
13742}
13743
13744#[cfg(all(test, unix))]
13745mod health_event_tests {
13746 use super::*;
13747 use cortexkit_test_support::ScratchDir;
13748 use subc_protocol::{manifest::Concurrency, session::ModuleControlResponse};
13749
13750 struct Actor {
13751 module: SupervisedModule,
13752 registry: Arc<Registry>,
13753 forwarding: Arc<ForwardingTable>,
13754 spec: ModuleSpec,
13755 health: HealthConfig,
13756 rx: mpsc::Receiver<crate::router::OutboundFrame>,
13757 _home: ScratchDir,
13758 }
13759
13760 async fn wait_for<T>(reason: &str, mut observe: impl FnMut() -> Option<T>) -> T {
13761 let deadline = std::time::Instant::now() + Duration::from_secs(10);
13765 let tick = Instant::now();
13766 loop {
13767 if let Some(value) = observe() {
13768 assert_eq!(Instant::now(), tick, "{reason} must not wait for a timer");
13769 return value;
13770 }
13771 assert!(
13772 std::time::Instant::now() < deadline,
13773 "timed out waiting for {reason}"
13774 );
13775 tokio::task::yield_now().await;
13776 }
13777 }
13778
13779 impl Actor {
13780 async fn start(protocol: ModuleProtocol, cadence: Duration) -> Self {
13781 let home = ScratchDir::new("health-events");
13782 let registry = Arc::new(Registry::default());
13783 let forwarding = Arc::new(ForwardingTable::default());
13784 let health = HealthConfig {
13785 cadence,
13786 deadline: Duration::from_secs(3600),
13787 ..HealthConfig::default()
13788 };
13789 let supervisor =
13790 Supervisor::new_for_test(registry.clone(), RestartPolicy::new(3, Duration::ZERO))
13791 .with_forwarding(forwarding.clone())
13792 .with_health_config(health.clone());
13793 let spec = ModuleSpec {
13794 module_id: "event-health-child".into(),
13795 program: PathBuf::from("/bin/sleep"),
13803 args: vec!["600".into()],
13806 env: ["XDG_DATA_HOME", "XDG_RUNTIME_DIR", "XDG_CONFIG_HOME"]
13807 .into_iter()
13808 .map(|key| (key.into(), home.path().display().to_string()))
13809 .collect(),
13810 reserved: false,
13811 reserved_prefixes: vec![],
13812 protocol,
13813 overlap: Default::default(),
13814 };
13815 let module = supervisor.spawn(spec.clone()).unwrap();
13816 let (_, rx) = mpsc::channel(8);
13817 let actor = Self {
13818 module,
13819 registry,
13820 forwarding,
13821 spec,
13822 health,
13823 rx,
13824 _home: home,
13825 };
13826 actor
13827 .wait_for_select(|checkpoint| checkpoint.generation > 0)
13828 .await;
13829 actor
13830 }
13831
13832 fn turns(&self) -> u64 {
13833 lock_snapshot(&self.module.inner.snapshot)
13834 .unwrap()
13835 .actor_turns
13836 }
13837
13838 async fn wait_for_select(
13839 &self,
13840 expected: impl Fn(&ActorSelectCheckpoint) -> bool,
13841 ) -> ActorSelectCheckpoint {
13842 wait_for("actor select checkpoint", || {
13843 let state = lock_snapshot(&self.module.inner.snapshot).unwrap();
13844 state.actor_select.filter(|checkpoint| {
13845 checkpoint.generation == state.spawn_generation
13846 && state.state == ModuleState::Running
13847 && state.process_alive
13848 && state.reported_pid().is_some()
13849 && expected(checkpoint)
13850 })
13851 })
13852 .await
13853 }
13854
13855 fn hello(&mut self, connection: u64, health: bool) {
13856 let manifest =
13857 subc_protocol::manifest::ModuleManifest::builder(&self.spec.module_id, "1.0.0")
13858 .protocol_ver(subc_protocol::PROTOCOL_VERSION)
13859 .build();
13860 let connection = ConnectionId::new(connection);
13861 let (tx, rx) = mpsc::channel(8);
13862 self.rx = rx;
13863 self.forwarding
13864 .register_module_connection(
13865 connection,
13866 self.spec.module_id.clone(),
13867 subc_protocol::PROTOCOL_VERSION,
13868 Concurrency::ModuleManaged,
13869 FrameSink::new(tx),
13870 )
13871 .unwrap();
13872 self.registry
13873 .register_with_control_ops(
13874 manifest,
13875 subc_protocol::PROTOCOL_VERSION,
13876 connection,
13877 if health {
13878 vec![MODULE_CONTROL_OP_HEALTH_CHECK.into()]
13879 } else {
13880 vec![]
13881 },
13882 )
13883 .unwrap();
13884 }
13885
13886 async fn probe(&mut self) -> crate::router::OutboundFrame {
13887 let frame = wait_for("outbound health probe", || match self.rx.try_recv() {
13888 Ok(frame) => Some(frame),
13889 Err(mpsc::error::TryRecvError::Empty) => None,
13890 Err(mpsc::error::TryRecvError::Disconnected) => panic!("health peer disconnected"),
13891 })
13892 .await;
13893 let body: Value = serde_json::from_slice(&frame.body).unwrap();
13894 assert_eq!(body["op"], MODULE_CONTROL_OP_HEALTH_CHECK);
13895 frame
13896 }
13897
13898 fn no_probe(&mut self) {
13899 assert!(matches!(
13900 self.rx.try_recv(),
13901 Err(mpsc::error::TryRecvError::Empty)
13902 ));
13903 }
13904
13905 fn answer(&self, connection: u64, frame: crate::router::OutboundFrame) {
13906 self.forwarding
13907 .complete_module_control_rpc(
13908 ConnectionId::new(connection),
13909 frame.header.corr,
13910 Some(MODULE_CONTROL_OP_HEALTH_CHECK),
13911 ModuleControlRpcOutcome::Response(ModuleControlResponse::HealthCheck {
13912 status: HealthStatus::Ok,
13913 detail: None,
13914 metrics: None,
13915 }),
13916 )
13917 .unwrap();
13918 }
13919 }
13920
13921 #[tokio::test(start_paused = true)]
13922 async fn no_health_children_park_for_sixty_seconds() {
13923 let none = Actor::start(ModuleProtocol::None, Duration::from_secs(30)).await;
13924 let unregistered = Actor::start(ModuleProtocol::Subc, Duration::from_secs(30)).await;
13925 let mut unadvertised = Actor::start(ModuleProtocol::Subc, Duration::from_secs(30)).await;
13926 unadvertised.hello(1, false);
13927 for _ in 0..6000 {
13932 tokio::time::advance(Duration::from_millis(10)).await;
13933 tokio::task::yield_now().await;
13934 }
13935 for actor in [&none, &unregistered, &unadvertised] {
13936 assert!(
13937 actor.turns() <= 5,
13938 "no-health actor woke {} times",
13939 actor.turns()
13940 );
13941 assert_eq!(actor.module.state().unwrap(), ModuleState::Running);
13942 }
13943 }
13944
13945 #[tokio::test(start_paused = true)]
13946 async fn late_hello_starts_a_due_probe_in_the_notification_tick() {
13947 let mut actor = Actor::start(ModuleProtocol::Subc, Duration::ZERO).await;
13951 let tick = Instant::now();
13952 actor.hello(1, true);
13953 actor.probe().await;
13954 assert_eq!(Instant::now(), tick, "HELLO must not wait for a poll");
13955 }
13956
13957 #[tokio::test(start_paused = true)]
13958 async fn hello_keeps_the_jittered_cadence_and_catalog_updates_do_not_reset_it() {
13959 let mut actor = Actor::start(ModuleProtocol::Subc, Duration::from_secs(30)).await;
13960 let hello_tick = Instant::now();
13961 actor.hello(1, true);
13962 let armed = actor
13963 .wait_for_select(|checkpoint| {
13964 checkpoint.registered_connection == Some(ConnectionId::new(1))
13965 && checkpoint.next_probe_at.is_some()
13966 })
13967 .await;
13968 let delay = jittered_health_delay(&actor.spec.module_id, 0, actor.health.cadence);
13969 assert!((Duration::from_secs(30)..Duration::from_secs(33)).contains(&delay));
13970 let first_deadline = hello_tick + delay;
13971 assert_eq!(armed.next_probe_at, Some(first_deadline));
13972 tokio::time::advance(Duration::from_secs(29)).await;
13973 let turns = actor.turns();
13974 actor
13975 .registry
13976 .replace_catalog_for_connection(ConnectionId::new(1), vec![], None, Some(true))
13977 .unwrap();
13978 let updated = actor
13979 .wait_for_select(|checkpoint| checkpoint.turn > turns)
13980 .await;
13981 assert_eq!(
13982 updated.next_probe_at,
13983 Some(first_deadline),
13984 "catalog update must not reset cadence"
13985 );
13986 actor.no_probe();
13987 tokio::time::advance(first_deadline - Instant::now()).await;
13988 let frame = actor.probe().await;
13989 actor.answer(1, frame);
13990 let next = jittered_health_delay(&actor.spec.module_id, 1, actor.health.cadence);
13991 let next_deadline = Instant::now() + next;
13992 let rearmed = actor
13993 .wait_for_select(|checkpoint| {
13994 checkpoint
13995 .next_probe_at
13996 .is_some_and(|deadline| deadline > first_deadline)
13997 })
13998 .await;
13999 assert_eq!(rearmed.next_probe_at, Some(next_deadline));
14000 tokio::time::advance(next - Duration::from_nanos(1)).await;
14001 actor.no_probe();
14002 tokio::time::advance(Duration::from_nanos(1)).await;
14003 actor.probe().await;
14004 }
14005
14006 #[tokio::test(start_paused = true)]
14007 async fn disconnect_disarms_and_reconnect_rearms_health() {
14008 let mut actor = Actor::start(ModuleProtocol::Subc, Duration::from_secs(30)).await;
14009 actor.hello(1, true);
14010 actor
14011 .wait_for_select(|checkpoint| {
14012 checkpoint.registered_connection == Some(ConnectionId::new(1))
14013 && checkpoint.next_probe_at.is_some()
14014 })
14015 .await;
14016 let turns = actor.turns();
14017 let tick = Instant::now();
14018 actor
14019 .registry
14020 .deregister_connection(ConnectionId::new(1))
14021 .unwrap();
14022 let disarmed = actor
14023 .wait_for_select(|checkpoint| {
14024 checkpoint.turn > turns && checkpoint.registered_connection.is_none()
14025 })
14026 .await;
14027 assert_eq!(disarmed.next_probe_at, None);
14028 assert_eq!(disarmed.wake_after, None);
14029 assert_eq!(
14030 Instant::now(),
14031 tick,
14032 "disconnect must disarm in the notification tick"
14033 );
14034 tokio::time::advance(Duration::from_secs(60)).await;
14035 actor.no_probe();
14036 let reconnect_tick = Instant::now();
14037 actor.hello(2, true);
14038 let rearmed = actor
14039 .wait_for_select(|checkpoint| {
14040 checkpoint.registered_connection == Some(ConnectionId::new(2))
14041 && checkpoint.next_probe_at.is_some()
14042 })
14043 .await;
14044 let delay = jittered_health_delay(&actor.spec.module_id, 0, actor.health.cadence);
14045 assert_eq!(rearmed.next_probe_at, Some(reconnect_tick + delay));
14046 tokio::time::advance(delay).await;
14047 actor.probe().await;
14048 }
14049
14050 #[tokio::test(start_paused = true)]
14051 async fn reregistration_without_health_disarms_the_old_schedule() {
14052 let mut actor = Actor::start(ModuleProtocol::Subc, Duration::from_secs(30)).await;
14053 actor.hello(1, true);
14054 actor
14055 .wait_for_select(|checkpoint| {
14056 checkpoint.registered_connection == Some(ConnectionId::new(1))
14057 && checkpoint.next_probe_at.is_some()
14058 })
14059 .await;
14060 actor
14061 .registry
14062 .deregister_connection(ConnectionId::new(1))
14063 .unwrap();
14064 actor.hello(2, false);
14065 let disarmed = actor
14066 .wait_for_select(|checkpoint| {
14067 checkpoint.registered_connection == Some(ConnectionId::new(2))
14068 })
14069 .await;
14070 assert_eq!(disarmed.next_probe_at, None);
14071 assert_eq!(disarmed.wake_after, None);
14072 tokio::time::advance(Duration::from_secs(60)).await;
14073 actor.no_probe();
14074 }
14075
14076 #[tokio::test(start_paused = true)]
14077 async fn restart_waits_for_its_own_hello_before_rearming_health() {
14078 let mut actor = Actor::start(ModuleProtocol::Subc, Duration::from_secs(30)).await;
14079 actor.hello(1, true);
14080 let armed = actor
14081 .wait_for_select(|checkpoint| {
14082 checkpoint.registered_connection == Some(ConnectionId::new(1))
14083 && checkpoint.next_probe_at.is_some()
14084 })
14085 .await;
14086 actor
14090 .registry
14091 .deregister_connection(ConnectionId::new(1))
14092 .unwrap();
14093 actor.module.restart(Some(0)).await.unwrap();
14094 let parked = actor
14095 .wait_for_select(|checkpoint| {
14096 checkpoint.generation > armed.generation
14097 && checkpoint.registered_connection.is_none()
14098 && checkpoint.next_probe_at.is_none()
14099 })
14100 .await;
14101 assert_eq!(parked.wake_after, None);
14102 actor.rx.close();
14103 while actor.rx.try_recv().is_ok() {}
14106 let turns = actor.turns();
14107 tokio::time::advance(Duration::from_secs(60)).await;
14108 let after = actor.turns();
14109 eprintln!(
14110 "restart park turns: before={turns} after={after} delta={}",
14111 after - turns
14112 );
14113 assert_eq!(after, turns, "replacement must park before HELLO");
14114 let hello_tick = Instant::now();
14115 actor.hello(2, true);
14116 let rearmed = actor
14117 .wait_for_select(|checkpoint| {
14118 checkpoint.registered_connection == Some(ConnectionId::new(2))
14119 && checkpoint.next_probe_at.is_some()
14120 })
14121 .await;
14122 let delay = jittered_health_delay(&actor.spec.module_id, 0, actor.health.cadence);
14123 assert_eq!(rearmed.next_probe_at, Some(hello_tick + delay));
14124 tokio::time::advance(delay).await;
14125 actor.probe().await;
14126 }
14127
14128 #[tokio::test(start_paused = true)]
14129 async fn http_probe_waits_a_full_cadence_for_each_spawn() {
14130 use tokio::{
14131 io::{AsyncReadExt, AsyncWriteExt},
14132 net::TcpListener,
14133 };
14134 let mut actor = Actor::start(ModuleProtocol::None, Duration::from_secs(30)).await;
14135 let listener = TcpListener::bind("127.0.0.1:0").await.unwrap();
14136 actor.health.http = Some(format!("http://{}/healthz", listener.local_addr().unwrap()));
14137 let (started_tx, mut started_rx) = mpsc::channel(8);
14138 let peer = tokio::spawn(async move {
14139 loop {
14140 let (mut socket, _) = listener.accept().await.unwrap();
14141 let mut request = [0; 1024];
14142 assert!(socket.read(&mut request).await.unwrap() > 0);
14143 started_tx.send(Instant::now()).await.unwrap();
14144 socket
14145 .write_all(b"HTTP/1.1 200 OK\r\nContent-Length: 0\r\n\r\n")
14146 .await
14147 .unwrap();
14148 }
14149 });
14150 actor
14151 .module
14152 .update_configuration(actor.spec.clone(), actor.health.clone(), None)
14153 .await
14154 .unwrap();
14155 let first_tick = Instant::now();
14156 let first = actor
14157 .wait_for_select(|checkpoint| checkpoint.next_probe_at.is_some())
14158 .await;
14159 assert!(first.next_probe_at.unwrap() >= first_tick + actor.health.cadence);
14160 assert!(started_rx.try_recv().is_err());
14161
14162 tokio::time::advance(
14165 first.next_probe_at.unwrap() - Instant::now() - Duration::from_secs(1),
14166 )
14167 .await;
14168 actor.module.restart(Some(0)).await.unwrap();
14169 let spawn_tick = Instant::now();
14170 let replacement = actor
14171 .wait_for_select(|checkpoint| checkpoint.generation > first.generation)
14172 .await;
14173 let deadline = replacement.next_probe_at.unwrap();
14174 assert!(
14175 deadline >= spawn_tick + actor.health.cadence,
14176 "a replacement must not inherit its predecessor's probe deadline"
14177 );
14178 tokio::time::advance(actor.health.cadence - Duration::from_nanos(1)).await;
14179 assert!(
14180 started_rx.try_recv().is_err(),
14181 "no HTTP probe during startup grace"
14182 );
14183 tokio::time::advance(deadline - Instant::now()).await;
14184 let landed = wait_for("first replacement HTTP probe", || {
14185 started_rx.try_recv().ok()
14186 })
14187 .await;
14188 assert!(landed >= spawn_tick + actor.health.cadence);
14189 wait_for("healthy replacement", || {
14190 (actor.module.status().unwrap().health.status == SupervisorHealthStatus::Ok)
14191 .then_some(())
14192 })
14193 .await;
14194 peer.abort();
14195 }
14196
14197 #[tokio::test(start_paused = true)]
14198 async fn rescan_adds_and_removes_http_health_without_polling() {
14199 use tokio::{
14200 io::{AsyncReadExt, AsyncWriteExt},
14201 net::TcpListener,
14202 };
14203 let mut actor = Actor::start(ModuleProtocol::None, Duration::from_secs(30)).await;
14204 let listener = TcpListener::bind("127.0.0.1:0").await.unwrap();
14205 actor.health.http = Some(format!("http://{}/health", listener.local_addr().unwrap()));
14206 actor.health.cadence = Duration::from_millis(1);
14208 let (started_tx, mut started_rx) = mpsc::channel(8);
14209 let peer = tokio::spawn(async move {
14210 loop {
14211 let (mut socket, _) = listener.accept().await.unwrap();
14212 let mut request = [0; 1024];
14213 assert!(socket.read(&mut request).await.unwrap() > 0);
14214 started_tx.send(()).await.unwrap();
14215 socket
14216 .write_all(b"HTTP/1.1 200 OK\r\nContent-Length: 0\r\n\r\n")
14217 .await
14218 .unwrap();
14219 }
14220 });
14221 let tick = Instant::now();
14222 let turns = actor.turns();
14223 actor
14224 .module
14225 .update_configuration(actor.spec.clone(), actor.health.clone(), None)
14226 .await
14227 .unwrap();
14228 let armed = actor
14229 .wait_for_select(|checkpoint| {
14230 checkpoint.turn > turns && checkpoint.next_probe_at.is_some()
14231 })
14232 .await;
14233 assert_eq!(armed.next_probe_at, Some(tick + Duration::from_millis(1)));
14234 assert_eq!(Instant::now(), tick);
14235 tokio::time::advance(Duration::from_millis(1)).await;
14236 let mut started = false;
14237 wait_for("successful HTTP probe", || {
14238 started |= started_rx.try_recv().is_ok();
14239 (started && actor.module.status().unwrap().health.status == SupervisorHealthStatus::Ok)
14240 .then_some(())
14241 })
14242 .await;
14243 assert!(started, "added HTTP check must start at its first deadline");
14244 assert_eq!(
14245 actor.module.status().unwrap().health.status,
14246 SupervisorHealthStatus::Ok
14247 );
14248 assert_eq!(Instant::now(), tick + Duration::from_millis(1));
14249 actor.health.http = None;
14250 let turns = actor.turns();
14251 actor
14252 .module
14253 .update_configuration(actor.spec.clone(), actor.health.clone(), None)
14254 .await
14255 .unwrap();
14256 let parked = actor
14257 .wait_for_select(|checkpoint| {
14258 checkpoint.turn > turns && checkpoint.next_probe_at.is_none()
14259 })
14260 .await;
14261 assert_eq!(parked.wake_after, None);
14262 let turns = actor.turns();
14263 tokio::time::advance(Duration::from_secs(60)).await;
14264 assert_eq!(
14265 actor.turns(),
14266 turns,
14267 "removed HTTP check must park the actor"
14268 );
14269 assert!(
14270 started_rx.try_recv().is_err(),
14271 "removed HTTP check must stay disarmed"
14272 );
14273 assert_eq!(
14274 actor.module.status().unwrap().health,
14275 ModuleHealthStatus::default()
14276 );
14277 peer.abort();
14278 }
14279}
14280
14281#[cfg(test)]
14282mod drain_busy_reprobe_tests {
14283 use super::*;
14287 use std::collections::BTreeMap;
14288 use std::sync::Mutex as StdMutex;
14289 use subc_protocol::{manifest::Concurrency, session::ModuleControlResponse};
14290
14291 const MODULE_ID: &str = "drain-reprobe-module";
14292 const CONNECTION: u64 = 7;
14293 const GAUGE: &str = "runs_in_flight";
14294 const CADENCE: Duration = Duration::from_secs(30);
14297 const CEILING: Duration = Duration::from_secs(30);
14298
14299 #[derive(Clone, Copy)]
14301 enum Answer {
14302 BusyUntil { until: Instant, latency: Duration },
14305 OmitsGauge,
14307 Silent,
14309 }
14310
14311 struct Drain {
14312 forwarding: Arc<ForwardingTable>,
14313 runtime: SupervisorRuntimeConfig,
14314 endpoint: crate::ModuleEndpointId,
14315 probes: Arc<StdMutex<Vec<Instant>>>,
14317 responder: tokio::task::JoinHandle<()>,
14318 }
14319
14320 impl Drain {
14321 fn start(answer: Answer) -> Self {
14322 let registry = Arc::new(Registry::default());
14323 let forwarding = Arc::new(ForwardingTable::default());
14324 let health = HealthConfig {
14325 cadence: CADENCE,
14326 ..HealthConfig::default()
14327 };
14328 let runtime = Supervisor::new_for_test(registry, RestartPolicy::new(3, Duration::ZERO))
14329 .with_forwarding(forwarding.clone())
14330 .with_health_config(health)
14331 .runtime_config();
14332 let (tx, mut rx) = mpsc::channel::<crate::router::OutboundFrame>(8);
14333 let endpoint = forwarding
14334 .register_module_connection(
14335 ConnectionId::new(CONNECTION),
14336 MODULE_ID.to_string(),
14337 subc_protocol::PROTOCOL_VERSION,
14338 Concurrency::ModuleManaged,
14339 FrameSink::new(tx),
14340 )
14341 .unwrap();
14342 let probes = Arc::new(StdMutex::new(Vec::new()));
14343 let responder = tokio::spawn({
14344 let forwarding = forwarding.clone();
14345 let probes = probes.clone();
14346 async move {
14347 while let Some(frame) = rx.recv().await {
14348 let body: Value = serde_json::from_slice(&frame.body).unwrap();
14349 assert_eq!(body["op"], MODULE_CONTROL_OP_HEALTH_CHECK);
14350 probes.lock().unwrap().push(Instant::now());
14351 let metrics = match answer {
14352 Answer::BusyUntil { until, latency } => {
14353 sleep(latency).await;
14354 let busy = u64::from(Instant::now() < until);
14355 serde_json::json!({ GAUGE: busy })
14356 }
14357 Answer::OmitsGauge => serde_json::json!({ "queue_depth": 0 }),
14358 Answer::Silent => continue,
14359 };
14360 let _ = forwarding.complete_module_control_rpc(
14363 ConnectionId::new(CONNECTION),
14364 frame.header.corr,
14365 Some(MODULE_CONTROL_OP_HEALTH_CHECK),
14366 ModuleControlRpcOutcome::Response(ModuleControlResponse::HealthCheck {
14367 status: HealthStatus::Ok,
14368 detail: None,
14369 metrics: Some(metrics),
14370 }),
14371 );
14372 }
14373 }
14374 });
14375 Self {
14376 forwarding,
14377 runtime,
14378 endpoint,
14379 probes,
14380 responder,
14381 }
14382 }
14383
14384 async fn run(&self, ceiling: Duration) -> (bool, Duration) {
14387 let (drained, elapsed, _) = self.run_traced(ceiling).await;
14388 (drained, elapsed)
14389 }
14390
14391 async fn run_traced(&self, ceiling: Duration) -> (bool, Duration, DrainGaugeTrace) {
14393 let started = Instant::now();
14394 let mut trace = DrainGaugeTrace::new(&[GAUGE.to_string()]);
14395 let drained = wait_for_forwarding_quiescence(
14396 &self.forwarding,
14397 MODULE_ID,
14398 &self.runtime,
14399 self.endpoint,
14400 started + ceiling,
14401 &[GAUGE.to_string()],
14402 DrainScope::Active,
14403 &mut trace,
14404 )
14405 .await
14406 .unwrap();
14407 (drained, started.elapsed(), trace)
14408 }
14409
14410 fn probes(&self) -> Vec<Instant> {
14411 self.probes.lock().unwrap().clone()
14412 }
14413 }
14414
14415 impl Drop for Drain {
14416 fn drop(&mut self) {
14417 self.responder.abort();
14418 }
14419 }
14420
14421 #[tokio::test(start_paused = true)]
14422 async fn busy_at_first_probe_then_quiet_drains_long_before_the_ceiling() {
14423 let drain = Drain::start(Answer::BusyUntil {
14424 until: Instant::now() + Duration::from_secs(1),
14425 latency: Duration::ZERO,
14426 });
14427 let (drained, elapsed) = drain.run(CEILING).await;
14428 assert!(
14429 drained,
14430 "gauges that read zero during the drain must let it finish drained, \
14431 not wait out the {CADENCE:?} health cadence (elapsed {elapsed:?})"
14432 );
14433 assert!(
14434 elapsed < Duration::from_secs(3),
14435 "drain took {elapsed:?} after the gauges went quiet at 1s"
14436 );
14437 let probes = drain.probes();
14438 assert!(
14439 probes.len() >= 2,
14440 "the module was busy at the first probe, so a quiet verdict needs a second: {probes:?}"
14441 );
14442 }
14443
14444 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
14449 async fn an_instant_busy_first_answer_does_not_stop_later_probes_in_real_time() {
14450 let drain = Drain::start(Answer::BusyUntil {
14451 until: Instant::now() + Duration::from_millis(600),
14452 latency: Duration::ZERO,
14453 });
14454 let (drained, elapsed, trace) = drain.run_traced(CEILING).await;
14455 assert!(drained, "the gauge fell at 600ms: {trace}");
14456 assert!(
14457 elapsed < Duration::from_secs(3),
14458 "drain took {elapsed:?} after the gauge fell at 600ms: {trace}"
14459 );
14460 let probes = drain.probes();
14461 assert!(
14462 probes.len() >= 2 && trace.probes as usize == probes.len(),
14463 "the drain must re-probe after the first busy answer: {probes:?}, {trace}"
14464 );
14465 }
14466
14467 #[tokio::test(start_paused = true)]
14468 async fn busy_throughout_ends_undrained_at_the_ceiling_with_bounded_probes() {
14469 let started = Instant::now();
14470 let drain = Drain::start(Answer::BusyUntil {
14471 until: started + Duration::from_secs(3600),
14472 latency: Duration::ZERO,
14473 });
14474 let (drained, elapsed) = drain.run(CEILING).await;
14475 assert!(!drained, "a module busy throughout must not drain");
14476 assert!(
14477 elapsed >= CEILING && elapsed < CEILING + drain_busy_reprobe_interval(),
14478 "a busy module is held until the ceiling and no longer: {elapsed:?}"
14479 );
14480 let probes = drain.probes();
14483 let max_probes = (CEILING.as_millis() / drain_busy_reprobe_interval().as_millis()) + 1;
14484 assert!(
14485 probes.len() as u128 <= max_probes,
14486 "{} probes in a {CEILING:?} drain exceeds the {max_probes} the interval allows",
14487 probes.len()
14488 );
14489 for pair in probes.windows(2) {
14490 assert!(
14491 pair[1] - pair[0] >= drain_busy_reprobe_interval(),
14492 "consecutive probes {:?} apart",
14493 pair[1] - pair[0]
14494 );
14495 }
14496 }
14497
14498 #[tokio::test(start_paused = true)]
14499 async fn a_slow_probe_is_never_overlapped_by_the_next() {
14500 let latency = Duration::from_secs(1);
14503 let drain = Drain::start(Answer::BusyUntil {
14504 until: Instant::now() + Duration::from_secs(3600),
14505 latency,
14506 });
14507 let (drained, _) = drain.run(Duration::from_secs(5)).await;
14508 assert!(!drained);
14509 let probes = drain.probes();
14510 assert!(probes.len() >= 2, "the drain must re-probe: {probes:?}");
14511 for pair in probes.windows(2) {
14512 assert!(
14513 pair[1] - pair[0] >= latency + drain_busy_reprobe_interval(),
14514 "a probe started {:?} after the previous one, before it could finish",
14515 pair[1] - pair[0]
14516 );
14517 }
14518 }
14519
14520 #[tokio::test(start_paused = true)]
14521 async fn an_unanswered_probe_cannot_hold_the_drain_past_its_ceiling() {
14522 let drain = Drain::start(Answer::Silent);
14523 let ceiling = Duration::from_secs(2);
14524 let (drained, elapsed) = drain.run(ceiling).await;
14525 assert!(!drained, "an unanswered probe counts as busy");
14526 assert!(
14527 elapsed >= ceiling && elapsed < ceiling + drain_busy_reprobe_interval(),
14528 "the probe deadline is capped at the drain ceiling: {elapsed:?}"
14529 );
14530 assert_eq!(
14531 drain.probes().len(),
14532 1,
14533 "the unanswered probe holds the only slot until the ceiling"
14534 );
14535 }
14536
14537 #[derive(Clone, Default)]
14539 struct LogCapture {
14540 events: Arc<StdMutex<Vec<BTreeMap<String, String>>>>,
14541 }
14542
14543 impl<S: tracing::Subscriber> tracing_subscriber::Layer<S> for LogCapture {
14544 fn on_event(
14545 &self,
14546 event: &tracing::Event<'_>,
14547 _context: tracing_subscriber::layer::Context<'_, S>,
14548 ) {
14549 struct Fields(BTreeMap<String, String>);
14550 impl tracing::field::Visit for Fields {
14551 fn record_debug(&mut self, field: &tracing::field::Field, value: &dyn fmt::Debug) {
14552 self.0
14553 .insert(field.name().to_string(), format!("{value:?}"));
14554 }
14555 }
14556 let mut fields = Fields(BTreeMap::new());
14557 event.record(&mut fields);
14558 self.events.lock().unwrap().push(fields.0);
14559 }
14560 }
14561
14562 impl LogCapture {
14563 fn install(&self) -> tracing::subscriber::DefaultGuard {
14564 use tracing_subscriber::layer::SubscriberExt;
14565 tracing::subscriber::set_default(tracing_subscriber::registry().with(self.clone()))
14566 }
14567
14568 fn with_message(&self, needle: &str) -> Vec<BTreeMap<String, String>> {
14569 self.events
14570 .lock()
14571 .unwrap()
14572 .iter()
14573 .filter(|fields| fields.get("message").is_some_and(|m| m.contains(needle)))
14574 .cloned()
14575 .collect()
14576 }
14577 }
14578
14579 #[tokio::test(start_paused = true)]
14584 async fn a_missing_declared_gauge_is_named_on_the_drain_timeout_line() {
14585 let capture = LogCapture::default();
14586 let _guard = capture.install();
14587 let drain = Drain::start(Answer::OmitsGauge);
14588 let (drained, _, trace) = drain.run_traced(Duration::from_secs(2)).await;
14589 assert!(!drained, "a gauge never reported cannot read quiet");
14590 assert!(trace.probes >= 2, "the drain must re-probe: {trace:?}");
14591
14592 let missing = capture.with_message("lacks a declared busy gauge");
14593 assert_eq!(missing.len(), 1, "one warning per drain: {missing:?}");
14594 assert_eq!(
14595 missing[0].get("missing_gauges").map(String::as_str),
14596 Some(GAUGE)
14597 );
14598
14599 warn_drain_timed_out(
14600 MODULE_ID,
14601 Duration::from_secs(2),
14602 RouteCloseReason::Restart,
14603 &Default::default(),
14604 &trace,
14605 );
14606 let timeouts = capture.with_message("route drain timed out");
14607 assert_eq!(timeouts.len(), 1, "{timeouts:?}");
14608 let gauges = &timeouts[0]["busy_gauges"];
14609 assert!(
14610 gauges.starts_with(&format!("{GAUGE}: missing_from_health_body @")),
14611 "the timeout line must name the gauge and why it read busy: {gauges}"
14612 );
14613 assert!(
14614 gauges.ends_with("; no value ever read"),
14615 "no value was ever read for the gauge: {gauges}"
14616 );
14617 assert_eq!(timeouts[0]["gauge_probes"], trace.probes.to_string());
14618 }
14619
14620 #[tokio::test(start_paused = true)]
14624 async fn the_drain_trace_names_a_busy_value_and_an_unanswered_probe() {
14625 let busy = Drain::start(Answer::BusyUntil {
14626 until: Instant::now() + Duration::from_secs(3600),
14627 latency: Duration::ZERO,
14628 });
14629 let (_, _, trace) = busy.run_traced(Duration::from_secs(1)).await;
14630 let rendered = trace.to_string();
14631 assert!(
14632 rendered.starts_with(&format!("{GAUGE}: non_zero(1) @"))
14633 && rendered.contains("; last value non_zero(1) @"),
14634 "{rendered}"
14635 );
14636
14637 let silent = Drain::start(Answer::Silent);
14638 let (_, _, trace) = silent.run_traced(Duration::from_secs(1)).await;
14639 assert_eq!(
14642 trace.to_string(),
14643 format!("{GAUGE}: probe_failed(no-answer) @1000ms; no value ever read")
14644 );
14645 assert_eq!(trace.probes, 1);
14646 }
14647
14648 #[test]
14650 fn a_gauge_that_is_not_a_count_is_busy_and_shown_as_sent() {
14651 let gauges = [GAUGE.to_string()];
14652 let verdicts = gauge_verdicts(Some(&serde_json::json!({ GAUGE: 0.0 })), &gauges);
14653 assert_eq!(verdicts, vec![GaugeVerdict::NotACount("0.0".to_string())]);
14654 assert_eq!(
14655 busy_gauge_observation(&verdicts),
14656 BusyGaugeObservation::Busy
14657 );
14658 let zero = gauge_verdicts(Some(&serde_json::json!({ GAUGE: 0 })), &gauges);
14659 assert_eq!(
14660 busy_gauge_observation(&zero),
14661 BusyGaugeObservation::Quiescent
14662 );
14663 let none = gauge_verdicts(None, &gauges);
14664 assert_eq!(busy_gauge_observation(&none), BusyGaugeObservation::Omitted);
14665 }
14666}
14667
14668#[cfg(test)]
14669mod health_evidence_tests {
14670 use super::{HealthProbeError, HealthProbeEvidence};
14671 use std::collections::HashSet;
14672
14673 #[test]
14681 fn only_a_dead_lane_is_proof_of_death() {
14682 assert!(HealthProbeError::lane_dead("gone").is_proof_of_death());
14683 assert!(!HealthProbeError::no_answer("timed out").is_proof_of_death());
14687 assert!(!HealthProbeError::bad_answer("garbage").is_proof_of_death());
14688 assert!(!HealthProbeError::misconfigured("no table").is_proof_of_death());
14689 }
14690
14691 #[test]
14697 fn every_evidence_class_has_a_distinct_label() {
14698 let labels = [
14699 HealthProbeError::lane_dead("").label(),
14700 HealthProbeError::no_answer("").label(),
14701 HealthProbeError::bad_answer("").label(),
14702 HealthProbeError::misconfigured("").label(),
14703 ];
14704 let unique: HashSet<_> = labels.iter().collect();
14705 assert_eq!(unique.len(), labels.len(), "labels collided: {labels:?}");
14706 }
14707
14708 #[test]
14714 fn classification_preserves_the_original_message() {
14715 let err = HealthProbeError::no_answer("module did not answer within 5s");
14716 assert_eq!(err.to_string(), "module did not answer within 5s");
14717 assert!(matches!(err.evidence, HealthProbeEvidence::NoAnswer));
14718 }
14719}
14720
14721#[cfg(test)]
14722mod health_tombstone_tests {
14723 use std::{path::PathBuf, sync::Arc, time::Duration};
14724
14725 use subc_protocol::{
14726 manifest::Concurrency,
14727 session::{HealthStatus, ModuleControlResponse},
14728 };
14729 use tokio::sync::mpsc;
14730
14731 use super::{
14732 probe_module_health, HealthAction, HealthConfig, HealthProbeEvidence, ModuleProtocol,
14733 ModuleSpec, RestartPolicy, Supervisor, SupervisorRuntimeConfig,
14734 };
14735 use crate::{
14736 control::ControlHandler,
14737 forwarding::{ForwardingTable, ModuleControlRpcCompletion, ModuleControlRpcOutcome},
14738 registry::{ConnectionId, Registry},
14739 router::FrameSink,
14740 };
14741
14742 struct ProbeHarness {
14743 spec: ModuleSpec,
14744 runtime: SupervisorRuntimeConfig,
14745 forwarding: Arc<ForwardingTable>,
14746 module_connection: ConnectionId,
14747 module_rx: mpsc::Receiver<crate::router::OutboundFrame>,
14748 handler: ControlHandler,
14749 module: super::SupervisedModule,
14750 }
14751
14752 fn probe_harness() -> ProbeHarness {
14753 let registry = Arc::new(Registry::default());
14754 let forwarding = Arc::new(ForwardingTable::default());
14755 let supervisor_handle = super::SupervisorHandle::new();
14756 let health = HealthConfig {
14757 http: None,
14758 cadence: Duration::from_secs(30),
14759 deadline: Duration::from_secs(5),
14760 failure_threshold: 3,
14761 on_degraded: HealthAction::Report,
14762 on_failing: HealthAction::Report,
14763 critical: false,
14764 };
14765 let supervisor = Supervisor::new_for_test(Arc::clone(®istry), RestartPolicy::default())
14766 .with_forwarding(Arc::clone(&forwarding))
14767 .with_handle(supervisor_handle.clone())
14768 .with_health_config(health);
14769 let spec = ModuleSpec {
14770 module_id: "late-health-module".to_string(),
14771 program: PathBuf::from("disabled-module"),
14772 args: Vec::new(),
14773 env: Vec::new(),
14774 reserved: false,
14775 reserved_prefixes: Vec::new(),
14776 protocol: ModuleProtocol::Subc,
14777 overlap: Default::default(),
14778 };
14779 let module = supervisor
14780 .supervise_configured(spec.clone(), false)
14781 .unwrap();
14782 let runtime = supervisor.runtime_config();
14783 let handler = ControlHandler::with_forwarding(registry, Arc::clone(&forwarding))
14784 .with_supervisor(supervisor_handle);
14785 let module_connection = ConnectionId::new(700);
14786 let (module_tx, module_rx) = mpsc::channel(8);
14787 forwarding
14788 .register_module_connection(
14789 module_connection,
14790 spec.module_id.clone(),
14791 subc_protocol::PROTOCOL_VERSION,
14792 Concurrency::ModuleManaged,
14793 FrameSink::new(module_tx),
14794 )
14795 .unwrap();
14796
14797 ProbeHarness {
14798 spec,
14799 runtime,
14800 forwarding,
14801 module_connection,
14802 module_rx,
14803 handler,
14804 module,
14805 }
14806 }
14807
14808 async fn finish_after(
14809 harness: &mut ProbeHarness,
14810 stall: Duration,
14811 ) -> ModuleControlRpcCompletion {
14812 assert!(stall > harness.runtime.health.deadline);
14813 let deadline = harness.runtime.health.deadline;
14814 let probe = probe_module_health(&harness.spec.module_id, &harness.runtime, None);
14815 let answer = async {
14816 let frame = harness.module_rx.recv().await.expect("health.check frame");
14817 tokio::time::advance(deadline).await;
14818 tokio::task::yield_now().await;
14819 tokio::time::advance(stall - deadline).await;
14820 harness
14821 .forwarding
14822 .complete_module_control_rpc(
14823 harness.module_connection,
14824 frame.header.corr,
14825 Some("health.check"),
14826 ModuleControlRpcOutcome::Response(ModuleControlResponse::HealthCheck {
14827 status: HealthStatus::Ok,
14828 detail: None,
14829 metrics: None,
14830 }),
14831 )
14832 .unwrap()
14833 };
14834 let (probe_result, completion) = tokio::join!(probe, answer);
14835 let err = probe_result.expect_err("probe must miss its deadline");
14836 assert!(matches!(err.evidence, HealthProbeEvidence::NoAnswer));
14837 completion
14838 }
14839
14840 async fn time_out_without_answer(harness: &mut ProbeHarness) {
14841 let deadline = harness.runtime.health.deadline;
14842 let probe = probe_module_health(&harness.spec.module_id, &harness.runtime, None);
14843 let exhaust_deadline = async {
14844 let _frame = harness.module_rx.recv().await.expect("health.check frame");
14845 tokio::time::advance(deadline).await;
14846 tokio::task::yield_now().await;
14847 };
14848 let (probe_result, ()) = tokio::join!(probe, exhaust_deadline);
14849 let err = probe_result.expect_err("probe must miss its deadline");
14850 assert!(matches!(err.evidence, HealthProbeEvidence::NoAnswer));
14851 }
14852
14853 async fn run_probe_cycle(harness: &mut ProbeHarness, answer: bool) {
14854 let registry = Arc::clone(&harness.module.inner.registry);
14855 let snapshot = Arc::clone(&harness.module.inner.snapshot);
14856 let process_liveness = super::SupervisorProcessLiveness::default();
14857 let mut child = None;
14858 let cycle = super::run_health_probe_cycle(
14859 &harness.spec,
14860 &harness.runtime,
14861 ®istry,
14862 &process_liveness,
14863 &snapshot,
14864 &mut child,
14865 );
14866 let peer = async {
14867 let frame = harness.module_rx.recv().await.expect("health.check frame");
14868 if answer {
14869 harness
14870 .forwarding
14871 .complete_module_control_rpc(
14872 harness.module_connection,
14873 frame.header.corr,
14874 Some("health.check"),
14875 ModuleControlRpcOutcome::Response(ModuleControlResponse::HealthCheck {
14876 status: HealthStatus::Ok,
14877 detail: None,
14878 metrics: Some(serde_json::json!({"ready": true})),
14879 }),
14880 )
14881 .unwrap();
14882 } else {
14883 tokio::time::advance(harness.runtime.health.deadline).await;
14884 tokio::task::yield_now().await;
14885 }
14886 };
14887 tokio::join!(cycle, peer);
14888 }
14889
14890 #[tokio::test(start_paused = true)]
14891 async fn unanswered_probe_is_unknown_until_threshold_and_ok_report_recovers() {
14892 let mut harness = probe_harness();
14893 let monitor = harness.module.inner.monitor.lock().unwrap().take().unwrap();
14897 monitor.abort();
14898 let _ = monitor.await;
14899 super::update_snapshot(&harness.module.inner.snapshot, None, |state| {
14900 state.enabled = true;
14901 state.state = super::ModuleState::Running;
14902 state.process_alive = true;
14903 })
14904 .unwrap();
14905
14906 run_probe_cycle(&mut harness, true).await;
14907 assert_eq!(
14908 harness.module.status().unwrap().health.status,
14909 super::SupervisorHealthStatus::Ok
14910 );
14911
14912 for failures in 1..harness.runtime.health.failure_threshold {
14913 run_probe_cycle(&mut harness, false).await;
14914 let status = harness.module.status().unwrap();
14915 assert_eq!(status.health.status, super::SupervisorHealthStatus::Unknown);
14916 assert_eq!(status.health.consecutive_failures, failures);
14917 assert!(status.health.last_probe_ms.is_some());
14918 assert!(status.health.detail.unwrap().starts_with("[no-answer]"));
14919 assert!(status.health.metrics.is_none());
14920 assert_eq!(status.state, super::ModuleState::Running);
14921 assert!(status.process_alive);
14922 assert_eq!(status.restart_count, 0);
14923 assert_eq!(status.lifetime_restarts, 0);
14924 assert!(status.health.last_action.is_none());
14925 assert!(harness.runtime.scheduled_respawn.lock().unwrap().is_none());
14926 }
14927
14928 run_probe_cycle(&mut harness, true).await;
14929 let recovered = harness.module.status().unwrap();
14930 assert_eq!(recovered.health.status, super::SupervisorHealthStatus::Ok);
14931 assert_eq!(recovered.health.consecutive_failures, 0);
14932 assert!(recovered.health.detail.is_none());
14933 assert_eq!(
14934 recovered.health.metrics,
14935 Some(serde_json::json!({"ready": true}))
14936 );
14937 assert_eq!(recovered.lifetime_restarts, 0);
14938
14939 for failures in 1..=harness.runtime.health.failure_threshold {
14940 run_probe_cycle(&mut harness, false).await;
14941 let status = harness.module.status().unwrap();
14942 assert_eq!(status.health.consecutive_failures, failures);
14943 if failures < harness.runtime.health.failure_threshold {
14944 assert_eq!(status.health.status, super::SupervisorHealthStatus::Unknown);
14945 assert_eq!(status.state, super::ModuleState::Running);
14946 assert_eq!(status.lifetime_restarts, 0);
14947 assert!(harness.runtime.scheduled_respawn.lock().unwrap().is_none());
14948 } else {
14949 assert_eq!(
14950 status.health.status,
14951 super::SupervisorHealthStatus::Unresponsive
14952 );
14953 assert_eq!(status.state, super::ModuleState::Restarting);
14954 assert_eq!(status.restart_count, 1);
14955 assert_eq!(status.lifetime_restarts, 1);
14956 assert_eq!(status.health.last_action.as_deref(), Some("restart"));
14957 assert!(harness.runtime.scheduled_respawn.lock().unwrap().is_some());
14958 }
14959 }
14960 }
14961
14962 #[tokio::test(start_paused = true)]
14963 async fn late_health_answers_record_start_anchored_latency_for_two_stalls() {
14964 let mut harness = probe_harness();
14965
14966 let first = finish_after(&mut harness, Duration::from_secs(8)).await;
14967 let first_latency = match &first {
14968 ModuleControlRpcCompletion::LateHealthAnswer { latency, .. } => *latency,
14969 other => panic!("late answer was not retained: {other:?}"),
14970 };
14971 assert!(harness.handler.observe_module_control_completion(first));
14972
14973 let second = finish_after(&mut harness, Duration::from_secs(11)).await;
14974 let second_latency = match &second {
14975 ModuleControlRpcCompletion::LateHealthAnswer { latency, .. } => *latency,
14976 other => panic!("late answer was not retained: {other:?}"),
14977 };
14978 assert!(harness.handler.observe_module_control_completion(second));
14979
14980 assert_eq!(first_latency, Duration::from_secs(8));
14981 assert_eq!(
14982 second_latency - first_latency,
14983 Duration::from_secs(3),
14984 "latency must grow linearly with the additional stall"
14985 );
14986 let health = harness.module.status().unwrap().health;
14987 assert_eq!(health.late_answer_count, 2);
14988 assert_eq!(health.last_late_answer_latency_ms, Some(11_000));
14989 }
14990
14991 #[tokio::test(start_paused = true)]
14999 async fn late_answer_clears_the_consecutive_failure_streak() {
15000 let mut harness = probe_harness();
15001
15002 time_out_without_answer(&mut harness).await;
15004 harness
15005 .module
15006 .record_health_probe_failure_for_test("[no-answer] test miss")
15007 .unwrap();
15008 assert_eq!(
15009 harness.module.status().unwrap().health.consecutive_failures,
15010 1,
15011 "precondition: the miss must be on the streak before the late answer"
15012 );
15013
15014 let late = finish_after(&mut harness, Duration::from_secs(9)).await;
15016 assert!(matches!(
15017 late,
15018 ModuleControlRpcCompletion::LateHealthAnswer { .. }
15019 ));
15020 assert!(harness.handler.observe_module_control_completion(late));
15021
15022 let health = harness.module.status().unwrap().health;
15023 assert_eq!(
15024 health.consecutive_failures, 0,
15025 "a late answer is an answer: the streak must reset"
15026 );
15027 assert_eq!(health.late_answer_count, 1);
15028 }
15029
15030 #[tokio::test(start_paused = true)]
15031 async fn repeated_serial_probe_cycles_keep_one_tombstone_per_endpoint() {
15032 let mut harness = probe_harness();
15033
15034 for _ in 0..20 {
15035 time_out_without_answer(&mut harness).await;
15036 assert_eq!(
15037 harness.forwarding.health_probe_tombstone_count().unwrap(),
15038 1
15039 );
15040 }
15041 }
15042}
15043
15044#[cfg(test)]
15045mod child_env_tests {
15046 #[cfg(windows)]
15047 use super::{
15048 clear_current_process_facts, observed_launch_nonce_source, ModuleState, SupervisorSnapshot,
15049 };
15050 #[cfg(windows)]
15051 use std::sync::{Arc, OnceLock};
15052 #[cfg(windows)]
15053 #[test]
15054 fn windows_wire_spawn_prepares_pipe_and_keeps_phase_one_env() {
15055 let spec = spec(Vec::new());
15056 let mut command = Command::new("unused");
15057 let handoff = apply_wire_spawn_args(&mut command, &spec, None, None)
15058 .unwrap()
15059 .unwrap();
15060 let name = command
15061 .as_std()
15062 .get_envs()
15063 .find(|(key, _)| *key == OsStr::new(subc_os::launch_nonce::LAUNCH_NONCE_PIPE_ENV))
15064 .and_then(|(_, value)| value)
15065 .unwrap();
15066 assert_eq!(name, OsStr::new(handoff.name()));
15067 assert!(handoff.name().starts_with(r"\\.\pipe\subc-launch-"));
15068 assert!(command.as_std().get_envs().any(|(key, value)| key
15069 == OsStr::new(subc_os::LAUNCH_NONCE_PIPE_FALLBACK_ENV)
15070 && value == Some(OsStr::new("env"))));
15071 assert!(command
15072 .as_std()
15073 .get_envs()
15074 .any(|(key, value)| key == OsStr::new(SUBC_LAUNCH_NONCE_ENV) && value.is_some()));
15075 }
15076
15077 #[cfg(windows)]
15078 #[test]
15079 fn windows_source_reporting_tracks_consumption_and_resets_per_spawn() {
15080 let mut snapshot = SupervisorSnapshot::new(ModuleState::Running, true);
15081 snapshot.process_alive = true;
15082 snapshot.spawned_protocol = Some(ModuleProtocol::Subc);
15083 assert_eq!(
15084 observed_launch_nonce_source(&snapshot).as_deref(),
15085 Some("env")
15086 );
15087 let consumed = Arc::new(OnceLock::new());
15088 snapshot.pipe_consumed = Some(consumed.clone());
15089 assert_eq!(
15090 observed_launch_nonce_source(&snapshot).as_deref(),
15091 Some("env")
15092 );
15093 consumed.set(()).unwrap();
15094 assert_eq!(
15095 observed_launch_nonce_source(&snapshot).as_deref(),
15096 Some("pipe")
15097 );
15098 snapshot.pipe_consumed = Some(Arc::new(OnceLock::new()));
15099 assert_eq!(
15100 observed_launch_nonce_source(&snapshot).as_deref(),
15101 Some("env")
15102 );
15103 clear_current_process_facts(&mut snapshot);
15104 assert_eq!(observed_launch_nonce_source(&snapshot), None);
15105 }
15106 use super::{
15107 apply_child_env, apply_spawn_role, apply_wire_spawn_args, LaunchSpecDigest, ModuleProtocol,
15108 ModuleSpec, SpawnRole, SupervisorHandle, SPAWN_ROLE_SWAP_CANDIDATE, SUBC_ARG,
15109 SUBC_LAUNCH_NONCE_ENV, SUBC_MODULE_ID_ENV, SUBC_SPAWN_ROLE_ENV,
15110 };
15111 use std::{ffi::OsStr, path::PathBuf};
15112 use tokio::process::Command;
15113
15114 fn spec(env: Vec<(String, String)>) -> ModuleSpec {
15115 ModuleSpec {
15116 module_id: "env-plan".to_string(),
15117 program: PathBuf::from("/nonexistent"),
15118 args: Vec::new(),
15119 env,
15120 reserved: false,
15121 reserved_prefixes: Vec::new(),
15122 protocol: ModuleProtocol::Subc,
15123 overlap: Default::default(),
15124 }
15125 }
15126
15127 #[test]
15128 fn configured_launch_spec_digest_is_canonical_and_excludes_injected_inputs() {
15129 let base = spec(vec![("A".into(), "B=C".into()), ("Z".into(), "v".into())]);
15130 let digest = LaunchSpecDigest::from_spec(&base);
15131 let mut reordered = base.clone();
15132 reordered.env.reverse();
15133 reordered.module_id = "different-id".into();
15134 reordered.reserved = true;
15135 reordered.overlap = Default::default();
15136 assert_eq!(digest, LaunchSpecDigest::from_spec(&reordered));
15137 let mut delimiter_collision = base.clone();
15138 delimiter_collision.env[0] = ("A=B".into(), "C".into());
15139 assert_ne!(digest, LaunchSpecDigest::from_spec(&delimiter_collision));
15140 let mut changed = base.clone();
15141 changed.protocol = ModuleProtocol::None;
15142 assert_ne!(digest, LaunchSpecDigest::from_spec(&changed));
15143 changed = base.clone();
15144 changed.args.push("--flag".into());
15145 assert_ne!(digest, LaunchSpecDigest::from_spec(&changed));
15146 changed = base.clone();
15147 changed.program = PathBuf::from("/other");
15148 assert_ne!(digest, LaunchSpecDigest::from_spec(&changed));
15149 changed = base.clone();
15150 changed.env[1].1 = "other".into();
15151 assert_ne!(digest, LaunchSpecDigest::from_spec(&changed));
15152 assert!(!format!("{digest:?}").contains("B=C"));
15153 assert!(!format!("{digest:?}").contains("[u8"));
15154 }
15155
15156 #[cfg(unix)]
15157 #[test]
15158 fn native_program_bytes_do_not_collapse_through_lossy_utf8() {
15159 use std::os::unix::ffi::OsStringExt;
15160 let mut left = spec(Vec::new());
15161 left.program = std::ffi::OsString::from_vec(vec![b'/', 0xff]).into();
15162 let mut right = left.clone();
15163 right.program = std::ffi::OsString::from_vec(vec![b'/', 0xfe]).into();
15164 assert_eq!(
15165 left.program.to_string_lossy(),
15166 right.program.to_string_lossy()
15167 );
15168 assert_ne!(
15169 LaunchSpecDigest::from_spec(&left),
15170 LaunchSpecDigest::from_spec(&right)
15171 );
15172 }
15173
15174 #[test]
15188 fn ambient_ck_log_is_removed_and_a_configured_one_survives() {
15189 let mut command = Command::new("/nonexistent");
15190 apply_child_env(&mut command, &spec(Vec::new()));
15191 let removed = command
15192 .as_std()
15193 .get_envs()
15194 .any(|(key, value)| key == OsStr::new("CK_LOG") && value.is_none());
15195 assert!(
15196 removed,
15197 "ambient CK_LOG must be explicitly removed for an unconfigured module"
15198 );
15199
15200 let mut configured = Command::new("/nonexistent");
15201 apply_child_env(
15202 &mut configured,
15203 &spec(vec![("CK_LOG".to_string(), "debug".to_string())]),
15204 );
15205 let effective = configured
15206 .as_std()
15207 .get_envs()
15208 .filter(|(key, _)| *key == OsStr::new("CK_LOG"))
15209 .last()
15210 .map(|(_, value)| value.map(|v| v.to_string_lossy().into_owned()));
15211 assert_eq!(
15212 effective,
15213 Some(Some("debug".to_string())),
15214 "a module's configured CK_LOG must survive the ambient removal"
15215 );
15216 }
15217
15218 #[test]
15227 fn protocol_none_spawn_carries_no_subc_argument_and_no_nonce() {
15228 let connection_file = std::path::Path::new("/run/subc-connection.json");
15229 let handle = SupervisorHandle::new();
15230
15231 let mut none_spec = spec(Vec::new());
15232 none_spec.protocol = ModuleProtocol::None;
15233 let mut none = Command::new("/nonexistent");
15234 let none_handoff =
15235 apply_wire_spawn_args(&mut none, &none_spec, Some(connection_file), Some(&handle))
15236 .expect("protocol-none spawn args apply");
15237 assert!(
15238 none_handoff.is_none(),
15239 "protocol:none spawn must not receive a nonce descriptor"
15240 );
15241 #[cfg(windows)]
15242 assert!(!none.as_std().get_envs().any(|(key, value)| key
15243 == OsStr::new(subc_os::LAUNCH_NONCE_PIPE_FALLBACK_ENV)
15244 && value.is_some()));
15245 assert!(
15246 !none.as_std().get_envs().any(|(key, value)| key
15247 == OsStr::new(subc_os::LAUNCH_NONCE_FD_ENV)
15248 && value.is_some()),
15249 "protocol:none spawn must not name a nonce descriptor"
15250 );
15251 let none_args: Vec<String> = none
15252 .as_std()
15253 .get_args()
15254 .map(|a| a.to_string_lossy().into_owned())
15255 .collect();
15256 assert!(
15257 !none_args.iter().any(|a| a == SUBC_ARG),
15258 "protocol:none argv must not carry --subc; got {none_args:?}"
15259 );
15260 let none_has_nonce = none
15261 .as_std()
15262 .get_envs()
15263 .any(|(key, value)| key == OsStr::new(SUBC_LAUNCH_NONCE_ENV) && value.is_some());
15264 assert!(
15265 !none_has_nonce,
15266 "protocol:none spawn must not receive a launch nonce"
15267 );
15268 let none_has_module_id = none
15269 .as_std()
15270 .get_envs()
15271 .any(|(key, value)| key == OsStr::new(SUBC_MODULE_ID_ENV) && value.is_some());
15272 assert!(
15273 none_has_module_id,
15274 "SUBC_MODULE_ID is inert and stays on every path"
15275 );
15276 assert!(
15277 handle.spawn_nonce(&none_spec.module_id).is_none(),
15278 "no nonce record for a process that will never present one"
15279 );
15280
15281 let wire_spec = spec(Vec::new());
15283 let mut wire = Command::new("/nonexistent");
15284 let wire_handoff =
15285 apply_wire_spawn_args(&mut wire, &wire_spec, Some(connection_file), Some(&handle))
15286 .expect("subc-wire spawn args apply");
15287 let wire_fd_env = wire
15288 .as_std()
15289 .get_envs()
15290 .find(|(key, _)| *key == OsStr::new(subc_os::LAUNCH_NONCE_FD_ENV))
15291 .and_then(|(_, value)| value.map(|v| v.to_string_lossy().into_owned()));
15292 #[cfg(unix)]
15293 assert_eq!(
15294 wire_fd_env,
15295 Some(wire_handoff.expect("a descriptor handoff").fd_env_value()),
15296 "a subc-wire spawn names the pipe it will receive at descriptor 3"
15297 );
15298 #[cfg(windows)]
15299 assert!(wire_handoff.is_some() && wire_fd_env.is_none());
15300 #[cfg(not(any(unix, windows)))]
15301 assert!(wire_handoff.is_none() && wire_fd_env.is_none());
15302 let wire_args: Vec<String> = wire
15303 .as_std()
15304 .get_args()
15305 .map(|a| a.to_string_lossy().into_owned())
15306 .collect();
15307 assert_eq!(
15308 wire_args,
15309 vec![
15310 SUBC_ARG.to_string(),
15311 connection_file.to_string_lossy().into_owned()
15312 ],
15313 "a subc-wire spawn still carries --subc <path>"
15314 );
15315 assert_eq!(
15316 wire.as_std()
15317 .get_envs()
15318 .any(|(key, value)| key == OsStr::new(SUBC_LAUNCH_NONCE_ENV) && value.is_some()),
15319 !cfg!(unix),
15320 "only Windows supplies the environment nonce"
15321 );
15322 assert!(handle.spawn_nonce(&wire_spec.module_id).is_some());
15323 }
15324
15325 #[test]
15335 fn plain_spawn_removes_the_spawn_role_even_when_the_spec_sets_it() {
15336 let role = |command: &Command| {
15337 command
15338 .as_std()
15339 .get_envs()
15340 .filter(|(key, _)| *key == OsStr::new(SUBC_SPAWN_ROLE_ENV))
15341 .last()
15342 .map(|(_, value)| value.map(|v| v.to_string_lossy().into_owned()))
15343 };
15344 let forged = spec(vec![(
15345 SUBC_SPAWN_ROLE_ENV.to_string(),
15346 SPAWN_ROLE_SWAP_CANDIDATE.to_string(),
15347 )]);
15348
15349 let mut plain = Command::new("/nonexistent");
15350 apply_child_env(&mut plain, &forged);
15351 apply_spawn_role(&mut plain, SpawnRole::Plain);
15352 assert_eq!(
15353 role(&plain),
15354 Some(None),
15355 "a plain spawn must remove SUBC_SPAWN_ROLE, whatever the spec says"
15356 );
15357
15358 let mut candidate = Command::new("/nonexistent");
15359 apply_child_env(&mut candidate, &spec(Vec::new()));
15360 apply_spawn_role(&mut candidate, SpawnRole::SwapCandidate);
15361 assert_eq!(
15362 role(&candidate),
15363 Some(Some(SPAWN_ROLE_SWAP_CANDIDATE.to_string()))
15364 );
15365 }
15366
15367 #[test]
15373 fn daemon_private_capture_keys_are_not_passed_to_the_child() {
15374 let mut command = Command::new("/nonexistent");
15375 apply_child_env(
15376 &mut command,
15377 &spec(vec![
15378 (super::CAPTURE_KEEP_ENV.to_string(), "5".to_string()),
15379 ("KEPT".to_string(), "yes".to_string()),
15380 ]),
15381 );
15382 let keys: Vec<String> = command
15383 .as_std()
15384 .get_envs()
15385 .filter(|(_, value)| value.is_some())
15386 .map(|(key, _)| key.to_string_lossy().into_owned())
15387 .collect();
15388 assert!(keys.contains(&"KEPT".to_string()), "got {keys:?}");
15389 assert!(
15390 !keys.contains(&super::CAPTURE_KEEP_ENV.to_string()),
15391 "daemon-private capture key leaked to the child: {keys:?}"
15392 );
15393 }
15394}
15395
15396#[cfg(test)]
15397mod jitter_tests {
15398 use super::jittered_health_delay;
15399 use std::{collections::HashSet, time::Duration};
15400
15401 const FLEET: [&str; 14] = [
15410 "aft",
15411 "alfonso-core",
15412 "magic-context",
15413 "broca",
15414 "thalamus",
15415 "quota",
15416 "engram",
15417 "plexus",
15418 "cerebellum",
15419 "astrocyte",
15420 "synapse",
15421 "subc-mcp",
15422 "cortexkit-credentials",
15423 "subc-federation",
15424 ];
15425
15426 #[test]
15434 fn probe_delays_disperse_across_the_fleet() {
15435 let cadence = Duration::from_secs(30);
15436 let delays: HashSet<Duration> = FLEET
15437 .iter()
15438 .map(|id| jittered_health_delay(id, 0, cadence))
15439 .collect();
15440 assert_eq!(
15441 delays.len(),
15442 FLEET.len(),
15443 "every supervised module must land on its own probe offset"
15444 );
15445 }
15446
15447 #[test]
15453 fn jitter_only_delays_and_stays_within_one_tenth_of_cadence() {
15454 let cadence = Duration::from_secs(30);
15455 let span = cadence / 10;
15456 for id in FLEET {
15457 for probe_index in 0..8 {
15458 let delay = jittered_health_delay(id, probe_index, cadence);
15459 assert!(
15460 delay >= cadence,
15461 "{id}#{probe_index}: jitter must not shorten the cadence"
15462 );
15463 assert!(
15464 delay < cadence + span,
15465 "{id}#{probe_index}: jitter must stay inside one tenth of the cadence"
15466 );
15467 }
15468 }
15469 }
15470
15471 #[test]
15477 fn a_module_offset_is_stable_across_restarts() {
15478 let cadence = Duration::from_secs(30);
15479 for id in FLEET {
15480 assert_eq!(
15481 jittered_health_delay(id, 0, cadence),
15482 jittered_health_delay(id, 0, cadence),
15483 "{id}: the same module and probe index must produce the same offset"
15484 );
15485 }
15486 }
15487
15488 #[test]
15490 fn zero_cadence_yields_zero_delay() {
15491 assert_eq!(
15492 jittered_health_delay("aft", 0, Duration::ZERO),
15493 Duration::ZERO
15494 );
15495 }
15496}
15497
15498#[cfg(all(test, target_os = "linux"))]
15499mod cgroup_placement_tests {
15500 use super::{
15501 apply_cgroup_placement, remove_module_cgroup, LaunchSpecDigest, ModuleProtocol, ModuleSpec,
15502 SuperviseError, SupervisedChild,
15503 };
15504 use crate::stderr_tail::{StderrRing, StderrTailConfig};
15505 use cortexkit_test_support::ScratchDir;
15506 use std::{
15507 fs, io,
15508 path::{Path, PathBuf},
15509 sync::{Arc, Mutex},
15510 };
15511 use tokio::process::Command;
15512
15513 #[tokio::test]
15514 async fn unique_spawn_cgroups_clean_up_across_restarts_and_shutdown() {
15515 use super::*;
15516 let dir = ScratchDir::new("unique-spawn-cgroups");
15517 let root = PathBuf::from(format!(
15518 "/sys/fs/cgroup/subc-unique-test-{}-{}",
15519 std::process::id(),
15520 unix_ms_now()
15521 ));
15522 if let Err(error) = fs::create_dir(&root) {
15523 assert!(
15524 std::env::var_os("SUBC_REQUIRE_CGROUP_TEST").is_none(),
15525 "required cgroup test cannot execute: {error}"
15526 );
15527 eprintln!("SKIP unique_spawn_cgroups_clean_up_across_restarts_and_shutdown: {error}");
15528 return;
15529 }
15530 let placement = subc_cgroup::prepare_at(&root).unwrap().unwrap();
15531 let group_count = || {
15532 fs::read_dir(root.join("subc-modules"))
15533 .unwrap()
15534 .map(|entry| entry.unwrap().file_type().unwrap())
15535 .filter(|kind| kind.is_dir())
15536 .count()
15537 };
15538 let supervisor =
15539 Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default())
15540 .with_cgroup_placement(Some(placement.clone()));
15541 let runtime = supervisor.runtime_config();
15542 let mut spec = ModuleSpec {
15543 module_id: "unique-spawn".into(),
15544 program: PathBuf::from("/bin/sleep"),
15545 args: vec!["60".into()],
15546 env: ["XDG_DATA_HOME", "XDG_RUNTIME_DIR", "XDG_CONFIG_HOME"]
15547 .into_iter()
15548 .map(|key| (key.into(), dir.to_string_lossy().into_owned()))
15549 .collect(),
15550 reserved: false,
15551 reserved_prefixes: vec![],
15552 protocol: ModuleProtocol::None,
15553 overlap: Default::default(),
15554 };
15555 let spawn = |spec: &ModuleSpec| {
15556 spawn_child(
15557 spec,
15558 None,
15559 None,
15560 &runtime.stderr_ring,
15561 None,
15562 &runtime.child_roster,
15563 Some(&placement),
15564 )
15565 .unwrap()
15566 };
15567 let mut live = spawn(&spec);
15568 for _ in 0..3 {
15569 let next = spawn(&spec);
15571 assert_ne!(live.module_id, next.module_id);
15572 live.start_kill().unwrap();
15573 live.wait().await.unwrap();
15574 live = next;
15575 assert!(
15576 live.child.try_wait().unwrap().is_none(),
15577 "retiring the old slot must not kill the replacement"
15578 );
15579 assert_eq!(
15580 group_count(),
15581 1,
15582 "only the live spawn's cgroup should remain"
15583 );
15584 }
15585 supervisor.begin_daemon_shutdown();
15586 let reap = tokio::spawn(async move {
15587 live.wait().await.unwrap();
15588 });
15589 supervisor
15590 .end_children_for_daemon_shutdown(false, std::future::pending())
15591 .await;
15592 reap.await.unwrap();
15593 assert_eq!(group_count(), 0);
15594 spec.program = PathBuf::from("/bin/true");
15596 spec.args.clear();
15597 let fresh_roster = ChildRoster::default();
15598 let mut short = spawn_child(
15599 &spec,
15600 None,
15601 None,
15602 &runtime.stderr_ring,
15603 None,
15604 &fresh_roster,
15605 Some(&placement),
15606 )
15607 .unwrap();
15608 short.wait().await.unwrap();
15609 assert_eq!(group_count(), 0);
15610 spec.module_id = "_".repeat(255);
15611 let mut long_id = spawn_child(
15612 &spec,
15613 None,
15614 None,
15615 &runtime.stderr_ring,
15616 None,
15617 &fresh_roster,
15618 Some(&placement),
15619 )
15620 .unwrap();
15621 long_id.wait().await.unwrap();
15622 assert_eq!(
15623 group_count(),
15624 0,
15625 "valid long module IDs must not exceed cgroup NAME_MAX"
15626 );
15627 fs::remove_dir(root.join("subc-modules")).unwrap();
15628 fs::remove_dir(root).unwrap();
15629 eprintln!("EXECUTED unique_spawn_cgroups_clean_up_across_restarts_and_shutdown");
15630 }
15631
15632 #[test]
15633 fn failed_parent_cgroup_open_is_a_cgroup_supervision_error() {
15634 let path = Path::new("/definitely-missing-subc-cgroup");
15635 let mut command = Command::new("true");
15636 let error = apply_cgroup_placement(
15637 &mut command,
15638 &ModuleSpec {
15639 module_id: "broken-cgroup".to_string(),
15640 program: PathBuf::from("true"),
15641 args: Vec::new(),
15642 env: Vec::new(),
15643 reserved: false,
15644 reserved_prefixes: Vec::new(),
15645 protocol: ModuleProtocol::Subc,
15646 overlap: Default::default(),
15647 },
15648 path,
15649 )
15650 .expect_err("a parent cgroup open failure must reject the supervised spawn");
15651 let reason = error.to_string();
15652
15653 assert!(
15654 matches!(error, SuperviseError::Cgroup { .. }),
15655 "parent cgroup open must be reported as a cgroup supervision error: {reason}"
15656 );
15657 assert!(
15658 reason.contains("/definitely-missing-subc-cgroup/cgroup.procs"),
15659 "parent cgroup open failure must name cgroup.procs: {reason}"
15660 );
15661 }
15662
15663 #[tokio::test]
15664 async fn reaping_a_child_removes_its_empty_module_cgroup() {
15665 let root = ScratchDir::new("supervisor-reap-cgroup");
15666 fs::write(root.join("cgroup.procs"), b"").expect("write scratch cgroup marker");
15667 let placement = subc_cgroup::prepare_at(&root)
15668 .expect("prepare scratch cgroup root")
15669 .expect("scratch root has a cgroup.procs marker");
15670 let module_id = "reaped-module";
15671 let module = placement
15672 .module_path(module_id)
15673 .expect("create scratch module cgroup");
15674 let child = Command::new("true")
15675 .env("XDG_DATA_HOME", root.path())
15676 .env("XDG_RUNTIME_DIR", root.path())
15677 .env("XDG_CONFIG_HOME", root.path())
15678 .spawn()
15679 .expect("spawn short-lived child");
15680 let pid = child.id().expect("spawned child has pid");
15681 let mut child = SupervisedChild {
15682 child,
15683 exec: crate::activity::ExecOwner(Arc::new(crate::activity::ExecOutcome::new())),
15684 protocol: ModuleProtocol::Subc,
15685 module_id: module_id.to_string(),
15686 cgroup_placement: Some(placement),
15687 stdout_pump: None,
15688 stderr_pump: None,
15689 stderr_ring: Arc::new(Mutex::new(StderrRing::new(StderrTailConfig::default()))),
15690 spawned_at: tokio::time::Instant::now(),
15691 spawned_at_ms: 0,
15692 spawned_from: PathBuf::from("true"),
15693 launch_spec_digest: LaunchSpecDigest([0; 32]),
15694 spawned_file_identity: None,
15695 process_start_time: None,
15696 process_identity: None,
15697 pid,
15698 roster_guard: None,
15699 #[cfg(target_os = "macos")]
15700 privacy_exec: None,
15701 spawn_failure: None,
15702 };
15703
15704 child.wait().await.expect("reap short-lived child");
15705
15706 assert!(
15707 !module.exists(),
15708 "reaping the supervised child must remove its empty cgroup"
15709 );
15710 }
15711
15712 #[test]
15713 fn non_empty_cgroup_removal_is_reported_without_blocking_teardown() {
15714 let root = ScratchDir::new("supervisor-non-empty-cgroup");
15715 fs::write(root.join("cgroup.procs"), b"").expect("write scratch cgroup marker");
15716 let placement = subc_cgroup::prepare_at(&root)
15717 .expect("prepare scratch cgroup root")
15718 .expect("scratch root has a cgroup.procs marker");
15719 let module = placement
15720 .module_path("surviving-module")
15721 .expect("create scratch module cgroup");
15722 fs::write(module.join("surviving-process"), b"still present")
15723 .expect("make scratch cgroup non-empty");
15724 let (logs, _guard) = crate::router::test_log::log_capture(tracing::Level::WARN);
15725
15726 remove_module_cgroup(&placement, "surviving-module");
15727
15728 let logs = crate::router::test_log::captured_logs(&logs);
15729 assert!(
15730 module.exists(),
15731 "failed removal must leave the cgroup intact"
15732 );
15733 assert!(
15734 logs.contains("could not remove module cgroup after process exit; continuing teardown")
15735 && logs.contains("surviving-module"),
15736 "best-effort removal must report the failure without returning it: {logs}"
15737 );
15738 }
15739
15740 #[test]
15741 fn cgroup_pre_exec_spawn_failure_names_the_cgroup_path() {
15742 let cgroup_path = PathBuf::from("/sys/fs/cgroup/subc-modules/broken-module");
15743 let reason = SuperviseError::Spawn {
15744 program: PathBuf::from("/bin/true"),
15745 source: io::Error::from_raw_os_error(13),
15746 cgroup_path: Some(cgroup_path.clone()),
15747 }
15748 .to_string();
15749
15750 assert!(
15751 reason.contains(&cgroup_path.display().to_string()),
15752 "a pre_exec spawn failure must name the cgroup path: {reason}"
15753 );
15754 }
15755}
15756
15757#[cfg(test)]
15758mod spawn_subscriber_lag_tests {
15759 use super::*;
15760
15761 #[tokio::test]
15766 async fn lagged_spawn_subscriber_receives_a_terminal_lagged_error_after_its_queued_frames() {
15767 let feed = SpawnEventFeed::default();
15768 feed.configure_incarnation("lag-incarnation".to_string());
15769 let (tx, mut rx) = mpsc::channel(1);
15772 feed.subscribe(ConnectionId::new(1), 7, 1, None, FrameSink::new(tx))
15773 .expect("subscribe");
15774 let emitted = SPAWN_SUBSCRIBER_BUFFER + 16;
15775 for index in 0..emitted {
15776 feed.emit_spawned(&format!("lag-module-{index}"), 1000, 0);
15777 tokio::task::yield_now().await;
15780 }
15781 assert_eq!(
15782 feed.subscriber_count(),
15783 0,
15784 "the lagged subscriber must be removed"
15785 );
15786
15787 let mut data = Vec::new();
15788 let mut last = None;
15789 loop {
15790 let next = tokio::time::timeout(Duration::from_secs(5), rx.recv())
15791 .await
15792 .expect("the forwarder must finish once the subscriber is dropped");
15793 let Some(outbound) = next else { break };
15794 let frame = outbound.frame;
15795 if frame.header.ty == FrameType::StreamData {
15796 assert!(last.is_none(), "no data may follow the terminal frame");
15797 let event: SpawnEvent = serde_json::from_slice(&frame.body).unwrap();
15798 data.push(event.cursor.seq);
15799 } else {
15800 assert!(last.is_none(), "exactly one terminal frame");
15801 last = Some(frame);
15802 }
15803 }
15804 assert!(!data.is_empty(), "queued frames drain before the terminal");
15805 for pair in data.windows(2) {
15806 assert_eq!(
15807 pair[1],
15808 pair[0] + 1,
15809 "queued frames arrive dense and in order"
15810 );
15811 }
15812 let terminal = last.expect("a lagged subscriber must receive a terminal frame");
15813 assert_eq!(terminal.header.ty, FrameType::Error);
15814 assert_eq!(terminal.header.corr, 7);
15815 let body: subc_protocol::ErrorBody = serde_json::from_slice(&terminal.body).unwrap();
15816 assert_eq!(body.code, SPAWN_SUBSCRIBER_LAGGED_CODE);
15817 let detail = body.detail.expect("lagged error carries detail");
15818 assert_eq!(
15819 detail["first_undelivered_cursor"]["seq"],
15820 data.last().unwrap() + 1,
15821 "the named cursor is the first event the subscriber did not receive"
15822 );
15823 assert_eq!(
15824 detail["first_undelivered_cursor"]["daemon_incarnation"],
15825 "lag-incarnation"
15826 );
15827 }
15828}
15829
15830#[cfg(test)]
15831mod terminal_history_read_concurrency_tests {
15832 use super::*;
15833 use crate::terminal_journal::read_pause;
15834 use cortexkit_test_support::ScratchDir;
15835 use std::sync::mpsc as std_mpsc;
15836
15837 fn journaled_ring(
15838 journal: &Arc<crate::terminal_journal::TerminalJournal>,
15839 ) -> Arc<Mutex<TerminalRing>> {
15840 Arc::new(Mutex::new(
15841 TerminalRing::new(TerminalRingConfig::default(), 1)
15842 .with_journal(Some(Arc::clone(journal))),
15843 ))
15844 }
15845
15846 fn crash(at_ms: u64) -> ExitReport {
15847 ExitReport {
15848 kind: ExitKind::Crash,
15849 code: Some(1),
15850 signal: None,
15851 at_ms,
15852 }
15853 }
15854
15855 fn record_within(
15858 module_id: &'static str,
15859 ring: &Arc<Mutex<TerminalRing>>,
15860 at_ms: u64,
15861 bound: Duration,
15862 ) -> bool {
15863 let ring = Arc::clone(ring);
15864 let (done, done_rx) = std_mpsc::channel();
15865 std::thread::spawn(move || {
15866 record_terminal(
15867 module_id,
15868 &ring,
15869 &SpawnEventFeed::default(),
15870 &crash(at_ms),
15871 TerminalDisposition::Restarting,
15872 );
15873 let _ = done.send(());
15874 });
15875 done_rx.recv_timeout(bound).is_ok()
15876 }
15877
15878 #[test]
15883 fn exits_recorded_during_a_paused_history_read_are_not_blocked_or_half_merged() {
15884 let dir = ScratchDir::new("terminal-history-concurrent-read");
15885 let path = dir.join("terminals.jsonl");
15886 let journal = Arc::new(crate::terminal_journal::TerminalJournal::open(
15887 path.clone(),
15888 "daemon".into(),
15889 ));
15890 let reader_ring = journaled_ring(&journal);
15891 let other_ring = journaled_ring(&journal);
15892 assert!(record_within(
15893 "reader-module",
15894 &reader_ring,
15895 10,
15896 Duration::from_secs(5)
15897 ));
15898
15899 let (started, release) = read_pause::install(&path);
15900 let reading = {
15901 let ring = Arc::clone(&reader_ring);
15902 std::thread::spawn(move || durable_terminal_history_of(&ring, "reader-module"))
15903 };
15904 started
15905 .recv_timeout(Duration::from_secs(5))
15906 .expect("the history read reached its pause");
15907
15908 let bound = Duration::from_secs(1);
15909 assert!(
15910 record_within("other-module", &other_ring, 20, bound),
15911 "another module's exit waited on a history read (journal writer held)"
15912 );
15913 assert!(
15914 record_within("reader-module", &reader_ring, 30, bound),
15915 "the read module's own exit waited on its history read (ring held)"
15916 );
15917
15918 drop(release);
15919 let paused = reading.join().unwrap();
15920 assert_eq!(
15921 paused.entries.iter().map(|e| e.at_ms).collect::<Vec<_>>(),
15922 vec![10],
15923 "an exit recorded after the read began lands in neither half of it"
15924 );
15925 assert_eq!(paused.journal_skipped_lines, 0);
15926 assert_eq!(paused.journal_read_errors, 0);
15927
15928 let after = durable_terminal_history_of(&reader_ring, "reader-module");
15929 assert_eq!(
15930 after.entries.iter().map(|e| e.at_ms).collect::<Vec<_>>(),
15931 vec![10, 30],
15932 "the next read merges ring and journal with no duplicate"
15933 );
15934 assert_eq!(after.journal_skipped_lines, 0);
15935 }
15936}
15937
15938#[cfg(test)]
15943mod stderr_settle_tests {
15944 use std::{
15945 future::Future,
15946 io,
15947 pin::Pin,
15948 sync::{Arc, Mutex},
15949 task::{Context, Poll},
15950 time::Duration,
15951 };
15952
15953 use tokio::{
15954 io::{AsyncRead, ReadBuf},
15955 sync::oneshot,
15956 time::Instant,
15957 };
15958
15959 use super::{settle_stderr_pump, settle_stdout_pump, StderrPump};
15960 use crate::stderr_tail::{
15961 pump_stderr_to, pump_stdout_to, split_capture_stamp, untimed, CaptureState,
15962 ChildOutputSink, OutputSink, StderrRing, StderrTailConfig, TailEntry,
15963 };
15964
15965 const BOUND: Duration = Duration::from_millis(250);
15966
15967 struct HeldReader {
15971 before: Option<Vec<u8>>,
15972 gate: Option<oneshot::Receiver<()>>,
15973 after: io::Cursor<Vec<u8>>,
15974 }
15975
15976 impl AsyncRead for HeldReader {
15977 fn poll_read(
15978 mut self: Pin<&mut Self>,
15979 cx: &mut Context<'_>,
15980 buf: &mut ReadBuf<'_>,
15981 ) -> Poll<io::Result<()>> {
15982 if let Some(bytes) = self.before.take() {
15983 buf.put_slice(&bytes);
15984 return Poll::Ready(Ok(()));
15985 }
15986 if let Some(gate) = self.gate.as_mut() {
15987 match Pin::new(gate).poll(cx) {
15988 Poll::Pending => return Poll::Pending,
15989 Poll::Ready(_) => self.gate = None,
15990 }
15991 }
15992 Pin::new(&mut self.after).poll_read(cx, buf)
15993 }
15994 }
15995
15996 struct DiscardSink;
15997
15998 impl OutputSink for DiscardSink {
15999 fn write_line(&mut self, _line: &[u8]) {}
16000 }
16001
16002 fn line(text: &str) -> TailEntry {
16003 TailEntry::Line {
16004 text: text.to_string(),
16005 truncated: false,
16006 at_ms: None,
16007 }
16008 }
16009
16010 fn lock(ring: &Arc<Mutex<StderrRing>>) -> std::sync::MutexGuard<'_, StderrRing> {
16011 ring.lock().unwrap()
16012 }
16013
16014 fn held_pump(
16018 ring: &Arc<Mutex<StderrRing>>,
16019 before: &str,
16020 after: &str,
16021 ) -> (StderrPump, oneshot::Sender<()>) {
16022 let generation = lock(ring).begin_process();
16023 let (release, gate) = oneshot::channel();
16024 let reader = HeldReader {
16025 before: Some(before.as_bytes().to_vec()),
16026 gate: Some(gate),
16027 after: io::Cursor::new(after.as_bytes().to_vec()),
16028 };
16029 let task = tokio::spawn(pump_stderr_to(
16030 reader,
16031 Arc::clone(ring),
16032 generation,
16033 DiscardSink,
16034 ));
16035 (StderrPump { task, generation }, release)
16036 }
16037
16038 async fn wait_until(ring: &Arc<Mutex<StderrRing>>, done: impl Fn(&StderrRing) -> bool) {
16039 for _ in 0..1000 {
16040 if done(&lock(ring)) {
16041 return;
16042 }
16043 tokio::time::sleep(Duration::from_millis(1)).await;
16044 }
16045 panic!(
16046 "ring never reached the expected state: {:?}",
16047 lock(ring).snapshot(None, None)
16048 );
16049 }
16050
16051 #[tokio::test(start_paused = true)]
16052 async fn a_crash_line_the_reader_had_not_reached_by_the_bound_is_kept_before_the_restart() {
16053 let ring = Arc::new(Mutex::new(StderrRing::new(StderrTailConfig::default())));
16054 let (pump, release) = held_pump(&ring, "booting\n", "config error: missing storage\n");
16055
16056 settle_stderr_pump("crasher", &ring, pump, BOUND).await;
16057 let before_release = lock(&ring).snapshot(None, None);
16058 assert!(
16059 matches!(before_release.capture, CaptureState::Incomplete { .. }),
16060 "a reader that has not reached EOF cannot claim a whole tail: {before_release:?}"
16061 );
16062
16063 let next = lock(&ring).begin_process();
16066 lock(&ring).push_line_from(next, "next process booting");
16067 release.send(()).unwrap();
16068 wait_until(&ring, |ring| {
16069 ring.snapshot(None, None).capture == CaptureState::Captured
16070 })
16071 .await;
16072
16073 assert_eq!(
16074 untimed(lock(&ring).snapshot(None, None).entries),
16075 vec![
16076 line("booting"),
16077 line("config error: missing storage"),
16078 TailEntry::ProcessStart,
16079 line("next process booting"),
16080 ],
16081 "the crash's last line must survive a slow reader and stay in the crashed process's section"
16082 );
16083 }
16084
16085 #[tokio::test(start_paused = true)]
16086 async fn a_pipe_held_open_by_a_descendant_reads_incomplete_without_delaying_the_restart_past_the_bound(
16087 ) {
16088 let ring = Arc::new(Mutex::new(StderrRing::new(StderrTailConfig::default())));
16089 let (pump, _held) = held_pump(&ring, "parent exiting\n", "");
16092
16093 let started = Instant::now();
16094 settle_stderr_pump("orphaning", &ring, pump, BOUND).await;
16095 assert_eq!(
16096 started.elapsed(),
16097 BOUND,
16098 "the restart must wait exactly the bound for a pipe that stays open, no longer"
16099 );
16100
16101 let next = lock(&ring).begin_process();
16102 lock(&ring).push_line_from(next, "next process booting");
16103 tokio::time::sleep(Duration::from_secs(60)).await;
16104
16105 let snapshot = lock(&ring).snapshot(None, None);
16106 match &snapshot.capture {
16107 CaptureState::Incomplete { reason } => assert!(
16108 reason.contains("had not reached EOF") && reason.contains("250ms"),
16109 "the reason must say what is missing and after how long: {reason}"
16110 ),
16111 other => panic!("expected Incomplete while the pipe is held open, got {other:?}"),
16112 }
16113 assert_eq!(
16114 untimed(snapshot.entries),
16115 vec![
16116 line("parent exiting"),
16117 TailEntry::ProcessStart,
16118 line("next process booting"),
16119 ]
16120 );
16121 }
16122
16123 #[tokio::test(start_paused = true)]
16124 async fn a_reader_that_reaches_eof_within_the_bound_leaves_the_tail_captured() {
16125 let ring = Arc::new(Mutex::new(StderrRing::new(StderrTailConfig::default())));
16126 let (pump, release) = held_pump(&ring, "one\n", "two\n");
16127 release.send(()).unwrap();
16128
16129 settle_stderr_pump("clean", &ring, pump, BOUND).await;
16130
16131 let snapshot = lock(&ring).snapshot(None, None);
16132 assert_eq!(snapshot.capture, CaptureState::Captured);
16133 assert_eq!(untimed(snapshot.entries), vec![line("one"), line("two")]);
16134 }
16135
16136 #[tokio::test(start_paused = true)]
16137 async fn stdout_buffered_past_the_exit_drain_bound_reaches_the_capture_file() {
16138 const BURST: usize = 150;
16139 let capture = cortexkit_test_support::ScratchDir::new("stdout-exit-drain");
16140 let path = capture.join("two-pipe.stderr.log");
16141 let sink = ChildOutputSink::open(&path, cortexkit_log::Retention::default()).unwrap();
16142 let ring = Arc::new(Mutex::new(StderrRing::new(StderrTailConfig::default())));
16143 let generation = lock(&ring).begin_process();
16144 let stdout = (0..BURST)
16145 .map(|i| format!("out-{i:04}-{}\n", "o".repeat(64)))
16146 .collect::<String>();
16147 let stderr = (0..BURST)
16148 .map(|i| format!("err-{i:04}-{}\n", "e".repeat(64)))
16149 .collect::<String>();
16150 let first_chunk = stdout.find('\n').unwrap() + 1 + "out-0001-".len();
16155 let (release, gate) = oneshot::channel();
16156 let reader = HeldReader {
16157 before: Some(stdout.as_bytes()[..first_chunk].to_vec()),
16158 gate: Some(gate),
16159 after: io::Cursor::new(stdout.as_bytes()[first_chunk..stdout.len() - 1].to_vec()),
16160 };
16161 let (finished, completion) = oneshot::channel();
16162 let stdout_sink = sink.clone();
16163 let pump = tokio::spawn(async move {
16164 pump_stdout_to(reader, stdout_sink).await;
16165 finished.send(()).unwrap();
16166 });
16167 pump_stderr_to(io::Cursor::new(stderr.into_bytes()), ring, generation, sink).await;
16168
16169 let started = Instant::now();
16170 settle_stdout_pump("two-pipe", pump, BOUND).await;
16171 assert_eq!(
16172 started.elapsed(),
16173 BOUND,
16174 "a late reader must not hold up restart"
16175 );
16176 assert_eq!(
16177 std::fs::read_to_string(&path).unwrap().lines().count(),
16178 BURST + 1
16179 );
16180
16181 release
16182 .send(())
16183 .expect("stdout reader must survive the exit drain bound");
16184 tokio::time::timeout(Duration::from_secs(1), completion)
16185 .await
16186 .expect("released stdout reader must reach EOF")
16187 .expect("stdout capture must finish at EOF, not be cancelled");
16188 let contents = std::fs::read_to_string(&path).unwrap();
16189 let mut lines = contents
16190 .lines()
16191 .map(|line| split_capture_stamp(line).expect("capture stamp").1)
16192 .collect::<Vec<_>>();
16193 let mut expected = (0..BURST)
16194 .flat_map(|i| {
16195 [
16196 format!("out-{i:04}-{}", "o".repeat(64)),
16197 format!("err-{i:04}-{}", "e".repeat(64)),
16198 ]
16199 })
16200 .collect::<Vec<_>>();
16201 lines.sort_unstable();
16202 expected.sort_unstable();
16203 assert_eq!(
16204 lines, expected,
16205 "both pipes must retain every complete line exactly once"
16206 );
16207 }
16208}
16209
16210#[cfg(all(test, windows))]
16224mod job_containment_tests {
16225 use super::*;
16226 use cortexkit_test_support::ScratchDir;
16227 use std::{
16228 path::{Path, PathBuf},
16229 sync::{Arc, Mutex},
16230 time::{Duration, Instant},
16231 };
16232
16233 fn stub_path() -> PathBuf {
16239 let mut path = std::env::current_exe().expect("current_exe available in tests");
16240 path.pop();
16241 path.pop();
16242 path.push("fake-aft-stub.exe");
16243 assert!(
16244 path.exists(),
16245 "fake-aft-stub not built at {}: run `cargo test -p subc-core` (which builds \
16246 [[bin]] targets) rather than `cargo test -p subc-core --lib` (which does not)",
16247 path.display()
16248 );
16249 path
16250 }
16251
16252 fn read_grandchild_pid(path: &Path) -> u32 {
16254 let deadline = Instant::now() + Duration::from_secs(10);
16255 loop {
16256 if let Ok(contents) = std::fs::read_to_string(path) {
16257 if let Ok(pid) = contents.trim().parse() {
16258 return pid;
16259 }
16260 }
16261 assert!(
16262 Instant::now() < deadline,
16263 "the stub never recorded a grandchild pid at {}",
16264 path.display()
16265 );
16266 std::thread::sleep(Duration::from_millis(10));
16267 }
16268 }
16269
16270 struct Fixture {
16273 _dir: ScratchDir,
16274 module_id: String,
16275 grandchild: u32,
16276 child: Option<SupervisedChild>,
16277 registry: Arc<Registry>,
16278 snapshot: Arc<Mutex<SupervisorSnapshot>>,
16279 terminal_ring: Arc<Mutex<TerminalRing>>,
16280 spawn_events: SpawnEventFeed,
16281 }
16282
16283 fn fixture(label: &str, module_id: &str) -> Fixture {
16284 let dir = ScratchDir::new(label);
16285 let pid_file = dir.join("grandchild.pid");
16286 let supervisor = Supervisor::new_for_test(
16287 Arc::new(Registry::default()),
16288 RestartPolicy::new(3, Duration::ZERO),
16289 );
16290 let runtime = supervisor.runtime_config();
16291 let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::starting()));
16292 let spec = ModuleSpec {
16293 module_id: module_id.to_string(),
16294 program: stub_path(),
16295 args: Vec::new(),
16299 env: vec![
16300 ("FAKE_AFT_NEVER_CONNECT".to_string(), "1".to_string()),
16301 (
16302 "FAKE_AFT_GRANDCHILD_PID_FILE".to_string(),
16303 pid_file.display().to_string(),
16304 ),
16305 ],
16306 reserved: false,
16307 reserved_prefixes: Vec::new(),
16308 protocol: ModuleProtocol::Subc,
16309 overlap: Default::default(),
16310 };
16311 let child = spawn_and_mark_running(&spec, &runtime, &snapshot)
16312 .expect("spawn the supervised fixture");
16313 let grandchild = read_grandchild_pid(&pid_file);
16314 Fixture {
16315 _dir: dir,
16316 module_id: module_id.to_string(),
16317 grandchild,
16318 child: Some(child),
16319 registry: Arc::new(Registry::default()),
16320 snapshot,
16321 terminal_ring: Arc::clone(&runtime.terminal_ring),
16322 spawn_events: SpawnEventFeed::default(),
16323 }
16324 }
16325
16326 impl Fixture {
16327 async fn drain(&mut self) {
16329 let child = self
16330 .child
16331 .take()
16332 .expect("the fixture child is still present");
16333 drain_child_to_state(
16334 &self.module_id,
16335 ModuleProtocol::Subc,
16336 StopNotice::NotSent,
16339 &self.registry,
16340 None,
16341 &self.snapshot,
16342 &self.terminal_ring,
16343 &self.spawn_events,
16344 child,
16345 Duration::from_millis(500),
16346 ModuleState::Stopped,
16347 Some(false),
16348 )
16349 .await
16350 .expect("drain the supervised fixture");
16351 }
16352 }
16353
16354 #[tokio::test]
16360 async fn teardown_reaps_the_grandchild() {
16361 let mut fixture = fixture("teardown-grandchild", "tree-teardown");
16362 let grandchild = fixture.grandchild;
16363
16364 assert!(
16365 subc_jobobject::process_exists(grandchild),
16366 "grandchild {grandchild} must be alive before teardown, or this proves nothing"
16367 );
16368
16369 fixture.drain().await;
16370
16371 assert!(
16372 subc_jobobject::wait_for_process_exit(grandchild, Duration::from_secs(10)),
16373 "grandchild {grandchild} outlived module teardown: the tree was not contained"
16374 );
16375 }
16376
16377 #[test]
16390 fn an_uncontained_grandchild_survives_a_direct_child_kill() {
16391 let dir = ScratchDir::new("teardown-uncontained");
16392 let pid_file = dir.join("grandchild.pid");
16393 let mut child = std::process::Command::new(stub_path())
16394 .env("FAKE_AFT_NEVER_CONNECT", "1")
16395 .env(
16396 "FAKE_AFT_GRANDCHILD_PID_FILE",
16397 pid_file.display().to_string(),
16398 )
16399 .stdin(std::process::Stdio::null())
16400 .stdout(std::process::Stdio::null())
16401 .stderr(std::process::Stdio::null())
16402 .spawn()
16403 .expect("spawn the uncontained fixture");
16404 let grandchild = read_grandchild_pid(&pid_file);
16405
16406 child.kill().expect("kill the direct child");
16408 let _ = child.wait();
16409
16410 assert!(
16411 subc_jobobject::process_exists(grandchild),
16412 "grandchild {grandchild} died with the direct child, so this control no longer \
16413 distinguishes contained from uncontained teardown and the regression test is \
16414 passing vacuously"
16415 );
16416
16417 kill_tree(grandchild);
16420 }
16421
16422 #[tokio::test]
16431 async fn dropping_containment_reaps_the_grandchild() {
16432 let mut fixture = fixture("drop-containment", "tree-drop");
16433 let grandchild = fixture.grandchild;
16434
16435 assert!(subc_jobobject::process_exists(grandchild));
16436
16437 fixture.child.as_mut().expect("child present").job = None;
16439
16440 assert!(
16441 subc_jobobject::wait_for_process_exit(grandchild, Duration::from_secs(10)),
16442 "grandchild {grandchild} survived the containment handle closing, so a daemon \
16443 crash would leave the tree behind"
16444 );
16445 }
16446
16447 fn kill_tree(pid: u32) {
16449 let _ = std::process::Command::new("taskkill.exe")
16450 .args(["/PID", &pid.to_string(), "/T", "/F"])
16451 .stdin(std::process::Stdio::null())
16452 .stdout(std::process::Stdio::null())
16453 .stderr(std::process::Stdio::null())
16454 .status();
16455 assert!(
16456 subc_jobobject::wait_for_process_exit(pid, Duration::from_secs(10)),
16457 "could not clean up grandchild {pid}"
16458 );
16459 }
16460}
16461
16462#[cfg(test)]
16463mod privacy_trampoline_configuration_tests {
16464 #[tokio::test]
16465 #[cfg_attr(not(target_os = "macos"), ignore = "requires macOS privacy trampoline")]
16466 async fn macos_spawn_without_a_configured_trampoline_refuses_by_name() {
16467 #[cfg(target_os = "macos")]
16468 {
16469 let supervisor = super::Supervisor::new(
16470 std::sync::Arc::new(crate::Registry::default()),
16471 super::RestartPolicy::default(),
16472 );
16473 let error = supervisor.spawn(spec()).unwrap_err();
16474 assert!(
16475 error
16476 .to_string()
16477 .contains("no privacy trampoline configured"),
16478 "{error}"
16479 );
16480 }
16481 }
16482
16483 #[tokio::test]
16484 #[cfg_attr(not(target_os = "macos"), ignore = "requires macOS privacy trampoline")]
16485 async fn macos_wrong_trampoline_is_rejected_by_the_startup_probe() {
16486 #[cfg(target_os = "macos")]
16487 {
16488 let supervisor = super::Supervisor::new(
16489 std::sync::Arc::new(crate::Registry::default()),
16490 super::RestartPolicy::default(),
16491 )
16492 .with_privacy_trampoline(std::env::current_exe().unwrap());
16493 let error = supervisor.spawn(spec()).unwrap_err();
16494 assert!(
16495 error
16496 .to_string()
16497 .contains("binary does not implement the privacy trampoline protocol"),
16498 "{error}"
16499 );
16500 }
16501 }
16502
16503 #[cfg(target_os = "macos")]
16504 fn spec() -> super::ModuleSpec {
16505 super::ModuleSpec {
16506 module_id: "privacy-configuration".into(),
16507 program: "/bin/sleep".into(),
16508 args: vec!["30".into()],
16509 env: vec![],
16510 reserved: false,
16511 reserved_prefixes: vec![],
16512 protocol: subc_control::ModuleProtocol::None,
16513 overlap: super::ModuleOverlap::Exclusive,
16514 }
16515 }
16516}
16517
16518#[cfg(test)]
16519mod privacy_exec_boundary_tests {
16520 #[cfg(target_os = "macos")]
16521 use super::*;
16522 #[cfg(target_os = "macos")]
16523 use std::{
16524 io::{Read, Write},
16525 net::{TcpListener, TcpStream},
16526 };
16527
16528 #[cfg(target_os = "macos")]
16531 pub(super) fn before_image_sample(spec: &ModuleSpec, pid: u32) {
16532 if let Some((_, path)) = spec
16533 .env
16534 .iter()
16535 .find(|(key, _)| key == "SUBC_TEST_PRIVACY_SAMPLE_BARRIER")
16536 {
16537 let mut barrier = TcpStream::connect(path).unwrap();
16538 barrier
16539 .set_read_timeout(Some(Duration::from_secs(30)))
16540 .unwrap();
16541 barrier.write_all(&pid.to_ne_bytes()).unwrap();
16542 let mut release = [0];
16543 barrier.read_exact(&mut release).unwrap();
16544 assert_eq!(&release, b"X");
16545 }
16546 }
16547
16548 #[cfg(target_os = "macos")]
16549 fn spec(program: &str, args: &[&str]) -> ModuleSpec {
16550 ModuleSpec {
16551 module_id: "privacy-boundary".into(),
16552 program: program.into(),
16553 args: args.iter().map(|arg| (*arg).into()).collect(),
16554 env: vec![],
16555 reserved: false,
16556 reserved_prefixes: vec![],
16557 protocol: ModuleProtocol::None,
16558 overlap: ModuleOverlap::Exclusive,
16559 }
16560 }
16561
16562 #[cfg(target_os = "macos")]
16563 async fn accept(listener: TcpListener) -> TcpStream {
16564 let listener = tokio::net::TcpListener::from_std({
16566 listener.set_nonblocking(true).unwrap();
16567 listener
16568 })
16569 .unwrap();
16570 let (stream, _) = tokio::time::timeout(Duration::from_secs(30), listener.accept())
16571 .await
16572 .unwrap()
16573 .unwrap();
16574 let stream = stream.into_std().unwrap();
16575 stream.set_nonblocking(false).unwrap();
16576 stream
16577 .set_read_timeout(Some(Duration::from_secs(30)))
16578 .unwrap();
16579 stream
16580 }
16581
16582 #[tokio::test]
16583 #[cfg_attr(not(target_os = "macos"), ignore = "requires macOS privacy trampoline")]
16584 async fn macos_roster_withholds_a_nonnull_trampoline_image_until_exec_confirmation() {
16585 #[cfg(target_os = "macos")]
16586 {
16587 let root = cortexkit_test_support::ScratchDir::new("privacy-roster-barrier");
16588 let exec_listener = TcpListener::bind("127.0.0.1:0").unwrap();
16591 let sample_listener = TcpListener::bind("127.0.0.1:0").unwrap();
16592 let record = root.join("live-children.json");
16593 let supervisor =
16594 Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default())
16595 .with_live_children_record(&record);
16596 let runtime = supervisor.runtime_config();
16597 let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::starting()));
16598 let mut spec = spec("/bin/sleep", &["30"]);
16599 spec.env = vec![
16600 (
16601 "SUBC_TEST_PRIVACY_EXEC_BARRIER".into(),
16602 exec_listener.local_addr().unwrap().to_string(),
16603 ),
16604 (
16605 "SUBC_TEST_PRIVACY_SAMPLE_BARRIER".into(),
16606 sample_listener.local_addr().unwrap().to_string(),
16607 ),
16608 ];
16609 let spawn = tokio::task::spawn_blocking(move || {
16610 spawn_and_mark_running(&spec, &runtime, &snapshot).unwrap()
16611 });
16612 let mut sample = accept(sample_listener).await;
16613 let mut pid = [0; 4];
16614 sample.read_exact(&mut pid).unwrap();
16615 let pid = u32::from_ne_bytes(pid);
16616 let mut exec = accept(exec_listener).await;
16617 let mut ready = [0];
16618 exec.read_exact(&mut ready).unwrap();
16619 assert_eq!(&ready, b"R");
16620 let trampoline = subc_os::file_identity(&test_privacy_trampoline()).unwrap();
16623 assert_eq!(
16624 observe_spawned_image(pid).unwrap().executable,
16625 Some(trampoline)
16626 );
16627 sample.write_all(b"X").unwrap();
16628 let mut child = spawn.await.unwrap();
16629 let early = crate::live_children::read_record(&record).unwrap();
16630 assert_eq!(early.len(), 1);
16631 assert_eq!(early[0].pid, pid);
16632 assert_eq!(
16633 early[0].executable, None,
16634 "unconfirmed trampoline image entered the roster"
16635 );
16636 assert!(child.report_ready.get().is_none());
16637 child.privacy_exec.as_mut().unwrap().deadline =
16640 tokio::time::Instant::now() + Duration::from_secs(30);
16641 exec.write_all(b"X").unwrap();
16642 child.confirm_privacy_exec().await;
16643 assert_eq!(child.spawn_failure, None);
16644 assert!(child.report_ready.get().is_some());
16645 let confirmed = crate::live_children::read_record(&record).unwrap();
16646 let module = subc_os::file_identity(std::path::Path::new("/bin/sleep")).unwrap();
16647 assert_ne!(module, trampoline);
16648 assert_eq!(confirmed[0].executable, Some(module.into()));
16649 child.start_kill().unwrap();
16650 child.wait().await.unwrap();
16651 child.release_roster();
16652 }
16653 }
16654
16655 #[tokio::test]
16656 #[cfg_attr(not(target_os = "macos"), ignore = "requires macOS privacy trampoline")]
16657 async fn macos_already_exited_121_in_try_wait_is_a_module_exit_not_a_trampoline_refusal() {
16658 #[cfg(target_os = "macos")]
16659 {
16660 let registry = Arc::new(Registry::default());
16661 let policy = RestartPolicy::new(0, Duration::ZERO);
16662 let supervisor = Supervisor::new_for_test(Arc::clone(®istry), policy);
16663 let runtime = supervisor.runtime_config();
16664 let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::starting()));
16665 let spec = spec("/bin/sh", &["-c", "exit 121"]);
16666 let mut child = spawn_and_mark_running(&spec, &runtime, &snapshot).unwrap();
16670 let pid = child.pid;
16671 tokio::task::spawn_blocking(move || {
16672 subc_os::fork_exec_test::wait_for_child_exit_without_reaping(pid)
16673 })
16674 .await
16675 .unwrap()
16676 .unwrap();
16677 child.privacy_exec.as_mut().unwrap().deadline =
16678 tokio::time::Instant::now() + Duration::from_secs(30);
16679 let status = child.wait().await.unwrap();
16680 assert_eq!(status.code(), Some(121));
16681 assert!(child.privacy_exec.is_none());
16682 assert!(
16683 child.report_ready.get().is_none(),
16684 "an exited module must not publish a live pid"
16685 );
16686 let report = classify_reaped_child_exit(&snapshot, &child, &status);
16687 on_child_exit(
16688 &spec,
16689 policy,
16690 ®istry,
16691 &snapshot,
16692 &runtime.terminal_ring,
16693 &runtime.spawn_events,
16694 &runtime.child_roster,
16695 report,
16696 )
16697 .await;
16698 let state = lock_snapshot(&snapshot).unwrap();
16699 assert_eq!(state.state, ModuleState::Failed);
16700 assert_eq!(state.last_exit.as_ref().unwrap().code, Some(121));
16701 assert_eq!(state.reported_pid(), None);
16702 drop(state);
16703 let history = runtime.terminal_ring.lock().unwrap().snapshot();
16704 assert_eq!(history.entries.len(), 1);
16705 let terminal = &history.entries[0];
16706 assert_eq!(terminal.exit_code, Some(121));
16707 assert_eq!(terminal.exit_kind, subc_control::TerminalExitKind::Crash);
16708 assert_eq!(terminal.disposition, TerminalDisposition::Failed);
16709 assert_eq!(
16710 terminal.disposition_detail.as_deref(),
16711 Some(policy.budget_exhausted_detail().as_str()),
16712 "module exit 121 was classified as a trampoline refusal: {terminal:?}"
16713 );
16714 assert_eq!(child.spawn_failure, None);
16715 child.release_roster();
16716 }
16717 }
16718}
16719
16720#[cfg(all(test, unix))]
16724mod launch_nonce_descriptor_tests {
16725 use super::{ChildRoster, ModuleProtocol, ModuleSpec, SupervisorHandle};
16726 use crate::stderr_tail::{StderrRing, StderrTailConfig};
16727 use cortexkit_test_support::ScratchDir;
16728 use std::{
16729 path::PathBuf,
16730 sync::{Arc, Mutex},
16731 time::{Duration, Instant},
16732 };
16733
16734 async fn probe(role: super::SpawnRole) {
16735 let scratch = ScratchDir::new("launch-nonce-descriptor");
16736 let fd_copy = scratch.join("from-descriptor");
16737 let env_copy = scratch.join("environment");
16738 let script = format!(
16739 "env > '{env}.tmp' && mv '{env}.tmp' '{env}'; cat <&3 > '{fd}.tmp' && mv '{fd}.tmp' '{fd}'; sleep 30",
16740 fd = fd_copy.display(), env = env_copy.display(),
16741 );
16742 let xdg = |name: &str| (name.to_string(), scratch.join(name).display().to_string());
16743 let spec = ModuleSpec {
16744 module_id: "nonce-descriptor-probe".to_string(),
16745 program: PathBuf::from("/bin/sh"),
16746 args: vec!["-c".to_string(), script],
16747 env: vec![
16748 xdg("XDG_DATA_HOME"),
16749 xdg("XDG_RUNTIME_DIR"),
16750 xdg("XDG_CONFIG_HOME"),
16751 ("SUBC_LAUNCH_NONCE".to_string(), "must-not-leak".to_string()),
16752 ],
16753 reserved: true,
16754 reserved_prefixes: Vec::new(),
16755 protocol: ModuleProtocol::Subc,
16756 overlap: Default::default(),
16757 };
16758 let handle = SupervisorHandle::new();
16759 let ring = Arc::new(Mutex::new(StderrRing::new(StderrTailConfig::default())));
16760 let roster = ChildRoster::default();
16761 #[cfg(target_os = "macos")]
16762 {
16763 let path = super::test_privacy_trampoline();
16764 roster.set_privacy_trampoline(super::probe_privacy_trampoline(&path).map(|()| path));
16765 }
16766 let child = super::spawn_child_in_slot(
16767 &spec,
16768 None,
16769 Some(&handle),
16770 &ring,
16771 None,
16772 &roster,
16773 #[cfg(target_os = "linux")]
16774 None,
16775 role,
16776 matches!(role, super::SpawnRole::SwapCandidate),
16777 )
16778 .expect("spawn probe");
16779 let deadline = Instant::now() + Duration::from_secs(10);
16780 while !(fd_copy.exists() && env_copy.exists()) {
16781 assert!(Instant::now() < deadline, "probe never wrote its copies");
16782 tokio::time::sleep(Duration::from_millis(20)).await;
16783 }
16784 let nonce = std::fs::read_to_string(fd_copy).unwrap();
16785 assert!(!nonce.is_empty());
16786 let environment = std::fs::read_to_string(env_copy).unwrap();
16787 assert!(environment
16788 .lines()
16789 .any(|line| line.starts_with("SUBC_LAUNCH_NONCE_FD=3:")));
16790 let copy = environment
16791 .lines()
16792 .find_map(|line| line.strip_prefix("SUBC_LAUNCH_NONCE="));
16793 assert_eq!(
16794 copy, None,
16795 "Unix children must never receive the environment nonce"
16796 );
16797 if matches!(role, super::SpawnRole::Plain) {
16798 assert_eq!(
16799 handle.spawn_nonce(&spec.module_id).as_deref(),
16800 Some(nonce.as_str())
16801 );
16802 }
16803 drop(child);
16804 }
16805
16806 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
16807 async fn a_spawned_module_receives_its_nonce_only_on_descriptor_3() {
16808 probe(super::SpawnRole::Plain).await;
16809 }
16810
16811 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
16812 async fn swap_candidate_receives_its_nonce_only_on_descriptor_3() {
16813 probe(super::SpawnRole::SwapCandidate).await;
16814 }
16815}
16816
16817#[cfg(all(test, target_os = "linux"))]
16818mod cgroup_containment_tests {
16819 use super::*;
16820 use cortexkit_test_support::ScratchDir;
16821
16822 fn running(pid: u32) -> bool {
16823 std::fs::read_to_string(format!("/proc/{pid}/stat"))
16825 .ok()
16826 .and_then(|stat| {
16827 stat.rsplit_once(") ")
16828 .map(|(_, rest)| rest.starts_with('Z'))
16829 })
16830 .is_some_and(|zombie| !zombie)
16831 }
16832
16833 #[tokio::test]
16834 async fn linux_teardown_reaps_the_grandchild() {
16835 teardown_tree("linux_teardown_reaps_the_grandchild", false).await;
16836 }
16837
16838 #[tokio::test]
16839 async fn linux_shutdown_straggler_reaps_the_grandchild() {
16840 teardown_tree("linux_shutdown_straggler_reaps_the_grandchild", true).await;
16841 }
16842
16843 async fn teardown_tree(test_name: &str, shutdown: bool) {
16844 let dir = ScratchDir::new(test_name);
16845 let root = PathBuf::from(format!(
16846 "/sys/fs/cgroup/subc-tree-test-{test_name}-{}-{}",
16847 std::process::id(),
16848 unix_ms_now()
16849 ));
16850 if let Err(error) = std::fs::create_dir(&root) {
16851 assert!(
16852 std::env::var_os("SUBC_REQUIRE_CGROUP_TEST").is_none(),
16853 "required cgroup test cannot execute: {error}"
16854 );
16855 eprintln!(
16856 "SKIP {test_name}: no writable delegated cgroup at {}: {error}",
16857 root.display()
16858 );
16859 return;
16860 }
16861 let placement = subc_cgroup::prepare_at(&root)
16862 .expect("prepare isolated kernel cgroup")
16863 .expect("isolated cgroup is delegated");
16864 let module_id = "tree-teardown";
16865 let module = placement
16866 .module_path(module_id)
16867 .expect("create isolated module cgroup");
16868 if !module.join("cgroup.kill").exists() {
16869 std::fs::remove_dir(&module).unwrap();
16870 std::fs::remove_dir(root.join("subc-modules")).unwrap();
16871 std::fs::remove_dir(&root).unwrap();
16872 assert!(
16873 std::env::var_os("SUBC_REQUIRE_CGROUP_TEST").is_none(),
16874 "required cgroup.kill interface unavailable"
16875 );
16876 eprintln!("SKIP {test_name}: cgroup.kill unavailable (kernel < 5.14)");
16877 return;
16878 }
16879 let supervisor = Supervisor::new_for_test(
16880 Arc::new(Registry::default()),
16881 RestartPolicy::new(3, Duration::ZERO),
16882 )
16883 .with_cgroup_placement(Some(placement));
16884 let mut runtime = supervisor.runtime_config();
16885 runtime.child_roster = runtime
16886 .child_roster
16887 .for_module(Arc::new(Mutex::new(Duration::from_millis(100))));
16888 let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::starting()));
16889 let pid_file = dir.join("grandchild.pid");
16890 let spec = ModuleSpec {
16891 module_id: module_id.to_string(),
16892 program: PathBuf::from("/bin/sh"),
16893 args: vec![
16894 "-c".into(),
16895 "trap '' TERM; sleep 600 & echo $! > \"$1\"; wait".into(),
16896 "fixture".into(),
16897 pid_file.display().to_string(),
16898 ],
16899 env: ["XDG_DATA_HOME", "XDG_RUNTIME_DIR", "XDG_CONFIG_HOME"]
16900 .into_iter()
16901 .map(|key| (key.to_string(), dir.display().to_string()))
16902 .collect(),
16903 reserved: false,
16904 reserved_prefixes: Vec::new(),
16905 protocol: ModuleProtocol::None,
16906 overlap: Default::default(),
16907 };
16908 let child =
16909 spawn_and_mark_running(&spec, &runtime, &snapshot).expect("spawn supervised tree");
16910 let deadline = tokio::time::Instant::now() + Duration::from_secs(10);
16911 let grandchild: u32 = loop {
16912 if let Ok(contents) = std::fs::read_to_string(&pid_file) {
16913 if let Ok(pid) = contents.trim().parse() {
16914 break pid;
16915 }
16916 }
16917 assert!(
16918 tokio::time::Instant::now() < deadline,
16919 "grandchild pid was not recorded"
16920 );
16921 tokio::time::sleep(Duration::from_millis(10)).await;
16922 };
16923 assert!(
16924 running(grandchild),
16925 "grandchild must be alive before teardown"
16926 );
16927 if shutdown {
16928 let mut child = child;
16929 crate::child_roster::end_children_for_daemon_shutdown(
16930 &runtime.child_roster,
16931 false,
16932 std::future::pending(),
16933 )
16934 .await;
16935 child.wait().await.expect("reap shutdown straggler");
16936 } else {
16937 drain_child_to_state(
16938 module_id,
16939 ModuleProtocol::None,
16940 StopNotice::NotSent,
16941 &Registry::default(),
16942 None,
16943 &snapshot,
16944 &runtime.terminal_ring,
16945 &SpawnEventFeed::default(),
16946 child,
16947 Duration::from_millis(100),
16948 ModuleState::Stopped,
16949 Some(false),
16950 )
16951 .await
16952 .expect("real supervisor teardown");
16953 }
16954 let deadline = tokio::time::Instant::now() + Duration::from_secs(5);
16955 while running(grandchild) && tokio::time::Instant::now() < deadline {
16956 tokio::time::sleep(Duration::from_millis(10)).await;
16957 }
16958 let survived = running(grandchild);
16959 if survived {
16961 let pid = rustix::process::Pid::from_raw(grandchild as i32).unwrap();
16962 let _ = rustix::process::kill_process(pid, rustix::process::Signal::KILL);
16963 tokio::time::sleep(Duration::from_millis(100)).await;
16964 }
16965 if module.exists() {
16966 std::fs::remove_dir(&module).expect("remove empty module cgroup");
16967 }
16968 std::fs::remove_dir(root.join("subc-modules")).unwrap();
16969 std::fs::remove_dir(&root).unwrap();
16970 assert!(
16971 !survived,
16972 "grandchild {grandchild} outlived module teardown"
16973 );
16974 eprintln!("EXECUTED {test_name}: grandchild {grandchild} killed in isolated cgroup");
16975 }
16976}
16977
16978#[cfg(all(test, unix))]
16981mod record_write_stall_tests {
16982 use super::*;
16983 use std::sync::atomic::{AtomicBool, Ordering};
16984 use subc_transport::{authenticate_client, ConnectionInfo, Endpoint, SCHEMA_VERSION};
16985
16986 const MODULE_AUTH_DEADLINE: Duration = Duration::from_secs(2);
16989 const HELD_RECORD_WRITE: Duration = Duration::from_secs(3);
16993
16994 fn sleeping_spec() -> ModuleSpec {
17006 ModuleSpec {
17007 module_id: "record-write-stall".into(),
17008 program: "/bin/sleep".into(),
17009 args: vec!["600".into()],
17010 env: vec![],
17011 reserved: false,
17012 reserved_prefixes: vec![],
17013 protocol: ModuleProtocol::None,
17014 overlap: Default::default(),
17015 }
17016 }
17017
17018 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
17019 async fn module_auth_completes_while_the_restart_waits_on_the_record_write() {
17020 let home = cortexkit_test_support::ScratchDir::new("record-write-stall");
17021 let supervisor =
17022 Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default())
17023 .with_live_children_record(home.join("live-children.json"));
17024 let module = supervisor.spawn(sleeping_spec()).unwrap();
17025 let old_pid = tokio::time::timeout(Duration::from_secs(10), async {
17030 loop {
17031 if let Some(pid) = lock_snapshot(&module.inner.snapshot)
17032 .unwrap()
17033 .reported_pid()
17034 {
17035 return pid;
17036 }
17037 tokio::time::sleep(Duration::from_millis(10)).await;
17038 }
17039 })
17040 .await
17041 .expect("the first process was admitted as the module");
17042
17043 let key = vec![0x42; 32];
17046 let daemon_id = [0x24; 16];
17047 let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
17048 let port = listener.local_addr().unwrap().port();
17049 let server = tokio::spawn(crate::server::serve_listener(
17050 listener,
17051 Arc::new(crate::router::Router::with_default_self_handler()),
17052 crate::server::ServerAuth::with_limits(
17053 key.clone(),
17054 daemon_id,
17055 "test",
17056 crate::server::DEFAULT_AUTH_DEADLINE,
17057 4,
17058 ),
17059 ));
17060
17061 let (held_tx, held_rx) = std::sync::mpsc::channel();
17064 let armed = AtomicBool::new(true);
17065 supervisor
17066 .child_roster
17067 .set_record_write_hook_for_test(move |live| {
17068 let admitting_replacement = live.values().any(|entry| entry.pid != old_pid);
17069 if admitting_replacement && armed.swap(false, Ordering::SeqCst) {
17070 let _ = held_tx.send(());
17071 std::thread::sleep(HELD_RECORD_WRITE);
17072 }
17073 });
17074
17075 let module_process = std::thread::spawn(move || {
17078 held_rx
17079 .recv_timeout(Duration::from_secs(30))
17080 .expect("the restart reached the replacement's record write");
17081 let conn = ConnectionInfo {
17082 schema: SCHEMA_VERSION,
17083 wire_version: None,
17084 endpoints: vec![Endpoint {
17085 host: "127.0.0.1".to_owned(),
17086 port,
17087 }],
17088 key,
17089 daemon_id,
17090 pid: std::process::id(),
17091 daemon_ver: "test".to_owned(),
17092 };
17093 tokio::runtime::Builder::new_current_thread()
17094 .enable_all()
17095 .build()
17096 .unwrap()
17097 .block_on(async {
17098 let started = Instant::now();
17099 let mut stream = tokio::net::TcpStream::connect(("127.0.0.1", port))
17100 .await
17101 .unwrap();
17102 let result =
17103 authenticate_client(&mut stream, &conn, MODULE_AUTH_DEADLINE).await;
17104 (result, started.elapsed())
17105 })
17106 });
17107
17108 module.restart(Some(0)).await.unwrap();
17109 let (auth, elapsed) = tokio::task::spawn_blocking(move || module_process.join().unwrap())
17110 .await
17111 .unwrap();
17112 assert!(
17113 auth.is_ok(),
17114 "the daemon must answer a module's handshake while a restart waits on the \
17115 live-children record write; got {auth:?} after {elapsed:?}"
17116 );
17117
17118 tokio::time::timeout(Duration::from_secs(20), async {
17119 while lock_snapshot(&module.inner.snapshot)
17120 .unwrap()
17121 .reported_pid()
17122 .is_none_or(|pid| pid == old_pid)
17123 {
17124 tokio::time::sleep(Duration::from_millis(10)).await;
17125 }
17126 })
17127 .await
17128 .expect("the restart published its replacement");
17129 module.stop().await.unwrap();
17130 server.abort();
17131 }
17132}