1use std::{
2 collections::{HashMap, HashSet, VecDeque},
3 error::Error,
4 fmt, io,
5 path::PathBuf,
6 process::{ExitStatus, Stdio},
7 sync::{Arc, Mutex, OnceLock},
8 time::{Duration, SystemTime, UNIX_EPOCH},
9};
10
11use cortexkit_log::Retention;
12use serde_json::Value;
13use sha2::{Digest, Sha256};
14use subc_control::{
15 ClientControlPush, LiveSpawn, ModuleProtocol, RouteCloseReason, SpawnCursor, SpawnEvent,
16 SpawnEventKind, SpawnSnapshot, SupervisorHealthStatus, TerminalDisposition, TerminalExitKind,
17};
18use subc_protocol::{
19 manifest::{SelfSignalKind, SignalAnchor},
20 session::{
21 HealthReport, HealthStatus, ModuleControlCommand, ModuleControlRequest,
22 MODULE_CONTROL_OP_HEALTH_CHECK,
23 },
24 Flags, FrameType, Principal, Priority, SUBC_LAUNCH_NONCE_ENV, SUBC_MODULE_ID_ENV,
25};
26use tokio::{
27 process::{Child, Command},
28 sync::{mpsc, oneshot, watch, Mutex as AsyncMutex},
29 task::JoinHandle,
30 time::{sleep, sleep_until, timeout, timeout_at, Instant},
31};
32use tracing::{debug, error, info, warn};
33
34use crate::{
35 child_roster::ChildRoster,
36 daemon_config::{
37 CAPTURE_KEEP_ENV, CAPTURE_MAX_AGE_DAYS_ENV, CAPTURE_MAX_FILE_MB_ENV, CK_LOG_ENV,
38 },
39 forwarding::{
40 CloseReason, ForwardingError, ForwardingTable, GoodbyeTarget, ModuleControlRpcOutcome,
41 ModuleDrainTarget, PendingModuleControlRpc,
42 },
43 provenance::{ExecutableIdentityProbe, SpawnedFileIdentity},
44 registry::{ConnectionId, RegistrationEndReason, RegistryError},
45 stderr_tail::{
46 pump_stderr_to, pump_stdout_to, ChildOutputSink, StderrRing, StderrTailConfig,
47 StderrTailSnapshot,
48 },
49 terminal_ring::{TerminalHistorySnapshot, TerminalRecord, TerminalRing, TerminalRingConfig},
50 Frame, FrameSink, Registry,
51};
52
53#[cfg(not(windows))]
54use crate::provenance::spawned_file_identity;
55
56#[path = "supervise_swap.rs"]
57mod swap;
58
59#[path = "supervise_activity.rs"]
60mod activity;
61
62pub const SUBC_ARG: &str = "--subc";
68
69const DEFAULT_MAX_RESTARTS: u32 = 3;
70const DEFAULT_BACKOFF: Duration = Duration::from_millis(100);
71const DEFAULT_MAX_BACKOFF: Duration = Duration::from_secs(30);
72const DEFAULT_RESTART_WINDOW: Duration = Duration::from_secs(600);
76pub const DEFAULT_DRAIN_TIMEOUT: Duration = Duration::from_secs(30);
87const REGISTRY_RELEASE_TIMEOUT: Duration = Duration::from_secs(1);
88const REGISTRY_RELEASE_POLL: Duration = Duration::from_millis(10);
89const DRAIN_BUSY_REPROBE_INTERVAL: Duration = Duration::from_millis(250);
101const STDERR_PUMP_DRAIN_TIMEOUT: Duration = Duration::from_millis(250);
124pub const SPAWN_EVENT_RING_CAPACITY: usize = 4096;
126const SPAWN_SUBSCRIBER_BUFFER: usize = SPAWN_EVENT_RING_CAPACITY + 1;
127pub(crate) const SPAWN_SUBSCRIBER_LAGGED_CODE: &str = "spawn_subscriber_lagged";
132
133#[cfg(target_os = "macos")]
134struct PrivacyExec {
135 reader: tokio::io::unix::AsyncFd<std::io::PipeReader>,
136 deadline: tokio::time::Instant,
137 expected: Option<subc_os::FileIdentity>,
138 trampoline: Option<subc_os::FileIdentity>,
139 script: bool,
140 module_id: String,
141 #[cfg(test)]
142 confirmation_pause: Option<(String, Option<tokio::net::TcpStream>)>,
143}
144
145#[cfg(all(test, target_os = "macos"))]
146pub(crate) fn test_privacy_trampoline() -> PathBuf {
147 let path = std::env::current_exe()
150 .unwrap()
151 .parent()
152 .unwrap()
153 .parent()
154 .unwrap()
155 .join("privacy-trampoline-fixture");
156 assert!(
160 path.exists(),
161 "privacy-trampoline-fixture not built at {}: run `cargo build -p subc-daemon \
162 --bins --features test-support` or `cargo test -p subc-daemon` first",
163 path.display()
164 );
165 path
166}
167
168#[cfg(target_os = "macos")]
169fn probe_privacy_trampoline(path: &std::path::Path) -> Result<(), String> {
170 use std::io::Read;
171 let mut probe = std::process::Command::new(path)
172 .args(["__disclaim-exec", "--probe"])
173 .stdin(Stdio::null())
174 .stdout(Stdio::piped())
175 .stderr(Stdio::piped())
176 .spawn()
177 .map_err(|error| {
178 format!(
179 "privacy trampoline probe failed for {}: {error}",
180 path.display()
181 )
182 })?;
183 let deadline = std::time::Instant::now() + Duration::from_secs(5);
184 let status = loop {
185 match probe.try_wait() {
186 Ok(Some(status)) => break status,
187 Ok(None) if std::time::Instant::now() < deadline => {
188 std::thread::sleep(Duration::from_millis(5))
189 }
190 result => {
191 let _ = probe.kill();
192 let _ = probe.wait();
193 return Err(format!(
194 "privacy trampoline probe failed or timed out for {}: {result:?}",
195 path.display()
196 ));
197 }
198 }
199 };
200 let mut answer = String::new();
201 if let Some(stdout) = probe.stdout.take() {
202 let _ = stdout.take(256).read_to_string(&mut answer);
203 }
204 if status.success() && answer.trim() == subc_os::privacy_identity::TRAMPOLINE_PROBE {
205 return Ok(());
206 }
207 let mut diagnostic = String::new();
208 if let Some(stderr) = probe.stderr.take() {
209 let _ = stderr.take(1024).read_to_string(&mut diagnostic);
210 }
211 let cause = diagnostic
212 .trim()
213 .strip_prefix("ck-subc: own privacy identity refused: ")
214 .unwrap_or("binary does not implement the privacy trampoline protocol");
215 Err(format!(
216 "{cause}: probe of {} exited {status}",
217 path.display()
218 ))
219}
220
221#[cfg(target_os = "macos")]
222fn privacy_command(
223 spec: &ModuleSpec,
224 roster: &ChildRoster,
225) -> Result<
226 (
227 Command,
228 Option<PrivacyExec>,
229 subc_os::privacy_identity::ExecAcknowledgement,
230 ),
231 SuperviseError,
232> {
233 let failure = |cause: String| {
234 warn!(module_id = %spec.module_id, %cause, "privacy identity trampoline refused module spawn");
235 SuperviseError::Spawn {
236 program: spec.program.clone(),
237 source: io::Error::other(cause),
238 cgroup_path: None,
239 }
240 };
241 let trampoline = roster.privacy_trampoline().map_err(failure)?;
242 let program = if spec.program.components().count() == 1 && !spec.program.is_absolute() {
247 let path = spec
248 .env
249 .iter()
250 .find(|(key, _)| key == "PATH")
251 .map(|(_, value)| std::ffi::OsString::from(value))
252 .or_else(|| std::env::var_os("PATH"))
253 .unwrap_or_else(|| "/usr/bin:/bin".into());
254 std::env::split_paths(&path)
255 .map(|dir| dir.join(&spec.program))
256 .find(|path| path.is_file())
257 .unwrap_or_else(|| spec.program.clone())
258 } else {
259 spec.program.clone()
260 };
261 let expected = subc_os::file_identity(&program);
262 let trampoline_image = subc_os::file_identity(&trampoline);
263 let script = {
264 use std::io::Read;
265 let mut prefix = [0u8; 2];
266 std::fs::File::open(&program)
267 .is_ok_and(|mut file| file.read_exact(&mut prefix).is_ok() && &prefix == b"#!")
268 };
269 if expected.is_none() || expected == subc_os::file_identity(&trampoline) {
270 return Err(failure(
271 "privacy identity module executable is missing or is the trampoline itself".to_string(),
272 ));
273 }
274 let (reader, ack) = subc_os::privacy_identity::ExecAcknowledgement::pipe()
275 .map_err(|error| failure(error.to_string()))?;
276 let reader =
277 tokio::io::unix::AsyncFd::new(reader).map_err(|error| failure(error.to_string()))?;
278 let mut command = Command::new(&trampoline);
279 command
280 .arg("__disclaim-exec")
281 .arg(ack.fd().to_string())
282 .arg(&program);
283 ack.install(command.as_std_mut());
284 Ok((
285 command,
286 Some(PrivacyExec {
287 reader,
288 deadline: tokio::time::Instant::now() + Duration::from_secs(5),
289 expected,
290 trampoline: trampoline_image,
291 script,
292 module_id: spec.module_id.clone(),
293 #[cfg(test)]
294 confirmation_pause: spec
295 .env
296 .iter()
297 .find(|(key, _)| key == "SUBC_TEST_BEFORE_EXEC_CONFIRM")
298 .map(|(_, address)| (address.clone(), None)),
299 }),
300 ack,
301 ))
302}
303
304struct SupervisedChild {
305 child: Child,
306 exec: crate::activity::ExecOwner,
307 #[cfg(windows)]
308 nonce_delivery: Option<subc_os::LaunchNoncePipeDelivery>,
309 #[cfg(target_os = "macos")]
310 privacy_exec: Option<PrivacyExec>,
311 #[cfg(target_os = "macos")]
318 report_ready: Arc<OnceLock<()>>,
319 spawn_failure: Option<String>,
321 protocol: ModuleProtocol,
326 #[cfg(target_os = "linux")]
332 module_id: String,
333 #[cfg(target_os = "linux")]
334 cgroup_placement: Option<subc_cgroup::Placement>,
335 #[cfg(windows)]
357 job: Option<subc_jobobject::JobObject>,
358 stdout_pump: Option<JoinHandle<()>>,
359 stderr_pump: Option<StderrPump>,
360 stderr_ring: Arc<Mutex<StderrRing>>,
361 spawned_at: Instant,
364 spawned_at_ms: u64,
365 spawned_from: PathBuf,
366 launch_spec_digest: LaunchSpecDigest,
367 spawned_file_identity: Option<SpawnedFileIdentity>,
368 process_start_time: Option<u64>,
369 process_identity: Option<ProcessIdentity>,
370 pid: u32,
371 roster_guard: Option<crate::child_roster::RosterGuard>,
374}
375
376impl SupervisedChild {
377 fn id(&self) -> Option<u32> {
378 Some(self.pid)
379 }
380
381 fn process_identity(&self) -> Option<ProcessIdentity> {
382 self.process_identity
383 }
384
385 async fn wait(&mut self) -> io::Result<ExitStatus> {
386 #[cfg(target_os = "macos")]
387 self.confirm_privacy_exec().await;
388 let result = self.child.wait().await;
396 #[cfg(target_os = "linux")]
397 if result.is_ok() {
398 if let Some(placement) = self.cgroup_placement.as_ref() {
399 cleanup_reaped_cgroup(placement, &self.module_id).await;
400 self.cgroup_placement = None;
403 }
404 }
405 result
406 }
407
408 #[cfg(target_os = "macos")]
409 async fn confirm_privacy_exec(&mut self) {
410 let Some(pending) = &mut self.privacy_exec else {
411 return;
412 };
413 let result = tokio::time::timeout_at(pending.deadline, async {
414 let mut record = Vec::new();
415 loop {
416 let mut ready = pending.reader.readable().await?;
417 let read = ready.try_io(|reader| {
418 use std::io::Read;
419 let mut reader = reader.get_ref();
420 let mut buffer = [0u8; 256];
421 reader.read(&mut buffer).map(|count| (count, buffer))
422 });
423 match read {
424 Ok(Ok((0, _))) => return Ok::<_, io::Error>(record),
425 Ok(Ok((count, buffer))) => {
426 if record.len() + count > 1024 {
427 return Err(io::Error::other(
428 "privacy exec refusal record is too long",
429 ));
430 }
431 record.extend_from_slice(&buffer[..count]);
432 }
433 Ok(Err(error)) => return Err(error),
434 Err(_) => continue,
435 }
436 }
437 })
438 .await;
439 #[cfg(test)]
440 let result = if let Some((address, stream)) = &mut pending.confirmation_pause {
441 use tokio::io::{AsyncReadExt, AsyncWriteExt};
442 if stream.is_none() {
443 *stream = Some(
444 tokio::net::TcpStream::connect(address.as_str())
445 .await
446 .unwrap(),
447 );
448 stream.as_mut().unwrap().write_all(b"R").await.unwrap();
449 stream
450 .as_mut()
451 .unwrap()
452 .write_all(&self.pid.to_ne_bytes())
453 .await
454 .unwrap();
455 }
456 let mut reply = [0];
457 stream
458 .as_mut()
459 .unwrap()
460 .read_exact(&mut reply)
461 .await
462 .unwrap();
463 pending.confirmation_pause = None;
464 if &reply == b"F" {
465 Ok(Err(io::Error::other("injected late exec refusal")))
466 } else {
467 assert_eq!(&reply, b"X");
468 result
469 }
470 } else {
471 result
472 };
473 let pending = self.privacy_exec.as_ref().expect("pending exec");
476 let cause = match result {
477 Err(_) => Some("privacy identity trampoline did not exec within 5s".to_string()),
478 Ok(Err(error)) => Some(format!(
479 "privacy identity exec acknowledgement failed: {error}"
480 )),
481 Ok(Ok(record)) if !record.is_empty() => Some(
482 std::str::from_utf8(&record)
483 .ok()
484 .and_then(|record| {
485 record
486 .trim()
487 .strip_prefix(subc_os::privacy_identity::EXEC_REFUSAL_TAG)
488 })
489 .filter(|cause| !cause.is_empty())
490 .unwrap_or("invalid privacy exec refusal record")
491 .to_string(),
492 ),
493 Ok(Ok(_)) => match self.child.try_wait() {
494 Ok(Some(_status)) => None,
498 Err(error) => Some(format!("privacy identity trampoline wait failed: {error}")),
499 Ok(None) => {
500 let image = observe_spawned_image(self.pid);
501 if let Some(image) = image.filter(|image| {
502 image.executable.is_some()
503 && image.executable != pending.trampoline
504 && (image.executable == pending.expected || pending.script)
505 }) {
506 if let Err(error) = self
507 .roster_guard
508 .as_ref()
509 .map(|guard| guard.confirm_executable(image))
510 .transpose()
511 {
512 Some(format!(
513 "could not durably record confirmed module image: {error}"
514 ))
515 } else {
516 let _ = self.report_ready.set(());
517 info!(module_id = %pending.module_id, pid = self.pid,
518 "module spawned with own privacy identity (responsibility disclaimed)");
519 None
520 }
521 } else if image.is_none()
522 || image.is_some_and(|image| image.executable.is_none())
523 {
524 match tokio::time::timeout_at(pending.deadline, self.child.wait()).await {
531 Ok(Ok(_status)) => None,
532 Ok(Err(error)) => Some(format!("privacy identity trampoline wait failed: {error}")),
533 Err(_) => Some("privacy identity module executable remained unreadable until the 5s exec deadline".to_string()),
534 }
535 } else {
536 Some("privacy identity trampoline executable mismatch: expected the module program, not ck-subc".to_string())
537 }
538 }
539 },
540 };
541 let pending = self.privacy_exec.take().expect("pending exec");
542 self.exec.0.finish(if cause.is_some() {
545 crate::activity::ExecState::Refused
546 } else {
547 crate::activity::ExecState::CommittedExec
548 });
549 if let Some(cause) = cause {
550 warn!(module_id = %pending.module_id, pid = self.pid, %cause, "privacy identity trampoline refused module spawn");
551 self.spawn_failure = Some(cause);
552 if let Some(pid) = rustix::process::Pid::from_raw(self.pid as i32) {
555 let _ = rustix::process::kill_process_group(pid, rustix::process::Signal::KILL);
556 }
557 let _ = self.child.start_kill();
558 }
559 }
560
561 fn release_roster(&mut self) {
565 self.roster_guard = None;
566 }
567
568 fn start_kill(&mut self) -> io::Result<()> {
581 #[cfg(windows)]
582 if let Some(job) = &self.job {
583 if let Err(error) = job.terminate() {
584 debug!(
585 error = %error,
586 "job termination failed; the direct-child kill still owns the outcome"
587 );
588 }
589 }
590 #[cfg(target_os = "linux")]
591 kill_module_cgroup(self.cgroup_placement.as_ref(), &self.module_id);
592 self.child.start_kill()
593 }
594
595 async fn drain_stderr(&mut self, module_id: &str) {
596 if let Some(pump) = self.stdout_pump.take() {
597 settle_stdout_pump(module_id, pump, STDERR_PUMP_DRAIN_TIMEOUT).await;
598 }
599
600 let Some(pump) = self.stderr_pump.take() else {
601 return;
602 };
603 settle_stderr_pump(
604 module_id,
605 &self.stderr_ring,
606 pump,
607 STDERR_PUMP_DRAIN_TIMEOUT,
608 )
609 .await;
610 }
611}
612
613async fn settle_stdout_pump(module_id: &str, mut pump: JoinHandle<()>, bound: Duration) {
617 match timeout(bound, &mut pump).await {
618 Ok(Ok(())) => {}
619 Ok(Err(error)) => {
620 warn!(module_id, error = %error, "stdout pump ended unexpectedly");
621 }
622 Err(_) => {
623 drop(pump);
624 warn!(
625 module_id,
626 waited = ?bound,
627 "stdout pipe of the exited process has not drained; its reader keeps running without delaying the restart"
628 );
629 }
630 }
631}
632
633struct StderrPump {
636 task: JoinHandle<()>,
637 generation: u64,
638}
639
640async fn settle_stderr_pump(
646 module_id: &str,
647 ring: &Arc<Mutex<StderrRing>>,
648 pump: StderrPump,
649 bound: Duration,
650) {
651 let lock = || ring.lock().unwrap_or_else(|poisoned| poisoned.into_inner());
652 let StderrPump {
653 mut task,
654 generation,
655 } = pump;
656 lock().retire_pump(generation);
657 match timeout(bound, &mut task).await {
658 Ok(Ok(())) => {}
659 Ok(Err(err)) => {
660 let mut ring = lock();
661 ring.mark_incomplete_from(generation, format!("stderr pump ended unexpectedly: {err}"));
662 ring.finish_pump(generation);
663 warn!(module_id, error = %err, "stderr pump ended before clean EOF");
664 }
665 Err(_) => {
666 drop(task);
668 lock().mark_pump_late(
669 generation,
670 format!(
671 "stderr of the exited process had not reached EOF {bound:?} after it was \
672 retired (a descendant may still hold the pipe open); lines it still \
673 writes are kept in that process's section"
674 ),
675 );
676 warn!(
677 module_id,
678 waited = ?bound,
679 "stderr pipe of the exited process is still open; its reader keeps running without delaying the restart"
680 );
681 }
682 }
683}
684
685fn registration_release_events() -> &'static watch::Sender<u64> {
686 static EVENTS: OnceLock<watch::Sender<u64>> = OnceLock::new();
687 EVENTS.get_or_init(|| {
688 let (sender, _receiver) = watch::channel(0);
689 sender
690 })
691}
692
693pub(crate) fn notify_registration_release() {
694 let events = registration_release_events();
695 let next_generation = (*events.borrow()).wrapping_add(1);
696 events.send_replace(next_generation);
697}
698
699#[derive(Debug, Clone, PartialEq, Eq)]
701pub struct ModuleSpec {
702 pub module_id: String,
703 pub program: PathBuf,
704 pub args: Vec<String>,
705 pub env: Vec<(String, String)>,
706 pub reserved: bool,
711 pub reserved_prefixes: Vec<String>,
716 pub protocol: ModuleProtocol,
735 pub overlap: ModuleOverlap,
740}
741
742#[derive(Clone, PartialEq, Eq)]
745pub(crate) struct LaunchSpecDigest([u8; 32]);
746
747impl fmt::Debug for LaunchSpecDigest {
748 fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
749 f.write_str("LaunchSpecDigest([redacted])")
750 }
751}
752
753impl LaunchSpecDigest {
754 pub(crate) fn from_spec(spec: &ModuleSpec) -> Self {
755 fn field(hash: &mut Sha256, bytes: &[u8]) {
756 hash.update((bytes.len() as u64).to_le_bytes());
757 hash.update(bytes);
758 }
759 let mut hash = Sha256::new();
760 if let Some(program) = spec.program.to_str() {
761 field(&mut hash, b"utf8-program");
762 field(&mut hash, program.as_bytes());
763 } else {
764 #[cfg(unix)]
767 {
768 use std::os::unix::ffi::OsStrExt;
769 field(&mut hash, b"unix-program");
770 field(&mut hash, spec.program.as_os_str().as_bytes());
771 }
772 #[cfg(windows)]
773 {
774 use std::os::windows::ffi::OsStrExt;
775 field(&mut hash, b"windows-program");
776 let bytes = spec
777 .program
778 .as_os_str()
779 .encode_wide()
780 .flat_map(u16::to_le_bytes)
781 .collect::<Vec<_>>();
782 field(&mut hash, &bytes);
783 }
784 #[cfg(not(any(unix, windows)))]
785 field(&mut hash, spec.program.to_string_lossy().as_bytes());
786 }
787 hash.update((spec.args.len() as u64).to_le_bytes());
788 for arg in &spec.args {
789 field(&mut hash, arg.as_bytes());
790 }
791 let mut env = spec.env.iter().collect::<Vec<_>>();
792 env.sort_unstable();
793 hash.update((env.len() as u64).to_le_bytes());
794 for (key, value) in env {
795 field(&mut hash, key.as_bytes());
796 field(&mut hash, value.as_bytes());
797 }
798 field(
799 &mut hash,
800 match spec.protocol {
801 ModuleProtocol::Subc => b"subc",
802 ModuleProtocol::None => b"none",
803 },
804 );
805 Self(hash.finalize().into())
806 }
807}
808
809#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)]
816pub enum ModuleOverlap {
817 #[default]
819 Exclusive,
820 Safe,
832}
833
834impl ModuleOverlap {
835 pub fn as_str(self) -> &'static str {
836 match self {
837 Self::Exclusive => "exclusive",
838 Self::Safe => "safe",
839 }
840 }
841}
842
843pub const SUBC_SPAWN_ROLE_ENV: &str = "SUBC_SPAWN_ROLE";
853pub const SPAWN_ROLE_SWAP_CANDIDATE: &str = "swap_candidate";
855pub const DEFAULT_SWAP_READY_TIMEOUT: Duration = Duration::from_secs(100);
860
861#[derive(Debug, Clone, Copy, PartialEq, Eq)]
879pub struct RestartPolicy {
880 pub max_restarts: u32,
881 pub backoff: Duration,
884 pub max_backoff: Duration,
886 pub window: Duration,
890}
891
892impl RestartPolicy {
893 pub fn new(max_restarts: u32, backoff: Duration) -> Self {
897 Self {
898 max_restarts,
899 backoff,
900 max_backoff: DEFAULT_MAX_BACKOFF,
901 window: DEFAULT_RESTART_WINDOW,
902 }
903 }
904
905 pub fn with_max_backoff(mut self, max_backoff: Duration) -> Self {
906 self.max_backoff = max_backoff;
907 self
908 }
909
910 pub fn with_window(mut self, window: Duration) -> Self {
911 self.window = window;
912 self
913 }
914
915 fn delay_for_restart(&self, restart_in_window: u32) -> Duration {
920 if self.backoff.is_zero() || self.max_backoff.is_zero() {
921 return Duration::ZERO;
922 }
923
924 let mut delay = self.backoff;
925 for _ in 0..restart_in_window {
926 if delay >= self.max_backoff {
927 return self.max_backoff;
928 }
929 delay = delay
930 .checked_mul(10)
931 .unwrap_or(self.max_backoff)
932 .min(self.max_backoff);
933 }
934 delay.min(self.max_backoff)
935 }
936
937 fn budget_exhausted_detail(&self) -> String {
942 format!(
943 "crash budget exhausted: max_restarts={} within window_secs={}",
944 self.max_restarts,
945 self.window.as_secs()
946 )
947 }
948}
949
950impl Default for RestartPolicy {
951 fn default() -> Self {
952 Self {
953 max_restarts: DEFAULT_MAX_RESTARTS,
954 backoff: DEFAULT_BACKOFF,
955 max_backoff: DEFAULT_MAX_BACKOFF,
956 window: DEFAULT_RESTART_WINDOW,
957 }
958 }
959}
960
961#[derive(Debug, Clone, Copy, PartialEq, Eq)]
962struct CrashRestartSchedule {
963 restart_in_window: u32,
964 delay: Duration,
965}
966
967fn daemon_will_restart(
974 state: &mut SupervisorSnapshot,
975 policy: &RestartPolicy,
976 now: Instant,
977) -> bool {
978 state.enabled && state.crash_restarts_in_window(policy.window, now) < policy.max_restarts
979}
980
981const DEFAULT_HEALTH_CADENCE: Duration = Duration::from_secs(30);
982const DEFAULT_HEALTH_DEADLINE: Duration = Duration::from_secs(5);
983const DEFAULT_HEALTH_FAILURE_THRESHOLD: u32 = 3;
984const MAX_HEALTH_METRICS_BYTES: usize = 16 * 1024;
985
986#[derive(Debug, Clone, Copy, PartialEq, Eq)]
987pub enum HealthAction {
988 Report,
989 Restart,
990 Alert,
991}
992
993impl fmt::Display for HealthAction {
994 fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
995 f.write_str(match self {
996 Self::Report => "report",
997 Self::Restart => "restart",
998 Self::Alert => "alert",
999 })
1000 }
1001}
1002
1003#[derive(Debug, Clone, PartialEq, Eq)]
1004pub struct HealthConfig {
1005 pub http: Option<String>,
1009 pub cadence: Duration,
1010 pub deadline: Duration,
1011 pub failure_threshold: u32,
1012 pub on_degraded: HealthAction,
1013 pub on_failing: HealthAction,
1014 pub critical: bool,
1015}
1016
1017impl Default for HealthConfig {
1018 fn default() -> Self {
1019 Self {
1020 http: None,
1021 cadence: DEFAULT_HEALTH_CADENCE,
1022 deadline: DEFAULT_HEALTH_DEADLINE,
1023 failure_threshold: DEFAULT_HEALTH_FAILURE_THRESHOLD,
1024 on_degraded: HealthAction::Report,
1025 on_failing: HealthAction::Report,
1026 critical: false,
1027 }
1028 }
1029}
1030
1031#[derive(Debug, Clone, PartialEq)]
1049pub struct ModuleHealthStatus {
1050 pub status: SupervisorHealthStatus,
1051 pub last_probe_ms: Option<u64>,
1052 pub detail: Option<String>,
1053 pub metrics: Option<Value>,
1054 pub consecutive_failures: u32,
1055 pub late_answer_count: u64,
1058 pub last_late_answer_latency_ms: Option<u64>,
1060 pub last_action: Option<String>,
1061 pub last_action_ms: Option<u64>,
1065}
1066
1067impl Default for ModuleHealthStatus {
1068 fn default() -> Self {
1069 Self {
1070 status: SupervisorHealthStatus::Unknown,
1071 last_probe_ms: None,
1072 detail: None,
1073 metrics: None,
1074 consecutive_failures: 0,
1075 late_answer_count: 0,
1076 last_late_answer_latency_ms: None,
1077 last_action: None,
1078 last_action_ms: None,
1079 }
1080 }
1081}
1082
1083#[derive(Debug, Clone, Copy, PartialEq, Eq)]
1085pub enum ModuleState {
1086 Starting,
1087 Running,
1088 Unresponsive,
1089 Restarting,
1090 Draining,
1091 Stopped,
1092 Failed,
1093 Disabled,
1094}
1095
1096impl fmt::Display for ModuleState {
1097 fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
1098 f.write_str(match self {
1099 Self::Starting => "starting",
1100 Self::Running => "running",
1101 Self::Unresponsive => "unresponsive",
1102 Self::Restarting => "restarting",
1103 Self::Draining => "draining",
1104 Self::Stopped => "stopped",
1105 Self::Failed => "failed",
1106 Self::Disabled => "disabled",
1107 })
1108 }
1109}
1110
1111#[derive(Debug, Clone, Copy, PartialEq, Eq)]
1113pub enum ExitKind {
1114 Clean,
1115 Crash,
1116 DeliberateSeverance,
1117}
1118
1119impl From<ExitKind> for TerminalExitKind {
1120 fn from(kind: ExitKind) -> Self {
1121 match kind {
1122 ExitKind::Clean => Self::Clean,
1123 ExitKind::Crash => Self::Crash,
1124 ExitKind::DeliberateSeverance => Self::DeliberateSeverance,
1125 }
1126 }
1127}
1128
1129#[derive(Debug, Clone, Copy, PartialEq, Eq)]
1132pub(crate) struct ProcessIdentity {
1133 pub(crate) pid: u32,
1134 pub(crate) start_time: u64,
1135}
1136
1137#[derive(Debug, Clone, PartialEq, Eq)]
1139pub struct ExitReport {
1140 pub kind: ExitKind,
1141 pub code: Option<i32>,
1142 pub signal: Option<i32>,
1143 pub at_ms: u64,
1144}
1145
1146#[derive(Debug, Clone, PartialEq)]
1149pub struct ModuleStatus {
1150 pub module_id: String,
1151 pub waiting_for_previous_pid: Option<u32>,
1153 pub waiting_for_previous_detail: Option<String>,
1155 pub launch_nonce_source: Option<String>,
1160 pub state: ModuleState,
1161 pub enabled: bool,
1162 pub process_alive: bool,
1163 pub registration_active: bool,
1164 pub protocol: ModuleProtocol,
1169 pub live: bool,
1180 pub restart_count: u32,
1184 pub lifetime_restarts: u32,
1188 pub spawn_generation: u64,
1189 pub max_restarts: u32,
1194 pub restart_window: Duration,
1198 pub drain_timeout: Duration,
1202 pub restart_backoff: Duration,
1203 pub restart_max_backoff: Duration,
1204 pub pid: Option<u32>,
1209 pub spawned_at_ms: Option<u64>,
1210 pub spawned_from: Option<PathBuf>,
1211 pub(crate) launch_spec_digest: Option<LaunchSpecDigest>,
1212 pub process_start_time: Option<u64>,
1213 pub last_exit: Option<ExitReport>,
1214 pub health: ModuleHealthStatus,
1215}
1216
1217#[cfg(test)]
1218#[derive(Debug, Clone, Copy, PartialEq)]
1219struct ActorSelectCheckpoint {
1220 generation: u64,
1221 turn: u64,
1222 registered_connection: Option<ConnectionId>,
1223 next_probe_at: Option<Instant>,
1224 wake_after: Option<Duration>,
1225}
1226
1227#[derive(Debug, Clone, PartialEq)]
1228struct SupervisorSnapshot {
1229 exec_outcome: Option<Arc<crate::activity::ExecOutcome>>,
1230 #[cfg(windows)]
1231 pipe_consumed: Option<Arc<OnceLock<()>>>,
1232 #[cfg(test)]
1234 actor_turns: u64,
1235 #[cfg(test)]
1238 actor_select: Option<ActorSelectCheckpoint>,
1239 state: ModuleState,
1240 enabled: bool,
1241 process_alive: bool,
1242 spawned_protocol: Option<ModuleProtocol>,
1243 spawn_failure: Option<String>,
1244 waiting_for_previous_pid: Option<u32>,
1245 waiting_for_previous_detail: Option<String>,
1246 crash_restarts: VecDeque<Instant>,
1252 lifetime_restarts: u32,
1253 spawn_generation: u64,
1262 pid: Option<u32>,
1263 #[cfg(target_os = "macos")]
1264 report_ready: Option<Arc<OnceLock<()>>>,
1265 reaped_pid: Option<u32>,
1267 respawn_pending: bool,
1269 coalesced_restart_pending: bool,
1271 spawned_at_ms: Option<u64>,
1272 spawned_from: Option<PathBuf>,
1273 launch_spec_digest: Option<LaunchSpecDigest>,
1274 spawned_file_identity: Option<SpawnedFileIdentity>,
1275 process_start_time: Option<u64>,
1276 deliberate_severance: Option<ProcessIdentity>,
1277 last_exit: Option<ExitReport>,
1278 drain_disposition_detail: Option<String>,
1280 health: ModuleHealthStatus,
1281 in_alternate_slot: bool,
1286 draining_to_replace: bool,
1293 configuration_updated_since_spawn: bool,
1299}
1300
1301impl SupervisorSnapshot {
1302 fn reported_pid(&self) -> Option<u32> {
1307 #[cfg(target_os = "macos")]
1308 if self
1309 .report_ready
1310 .as_ref()
1311 .is_some_and(|ready| ready.get().is_none())
1312 {
1313 return None;
1314 }
1315 self.pid
1316 }
1317
1318 fn starting() -> Self {
1319 Self::new(ModuleState::Starting, true)
1320 }
1321
1322 fn disabled() -> Self {
1323 Self::new(ModuleState::Disabled, false)
1324 }
1325
1326 fn crash_restarts_in_window(&mut self, window: Duration, now: Instant) -> u32 {
1330 while let Some(oldest) = self.crash_restarts.front() {
1331 if now.duration_since(*oldest) > window {
1332 self.crash_restarts.pop_front();
1333 } else {
1334 break;
1335 }
1336 }
1337 u32::try_from(self.crash_restarts.len()).unwrap_or(u32::MAX)
1338 }
1339
1340 fn record_crash_restart(&mut self, policy: &RestartPolicy, now: Instant) {
1346 self.crash_restarts.push_back(now);
1347 while self.crash_restarts.len() > policy.max_restarts as usize {
1348 self.crash_restarts.pop_front();
1349 }
1350 self.lifetime_restarts += 1;
1351 }
1352
1353 fn next_crash_restart(
1357 &mut self,
1358 policy: &RestartPolicy,
1359 now: Instant,
1360 ) -> Option<CrashRestartSchedule> {
1361 let restart_in_window = self.crash_restarts_in_window(policy.window, now);
1362 if restart_in_window >= policy.max_restarts {
1363 return None;
1364 }
1365 self.record_crash_restart(policy, now);
1366 Some(CrashRestartSchedule {
1367 restart_in_window,
1368 delay: policy.delay_for_restart(restart_in_window),
1369 })
1370 }
1371
1372 fn clear_crash_restarts(&mut self) {
1377 self.crash_restarts.clear();
1378 }
1379
1380 fn new(state: ModuleState, enabled: bool) -> Self {
1381 Self {
1382 exec_outcome: None,
1383 #[cfg(test)]
1384 actor_turns: 0,
1385 #[cfg(test)]
1386 actor_select: None,
1387 state,
1388 enabled,
1389 process_alive: false,
1390 #[cfg(windows)]
1391 pipe_consumed: None,
1392 spawned_protocol: None,
1393 spawn_failure: None,
1394 waiting_for_previous_pid: None,
1395 waiting_for_previous_detail: None,
1396 crash_restarts: VecDeque::new(),
1397 lifetime_restarts: 0,
1398 spawn_generation: 0,
1399 pid: None,
1400 #[cfg(target_os = "macos")]
1401 report_ready: None,
1402 reaped_pid: None,
1403 respawn_pending: false,
1404 coalesced_restart_pending: false,
1405 spawned_at_ms: None,
1406 spawned_from: None,
1407 launch_spec_digest: None,
1408 spawned_file_identity: None,
1409 process_start_time: None,
1410 deliberate_severance: None,
1411 last_exit: None,
1412 drain_disposition_detail: None,
1413 health: ModuleHealthStatus::default(),
1414 in_alternate_slot: false,
1415 draining_to_replace: false,
1416 configuration_updated_since_spawn: false,
1417 }
1418 }
1419}
1420
1421type SharedSnapshot = Arc<Mutex<SupervisorSnapshot>>;
1422
1423type SpawnSubscriberKey = (ConnectionId, u64);
1424
1425#[derive(Debug)]
1426struct SpawnSubscriber {
1427 version: u8,
1428 frames: mpsc::Sender<Frame>,
1429 lagged: Option<oneshot::Sender<SpawnCursor>>,
1433}
1434
1435#[derive(Debug)]
1436struct SpawnEventState {
1437 daemon_incarnation: String,
1438 seq: u64,
1439 capacity: usize,
1440 live: HashMap<String, LiveSpawn>,
1441 generations: HashMap<String, u64>,
1442 events: VecDeque<SpawnEvent>,
1443 subscribers: HashMap<SpawnSubscriberKey, SpawnSubscriber>,
1444}
1445
1446impl Default for SpawnEventState {
1447 fn default() -> Self {
1448 Self {
1449 daemon_incarnation: "unconfigured".to_string(),
1450 seq: 0,
1451 capacity: SPAWN_EVENT_RING_CAPACITY,
1452 live: HashMap::new(),
1453 generations: HashMap::new(),
1454 events: VecDeque::new(),
1455 subscribers: HashMap::new(),
1456 }
1457 }
1458}
1459
1460#[derive(Debug, Clone, Default)]
1461struct SpawnEventFeed(
1462 Arc<Mutex<SpawnEventState>>,
1463 #[cfg(test)] Arc<Mutex<Option<std::net::TcpStream>>>,
1464);
1465
1466#[derive(Debug, Clone, PartialEq, Eq)]
1467pub(crate) enum SpawnSubscribeRefusal {
1468 ForeignIncarnation { current: String },
1469 TooOld { oldest: SpawnCursor },
1470 Frame(String),
1471}
1472
1473impl SpawnEventFeed {
1474 fn configure_incarnation(&self, daemon_incarnation: String) {
1475 let mut state = self.0.lock().unwrap_or_else(|p| p.into_inner());
1476 state.daemon_incarnation = daemon_incarnation;
1477 state.seq = 0;
1478 state.live.clear();
1479 state.generations.clear();
1480 state.events.clear();
1481 state.subscribers.clear();
1482 }
1483
1484 fn cursor(state: &SpawnEventState) -> SpawnCursor {
1485 SpawnCursor {
1486 daemon_incarnation: state.daemon_incarnation.clone(),
1487 seq: state.seq,
1488 }
1489 }
1490
1491 fn snapshot(&self) -> SpawnSnapshot {
1492 let state = self.0.lock().unwrap_or_else(|p| p.into_inner());
1493 let mut live = state.live.values().cloned().collect::<Vec<_>>();
1494 live.sort_by(|left, right| left.module_id.cmp(&right.module_id));
1495 SpawnSnapshot {
1496 cursor: Self::cursor(&state),
1497 ring_bound: state.capacity as u64,
1498 live,
1499 }
1500 }
1501
1502 fn emit_spawned(&self, module_id: &str, pid: u32, spawned_at_ms: u64) -> u64 {
1503 let mut state = self.0.lock().unwrap_or_else(|p| p.into_inner());
1504 let generation = state
1505 .generations
1506 .get(module_id)
1507 .copied()
1508 .unwrap_or(0)
1509 .checked_add(1)
1510 .expect("spawn generation exhausted");
1511 state.generations.insert(module_id.to_string(), generation);
1512 let live = LiveSpawn {
1513 module_id: module_id.to_string(),
1514 spawn_generation: generation,
1515 pid,
1516 spawned_at_ms,
1517 };
1518 state.live.insert(module_id.to_string(), live);
1519 Self::emit_locked(
1520 &mut state,
1521 SpawnEventKind::Spawned,
1522 module_id.to_string(),
1523 generation,
1524 pid,
1525 None,
1526 None,
1527 );
1528 generation
1529 }
1530
1531 fn emit_exited(&self, module_id: &str, exit_code: Option<i32>, exit_signal: Option<i32>) {
1532 let mut state = self.0.lock().unwrap_or_else(|p| p.into_inner());
1533 let Some(live) = state.live.remove(module_id) else {
1534 warn!(
1535 module_id,
1536 "terminal record had no live spawn event identity"
1537 );
1538 return;
1539 };
1540 Self::emit_locked(
1541 &mut state,
1542 SpawnEventKind::Exited,
1543 module_id.to_string(),
1544 live.spawn_generation,
1545 live.pid,
1546 exit_code,
1547 exit_signal,
1548 );
1549 }
1550
1551 fn emit_superseded_exited(
1558 &self,
1559 module_id: &str,
1560 spawn_generation: u64,
1561 pid: u32,
1562 exit_code: Option<i32>,
1563 exit_signal: Option<i32>,
1564 ) {
1565 let mut state = self.0.lock().unwrap_or_else(|p| p.into_inner());
1566 if state
1567 .live
1568 .get(module_id)
1569 .is_some_and(|live| live.spawn_generation == spawn_generation)
1570 {
1571 state.live.remove(module_id);
1572 }
1573 Self::emit_locked(
1574 &mut state,
1575 SpawnEventKind::Exited,
1576 module_id.to_string(),
1577 spawn_generation,
1578 pid,
1579 exit_code,
1580 exit_signal,
1581 );
1582 }
1583
1584 #[allow(clippy::too_many_arguments)]
1585 fn emit_locked(
1586 state: &mut SpawnEventState,
1587 kind: SpawnEventKind,
1588 module_id: String,
1589 spawn_generation: u64,
1590 pid: u32,
1591 exit_code: Option<i32>,
1592 exit_signal: Option<i32>,
1593 ) {
1594 state.seq = state
1595 .seq
1596 .checked_add(1)
1597 .expect("spawn event sequence exhausted");
1598 let event = SpawnEvent {
1599 cursor: Self::cursor(state),
1600 kind,
1601 module_id,
1602 spawn_generation,
1603 pid,
1604 exit_code,
1605 exit_signal,
1606 };
1607 state.events.push_back(event.clone());
1608 while state.events.len() > state.capacity {
1609 state.events.pop_front();
1610 }
1611 let body = match serde_json::to_vec(&event) {
1612 Ok(body) => body,
1613 Err(error) => {
1614 error!(%error, "failed to serialize supervisor spawn event");
1615 return;
1616 }
1617 };
1618 state.subscribers.retain(|(connection_id, corr), subscriber| {
1619 let frame = Frame::build_with_version(
1620 subscriber.version,
1621 FrameType::StreamData,
1622 control_flags(),
1623 0,
1624 0,
1625 *corr,
1626 body.clone(),
1627 );
1628 match frame {
1629 Ok(frame) => {
1630 if subscriber.frames.try_send(frame).is_ok() {
1631 true
1632 } else {
1633 warn!(connection_id = connection_id.get(), corr, "dropping lagged supervisor spawn subscriber");
1634 if let Some(lagged) = subscriber.lagged.take() {
1635 let _ = lagged.send(event.cursor.clone());
1636 }
1637 false
1638 }
1639 }
1640 Err(error) => {
1641 warn!(connection_id = connection_id.get(), corr, %error, "dropping supervisor spawn subscriber after frame build failure");
1642 false
1643 }
1644 }
1645 });
1646 }
1647
1648 fn subscribe(
1649 &self,
1650 connection_id: ConnectionId,
1651 corr: u64,
1652 version: u8,
1653 since: Option<SpawnCursor>,
1654 sink: FrameSink,
1655 ) -> Result<(), SpawnSubscribeRefusal> {
1656 let (frames, mut receiver) = mpsc::channel(SPAWN_SUBSCRIBER_BUFFER);
1657 let (lagged, mut lagged_rx) = oneshot::channel::<SpawnCursor>();
1658 {
1659 let mut state = self.0.lock().unwrap_or_else(|p| p.into_inner());
1660 let replay = if let Some(since) = since {
1661 if since.daemon_incarnation != state.daemon_incarnation {
1662 return Err(SpawnSubscribeRefusal::ForeignIncarnation {
1663 current: state.daemon_incarnation.clone(),
1664 });
1665 }
1666 if let Some(oldest) = state.events.front().map(|event| event.cursor.clone()) {
1667 if since.seq < oldest.seq.saturating_sub(1) {
1668 return Err(SpawnSubscribeRefusal::TooOld { oldest });
1669 }
1670 }
1671 state
1672 .events
1673 .iter()
1674 .filter(|event| event.cursor.seq > since.seq)
1675 .cloned()
1676 .collect::<Vec<_>>()
1677 } else {
1678 Vec::new()
1679 };
1680 for event in replay {
1681 let body = serde_json::to_vec(&event)
1682 .map_err(|error| SpawnSubscribeRefusal::Frame(error.to_string()))?;
1683 let frame = Frame::build_with_version(
1684 version,
1685 FrameType::StreamData,
1686 control_flags(),
1687 0,
1688 0,
1689 corr,
1690 body,
1691 )
1692 .map_err(|error| SpawnSubscribeRefusal::Frame(error.to_string()))?;
1693 frames
1694 .try_send(frame)
1695 .map_err(|error| SpawnSubscribeRefusal::Frame(error.to_string()))?;
1696 }
1697 state.subscribers.insert(
1698 (connection_id, corr),
1699 SpawnSubscriber {
1700 version,
1701 frames,
1702 lagged: Some(lagged),
1703 },
1704 );
1705 }
1706 tokio::spawn(async move {
1717 while let Some(frame) = receiver.recv().await {
1718 if sink.send(frame).await.is_err() {
1719 return;
1720 }
1721 }
1722 let Ok(first_undelivered) = lagged_rx.try_recv() else {
1723 return;
1724 };
1725 match spawn_subscriber_lagged_frame(version, corr, first_undelivered) {
1726 Ok(frame) => {
1727 let _ = sink.send(frame).await;
1728 }
1729 Err(error) => {
1730 error!(%error, corr, "failed to build lagged spawn subscriber terminal frame");
1731 }
1732 }
1733 });
1734 Ok(())
1735 }
1736
1737 fn cancel(&self, connection_id: ConnectionId, corr: u64) -> bool {
1738 let Some(subscriber) = self
1739 .0
1740 .lock()
1741 .unwrap_or_else(|p| p.into_inner())
1742 .subscribers
1743 .remove(&(connection_id, corr))
1744 else {
1745 return false;
1746 };
1747 if let Ok(frame) = Frame::build_with_version(
1748 subscriber.version,
1749 FrameType::StreamEnd,
1750 control_flags(),
1751 0,
1752 0,
1753 corr,
1754 Vec::new(),
1755 ) {
1756 tokio::spawn(async move {
1757 let _ = subscriber.frames.send(frame).await;
1758 });
1759 }
1760 true
1761 }
1762
1763 fn remove_connection(&self, connection_id: ConnectionId) {
1764 self.0
1765 .lock()
1766 .unwrap_or_else(|p| p.into_inner())
1767 .subscribers
1768 .retain(|(subscriber_connection, _), _| *subscriber_connection != connection_id);
1769 }
1770
1771 #[cfg(any(test, feature = "test-support"))]
1772 fn set_capacity(&self, capacity: usize) {
1773 self.0.lock().unwrap_or_else(|p| p.into_inner()).capacity = capacity;
1774 }
1775
1776 #[cfg(any(test, feature = "test-support"))]
1777 fn subscriber_count(&self) -> usize {
1778 self.0
1779 .lock()
1780 .unwrap_or_else(|p| p.into_inner())
1781 .subscribers
1782 .len()
1783 }
1784}
1785
1786fn spawn_subscriber_lagged_frame(
1789 version: u8,
1790 corr: u64,
1791 first_undelivered: SpawnCursor,
1792) -> Result<Frame, String> {
1793 let body = serde_json::to_vec(&subc_protocol::ErrorBody {
1794 code: SPAWN_SUBSCRIBER_LAGGED_CODE.to_string(),
1795 message: "spawn subscriber fell behind and was dropped; resubscribe from the last cursor received"
1796 .to_string(),
1797 detail: Some(serde_json::json!({
1798 "first_undelivered_cursor": first_undelivered
1799 })),
1800 })
1801 .map_err(|error| error.to_string())?;
1802 Frame::build_with_version(version, FrameType::Error, control_flags(), 0, 0, corr, body)
1803 .map_err(|error| error.to_string())
1804}
1805
1806pub trait ModuleProcessLiveness: Send + Sync {
1807 fn process_live(&self, module_id: &str) -> Option<bool>;
1808
1809 fn process_replacing(&self, _module_id: &str) -> bool {
1815 false
1816 }
1817}
1818
1819#[derive(Debug, Clone, Default)]
1821pub struct SupervisorProcessLiveness {
1822 snapshots: Arc<Mutex<HashMap<String, SharedSnapshot>>>,
1823}
1824
1825impl SupervisorProcessLiveness {
1826 pub fn new() -> Self {
1827 Self::default()
1828 }
1829
1830 fn track(&self, module_id: String, snapshot: SharedSnapshot) {
1831 let mut snapshots = self
1832 .snapshots
1833 .lock()
1834 .unwrap_or_else(|poisoned| poisoned.into_inner());
1835 snapshots.insert(module_id, snapshot);
1836 }
1837
1838 fn untrack_if_current(&self, module_id: &str, snapshot: &SharedSnapshot) {
1839 let mut snapshots = self
1840 .snapshots
1841 .lock()
1842 .unwrap_or_else(|poisoned| poisoned.into_inner());
1843 let is_current = snapshots
1844 .get(module_id)
1845 .map(|tracked| Arc::ptr_eq(tracked, snapshot))
1846 .unwrap_or(false);
1847 if is_current {
1848 snapshots.remove(module_id);
1849 }
1850 }
1851}
1852
1853impl ModuleProcessLiveness for SupervisorProcessLiveness {
1854 fn process_live(&self, module_id: &str) -> Option<bool> {
1855 let snapshot = {
1856 let snapshots = self
1857 .snapshots
1858 .lock()
1859 .unwrap_or_else(|poisoned| poisoned.into_inner());
1860 snapshots.get(module_id).cloned()
1861 }?;
1862 let snapshot = snapshot
1863 .lock()
1864 .unwrap_or_else(|poisoned| poisoned.into_inner());
1865 Some(snapshot.state == ModuleState::Running && snapshot.process_alive)
1866 }
1867
1868 fn process_replacing(&self, module_id: &str) -> bool {
1869 let Some(snapshot) = self
1870 .snapshots
1871 .lock()
1872 .unwrap_or_else(|poisoned| poisoned.into_inner())
1873 .get(module_id)
1874 .cloned()
1875 else {
1876 return false;
1877 };
1878 let snapshot = snapshot
1879 .lock()
1880 .unwrap_or_else(|poisoned| poisoned.into_inner());
1881 snapshot.enabled
1882 && match snapshot.state {
1883 ModuleState::Restarting => true,
1884 ModuleState::Draining => snapshot.draining_to_replace,
1885 ModuleState::Starting
1886 | ModuleState::Running
1887 | ModuleState::Unresponsive
1888 | ModuleState::Stopped
1889 | ModuleState::Failed
1890 | ModuleState::Disabled => false,
1891 }
1892 }
1893}
1894
1895#[cfg(test)]
1896#[derive(Debug, Default)]
1897struct ReloadExitRecordGate {
1898 reached: tokio::sync::Notify,
1899 resume: tokio::sync::Notify,
1900}
1901
1902#[derive(Debug, Clone, Copy)]
1903enum RespawnKind {
1904 Spawn,
1905 Reload,
1906}
1907
1908#[derive(Debug, Clone, Copy)]
1909struct PendingRespawn {
1910 deadline: Instant,
1911 kind: RespawnKind,
1912}
1913
1914type ReloadReply = oneshot::Sender<Result<(), SuperviseError>>;
1915
1916#[derive(Debug, Clone)]
1917struct SupervisorRuntimeConfig {
1918 start_requester: Option<Principal>,
1919 pending_activity_restart: bool,
1920 provenance_probe: ExecutableIdentityProbe,
1921 scheduled_respawn: Arc<Mutex<Option<PendingRespawn>>>,
1923 deferred_reload_reply: Arc<Mutex<Option<ReloadReply>>>,
1925 restart_policy: RestartPolicy,
1926 drain_timeout: Duration,
1929 effective_drain_timeout: Arc<Mutex<Duration>>,
1932 default_drain_timeout: Duration,
1935 health: HealthConfig,
1936 connection_file_path: Option<PathBuf>,
1937 capture_logs_dir: Option<PathBuf>,
1938 forwarding: Option<Arc<ForwardingTable>>,
1939 supervisor_handle: Option<SupervisorHandle>,
1942 stderr_ring: Arc<Mutex<StderrRing>>,
1949 terminal_ring: Arc<Mutex<TerminalRing>>,
1950 spawn_events: SpawnEventFeed,
1951 child_roster: ChildRoster,
1952 #[cfg(target_os = "linux")]
1953 cgroup_placement: Option<subc_cgroup::Placement>,
1954 #[cfg(test)]
1955 test_seed_stale_facts_before_enable_spawn: bool,
1956 #[cfg(test)]
1957 test_reload_exit_record_gate: Option<Arc<ReloadExitRecordGate>>,
1958}
1959
1960#[derive(Debug, Clone, PartialEq, Eq)]
1961struct SupervisedConfiguration {
1962 spec: ModuleSpec,
1963 health: HealthConfig,
1964}
1965
1966#[derive(Debug, Clone, Default)]
1972pub struct SupervisorHandle {
1973 activity: Arc<Mutex<Option<crate::activity::ActivityLedger>>>,
1974 activity_images: Arc<Mutex<HashMap<String, activity::ActivityImage>>>,
1975 modules: Arc<Mutex<HashMap<String, SupervisedModule>>>,
1976 configured_ids: Arc<Mutex<HashSet<String>>>,
1988 spawn_events: SpawnEventFeed,
1989 reserved_nonces: Arc<Mutex<HashMap<String, Option<String>>>>,
2000 removal_tombstones: Arc<Mutex<HashMap<String, u64>>>,
2006 spawn_nonces: Arc<Mutex<HashMap<String, String>>>,
2010 reserved_prefix_owners: Arc<Mutex<HashMap<String, String>>>,
2018 swaps: Arc<Mutex<HashMap<String, OpenSwap>>>,
2024 promotion_observer: PromotionObserverSlot,
2026 operation_lock: Arc<AsyncMutex<()>>,
2030}
2031
2032pub(crate) trait SwapPromotionObserver: Send + Sync {
2041 fn swap_promoted(&self, registration: &crate::registry::ModuleRegistration);
2042}
2043
2044#[derive(Clone, Default)]
2048struct PromotionObserverSlot(Arc<Mutex<Option<std::sync::Weak<dyn SwapPromotionObserver>>>>);
2049
2050impl fmt::Debug for PromotionObserverSlot {
2051 fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
2052 f.write_str("PromotionObserverSlot")
2053 }
2054}
2055
2056#[derive(Debug, Clone)]
2058struct OpenSwap {
2059 candidate_nonce: String,
2062 incumbent_nonce: Option<String>,
2067 candidate_admitted: bool,
2071}
2072
2073#[derive(Debug, Clone, Copy, PartialEq, Eq)]
2076pub(crate) enum SwapHelloAdmission {
2077 NotSwapping,
2080 Candidate,
2082 Refused,
2085}
2086
2087#[derive(Debug, Clone, PartialEq, Eq)]
2088pub(crate) enum ReservedHelloRejection {
2089 Exact {
2090 module_id: String,
2091 },
2092 Prefix {
2093 prefix: String,
2094 owner_module_id: String,
2095 },
2096}
2097
2098impl SupervisorHandle {
2099 pub fn new() -> Self {
2100 Self::default()
2101 }
2102
2103 pub(crate) fn spawn_snapshot(&self) -> SpawnSnapshot {
2104 self.spawn_events.snapshot()
2105 }
2106
2107 pub(crate) fn subscribe_spawns(
2108 &self,
2109 connection_id: ConnectionId,
2110 corr: u64,
2111 version: u8,
2112 since: Option<SpawnCursor>,
2113 sink: FrameSink,
2114 ) -> Result<(), SpawnSubscribeRefusal> {
2115 self.spawn_events
2116 .subscribe(connection_id, corr, version, since, sink)
2117 }
2118
2119 pub(crate) fn cancel_spawn_subscription(&self, connection_id: ConnectionId, corr: u64) -> bool {
2120 self.spawn_events.cancel(connection_id, corr)
2121 }
2122
2123 pub(crate) fn remove_spawn_subscribers(&self, connection_id: ConnectionId) {
2124 self.spawn_events.remove_connection(connection_id);
2125 }
2126
2127 #[cfg(any(test, feature = "test-support"))]
2128 pub fn set_spawn_event_capacity_for_test(&self, capacity: usize) {
2129 assert!(capacity > 0, "spawn event capacity must be non-zero");
2130 self.spawn_events.set_capacity(capacity);
2131 }
2132
2133 #[cfg(any(test, feature = "test-support"))]
2134 pub fn spawn_subscriber_count_for_test(&self) -> usize {
2135 self.spawn_events.subscriber_count()
2136 }
2137
2138 pub fn set_spawn_nonce(&self, module_id: &str, nonce: String) {
2141 self.spawn_nonces
2142 .lock()
2143 .unwrap_or_else(|poisoned| poisoned.into_inner())
2144 .insert(module_id.to_string(), nonce);
2145 }
2146
2147 pub fn set_reserved_nonce(&self, module_id: &str, nonce: String) {
2150 self.reserved_nonces
2151 .lock()
2152 .unwrap_or_else(|poisoned| poisoned.into_inner())
2153 .insert(module_id.to_string(), Some(nonce));
2154 }
2155
2156 pub fn set_reserved_prefixes(&self, owner_module_id: &str, prefixes: &[String]) {
2158 let mut owners = self
2159 .reserved_prefix_owners
2160 .lock()
2161 .unwrap_or_else(|poisoned| poisoned.into_inner());
2162 owners.retain(|_, owner| owner != owner_module_id);
2163 for prefix in prefixes {
2164 owners.insert(prefix.clone(), owner_module_id.to_string());
2165 }
2166 }
2167
2168 #[cfg(test)]
2170 pub(crate) fn spawn_nonce(&self, module_id: &str) -> Option<String> {
2171 self.spawn_nonces
2172 .lock()
2173 .unwrap_or_else(|poisoned| poisoned.into_inner())
2174 .get(module_id)
2175 .cloned()
2176 }
2177
2178 fn apply_identity_configuration(&self, spec: &ModuleSpec) {
2179 self.set_reserved_prefixes(&spec.module_id, &spec.reserved_prefixes);
2180 let spawn_nonce = self
2181 .spawn_nonces
2182 .lock()
2183 .unwrap_or_else(|poisoned| poisoned.into_inner())
2184 .get(&spec.module_id)
2185 .cloned();
2186 let mut reserved_nonces = self
2187 .reserved_nonces
2188 .lock()
2189 .unwrap_or_else(|poisoned| poisoned.into_inner());
2190 if spec.reserved {
2191 reserved_nonces.insert(spec.module_id.clone(), spawn_nonce);
2196 }
2197 drop(reserved_nonces);
2198 self.removal_tombstones
2202 .lock()
2203 .unwrap_or_else(|poisoned| poisoned.into_inner())
2204 .remove(&spec.module_id);
2205 }
2206
2207 pub fn reserved_hello_authorized(&self, module_id: &str, presented: Option<&str>) -> bool {
2212 self.reserved_hello_verification(module_id, presented)
2213 .is_ok()
2214 }
2215
2216 pub(crate) fn reserved_hello_verification(
2219 &self,
2220 module_id: &str,
2221 presented: Option<&str>,
2222 ) -> Result<bool, ReservedHelloRejection> {
2223 let nonces = self
2224 .reserved_nonces
2225 .lock()
2226 .unwrap_or_else(|poisoned| poisoned.into_inner());
2227 if let Some(expected) = nonces.get(module_id) {
2228 let authorized = match expected {
2232 Some(expected) => {
2233 presented.is_some_and(|p| constant_time_eq(expected.as_bytes(), p.as_bytes()))
2234 }
2235 None => false,
2236 };
2237 if authorized {
2238 return Ok(true);
2239 }
2240 return Err(ReservedHelloRejection::Exact {
2241 module_id: module_id.to_string(),
2242 });
2243 }
2244 drop(nonces);
2245
2246 let matched_prefix = self
2247 .reserved_prefix_owners
2248 .lock()
2249 .unwrap_or_else(|poisoned| poisoned.into_inner())
2250 .iter()
2251 .filter(|(prefix, _)| module_id.starts_with(prefix.as_str()))
2252 .max_by_key(|(prefix, _)| prefix.len())
2253 .map(|(prefix, owner)| (prefix.clone(), owner.clone()));
2254 let Some((prefix, owner_module_id)) = matched_prefix else {
2255 return Ok(false);
2256 };
2257
2258 let authorized = presented.is_some_and(|presented| {
2259 self.spawn_nonces
2260 .lock()
2261 .unwrap_or_else(|poisoned| poisoned.into_inner())
2262 .get(&owner_module_id)
2263 .is_some_and(|expected| constant_time_eq(expected.as_bytes(), presented.as_bytes()))
2264 || self.swap_nonce_matches(&owner_module_id, presented)
2267 });
2268 if authorized {
2269 Ok(true)
2270 } else {
2271 Err(ReservedHelloRejection::Prefix {
2272 prefix,
2273 owner_module_id,
2274 })
2275 }
2276 }
2277
2278 pub fn spawned_consumer_authorized(&self, module_id: &str, presented: &str) -> bool {
2283 if presented.is_empty() {
2284 return false;
2285 }
2286 let nonces = self
2287 .spawn_nonces
2288 .lock()
2289 .unwrap_or_else(|poisoned| poisoned.into_inner());
2290 let current = nonces
2291 .get(module_id)
2292 .is_some_and(|expected| constant_time_eq(expected.as_bytes(), presented.as_bytes()));
2293 drop(nonces);
2294 current || self.swap_nonce_matches(module_id, presented)
2299 }
2300
2301 fn swap_nonce_matches(&self, module_id: &str, presented: &str) -> bool {
2303 let swaps = self
2304 .swaps
2305 .lock()
2306 .unwrap_or_else(|poisoned| poisoned.into_inner());
2307 swaps.get(module_id).is_some_and(|swap| {
2308 constant_time_eq(swap.candidate_nonce.as_bytes(), presented.as_bytes())
2309 || swap.incumbent_nonce.as_deref().is_some_and(|incumbent| {
2310 constant_time_eq(incumbent.as_bytes(), presented.as_bytes())
2311 })
2312 })
2313 }
2314
2315 pub(crate) fn open_swap(&self, module_id: &str, candidate_nonce: String) {
2318 let incumbent_nonce = self
2319 .spawn_nonces
2320 .lock()
2321 .unwrap_or_else(|poisoned| poisoned.into_inner())
2322 .get(module_id)
2323 .cloned();
2324 self.swaps
2325 .lock()
2326 .unwrap_or_else(|poisoned| poisoned.into_inner())
2327 .insert(
2328 module_id.to_string(),
2329 OpenSwap {
2330 candidate_nonce,
2331 incumbent_nonce,
2332 candidate_admitted: false,
2333 },
2334 );
2335 }
2336
2337 pub(crate) fn close_swap(&self, module_id: &str) {
2340 self.swaps
2341 .lock()
2342 .unwrap_or_else(|poisoned| poisoned.into_inner())
2343 .remove(module_id);
2344 }
2345
2346 pub(crate) fn set_swap_promotion_observer(
2349 &self,
2350 observer: std::sync::Weak<dyn SwapPromotionObserver>,
2351 ) {
2352 *self
2353 .promotion_observer
2354 .0
2355 .lock()
2356 .unwrap_or_else(|poisoned| poisoned.into_inner()) = Some(observer);
2357 }
2358
2359 fn notify_swap_promoted(&self, registration: &crate::registry::ModuleRegistration) {
2362 let observer = self
2363 .promotion_observer
2364 .0
2365 .lock()
2366 .unwrap_or_else(|poisoned| poisoned.into_inner())
2367 .as_ref()
2368 .and_then(std::sync::Weak::upgrade);
2369 if let Some(observer) = observer {
2370 observer.swap_promoted(registration);
2371 }
2372 }
2373
2374 pub(crate) fn swap_open(&self, module_id: &str) -> bool {
2376 self.swaps
2377 .lock()
2378 .unwrap_or_else(|poisoned| poisoned.into_inner())
2379 .contains_key(module_id)
2380 }
2381
2382 fn promote_swap_nonce(&self, module_id: &str, reserved: bool) {
2387 let candidate_nonce = self
2388 .swaps
2389 .lock()
2390 .unwrap_or_else(|poisoned| poisoned.into_inner())
2391 .get(module_id)
2392 .map(|swap| swap.candidate_nonce.clone());
2393 let Some(nonce) = candidate_nonce else {
2394 return;
2395 };
2396 self.set_spawn_nonce(module_id, nonce.clone());
2397 if reserved {
2398 self.set_reserved_nonce(module_id, nonce);
2399 }
2400 }
2401
2402 pub(crate) fn swap_hello_admission(
2417 &self,
2418 module_id: &str,
2419 presented: Option<&str>,
2420 ) -> SwapHelloAdmission {
2421 let swaps = self
2422 .swaps
2423 .lock()
2424 .unwrap_or_else(|poisoned| poisoned.into_inner());
2425 let Some(swap) = swaps.get(module_id) else {
2426 return SwapHelloAdmission::NotSwapping;
2427 };
2428 let Some(presented) = presented else {
2429 return SwapHelloAdmission::Refused;
2430 };
2431 if constant_time_eq(swap.candidate_nonce.as_bytes(), presented.as_bytes()) {
2432 return if swap.candidate_admitted {
2433 SwapHelloAdmission::Refused
2434 } else {
2435 SwapHelloAdmission::Candidate
2436 };
2437 }
2438 if swap
2439 .incumbent_nonce
2440 .as_deref()
2441 .is_some_and(|incumbent| constant_time_eq(incumbent.as_bytes(), presented.as_bytes()))
2442 {
2443 return SwapHelloAdmission::NotSwapping;
2444 }
2445 SwapHelloAdmission::Refused
2446 }
2447
2448 pub(crate) fn mark_swap_candidate_admitted(&self, module_id: &str) {
2451 if let Some(swap) = self
2452 .swaps
2453 .lock()
2454 .unwrap_or_else(|poisoned| poisoned.into_inner())
2455 .get_mut(module_id)
2456 {
2457 swap.candidate_admitted = true;
2458 }
2459 }
2460
2461 pub fn spawn_launch_nonce_for(&self, module_id: &str) -> Option<String> {
2463 self.spawn_nonces
2464 .lock()
2465 .unwrap_or_else(|poisoned| poisoned.into_inner())
2466 .get(module_id)
2467 .cloned()
2468 }
2469
2470 pub fn reserved_launch_nonce_for(&self, module_id: &str) -> Option<String> {
2472 self.reserved_nonces
2473 .lock()
2474 .unwrap_or_else(|poisoned| poisoned.into_inner())
2475 .get(module_id)
2476 .cloned()
2477 .flatten()
2478 }
2479
2480 pub fn insert(&self, module: SupervisedModule) -> Option<SupervisedModule> {
2481 self.mark_configured(module.module_id());
2485 let mut modules = self
2486 .modules
2487 .lock()
2488 .unwrap_or_else(|poisoned| poisoned.into_inner());
2489 modules.insert(module.module_id().to_string(), module)
2490 }
2491
2492 fn mark_configured(&self, module_id: &str) {
2496 self.configured_ids
2497 .lock()
2498 .unwrap_or_else(|poisoned| poisoned.into_inner())
2499 .insert(module_id.to_string());
2500 }
2501
2502 fn unmark_configured_unless_rostered(&self, module_id: &str) {
2506 let modules = self
2507 .modules
2508 .lock()
2509 .unwrap_or_else(|poisoned| poisoned.into_inner());
2510 if !modules.contains_key(module_id) {
2511 self.configured_ids
2512 .lock()
2513 .unwrap_or_else(|poisoned| poisoned.into_inner())
2514 .remove(module_id);
2515 }
2516 }
2517
2518 pub(crate) fn is_configured(&self, module_id: &str) -> bool {
2526 self.configured_ids
2527 .lock()
2528 .unwrap_or_else(|poisoned| poisoned.into_inner())
2529 .contains(module_id)
2530 }
2531
2532 pub fn get(&self, module_id: &str) -> Option<SupervisedModule> {
2533 let modules = self
2534 .modules
2535 .lock()
2536 .unwrap_or_else(|poisoned| poisoned.into_inner());
2537 modules.get(module_id).cloned()
2538 }
2539
2540 pub(crate) fn record_late_health_answer(
2541 &self,
2542 module_id: &str,
2543 latency_ms: u64,
2544 ) -> Result<bool, SuperviseError> {
2545 let Some(module) = self.get(module_id) else {
2546 return Ok(false);
2547 };
2548 update_snapshot(&module.inner.snapshot, Some(module_id), |state| {
2549 state.health.late_answer_count = state.health.late_answer_count.saturating_add(1);
2550 state.health.last_late_answer_latency_ms = Some(latency_ms);
2551 state.health.consecutive_failures = 0;
2559 })?;
2560 Ok(true)
2561 }
2562
2563 pub fn record_deliberate_severance(&self, module_id: &str) -> Result<bool, SuperviseError> {
2569 let Some(module) = self.get(module_id) else {
2570 return Ok(false);
2571 };
2572 let snapshot = lock_snapshot(&module.inner.snapshot)?;
2573 let Some((pid, start_time)) = snapshot.pid.zip(snapshot.process_start_time) else {
2574 return Ok(false);
2575 };
2576 drop(snapshot);
2577 module.record_deliberate_severance(ProcessIdentity { pid, start_time })
2578 }
2579
2580 pub fn list(&self) -> Vec<SupervisedModule> {
2581 let modules = self
2582 .modules
2583 .lock()
2584 .unwrap_or_else(|poisoned| poisoned.into_inner());
2585 let mut modules = modules.values().cloned().collect::<Vec<_>>();
2586 modules.sort_by(|left, right| left.module_id().cmp(right.module_id()));
2587 modules
2588 }
2589
2590 pub(crate) fn retire(&self, module_id: &str) -> Option<SupervisedModule> {
2591 self.activity_images
2592 .lock()
2593 .unwrap_or_else(|p| p.into_inner())
2594 .remove(module_id);
2595 self.spawn_nonces
2596 .lock()
2597 .unwrap_or_else(|poisoned| poisoned.into_inner())
2598 .remove(module_id);
2599 self.close_swap(module_id);
2600 let mut reserved_nonces = self
2601 .reserved_nonces
2602 .lock()
2603 .unwrap_or_else(|poisoned| poisoned.into_inner());
2604 if reserved_nonces.contains_key(module_id) {
2605 reserved_nonces.insert(module_id.to_string(), None);
2608 }
2609 drop(reserved_nonces);
2610 self.reserved_prefix_owners
2611 .lock()
2612 .unwrap_or_else(|poisoned| poisoned.into_inner())
2613 .retain(|_, owner| owner != module_id);
2614 let removed = self
2615 .modules
2616 .lock()
2617 .unwrap_or_else(|poisoned| poisoned.into_inner())
2618 .remove(module_id);
2619 self.configured_ids
2620 .lock()
2621 .unwrap_or_else(|poisoned| poisoned.into_inner())
2622 .remove(module_id);
2623 removed
2624 }
2625
2626 pub(crate) fn record_rescan_removal(&self, module_id: &str) {
2629 self.removal_tombstones
2630 .lock()
2631 .unwrap_or_else(|poisoned| poisoned.into_inner())
2632 .insert(module_id.to_string(), unix_ms_now());
2633 }
2634
2635 pub(crate) fn removal_tombstone_age_ms(&self, module_id: &str) -> Option<u64> {
2637 self.removal_tombstones
2638 .lock()
2639 .unwrap_or_else(|poisoned| poisoned.into_inner())
2640 .get(module_id)
2641 .copied()
2642 .map(|removed_at_ms| unix_ms_now().saturating_sub(removed_at_ms))
2643 }
2644
2645 pub(crate) fn release_retained_reserved_gate(&self, module_id: &str) -> bool {
2650 if self.get(module_id).is_some() {
2651 return false;
2652 }
2653 let mut reserved_nonces = self
2654 .reserved_nonces
2655 .lock()
2656 .unwrap_or_else(|poisoned| poisoned.into_inner());
2657 if !matches!(reserved_nonces.get(module_id), Some(None)) {
2658 return false;
2659 }
2660 reserved_nonces.remove(module_id);
2661 true
2662 }
2663
2664 pub(crate) fn operation_lock(&self) -> Arc<AsyncMutex<()>> {
2665 Arc::clone(&self.operation_lock)
2666 }
2667}
2668
2669#[derive(Debug, Clone)]
2671pub struct Supervisor {
2672 registry: Arc<Registry>,
2673 restart_policy: RestartPolicy,
2674 drain_timeout: Duration,
2675 connection_file_path: Option<PathBuf>,
2676 capture_logs_dir: Option<PathBuf>,
2677 forwarding: Option<Arc<ForwardingTable>>,
2678 process_liveness: Arc<SupervisorProcessLiveness>,
2679 supervisor_handle: Option<SupervisorHandle>,
2680 health: HealthConfig,
2681 daemon_start_clock: crate::clock::StartClock,
2682 terminal_journal: Option<Arc<crate::terminal_journal::TerminalJournal>>,
2683 spawn_events: SpawnEventFeed,
2684 provenance_probe: ExecutableIdentityProbe,
2685 child_roster: ChildRoster,
2688 #[cfg(target_os = "linux")]
2689 cgroup_placement: Option<subc_cgroup::Placement>,
2690 #[cfg(test)]
2691 test_after_first_spawn: AfterFirstSpawnHook,
2692}
2693
2694pub(crate) struct RegisteredConfiguredModule {
2698 spec: ModuleSpec,
2699 runtime: SupervisorRuntimeConfig,
2700 module: SupervisedModule,
2701 commands: mpsc::Receiver<SupervisorCommand>,
2702}
2703
2704#[cfg(test)]
2710#[derive(Clone, Default)]
2711struct AfterFirstSpawnHook(Option<AfterFirstSpawnFn>);
2712
2713#[cfg(test)]
2714type AfterFirstSpawnFn = Arc<dyn Fn(&str) + Send + Sync>;
2715
2716#[cfg(test)]
2717impl fmt::Debug for AfterFirstSpawnHook {
2718 fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
2719 f.write_str("AfterFirstSpawnHook")
2720 }
2721}
2722
2723#[cfg(test)]
2724impl AfterFirstSpawnHook {
2725 fn run(&self, module_id: &str) {
2726 if let Some(hook) = &self.0 {
2727 hook(module_id);
2728 }
2729 }
2730}
2731
2732impl Supervisor {
2733 #[cfg(test)]
2734 pub(crate) fn new_for_test(registry: Arc<Registry>, policy: RestartPolicy) -> Self {
2735 let supervisor = Self::new(registry, policy);
2736 #[cfg(target_os = "macos")]
2737 let supervisor = supervisor.with_privacy_trampoline(test_privacy_trampoline());
2738 supervisor
2739 }
2740 pub fn with_privacy_trampoline(self, path: impl Into<PathBuf>) -> Self {
2745 let path = path.into();
2746 #[cfg(target_os = "macos")]
2747 {
2748 let result = probe_privacy_trampoline(&path).map(|()| path);
2749 if let Err(cause) = &result {
2750 error!(%cause, "macOS privacy identity unavailable; supervised launches will refuse");
2751 }
2752 self.child_roster.set_privacy_trampoline(result);
2753 }
2754 #[cfg(not(target_os = "macos"))]
2755 let _ = path;
2756 self
2757 }
2758 pub(crate) fn daemon_shutdown_flag(&self) -> crate::child_roster::DaemonShutdownFlag {
2761 self.child_roster.shutdown_flag()
2762 }
2763
2764 #[cfg(unix)]
2775 pub(crate) fn begin_daemon_shutdown(&self) {
2776 self.child_roster.close();
2777 if let Some(journal) = &self.terminal_journal {
2778 journal.stamp_shutdown();
2779 }
2780 }
2781
2782 #[cfg(unix)]
2786 pub(crate) async fn drain_for_daemon_shutdown(&self) -> Result<(), SuperviseError> {
2787 const NOTICE_BUDGET: Duration = Duration::from_millis(500);
2788 const DRAIN_BUDGET: Duration = Duration::from_secs(2);
2789 let Some(forwarding) = &self.forwarding else {
2790 return Ok(());
2791 };
2792 let modules = forwarding
2793 .begin_daemon_drain()
2794 .map_err(SuperviseError::Forwarding)?;
2795 let deadline_ms =
2796 unix_ms_now().saturating_add((NOTICE_BUDGET + DRAIN_BUDGET).as_millis() as u64);
2797 let mut notices = tokio::task::JoinSet::new();
2798 let mut drains = Vec::new();
2799 for (module_id, endpoint) in modules {
2800 let Some(target) = forwarding
2801 .begin_endpoint_drain(endpoint, RouteCloseReason::Restart)
2802 .map_err(SuperviseError::Forwarding)?
2803 else {
2804 continue;
2805 };
2806 let routes = forwarding
2807 .endpoint_routes(target.endpoint)
2808 .map_err(SuperviseError::Forwarding)?;
2809 let command = serde_json::to_vec(&ModuleControlCommand::Draining {
2815 reason: RouteCloseReason::Restart,
2816 deadline_ms,
2817 })
2818 .expect("module draining serializes");
2819 let mut recipients = vec![(target.sink.clone(), target.negotiated_ver, command)];
2820 let mut clients: Vec<(crate::forwarding::GoodbyeTarget, Vec<u16>)> = Vec::new();
2821 for route in routes {
2822 let client = route.goodbye_target;
2823 if let Some((_, channels)) = clients
2824 .iter_mut()
2825 .find(|(existing, _)| existing.connection_id == client.connection_id)
2826 {
2827 channels.push(client.channel);
2828 } else {
2829 let channel = client.channel;
2830 clients.push((client, vec![channel]));
2831 }
2832 }
2833 for (client, mut channels) in clients {
2834 channels.sort_unstable();
2835 channels.dedup();
2836 let closing = serde_json::to_vec(&ClientControlPush::RouteClosing {
2837 module_id: module_id.clone(),
2838 channels,
2839 reason: RouteCloseReason::Restart,
2840 })
2841 .expect("route closing serializes");
2842 recipients.push((client.sink, client.negotiated_ver, closing));
2843 }
2844 for (sink, version, body) in recipients {
2845 notices.spawn(async move {
2846 let frame = Frame::build_with_version(
2847 version,
2848 FrameType::Push,
2849 control_flags(),
2850 0,
2851 0,
2852 0,
2853 body,
2854 )
2855 .expect("bounded lifecycle notice frame builds");
2856 sink.send_flushed(frame).await
2857 });
2858 }
2859 let gauges = declared_busy_gauges(&self.registry, &module_id)?;
2860 drains.push((module_id, target.endpoint, gauges));
2861 }
2862 let notice_deadline = Instant::now() + NOTICE_BUDGET;
2865 while let Ok(Some(result)) = timeout_at(notice_deadline, notices.join_next()).await {
2866 if !matches!(result, Ok(Ok(()))) {
2867 warn!(?result, "daemon shutdown notice delivery failed");
2868 }
2869 }
2870 notices.abort_all();
2871 let deadline = Instant::now() + DRAIN_BUDGET;
2872 let mut waits = tokio::task::JoinSet::new();
2873 for (module_id, endpoint, gauges) in drains {
2874 let forwarding = Arc::clone(forwarding);
2875 let mut runtime = self.runtime_config();
2876 runtime.health.cadence = Duration::from_millis(100);
2877 waits.spawn(async move {
2878 let wait_result = wait_for_forwarding_quiescence(
2879 &forwarding,
2880 &module_id,
2881 &runtime,
2882 endpoint,
2883 deadline,
2884 &gauges,
2885 DrainScope::Endpoint(endpoint),
2886 &mut DrainGaugeTrace::new(&gauges),
2887 )
2888 .await;
2889 forwarding
2893 .record_daemon_drain_result(
2894 endpoint,
2895 drained_after_quiescence_wait(&wait_result),
2896 )
2897 .map_err(SuperviseError::Forwarding)?;
2898 wait_result
2899 });
2900 }
2901 while let Ok(Some(result)) = timeout_at(deadline, waits.join_next()).await {
2902 if !matches!(result, Ok(Ok(true))) {
2903 warn!(?result, "daemon shutdown drain did not reach quiescence");
2904 }
2905 }
2906 Ok(())
2907 }
2908
2909 #[cfg(unix)]
2921 pub(crate) async fn end_children_for_daemon_shutdown(
2922 &self,
2923 already_escalated: bool,
2924 escalate: impl std::future::Future<Output = ()>,
2925 ) {
2926 tokio::pin!(escalate);
2927 let mut escalated = already_escalated;
2928 if let Some(forwarding) = &self.forwarding {
2929 let reason = CloseReason::new(
2930 "daemon_shutdown",
2931 "the daemon is exiting after its shutdown notice and drain",
2932 );
2933 if escalated {
2934 send_module_goodbyes_for_daemon_shutdown(forwarding, &reason, false).await;
2937 } else {
2938 tokio::select! {
2939 biased;
2940 _ = escalate.as_mut() => {
2941 info!("second SIGTERM: abandoning module GOODBYE delivery");
2942 escalated = true;
2943 }
2944 _ = send_module_goodbyes_for_daemon_shutdown(forwarding, &reason, true) => {}
2945 }
2946 }
2947 let closed = forwarding.close_all_connections(&reason);
2948 debug!(closed, "closed established connections for daemon shutdown");
2949 }
2950 let escalated_here = escalated && !already_escalated;
2954 let remaining_escalate = async move {
2955 if escalated_here {
2956 std::future::pending::<()>().await;
2957 } else {
2958 escalate.await;
2959 }
2960 };
2961 crate::child_roster::end_children_for_daemon_shutdown(
2962 &self.child_roster,
2963 escalated,
2964 remaining_escalate,
2965 )
2966 .await;
2967 }
2968
2969 pub fn new(registry: Arc<Registry>, restart_policy: RestartPolicy) -> Self {
2970 Self {
2971 registry,
2972 restart_policy,
2973 drain_timeout: DEFAULT_DRAIN_TIMEOUT,
2974 connection_file_path: None,
2975 capture_logs_dir: None,
2976 forwarding: None,
2977 process_liveness: Arc::new(SupervisorProcessLiveness::default()),
2978 supervisor_handle: None,
2979 health: HealthConfig::default(),
2980 daemon_start_clock: crate::clock::StartClock::capture(),
2981 terminal_journal: None,
2982 spawn_events: SpawnEventFeed::default(),
2983 provenance_probe: ExecutableIdentityProbe::default(),
2984 child_roster: ChildRoster::default(),
2985 #[cfg(target_os = "linux")]
2986 cgroup_placement: None,
2987 #[cfg(test)]
2988 test_after_first_spawn: AfterFirstSpawnHook::default(),
2989 }
2990 }
2991
2992 pub fn with_drain_timeout(mut self, drain_timeout: Duration) -> Self {
2993 self.drain_timeout = drain_timeout;
2994 self
2995 }
2996
2997 pub(crate) fn default_drain_timeout(&self) -> Duration {
2999 self.drain_timeout
3000 }
3001
3002 pub fn with_process_liveness(
3003 mut self,
3004 process_liveness: Arc<SupervisorProcessLiveness>,
3005 ) -> Self {
3006 self.process_liveness = process_liveness;
3007 self
3008 }
3009
3010 pub fn with_connection_file_path(mut self, connection_file_path: impl Into<PathBuf>) -> Self {
3011 self.connection_file_path = Some(connection_file_path.into());
3012 self
3013 }
3014
3015 pub fn with_capture_logs_dir(mut self, logs_dir: impl Into<PathBuf>) -> Self {
3017 self.capture_logs_dir = Some(logs_dir.into());
3018 self
3019 }
3020
3021 pub fn with_daemon_incarnation(self, daemon_incarnation: String) -> Self {
3024 self.spawn_events.configure_incarnation(daemon_incarnation);
3028 self
3029 }
3030
3031 pub fn with_terminal_journal(self, path: PathBuf, daemon_incarnation: String) -> Self {
3034 let mut this = self.with_daemon_incarnation(daemon_incarnation.clone());
3035 this.terminal_journal = Some(Arc::new(crate::terminal_journal::TerminalJournal::open(
3036 path,
3037 daemon_incarnation,
3038 )));
3039 this
3040 }
3041
3042 pub fn with_forwarding(mut self, forwarding: Arc<ForwardingTable>) -> Self {
3043 self.forwarding = Some(forwarding);
3044 self
3045 }
3046
3047 pub fn with_handle(mut self, supervisor_handle: SupervisorHandle) -> Self {
3048 self.spawn_events = supervisor_handle.spawn_events.clone();
3049 self.supervisor_handle = Some(supervisor_handle);
3050 self
3051 }
3052
3053 pub fn with_health_config(mut self, health: HealthConfig) -> Self {
3054 self.health = health;
3055 self
3056 }
3057
3058 pub fn with_live_children_record(self, path: impl Into<PathBuf>) -> Self {
3062 self.child_roster.record_to(path.into());
3063 self
3064 }
3065
3066 #[cfg(target_os = "linux")]
3067 pub fn with_cgroup_placement(
3068 mut self,
3069 cgroup_placement: Option<subc_cgroup::Placement>,
3070 ) -> Self {
3071 self.cgroup_placement = cgroup_placement;
3072 self
3073 }
3074
3075 pub fn spawn(&self, spec: ModuleSpec) -> Result<SupervisedModule, SuperviseError> {
3081 validate_spec(&spec)?;
3082 self.establish_identity(&spec);
3083
3084 let runtime = self.runtime_config();
3085 let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::starting()));
3086 let activity_spawn = runtime.begin_activity_spawn(&spec.module_id);
3087 let spawned = spawn_child(
3088 &spec,
3089 runtime.connection_file_path.as_deref(),
3090 self.supervisor_handle.as_ref(),
3091 &runtime.stderr_ring,
3092 runtime.capture_logs_dir.as_deref(),
3093 &runtime.child_roster,
3094 #[cfg(target_os = "linux")]
3095 runtime.cgroup_placement.as_ref(),
3096 );
3097 #[cfg(test)]
3098 self.test_after_first_spawn.run(&spec.module_id);
3099 let child = match spawned {
3100 Ok(child) => child,
3101 Err(err) => {
3102 self.abandon_unrostered(&spec.module_id);
3105 return Err(err);
3106 }
3107 };
3108 self.mark_first_process_running(&spec, &runtime, &snapshot, &child)?;
3109 if let Some(spawn) = activity_spawn {
3110 spawn.complete(&snapshot);
3111 }
3112
3113 Ok(self.supervised_module(spec, runtime, snapshot, Some(child)))
3114 }
3115
3116 fn establish_identity(&self, spec: &ModuleSpec) {
3129 if let Some(supervisor_handle) = &self.supervisor_handle {
3130 supervisor_handle.apply_identity_configuration(spec);
3131 supervisor_handle.mark_configured(&spec.module_id);
3132 }
3133 }
3134
3135 fn abandon_unrostered(&self, module_id: &str) {
3138 if let Some(supervisor_handle) = &self.supervisor_handle {
3139 supervisor_handle.unmark_configured_unless_rostered(module_id);
3140 }
3141 }
3142
3143 fn mark_first_process_running(
3146 &self,
3147 spec: &ModuleSpec,
3148 runtime: &SupervisorRuntimeConfig,
3149 snapshot: &SharedSnapshot,
3150 child: &SupervisedChild,
3151 ) -> Result<(), SuperviseError> {
3152 if let Err(err) = set_running(snapshot, child, &spec.module_id, &runtime.spawn_events) {
3153 self.abandon_unrostered(&spec.module_id);
3154 return Err(err);
3155 }
3156 self.process_liveness
3157 .track(spec.module_id.clone(), Arc::clone(snapshot));
3158 Ok(())
3159 }
3160
3161 pub fn supervise_configured(
3167 &self,
3168 spec: ModuleSpec,
3169 enabled: bool,
3170 ) -> Result<SupervisedModule, SuperviseError> {
3171 self.supervise_configured_requested(
3172 spec,
3173 enabled,
3174 self.health.clone(),
3175 None,
3176 self.restart_policy,
3177 None,
3178 )
3179 }
3180
3181 pub fn supervise_configured_with_health(
3187 &self,
3188 spec: ModuleSpec,
3189 enabled: bool,
3190 health: HealthConfig,
3191 drain_timeout_ms: Option<u64>,
3192 restart_policy: RestartPolicy,
3193 ) -> Result<SupervisedModule, SuperviseError> {
3194 self.supervise_configured_requested(
3195 spec,
3196 enabled,
3197 health,
3198 drain_timeout_ms,
3199 restart_policy,
3200 None,
3201 )
3202 }
3203
3204 pub(crate) fn supervise_configured_requested(
3205 &self,
3206 spec: ModuleSpec,
3207 enabled: bool,
3208 health: HealthConfig,
3209 drain_timeout_ms: Option<u64>,
3210 restart_policy: RestartPolicy,
3211 requester: Option<Principal>,
3212 ) -> Result<SupervisedModule, SuperviseError> {
3213 let registered = self.register_configured_with_health(
3214 spec,
3215 enabled,
3216 health,
3217 drain_timeout_ms,
3218 restart_policy,
3219 )?;
3220 self.start_registered_configured(registered, requester)
3221 }
3222
3223 pub(crate) fn register_configured_with_health(
3228 &self,
3229 spec: ModuleSpec,
3230 enabled: bool,
3231 health: HealthConfig,
3232 drain_timeout_ms: Option<u64>,
3233 restart_policy: RestartPolicy,
3234 ) -> Result<RegisteredConfiguredModule, SuperviseError> {
3235 validate_spec(&spec)?;
3236 self.establish_identity(&spec);
3237
3238 let mut runtime = self.runtime_config();
3239 runtime.health = health;
3240 runtime.restart_policy = restart_policy;
3241 if let Some(ms) = drain_timeout_ms {
3242 runtime.drain_timeout = Duration::from_millis(ms);
3243 *runtime
3244 .effective_drain_timeout
3245 .lock()
3246 .unwrap_or_else(|poisoned| poisoned.into_inner()) = runtime.drain_timeout;
3247 }
3248 let snapshot = if enabled {
3249 SupervisorSnapshot::starting()
3250 } else {
3251 SupervisorSnapshot::disabled()
3252 };
3253 let snapshot = {
3254 let mut snapshot = snapshot;
3255 if enabled {
3256 if let Some((pid, reason)) = runtime.child_roster.previous_instance(&spec.module_id)
3257 {
3258 snapshot.waiting_for_previous_pid = Some(pid);
3259 snapshot.waiting_for_previous_detail = Some(reason.waiting_message(pid));
3260 }
3261 }
3262 snapshot
3263 };
3264 let (module, commands) =
3265 self.register_module(&spec, &runtime, Arc::new(Mutex::new(snapshot)));
3266 Ok(RegisteredConfiguredModule {
3267 spec,
3268 runtime,
3269 module,
3270 commands,
3271 })
3272 }
3273
3274 pub(crate) fn start_registered_configured(
3277 &self,
3278 registered: RegisteredConfiguredModule,
3279 requester: Option<Principal>,
3280 ) -> Result<SupervisedModule, SuperviseError> {
3281 let RegisteredConfiguredModule {
3282 spec,
3283 mut runtime,
3284 module,
3285 commands,
3286 } = registered;
3287 runtime.start_requester = requester;
3288 let snapshot = &module.inner.snapshot;
3289 let should_spawn = {
3290 let state = lock_snapshot(snapshot)?;
3291 state.enabled && state.waiting_for_previous_pid.is_none()
3292 };
3293 let child = if should_spawn {
3294 let activity_spawn = runtime.begin_activity_spawn(&spec.module_id);
3295 let spawned = spawn_child(
3296 &spec,
3297 runtime.connection_file_path.as_deref(),
3298 self.supervisor_handle.as_ref(),
3299 &runtime.stderr_ring,
3300 runtime.capture_logs_dir.as_deref(),
3301 &runtime.child_roster,
3302 #[cfg(target_os = "linux")]
3303 runtime.cgroup_placement.as_ref(),
3304 );
3305 #[cfg(test)]
3306 self.test_after_first_spawn.run(&spec.module_id);
3307 match spawned {
3308 Ok(child) => {
3309 self.mark_first_process_running(&spec, &runtime, snapshot, &child)?;
3310 if let Some(spawn) = activity_spawn {
3311 spawn.complete(snapshot);
3312 }
3313 Some(child)
3314 }
3315 Err(err) => {
3316 if runtime.health.critical {
3317 error!(
3318 module_id = %spec.module_id,
3319 program = %spec.program.display(),
3320 error = %err,
3321 "critical configured module failed to spawn; marking failed and alerting"
3322 );
3323 } else {
3324 error!(
3325 module_id = %spec.module_id,
3326 program = %spec.program.display(),
3327 error = %err,
3328 "configured module failed to spawn; marking failed and continuing"
3329 );
3330 }
3331 update_snapshot(snapshot, Some(&spec.module_id), |state| {
3332 state.state = ModuleState::Failed;
3333 clear_current_process_facts(state);
3334 })?;
3335 None
3336 }
3337 }
3338 } else {
3339 None
3340 };
3341 self.start_monitor(spec, runtime, &module, child, commands);
3342 Ok(module)
3343 }
3344
3345 fn runtime_config(&self) -> SupervisorRuntimeConfig {
3346 let effective_drain_timeout = Arc::new(Mutex::new(self.drain_timeout));
3347 SupervisorRuntimeConfig {
3348 start_requester: None,
3349 pending_activity_restart: false,
3350 provenance_probe: self.provenance_probe.clone(),
3351 scheduled_respawn: Arc::default(),
3352 deferred_reload_reply: Arc::default(),
3353 restart_policy: self.restart_policy,
3354 drain_timeout: self.drain_timeout,
3355 child_roster: self
3358 .child_roster
3359 .for_module(Arc::clone(&effective_drain_timeout)),
3360 effective_drain_timeout,
3361 default_drain_timeout: self.drain_timeout,
3362 health: self.health.clone(),
3363 connection_file_path: self.connection_file_path.clone(),
3364 capture_logs_dir: self.capture_logs_dir.clone(),
3365 forwarding: self.forwarding.clone(),
3366 supervisor_handle: self.supervisor_handle.clone(),
3367 stderr_ring: Arc::new(Mutex::new(StderrRing::new(StderrTailConfig::default()))),
3368 terminal_ring: Arc::new(Mutex::new(
3369 TerminalRing::new(
3370 TerminalRingConfig::default(),
3371 self.daemon_start_clock.started_at_ms(),
3372 )
3373 .with_start_clock(self.daemon_start_clock)
3374 .with_journal(self.terminal_journal.clone())
3375 .with_daemon_shutdown(self.child_roster.shutdown_flag()),
3376 )),
3377 spawn_events: self.spawn_events.clone(),
3378 #[cfg(target_os = "linux")]
3379 cgroup_placement: self.cgroup_placement.clone(),
3380 #[cfg(test)]
3381 test_seed_stale_facts_before_enable_spawn: false,
3382 #[cfg(test)]
3383 test_reload_exit_record_gate: None,
3384 }
3385 }
3386
3387 fn supervised_module(
3388 &self,
3389 spec: ModuleSpec,
3390 runtime: SupervisorRuntimeConfig,
3391 snapshot: SharedSnapshot,
3392 child: Option<SupervisedChild>,
3393 ) -> SupervisedModule {
3394 let (module, commands) = self.register_module(&spec, &runtime, snapshot);
3395 self.start_monitor(spec, runtime, &module, child, commands);
3396 module
3397 }
3398
3399 fn register_module(
3400 &self,
3401 spec: &ModuleSpec,
3402 runtime: &SupervisorRuntimeConfig,
3403 snapshot: SharedSnapshot,
3404 ) -> (SupervisedModule, mpsc::Receiver<SupervisorCommand>) {
3405 let configuration = Arc::new(Mutex::new(SupervisedConfiguration {
3406 spec: spec.clone(),
3407 health: runtime.health.clone(),
3408 }));
3409 let stderr_ring = Arc::clone(&runtime.stderr_ring);
3410 let terminal_ring = Arc::clone(&runtime.terminal_ring);
3411 let restart_policy = runtime.restart_policy;
3415 let effective_drain_timeout = Arc::clone(&runtime.effective_drain_timeout);
3416 let (tx, rx) = mpsc::channel(4);
3417 let module_id = spec.module_id.clone();
3418 let module = SupervisedModule {
3419 inner: Arc::new(SupervisedModuleInner {
3420 module_id: module_id.clone(),
3421 registry: Arc::clone(&self.registry),
3422 snapshot,
3423 configuration,
3424 stderr_ring,
3425 terminal_ring,
3426 commands: tx,
3427 monitor: Mutex::new(None),
3428 restart_policy,
3429 effective_drain_timeout,
3430 provenance_probe: self.provenance_probe.clone(),
3431 }),
3432 };
3433 if let Some(supervisor_handle) = &self.supervisor_handle {
3436 supervisor_handle.insert(module.clone());
3437 }
3438 (module, rx)
3439 }
3440
3441 fn start_monitor(
3442 &self,
3443 spec: ModuleSpec,
3444 runtime: SupervisorRuntimeConfig,
3445 module: &SupervisedModule,
3446 child: Option<SupervisedChild>,
3447 commands: mpsc::Receiver<SupervisorCommand>,
3448 ) {
3449 let monitor = tokio::spawn(supervise_loop(
3450 spec,
3451 runtime,
3452 Arc::clone(&self.registry),
3453 Arc::clone(&self.process_liveness),
3454 Arc::clone(&module.inner.snapshot),
3455 child,
3456 commands,
3457 ));
3458 *module
3459 .inner
3460 .monitor
3461 .lock()
3462 .unwrap_or_else(|p| p.into_inner()) = Some(monitor);
3463 }
3464}
3465
3466impl Default for Supervisor {
3467 fn default() -> Self {
3468 Self::new(Arc::new(Registry::default()), RestartPolicy::default())
3469 }
3470}
3471
3472#[derive(Clone)]
3474pub struct SupervisedModule {
3475 inner: Arc<SupervisedModuleInner>,
3476}
3477
3478struct SupervisedModuleInner {
3479 module_id: String,
3480 registry: Arc<Registry>,
3481 snapshot: SharedSnapshot,
3482 configuration: Arc<Mutex<SupervisedConfiguration>>,
3483 stderr_ring: Arc<Mutex<StderrRing>>,
3484 terminal_ring: Arc<Mutex<TerminalRing>>,
3485 commands: mpsc::Sender<SupervisorCommand>,
3486 monitor: Mutex<Option<JoinHandle<()>>>,
3487 restart_policy: RestartPolicy,
3491 effective_drain_timeout: Arc<Mutex<Duration>>,
3492 provenance_probe: ExecutableIdentityProbe,
3493}
3494
3495impl fmt::Debug for SupervisedModule {
3496 fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
3497 f.debug_struct("SupervisedModule")
3498 .field("module_id", &self.inner.module_id)
3499 .field("status", &self.status())
3500 .finish_non_exhaustive()
3501 }
3502}
3503
3504impl SupervisedModule {
3505 pub fn module_id(&self) -> &str {
3506 &self.inner.module_id
3507 }
3508
3509 #[cfg(test)]
3513 pub(crate) fn record_health_probe_failure_for_test(
3514 &self,
3515 detail: &str,
3516 ) -> Result<(), SuperviseError> {
3517 update_snapshot(&self.inner.snapshot, Some(&self.inner.module_id), |state| {
3518 state.health.consecutive_failures = state.health.consecutive_failures.saturating_add(1);
3519 state.health.detail = Some(detail.to_string());
3520 })
3521 }
3522
3523 pub fn state(&self) -> Result<ModuleState, SuperviseError> {
3524 Ok(lock_snapshot(&self.inner.snapshot)?.state)
3525 }
3526
3527 pub(crate) fn registration_end_reason(
3528 &self,
3529 ) -> Result<Option<RegistrationEndReason>, SuperviseError> {
3530 let snapshot = lock_snapshot(&self.inner.snapshot)?;
3531 Ok(match snapshot.state {
3532 ModuleState::Draining if snapshot.draining_to_replace => {
3533 Some(RegistrationEndReason::SupervisorRestart)
3534 }
3535 ModuleState::Draining => Some(RegistrationEndReason::SupervisorStop),
3536 ModuleState::Restarting if snapshot.draining_to_replace => {
3537 Some(RegistrationEndReason::SupervisorRestart)
3538 }
3539 _ => None,
3540 })
3541 }
3542
3543 pub fn stderr_tail(
3550 &self,
3551 max_lines: Option<usize>,
3552 max_bytes: Option<usize>,
3553 ) -> StderrTailSnapshot {
3554 self.inner
3555 .stderr_ring
3556 .lock()
3557 .unwrap_or_else(|poisoned| poisoned.into_inner())
3558 .snapshot(max_lines, max_bytes)
3559 }
3560
3561 pub fn terminal_history(&self) -> TerminalHistorySnapshot {
3566 self.inner
3567 .terminal_ring
3568 .lock()
3569 .unwrap_or_else(|poisoned| poisoned.into_inner())
3570 .snapshot()
3571 }
3572
3573 pub fn durable_terminal_history(&self) -> subc_control::TerminalHistory {
3578 durable_terminal_history_of(&self.inner.terminal_ring, &self.inner.module_id)
3579 }
3580
3581 pub(crate) async fn read_durable_terminal_history(
3586 &self,
3587 ) -> Result<subc_control::TerminalHistory, tokio::task::JoinError> {
3588 let terminal_ring = Arc::clone(&self.inner.terminal_ring);
3589 let module_id = self.inner.module_id.clone();
3590 tokio::task::spawn_blocking(move || durable_terminal_history_of(&terminal_ring, &module_id))
3591 .await
3592 }
3593
3594 pub fn status(&self) -> Result<ModuleStatus, SuperviseError> {
3595 self.status_with_snapshot_lock(&self.inner.snapshot, None)
3596 .map(|(status, _)| status)
3597 }
3598
3599 pub(crate) fn record_deliberate_severance(
3600 &self,
3601 identity: ProcessIdentity,
3602 ) -> Result<bool, SuperviseError> {
3603 let mut snapshot = lock_snapshot(&self.inner.snapshot)?;
3604 if snapshot.pid != Some(identity.pid)
3605 || snapshot.process_start_time != Some(identity.start_time)
3606 {
3607 return Ok(false);
3608 }
3609 snapshot.deliberate_severance = Some(identity);
3610 Ok(true)
3611 }
3612
3613 pub(crate) fn status_for_control(
3618 &self,
3619 caller: &'static str,
3620 ) -> Result<ModuleStatus, SuperviseError> {
3621 self.status_with_snapshot_lock(&self.inner.snapshot, Some(caller))
3622 .map(|(status, _)| status)
3623 }
3624
3625 fn status_with_snapshot_lock(
3626 &self,
3627 snapshot: &SharedSnapshot,
3628 caller: Option<&'static str>,
3629 ) -> Result<(ModuleStatus, Option<SpawnedFileIdentity>), SuperviseError> {
3630 let mut guard = match caller {
3631 Some(caller) => lock_snapshot_for_control(snapshot, &self.inner.module_id, caller)?,
3632 None => lock_snapshot(snapshot)?,
3633 };
3634 let restart_count =
3637 guard.crash_restarts_in_window(self.inner.restart_policy.window, Instant::now());
3638 let snapshot = guard.clone();
3639 drop(guard);
3640 let drain_timeout = *self.inner.effective_drain_timeout.lock().map_err(|_| {
3641 SuperviseError::StatePoisoned {
3642 module_id: Some(self.inner.module_id.clone()),
3643 }
3644 })?;
3645 let registration_active = self
3646 .inner
3647 .registry
3648 .get_module(&self.inner.module_id)
3649 .map_err(SuperviseError::Registry)?
3650 .is_some();
3651 let protocol = snapshot
3652 .spawned_protocol
3653 .unwrap_or(self.declared_protocol()?);
3654 let running_process =
3655 snapshot.enabled && snapshot.state == ModuleState::Running && snapshot.process_alive;
3656 let live = match protocol {
3662 ModuleProtocol::Subc => running_process && registration_active,
3663 ModuleProtocol::None => running_process,
3664 };
3665
3666 Ok((
3667 ModuleStatus {
3668 waiting_for_previous_pid: snapshot.waiting_for_previous_pid,
3669 waiting_for_previous_detail: snapshot.waiting_for_previous_detail.clone(),
3670 module_id: self.inner.module_id.clone(),
3671 launch_nonce_source: observed_launch_nonce_source(&snapshot),
3672 state: snapshot.state,
3673 enabled: snapshot.enabled,
3674 process_alive: snapshot.process_alive,
3675 registration_active,
3676 protocol,
3677 live,
3678 restart_count,
3679 lifetime_restarts: snapshot.lifetime_restarts,
3680 spawn_generation: snapshot.spawn_generation,
3681 max_restarts: self.inner.restart_policy.max_restarts,
3682 restart_window: self.inner.restart_policy.window,
3683 drain_timeout,
3684 restart_backoff: self.inner.restart_policy.backoff,
3685 restart_max_backoff: self.inner.restart_policy.max_backoff,
3686 pid: snapshot.reported_pid(),
3687 spawned_at_ms: snapshot.spawned_at_ms,
3688 spawned_from: snapshot.spawned_from,
3689 launch_spec_digest: snapshot.launch_spec_digest,
3690 process_start_time: snapshot.process_start_time,
3691 last_exit: snapshot.last_exit,
3692 health: snapshot.health,
3693 },
3694 snapshot.spawned_file_identity.clone(),
3695 ))
3696 }
3697
3698 #[cfg(test)]
3699 pub(crate) fn hold_snapshot_for_test(
3700 &self,
3701 acquired: std::sync::mpsc::Sender<()>,
3702 hold: Duration,
3703 ) -> std::thread::JoinHandle<()> {
3704 let snapshot = Arc::clone(&self.inner.snapshot);
3705 std::thread::spawn(move || {
3706 let _guard = snapshot.lock().expect("test snapshot lock is not poisoned");
3707 acquired
3708 .send(())
3709 .expect("test receiver waits for snapshot lock");
3710 std::thread::sleep(hold);
3711 })
3712 }
3713
3714 pub(crate) async fn status_and_running_image_agreement(
3720 &self,
3721 ) -> Result<(ModuleStatus, subc_control::RunningImageAgreement), SuperviseError> {
3722 let (status, identity) = self.status_with_snapshot_lock(&self.inner.snapshot, None)?;
3723 let image = self
3724 .inner
3725 .provenance_probe
3726 .observe(
3727 status.pid,
3728 status.spawned_from.as_deref(),
3729 identity,
3730 status.process_start_time,
3731 )
3732 .await;
3733 Ok((status, image))
3734 }
3735
3736 pub(crate) async fn running_image_agreement(&self) -> subc_control::RunningImageAgreement {
3737 let snapshot = match lock_snapshot(&self.inner.snapshot) {
3738 Ok(snapshot) => snapshot.clone(),
3739 Err(_) => {
3740 return subc_control::RunningImageAgreement::Unavailable {
3741 reason: subc_control::RunningImageUnavailableReason::NotRunning,
3742 };
3743 }
3744 };
3745 self.inner
3746 .provenance_probe
3747 .observe(
3748 snapshot.reported_pid(),
3749 snapshot.spawned_from.as_deref(),
3750 snapshot.spawned_file_identity.clone(),
3751 snapshot.process_start_time,
3752 )
3753 .await
3754 }
3755
3756 pub(crate) fn child_resource_usage(&self) -> subc_control::ChildResourceUsage {
3759 #[cfg(windows)]
3760 {
3761 let image = lock_snapshot(&self.inner.snapshot)
3762 .ok()
3763 .and_then(|snapshot| {
3764 snapshot
3765 .spawned_file_identity
3766 .as_ref()
3767 .and_then(|identity| identity.windows_image.clone())
3768 });
3769 match image {
3770 Some(image) => {
3771 crate::child_resources::read_windows_handle(image.process(), image.start_time())
3772 }
3773 None => subc_control::ChildResourceUsage::Unavailable {
3774 reason: subc_control::ChildResourceUnavailableReason::NotRunning,
3775 },
3776 }
3777 }
3778 #[cfg(not(windows))]
3779 {
3780 let (pid, start_time) = match lock_snapshot(&self.inner.snapshot) {
3781 Ok(snapshot) => (snapshot.reported_pid(), snapshot.process_start_time),
3782 Err(_) => {
3783 return subc_control::ChildResourceUsage::Unavailable {
3784 reason: subc_control::ChildResourceUnavailableReason::Unreadable,
3785 }
3786 }
3787 };
3788 crate::child_resources::read(pid, start_time)
3789 }
3790 }
3791
3792 pub(crate) fn will_recover_after_connection_loss(&self) -> Result<bool, SuperviseError> {
3793 let mut snapshot = lock_snapshot(&self.inner.snapshot)?;
3794 Ok(match snapshot.state {
3795 ModuleState::Restarting => true,
3796 ModuleState::Failed | ModuleState::Disabled => false,
3797 _ if snapshot.enabled && snapshot.respawn_pending => true,
3798 _ => daemon_will_restart(&mut snapshot, &self.inner.restart_policy, Instant::now()),
3799 })
3800 }
3801
3802 #[cfg(test)]
3803 pub(crate) fn is_warming(&self) -> Result<bool, SuperviseError> {
3804 self.is_warming_with_snapshot_lock(None)
3805 }
3806
3807 pub(crate) fn is_warming_for_control(
3808 &self,
3809 caller: &'static str,
3810 ) -> Result<bool, SuperviseError> {
3811 self.is_warming_with_snapshot_lock(Some(caller))
3812 }
3813
3814 fn is_warming_with_snapshot_lock(
3815 &self,
3816 caller: Option<&'static str>,
3817 ) -> Result<bool, SuperviseError> {
3818 let snapshot = match caller {
3819 Some(caller) => {
3820 lock_snapshot_for_control(&self.inner.snapshot, &self.inner.module_id, caller)?
3821 }
3822 None => lock_snapshot(&self.inner.snapshot)?,
3823 }
3824 .clone();
3825 Ok(matches!(
3826 snapshot.state,
3827 ModuleState::Starting | ModuleState::Running | ModuleState::Restarting
3828 ))
3829 }
3830
3831 pub async fn drain(&self) -> Result<(), SuperviseError> {
3833 self.stop().await
3834 }
3835
3836 pub(crate) async fn retire_requested(
3837 &self,
3838 requester: Option<Principal>,
3839 ) -> Result<(), SuperviseError> {
3840 match self.state()? {
3841 ModuleState::Stopped | ModuleState::Failed => return Ok(()),
3842 ModuleState::Starting
3843 | ModuleState::Running
3844 | ModuleState::Unresponsive
3845 | ModuleState::Restarting
3846 | ModuleState::Draining
3847 | ModuleState::Disabled => {}
3848 }
3849
3850 let (reply_tx, reply_rx) = oneshot::channel();
3851 self.inner
3852 .commands
3853 .send(SupervisorCommand::Retire {
3854 requester,
3855 reply: reply_tx,
3856 })
3857 .await
3858 .map_err(|_| SuperviseError::CommandClosed {
3859 module_id: self.inner.module_id.clone(),
3860 })?;
3861 reply_rx.await.map_err(|_| SuperviseError::CommandClosed {
3862 module_id: self.inner.module_id.clone(),
3863 })?
3864 }
3865
3866 pub async fn stop(&self) -> Result<(), SuperviseError> {
3867 self.stop_requested(None).await
3868 }
3869
3870 pub(crate) async fn stop_requested(
3871 &self,
3872 requester: Option<Principal>,
3873 ) -> Result<(), SuperviseError> {
3874 match self.state()? {
3875 ModuleState::Stopped | ModuleState::Failed => return Ok(()),
3876 ModuleState::Starting
3877 | ModuleState::Running
3878 | ModuleState::Unresponsive
3879 | ModuleState::Restarting
3880 | ModuleState::Draining
3881 | ModuleState::Disabled => {}
3882 }
3883
3884 let (reply_tx, reply_rx) = oneshot::channel();
3885 self.inner
3886 .commands
3887 .send(SupervisorCommand::Drain {
3888 requester,
3889 reply: reply_tx,
3890 })
3891 .await
3892 .map_err(|_| SuperviseError::CommandClosed {
3893 module_id: self.inner.module_id.clone(),
3894 })?;
3895 reply_rx.await.map_err(|_| SuperviseError::CommandClosed {
3896 module_id: self.inner.module_id.clone(),
3897 })?
3898 }
3899
3900 pub async fn restart(&self, drain_timeout_ms: Option<u64>) -> Result<(), SuperviseError> {
3901 self.restart_requested(drain_timeout_ms, None).await
3902 }
3903
3904 pub(crate) async fn restart_requested(
3905 &self,
3906 drain_timeout_ms: Option<u64>,
3907 requester: Option<Principal>,
3908 ) -> Result<(), SuperviseError> {
3909 let queued_at = Instant::now();
3910 let received_at_generation = lock_snapshot(&self.inner.snapshot)?.spawn_generation;
3911 let (reply_tx, reply_rx) = oneshot::channel();
3912 self.inner
3913 .commands
3914 .send(SupervisorCommand::Restart {
3915 requester,
3916 drain_timeout_ms,
3917 received_at_generation,
3918 queued_at,
3919 reply: reply_tx,
3920 })
3921 .await
3922 .map_err(|_| SuperviseError::CommandClosed {
3923 module_id: self.inner.module_id.clone(),
3924 })?;
3925 reply_rx.await.map_err(|_| SuperviseError::CommandClosed {
3926 module_id: self.inner.module_id.clone(),
3927 })?
3928 }
3929
3930 pub async fn swap(&self, ready_timeout: Option<Duration>) -> Result<(), SuperviseError> {
3935 self.swap_requested(ready_timeout, None).await
3936 }
3937
3938 pub(crate) async fn swap_requested(
3939 &self,
3940 ready_timeout: Option<Duration>,
3941 requester: Option<Principal>,
3942 ) -> Result<(), SuperviseError> {
3943 let (reply_tx, reply_rx) = oneshot::channel();
3944 self.inner
3945 .commands
3946 .send(SupervisorCommand::Swap {
3947 requester,
3948 ready_timeout,
3949 reply: reply_tx,
3950 })
3951 .await
3952 .map_err(|_| SuperviseError::CommandClosed {
3953 module_id: self.inner.module_id.clone(),
3954 })?;
3955 reply_rx.await.map_err(|_| SuperviseError::CommandClosed {
3956 module_id: self.inner.module_id.clone(),
3957 })?
3958 }
3959
3960 pub async fn reload(&self) -> Result<(), SuperviseError> {
3961 self.reload_requested(None).await
3962 }
3963
3964 pub(crate) async fn reload_requested(
3965 &self,
3966 requester: Option<Principal>,
3967 ) -> Result<(), SuperviseError> {
3968 let (reply_tx, reply_rx) = oneshot::channel();
3969 self.inner
3970 .commands
3971 .send(SupervisorCommand::Reload {
3972 requester,
3973 reply: reply_tx,
3974 })
3975 .await
3976 .map_err(|_| SuperviseError::CommandClosed {
3977 module_id: self.inner.module_id.clone(),
3978 })?;
3979 reply_rx.await.map_err(|_| SuperviseError::CommandClosed {
3980 module_id: self.inner.module_id.clone(),
3981 })?
3982 }
3983
3984 pub async fn set_enabled(&self, enabled: bool) -> Result<bool, SuperviseError> {
3985 self.set_enabled_requested(enabled, None).await
3986 }
3987
3988 pub(crate) async fn set_enabled_requested(
3989 &self,
3990 enabled: bool,
3991 requester: Option<Principal>,
3992 ) -> Result<bool, SuperviseError> {
3993 let (reply_tx, reply_rx) = oneshot::channel();
3994 self.inner
3995 .commands
3996 .send(SupervisorCommand::SetEnabled {
3997 requester,
3998 enabled,
3999 reply: reply_tx,
4000 })
4001 .await
4002 .map_err(|_| SuperviseError::CommandClosed {
4003 module_id: self.inner.module_id.clone(),
4004 })?;
4005 reply_rx.await.map_err(|_| SuperviseError::CommandClosed {
4006 module_id: self.inner.module_id.clone(),
4007 })?
4008 }
4009
4010 pub(crate) fn declared_protocol(&self) -> Result<ModuleProtocol, SuperviseError> {
4014 let configured = self
4015 .inner
4016 .configuration
4017 .lock()
4018 .map_err(|_| SuperviseError::StatePoisoned {
4019 module_id: Some(self.inner.module_id.clone()),
4020 })?
4021 .spec
4022 .protocol;
4023 let state = lock_snapshot(&self.inner.snapshot)?;
4024 Ok(state.spawned_protocol.unwrap_or(configured))
4025 }
4026
4027 pub(crate) fn configuration(&self) -> Result<(ModuleSpec, HealthConfig), SuperviseError> {
4028 let configuration =
4029 self.inner
4030 .configuration
4031 .lock()
4032 .map_err(|_| SuperviseError::StatePoisoned {
4033 module_id: Some(self.inner.module_id.clone()),
4034 })?;
4035 Ok((configuration.spec.clone(), configuration.health.clone()))
4036 }
4037
4038 #[cfg(any(test, feature = "test-support"))]
4042 pub async fn update_spec_for_test(&self, spec: ModuleSpec) -> Result<(), SuperviseError> {
4043 let (_, health) = self.configuration()?;
4044 let drain_timeout_ms = u64::try_from(
4045 self.inner
4046 .effective_drain_timeout
4047 .lock()
4048 .unwrap_or_else(|poisoned| poisoned.into_inner())
4049 .as_millis(),
4050 )
4051 .ok();
4052 self.update_configuration(spec, health, drain_timeout_ms)
4053 .await
4054 }
4055
4056 pub(crate) async fn update_configuration(
4057 &self,
4058 spec: ModuleSpec,
4059 health: HealthConfig,
4060 drain_timeout_ms: Option<u64>,
4061 ) -> Result<(), SuperviseError> {
4062 if spec.module_id != self.inner.module_id {
4063 return Err(SuperviseError::InvalidSpec {
4064 reason: "a supervised module's module_id cannot be changed".to_string(),
4065 });
4066 }
4067 validate_spec(&spec)?;
4068 let (reply_tx, reply_rx) = oneshot::channel();
4069 self.inner
4070 .commands
4071 .send(SupervisorCommand::UpdateConfiguration {
4072 spec: spec.clone(),
4073 health: health.clone(),
4074 drain_timeout_ms,
4075 reply: reply_tx,
4076 })
4077 .await
4078 .map_err(|_| SuperviseError::CommandClosed {
4079 module_id: self.inner.module_id.clone(),
4080 })?;
4081 reply_rx.await.map_err(|_| SuperviseError::CommandClosed {
4082 module_id: self.inner.module_id.clone(),
4083 })?;
4084 let mut configuration =
4085 self.inner
4086 .configuration
4087 .lock()
4088 .map_err(|_| SuperviseError::StatePoisoned {
4089 module_id: Some(self.inner.module_id.clone()),
4090 })?;
4091 configuration.spec = spec;
4092 configuration.health = health;
4093 Ok(())
4094 }
4095}
4096
4097impl Drop for SupervisedModuleInner {
4098 fn drop(&mut self) {
4099 let Ok(mut monitor) = self.monitor.lock() else {
4100 return;
4101 };
4102 if let Some(monitor) = monitor.as_ref().filter(|monitor| !monitor.is_finished()) {
4103 let _ = update_snapshot(&self.snapshot, Some(&self.module_id), |state| {
4104 state.state = ModuleState::Stopped;
4105 clear_current_process_facts(state);
4106 });
4107 monitor.abort();
4108 }
4109 let _ = monitor.take();
4110 }
4111}
4112
4113#[derive(Debug)]
4114enum SupervisorCommand {
4115 Drain {
4116 requester: Option<Principal>,
4117 reply: oneshot::Sender<Result<(), SuperviseError>>,
4118 },
4119 Retire {
4120 requester: Option<Principal>,
4121 reply: oneshot::Sender<Result<(), SuperviseError>>,
4122 },
4123 Restart {
4124 requester: Option<Principal>,
4125 drain_timeout_ms: Option<u64>,
4130 received_at_generation: u64,
4134 queued_at: Instant,
4137 reply: oneshot::Sender<Result<(), SuperviseError>>,
4138 },
4139 Reload {
4140 requester: Option<Principal>,
4141 reply: oneshot::Sender<Result<(), SuperviseError>>,
4142 },
4143 SetEnabled {
4144 requester: Option<Principal>,
4145 enabled: bool,
4146 reply: oneshot::Sender<Result<bool, SuperviseError>>,
4147 },
4148 UpdateConfiguration {
4149 spec: ModuleSpec,
4150 health: HealthConfig,
4151 drain_timeout_ms: Option<u64>,
4154 reply: oneshot::Sender<()>,
4155 },
4156 Swap {
4157 requester: Option<Principal>,
4158 ready_timeout: Option<Duration>,
4161 reply: oneshot::Sender<Result<(), SuperviseError>>,
4163 },
4164}
4165
4166#[derive(Debug)]
4167pub enum SuperviseError {
4168 InvalidSpec {
4169 reason: String,
4170 },
4171 Spawn {
4172 program: PathBuf,
4173 source: io::Error,
4174 cgroup_path: Option<PathBuf>,
4175 },
4176 Cgroup {
4177 module_id: String,
4178 source: io::Error,
4179 },
4180 LaunchNonce {
4183 reason: String,
4184 },
4185 Wait {
4186 module_id: String,
4187 source: io::Error,
4188 },
4189 Kill {
4190 module_id: String,
4191 source: io::Error,
4192 },
4193 Forwarding(ForwardingError),
4194 Registry(RegistryError),
4195 ReloadUnavailable {
4196 module_id: String,
4197 reason: String,
4198 },
4199 Disabled {
4204 module_id: String,
4205 },
4206 ReloadFailed {
4207 module_id: String,
4208 reason: String,
4209 },
4210 RegistrationStillActive {
4211 module_id: String,
4212 waited: Duration,
4213 },
4214 StatePoisoned {
4215 module_id: Option<String>,
4216 },
4217 CommandClosed {
4218 module_id: String,
4219 },
4220 SwapInProgress {
4224 module_id: String,
4225 },
4226 SwapRefused {
4228 module_id: String,
4229 reason: SwapRefusal,
4230 },
4231 SwapFailed {
4235 module_id: String,
4236 arm: SwapFailureArm,
4237 detail: String,
4238 candidate_exit: Option<ExitReport>,
4241 },
4242}
4243
4244#[derive(Debug, Clone, Copy, PartialEq, Eq)]
4246pub enum SwapRefusal {
4247 OverlapExclusive,
4249 NotRegistered,
4252 ProtocolNone,
4255 NotConfigured,
4258 AlreadySwapping,
4260}
4261
4262impl SwapRefusal {
4263 pub fn as_str(self) -> &'static str {
4264 match self {
4265 Self::OverlapExclusive => "overlap_exclusive",
4266 Self::NotRegistered => "not_registered",
4267 Self::ProtocolNone => "protocol_none",
4268 Self::NotConfigured => "not_configured",
4269 Self::AlreadySwapping => "already_swapping",
4270 }
4271 }
4272}
4273
4274#[derive(Debug, Clone, Copy, PartialEq, Eq)]
4277pub enum SwapFailureArm {
4278 SpawnFailed,
4280 NeverRegistered,
4282 NeverReady,
4284 CandidateExited,
4286 CandidateUnhealthy,
4288 Interrupted,
4292 CutoverLost,
4297}
4298
4299impl SwapFailureArm {
4300 pub fn as_str(self) -> &'static str {
4301 match self {
4302 Self::SpawnFailed => "spawn_failed",
4303 Self::NeverRegistered => "never_registered",
4304 Self::NeverReady => "never_ready",
4305 Self::CandidateExited => "candidate_exited",
4306 Self::CandidateUnhealthy => "candidate_unhealthy",
4307 Self::Interrupted => "interrupted",
4308 Self::CutoverLost => "cutover_lost",
4309 }
4310 }
4311}
4312
4313impl fmt::Display for SuperviseError {
4314 fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
4315 match self {
4316 Self::InvalidSpec { reason } => write!(f, "invalid module spec: {reason}"),
4317 Self::Spawn {
4318 program,
4319 source,
4320 cgroup_path: Some(cgroup_path),
4321 } => write!(
4322 f,
4323 "failed to place module in cgroup '{}' while spawning '{}': {source}",
4324 cgroup_path.display(),
4325 program.display()
4326 ),
4327 Self::Spawn {
4328 program,
4329 source,
4330 cgroup_path: None,
4331 } => write!(
4332 f,
4333 "failed to spawn module '{}': {source}",
4334 program.display()
4335 ),
4336 Self::Cgroup { module_id, source } => {
4337 write!(
4338 f,
4339 "failed to prepare cgroup for module '{module_id}': {source}"
4340 )
4341 }
4342 Self::LaunchNonce { reason } => {
4343 write!(
4344 f,
4345 "failed to generate reserved-module launch nonce: {reason}"
4346 )
4347 }
4348 Self::Wait { module_id, source } => {
4349 write!(f, "failed to wait for module '{module_id}': {source}")
4350 }
4351 Self::Kill { module_id, source } => {
4352 write!(f, "failed to kill module '{module_id}': {source}")
4353 }
4354 Self::Forwarding(err) => write!(f, "forwarding error: {err}"),
4355 Self::Registry(err) => write!(f, "registry error: {err}"),
4356 Self::ReloadUnavailable { module_id, reason } => {
4357 write!(f, "reload unavailable for module '{module_id}': {reason}")
4358 }
4359 Self::Disabled { module_id } => {
4360 write!(
4361 f,
4362 "module '{module_id}' is disabled; enable it before restart or reload"
4363 )
4364 }
4365 Self::ReloadFailed { module_id, reason } => {
4366 write!(f, "reload failed for module '{module_id}': {reason}")
4367 }
4368 Self::RegistrationStillActive { module_id, waited } => write!(
4369 f,
4370 "module '{module_id}' registration remained active after waiting {waited:?}"
4371 ),
4372 Self::StatePoisoned { module_id } => match module_id {
4373 Some(module_id) => {
4374 write!(f, "supervisor state for module '{module_id}' was poisoned")
4375 }
4376 None => write!(f, "supervisor state was poisoned"),
4377 },
4378 Self::CommandClosed { module_id } => {
4379 write!(
4380 f,
4381 "supervisor command channel for module '{module_id}' is closed"
4382 )
4383 }
4384 Self::SwapInProgress { module_id } => write!(
4385 f,
4386 "module '{module_id}' is being swapped; retry once the swap has cut over or failed, or stop the module to abort the swap"
4387 ),
4388 Self::SwapRefused { module_id, reason } => match reason {
4389 SwapRefusal::OverlapExclusive => write!(
4390 f,
4391 "module '{module_id}' is declared overlap: \"exclusive\" (the default): two processes of it must not run at once, so it cannot be swapped; use a plain restart, or declare overlap: \"safe\" in its config if it really tolerates a second process"
4392 ),
4393 SwapRefusal::NotRegistered => write!(
4394 f,
4395 "module '{module_id}' is not registered, so there is no serving process to keep while a replacement warms; use a plain restart"
4396 ),
4397 SwapRefusal::ProtocolNone => write!(
4398 f,
4399 "module '{module_id}' is protocol: \"none\" and never registers, so a swap could never see its replacement become ready; use a plain restart"
4400 ),
4401 SwapRefusal::NotConfigured => write!(
4402 f,
4403 "module '{module_id}' cannot be swapped: the supervisor was built without the forwarding table or shared handle a swap needs"
4404 ),
4405 SwapRefusal::AlreadySwapping => {
4406 write!(f, "module '{module_id}' is already being swapped")
4407 }
4408 },
4409 Self::SwapFailed {
4410 module_id,
4411 arm,
4412 detail,
4413 ..
4414 } => write!(
4415 f,
4416 "swap of module '{module_id}' failed ({}): {detail}; the running process was left serving",
4417 arm.as_str()
4418 ),
4419 }
4420 }
4421}
4422
4423impl Error for SuperviseError {
4424 fn source(&self) -> Option<&(dyn Error + 'static)> {
4425 match self {
4426 Self::Spawn { source, .. }
4427 | Self::Cgroup { source, .. }
4428 | Self::Wait { source, .. }
4429 | Self::Kill { source, .. } => Some(source),
4430 Self::Forwarding(err) => Some(err),
4431 Self::Registry(err) => Some(err),
4432 Self::LaunchNonce { .. }
4433 | Self::InvalidSpec { .. }
4434 | Self::ReloadUnavailable { .. }
4435 | Self::Disabled { .. }
4436 | Self::ReloadFailed { .. }
4437 | Self::RegistrationStillActive { .. }
4438 | Self::StatePoisoned { .. }
4439 | Self::CommandClosed { .. }
4440 | Self::SwapInProgress { .. }
4441 | Self::SwapRefused { .. }
4442 | Self::SwapFailed { .. } => None,
4443 }
4444 }
4445}
4446
4447pub(crate) fn validate_spec(spec: &ModuleSpec) -> Result<(), SuperviseError> {
4448 if spec.module_id.trim().is_empty() {
4449 return Err(SuperviseError::InvalidSpec {
4450 reason: "module_id must not be empty".to_string(),
4451 });
4452 }
4453
4454 Ok(())
4455}
4456
4457#[derive(Debug, Default)]
4458struct HealthProbeRuntime {
4459 configured_health: Option<HealthConfig>,
4460 registered_connection: Option<crate::ConnectionId>,
4461 advertised: bool,
4462 next_probe_at: Option<Instant>,
4463 probe_index: u64,
4464}
4465
4466fn running_protocol(spec: &ModuleSpec, snapshot: &SharedSnapshot) -> ModuleProtocol {
4467 lock_snapshot(snapshot)
4468 .ok()
4469 .and_then(|state| state.spawned_protocol)
4470 .unwrap_or(spec.protocol)
4471}
4472
4473impl HealthProbeRuntime {
4474 fn refresh_registration(
4475 &mut self,
4476 spec: &ModuleSpec,
4477 runtime: &SupervisorRuntimeConfig,
4478 registry: &Registry,
4479 snapshot: &SharedSnapshot,
4480 ) {
4481 if self.configured_health.as_ref() != Some(&runtime.health) {
4482 self.configured_health = Some(runtime.health.clone());
4483 self.next_probe_at = None;
4484 self.registered_connection = None;
4485 self.probe_index = 0;
4486 }
4487 if running_protocol(spec, snapshot) == ModuleProtocol::None {
4491 self.registered_connection = None;
4492 self.advertised = runtime.health.http.is_some();
4493 if !self.advertised {
4494 self.next_probe_at = None;
4495 let _ = update_snapshot(snapshot, Some(&spec.module_id), |state| {
4496 let unknown = ModuleHealthStatus::default();
4497 if state.health != unknown {
4498 state.health = unknown;
4499 }
4500 });
4501 } else if self.next_probe_at.is_none() {
4502 self.next_probe_at = Some(
4503 Instant::now()
4504 + jittered_health_delay(&spec.module_id, 0, runtime.health.cadence),
4505 );
4506 }
4507 return;
4508 }
4509
4510 let registration = match registry.get_module(&spec.module_id) {
4511 Ok(registration) => registration,
4512 Err(err) => {
4513 warn!(module_id = %spec.module_id, error = %err, "health prober could not read registry");
4514 self.advertised = false;
4515 self.next_probe_at = None;
4516 return;
4517 }
4518 };
4519
4520 let Some(registration) = registration else {
4521 self.registered_connection = None;
4522 self.advertised = false;
4523 self.next_probe_at = None;
4524 return;
4525 };
4526
4527 let advertised = registration
4528 .control_ops
4529 .iter()
4530 .any(|op| op == MODULE_CONTROL_OP_HEALTH_CHECK);
4531 if !advertised {
4532 self.registered_connection = Some(registration.connection_id);
4533 self.advertised = false;
4534 self.next_probe_at = None;
4535 let _ = update_snapshot(snapshot, Some(&spec.module_id), |state| {
4536 if state.health.status != SupervisorHealthStatus::Unknown
4537 || state.health.consecutive_failures != 0
4538 || state.health.last_probe_ms.is_some()
4539 || state.health.detail.is_some()
4540 || state.health.metrics.is_some()
4541 {
4542 state.health.status = SupervisorHealthStatus::Unknown;
4543 state.health.consecutive_failures = 0;
4544 state.health.last_probe_ms = None;
4545 state.health.detail = None;
4546 state.health.metrics = None;
4547 }
4548 });
4549 return;
4550 }
4551
4552 let reregistered = self.registered_connection != Some(registration.connection_id);
4553 self.registered_connection = Some(registration.connection_id);
4554 self.advertised = true;
4555 if reregistered || self.next_probe_at.is_none() {
4556 self.probe_index = 0;
4557 self.next_probe_at = Some(
4558 Instant::now() + jittered_health_delay(&spec.module_id, 0, runtime.health.cadence),
4559 );
4560 let _ = update_snapshot(snapshot, Some(&spec.module_id), |state| {
4561 state.health.status = SupervisorHealthStatus::Unknown;
4562 state.health.consecutive_failures = 0;
4563 state.health.detail = None;
4564 state.health.metrics = None;
4565 });
4566 }
4567 }
4568
4569 fn wake_after(&self) -> Option<Duration> {
4570 if !self.advertised {
4571 return None;
4572 }
4573 self.next_probe_at
4574 .map(|next| next.saturating_duration_since(Instant::now()))
4575 }
4576
4577 fn due(&self) -> bool {
4578 self.advertised
4579 && self
4580 .next_probe_at
4581 .is_some_and(|next| Instant::now() >= next)
4582 }
4583
4584 fn schedule_next(&mut self, spec: &ModuleSpec, cadence: Duration) {
4585 self.probe_index = self.probe_index.wrapping_add(1);
4586 self.next_probe_at = Some(
4587 Instant::now() + jittered_health_delay(&spec.module_id, self.probe_index, cadence),
4588 );
4589 }
4590}
4591
4592#[derive(Debug)]
4627enum HealthProbeEvidence {
4628 LaneDead,
4630 NoAnswer,
4632 BadAnswer,
4634 Misconfigured,
4636}
4637
4638#[derive(Debug)]
4639struct HealthProbeError {
4640 evidence: HealthProbeEvidence,
4641 message: String,
4642}
4643
4644impl HealthProbeError {
4645 fn lane_dead(message: impl Into<String>) -> Self {
4646 Self::with(HealthProbeEvidence::LaneDead, message)
4647 }
4648
4649 fn no_answer(message: impl Into<String>) -> Self {
4650 Self::with(HealthProbeEvidence::NoAnswer, message)
4651 }
4652
4653 fn bad_answer(message: impl Into<String>) -> Self {
4654 Self::with(HealthProbeEvidence::BadAnswer, message)
4655 }
4656
4657 fn misconfigured(message: impl Into<String>) -> Self {
4658 Self::with(HealthProbeEvidence::Misconfigured, message)
4659 }
4660
4661 fn with(evidence: HealthProbeEvidence, message: impl Into<String>) -> Self {
4662 Self {
4663 evidence,
4664 message: message.into(),
4665 }
4666 }
4667
4668 #[allow(dead_code)]
4682 fn is_proof_of_death(&self) -> bool {
4683 matches!(self.evidence, HealthProbeEvidence::LaneDead)
4684 }
4685
4686 fn label(&self) -> &'static str {
4694 match self.evidence {
4695 HealthProbeEvidence::LaneDead => "lane-dead",
4696 HealthProbeEvidence::NoAnswer => "no-answer",
4697 HealthProbeEvidence::BadAnswer => "bad-answer",
4698 HealthProbeEvidence::Misconfigured => "daemon-misconfigured",
4699 }
4700 }
4701}
4702
4703impl fmt::Display for HealthProbeError {
4704 fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
4705 f.write_str(&self.message)
4706 }
4707}
4708
4709async fn run_health_probe_cycle(
4710 spec: &ModuleSpec,
4711 runtime: &SupervisorRuntimeConfig,
4712 registry: &Registry,
4713 process_liveness: &SupervisorProcessLiveness,
4714 snapshot: &SharedSnapshot,
4715 child: &mut Option<SupervisedChild>,
4716) {
4717 let now_ms = unix_ms_now();
4718 let http = (running_protocol(spec, snapshot) == ModuleProtocol::None)
4719 .then_some(runtime.health.http.as_deref())
4720 .flatten();
4721 let result = match http {
4722 Some(url) => probe_http_health(url, runtime.health.deadline).await,
4723 None => probe_module_health(&spec.module_id, runtime, None).await,
4724 };
4725 match result {
4726 Ok(report) => {
4727 handle_health_report(
4728 spec,
4729 runtime,
4730 registry,
4731 process_liveness,
4732 snapshot,
4733 child,
4734 report,
4735 now_ms,
4736 )
4737 .await;
4738 }
4739 Err(err) => {
4740 if http.is_some() {
4741 let _ = update_snapshot(snapshot, Some(&spec.module_id), |state| {
4742 state.health.status = SupervisorHealthStatus::Failing;
4743 });
4744 }
4745 handle_health_probe_failure(
4746 spec,
4747 runtime,
4748 registry,
4749 process_liveness,
4750 snapshot,
4751 child,
4752 err,
4753 now_ms,
4754 )
4755 .await;
4756 }
4757 }
4758}
4759
4760pub(crate) struct HttpProbeTarget<'a> {
4761 address: std::net::SocketAddr,
4762 localhost: bool,
4763 authority: &'a str,
4764 path: String,
4765}
4766
4767pub(crate) fn parse_http_probe_url(url: &str) -> Result<HttpProbeTarget<'_>, String> {
4770 if url.bytes().any(|b| b <= b' ' || b == 127) || url.contains('#') {
4771 return Err("must not contain whitespace, controls, or a fragment".into());
4772 }
4773 let rest = url
4774 .strip_prefix("http://")
4775 .ok_or("must use plain http://")?;
4776 let split = rest.find(['/', '?']).unwrap_or(rest.len());
4777 let (authority, suffix) = rest.split_at(split);
4778 let (host, port) = if let Some(rest) = authority.strip_prefix("[::1]") {
4779 ("::1", rest)
4780 } else {
4781 let split = authority.find(':').unwrap_or(authority.len());
4782 authority.split_at(split)
4783 };
4784 let ip: std::net::IpAddr = match host {
4785 "127.0.0.1" | "localhost" => std::net::Ipv4Addr::LOCALHOST.into(),
4786 "::1" => std::net::Ipv6Addr::LOCALHOST.into(),
4787 _ => return Err("host must be 127.0.0.1, [::1], or localhost".into()),
4788 };
4789 let port = if port.is_empty() {
4790 80
4791 } else {
4792 port.strip_prefix(':')
4793 .and_then(|p| p.parse::<u16>().ok())
4794 .filter(|p| *p > 0)
4795 .ok_or("must have a valid nonzero TCP port")?
4796 };
4797 let path = if suffix.is_empty() {
4798 "/".into()
4799 } else if suffix.starts_with('?') {
4800 format!("/{suffix}")
4801 } else {
4802 suffix.into()
4803 };
4804 Ok(HttpProbeTarget {
4805 address: std::net::SocketAddr::new(ip, port),
4806 localhost: host == "localhost",
4807 authority,
4808 path,
4809 })
4810}
4811
4812async fn probe_http_health(
4813 url: &str,
4814 deadline: Duration,
4815) -> Result<HealthReport, HealthProbeError> {
4816 use tokio::io::{AsyncReadExt, AsyncWriteExt, BufReader};
4817 let target = parse_http_probe_url(url).map_err(HealthProbeError::misconfigured)?;
4818 let mut response_status = String::new();
4821 let mut body = Vec::new();
4822 let probe = async {
4823 let connection = match tokio::net::TcpStream::connect(target.address).await {
4826 Err(_) if target.localhost => {
4827 tokio::net::TcpStream::connect((
4828 std::net::Ipv6Addr::LOCALHOST,
4829 target.address.port(),
4830 ))
4831 .await
4832 }
4833 result => result,
4834 };
4835 let mut stream = connection.map_err(|error| {
4836 HealthProbeError::no_answer(format!("HTTP connect failed: {error}"))
4837 })?;
4838 stream.write_all(format!("GET {} HTTP/1.1\r\nHost: {}\r\nAccept-Encoding: identity\r\nConnection: close\r\n\r\n", target.path, target.authority).as_bytes()).await
4839 .map_err(|error| HealthProbeError::no_answer(format!("HTTP write failed: {error}")))?;
4840 let mut reader = BufReader::new(stream);
4841 let mut budget = 16 * 1024;
4842 let status = http_line(&mut reader, &mut budget).await?;
4843 let mut words = status.split_ascii_whitespace();
4844 let version = words.next();
4845 let code = words
4846 .next()
4847 .filter(|word| word.len() == 3)
4848 .and_then(|word| word.parse::<u16>().ok());
4849 if !matches!(version, Some("HTTP/1.1" | "HTTP/1.0"))
4850 || !code.is_some_and(|code| (100..600).contains(&code))
4851 {
4852 return Err(HealthProbeError::bad_answer(format!(
4853 "invalid HTTP status: {status}"
4854 )));
4855 }
4856 let code = code.expect("validated status code");
4857 response_status = status.clone();
4858 let mut length = None;
4859 let mut chunked = false;
4860 loop {
4861 let line = http_line(&mut reader, &mut budget).await?;
4862 if line.is_empty() {
4863 break;
4864 }
4865 if let Some((name, value)) = line.split_once(':') {
4866 if name.eq_ignore_ascii_case("content-length") {
4867 length = Some(value.trim().parse::<u64>().map_err(|_| {
4868 HealthProbeError::bad_answer("invalid HTTP Content-Length")
4869 })?);
4870 } else if name.eq_ignore_ascii_case("transfer-encoding") {
4871 chunked = value.trim().eq_ignore_ascii_case("chunked");
4872 }
4873 }
4874 }
4875 if chunked {
4876 while body.len() < 200 {
4877 let line = http_line(&mut reader, &mut budget).await?;
4878 let size = u64::from_str_radix(line.split(';').next().unwrap_or("").trim(), 16)
4879 .map_err(|_| HealthProbeError::bad_answer("invalid HTTP chunk size"))?;
4880 if size == 0 {
4881 break;
4882 }
4883 let count = size.min((200 - body.len()) as u64) as usize;
4884 let start = body.len();
4885 (&mut reader)
4886 .take(count as u64)
4887 .read_to_end(&mut body)
4888 .await
4889 .map_err(|error| {
4890 HealthProbeError::no_answer(format!("HTTP body read failed: {error}"))
4891 })?;
4892 if body.len() - start != count {
4893 return Err(HealthProbeError::bad_answer("truncated HTTP chunk"));
4894 }
4895 if size > count as u64 || body.len() == 200 {
4896 break;
4897 }
4898 if !http_line(&mut reader, &mut budget).await?.is_empty() {
4899 return Err(HealthProbeError::bad_answer("invalid HTTP chunk delimiter"));
4900 }
4901 }
4902 } else {
4903 reader
4904 .take(length.unwrap_or(200).min(200))
4905 .read_to_end(&mut body)
4906 .await
4907 .map_err(|error| {
4908 HealthProbeError::no_answer(format!("HTTP body read failed: {error}"))
4909 })?;
4910 }
4911 if (200..300).contains(&code) {
4912 Ok(HealthReport::ok())
4913 } else {
4914 Err(HealthProbeError::bad_answer(
4915 "HTTP health endpoint returned non-2xx",
4916 ))
4917 }
4918 };
4919 let mut result = timeout(deadline, probe).await.unwrap_or_else(|_| {
4920 Err(HealthProbeError::no_answer(format!(
4921 "HTTP probe timed out after {deadline:?}"
4922 )))
4923 });
4924 if let Err(error) = &mut result {
4925 if !response_status.is_empty() {
4926 error.message = format!(
4927 "{}; {response_status}: {}",
4928 error.message,
4929 String::from_utf8_lossy(&body)
4930 );
4931 }
4932 }
4933 result
4934}
4935
4936async fn http_line(
4937 reader: &mut tokio::io::BufReader<tokio::net::TcpStream>,
4938 remaining: &mut usize,
4939) -> Result<String, HealthProbeError> {
4940 use tokio::io::{AsyncBufReadExt, AsyncReadExt};
4941 let mut line = Vec::new();
4942 (&mut *reader)
4943 .take(*remaining as u64)
4944 .read_until(b'\n', &mut line)
4945 .await
4946 .map_err(|error| {
4947 HealthProbeError::no_answer(format!("HTTP header read failed: {error}"))
4948 })?;
4949 *remaining -= line.len();
4950 if !line.ends_with(b"\r\n") {
4951 return Err(HealthProbeError::bad_answer(
4952 "HTTP headers are incomplete or exceed 16 KiB",
4953 ));
4954 }
4955 line.truncate(line.len() - 2);
4956 String::from_utf8(line).map_err(|_| HealthProbeError::bad_answer("HTTP header is not UTF-8"))
4957}
4958
4959async fn probe_module_health(
4960 module_id: &str,
4961 runtime: &SupervisorRuntimeConfig,
4962 drain_deadline: Option<Instant>,
4963) -> Result<HealthReport, HealthProbeError> {
4964 let Some(forwarding) = runtime.forwarding.as_ref() else {
4965 return Err(HealthProbeError::misconfigured(
4966 "supervisor was not configured with a forwarding table",
4967 ));
4968 };
4969 let probe_started_at = Instant::now();
4970 let mut deadline = probe_started_at + runtime.health.deadline;
4971 if let Some(drain_deadline) = drain_deadline {
4972 deadline = deadline.min(drain_deadline);
4973 }
4974 let pending = if drain_deadline.is_some() {
4975 forwarding.begin_drain_health_probe_rpc_for(
4976 module_id,
4977 MODULE_CONTROL_OP_HEALTH_CHECK,
4978 probe_started_at,
4979 deadline,
4980 )
4981 } else {
4982 forwarding.begin_health_probe_rpc_for(
4983 module_id,
4984 MODULE_CONTROL_OP_HEALTH_CHECK,
4985 probe_started_at,
4986 deadline,
4987 )
4988 }
4989 .map_err(|err| {
4990 HealthProbeError::lane_dead(format!("failed to begin health.check RPC: {err}"))
4993 })?;
4994 await_health_probe(forwarding, pending, deadline, runtime.health.deadline).await
4995}
4996
4997async fn probe_endpoint_health(
5004 endpoint: crate::ModuleEndpointId,
5005 runtime: &SupervisorRuntimeConfig,
5006 deadline_cap: Option<Instant>,
5007) -> Result<HealthReport, HealthProbeError> {
5008 let Some(forwarding) = runtime.forwarding.as_ref() else {
5009 return Err(HealthProbeError::misconfigured(
5010 "supervisor was not configured with a forwarding table",
5011 ));
5012 };
5013 let probe_started_at = Instant::now();
5014 let mut deadline = probe_started_at + runtime.health.deadline;
5015 if let Some(cap) = deadline_cap {
5016 deadline = deadline.min(cap);
5017 }
5018 let pending = forwarding
5019 .begin_endpoint_health_probe_rpc_for(
5020 endpoint,
5021 MODULE_CONTROL_OP_HEALTH_CHECK,
5022 probe_started_at,
5023 deadline,
5024 )
5025 .map_err(|err| {
5026 HealthProbeError::lane_dead(format!("failed to begin health.check RPC: {err}"))
5027 })?;
5028 await_health_probe(forwarding, pending, deadline, runtime.health.deadline).await
5029}
5030
5031async fn await_health_probe(
5033 forwarding: &ForwardingTable,
5034 pending: PendingModuleControlRpc,
5035 deadline: Instant,
5036 probe_budget: Duration,
5037) -> Result<HealthReport, HealthProbeError> {
5038 let PendingModuleControlRpc {
5039 endpoint,
5040 module_sink,
5041 negotiated_ver,
5042 corr,
5043 receiver,
5044 } = pending;
5045 let body = serde_json::to_vec(&ModuleControlRequest::HealthCheck {}).map_err(|err| {
5046 HealthProbeError::misconfigured(format!("failed to encode health.check: {err}"))
5047 })?;
5048 let frame = Frame::build_with_version(
5049 negotiated_ver,
5050 FrameType::Request,
5051 control_flags(),
5052 0,
5053 0,
5054 corr,
5055 body,
5056 )
5057 .map_err(|err| {
5058 HealthProbeError::misconfigured(format!("failed to build health.check frame: {err}"))
5059 })?;
5060
5061 match timeout_at(deadline, module_sink.send(frame)).await {
5067 Ok(Ok(())) => {}
5068 Ok(Err(err)) => {
5069 let _ = forwarding.cancel_module_control_rpc(endpoint, corr);
5070 return Err(HealthProbeError::lane_dead(format!(
5073 "failed to send health.check: {err}"
5074 )));
5075 }
5076 Err(_elapsed) => {
5077 let _ = forwarding.cancel_module_control_rpc(endpoint, corr);
5078 return Err(HealthProbeError::no_answer(
5082 "health.check send timed out before enqueue (module egress full)",
5083 ));
5084 }
5085 }
5086
5087 match timeout_at(deadline, receiver).await {
5088 Ok(Ok(ModuleControlRpcOutcome::Response(response))) => {
5092 response.health_report().ok_or_else(|| {
5093 HealthProbeError::bad_answer("health.check RPC returned a non-health response")
5094 })
5095 }
5096 Ok(Ok(ModuleControlRpcOutcome::Rejected(body))) => Err(HealthProbeError::bad_answer(
5097 format!("health.check rejected: {}", body.message),
5098 )),
5099 Ok(Ok(ModuleControlRpcOutcome::ModuleGone(message))) => {
5100 Err(HealthProbeError::lane_dead(message))
5101 }
5102 Ok(Ok(ModuleControlRpcOutcome::MalformedResponse(message))) => {
5103 Err(HealthProbeError::bad_answer(message))
5104 }
5105 Ok(Ok(ModuleControlRpcOutcome::UnexpectedOp { expected, actual })) => {
5106 Err(HealthProbeError::bad_answer(format!(
5107 "expected module-control op '{expected}', got '{actual}'"
5108 )))
5109 }
5110 Ok(Ok(ModuleControlRpcOutcome::DeadlineElapsed)) => Err(HealthProbeError::bad_answer(
5114 "module answered health.check after its daemon deadline",
5115 )),
5116 Ok(Err(_)) => Err(HealthProbeError::misconfigured(
5117 "health.check waiter was canceled before the module responded",
5118 )),
5119 Err(_) => {
5120 let _ = forwarding.tombstone_health_probe_rpc(endpoint, corr);
5121 Err(HealthProbeError::no_answer(format!(
5122 "module did not answer health.check within {probe_budget:?}"
5123 )))
5124 }
5125 }
5126}
5127
5128#[allow(clippy::too_many_arguments)]
5129async fn handle_health_report(
5130 spec: &ModuleSpec,
5131 runtime: &SupervisorRuntimeConfig,
5132 registry: &Registry,
5133 process_liveness: &SupervisorProcessLiveness,
5134 snapshot: &SharedSnapshot,
5135 child: &mut Option<SupervisedChild>,
5136 report: HealthReport,
5137 now_ms: u64,
5138) {
5139 let status = supervisor_health_status(report.status);
5140 let detail = report.detail.clone();
5141 let metrics = truncate_health_metrics(report.metrics);
5142 let _ = update_snapshot(snapshot, Some(&spec.module_id), |state| {
5143 state.health.status = status;
5144 state.health.last_probe_ms = Some(now_ms);
5145 state.health.detail = detail.clone();
5146 state.health.metrics = metrics.clone();
5147 state.health.consecutive_failures = 0;
5148 });
5149
5150 let action = match report.status {
5151 HealthStatus::Ok => return,
5152 HealthStatus::Degraded => runtime.health.on_degraded,
5153 HealthStatus::Failing => runtime.health.on_failing,
5154 };
5155 apply_l3_health_action(
5156 spec,
5157 runtime,
5158 registry,
5159 process_liveness,
5160 snapshot,
5161 child,
5162 status,
5163 detail.as_deref(),
5164 action,
5165 now_ms,
5166 )
5167 .await;
5168}
5169
5170#[allow(clippy::too_many_arguments)]
5171async fn handle_health_probe_failure(
5172 spec: &ModuleSpec,
5173 runtime: &SupervisorRuntimeConfig,
5174 registry: &Registry,
5175 process_liveness: &SupervisorProcessLiveness,
5176 snapshot: &SharedSnapshot,
5177 child: &mut Option<SupervisedChild>,
5178 err: HealthProbeError,
5179 now_ms: u64,
5180) {
5181 let threshold = runtime.health.failure_threshold.max(1);
5182 let mut failures = 0;
5183 let detail = format!("[{}] {err}", err.label());
5188 let _ = update_snapshot(snapshot, Some(&spec.module_id), |state| {
5189 if state.spawned_protocol.unwrap_or(spec.protocol) == ModuleProtocol::Subc {
5192 state.health.status = SupervisorHealthStatus::Unknown;
5193 }
5194 state.health.last_probe_ms = Some(now_ms);
5195 state.health.consecutive_failures = state.health.consecutive_failures.saturating_add(1);
5196 state.health.detail = Some(detail.clone());
5197 state.health.metrics = None;
5198 failures = state.health.consecutive_failures;
5199 });
5200
5201 if failures < threshold {
5202 warn!(
5203 module_id = %spec.module_id,
5204 consecutive_failures = failures,
5205 threshold,
5206 evidence = err.label(),
5207 detail = %detail,
5208 "health.check probe failed"
5209 );
5210 return;
5211 }
5212
5213 let _ = update_snapshot(snapshot, Some(&spec.module_id), |state| {
5214 state.state = ModuleState::Unresponsive;
5215 state.health.status = SupervisorHealthStatus::Unresponsive;
5216 });
5217 if runtime.health.critical {
5221 error!(
5222 module_id = %spec.module_id,
5223 status = "unresponsive",
5224 evidence = err.label(),
5225 detail = %detail,
5226 "critical module health alert"
5227 );
5228 } else {
5229 warn!(
5230 module_id = %spec.module_id,
5231 status = "unresponsive",
5232 evidence = err.label(),
5233 detail = %detail,
5234 "module health threshold breached"
5235 );
5236 }
5237 if let Err(err) = health_restart_child(
5238 spec,
5239 runtime,
5240 registry,
5241 process_liveness,
5242 snapshot,
5243 child,
5244 SupervisorHealthStatus::Unresponsive,
5245 Some(&detail),
5246 now_ms,
5247 )
5248 .await
5249 {
5250 error!(module_id = %spec.module_id, error = %err, "health-triggered restart failed");
5251 }
5252}
5253
5254#[allow(clippy::too_many_arguments)]
5255async fn apply_l3_health_action(
5256 spec: &ModuleSpec,
5257 runtime: &SupervisorRuntimeConfig,
5258 registry: &Registry,
5259 process_liveness: &SupervisorProcessLiveness,
5260 snapshot: &SharedSnapshot,
5261 child: &mut Option<SupervisedChild>,
5262 status: SupervisorHealthStatus,
5263 detail: Option<&str>,
5264 action: HealthAction,
5265 now_ms: u64,
5266) {
5267 record_health_action(snapshot, &spec.module_id, action.to_string(), now_ms);
5268 match action {
5269 HealthAction::Report => {
5270 info!(
5271 module_id = %spec.module_id,
5272 status = ?status,
5273 detail,
5274 "module reported non-ok health"
5275 );
5276 }
5277 HealthAction::Alert => {
5278 error!(
5279 module_id = %spec.module_id,
5280 status = ?status,
5281 detail,
5282 "module health alert"
5283 );
5284 }
5285 HealthAction::Restart => {
5286 if let Err(err) = health_restart_child(
5287 spec,
5288 runtime,
5289 registry,
5290 process_liveness,
5291 snapshot,
5292 child,
5293 status,
5294 detail,
5295 now_ms,
5296 )
5297 .await
5298 {
5299 error!(module_id = %spec.module_id, error = %err, "health-triggered restart failed");
5300 }
5301 }
5302 }
5303}
5304
5305#[allow(clippy::too_many_arguments)]
5306async fn health_restart_child(
5307 spec: &ModuleSpec,
5308 runtime: &SupervisorRuntimeConfig,
5309 registry: &Registry,
5310 process_liveness: &SupervisorProcessLiveness,
5311 snapshot: &SharedSnapshot,
5312 child: &mut Option<SupervisedChild>,
5313 status: SupervisorHealthStatus,
5314 detail: Option<&str>,
5315 now_ms: u64,
5316) -> Result<(), SuperviseError> {
5317 let (enabled, schedule) = {
5318 let mut state = lock_snapshot(snapshot)?;
5319 let enabled = state.enabled;
5320 let schedule = if enabled {
5321 state.next_crash_restart(&runtime.restart_policy, Instant::now())
5322 } else {
5323 None
5324 };
5325 (enabled, schedule)
5326 };
5327
5328 if !enabled {
5329 return Err(SuperviseError::Disabled {
5330 module_id: spec.module_id.clone(),
5331 });
5332 }
5333
5334 if schedule.is_none() {
5335 record_health_action(snapshot, &spec.module_id, "failed".to_string(), now_ms);
5336 error!(
5337 module_id = %spec.module_id,
5338 status = ?status,
5339 detail,
5340 max_restarts = runtime.restart_policy.max_restarts,
5341 window_secs = runtime.restart_policy.window.as_secs(),
5342 reason = %runtime.restart_policy.budget_exhausted_detail(),
5343 "health restart budget exhausted; marking module failed"
5344 );
5345 let stop_notice = begin_forwarding_drain_if_configured(
5346 spec,
5347 runtime,
5348 registry,
5349 snapshot,
5350 Some(true),
5351 RouteCloseReason::Disable,
5352 )
5353 .await?;
5354 update_snapshot(snapshot, Some(&spec.module_id), |state| {
5355 state.drain_disposition_detail = Some(runtime.restart_policy.budget_exhausted_detail());
5356 })?;
5357 drain_optional_child(
5358 &spec.module_id,
5359 spec.protocol,
5360 stop_notice,
5361 registry,
5362 runtime.forwarding.as_deref(),
5363 snapshot,
5364 &runtime.terminal_ring,
5365 &runtime.spawn_events,
5366 child,
5367 runtime.drain_timeout,
5368 ModuleState::Failed,
5369 Some(true),
5370 )
5371 .await?;
5372 process_liveness.untrack_if_current(&spec.module_id, snapshot);
5373 return Ok(());
5374 }
5375
5376 let schedule = schedule.expect("a health restart must have a crash-restart schedule");
5377 let mut restart_count = 0;
5378 update_snapshot(snapshot, Some(&spec.module_id), |state| {
5379 restart_count = state.crash_restarts.len();
5380 state.state = ModuleState::Unresponsive;
5381 state.health.status = status;
5382 state.health.last_action = Some(HealthAction::Restart.to_string());
5383 state.health.last_action_ms = Some(now_ms);
5384 })?;
5385 warn!(
5386 module_id = %spec.module_id,
5387 status = ?status,
5388 detail,
5389 restart_count,
5390 restart_in_window = schedule.restart_in_window,
5391 delay_ms = schedule.delay.as_millis() as u64,
5392 "health-triggered module restart"
5393 );
5394
5395 let stop_notice = begin_forwarding_drain_if_configured(
5396 spec,
5397 runtime,
5398 registry,
5399 snapshot,
5400 Some(true),
5401 RouteCloseReason::Restart,
5402 )
5403 .await?;
5404 drain_optional_child(
5405 &spec.module_id,
5406 spec.protocol,
5407 stop_notice,
5408 registry,
5409 runtime.forwarding.as_deref(),
5410 snapshot,
5411 &runtime.terminal_ring,
5412 &runtime.spawn_events,
5413 child,
5414 runtime.drain_timeout,
5415 ModuleState::Restarting,
5416 Some(true),
5417 )
5418 .await?;
5419 schedule_respawn(
5420 runtime,
5421 snapshot,
5422 &spec.module_id,
5423 schedule.delay,
5424 RespawnKind::Spawn,
5425 )
5426}
5427
5428fn cancel_deferred_reload(runtime: &SupervisorRuntimeConfig, module_id: &str, reason: &str) {
5429 if let Some(reply) = runtime
5430 .deferred_reload_reply
5431 .lock()
5432 .unwrap_or_else(|p| p.into_inner())
5433 .take()
5434 {
5435 let _ = reply.send(Err(SuperviseError::ReloadFailed {
5436 module_id: module_id.to_string(),
5437 reason: reason.to_string(),
5438 }));
5439 }
5440}
5441
5442fn schedule_respawn(
5443 runtime: &SupervisorRuntimeConfig,
5444 snapshot: &SharedSnapshot,
5445 module_id: &str,
5446 delay: Duration,
5447 kind: RespawnKind,
5448) -> Result<(), SuperviseError> {
5449 cancel_deferred_reload(runtime, module_id, "respawn superseded by another restart");
5450 update_snapshot(snapshot, Some(module_id), |state| {
5451 state.respawn_pending = true
5452 })?;
5453 *runtime
5454 .scheduled_respawn
5455 .lock()
5456 .unwrap_or_else(|p| p.into_inner()) = Some(PendingRespawn {
5457 deadline: Instant::now() + delay,
5458 kind,
5459 });
5460 Ok(())
5461}
5462
5463fn record_health_action(snapshot: &SharedSnapshot, module_id: &str, action: String, now_ms: u64) {
5464 let _ = update_snapshot(snapshot, Some(module_id), |state| {
5465 state.health.last_action = Some(action);
5466 state.health.last_action_ms = Some(now_ms);
5467 });
5468}
5469
5470fn supervisor_health_status(status: HealthStatus) -> SupervisorHealthStatus {
5471 match status {
5472 HealthStatus::Ok => SupervisorHealthStatus::Ok,
5473 HealthStatus::Degraded => SupervisorHealthStatus::Degraded,
5474 HealthStatus::Failing => SupervisorHealthStatus::Failing,
5475 }
5476}
5477
5478fn truncate_health_metrics(metrics: Option<Value>) -> Option<Value> {
5490 let metrics = metrics?;
5491 match serde_json::to_vec(&metrics) {
5492 Ok(encoded) if encoded.len() > MAX_HEALTH_METRICS_BYTES => Some(serde_json::json!({
5493 "truncated": true,
5494 "original_bytes": encoded.len(),
5495 })),
5496 Ok(_) | Err(_) => Some(metrics),
5497 }
5498}
5499
5500fn jittered_health_delay(module_id: &str, probe_index: u64, cadence: Duration) -> Duration {
5506 if cadence.is_zero() {
5507 return Duration::ZERO;
5508 }
5509 let cadence_ms = cadence.as_millis() as u64;
5510 if cadence_ms == 0 {
5526 return cadence;
5527 }
5528 let jitter_span = (cadence_ms / 10).max(1);
5543 let hash = module_id.as_bytes().iter().fold(
5544 probe_index.wrapping_mul(0x9E37_79B9_7F4A_7C15),
5545 |acc, byte| {
5546 acc.wrapping_mul(1099511628211)
5547 .wrapping_add(u64::from(*byte))
5548 },
5549 );
5550 cadence + Duration::from_millis(hash % jitter_span)
5551}
5552
5553#[cfg(test)]
5554mod tests {
5555 use super::*;
5556
5557 #[test]
5558 fn readding_a_module_clears_its_rescan_removal_tombstone() {
5559 let handle = SupervisorHandle::new();
5560 let module_id = "readded-tombstone";
5561 handle.record_rescan_removal(module_id);
5562 assert!(handle.removal_tombstone_age_ms(module_id).is_some());
5563
5564 handle.apply_identity_configuration(&ModuleSpec {
5565 module_id: module_id.to_string(),
5566 program: PathBuf::from("/test/module"),
5567 args: Vec::new(),
5568 env: Vec::new(),
5569 reserved: false,
5570 reserved_prefixes: Vec::new(),
5571 protocol: ModuleProtocol::Subc,
5572 overlap: Default::default(),
5573 });
5574
5575 assert!(
5576 handle.removal_tombstone_age_ms(module_id).is_none(),
5577 "a re-added module must not retain a stale removal tombstone"
5578 );
5579 }
5580
5581 #[derive(Debug, PartialEq, Eq)]
5584 struct OwnerInSpawnWindow {
5585 module_id: String,
5586 configured: bool,
5587 on_roster: bool,
5588 admission_refusal: Option<&'static str>,
5589 }
5590
5591 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
5602 async fn a_new_module_is_configured_before_its_first_process_can_register() {
5603 use crate::scopes::ScopeTable;
5604 use subc_protocol::{error_codes, scope::ScopeSelector, Principal};
5605
5606 let dir = cortexkit_test_support::ScratchDir::new("configured-before-spawn");
5607 let stub = |module_id: &str, program: PathBuf| ModuleSpec {
5608 module_id: module_id.to_string(),
5609 program,
5610 args: Vec::new(),
5611 env: ["XDG_DATA_HOME", "XDG_RUNTIME_DIR", "XDG_CONFIG_HOME"]
5612 .into_iter()
5613 .map(|key| (key.to_string(), dir.path().display().to_string()))
5614 .chain([("FAKE_AFT_NEVER_CONNECT".to_string(), "1".to_string())])
5615 .collect(),
5616 reserved: false,
5617 reserved_prefixes: Vec::new(),
5618 protocol: ModuleProtocol::Subc,
5619 overlap: Default::default(),
5620 };
5621 let live = super::terminal_history_tests::fake_aft_stub_path();
5622 let missing = dir.path().join("definitely-missing-module");
5623
5624 let handle = SupervisorHandle::new();
5625 let mut supervisor =
5626 Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default())
5627 .with_handle(handle.clone());
5628 let observed = Arc::new(Mutex::new(Vec::<OwnerInSpawnWindow>::new()));
5629 let hook_handle = handle.clone();
5630 let hook_observed = Arc::clone(&observed);
5631 supervisor.test_after_first_spawn = AfterFirstSpawnHook(Some(Arc::new(move |module_id| {
5632 let configured = hook_handle.is_configured(module_id);
5635 let selector = ScopeSelector::new(
5636 Principal::Reserved {
5637 module_id: module_id.to_string(),
5638 },
5639 "s".to_string(),
5640 Some(1),
5641 );
5642 let carrier = Principal::Reserved {
5643 module_id: "carrier".to_string(),
5644 };
5645 let admission_refusal = match ScopeTable::new(Vec::<String>::new())
5646 .admit(&carrier, module_id, &selector, configured)
5647 {
5648 Ok(_) => None,
5649 Err(refusal) => Some(refusal.code),
5650 };
5651 hook_observed.lock().unwrap().push(OwnerInSpawnWindow {
5652 module_id: module_id.to_string(),
5653 configured,
5654 on_roster: hook_handle.get(module_id).is_some(),
5655 admission_refusal,
5656 });
5657 })));
5658
5659 let plain = supervisor.spawn(stub("plain", live.clone())).unwrap();
5660 let configured = supervisor
5661 .supervise_configured(stub("configured", live.clone()), true)
5662 .unwrap();
5663 let with_health = supervisor
5664 .supervise_configured_with_health(
5665 stub("with-health", live.clone()),
5666 true,
5667 HealthConfig::default(),
5668 None,
5669 RestartPolicy::default(),
5670 )
5671 .unwrap();
5672 let failed = supervisor
5675 .supervise_configured_with_health(
5676 stub("failed-spawn", missing.clone()),
5677 true,
5678 HealthConfig::default(),
5679 None,
5680 RestartPolicy::default(),
5681 )
5682 .unwrap();
5683 assert!(supervisor.spawn(stub("spawn-error", missing)).is_err());
5686
5687 let expected = [
5688 "plain",
5689 "configured",
5690 "with-health",
5691 "failed-spawn",
5692 "spawn-error",
5693 ]
5694 .into_iter()
5695 .map(|module_id| OwnerInSpawnWindow {
5696 module_id: module_id.to_string(),
5697 configured: true,
5698 on_roster: !matches!(module_id, "plain" | "spawn-error"),
5699 admission_refusal: Some(error_codes::SCOPE_NOT_SYNCED),
5700 })
5701 .collect::<Vec<_>>();
5702 assert_eq!(*observed.lock().unwrap(), expected);
5703
5704 for module_id in ["plain", "configured", "with-health", "failed-spawn"] {
5705 assert!(
5706 handle.get(module_id).is_some(),
5707 "{module_id} is on the roster"
5708 );
5709 assert!(
5710 handle.is_configured(module_id),
5711 "{module_id} stays configured"
5712 );
5713 }
5714 assert!(handle.get("spawn-error").is_none());
5715 assert!(
5716 !handle.is_configured("spawn-error"),
5717 "a plain spawn that failed must not leave its module marked configured"
5718 );
5719
5720 handle.retire("failed-spawn");
5722 assert!(!handle.is_configured("failed-spawn"));
5723
5724 for module in [plain, configured, with_health] {
5725 module.stop().await.unwrap();
5726 }
5727 drop(failed);
5728 }
5729
5730 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
5737 async fn a_configured_module_reads_starting_while_its_first_process_already_runs() {
5738 let dir = cortexkit_test_support::ScratchDir::new("starting-while-spawned");
5739 let spec = ModuleSpec {
5740 module_id: "spawned-not-admitted".to_string(),
5741 program: super::terminal_history_tests::fake_aft_stub_path(),
5742 args: Vec::new(),
5743 env: ["XDG_DATA_HOME", "XDG_RUNTIME_DIR", "XDG_CONFIG_HOME"]
5744 .into_iter()
5745 .map(|key| (key.to_string(), dir.path().display().to_string()))
5746 .chain([("FAKE_AFT_NEVER_CONNECT".to_string(), "1".to_string())])
5747 .collect(),
5748 reserved: false,
5749 reserved_prefixes: Vec::new(),
5750 protocol: ModuleProtocol::Subc,
5751 overlap: Default::default(),
5752 };
5753 let handle = SupervisorHandle::new();
5754 let mut supervisor =
5755 Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default())
5756 .with_handle(handle.clone());
5757 let observed = Arc::new(Mutex::new(None));
5758 let hook_handle = handle.clone();
5759 let hook_observed = Arc::clone(&observed);
5760 supervisor.test_after_first_spawn = AfterFirstSpawnHook(Some(Arc::new(move |module_id| {
5761 let state = hook_handle
5762 .get(module_id)
5763 .expect("a configured module is on the roster before it is spawned")
5764 .state()
5765 .unwrap();
5766 *hook_observed.lock().unwrap() = Some(state);
5767 })));
5768
5769 let module = supervisor.supervise_configured(spec, true).unwrap();
5770 assert_eq!(
5771 *observed.lock().unwrap(),
5772 Some(ModuleState::Starting),
5773 "between the spawn and its admission the supervisor still reports starting"
5774 );
5775 assert_eq!(module.state().unwrap(), ModuleState::Running);
5776 module.stop().await.unwrap();
5777 }
5778
5779 fn stale_process_snapshot(state: ModuleState, enabled: bool) -> SharedSnapshot {
5780 let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::new(state, enabled)));
5781 update_snapshot(&snapshot, Some("stale-process-facts"), |snapshot| {
5782 snapshot.process_alive = true;
5783 snapshot.pid = Some(41);
5784 snapshot.spawned_at_ms = Some(42);
5785 snapshot.spawned_from = Some(PathBuf::from("/spawned/module"));
5786 snapshot.launch_spec_digest = Some(LaunchSpecDigest([0xA5; 32]));
5787 snapshot.spawned_file_identity = Some(SpawnedFileIdentity {
5788 device: 43,
5789 inode: 44,
5790 #[cfg(windows)]
5791 windows_image: None,
5792 });
5793 })
5794 .unwrap();
5795 snapshot
5796 }
5797
5798 fn assert_snapshot_process_facts_cleared(snapshot: &SharedSnapshot) {
5799 let snapshot = lock_snapshot(snapshot).unwrap();
5800 assert!(!snapshot.process_alive);
5801 assert_eq!(snapshot.pid, None);
5802 assert_eq!(snapshot.spawned_at_ms, None);
5803 assert_eq!(snapshot.spawned_from, None);
5804 assert_eq!(snapshot.launch_spec_digest, None);
5805 assert_eq!(snapshot.spawned_file_identity, None);
5806 }
5807
5808 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
5809 async fn failed_enable_spawn_clears_preexisting_current_process_facts() {
5810 let supervisor =
5811 Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default());
5812 let mut runtime = supervisor.runtime_config();
5813 runtime.test_seed_stale_facts_before_enable_spawn = true;
5814 let snapshot = stale_process_snapshot(ModuleState::Disabled, false);
5815 let mut child = None;
5816 let spec = ModuleSpec {
5817 module_id: "failed-enable-clears-facts".to_string(),
5818 program: PathBuf::from("/definitely/missing/failed-enable-module"),
5819 args: Vec::new(),
5820 env: Vec::new(),
5821 reserved: false,
5822 reserved_prefixes: Vec::new(),
5823 protocol: ModuleProtocol::Subc,
5824 overlap: Default::default(),
5825 };
5826
5827 let result = set_child_enabled(
5828 &spec,
5829 &runtime,
5830 &supervisor.registry,
5831 &supervisor.process_liveness,
5832 &snapshot,
5833 &mut child,
5834 true,
5835 )
5836 .await;
5837
5838 assert!(matches!(result, Err(SuperviseError::Spawn { .. })));
5839 assert_eq!(lock_snapshot(&snapshot).unwrap().state, ModuleState::Failed);
5840 assert_snapshot_process_facts_cleared(&snapshot);
5841 }
5842
5843 #[tokio::test]
5844 async fn start_revives_stranded_restarting_but_not_pending_backoff() {
5845 let supervisor =
5846 Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default());
5847 let runtime = supervisor.runtime_config();
5848 let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::new(
5849 ModuleState::Restarting,
5850 true,
5851 )));
5852 let spec = ModuleSpec {
5853 module_id: "start-stranded-restarting".to_string(),
5854 program: super::terminal_history_tests::fake_aft_stub_path(),
5855 args: Vec::new(),
5856 env: vec![("FAKE_AFT_NEVER_CONNECT".to_string(), "1".to_string())],
5857 reserved: false,
5858 reserved_prefixes: Vec::new(),
5859 protocol: ModuleProtocol::None,
5860 overlap: Default::default(),
5861 };
5862 let mut child = None;
5863 lock_snapshot(&snapshot).unwrap().respawn_pending = true;
5864 assert!(!super::set_child_enabled(
5865 &spec,
5866 &runtime,
5867 &Registry::default(),
5868 &supervisor.process_liveness,
5869 &snapshot,
5870 &mut child,
5871 true
5872 )
5873 .await
5874 .unwrap());
5875 assert!(child.is_none());
5876 lock_snapshot(&snapshot).unwrap().respawn_pending = false;
5877 assert!(super::set_child_enabled(
5878 &spec,
5879 &runtime,
5880 &Registry::default(),
5881 &supervisor.process_liveness,
5882 &snapshot,
5883 &mut child,
5884 true
5885 )
5886 .await
5887 .unwrap());
5888 assert_eq!(
5889 lock_snapshot(&snapshot).unwrap().state,
5890 ModuleState::Running
5891 );
5892 let mut child = child.unwrap();
5893 child.start_kill().unwrap();
5894 child.wait().await.unwrap();
5895 }
5896
5897 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
5898 async fn failed_reload_spawn_clears_current_process_facts() {
5899 let supervisor =
5900 Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default());
5901 let mut runtime = supervisor.runtime_config();
5902 runtime.restart_policy = RestartPolicy::new(0, Duration::ZERO);
5903 let snapshot = stale_process_snapshot(ModuleState::Running, true);
5904 let mut child = None;
5905 let spec = ModuleSpec {
5906 module_id: "failed-reload-clears-facts".to_string(),
5907 program: PathBuf::from("/unused/failed-reload-module"),
5908 args: Vec::new(),
5909 env: Vec::new(),
5910 reserved: false,
5911 reserved_prefixes: Vec::new(),
5912 protocol: ModuleProtocol::Subc,
5913 overlap: Default::default(),
5914 };
5915
5916 let result = handle_reload_spawn_failure(
5917 &spec,
5918 &runtime,
5919 &supervisor.process_liveness,
5920 &snapshot,
5921 &mut child,
5922 "forced reload spawn failure".to_string(),
5923 )
5924 .await;
5925
5926 assert!(matches!(result, Err(SuperviseError::ReloadFailed { .. })));
5927 assert_eq!(lock_snapshot(&snapshot).unwrap().state, ModuleState::Failed);
5928 assert_snapshot_process_facts_cleared(&snapshot);
5929 }
5930
5931 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
5932 async fn dropping_a_module_with_an_active_monitor_clears_current_process_facts() {
5933 let supervisor =
5934 Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default());
5935 let snapshot = stale_process_snapshot(ModuleState::Running, true);
5936 let module = supervisor.supervised_module(
5937 ModuleSpec {
5938 module_id: "drop-clears-facts".to_string(),
5939 program: PathBuf::from("/unused/drop-module"),
5940 args: Vec::new(),
5941 env: Vec::new(),
5942 reserved: false,
5943 reserved_prefixes: Vec::new(),
5944 protocol: ModuleProtocol::Subc,
5945 overlap: Default::default(),
5946 },
5947 supervisor.runtime_config(),
5948 Arc::clone(&snapshot),
5949 None,
5950 );
5951 assert!(!module
5952 .inner
5953 .monitor
5954 .lock()
5955 .unwrap()
5956 .as_ref()
5957 .unwrap()
5958 .is_finished());
5959
5960 drop(module);
5961
5962 assert_eq!(
5963 lock_snapshot(&snapshot).unwrap().state,
5964 ModuleState::Stopped
5965 );
5966 assert_snapshot_process_facts_cleared(&snapshot);
5967 }
5968
5969 #[cfg(unix)]
5970 #[tokio::test]
5971 async fn rescan_preserves_running_protocol_until_respawn() {
5972 let dir = cortexkit_test_support::ScratchDir::new("rescan-protocol");
5973 let initial = ModuleSpec {
5974 module_id: "rescan-protocol".into(),
5975 program: PathBuf::from("/bin/sleep"),
5976 args: vec!["60".into()],
5977 env: ["XDG_DATA_HOME", "XDG_RUNTIME_DIR", "XDG_CONFIG_HOME"]
5978 .into_iter()
5979 .map(|key| (key.into(), dir.path().to_string_lossy().into_owned()))
5980 .collect(),
5981 reserved: false,
5982 reserved_prefixes: vec![],
5983 protocol: ModuleProtocol::None,
5984 overlap: Default::default(),
5985 };
5986 let supervisor =
5987 Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default());
5988 let module = supervisor.spawn(initial.clone()).unwrap();
5989 assert!(module.status().unwrap().live);
5990 let mut next = initial;
5991 next.protocol = ModuleProtocol::Subc;
5992 module
5993 .update_configuration(next.clone(), HealthConfig::default(), None)
5994 .await
5995 .unwrap();
5996 assert!(
5997 module.status().unwrap().live,
5998 "rescan must not require HELLO from the old non-wire process"
5999 );
6000 let runtime = supervisor.runtime_config();
6001 let action = on_child_exit(
6002 &next,
6003 RestartPolicy::default(),
6004 &supervisor.registry,
6005 &module.inner.snapshot,
6006 &runtime.terminal_ring,
6007 &runtime.spawn_events,
6008 &runtime.child_roster,
6009 ExitReport {
6010 kind: ExitKind::Clean,
6011 code: Some(0),
6012 signal: None,
6013 at_ms: unix_ms_now(),
6014 },
6015 )
6016 .await;
6017 assert!(
6018 matches!(action, NextAction::Restart { .. }),
6019 "the old non-wire process's clean exit must restart"
6020 );
6021 module.drain().await.unwrap();
6022 }
6023
6024 #[cfg(unix)]
6025 #[tokio::test]
6026 async fn orphan_identity_matches_path_names_and_shebang_interpreters() {
6027 use std::os::unix::fs::PermissionsExt;
6028 let dir = cortexkit_test_support::ScratchDir::new("spawn-image-identity");
6029 let script = dir.join("module.sh");
6030 std::fs::write(&script, "#!/bin/sh\nwhile :; do sleep 0.1; done\n").unwrap();
6031 std::fs::set_permissions(&script, std::fs::Permissions::from_mode(0o700)).unwrap();
6032 let record_path = dir.join("live-children.json");
6033 let supervisor =
6034 Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default())
6035 .with_live_children_record(&record_path);
6036 for (program, args) in [
6037 (PathBuf::from("sleep"), vec!["60".into()]),
6038 (script, vec![]),
6039 ] {
6040 let spec = ModuleSpec {
6041 module_id: "image-identity".into(),
6042 program: program.clone(),
6043 args,
6044 env: ["XDG_DATA_HOME", "XDG_RUNTIME_DIR", "XDG_CONFIG_HOME"]
6045 .into_iter()
6046 .map(|key| (key.into(), dir.to_string_lossy().into_owned()))
6047 .collect(),
6048 reserved: false,
6049 reserved_prefixes: vec![],
6050 protocol: ModuleProtocol::None,
6051 overlap: Default::default(),
6052 };
6053 let module = supervisor.spawn(spec).unwrap();
6054 #[cfg(target_os = "macos")]
6055 {
6056 let deadline = Instant::now() + Duration::from_secs(5);
6059 while crate::live_children::read_record(&record_path)
6060 .unwrap()
6061 .iter()
6062 .all(|entry| entry.executable.is_none())
6063 {
6064 assert!(Instant::now() < deadline, "module image was not confirmed");
6065 tokio::time::sleep(Duration::from_millis(5)).await;
6066 }
6067 }
6068 let entry = crate::live_children::read_record(&record_path)
6069 .unwrap()
6070 .pop()
6071 .unwrap();
6072 let observed = subc_os::Process::open(entry.pid)
6073 .unwrap()
6074 .unwrap()
6075 .observe()
6076 .unwrap();
6077 let verdict = crate::live_children::identity_verdict(&entry, entry.pid, &observed);
6078 module.drain().await.unwrap();
6079 assert_eq!(
6080 verdict,
6081 crate::live_children::IdentityVerdict::Matches,
6082 "program {program:?}: recorded {entry:?}, observed {observed:?}"
6083 );
6084 }
6085 }
6086
6087 #[cfg(unix)]
6088 fn http_fixture(
6089 dir: &std::path::Path,
6090 url: &str,
6091 threshold: u32,
6092 ) -> crate::daemon_config::ConfiguredModule {
6093 let path = dir.join("subc.jsonc");
6094 std::fs::write(&path, serde_json::json!({"version": 1, "modules": {"http-process": {
6095 "program": "/bin/sleep", "args": ["60"], "protocol": "none",
6096 "env": {"XDG_DATA_HOME":dir,"XDG_RUNTIME_DIR":dir,"XDG_CONFIG_HOME":dir},
6097 "health": {"http":url,"cadence_ms":20,"deadline_ms":200,"failure_threshold":threshold},
6098 "restart": {"backoff_ms":1,"max_backoff_ms":1}, "drain_timeout_ms":10
6099 }}}).to_string()).unwrap();
6100 crate::daemon_config::load(&path)
6101 .unwrap()
6102 .unwrap()
6103 .modules
6104 .pop()
6105 .unwrap()
6106 }
6107
6108 #[cfg(unix)]
6109 async fn wait_http_health(module: &SupervisedModule, status: SupervisorHealthStatus) {
6110 timeout(Duration::from_secs(5), async {
6111 loop {
6112 if module.status().unwrap().health.status == status {
6113 break;
6114 }
6115 sleep(Duration::from_millis(5)).await;
6116 }
6117 })
6118 .await
6119 .unwrap_or_else(|_| {
6120 panic!(
6121 "expected {status:?}, got {:?}",
6122 module.status().unwrap().health
6123 )
6124 });
6125 }
6126
6127 #[cfg(unix)]
6128 #[tokio::test]
6129 async fn http_health_status_flips_ok_failing_ok() {
6130 use tokio::io::{AsyncReadExt, AsyncWriteExt};
6131 let dir = cortexkit_test_support::ScratchDir::new("http-status-flips");
6132 let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
6133 let status = Arc::new(std::sync::atomic::AtomicU16::new(200));
6134 let serving_status = status.clone();
6135 let url = format!("http://{}/healthz", listener.local_addr().unwrap());
6136 let server = tokio::spawn(async move {
6137 loop {
6138 let (mut stream, _) = listener.accept().await.unwrap();
6139 let mut request = [0u8; 2048];
6140 let count = stream.read(&mut request).await.unwrap();
6141 assert!(count > 0, "a probe must send an HTTP request");
6142 let code = serving_status.load(std::sync::atomic::Ordering::SeqCst);
6143 let body = if code == 200 {
6144 "ready"
6145 } else {
6146 "{\"status\":\"unavailable\",\"error\":\"scratch failure\"}"
6147 };
6148 let response = format!("HTTP/1.1 {code} scratch\r\nContent-Length: {}\r\nConnection: close\r\n\r\n{body}", body.len());
6149 let _ = stream.write_all(response.as_bytes()).await;
6150 }
6151 });
6152 let configured = http_fixture(&dir, &url, 1000);
6153 let module =
6154 Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default())
6155 .supervise_configured_with_health(
6156 configured.module_spec(),
6157 true,
6158 configured.health,
6159 configured.drain_timeout_ms,
6160 configured.restart,
6161 )
6162 .unwrap();
6163 wait_http_health(&module, SupervisorHealthStatus::Ok).await;
6164 status.store(503, std::sync::atomic::Ordering::SeqCst);
6165 wait_http_health(&module, SupervisorHealthStatus::Failing).await;
6166 assert!(module
6167 .status()
6168 .unwrap()
6169 .health
6170 .detail
6171 .unwrap()
6172 .contains("scratch failure"));
6173 status.store(200, std::sync::atomic::Ordering::SeqCst);
6174 wait_http_health(&module, SupervisorHealthStatus::Ok).await;
6175 assert_eq!(module.status().unwrap().health.consecutive_failures, 0);
6176 let before = module.status().unwrap();
6177 let (spec, mut health) = module.configuration().unwrap();
6178 health.http = None;
6179 module
6180 .update_configuration(spec.clone(), health.clone(), Some(10))
6181 .await
6182 .unwrap();
6183 wait_http_health(&module, SupervisorHealthStatus::Unknown).await;
6184 health.http = Some(url);
6185 module
6186 .update_configuration(spec, health, Some(10))
6187 .await
6188 .unwrap();
6189 wait_http_health(&module, SupervisorHealthStatus::Ok).await;
6190 assert_eq!(
6191 module.status().unwrap().pid,
6192 before.pid,
6193 "changing a probe must apply live, not restart its process"
6194 );
6195 let (spec, mut health) = module.configuration().unwrap();
6196 health.failure_threshold = 2;
6197 module
6198 .update_configuration(spec, health, Some(10))
6199 .await
6200 .unwrap();
6201 status.store(503, std::sync::atomic::Ordering::SeqCst);
6202 timeout(Duration::from_secs(5), async {
6203 while module.status().unwrap().spawn_generation == before.spawn_generation {
6204 sleep(Duration::from_millis(5)).await;
6205 }
6206 })
6207 .await
6208 .expect("sustained HTTP 503 responses must trigger the health restart policy");
6209 module.drain().await.unwrap();
6210 server.abort();
6211 }
6212
6213 #[cfg(unix)]
6214 #[tokio::test]
6215 async fn http_health_no_listener_restarts_after_consecutive_failures() {
6216 let dir = cortexkit_test_support::ScratchDir::new("http-refused");
6217 let unused = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
6218 let url = format!("http://{}/healthz", unused.local_addr().unwrap());
6219 drop(unused);
6220 let configured = http_fixture(&dir, &url, 2);
6221 let module =
6222 Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default())
6223 .supervise_configured_with_health(
6224 configured.module_spec(),
6225 true,
6226 configured.health,
6227 configured.drain_timeout_ms,
6228 configured.restart,
6229 )
6230 .unwrap();
6231 let before = module.status().unwrap().spawn_generation;
6232 timeout(Duration::from_secs(5), async {
6233 loop {
6234 let status = module.status().unwrap();
6235 if status.spawn_generation > before {
6236 assert!(status.lifetime_restarts > 0);
6237 break;
6238 }
6239 sleep(Duration::from_millis(5)).await;
6240 }
6241 })
6242 .await
6243 .expect("sustained HTTP refusal must trigger the health restart policy");
6244 module.drain().await.unwrap();
6245 }
6246
6247 #[cfg(unix)]
6248 #[tokio::test]
6249 async fn http_health_timeout_honours_deadline() {
6250 let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
6251 let url = format!("http://{}/healthz", listener.local_addr().unwrap());
6252 let server = tokio::spawn(async move {
6253 let _held = listener.accept().await.unwrap();
6254 std::future::pending::<()>().await;
6255 });
6256 let error = timeout(
6257 Duration::from_secs(1),
6258 probe_http_health(&url, Duration::from_millis(10)),
6259 )
6260 .await
6261 .expect("the probe must enforce its own deadline")
6262 .unwrap_err();
6263 server.abort();
6264 assert!(error.to_string().contains("timed out"));
6265 }
6266
6267 #[cfg(unix)]
6268 #[tokio::test]
6269 async fn http_health_localhost_reaches_an_ipv6_only_listener() {
6270 use tokio::io::{AsyncReadExt, AsyncWriteExt};
6271 let listener = tokio::net::TcpListener::bind("[::1]:0").await.unwrap();
6272 let url = format!(
6273 "http://localhost:{}/healthz",
6274 listener.local_addr().unwrap().port()
6275 );
6276 let server = tokio::spawn(async move {
6277 let (mut stream, _) = listener.accept().await.unwrap();
6278 let mut request = [0u8; 2048];
6279 assert!(stream.read(&mut request).await.unwrap() > 0);
6280 stream
6281 .write_all(b"HTTP/1.1 200 OK\r\nContent-Length: 2\r\n\r\nok")
6282 .await
6283 .unwrap();
6284 });
6285 assert_eq!(
6290 probe_http_health(&url, Duration::from_secs(10))
6291 .await
6292 .unwrap()
6293 .status,
6294 HealthStatus::Ok
6295 );
6296 server.await.unwrap();
6297 }
6298
6299 #[cfg(unix)]
6300 #[tokio::test]
6301 async fn http_health_timeout_keeps_partial_status_and_body() {
6302 use tokio::io::{AsyncReadExt, AsyncWriteExt};
6303 let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
6304 let url = format!("http://{}/healthz", listener.local_addr().unwrap());
6305 let server = tokio::spawn(async move {
6306 let (mut stream, _) = listener.accept().await.unwrap();
6307 let mut request = [0u8; 2048];
6308 assert!(stream.read(&mut request).await.unwrap() > 0);
6309 stream
6310 .write_all(
6311 b"HTTP/1.1 503 Unavailable\r\nContent-Length: 100\r\n\r\npartial diagnostic",
6312 )
6313 .await
6314 .unwrap();
6315 std::future::pending::<()>().await;
6316 });
6317 let error = probe_http_health(&url, Duration::from_secs(1))
6318 .await
6319 .unwrap_err()
6320 .to_string();
6321 server.abort();
6322 assert!(
6323 error.contains("timed out")
6324 && error.contains("503 Unavailable")
6325 && error.contains("partial diagnostic"),
6326 "{error}"
6327 );
6328 }
6329
6330 #[cfg(unix)]
6331 #[tokio::test]
6332 async fn http_health_diagnostic_body_is_bounded_to_200_bytes() {
6333 use tokio::io::{AsyncReadExt, AsyncWriteExt};
6334 let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
6335 let url = format!("http://{}/healthz", listener.local_addr().unwrap());
6336 let server = tokio::spawn(async move {
6337 let (mut stream, _) = listener.accept().await.unwrap();
6338 let mut request = [0u8; 2048];
6339 assert!(stream.read(&mut request).await.unwrap() > 0);
6340 let body = format!("{}not-in-diagnostic", "x".repeat(400));
6341 let response = format!(
6342 "HTTP/1.1 503 Unavailable\r\nContent-Length: {}\r\n\r\n{body}",
6343 body.len()
6344 );
6345 stream.write_all(response.as_bytes()).await.unwrap();
6346 });
6347 let error = probe_http_health(&url, Duration::from_secs(1))
6348 .await
6349 .unwrap_err()
6350 .to_string();
6351 server.await.unwrap();
6352 assert!(error.ends_with(&"x".repeat(200)), "{error}");
6353 assert_eq!(error.split(": ").last().unwrap().len(), 200);
6354 assert!(!error.contains("not-in-diagnostic"));
6355 }
6356
6357 #[cfg(unix)]
6358 #[tokio::test]
6359 async fn http_health_real_nats_server_monitoring() {
6360 if std::process::Command::new("nats-server")
6361 .arg("--version")
6362 .env("XDG_DATA_HOME", std::env::temp_dir())
6363 .env("XDG_RUNTIME_DIR", std::env::temp_dir())
6364 .env("XDG_CONFIG_HOME", std::env::temp_dir())
6365 .output()
6366 .is_err()
6367 {
6368 eprintln!(
6369 "skipped: http_health_real_nats_server_monitoring (nats-server is not on PATH)"
6370 );
6371 return;
6372 }
6373 let dir = cortexkit_test_support::ScratchDir::new("nats-http-monitoring");
6374 let monitor = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
6375 let port = monitor.local_addr().unwrap().port();
6376 let client = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
6377 let client_port = client.local_addr().unwrap().port();
6378 let config = dir.join("server.conf");
6379 std::fs::write(&config, format!("listen: 127.0.0.1:{client_port}\nhttp: 127.0.0.1:{port}\njetstream {{ store_dir: \"{}\" }}\n", dir.join("jetstream").display())).unwrap();
6380 let mut configured = http_fixture(&dir, &format!("http://127.0.0.1:{port}/healthz"), 1000);
6381 configured.program = PathBuf::from("nats-server");
6382 configured.args = vec!["-c".into(), config.to_string_lossy().into_owned()];
6383 drop(monitor);
6384 drop(client);
6385 let module =
6386 Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default())
6387 .supervise_configured_with_health(
6388 configured.module_spec(),
6389 true,
6390 configured.health,
6391 configured.drain_timeout_ms,
6392 configured.restart,
6393 )
6394 .unwrap();
6395 wait_http_health(&module, SupervisorHealthStatus::Ok).await;
6396 module.drain().await.unwrap();
6397 }
6398
6399 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
6400 async fn configuration_update_does_not_replace_captured_running_process_facts() {
6401 let supervisor =
6402 Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default());
6403 let snapshot = stale_process_snapshot(ModuleState::Running, true);
6404 let initial = ModuleSpec {
6405 module_id: "rescan-preserves-spawn-facts".to_string(),
6406 program: PathBuf::from("/spawned/module"),
6407 args: Vec::new(),
6408 env: Vec::new(),
6409 reserved: false,
6410 reserved_prefixes: Vec::new(),
6411 protocol: ModuleProtocol::Subc,
6412 overlap: Default::default(),
6413 };
6414 let module = supervisor.supervised_module(
6415 initial.clone(),
6416 supervisor.runtime_config(),
6417 snapshot,
6418 None,
6419 );
6420 let before = module.status().unwrap();
6421 let mut replacement = initial;
6422 replacement.program = PathBuf::from("/rescanned/replacement-module");
6423
6424 module
6425 .update_configuration(replacement, HealthConfig::default(), None)
6426 .await
6427 .unwrap();
6428
6429 let after = module.status().unwrap();
6430 assert_eq!(after.pid, before.pid);
6431 assert_eq!(after.spawned_at_ms, before.spawned_at_ms);
6432 assert_eq!(after.spawned_from, before.spawned_from);
6433 drop(module);
6434 }
6435}
6436
6437fn unix_ms_now() -> u64 {
6438 SystemTime::now()
6439 .duration_since(UNIX_EPOCH)
6440 .map(|duration| duration.as_millis().min(u128::from(u64::MAX)) as u64)
6441 .unwrap_or(0)
6442}
6443
6444async fn supervise_loop(
6445 mut spec: ModuleSpec,
6446 mut runtime: SupervisorRuntimeConfig,
6447 registry: Arc<Registry>,
6448 process_liveness: Arc<SupervisorProcessLiveness>,
6449 snapshot: SharedSnapshot,
6450 mut child: Option<SupervisedChild>,
6451 mut commands: mpsc::Receiver<SupervisorCommand>,
6452) {
6453 let mut health_probe = HealthProbeRuntime::default();
6454 let mut registration_changes = match registry.subscribe_module_changes(&spec.module_id) {
6458 Ok(changes) => Some(changes),
6459 Err(err) => {
6460 warn!(module_id = %spec.module_id, error = %err, "health prober could not subscribe to registry");
6463 None
6464 }
6465 };
6466 let mut pending_respawn: Option<PendingRespawn> = None;
6470 let mut requeued: VecDeque<SupervisorCommand> = VecDeque::new();
6473 let mut previous_instance_recheck = tokio::time::interval_at(
6474 Instant::now() + Duration::from_secs(5),
6475 Duration::from_secs(5),
6476 );
6477 previous_instance_recheck.set_missed_tick_behavior(tokio::time::MissedTickBehavior::Skip);
6478 loop {
6479 #[cfg(test)]
6480 {
6481 lock_snapshot(&snapshot).unwrap().actor_select = None;
6482 }
6483 #[cfg(target_os = "macos")]
6484 if let Some(active) = child.as_mut() {
6485 active.confirm_privacy_exec().await;
6486 }
6487 if let Some(scheduled) = runtime
6488 .scheduled_respawn
6489 .lock()
6490 .unwrap_or_else(|p| p.into_inner())
6491 .take()
6492 {
6493 pending_respawn = Some(scheduled);
6494 }
6495 if pending_respawn.is_some() && (child.is_some() || !respawn_still_pending(&snapshot)) {
6496 pending_respawn = None;
6497 cancel_deferred_reload(
6498 &runtime,
6499 &spec.module_id,
6500 "respawn cancelled by a supervisor command",
6501 );
6502 }
6503 if child.is_none()
6504 && pending_respawn.is_none()
6505 && lock_snapshot(&snapshot).is_ok_and(|state| state.waiting_for_previous_pid.is_none())
6506 {
6507 cancel_deferred_reload(
6508 &runtime,
6509 &spec.module_id,
6510 "respawn cancelled before a replacement was spawned",
6511 );
6512 let _ = update_snapshot(&snapshot, Some(&spec.module_id), |state| {
6513 state.respawn_pending = false;
6514 state.coalesced_restart_pending = false;
6515 if matches!(
6516 state.state,
6517 ModuleState::Restarting
6518 | ModuleState::Starting
6519 | ModuleState::Draining
6520 | ModuleState::Unresponsive
6521 ) {
6522 error!(module_id = %spec.module_id, state = ?state.state,
6523 "supervision operation ended without a child or pending respawn; marking failed so start can retry");
6524 state.state = ModuleState::Failed;
6525 clear_current_process_facts(state);
6526 }
6527 });
6528 }
6529 if let Some(command) = requeued.pop_front() {
6530 if !handle_supervisor_command(
6531 command,
6532 &mut spec,
6533 &mut runtime,
6534 ®istry,
6535 &process_liveness,
6536 &snapshot,
6537 &mut child,
6538 &mut commands,
6539 &mut requeued,
6540 )
6541 .await
6542 {
6543 return;
6544 }
6545 if child.is_some() || !respawn_still_pending(&snapshot) {
6546 pending_respawn = None;
6547 let _ = update_snapshot(&snapshot, Some(&spec.module_id), |state| {
6548 state.respawn_pending = false
6549 });
6550 }
6551 continue;
6552 }
6553 if child.is_some() {
6554 #[cfg(test)]
6555 {
6556 lock_snapshot(&snapshot).unwrap().actor_turns += 1;
6557 }
6558 health_probe.refresh_registration(&spec, &runtime, ®istry, &snapshot);
6559 let wake_after = health_probe.wake_after();
6560 let probe_sleep = async move {
6561 match wake_after {
6562 Some(delay) => sleep(delay).await,
6563 None => std::future::pending().await,
6564 }
6565 };
6566 tokio::pin!(probe_sleep);
6567 let wire_child = running_protocol(&spec, &snapshot) == ModuleProtocol::Subc;
6572 #[cfg(test)]
6573 {
6574 let registration_pending = wire_child
6575 && registration_changes
6576 .as_ref()
6577 .is_some_and(|changes| changes.has_changed().unwrap());
6578 let mut state = lock_snapshot(&snapshot).unwrap();
6579 state.actor_select = (!registration_pending).then_some(ActorSelectCheckpoint {
6580 generation: state.spawn_generation,
6581 turn: state.actor_turns,
6582 registered_connection: health_probe.registered_connection,
6583 next_probe_at: health_probe.next_probe_at,
6584 wake_after,
6585 });
6586 }
6587 let active_child = child.as_mut().expect("child checked above");
6588 tokio::select! {
6589 wait_result = active_child.wait() => {
6590 #[cfg(test)]
6591 { lock_snapshot(&snapshot).unwrap().actor_select = None; }
6592 let exit_report = match wait_result {
6601 Ok(status) => classify_reaped_child_exit(&snapshot, active_child, &status),
6602 Err(err) => {
6603 active_child.drain_stderr(&spec.module_id).await;
6604 fail_after_wait_error(
6605 &spec.module_id,
6606 &snapshot,
6607 &runtime.terminal_ring,
6608 &runtime.spawn_events,
6609 format!("failed to wait for supervised module: {err}"),
6610 );
6611 untrack_if_registration_released(
6612 &process_liveness,
6613 ®istry,
6614 &spec.module_id,
6615 &snapshot,
6616 );
6617 error!(module_id = %spec.module_id, error = %err, "failed to wait for supervised module");
6618 child = None;
6619 continue;
6620 }
6621 };
6622 active_child.drain_stderr(&spec.module_id).await;
6623
6624 let next = on_child_exit(
6625 &spec,
6626 runtime.restart_policy,
6627 ®istry,
6628 &snapshot,
6629 &runtime.terminal_ring,
6630 &runtime.spawn_events,
6631 &runtime.child_roster,
6632 exit_report,
6633 ).await;
6634 runtime.start_requester = None;
6635 runtime.pending_activity_restart = false;
6636 active_child.release_roster();
6639 match next {
6640 NextAction::Stop { registration_released } => {
6641 if registration_released {
6642 process_liveness.untrack_if_current(&spec.module_id, &snapshot);
6643 }
6644 child = None;
6645 }
6646 NextAction::Restart { schedule } => {
6647 let delay = schedule.map_or(
6648 runtime.restart_policy.delay_for_restart(0),
6649 |schedule| schedule.delay,
6650 );
6651 if let Some(schedule) = schedule {
6652 log_crash_respawn(&spec.module_id, schedule);
6653 }
6654 child = None;
6662 pending_respawn = Some(PendingRespawn { deadline: Instant::now() + delay, kind: RespawnKind::Spawn });
6663 let _ = update_snapshot(&snapshot, Some(&spec.module_id), |state| state.respawn_pending = true);
6664 }
6665 }
6666 }
6667 command = commands.recv() => {
6668 #[cfg(test)]
6669 { lock_snapshot(&snapshot).unwrap().actor_select = None; }
6670 let Some(command) = command else {
6671 return;
6672 };
6673 if !handle_supervisor_command(
6674 command,
6675 &mut spec,
6676 &mut runtime,
6677 ®istry,
6678 &process_liveness,
6679 &snapshot,
6680 &mut child,
6681 &mut commands,
6682 &mut requeued,
6683 ).await {
6684 return;
6685 }
6686 }
6687 _ = async {
6688 match registration_changes.as_mut() {
6689 Some(changes) => { let _ = changes.changed().await; }
6690 None => std::future::pending().await,
6691 }
6692 }, if wire_child => {
6693 #[cfg(test)]
6694 { lock_snapshot(&snapshot).unwrap().actor_select = None; }
6695 }
6696 _ = &mut probe_sleep => {
6697 #[cfg(test)]
6698 { lock_snapshot(&snapshot).unwrap().actor_select = None; }
6699 if health_probe.due() {
6700 run_health_probe_cycle(
6701 &spec,
6702 &runtime,
6703 ®istry,
6704 &process_liveness,
6705 &snapshot,
6706 &mut child,
6707 ).await;
6708 if child.is_none() {
6709 runtime.start_requester = None;
6710 runtime.pending_activity_restart = false;
6711 }
6712 if child.is_some() {
6713 health_probe.schedule_next(&spec, runtime.health.cadence);
6714 }
6715 }
6716 }
6717 }
6718 } else if let Some(pending) = pending_respawn {
6719 tokio::select! {
6720 _ = sleep_until(pending.deadline) => {
6721 pending_respawn = None;
6722 let _ = update_snapshot(&snapshot, Some(&spec.module_id), |state| state.respawn_pending = false);
6723 if !respawn_still_pending(&snapshot) {
6727 continue;
6728 }
6729 if runtime.child_roster.is_closed() {
6734 let _ = update_snapshot(&snapshot, Some(&spec.module_id), |state| {
6735 state.state = ModuleState::Stopped;
6736 });
6737 debug!(module_id = %spec.module_id, "crash respawn cancelled by daemon shutdown");
6738 continue;
6739 }
6740 if let Err(err) = release_dead_registration(
6741 ®istry,
6742 runtime.forwarding.as_deref(),
6743 &snapshot,
6744 &spec.module_id,
6745 ).await {
6746 fail_snapshot(&snapshot, Some(&spec.module_id), None);
6747 error!(module_id = %spec.module_id, error = %err, "registration did not release before restart");
6748 continue;
6749 }
6750
6751 if matches!(pending.kind, RespawnKind::Reload) {
6752 let reply = runtime.deferred_reload_reply.lock().unwrap_or_else(|p| p.into_inner()).take();
6753 let result = finish_reload_child(&spec, &runtime, ®istry, &process_liveness, &snapshot, &mut child).await;
6754 if result.is_ok() && runtime.pending_activity_restart {
6755 runtime.record_activity_transition("subc.module_restarted", &spec.module_id, runtime.start_requester.clone());
6756 }
6757 runtime.pending_activity_restart = false;
6758 if let Some(reply) = reply { let _ = reply.send(result); }
6759 continue;
6760 }
6761 process_liveness.track(spec.module_id.clone(), Arc::clone(&snapshot));
6762 match spawn_and_mark_running(&spec, &runtime, &snapshot) {
6763 Ok(next_child) => {
6764 child = Some(next_child);
6765 if runtime.pending_activity_restart {
6766 runtime.record_activity_transition("subc.module_restarted", &spec.module_id, runtime.start_requester.clone());
6767 runtime.pending_activity_restart = false;
6768 }
6769 debug!(module_id = %spec.module_id, "supervised module restarted after crash");
6770 }
6771 Err(err) => {
6772 fail_snapshot(&snapshot, Some(&spec.module_id), None);
6773 process_liveness.untrack_if_current(&spec.module_id, &snapshot);
6774 error!(module_id = %spec.module_id, error = %err, "failed to restart supervised module");
6775 }
6776 }
6777 }
6778 command = commands.recv() => {
6779 let Some(command) = command else {
6780 return;
6781 };
6782 if !handle_supervisor_command(
6783 command,
6784 &mut spec,
6785 &mut runtime,
6786 ®istry,
6787 &process_liveness,
6788 &snapshot,
6789 &mut child,
6790 &mut commands,
6791 &mut requeued,
6792 ).await {
6793 return;
6794 }
6795 if child.is_some() || !respawn_still_pending(&snapshot) {
6800 pending_respawn = None;
6801 let _ = update_snapshot(&snapshot, Some(&spec.module_id), |state| state.respawn_pending = false);
6802 }
6803 }
6804 }
6805 } else {
6806 let command = if lock_snapshot(&snapshot)
6807 .is_ok_and(|state| state.waiting_for_previous_pid.is_some())
6808 {
6809 tokio::select! {
6810 command = commands.recv() => command,
6811 _ = previous_instance_recheck.tick() => {
6812 if runtime.child_roster.previous_instance_pid(&spec.module_id).is_none() {
6813 let enabled = lock_snapshot(&snapshot).is_ok_and(|state| state.enabled);
6814 if enabled && !runtime.child_roster.is_closed() {
6815 process_liveness.track(spec.module_id.clone(), Arc::clone(&snapshot));
6816 match spawn_and_mark_running(&spec, &runtime, &snapshot) {
6817 Ok(next_child) => child = Some(next_child),
6818 Err(error) => {
6819 fail_snapshot(&snapshot, Some(&spec.module_id), None);
6820 error!(module_id = %spec.module_id, %error, "failed to start module after its previous instance exited");
6821 }
6822 }
6823 } else {
6824 let _ = update_snapshot(&snapshot, Some(&spec.module_id), |state| {
6825 state.waiting_for_previous_pid = None;
6826 state.waiting_for_previous_detail = None;
6827 });
6828 }
6829 }
6830 continue;
6831 }
6832 }
6833 } else {
6834 commands.recv().await
6835 };
6836 let Some(command) = command else {
6837 return;
6838 };
6839 if !handle_supervisor_command(
6840 command,
6841 &mut spec,
6842 &mut runtime,
6843 ®istry,
6844 &process_liveness,
6845 &snapshot,
6846 &mut child,
6847 &mut commands,
6848 &mut requeued,
6849 )
6850 .await
6851 {
6852 return;
6853 }
6854 }
6855 }
6856}
6857
6858fn log_crash_respawn(module_id: &str, schedule: CrashRestartSchedule) {
6859 info!(
6860 module_id,
6861 restart_in_window = schedule.restart_in_window,
6862 delay_ms = schedule.delay.as_millis() as u64,
6863 "respawning after crash"
6864 );
6865}
6866
6867fn respawn_still_pending(snapshot: &SharedSnapshot) -> bool {
6873 matches!(
6874 lock_snapshot(snapshot),
6875 Ok(state) if state.enabled && state.state == ModuleState::Restarting
6876 )
6877}
6878
6879enum NextAction {
6880 Stop {
6881 registration_released: bool,
6882 },
6883 Restart {
6884 schedule: Option<CrashRestartSchedule>,
6885 },
6886}
6887
6888#[allow(clippy::too_many_arguments)]
6889async fn handle_supervisor_command(
6890 command: SupervisorCommand,
6891 spec: &mut ModuleSpec,
6892 runtime: &mut SupervisorRuntimeConfig,
6893 registry: &Arc<Registry>,
6894 process_liveness: &SupervisorProcessLiveness,
6895 snapshot: &SharedSnapshot,
6896 child: &mut Option<SupervisedChild>,
6897 commands: &mut mpsc::Receiver<SupervisorCommand>,
6898 requeued: &mut VecDeque<SupervisorCommand>,
6899) -> bool {
6900 let command =
6901 if let Some((pid, reason)) = runtime.child_roster.previous_instance(&spec.module_id) {
6902 match command {
6903 SupervisorCommand::Restart { reply, .. }
6904 | SupervisorCommand::Reload { reply, .. }
6905 | SupervisorCommand::Swap { reply, .. } => {
6906 let _ = reply.send(Err(previous_instance_error(spec, pid, reason)));
6907 return true;
6908 }
6909 SupervisorCommand::SetEnabled {
6910 enabled: true,
6911 reply,
6912 ..
6913 } => {
6914 let _ = reply.send(Err(previous_instance_error(spec, pid, reason)));
6915 return true;
6916 }
6917 other => other,
6918 }
6919 } else {
6920 command
6921 };
6922 match command {
6923 SupervisorCommand::Drain { requester, reply } => {
6924 let result = drain_optional_child(
6927 &spec.module_id,
6928 spec.protocol,
6929 StopNotice::NotSent,
6930 registry,
6931 runtime.forwarding.as_deref(),
6932 snapshot,
6933 &runtime.terminal_ring,
6934 &runtime.spawn_events,
6935 child,
6936 runtime.drain_timeout,
6937 ModuleState::Stopped,
6938 None,
6939 )
6940 .await;
6941 let registration_released = result.is_ok();
6942 if registration_released {
6943 runtime.record_activity_transition(
6944 "subc.module_stopped",
6945 &spec.module_id,
6946 requester,
6947 );
6948 }
6949 let _ = reply.send(result);
6950 if registration_released {
6951 process_liveness.untrack_if_current(&spec.module_id, snapshot);
6952 }
6953 false
6954 }
6955 SupervisorCommand::Retire { requester, reply } => {
6956 let result = async {
6957 let stop_notice = begin_forwarding_drain_if_configured(
6958 spec,
6959 runtime,
6960 registry,
6961 snapshot,
6962 None,
6963 RouteCloseReason::Disable,
6964 )
6965 .await?;
6966 drain_optional_child(
6967 &spec.module_id,
6968 spec.protocol,
6969 stop_notice,
6970 registry,
6971 runtime.forwarding.as_deref(),
6972 snapshot,
6973 &runtime.terminal_ring,
6974 &runtime.spawn_events,
6975 child,
6976 runtime.drain_timeout,
6977 ModuleState::Stopped,
6978 None,
6979 )
6980 .await
6981 }
6982 .await;
6983 let registration_released = result.is_ok();
6984 if registration_released {
6985 runtime.record_activity_transition(
6986 "subc.module_stopped",
6987 &spec.module_id,
6988 requester,
6989 );
6990 }
6991 let _ = reply.send(result);
6992 if registration_released {
6993 process_liveness.untrack_if_current(&spec.module_id, snapshot);
6994 }
6995 false
6996 }
6997 SupervisorCommand::Restart {
6998 requester,
6999 drain_timeout_ms,
7000 received_at_generation,
7001 queued_at,
7002 reply,
7003 } => {
7004 info!(
7008 module_id = %spec.module_id,
7009 queued_ms = u64::try_from(queued_at.elapsed().as_millis()).unwrap_or(u64::MAX),
7010 "restart command dequeued"
7011 );
7012 let validation = match lock_snapshot(snapshot) {
7024 Ok(state) if !state.enabled => Err(SuperviseError::Disabled {
7025 module_id: spec.module_id.clone(),
7026 }),
7027 Ok(_) => Ok(()),
7028 Err(err) => Err(err),
7029 };
7030 let initiated = validation.is_ok();
7031 let _ = reply.send(validation);
7032 let satisfied_by_generation = if initiated
7046 && child
7047 .as_ref()
7048 .is_some_and(|child| child.spawned_at > queued_at)
7049 {
7050 lock_snapshot(snapshot).ok().and_then(|state| {
7051 (!state.configuration_updated_since_spawn).then_some(state.spawn_generation)
7052 })
7053 } else {
7054 None
7055 };
7056 let satisfied_by_pending = initiated
7057 && child.is_none()
7058 && lock_snapshot(snapshot).ok().is_some_and(|mut state| {
7059 let pending = state.respawn_pending && !state.configuration_updated_since_spawn;
7063 if pending {
7064 state.coalesced_restart_pending = true;
7065 }
7066 pending
7067 });
7068 if satisfied_by_pending {
7069 debug!(module_id = %spec.module_id, "restart coalesced into the pending replacement");
7070 } else if let Some(generation) = satisfied_by_generation {
7071 info!(
7072 module_id = %spec.module_id,
7073 received_at_generation,
7074 "restart already satisfied by generation {generation}; not restarting again"
7075 );
7076 } else if initiated {
7077 let drain_timeout = drain_timeout_ms
7080 .map(Duration::from_millis)
7081 .unwrap_or(runtime.drain_timeout);
7082 if let Err(err) = restart_child(
7083 spec,
7084 runtime,
7085 registry,
7086 process_liveness,
7087 snapshot,
7088 child,
7089 drain_timeout,
7090 )
7091 .await
7092 {
7093 warn!(
7094 module_id = %spec.module_id,
7095 error = %err,
7096 "operator restart failed after initiation ack; module state carries the outcome"
7097 );
7098 let _ = update_snapshot(snapshot, Some(&spec.module_id), |state| {
7099 state.state = ModuleState::Failed;
7100 clear_current_process_facts(state);
7101 });
7102 } else {
7103 runtime.start_requester = requester;
7104 runtime.pending_activity_restart = true;
7105 }
7106 }
7107 true
7108 }
7109 SupervisorCommand::Reload { requester, reply } => {
7110 let result =
7111 reload_child(spec, runtime, registry, process_liveness, snapshot, child).await;
7112 if result.is_ok() {
7113 runtime.start_requester = requester;
7114 runtime.pending_activity_restart = true;
7115 }
7116 if result.is_ok()
7117 && runtime
7118 .scheduled_respawn
7119 .lock()
7120 .unwrap_or_else(|p| p.into_inner())
7121 .as_ref()
7122 .is_some_and(|pending| matches!(pending.kind, RespawnKind::Reload))
7123 {
7124 *runtime
7125 .deferred_reload_reply
7126 .lock()
7127 .unwrap_or_else(|p| p.into_inner()) = Some(reply);
7128 } else {
7129 let _ = reply.send(result);
7130 }
7131 true
7132 }
7133 SupervisorCommand::SetEnabled {
7134 enabled,
7135 requester,
7136 reply,
7137 } => {
7138 let previous_requester = runtime.start_requester.clone();
7139 if enabled {
7140 runtime.start_requester = requester.clone();
7141 }
7142 let result = set_child_enabled(
7143 spec,
7144 runtime,
7145 registry,
7146 process_liveness,
7147 snapshot,
7148 child,
7149 enabled,
7150 )
7151 .await;
7152 if matches!(result, Ok(true)) {
7153 runtime.pending_activity_restart = false;
7154 if enabled {
7155 runtime.start_requester = requester;
7156 } else {
7157 runtime.start_requester = None;
7158 runtime.record_activity_transition(
7159 "subc.module_disabled",
7160 &spec.module_id,
7161 requester,
7162 );
7163 }
7164 } else {
7165 runtime.start_requester = previous_requester;
7166 }
7167 let _ = reply.send(result);
7168 true
7169 }
7170 SupervisorCommand::UpdateConfiguration {
7171 spec: next_spec,
7172 health,
7173 drain_timeout_ms,
7174 reply,
7175 } => {
7176 if let Some(handle) = &runtime.supervisor_handle {
7177 handle.apply_identity_configuration(&next_spec);
7178 }
7179 *spec = next_spec;
7180 let _ = update_snapshot(snapshot, Some(&spec.module_id), |state| {
7181 state.configuration_updated_since_spawn = true;
7182 });
7183 let health_changed = runtime.health != health;
7184 runtime.health = health;
7185 if health_changed {
7188 let _ = update_snapshot(snapshot, Some(&spec.module_id), |state| {
7189 state.health = ModuleHealthStatus::default();
7190 });
7191 }
7192 runtime.drain_timeout = drain_timeout_ms
7193 .map(Duration::from_millis)
7194 .unwrap_or(runtime.default_drain_timeout);
7195 *runtime
7196 .effective_drain_timeout
7197 .lock()
7198 .unwrap_or_else(|poisoned| poisoned.into_inner()) = runtime.drain_timeout;
7199 let _ = reply.send(());
7200 true
7201 }
7202 SupervisorCommand::Swap {
7203 requester,
7204 ready_timeout,
7205 reply,
7206 } => {
7207 let mut swap_runtime = runtime.clone();
7208 swap_runtime.start_requester = requester;
7209 let before = lock_snapshot(snapshot)
7210 .ok()
7211 .map(|state| state.spawn_generation);
7212 let end = swap::run_swap(
7213 spec,
7214 &swap_runtime,
7215 registry,
7216 process_liveness,
7217 snapshot,
7218 child,
7219 commands,
7220 ready_timeout.unwrap_or(DEFAULT_SWAP_READY_TIMEOUT),
7221 reply,
7222 )
7223 .await;
7224 if before
7225 != lock_snapshot(snapshot)
7226 .ok()
7227 .map(|state| state.spawn_generation)
7228 {
7229 runtime.start_requester = swap_runtime.start_requester;
7230 }
7231 requeued.extend(end.requeue);
7232 true
7233 }
7234 }
7235}
7236
7237async fn restart_child(
7238 spec: &ModuleSpec,
7239 runtime: &SupervisorRuntimeConfig,
7240 registry: &Registry,
7241 process_liveness: &SupervisorProcessLiveness,
7242 snapshot: &SharedSnapshot,
7243 child: &mut Option<SupervisedChild>,
7244 drain_timeout: Duration,
7245) -> Result<(), SuperviseError> {
7246 if !lock_snapshot(snapshot)?.enabled {
7248 return Err(SuperviseError::Disabled {
7249 module_id: spec.module_id.clone(),
7250 });
7251 }
7252 let stop_notice = begin_forwarding_drain_with_timeout(
7253 spec,
7254 runtime,
7255 registry,
7256 snapshot,
7257 None,
7258 RouteCloseReason::Restart,
7259 drain_timeout,
7260 )
7261 .await?;
7262
7263 if child.is_some() {
7264 drain_optional_child(
7265 &spec.module_id,
7266 spec.protocol,
7267 stop_notice,
7268 registry,
7269 runtime.forwarding.as_deref(),
7270 snapshot,
7271 &runtime.terminal_ring,
7272 &runtime.spawn_events,
7273 child,
7274 drain_timeout,
7275 ModuleState::Restarting,
7276 Some(true),
7277 )
7278 .await?;
7279 } else {
7280 update_snapshot(snapshot, Some(&spec.module_id), |state| {
7281 state.enabled = true;
7282 state.state = ModuleState::Restarting;
7283 clear_current_process_facts(state);
7284 })?;
7285 release_dead_registration(
7286 registry,
7287 runtime.forwarding.as_deref(),
7288 snapshot,
7289 &spec.module_id,
7290 )
7291 .await?;
7292 }
7293
7294 reset_restart_count(snapshot, &spec.module_id)?;
7295 process_liveness.track(spec.module_id.clone(), Arc::clone(snapshot));
7296 schedule_respawn(
7297 runtime,
7298 snapshot,
7299 &spec.module_id,
7300 runtime.restart_policy.backoff,
7301 RespawnKind::Spawn,
7302 )
7303}
7304
7305async fn reload_child(
7306 spec: &ModuleSpec,
7307 runtime: &SupervisorRuntimeConfig,
7308 registry: &Registry,
7309 process_liveness: &SupervisorProcessLiveness,
7310 snapshot: &SharedSnapshot,
7311 child: &mut Option<SupervisedChild>,
7312) -> Result<(), SuperviseError> {
7313 if !lock_snapshot(snapshot)?.enabled {
7315 return Err(SuperviseError::Disabled {
7316 module_id: spec.module_id.clone(),
7317 });
7318 }
7319 let stop_notice = begin_forwarding_drain(
7320 spec,
7321 runtime,
7322 registry,
7323 snapshot,
7324 Some(true),
7325 RouteCloseReason::Reload,
7326 )
7327 .await?;
7328
7329 if child.is_some() {
7330 drain_optional_child(
7331 &spec.module_id,
7332 spec.protocol,
7333 stop_notice,
7334 registry,
7335 runtime.forwarding.as_deref(),
7336 snapshot,
7337 &runtime.terminal_ring,
7338 &runtime.spawn_events,
7339 child,
7340 runtime.drain_timeout,
7341 ModuleState::Restarting,
7342 Some(true),
7343 )
7344 .await?;
7345 } else {
7346 update_snapshot(snapshot, Some(&spec.module_id), |state| {
7347 state.enabled = true;
7348 state.state = ModuleState::Restarting;
7349 clear_current_process_facts(state);
7350 })?;
7351 release_dead_registration(
7352 registry,
7353 runtime.forwarding.as_deref(),
7354 snapshot,
7355 &spec.module_id,
7356 )
7357 .await?;
7358 }
7359
7360 reset_restart_count(snapshot, &spec.module_id)?;
7361 process_liveness.track(spec.module_id.clone(), Arc::clone(snapshot));
7362 schedule_respawn(
7363 runtime,
7364 snapshot,
7365 &spec.module_id,
7366 runtime.restart_policy.backoff,
7367 RespawnKind::Reload,
7368 )
7369}
7370
7371async fn finish_reload_child(
7372 spec: &ModuleSpec,
7373 runtime: &SupervisorRuntimeConfig,
7374 registry: &Registry,
7375 process_liveness: &SupervisorProcessLiveness,
7376 snapshot: &SharedSnapshot,
7377 child: &mut Option<SupervisedChild>,
7378) -> Result<(), SuperviseError> {
7379 process_liveness.track(spec.module_id.clone(), Arc::clone(snapshot));
7380 let next_child = match spawn_and_mark_running(spec, runtime, snapshot) {
7381 Ok(next_child) => next_child,
7382 Err(err) => {
7383 return handle_reload_spawn_failure(
7384 spec,
7385 runtime,
7386 process_liveness,
7387 snapshot,
7388 child,
7389 format!("new child failed to spawn: {err}"),
7390 )
7391 .await;
7392 }
7393 };
7394 *child = Some(next_child);
7395
7396 let wait_outcome = {
7397 let active_child = child.as_mut().expect("new reload child was just stored");
7398 wait_for_registration_after_reload(
7399 registry,
7400 &spec.module_id,
7401 snapshot,
7402 active_child,
7403 REGISTRY_RELEASE_TIMEOUT,
7404 )
7405 .await?
7406 };
7407
7408 match wait_outcome {
7409 RegistrationWaitOutcome::Registered => {
7410 debug!(module_id = %spec.module_id, "supervised module reloaded and registered");
7411 Ok(())
7412 }
7413 RegistrationWaitOutcome::Exited(exit_report) => {
7414 if let Some(active_child) = child.as_mut() {
7415 active_child.drain_stderr(&spec.module_id).await;
7416 }
7417 let mut exited_child = child.take().expect("exited reload child is still stored");
7420 #[cfg(test)]
7421 if let Some(gate) = &runtime.test_reload_exit_record_gate {
7422 gate.reached.notify_one();
7423 gate.resume.notified().await;
7424 }
7425 let result = handle_reload_child_registration_failure(
7426 spec,
7427 runtime,
7428 registry,
7429 process_liveness,
7430 snapshot,
7431 child,
7432 ReloadRegistrationFailure {
7433 exit_report: registration_failure_exit_report(exit_report),
7434 reason: exited_child
7435 .spawn_failure
7436 .clone()
7437 .unwrap_or_else(|| "new child exited before registering".to_string()),
7438 },
7439 )
7440 .await;
7441 exited_child.release_roster();
7442 result
7443 }
7444 RegistrationWaitOutcome::TimedOut => {
7445 let mut timed_out_child = child
7446 .take()
7447 .expect("timed-out reload child is still running");
7448 timed_out_child
7449 .start_kill()
7450 .map_err(|source| SuperviseError::Kill {
7451 module_id: spec.module_id.clone(),
7452 source,
7453 })?;
7454 let status = timed_out_child
7455 .wait()
7456 .await
7457 .map_err(|source| SuperviseError::Wait {
7458 module_id: spec.module_id.clone(),
7459 source,
7460 })?;
7461 timed_out_child.drain_stderr(&spec.module_id).await;
7462 handle_reload_child_registration_failure(
7463 spec,
7464 runtime,
7465 registry,
7466 process_liveness,
7467 snapshot,
7468 child,
7469 ReloadRegistrationFailure {
7470 exit_report: registration_failure_exit_report(classify_reaped_child_exit(
7471 snapshot,
7472 &timed_out_child,
7473 &status,
7474 )),
7475 reason: format!(
7476 "new child did not register within {:?}",
7477 REGISTRY_RELEASE_TIMEOUT
7478 ),
7479 },
7480 )
7481 .await
7482 }
7483 }
7484}
7485
7486async fn set_child_enabled(
7487 spec: &ModuleSpec,
7488 runtime: &SupervisorRuntimeConfig,
7489 registry: &Registry,
7490 process_liveness: &SupervisorProcessLiveness,
7491 snapshot: &SharedSnapshot,
7492 child: &mut Option<SupervisedChild>,
7493 enabled: bool,
7494) -> Result<bool, SuperviseError> {
7495 let (current_enabled, current_state, respawn_pending) = {
7496 let state = lock_snapshot(snapshot)?;
7497 (state.enabled, state.state, state.respawn_pending)
7498 };
7499 let revive_terminal = enabled
7507 && current_enabled
7508 && child.is_none()
7509 && (matches!(current_state, ModuleState::Failed | ModuleState::Stopped)
7510 || (current_state == ModuleState::Restarting && !respawn_pending));
7511 if current_enabled == enabled && !revive_terminal {
7512 return Ok(false);
7513 }
7514
7515 if enabled {
7516 update_snapshot(snapshot, Some(&spec.module_id), |state| {
7517 state.enabled = true;
7518 state.state = ModuleState::Starting;
7519 clear_current_process_facts(state);
7520 })?;
7521 #[cfg(test)]
7522 if runtime.test_seed_stale_facts_before_enable_spawn {
7523 update_snapshot(snapshot, Some(&spec.module_id), |state| {
7524 state.process_alive = true;
7525 state.pid = Some(41);
7526 state.spawned_at_ms = Some(42);
7527 state.spawned_from = Some(PathBuf::from("/spawned/module"));
7528 state.spawned_file_identity = Some(SpawnedFileIdentity {
7529 device: 43,
7530 inode: 44,
7531 #[cfg(windows)]
7532 windows_image: None,
7533 });
7534 })?;
7535 }
7536 release_dead_registration(
7537 registry,
7538 runtime.forwarding.as_deref(),
7539 snapshot,
7540 &spec.module_id,
7541 )
7542 .await?;
7543 reset_restart_count(snapshot, &spec.module_id)?;
7544 process_liveness.track(spec.module_id.clone(), Arc::clone(snapshot));
7545 let next_child = match spawn_and_mark_running(spec, runtime, snapshot) {
7546 Ok(next_child) => next_child,
7547 Err(err) => {
7548 if let Err(state_err) = update_snapshot(snapshot, Some(&spec.module_id), |state| {
7549 state.state = ModuleState::Failed;
7550 clear_current_process_facts(state);
7551 }) {
7552 error!(module_id = %spec.module_id, error = %state_err, "failed to record enable spawn failure");
7553 }
7554 process_liveness.untrack_if_current(&spec.module_id, snapshot);
7555 return Err(err);
7556 }
7557 };
7558 *child = Some(next_child);
7559 debug!(module_id = %spec.module_id, "supervised module enabled");
7560 Ok(true)
7561 } else {
7562 let stop_notice = begin_forwarding_drain_if_configured(
7563 spec,
7564 runtime,
7565 registry,
7566 snapshot,
7567 Some(false),
7568 RouteCloseReason::Disable,
7569 )
7570 .await?;
7571 drain_optional_child(
7572 &spec.module_id,
7573 spec.protocol,
7574 stop_notice,
7575 registry,
7576 runtime.forwarding.as_deref(),
7577 snapshot,
7578 &runtime.terminal_ring,
7579 &runtime.spawn_events,
7580 child,
7581 runtime.drain_timeout,
7582 ModuleState::Disabled,
7583 Some(false),
7584 )
7585 .await?;
7586 debug!(module_id = %spec.module_id, "supervised module disabled");
7587 Ok(true)
7588 }
7589}
7590
7591#[allow(clippy::too_many_arguments)]
7592async fn on_child_exit(
7593 spec: &ModuleSpec,
7594 policy: RestartPolicy,
7595 registry: &Registry,
7596 snapshot: &SharedSnapshot,
7597 terminal_ring: &Arc<Mutex<TerminalRing>>,
7598 spawn_events: &SpawnEventFeed,
7599 roster: &ChildRoster,
7600 exit_report: ExitReport,
7601) -> NextAction {
7602 if roster.is_closed() {
7608 return on_child_exit_during_daemon_shutdown(
7609 spec,
7610 registry,
7611 snapshot,
7612 terminal_ring,
7613 spawn_events,
7614 exit_report,
7615 )
7616 .await;
7617 }
7618 let unrequested_clean_exit_of_protocol_none = exit_report.kind == ExitKind::Clean
7633 && running_protocol(spec, snapshot) == ModuleProtocol::None;
7634 match exit_report.kind {
7635 ExitKind::Clean if !unrequested_clean_exit_of_protocol_none => {
7636 info!(
7637 module_id = %spec.module_id,
7638 exit_code = ?exit_report.code,
7639 exit_signal = ?exit_report.signal,
7640 "supervised module exited cleanly"
7641 );
7642 if let Err(err) = publish_after_terminal_record(
7643 &spec.module_id,
7644 || {
7645 record_terminal(
7646 &spec.module_id,
7647 terminal_ring,
7648 spawn_events,
7649 &exit_report,
7650 TerminalDisposition::Stopped,
7651 )
7652 },
7653 || {
7654 update_snapshot(snapshot, Some(&spec.module_id), |state| {
7655 state.state = ModuleState::Stopped;
7656 clear_current_process_facts(state);
7657 state.last_exit = Some(exit_report.clone());
7658 })
7659 },
7660 ) {
7661 error!(module_id = %spec.module_id, error = %err, "failed to publish clean module exit");
7662 }
7663 let registration_released = match wait_for_registration_release(
7664 registry,
7665 &spec.module_id,
7666 REGISTRY_RELEASE_TIMEOUT,
7667 )
7668 .await
7669 {
7670 Ok(()) => true,
7671 Err(err) => {
7672 warn!(module_id = %spec.module_id, error = %err, "registration still active after clean exit");
7673 false
7674 }
7675 };
7676 NextAction::Stop {
7677 registration_released,
7678 }
7679 }
7680 ExitKind::Clean | ExitKind::Crash => {
7681 if unrequested_clean_exit_of_protocol_none {
7682 warn!(
7683 module_id = %spec.module_id,
7684 exit_code = ?exit_report.code,
7685 exit_signal = ?exit_report.signal,
7686 "protocol-none module exited cleanly without a stop request; handling it as a crash"
7687 );
7688 } else {
7689 warn!(
7690 module_id = %spec.module_id,
7691 exit_code = ?exit_report.code,
7692 exit_signal = ?exit_report.signal,
7693 "supervised module exited abnormally (crash)"
7694 );
7695 }
7696 let mut restart_schedule = None;
7697 let mut disposition = TerminalDisposition::Disabled;
7698 let mut final_state = ModuleState::Disabled;
7699 let mut disposition_detail = lock_snapshot(snapshot)
7703 .ok()
7704 .and_then(|mut state| state.spawn_failure.take());
7705 let now = Instant::now();
7706 if let Err(err) = update_snapshot(snapshot, Some(&spec.module_id), |state| {
7707 if state.enabled {
7708 if let Some(schedule) = state.next_crash_restart(&policy, now) {
7709 state.respawn_pending = true;
7715 final_state = ModuleState::Restarting;
7716 restart_schedule = Some(schedule);
7717 disposition = TerminalDisposition::Restarting;
7718 } else {
7719 final_state = ModuleState::Failed;
7720 disposition = TerminalDisposition::Failed;
7721 let budget = policy.budget_exhausted_detail();
7722 disposition_detail =
7723 Some(disposition_detail.take().map_or_else(
7724 || budget.clone(),
7725 |cause| format!("{cause}; {budget}"),
7726 ));
7727 }
7728 } else {
7729 disposition = TerminalDisposition::Disabled;
7730 }
7731 }) {
7732 error!(module_id = %spec.module_id, error = %err, "failed to record crashed module exit");
7733 return NextAction::Stop {
7734 registration_released: false,
7735 };
7736 }
7737 if disposition == TerminalDisposition::Failed {
7738 error!(
7743 module_id = %spec.module_id,
7744 max_restarts = policy.max_restarts,
7745 window_secs = policy.window.as_secs(),
7746 "module stopped: {}",
7747 policy.budget_exhausted_detail()
7748 );
7749 }
7750 if let Err(err) = publish_after_terminal_record(
7751 &spec.module_id,
7752 || {
7753 record_terminal_with_detail(
7754 &spec.module_id,
7755 terminal_ring,
7756 spawn_events,
7757 &exit_report,
7758 disposition,
7759 disposition_detail,
7760 )
7761 },
7762 || {
7763 update_snapshot(snapshot, Some(&spec.module_id), |state| {
7764 state.state = final_state;
7765 clear_current_process_facts(state);
7766 state.last_exit = Some(exit_report.clone());
7767 })
7768 },
7769 ) {
7770 error!(module_id = %spec.module_id, error = %err, "failed to publish crashed module exit");
7771 }
7772
7773 if let Some(schedule) = restart_schedule {
7774 NextAction::Restart {
7775 schedule: Some(schedule),
7776 }
7777 } else {
7778 let registration_released = match wait_for_registration_release(
7779 registry,
7780 &spec.module_id,
7781 REGISTRY_RELEASE_TIMEOUT,
7782 )
7783 .await
7784 {
7785 Ok(()) => true,
7786 Err(err) => {
7787 warn!(module_id = %spec.module_id, error = %err, "registration still active after failed module");
7788 false
7789 }
7790 };
7791 NextAction::Stop {
7792 registration_released,
7793 }
7794 }
7795 }
7796 ExitKind::DeliberateSeverance => {
7797 warn!(
7798 module_id = %spec.module_id,
7799 exit_code = ?exit_report.code,
7800 exit_signal = ?exit_report.signal,
7801 "supervised module exited after deliberate connection severance"
7802 );
7803 let should_restart = match lock_snapshot(snapshot) {
7804 Ok(state) => state.enabled,
7805 Err(err) => {
7806 error!(module_id = %spec.module_id, error = %err, "failed to read deliberately severed module state");
7807 return NextAction::Stop {
7808 registration_released: false,
7809 };
7810 }
7811 };
7812 let final_state = if should_restart {
7813 ModuleState::Restarting
7814 } else {
7815 ModuleState::Disabled
7816 };
7817 if let Err(err) = publish_after_terminal_record(
7818 &spec.module_id,
7819 || {
7820 record_terminal(
7821 &spec.module_id,
7822 terminal_ring,
7823 spawn_events,
7824 &exit_report,
7825 terminal_disposition(final_state),
7826 )
7827 },
7828 || {
7829 update_snapshot(snapshot, Some(&spec.module_id), |state| {
7830 state.state = final_state;
7831 clear_current_process_facts(state);
7832 state.last_exit = Some(exit_report.clone());
7833 state.lifetime_restarts += 1;
7834 })
7835 },
7836 ) {
7837 error!(module_id = %spec.module_id, error = %err, "failed to publish deliberately severed module exit");
7838 return NextAction::Stop {
7839 registration_released: false,
7840 };
7841 }
7842
7843 if should_restart {
7844 NextAction::Restart { schedule: None }
7845 } else {
7846 let registration_released = match wait_for_registration_release(
7847 registry,
7848 &spec.module_id,
7849 REGISTRY_RELEASE_TIMEOUT,
7850 )
7851 .await
7852 {
7853 Ok(()) => true,
7854 Err(err) => {
7855 warn!(module_id = %spec.module_id, error = %err, "registration still active after deliberately severed module exit");
7856 false
7857 }
7858 };
7859 NextAction::Stop {
7860 registration_released,
7861 }
7862 }
7863 }
7864 }
7865}
7866
7867async fn on_child_exit_during_daemon_shutdown(
7868 spec: &ModuleSpec,
7869 registry: &Registry,
7870 snapshot: &SharedSnapshot,
7871 terminal_ring: &Arc<Mutex<TerminalRing>>,
7872 spawn_events: &SpawnEventFeed,
7873 exit_report: ExitReport,
7874) -> NextAction {
7875 info!(
7876 module_id = %spec.module_id,
7877 exit_code = ?exit_report.code,
7878 exit_signal = ?exit_report.signal,
7879 exit_kind = ?exit_report.kind,
7880 "supervised module exited during daemon shutdown; not restarting it"
7881 );
7882 if let Err(err) = publish_after_terminal_record(
7883 &spec.module_id,
7884 || {
7885 record_terminal(
7886 &spec.module_id,
7887 terminal_ring,
7888 spawn_events,
7889 &exit_report,
7890 TerminalDisposition::DaemonShutdown,
7891 )
7892 },
7893 || {
7894 update_snapshot(snapshot, Some(&spec.module_id), |state| {
7895 state.state = ModuleState::Stopped;
7896 clear_current_process_facts(state);
7897 state.last_exit = Some(exit_report.clone());
7898 })
7899 },
7900 ) {
7901 error!(module_id = %spec.module_id, error = %err, "failed to publish module exit during daemon shutdown");
7902 }
7903 let registration_released =
7904 wait_for_registration_release(registry, &spec.module_id, REGISTRY_RELEASE_TIMEOUT)
7905 .await
7906 .is_ok();
7907 NextAction::Stop {
7908 registration_released,
7909 }
7910}
7911
7912fn publish_after_terminal_record<T>(
7917 module_id: &str,
7918 record: impl FnOnce(),
7919 publish: impl FnOnce() -> T,
7920) -> T {
7921 let recorded = std::panic::catch_unwind(std::panic::AssertUnwindSafe(record));
7922 let published = publish();
7923 if let Err(panic) = recorded {
7924 error!(
7925 module_id,
7926 "terminal recording panicked; published final state before resuming unwind"
7927 );
7928 std::panic::resume_unwind(panic);
7929 }
7930 published
7931}
7932
7933fn fail_after_wait_error(
7934 module_id: &str,
7935 snapshot: &SharedSnapshot,
7936 terminal_ring: &Arc<Mutex<TerminalRing>>,
7937 spawn_events: &SpawnEventFeed,
7938 detail: String,
7939) {
7940 let report = wait_error_exit_report();
7941 publish_after_terminal_record(
7942 module_id,
7943 || {
7944 record_terminal_with_detail(
7945 module_id,
7946 terminal_ring,
7947 spawn_events,
7948 &report,
7949 TerminalDisposition::Failed,
7950 Some(detail),
7951 )
7952 },
7953 || fail_snapshot(snapshot, Some(module_id), Some(report.clone())),
7954 );
7955}
7956
7957#[cfg(test)]
7958fn record_wait_error_terminal(
7959 module_id: &str,
7960 terminal_ring: &Arc<Mutex<TerminalRing>>,
7961 spawn_events: &SpawnEventFeed,
7962) {
7963 record_terminal(
7964 module_id,
7965 terminal_ring,
7966 spawn_events,
7967 &wait_error_exit_report(),
7968 TerminalDisposition::Failed,
7969 );
7970}
7971
7972fn record_terminal(
7973 module_id: &str,
7974 terminal_ring: &Arc<Mutex<TerminalRing>>,
7975 spawn_events: &SpawnEventFeed,
7976 exit_report: &ExitReport,
7977 disposition: TerminalDisposition,
7978) {
7979 record_terminal_with_detail(
7980 module_id,
7981 terminal_ring,
7982 spawn_events,
7983 exit_report,
7984 disposition,
7985 None,
7986 );
7987}
7988
7989fn durable_terminal_history_of(
7993 terminal_ring: &Mutex<TerminalRing>,
7994 module_id: &str,
7995) -> subc_control::TerminalHistory {
7996 let read = terminal_ring
7997 .lock()
7998 .unwrap_or_else(|p| p.into_inner())
7999 .capture_durable_history();
8000 read.read(module_id)
8001}
8002
8003fn record_terminal_with_detail(
8004 module_id: &str,
8005 terminal_ring: &Arc<Mutex<TerminalRing>>,
8006 spawn_events: &SpawnEventFeed,
8007 exit_report: &ExitReport,
8008 disposition: TerminalDisposition,
8009 disposition_detail: Option<String>,
8010) {
8011 #[cfg(test)]
8012 terminal_history_tests::before_terminal_record(spawn_events);
8013 spawn_events.emit_exited(module_id, exit_report.code, exit_report.signal);
8014 let record = TerminalRecord {
8015 exit_code: exit_report.code,
8016 exit_signal: exit_report.signal,
8017 at_ms: exit_report.at_ms,
8018 disposition,
8019 exit_kind: exit_report.kind.into(),
8020 disposition_detail,
8021 };
8022 terminal_ring
8023 .lock()
8024 .unwrap_or_else(|poisoned| poisoned.into_inner())
8025 .record_exit(module_id, record);
8026}
8027
8028fn untrack_if_registration_released(
8029 process_liveness: &SupervisorProcessLiveness,
8030 registry: &Registry,
8031 module_id: &str,
8032 snapshot: &SharedSnapshot,
8033) {
8034 match registry.get_module(module_id) {
8035 Ok(None) => process_liveness.untrack_if_current(module_id, snapshot),
8036 Ok(Some(_)) => {}
8037 Err(err) => {
8038 warn!(module_id, error = %err, "could not determine whether supervisor liveness can be untracked");
8039 }
8040 }
8041}
8042
8043#[cfg(test)]
8057fn apply_wire_spawn_args(
8058 command: &mut Command,
8059 spec: &ModuleSpec,
8060 connection_file_path: Option<&std::path::Path>,
8061 handle: Option<&SupervisorHandle>,
8062) -> Result<Option<NonceHandoff>, SuperviseError> {
8063 apply_wire_spawn_args_for_role(
8064 command,
8065 spec,
8066 connection_file_path,
8067 handle,
8068 SpawnRole::Plain,
8069 )
8070}
8071
8072#[cfg(unix)]
8077type NonceHandoff = subc_os::LaunchNonceHandoff;
8078#[cfg(windows)]
8079type NonceHandoff = subc_os::LaunchNoncePipeHandoff;
8080#[cfg(not(any(unix, windows)))]
8081type NonceHandoff = std::convert::Infallible;
8082
8083fn apply_wire_spawn_args_for_role(
8104 command: &mut Command,
8105 spec: &ModuleSpec,
8106 connection_file_path: Option<&std::path::Path>,
8107 handle: Option<&SupervisorHandle>,
8108 role: SpawnRole,
8109) -> Result<Option<NonceHandoff>, SuperviseError> {
8110 command.env(SUBC_MODULE_ID_ENV, &spec.module_id);
8111 command.env_remove(subc_os::LAUNCH_NONCE_FD_ENV);
8116 #[cfg(windows)]
8117 {
8118 command.env_remove(subc_os::LAUNCH_NONCE_PIPE_ENV);
8119 command.env_remove(subc_os::LAUNCH_NONCE_PIPE_FALLBACK_ENV);
8120 }
8121 command.env_remove(SUBC_LAUNCH_NONCE_ENV);
8123 if spec.protocol == ModuleProtocol::None {
8124 return Ok(None);
8125 }
8126 if let Some(connection_file_path) = connection_file_path {
8127 command.arg(SUBC_ARG).arg(connection_file_path);
8128 }
8129
8130 let nonce = generate_launch_nonce()?;
8134 if let Some(handle) = handle {
8135 match role {
8136 SpawnRole::Plain => {
8137 handle.set_spawn_nonce(&spec.module_id, nonce.clone());
8138 if spec.reserved {
8139 handle.set_reserved_nonce(&spec.module_id, nonce.clone());
8140 }
8141 }
8142 SpawnRole::SwapCandidate => handle.open_swap(&spec.module_id, nonce.clone()),
8143 }
8144 }
8145 #[cfg(unix)]
8146 let handoff = {
8147 let handoff =
8148 subc_os::LaunchNonceHandoff::new(&nonce).map_err(|source| SuperviseError::Spawn {
8149 program: spec.program.clone(),
8150 source,
8151 cgroup_path: None,
8152 })?;
8153 command.env(subc_os::LAUNCH_NONCE_FD_ENV, handoff.fd_env_value());
8154 Some(handoff)
8155 };
8156 #[cfg(windows)]
8157 let handoff = prepare_windows_nonce_pipe(command, spec, &nonce);
8158 #[cfg(not(any(unix, windows)))]
8159 let handoff = None;
8160 #[cfg(not(unix))]
8164 command.env(SUBC_LAUNCH_NONCE_ENV, nonce);
8165 Ok(handoff)
8166}
8167
8168fn apply_child_env(command: &mut Command, spec: &ModuleSpec) {
8169 command.env_remove(CK_LOG_ENV);
8170 command.env_remove(SUBC_SPAWN_ROLE_ENV);
8177 for (key, value) in &spec.env {
8178 if matches!(
8182 key.as_str(),
8183 CAPTURE_MAX_FILE_MB_ENV | CAPTURE_KEEP_ENV | CAPTURE_MAX_AGE_DAYS_ENV
8184 ) || key == SUBC_SPAWN_ROLE_ENV
8185 {
8186 continue;
8187 }
8188 command.env(key, value);
8189 }
8190}
8191
8192#[cfg(windows)]
8193fn prepare_windows_nonce_pipe(
8194 command: &mut Command,
8195 spec: &ModuleSpec,
8196 nonce: &str,
8197) -> Option<NonceHandoff> {
8198 match subc_os::LaunchNoncePipeHandoff::new(nonce) {
8199 Ok(handoff) => {
8200 command.env(subc_os::LAUNCH_NONCE_PIPE_ENV, handoff.name());
8201 command.env(subc_os::LAUNCH_NONCE_PIPE_FALLBACK_ENV, "env");
8206 Some(handoff)
8207 }
8208 Err(error) => {
8209 warn!(module_id = %spec.module_id, %error, "could not prepare launch nonce pipe; retaining the environment handoff");
8210 None
8211 }
8212 }
8213}
8214
8215#[cfg(windows)]
8216fn start_windows_nonce_pipe(
8217 handoff: Option<NonceHandoff>,
8218 pid: u32,
8219 spec: &ModuleSpec,
8220) -> Option<subc_os::LaunchNoncePipeDelivery> {
8221 handoff.and_then(|handoff| match handoff.serve(pid, REGISTRY_RELEASE_TIMEOUT) {
8225 Ok(delivery) => Some(delivery),
8226 Err(error) => {
8227 warn!(module_id = %spec.module_id, %error, "could not serve launch nonce pipe; retaining the environment handoff");
8228 None
8229 }
8230 })
8231}
8232
8233fn observed_launch_nonce_source(snapshot: &SupervisorSnapshot) -> Option<String> {
8234 #[cfg(windows)]
8235 if snapshot.process_alive && snapshot.spawned_protocol == Some(ModuleProtocol::Subc) {
8236 return Some(
8237 if snapshot
8238 .pipe_consumed
8239 .as_ref()
8240 .is_some_and(|consumed| consumed.get().is_some())
8241 {
8242 "pipe"
8243 } else {
8244 "env"
8245 }
8246 .to_owned(),
8247 );
8248 }
8249 let _ = snapshot;
8250 None
8251}
8252
8253#[derive(Debug, Clone, Copy, PartialEq, Eq)]
8256enum SpawnRole {
8257 Plain,
8258 SwapCandidate,
8259}
8260
8261fn apply_spawn_role(command: &mut Command, role: SpawnRole) {
8264 if role == SpawnRole::SwapCandidate {
8265 command.env(SUBC_SPAWN_ROLE_ENV, SPAWN_ROLE_SWAP_CANDIDATE);
8266 }
8267}
8268
8269fn previous_instance_error(
8270 spec: &ModuleSpec,
8271 pid: u32,
8272 reason: crate::live_children::RetainedReason,
8273) -> SuperviseError {
8274 SuperviseError::Spawn {
8275 program: spec.program.clone(),
8276 source: io::Error::new(io::ErrorKind::WouldBlock, reason.waiting_message(pid)),
8277 cgroup_path: None,
8278 }
8279}
8280
8281fn spawn_child(
8282 spec: &ModuleSpec,
8283 connection_file_path: Option<&std::path::Path>,
8284 handle: Option<&SupervisorHandle>,
8285 ring: &Arc<Mutex<StderrRing>>,
8286 capture_logs_dir: Option<&std::path::Path>,
8287 roster: &ChildRoster,
8288 #[cfg(target_os = "linux")] cgroup_placement: Option<&subc_cgroup::Placement>,
8289) -> Result<SupervisedChild, SuperviseError> {
8290 spawn_child_in_slot(
8291 spec,
8292 connection_file_path,
8293 handle,
8294 ring,
8295 capture_logs_dir,
8296 roster,
8297 #[cfg(target_os = "linux")]
8298 cgroup_placement,
8299 SpawnRole::Plain,
8300 false,
8301 )
8302}
8303
8304#[allow(clippy::too_many_arguments)]
8317fn spawn_child_in_slot(
8318 spec: &ModuleSpec,
8319 connection_file_path: Option<&std::path::Path>,
8320 handle: Option<&SupervisorHandle>,
8321 ring: &Arc<Mutex<StderrRing>>,
8322 capture_logs_dir: Option<&std::path::Path>,
8323 roster: &ChildRoster,
8324 #[cfg(target_os = "linux")] cgroup_placement: Option<&subc_cgroup::Placement>,
8325 role: SpawnRole,
8326 alternate_slot: bool,
8327) -> Result<SupervisedChild, SuperviseError> {
8328 if let Some((pid, reason)) = roster.previous_instance(&spec.module_id) {
8329 return Err(previous_instance_error(spec, pid, reason));
8330 }
8331 if roster.is_closed() {
8332 return Err(SuperviseError::Spawn {
8333 program: spec.program.clone(),
8334 source: io::Error::other("the daemon is shutting down; not starting a new process"),
8335 cgroup_path: None,
8336 });
8337 }
8338 #[cfg(target_os = "linux")]
8339 let cgroup_name = {
8340 if cgroup_placement.is_none() {
8345 swap::cgroup_name(&spec.module_id, alternate_slot)
8346 } else {
8347 let nonce = generate_launch_nonce()?;
8348 let suffix = u128::from_str_radix(&nonce[..32], 16).expect("hex launch nonce");
8349 let mut end = spec.module_id.len().min(64);
8352 while !spec.module_id.is_char_boundary(end) {
8353 end -= 1;
8354 }
8355 format!(
8356 "{}_{suffix}",
8357 swap::cgroup_name(&spec.module_id[..end], alternate_slot)
8358 )
8359 }
8360 };
8361 #[cfg(not(target_os = "linux"))]
8362 let _ = alternate_slot;
8363 #[cfg(target_os = "macos")]
8364 let (mut command, privacy_exec, exec_ack) = privacy_command(spec, roster)?;
8365 #[cfg(windows)]
8366 let image_capture = windows_spawn_capture(spec).map_err(|source| SuperviseError::Spawn {
8367 program: spec.program.clone(),
8368 source,
8369 cgroup_path: None,
8370 })?;
8371 #[cfg(windows)]
8372 let mut command = Command::new(image_capture.path());
8373 #[cfg(not(any(target_os = "macos", windows)))]
8374 let mut command = Command::new(&spec.program);
8375 command.args(&spec.args);
8376 apply_child_env(&mut command, spec);
8406 apply_spawn_role(&mut command, role);
8407 let nonce_handoff =
8408 apply_wire_spawn_args_for_role(&mut command, spec, connection_file_path, handle, role)?;
8409
8410 #[cfg(target_os = "linux")]
8411 let cgroup_path = cgroup_placement
8412 .map(|placement| placement.module_path(&cgroup_name))
8413 .transpose()
8414 .map_err(|source| SuperviseError::Cgroup {
8415 module_id: spec.module_id.clone(),
8416 source,
8417 })?;
8418 #[cfg(not(target_os = "linux"))]
8419 let cgroup_path: Option<PathBuf> = None;
8420 #[cfg(target_os = "linux")]
8421 if let Some(path) = &cgroup_path {
8422 if let Err(error) = apply_cgroup_placement(&mut command, spec, path) {
8423 if let Some(placement) = cgroup_placement {
8424 remove_module_cgroup(placement, &cgroup_name);
8425 }
8426 return Err(error);
8427 }
8428 }
8429
8430 let output_sink = if let Some(logs_dir) = capture_logs_dir {
8431 let path = logs_dir.join(format!("{}.stderr.log", spec.module_id));
8432 match ChildOutputSink::open(&path, capture_retention(spec)) {
8433 Ok(sink) => sink,
8434 Err(error) => {
8435 warn!(
8436 module_id = %spec.module_id,
8437 path = %path.display(),
8438 error = %error,
8439 "could not open child output capture file; forwarding to stderr"
8440 );
8441 ChildOutputSink::Stderr
8442 }
8443 }
8444 } else {
8445 ChildOutputSink::Stderr
8446 };
8447
8448 command.stdout(Stdio::piped());
8449 command.stderr(Stdio::piped());
8450 command.kill_on_drop(true);
8451 #[cfg(unix)]
8468 command.process_group(0);
8469 command.stdin(Stdio::null());
8470 #[cfg(unix)]
8474 if let Some(handoff) = nonce_handoff {
8475 handoff.install_last(command.as_std_mut());
8476 }
8477 #[cfg(not(any(unix, windows)))]
8478 let _ = nonce_handoff;
8479
8480 #[cfg(windows)]
8485 {
8486 subc_jobobject::ensure_root_job().map_err(|source| SuperviseError::Spawn {
8487 program: spec.program.clone(),
8488 source,
8489 cgroup_path: None,
8490 })?;
8491 subc_jobobject::suspend_on_create_async(&mut command);
8492 }
8493 let spawned_at = Instant::now();
8497 let mut child = match command.spawn() {
8498 Ok(child) => child,
8499 Err(source) => {
8500 #[cfg(target_os = "linux")]
8501 if let Some(placement) = cgroup_placement {
8502 remove_module_cgroup(placement, &cgroup_name);
8503 }
8504 return Err(SuperviseError::Spawn {
8505 program: spec.program.clone(),
8506 source,
8507 cgroup_path,
8508 });
8509 }
8510 };
8511 #[cfg(test)]
8512 spawn_record_tests::after_create(&child, spec);
8513 #[cfg(target_os = "macos")]
8518 drop(exec_ack);
8519
8520 #[cfg(windows)]
8525 let windows_image = match image_capture.bind_tokio(&child) {
8526 Ok(image) => Arc::new(image),
8527 Err(source) => {
8528 kill_and_reap_rejected_child(&mut child);
8529 return Err(SuperviseError::Spawn {
8530 program: spec.program.clone(),
8531 source,
8532 cgroup_path: None,
8533 });
8534 }
8535 };
8536 #[cfg(windows)]
8539 let job = contain_spawned_child(&mut child, spec)?;
8540 let spawned_at_ms = unix_ms_now();
8541 #[cfg(not(windows))]
8542 let spawned_from = spec.program.clone();
8543 #[cfg(windows)]
8544 let spawned_from = windows_image.path().to_path_buf();
8545 #[cfg(not(windows))]
8546 let spawned_file_identity = spawned_file_identity(&spawned_from);
8547 #[cfg(windows)]
8548 let spawned_file_identity = Some(SpawnedFileIdentity {
8549 device: 0,
8550 inode: 0,
8551 windows_image: Some(Arc::clone(&windows_image)),
8552 });
8553 let pid = child.id().ok_or_else(|| SuperviseError::Spawn {
8554 program: spec.program.clone(),
8555 source: io::Error::other("spawned child exposed no live pid"),
8556 cgroup_path: cgroup_path.clone(),
8557 })?;
8558 #[cfg(windows)]
8559 let nonce_delivery = start_windows_nonce_pipe(nonce_handoff, pid, spec);
8560 #[cfg(not(windows))]
8561 let process_start_time = crate::provenance::process_start_time(pid);
8562 #[cfg(windows)]
8563 let process_start_time = Some(windows_image.start_time());
8564 let process_identity = process_start_time.map(|start_time| ProcessIdentity { pid, start_time });
8565 #[cfg(all(test, target_os = "macos"))]
8566 privacy_exec_boundary_tests::before_image_sample(spec, pid);
8567 #[cfg(not(windows))]
8572 let recorded_image = observe_spawned_image(pid);
8573 #[cfg(target_os = "macos")]
8577 let recorded_image = if privacy_exec.is_some() {
8578 None
8579 } else {
8580 recorded_image
8581 };
8582 #[cfg(target_os = "linux")]
8583 let recorded_cgroup_name = cgroup_path.as_ref().map(|_| cgroup_name.clone());
8584 #[cfg(not(target_os = "linux"))]
8585 let recorded_cgroup_name = None;
8586 let roster_guard = roster.admit(
8587 spec.module_id.clone(),
8588 pid,
8589 spec.protocol,
8590 process_start_time,
8591 crate::child_roster::RecordedIdentity {
8592 #[cfg(not(windows))]
8593 start_time: recorded_image.map(|image| image.start_time),
8594 #[cfg(windows)]
8595 start_time: process_start_time,
8596 #[cfg(not(windows))]
8597 executable: recorded_image
8598 .and_then(|image| image.executable)
8599 .map(crate::live_children::ExecutableIdentity::from),
8600 #[cfg(windows)]
8601 executable: Some(crate::live_children::ExecutableIdentity::from(
8602 windows_image.identity(),
8603 )),
8604 cgroup_name: recorded_cgroup_name,
8605 #[cfg(target_os = "linux")]
8606 cgroup_placement: cgroup_placement.cloned(),
8607 },
8608 );
8609 let roster_guard = match roster_guard {
8610 Ok(guard) => guard,
8611 Err(source) => {
8612 #[cfg(windows)]
8613 if let Some(job) = &job {
8614 let _ = job.terminate();
8615 }
8616 #[cfg(target_os = "linux")]
8617 kill_module_cgroup(cgroup_placement, &cgroup_name);
8618 kill_and_reap_rejected_child(&mut child);
8619 #[cfg(target_os = "linux")]
8620 if let Some(placement) = cgroup_placement {
8621 remove_module_cgroup(placement, &cgroup_name);
8622 }
8623 return Err(SuperviseError::Spawn {
8624 program: spec.program.clone(),
8625 source: io::Error::other(format!(
8626 "could not durably record module child: {source}"
8627 )),
8628 cgroup_path,
8629 });
8630 }
8631 };
8632 if roster.is_closed() {
8641 #[cfg(target_os = "linux")]
8643 kill_module_cgroup(cgroup_placement, &cgroup_name);
8644 if let Err(error) = child.start_kill() {
8645 debug!(module_id = %spec.module_id, pid, %error, "kill of a process spawned during daemon shutdown failed; it may already have exited");
8646 }
8647 #[cfg(target_os = "linux")]
8648 if let Some(placement) = cgroup_placement {
8649 while matches!(child.try_wait(), Ok(None)) {
8653 std::thread::yield_now();
8654 }
8655 if matches!(
8656 subc_cgroup::kill_module(Some(placement), &cgroup_name),
8657 subc_cgroup::KillOutcome::Killed
8658 ) {
8659 if let Ok(path) = placement.module_path(&cgroup_name) {
8660 while std::fs::read_to_string(path.join("cgroup.events"))
8661 .ok()
8662 .is_some_and(|events| events.lines().any(|line| line == "populated 1"))
8663 {
8664 std::thread::yield_now();
8665 }
8666 }
8667 }
8668 remove_module_cgroup(placement, &cgroup_name);
8669 }
8670 drop(roster_guard);
8671 return Err(SuperviseError::Spawn {
8672 program: spec.program.clone(),
8673 source: io::Error::other(
8674 "the daemon began shutting down while this process was starting; ended it",
8675 ),
8676 cgroup_path,
8677 });
8678 }
8679
8680 #[cfg(windows)]
8681 if let Err(error) = resume_suspended_child(pid, spec) {
8682 kill_and_reap_rejected_child(&mut child);
8683 return Err(error);
8684 }
8685
8686 let stdout_pump = match child.stdout.take() {
8687 Some(stdout) => Some(tokio::spawn(pump_stdout_to(stdout, output_sink.clone()))),
8688 None => {
8689 warn!(
8690 module_id = %spec.module_id,
8691 "spawned child exposed no stdout pipe; file capture will be incomplete"
8692 );
8693 None
8694 }
8695 };
8696 let stderr_pump = match child.stderr.take() {
8697 Some(stderr) => {
8698 let generation = ring
8699 .lock()
8700 .unwrap_or_else(|poisoned| poisoned.into_inner())
8701 .begin_process();
8702 Some(StderrPump {
8703 task: tokio::spawn(pump_stderr_to(
8704 stderr,
8705 Arc::clone(ring),
8706 generation,
8707 output_sink,
8708 )),
8709 generation,
8710 })
8711 }
8712 None => {
8713 ring.lock()
8717 .unwrap_or_else(|poisoned| poisoned.into_inner())
8718 .mark_not_captured("stderr pipe was not available on spawn");
8719 warn!(
8720 module_id = %spec.module_id,
8721 "spawned child exposed no stderr pipe; tail will be unavailable"
8722 );
8723 None
8724 }
8725 };
8726
8727 let exec = Arc::new(crate::activity::ExecOutcome::new());
8728 #[cfg(target_os = "macos")]
8729 let pending_exec = privacy_exec.is_some();
8730 #[cfg(not(target_os = "macos"))]
8731 let pending_exec = false;
8732 if !pending_exec {
8733 exec.finish(crate::activity::ExecState::CommittedExec);
8734 }
8735 Ok(SupervisedChild {
8736 child,
8737 exec: crate::activity::ExecOwner(exec),
8738 #[cfg(windows)]
8739 nonce_delivery,
8740 protocol: spec.protocol,
8741 #[cfg(target_os = "linux")]
8742 module_id: cgroup_name,
8743 #[cfg(target_os = "linux")]
8744 cgroup_placement: cgroup_placement.cloned(),
8745 #[cfg(windows)]
8746 job,
8747 stdout_pump,
8748 stderr_pump,
8749 stderr_ring: Arc::clone(ring),
8750 spawned_at,
8751 spawned_at_ms,
8752 spawned_from,
8753 launch_spec_digest: LaunchSpecDigest::from_spec(spec),
8754 spawned_file_identity,
8755 process_start_time,
8756 process_identity,
8757 pid,
8758 roster_guard: Some(roster_guard),
8759 #[cfg(target_os = "macos")]
8760 privacy_exec,
8761 #[cfg(target_os = "macos")]
8762 report_ready: Arc::new(OnceLock::new()),
8763 spawn_failure: None,
8764 })
8765}
8766
8767#[cfg(test)]
8768mod spawn_record_tests {
8769 use super::*;
8770
8771 thread_local! {
8772 static LAST_PID: std::cell::Cell<Option<u32>> = const { std::cell::Cell::new(None) };
8773 }
8774
8775 pub(super) fn after_create(child: &Child, _spec: &ModuleSpec) {
8776 LAST_PID.set(child.id());
8777 #[cfg(windows)]
8778 if let Some((_, address)) = _spec
8779 .env
8780 .iter()
8781 .find(|(key, _)| key == "SUBC_TEST_AFTER_CREATE")
8782 {
8783 use std::io::{Read, Write};
8784 let mut stream = std::net::TcpStream::connect(address).unwrap();
8785 stream
8786 .write_all(&child.id().unwrap().to_ne_bytes())
8787 .unwrap();
8788 let mut release = [0];
8789 stream.read_exact(&mut release).unwrap();
8790 }
8791 }
8792
8793 #[cfg(windows)]
8794 pub(super) fn assignment_failpoint(spec: &ModuleSpec) -> io::Result<()> {
8795 if spec
8796 .env
8797 .iter()
8798 .any(|(key, _)| key == "SUBC_TEST_ASSIGNMENT_FAILURE")
8799 {
8800 Err(io::Error::other("injected module job assignment failure"))
8801 } else {
8802 Ok(())
8803 }
8804 }
8805
8806 fn spec() -> ModuleSpec {
8807 ModuleSpec {
8808 module_id: "spawn-record-worker".into(),
8809 #[cfg(unix)]
8810 program: "/bin/sleep".into(),
8811 #[cfg(windows)]
8812 program: std::env::current_exe().unwrap(),
8813 #[cfg(unix)]
8814 args: vec!["60".into()],
8815 #[cfg(windows)]
8816 args: vec![
8817 "--exact".into(),
8818 "supervise::spawn_record_tests::windows_store_touch_fixture".into(),
8819 "--nocapture".into(),
8820 ],
8821 env: vec![],
8822 reserved: false,
8823 reserved_prefixes: vec![],
8824 protocol: ModuleProtocol::None,
8825 overlap: Default::default(),
8826 }
8827 }
8828
8829 #[tokio::test]
8830 async fn record_sync_failure_fails_spawn_and_reaps_child() {
8831 let home = cortexkit_test_support::ScratchDir::new("record-sync-failure");
8832 let supervisor =
8833 Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default())
8834 .with_live_children_record(home.join("live-children.json"));
8835 crate::live_children::fail_next_file_sync();
8836 let result = supervisor.spawn(spec());
8837 let failed = result.is_err();
8838 if let Ok(module) = result {
8839 module.stop().await.unwrap();
8840 }
8841 assert!(failed, "record failure must fail spawn");
8842 let pid = LAST_PID.get().expect("the real OS child was created");
8843 assert!(
8844 subc_os::Process::open(pid).unwrap().is_none(),
8845 "child {pid} survived rejected admission or was not reaped"
8846 );
8847 }
8848
8849 #[cfg(target_os = "macos")]
8850 #[test]
8851 #[ignore = "manual on-device latency measurement"]
8852 fn durable_record_write_latency_50() {
8853 let home = cortexkit_test_support::ScratchDir::new("record-latency");
8854 let path = home.join("live-children.json");
8855 let children = vec![crate::live_children::LiveChild {
8856 module_id: "latency-worker".into(),
8857 pid: std::process::id(),
8858 protocol: ModuleProtocol::None,
8859 start_time: Some(1),
8860 executable: None,
8861 cgroup_name: None,
8862 }];
8863 let mut samples = Vec::new();
8864 for _ in 0..50 {
8865 let start = std::time::Instant::now();
8866 crate::live_children::write_record(&path, &children).unwrap();
8867 samples.push(start.elapsed());
8868 }
8869 samples.sort();
8870 eprintln!(
8871 "durable_record_write_latency_50: n=50 median={:?} p95={:?}",
8872 (samples[24] + samples[25]) / 2,
8873 samples[47]
8874 );
8875 }
8876
8877 #[cfg(windows)]
8878 #[test]
8879 fn windows_store_touch_fixture() {
8880 if let Some(path) = std::env::var_os("SUBC_TEST_STORE_TOUCH") {
8881 std::fs::write(path, b"module ran").unwrap();
8882 }
8883 }
8884
8885 #[cfg(windows)]
8886 #[tokio::test]
8887 async fn windows_daemon_fixture() {
8888 let Ok(address) = std::env::var("SUBC_TEST_DAEMON_BARRIER") else {
8889 return;
8890 };
8891 let mut spec = spec();
8892 spec.env.push(("SUBC_TEST_AFTER_CREATE".into(), address));
8893 let supervisor =
8894 Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default());
8895 let _module = supervisor.spawn(spec).unwrap();
8896 }
8897
8898 #[cfg(windows)]
8899 #[test]
8900 fn windows_daemon_death_before_module_job_assignment_leaves_no_child() {
8901 use std::io::Read;
8902 let listener = std::net::TcpListener::bind("127.0.0.1:0").unwrap();
8903 let mut command = std::process::Command::new(std::env::current_exe().unwrap());
8906 command
8907 .args([
8908 "--exact",
8909 "supervise::spawn_record_tests::windows_daemon_fixture",
8910 "--nocapture",
8911 ])
8912 .env(
8913 "SUBC_TEST_DAEMON_BARRIER",
8914 listener.local_addr().unwrap().to_string(),
8915 );
8916 let mut daemon = subc_jobobject::spawn_contained(&mut command).unwrap();
8917 listener.set_nonblocking(true).unwrap();
8918 let deadline = std::time::Instant::now() + Duration::from_secs(30);
8919 let mut stream = loop {
8920 if let Ok((stream, _)) = listener.accept() {
8921 break stream;
8922 }
8923 assert!(
8924 daemon.child.try_wait().unwrap().is_none(),
8925 "daemon failed before rendezvous"
8926 );
8927 assert!(
8928 std::time::Instant::now() < deadline,
8929 "daemon did not reach CreateProcess failpoint"
8930 );
8931 std::thread::yield_now();
8932 };
8933 stream
8934 .set_read_timeout(Some(Duration::from_secs(30)))
8935 .unwrap();
8936 let mut pid = [0; 4];
8937 stream.read_exact(&mut pid).unwrap();
8938 let pid = u32::from_ne_bytes(pid);
8939 assert!(subc_jobobject::process_exists(pid));
8940 daemon.child.kill().unwrap();
8943 daemon.child.wait().unwrap();
8944 let exited = subc_jobobject::wait_for_process_exit(pid, Duration::from_secs(10));
8945 if !exited {
8946 daemon.job.terminate().unwrap();
8947 }
8948 assert!(exited, "suspended child {pid} outlived hard-killed daemon");
8949 }
8950
8951 #[cfg(windows)]
8952 #[tokio::test]
8953 async fn windows_containment_failure_never_resumes() {
8954 let home = cortexkit_test_support::ScratchDir::new("assignment-refusal");
8955 let sentinel = home.join("store-touched");
8956 let mut spec = spec();
8957 spec.env
8958 .push(("SUBC_TEST_ASSIGNMENT_FAILURE".into(), "1".into()));
8959 spec.env.push((
8960 "SUBC_TEST_STORE_TOUCH".into(),
8961 sentinel.display().to_string(),
8962 ));
8963 let supervisor =
8964 Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default());
8965 let result = supervisor.spawn(spec);
8966 if let Ok(module) = result {
8969 let _ = tokio::time::timeout(Duration::from_secs(10), async {
8970 while !sentinel.exists() && module.state().unwrap() == ModuleState::Running {
8971 tokio::task::yield_now().await;
8972 }
8973 })
8974 .await;
8975 let _ = module.stop().await;
8976 panic!(
8977 "assignment failure resumed a module; store touched={}",
8978 sentinel.exists()
8979 );
8980 }
8981 assert!(
8982 !sentinel.exists(),
8983 "refused module executed store-touch sentinel"
8984 );
8985 assert!(!subc_jobobject::process_exists(LAST_PID.get().unwrap()));
8986 }
8987
8988 #[cfg(windows)]
8989 #[tokio::test]
8990 async fn windows_nested_daemon_fixture() {
8991 let Some(sentinel) = std::env::var_os("SUBC_TEST_NESTED_STORE") else {
8992 return;
8993 };
8994 let mut spec = spec();
8995 spec.env.push((
8996 "SUBC_TEST_STORE_TOUCH".into(),
8997 sentinel.to_str().unwrap().into(),
8998 ));
8999 let supervisor =
9000 Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default());
9001 let module = supervisor
9002 .spawn(spec)
9003 .expect("per-module job must nest below daemon and host jobs");
9004 tokio::time::timeout(Duration::from_secs(10), async {
9005 while !PathBuf::from(&sentinel).exists() {
9006 tokio::task::yield_now().await;
9007 }
9008 })
9009 .await
9010 .unwrap();
9011 module.stop().await.unwrap();
9012 }
9013
9014 #[cfg(windows)]
9015 #[test]
9016 fn windows_module_job_nests_under_host_job() {
9017 let home = cortexkit_test_support::ScratchDir::new("nested-jobs");
9018 let sentinel = home.join("nested-store-touched");
9019 let mut command = std::process::Command::new(std::env::current_exe().unwrap());
9020 command
9021 .args([
9022 "--exact",
9023 "supervise::spawn_record_tests::windows_nested_daemon_fixture",
9024 "--nocapture",
9025 ])
9026 .env("SUBC_TEST_NESTED_STORE", &sentinel);
9027 let mut daemon = subc_jobobject::spawn_contained(&mut command).unwrap();
9028 assert!(daemon.child.wait().unwrap().success());
9029 assert!(sentinel.exists(), "nested protocol:none module never ran");
9030 }
9031}
9032
9033#[cfg(target_os = "linux")]
9034pub(crate) fn kill_module_cgroup(placement: Option<&subc_cgroup::Placement>, module_id: &str) {
9035 use subc_cgroup::KillOutcome;
9036 match subc_cgroup::kill_module(placement, module_id) {
9037 KillOutcome::Killed => {}
9038 KillOutcome::NotPlaced | KillOutcome::Unsupported => {
9039 debug!(
9040 module_id,
9041 "cgroup tree kill unavailable; using direct-child kill"
9042 );
9043 }
9044 KillOutcome::IoError { path, error } => {
9045 warn!(module_id, path = %path.display(), %error, "cgroup tree kill failed; using direct-child kill");
9046 }
9047 }
9048}
9049
9050#[cfg(windows)]
9058fn contain_spawned_child(
9059 child: &mut Child,
9060 spec: &ModuleSpec,
9061) -> Result<Option<subc_jobobject::JobObject>, SuperviseError> {
9062 let assignment = || {
9063 let job = subc_jobobject::JobObject::new()?;
9064 #[cfg(test)]
9065 spawn_record_tests::assignment_failpoint(spec)?;
9066 job.assign(child)?;
9067 Ok::<_, io::Error>(job)
9068 };
9069 match assignment() {
9070 Ok(job) => Ok(Some(job)),
9071 Err(source) => {
9072 kill_and_reap_rejected_child(child);
9073 Err(SuperviseError::Spawn {
9074 program: spec.program.clone(),
9075 source: io::Error::other(format!(
9076 "Windows module job containment failed: {source}"
9077 )),
9078 cgroup_path: None,
9079 })
9080 }
9081 }
9082}
9083
9084fn kill_and_reap_rejected_child(child: &mut Child) {
9087 #[cfg(unix)]
9088 if let Some(pid) = child
9089 .id()
9090 .and_then(|pid| rustix::process::Pid::from_raw(pid as i32))
9091 {
9092 let _ = rustix::process::kill_process_group(pid, rustix::process::Signal::KILL);
9093 }
9094 let _ = child.start_kill();
9095 while matches!(child.try_wait(), Ok(None)) {
9096 std::thread::yield_now();
9097 }
9098}
9099
9100#[cfg(windows)]
9105fn resume_suspended_child(pid: u32, spec: &ModuleSpec) -> Result<(), SuperviseError> {
9106 if let Err(source) = subc_jobobject::resume_main_thread(pid) {
9107 let _ = std::process::Command::new("taskkill.exe")
9111 .args(["/PID", &pid.to_string(), "/T", "/F"])
9112 .stdin(Stdio::null())
9113 .stdout(Stdio::null())
9114 .stderr(Stdio::null())
9115 .status();
9116 return Err(SuperviseError::Spawn {
9117 program: spec.program.clone(),
9118 source,
9119 cgroup_path: None,
9120 });
9121 }
9122 Ok(())
9123}
9124
9125#[cfg(target_os = "linux")]
9126fn remove_module_cgroup(placement: &subc_cgroup::Placement, module_id: &str) {
9127 match placement.remove_module(module_id) {
9128 Ok(()) => debug!(module_id, "removed module cgroup after process exit"),
9129 Err(error) => warn!(
9130 module_id,
9131 error = %error,
9132 "could not remove module cgroup after process exit; continuing teardown"
9133 ),
9134 }
9135}
9136
9137#[cfg(target_os = "linux")]
9138async fn cleanup_reaped_cgroup(placement: &subc_cgroup::Placement, module_id: &str) {
9139 if matches!(
9143 subc_cgroup::kill_module(Some(placement), module_id),
9144 subc_cgroup::KillOutcome::Killed
9145 ) {
9146 if let Ok(path) = placement.module_path(module_id) {
9147 while std::fs::read_to_string(path.join("cgroup.events"))
9148 .ok()
9149 .is_some_and(|events| events.lines().any(|line| line == "populated 1"))
9150 {
9151 sleep(Duration::from_millis(1)).await;
9152 }
9153 }
9154 }
9155 remove_module_cgroup(placement, module_id);
9156}
9157
9158#[cfg(target_os = "linux")]
9159fn apply_cgroup_placement(
9160 command: &mut Command,
9161 spec: &ModuleSpec,
9162 path: &std::path::Path,
9163) -> Result<(), SuperviseError> {
9164 subc_cgroup::apply(command, path).map_err(|source| SuperviseError::Cgroup {
9165 module_id: spec.module_id.clone(),
9166 source,
9167 })
9168}
9169
9170fn capture_retention(spec: &ModuleSpec) -> Retention {
9171 let defaults = Retention::default();
9172 let value = |name: &str| {
9173 spec.env
9174 .iter()
9175 .rev()
9176 .find_map(|(key, value)| (key == name).then_some(value.as_str()))
9177 };
9178 Retention {
9179 max_file_mb: value(CAPTURE_MAX_FILE_MB_ENV)
9180 .and_then(|value| value.parse().ok())
9181 .unwrap_or(defaults.max_file_mb),
9182 keep: value(CAPTURE_KEEP_ENV)
9183 .and_then(|value| value.parse().ok())
9184 .unwrap_or(defaults.keep),
9185 max_age_days: value(CAPTURE_MAX_AGE_DAYS_ENV)
9186 .and_then(|value| value.parse().ok())
9187 .unwrap_or(defaults.max_age_days),
9188 }
9189}
9190
9191fn generate_launch_nonce() -> Result<String, SuperviseError> {
9194 let mut bytes = [0u8; 32];
9195 getrandom::getrandom(&mut bytes).map_err(|source| SuperviseError::LaunchNonce {
9196 reason: source.to_string(),
9197 })?;
9198 let mut hex = String::with_capacity(64);
9199 for b in bytes {
9200 use std::fmt::Write;
9201 let _ = write!(hex, "{b:02x}");
9202 }
9203 Ok(hex)
9204}
9205
9206fn constant_time_eq(a: &[u8], b: &[u8]) -> bool {
9209 if a.len() != b.len() {
9210 return false;
9211 }
9212 let mut diff = 0u8;
9213 for (x, y) in a.iter().zip(b.iter()) {
9214 diff |= x ^ y;
9215 }
9216 diff == 0
9217}
9218
9219#[cfg(not(windows))]
9223fn observe_spawned_image(pid: u32) -> Option<subc_os::Observation> {
9224 let process = subc_os::Process::open(pid).ok().flatten()?;
9225 #[cfg(target_os = "linux")]
9226 return sample_after_linux_exec(
9227 || {
9228 let stat = std::fs::read_to_string(format!("/proc/{pid}/stat")).ok()?;
9229 let committed = linux_exec_committed(&stat);
9230 #[cfg(test)]
9231 spawn_image_tests::report_exec_state(committed);
9232 committed
9233 },
9234 || process.observe(),
9235 );
9236 #[cfg(not(target_os = "linux"))]
9237 process.observe()
9238}
9239
9240#[cfg(target_os = "linux")]
9244fn linux_exec_committed(stat: &str) -> Option<bool> {
9245 const PF_FORKNOEXEC: u64 = 0x40;
9246 let flags = stat.rsplit_once(')')?.1.split_whitespace().nth(6)?;
9247 Some(flags.parse::<u64>().ok()? & PF_FORKNOEXEC == 0)
9248}
9249
9250#[cfg(target_os = "linux")]
9251fn sample_after_linux_exec(
9252 mut exec_committed: impl FnMut() -> Option<bool>,
9253 observe: impl FnOnce() -> Option<subc_os::Observation>,
9254) -> Option<subc_os::Observation> {
9255 let deadline = std::time::Instant::now() + Duration::from_secs(1);
9256 while !exec_committed()? {
9257 if std::time::Instant::now() >= deadline {
9261 return None;
9262 }
9263 std::thread::sleep(Duration::from_micros(50));
9264 }
9265 observe()
9266}
9267
9268#[cfg(all(test, target_os = "linux"))]
9269mod spawn_image_tests {
9270 use super::*;
9271 use std::cell::{Cell, RefCell};
9272
9273 enum ExecEvent {
9274 Uncommitted,
9275 Image(Option<subc_os::Observation>),
9276 }
9277
9278 thread_local! {
9279 static EXEC_EVENTS: RefCell<Option<std::sync::mpsc::Sender<ExecEvent>>> = const { RefCell::new(None) };
9280 }
9281
9282 pub(super) fn report_exec_state(committed: Option<bool>) {
9283 if committed == Some(false) {
9284 EXEC_EVENTS.with(|events| {
9285 if let Some(events) = events.borrow_mut().take() {
9286 let _ = events.send(ExecEvent::Uncommitted);
9287 }
9288 });
9289 }
9290 }
9291
9292 #[test]
9293 fn linux_spawn_observation_waits_for_a_paused_fork_to_exec() {
9294 use std::io::{Read, Write};
9295 use std::os::fd::AsRawFd;
9296
9297 let descriptor = std::fs::File::open("/dev/null").unwrap();
9298 let mut command = std::process::Command::new("/bin/sleep");
9299 command.arg("30");
9300 let (mut ready, mut release) =
9301 subc_os::fork_exec_test::pause_before_exec(&mut command, descriptor.as_raw_fd())
9302 .unwrap();
9303 let spawn = std::thread::spawn(move || command.spawn());
9304 let mut message = [0u8; 8];
9305 ready.read_exact(&mut message).unwrap();
9306 let pid = i32::from_ne_bytes(message[..4].try_into().unwrap()) as u32;
9307 let before = std::fs::read_to_string(format!("/proc/{pid}/stat"));
9308 let inherited = subc_os::Process::open(pid).unwrap().unwrap().observe();
9309 let (events, received) = std::sync::mpsc::channel();
9310 let observer = std::thread::spawn(move || {
9311 EXEC_EVENTS.with(|slot| *slot.borrow_mut() = Some(events.clone()));
9312 let image = observe_spawned_image(pid);
9313 let _ = events.send(ExecEvent::Image(image));
9314 });
9315 let first = received.recv_timeout(Duration::from_secs(5));
9319 release.write_all(&[1]).unwrap();
9320 let mut child = spawn.join().unwrap().unwrap();
9321 let after = match first {
9322 Ok(ExecEvent::Uncommitted) => match received.recv_timeout(Duration::from_secs(5)) {
9323 Ok(ExecEvent::Image(image)) => image,
9324 _ => None,
9325 },
9326 Ok(ExecEvent::Image(image)) => image,
9327 Err(_) => None,
9328 };
9329 observer.join().unwrap();
9330 child.kill().unwrap();
9331 child.wait().unwrap();
9332
9333 assert_eq!(linux_exec_committed(&before.unwrap()), Some(false));
9334 assert_eq!(
9335 inherited.unwrap().executable,
9336 subc_os::file_identity(&std::env::current_exe().unwrap())
9337 );
9338 assert_eq!(
9339 after.unwrap().executable,
9340 subc_os::file_identity(std::path::Path::new("/bin/sleep"))
9341 );
9342 }
9343
9344 #[test]
9345 fn spawned_image_is_not_sampled_until_linux_exec_commits() {
9346 let committed = Cell::new(false);
9347 let inherited = subc_os::Observation {
9348 start_time: 23,
9349 executable: Some(subc_os::FileIdentity {
9350 device: 29,
9351 inode: 31,
9352 }),
9353 };
9354 let module = subc_os::Observation {
9355 start_time: 23,
9356 executable: Some(subc_os::FileIdentity {
9357 device: 29,
9358 inode: 37,
9359 }),
9360 };
9361 let polls = Cell::new(0);
9362 let image = sample_after_linux_exec(
9363 || {
9364 polls.set(polls.get() + 1);
9365 committed.set(polls.get() == 2);
9368 Some(committed.get())
9369 },
9370 || Some(if committed.get() { module } else { inherited }),
9371 );
9372 assert_eq!(image, Some(module));
9373 assert_eq!(polls.get(), 2);
9374 }
9375
9376 #[test]
9377 fn linux_exec_flag_is_read_from_flags_after_the_last_parenthesis() {
9378 assert_eq!(
9379 linux_exec_committed("17 (a ) b) R 1 2 3 4 5 4194368"),
9380 Some(false)
9381 );
9382 assert_eq!(
9383 linux_exec_committed("17 (a ) b) R 1 2 3 4 5 4194304"),
9384 Some(true)
9385 );
9386 assert_eq!(linux_exec_committed("17 (a) R 1 2 3 4 5 invalid"), None);
9387 assert_eq!(linux_exec_committed("17 (a) R"), None);
9388 }
9389
9390 #[test]
9391 fn unreadable_exec_state_never_samples_an_inherited_image() {
9392 assert_eq!(
9393 sample_after_linux_exec(
9394 || None,
9395 || panic!("must not sample before exec confirmation")
9396 ),
9397 None
9398 );
9399 }
9400
9401 #[test]
9402 fn uncommitted_exec_times_out_without_sampling_an_inherited_image() {
9403 assert_eq!(
9404 sample_after_linux_exec(
9405 || Some(false),
9406 || panic!("must not sample after an exec timeout")
9407 ),
9408 None
9409 );
9410 }
9411}
9412
9413#[cfg(windows)]
9416fn windows_spawn_capture(spec: &ModuleSpec) -> io::Result<subc_os::ExecutableCapture> {
9417 let mut program = spec.program.clone();
9418 if program.is_absolute() || program.components().count() > 1 {
9419 match subc_os::ExecutableCapture::open(&program) {
9422 Ok(capture) => return Ok(capture),
9423 Err(error)
9424 if error.kind() == io::ErrorKind::NotFound && program.extension().is_none() => {}
9425 Err(error) => return Err(error),
9426 }
9427 program.set_extension("exe");
9428 return subc_os::ExecutableCapture::open(&program);
9429 }
9430 if program.extension().is_none() {
9431 program.set_extension("exe");
9432 }
9433 let custom_path = spec
9434 .env
9435 .iter()
9436 .rev()
9437 .find(|(key, _)| key.eq_ignore_ascii_case("PATH"));
9438 let mut directories = Vec::new();
9439 if let Some(parent) = std::env::current_exe()?.parent() {
9442 directories.push(parent.to_path_buf());
9443 }
9444 directories.push(std::env::current_dir()?);
9445 if let Some(windows) = std::env::var_os("SystemRoot") {
9446 let windows = PathBuf::from(windows);
9447 directories.push(windows.join("System32"));
9448 directories.push(windows.clone());
9449 }
9450 let path = custom_path
9451 .map(|(_, value)| std::ffi::OsString::from(value))
9452 .or_else(|| std::env::var_os("PATH"));
9453 if let Some(path) = path {
9454 directories.extend(std::env::split_paths(&path));
9455 }
9456 for directory in directories {
9457 match subc_os::ExecutableCapture::open(&directory.join(&program)) {
9458 Ok(capture) => return Ok(capture),
9459 Err(error) if error.kind() == io::ErrorKind::NotFound => continue,
9460 Err(error) => return Err(error),
9461 }
9462 }
9463 Err(io::Error::new(
9464 io::ErrorKind::NotFound,
9465 "resolved executable not found",
9466 ))
9467}
9468
9469fn spawn_and_mark_running(
9470 spec: &ModuleSpec,
9471 runtime: &SupervisorRuntimeConfig,
9472 snapshot: &SharedSnapshot,
9473) -> Result<SupervisedChild, SuperviseError> {
9474 let activity_spawn = runtime.begin_activity_spawn(&spec.module_id);
9475 let child = spawn_child(
9476 spec,
9477 runtime.connection_file_path.as_deref(),
9478 runtime.supervisor_handle.as_ref(),
9479 &runtime.stderr_ring,
9480 runtime.capture_logs_dir.as_deref(),
9481 &runtime.child_roster,
9482 #[cfg(target_os = "linux")]
9483 runtime.cgroup_placement.as_ref(),
9484 )?;
9485 set_running(snapshot, &child, &spec.module_id, &runtime.spawn_events)?;
9486 if let Some(spawn) = activity_spawn {
9487 spawn.complete(snapshot);
9488 }
9489 Ok(child)
9490}
9491
9492enum RegistrationWaitOutcome {
9493 Registered,
9494 Exited(ExitReport),
9495 TimedOut,
9496}
9497
9498struct ReloadRegistrationFailure {
9499 exit_report: ExitReport,
9500 reason: String,
9501}
9502
9503#[derive(Debug, Clone, Copy, PartialEq, Eq)]
9504enum BusyGaugeObservation {
9505 Quiescent,
9506 Busy,
9507 Omitted,
9508}
9509
9510#[derive(Debug, Clone, PartialEq, Eq)]
9512enum GaugeVerdict {
9513 Zero,
9515 NonZero(u64),
9517 NotACount(String),
9521 Missing,
9523 ProbeFailed(&'static str),
9526}
9527
9528impl GaugeVerdict {
9529 fn reads_quiet(&self) -> bool {
9530 matches!(self, Self::Zero)
9531 }
9532}
9533
9534impl fmt::Display for GaugeVerdict {
9535 fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
9536 match self {
9537 Self::Zero => f.write_str("zero"),
9538 Self::NonZero(value) => write!(f, "non_zero({value})"),
9539 Self::NotACount(value) => write!(f, "not_a_count({value})"),
9540 Self::Missing => f.write_str("missing_from_health_body"),
9541 Self::ProbeFailed(kind) => write!(f, "probe_failed({kind})"),
9542 }
9543 }
9544}
9545
9546const GAUGE_VALUE_TEXT_LIMIT: usize = 32;
9549
9550fn gauge_verdicts(metrics: Option<&Value>, gauges: &[String]) -> Vec<GaugeVerdict> {
9552 let metrics = metrics.and_then(Value::as_object);
9553 gauges
9554 .iter()
9555 .map(
9556 |gauge| match metrics.and_then(|metrics| metrics.get(gauge)) {
9557 None => GaugeVerdict::Missing,
9558 Some(value) => match value.as_u64() {
9559 Some(0) => GaugeVerdict::Zero,
9560 Some(count) => GaugeVerdict::NonZero(count),
9561 None => {
9562 let mut text = value.to_string();
9563 if text.len() > GAUGE_VALUE_TEXT_LIMIT {
9564 let mut end = GAUGE_VALUE_TEXT_LIMIT;
9565 while !text.is_char_boundary(end) {
9566 end -= 1;
9567 }
9568 text.truncate(end);
9569 text.push_str("...");
9570 }
9571 GaugeVerdict::NotACount(text)
9572 }
9573 },
9574 },
9575 )
9576 .collect()
9577}
9578
9579fn busy_gauge_observation(verdicts: &[GaugeVerdict]) -> BusyGaugeObservation {
9582 if verdicts
9583 .iter()
9584 .any(|verdict| matches!(verdict, GaugeVerdict::Missing))
9585 {
9586 BusyGaugeObservation::Omitted
9587 } else if verdicts.iter().all(GaugeVerdict::reads_quiet) {
9588 BusyGaugeObservation::Quiescent
9589 } else {
9590 BusyGaugeObservation::Busy
9591 }
9592}
9593
9594#[derive(Debug, Clone, Default, PartialEq, Eq)]
9599struct GaugeTrace {
9600 latest: Option<(GaugeVerdict, Duration)>,
9601 last_value: Option<(GaugeVerdict, Duration)>,
9602}
9603
9604#[derive(Debug, Clone, Default, PartialEq, Eq)]
9610struct DrainGaugeTrace {
9611 probes: u32,
9613 gauges: Vec<(String, GaugeTrace)>,
9615}
9616
9617impl DrainGaugeTrace {
9618 fn new(gauges: &[String]) -> Self {
9619 Self {
9620 probes: 0,
9621 gauges: gauges
9622 .iter()
9623 .map(|gauge| (gauge.clone(), GaugeTrace::default()))
9624 .collect(),
9625 }
9626 }
9627
9628 fn record(&mut self, verdicts: &[GaugeVerdict], at: Duration) {
9630 self.probes = self.probes.saturating_add(1);
9631 for ((_, trace), verdict) in self.gauges.iter_mut().zip(verdicts) {
9632 trace.latest = Some((verdict.clone(), at));
9633 if matches!(
9634 verdict,
9635 GaugeVerdict::Zero | GaugeVerdict::NonZero(_) | GaugeVerdict::NotACount(_)
9636 ) {
9637 trace.last_value = Some((verdict.clone(), at));
9638 }
9639 }
9640 }
9641}
9642
9643impl fmt::Display for DrainGaugeTrace {
9648 fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
9649 if self.gauges.is_empty() {
9650 return f.write_str("none declared");
9651 }
9652 for (index, (name, trace)) in self.gauges.iter().enumerate() {
9653 if index > 0 {
9654 f.write_str(" | ")?;
9655 }
9656 write!(f, "{name}: ")?;
9657 match &trace.latest {
9658 Some((verdict, at)) => write!(f, "{verdict} @{}ms", at.as_millis())?,
9659 None => f.write_str("never probed")?,
9660 }
9661 match &trace.last_value {
9662 Some((value, at)) => write!(f, "; last value {value} @{}ms", at.as_millis())?,
9663 None => f.write_str("; no value ever read")?,
9664 }
9665 }
9666 Ok(())
9667 }
9668}
9669
9670pub(crate) fn declared_busy_gauges(
9671 registry: &Registry,
9672 module_id: &str,
9673) -> Result<Vec<String>, SuperviseError> {
9674 busy_gauges_of(
9675 registry
9676 .get_module(module_id)
9677 .map_err(SuperviseError::Registry)?,
9678 )
9679}
9680
9681fn declared_busy_gauges_for_connection(
9685 registry: &Registry,
9686 connection_id: ConnectionId,
9687) -> Result<Vec<String>, SuperviseError> {
9688 busy_gauges_of(
9689 registry
9690 .get_module_by_connection(connection_id)
9691 .map_err(SuperviseError::Registry)?,
9692 )
9693}
9694
9695fn busy_gauges_of(
9696 registration: Option<crate::registry::ModuleRegistration>,
9697) -> Result<Vec<String>, SuperviseError> {
9698 let Some(registration) = registration else {
9699 return Ok(Vec::new());
9700 };
9701 let Some(self_signals) = registration.manifest.self_signals else {
9702 return Ok(Vec::new());
9703 };
9704
9705 let mut gauges = Vec::new();
9706 for declaration in self_signals {
9707 if declaration.kind != SelfSignalKind::Busy {
9708 continue;
9709 }
9710 match declaration.anchored_to {
9711 SignalAnchor::HealthGauges { gauges: declared } if !declared.is_empty() => {
9712 gauges.extend(declared)
9713 }
9714 _ => {
9715 gauges.push(String::new());
9718 }
9719 }
9720 }
9721 Ok(gauges)
9722}
9723
9724#[allow(clippy::too_many_arguments)]
9733async fn wait_for_forwarding_quiescence(
9734 forwarding: &ForwardingTable,
9735 module_id: &str,
9736 runtime: &SupervisorRuntimeConfig,
9737 endpoint: crate::ModuleEndpointId,
9738 deadline: Instant,
9739 busy_gauges: &[String],
9740 scope: DrainScope,
9741 trace: &mut DrainGaugeTrace,
9742) -> Result<bool, SuperviseError> {
9743 let wait_started = Instant::now();
9744 let mut gauges_quiescent = busy_gauges.is_empty();
9745 let mut next_probe_at = wait_started;
9746 let mut omission_reported = false;
9747
9748 loop {
9749 let now = Instant::now();
9750 if !busy_gauges.is_empty() && now >= next_probe_at && now < deadline {
9751 let report = match scope {
9752 DrainScope::Active => probe_module_health(module_id, runtime, Some(deadline)).await,
9753 DrainScope::Endpoint(endpoint) => {
9754 probe_endpoint_health(endpoint, runtime, Some(deadline)).await
9755 }
9756 };
9757 let verdicts = match &report {
9758 Ok(report) => gauge_verdicts(report.metrics.as_ref(), busy_gauges),
9759 Err(err) => vec![GaugeVerdict::ProbeFailed(err.label()); busy_gauges.len()],
9760 };
9761 trace.record(&verdicts, wait_started.elapsed());
9762 gauges_quiescent = match report {
9763 Ok(_) => match busy_gauge_observation(&verdicts) {
9764 BusyGaugeObservation::Quiescent => true,
9765 BusyGaugeObservation::Busy => false,
9766 BusyGaugeObservation::Omitted => {
9767 if !omission_reported {
9770 forwarding
9771 .counters()
9772 .increment_drains_with_undeclared_gauge();
9773 let missing = busy_gauges
9774 .iter()
9775 .zip(&verdicts)
9776 .filter(|(_, verdict)| matches!(verdict, GaugeVerdict::Missing))
9777 .map(|(gauge, _)| gauge.as_str())
9778 .collect::<Vec<_>>()
9779 .join(",");
9780 warn!(
9781 module_id,
9782 missing_gauges = %missing,
9783 "drain health.check answer lacks a declared busy gauge; the drain treats the module as busy until it reports it"
9784 );
9785 omission_reported = true;
9786 }
9787 false
9788 }
9789 },
9790 Err(err) => {
9791 warn!(
9792 module_id,
9793 error = %err,
9794 "drain health.check did not produce declared busy gauges; treating module as busy"
9795 );
9796 false
9797 }
9798 };
9799 next_probe_at = Instant::now() + drain_busy_reprobe_interval();
9802 }
9803
9804 let in_flight = forwarding
9805 .endpoint_in_flight_count(endpoint)
9806 .map_err(SuperviseError::Forwarding)?;
9807 if in_flight == 0 && gauges_quiescent {
9808 return Ok(true);
9809 }
9810
9811 let now = Instant::now();
9812 if now >= deadline {
9813 return Ok(false);
9814 }
9815 let mut wait = deadline
9816 .saturating_duration_since(now)
9817 .min(REGISTRY_RELEASE_POLL);
9818 if !busy_gauges.is_empty() {
9819 wait = wait.min(next_probe_at.saturating_duration_since(now));
9820 }
9821 sleep(wait).await;
9822 }
9823}
9824
9825fn drain_busy_reprobe_interval() -> Duration {
9829 DRAIN_BUSY_REPROBE_INTERVAL.max(REGISTRY_RELEASE_POLL)
9830}
9831
9832fn warn_drain_timed_out(
9847 module_id: &str,
9848 drain_timeout: Duration,
9849 reason: RouteCloseReason,
9850 holdouts: &crate::forwarding::DrainHoldouts,
9851 gauges: &DrainGaugeTrace,
9852) {
9853 warn!(
9854 module_id = %module_id,
9855 waited = ?drain_timeout,
9856 ?reason,
9857 held_requests = holdouts.requests,
9858 held_routes = holdouts.routes,
9859 total_routes = holdouts.total_routes,
9860 top_connections = ?holdouts.top_connections,
9861 held = %holdouts
9864 .held
9865 .iter()
9866 .map(|(channel, corr)| format!("{channel}:{corr}"))
9867 .collect::<Vec<_>>()
9868 .join(","),
9869 gauge_probes = gauges.probes,
9870 busy_gauges = %gauges,
9871 "route drain timed out before request quiescence; forcing teardown"
9872 );
9873}
9874
9875fn drained_after_quiescence_wait(wait_result: &Result<bool, SuperviseError>) -> bool {
9883 match wait_result {
9884 Ok(drained) => *drained,
9885 Err(_) => false,
9886 }
9887}
9888
9889fn send_route_goodbyes(forwarding: &ForwardingTable, released_routes: Vec<GoodbyeTarget>) {
9890 for released in released_routes {
9891 let frame = match Frame::build_with_version(
9892 released.negotiated_ver,
9893 FrameType::Goodbye,
9894 control_flags(),
9895 released.channel,
9896 released.epoch,
9897 0,
9898 Vec::new(),
9899 ) {
9900 Ok(frame) => frame,
9901 Err(err) => {
9902 warn!(
9903 route_channel = released.channel,
9904 error = %err,
9905 "failed to build supervisor drain route GOODBYE frame"
9906 );
9907 continue;
9908 }
9909 };
9910 if !released.close_on_delivery_failure() {
9911 crate::forwarding::send_module_route_goodbye(
9912 &forwarding.counters(),
9913 &released.sink,
9914 frame,
9915 released.module_id.as_deref(),
9916 "supervisor drain",
9917 );
9918 continue;
9919 }
9920 if let Err(err) = released.sink.try_send(frame) {
9921 warn!(
9922 target_connection_id = released.connection_id.get(),
9923 route_channel = released.channel,
9924 error = %err,
9925 "supervisor drain route GOODBYE was not delivered to client; closing target connection"
9926 );
9927 let _ = forwarding.escalate_client_delivery_failure(
9928 released.connection_id,
9929 released.channel,
9930 released.epoch,
9931 CloseReason::new(
9932 "route_goodbye_delivery_failed",
9933 format!(
9934 "failed to enqueue supervisor drain route GOODBYE for channel {}: {err}",
9935 released.channel
9936 ),
9937 ),
9938 crate::forwarding::UndeliveredFrame {
9939 module_id: released.module_id.as_deref(),
9940 sink: &released.sink,
9941 },
9942 );
9943 }
9944 }
9945}
9946
9947fn send_module_draining(
9948 module_id: &str,
9949 reason: RouteCloseReason,
9950 deadline_ms: u64,
9951 target: &ModuleDrainTarget,
9952) {
9953 let body = match serde_json::to_vec(&ModuleControlCommand::Draining {
9954 reason,
9955 deadline_ms,
9956 }) {
9957 Ok(body) => body,
9958 Err(err) => {
9959 warn!(
9960 module_id,
9961 error = %err,
9962 "failed to encode module draining command"
9963 );
9964 return;
9965 }
9966 };
9967 let frame = match Frame::build_with_version(
9968 target.negotiated_ver,
9969 FrameType::Push,
9970 control_flags(),
9971 0,
9972 0,
9973 0,
9974 body,
9975 ) {
9976 Ok(frame) => frame,
9977 Err(err) => {
9978 warn!(
9979 module_id,
9980 error = %err,
9981 "failed to build module draining command frame"
9982 );
9983 return;
9984 }
9985 };
9986 if let Err(err) = target.sink.try_send(frame) {
9987 warn!(
9988 module_id,
9989 target_connection_id = target.endpoint.connection_id.get(),
9990 error = %err,
9991 "module draining command was not delivered to peer"
9992 );
9993 }
9994}
9995
9996fn module_goodbye_frame(module_id: &str, negotiated_ver: u8) -> Option<Frame> {
9998 match Frame::build_with_version(
9999 negotiated_ver,
10000 FrameType::Goodbye,
10001 control_flags(),
10002 0,
10003 0,
10004 0,
10005 Vec::new(),
10006 ) {
10007 Ok(frame) => Some(frame),
10008 Err(err) => {
10009 warn!(
10010 module_id,
10011 error = %err,
10012 "failed to build module GOODBYE frame"
10013 );
10014 None
10015 }
10016 }
10017}
10018
10019#[cfg(unix)]
10033async fn send_module_goodbyes_for_daemon_shutdown(
10034 forwarding: &Arc<ForwardingTable>,
10035 reason: &CloseReason,
10036 wait_for_flush: bool,
10037) {
10038 const GOODBYE_BUDGET: Duration = Duration::from_millis(500);
10039 let targets = match forwarding.module_connections() {
10040 Ok(targets) => targets,
10041 Err(err) => {
10042 warn!(error = %err, "could not list module connections for shutdown GOODBYE");
10043 return;
10044 }
10045 };
10046 let deadline = Instant::now() + GOODBYE_BUDGET;
10047 let mut sends = tokio::task::JoinSet::new();
10048 for target in targets {
10049 let Some(frame) = module_goodbye_frame(&target.module_id, target.negotiated_ver) else {
10050 continue;
10051 };
10052 if !wait_for_flush {
10053 if let Err(err) = target.sink.try_send(frame) {
10054 debug!(
10055 module_id = %target.module_id,
10056 error = %err,
10057 "shutdown module GOODBYE was not queued"
10058 );
10059 }
10060 continue;
10061 }
10062 let forwarding = Arc::clone(forwarding);
10063 let reason = reason.clone();
10064 sends.spawn(async move {
10065 match timeout_at(deadline, target.sink.send_flushed(frame)).await {
10066 Ok(Ok(())) => {}
10067 Ok(Err(err)) => debug!(
10068 module_id = %target.module_id,
10069 error = %err,
10070 "module connection closed before its shutdown GOODBYE was written"
10071 ),
10072 Err(_) => warn!(
10073 module_id = %target.module_id,
10074 budget = ?GOODBYE_BUDGET,
10075 "shutdown module GOODBYE was not written within its budget; closing anyway"
10076 ),
10077 }
10078 forwarding.request_connection_close(target.endpoint.connection_id, reason);
10079 });
10080 }
10081 while sends.join_next().await.is_some() {}
10083}
10084
10085fn send_module_goodbye(module_id: &str, forwarding: &ForwardingTable, target: &ModuleDrainTarget) {
10086 let Some(frame) = module_goodbye_frame(module_id, target.negotiated_ver) else {
10087 return;
10088 };
10089 if let Err(err) = target.sink.try_send(frame) {
10090 warn!(
10091 module_id,
10092 target_connection_id = target.endpoint.connection_id.get(),
10093 error = %err,
10094 "supervisor drain module GOODBYE was not delivered to peer; closing module connection"
10095 );
10096 forwarding.request_connection_close(
10097 target.endpoint.connection_id,
10098 CloseReason::new(
10099 "module_goodbye_delivery_failed",
10100 format!("failed to enqueue supervisor drain module GOODBYE for module '{module_id}': {err}"),
10101 ),
10102 );
10103 }
10104}
10105
10106#[derive(Clone, Copy)]
10107struct ForwardingDrainContext<'a> {
10108 spec: &'a ModuleSpec,
10109 runtime: &'a SupervisorRuntimeConfig,
10110 registry: &'a Registry,
10111 scope: DrainScope,
10112}
10113
10114#[derive(Debug, Clone, Copy, PartialEq, Eq)]
10116enum DrainScope {
10117 Active,
10120 Endpoint(crate::ModuleEndpointId),
10125}
10126
10127#[derive(Debug, Clone, Copy, PartialEq, Eq)]
10135enum StopNotice {
10136 SentOverConnection,
10139 NoConnection,
10143 NotSent,
10147}
10148
10149async fn begin_forwarding_drain(
10150 spec: &ModuleSpec,
10151 runtime: &SupervisorRuntimeConfig,
10152 registry: &Registry,
10153 snapshot: &SharedSnapshot,
10154 enabled: Option<bool>,
10155 reason: RouteCloseReason,
10156) -> Result<StopNotice, SuperviseError> {
10157 let Some(forwarding) = runtime.forwarding.as_ref() else {
10158 return Err(SuperviseError::ReloadUnavailable {
10159 module_id: spec.module_id.clone(),
10160 reason: "supervisor was not configured with a forwarding table".to_string(),
10161 });
10162 };
10163
10164 begin_forwarding_drain_with(
10165 forwarding,
10166 ForwardingDrainContext {
10167 spec,
10168 runtime,
10169 registry,
10170 scope: DrainScope::Active,
10171 },
10172 snapshot,
10173 enabled,
10174 reason,
10175 runtime.drain_timeout,
10176 )
10177 .await
10178}
10179
10180async fn begin_forwarding_drain_if_configured(
10181 spec: &ModuleSpec,
10182 runtime: &SupervisorRuntimeConfig,
10183 registry: &Registry,
10184 snapshot: &SharedSnapshot,
10185 enabled: Option<bool>,
10186 reason: RouteCloseReason,
10187) -> Result<StopNotice, SuperviseError> {
10188 begin_forwarding_drain_with_timeout(
10189 spec,
10190 runtime,
10191 registry,
10192 snapshot,
10193 enabled,
10194 reason,
10195 runtime.drain_timeout,
10196 )
10197 .await
10198}
10199
10200async fn begin_forwarding_drain_with_timeout(
10204 spec: &ModuleSpec,
10205 runtime: &SupervisorRuntimeConfig,
10206 registry: &Registry,
10207 snapshot: &SharedSnapshot,
10208 enabled: Option<bool>,
10209 reason: RouteCloseReason,
10210 drain_timeout: Duration,
10211) -> Result<StopNotice, SuperviseError> {
10212 let Some(forwarding) = runtime.forwarding.as_ref() else {
10213 return Ok(StopNotice::NotSent);
10214 };
10215
10216 begin_forwarding_drain_with(
10217 forwarding,
10218 ForwardingDrainContext {
10219 spec,
10220 runtime,
10221 registry,
10222 scope: DrainScope::Active,
10223 },
10224 snapshot,
10225 enabled,
10226 reason,
10227 drain_timeout,
10228 )
10229 .await
10230}
10231
10232async fn begin_forwarding_drain_with(
10233 forwarding: &ForwardingTable,
10234 context: ForwardingDrainContext<'_>,
10235 snapshot: &SharedSnapshot,
10236 enabled: Option<bool>,
10237 reason: RouteCloseReason,
10238 drain_timeout: Duration,
10239) -> Result<StopNotice, SuperviseError> {
10240 let ForwardingDrainContext {
10241 spec,
10242 runtime,
10243 registry,
10244 scope,
10245 } = context;
10246 debug_assert_ne!(reason, RouteCloseReason::Crash);
10247 let terminal = matches!(reason, RouteCloseReason::Disable);
10248 let drain_started_at = Instant::now();
10249 let drain_deadline = drain_started_at + drain_timeout;
10250 let deadline_ms =
10251 unix_ms_now().saturating_add(u64::try_from(drain_timeout.as_millis()).unwrap_or(u64::MAX));
10252 let busy_gauges = match scope {
10253 DrainScope::Active => declared_busy_gauges(registry, &spec.module_id)?,
10254 DrainScope::Endpoint(endpoint) => {
10255 declared_busy_gauges_for_connection(registry, endpoint.connection_id)?
10256 }
10257 };
10258
10259 let gate_started = Instant::now();
10262 let drain_target = match scope {
10263 DrainScope::Active => forwarding.begin_module_drain(&spec.module_id, reason),
10264 DrainScope::Endpoint(endpoint) => forwarding.begin_endpoint_drain(endpoint, reason),
10265 }
10266 .map_err(SuperviseError::Forwarding)?;
10267 info!(
10272 module_id = %spec.module_id,
10273 ?reason,
10274 gate_ms = u64::try_from(gate_started.elapsed().as_millis()).unwrap_or(u64::MAX),
10275 connected = drain_target.is_some(),
10276 "module drain began; route admission closed"
10277 );
10278 if scope == DrainScope::Active {
10279 update_snapshot(snapshot, Some(&spec.module_id), |state| {
10280 state.state = ModuleState::Draining;
10281 state.draining_to_replace =
10282 matches!(reason, RouteCloseReason::Restart | RouteCloseReason::Reload);
10283 if let Some(enabled) = enabled {
10284 state.enabled = enabled;
10285 }
10286 })?;
10287 }
10288
10289 let Some(target) = drain_target.as_ref() else {
10290 return Ok(StopNotice::NoConnection);
10294 };
10295 {
10296 send_module_draining(&spec.module_id, reason, deadline_ms, target);
10297 #[cfg(feature = "test-support")]
10298 forwarding.wait_before_route_closing_for_test().await;
10299 let routes = forwarding
10300 .endpoint_routes(target.endpoint)
10301 .map_err(SuperviseError::Forwarding)?;
10302 let routes_notified = routes.len();
10303 crate::control::send_route_control_pushes(
10304 forwarding,
10305 routes.clone(),
10306 ClientControlPush::RouteClosing {
10307 module_id: spec.module_id.clone(),
10308 channels: Vec::new(),
10309 reason,
10310 },
10311 );
10312 send_route_goodbyes(forwarding, target.abandoned_bindings.clone());
10313
10314 let mut gauge_trace = DrainGaugeTrace::new(&busy_gauges);
10320 let wait_result = wait_for_forwarding_quiescence(
10321 forwarding,
10322 &spec.module_id,
10323 runtime,
10324 target.endpoint,
10325 drain_deadline,
10326 &busy_gauges,
10327 scope,
10328 &mut gauge_trace,
10329 )
10330 .await;
10331 let drained = drained_after_quiescence_wait(&wait_result);
10332 if let Err(err) = &wait_result {
10333 error!(
10334 module_id = %spec.module_id,
10335 ?reason,
10336 error = %err,
10337 "forwarding quiescence wait failed after route.closing; forcing route.closed(drained: false) so the client is not left waiting on an unfulfilled promise"
10338 );
10339 } else if !drained {
10340 let holdouts = forwarding
10342 .endpoint_drain_holdouts(target.endpoint)
10343 .unwrap_or_default();
10344 warn_drain_timed_out(
10345 &spec.module_id,
10346 drain_timeout,
10347 reason,
10348 &holdouts,
10349 &gauge_trace,
10350 );
10351 }
10352 crate::control::send_route_control_pushes(
10353 forwarding,
10354 routes,
10355 ClientControlPush::RouteClosed {
10356 module_id: spec.module_id.clone(),
10357 channels: Vec::new(),
10358 reason,
10359 drained,
10360 abandoned: target.abandoned_bindings.len() as u32,
10361 excluded_subscriptions: target.excluded_subscriptions,
10362 terminal: Some(terminal),
10363 },
10364 );
10365 wait_result?;
10366
10367 let released_routes = match forwarding.release_module_endpoint_routes(target.endpoint) {
10373 Ok(routes) => routes,
10374 Err(err) => {
10375 warn!(
10376 module_id = %spec.module_id,
10377 ?reason,
10378 error = %err,
10379 "failed to release module endpoint routes after route.closed; module GOODBYE will still be sent"
10380 );
10381 send_module_goodbye(&spec.module_id, forwarding, target);
10382 return Err(SuperviseError::Forwarding(err));
10383 }
10384 };
10385 let route_goodbye_count = released_routes.len();
10386 send_route_goodbyes(forwarding, released_routes);
10387 send_module_goodbye(&spec.module_id, forwarding, target);
10388
10389 info!(
10395 module_id = %spec.module_id,
10396 ?reason,
10397 routes_notified,
10398 route_goodbyes = route_goodbye_count,
10399 abandoned_reservations = target.abandoned_bindings.len(),
10400 excluded_subscriptions = target.excluded_subscriptions,
10401 drained,
10402 "module drain complete; consumers notified via route.closing/route.closed pushes and per-route GOODBYE frames"
10403 );
10404 }
10405
10406 Ok(StopNotice::SentOverConnection)
10407}
10408
10409async fn wait_for_registration_after_reload(
10412 registry: &Registry,
10413 module_id: &str,
10414 snapshot: &SharedSnapshot,
10415 child: &mut SupervisedChild,
10416 wait: Duration,
10417) -> Result<RegistrationWaitOutcome, SuperviseError> {
10418 wait_for_slot_registration(
10419 registry,
10420 crate::registry::RegistrationSlot::Active(module_id),
10421 module_id,
10422 snapshot,
10423 child,
10424 wait,
10425 )
10426 .await
10427}
10428
10429async fn wait_for_slot_registration(
10437 registry: &Registry,
10438 slot: crate::registry::RegistrationSlot<'_>,
10439 module_id: &str,
10440 snapshot: &SharedSnapshot,
10441 child: &mut SupervisedChild,
10442 wait: Duration,
10443) -> Result<RegistrationWaitOutcome, SuperviseError> {
10444 let deadline = Instant::now() + wait;
10445 loop {
10446 if registry
10447 .registration(slot)
10448 .map_err(SuperviseError::Registry)?
10449 .is_some()
10450 {
10451 return Ok(RegistrationWaitOutcome::Registered);
10452 }
10453
10454 let now = Instant::now();
10455 if now >= deadline {
10456 return Ok(RegistrationWaitOutcome::TimedOut);
10457 }
10458 let remaining = deadline.saturating_duration_since(now);
10459 let poll = remaining.min(REGISTRY_RELEASE_POLL);
10460
10461 tokio::select! {
10462 wait_result = child.wait() => {
10463 let status = wait_result.map_err(|source| SuperviseError::Wait {
10464 module_id: module_id.to_string(),
10465 source,
10466 })?;
10467 return Ok(RegistrationWaitOutcome::Exited(classify_reaped_child_exit(
10468 snapshot,
10469 child,
10470 &status,
10471 )));
10472 }
10473 _ = sleep(poll) => {}
10474 }
10475 }
10476}
10477
10478fn registration_failure_exit_report(mut exit_report: ExitReport) -> ExitReport {
10479 if exit_report.kind != ExitKind::DeliberateSeverance {
10482 exit_report.kind = ExitKind::Crash;
10483 }
10484 exit_report
10485}
10486
10487async fn handle_reload_child_registration_failure(
10488 spec: &ModuleSpec,
10489 runtime: &SupervisorRuntimeConfig,
10490 registry: &Registry,
10491 process_liveness: &SupervisorProcessLiveness,
10492 snapshot: &SharedSnapshot,
10493 _child: &mut Option<SupervisedChild>,
10494 failure: ReloadRegistrationFailure,
10495) -> Result<(), SuperviseError> {
10496 let ReloadRegistrationFailure {
10497 exit_report,
10498 reason,
10499 } = failure;
10500 match on_child_exit(
10501 spec,
10502 runtime.restart_policy,
10503 registry,
10504 snapshot,
10505 &runtime.terminal_ring,
10506 &runtime.spawn_events,
10507 &runtime.child_roster,
10508 exit_report,
10509 )
10510 .await
10511 {
10512 NextAction::Stop {
10513 registration_released,
10514 } => {
10515 if registration_released {
10516 process_liveness.untrack_if_current(&spec.module_id, snapshot);
10517 }
10518 }
10519 NextAction::Restart { schedule } => {
10520 let delay = schedule.map_or(runtime.restart_policy.delay_for_restart(0), |schedule| {
10521 schedule.delay
10522 });
10523 if let Some(schedule) = schedule {
10524 log_crash_respawn(&spec.module_id, schedule);
10525 }
10526 schedule_respawn(
10527 runtime,
10528 snapshot,
10529 &spec.module_id,
10530 delay,
10531 RespawnKind::Spawn,
10532 )?;
10533 }
10534 }
10535 Err(SuperviseError::ReloadFailed {
10536 module_id: spec.module_id.clone(),
10537 reason,
10538 })
10539}
10540
10541async fn handle_reload_spawn_failure(
10542 spec: &ModuleSpec,
10543 runtime: &SupervisorRuntimeConfig,
10544 process_liveness: &SupervisorProcessLiveness,
10545 snapshot: &SharedSnapshot,
10546 _child: &mut Option<SupervisedChild>,
10547 reason: String,
10548) -> Result<(), SuperviseError> {
10549 let now = Instant::now();
10550 let mut schedule = None;
10551 update_snapshot(snapshot, Some(&spec.module_id), |state| {
10552 clear_current_process_facts(state);
10553 if state.enabled {
10554 schedule = state.next_crash_restart(&runtime.restart_policy, now);
10555 state.state = if schedule.is_some() {
10556 ModuleState::Restarting
10557 } else {
10558 ModuleState::Failed
10559 };
10560 } else {
10561 state.state = ModuleState::Disabled;
10562 }
10563 })?;
10564 if let Some(schedule) = schedule {
10565 schedule_respawn(
10566 runtime,
10567 snapshot,
10568 &spec.module_id,
10569 schedule.delay,
10570 RespawnKind::Spawn,
10571 )?;
10572 } else {
10573 process_liveness.untrack_if_current(&spec.module_id, snapshot);
10574 }
10575 Err(SuperviseError::ReloadFailed {
10576 module_id: spec.module_id.clone(),
10577 reason,
10578 })
10579}
10580
10581fn control_flags() -> Flags {
10582 Flags::new(false, Priority::Passive, false)
10583}
10584
10585#[allow(clippy::too_many_arguments)]
10586async fn drain_optional_child(
10587 module_id: &str,
10588 protocol: ModuleProtocol,
10589 stop_notice: StopNotice,
10590 registry: &Registry,
10591 forwarding: Option<&ForwardingTable>,
10592 snapshot: &SharedSnapshot,
10593 terminal_ring: &Arc<Mutex<TerminalRing>>,
10594 spawn_events: &SpawnEventFeed,
10595 child: &mut Option<SupervisedChild>,
10596 drain_timeout: Duration,
10597 final_state: ModuleState,
10598 enabled: Option<bool>,
10599) -> Result<(), SuperviseError> {
10600 if let Some(child) = child.take() {
10601 drain_child_to_state(
10602 module_id,
10603 protocol,
10604 stop_notice,
10605 registry,
10606 forwarding,
10607 snapshot,
10608 terminal_ring,
10609 spawn_events,
10610 child,
10611 drain_timeout,
10612 final_state,
10613 enabled,
10614 )
10615 .await
10616 } else {
10617 update_snapshot(snapshot, Some(module_id), |state| {
10618 state.state = final_state;
10619 if let Some(enabled) = enabled {
10620 state.enabled = enabled;
10621 }
10622 clear_current_process_facts(state);
10623 })?;
10624 release_dead_registration(registry, forwarding, snapshot, module_id).await
10625 }
10626}
10627
10628#[allow(clippy::too_many_arguments)]
10629async fn drain_child_to_state(
10630 module_id: &str,
10631 _protocol: ModuleProtocol,
10632 stop_notice: StopNotice,
10633 registry: &Registry,
10634 forwarding: Option<&ForwardingTable>,
10635 snapshot: &SharedSnapshot,
10636 terminal_ring: &Arc<Mutex<TerminalRing>>,
10637 spawn_events: &SpawnEventFeed,
10638 mut child: SupervisedChild,
10639 drain_timeout: Duration,
10640 final_state: ModuleState,
10641 enabled: Option<bool>,
10642) -> Result<(), SuperviseError> {
10643 let protocol = child.protocol;
10644 update_snapshot(snapshot, Some(module_id), |state| {
10645 state.state = ModuleState::Draining;
10646 state.draining_to_replace = final_state == ModuleState::Restarting;
10647 if let Some(enabled) = enabled {
10648 state.enabled = enabled;
10649 }
10650 })?;
10651
10652 if stop_notice != StopNotice::SentOverConnection {
10663 if protocol == ModuleProtocol::Subc && stop_notice == StopNotice::NoConnection {
10664 info!(
10665 module_id,
10666 pid = child.pid,
10667 budget_ms = u64::try_from(drain_timeout.as_millis()).unwrap_or(u64::MAX),
10668 "module has no connection yet; requesting stop by signal"
10669 );
10670 }
10671 request_graceful_stop(module_id, &child);
10672 }
10673
10674 let exit_report = match timeout(drain_timeout, child.wait()).await {
10675 Ok(Ok(status)) => classify_reaped_child_exit(snapshot, &child, &status),
10676 Ok(Err(source)) => {
10677 fail_after_wait_error(
10678 module_id,
10679 snapshot,
10680 terminal_ring,
10681 spawn_events,
10682 format!("failed to wait during drain: {source}"),
10683 );
10684 return Err(SuperviseError::Wait {
10685 module_id: module_id.to_string(),
10686 source,
10687 });
10688 }
10689 Err(_) => {
10690 warn!(
10703 module_id,
10704 pid = child.pid,
10705 budget_ms = u64::try_from(drain_timeout.as_millis()).unwrap_or(u64::MAX),
10706 reason = ?final_state,
10707 ?stop_notice,
10708 "drain budget expired before the module exited; killing it"
10709 );
10710 child.start_kill().map_err(|source| {
10711 fail_after_wait_error(
10712 module_id,
10713 snapshot,
10714 terminal_ring,
10715 spawn_events,
10716 format!("failed to kill during drain: {source}"),
10717 );
10718 SuperviseError::Kill {
10719 module_id: module_id.to_string(),
10720 source,
10721 }
10722 })?;
10723 let status = child.wait().await.map_err(|source| {
10724 fail_after_wait_error(
10725 module_id,
10726 snapshot,
10727 terminal_ring,
10728 spawn_events,
10729 format!("failed to wait after drain kill: {source}"),
10730 );
10731 SuperviseError::Wait {
10732 module_id: module_id.to_string(),
10733 source,
10734 }
10735 })?;
10736 classify_reaped_child_exit(snapshot, &child, &status)
10737 }
10738 };
10739
10740 let detail = lock_snapshot(snapshot)?.drain_disposition_detail.take();
10741 publish_after_terminal_record(
10742 module_id,
10743 || {
10744 record_terminal_with_detail(
10745 module_id,
10746 terminal_ring,
10747 spawn_events,
10748 &exit_report,
10749 terminal_disposition(final_state),
10750 detail,
10751 )
10752 },
10753 || {
10754 update_snapshot(snapshot, Some(module_id), |state| {
10755 state.state = final_state;
10756 if let Some(enabled) = enabled {
10757 state.enabled = enabled;
10758 }
10759 clear_current_process_facts(state);
10760 state.last_exit = Some(exit_report.clone());
10761 if exit_report.kind == ExitKind::DeliberateSeverance {
10762 state.lifetime_restarts += 1;
10763 }
10764 })
10765 },
10766 )?;
10767 child.drain_stderr(module_id).await;
10768
10769 release_dead_registration(registry, forwarding, snapshot, module_id).await
10770}
10771
10772#[cfg(unix)]
10792fn request_graceful_stop(module_id: &str, child: &SupervisedChild) {
10793 let Some(pid) = child
10794 .id()
10795 .and_then(|pid| i32::try_from(pid).ok())
10796 .and_then(rustix::process::Pid::from_raw)
10797 else {
10798 debug!(
10799 module_id,
10800 "no pid to signal for teardown; falling through to the drain wait"
10801 );
10802 return;
10803 };
10804 match rustix::process::kill_process(pid, rustix::process::Signal::TERM) {
10805 Ok(()) => debug!(
10806 module_id,
10807 "sent SIGTERM to a module nothing else asked to stop"
10808 ),
10809 Err(err) => debug!(
10810 module_id,
10811 error = %err,
10812 "SIGTERM to module failed; the drain wait and kill still apply"
10813 ),
10814 }
10815}
10816
10817#[cfg(not(unix))]
10825fn request_graceful_stop(module_id: &str, _child: &SupervisedChild) {
10826 debug!(
10827 module_id,
10828 "no graceful stop signal exists on this platform; teardown of a module nothing asked to stop waits, then kills"
10829 );
10830}
10831
10832fn terminal_disposition(final_state: ModuleState) -> TerminalDisposition {
10833 match final_state {
10834 ModuleState::Stopped => TerminalDisposition::Stopped,
10835 ModuleState::Disabled => TerminalDisposition::Disabled,
10836 ModuleState::Restarting => TerminalDisposition::Restarting,
10837 ModuleState::Failed => TerminalDisposition::Failed,
10838 ModuleState::Starting
10839 | ModuleState::Running
10840 | ModuleState::Unresponsive
10841 | ModuleState::Draining => {
10842 unreachable!("terminal exits only finish in terminal or restarting states")
10843 }
10844 }
10845}
10846
10847async fn release_dead_registration(
10855 registry: &Registry,
10856 forwarding: Option<&ForwardingTable>,
10857 snapshot: &SharedSnapshot,
10858 module_id: &str,
10859) -> Result<(), SuperviseError> {
10860 let result = async {
10861 let registration = registry
10862 .get_module(module_id)
10863 .map_err(SuperviseError::Registry)?;
10864 match wait_for_registration_release(registry, module_id, REGISTRY_RELEASE_TIMEOUT).await {
10865 Ok(()) => return Ok(()),
10866 Err(SuperviseError::RegistrationStillActive { .. }) => {}
10867 Err(err) => return Err(err),
10868 }
10869 let pid = lock_snapshot(snapshot)?.reaped_pid;
10870 if let (Some(registration), Some(forwarding), Some(pid)) = (registration, forwarding, pid) {
10871 warn!(
10872 module_id,
10873 pid,
10874 connection_id = registration.connection_id.get(),
10875 "reaped module registration outlived release grace; closing dead connection"
10876 );
10877 forwarding.request_connection_close(
10878 registration.connection_id,
10879 CloseReason::new(
10880 "supervised_process_reaped",
10881 format!("module '{module_id}' pid {pid} exited"),
10882 ),
10883 );
10884 wait_for_slot_registration_release(
10885 registry,
10886 crate::registry::RegistrationSlot::Connection(registration.connection_id),
10887 REGISTRY_RELEASE_TIMEOUT,
10888 )
10889 .await?;
10890 }
10891 wait_for_registration_release(registry, module_id, Duration::ZERO).await
10892 }
10893 .await;
10894 if let Err(err) = &result {
10895 fail_snapshot(snapshot, Some(module_id), None);
10896 error!(module_id, error = %err, "registration release failed after child exit; module is failed and start can retry");
10897 }
10898 result
10899}
10900
10901async fn wait_for_registration_release(
10904 registry: &Registry,
10905 module_id: &str,
10906 wait: Duration,
10907) -> Result<(), SuperviseError> {
10908 wait_for_slot_registration_release(
10909 registry,
10910 crate::registry::RegistrationSlot::Active(module_id),
10911 wait,
10912 )
10913 .await
10914}
10915
10916async fn wait_for_slot_registration_release(
10924 registry: &Registry,
10925 slot: crate::registry::RegistrationSlot<'_>,
10926 wait: Duration,
10927) -> Result<(), SuperviseError> {
10928 let deadline = Instant::now() + wait;
10929 let mut release_events = registration_release_events().subscribe();
10930 let still_active = |registration: &crate::registry::ModuleRegistration| {
10931 SuperviseError::RegistrationStillActive {
10932 module_id: registration.manifest.module_id.clone(),
10933 waited: wait,
10934 }
10935 };
10936 loop {
10937 let _observed_generation = *release_events.borrow_and_update();
10938 let Some(registration) = registry
10939 .registration(slot)
10940 .map_err(SuperviseError::Registry)?
10941 else {
10942 return Ok(());
10943 };
10944
10945 let now = Instant::now();
10946 if now >= deadline {
10947 return Err(still_active(®istration));
10948 }
10949
10950 let remaining = deadline.saturating_duration_since(now);
10951 match timeout(remaining, release_events.changed()).await {
10952 Ok(Ok(())) | Ok(Err(_)) => {}
10953 Err(_) => return Err(still_active(®istration)),
10954 }
10955 }
10956}
10957
10958#[cfg(test)]
10959mod slot_registration_wait_tests {
10960 use super::*;
10961 use crate::registry::{ConnectionId, RegistrationSlot};
10962 use subc_protocol::manifest::ModuleManifest;
10963
10964 #[tokio::test]
10965 async fn enable_release_failure_is_failed_and_a_second_enable_retries() {
10966 let registry = Arc::new(Registry::default());
10967 let supervisor = Supervisor::new_for_test(Arc::clone(®istry), RestartPolicy::default());
10968 let runtime = supervisor.runtime_config();
10969 let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::disabled()));
10970 let spec = ModuleSpec {
10971 module_id: "enable-stale-registration".to_string(),
10972 program: PathBuf::from("/missing/enable-retry-test"),
10973 args: Vec::new(),
10974 env: Vec::new(),
10975 reserved: false,
10976 reserved_prefixes: Vec::new(),
10977 protocol: ModuleProtocol::Subc,
10978 overlap: Default::default(),
10979 };
10980 let connection = ConnectionId::new(90);
10981 registry
10982 .register_with_control_ops(
10983 ModuleManifest::builder(&spec.module_id, "0.1.0").build(),
10984 1,
10985 connection,
10986 Vec::new(),
10987 )
10988 .unwrap();
10989 let mut child = None;
10990 let err = set_child_enabled(
10991 &spec,
10992 &runtime,
10993 ®istry,
10994 &supervisor.process_liveness,
10995 &snapshot,
10996 &mut child,
10997 true,
10998 )
10999 .await
11000 .unwrap_err();
11001 assert!(matches!(
11002 err,
11003 SuperviseError::RegistrationStillActive { .. }
11004 ));
11005 assert_eq!(lock_snapshot(&snapshot).unwrap().state, ModuleState::Failed);
11006 assert!(child.is_none());
11007 registry.deregister_connection(connection).unwrap();
11008 let err = set_child_enabled(
11009 &spec,
11010 &runtime,
11011 ®istry,
11012 &supervisor.process_liveness,
11013 &snapshot,
11014 &mut child,
11015 true,
11016 )
11017 .await
11018 .unwrap_err();
11019 assert!(
11020 matches!(err, SuperviseError::Spawn { .. }),
11021 "second enable must attempt a spawn: {err}"
11022 );
11023 assert_eq!(lock_snapshot(&snapshot).unwrap().state, ModuleState::Failed);
11024 }
11025
11026 const INCUMBENT: u64 = 1;
11027 const CANDIDATE: u64 = 2;
11028
11029 fn swapped_registry() -> Arc<Registry> {
11030 let registry = Arc::new(Registry::default());
11031 let manifest = ModuleManifest::builder("m", "0.1.0").build();
11032 registry
11033 .register_with_control_ops(
11034 manifest.clone(),
11035 1,
11036 ConnectionId::new(INCUMBENT),
11037 Vec::new(),
11038 )
11039 .unwrap();
11040 registry
11041 .register_candidate_with_control_ops(
11042 manifest,
11043 1,
11044 ConnectionId::new(CANDIDATE),
11045 Vec::new(),
11046 )
11047 .unwrap();
11048 registry
11049 }
11050
11051 #[tokio::test]
11055 async fn incumbent_release_is_awaited_by_connection_not_by_module_id() {
11056 let registry = swapped_registry();
11057 registry.promote_candidate("m").unwrap().unwrap();
11058
11059 assert!(matches!(
11060 wait_for_registration_release(®istry, "m", Duration::from_millis(50)).await,
11061 Err(SuperviseError::RegistrationStillActive { .. })
11062 ));
11063
11064 assert!(matches!(
11066 wait_for_slot_registration_release(
11067 ®istry,
11068 RegistrationSlot::Connection(ConnectionId::new(INCUMBENT)),
11069 Duration::from_millis(50),
11070 )
11071 .await,
11072 Err(SuperviseError::RegistrationStillActive { .. })
11073 ));
11074
11075 let releaser = Arc::clone(®istry);
11076 let release = tokio::spawn(async move {
11077 sleep(Duration::from_millis(20)).await;
11078 releaser
11079 .deregister_connection(ConnectionId::new(INCUMBENT))
11080 .unwrap();
11081 notify_registration_release();
11082 });
11083 wait_for_slot_registration_release(
11084 ®istry,
11085 RegistrationSlot::Connection(ConnectionId::new(INCUMBENT)),
11086 Duration::from_secs(5),
11087 )
11088 .await
11089 .expect("the incumbent's own registration is released");
11090 release.await.unwrap();
11091 assert!(registry.get_module("m").unwrap().is_some());
11092 }
11093
11094 #[tokio::test]
11097 async fn candidate_slot_wait_ignores_the_incumbents_registration() {
11098 let registry = swapped_registry();
11099 assert!(matches!(
11100 wait_for_slot_registration_release(
11101 ®istry,
11102 RegistrationSlot::Candidate("m"),
11103 Duration::from_millis(50),
11104 )
11105 .await,
11106 Err(SuperviseError::RegistrationStillActive { .. })
11107 ));
11108 registry
11109 .deregister_connection(ConnectionId::new(CANDIDATE))
11110 .unwrap();
11111 wait_for_slot_registration_release(
11112 ®istry,
11113 RegistrationSlot::Candidate("m"),
11114 Duration::from_millis(50),
11115 )
11116 .await
11117 .expect("a candidate slot with no candidate is released");
11118 assert!(registry
11119 .registration(RegistrationSlot::Active("m"))
11120 .unwrap()
11121 .is_some());
11122 }
11123}
11124
11125fn classify_exit(status: &ExitStatus) -> ExitReport {
11126 ExitReport {
11127 kind: if status.success() {
11128 ExitKind::Clean
11129 } else {
11130 ExitKind::Crash
11131 },
11132 code: status.code(),
11133 signal: exit_signal(status),
11134 at_ms: unix_ms_now(),
11135 }
11136}
11137
11138fn wait_error_exit_report() -> ExitReport {
11144 ExitReport {
11145 kind: ExitKind::Crash,
11146 code: None,
11147 signal: None,
11148 at_ms: unix_ms_now(),
11149 }
11150}
11151
11152#[cfg(unix)]
11153fn exit_signal(status: &ExitStatus) -> Option<i32> {
11154 use std::os::unix::process::ExitStatusExt;
11155
11156 status.signal()
11157}
11158
11159#[cfg(not(unix))]
11160fn exit_signal(_status: &ExitStatus) -> Option<i32> {
11161 None
11162}
11163
11164fn reset_restart_count(snapshot: &SharedSnapshot, module_id: &str) -> Result<(), SuperviseError> {
11170 update_snapshot(snapshot, Some(module_id), |state| {
11171 state.clear_crash_restarts();
11172 })
11173}
11174
11175fn set_running(
11176 snapshot: &SharedSnapshot,
11177 child: &SupervisedChild,
11178 module_id: &str,
11179 spawn_events: &SpawnEventFeed,
11180) -> Result<(), SuperviseError> {
11181 let mut state = snapshot.lock().map_err(|_| SuperviseError::StatePoisoned {
11182 module_id: Some(module_id.to_string()),
11183 })?;
11184 state.spawn_generation = spawn_events.emit_spawned(module_id, child.pid, child.spawned_at_ms);
11185 if std::mem::take(&mut state.coalesced_restart_pending) {
11186 let generation = state.spawn_generation;
11187 info!(module_id, "restart already satisfied by generation {generation}; coalesced pending request completed");
11188 }
11189 state.drain_disposition_detail = None;
11190 state.spawn_failure = None;
11191 state.waiting_for_previous_pid = None;
11192 state.waiting_for_previous_detail = None;
11193 state.in_alternate_slot = false;
11196 state.configuration_updated_since_spawn = false;
11197 state.spawned_protocol = Some(child.protocol);
11198 state.state = ModuleState::Running;
11199 state.enabled = true;
11200 state.process_alive = true;
11201 state.pid = child.id();
11202 #[cfg(windows)]
11203 {
11204 state.pipe_consumed = child
11205 .nonce_delivery
11206 .as_ref()
11207 .map(|delivery| delivery.consumed());
11208 }
11209 #[cfg(target_os = "macos")]
11210 {
11211 state.report_ready = Some(Arc::clone(&child.report_ready));
11212 }
11213 state.spawned_at_ms = Some(child.spawned_at_ms);
11214 state.exec_outcome = Some(Arc::clone(&child.exec.0));
11215 state.spawned_from = Some(child.spawned_from.clone());
11216 state.launch_spec_digest = Some(child.launch_spec_digest.clone());
11217 state.spawned_file_identity = child.spawned_file_identity.clone();
11218 state.process_start_time = child.process_start_time;
11219 Ok(())
11220}
11221
11222fn clear_current_process_facts(state: &mut SupervisorSnapshot) {
11223 state.waiting_for_previous_pid = None;
11224 state.waiting_for_previous_detail = None;
11225 state.process_alive = false;
11226 state.spawned_protocol = None;
11227 state.pid = None;
11228 #[cfg(windows)]
11229 {
11230 state.pipe_consumed = None;
11231 }
11232 #[cfg(target_os = "macos")]
11233 {
11234 state.report_ready = None;
11235 }
11236 state.spawned_at_ms = None;
11237 state.exec_outcome = None;
11238 state.spawned_from = None;
11239 state.launch_spec_digest = None;
11240 state.spawned_file_identity = None;
11241 state.process_start_time = None;
11242 state.deliberate_severance = None;
11243}
11244
11245#[cfg(test)]
11246fn record_deliberate_severance(
11247 snapshot: &SharedSnapshot,
11248 identity: ProcessIdentity,
11249) -> Result<(), SuperviseError> {
11250 update_snapshot(snapshot, None, |state| {
11251 state.deliberate_severance = Some(identity);
11252 })
11253}
11254
11255fn apply_deliberate_severance_marker(
11256 snapshot: &SharedSnapshot,
11257 exited_identity: Option<ProcessIdentity>,
11258 mut exit_report: ExitReport,
11259) -> ExitReport {
11260 let marker = lock_snapshot(snapshot)
11261 .ok()
11262 .and_then(|mut state| state.deliberate_severance.take());
11263 if marker.is_some() && marker == exited_identity {
11264 exit_report.kind = ExitKind::DeliberateSeverance;
11265 }
11266 exit_report
11267}
11268
11269fn classify_reaped_child_exit(
11270 snapshot: &SharedSnapshot,
11271 child: &SupervisedChild,
11272 status: &ExitStatus,
11273) -> ExitReport {
11274 let _ = update_snapshot(snapshot, None, |state| {
11275 state.reaped_pid = Some(child.pid);
11276 state.spawn_failure = child.spawn_failure.clone();
11277 });
11278 apply_deliberate_severance_marker(snapshot, child.process_identity(), classify_exit(status))
11279}
11280
11281fn fail_snapshot(
11282 snapshot: &SharedSnapshot,
11283 module_id: Option<&str>,
11284 last_exit: Option<ExitReport>,
11285) {
11286 if let Err(err) = update_snapshot(snapshot, module_id, |state| {
11287 state.state = ModuleState::Failed;
11288 clear_current_process_facts(state);
11289 if let Some(last_exit) = last_exit {
11290 state.last_exit = Some(last_exit);
11291 }
11292 }) {
11293 error!(error = %err, "failed to mark supervisor state failed");
11294 }
11295}
11296
11297fn update_snapshot(
11298 snapshot: &SharedSnapshot,
11299 module_id: Option<&str>,
11300 update: impl FnOnce(&mut SupervisorSnapshot),
11301) -> Result<(), SuperviseError> {
11302 let mut state = snapshot.lock().map_err(|_| SuperviseError::StatePoisoned {
11303 module_id: module_id.map(ToOwned::to_owned),
11304 })?;
11305 update(&mut state);
11306 Ok(())
11307}
11308
11309const SLOW_SNAPSHOT_LOCK_THRESHOLD: Duration = Duration::from_millis(250);
11310
11311fn lock_snapshot_for_control<'a>(
11312 snapshot: &'a SharedSnapshot,
11313 module_id: &str,
11314 caller: &'static str,
11315) -> Result<std::sync::MutexGuard<'a, SupervisorSnapshot>, SuperviseError> {
11316 let started_at = Instant::now();
11317 let guard = lock_snapshot(snapshot)?;
11318 let waited = started_at.elapsed();
11319 if waited >= SLOW_SNAPSHOT_LOCK_THRESHOLD {
11320 warn!(
11321 module_id = %module_id,
11322 waited_ms = waited.as_millis() as u64,
11323 caller = %caller,
11324 "slow snapshot lock"
11325 );
11326 }
11327 Ok(guard)
11328}
11329
11330fn lock_snapshot(
11331 snapshot: &SharedSnapshot,
11332) -> Result<std::sync::MutexGuard<'_, SupervisorSnapshot>, SuperviseError> {
11333 snapshot
11334 .lock()
11335 .map_err(|_| SuperviseError::StatePoisoned { module_id: None })
11336}
11337
11338#[cfg(test)]
11339mod terminal_history_tests {
11340 use std::{
11341 path::PathBuf,
11342 sync::Arc,
11343 time::{Duration, Instant},
11344 };
11345
11346 use tokio::{sync::mpsc, time::sleep};
11347
11348 use super::{
11349 apply_deliberate_severance_marker, daemon_will_restart, drain_child_to_state,
11350 drained_after_quiescence_wait, handle_reload_spawn_failure, health_restart_child,
11351 lock_snapshot, on_child_exit, record_deliberate_severance, record_wait_error_terminal,
11352 reset_restart_count, spawn_and_mark_running, update_snapshot, wait_error_exit_report,
11353 ExitKind, ExitReport, ModuleProtocol, ModuleSpec, ModuleState, NextAction, ProcessIdentity,
11354 RestartPolicy, SpawnEventKind, StopNotice, SuperviseError, SupervisedConfiguration,
11355 SupervisedModule, SupervisedModuleInner, Supervisor, SupervisorHandle,
11356 SupervisorHealthStatus, SupervisorSnapshot,
11357 };
11358 use super::Instant as ClockInstant;
11363 use crate::{
11364 registry::Registry,
11365 terminal_ring::{TerminalRing, TerminalRingConfig},
11366 };
11367 use std::sync::Mutex;
11368 use subc_control::TerminalDisposition;
11369
11370 pub(super) fn fake_aft_stub_path() -> PathBuf {
11375 let mut path = std::env::current_exe().expect("current_exe available in tests");
11376 path.pop();
11377 path.pop();
11378 path.push(if cfg!(windows) {
11379 "fake-aft-stub.exe"
11380 } else {
11381 "fake-aft-stub"
11382 });
11383 assert!(
11384 path.exists(),
11385 "fake-aft-stub not built at {}: run `cargo test -p subc-core` (which builds \
11386 [[bin]] targets) rather than `cargo test -p subc-core --lib` (which does not)",
11387 path.display()
11388 );
11389 path
11390 }
11391
11392 #[test]
11393 fn reserved_never_spawned_refuses_every_hello() {
11394 let supervisor = SupervisorHandle::default();
11399 supervisor.apply_identity_configuration(&ModuleSpec {
11400 module_id: "never-spawned".to_string(),
11401 program: PathBuf::from("/usr/bin/false"),
11402 args: Vec::new(),
11403 env: Vec::new(),
11404 reserved: true,
11405 reserved_prefixes: Vec::new(),
11406 protocol: ModuleProtocol::Subc,
11407 overlap: Default::default(),
11408 });
11409 assert!(
11410 supervisor
11411 .reserved_hello_verification("never-spawned", Some("any-forged-nonce"))
11412 .is_err(),
11413 "forged nonce must refuse on a reserved never-spawned id"
11414 );
11415 assert!(
11416 supervisor
11417 .reserved_hello_verification("never-spawned", None)
11418 .is_err(),
11419 "absent nonce must refuse on a reserved never-spawned id"
11420 );
11421 supervisor.set_spawn_nonce("never-spawned", "minted".to_string());
11423 supervisor.apply_identity_configuration(&ModuleSpec {
11424 module_id: "never-spawned".to_string(),
11425 program: PathBuf::from("/usr/bin/false"),
11426 args: Vec::new(),
11427 env: Vec::new(),
11428 reserved: true,
11429 reserved_prefixes: Vec::new(),
11430 protocol: ModuleProtocol::Subc,
11431 overlap: Default::default(),
11432 });
11433 assert!(supervisor
11434 .reserved_hello_verification("never-spawned", Some("minted"))
11435 .is_ok());
11436 assert!(supervisor
11437 .reserved_hello_verification("never-spawned", Some("forged"))
11438 .is_err());
11439 }
11440
11441 fn seed_crash_restarts(state: &mut SupervisorSnapshot, count: u32) {
11444 let now = ClockInstant::now();
11445 for _ in 0..count {
11446 state.crash_restarts.push_back(now);
11447 }
11448 }
11449
11450 fn age_oldest_crash_restart_out_of_window(state: &mut SupervisorSnapshot, window: Duration) {
11454 let aged = state
11455 .crash_restarts
11456 .front()
11457 .expect("a crash restart must be recorded before it can be aged")
11458 .checked_sub(window + Duration::from_secs(1))
11459 .expect("the test clock is far enough from its origin to age an instant");
11460 state.crash_restarts[0] = aged;
11461 }
11462
11463 fn snapshot_with_restarts(enabled: bool, count: u32) -> SupervisorSnapshot {
11464 let mut state = SupervisorSnapshot::new(ModuleState::Running, enabled);
11465 seed_crash_restarts(&mut state, count);
11466 state
11467 }
11468
11469 #[test]
11470 fn daemon_owned_recovery_predicate_uses_the_pre_increment_budget() {
11471 let policy = RestartPolicy::new(3, Duration::ZERO);
11472 let now = ClockInstant::now();
11473 assert!(daemon_will_restart(
11474 &mut snapshot_with_restarts(true, 2),
11475 &policy,
11476 now
11477 ));
11478 assert!(!daemon_will_restart(
11479 &mut snapshot_with_restarts(true, 3),
11480 &policy,
11481 now
11482 ));
11483 assert!(!daemon_will_restart(
11484 &mut snapshot_with_restarts(false, 0),
11485 &policy,
11486 now
11487 ));
11488 }
11489
11490 #[test]
11491 fn crash_restart_backoff_escalates_with_in_window_count() {
11492 let policy = RestartPolicy::new(4, Duration::from_millis(100))
11493 .with_max_backoff(Duration::from_secs(30));
11494 let now = ClockInstant::now();
11495 let mut state = SupervisorSnapshot::new(ModuleState::Running, true);
11496 let schedules = (0..4)
11497 .map(|_| {
11498 state
11499 .next_crash_restart(&policy, now)
11500 .expect("the test policy allows four crash restarts")
11501 })
11502 .collect::<Vec<_>>();
11503
11504 assert_eq!(
11505 schedules
11506 .iter()
11507 .map(|schedule| schedule.restart_in_window)
11508 .collect::<Vec<_>>(),
11509 vec![0, 1, 2, 3]
11510 );
11511 assert_eq!(
11512 schedules
11513 .iter()
11514 .map(|schedule| schedule.delay)
11515 .collect::<Vec<_>>(),
11516 vec![
11517 Duration::from_millis(100),
11518 Duration::from_secs(1),
11519 Duration::from_secs(10),
11520 Duration::from_secs(30),
11521 ]
11522 );
11523 }
11524
11525 #[test]
11526 fn crash_restart_backoff_resets_after_ring_clear() {
11527 let policy = RestartPolicy::new(3, Duration::from_millis(100));
11528 let now = ClockInstant::now();
11529 let mut state = SupervisorSnapshot::new(ModuleState::Running, true);
11530 assert_eq!(
11531 state.next_crash_restart(&policy, now).unwrap().delay,
11532 Duration::from_millis(100)
11533 );
11534 assert_eq!(
11535 state.next_crash_restart(&policy, now).unwrap().delay,
11536 Duration::from_secs(1)
11537 );
11538
11539 state.clear_crash_restarts();
11540 let schedule = state
11541 .next_crash_restart(&policy, now)
11542 .expect("a cleared ring must allow another restart");
11543 assert_eq!(schedule.restart_in_window, 0);
11544 assert_eq!(schedule.delay, Duration::from_millis(100));
11545 }
11546
11547 #[test]
11548 fn crash_restart_backoff_ignores_aged_restarts() {
11549 let policy = RestartPolicy::new(3, Duration::from_millis(100));
11550 let now = ClockInstant::now();
11551 let mut state = SupervisorSnapshot::new(ModuleState::Running, true);
11552 state
11553 .next_crash_restart(&policy, now)
11554 .expect("the first restart is allowed");
11555 state
11556 .next_crash_restart(&policy, now)
11557 .expect("the second restart is allowed");
11558 state.crash_restarts[0] = now
11559 .checked_sub(policy.window + Duration::from_secs(1))
11560 .expect("the fake clock can age a restart past the window");
11561
11562 let schedule = state
11563 .next_crash_restart(&policy, now)
11564 .expect("an aged restart must release its slot");
11565 assert_eq!(schedule.restart_in_window, 1);
11566 assert_eq!(schedule.delay, Duration::from_secs(1));
11567 assert_eq!(state.crash_restarts.len(), 2);
11568 }
11569
11570 #[test]
11574 fn a_budget_spent_before_the_window_no_longer_refuses() {
11575 let policy = RestartPolicy::new(3, Duration::ZERO);
11576 let mut state = snapshot_with_restarts(true, 3);
11577 let now = ClockInstant::now();
11578 assert!(!daemon_will_restart(&mut state, &policy, now));
11579
11580 assert!(daemon_will_restart(
11581 &mut state,
11582 &policy,
11583 now + policy.window + Duration::from_secs(1)
11584 ));
11585 assert!(
11586 state.crash_restarts.is_empty(),
11587 "reading the budget must drop the instants that left the window"
11588 );
11589 }
11590
11591 fn module_with_recovery_snapshot(
11592 state: ModuleState,
11593 enabled: bool,
11594 restart_count: u32,
11595 ) -> SupervisedModule {
11596 let registry = Arc::new(Registry::default());
11597 let supervisor =
11598 Supervisor::new_for_test(Arc::clone(®istry), RestartPolicy::new(3, Duration::ZERO));
11599 let runtime = supervisor.runtime_config();
11600 let spec = ModuleSpec {
11601 module_id: "recovery-snapshot".to_string(),
11602 program: fake_aft_stub_path(),
11603 args: Vec::new(),
11604 env: Vec::new(),
11605 reserved: false,
11606 reserved_prefixes: Vec::new(),
11607 protocol: ModuleProtocol::Subc,
11608 overlap: Default::default(),
11609 };
11610 let mut snapshot = SupervisorSnapshot::new(state, enabled);
11611 seed_crash_restarts(&mut snapshot, restart_count);
11612 let (commands, _rx) = mpsc::channel(4);
11615 SupervisedModule {
11616 inner: Arc::new(SupervisedModuleInner {
11617 module_id: spec.module_id.clone(),
11618 registry,
11619 snapshot: Arc::new(Mutex::new(snapshot)),
11620 configuration: Arc::new(Mutex::new(SupervisedConfiguration {
11621 spec,
11622 health: runtime.health,
11623 })),
11624 stderr_ring: runtime.stderr_ring,
11625 terminal_ring: runtime.terminal_ring,
11626 commands,
11627 monitor: Mutex::new(None),
11628 restart_policy: runtime.restart_policy,
11629 effective_drain_timeout: runtime.effective_drain_timeout,
11630 provenance_probe: supervisor.provenance_probe.clone(),
11631 }),
11632 }
11633 }
11634
11635 #[cfg(target_os = "linux")]
11636 #[tokio::test]
11637 async fn no_cgroup_placement_does_not_block_fake_aft_stub_spawn() {
11638 let supervisor =
11639 Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default())
11640 .with_cgroup_placement(None);
11641 let result = supervisor.spawn(ModuleSpec {
11642 module_id: "no-cgroup-placement".to_string(),
11643 program: fake_aft_stub_path(),
11644 args: Vec::new(),
11645 env: Vec::new(),
11646 reserved: false,
11647 reserved_prefixes: Vec::new(),
11648 protocol: ModuleProtocol::Subc,
11649 overlap: Default::default(),
11650 });
11651
11652 assert!(
11653 result.is_ok(),
11654 "no delegation must not turn an otherwise valid spawn into a failure: {result:?}"
11655 );
11656 }
11657
11658 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
11659 async fn undecided_snapshot_uses_shared_restart_predicate() {
11660 assert!(module_with_recovery_snapshot(ModuleState::Running, true, 2)
11661 .will_recover_after_connection_loss()
11662 .unwrap());
11663 assert!(
11664 !module_with_recovery_snapshot(ModuleState::Running, true, 3)
11665 .will_recover_after_connection_loss()
11666 .unwrap()
11667 );
11668 }
11669
11670 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
11671 async fn restarting_snapshot_at_exhausted_budget_is_non_terminal() {
11672 assert!(
11673 module_with_recovery_snapshot(ModuleState::Restarting, true, 3)
11674 .will_recover_after_connection_loss()
11675 .unwrap()
11676 );
11677 }
11678
11679 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
11680 async fn terminal_phase_snapshots_are_terminal_before_budget_exhaustion() {
11681 assert!(!module_with_recovery_snapshot(ModuleState::Failed, true, 0)
11682 .will_recover_after_connection_loss()
11683 .unwrap());
11684 assert!(
11685 !module_with_recovery_snapshot(ModuleState::Disabled, true, 0)
11686 .will_recover_after_connection_loss()
11687 .unwrap()
11688 );
11689 }
11690
11691 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
11692 async fn warming_snapshot_is_limited_to_startup_phases() {
11693 for state in [
11694 ModuleState::Starting,
11695 ModuleState::Running,
11696 ModuleState::Restarting,
11697 ] {
11698 assert!(
11699 module_with_recovery_snapshot(state, true, 0)
11700 .is_warming()
11701 .unwrap(),
11702 "{state:?} should be warming"
11703 );
11704 }
11705 for state in [
11706 ModuleState::Unresponsive,
11707 ModuleState::Draining,
11708 ModuleState::Stopped,
11709 ModuleState::Failed,
11710 ModuleState::Disabled,
11711 ] {
11712 assert!(
11713 !module_with_recovery_snapshot(state, true, 0)
11714 .is_warming()
11715 .unwrap(),
11716 "{state:?} should not be warming"
11717 );
11718 }
11719 }
11720
11721 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
11722 async fn terminal_history_survives_respawn_and_keeps_both_crashes_in_order() {
11723 let registry = Arc::new(Registry::default());
11724 let supervisor =
11725 Supervisor::new_for_test(Arc::clone(®istry), RestartPolicy::new(1, Duration::ZERO));
11726 let module = supervisor
11727 .spawn(ModuleSpec {
11728 module_id: "terminal-history".to_string(),
11729 program: fake_aft_stub_path(),
11730 args: Vec::new(),
11731 env: vec![("FAKE_AFT_EXIT_CODE".to_string(), "23".to_string())],
11732 reserved: false,
11733 reserved_prefixes: Vec::new(),
11734 protocol: ModuleProtocol::Subc,
11735 overlap: Default::default(),
11736 })
11737 .unwrap();
11738
11739 let deadline = Instant::now() + Duration::from_secs(5);
11740 loop {
11741 let history = module.terminal_history();
11742 if history.entries.len() == 2 {
11743 assert_eq!(module.status().unwrap().state, ModuleState::Failed);
11744 assert_eq!(history.dropped, 0);
11745 assert_eq!(
11746 history
11747 .entries
11748 .iter()
11749 .map(|entry| entry.exit_code)
11750 .collect::<Vec<_>>(),
11751 vec![Some(23), Some(23)]
11752 );
11753 assert!(history.entries[0].at_ms <= history.entries[1].at_ms);
11754 return;
11755 }
11756 assert!(
11757 Instant::now() < deadline,
11758 "module did not retain two terminal exits: {history:?}"
11759 );
11760 sleep(Duration::from_millis(10)).await;
11761 }
11762 }
11763
11764 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
11768 async fn disable_during_crash_backoff_cancels_pending_respawn() {
11769 let backoff = Duration::from_secs(2);
11770 let supervisor = Supervisor::new_for_test(
11771 Arc::new(Registry::default()),
11772 RestartPolicy::new(10, backoff),
11773 );
11774 let module = supervisor
11775 .spawn(ModuleSpec {
11776 module_id: "disable-during-backoff".to_string(),
11777 program: fake_aft_stub_path(),
11778 args: Vec::new(),
11779 env: vec![("FAKE_AFT_EXIT_CODE".to_string(), "23".to_string())],
11780 reserved: false,
11781 reserved_prefixes: Vec::new(),
11782 protocol: ModuleProtocol::Subc,
11783 overlap: Default::default(),
11784 })
11785 .unwrap();
11786
11787 let deadline = Instant::now() + Duration::from_secs(5);
11789 loop {
11790 if module.status().unwrap().state == ModuleState::Restarting {
11791 break;
11792 }
11793 assert!(
11794 Instant::now() < deadline,
11795 "module never entered the crash backoff"
11796 );
11797 sleep(Duration::from_millis(10)).await;
11798 }
11799
11800 let started = Instant::now();
11801 module.set_enabled(false).await.unwrap();
11802 let waited = started.elapsed();
11803
11804 assert!(
11805 waited < backoff / 2,
11806 "disable waited {waited:?} behind the {backoff:?} crash backoff; the operator command must preempt the pending respawn"
11807 );
11808 assert_eq!(module.status().unwrap().state, ModuleState::Disabled);
11809
11810 sleep(backoff + Duration::from_millis(500)).await;
11812 let status = module.status().unwrap();
11813 assert_eq!(status.state, ModuleState::Disabled);
11814 assert_eq!(
11815 status.spawn_generation, 1,
11816 "module respawned after the operator disabled it"
11817 );
11818 }
11819
11820 #[cfg(unix)]
11823 fn protocol_none_sigterm_exits_clean_spec(
11824 module_id: &str,
11825 dir: &std::path::Path,
11826 ) -> (ModuleSpec, PathBuf, PathBuf) {
11827 let ready = dir.join("ready");
11828 let marker = dir.join("sigterm");
11829 let spec = ModuleSpec {
11830 module_id: module_id.to_string(),
11831 program: fake_aft_stub_path(),
11832 args: Vec::new(),
11833 env: vec![
11834 ("FAKE_AFT_NEVER_CONNECT".to_string(), "1".to_string()),
11835 (
11836 "FAKE_AFT_SIGTERM_MARKER_PATH".to_string(),
11837 marker.display().to_string(),
11838 ),
11839 (
11840 "FAKE_AFT_NEVER_CONNECT_READY_PATH".to_string(),
11841 ready.display().to_string(),
11842 ),
11843 ],
11844 reserved: false,
11845 reserved_prefixes: Vec::new(),
11846 protocol: ModuleProtocol::None,
11847 overlap: Default::default(),
11848 };
11849 (spec, ready, marker)
11850 }
11851
11852 #[cfg(unix)]
11856 async fn wait_for_file(path: &std::path::Path) {
11857 let deadline = Instant::now() + Duration::from_secs(10);
11858 while !path.exists() {
11859 assert!(
11860 Instant::now() < deadline,
11861 "{} never appeared",
11862 path.display()
11863 );
11864 sleep(Duration::from_millis(10)).await;
11865 }
11866 }
11867
11868 #[cfg(unix)]
11872 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
11873 async fn protocol_none_unrequested_clean_exit_restarts_as_a_crash() {
11874 let dir = cortexkit_test_support::ScratchDir::new("none-unrequested-clean-exit");
11875 let (spec, ready, marker) =
11876 protocol_none_sigterm_exits_clean_spec("none-unrequested-clean-exit", dir.path());
11877 let supervisor = Supervisor::new_for_test(
11878 Arc::new(Registry::default()),
11879 RestartPolicy::new(3, Duration::ZERO),
11880 );
11881 let module = supervisor.spawn(spec).unwrap();
11882 wait_for_file(&ready).await;
11883 let deadline = Instant::now() + Duration::from_secs(10);
11887 let first_pid = loop {
11888 let status = module.status().unwrap();
11889 if status.state == ModuleState::Running {
11890 if let Some(pid) = status.pid {
11891 break pid;
11892 }
11893 }
11894 assert!(
11895 Instant::now() < deadline,
11896 "a running module must report its pid after exec confirmation: {status:?}"
11897 );
11898 sleep(Duration::from_millis(10)).await;
11899 };
11900
11901 rustix::process::kill_process(
11902 rustix::process::Pid::from_raw(i32::try_from(first_pid).unwrap()).unwrap(),
11903 rustix::process::Signal::TERM,
11904 )
11905 .unwrap();
11906
11907 let deadline = Instant::now() + Duration::from_secs(10);
11908 let respawned = loop {
11909 let status = module.status().unwrap();
11910 if status.state == ModuleState::Running
11911 && status.pid.is_some_and(|pid| pid != first_pid)
11912 {
11913 break status;
11914 }
11915 assert!(
11916 Instant::now() < deadline,
11917 "protocol-none module was not respawned after an unrequested clean exit: {status:?}"
11918 );
11919 sleep(Duration::from_millis(10)).await;
11920 };
11921 assert_eq!(respawned.spawn_generation, 2);
11922 assert!(
11923 marker.exists(),
11924 "the child must have exited through its SIGTERM handler (exit 0), or this proves nothing about clean exits"
11925 );
11926
11927 let history = module.terminal_history();
11928 assert_eq!(history.entries.len(), 1, "{history:?}");
11929 let entry = &history.entries[0];
11930 assert_eq!(entry.exit_code, Some(0));
11931 assert_eq!(entry.exit_kind, subc_control::TerminalExitKind::Clean);
11932 assert_eq!(entry.disposition, TerminalDisposition::Restarting);
11933
11934 module.stop().await.unwrap();
11935 }
11936
11937 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
11941 async fn protocol_none_repeated_clean_exits_exhaust_the_restart_budget() {
11942 let supervisor = Supervisor::new_for_test(
11943 Arc::new(Registry::default()),
11944 RestartPolicy::new(1, Duration::ZERO),
11945 );
11946 let module = supervisor
11947 .spawn(ModuleSpec {
11948 module_id: "none-clean-exit-budget".to_string(),
11949 program: fake_aft_stub_path(),
11950 args: Vec::new(),
11951 env: vec![("FAKE_AFT_EXIT_CODE".to_string(), "0".to_string())],
11952 reserved: false,
11953 reserved_prefixes: Vec::new(),
11954 protocol: ModuleProtocol::None,
11955 overlap: Default::default(),
11956 })
11957 .unwrap();
11958
11959 let deadline = Instant::now() + Duration::from_secs(10);
11962 loop {
11963 let status = module.status().unwrap();
11964 if status.state == ModuleState::Failed {
11965 break;
11966 }
11967 assert!(
11968 Instant::now() < deadline,
11969 "module never exhausted its budget: {status:?} {:?}",
11970 module.terminal_history()
11971 );
11972 sleep(Duration::from_millis(10)).await;
11973 }
11974 let history = module.terminal_history();
11975 assert_eq!(
11976 history
11977 .entries
11978 .iter()
11979 .map(|entry| (entry.exit_code, entry.disposition.clone()))
11980 .collect::<Vec<_>>(),
11981 vec![
11982 (Some(0), TerminalDisposition::Restarting),
11983 (Some(0), TerminalDisposition::Failed),
11984 ]
11985 );
11986 let detail = history.entries[1]
11987 .disposition_detail
11988 .as_deref()
11989 .expect("a budget failure names the budget");
11990 assert!(detail.contains("max_restarts=1"), "{detail}");
11991 assert_eq!(module.status().unwrap().spawn_generation, 2);
11992 }
11993
11994 #[test]
11995 fn restart_budget_failure_is_published_after_its_terminal_record() {
11996 use std::io::{Read, Write};
11997 let supervisor = Supervisor::new_for_test(
11998 Arc::new(Registry::default()),
11999 RestartPolicy::new(0, Duration::ZERO),
12000 );
12001 let runtime = supervisor.runtime_config();
12002 let spec = ModuleSpec {
12003 protocol: ModuleProtocol::None,
12004 ..windowed_crash_spec("budget-publication-order")
12005 };
12006 let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::new(
12007 ModuleState::Running,
12008 true,
12009 )));
12010 lock_snapshot(&snapshot).unwrap().process_alive = true;
12011 let reaped_snapshot = snapshot.clone();
12012 let mut barrier = terminal_record_barrier(&runtime.spawn_events);
12013 let history = runtime.terminal_ring.clone();
12014 let reap = std::thread::spawn(move || {
12015 tokio::runtime::Builder::new_current_thread()
12016 .enable_all()
12017 .build()
12018 .unwrap()
12019 .block_on(on_child_exit(
12020 &spec,
12021 runtime.restart_policy,
12022 &supervisor.registry,
12023 &reaped_snapshot,
12024 &runtime.terminal_ring,
12025 &runtime.spawn_events,
12026 &runtime.child_roster,
12027 ExitReport {
12028 kind: ExitKind::Clean,
12029 code: Some(0),
12030 signal: None,
12031 at_ms: 1,
12032 },
12033 ))
12034 });
12035 let mut reached = [0];
12036 barrier.read_exact(&mut reached).unwrap();
12037 assert_eq!(&reached, b"R");
12038 let (before_state, before_alive) = {
12039 let state = lock_snapshot(&snapshot).unwrap();
12040 (state.state, state.process_alive)
12041 };
12042 let before_history = history.lock().unwrap().snapshot();
12043 barrier.write_all(b"X").unwrap();
12044 assert!(matches!(reap.join().unwrap(), NextAction::Stop { .. }));
12045 assert!(before_history.entries.is_empty());
12046 assert_ne!(
12047 before_state,
12048 ModuleState::Failed,
12049 "Failed was visible before its terminal record could be written"
12050 );
12051 assert!(before_alive);
12052 assert_eq!(lock_snapshot(&snapshot).unwrap().state, ModuleState::Failed);
12053 let history = history.lock().unwrap().snapshot();
12054 assert_eq!(history.entries.len(), 1);
12055 assert_eq!(history.entries[0].exit_code, Some(0));
12056 assert_eq!(history.entries[0].disposition, TerminalDisposition::Failed);
12057 }
12058
12059 #[test]
12060 fn crash_restart_at_budget_limit_remains_recoverable_while_recording() {
12061 use std::io::{Read, Write};
12062
12063 let supervisor = Supervisor::new_for_test(
12064 Arc::new(Registry::default()),
12065 RestartPolicy::new(3, Duration::ZERO),
12066 );
12067 let mut runtime = supervisor.runtime_config();
12068 let module = module_with_recovery_snapshot(ModuleState::Running, true, 2);
12069 let snapshot = module.inner.snapshot.clone();
12070 lock_snapshot(&snapshot).unwrap().process_alive = true;
12071 runtime.terminal_ring = module.inner.terminal_ring.clone();
12072 let spec = windowed_crash_spec(module.module_id());
12073 let mut barrier = terminal_record_barrier(&runtime.spawn_events);
12074 let reaped_snapshot = snapshot.clone();
12075 let reap = std::thread::spawn(move || {
12076 tokio::runtime::Builder::new_current_thread()
12077 .enable_all()
12078 .build()
12079 .unwrap()
12080 .block_on(on_child_exit(
12081 &spec,
12082 runtime.restart_policy,
12083 &supervisor.registry,
12084 &reaped_snapshot,
12085 &runtime.terminal_ring,
12086 &runtime.spawn_events,
12087 &runtime.child_roster,
12088 ExitReport {
12089 kind: ExitKind::Crash,
12090 code: Some(2),
12091 signal: None,
12092 at_ms: 1,
12093 },
12094 ))
12095 });
12096
12097 let mut reached = [0];
12101 barrier.read_exact(&mut reached).unwrap();
12102 assert_eq!(&reached, b"R");
12103 let before_status = module.status().unwrap();
12104 let before_history = module.terminal_history();
12105 let before_recovers = module.will_recover_after_connection_loss().unwrap();
12106 barrier.write_all(b"X").unwrap();
12107 assert!(matches!(reap.join().unwrap(), NextAction::Restart { .. }));
12108
12109 assert_eq!(before_status.state, ModuleState::Running);
12110 assert!(before_status.process_alive);
12111 assert_eq!(before_status.restart_count, 3);
12112 assert!(before_history.entries.is_empty());
12113 assert!(
12114 before_recovers,
12115 "connection cleanup would report a terminal close for a scheduled crash restart"
12116 );
12117 let restarting = module.status().unwrap();
12118 assert_eq!(restarting.state, ModuleState::Restarting);
12119 assert_eq!(restarting.restart_count, 3);
12120 assert!(module.will_recover_after_connection_loss().unwrap());
12121 let history = module.terminal_history();
12122 assert_eq!(history.entries.len(), 1);
12123 assert_eq!(
12124 history.entries[0].disposition,
12125 TerminalDisposition::Restarting
12126 );
12127
12128 update_snapshot(&snapshot, Some(module.module_id()), |state| {
12131 state.state = ModuleState::Running;
12132 state.respawn_pending = false;
12133 })
12134 .unwrap();
12135 assert!(!module.will_recover_after_connection_loss().unwrap());
12136 update_snapshot(&snapshot, Some(module.module_id()), |state| {
12137 state.enabled = false;
12138 state.respawn_pending = true;
12139 })
12140 .unwrap();
12141 assert!(!module.will_recover_after_connection_loss().unwrap());
12142 }
12143
12144 fn terminal_record_barrier(events: &super::SpawnEventFeed) -> std::net::TcpStream {
12145 use std::net::{TcpListener, TcpStream};
12146 let listener = TcpListener::bind("127.0.0.1:0").unwrap();
12147 let writer = TcpStream::connect(listener.local_addr().unwrap()).unwrap();
12148 let (barrier, _) = listener.accept().unwrap();
12149 for stream in [&writer, &barrier] {
12150 stream
12151 .set_read_timeout(Some(Duration::from_secs(10)))
12152 .unwrap();
12153 stream
12154 .set_write_timeout(Some(Duration::from_secs(10)))
12155 .unwrap();
12156 }
12157 *events.1.lock().unwrap() = Some(writer);
12158 barrier
12159 }
12160
12161 pub(super) fn before_terminal_record(events: &super::SpawnEventFeed) {
12162 use std::io::{Read, Write};
12163 let barrier = events.1.lock().unwrap().take();
12164 if let Some(mut barrier) = barrier {
12165 barrier.write_all(b"R").unwrap();
12166 let mut release = [0];
12167 barrier.read_exact(&mut release).unwrap();
12168 assert_eq!(&release, b"X");
12169 }
12170 }
12171
12172 #[test]
12173 fn health_restart_budget_failure_is_published_after_its_terminal_record() {
12174 use std::io::{Read, Write};
12175
12176 let supervisor = Supervisor::new_for_test(
12177 Arc::new(Registry::default()),
12178 RestartPolicy::new(0, Duration::ZERO),
12179 );
12180 let mut runtime = supervisor.runtime_config();
12181 let module = module_with_recovery_snapshot(ModuleState::Running, true, 0);
12182 let snapshot = module.inner.snapshot.clone();
12183 runtime.terminal_ring = module.inner.terminal_ring.clone();
12184 let spec = ModuleSpec {
12185 module_id: module.module_id().into(),
12186 program: fake_aft_stub_path(),
12187 protocol: ModuleProtocol::None,
12188 env: vec![("FAKE_AFT_EXIT_CODE".into(), "17".into())],
12189 ..windowed_crash_spec(module.module_id())
12190 };
12191 let mut barrier = terminal_record_barrier(&runtime.spawn_events);
12192 let reap = std::thread::spawn(move || {
12193 tokio::runtime::Builder::new_current_thread()
12194 .enable_all()
12195 .build()
12196 .unwrap()
12197 .block_on(async {
12198 let mut child =
12199 Some(spawn_and_mark_running(&spec, &runtime, &snapshot).unwrap());
12200 health_restart_child(
12201 &spec,
12202 &runtime,
12203 &supervisor.registry,
12204 &supervisor.process_liveness,
12205 &snapshot,
12206 &mut child,
12207 SupervisorHealthStatus::Failing,
12208 Some("health probe did not answer"),
12209 1,
12210 )
12211 .await
12212 .unwrap();
12213 assert!(child.is_none());
12214 });
12215 });
12216 let mut reached = [0];
12221 barrier.read_exact(&mut reached).unwrap();
12222 assert_eq!(&reached, b"R");
12223 let before_status = module.status().unwrap();
12224 let before_history = module.terminal_history();
12225 barrier.write_all(b"X").unwrap();
12226 reap.join().unwrap();
12227 assert!(before_history.entries.is_empty());
12228 assert_ne!(
12229 before_status.state,
12230 ModuleState::Failed,
12231 "Failed was visible before its terminal record could be written"
12232 );
12233 assert!(
12234 before_status.process_alive,
12235 "process death was visible before its terminal record could be written"
12236 );
12237 let failed = module.status().unwrap();
12238 assert_eq!(failed.state, ModuleState::Failed);
12239 assert!(!failed.process_alive);
12240 let history = module.terminal_history();
12241 assert_eq!(history.entries.len(), 1);
12242 assert_eq!(history.entries[0].disposition, TerminalDisposition::Failed);
12243 assert_eq!(
12244 history.entries[0].disposition_detail.as_deref(),
12245 Some("crash budget exhausted: max_restarts=0 within window_secs=600")
12246 );
12247 }
12248
12249 #[test]
12250 fn observed_exits_publish_process_death_after_terminal_history() {
12251 use std::io::{Read, Write};
12252 for (kind, enabled, shutdown, wait_error, expected_state) in [
12253 (ExitKind::Clean, true, false, false, ModuleState::Stopped),
12254 (ExitKind::Crash, true, false, false, ModuleState::Restarting),
12255 (ExitKind::Crash, false, false, false, ModuleState::Disabled),
12256 (
12257 ExitKind::DeliberateSeverance,
12258 true,
12259 false,
12260 false,
12261 ModuleState::Restarting,
12262 ),
12263 #[cfg(unix)]
12264 (ExitKind::Crash, true, true, false, ModuleState::Stopped),
12265 (ExitKind::Crash, true, false, true, ModuleState::Failed),
12266 ] {
12267 let supervisor = Supervisor::new_for_test(
12268 Arc::new(Registry::default()),
12269 RestartPolicy::new(1, Duration::ZERO),
12270 );
12271 let runtime = supervisor.runtime_config();
12272 #[cfg(unix)]
12273 if shutdown {
12274 runtime.child_roster.close();
12275 }
12276 #[cfg(not(unix))]
12277 let _ = shutdown;
12278 let spec = windowed_crash_spec("exit-publication-order");
12279 let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::new(
12280 ModuleState::Running,
12281 enabled,
12282 )));
12283 lock_snapshot(&snapshot).unwrap().process_alive = true;
12284 let reaped_snapshot = snapshot.clone();
12285 let history = runtime.terminal_ring.clone();
12286 let mut barrier = terminal_record_barrier(&runtime.spawn_events);
12287 let reap = std::thread::spawn(move || {
12288 if wait_error {
12289 super::fail_after_wait_error(
12290 &spec.module_id,
12291 &reaped_snapshot,
12292 &runtime.terminal_ring,
12293 &runtime.spawn_events,
12294 "wait failed".into(),
12295 );
12296 } else {
12297 tokio::runtime::Builder::new_current_thread()
12298 .enable_all()
12299 .build()
12300 .unwrap()
12301 .block_on(on_child_exit(
12302 &spec,
12303 runtime.restart_policy,
12304 &supervisor.registry,
12305 &reaped_snapshot,
12306 &runtime.terminal_ring,
12307 &runtime.spawn_events,
12308 &runtime.child_roster,
12309 ExitReport {
12310 kind,
12311 code: Some(0),
12312 signal: None,
12313 at_ms: 1,
12314 },
12315 ));
12316 }
12317 });
12318 let mut reached = [0];
12319 barrier.read_exact(&mut reached).unwrap();
12320 assert_eq!(&reached, b"R");
12321 let before_alive = lock_snapshot(&snapshot).unwrap().process_alive;
12322 let before_history = history.lock().unwrap().snapshot();
12323 barrier.write_all(b"X").unwrap();
12324 reap.join().unwrap();
12325 assert!(
12326 before_alive,
12327 "{kind:?}: death was published before recording"
12328 );
12329 assert!(before_history.entries.is_empty());
12330 assert_eq!(lock_snapshot(&snapshot).unwrap().state, expected_state);
12331 assert!(!lock_snapshot(&snapshot).unwrap().process_alive);
12332 assert_eq!(history.lock().unwrap().snapshot().entries.len(), 1);
12333 }
12334 }
12335
12336 #[tokio::test]
12337 async fn health_restart_budget_failure_remains_failed_when_journal_append_fails() {
12338 let dir = cortexkit_test_support::ScratchDir::new("health-budget-journal-failure");
12339 let path = dir.join("terminals.jsonl");
12340 std::fs::create_dir(&path).unwrap();
12341 let supervisor = Supervisor::new_for_test(
12342 Arc::new(Registry::default()),
12343 RestartPolicy::new(0, Duration::ZERO),
12344 )
12345 .with_terminal_journal(path, "health-budget-journal-failure".into());
12346 let runtime = supervisor.runtime_config();
12347 let spec = ModuleSpec {
12348 program: fake_aft_stub_path(),
12349 protocol: ModuleProtocol::None,
12350 env: vec![("FAKE_AFT_EXIT_CODE".into(), "17".into())],
12351 ..windowed_crash_spec("health-budget-journal-failure")
12352 };
12353 let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::starting()));
12354 let mut child = Some(spawn_and_mark_running(&spec, &runtime, &snapshot).unwrap());
12355 health_restart_child(
12356 &spec,
12357 &runtime,
12358 &supervisor.registry,
12359 &supervisor.process_liveness,
12360 &snapshot,
12361 &mut child,
12362 SupervisorHealthStatus::Failing,
12363 None,
12364 1,
12365 )
12366 .await
12367 .unwrap();
12368 assert_eq!(lock_snapshot(&snapshot).unwrap().state, ModuleState::Failed);
12369 assert!(!lock_snapshot(&snapshot).unwrap().process_alive);
12370 let history = runtime
12371 .terminal_ring
12372 .lock()
12373 .unwrap()
12374 .durable_history(&spec.module_id);
12375 assert!(history.journal_write_failures > 0);
12376 assert_eq!(history.entries.len(), 1);
12377 assert_eq!(history.entries[0].disposition, TerminalDisposition::Failed);
12378 }
12379
12380 #[test]
12381 fn health_restart_budget_failure_remains_failed_when_exit_recording_panics() {
12382 let supervisor = Supervisor::new_for_test(
12383 Arc::new(Registry::default()),
12384 RestartPolicy::new(0, Duration::ZERO),
12385 );
12386 let runtime = supervisor.runtime_config();
12387 let spec = ModuleSpec {
12388 program: fake_aft_stub_path(),
12389 protocol: ModuleProtocol::None,
12390 env: vec![("FAKE_AFT_EXIT_CODE".into(), "17".into())],
12391 ..windowed_crash_spec("health-budget-recording-panic")
12392 };
12393 let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::starting()));
12394 let result = std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| {
12395 tokio::runtime::Builder::new_current_thread()
12396 .enable_all()
12397 .build()
12398 .unwrap()
12399 .block_on(async {
12400 let mut child =
12401 Some(spawn_and_mark_running(&spec, &runtime, &snapshot).unwrap());
12402 runtime.spawn_events.0.lock().unwrap().seq = u64::MAX;
12403 health_restart_child(
12404 &spec,
12405 &runtime,
12406 &supervisor.registry,
12407 &supervisor.process_liveness,
12408 &snapshot,
12409 &mut child,
12410 SupervisorHealthStatus::Failing,
12411 None,
12412 1,
12413 )
12414 .await
12415 .unwrap();
12416 });
12417 }));
12418 let panic = result.expect_err("recording must still unwind");
12419 assert_eq!(
12420 panic.downcast_ref::<String>().map(String::as_str),
12421 Some("spawn event sequence exhausted")
12422 );
12423 let state = lock_snapshot(&snapshot).unwrap();
12424 assert_eq!(state.state, ModuleState::Failed);
12425 assert!(!state.process_alive);
12426 assert!(state.last_exit.is_some());
12427 }
12428
12429 #[tokio::test]
12430 async fn restart_budget_failure_remains_failed_when_journal_append_fails() {
12431 let dir = cortexkit_test_support::ScratchDir::new("budget-journal-failure");
12432 let path = dir.join("terminals.jsonl");
12433 std::fs::create_dir(&path).unwrap();
12434 let supervisor = Supervisor::new_for_test(
12435 Arc::new(Registry::default()),
12436 RestartPolicy::new(0, Duration::ZERO),
12437 )
12438 .with_terminal_journal(path, "budget-journal-failure".into());
12439 let runtime = supervisor.runtime_config();
12440 let spec = windowed_crash_spec("budget-journal-failure");
12441 let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::starting()));
12442 assert!(matches!(
12443 on_child_exit(
12444 &spec,
12445 runtime.restart_policy,
12446 &supervisor.registry,
12447 &snapshot,
12448 &runtime.terminal_ring,
12449 &runtime.spawn_events,
12450 &runtime.child_roster,
12451 crash_exit_report(1),
12452 )
12453 .await,
12454 NextAction::Stop { .. }
12455 ));
12456 assert_eq!(lock_snapshot(&snapshot).unwrap().state, ModuleState::Failed);
12457 let history = runtime
12458 .terminal_ring
12459 .lock()
12460 .unwrap()
12461 .durable_history(&spec.module_id);
12462 assert!(history.journal_write_failures > 0);
12463 assert_eq!(history.entries.len(), 1);
12464 assert_eq!(history.entries[0].disposition, TerminalDisposition::Failed);
12465 }
12466
12467 #[test]
12468 fn restart_budget_failure_remains_failed_when_exit_recording_panics() {
12469 let supervisor = Supervisor::new_for_test(
12470 Arc::new(Registry::default()),
12471 RestartPolicy::new(0, Duration::ZERO),
12472 );
12473 let runtime = supervisor.runtime_config();
12474 let spec = windowed_crash_spec("budget-recording-panic");
12475 let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::starting()));
12476 runtime.spawn_events.emit_spawned(&spec.module_id, 1, 1);
12477 runtime.spawn_events.0.lock().unwrap().seq = u64::MAX;
12480 let result = std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| {
12481 tokio::runtime::Builder::new_current_thread()
12482 .enable_all()
12483 .build()
12484 .unwrap()
12485 .block_on(on_child_exit(
12486 &spec,
12487 runtime.restart_policy,
12488 &supervisor.registry,
12489 &snapshot,
12490 &runtime.terminal_ring,
12491 &runtime.spawn_events,
12492 &runtime.child_roster,
12493 crash_exit_report(1),
12494 ))
12495 }));
12496 let panic = result.err().expect("recording must still unwind");
12497 assert_eq!(
12498 panic.downcast_ref::<String>().map(String::as_str),
12499 Some("spawn event sequence exhausted")
12500 );
12501 assert_eq!(lock_snapshot(&snapshot).unwrap().state, ModuleState::Failed);
12502 }
12503
12504 #[cfg(unix)]
12507 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
12508 async fn protocol_none_requested_stop_and_disable_do_not_respawn() {
12509 for disable in [false, true] {
12510 let label = if disable {
12511 "none-requested-disable"
12512 } else {
12513 "none-requested-stop"
12514 };
12515 let dir = cortexkit_test_support::ScratchDir::new(label);
12516 let (spec, ready, marker) = protocol_none_sigterm_exits_clean_spec(label, dir.path());
12517 let supervisor = Supervisor::new_for_test(
12518 Arc::new(Registry::default()),
12519 RestartPolicy::new(3, Duration::ZERO),
12520 );
12521 let module = supervisor.spawn(spec).unwrap();
12522 wait_for_file(&ready).await;
12523
12524 if disable {
12525 module.set_enabled(false).await.unwrap();
12526 } else {
12527 module.stop().await.unwrap();
12528 }
12529 assert!(
12530 marker.exists(),
12531 "{label}: the child must have left through its SIGTERM handler with exit 0"
12532 );
12533
12534 sleep(Duration::from_millis(500)).await;
12537 let status = module.status().unwrap();
12538 let expected = if disable {
12539 ModuleState::Disabled
12540 } else {
12541 ModuleState::Stopped
12542 };
12543 assert_eq!(status.state, expected, "{label}");
12544 assert_eq!(
12545 status.spawn_generation, 1,
12546 "{label}: respawned after a requested stop"
12547 );
12548 let history = module.terminal_history();
12549 assert_eq!(history.entries.len(), 1, "{label}: {history:?}");
12550 assert_eq!(history.entries[0].exit_code, Some(0), "{label}");
12551 assert_ne!(
12552 history.entries[0].disposition,
12553 TerminalDisposition::Restarting,
12554 "{label}"
12555 );
12556 }
12557 }
12558
12559 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
12562 async fn subc_wire_clean_exit_is_still_a_stop() {
12563 let supervisor = Supervisor::new_for_test(
12564 Arc::new(Registry::default()),
12565 RestartPolicy::new(3, Duration::ZERO),
12566 );
12567 let module = supervisor
12568 .spawn(ModuleSpec {
12569 module_id: "wire-clean-exit".to_string(),
12570 program: fake_aft_stub_path(),
12571 args: Vec::new(),
12572 env: vec![("FAKE_AFT_EXIT_CODE".to_string(), "0".to_string())],
12573 reserved: false,
12574 reserved_prefixes: Vec::new(),
12575 protocol: ModuleProtocol::Subc,
12576 overlap: Default::default(),
12577 })
12578 .unwrap();
12579
12580 let deadline = Instant::now() + Duration::from_secs(10);
12581 while module.terminal_history().entries.is_empty() {
12582 assert!(Instant::now() < deadline, "module never exited");
12583 sleep(Duration::from_millis(10)).await;
12584 }
12585 sleep(Duration::from_millis(500)).await;
12587 let status = module.status().unwrap();
12588 assert_eq!(status.state, ModuleState::Stopped);
12589 assert_eq!(status.spawn_generation, 1);
12590 let history = module.terminal_history();
12591 assert_eq!(history.entries.len(), 1, "{history:?}");
12592 assert_eq!(history.entries[0].exit_code, Some(0));
12593 assert_eq!(history.entries[0].disposition, TerminalDisposition::Stopped);
12594 }
12595
12596 #[cfg(unix)]
12597 #[tokio::test]
12598 async fn reload_exit_keeps_roster_until_terminal_is_recorded_during_shutdown() {
12599 let dir = cortexkit_test_support::ScratchDir::new("reload-roster-terminal-order");
12600 let record = dir.join("live-children.json");
12601 let supervisor = Supervisor::new_for_test(
12602 Arc::new(Registry::default()),
12603 RestartPolicy::new(0, Duration::ZERO),
12604 );
12605 let mut runtime = supervisor.runtime_config();
12606 runtime.child_roster.record_to(record.clone());
12607 let gate = Arc::new(super::ReloadExitRecordGate::default());
12608 runtime.test_reload_exit_record_gate = Some(Arc::clone(&gate));
12609 let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::starting()));
12610 let spec = ModuleSpec {
12611 module_id: "reload-exit-roster".into(),
12612 program: fake_aft_stub_path(),
12613 args: Vec::new(),
12614 env: vec![("FAKE_AFT_EXIT_CODE".into(), "23".into())],
12615 reserved: false,
12616 reserved_prefixes: Vec::new(),
12617 protocol: ModuleProtocol::Subc,
12618 overlap: Default::default(),
12619 };
12620 let mut child = None;
12621 let reload = super::finish_reload_child(
12622 &spec,
12623 &runtime,
12624 &supervisor.registry,
12625 &supervisor.process_liveness,
12626 &snapshot,
12627 &mut child,
12628 );
12629 tokio::pin!(reload);
12630 tokio::select! {
12631 result = &mut reload => panic!("reload missed the before-record gate: {result:?}"),
12632 _ = gate.reached.notified() => {}
12633 }
12634 assert!(runtime
12635 .terminal_ring
12636 .lock()
12637 .unwrap()
12638 .snapshot()
12639 .entries
12640 .is_empty());
12641 assert_eq!(
12642 crate::live_children::read_record(&record).unwrap().len(),
12643 1,
12644 "shutdown must still wait for the reaped child until its terminal record exists"
12645 );
12646 runtime.child_roster.close();
12647 gate.resume.notify_one();
12648 assert!(reload.await.is_err());
12649 assert!(crate::live_children::read_record(&record)
12650 .unwrap()
12651 .is_empty());
12652 let history = runtime.terminal_ring.lock().unwrap().snapshot();
12653 assert_eq!(history.entries.len(), 1);
12654 assert_eq!(
12655 history.entries[0].disposition,
12656 TerminalDisposition::DaemonShutdown
12657 );
12658 }
12659
12660 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
12664 async fn every_restart_increment_path_advances_lifetime_count() {
12665 let supervisor = Supervisor::new_for_test(
12666 Arc::new(Registry::default()),
12667 RestartPolicy::new(1, Duration::ZERO),
12668 );
12669 let runtime = supervisor.runtime_config();
12670 let spec = ModuleSpec {
12671 module_id: "lifetime-increment-path".to_string(),
12672 program: PathBuf::from("/unused/lifetime-increment-path"),
12673 args: Vec::new(),
12674 env: Vec::new(),
12675 reserved: false,
12676 reserved_prefixes: Vec::new(),
12677 protocol: ModuleProtocol::Subc,
12678 overlap: Default::default(),
12679 };
12680
12681 let crash_snapshot = Arc::new(Mutex::new(SupervisorSnapshot::starting()));
12682 assert!(matches!(
12683 on_child_exit(
12684 &spec,
12685 runtime.restart_policy,
12686 &supervisor.registry,
12687 &crash_snapshot,
12688 &runtime.terminal_ring,
12689 &runtime.spawn_events,
12690 &runtime.child_roster,
12691 ExitReport {
12692 kind: ExitKind::Crash,
12693 code: Some(1),
12694 signal: None,
12695 at_ms: 1,
12696 },
12697 )
12698 .await,
12699 NextAction::Restart { schedule: _ }
12700 ));
12701 let (crash_restarts, crash_lifetime) = {
12702 let state = lock_snapshot(&crash_snapshot).unwrap();
12703 (state.crash_restarts.len(), state.lifetime_restarts)
12704 };
12705 assert_eq!(crash_restarts, 1);
12706 assert_eq!(crash_lifetime, 1);
12707
12708 let health_snapshot = Arc::new(Mutex::new(SupervisorSnapshot::starting()));
12709 let mut health_child = None;
12710 assert!(matches!(
12711 health_restart_child(
12712 &spec,
12713 &runtime,
12714 &supervisor.registry,
12715 &supervisor.process_liveness,
12716 &health_snapshot,
12717 &mut health_child,
12718 SupervisorHealthStatus::Failing,
12719 None,
12720 2,
12721 )
12722 .await,
12723 Ok(())
12724 ));
12725 assert!(health_child.is_none());
12726 assert!(lock_snapshot(&health_snapshot).unwrap().respawn_pending);
12727 assert!(runtime.scheduled_respawn.lock().unwrap().take().is_some());
12728 let (health_restarts, health_lifetime) = {
12729 let state = lock_snapshot(&health_snapshot).unwrap();
12730 (state.crash_restarts.len(), state.lifetime_restarts)
12731 };
12732 assert_eq!(health_restarts, 1);
12733 assert_eq!(health_lifetime, 1);
12734
12735 let reload_snapshot = Arc::new(Mutex::new(SupervisorSnapshot::starting()));
12736 let mut reload_child = None;
12737 assert!(matches!(
12738 handle_reload_spawn_failure(
12739 &spec,
12740 &runtime,
12741 &supervisor.process_liveness,
12742 &reload_snapshot,
12743 &mut reload_child,
12744 "forced reload spawn failure".to_string(),
12745 )
12746 .await,
12747 Err(SuperviseError::ReloadFailed { .. })
12748 ));
12749 let (reload_restarts, reload_lifetime) = {
12750 let state = lock_snapshot(&reload_snapshot).unwrap();
12751 (state.crash_restarts.len(), state.lifetime_restarts)
12752 };
12753 assert_eq!(reload_restarts, 1);
12754 assert_eq!(reload_lifetime, 1);
12755 }
12756
12757 #[tokio::test]
12758 async fn deliberately_severed_live_child_records_lifetime_without_spending_restart_budget() {
12759 let supervisor = Supervisor::new_for_test(
12760 Arc::new(Registry::default()),
12761 RestartPolicy::new(3, Duration::ZERO),
12762 );
12763 let runtime = supervisor.runtime_config();
12764 let spec = ModuleSpec {
12765 module_id: "deliberately-severed".to_string(),
12766 program: PathBuf::from("/unused/deliberately-severed"),
12767 args: Vec::new(),
12768 env: Vec::new(),
12769 reserved: false,
12770 reserved_prefixes: Vec::new(),
12771 protocol: ModuleProtocol::Subc,
12772 overlap: Default::default(),
12773 };
12774 let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::starting()));
12775 let process = ProcessIdentity {
12776 pid: 41,
12777 start_time: 101,
12778 };
12779 record_deliberate_severance(&snapshot, process).unwrap();
12780 let exit_report = apply_deliberate_severance_marker(
12781 &snapshot,
12782 Some(process),
12783 ExitReport {
12784 kind: ExitKind::Crash,
12785 code: Some(1),
12786 signal: None,
12787 at_ms: 1,
12788 },
12789 );
12790 assert_eq!(exit_report.kind, ExitKind::DeliberateSeverance);
12791
12792 assert!(matches!(
12793 on_child_exit(
12794 &spec,
12795 runtime.restart_policy,
12796 &supervisor.registry,
12797 &snapshot,
12798 &runtime.terminal_ring,
12799 &runtime.spawn_events,
12800 &runtime.child_roster,
12801 exit_report,
12802 )
12803 .await,
12804 NextAction::Restart { schedule: _ }
12805 ));
12806 let state = lock_snapshot(&snapshot).unwrap();
12807 assert_eq!(state.lifetime_restarts, 1);
12808 assert_eq!(state.crash_restarts.len(), 0);
12809 }
12810
12811 #[tokio::test]
12812 async fn genuine_crash_spends_restart_budget_and_records_lifetime() {
12813 let supervisor = Supervisor::new_for_test(
12814 Arc::new(Registry::default()),
12815 RestartPolicy::new(3, Duration::ZERO),
12816 );
12817 let runtime = supervisor.runtime_config();
12818 let spec = ModuleSpec {
12819 module_id: "genuine-crash".to_string(),
12820 program: PathBuf::from("/unused/genuine-crash"),
12821 args: Vec::new(),
12822 env: Vec::new(),
12823 reserved: false,
12824 reserved_prefixes: Vec::new(),
12825 protocol: ModuleProtocol::Subc,
12826 overlap: Default::default(),
12827 };
12828 let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::starting()));
12829
12830 assert!(matches!(
12831 on_child_exit(
12832 &spec,
12833 runtime.restart_policy,
12834 &supervisor.registry,
12835 &snapshot,
12836 &runtime.terminal_ring,
12837 &runtime.spawn_events,
12838 &runtime.child_roster,
12839 ExitReport {
12840 kind: ExitKind::Crash,
12841 code: Some(1),
12842 signal: None,
12843 at_ms: 1,
12844 },
12845 )
12846 .await,
12847 NextAction::Restart { schedule: _ }
12848 ));
12849 let state = lock_snapshot(&snapshot).unwrap();
12850 assert_eq!(state.lifetime_restarts, 1);
12851 assert_eq!(state.crash_restarts.len(), 1);
12852 }
12853
12854 fn crash_exit_report(at_ms: u64) -> ExitReport {
12855 ExitReport {
12856 kind: ExitKind::Crash,
12857 code: Some(1),
12858 signal: None,
12859 at_ms,
12860 }
12861 }
12862
12863 fn windowed_crash_spec(module_id: &str) -> ModuleSpec {
12864 ModuleSpec {
12865 module_id: module_id.to_string(),
12866 program: PathBuf::from("/unused").join(module_id),
12867 args: Vec::new(),
12868 env: Vec::new(),
12869 reserved: false,
12870 reserved_prefixes: Vec::new(),
12871 protocol: ModuleProtocol::Subc,
12872 overlap: Default::default(),
12873 }
12874 }
12875
12876 #[tokio::test]
12882 async fn three_crashes_inside_the_window_stop_the_module_and_name_the_window() {
12883 let (logs, _guard) = crate::router::test_log::log_capture(tracing::Level::ERROR);
12884 let supervisor = Supervisor::new_for_test(
12885 Arc::new(Registry::default()),
12886 RestartPolicy::new(2, Duration::ZERO),
12887 );
12888 let runtime = supervisor.runtime_config();
12889 let spec = windowed_crash_spec("crash-loop-in-window");
12890 let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::starting()));
12891
12892 for attempt in 1..=2 {
12893 assert!(
12894 matches!(
12895 on_child_exit(
12896 &spec,
12897 runtime.restart_policy,
12898 &supervisor.registry,
12899 &snapshot,
12900 &runtime.terminal_ring,
12901 &runtime.spawn_events,
12902 &runtime.child_roster,
12903 crash_exit_report(attempt),
12904 )
12905 .await,
12906 NextAction::Restart { schedule: _ }
12907 ),
12908 "crash {attempt} is inside the budget and must respawn"
12909 );
12910 }
12911
12912 assert!(matches!(
12913 on_child_exit(
12914 &spec,
12915 runtime.restart_policy,
12916 &supervisor.registry,
12917 &snapshot,
12918 &runtime.terminal_ring,
12919 &runtime.spawn_events,
12920 &runtime.child_roster,
12921 crash_exit_report(3),
12922 )
12923 .await,
12924 NextAction::Stop { .. }
12925 ));
12926
12927 {
12928 let state = lock_snapshot(&snapshot).unwrap();
12929 assert_eq!(state.state, ModuleState::Failed);
12930 assert_eq!(state.crash_restarts.len(), 2);
12931 assert_eq!(state.lifetime_restarts, 2);
12932 }
12933
12934 let history = runtime
12935 .terminal_ring
12936 .lock()
12937 .expect("terminal ring is not poisoned")
12938 .snapshot();
12939 let last = history
12940 .entries
12941 .last()
12942 .expect("the refused crash is retained");
12943 assert_eq!(last.disposition, TerminalDisposition::Failed);
12944 assert_eq!(
12945 last.disposition_detail.as_deref(),
12946 Some("crash budget exhausted: max_restarts=2 within window_secs=600")
12947 );
12948
12949 let captured = crate::router::test_log::captured_logs(&logs);
12950 assert!(
12951 captured.contains("crash budget exhausted: max_restarts=2 within window_secs=600"),
12952 "the stop must be logged with its window: {captured}"
12953 );
12954 }
12955
12956 #[tokio::test]
12964 async fn a_crash_older_than_the_window_frees_its_slot_for_a_later_crash() {
12965 let supervisor = Supervisor::new_for_test(
12966 Arc::new(Registry::default()),
12967 RestartPolicy::new(2, Duration::ZERO),
12968 );
12969 let runtime = supervisor.runtime_config();
12970 let spec = windowed_crash_spec("crash-across-windows");
12971 let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::starting()));
12972
12973 for attempt in 1..=2 {
12974 assert!(matches!(
12975 on_child_exit(
12976 &spec,
12977 runtime.restart_policy,
12978 &supervisor.registry,
12979 &snapshot,
12980 &runtime.terminal_ring,
12981 &runtime.spawn_events,
12982 &runtime.child_roster,
12983 crash_exit_report(attempt),
12984 )
12985 .await,
12986 NextAction::Restart { schedule: _ }
12987 ));
12988 }
12989
12990 update_snapshot(&snapshot, Some(&spec.module_id), |state| {
12993 age_oldest_crash_restart_out_of_window(state, runtime.restart_policy.window);
12994 })
12995 .unwrap();
12996
12997 assert!(
12998 matches!(
12999 on_child_exit(
13000 &spec,
13001 runtime.restart_policy,
13002 &supervisor.registry,
13003 &snapshot,
13004 &runtime.terminal_ring,
13005 &runtime.spawn_events,
13006 &runtime.child_roster,
13007 crash_exit_report(3),
13008 )
13009 .await,
13010 NextAction::Restart { schedule: _ }
13011 ),
13012 "a crash older than the window must not hold a budget slot"
13013 );
13014
13015 let state = lock_snapshot(&snapshot).unwrap();
13016 assert_eq!(state.state, ModuleState::Restarting);
13017 assert_eq!(
13018 state.crash_restarts.len(),
13019 2,
13020 "the aged instant is dropped and the new one takes its place"
13021 );
13022 assert_eq!(
13023 state.lifetime_restarts, 3,
13024 "the ledger counts every restart, including the ones the window forgot"
13025 );
13026 }
13027
13028 #[tokio::test]
13033 async fn an_operator_restart_clears_the_ring_and_leaves_the_ledger_alone() {
13034 let supervisor = Supervisor::new_for_test(
13035 Arc::new(Registry::default()),
13036 RestartPolicy::new(2, Duration::ZERO),
13037 );
13038 let runtime = supervisor.runtime_config();
13039 let spec = windowed_crash_spec("operator-cleared-budget");
13040 let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::starting()));
13041
13042 for attempt in 1..=2 {
13043 assert!(matches!(
13044 on_child_exit(
13045 &spec,
13046 runtime.restart_policy,
13047 &supervisor.registry,
13048 &snapshot,
13049 &runtime.terminal_ring,
13050 &runtime.spawn_events,
13051 &runtime.child_roster,
13052 crash_exit_report(attempt),
13053 )
13054 .await,
13055 NextAction::Restart { schedule: _ }
13056 ));
13057 }
13058
13059 reset_restart_count(&snapshot, &spec.module_id).unwrap();
13060 {
13061 let state = lock_snapshot(&snapshot).unwrap();
13062 assert!(
13063 state.crash_restarts.is_empty(),
13064 "an operator restart returns the full budget"
13065 );
13066 assert_eq!(
13067 state.lifetime_restarts, 2,
13068 "clearing the budget must not unmake the crashes"
13069 );
13070 }
13071
13072 assert!(
13073 matches!(
13074 on_child_exit(
13075 &spec,
13076 runtime.restart_policy,
13077 &supervisor.registry,
13078 &snapshot,
13079 &runtime.terminal_ring,
13080 &runtime.spawn_events,
13081 &runtime.child_roster,
13082 crash_exit_report(3),
13083 )
13084 .await,
13085 NextAction::Restart { schedule: _ }
13086 ),
13087 "the cleared budget must be spendable again"
13088 );
13089 let state = lock_snapshot(&snapshot).unwrap();
13090 assert_eq!(state.crash_restarts.len(), 1);
13091 assert_eq!(state.lifetime_restarts, 3);
13092 }
13093
13094 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
13095 async fn severance_marker_for_a_dead_child_does_not_label_its_successor() {
13096 let severed = ProcessIdentity {
13097 pid: 41,
13098 start_time: 101,
13099 };
13100 let successor = ProcessIdentity {
13101 pid: 41,
13102 start_time: 202,
13103 };
13104 let module = module_with_recovery_snapshot(ModuleState::Running, true, 0);
13105 update_snapshot(&module.inner.snapshot, Some("recovery-snapshot"), |state| {
13106 state.pid = Some(successor.pid);
13107 state.process_start_time = Some(successor.start_time);
13108 })
13109 .unwrap();
13110 assert!(!module.record_deliberate_severance(severed).unwrap());
13111
13112 let exit_report = apply_deliberate_severance_marker(
13113 &module.inner.snapshot,
13114 Some(successor),
13115 ExitReport {
13116 kind: ExitKind::Crash,
13117 code: Some(1),
13118 signal: None,
13119 at_ms: 1,
13120 },
13121 );
13122
13123 assert_eq!(exit_report.kind, ExitKind::Crash);
13124 }
13125
13126 #[tokio::test]
13127 async fn drain_reap_marks_deliberate_severance_and_records_lifetime_without_budget() {
13128 let registry = Registry::default();
13129 let supervisor = Supervisor::new_for_test(
13130 Arc::new(Registry::default()),
13131 RestartPolicy::new(3, Duration::ZERO),
13132 );
13133 let runtime = supervisor.runtime_config();
13134 let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::starting()));
13135 let spec = ModuleSpec {
13136 module_id: "drain-deliberate-severance".to_string(),
13137 program: fake_aft_stub_path(),
13138 args: Vec::new(),
13139 env: vec![("FAKE_AFT_EXIT_CODE".to_string(), "23".to_string())],
13140 reserved: false,
13141 reserved_prefixes: Vec::new(),
13142 protocol: ModuleProtocol::Subc,
13143 overlap: Default::default(),
13144 };
13145 let mut child = spawn_and_mark_running(&spec, &runtime, &snapshot).unwrap();
13146 let process = ProcessIdentity {
13147 pid: 41,
13148 start_time: 101,
13149 };
13150 child.process_identity = Some(process);
13151 update_snapshot(&snapshot, Some(&spec.module_id), |state| {
13152 state.pid = Some(process.pid);
13153 state.process_start_time = Some(process.start_time);
13154 })
13155 .unwrap();
13156 record_deliberate_severance(&snapshot, process).unwrap();
13157
13158 drain_child_to_state(
13159 &spec.module_id,
13160 spec.protocol,
13161 StopNotice::SentOverConnection,
13164 ®istry,
13165 None,
13166 &snapshot,
13167 &runtime.terminal_ring,
13168 &runtime.spawn_events,
13169 child,
13170 Duration::from_secs(1),
13171 ModuleState::Stopped,
13172 Some(false),
13173 )
13174 .await
13175 .unwrap();
13176
13177 let state = lock_snapshot(&snapshot).unwrap();
13178 assert_eq!(
13179 state.last_exit.as_ref().map(|exit| exit.kind),
13180 Some(ExitKind::DeliberateSeverance)
13181 );
13182 assert_eq!(state.lifetime_restarts, 1);
13183 assert_eq!(state.crash_restarts.len(), 0);
13184 drop(state);
13185 let history = runtime.terminal_ring.lock().unwrap().snapshot();
13186 assert_eq!(
13187 history.entries[0].exit_kind,
13188 subc_control::TerminalExitKind::DeliberateSeverance
13189 );
13190 }
13191
13192 #[tokio::test]
13193 async fn ordinary_drain_reap_does_not_record_a_lifetime_restart() {
13194 let registry = Registry::default();
13195 let supervisor = Supervisor::new_for_test(
13196 Arc::new(Registry::default()),
13197 RestartPolicy::new(3, Duration::ZERO),
13198 );
13199 let runtime = supervisor.runtime_config();
13200 let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::starting()));
13201 let spec = ModuleSpec {
13202 module_id: "ordinary-drain".to_string(),
13203 program: fake_aft_stub_path(),
13204 args: Vec::new(),
13205 env: vec![("FAKE_AFT_EXIT_CODE".to_string(), "23".to_string())],
13206 reserved: false,
13207 reserved_prefixes: Vec::new(),
13208 protocol: ModuleProtocol::Subc,
13209 overlap: Default::default(),
13210 };
13211 let child = spawn_and_mark_running(&spec, &runtime, &snapshot).unwrap();
13212
13213 drain_child_to_state(
13214 &spec.module_id,
13215 spec.protocol,
13216 StopNotice::SentOverConnection,
13219 ®istry,
13220 None,
13221 &snapshot,
13222 &runtime.terminal_ring,
13223 &runtime.spawn_events,
13224 child,
13225 Duration::from_secs(1),
13226 ModuleState::Stopped,
13227 Some(false),
13228 )
13229 .await
13230 .unwrap();
13231
13232 let state = lock_snapshot(&snapshot).unwrap();
13233 assert_eq!(
13234 state.last_exit.as_ref().map(|exit| exit.kind),
13235 Some(ExitKind::Crash)
13236 );
13237 assert_eq!(state.lifetime_restarts, 0);
13238 assert_eq!(state.crash_restarts.len(), 0);
13239 }
13240
13241 #[test]
13242 fn fatal_connection_teardown_cannot_arm_a_marker_for_a_surviving_process() {
13243 assert!(!include_str!("server.rs")
13249 .contains("router.record_deliberate_connection_severance(ctx.connection_id)"));
13250 }
13251
13252 #[test]
13259 fn drained_after_quiescence_wait_passes_ok_through_and_forces_false_on_err() {
13260 assert!(drained_after_quiescence_wait(&Ok(true)));
13261 assert!(!drained_after_quiescence_wait(&Ok(false)));
13262 assert!(!drained_after_quiescence_wait(&Err(
13263 SuperviseError::StatePoisoned { module_id: None }
13264 )));
13265 }
13266
13267 #[test]
13276 fn wait_error_exit_report_records_a_failed_terminal_with_no_code_or_signal() {
13277 let ring = Arc::new(Mutex::new(TerminalRing::new(
13278 TerminalRingConfig::default(),
13279 0,
13280 )));
13281 record_wait_error_terminal("wait-error", &ring, &super::SpawnEventFeed::default());
13282
13283 let snapshot = ring.lock().unwrap().snapshot();
13284 assert_eq!(snapshot.entries.len(), 1);
13285 let entry = &snapshot.entries[0];
13286 assert_eq!(entry.exit_code, None);
13287 assert_eq!(entry.exit_signal, None);
13288 assert_eq!(entry.disposition, TerminalDisposition::Failed);
13289 }
13290
13291 #[test]
13292 fn wait_error_exit_path_preserves_spawn_event_density() {
13293 let feed = super::SpawnEventFeed::default();
13294 feed.configure_incarnation("wait-error-density".to_string());
13295 feed.emit_spawned("wait-error", 41, 1);
13296 let ring = Arc::new(Mutex::new(TerminalRing::new(
13297 TerminalRingConfig::default(),
13298 0,
13299 )));
13300
13301 record_wait_error_terminal("wait-error", &ring, &feed);
13302 feed.emit_spawned("after-wait-error", 42, 2);
13303
13304 let state = feed.0.lock().unwrap();
13305 let sequences = state
13306 .events
13307 .iter()
13308 .map(|event| event.cursor.seq)
13309 .collect::<Vec<_>>();
13310 assert_eq!(sequences, vec![1, 2, 3]);
13311 assert_eq!(state.events[1].kind, SpawnEventKind::Exited);
13312 assert_eq!(state.events[1].exit_code, None);
13313 assert_eq!(state.events[1].exit_signal, None);
13314 }
13315
13316 #[test]
13320 fn wait_error_exit_report_is_classified_as_a_crash() {
13321 assert_eq!(wait_error_exit_report().kind, ExitKind::Crash);
13322 }
13323}
13324
13325#[cfg(all(test, unix))]
13326mod restart_ordering_tests {
13327 use super::*;
13328 use std::io::Write;
13329 use tokio::io::AsyncReadExt;
13330
13331 #[test]
13332 #[ignore = "child process fixture"]
13333 fn serving_child() {
13334 let Ok(address_file) = std::env::var("RESTART_TEST_ADDRESS_FILE") else {
13335 return;
13336 };
13337 let listener = std::net::TcpListener::bind("127.0.0.1:0").unwrap();
13338 std::fs::write(address_file, listener.local_addr().unwrap().to_string()).unwrap();
13339 for stream in listener.incoming() {
13340 writeln!(stream.unwrap(), "{}", std::process::id()).unwrap();
13341 }
13342 }
13343
13344 async fn restart_before_publication(requests: usize) {
13345 let home = cortexkit_test_support::ScratchDir::new("restart-before-publication");
13346 let address_file = home.path().join("address");
13347 let spec = ModuleSpec {
13348 module_id: "restart-serving-child".into(),
13349 program: std::env::current_exe().unwrap(),
13350 args: vec![
13351 "--exact".into(),
13352 "supervise::restart_ordering_tests::serving_child".into(),
13353 "--ignored".into(),
13354 "--nocapture".into(),
13355 ],
13356 env: vec![(
13357 "RESTART_TEST_ADDRESS_FILE".into(),
13358 address_file.display().to_string(),
13359 )],
13360 reserved: false,
13361 reserved_prefixes: vec![],
13362 protocol: ModuleProtocol::None,
13363 overlap: Default::default(),
13364 };
13365 let mut supervisor =
13366 Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default());
13367 let (spawned_tx, spawned_rx) = oneshot::channel();
13368 let spawned_tx = Mutex::new(Some(spawned_tx));
13369 let (release_tx, release_rx) = std::sync::mpsc::channel();
13370 let release_rx = Mutex::new(release_rx);
13371 supervisor.test_after_first_spawn = AfterFirstSpawnHook(Some(Arc::new(move |_| {
13374 spawned_tx.lock().unwrap().take().unwrap().send(()).unwrap();
13375 release_rx
13376 .lock()
13377 .unwrap()
13378 .recv_timeout(Duration::from_secs(20))
13379 .unwrap();
13380 })));
13381 let registered = supervisor
13382 .register_configured_with_health(
13383 spec.clone(),
13384 true,
13385 HealthConfig::default(),
13386 None,
13387 RestartPolicy::new(3, Duration::ZERO),
13388 )
13389 .unwrap();
13390 let module = registered.module.clone();
13391 let startup = tokio::task::spawn_blocking(move || {
13392 supervisor.start_registered_configured(registered, None)
13393 });
13394 spawned_rx.await.unwrap();
13395 let old_pid = tokio::time::timeout(Duration::from_secs(10), async {
13396 let address = loop {
13397 if let Ok(address) = std::fs::read_to_string(&address_file) {
13398 if !address.is_empty() {
13399 break address;
13400 }
13401 }
13402 tokio::time::sleep(Duration::from_millis(1)).await;
13403 };
13404 let mut stream = tokio::net::TcpStream::connect(address).await.unwrap();
13405 let mut reply = String::new();
13406 stream.read_to_string(&mut reply).await.unwrap();
13407 reply.trim().parse::<u32>().unwrap()
13408 })
13409 .await
13410 .expect("child must serve before generation publication");
13411 assert_eq!(
13412 lock_snapshot(&module.inner.snapshot)
13413 .unwrap()
13414 .spawn_generation,
13415 0
13416 );
13417 let mut replies = Vec::new();
13418 for _ in 0..requests {
13419 let mut restart = Box::pin(module.restart(Some(0)));
13420 std::future::poll_fn(|cx| {
13423 assert!(std::future::Future::poll(restart.as_mut(), cx).is_pending());
13424 std::task::Poll::Ready(())
13425 })
13426 .await;
13427 replies.push(restart);
13428 }
13429 release_tx.send(()).unwrap();
13430 startup.await.unwrap().unwrap();
13431 for reply in replies {
13432 reply.await.unwrap();
13433 }
13434 let replacement = tokio::time::timeout(Duration::from_secs(10), async {
13437 loop {
13438 if lock_snapshot(&module.inner.snapshot)
13439 .unwrap()
13440 .spawn_generation
13441 > 1
13442 {
13443 break;
13444 }
13445 tokio::time::sleep(Duration::from_millis(1)).await;
13446 }
13447 })
13448 .await;
13449 module.update_spec_for_test(spec).await.unwrap();
13453 let (generation, pid) = {
13454 let state = lock_snapshot(&module.inner.snapshot).unwrap();
13455 (state.spawn_generation, state.pid)
13456 };
13457 module.stop().await.unwrap();
13458 replacement.expect("restart must publish a replacement for the already-serving child");
13459 assert_ne!(
13460 pid,
13461 Some(old_pid),
13462 "a serving child predating the restart must be replaced"
13463 );
13464 assert_eq!(generation, 2, "queued restarts must share one replacement");
13465 }
13466
13467 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
13468 async fn restart_received_after_serving_before_generation_publication_replaces_child() {
13469 restart_before_publication(1).await;
13470 }
13471
13472 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
13473 async fn restart_queued_behind_restart_does_not_kill_its_replacement() {
13474 restart_before_publication(2).await;
13475 }
13476
13477 #[tokio::test]
13478 async fn restart_queued_before_replacement_spawn_keeps_the_new_child() {
13479 let supervisor =
13480 Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default());
13481 let registered = supervisor
13482 .register_configured_with_health(
13483 ModuleSpec {
13484 module_id: "queued-before-replacement".into(),
13485 program: "/bin/sleep".into(),
13490 args: vec!["600".into()],
13491 env: vec![],
13492 reserved: false,
13493 reserved_prefixes: vec![],
13494 protocol: ModuleProtocol::None,
13495 overlap: Default::default(),
13496 },
13497 true,
13498 HealthConfig::default(),
13499 None,
13500 RestartPolicy::default(),
13501 )
13502 .unwrap();
13503 let RegisteredConfiguredModule {
13504 mut spec,
13505 mut runtime,
13506 module,
13507 mut commands,
13508 } = registered;
13509 let snapshot = &module.inner.snapshot;
13510 let mut restart = Box::pin(module.restart(Some(0)));
13511 std::future::poll_fn(|cx| {
13512 assert!(std::future::Future::poll(restart.as_mut(), cx).is_pending());
13513 std::task::Poll::Ready(())
13514 })
13515 .await;
13516 let mut child = Some(spawn_and_mark_running(&spec, &runtime, snapshot).unwrap());
13519 let replacement_pid = child.as_ref().unwrap().id();
13520 let command = commands.recv().await.unwrap();
13521 assert!(
13522 handle_supervisor_command(
13523 command,
13524 &mut spec,
13525 &mut runtime,
13526 &supervisor.registry,
13527 &supervisor.process_liveness,
13528 snapshot,
13529 &mut child,
13530 &mut commands,
13531 &mut VecDeque::new(),
13532 )
13533 .await
13534 );
13535 restart.await.unwrap();
13536 let kept_pid = child.as_ref().and_then(SupervisedChild::id);
13537 let pending = lock_snapshot(snapshot).unwrap().respawn_pending;
13538 if let Some(mut child) = child {
13539 child.start_kill().unwrap();
13540 child.wait().await.unwrap();
13541 }
13542 assert_eq!(
13543 kept_pid, replacement_pid,
13544 "a newer replacement must survive"
13545 );
13546 assert!(
13547 !pending,
13548 "a satisfied restart must not schedule another spawn"
13549 );
13550 }
13551}
13552
13553#[cfg(all(test, unix))]
13554mod health_event_tests {
13555 use super::*;
13556 use cortexkit_test_support::ScratchDir;
13557 use subc_protocol::{manifest::Concurrency, session::ModuleControlResponse};
13558
13559 struct Actor {
13560 module: SupervisedModule,
13561 registry: Arc<Registry>,
13562 forwarding: Arc<ForwardingTable>,
13563 spec: ModuleSpec,
13564 health: HealthConfig,
13565 rx: mpsc::Receiver<crate::router::OutboundFrame>,
13566 _home: ScratchDir,
13567 }
13568
13569 async fn wait_for<T>(reason: &str, mut observe: impl FnMut() -> Option<T>) -> T {
13570 let deadline = std::time::Instant::now() + Duration::from_secs(10);
13574 let tick = Instant::now();
13575 loop {
13576 if let Some(value) = observe() {
13577 assert_eq!(Instant::now(), tick, "{reason} must not wait for a timer");
13578 return value;
13579 }
13580 assert!(
13581 std::time::Instant::now() < deadline,
13582 "timed out waiting for {reason}"
13583 );
13584 tokio::task::yield_now().await;
13585 }
13586 }
13587
13588 impl Actor {
13589 async fn start(protocol: ModuleProtocol, cadence: Duration) -> Self {
13590 let home = ScratchDir::new("health-events");
13591 let registry = Arc::new(Registry::default());
13592 let forwarding = Arc::new(ForwardingTable::default());
13593 let health = HealthConfig {
13594 cadence,
13595 deadline: Duration::from_secs(3600),
13596 ..HealthConfig::default()
13597 };
13598 let supervisor =
13599 Supervisor::new_for_test(registry.clone(), RestartPolicy::new(3, Duration::ZERO))
13600 .with_forwarding(forwarding.clone())
13601 .with_health_config(health.clone());
13602 let spec = ModuleSpec {
13603 module_id: "event-health-child".into(),
13604 program: PathBuf::from("/bin/sleep"),
13612 args: vec!["600".into()],
13615 env: ["XDG_DATA_HOME", "XDG_RUNTIME_DIR", "XDG_CONFIG_HOME"]
13616 .into_iter()
13617 .map(|key| (key.into(), home.path().display().to_string()))
13618 .collect(),
13619 reserved: false,
13620 reserved_prefixes: vec![],
13621 protocol,
13622 overlap: Default::default(),
13623 };
13624 let module = supervisor.spawn(spec.clone()).unwrap();
13625 let (_, rx) = mpsc::channel(8);
13626 let actor = Self {
13627 module,
13628 registry,
13629 forwarding,
13630 spec,
13631 health,
13632 rx,
13633 _home: home,
13634 };
13635 actor
13636 .wait_for_select(|checkpoint| checkpoint.generation > 0)
13637 .await;
13638 actor
13639 }
13640
13641 fn turns(&self) -> u64 {
13642 lock_snapshot(&self.module.inner.snapshot)
13643 .unwrap()
13644 .actor_turns
13645 }
13646
13647 async fn wait_for_select(
13648 &self,
13649 expected: impl Fn(&ActorSelectCheckpoint) -> bool,
13650 ) -> ActorSelectCheckpoint {
13651 wait_for("actor select checkpoint", || {
13652 let state = lock_snapshot(&self.module.inner.snapshot).unwrap();
13653 state.actor_select.filter(|checkpoint| {
13654 checkpoint.generation == state.spawn_generation
13655 && state.state == ModuleState::Running
13656 && state.process_alive
13657 && state.reported_pid().is_some()
13658 && expected(checkpoint)
13659 })
13660 })
13661 .await
13662 }
13663
13664 fn hello(&mut self, connection: u64, health: bool) {
13665 let manifest =
13666 subc_protocol::manifest::ModuleManifest::builder(&self.spec.module_id, "1.0.0")
13667 .protocol_ver(subc_protocol::PROTOCOL_VERSION)
13668 .build();
13669 let connection = ConnectionId::new(connection);
13670 let (tx, rx) = mpsc::channel(8);
13671 self.rx = rx;
13672 self.forwarding
13673 .register_module_connection(
13674 connection,
13675 self.spec.module_id.clone(),
13676 subc_protocol::PROTOCOL_VERSION,
13677 Concurrency::ModuleManaged,
13678 FrameSink::new(tx),
13679 )
13680 .unwrap();
13681 self.registry
13682 .register_with_control_ops(
13683 manifest,
13684 subc_protocol::PROTOCOL_VERSION,
13685 connection,
13686 if health {
13687 vec![MODULE_CONTROL_OP_HEALTH_CHECK.into()]
13688 } else {
13689 vec![]
13690 },
13691 )
13692 .unwrap();
13693 }
13694
13695 async fn probe(&mut self) -> crate::router::OutboundFrame {
13696 let frame = wait_for("outbound health probe", || match self.rx.try_recv() {
13697 Ok(frame) => Some(frame),
13698 Err(mpsc::error::TryRecvError::Empty) => None,
13699 Err(mpsc::error::TryRecvError::Disconnected) => panic!("health peer disconnected"),
13700 })
13701 .await;
13702 let body: Value = serde_json::from_slice(&frame.body).unwrap();
13703 assert_eq!(body["op"], MODULE_CONTROL_OP_HEALTH_CHECK);
13704 frame
13705 }
13706
13707 fn no_probe(&mut self) {
13708 assert!(matches!(
13709 self.rx.try_recv(),
13710 Err(mpsc::error::TryRecvError::Empty)
13711 ));
13712 }
13713
13714 fn answer(&self, connection: u64, frame: crate::router::OutboundFrame) {
13715 self.forwarding
13716 .complete_module_control_rpc(
13717 ConnectionId::new(connection),
13718 frame.header.corr,
13719 Some(MODULE_CONTROL_OP_HEALTH_CHECK),
13720 ModuleControlRpcOutcome::Response(ModuleControlResponse::HealthCheck {
13721 status: HealthStatus::Ok,
13722 detail: None,
13723 metrics: None,
13724 }),
13725 )
13726 .unwrap();
13727 }
13728 }
13729
13730 #[tokio::test(start_paused = true)]
13731 async fn no_health_children_park_for_sixty_seconds() {
13732 let none = Actor::start(ModuleProtocol::None, Duration::from_secs(30)).await;
13733 let unregistered = Actor::start(ModuleProtocol::Subc, Duration::from_secs(30)).await;
13734 let mut unadvertised = Actor::start(ModuleProtocol::Subc, Duration::from_secs(30)).await;
13735 unadvertised.hello(1, false);
13736 for _ in 0..6000 {
13741 tokio::time::advance(Duration::from_millis(10)).await;
13742 tokio::task::yield_now().await;
13743 }
13744 for actor in [&none, &unregistered, &unadvertised] {
13745 assert!(
13746 actor.turns() <= 5,
13747 "no-health actor woke {} times",
13748 actor.turns()
13749 );
13750 assert_eq!(actor.module.state().unwrap(), ModuleState::Running);
13751 }
13752 }
13753
13754 #[tokio::test(start_paused = true)]
13755 async fn late_hello_starts_a_due_probe_in_the_notification_tick() {
13756 let mut actor = Actor::start(ModuleProtocol::Subc, Duration::ZERO).await;
13760 let tick = Instant::now();
13761 actor.hello(1, true);
13762 actor.probe().await;
13763 assert_eq!(Instant::now(), tick, "HELLO must not wait for a poll");
13764 }
13765
13766 #[tokio::test(start_paused = true)]
13767 async fn hello_keeps_the_jittered_cadence_and_catalog_updates_do_not_reset_it() {
13768 let mut actor = Actor::start(ModuleProtocol::Subc, Duration::from_secs(30)).await;
13769 let hello_tick = Instant::now();
13770 actor.hello(1, true);
13771 let armed = actor
13772 .wait_for_select(|checkpoint| {
13773 checkpoint.registered_connection == Some(ConnectionId::new(1))
13774 && checkpoint.next_probe_at.is_some()
13775 })
13776 .await;
13777 let delay = jittered_health_delay(&actor.spec.module_id, 0, actor.health.cadence);
13778 assert!((Duration::from_secs(30)..Duration::from_secs(33)).contains(&delay));
13779 let first_deadline = hello_tick + delay;
13780 assert_eq!(armed.next_probe_at, Some(first_deadline));
13781 tokio::time::advance(Duration::from_secs(29)).await;
13782 let turns = actor.turns();
13783 actor
13784 .registry
13785 .replace_catalog_for_connection(ConnectionId::new(1), vec![], None, Some(true))
13786 .unwrap();
13787 let updated = actor
13788 .wait_for_select(|checkpoint| checkpoint.turn > turns)
13789 .await;
13790 assert_eq!(
13791 updated.next_probe_at,
13792 Some(first_deadline),
13793 "catalog update must not reset cadence"
13794 );
13795 actor.no_probe();
13796 tokio::time::advance(first_deadline - Instant::now()).await;
13797 let frame = actor.probe().await;
13798 actor.answer(1, frame);
13799 let next = jittered_health_delay(&actor.spec.module_id, 1, actor.health.cadence);
13800 let next_deadline = Instant::now() + next;
13801 let rearmed = actor
13802 .wait_for_select(|checkpoint| {
13803 checkpoint
13804 .next_probe_at
13805 .is_some_and(|deadline| deadline > first_deadline)
13806 })
13807 .await;
13808 assert_eq!(rearmed.next_probe_at, Some(next_deadline));
13809 tokio::time::advance(next - Duration::from_nanos(1)).await;
13810 actor.no_probe();
13811 tokio::time::advance(Duration::from_nanos(1)).await;
13812 actor.probe().await;
13813 }
13814
13815 #[tokio::test(start_paused = true)]
13816 async fn disconnect_disarms_and_reconnect_rearms_health() {
13817 let mut actor = Actor::start(ModuleProtocol::Subc, Duration::from_secs(30)).await;
13818 actor.hello(1, true);
13819 actor
13820 .wait_for_select(|checkpoint| {
13821 checkpoint.registered_connection == Some(ConnectionId::new(1))
13822 && checkpoint.next_probe_at.is_some()
13823 })
13824 .await;
13825 let turns = actor.turns();
13826 let tick = Instant::now();
13827 actor
13828 .registry
13829 .deregister_connection(ConnectionId::new(1))
13830 .unwrap();
13831 let disarmed = actor
13832 .wait_for_select(|checkpoint| {
13833 checkpoint.turn > turns && checkpoint.registered_connection.is_none()
13834 })
13835 .await;
13836 assert_eq!(disarmed.next_probe_at, None);
13837 assert_eq!(disarmed.wake_after, None);
13838 assert_eq!(
13839 Instant::now(),
13840 tick,
13841 "disconnect must disarm in the notification tick"
13842 );
13843 tokio::time::advance(Duration::from_secs(60)).await;
13844 actor.no_probe();
13845 let reconnect_tick = Instant::now();
13846 actor.hello(2, true);
13847 let rearmed = actor
13848 .wait_for_select(|checkpoint| {
13849 checkpoint.registered_connection == Some(ConnectionId::new(2))
13850 && checkpoint.next_probe_at.is_some()
13851 })
13852 .await;
13853 let delay = jittered_health_delay(&actor.spec.module_id, 0, actor.health.cadence);
13854 assert_eq!(rearmed.next_probe_at, Some(reconnect_tick + delay));
13855 tokio::time::advance(delay).await;
13856 actor.probe().await;
13857 }
13858
13859 #[tokio::test(start_paused = true)]
13860 async fn reregistration_without_health_disarms_the_old_schedule() {
13861 let mut actor = Actor::start(ModuleProtocol::Subc, Duration::from_secs(30)).await;
13862 actor.hello(1, true);
13863 actor
13864 .wait_for_select(|checkpoint| {
13865 checkpoint.registered_connection == Some(ConnectionId::new(1))
13866 && checkpoint.next_probe_at.is_some()
13867 })
13868 .await;
13869 actor
13870 .registry
13871 .deregister_connection(ConnectionId::new(1))
13872 .unwrap();
13873 actor.hello(2, false);
13874 let disarmed = actor
13875 .wait_for_select(|checkpoint| {
13876 checkpoint.registered_connection == Some(ConnectionId::new(2))
13877 })
13878 .await;
13879 assert_eq!(disarmed.next_probe_at, None);
13880 assert_eq!(disarmed.wake_after, None);
13881 tokio::time::advance(Duration::from_secs(60)).await;
13882 actor.no_probe();
13883 }
13884
13885 #[tokio::test(start_paused = true)]
13886 async fn restart_waits_for_its_own_hello_before_rearming_health() {
13887 let mut actor = Actor::start(ModuleProtocol::Subc, Duration::from_secs(30)).await;
13888 actor.hello(1, true);
13889 let armed = actor
13890 .wait_for_select(|checkpoint| {
13891 checkpoint.registered_connection == Some(ConnectionId::new(1))
13892 && checkpoint.next_probe_at.is_some()
13893 })
13894 .await;
13895 actor
13899 .registry
13900 .deregister_connection(ConnectionId::new(1))
13901 .unwrap();
13902 actor.module.restart(Some(0)).await.unwrap();
13903 let parked = actor
13904 .wait_for_select(|checkpoint| {
13905 checkpoint.generation > armed.generation
13906 && checkpoint.registered_connection.is_none()
13907 && checkpoint.next_probe_at.is_none()
13908 })
13909 .await;
13910 assert_eq!(parked.wake_after, None);
13911 actor.rx.close();
13912 while actor.rx.try_recv().is_ok() {}
13915 let turns = actor.turns();
13916 tokio::time::advance(Duration::from_secs(60)).await;
13917 let after = actor.turns();
13918 eprintln!(
13919 "restart park turns: before={turns} after={after} delta={}",
13920 after - turns
13921 );
13922 assert_eq!(after, turns, "replacement must park before HELLO");
13923 let hello_tick = Instant::now();
13924 actor.hello(2, true);
13925 let rearmed = actor
13926 .wait_for_select(|checkpoint| {
13927 checkpoint.registered_connection == Some(ConnectionId::new(2))
13928 && checkpoint.next_probe_at.is_some()
13929 })
13930 .await;
13931 let delay = jittered_health_delay(&actor.spec.module_id, 0, actor.health.cadence);
13932 assert_eq!(rearmed.next_probe_at, Some(hello_tick + delay));
13933 tokio::time::advance(delay).await;
13934 actor.probe().await;
13935 }
13936
13937 #[tokio::test(start_paused = true)]
13938 async fn rescan_adds_and_removes_http_health_without_polling() {
13939 use tokio::{
13940 io::{AsyncReadExt, AsyncWriteExt},
13941 net::TcpListener,
13942 };
13943 let mut actor = Actor::start(ModuleProtocol::None, Duration::from_secs(30)).await;
13944 let listener = TcpListener::bind("127.0.0.1:0").await.unwrap();
13945 actor.health.http = Some(format!("http://{}/health", listener.local_addr().unwrap()));
13946 actor.health.cadence = Duration::from_millis(1);
13948 let (started_tx, mut started_rx) = mpsc::channel(8);
13949 let peer = tokio::spawn(async move {
13950 loop {
13951 let (mut socket, _) = listener.accept().await.unwrap();
13952 let mut request = [0; 1024];
13953 assert!(socket.read(&mut request).await.unwrap() > 0);
13954 started_tx.send(()).await.unwrap();
13955 socket
13956 .write_all(b"HTTP/1.1 200 OK\r\nContent-Length: 0\r\n\r\n")
13957 .await
13958 .unwrap();
13959 }
13960 });
13961 let tick = Instant::now();
13962 let turns = actor.turns();
13963 actor
13964 .module
13965 .update_configuration(actor.spec.clone(), actor.health.clone(), None)
13966 .await
13967 .unwrap();
13968 let armed = actor
13969 .wait_for_select(|checkpoint| {
13970 checkpoint.turn > turns && checkpoint.next_probe_at.is_some()
13971 })
13972 .await;
13973 assert_eq!(armed.next_probe_at, Some(tick + Duration::from_millis(1)));
13974 assert_eq!(Instant::now(), tick);
13975 tokio::time::advance(Duration::from_millis(1)).await;
13976 let mut started = false;
13977 wait_for("successful HTTP probe", || {
13978 started |= started_rx.try_recv().is_ok();
13979 (started && actor.module.status().unwrap().health.status == SupervisorHealthStatus::Ok)
13980 .then_some(())
13981 })
13982 .await;
13983 assert!(started, "added HTTP check must start at its first deadline");
13984 assert_eq!(
13985 actor.module.status().unwrap().health.status,
13986 SupervisorHealthStatus::Ok
13987 );
13988 assert_eq!(Instant::now(), tick + Duration::from_millis(1));
13989 actor.health.http = None;
13990 let turns = actor.turns();
13991 actor
13992 .module
13993 .update_configuration(actor.spec.clone(), actor.health.clone(), None)
13994 .await
13995 .unwrap();
13996 let parked = actor
13997 .wait_for_select(|checkpoint| {
13998 checkpoint.turn > turns && checkpoint.next_probe_at.is_none()
13999 })
14000 .await;
14001 assert_eq!(parked.wake_after, None);
14002 let turns = actor.turns();
14003 tokio::time::advance(Duration::from_secs(60)).await;
14004 assert_eq!(
14005 actor.turns(),
14006 turns,
14007 "removed HTTP check must park the actor"
14008 );
14009 assert!(
14010 started_rx.try_recv().is_err(),
14011 "removed HTTP check must stay disarmed"
14012 );
14013 assert_eq!(
14014 actor.module.status().unwrap().health,
14015 ModuleHealthStatus::default()
14016 );
14017 peer.abort();
14018 }
14019}
14020
14021#[cfg(test)]
14022mod drain_busy_reprobe_tests {
14023 use super::*;
14027 use std::collections::BTreeMap;
14028 use std::sync::Mutex as StdMutex;
14029 use subc_protocol::{manifest::Concurrency, session::ModuleControlResponse};
14030
14031 const MODULE_ID: &str = "drain-reprobe-module";
14032 const CONNECTION: u64 = 7;
14033 const GAUGE: &str = "runs_in_flight";
14034 const CADENCE: Duration = Duration::from_secs(30);
14037 const CEILING: Duration = Duration::from_secs(30);
14038
14039 #[derive(Clone, Copy)]
14041 enum Answer {
14042 BusyUntil { until: Instant, latency: Duration },
14045 OmitsGauge,
14047 Silent,
14049 }
14050
14051 struct Drain {
14052 forwarding: Arc<ForwardingTable>,
14053 runtime: SupervisorRuntimeConfig,
14054 endpoint: crate::ModuleEndpointId,
14055 probes: Arc<StdMutex<Vec<Instant>>>,
14057 responder: tokio::task::JoinHandle<()>,
14058 }
14059
14060 impl Drain {
14061 fn start(answer: Answer) -> Self {
14062 let registry = Arc::new(Registry::default());
14063 let forwarding = Arc::new(ForwardingTable::default());
14064 let health = HealthConfig {
14065 cadence: CADENCE,
14066 ..HealthConfig::default()
14067 };
14068 let runtime = Supervisor::new_for_test(registry, RestartPolicy::new(3, Duration::ZERO))
14069 .with_forwarding(forwarding.clone())
14070 .with_health_config(health)
14071 .runtime_config();
14072 let (tx, mut rx) = mpsc::channel::<crate::router::OutboundFrame>(8);
14073 let endpoint = forwarding
14074 .register_module_connection(
14075 ConnectionId::new(CONNECTION),
14076 MODULE_ID.to_string(),
14077 subc_protocol::PROTOCOL_VERSION,
14078 Concurrency::ModuleManaged,
14079 FrameSink::new(tx),
14080 )
14081 .unwrap();
14082 let probes = Arc::new(StdMutex::new(Vec::new()));
14083 let responder = tokio::spawn({
14084 let forwarding = forwarding.clone();
14085 let probes = probes.clone();
14086 async move {
14087 while let Some(frame) = rx.recv().await {
14088 let body: Value = serde_json::from_slice(&frame.body).unwrap();
14089 assert_eq!(body["op"], MODULE_CONTROL_OP_HEALTH_CHECK);
14090 probes.lock().unwrap().push(Instant::now());
14091 let metrics = match answer {
14092 Answer::BusyUntil { until, latency } => {
14093 sleep(latency).await;
14094 let busy = u64::from(Instant::now() < until);
14095 serde_json::json!({ GAUGE: busy })
14096 }
14097 Answer::OmitsGauge => serde_json::json!({ "queue_depth": 0 }),
14098 Answer::Silent => continue,
14099 };
14100 let _ = forwarding.complete_module_control_rpc(
14103 ConnectionId::new(CONNECTION),
14104 frame.header.corr,
14105 Some(MODULE_CONTROL_OP_HEALTH_CHECK),
14106 ModuleControlRpcOutcome::Response(ModuleControlResponse::HealthCheck {
14107 status: HealthStatus::Ok,
14108 detail: None,
14109 metrics: Some(metrics),
14110 }),
14111 );
14112 }
14113 }
14114 });
14115 Self {
14116 forwarding,
14117 runtime,
14118 endpoint,
14119 probes,
14120 responder,
14121 }
14122 }
14123
14124 async fn run(&self, ceiling: Duration) -> (bool, Duration) {
14127 let (drained, elapsed, _) = self.run_traced(ceiling).await;
14128 (drained, elapsed)
14129 }
14130
14131 async fn run_traced(&self, ceiling: Duration) -> (bool, Duration, DrainGaugeTrace) {
14133 let started = Instant::now();
14134 let mut trace = DrainGaugeTrace::new(&[GAUGE.to_string()]);
14135 let drained = wait_for_forwarding_quiescence(
14136 &self.forwarding,
14137 MODULE_ID,
14138 &self.runtime,
14139 self.endpoint,
14140 started + ceiling,
14141 &[GAUGE.to_string()],
14142 DrainScope::Active,
14143 &mut trace,
14144 )
14145 .await
14146 .unwrap();
14147 (drained, started.elapsed(), trace)
14148 }
14149
14150 fn probes(&self) -> Vec<Instant> {
14151 self.probes.lock().unwrap().clone()
14152 }
14153 }
14154
14155 impl Drop for Drain {
14156 fn drop(&mut self) {
14157 self.responder.abort();
14158 }
14159 }
14160
14161 #[tokio::test(start_paused = true)]
14162 async fn busy_at_first_probe_then_quiet_drains_long_before_the_ceiling() {
14163 let drain = Drain::start(Answer::BusyUntil {
14164 until: Instant::now() + Duration::from_secs(1),
14165 latency: Duration::ZERO,
14166 });
14167 let (drained, elapsed) = drain.run(CEILING).await;
14168 assert!(
14169 drained,
14170 "gauges that read zero during the drain must let it finish drained, \
14171 not wait out the {CADENCE:?} health cadence (elapsed {elapsed:?})"
14172 );
14173 assert!(
14174 elapsed < Duration::from_secs(3),
14175 "drain took {elapsed:?} after the gauges went quiet at 1s"
14176 );
14177 let probes = drain.probes();
14178 assert!(
14179 probes.len() >= 2,
14180 "the module was busy at the first probe, so a quiet verdict needs a second: {probes:?}"
14181 );
14182 }
14183
14184 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
14189 async fn an_instant_busy_first_answer_does_not_stop_later_probes_in_real_time() {
14190 let drain = Drain::start(Answer::BusyUntil {
14191 until: Instant::now() + Duration::from_millis(600),
14192 latency: Duration::ZERO,
14193 });
14194 let (drained, elapsed, trace) = drain.run_traced(CEILING).await;
14195 assert!(drained, "the gauge fell at 600ms: {trace}");
14196 assert!(
14197 elapsed < Duration::from_secs(3),
14198 "drain took {elapsed:?} after the gauge fell at 600ms: {trace}"
14199 );
14200 let probes = drain.probes();
14201 assert!(
14202 probes.len() >= 2 && trace.probes as usize == probes.len(),
14203 "the drain must re-probe after the first busy answer: {probes:?}, {trace}"
14204 );
14205 }
14206
14207 #[tokio::test(start_paused = true)]
14208 async fn busy_throughout_ends_undrained_at_the_ceiling_with_bounded_probes() {
14209 let started = Instant::now();
14210 let drain = Drain::start(Answer::BusyUntil {
14211 until: started + Duration::from_secs(3600),
14212 latency: Duration::ZERO,
14213 });
14214 let (drained, elapsed) = drain.run(CEILING).await;
14215 assert!(!drained, "a module busy throughout must not drain");
14216 assert!(
14217 elapsed >= CEILING && elapsed < CEILING + drain_busy_reprobe_interval(),
14218 "a busy module is held until the ceiling and no longer: {elapsed:?}"
14219 );
14220 let probes = drain.probes();
14223 let max_probes = (CEILING.as_millis() / drain_busy_reprobe_interval().as_millis()) + 1;
14224 assert!(
14225 probes.len() as u128 <= max_probes,
14226 "{} probes in a {CEILING:?} drain exceeds the {max_probes} the interval allows",
14227 probes.len()
14228 );
14229 for pair in probes.windows(2) {
14230 assert!(
14231 pair[1] - pair[0] >= drain_busy_reprobe_interval(),
14232 "consecutive probes {:?} apart",
14233 pair[1] - pair[0]
14234 );
14235 }
14236 }
14237
14238 #[tokio::test(start_paused = true)]
14239 async fn a_slow_probe_is_never_overlapped_by_the_next() {
14240 let latency = Duration::from_secs(1);
14243 let drain = Drain::start(Answer::BusyUntil {
14244 until: Instant::now() + Duration::from_secs(3600),
14245 latency,
14246 });
14247 let (drained, _) = drain.run(Duration::from_secs(5)).await;
14248 assert!(!drained);
14249 let probes = drain.probes();
14250 assert!(probes.len() >= 2, "the drain must re-probe: {probes:?}");
14251 for pair in probes.windows(2) {
14252 assert!(
14253 pair[1] - pair[0] >= latency + drain_busy_reprobe_interval(),
14254 "a probe started {:?} after the previous one, before it could finish",
14255 pair[1] - pair[0]
14256 );
14257 }
14258 }
14259
14260 #[tokio::test(start_paused = true)]
14261 async fn an_unanswered_probe_cannot_hold_the_drain_past_its_ceiling() {
14262 let drain = Drain::start(Answer::Silent);
14263 let ceiling = Duration::from_secs(2);
14264 let (drained, elapsed) = drain.run(ceiling).await;
14265 assert!(!drained, "an unanswered probe counts as busy");
14266 assert!(
14267 elapsed >= ceiling && elapsed < ceiling + drain_busy_reprobe_interval(),
14268 "the probe deadline is capped at the drain ceiling: {elapsed:?}"
14269 );
14270 assert_eq!(
14271 drain.probes().len(),
14272 1,
14273 "the unanswered probe holds the only slot until the ceiling"
14274 );
14275 }
14276
14277 #[derive(Clone, Default)]
14279 struct LogCapture {
14280 events: Arc<StdMutex<Vec<BTreeMap<String, String>>>>,
14281 }
14282
14283 impl<S: tracing::Subscriber> tracing_subscriber::Layer<S> for LogCapture {
14284 fn on_event(
14285 &self,
14286 event: &tracing::Event<'_>,
14287 _context: tracing_subscriber::layer::Context<'_, S>,
14288 ) {
14289 struct Fields(BTreeMap<String, String>);
14290 impl tracing::field::Visit for Fields {
14291 fn record_debug(&mut self, field: &tracing::field::Field, value: &dyn fmt::Debug) {
14292 self.0
14293 .insert(field.name().to_string(), format!("{value:?}"));
14294 }
14295 }
14296 let mut fields = Fields(BTreeMap::new());
14297 event.record(&mut fields);
14298 self.events.lock().unwrap().push(fields.0);
14299 }
14300 }
14301
14302 impl LogCapture {
14303 fn install(&self) -> tracing::subscriber::DefaultGuard {
14304 use tracing_subscriber::layer::SubscriberExt;
14305 tracing::subscriber::set_default(tracing_subscriber::registry().with(self.clone()))
14306 }
14307
14308 fn with_message(&self, needle: &str) -> Vec<BTreeMap<String, String>> {
14309 self.events
14310 .lock()
14311 .unwrap()
14312 .iter()
14313 .filter(|fields| fields.get("message").is_some_and(|m| m.contains(needle)))
14314 .cloned()
14315 .collect()
14316 }
14317 }
14318
14319 #[tokio::test(start_paused = true)]
14324 async fn a_missing_declared_gauge_is_named_on_the_drain_timeout_line() {
14325 let capture = LogCapture::default();
14326 let _guard = capture.install();
14327 let drain = Drain::start(Answer::OmitsGauge);
14328 let (drained, _, trace) = drain.run_traced(Duration::from_secs(2)).await;
14329 assert!(!drained, "a gauge never reported cannot read quiet");
14330 assert!(trace.probes >= 2, "the drain must re-probe: {trace:?}");
14331
14332 let missing = capture.with_message("lacks a declared busy gauge");
14333 assert_eq!(missing.len(), 1, "one warning per drain: {missing:?}");
14334 assert_eq!(
14335 missing[0].get("missing_gauges").map(String::as_str),
14336 Some(GAUGE)
14337 );
14338
14339 warn_drain_timed_out(
14340 MODULE_ID,
14341 Duration::from_secs(2),
14342 RouteCloseReason::Restart,
14343 &Default::default(),
14344 &trace,
14345 );
14346 let timeouts = capture.with_message("route drain timed out");
14347 assert_eq!(timeouts.len(), 1, "{timeouts:?}");
14348 let gauges = &timeouts[0]["busy_gauges"];
14349 assert!(
14350 gauges.starts_with(&format!("{GAUGE}: missing_from_health_body @")),
14351 "the timeout line must name the gauge and why it read busy: {gauges}"
14352 );
14353 assert!(
14354 gauges.ends_with("; no value ever read"),
14355 "no value was ever read for the gauge: {gauges}"
14356 );
14357 assert_eq!(timeouts[0]["gauge_probes"], trace.probes.to_string());
14358 }
14359
14360 #[tokio::test(start_paused = true)]
14364 async fn the_drain_trace_names_a_busy_value_and_an_unanswered_probe() {
14365 let busy = Drain::start(Answer::BusyUntil {
14366 until: Instant::now() + Duration::from_secs(3600),
14367 latency: Duration::ZERO,
14368 });
14369 let (_, _, trace) = busy.run_traced(Duration::from_secs(1)).await;
14370 let rendered = trace.to_string();
14371 assert!(
14372 rendered.starts_with(&format!("{GAUGE}: non_zero(1) @"))
14373 && rendered.contains("; last value non_zero(1) @"),
14374 "{rendered}"
14375 );
14376
14377 let silent = Drain::start(Answer::Silent);
14378 let (_, _, trace) = silent.run_traced(Duration::from_secs(1)).await;
14379 assert_eq!(
14382 trace.to_string(),
14383 format!("{GAUGE}: probe_failed(no-answer) @1000ms; no value ever read")
14384 );
14385 assert_eq!(trace.probes, 1);
14386 }
14387
14388 #[test]
14390 fn a_gauge_that_is_not_a_count_is_busy_and_shown_as_sent() {
14391 let gauges = [GAUGE.to_string()];
14392 let verdicts = gauge_verdicts(Some(&serde_json::json!({ GAUGE: 0.0 })), &gauges);
14393 assert_eq!(verdicts, vec![GaugeVerdict::NotACount("0.0".to_string())]);
14394 assert_eq!(
14395 busy_gauge_observation(&verdicts),
14396 BusyGaugeObservation::Busy
14397 );
14398 let zero = gauge_verdicts(Some(&serde_json::json!({ GAUGE: 0 })), &gauges);
14399 assert_eq!(
14400 busy_gauge_observation(&zero),
14401 BusyGaugeObservation::Quiescent
14402 );
14403 let none = gauge_verdicts(None, &gauges);
14404 assert_eq!(busy_gauge_observation(&none), BusyGaugeObservation::Omitted);
14405 }
14406}
14407
14408#[cfg(test)]
14409mod health_evidence_tests {
14410 use super::{HealthProbeError, HealthProbeEvidence};
14411 use std::collections::HashSet;
14412
14413 #[test]
14421 fn only_a_dead_lane_is_proof_of_death() {
14422 assert!(HealthProbeError::lane_dead("gone").is_proof_of_death());
14423 assert!(!HealthProbeError::no_answer("timed out").is_proof_of_death());
14427 assert!(!HealthProbeError::bad_answer("garbage").is_proof_of_death());
14428 assert!(!HealthProbeError::misconfigured("no table").is_proof_of_death());
14429 }
14430
14431 #[test]
14437 fn every_evidence_class_has_a_distinct_label() {
14438 let labels = [
14439 HealthProbeError::lane_dead("").label(),
14440 HealthProbeError::no_answer("").label(),
14441 HealthProbeError::bad_answer("").label(),
14442 HealthProbeError::misconfigured("").label(),
14443 ];
14444 let unique: HashSet<_> = labels.iter().collect();
14445 assert_eq!(unique.len(), labels.len(), "labels collided: {labels:?}");
14446 }
14447
14448 #[test]
14454 fn classification_preserves_the_original_message() {
14455 let err = HealthProbeError::no_answer("module did not answer within 5s");
14456 assert_eq!(err.to_string(), "module did not answer within 5s");
14457 assert!(matches!(err.evidence, HealthProbeEvidence::NoAnswer));
14458 }
14459}
14460
14461#[cfg(test)]
14462mod health_tombstone_tests {
14463 use std::{path::PathBuf, sync::Arc, time::Duration};
14464
14465 use subc_protocol::{
14466 manifest::Concurrency,
14467 session::{HealthStatus, ModuleControlResponse},
14468 };
14469 use tokio::sync::mpsc;
14470
14471 use super::{
14472 probe_module_health, HealthAction, HealthConfig, HealthProbeEvidence, ModuleProtocol,
14473 ModuleSpec, RestartPolicy, Supervisor, SupervisorRuntimeConfig,
14474 };
14475 use crate::{
14476 control::ControlHandler,
14477 forwarding::{ForwardingTable, ModuleControlRpcCompletion, ModuleControlRpcOutcome},
14478 registry::{ConnectionId, Registry},
14479 router::FrameSink,
14480 };
14481
14482 struct ProbeHarness {
14483 spec: ModuleSpec,
14484 runtime: SupervisorRuntimeConfig,
14485 forwarding: Arc<ForwardingTable>,
14486 module_connection: ConnectionId,
14487 module_rx: mpsc::Receiver<crate::router::OutboundFrame>,
14488 handler: ControlHandler,
14489 module: super::SupervisedModule,
14490 }
14491
14492 fn probe_harness() -> ProbeHarness {
14493 let registry = Arc::new(Registry::default());
14494 let forwarding = Arc::new(ForwardingTable::default());
14495 let supervisor_handle = super::SupervisorHandle::new();
14496 let health = HealthConfig {
14497 http: None,
14498 cadence: Duration::from_secs(30),
14499 deadline: Duration::from_secs(5),
14500 failure_threshold: 3,
14501 on_degraded: HealthAction::Report,
14502 on_failing: HealthAction::Report,
14503 critical: false,
14504 };
14505 let supervisor = Supervisor::new_for_test(Arc::clone(®istry), RestartPolicy::default())
14506 .with_forwarding(Arc::clone(&forwarding))
14507 .with_handle(supervisor_handle.clone())
14508 .with_health_config(health);
14509 let spec = ModuleSpec {
14510 module_id: "late-health-module".to_string(),
14511 program: PathBuf::from("disabled-module"),
14512 args: Vec::new(),
14513 env: Vec::new(),
14514 reserved: false,
14515 reserved_prefixes: Vec::new(),
14516 protocol: ModuleProtocol::Subc,
14517 overlap: Default::default(),
14518 };
14519 let module = supervisor
14520 .supervise_configured(spec.clone(), false)
14521 .unwrap();
14522 let runtime = supervisor.runtime_config();
14523 let handler = ControlHandler::with_forwarding(registry, Arc::clone(&forwarding))
14524 .with_supervisor(supervisor_handle);
14525 let module_connection = ConnectionId::new(700);
14526 let (module_tx, module_rx) = mpsc::channel(8);
14527 forwarding
14528 .register_module_connection(
14529 module_connection,
14530 spec.module_id.clone(),
14531 subc_protocol::PROTOCOL_VERSION,
14532 Concurrency::ModuleManaged,
14533 FrameSink::new(module_tx),
14534 )
14535 .unwrap();
14536
14537 ProbeHarness {
14538 spec,
14539 runtime,
14540 forwarding,
14541 module_connection,
14542 module_rx,
14543 handler,
14544 module,
14545 }
14546 }
14547
14548 async fn finish_after(
14549 harness: &mut ProbeHarness,
14550 stall: Duration,
14551 ) -> ModuleControlRpcCompletion {
14552 assert!(stall > harness.runtime.health.deadline);
14553 let deadline = harness.runtime.health.deadline;
14554 let probe = probe_module_health(&harness.spec.module_id, &harness.runtime, None);
14555 let answer = async {
14556 let frame = harness.module_rx.recv().await.expect("health.check frame");
14557 tokio::time::advance(deadline).await;
14558 tokio::task::yield_now().await;
14559 tokio::time::advance(stall - deadline).await;
14560 harness
14561 .forwarding
14562 .complete_module_control_rpc(
14563 harness.module_connection,
14564 frame.header.corr,
14565 Some("health.check"),
14566 ModuleControlRpcOutcome::Response(ModuleControlResponse::HealthCheck {
14567 status: HealthStatus::Ok,
14568 detail: None,
14569 metrics: None,
14570 }),
14571 )
14572 .unwrap()
14573 };
14574 let (probe_result, completion) = tokio::join!(probe, answer);
14575 let err = probe_result.expect_err("probe must miss its deadline");
14576 assert!(matches!(err.evidence, HealthProbeEvidence::NoAnswer));
14577 completion
14578 }
14579
14580 async fn time_out_without_answer(harness: &mut ProbeHarness) {
14581 let deadline = harness.runtime.health.deadline;
14582 let probe = probe_module_health(&harness.spec.module_id, &harness.runtime, None);
14583 let exhaust_deadline = async {
14584 let _frame = harness.module_rx.recv().await.expect("health.check frame");
14585 tokio::time::advance(deadline).await;
14586 tokio::task::yield_now().await;
14587 };
14588 let (probe_result, ()) = tokio::join!(probe, exhaust_deadline);
14589 let err = probe_result.expect_err("probe must miss its deadline");
14590 assert!(matches!(err.evidence, HealthProbeEvidence::NoAnswer));
14591 }
14592
14593 async fn run_probe_cycle(harness: &mut ProbeHarness, answer: bool) {
14594 let registry = Arc::clone(&harness.module.inner.registry);
14595 let snapshot = Arc::clone(&harness.module.inner.snapshot);
14596 let process_liveness = super::SupervisorProcessLiveness::default();
14597 let mut child = None;
14598 let cycle = super::run_health_probe_cycle(
14599 &harness.spec,
14600 &harness.runtime,
14601 ®istry,
14602 &process_liveness,
14603 &snapshot,
14604 &mut child,
14605 );
14606 let peer = async {
14607 let frame = harness.module_rx.recv().await.expect("health.check frame");
14608 if answer {
14609 harness
14610 .forwarding
14611 .complete_module_control_rpc(
14612 harness.module_connection,
14613 frame.header.corr,
14614 Some("health.check"),
14615 ModuleControlRpcOutcome::Response(ModuleControlResponse::HealthCheck {
14616 status: HealthStatus::Ok,
14617 detail: None,
14618 metrics: Some(serde_json::json!({"ready": true})),
14619 }),
14620 )
14621 .unwrap();
14622 } else {
14623 tokio::time::advance(harness.runtime.health.deadline).await;
14624 tokio::task::yield_now().await;
14625 }
14626 };
14627 tokio::join!(cycle, peer);
14628 }
14629
14630 #[tokio::test(start_paused = true)]
14631 async fn unanswered_probe_is_unknown_until_threshold_and_ok_report_recovers() {
14632 let mut harness = probe_harness();
14633 let monitor = harness.module.inner.monitor.lock().unwrap().take().unwrap();
14637 monitor.abort();
14638 let _ = monitor.await;
14639 super::update_snapshot(&harness.module.inner.snapshot, None, |state| {
14640 state.enabled = true;
14641 state.state = super::ModuleState::Running;
14642 state.process_alive = true;
14643 })
14644 .unwrap();
14645
14646 run_probe_cycle(&mut harness, true).await;
14647 assert_eq!(
14648 harness.module.status().unwrap().health.status,
14649 super::SupervisorHealthStatus::Ok
14650 );
14651
14652 for failures in 1..harness.runtime.health.failure_threshold {
14653 run_probe_cycle(&mut harness, false).await;
14654 let status = harness.module.status().unwrap();
14655 assert_eq!(status.health.status, super::SupervisorHealthStatus::Unknown);
14656 assert_eq!(status.health.consecutive_failures, failures);
14657 assert!(status.health.last_probe_ms.is_some());
14658 assert!(status.health.detail.unwrap().starts_with("[no-answer]"));
14659 assert!(status.health.metrics.is_none());
14660 assert_eq!(status.state, super::ModuleState::Running);
14661 assert!(status.process_alive);
14662 assert_eq!(status.restart_count, 0);
14663 assert_eq!(status.lifetime_restarts, 0);
14664 assert!(status.health.last_action.is_none());
14665 assert!(harness.runtime.scheduled_respawn.lock().unwrap().is_none());
14666 }
14667
14668 run_probe_cycle(&mut harness, true).await;
14669 let recovered = harness.module.status().unwrap();
14670 assert_eq!(recovered.health.status, super::SupervisorHealthStatus::Ok);
14671 assert_eq!(recovered.health.consecutive_failures, 0);
14672 assert!(recovered.health.detail.is_none());
14673 assert_eq!(
14674 recovered.health.metrics,
14675 Some(serde_json::json!({"ready": true}))
14676 );
14677 assert_eq!(recovered.lifetime_restarts, 0);
14678
14679 for failures in 1..=harness.runtime.health.failure_threshold {
14680 run_probe_cycle(&mut harness, false).await;
14681 let status = harness.module.status().unwrap();
14682 assert_eq!(status.health.consecutive_failures, failures);
14683 if failures < harness.runtime.health.failure_threshold {
14684 assert_eq!(status.health.status, super::SupervisorHealthStatus::Unknown);
14685 assert_eq!(status.state, super::ModuleState::Running);
14686 assert_eq!(status.lifetime_restarts, 0);
14687 assert!(harness.runtime.scheduled_respawn.lock().unwrap().is_none());
14688 } else {
14689 assert_eq!(
14690 status.health.status,
14691 super::SupervisorHealthStatus::Unresponsive
14692 );
14693 assert_eq!(status.state, super::ModuleState::Restarting);
14694 assert_eq!(status.restart_count, 1);
14695 assert_eq!(status.lifetime_restarts, 1);
14696 assert_eq!(status.health.last_action.as_deref(), Some("restart"));
14697 assert!(harness.runtime.scheduled_respawn.lock().unwrap().is_some());
14698 }
14699 }
14700 }
14701
14702 #[tokio::test(start_paused = true)]
14703 async fn late_health_answers_record_start_anchored_latency_for_two_stalls() {
14704 let mut harness = probe_harness();
14705
14706 let first = finish_after(&mut harness, Duration::from_secs(8)).await;
14707 let first_latency = match &first {
14708 ModuleControlRpcCompletion::LateHealthAnswer { latency, .. } => *latency,
14709 other => panic!("late answer was not retained: {other:?}"),
14710 };
14711 assert!(harness.handler.observe_module_control_completion(first));
14712
14713 let second = finish_after(&mut harness, Duration::from_secs(11)).await;
14714 let second_latency = match &second {
14715 ModuleControlRpcCompletion::LateHealthAnswer { latency, .. } => *latency,
14716 other => panic!("late answer was not retained: {other:?}"),
14717 };
14718 assert!(harness.handler.observe_module_control_completion(second));
14719
14720 assert_eq!(first_latency, Duration::from_secs(8));
14721 assert_eq!(
14722 second_latency - first_latency,
14723 Duration::from_secs(3),
14724 "latency must grow linearly with the additional stall"
14725 );
14726 let health = harness.module.status().unwrap().health;
14727 assert_eq!(health.late_answer_count, 2);
14728 assert_eq!(health.last_late_answer_latency_ms, Some(11_000));
14729 }
14730
14731 #[tokio::test(start_paused = true)]
14739 async fn late_answer_clears_the_consecutive_failure_streak() {
14740 let mut harness = probe_harness();
14741
14742 time_out_without_answer(&mut harness).await;
14744 harness
14745 .module
14746 .record_health_probe_failure_for_test("[no-answer] test miss")
14747 .unwrap();
14748 assert_eq!(
14749 harness.module.status().unwrap().health.consecutive_failures,
14750 1,
14751 "precondition: the miss must be on the streak before the late answer"
14752 );
14753
14754 let late = finish_after(&mut harness, Duration::from_secs(9)).await;
14756 assert!(matches!(
14757 late,
14758 ModuleControlRpcCompletion::LateHealthAnswer { .. }
14759 ));
14760 assert!(harness.handler.observe_module_control_completion(late));
14761
14762 let health = harness.module.status().unwrap().health;
14763 assert_eq!(
14764 health.consecutive_failures, 0,
14765 "a late answer is an answer: the streak must reset"
14766 );
14767 assert_eq!(health.late_answer_count, 1);
14768 }
14769
14770 #[tokio::test(start_paused = true)]
14771 async fn repeated_serial_probe_cycles_keep_one_tombstone_per_endpoint() {
14772 let mut harness = probe_harness();
14773
14774 for _ in 0..20 {
14775 time_out_without_answer(&mut harness).await;
14776 assert_eq!(
14777 harness.forwarding.health_probe_tombstone_count().unwrap(),
14778 1
14779 );
14780 }
14781 }
14782}
14783
14784#[cfg(test)]
14785mod child_env_tests {
14786 #[cfg(windows)]
14787 use super::{
14788 clear_current_process_facts, observed_launch_nonce_source, ModuleState, SupervisorSnapshot,
14789 };
14790 #[cfg(windows)]
14791 use std::sync::{Arc, OnceLock};
14792 #[cfg(windows)]
14793 #[test]
14794 fn windows_wire_spawn_prepares_pipe_and_keeps_phase_one_env() {
14795 let spec = spec(Vec::new());
14796 let mut command = Command::new("unused");
14797 let handoff = apply_wire_spawn_args(&mut command, &spec, None, None)
14798 .unwrap()
14799 .unwrap();
14800 let name = command
14801 .as_std()
14802 .get_envs()
14803 .find(|(key, _)| *key == OsStr::new(subc_os::launch_nonce::LAUNCH_NONCE_PIPE_ENV))
14804 .and_then(|(_, value)| value)
14805 .unwrap();
14806 assert_eq!(name, OsStr::new(handoff.name()));
14807 assert!(handoff.name().starts_with(r"\\.\pipe\subc-launch-"));
14808 assert!(command.as_std().get_envs().any(|(key, value)| key
14809 == OsStr::new(subc_os::LAUNCH_NONCE_PIPE_FALLBACK_ENV)
14810 && value == Some(OsStr::new("env"))));
14811 assert!(command
14812 .as_std()
14813 .get_envs()
14814 .any(|(key, value)| key == OsStr::new(SUBC_LAUNCH_NONCE_ENV) && value.is_some()));
14815 }
14816
14817 #[cfg(windows)]
14818 #[test]
14819 fn windows_source_reporting_tracks_consumption_and_resets_per_spawn() {
14820 let mut snapshot = SupervisorSnapshot::new(ModuleState::Running, true);
14821 snapshot.process_alive = true;
14822 snapshot.spawned_protocol = Some(ModuleProtocol::Subc);
14823 assert_eq!(
14824 observed_launch_nonce_source(&snapshot).as_deref(),
14825 Some("env")
14826 );
14827 let consumed = Arc::new(OnceLock::new());
14828 snapshot.pipe_consumed = Some(consumed.clone());
14829 assert_eq!(
14830 observed_launch_nonce_source(&snapshot).as_deref(),
14831 Some("env")
14832 );
14833 consumed.set(()).unwrap();
14834 assert_eq!(
14835 observed_launch_nonce_source(&snapshot).as_deref(),
14836 Some("pipe")
14837 );
14838 snapshot.pipe_consumed = Some(Arc::new(OnceLock::new()));
14839 assert_eq!(
14840 observed_launch_nonce_source(&snapshot).as_deref(),
14841 Some("env")
14842 );
14843 clear_current_process_facts(&mut snapshot);
14844 assert_eq!(observed_launch_nonce_source(&snapshot), None);
14845 }
14846 use super::{
14847 apply_child_env, apply_spawn_role, apply_wire_spawn_args, LaunchSpecDigest, ModuleProtocol,
14848 ModuleSpec, SpawnRole, SupervisorHandle, SPAWN_ROLE_SWAP_CANDIDATE, SUBC_ARG,
14849 SUBC_LAUNCH_NONCE_ENV, SUBC_MODULE_ID_ENV, SUBC_SPAWN_ROLE_ENV,
14850 };
14851 use std::{ffi::OsStr, path::PathBuf};
14852 use tokio::process::Command;
14853
14854 fn spec(env: Vec<(String, String)>) -> ModuleSpec {
14855 ModuleSpec {
14856 module_id: "env-plan".to_string(),
14857 program: PathBuf::from("/nonexistent"),
14858 args: Vec::new(),
14859 env,
14860 reserved: false,
14861 reserved_prefixes: Vec::new(),
14862 protocol: ModuleProtocol::Subc,
14863 overlap: Default::default(),
14864 }
14865 }
14866
14867 #[test]
14868 fn configured_launch_spec_digest_is_canonical_and_excludes_injected_inputs() {
14869 let base = spec(vec![("A".into(), "B=C".into()), ("Z".into(), "v".into())]);
14870 let digest = LaunchSpecDigest::from_spec(&base);
14871 let mut reordered = base.clone();
14872 reordered.env.reverse();
14873 reordered.module_id = "different-id".into();
14874 reordered.reserved = true;
14875 reordered.overlap = Default::default();
14876 assert_eq!(digest, LaunchSpecDigest::from_spec(&reordered));
14877 let mut delimiter_collision = base.clone();
14878 delimiter_collision.env[0] = ("A=B".into(), "C".into());
14879 assert_ne!(digest, LaunchSpecDigest::from_spec(&delimiter_collision));
14880 let mut changed = base.clone();
14881 changed.protocol = ModuleProtocol::None;
14882 assert_ne!(digest, LaunchSpecDigest::from_spec(&changed));
14883 changed = base.clone();
14884 changed.args.push("--flag".into());
14885 assert_ne!(digest, LaunchSpecDigest::from_spec(&changed));
14886 changed = base.clone();
14887 changed.program = PathBuf::from("/other");
14888 assert_ne!(digest, LaunchSpecDigest::from_spec(&changed));
14889 changed = base.clone();
14890 changed.env[1].1 = "other".into();
14891 assert_ne!(digest, LaunchSpecDigest::from_spec(&changed));
14892 assert!(!format!("{digest:?}").contains("B=C"));
14893 assert!(!format!("{digest:?}").contains("[u8"));
14894 }
14895
14896 #[cfg(unix)]
14897 #[test]
14898 fn native_program_bytes_do_not_collapse_through_lossy_utf8() {
14899 use std::os::unix::ffi::OsStringExt;
14900 let mut left = spec(Vec::new());
14901 left.program = std::ffi::OsString::from_vec(vec![b'/', 0xff]).into();
14902 let mut right = left.clone();
14903 right.program = std::ffi::OsString::from_vec(vec![b'/', 0xfe]).into();
14904 assert_eq!(
14905 left.program.to_string_lossy(),
14906 right.program.to_string_lossy()
14907 );
14908 assert_ne!(
14909 LaunchSpecDigest::from_spec(&left),
14910 LaunchSpecDigest::from_spec(&right)
14911 );
14912 }
14913
14914 #[test]
14928 fn ambient_ck_log_is_removed_and_a_configured_one_survives() {
14929 let mut command = Command::new("/nonexistent");
14930 apply_child_env(&mut command, &spec(Vec::new()));
14931 let removed = command
14932 .as_std()
14933 .get_envs()
14934 .any(|(key, value)| key == OsStr::new("CK_LOG") && value.is_none());
14935 assert!(
14936 removed,
14937 "ambient CK_LOG must be explicitly removed for an unconfigured module"
14938 );
14939
14940 let mut configured = Command::new("/nonexistent");
14941 apply_child_env(
14942 &mut configured,
14943 &spec(vec![("CK_LOG".to_string(), "debug".to_string())]),
14944 );
14945 let effective = configured
14946 .as_std()
14947 .get_envs()
14948 .filter(|(key, _)| *key == OsStr::new("CK_LOG"))
14949 .last()
14950 .map(|(_, value)| value.map(|v| v.to_string_lossy().into_owned()));
14951 assert_eq!(
14952 effective,
14953 Some(Some("debug".to_string())),
14954 "a module's configured CK_LOG must survive the ambient removal"
14955 );
14956 }
14957
14958 #[test]
14967 fn protocol_none_spawn_carries_no_subc_argument_and_no_nonce() {
14968 let connection_file = std::path::Path::new("/run/subc-connection.json");
14969 let handle = SupervisorHandle::new();
14970
14971 let mut none_spec = spec(Vec::new());
14972 none_spec.protocol = ModuleProtocol::None;
14973 let mut none = Command::new("/nonexistent");
14974 let none_handoff =
14975 apply_wire_spawn_args(&mut none, &none_spec, Some(connection_file), Some(&handle))
14976 .expect("protocol-none spawn args apply");
14977 assert!(
14978 none_handoff.is_none(),
14979 "protocol:none spawn must not receive a nonce descriptor"
14980 );
14981 #[cfg(windows)]
14982 assert!(!none.as_std().get_envs().any(|(key, value)| key
14983 == OsStr::new(subc_os::LAUNCH_NONCE_PIPE_FALLBACK_ENV)
14984 && value.is_some()));
14985 assert!(
14986 !none.as_std().get_envs().any(|(key, value)| key
14987 == OsStr::new(subc_os::LAUNCH_NONCE_FD_ENV)
14988 && value.is_some()),
14989 "protocol:none spawn must not name a nonce descriptor"
14990 );
14991 let none_args: Vec<String> = none
14992 .as_std()
14993 .get_args()
14994 .map(|a| a.to_string_lossy().into_owned())
14995 .collect();
14996 assert!(
14997 !none_args.iter().any(|a| a == SUBC_ARG),
14998 "protocol:none argv must not carry --subc; got {none_args:?}"
14999 );
15000 let none_has_nonce = none
15001 .as_std()
15002 .get_envs()
15003 .any(|(key, value)| key == OsStr::new(SUBC_LAUNCH_NONCE_ENV) && value.is_some());
15004 assert!(
15005 !none_has_nonce,
15006 "protocol:none spawn must not receive a launch nonce"
15007 );
15008 let none_has_module_id = none
15009 .as_std()
15010 .get_envs()
15011 .any(|(key, value)| key == OsStr::new(SUBC_MODULE_ID_ENV) && value.is_some());
15012 assert!(
15013 none_has_module_id,
15014 "SUBC_MODULE_ID is inert and stays on every path"
15015 );
15016 assert!(
15017 handle.spawn_nonce(&none_spec.module_id).is_none(),
15018 "no nonce record for a process that will never present one"
15019 );
15020
15021 let wire_spec = spec(Vec::new());
15023 let mut wire = Command::new("/nonexistent");
15024 let wire_handoff =
15025 apply_wire_spawn_args(&mut wire, &wire_spec, Some(connection_file), Some(&handle))
15026 .expect("subc-wire spawn args apply");
15027 let wire_fd_env = wire
15028 .as_std()
15029 .get_envs()
15030 .find(|(key, _)| *key == OsStr::new(subc_os::LAUNCH_NONCE_FD_ENV))
15031 .and_then(|(_, value)| value.map(|v| v.to_string_lossy().into_owned()));
15032 #[cfg(unix)]
15033 assert_eq!(
15034 wire_fd_env,
15035 Some(wire_handoff.expect("a descriptor handoff").fd_env_value()),
15036 "a subc-wire spawn names the pipe it will receive at descriptor 3"
15037 );
15038 #[cfg(windows)]
15039 assert!(wire_handoff.is_some() && wire_fd_env.is_none());
15040 #[cfg(not(any(unix, windows)))]
15041 assert!(wire_handoff.is_none() && wire_fd_env.is_none());
15042 let wire_args: Vec<String> = wire
15043 .as_std()
15044 .get_args()
15045 .map(|a| a.to_string_lossy().into_owned())
15046 .collect();
15047 assert_eq!(
15048 wire_args,
15049 vec![
15050 SUBC_ARG.to_string(),
15051 connection_file.to_string_lossy().into_owned()
15052 ],
15053 "a subc-wire spawn still carries --subc <path>"
15054 );
15055 assert_eq!(
15056 wire.as_std()
15057 .get_envs()
15058 .any(|(key, value)| key == OsStr::new(SUBC_LAUNCH_NONCE_ENV) && value.is_some()),
15059 !cfg!(unix),
15060 "only Windows supplies the environment nonce"
15061 );
15062 assert!(handle.spawn_nonce(&wire_spec.module_id).is_some());
15063 }
15064
15065 #[test]
15075 fn plain_spawn_removes_the_spawn_role_even_when_the_spec_sets_it() {
15076 let role = |command: &Command| {
15077 command
15078 .as_std()
15079 .get_envs()
15080 .filter(|(key, _)| *key == OsStr::new(SUBC_SPAWN_ROLE_ENV))
15081 .last()
15082 .map(|(_, value)| value.map(|v| v.to_string_lossy().into_owned()))
15083 };
15084 let forged = spec(vec![(
15085 SUBC_SPAWN_ROLE_ENV.to_string(),
15086 SPAWN_ROLE_SWAP_CANDIDATE.to_string(),
15087 )]);
15088
15089 let mut plain = Command::new("/nonexistent");
15090 apply_child_env(&mut plain, &forged);
15091 apply_spawn_role(&mut plain, SpawnRole::Plain);
15092 assert_eq!(
15093 role(&plain),
15094 Some(None),
15095 "a plain spawn must remove SUBC_SPAWN_ROLE, whatever the spec says"
15096 );
15097
15098 let mut candidate = Command::new("/nonexistent");
15099 apply_child_env(&mut candidate, &spec(Vec::new()));
15100 apply_spawn_role(&mut candidate, SpawnRole::SwapCandidate);
15101 assert_eq!(
15102 role(&candidate),
15103 Some(Some(SPAWN_ROLE_SWAP_CANDIDATE.to_string()))
15104 );
15105 }
15106
15107 #[test]
15113 fn daemon_private_capture_keys_are_not_passed_to_the_child() {
15114 let mut command = Command::new("/nonexistent");
15115 apply_child_env(
15116 &mut command,
15117 &spec(vec![
15118 (super::CAPTURE_KEEP_ENV.to_string(), "5".to_string()),
15119 ("KEPT".to_string(), "yes".to_string()),
15120 ]),
15121 );
15122 let keys: Vec<String> = command
15123 .as_std()
15124 .get_envs()
15125 .filter(|(_, value)| value.is_some())
15126 .map(|(key, _)| key.to_string_lossy().into_owned())
15127 .collect();
15128 assert!(keys.contains(&"KEPT".to_string()), "got {keys:?}");
15129 assert!(
15130 !keys.contains(&super::CAPTURE_KEEP_ENV.to_string()),
15131 "daemon-private capture key leaked to the child: {keys:?}"
15132 );
15133 }
15134}
15135
15136#[cfg(test)]
15137mod jitter_tests {
15138 use super::jittered_health_delay;
15139 use std::{collections::HashSet, time::Duration};
15140
15141 const FLEET: [&str; 14] = [
15150 "aft",
15151 "alfonso-core",
15152 "magic-context",
15153 "broca",
15154 "thalamus",
15155 "quota",
15156 "engram",
15157 "plexus",
15158 "cerebellum",
15159 "astrocyte",
15160 "synapse",
15161 "subc-mcp",
15162 "cortexkit-credentials",
15163 "subc-federation",
15164 ];
15165
15166 #[test]
15174 fn probe_delays_disperse_across_the_fleet() {
15175 let cadence = Duration::from_secs(30);
15176 let delays: HashSet<Duration> = FLEET
15177 .iter()
15178 .map(|id| jittered_health_delay(id, 0, cadence))
15179 .collect();
15180 assert_eq!(
15181 delays.len(),
15182 FLEET.len(),
15183 "every supervised module must land on its own probe offset"
15184 );
15185 }
15186
15187 #[test]
15193 fn jitter_only_delays_and_stays_within_one_tenth_of_cadence() {
15194 let cadence = Duration::from_secs(30);
15195 let span = cadence / 10;
15196 for id in FLEET {
15197 for probe_index in 0..8 {
15198 let delay = jittered_health_delay(id, probe_index, cadence);
15199 assert!(
15200 delay >= cadence,
15201 "{id}#{probe_index}: jitter must not shorten the cadence"
15202 );
15203 assert!(
15204 delay < cadence + span,
15205 "{id}#{probe_index}: jitter must stay inside one tenth of the cadence"
15206 );
15207 }
15208 }
15209 }
15210
15211 #[test]
15217 fn a_module_offset_is_stable_across_restarts() {
15218 let cadence = Duration::from_secs(30);
15219 for id in FLEET {
15220 assert_eq!(
15221 jittered_health_delay(id, 0, cadence),
15222 jittered_health_delay(id, 0, cadence),
15223 "{id}: the same module and probe index must produce the same offset"
15224 );
15225 }
15226 }
15227
15228 #[test]
15230 fn zero_cadence_yields_zero_delay() {
15231 assert_eq!(
15232 jittered_health_delay("aft", 0, Duration::ZERO),
15233 Duration::ZERO
15234 );
15235 }
15236}
15237
15238#[cfg(all(test, target_os = "linux"))]
15239mod cgroup_placement_tests {
15240 use super::{
15241 apply_cgroup_placement, remove_module_cgroup, LaunchSpecDigest, ModuleProtocol, ModuleSpec,
15242 SuperviseError, SupervisedChild,
15243 };
15244 use crate::stderr_tail::{StderrRing, StderrTailConfig};
15245 use cortexkit_test_support::ScratchDir;
15246 use std::{
15247 fs, io,
15248 path::{Path, PathBuf},
15249 sync::{Arc, Mutex},
15250 };
15251 use tokio::process::Command;
15252
15253 #[tokio::test]
15254 async fn unique_spawn_cgroups_clean_up_across_restarts_and_shutdown() {
15255 use super::*;
15256 let dir = ScratchDir::new("unique-spawn-cgroups");
15257 let root = PathBuf::from(format!(
15258 "/sys/fs/cgroup/subc-unique-test-{}-{}",
15259 std::process::id(),
15260 unix_ms_now()
15261 ));
15262 if let Err(error) = fs::create_dir(&root) {
15263 assert!(
15264 std::env::var_os("SUBC_REQUIRE_CGROUP_TEST").is_none(),
15265 "required cgroup test cannot execute: {error}"
15266 );
15267 eprintln!("SKIP unique_spawn_cgroups_clean_up_across_restarts_and_shutdown: {error}");
15268 return;
15269 }
15270 let placement = subc_cgroup::prepare_at(&root).unwrap().unwrap();
15271 let group_count = || {
15272 fs::read_dir(root.join("subc-modules"))
15273 .unwrap()
15274 .map(|entry| entry.unwrap().file_type().unwrap())
15275 .filter(|kind| kind.is_dir())
15276 .count()
15277 };
15278 let supervisor =
15279 Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default())
15280 .with_cgroup_placement(Some(placement.clone()));
15281 let runtime = supervisor.runtime_config();
15282 let mut spec = ModuleSpec {
15283 module_id: "unique-spawn".into(),
15284 program: PathBuf::from("/bin/sleep"),
15285 args: vec!["60".into()],
15286 env: ["XDG_DATA_HOME", "XDG_RUNTIME_DIR", "XDG_CONFIG_HOME"]
15287 .into_iter()
15288 .map(|key| (key.into(), dir.to_string_lossy().into_owned()))
15289 .collect(),
15290 reserved: false,
15291 reserved_prefixes: vec![],
15292 protocol: ModuleProtocol::None,
15293 overlap: Default::default(),
15294 };
15295 let spawn = |spec: &ModuleSpec| {
15296 spawn_child(
15297 spec,
15298 None,
15299 None,
15300 &runtime.stderr_ring,
15301 None,
15302 &runtime.child_roster,
15303 Some(&placement),
15304 )
15305 .unwrap()
15306 };
15307 let mut live = spawn(&spec);
15308 for _ in 0..3 {
15309 let next = spawn(&spec);
15311 assert_ne!(live.module_id, next.module_id);
15312 live.start_kill().unwrap();
15313 live.wait().await.unwrap();
15314 live = next;
15315 assert!(
15316 live.child.try_wait().unwrap().is_none(),
15317 "retiring the old slot must not kill the replacement"
15318 );
15319 assert_eq!(
15320 group_count(),
15321 1,
15322 "only the live spawn's cgroup should remain"
15323 );
15324 }
15325 supervisor.begin_daemon_shutdown();
15326 let reap = tokio::spawn(async move {
15327 live.wait().await.unwrap();
15328 });
15329 supervisor
15330 .end_children_for_daemon_shutdown(false, std::future::pending())
15331 .await;
15332 reap.await.unwrap();
15333 assert_eq!(group_count(), 0);
15334 spec.program = PathBuf::from("/bin/true");
15336 spec.args.clear();
15337 let fresh_roster = ChildRoster::default();
15338 let mut short = spawn_child(
15339 &spec,
15340 None,
15341 None,
15342 &runtime.stderr_ring,
15343 None,
15344 &fresh_roster,
15345 Some(&placement),
15346 )
15347 .unwrap();
15348 short.wait().await.unwrap();
15349 assert_eq!(group_count(), 0);
15350 spec.module_id = "_".repeat(255);
15351 let mut long_id = spawn_child(
15352 &spec,
15353 None,
15354 None,
15355 &runtime.stderr_ring,
15356 None,
15357 &fresh_roster,
15358 Some(&placement),
15359 )
15360 .unwrap();
15361 long_id.wait().await.unwrap();
15362 assert_eq!(
15363 group_count(),
15364 0,
15365 "valid long module IDs must not exceed cgroup NAME_MAX"
15366 );
15367 fs::remove_dir(root.join("subc-modules")).unwrap();
15368 fs::remove_dir(root).unwrap();
15369 eprintln!("EXECUTED unique_spawn_cgroups_clean_up_across_restarts_and_shutdown");
15370 }
15371
15372 #[test]
15373 fn failed_parent_cgroup_open_is_a_cgroup_supervision_error() {
15374 let path = Path::new("/definitely-missing-subc-cgroup");
15375 let mut command = Command::new("true");
15376 let error = apply_cgroup_placement(
15377 &mut command,
15378 &ModuleSpec {
15379 module_id: "broken-cgroup".to_string(),
15380 program: PathBuf::from("true"),
15381 args: Vec::new(),
15382 env: Vec::new(),
15383 reserved: false,
15384 reserved_prefixes: Vec::new(),
15385 protocol: ModuleProtocol::Subc,
15386 overlap: Default::default(),
15387 },
15388 path,
15389 )
15390 .expect_err("a parent cgroup open failure must reject the supervised spawn");
15391 let reason = error.to_string();
15392
15393 assert!(
15394 matches!(error, SuperviseError::Cgroup { .. }),
15395 "parent cgroup open must be reported as a cgroup supervision error: {reason}"
15396 );
15397 assert!(
15398 reason.contains("/definitely-missing-subc-cgroup/cgroup.procs"),
15399 "parent cgroup open failure must name cgroup.procs: {reason}"
15400 );
15401 }
15402
15403 #[tokio::test]
15404 async fn reaping_a_child_removes_its_empty_module_cgroup() {
15405 let root = ScratchDir::new("supervisor-reap-cgroup");
15406 fs::write(root.join("cgroup.procs"), b"").expect("write scratch cgroup marker");
15407 let placement = subc_cgroup::prepare_at(&root)
15408 .expect("prepare scratch cgroup root")
15409 .expect("scratch root has a cgroup.procs marker");
15410 let module_id = "reaped-module";
15411 let module = placement
15412 .module_path(module_id)
15413 .expect("create scratch module cgroup");
15414 let child = Command::new("true")
15415 .env("XDG_DATA_HOME", root.path())
15416 .env("XDG_RUNTIME_DIR", root.path())
15417 .env("XDG_CONFIG_HOME", root.path())
15418 .spawn()
15419 .expect("spawn short-lived child");
15420 let pid = child.id().expect("spawned child has pid");
15421 let mut child = SupervisedChild {
15422 child,
15423 exec: crate::activity::ExecOwner(Arc::new(crate::activity::ExecOutcome::new())),
15424 protocol: ModuleProtocol::Subc,
15425 module_id: module_id.to_string(),
15426 cgroup_placement: Some(placement),
15427 stdout_pump: None,
15428 stderr_pump: None,
15429 stderr_ring: Arc::new(Mutex::new(StderrRing::new(StderrTailConfig::default()))),
15430 spawned_at: tokio::time::Instant::now(),
15431 spawned_at_ms: 0,
15432 spawned_from: PathBuf::from("true"),
15433 launch_spec_digest: LaunchSpecDigest([0; 32]),
15434 spawned_file_identity: None,
15435 process_start_time: None,
15436 process_identity: None,
15437 pid,
15438 roster_guard: None,
15439 #[cfg(target_os = "macos")]
15440 privacy_exec: None,
15441 spawn_failure: None,
15442 };
15443
15444 child.wait().await.expect("reap short-lived child");
15445
15446 assert!(
15447 !module.exists(),
15448 "reaping the supervised child must remove its empty cgroup"
15449 );
15450 }
15451
15452 #[test]
15453 fn non_empty_cgroup_removal_is_reported_without_blocking_teardown() {
15454 let root = ScratchDir::new("supervisor-non-empty-cgroup");
15455 fs::write(root.join("cgroup.procs"), b"").expect("write scratch cgroup marker");
15456 let placement = subc_cgroup::prepare_at(&root)
15457 .expect("prepare scratch cgroup root")
15458 .expect("scratch root has a cgroup.procs marker");
15459 let module = placement
15460 .module_path("surviving-module")
15461 .expect("create scratch module cgroup");
15462 fs::write(module.join("surviving-process"), b"still present")
15463 .expect("make scratch cgroup non-empty");
15464 let (logs, _guard) = crate::router::test_log::log_capture(tracing::Level::WARN);
15465
15466 remove_module_cgroup(&placement, "surviving-module");
15467
15468 let logs = crate::router::test_log::captured_logs(&logs);
15469 assert!(
15470 module.exists(),
15471 "failed removal must leave the cgroup intact"
15472 );
15473 assert!(
15474 logs.contains("could not remove module cgroup after process exit; continuing teardown")
15475 && logs.contains("surviving-module"),
15476 "best-effort removal must report the failure without returning it: {logs}"
15477 );
15478 }
15479
15480 #[test]
15481 fn cgroup_pre_exec_spawn_failure_names_the_cgroup_path() {
15482 let cgroup_path = PathBuf::from("/sys/fs/cgroup/subc-modules/broken-module");
15483 let reason = SuperviseError::Spawn {
15484 program: PathBuf::from("/bin/true"),
15485 source: io::Error::from_raw_os_error(13),
15486 cgroup_path: Some(cgroup_path.clone()),
15487 }
15488 .to_string();
15489
15490 assert!(
15491 reason.contains(&cgroup_path.display().to_string()),
15492 "a pre_exec spawn failure must name the cgroup path: {reason}"
15493 );
15494 }
15495}
15496
15497#[cfg(test)]
15498mod spawn_subscriber_lag_tests {
15499 use super::*;
15500
15501 #[tokio::test]
15506 async fn lagged_spawn_subscriber_receives_a_terminal_lagged_error_after_its_queued_frames() {
15507 let feed = SpawnEventFeed::default();
15508 feed.configure_incarnation("lag-incarnation".to_string());
15509 let (tx, mut rx) = mpsc::channel(1);
15512 feed.subscribe(ConnectionId::new(1), 7, 1, None, FrameSink::new(tx))
15513 .expect("subscribe");
15514 let emitted = SPAWN_SUBSCRIBER_BUFFER + 16;
15515 for index in 0..emitted {
15516 feed.emit_spawned(&format!("lag-module-{index}"), 1000, 0);
15517 tokio::task::yield_now().await;
15520 }
15521 assert_eq!(
15522 feed.subscriber_count(),
15523 0,
15524 "the lagged subscriber must be removed"
15525 );
15526
15527 let mut data = Vec::new();
15528 let mut last = None;
15529 loop {
15530 let next = tokio::time::timeout(Duration::from_secs(5), rx.recv())
15531 .await
15532 .expect("the forwarder must finish once the subscriber is dropped");
15533 let Some(outbound) = next else { break };
15534 let frame = outbound.frame;
15535 if frame.header.ty == FrameType::StreamData {
15536 assert!(last.is_none(), "no data may follow the terminal frame");
15537 let event: SpawnEvent = serde_json::from_slice(&frame.body).unwrap();
15538 data.push(event.cursor.seq);
15539 } else {
15540 assert!(last.is_none(), "exactly one terminal frame");
15541 last = Some(frame);
15542 }
15543 }
15544 assert!(!data.is_empty(), "queued frames drain before the terminal");
15545 for pair in data.windows(2) {
15546 assert_eq!(
15547 pair[1],
15548 pair[0] + 1,
15549 "queued frames arrive dense and in order"
15550 );
15551 }
15552 let terminal = last.expect("a lagged subscriber must receive a terminal frame");
15553 assert_eq!(terminal.header.ty, FrameType::Error);
15554 assert_eq!(terminal.header.corr, 7);
15555 let body: subc_protocol::ErrorBody = serde_json::from_slice(&terminal.body).unwrap();
15556 assert_eq!(body.code, SPAWN_SUBSCRIBER_LAGGED_CODE);
15557 let detail = body.detail.expect("lagged error carries detail");
15558 assert_eq!(
15559 detail["first_undelivered_cursor"]["seq"],
15560 data.last().unwrap() + 1,
15561 "the named cursor is the first event the subscriber did not receive"
15562 );
15563 assert_eq!(
15564 detail["first_undelivered_cursor"]["daemon_incarnation"],
15565 "lag-incarnation"
15566 );
15567 }
15568}
15569
15570#[cfg(test)]
15571mod terminal_history_read_concurrency_tests {
15572 use super::*;
15573 use crate::terminal_journal::read_pause;
15574 use cortexkit_test_support::ScratchDir;
15575 use std::sync::mpsc as std_mpsc;
15576
15577 fn journaled_ring(
15578 journal: &Arc<crate::terminal_journal::TerminalJournal>,
15579 ) -> Arc<Mutex<TerminalRing>> {
15580 Arc::new(Mutex::new(
15581 TerminalRing::new(TerminalRingConfig::default(), 1)
15582 .with_journal(Some(Arc::clone(journal))),
15583 ))
15584 }
15585
15586 fn crash(at_ms: u64) -> ExitReport {
15587 ExitReport {
15588 kind: ExitKind::Crash,
15589 code: Some(1),
15590 signal: None,
15591 at_ms,
15592 }
15593 }
15594
15595 fn record_within(
15598 module_id: &'static str,
15599 ring: &Arc<Mutex<TerminalRing>>,
15600 at_ms: u64,
15601 bound: Duration,
15602 ) -> bool {
15603 let ring = Arc::clone(ring);
15604 let (done, done_rx) = std_mpsc::channel();
15605 std::thread::spawn(move || {
15606 record_terminal(
15607 module_id,
15608 &ring,
15609 &SpawnEventFeed::default(),
15610 &crash(at_ms),
15611 TerminalDisposition::Restarting,
15612 );
15613 let _ = done.send(());
15614 });
15615 done_rx.recv_timeout(bound).is_ok()
15616 }
15617
15618 #[test]
15623 fn exits_recorded_during_a_paused_history_read_are_not_blocked_or_half_merged() {
15624 let dir = ScratchDir::new("terminal-history-concurrent-read");
15625 let path = dir.join("terminals.jsonl");
15626 let journal = Arc::new(crate::terminal_journal::TerminalJournal::open(
15627 path.clone(),
15628 "daemon".into(),
15629 ));
15630 let reader_ring = journaled_ring(&journal);
15631 let other_ring = journaled_ring(&journal);
15632 assert!(record_within(
15633 "reader-module",
15634 &reader_ring,
15635 10,
15636 Duration::from_secs(5)
15637 ));
15638
15639 let (started, release) = read_pause::install(&path);
15640 let reading = {
15641 let ring = Arc::clone(&reader_ring);
15642 std::thread::spawn(move || durable_terminal_history_of(&ring, "reader-module"))
15643 };
15644 started
15645 .recv_timeout(Duration::from_secs(5))
15646 .expect("the history read reached its pause");
15647
15648 let bound = Duration::from_secs(1);
15649 assert!(
15650 record_within("other-module", &other_ring, 20, bound),
15651 "another module's exit waited on a history read (journal writer held)"
15652 );
15653 assert!(
15654 record_within("reader-module", &reader_ring, 30, bound),
15655 "the read module's own exit waited on its history read (ring held)"
15656 );
15657
15658 drop(release);
15659 let paused = reading.join().unwrap();
15660 assert_eq!(
15661 paused.entries.iter().map(|e| e.at_ms).collect::<Vec<_>>(),
15662 vec![10],
15663 "an exit recorded after the read began lands in neither half of it"
15664 );
15665 assert_eq!(paused.journal_skipped_lines, 0);
15666 assert_eq!(paused.journal_read_errors, 0);
15667
15668 let after = durable_terminal_history_of(&reader_ring, "reader-module");
15669 assert_eq!(
15670 after.entries.iter().map(|e| e.at_ms).collect::<Vec<_>>(),
15671 vec![10, 30],
15672 "the next read merges ring and journal with no duplicate"
15673 );
15674 assert_eq!(after.journal_skipped_lines, 0);
15675 }
15676}
15677
15678#[cfg(test)]
15683mod stderr_settle_tests {
15684 use std::{
15685 future::Future,
15686 io,
15687 pin::Pin,
15688 sync::{Arc, Mutex},
15689 task::{Context, Poll},
15690 time::Duration,
15691 };
15692
15693 use tokio::{
15694 io::{AsyncRead, ReadBuf},
15695 sync::oneshot,
15696 time::Instant,
15697 };
15698
15699 use super::{settle_stderr_pump, settle_stdout_pump, StderrPump};
15700 use crate::stderr_tail::{
15701 pump_stderr_to, pump_stdout_to, split_capture_stamp, untimed, CaptureState,
15702 ChildOutputSink, OutputSink, StderrRing, StderrTailConfig, TailEntry,
15703 };
15704
15705 const BOUND: Duration = Duration::from_millis(250);
15706
15707 struct HeldReader {
15711 before: Option<Vec<u8>>,
15712 gate: Option<oneshot::Receiver<()>>,
15713 after: io::Cursor<Vec<u8>>,
15714 }
15715
15716 impl AsyncRead for HeldReader {
15717 fn poll_read(
15718 mut self: Pin<&mut Self>,
15719 cx: &mut Context<'_>,
15720 buf: &mut ReadBuf<'_>,
15721 ) -> Poll<io::Result<()>> {
15722 if let Some(bytes) = self.before.take() {
15723 buf.put_slice(&bytes);
15724 return Poll::Ready(Ok(()));
15725 }
15726 if let Some(gate) = self.gate.as_mut() {
15727 match Pin::new(gate).poll(cx) {
15728 Poll::Pending => return Poll::Pending,
15729 Poll::Ready(_) => self.gate = None,
15730 }
15731 }
15732 Pin::new(&mut self.after).poll_read(cx, buf)
15733 }
15734 }
15735
15736 struct DiscardSink;
15737
15738 impl OutputSink for DiscardSink {
15739 fn write_line(&mut self, _line: &[u8]) {}
15740 }
15741
15742 fn line(text: &str) -> TailEntry {
15743 TailEntry::Line {
15744 text: text.to_string(),
15745 truncated: false,
15746 at_ms: None,
15747 }
15748 }
15749
15750 fn lock(ring: &Arc<Mutex<StderrRing>>) -> std::sync::MutexGuard<'_, StderrRing> {
15751 ring.lock().unwrap()
15752 }
15753
15754 fn held_pump(
15758 ring: &Arc<Mutex<StderrRing>>,
15759 before: &str,
15760 after: &str,
15761 ) -> (StderrPump, oneshot::Sender<()>) {
15762 let generation = lock(ring).begin_process();
15763 let (release, gate) = oneshot::channel();
15764 let reader = HeldReader {
15765 before: Some(before.as_bytes().to_vec()),
15766 gate: Some(gate),
15767 after: io::Cursor::new(after.as_bytes().to_vec()),
15768 };
15769 let task = tokio::spawn(pump_stderr_to(
15770 reader,
15771 Arc::clone(ring),
15772 generation,
15773 DiscardSink,
15774 ));
15775 (StderrPump { task, generation }, release)
15776 }
15777
15778 async fn wait_until(ring: &Arc<Mutex<StderrRing>>, done: impl Fn(&StderrRing) -> bool) {
15779 for _ in 0..1000 {
15780 if done(&lock(ring)) {
15781 return;
15782 }
15783 tokio::time::sleep(Duration::from_millis(1)).await;
15784 }
15785 panic!(
15786 "ring never reached the expected state: {:?}",
15787 lock(ring).snapshot(None, None)
15788 );
15789 }
15790
15791 #[tokio::test(start_paused = true)]
15792 async fn a_crash_line_the_reader_had_not_reached_by_the_bound_is_kept_before_the_restart() {
15793 let ring = Arc::new(Mutex::new(StderrRing::new(StderrTailConfig::default())));
15794 let (pump, release) = held_pump(&ring, "booting\n", "config error: missing storage\n");
15795
15796 settle_stderr_pump("crasher", &ring, pump, BOUND).await;
15797 let before_release = lock(&ring).snapshot(None, None);
15798 assert!(
15799 matches!(before_release.capture, CaptureState::Incomplete { .. }),
15800 "a reader that has not reached EOF cannot claim a whole tail: {before_release:?}"
15801 );
15802
15803 let next = lock(&ring).begin_process();
15806 lock(&ring).push_line_from(next, "next process booting");
15807 release.send(()).unwrap();
15808 wait_until(&ring, |ring| {
15809 ring.snapshot(None, None).capture == CaptureState::Captured
15810 })
15811 .await;
15812
15813 assert_eq!(
15814 untimed(lock(&ring).snapshot(None, None).entries),
15815 vec![
15816 line("booting"),
15817 line("config error: missing storage"),
15818 TailEntry::ProcessStart,
15819 line("next process booting"),
15820 ],
15821 "the crash's last line must survive a slow reader and stay in the crashed process's section"
15822 );
15823 }
15824
15825 #[tokio::test(start_paused = true)]
15826 async fn a_pipe_held_open_by_a_descendant_reads_incomplete_without_delaying_the_restart_past_the_bound(
15827 ) {
15828 let ring = Arc::new(Mutex::new(StderrRing::new(StderrTailConfig::default())));
15829 let (pump, _held) = held_pump(&ring, "parent exiting\n", "");
15832
15833 let started = Instant::now();
15834 settle_stderr_pump("orphaning", &ring, pump, BOUND).await;
15835 assert_eq!(
15836 started.elapsed(),
15837 BOUND,
15838 "the restart must wait exactly the bound for a pipe that stays open, no longer"
15839 );
15840
15841 let next = lock(&ring).begin_process();
15842 lock(&ring).push_line_from(next, "next process booting");
15843 tokio::time::sleep(Duration::from_secs(60)).await;
15844
15845 let snapshot = lock(&ring).snapshot(None, None);
15846 match &snapshot.capture {
15847 CaptureState::Incomplete { reason } => assert!(
15848 reason.contains("had not reached EOF") && reason.contains("250ms"),
15849 "the reason must say what is missing and after how long: {reason}"
15850 ),
15851 other => panic!("expected Incomplete while the pipe is held open, got {other:?}"),
15852 }
15853 assert_eq!(
15854 untimed(snapshot.entries),
15855 vec![
15856 line("parent exiting"),
15857 TailEntry::ProcessStart,
15858 line("next process booting"),
15859 ]
15860 );
15861 }
15862
15863 #[tokio::test(start_paused = true)]
15864 async fn a_reader_that_reaches_eof_within_the_bound_leaves_the_tail_captured() {
15865 let ring = Arc::new(Mutex::new(StderrRing::new(StderrTailConfig::default())));
15866 let (pump, release) = held_pump(&ring, "one\n", "two\n");
15867 release.send(()).unwrap();
15868
15869 settle_stderr_pump("clean", &ring, pump, BOUND).await;
15870
15871 let snapshot = lock(&ring).snapshot(None, None);
15872 assert_eq!(snapshot.capture, CaptureState::Captured);
15873 assert_eq!(untimed(snapshot.entries), vec![line("one"), line("two")]);
15874 }
15875
15876 #[tokio::test(start_paused = true)]
15877 async fn stdout_buffered_past_the_exit_drain_bound_reaches_the_capture_file() {
15878 const BURST: usize = 150;
15879 let capture = cortexkit_test_support::ScratchDir::new("stdout-exit-drain");
15880 let path = capture.join("two-pipe.stderr.log");
15881 let sink = ChildOutputSink::open(&path, cortexkit_log::Retention::default()).unwrap();
15882 let ring = Arc::new(Mutex::new(StderrRing::new(StderrTailConfig::default())));
15883 let generation = lock(&ring).begin_process();
15884 let stdout = (0..BURST)
15885 .map(|i| format!("out-{i:04}-{}\n", "o".repeat(64)))
15886 .collect::<String>();
15887 let stderr = (0..BURST)
15888 .map(|i| format!("err-{i:04}-{}\n", "e".repeat(64)))
15889 .collect::<String>();
15890 let first_chunk = stdout.find('\n').unwrap() + 1 + "out-0001-".len();
15895 let (release, gate) = oneshot::channel();
15896 let reader = HeldReader {
15897 before: Some(stdout.as_bytes()[..first_chunk].to_vec()),
15898 gate: Some(gate),
15899 after: io::Cursor::new(stdout.as_bytes()[first_chunk..stdout.len() - 1].to_vec()),
15900 };
15901 let (finished, completion) = oneshot::channel();
15902 let stdout_sink = sink.clone();
15903 let pump = tokio::spawn(async move {
15904 pump_stdout_to(reader, stdout_sink).await;
15905 finished.send(()).unwrap();
15906 });
15907 pump_stderr_to(io::Cursor::new(stderr.into_bytes()), ring, generation, sink).await;
15908
15909 let started = Instant::now();
15910 settle_stdout_pump("two-pipe", pump, BOUND).await;
15911 assert_eq!(
15912 started.elapsed(),
15913 BOUND,
15914 "a late reader must not hold up restart"
15915 );
15916 assert_eq!(
15917 std::fs::read_to_string(&path).unwrap().lines().count(),
15918 BURST + 1
15919 );
15920
15921 release
15922 .send(())
15923 .expect("stdout reader must survive the exit drain bound");
15924 tokio::time::timeout(Duration::from_secs(1), completion)
15925 .await
15926 .expect("released stdout reader must reach EOF")
15927 .expect("stdout capture must finish at EOF, not be cancelled");
15928 let contents = std::fs::read_to_string(&path).unwrap();
15929 let mut lines = contents
15930 .lines()
15931 .map(|line| split_capture_stamp(line).expect("capture stamp").1)
15932 .collect::<Vec<_>>();
15933 let mut expected = (0..BURST)
15934 .flat_map(|i| {
15935 [
15936 format!("out-{i:04}-{}", "o".repeat(64)),
15937 format!("err-{i:04}-{}", "e".repeat(64)),
15938 ]
15939 })
15940 .collect::<Vec<_>>();
15941 lines.sort_unstable();
15942 expected.sort_unstable();
15943 assert_eq!(
15944 lines, expected,
15945 "both pipes must retain every complete line exactly once"
15946 );
15947 }
15948}
15949
15950#[cfg(all(test, windows))]
15964mod job_containment_tests {
15965 use super::*;
15966 use cortexkit_test_support::ScratchDir;
15967 use std::{
15968 path::{Path, PathBuf},
15969 sync::{Arc, Mutex},
15970 time::{Duration, Instant},
15971 };
15972
15973 fn stub_path() -> PathBuf {
15979 let mut path = std::env::current_exe().expect("current_exe available in tests");
15980 path.pop();
15981 path.pop();
15982 path.push("fake-aft-stub.exe");
15983 assert!(
15984 path.exists(),
15985 "fake-aft-stub not built at {}: run `cargo test -p subc-core` (which builds \
15986 [[bin]] targets) rather than `cargo test -p subc-core --lib` (which does not)",
15987 path.display()
15988 );
15989 path
15990 }
15991
15992 fn read_grandchild_pid(path: &Path) -> u32 {
15994 let deadline = Instant::now() + Duration::from_secs(10);
15995 loop {
15996 if let Ok(contents) = std::fs::read_to_string(path) {
15997 if let Ok(pid) = contents.trim().parse() {
15998 return pid;
15999 }
16000 }
16001 assert!(
16002 Instant::now() < deadline,
16003 "the stub never recorded a grandchild pid at {}",
16004 path.display()
16005 );
16006 std::thread::sleep(Duration::from_millis(10));
16007 }
16008 }
16009
16010 struct Fixture {
16013 _dir: ScratchDir,
16014 module_id: String,
16015 grandchild: u32,
16016 child: Option<SupervisedChild>,
16017 registry: Arc<Registry>,
16018 snapshot: Arc<Mutex<SupervisorSnapshot>>,
16019 terminal_ring: Arc<Mutex<TerminalRing>>,
16020 spawn_events: SpawnEventFeed,
16021 }
16022
16023 fn fixture(label: &str, module_id: &str) -> Fixture {
16024 let dir = ScratchDir::new(label);
16025 let pid_file = dir.join("grandchild.pid");
16026 let supervisor = Supervisor::new_for_test(
16027 Arc::new(Registry::default()),
16028 RestartPolicy::new(3, Duration::ZERO),
16029 );
16030 let runtime = supervisor.runtime_config();
16031 let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::starting()));
16032 let spec = ModuleSpec {
16033 module_id: module_id.to_string(),
16034 program: stub_path(),
16035 args: Vec::new(),
16039 env: vec![
16040 ("FAKE_AFT_NEVER_CONNECT".to_string(), "1".to_string()),
16041 (
16042 "FAKE_AFT_GRANDCHILD_PID_FILE".to_string(),
16043 pid_file.display().to_string(),
16044 ),
16045 ],
16046 reserved: false,
16047 reserved_prefixes: Vec::new(),
16048 protocol: ModuleProtocol::Subc,
16049 overlap: Default::default(),
16050 };
16051 let child = spawn_and_mark_running(&spec, &runtime, &snapshot)
16052 .expect("spawn the supervised fixture");
16053 let grandchild = read_grandchild_pid(&pid_file);
16054 Fixture {
16055 _dir: dir,
16056 module_id: module_id.to_string(),
16057 grandchild,
16058 child: Some(child),
16059 registry: Arc::new(Registry::default()),
16060 snapshot,
16061 terminal_ring: Arc::clone(&runtime.terminal_ring),
16062 spawn_events: SpawnEventFeed::default(),
16063 }
16064 }
16065
16066 impl Fixture {
16067 async fn drain(&mut self) {
16069 let child = self
16070 .child
16071 .take()
16072 .expect("the fixture child is still present");
16073 drain_child_to_state(
16074 &self.module_id,
16075 ModuleProtocol::Subc,
16076 StopNotice::NotSent,
16079 &self.registry,
16080 None,
16081 &self.snapshot,
16082 &self.terminal_ring,
16083 &self.spawn_events,
16084 child,
16085 Duration::from_millis(500),
16086 ModuleState::Stopped,
16087 Some(false),
16088 )
16089 .await
16090 .expect("drain the supervised fixture");
16091 }
16092 }
16093
16094 #[tokio::test]
16100 async fn teardown_reaps_the_grandchild() {
16101 let mut fixture = fixture("teardown-grandchild", "tree-teardown");
16102 let grandchild = fixture.grandchild;
16103
16104 assert!(
16105 subc_jobobject::process_exists(grandchild),
16106 "grandchild {grandchild} must be alive before teardown, or this proves nothing"
16107 );
16108
16109 fixture.drain().await;
16110
16111 assert!(
16112 subc_jobobject::wait_for_process_exit(grandchild, Duration::from_secs(10)),
16113 "grandchild {grandchild} outlived module teardown: the tree was not contained"
16114 );
16115 }
16116
16117 #[test]
16130 fn an_uncontained_grandchild_survives_a_direct_child_kill() {
16131 let dir = ScratchDir::new("teardown-uncontained");
16132 let pid_file = dir.join("grandchild.pid");
16133 let mut child = std::process::Command::new(stub_path())
16134 .env("FAKE_AFT_NEVER_CONNECT", "1")
16135 .env(
16136 "FAKE_AFT_GRANDCHILD_PID_FILE",
16137 pid_file.display().to_string(),
16138 )
16139 .stdin(std::process::Stdio::null())
16140 .stdout(std::process::Stdio::null())
16141 .stderr(std::process::Stdio::null())
16142 .spawn()
16143 .expect("spawn the uncontained fixture");
16144 let grandchild = read_grandchild_pid(&pid_file);
16145
16146 child.kill().expect("kill the direct child");
16148 let _ = child.wait();
16149
16150 assert!(
16151 subc_jobobject::process_exists(grandchild),
16152 "grandchild {grandchild} died with the direct child, so this control no longer \
16153 distinguishes contained from uncontained teardown and the regression test is \
16154 passing vacuously"
16155 );
16156
16157 kill_tree(grandchild);
16160 }
16161
16162 #[tokio::test]
16171 async fn dropping_containment_reaps_the_grandchild() {
16172 let mut fixture = fixture("drop-containment", "tree-drop");
16173 let grandchild = fixture.grandchild;
16174
16175 assert!(subc_jobobject::process_exists(grandchild));
16176
16177 fixture.child.as_mut().expect("child present").job = None;
16179
16180 assert!(
16181 subc_jobobject::wait_for_process_exit(grandchild, Duration::from_secs(10)),
16182 "grandchild {grandchild} survived the containment handle closing, so a daemon \
16183 crash would leave the tree behind"
16184 );
16185 }
16186
16187 fn kill_tree(pid: u32) {
16189 let _ = std::process::Command::new("taskkill.exe")
16190 .args(["/PID", &pid.to_string(), "/T", "/F"])
16191 .stdin(std::process::Stdio::null())
16192 .stdout(std::process::Stdio::null())
16193 .stderr(std::process::Stdio::null())
16194 .status();
16195 assert!(
16196 subc_jobobject::wait_for_process_exit(pid, Duration::from_secs(10)),
16197 "could not clean up grandchild {pid}"
16198 );
16199 }
16200}
16201
16202#[cfg(test)]
16203mod privacy_trampoline_configuration_tests {
16204 #[tokio::test]
16205 #[cfg_attr(not(target_os = "macos"), ignore = "requires macOS privacy trampoline")]
16206 async fn macos_spawn_without_a_configured_trampoline_refuses_by_name() {
16207 #[cfg(target_os = "macos")]
16208 {
16209 let supervisor = super::Supervisor::new(
16210 std::sync::Arc::new(crate::Registry::default()),
16211 super::RestartPolicy::default(),
16212 );
16213 let error = supervisor.spawn(spec()).unwrap_err();
16214 assert!(
16215 error
16216 .to_string()
16217 .contains("no privacy trampoline configured"),
16218 "{error}"
16219 );
16220 }
16221 }
16222
16223 #[tokio::test]
16224 #[cfg_attr(not(target_os = "macos"), ignore = "requires macOS privacy trampoline")]
16225 async fn macos_wrong_trampoline_is_rejected_by_the_startup_probe() {
16226 #[cfg(target_os = "macos")]
16227 {
16228 let supervisor = super::Supervisor::new(
16229 std::sync::Arc::new(crate::Registry::default()),
16230 super::RestartPolicy::default(),
16231 )
16232 .with_privacy_trampoline(std::env::current_exe().unwrap());
16233 let error = supervisor.spawn(spec()).unwrap_err();
16234 assert!(
16235 error
16236 .to_string()
16237 .contains("binary does not implement the privacy trampoline protocol"),
16238 "{error}"
16239 );
16240 }
16241 }
16242
16243 #[cfg(target_os = "macos")]
16244 fn spec() -> super::ModuleSpec {
16245 super::ModuleSpec {
16246 module_id: "privacy-configuration".into(),
16247 program: "/bin/sleep".into(),
16248 args: vec!["30".into()],
16249 env: vec![],
16250 reserved: false,
16251 reserved_prefixes: vec![],
16252 protocol: subc_control::ModuleProtocol::None,
16253 overlap: super::ModuleOverlap::Exclusive,
16254 }
16255 }
16256}
16257
16258#[cfg(test)]
16259mod privacy_exec_boundary_tests {
16260 #[cfg(target_os = "macos")]
16261 use super::*;
16262 #[cfg(target_os = "macos")]
16263 use std::{
16264 io::{Read, Write},
16265 net::{TcpListener, TcpStream},
16266 };
16267
16268 #[cfg(target_os = "macos")]
16271 pub(super) fn before_image_sample(spec: &ModuleSpec, pid: u32) {
16272 if let Some((_, path)) = spec
16273 .env
16274 .iter()
16275 .find(|(key, _)| key == "SUBC_TEST_PRIVACY_SAMPLE_BARRIER")
16276 {
16277 let mut barrier = TcpStream::connect(path).unwrap();
16278 barrier
16279 .set_read_timeout(Some(Duration::from_secs(30)))
16280 .unwrap();
16281 barrier.write_all(&pid.to_ne_bytes()).unwrap();
16282 let mut release = [0];
16283 barrier.read_exact(&mut release).unwrap();
16284 assert_eq!(&release, b"X");
16285 }
16286 }
16287
16288 #[cfg(target_os = "macos")]
16289 fn spec(program: &str, args: &[&str]) -> ModuleSpec {
16290 ModuleSpec {
16291 module_id: "privacy-boundary".into(),
16292 program: program.into(),
16293 args: args.iter().map(|arg| (*arg).into()).collect(),
16294 env: vec![],
16295 reserved: false,
16296 reserved_prefixes: vec![],
16297 protocol: ModuleProtocol::None,
16298 overlap: ModuleOverlap::Exclusive,
16299 }
16300 }
16301
16302 #[cfg(target_os = "macos")]
16303 async fn accept(listener: TcpListener) -> TcpStream {
16304 let listener = tokio::net::TcpListener::from_std({
16306 listener.set_nonblocking(true).unwrap();
16307 listener
16308 })
16309 .unwrap();
16310 let (stream, _) = tokio::time::timeout(Duration::from_secs(30), listener.accept())
16311 .await
16312 .unwrap()
16313 .unwrap();
16314 let stream = stream.into_std().unwrap();
16315 stream.set_nonblocking(false).unwrap();
16316 stream
16317 .set_read_timeout(Some(Duration::from_secs(30)))
16318 .unwrap();
16319 stream
16320 }
16321
16322 #[tokio::test]
16323 #[cfg_attr(not(target_os = "macos"), ignore = "requires macOS privacy trampoline")]
16324 async fn macos_roster_withholds_a_nonnull_trampoline_image_until_exec_confirmation() {
16325 #[cfg(target_os = "macos")]
16326 {
16327 let root = cortexkit_test_support::ScratchDir::new("privacy-roster-barrier");
16328 let exec_listener = TcpListener::bind("127.0.0.1:0").unwrap();
16331 let sample_listener = TcpListener::bind("127.0.0.1:0").unwrap();
16332 let record = root.join("live-children.json");
16333 let supervisor =
16334 Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default())
16335 .with_live_children_record(&record);
16336 let runtime = supervisor.runtime_config();
16337 let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::starting()));
16338 let mut spec = spec("/bin/sleep", &["30"]);
16339 spec.env = vec![
16340 (
16341 "SUBC_TEST_PRIVACY_EXEC_BARRIER".into(),
16342 exec_listener.local_addr().unwrap().to_string(),
16343 ),
16344 (
16345 "SUBC_TEST_PRIVACY_SAMPLE_BARRIER".into(),
16346 sample_listener.local_addr().unwrap().to_string(),
16347 ),
16348 ];
16349 let spawn = tokio::task::spawn_blocking(move || {
16350 spawn_and_mark_running(&spec, &runtime, &snapshot).unwrap()
16351 });
16352 let mut sample = accept(sample_listener).await;
16353 let mut pid = [0; 4];
16354 sample.read_exact(&mut pid).unwrap();
16355 let pid = u32::from_ne_bytes(pid);
16356 let mut exec = accept(exec_listener).await;
16357 let mut ready = [0];
16358 exec.read_exact(&mut ready).unwrap();
16359 assert_eq!(&ready, b"R");
16360 let trampoline = subc_os::file_identity(&test_privacy_trampoline()).unwrap();
16363 assert_eq!(
16364 observe_spawned_image(pid).unwrap().executable,
16365 Some(trampoline)
16366 );
16367 sample.write_all(b"X").unwrap();
16368 let mut child = spawn.await.unwrap();
16369 let early = crate::live_children::read_record(&record).unwrap();
16370 assert_eq!(early.len(), 1);
16371 assert_eq!(early[0].pid, pid);
16372 assert_eq!(
16373 early[0].executable, None,
16374 "unconfirmed trampoline image entered the roster"
16375 );
16376 assert!(child.report_ready.get().is_none());
16377 child.privacy_exec.as_mut().unwrap().deadline =
16380 tokio::time::Instant::now() + Duration::from_secs(30);
16381 exec.write_all(b"X").unwrap();
16382 child.confirm_privacy_exec().await;
16383 assert_eq!(child.spawn_failure, None);
16384 assert!(child.report_ready.get().is_some());
16385 let confirmed = crate::live_children::read_record(&record).unwrap();
16386 let module = subc_os::file_identity(std::path::Path::new("/bin/sleep")).unwrap();
16387 assert_ne!(module, trampoline);
16388 assert_eq!(confirmed[0].executable, Some(module.into()));
16389 child.start_kill().unwrap();
16390 child.wait().await.unwrap();
16391 child.release_roster();
16392 }
16393 }
16394
16395 #[tokio::test]
16396 #[cfg_attr(not(target_os = "macos"), ignore = "requires macOS privacy trampoline")]
16397 async fn macos_already_exited_121_in_try_wait_is_a_module_exit_not_a_trampoline_refusal() {
16398 #[cfg(target_os = "macos")]
16399 {
16400 let registry = Arc::new(Registry::default());
16401 let policy = RestartPolicy::new(0, Duration::ZERO);
16402 let supervisor = Supervisor::new_for_test(Arc::clone(®istry), policy);
16403 let runtime = supervisor.runtime_config();
16404 let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::starting()));
16405 let spec = spec("/bin/sh", &["-c", "exit 121"]);
16406 let mut child = spawn_and_mark_running(&spec, &runtime, &snapshot).unwrap();
16410 let pid = child.pid;
16411 tokio::task::spawn_blocking(move || {
16412 subc_os::fork_exec_test::wait_for_child_exit_without_reaping(pid)
16413 })
16414 .await
16415 .unwrap()
16416 .unwrap();
16417 child.privacy_exec.as_mut().unwrap().deadline =
16418 tokio::time::Instant::now() + Duration::from_secs(30);
16419 let status = child.wait().await.unwrap();
16420 assert_eq!(status.code(), Some(121));
16421 assert!(child.privacy_exec.is_none());
16422 assert!(
16423 child.report_ready.get().is_none(),
16424 "an exited module must not publish a live pid"
16425 );
16426 let report = classify_reaped_child_exit(&snapshot, &child, &status);
16427 on_child_exit(
16428 &spec,
16429 policy,
16430 ®istry,
16431 &snapshot,
16432 &runtime.terminal_ring,
16433 &runtime.spawn_events,
16434 &runtime.child_roster,
16435 report,
16436 )
16437 .await;
16438 let state = lock_snapshot(&snapshot).unwrap();
16439 assert_eq!(state.state, ModuleState::Failed);
16440 assert_eq!(state.last_exit.as_ref().unwrap().code, Some(121));
16441 assert_eq!(state.reported_pid(), None);
16442 drop(state);
16443 let history = runtime.terminal_ring.lock().unwrap().snapshot();
16444 assert_eq!(history.entries.len(), 1);
16445 let terminal = &history.entries[0];
16446 assert_eq!(terminal.exit_code, Some(121));
16447 assert_eq!(terminal.exit_kind, subc_control::TerminalExitKind::Crash);
16448 assert_eq!(terminal.disposition, TerminalDisposition::Failed);
16449 assert_eq!(
16450 terminal.disposition_detail.as_deref(),
16451 Some(policy.budget_exhausted_detail().as_str()),
16452 "module exit 121 was classified as a trampoline refusal: {terminal:?}"
16453 );
16454 assert_eq!(child.spawn_failure, None);
16455 child.release_roster();
16456 }
16457 }
16458}
16459
16460#[cfg(all(test, unix))]
16464mod launch_nonce_descriptor_tests {
16465 use super::{ChildRoster, ModuleProtocol, ModuleSpec, SupervisorHandle};
16466 use crate::stderr_tail::{StderrRing, StderrTailConfig};
16467 use cortexkit_test_support::ScratchDir;
16468 use std::{
16469 path::PathBuf,
16470 sync::{Arc, Mutex},
16471 time::{Duration, Instant},
16472 };
16473
16474 async fn probe(role: super::SpawnRole) {
16475 let scratch = ScratchDir::new("launch-nonce-descriptor");
16476 let fd_copy = scratch.join("from-descriptor");
16477 let env_copy = scratch.join("environment");
16478 let script = format!(
16479 "env > '{env}.tmp' && mv '{env}.tmp' '{env}'; cat <&3 > '{fd}.tmp' && mv '{fd}.tmp' '{fd}'; sleep 30",
16480 fd = fd_copy.display(), env = env_copy.display(),
16481 );
16482 let xdg = |name: &str| (name.to_string(), scratch.join(name).display().to_string());
16483 let spec = ModuleSpec {
16484 module_id: "nonce-descriptor-probe".to_string(),
16485 program: PathBuf::from("/bin/sh"),
16486 args: vec!["-c".to_string(), script],
16487 env: vec![
16488 xdg("XDG_DATA_HOME"),
16489 xdg("XDG_RUNTIME_DIR"),
16490 xdg("XDG_CONFIG_HOME"),
16491 ("SUBC_LAUNCH_NONCE".to_string(), "must-not-leak".to_string()),
16492 ],
16493 reserved: true,
16494 reserved_prefixes: Vec::new(),
16495 protocol: ModuleProtocol::Subc,
16496 overlap: Default::default(),
16497 };
16498 let handle = SupervisorHandle::new();
16499 let ring = Arc::new(Mutex::new(StderrRing::new(StderrTailConfig::default())));
16500 let roster = ChildRoster::default();
16501 #[cfg(target_os = "macos")]
16502 {
16503 let path = super::test_privacy_trampoline();
16504 roster.set_privacy_trampoline(super::probe_privacy_trampoline(&path).map(|()| path));
16505 }
16506 let child = super::spawn_child_in_slot(
16507 &spec,
16508 None,
16509 Some(&handle),
16510 &ring,
16511 None,
16512 &roster,
16513 #[cfg(target_os = "linux")]
16514 None,
16515 role,
16516 matches!(role, super::SpawnRole::SwapCandidate),
16517 )
16518 .expect("spawn probe");
16519 let deadline = Instant::now() + Duration::from_secs(10);
16520 while !(fd_copy.exists() && env_copy.exists()) {
16521 assert!(Instant::now() < deadline, "probe never wrote its copies");
16522 tokio::time::sleep(Duration::from_millis(20)).await;
16523 }
16524 let nonce = std::fs::read_to_string(fd_copy).unwrap();
16525 assert!(!nonce.is_empty());
16526 let environment = std::fs::read_to_string(env_copy).unwrap();
16527 assert!(environment
16528 .lines()
16529 .any(|line| line.starts_with("SUBC_LAUNCH_NONCE_FD=3:")));
16530 let copy = environment
16531 .lines()
16532 .find_map(|line| line.strip_prefix("SUBC_LAUNCH_NONCE="));
16533 assert_eq!(
16534 copy, None,
16535 "Unix children must never receive the environment nonce"
16536 );
16537 if matches!(role, super::SpawnRole::Plain) {
16538 assert_eq!(
16539 handle.spawn_nonce(&spec.module_id).as_deref(),
16540 Some(nonce.as_str())
16541 );
16542 }
16543 drop(child);
16544 }
16545
16546 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
16547 async fn a_spawned_module_receives_its_nonce_only_on_descriptor_3() {
16548 probe(super::SpawnRole::Plain).await;
16549 }
16550
16551 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
16552 async fn swap_candidate_receives_its_nonce_only_on_descriptor_3() {
16553 probe(super::SpawnRole::SwapCandidate).await;
16554 }
16555}
16556
16557#[cfg(all(test, target_os = "linux"))]
16558mod cgroup_containment_tests {
16559 use super::*;
16560 use cortexkit_test_support::ScratchDir;
16561
16562 fn running(pid: u32) -> bool {
16563 std::fs::read_to_string(format!("/proc/{pid}/stat"))
16565 .ok()
16566 .and_then(|stat| {
16567 stat.rsplit_once(") ")
16568 .map(|(_, rest)| rest.starts_with('Z'))
16569 })
16570 .is_some_and(|zombie| !zombie)
16571 }
16572
16573 #[tokio::test]
16574 async fn linux_teardown_reaps_the_grandchild() {
16575 teardown_tree("linux_teardown_reaps_the_grandchild", false).await;
16576 }
16577
16578 #[tokio::test]
16579 async fn linux_shutdown_straggler_reaps_the_grandchild() {
16580 teardown_tree("linux_shutdown_straggler_reaps_the_grandchild", true).await;
16581 }
16582
16583 async fn teardown_tree(test_name: &str, shutdown: bool) {
16584 let dir = ScratchDir::new(test_name);
16585 let root = PathBuf::from(format!(
16586 "/sys/fs/cgroup/subc-tree-test-{test_name}-{}-{}",
16587 std::process::id(),
16588 unix_ms_now()
16589 ));
16590 if let Err(error) = std::fs::create_dir(&root) {
16591 assert!(
16592 std::env::var_os("SUBC_REQUIRE_CGROUP_TEST").is_none(),
16593 "required cgroup test cannot execute: {error}"
16594 );
16595 eprintln!(
16596 "SKIP {test_name}: no writable delegated cgroup at {}: {error}",
16597 root.display()
16598 );
16599 return;
16600 }
16601 let placement = subc_cgroup::prepare_at(&root)
16602 .expect("prepare isolated kernel cgroup")
16603 .expect("isolated cgroup is delegated");
16604 let module_id = "tree-teardown";
16605 let module = placement
16606 .module_path(module_id)
16607 .expect("create isolated module cgroup");
16608 if !module.join("cgroup.kill").exists() {
16609 std::fs::remove_dir(&module).unwrap();
16610 std::fs::remove_dir(root.join("subc-modules")).unwrap();
16611 std::fs::remove_dir(&root).unwrap();
16612 assert!(
16613 std::env::var_os("SUBC_REQUIRE_CGROUP_TEST").is_none(),
16614 "required cgroup.kill interface unavailable"
16615 );
16616 eprintln!("SKIP {test_name}: cgroup.kill unavailable (kernel < 5.14)");
16617 return;
16618 }
16619 let supervisor = Supervisor::new_for_test(
16620 Arc::new(Registry::default()),
16621 RestartPolicy::new(3, Duration::ZERO),
16622 )
16623 .with_cgroup_placement(Some(placement));
16624 let mut runtime = supervisor.runtime_config();
16625 runtime.child_roster = runtime
16626 .child_roster
16627 .for_module(Arc::new(Mutex::new(Duration::from_millis(100))));
16628 let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::starting()));
16629 let pid_file = dir.join("grandchild.pid");
16630 let spec = ModuleSpec {
16631 module_id: module_id.to_string(),
16632 program: PathBuf::from("/bin/sh"),
16633 args: vec![
16634 "-c".into(),
16635 "trap '' TERM; sleep 600 & echo $! > \"$1\"; wait".into(),
16636 "fixture".into(),
16637 pid_file.display().to_string(),
16638 ],
16639 env: ["XDG_DATA_HOME", "XDG_RUNTIME_DIR", "XDG_CONFIG_HOME"]
16640 .into_iter()
16641 .map(|key| (key.to_string(), dir.display().to_string()))
16642 .collect(),
16643 reserved: false,
16644 reserved_prefixes: Vec::new(),
16645 protocol: ModuleProtocol::None,
16646 overlap: Default::default(),
16647 };
16648 let child =
16649 spawn_and_mark_running(&spec, &runtime, &snapshot).expect("spawn supervised tree");
16650 let deadline = tokio::time::Instant::now() + Duration::from_secs(10);
16651 let grandchild: u32 = loop {
16652 if let Ok(contents) = std::fs::read_to_string(&pid_file) {
16653 if let Ok(pid) = contents.trim().parse() {
16654 break pid;
16655 }
16656 }
16657 assert!(
16658 tokio::time::Instant::now() < deadline,
16659 "grandchild pid was not recorded"
16660 );
16661 tokio::time::sleep(Duration::from_millis(10)).await;
16662 };
16663 assert!(
16664 running(grandchild),
16665 "grandchild must be alive before teardown"
16666 );
16667 if shutdown {
16668 let mut child = child;
16669 crate::child_roster::end_children_for_daemon_shutdown(
16670 &runtime.child_roster,
16671 false,
16672 std::future::pending(),
16673 )
16674 .await;
16675 child.wait().await.expect("reap shutdown straggler");
16676 } else {
16677 drain_child_to_state(
16678 module_id,
16679 ModuleProtocol::None,
16680 StopNotice::NotSent,
16681 &Registry::default(),
16682 None,
16683 &snapshot,
16684 &runtime.terminal_ring,
16685 &SpawnEventFeed::default(),
16686 child,
16687 Duration::from_millis(100),
16688 ModuleState::Stopped,
16689 Some(false),
16690 )
16691 .await
16692 .expect("real supervisor teardown");
16693 }
16694 let deadline = tokio::time::Instant::now() + Duration::from_secs(5);
16695 while running(grandchild) && tokio::time::Instant::now() < deadline {
16696 tokio::time::sleep(Duration::from_millis(10)).await;
16697 }
16698 let survived = running(grandchild);
16699 if survived {
16701 let pid = rustix::process::Pid::from_raw(grandchild as i32).unwrap();
16702 let _ = rustix::process::kill_process(pid, rustix::process::Signal::KILL);
16703 tokio::time::sleep(Duration::from_millis(100)).await;
16704 }
16705 if module.exists() {
16706 std::fs::remove_dir(&module).expect("remove empty module cgroup");
16707 }
16708 std::fs::remove_dir(root.join("subc-modules")).unwrap();
16709 std::fs::remove_dir(&root).unwrap();
16710 assert!(
16711 !survived,
16712 "grandchild {grandchild} outlived module teardown"
16713 );
16714 eprintln!("EXECUTED {test_name}: grandchild {grandchild} killed in isolated cgroup");
16715 }
16716}
16717
16718#[cfg(all(test, unix))]
16721mod record_write_stall_tests {
16722 use super::*;
16723 use std::sync::atomic::{AtomicBool, Ordering};
16724 use subc_transport::{authenticate_client, ConnectionInfo, Endpoint, SCHEMA_VERSION};
16725
16726 const MODULE_AUTH_DEADLINE: Duration = Duration::from_secs(2);
16729 const HELD_RECORD_WRITE: Duration = Duration::from_secs(3);
16733
16734 fn sleeping_spec() -> ModuleSpec {
16746 ModuleSpec {
16747 module_id: "record-write-stall".into(),
16748 program: "/bin/sleep".into(),
16749 args: vec!["600".into()],
16750 env: vec![],
16751 reserved: false,
16752 reserved_prefixes: vec![],
16753 protocol: ModuleProtocol::None,
16754 overlap: Default::default(),
16755 }
16756 }
16757
16758 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
16759 async fn module_auth_completes_while_the_restart_waits_on_the_record_write() {
16760 let home = cortexkit_test_support::ScratchDir::new("record-write-stall");
16761 let supervisor =
16762 Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default())
16763 .with_live_children_record(home.join("live-children.json"));
16764 let module = supervisor.spawn(sleeping_spec()).unwrap();
16765 let old_pid = tokio::time::timeout(Duration::from_secs(10), async {
16770 loop {
16771 if let Some(pid) = lock_snapshot(&module.inner.snapshot)
16772 .unwrap()
16773 .reported_pid()
16774 {
16775 return pid;
16776 }
16777 tokio::time::sleep(Duration::from_millis(10)).await;
16778 }
16779 })
16780 .await
16781 .expect("the first process was admitted as the module");
16782
16783 let key = vec![0x42; 32];
16786 let daemon_id = [0x24; 16];
16787 let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
16788 let port = listener.local_addr().unwrap().port();
16789 let server = tokio::spawn(crate::server::serve_listener(
16790 listener,
16791 Arc::new(crate::router::Router::with_default_self_handler()),
16792 crate::server::ServerAuth::with_limits(
16793 key.clone(),
16794 daemon_id,
16795 "test",
16796 crate::server::DEFAULT_AUTH_DEADLINE,
16797 4,
16798 ),
16799 ));
16800
16801 let (held_tx, held_rx) = std::sync::mpsc::channel();
16804 let armed = AtomicBool::new(true);
16805 supervisor
16806 .child_roster
16807 .set_record_write_hook_for_test(move |live| {
16808 let admitting_replacement = live.values().any(|entry| entry.pid != old_pid);
16809 if admitting_replacement && armed.swap(false, Ordering::SeqCst) {
16810 let _ = held_tx.send(());
16811 std::thread::sleep(HELD_RECORD_WRITE);
16812 }
16813 });
16814
16815 let module_process = std::thread::spawn(move || {
16818 held_rx
16819 .recv_timeout(Duration::from_secs(30))
16820 .expect("the restart reached the replacement's record write");
16821 let conn = ConnectionInfo {
16822 schema: SCHEMA_VERSION,
16823 wire_version: None,
16824 endpoints: vec![Endpoint {
16825 host: "127.0.0.1".to_owned(),
16826 port,
16827 }],
16828 key,
16829 daemon_id,
16830 pid: std::process::id(),
16831 daemon_ver: "test".to_owned(),
16832 };
16833 tokio::runtime::Builder::new_current_thread()
16834 .enable_all()
16835 .build()
16836 .unwrap()
16837 .block_on(async {
16838 let started = Instant::now();
16839 let mut stream = tokio::net::TcpStream::connect(("127.0.0.1", port))
16840 .await
16841 .unwrap();
16842 let result =
16843 authenticate_client(&mut stream, &conn, MODULE_AUTH_DEADLINE).await;
16844 (result, started.elapsed())
16845 })
16846 });
16847
16848 module.restart(Some(0)).await.unwrap();
16849 let (auth, elapsed) = tokio::task::spawn_blocking(move || module_process.join().unwrap())
16850 .await
16851 .unwrap();
16852 assert!(
16853 auth.is_ok(),
16854 "the daemon must answer a module's handshake while a restart waits on the \
16855 live-children record write; got {auth:?} after {elapsed:?}"
16856 );
16857
16858 tokio::time::timeout(Duration::from_secs(20), async {
16859 while lock_snapshot(&module.inner.snapshot)
16860 .unwrap()
16861 .reported_pid()
16862 .is_none_or(|pid| pid == old_pid)
16863 {
16864 tokio::time::sleep(Duration::from_millis(10)).await;
16865 }
16866 })
16867 .await
16868 .expect("the restart published its replacement");
16869 module.stop().await.unwrap();
16870 server.abort();
16871 }
16872}