1use std::{
2 collections::{HashMap, HashSet, VecDeque},
3 error::Error,
4 fmt, io,
5 path::PathBuf,
6 process::{ExitStatus, Stdio},
7 sync::{Arc, Mutex, OnceLock},
8 time::{Duration, SystemTime, UNIX_EPOCH},
9};
10
11use cortexkit_log::Retention;
12use serde_json::Value;
13use sha2::{Digest, Sha256};
14use subc_control::{
15 ClientControlPush, LiveSpawn, ModuleProtocol, RouteCloseReason, SpawnCursor, SpawnEvent,
16 SpawnEventKind, SpawnSnapshot, SupervisorHealthStatus, TerminalDisposition, TerminalExitKind,
17};
18use subc_protocol::{
19 manifest::{SelfSignalKind, SignalAnchor},
20 session::{
21 HealthReport, HealthStatus, ModuleControlCommand, ModuleControlRequest,
22 MODULE_CONTROL_OP_HEALTH_CHECK,
23 },
24 Flags, FrameType, Principal, Priority, SUBC_LAUNCH_NONCE_ENV, SUBC_MODULE_ID_ENV,
25};
26use tokio::{
27 process::{Child, Command},
28 sync::{mpsc, oneshot, watch, Mutex as AsyncMutex},
29 task::JoinHandle,
30 time::{sleep, sleep_until, timeout, timeout_at, Instant},
31};
32use tracing::{debug, error, info, warn};
33
34use crate::{
35 child_roster::ChildRoster,
36 daemon_config::{
37 CAPTURE_KEEP_ENV, CAPTURE_MAX_AGE_DAYS_ENV, CAPTURE_MAX_FILE_MB_ENV, CK_LOG_ENV,
38 },
39 forwarding::{
40 CloseReason, ForwardingError, ForwardingTable, GoodbyeTarget, ModuleControlRpcOutcome,
41 ModuleDrainTarget, PendingModuleControlRpc,
42 },
43 provenance::{ExecutableIdentityProbe, SpawnedFileIdentity},
44 registry::{ConnectionId, RegistrationEndReason, RegistryError},
45 stderr_tail::{
46 pump_stderr_to, pump_stdout_to, ChildOutputSink, StderrRing, StderrTailConfig,
47 StderrTailSnapshot,
48 },
49 terminal_ring::{TerminalHistorySnapshot, TerminalRecord, TerminalRing, TerminalRingConfig},
50 Frame, FrameSink, Registry,
51};
52
53#[cfg(not(windows))]
54use crate::provenance::spawned_file_identity;
55
56#[path = "supervise_swap.rs"]
57mod swap;
58
59#[path = "supervise_activity.rs"]
60mod activity;
61
62pub const SUBC_ARG: &str = "--subc";
68
69const DEFAULT_MAX_RESTARTS: u32 = 3;
70const DEFAULT_BACKOFF: Duration = Duration::from_millis(100);
71const DEFAULT_MAX_BACKOFF: Duration = Duration::from_secs(30);
72const DEFAULT_RESTART_WINDOW: Duration = Duration::from_secs(600);
76pub const DEFAULT_DRAIN_TIMEOUT: Duration = Duration::from_secs(30);
87const REGISTRY_RELEASE_TIMEOUT: Duration = Duration::from_secs(1);
88const REGISTRY_RELEASE_POLL: Duration = Duration::from_millis(10);
89const DRAIN_BUSY_REPROBE_INTERVAL: Duration = Duration::from_millis(250);
101const STDERR_PUMP_DRAIN_TIMEOUT: Duration = Duration::from_millis(250);
124pub const SPAWN_EVENT_RING_CAPACITY: usize = 4096;
126const SPAWN_SUBSCRIBER_BUFFER: usize = SPAWN_EVENT_RING_CAPACITY + 1;
127pub(crate) const SPAWN_SUBSCRIBER_LAGGED_CODE: &str = "spawn_subscriber_lagged";
132
133#[cfg(target_os = "macos")]
134struct PrivacyExec {
135 reader: tokio::io::unix::AsyncFd<std::io::PipeReader>,
136 deadline: tokio::time::Instant,
137 expected: Option<subc_os::FileIdentity>,
138 trampoline: Option<subc_os::FileIdentity>,
139 script: bool,
140 module_id: String,
141 #[cfg(test)]
142 confirmation_pause: Option<(String, Option<tokio::net::TcpStream>)>,
143}
144
145#[cfg(all(test, target_os = "macos"))]
146pub(crate) fn test_privacy_trampoline() -> PathBuf {
147 let path = std::env::current_exe()
150 .unwrap()
151 .parent()
152 .unwrap()
153 .parent()
154 .unwrap()
155 .join("privacy-trampoline-fixture");
156 assert!(
160 path.exists(),
161 "privacy-trampoline-fixture not built at {}: run `cargo build -p subc-daemon \
162 --bins --features test-support` or `cargo test -p subc-daemon` first",
163 path.display()
164 );
165 path
166}
167
168#[cfg(target_os = "macos")]
169fn probe_privacy_trampoline(path: &std::path::Path) -> Result<(), String> {
170 use std::io::Read;
171 let mut probe = std::process::Command::new(path)
172 .args(["__disclaim-exec", "--probe"])
173 .stdin(Stdio::null())
174 .stdout(Stdio::piped())
175 .stderr(Stdio::piped())
176 .spawn()
177 .map_err(|error| {
178 format!(
179 "privacy trampoline probe failed for {}: {error}",
180 path.display()
181 )
182 })?;
183 let deadline = std::time::Instant::now() + Duration::from_secs(5);
184 let status = loop {
185 match probe.try_wait() {
186 Ok(Some(status)) => break status,
187 Ok(None) if std::time::Instant::now() < deadline => {
188 std::thread::sleep(Duration::from_millis(5))
189 }
190 result => {
191 let _ = probe.kill();
192 let _ = probe.wait();
193 return Err(format!(
194 "privacy trampoline probe failed or timed out for {}: {result:?}",
195 path.display()
196 ));
197 }
198 }
199 };
200 let mut answer = String::new();
201 if let Some(stdout) = probe.stdout.take() {
202 let _ = stdout.take(256).read_to_string(&mut answer);
203 }
204 if status.success() && answer.trim() == subc_os::privacy_identity::TRAMPOLINE_PROBE {
205 return Ok(());
206 }
207 let mut diagnostic = String::new();
208 if let Some(stderr) = probe.stderr.take() {
209 let _ = stderr.take(1024).read_to_string(&mut diagnostic);
210 }
211 let cause = diagnostic
212 .trim()
213 .strip_prefix("ck-subc: own privacy identity refused: ")
214 .unwrap_or("binary does not implement the privacy trampoline protocol");
215 Err(format!(
216 "{cause}: probe of {} exited {status}",
217 path.display()
218 ))
219}
220
221#[cfg(target_os = "macos")]
222fn privacy_command(
223 spec: &ModuleSpec,
224 roster: &ChildRoster,
225) -> Result<
226 (
227 Command,
228 Option<PrivacyExec>,
229 subc_os::privacy_identity::ExecAcknowledgement,
230 ),
231 SuperviseError,
232> {
233 let failure = |cause: String| {
234 warn!(module_id = %spec.module_id, %cause, "privacy identity trampoline refused module spawn");
235 SuperviseError::Spawn {
236 program: spec.program.clone(),
237 source: io::Error::other(cause),
238 cgroup_path: None,
239 }
240 };
241 let trampoline = roster.privacy_trampoline().map_err(failure)?;
242 let program = if spec.program.components().count() == 1 && !spec.program.is_absolute() {
247 let path = spec
248 .env
249 .iter()
250 .find(|(key, _)| key == "PATH")
251 .map(|(_, value)| std::ffi::OsString::from(value))
252 .or_else(|| std::env::var_os("PATH"))
253 .unwrap_or_else(|| "/usr/bin:/bin".into());
254 std::env::split_paths(&path)
255 .map(|dir| dir.join(&spec.program))
256 .find(|path| path.is_file())
257 .unwrap_or_else(|| spec.program.clone())
258 } else {
259 spec.program.clone()
260 };
261 let expected = subc_os::file_identity(&program);
262 let trampoline_image = subc_os::file_identity(&trampoline);
263 let script = {
264 use std::io::Read;
265 let mut prefix = [0u8; 2];
266 std::fs::File::open(&program)
267 .is_ok_and(|mut file| file.read_exact(&mut prefix).is_ok() && &prefix == b"#!")
268 };
269 if expected.is_none() || expected == subc_os::file_identity(&trampoline) {
270 return Err(failure(
271 "privacy identity module executable is missing or is the trampoline itself".to_string(),
272 ));
273 }
274 let (reader, ack) = subc_os::privacy_identity::ExecAcknowledgement::pipe()
275 .map_err(|error| failure(error.to_string()))?;
276 let reader =
277 tokio::io::unix::AsyncFd::new(reader).map_err(|error| failure(error.to_string()))?;
278 let mut command = Command::new(&trampoline);
279 command
280 .arg("__disclaim-exec")
281 .arg(ack.fd().to_string())
282 .arg(&program);
283 ack.install(command.as_std_mut());
284 Ok((
285 command,
286 Some(PrivacyExec {
287 reader,
288 deadline: tokio::time::Instant::now() + Duration::from_secs(5),
289 expected,
290 trampoline: trampoline_image,
291 script,
292 module_id: spec.module_id.clone(),
293 #[cfg(test)]
294 confirmation_pause: spec
295 .env
296 .iter()
297 .find(|(key, _)| key == "SUBC_TEST_BEFORE_EXEC_CONFIRM")
298 .map(|(_, address)| (address.clone(), None)),
299 }),
300 ack,
301 ))
302}
303
304struct SupervisedChild {
305 child: Child,
306 exec: crate::activity::ExecOwner,
307 #[cfg(windows)]
308 nonce_delivery: Option<subc_os::LaunchNoncePipeDelivery>,
309 #[cfg(target_os = "macos")]
310 privacy_exec: Option<PrivacyExec>,
311 #[cfg(target_os = "macos")]
318 report_ready: Arc<OnceLock<()>>,
319 spawn_failure: Option<String>,
321 protocol: ModuleProtocol,
326 #[cfg(target_os = "linux")]
332 module_id: String,
333 #[cfg(target_os = "linux")]
334 cgroup_placement: Option<subc_cgroup::Placement>,
335 #[cfg(windows)]
357 job: Option<subc_jobobject::JobObject>,
358 stdout_pump: Option<JoinHandle<()>>,
359 stderr_pump: Option<StderrPump>,
360 stderr_ring: Arc<Mutex<StderrRing>>,
361 spawned_at: Instant,
364 spawned_at_ms: u64,
365 spawned_from: PathBuf,
366 launch_spec_digest: LaunchSpecDigest,
367 spawned_file_identity: Option<SpawnedFileIdentity>,
368 process_start_time: Option<u64>,
369 process_identity: Option<ProcessIdentity>,
370 pid: u32,
371 roster_guard: Option<crate::child_roster::RosterGuard>,
374}
375
376impl SupervisedChild {
377 fn id(&self) -> Option<u32> {
378 Some(self.pid)
379 }
380
381 fn process_identity(&self) -> Option<ProcessIdentity> {
382 self.process_identity
383 }
384
385 async fn wait(&mut self) -> io::Result<ExitStatus> {
386 #[cfg(target_os = "macos")]
387 self.confirm_privacy_exec().await;
388 let result = self.child.wait().await;
396 #[cfg(target_os = "linux")]
397 if result.is_ok() {
398 if let Some(placement) = self.cgroup_placement.as_ref() {
399 cleanup_reaped_cgroup(placement, &self.module_id).await;
400 self.cgroup_placement = None;
403 }
404 }
405 result
406 }
407
408 #[cfg(target_os = "macos")]
409 async fn confirm_privacy_exec(&mut self) {
410 let Some(pending) = &mut self.privacy_exec else {
411 return;
412 };
413 let result = tokio::time::timeout_at(pending.deadline, async {
414 let mut record = Vec::new();
415 loop {
416 let mut ready = pending.reader.readable().await?;
417 let read = ready.try_io(|reader| {
418 use std::io::Read;
419 let mut reader = reader.get_ref();
420 let mut buffer = [0u8; 256];
421 reader.read(&mut buffer).map(|count| (count, buffer))
422 });
423 match read {
424 Ok(Ok((0, _))) => return Ok::<_, io::Error>(record),
425 Ok(Ok((count, buffer))) => {
426 if record.len() + count > 1024 {
427 return Err(io::Error::other(
428 "privacy exec refusal record is too long",
429 ));
430 }
431 record.extend_from_slice(&buffer[..count]);
432 }
433 Ok(Err(error)) => return Err(error),
434 Err(_) => continue,
435 }
436 }
437 })
438 .await;
439 #[cfg(test)]
440 let result = if let Some((address, stream)) = &mut pending.confirmation_pause {
441 use tokio::io::{AsyncReadExt, AsyncWriteExt};
442 if stream.is_none() {
443 *stream = Some(
444 tokio::net::TcpStream::connect(address.as_str())
445 .await
446 .unwrap(),
447 );
448 stream.as_mut().unwrap().write_all(b"R").await.unwrap();
449 stream
450 .as_mut()
451 .unwrap()
452 .write_all(&self.pid.to_ne_bytes())
453 .await
454 .unwrap();
455 }
456 let mut reply = [0];
457 stream
458 .as_mut()
459 .unwrap()
460 .read_exact(&mut reply)
461 .await
462 .unwrap();
463 pending.confirmation_pause = None;
464 if &reply == b"F" {
465 Ok(Err(io::Error::other("injected late exec refusal")))
466 } else {
467 assert_eq!(&reply, b"X");
468 result
469 }
470 } else {
471 result
472 };
473 let pending = self.privacy_exec.as_ref().expect("pending exec");
476 let cause = match result {
477 Err(_) => Some("privacy identity trampoline did not exec within 5s".to_string()),
478 Ok(Err(error)) => Some(format!(
479 "privacy identity exec acknowledgement failed: {error}"
480 )),
481 Ok(Ok(record)) if !record.is_empty() => Some(
482 std::str::from_utf8(&record)
483 .ok()
484 .and_then(|record| {
485 record
486 .trim()
487 .strip_prefix(subc_os::privacy_identity::EXEC_REFUSAL_TAG)
488 })
489 .filter(|cause| !cause.is_empty())
490 .unwrap_or("invalid privacy exec refusal record")
491 .to_string(),
492 ),
493 Ok(Ok(_)) => match self.child.try_wait() {
494 Ok(Some(_status)) => None,
498 Err(error) => Some(format!("privacy identity trampoline wait failed: {error}")),
499 Ok(None) => {
500 let image = observe_spawned_image(self.pid);
501 if let Some(image) = image.filter(|image| {
502 image.executable.is_some()
503 && image.executable != pending.trampoline
504 && (image.executable == pending.expected || pending.script)
505 }) {
506 if let Err(error) = self
507 .roster_guard
508 .as_ref()
509 .map(|guard| guard.confirm_executable(image))
510 .transpose()
511 {
512 Some(format!(
513 "could not durably record confirmed module image: {error}"
514 ))
515 } else {
516 let _ = self.report_ready.set(());
517 info!(module_id = %pending.module_id, pid = self.pid,
518 "module spawned with own privacy identity (responsibility disclaimed)");
519 None
520 }
521 } else if image.is_none()
522 || image.is_some_and(|image| image.executable.is_none())
523 {
524 match tokio::time::timeout_at(pending.deadline, self.child.wait()).await {
531 Ok(Ok(_status)) => None,
532 Ok(Err(error)) => Some(format!("privacy identity trampoline wait failed: {error}")),
533 Err(_) => Some("privacy identity module executable remained unreadable until the 5s exec deadline".to_string()),
534 }
535 } else {
536 Some("privacy identity trampoline executable mismatch: expected the module program, not ck-subc".to_string())
537 }
538 }
539 },
540 };
541 let pending = self.privacy_exec.take().expect("pending exec");
542 self.exec.0.finish(if cause.is_some() {
545 crate::activity::ExecState::Refused
546 } else {
547 crate::activity::ExecState::CommittedExec
548 });
549 if let Some(cause) = cause {
550 warn!(module_id = %pending.module_id, pid = self.pid, %cause, "privacy identity trampoline refused module spawn");
551 self.spawn_failure = Some(cause);
552 if let Some(pid) = rustix::process::Pid::from_raw(self.pid as i32) {
555 let _ = rustix::process::kill_process_group(pid, rustix::process::Signal::KILL);
556 }
557 let _ = self.child.start_kill();
558 }
559 }
560
561 fn release_roster(&mut self) {
565 self.roster_guard = None;
566 }
567
568 fn start_kill(&mut self) -> io::Result<()> {
581 #[cfg(windows)]
582 if let Some(job) = &self.job {
583 if let Err(error) = job.terminate() {
584 debug!(
585 error = %error,
586 "job termination failed; the direct-child kill still owns the outcome"
587 );
588 }
589 }
590 #[cfg(target_os = "linux")]
591 kill_module_cgroup(self.cgroup_placement.as_ref(), &self.module_id);
592 self.child.start_kill()
593 }
594
595 async fn drain_stderr(&mut self, module_id: &str) {
596 if let Some(pump) = self.stdout_pump.take() {
597 settle_stdout_pump(module_id, pump, STDERR_PUMP_DRAIN_TIMEOUT).await;
598 }
599
600 let Some(pump) = self.stderr_pump.take() else {
601 return;
602 };
603 settle_stderr_pump(
604 module_id,
605 &self.stderr_ring,
606 pump,
607 STDERR_PUMP_DRAIN_TIMEOUT,
608 )
609 .await;
610 }
611}
612
613async fn settle_stdout_pump(module_id: &str, mut pump: JoinHandle<()>, bound: Duration) {
617 match timeout(bound, &mut pump).await {
618 Ok(Ok(())) => {}
619 Ok(Err(error)) => {
620 warn!(module_id, error = %error, "stdout pump ended unexpectedly");
621 }
622 Err(_) => {
623 drop(pump);
624 warn!(
625 module_id,
626 waited = ?bound,
627 "stdout pipe of the exited process has not drained; its reader keeps running without delaying the restart"
628 );
629 }
630 }
631}
632
633struct StderrPump {
636 task: JoinHandle<()>,
637 generation: u64,
638}
639
640async fn settle_stderr_pump(
646 module_id: &str,
647 ring: &Arc<Mutex<StderrRing>>,
648 pump: StderrPump,
649 bound: Duration,
650) {
651 let lock = || ring.lock().unwrap_or_else(|poisoned| poisoned.into_inner());
652 let StderrPump {
653 mut task,
654 generation,
655 } = pump;
656 lock().retire_pump(generation);
657 match timeout(bound, &mut task).await {
658 Ok(Ok(())) => {}
659 Ok(Err(err)) => {
660 let mut ring = lock();
661 ring.mark_incomplete_from(generation, format!("stderr pump ended unexpectedly: {err}"));
662 ring.finish_pump(generation);
663 warn!(module_id, error = %err, "stderr pump ended before clean EOF");
664 }
665 Err(_) => {
666 drop(task);
668 lock().mark_pump_late(
669 generation,
670 format!(
671 "stderr of the exited process had not reached EOF {bound:?} after it was \
672 retired (a descendant may still hold the pipe open); lines it still \
673 writes are kept in that process's section"
674 ),
675 );
676 warn!(
677 module_id,
678 waited = ?bound,
679 "stderr pipe of the exited process is still open; its reader keeps running without delaying the restart"
680 );
681 }
682 }
683}
684
685fn registration_release_events() -> &'static watch::Sender<u64> {
686 static EVENTS: OnceLock<watch::Sender<u64>> = OnceLock::new();
687 EVENTS.get_or_init(|| {
688 let (sender, _receiver) = watch::channel(0);
689 sender
690 })
691}
692
693pub(crate) fn notify_registration_release() {
694 let events = registration_release_events();
695 let next_generation = (*events.borrow()).wrapping_add(1);
696 events.send_replace(next_generation);
697}
698
699#[derive(Debug, Clone, PartialEq, Eq)]
701pub struct ModuleSpec {
702 pub module_id: String,
703 pub program: PathBuf,
704 pub args: Vec<String>,
705 pub env: Vec<(String, String)>,
706 pub reserved: bool,
711 pub reserved_prefixes: Vec<String>,
716 pub protocol: ModuleProtocol,
735 pub overlap: ModuleOverlap,
740}
741
742#[derive(Clone, PartialEq, Eq)]
745pub(crate) struct LaunchSpecDigest([u8; 32]);
746
747impl fmt::Debug for LaunchSpecDigest {
748 fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
749 f.write_str("LaunchSpecDigest([redacted])")
750 }
751}
752
753impl LaunchSpecDigest {
754 pub(crate) fn from_spec(spec: &ModuleSpec) -> Self {
755 fn field(hash: &mut Sha256, bytes: &[u8]) {
756 hash.update((bytes.len() as u64).to_le_bytes());
757 hash.update(bytes);
758 }
759 let mut hash = Sha256::new();
760 if let Some(program) = spec.program.to_str() {
761 field(&mut hash, b"utf8-program");
762 field(&mut hash, program.as_bytes());
763 } else {
764 #[cfg(unix)]
767 {
768 use std::os::unix::ffi::OsStrExt;
769 field(&mut hash, b"unix-program");
770 field(&mut hash, spec.program.as_os_str().as_bytes());
771 }
772 #[cfg(windows)]
773 {
774 use std::os::windows::ffi::OsStrExt;
775 field(&mut hash, b"windows-program");
776 let bytes = spec
777 .program
778 .as_os_str()
779 .encode_wide()
780 .flat_map(u16::to_le_bytes)
781 .collect::<Vec<_>>();
782 field(&mut hash, &bytes);
783 }
784 #[cfg(not(any(unix, windows)))]
785 field(&mut hash, spec.program.to_string_lossy().as_bytes());
786 }
787 hash.update((spec.args.len() as u64).to_le_bytes());
788 for arg in &spec.args {
789 field(&mut hash, arg.as_bytes());
790 }
791 let mut env = spec.env.iter().collect::<Vec<_>>();
792 env.sort_unstable();
793 hash.update((env.len() as u64).to_le_bytes());
794 for (key, value) in env {
795 field(&mut hash, key.as_bytes());
796 field(&mut hash, value.as_bytes());
797 }
798 field(
799 &mut hash,
800 match spec.protocol {
801 ModuleProtocol::Subc => b"subc",
802 ModuleProtocol::None => b"none",
803 },
804 );
805 Self(hash.finalize().into())
806 }
807}
808
809#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)]
816pub enum ModuleOverlap {
817 #[default]
819 Exclusive,
820 Safe,
832}
833
834impl ModuleOverlap {
835 pub fn as_str(self) -> &'static str {
836 match self {
837 Self::Exclusive => "exclusive",
838 Self::Safe => "safe",
839 }
840 }
841}
842
843pub const SUBC_SPAWN_ROLE_ENV: &str = "SUBC_SPAWN_ROLE";
853pub const SPAWN_ROLE_SWAP_CANDIDATE: &str = "swap_candidate";
855pub const DEFAULT_SWAP_READY_TIMEOUT: Duration = Duration::from_secs(100);
860
861#[derive(Debug, Clone, Copy, PartialEq, Eq)]
879pub struct RestartPolicy {
880 pub max_restarts: u32,
881 pub backoff: Duration,
884 pub max_backoff: Duration,
886 pub window: Duration,
890}
891
892impl RestartPolicy {
893 pub fn new(max_restarts: u32, backoff: Duration) -> Self {
897 Self {
898 max_restarts,
899 backoff,
900 max_backoff: DEFAULT_MAX_BACKOFF,
901 window: DEFAULT_RESTART_WINDOW,
902 }
903 }
904
905 pub fn with_max_backoff(mut self, max_backoff: Duration) -> Self {
906 self.max_backoff = max_backoff;
907 self
908 }
909
910 pub fn with_window(mut self, window: Duration) -> Self {
911 self.window = window;
912 self
913 }
914
915 fn delay_for_restart(&self, restart_in_window: u32) -> Duration {
920 if self.backoff.is_zero() || self.max_backoff.is_zero() {
921 return Duration::ZERO;
922 }
923
924 let mut delay = self.backoff;
925 for _ in 0..restart_in_window {
926 if delay >= self.max_backoff {
927 return self.max_backoff;
928 }
929 delay = delay
930 .checked_mul(10)
931 .unwrap_or(self.max_backoff)
932 .min(self.max_backoff);
933 }
934 delay.min(self.max_backoff)
935 }
936
937 fn budget_exhausted_detail(&self) -> String {
942 format!(
943 "crash budget exhausted: max_restarts={} within window_secs={}",
944 self.max_restarts,
945 self.window.as_secs()
946 )
947 }
948}
949
950impl Default for RestartPolicy {
951 fn default() -> Self {
952 Self {
953 max_restarts: DEFAULT_MAX_RESTARTS,
954 backoff: DEFAULT_BACKOFF,
955 max_backoff: DEFAULT_MAX_BACKOFF,
956 window: DEFAULT_RESTART_WINDOW,
957 }
958 }
959}
960
961#[derive(Debug, Clone, Copy, PartialEq, Eq)]
962struct CrashRestartSchedule {
963 restart_in_window: u32,
964 delay: Duration,
965}
966
967fn daemon_will_restart(
974 state: &mut SupervisorSnapshot,
975 policy: &RestartPolicy,
976 now: Instant,
977) -> bool {
978 state.enabled && state.crash_restarts_in_window(policy.window, now) < policy.max_restarts
979}
980
981const DEFAULT_HEALTH_CADENCE: Duration = Duration::from_secs(30);
982const DEFAULT_HEALTH_DEADLINE: Duration = Duration::from_secs(5);
983const DEFAULT_HEALTH_FAILURE_THRESHOLD: u32 = 3;
984const MAX_HEALTH_METRICS_BYTES: usize = 16 * 1024;
985
986#[derive(Debug, Clone, Copy, PartialEq, Eq)]
987pub enum HealthAction {
988 Report,
989 Restart,
990 Alert,
991}
992
993impl fmt::Display for HealthAction {
994 fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
995 f.write_str(match self {
996 Self::Report => "report",
997 Self::Restart => "restart",
998 Self::Alert => "alert",
999 })
1000 }
1001}
1002
1003#[derive(Debug, Clone, PartialEq, Eq)]
1004pub struct HealthConfig {
1005 pub http: Option<String>,
1009 pub cadence: Duration,
1010 pub deadline: Duration,
1011 pub failure_threshold: u32,
1012 pub on_degraded: HealthAction,
1013 pub on_failing: HealthAction,
1014 pub critical: bool,
1015}
1016
1017impl Default for HealthConfig {
1018 fn default() -> Self {
1019 Self {
1020 http: None,
1021 cadence: DEFAULT_HEALTH_CADENCE,
1022 deadline: DEFAULT_HEALTH_DEADLINE,
1023 failure_threshold: DEFAULT_HEALTH_FAILURE_THRESHOLD,
1024 on_degraded: HealthAction::Report,
1025 on_failing: HealthAction::Report,
1026 critical: false,
1027 }
1028 }
1029}
1030
1031#[derive(Debug, Clone, PartialEq)]
1049pub struct ModuleHealthStatus {
1050 pub status: SupervisorHealthStatus,
1051 pub last_probe_ms: Option<u64>,
1052 pub detail: Option<String>,
1053 pub metrics: Option<Value>,
1054 pub consecutive_failures: u32,
1055 pub late_answer_count: u64,
1058 pub last_late_answer_latency_ms: Option<u64>,
1060 pub last_action: Option<String>,
1061 pub last_action_ms: Option<u64>,
1065}
1066
1067impl Default for ModuleHealthStatus {
1068 fn default() -> Self {
1069 Self {
1070 status: SupervisorHealthStatus::Unknown,
1071 last_probe_ms: None,
1072 detail: None,
1073 metrics: None,
1074 consecutive_failures: 0,
1075 late_answer_count: 0,
1076 last_late_answer_latency_ms: None,
1077 last_action: None,
1078 last_action_ms: None,
1079 }
1080 }
1081}
1082
1083#[derive(Debug, Clone, Copy, PartialEq, Eq)]
1085pub enum ModuleState {
1086 Starting,
1087 Running,
1088 Unresponsive,
1089 Restarting,
1090 Draining,
1091 Stopped,
1092 Failed,
1093 Disabled,
1094}
1095
1096impl fmt::Display for ModuleState {
1097 fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
1098 f.write_str(match self {
1099 Self::Starting => "starting",
1100 Self::Running => "running",
1101 Self::Unresponsive => "unresponsive",
1102 Self::Restarting => "restarting",
1103 Self::Draining => "draining",
1104 Self::Stopped => "stopped",
1105 Self::Failed => "failed",
1106 Self::Disabled => "disabled",
1107 })
1108 }
1109}
1110
1111#[derive(Debug, Clone, Copy, PartialEq, Eq)]
1113pub enum ExitKind {
1114 Clean,
1115 Crash,
1116 DeliberateSeverance,
1117}
1118
1119impl From<ExitKind> for TerminalExitKind {
1120 fn from(kind: ExitKind) -> Self {
1121 match kind {
1122 ExitKind::Clean => Self::Clean,
1123 ExitKind::Crash => Self::Crash,
1124 ExitKind::DeliberateSeverance => Self::DeliberateSeverance,
1125 }
1126 }
1127}
1128
1129#[derive(Debug, Clone, Copy, PartialEq, Eq)]
1132pub(crate) struct ProcessIdentity {
1133 pub(crate) pid: u32,
1134 pub(crate) start_time: u64,
1135}
1136
1137#[derive(Debug, Clone, PartialEq, Eq)]
1139pub struct ExitReport {
1140 pub kind: ExitKind,
1141 pub code: Option<i32>,
1142 pub signal: Option<i32>,
1143 pub at_ms: u64,
1144}
1145
1146#[derive(Debug, Clone, PartialEq)]
1149pub struct ModuleStatus {
1150 pub module_id: String,
1151 pub waiting_for_previous_pid: Option<u32>,
1153 pub waiting_for_previous_detail: Option<String>,
1155 pub launch_nonce_source: Option<String>,
1160 pub state: ModuleState,
1161 pub enabled: bool,
1162 pub process_alive: bool,
1163 pub registration_active: bool,
1164 pub protocol: ModuleProtocol,
1169 pub live: bool,
1180 pub restart_count: u32,
1184 pub lifetime_restarts: u32,
1188 pub spawn_generation: u64,
1189 pub max_restarts: u32,
1194 pub restart_window: Duration,
1198 pub drain_timeout: Duration,
1202 pub restart_backoff: Duration,
1203 pub restart_max_backoff: Duration,
1204 pub pid: Option<u32>,
1209 pub spawned_at_ms: Option<u64>,
1210 pub spawned_from: Option<PathBuf>,
1211 pub(crate) launch_spec_digest: Option<LaunchSpecDigest>,
1212 pub process_start_time: Option<u64>,
1213 pub last_exit: Option<ExitReport>,
1214 pub health: ModuleHealthStatus,
1215}
1216
1217#[cfg(test)]
1218#[derive(Debug, Clone, Copy, PartialEq)]
1219struct ActorSelectCheckpoint {
1220 generation: u64,
1221 turn: u64,
1222 registered_connection: Option<ConnectionId>,
1223 next_probe_at: Option<Instant>,
1224 wake_after: Option<Duration>,
1225}
1226
1227#[derive(Debug, Clone, PartialEq)]
1228struct SupervisorSnapshot {
1229 exec_outcome: Option<Arc<crate::activity::ExecOutcome>>,
1230 #[cfg(windows)]
1231 pipe_consumed: Option<Arc<OnceLock<()>>>,
1232 #[cfg(test)]
1234 actor_turns: u64,
1235 #[cfg(test)]
1238 actor_select: Option<ActorSelectCheckpoint>,
1239 state: ModuleState,
1240 enabled: bool,
1241 process_alive: bool,
1242 spawned_protocol: Option<ModuleProtocol>,
1243 spawn_failure: Option<String>,
1244 waiting_for_previous_pid: Option<u32>,
1245 waiting_for_previous_detail: Option<String>,
1246 crash_restarts: VecDeque<Instant>,
1252 lifetime_restarts: u32,
1253 spawn_generation: u64,
1262 pid: Option<u32>,
1263 #[cfg(target_os = "macos")]
1264 report_ready: Option<Arc<OnceLock<()>>>,
1265 reaped_pid: Option<u32>,
1267 respawn_pending: bool,
1269 coalesced_restart_pending: bool,
1271 spawned_at_ms: Option<u64>,
1272 spawned_from: Option<PathBuf>,
1273 launch_spec_digest: Option<LaunchSpecDigest>,
1274 spawned_file_identity: Option<SpawnedFileIdentity>,
1275 process_start_time: Option<u64>,
1276 deliberate_severance: Option<ProcessIdentity>,
1277 last_exit: Option<ExitReport>,
1278 drain_disposition_detail: Option<String>,
1280 health: ModuleHealthStatus,
1281 in_alternate_slot: bool,
1286 draining_to_replace: bool,
1293 configuration_updated_since_spawn: bool,
1299}
1300
1301impl SupervisorSnapshot {
1302 fn reported_pid(&self) -> Option<u32> {
1307 #[cfg(target_os = "macos")]
1308 if self
1309 .report_ready
1310 .as_ref()
1311 .is_some_and(|ready| ready.get().is_none())
1312 {
1313 return None;
1314 }
1315 self.pid
1316 }
1317
1318 fn starting() -> Self {
1319 Self::new(ModuleState::Starting, true)
1320 }
1321
1322 fn disabled() -> Self {
1323 Self::new(ModuleState::Disabled, false)
1324 }
1325
1326 fn crash_restarts_in_window(&mut self, window: Duration, now: Instant) -> u32 {
1330 while let Some(oldest) = self.crash_restarts.front() {
1331 if now.duration_since(*oldest) > window {
1332 self.crash_restarts.pop_front();
1333 } else {
1334 break;
1335 }
1336 }
1337 u32::try_from(self.crash_restarts.len()).unwrap_or(u32::MAX)
1338 }
1339
1340 fn record_crash_restart(&mut self, policy: &RestartPolicy, now: Instant) {
1346 self.crash_restarts.push_back(now);
1347 while self.crash_restarts.len() > policy.max_restarts as usize {
1348 self.crash_restarts.pop_front();
1349 }
1350 self.lifetime_restarts += 1;
1351 }
1352
1353 fn next_crash_restart(
1357 &mut self,
1358 policy: &RestartPolicy,
1359 now: Instant,
1360 ) -> Option<CrashRestartSchedule> {
1361 let restart_in_window = self.crash_restarts_in_window(policy.window, now);
1362 if restart_in_window >= policy.max_restarts {
1363 return None;
1364 }
1365 self.record_crash_restart(policy, now);
1366 Some(CrashRestartSchedule {
1367 restart_in_window,
1368 delay: policy.delay_for_restart(restart_in_window),
1369 })
1370 }
1371
1372 fn clear_crash_restarts(&mut self) {
1377 self.crash_restarts.clear();
1378 }
1379
1380 fn new(state: ModuleState, enabled: bool) -> Self {
1381 Self {
1382 exec_outcome: None,
1383 #[cfg(test)]
1384 actor_turns: 0,
1385 #[cfg(test)]
1386 actor_select: None,
1387 state,
1388 enabled,
1389 process_alive: false,
1390 #[cfg(windows)]
1391 pipe_consumed: None,
1392 spawned_protocol: None,
1393 spawn_failure: None,
1394 waiting_for_previous_pid: None,
1395 waiting_for_previous_detail: None,
1396 crash_restarts: VecDeque::new(),
1397 lifetime_restarts: 0,
1398 spawn_generation: 0,
1399 pid: None,
1400 #[cfg(target_os = "macos")]
1401 report_ready: None,
1402 reaped_pid: None,
1403 respawn_pending: false,
1404 coalesced_restart_pending: false,
1405 spawned_at_ms: None,
1406 spawned_from: None,
1407 launch_spec_digest: None,
1408 spawned_file_identity: None,
1409 process_start_time: None,
1410 deliberate_severance: None,
1411 last_exit: None,
1412 drain_disposition_detail: None,
1413 health: ModuleHealthStatus::default(),
1414 in_alternate_slot: false,
1415 draining_to_replace: false,
1416 configuration_updated_since_spawn: false,
1417 }
1418 }
1419}
1420
1421type SharedSnapshot = Arc<Mutex<SupervisorSnapshot>>;
1422
1423type SpawnSubscriberKey = (ConnectionId, u64);
1424
1425#[derive(Debug)]
1426struct SpawnSubscriber {
1427 version: u8,
1428 frames: mpsc::Sender<Frame>,
1429 lagged: Option<oneshot::Sender<SpawnCursor>>,
1433}
1434
1435#[derive(Debug)]
1436struct SpawnEventState {
1437 daemon_incarnation: String,
1438 seq: u64,
1439 capacity: usize,
1440 live: HashMap<String, LiveSpawn>,
1441 generations: HashMap<String, u64>,
1442 events: VecDeque<SpawnEvent>,
1443 subscribers: HashMap<SpawnSubscriberKey, SpawnSubscriber>,
1444}
1445
1446impl Default for SpawnEventState {
1447 fn default() -> Self {
1448 Self {
1449 daemon_incarnation: "unconfigured".to_string(),
1450 seq: 0,
1451 capacity: SPAWN_EVENT_RING_CAPACITY,
1452 live: HashMap::new(),
1453 generations: HashMap::new(),
1454 events: VecDeque::new(),
1455 subscribers: HashMap::new(),
1456 }
1457 }
1458}
1459
1460#[derive(Debug, Clone, Default)]
1461struct SpawnEventFeed(
1462 Arc<Mutex<SpawnEventState>>,
1463 #[cfg(test)] Arc<Mutex<Option<std::net::TcpStream>>>,
1464);
1465
1466#[derive(Debug, Clone, PartialEq, Eq)]
1467pub(crate) enum SpawnSubscribeRefusal {
1468 ForeignIncarnation { current: String },
1469 TooOld { oldest: SpawnCursor },
1470 Frame(String),
1471}
1472
1473impl SpawnEventFeed {
1474 fn configure_incarnation(&self, daemon_incarnation: String) {
1475 let mut state = self.0.lock().unwrap_or_else(|p| p.into_inner());
1476 state.daemon_incarnation = daemon_incarnation;
1477 state.seq = 0;
1478 state.live.clear();
1479 state.generations.clear();
1480 state.events.clear();
1481 state.subscribers.clear();
1482 }
1483
1484 fn cursor(state: &SpawnEventState) -> SpawnCursor {
1485 SpawnCursor {
1486 daemon_incarnation: state.daemon_incarnation.clone(),
1487 seq: state.seq,
1488 }
1489 }
1490
1491 fn snapshot(&self) -> SpawnSnapshot {
1492 let state = self.0.lock().unwrap_or_else(|p| p.into_inner());
1493 let mut live = state.live.values().cloned().collect::<Vec<_>>();
1494 live.sort_by(|left, right| left.module_id.cmp(&right.module_id));
1495 SpawnSnapshot {
1496 cursor: Self::cursor(&state),
1497 ring_bound: state.capacity as u64,
1498 live,
1499 }
1500 }
1501
1502 fn emit_spawned(&self, module_id: &str, pid: u32, spawned_at_ms: u64) -> u64 {
1503 let mut state = self.0.lock().unwrap_or_else(|p| p.into_inner());
1504 let generation = state
1505 .generations
1506 .get(module_id)
1507 .copied()
1508 .unwrap_or(0)
1509 .checked_add(1)
1510 .expect("spawn generation exhausted");
1511 state.generations.insert(module_id.to_string(), generation);
1512 let live = LiveSpawn {
1513 module_id: module_id.to_string(),
1514 spawn_generation: generation,
1515 pid,
1516 spawned_at_ms,
1517 };
1518 state.live.insert(module_id.to_string(), live);
1519 Self::emit_locked(
1520 &mut state,
1521 SpawnEventKind::Spawned,
1522 module_id.to_string(),
1523 generation,
1524 pid,
1525 None,
1526 None,
1527 );
1528 generation
1529 }
1530
1531 fn emit_exited(&self, module_id: &str, exit_code: Option<i32>, exit_signal: Option<i32>) {
1532 let mut state = self.0.lock().unwrap_or_else(|p| p.into_inner());
1533 let Some(live) = state.live.remove(module_id) else {
1534 warn!(
1535 module_id,
1536 "terminal record had no live spawn event identity"
1537 );
1538 return;
1539 };
1540 Self::emit_locked(
1541 &mut state,
1542 SpawnEventKind::Exited,
1543 module_id.to_string(),
1544 live.spawn_generation,
1545 live.pid,
1546 exit_code,
1547 exit_signal,
1548 );
1549 }
1550
1551 fn emit_superseded_exited(
1558 &self,
1559 module_id: &str,
1560 spawn_generation: u64,
1561 pid: u32,
1562 exit_code: Option<i32>,
1563 exit_signal: Option<i32>,
1564 ) {
1565 let mut state = self.0.lock().unwrap_or_else(|p| p.into_inner());
1566 if state
1567 .live
1568 .get(module_id)
1569 .is_some_and(|live| live.spawn_generation == spawn_generation)
1570 {
1571 state.live.remove(module_id);
1572 }
1573 Self::emit_locked(
1574 &mut state,
1575 SpawnEventKind::Exited,
1576 module_id.to_string(),
1577 spawn_generation,
1578 pid,
1579 exit_code,
1580 exit_signal,
1581 );
1582 }
1583
1584 #[allow(clippy::too_many_arguments)]
1585 fn emit_locked(
1586 state: &mut SpawnEventState,
1587 kind: SpawnEventKind,
1588 module_id: String,
1589 spawn_generation: u64,
1590 pid: u32,
1591 exit_code: Option<i32>,
1592 exit_signal: Option<i32>,
1593 ) {
1594 state.seq = state
1595 .seq
1596 .checked_add(1)
1597 .expect("spawn event sequence exhausted");
1598 let event = SpawnEvent {
1599 cursor: Self::cursor(state),
1600 kind,
1601 module_id,
1602 spawn_generation,
1603 pid,
1604 exit_code,
1605 exit_signal,
1606 };
1607 state.events.push_back(event.clone());
1608 while state.events.len() > state.capacity {
1609 state.events.pop_front();
1610 }
1611 let body = match serde_json::to_vec(&event) {
1612 Ok(body) => body,
1613 Err(error) => {
1614 error!(%error, "failed to serialize supervisor spawn event");
1615 return;
1616 }
1617 };
1618 state.subscribers.retain(|(connection_id, corr), subscriber| {
1619 let frame = Frame::build_with_version(
1620 subscriber.version,
1621 FrameType::StreamData,
1622 control_flags(),
1623 0,
1624 0,
1625 *corr,
1626 body.clone(),
1627 );
1628 match frame {
1629 Ok(frame) => {
1630 if subscriber.frames.try_send(frame).is_ok() {
1631 true
1632 } else {
1633 warn!(connection_id = connection_id.get(), corr, "dropping lagged supervisor spawn subscriber");
1634 if let Some(lagged) = subscriber.lagged.take() {
1635 let _ = lagged.send(event.cursor.clone());
1636 }
1637 false
1638 }
1639 }
1640 Err(error) => {
1641 warn!(connection_id = connection_id.get(), corr, %error, "dropping supervisor spawn subscriber after frame build failure");
1642 false
1643 }
1644 }
1645 });
1646 }
1647
1648 fn subscribe(
1649 &self,
1650 connection_id: ConnectionId,
1651 corr: u64,
1652 version: u8,
1653 since: Option<SpawnCursor>,
1654 sink: FrameSink,
1655 ) -> Result<(), SpawnSubscribeRefusal> {
1656 let (frames, mut receiver) = mpsc::channel(SPAWN_SUBSCRIBER_BUFFER);
1657 let (lagged, mut lagged_rx) = oneshot::channel::<SpawnCursor>();
1658 {
1659 let mut state = self.0.lock().unwrap_or_else(|p| p.into_inner());
1660 let replay = if let Some(since) = since {
1661 if since.daemon_incarnation != state.daemon_incarnation {
1662 return Err(SpawnSubscribeRefusal::ForeignIncarnation {
1663 current: state.daemon_incarnation.clone(),
1664 });
1665 }
1666 if let Some(oldest) = state.events.front().map(|event| event.cursor.clone()) {
1667 if since.seq < oldest.seq.saturating_sub(1) {
1668 return Err(SpawnSubscribeRefusal::TooOld { oldest });
1669 }
1670 }
1671 state
1672 .events
1673 .iter()
1674 .filter(|event| event.cursor.seq > since.seq)
1675 .cloned()
1676 .collect::<Vec<_>>()
1677 } else {
1678 Vec::new()
1679 };
1680 for event in replay {
1681 let body = serde_json::to_vec(&event)
1682 .map_err(|error| SpawnSubscribeRefusal::Frame(error.to_string()))?;
1683 let frame = Frame::build_with_version(
1684 version,
1685 FrameType::StreamData,
1686 control_flags(),
1687 0,
1688 0,
1689 corr,
1690 body,
1691 )
1692 .map_err(|error| SpawnSubscribeRefusal::Frame(error.to_string()))?;
1693 frames
1694 .try_send(frame)
1695 .map_err(|error| SpawnSubscribeRefusal::Frame(error.to_string()))?;
1696 }
1697 state.subscribers.insert(
1698 (connection_id, corr),
1699 SpawnSubscriber {
1700 version,
1701 frames,
1702 lagged: Some(lagged),
1703 },
1704 );
1705 }
1706 tokio::spawn(async move {
1717 while let Some(frame) = receiver.recv().await {
1718 if sink.send(frame).await.is_err() {
1719 return;
1720 }
1721 }
1722 let Ok(first_undelivered) = lagged_rx.try_recv() else {
1723 return;
1724 };
1725 match spawn_subscriber_lagged_frame(version, corr, first_undelivered) {
1726 Ok(frame) => {
1727 let _ = sink.send(frame).await;
1728 }
1729 Err(error) => {
1730 error!(%error, corr, "failed to build lagged spawn subscriber terminal frame");
1731 }
1732 }
1733 });
1734 Ok(())
1735 }
1736
1737 fn cancel(&self, connection_id: ConnectionId, corr: u64) -> bool {
1738 let Some(subscriber) = self
1739 .0
1740 .lock()
1741 .unwrap_or_else(|p| p.into_inner())
1742 .subscribers
1743 .remove(&(connection_id, corr))
1744 else {
1745 return false;
1746 };
1747 if let Ok(frame) = Frame::build_with_version(
1748 subscriber.version,
1749 FrameType::StreamEnd,
1750 control_flags(),
1751 0,
1752 0,
1753 corr,
1754 Vec::new(),
1755 ) {
1756 tokio::spawn(async move {
1757 let _ = subscriber.frames.send(frame).await;
1758 });
1759 }
1760 true
1761 }
1762
1763 fn remove_connection(&self, connection_id: ConnectionId) {
1764 self.0
1765 .lock()
1766 .unwrap_or_else(|p| p.into_inner())
1767 .subscribers
1768 .retain(|(subscriber_connection, _), _| *subscriber_connection != connection_id);
1769 }
1770
1771 #[cfg(any(test, feature = "test-support"))]
1772 fn set_capacity(&self, capacity: usize) {
1773 self.0.lock().unwrap_or_else(|p| p.into_inner()).capacity = capacity;
1774 }
1775
1776 #[cfg(any(test, feature = "test-support"))]
1777 fn subscriber_count(&self) -> usize {
1778 self.0
1779 .lock()
1780 .unwrap_or_else(|p| p.into_inner())
1781 .subscribers
1782 .len()
1783 }
1784}
1785
1786fn spawn_subscriber_lagged_frame(
1789 version: u8,
1790 corr: u64,
1791 first_undelivered: SpawnCursor,
1792) -> Result<Frame, String> {
1793 let body = serde_json::to_vec(&subc_protocol::ErrorBody {
1794 code: SPAWN_SUBSCRIBER_LAGGED_CODE.to_string(),
1795 message: "spawn subscriber fell behind and was dropped; resubscribe from the last cursor received"
1796 .to_string(),
1797 detail: Some(serde_json::json!({
1798 "first_undelivered_cursor": first_undelivered
1799 })),
1800 })
1801 .map_err(|error| error.to_string())?;
1802 Frame::build_with_version(version, FrameType::Error, control_flags(), 0, 0, corr, body)
1803 .map_err(|error| error.to_string())
1804}
1805
1806pub trait ModuleProcessLiveness: Send + Sync {
1807 fn process_live(&self, module_id: &str) -> Option<bool>;
1808
1809 fn process_replacing(&self, _module_id: &str) -> bool {
1815 false
1816 }
1817}
1818
1819#[derive(Debug, Clone, Default)]
1821pub struct SupervisorProcessLiveness {
1822 snapshots: Arc<Mutex<HashMap<String, SharedSnapshot>>>,
1823}
1824
1825impl SupervisorProcessLiveness {
1826 pub fn new() -> Self {
1827 Self::default()
1828 }
1829
1830 fn track(&self, module_id: String, snapshot: SharedSnapshot) {
1831 let mut snapshots = self
1832 .snapshots
1833 .lock()
1834 .unwrap_or_else(|poisoned| poisoned.into_inner());
1835 snapshots.insert(module_id, snapshot);
1836 }
1837
1838 fn untrack_if_current(&self, module_id: &str, snapshot: &SharedSnapshot) {
1839 let mut snapshots = self
1840 .snapshots
1841 .lock()
1842 .unwrap_or_else(|poisoned| poisoned.into_inner());
1843 let is_current = snapshots
1844 .get(module_id)
1845 .map(|tracked| Arc::ptr_eq(tracked, snapshot))
1846 .unwrap_or(false);
1847 if is_current {
1848 snapshots.remove(module_id);
1849 }
1850 }
1851}
1852
1853impl ModuleProcessLiveness for SupervisorProcessLiveness {
1854 fn process_live(&self, module_id: &str) -> Option<bool> {
1855 let snapshot = {
1856 let snapshots = self
1857 .snapshots
1858 .lock()
1859 .unwrap_or_else(|poisoned| poisoned.into_inner());
1860 snapshots.get(module_id).cloned()
1861 }?;
1862 let snapshot = snapshot
1863 .lock()
1864 .unwrap_or_else(|poisoned| poisoned.into_inner());
1865 Some(snapshot.state == ModuleState::Running && snapshot.process_alive)
1866 }
1867
1868 fn process_replacing(&self, module_id: &str) -> bool {
1869 let Some(snapshot) = self
1870 .snapshots
1871 .lock()
1872 .unwrap_or_else(|poisoned| poisoned.into_inner())
1873 .get(module_id)
1874 .cloned()
1875 else {
1876 return false;
1877 };
1878 let snapshot = snapshot
1879 .lock()
1880 .unwrap_or_else(|poisoned| poisoned.into_inner());
1881 snapshot.enabled
1882 && match snapshot.state {
1883 ModuleState::Restarting => true,
1884 ModuleState::Draining => snapshot.draining_to_replace,
1885 ModuleState::Starting
1886 | ModuleState::Running
1887 | ModuleState::Unresponsive
1888 | ModuleState::Stopped
1889 | ModuleState::Failed
1890 | ModuleState::Disabled => false,
1891 }
1892 }
1893}
1894
1895#[cfg(test)]
1896#[derive(Debug, Default)]
1897struct ReloadExitRecordGate {
1898 reached: tokio::sync::Notify,
1899 resume: tokio::sync::Notify,
1900}
1901
1902#[derive(Debug, Clone, Copy)]
1903enum RespawnKind {
1904 Spawn,
1905 Reload,
1906}
1907
1908#[derive(Debug, Clone, Copy)]
1909struct PendingRespawn {
1910 deadline: Instant,
1911 kind: RespawnKind,
1912}
1913
1914type ReloadReply = oneshot::Sender<Result<(), SuperviseError>>;
1915
1916#[derive(Debug, Clone)]
1917struct SupervisorRuntimeConfig {
1918 start_requester: Option<Principal>,
1919 pending_activity_restart: bool,
1920 provenance_probe: ExecutableIdentityProbe,
1921 scheduled_respawn: Arc<Mutex<Option<PendingRespawn>>>,
1923 deferred_reload_reply: Arc<Mutex<Option<ReloadReply>>>,
1925 restart_policy: RestartPolicy,
1926 drain_timeout: Duration,
1929 effective_drain_timeout: Arc<Mutex<Duration>>,
1932 default_drain_timeout: Duration,
1935 health: HealthConfig,
1936 connection_file_path: Option<PathBuf>,
1937 capture_logs_dir: Option<PathBuf>,
1938 forwarding: Option<Arc<ForwardingTable>>,
1939 supervisor_handle: Option<SupervisorHandle>,
1942 stderr_ring: Arc<Mutex<StderrRing>>,
1949 terminal_ring: Arc<Mutex<TerminalRing>>,
1950 spawn_events: SpawnEventFeed,
1951 child_roster: ChildRoster,
1952 #[cfg(target_os = "linux")]
1953 cgroup_placement: Option<subc_cgroup::Placement>,
1954 #[cfg(test)]
1955 test_seed_stale_facts_before_enable_spawn: bool,
1956 #[cfg(test)]
1957 test_reload_exit_record_gate: Option<Arc<ReloadExitRecordGate>>,
1958}
1959
1960#[derive(Debug, Clone, PartialEq, Eq)]
1961struct SupervisedConfiguration {
1962 spec: ModuleSpec,
1963 health: HealthConfig,
1964}
1965
1966#[derive(Debug, Clone, Default)]
1972pub struct SupervisorHandle {
1973 activity: Arc<Mutex<Option<crate::activity::ActivityLedger>>>,
1974 activity_images: Arc<Mutex<HashMap<String, activity::ActivityImage>>>,
1975 modules: Arc<Mutex<HashMap<String, SupervisedModule>>>,
1976 configured_ids: Arc<Mutex<HashSet<String>>>,
1988 spawn_events: SpawnEventFeed,
1989 reserved_nonces: Arc<Mutex<HashMap<String, Option<String>>>>,
2000 removal_tombstones: Arc<Mutex<HashMap<String, u64>>>,
2006 spawn_nonces: Arc<Mutex<HashMap<String, String>>>,
2010 reserved_prefix_owners: Arc<Mutex<HashMap<String, String>>>,
2018 swaps: Arc<Mutex<HashMap<String, OpenSwap>>>,
2024 promotion_observer: PromotionObserverSlot,
2026 operation_lock: Arc<AsyncMutex<()>>,
2030}
2031
2032pub(crate) trait SwapPromotionObserver: Send + Sync {
2041 fn swap_promoted(&self, registration: &crate::registry::ModuleRegistration);
2042}
2043
2044#[derive(Clone, Default)]
2048struct PromotionObserverSlot(Arc<Mutex<Option<std::sync::Weak<dyn SwapPromotionObserver>>>>);
2049
2050impl fmt::Debug for PromotionObserverSlot {
2051 fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
2052 f.write_str("PromotionObserverSlot")
2053 }
2054}
2055
2056#[derive(Debug, Clone)]
2058struct OpenSwap {
2059 candidate_nonce: String,
2062 incumbent_nonce: Option<String>,
2067 candidate_admitted: bool,
2071}
2072
2073#[derive(Debug, Clone, Copy, PartialEq, Eq)]
2076pub(crate) enum SwapHelloAdmission {
2077 NotSwapping,
2080 Candidate,
2082 Refused,
2085}
2086
2087#[derive(Debug, Clone, PartialEq, Eq)]
2088pub(crate) enum ReservedHelloRejection {
2089 Exact {
2090 module_id: String,
2091 },
2092 Prefix {
2093 prefix: String,
2094 owner_module_id: String,
2095 },
2096}
2097
2098impl SupervisorHandle {
2099 pub fn new() -> Self {
2100 Self::default()
2101 }
2102
2103 pub(crate) fn spawn_snapshot(&self) -> SpawnSnapshot {
2104 self.spawn_events.snapshot()
2105 }
2106
2107 pub(crate) fn subscribe_spawns(
2108 &self,
2109 connection_id: ConnectionId,
2110 corr: u64,
2111 version: u8,
2112 since: Option<SpawnCursor>,
2113 sink: FrameSink,
2114 ) -> Result<(), SpawnSubscribeRefusal> {
2115 self.spawn_events
2116 .subscribe(connection_id, corr, version, since, sink)
2117 }
2118
2119 pub(crate) fn cancel_spawn_subscription(&self, connection_id: ConnectionId, corr: u64) -> bool {
2120 self.spawn_events.cancel(connection_id, corr)
2121 }
2122
2123 pub(crate) fn remove_spawn_subscribers(&self, connection_id: ConnectionId) {
2124 self.spawn_events.remove_connection(connection_id);
2125 }
2126
2127 #[cfg(any(test, feature = "test-support"))]
2128 pub fn set_spawn_event_capacity_for_test(&self, capacity: usize) {
2129 assert!(capacity > 0, "spawn event capacity must be non-zero");
2130 self.spawn_events.set_capacity(capacity);
2131 }
2132
2133 #[cfg(any(test, feature = "test-support"))]
2134 pub fn spawn_subscriber_count_for_test(&self) -> usize {
2135 self.spawn_events.subscriber_count()
2136 }
2137
2138 pub fn set_spawn_nonce(&self, module_id: &str, nonce: String) {
2141 self.spawn_nonces
2142 .lock()
2143 .unwrap_or_else(|poisoned| poisoned.into_inner())
2144 .insert(module_id.to_string(), nonce);
2145 }
2146
2147 pub fn set_reserved_nonce(&self, module_id: &str, nonce: String) {
2150 self.reserved_nonces
2151 .lock()
2152 .unwrap_or_else(|poisoned| poisoned.into_inner())
2153 .insert(module_id.to_string(), Some(nonce));
2154 }
2155
2156 pub fn set_reserved_prefixes(&self, owner_module_id: &str, prefixes: &[String]) {
2158 let mut owners = self
2159 .reserved_prefix_owners
2160 .lock()
2161 .unwrap_or_else(|poisoned| poisoned.into_inner());
2162 owners.retain(|_, owner| owner != owner_module_id);
2163 for prefix in prefixes {
2164 owners.insert(prefix.clone(), owner_module_id.to_string());
2165 }
2166 }
2167
2168 #[cfg(test)]
2170 pub(crate) fn spawn_nonce(&self, module_id: &str) -> Option<String> {
2171 self.spawn_nonces
2172 .lock()
2173 .unwrap_or_else(|poisoned| poisoned.into_inner())
2174 .get(module_id)
2175 .cloned()
2176 }
2177
2178 fn apply_identity_configuration(&self, spec: &ModuleSpec) {
2179 self.set_reserved_prefixes(&spec.module_id, &spec.reserved_prefixes);
2180 let spawn_nonce = self
2181 .spawn_nonces
2182 .lock()
2183 .unwrap_or_else(|poisoned| poisoned.into_inner())
2184 .get(&spec.module_id)
2185 .cloned();
2186 let mut reserved_nonces = self
2187 .reserved_nonces
2188 .lock()
2189 .unwrap_or_else(|poisoned| poisoned.into_inner());
2190 if spec.reserved {
2191 reserved_nonces.insert(spec.module_id.clone(), spawn_nonce);
2196 }
2197 drop(reserved_nonces);
2198 self.removal_tombstones
2202 .lock()
2203 .unwrap_or_else(|poisoned| poisoned.into_inner())
2204 .remove(&spec.module_id);
2205 }
2206
2207 pub fn reserved_hello_authorized(&self, module_id: &str, presented: Option<&str>) -> bool {
2212 self.reserved_hello_rejection(module_id, presented)
2213 .is_none()
2214 }
2215
2216 pub(crate) fn reserved_hello_rejection(
2217 &self,
2218 module_id: &str,
2219 presented: Option<&str>,
2220 ) -> Option<ReservedHelloRejection> {
2221 let nonces = self
2222 .reserved_nonces
2223 .lock()
2224 .unwrap_or_else(|poisoned| poisoned.into_inner());
2225 if let Some(expected) = nonces.get(module_id) {
2226 let authorized = match expected {
2230 Some(expected) => {
2231 presented.is_some_and(|p| constant_time_eq(expected.as_bytes(), p.as_bytes()))
2232 }
2233 None => false,
2234 };
2235 if authorized {
2236 return None;
2237 }
2238 return Some(ReservedHelloRejection::Exact {
2239 module_id: module_id.to_string(),
2240 });
2241 }
2242 drop(nonces);
2243
2244 let matched_prefix = self
2245 .reserved_prefix_owners
2246 .lock()
2247 .unwrap_or_else(|poisoned| poisoned.into_inner())
2248 .iter()
2249 .filter(|(prefix, _)| module_id.starts_with(prefix.as_str()))
2250 .max_by_key(|(prefix, _)| prefix.len())
2251 .map(|(prefix, owner)| (prefix.clone(), owner.clone()));
2252 let (prefix, owner_module_id) = matched_prefix?;
2253
2254 let authorized = presented.is_some_and(|presented| {
2255 self.spawn_nonces
2256 .lock()
2257 .unwrap_or_else(|poisoned| poisoned.into_inner())
2258 .get(&owner_module_id)
2259 .is_some_and(|expected| constant_time_eq(expected.as_bytes(), presented.as_bytes()))
2260 || self.swap_nonce_matches(&owner_module_id, presented)
2263 });
2264 if authorized {
2265 None
2266 } else {
2267 Some(ReservedHelloRejection::Prefix {
2268 prefix,
2269 owner_module_id,
2270 })
2271 }
2272 }
2273
2274 pub fn spawned_consumer_authorized(&self, module_id: &str, presented: &str) -> bool {
2279 if presented.is_empty() {
2280 return false;
2281 }
2282 let nonces = self
2283 .spawn_nonces
2284 .lock()
2285 .unwrap_or_else(|poisoned| poisoned.into_inner());
2286 let current = nonces
2287 .get(module_id)
2288 .is_some_and(|expected| constant_time_eq(expected.as_bytes(), presented.as_bytes()));
2289 drop(nonces);
2290 current || self.swap_nonce_matches(module_id, presented)
2295 }
2296
2297 fn swap_nonce_matches(&self, module_id: &str, presented: &str) -> bool {
2299 let swaps = self
2300 .swaps
2301 .lock()
2302 .unwrap_or_else(|poisoned| poisoned.into_inner());
2303 swaps.get(module_id).is_some_and(|swap| {
2304 constant_time_eq(swap.candidate_nonce.as_bytes(), presented.as_bytes())
2305 || swap.incumbent_nonce.as_deref().is_some_and(|incumbent| {
2306 constant_time_eq(incumbent.as_bytes(), presented.as_bytes())
2307 })
2308 })
2309 }
2310
2311 pub(crate) fn open_swap(&self, module_id: &str, candidate_nonce: String) {
2314 let incumbent_nonce = self
2315 .spawn_nonces
2316 .lock()
2317 .unwrap_or_else(|poisoned| poisoned.into_inner())
2318 .get(module_id)
2319 .cloned();
2320 self.swaps
2321 .lock()
2322 .unwrap_or_else(|poisoned| poisoned.into_inner())
2323 .insert(
2324 module_id.to_string(),
2325 OpenSwap {
2326 candidate_nonce,
2327 incumbent_nonce,
2328 candidate_admitted: false,
2329 },
2330 );
2331 }
2332
2333 pub(crate) fn close_swap(&self, module_id: &str) {
2336 self.swaps
2337 .lock()
2338 .unwrap_or_else(|poisoned| poisoned.into_inner())
2339 .remove(module_id);
2340 }
2341
2342 pub(crate) fn set_swap_promotion_observer(
2345 &self,
2346 observer: std::sync::Weak<dyn SwapPromotionObserver>,
2347 ) {
2348 *self
2349 .promotion_observer
2350 .0
2351 .lock()
2352 .unwrap_or_else(|poisoned| poisoned.into_inner()) = Some(observer);
2353 }
2354
2355 fn notify_swap_promoted(&self, registration: &crate::registry::ModuleRegistration) {
2358 let observer = self
2359 .promotion_observer
2360 .0
2361 .lock()
2362 .unwrap_or_else(|poisoned| poisoned.into_inner())
2363 .as_ref()
2364 .and_then(std::sync::Weak::upgrade);
2365 if let Some(observer) = observer {
2366 observer.swap_promoted(registration);
2367 }
2368 }
2369
2370 pub(crate) fn swap_open(&self, module_id: &str) -> bool {
2372 self.swaps
2373 .lock()
2374 .unwrap_or_else(|poisoned| poisoned.into_inner())
2375 .contains_key(module_id)
2376 }
2377
2378 fn promote_swap_nonce(&self, module_id: &str, reserved: bool) {
2383 let candidate_nonce = self
2384 .swaps
2385 .lock()
2386 .unwrap_or_else(|poisoned| poisoned.into_inner())
2387 .get(module_id)
2388 .map(|swap| swap.candidate_nonce.clone());
2389 let Some(nonce) = candidate_nonce else {
2390 return;
2391 };
2392 self.set_spawn_nonce(module_id, nonce.clone());
2393 if reserved {
2394 self.set_reserved_nonce(module_id, nonce);
2395 }
2396 }
2397
2398 pub(crate) fn swap_hello_admission(
2413 &self,
2414 module_id: &str,
2415 presented: Option<&str>,
2416 ) -> SwapHelloAdmission {
2417 let swaps = self
2418 .swaps
2419 .lock()
2420 .unwrap_or_else(|poisoned| poisoned.into_inner());
2421 let Some(swap) = swaps.get(module_id) else {
2422 return SwapHelloAdmission::NotSwapping;
2423 };
2424 let Some(presented) = presented else {
2425 return SwapHelloAdmission::Refused;
2426 };
2427 if constant_time_eq(swap.candidate_nonce.as_bytes(), presented.as_bytes()) {
2428 return if swap.candidate_admitted {
2429 SwapHelloAdmission::Refused
2430 } else {
2431 SwapHelloAdmission::Candidate
2432 };
2433 }
2434 if swap
2435 .incumbent_nonce
2436 .as_deref()
2437 .is_some_and(|incumbent| constant_time_eq(incumbent.as_bytes(), presented.as_bytes()))
2438 {
2439 return SwapHelloAdmission::NotSwapping;
2440 }
2441 SwapHelloAdmission::Refused
2442 }
2443
2444 pub(crate) fn mark_swap_candidate_admitted(&self, module_id: &str) {
2447 if let Some(swap) = self
2448 .swaps
2449 .lock()
2450 .unwrap_or_else(|poisoned| poisoned.into_inner())
2451 .get_mut(module_id)
2452 {
2453 swap.candidate_admitted = true;
2454 }
2455 }
2456
2457 pub fn spawn_launch_nonce_for(&self, module_id: &str) -> Option<String> {
2459 self.spawn_nonces
2460 .lock()
2461 .unwrap_or_else(|poisoned| poisoned.into_inner())
2462 .get(module_id)
2463 .cloned()
2464 }
2465
2466 pub fn reserved_launch_nonce_for(&self, module_id: &str) -> Option<String> {
2468 self.reserved_nonces
2469 .lock()
2470 .unwrap_or_else(|poisoned| poisoned.into_inner())
2471 .get(module_id)
2472 .cloned()
2473 .flatten()
2474 }
2475
2476 pub fn insert(&self, module: SupervisedModule) -> Option<SupervisedModule> {
2477 self.mark_configured(module.module_id());
2481 let mut modules = self
2482 .modules
2483 .lock()
2484 .unwrap_or_else(|poisoned| poisoned.into_inner());
2485 modules.insert(module.module_id().to_string(), module)
2486 }
2487
2488 fn mark_configured(&self, module_id: &str) {
2492 self.configured_ids
2493 .lock()
2494 .unwrap_or_else(|poisoned| poisoned.into_inner())
2495 .insert(module_id.to_string());
2496 }
2497
2498 fn unmark_configured_unless_rostered(&self, module_id: &str) {
2502 let modules = self
2503 .modules
2504 .lock()
2505 .unwrap_or_else(|poisoned| poisoned.into_inner());
2506 if !modules.contains_key(module_id) {
2507 self.configured_ids
2508 .lock()
2509 .unwrap_or_else(|poisoned| poisoned.into_inner())
2510 .remove(module_id);
2511 }
2512 }
2513
2514 pub(crate) fn is_configured(&self, module_id: &str) -> bool {
2522 self.configured_ids
2523 .lock()
2524 .unwrap_or_else(|poisoned| poisoned.into_inner())
2525 .contains(module_id)
2526 }
2527
2528 pub fn get(&self, module_id: &str) -> Option<SupervisedModule> {
2529 let modules = self
2530 .modules
2531 .lock()
2532 .unwrap_or_else(|poisoned| poisoned.into_inner());
2533 modules.get(module_id).cloned()
2534 }
2535
2536 pub(crate) fn record_late_health_answer(
2537 &self,
2538 module_id: &str,
2539 latency_ms: u64,
2540 ) -> Result<bool, SuperviseError> {
2541 let Some(module) = self.get(module_id) else {
2542 return Ok(false);
2543 };
2544 update_snapshot(&module.inner.snapshot, Some(module_id), |state| {
2545 state.health.late_answer_count = state.health.late_answer_count.saturating_add(1);
2546 state.health.last_late_answer_latency_ms = Some(latency_ms);
2547 state.health.consecutive_failures = 0;
2555 })?;
2556 Ok(true)
2557 }
2558
2559 pub fn record_deliberate_severance(&self, module_id: &str) -> Result<bool, SuperviseError> {
2565 let Some(module) = self.get(module_id) else {
2566 return Ok(false);
2567 };
2568 let snapshot = lock_snapshot(&module.inner.snapshot)?;
2569 let Some((pid, start_time)) = snapshot.pid.zip(snapshot.process_start_time) else {
2570 return Ok(false);
2571 };
2572 drop(snapshot);
2573 module.record_deliberate_severance(ProcessIdentity { pid, start_time })
2574 }
2575
2576 pub fn list(&self) -> Vec<SupervisedModule> {
2577 let modules = self
2578 .modules
2579 .lock()
2580 .unwrap_or_else(|poisoned| poisoned.into_inner());
2581 let mut modules = modules.values().cloned().collect::<Vec<_>>();
2582 modules.sort_by(|left, right| left.module_id().cmp(right.module_id()));
2583 modules
2584 }
2585
2586 pub(crate) fn retire(&self, module_id: &str) -> Option<SupervisedModule> {
2587 self.activity_images
2588 .lock()
2589 .unwrap_or_else(|p| p.into_inner())
2590 .remove(module_id);
2591 self.spawn_nonces
2592 .lock()
2593 .unwrap_or_else(|poisoned| poisoned.into_inner())
2594 .remove(module_id);
2595 self.close_swap(module_id);
2596 let mut reserved_nonces = self
2597 .reserved_nonces
2598 .lock()
2599 .unwrap_or_else(|poisoned| poisoned.into_inner());
2600 if reserved_nonces.contains_key(module_id) {
2601 reserved_nonces.insert(module_id.to_string(), None);
2604 }
2605 drop(reserved_nonces);
2606 self.reserved_prefix_owners
2607 .lock()
2608 .unwrap_or_else(|poisoned| poisoned.into_inner())
2609 .retain(|_, owner| owner != module_id);
2610 let removed = self
2611 .modules
2612 .lock()
2613 .unwrap_or_else(|poisoned| poisoned.into_inner())
2614 .remove(module_id);
2615 self.configured_ids
2616 .lock()
2617 .unwrap_or_else(|poisoned| poisoned.into_inner())
2618 .remove(module_id);
2619 removed
2620 }
2621
2622 pub(crate) fn record_rescan_removal(&self, module_id: &str) {
2625 self.removal_tombstones
2626 .lock()
2627 .unwrap_or_else(|poisoned| poisoned.into_inner())
2628 .insert(module_id.to_string(), unix_ms_now());
2629 }
2630
2631 pub(crate) fn removal_tombstone_age_ms(&self, module_id: &str) -> Option<u64> {
2633 self.removal_tombstones
2634 .lock()
2635 .unwrap_or_else(|poisoned| poisoned.into_inner())
2636 .get(module_id)
2637 .copied()
2638 .map(|removed_at_ms| unix_ms_now().saturating_sub(removed_at_ms))
2639 }
2640
2641 pub(crate) fn release_retained_reserved_gate(&self, module_id: &str) -> bool {
2646 if self.get(module_id).is_some() {
2647 return false;
2648 }
2649 let mut reserved_nonces = self
2650 .reserved_nonces
2651 .lock()
2652 .unwrap_or_else(|poisoned| poisoned.into_inner());
2653 if !matches!(reserved_nonces.get(module_id), Some(None)) {
2654 return false;
2655 }
2656 reserved_nonces.remove(module_id);
2657 true
2658 }
2659
2660 pub(crate) fn operation_lock(&self) -> Arc<AsyncMutex<()>> {
2661 Arc::clone(&self.operation_lock)
2662 }
2663}
2664
2665#[derive(Debug, Clone)]
2667pub struct Supervisor {
2668 registry: Arc<Registry>,
2669 restart_policy: RestartPolicy,
2670 drain_timeout: Duration,
2671 connection_file_path: Option<PathBuf>,
2672 capture_logs_dir: Option<PathBuf>,
2673 forwarding: Option<Arc<ForwardingTable>>,
2674 process_liveness: Arc<SupervisorProcessLiveness>,
2675 supervisor_handle: Option<SupervisorHandle>,
2676 health: HealthConfig,
2677 daemon_start_clock: crate::clock::StartClock,
2678 terminal_journal: Option<Arc<crate::terminal_journal::TerminalJournal>>,
2679 spawn_events: SpawnEventFeed,
2680 provenance_probe: ExecutableIdentityProbe,
2681 child_roster: ChildRoster,
2684 #[cfg(target_os = "linux")]
2685 cgroup_placement: Option<subc_cgroup::Placement>,
2686 #[cfg(test)]
2687 test_after_first_spawn: AfterFirstSpawnHook,
2688}
2689
2690pub(crate) struct RegisteredConfiguredModule {
2694 spec: ModuleSpec,
2695 runtime: SupervisorRuntimeConfig,
2696 module: SupervisedModule,
2697 commands: mpsc::Receiver<SupervisorCommand>,
2698}
2699
2700#[cfg(test)]
2706#[derive(Clone, Default)]
2707struct AfterFirstSpawnHook(Option<AfterFirstSpawnFn>);
2708
2709#[cfg(test)]
2710type AfterFirstSpawnFn = Arc<dyn Fn(&str) + Send + Sync>;
2711
2712#[cfg(test)]
2713impl fmt::Debug for AfterFirstSpawnHook {
2714 fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
2715 f.write_str("AfterFirstSpawnHook")
2716 }
2717}
2718
2719#[cfg(test)]
2720impl AfterFirstSpawnHook {
2721 fn run(&self, module_id: &str) {
2722 if let Some(hook) = &self.0 {
2723 hook(module_id);
2724 }
2725 }
2726}
2727
2728impl Supervisor {
2729 #[cfg(test)]
2730 pub(crate) fn new_for_test(registry: Arc<Registry>, policy: RestartPolicy) -> Self {
2731 let supervisor = Self::new(registry, policy);
2732 #[cfg(target_os = "macos")]
2733 let supervisor = supervisor.with_privacy_trampoline(test_privacy_trampoline());
2734 supervisor
2735 }
2736 pub fn with_privacy_trampoline(self, path: impl Into<PathBuf>) -> Self {
2741 let path = path.into();
2742 #[cfg(target_os = "macos")]
2743 {
2744 let result = probe_privacy_trampoline(&path).map(|()| path);
2745 if let Err(cause) = &result {
2746 error!(%cause, "macOS privacy identity unavailable; supervised launches will refuse");
2747 }
2748 self.child_roster.set_privacy_trampoline(result);
2749 }
2750 #[cfg(not(target_os = "macos"))]
2751 let _ = path;
2752 self
2753 }
2754 #[cfg(unix)]
2765 pub(crate) fn begin_daemon_shutdown(&self) {
2766 self.child_roster.close();
2767 if let Some(journal) = &self.terminal_journal {
2768 journal.stamp_shutdown();
2769 }
2770 }
2771
2772 #[cfg(unix)]
2776 pub(crate) async fn drain_for_daemon_shutdown(&self) -> Result<(), SuperviseError> {
2777 const NOTICE_BUDGET: Duration = Duration::from_millis(500);
2778 const DRAIN_BUDGET: Duration = Duration::from_secs(2);
2779 let Some(forwarding) = &self.forwarding else {
2780 return Ok(());
2781 };
2782 let modules = forwarding
2783 .begin_daemon_drain()
2784 .map_err(SuperviseError::Forwarding)?;
2785 let deadline_ms =
2786 unix_ms_now().saturating_add((NOTICE_BUDGET + DRAIN_BUDGET).as_millis() as u64);
2787 let mut notices = tokio::task::JoinSet::new();
2788 let mut drains = Vec::new();
2789 for (module_id, endpoint) in modules {
2790 let Some(target) = forwarding
2791 .begin_endpoint_drain(endpoint, RouteCloseReason::Restart)
2792 .map_err(SuperviseError::Forwarding)?
2793 else {
2794 continue;
2795 };
2796 let routes = forwarding
2797 .endpoint_routes(target.endpoint)
2798 .map_err(SuperviseError::Forwarding)?;
2799 let command = serde_json::to_vec(&ModuleControlCommand::Draining {
2805 reason: RouteCloseReason::Restart,
2806 deadline_ms,
2807 })
2808 .expect("module draining serializes");
2809 let mut recipients = vec![(target.sink.clone(), target.negotiated_ver, command)];
2810 let mut clients: Vec<(crate::forwarding::GoodbyeTarget, Vec<u16>)> = Vec::new();
2811 for route in routes {
2812 let client = route.goodbye_target;
2813 if let Some((_, channels)) = clients
2814 .iter_mut()
2815 .find(|(existing, _)| existing.connection_id == client.connection_id)
2816 {
2817 channels.push(client.channel);
2818 } else {
2819 let channel = client.channel;
2820 clients.push((client, vec![channel]));
2821 }
2822 }
2823 for (client, mut channels) in clients {
2824 channels.sort_unstable();
2825 channels.dedup();
2826 let closing = serde_json::to_vec(&ClientControlPush::RouteClosing {
2827 module_id: module_id.clone(),
2828 channels,
2829 reason: RouteCloseReason::Restart,
2830 })
2831 .expect("route closing serializes");
2832 recipients.push((client.sink, client.negotiated_ver, closing));
2833 }
2834 for (sink, version, body) in recipients {
2835 notices.spawn(async move {
2836 let frame = Frame::build_with_version(
2837 version,
2838 FrameType::Push,
2839 control_flags(),
2840 0,
2841 0,
2842 0,
2843 body,
2844 )
2845 .expect("bounded lifecycle notice frame builds");
2846 sink.send_flushed(frame).await
2847 });
2848 }
2849 let gauges = declared_busy_gauges(&self.registry, &module_id)?;
2850 drains.push((module_id, target.endpoint, gauges));
2851 }
2852 let notice_deadline = Instant::now() + NOTICE_BUDGET;
2855 while let Ok(Some(result)) = timeout_at(notice_deadline, notices.join_next()).await {
2856 if !matches!(result, Ok(Ok(()))) {
2857 warn!(?result, "daemon shutdown notice delivery failed");
2858 }
2859 }
2860 notices.abort_all();
2861 let deadline = Instant::now() + DRAIN_BUDGET;
2862 let mut waits = tokio::task::JoinSet::new();
2863 for (module_id, endpoint, gauges) in drains {
2864 let forwarding = Arc::clone(forwarding);
2865 let mut runtime = self.runtime_config();
2866 runtime.health.cadence = Duration::from_millis(100);
2867 waits.spawn(async move {
2868 let wait_result = wait_for_forwarding_quiescence(
2869 &forwarding,
2870 &module_id,
2871 &runtime,
2872 endpoint,
2873 deadline,
2874 &gauges,
2875 DrainScope::Endpoint(endpoint),
2876 )
2877 .await;
2878 forwarding
2882 .record_daemon_drain_result(
2883 endpoint,
2884 drained_after_quiescence_wait(&wait_result),
2885 )
2886 .map_err(SuperviseError::Forwarding)?;
2887 wait_result
2888 });
2889 }
2890 while let Ok(Some(result)) = timeout_at(deadline, waits.join_next()).await {
2891 if !matches!(result, Ok(Ok(true))) {
2892 warn!(?result, "daemon shutdown drain did not reach quiescence");
2893 }
2894 }
2895 Ok(())
2896 }
2897
2898 #[cfg(unix)]
2910 pub(crate) async fn end_children_for_daemon_shutdown(
2911 &self,
2912 already_escalated: bool,
2913 escalate: impl std::future::Future<Output = ()>,
2914 ) {
2915 tokio::pin!(escalate);
2916 let mut escalated = already_escalated;
2917 if let Some(forwarding) = &self.forwarding {
2918 let reason = CloseReason::new(
2919 "daemon_shutdown",
2920 "the daemon is exiting after its shutdown notice and drain",
2921 );
2922 if escalated {
2923 send_module_goodbyes_for_daemon_shutdown(forwarding, &reason, false).await;
2926 } else {
2927 tokio::select! {
2928 biased;
2929 _ = escalate.as_mut() => {
2930 info!("second SIGTERM: abandoning module GOODBYE delivery");
2931 escalated = true;
2932 }
2933 _ = send_module_goodbyes_for_daemon_shutdown(forwarding, &reason, true) => {}
2934 }
2935 }
2936 let closed = forwarding.close_all_connections(&reason);
2937 debug!(closed, "closed established connections for daemon shutdown");
2938 }
2939 let escalated_here = escalated && !already_escalated;
2943 let remaining_escalate = async move {
2944 if escalated_here {
2945 std::future::pending::<()>().await;
2946 } else {
2947 escalate.await;
2948 }
2949 };
2950 crate::child_roster::end_children_for_daemon_shutdown(
2951 &self.child_roster,
2952 escalated,
2953 remaining_escalate,
2954 )
2955 .await;
2956 }
2957
2958 pub fn new(registry: Arc<Registry>, restart_policy: RestartPolicy) -> Self {
2959 Self {
2960 registry,
2961 restart_policy,
2962 drain_timeout: DEFAULT_DRAIN_TIMEOUT,
2963 connection_file_path: None,
2964 capture_logs_dir: None,
2965 forwarding: None,
2966 process_liveness: Arc::new(SupervisorProcessLiveness::default()),
2967 supervisor_handle: None,
2968 health: HealthConfig::default(),
2969 daemon_start_clock: crate::clock::StartClock::capture(),
2970 terminal_journal: None,
2971 spawn_events: SpawnEventFeed::default(),
2972 provenance_probe: ExecutableIdentityProbe::default(),
2973 child_roster: ChildRoster::default(),
2974 #[cfg(target_os = "linux")]
2975 cgroup_placement: None,
2976 #[cfg(test)]
2977 test_after_first_spawn: AfterFirstSpawnHook::default(),
2978 }
2979 }
2980
2981 pub fn with_drain_timeout(mut self, drain_timeout: Duration) -> Self {
2982 self.drain_timeout = drain_timeout;
2983 self
2984 }
2985
2986 pub fn with_process_liveness(
2987 mut self,
2988 process_liveness: Arc<SupervisorProcessLiveness>,
2989 ) -> Self {
2990 self.process_liveness = process_liveness;
2991 self
2992 }
2993
2994 pub fn with_connection_file_path(mut self, connection_file_path: impl Into<PathBuf>) -> Self {
2995 self.connection_file_path = Some(connection_file_path.into());
2996 self
2997 }
2998
2999 pub fn with_capture_logs_dir(mut self, logs_dir: impl Into<PathBuf>) -> Self {
3001 self.capture_logs_dir = Some(logs_dir.into());
3002 self
3003 }
3004
3005 pub fn with_daemon_incarnation(self, daemon_incarnation: String) -> Self {
3008 self.spawn_events.configure_incarnation(daemon_incarnation);
3012 self
3013 }
3014
3015 pub fn with_terminal_journal(self, path: PathBuf, daemon_incarnation: String) -> Self {
3018 let mut this = self.with_daemon_incarnation(daemon_incarnation.clone());
3019 this.terminal_journal = Some(Arc::new(crate::terminal_journal::TerminalJournal::open(
3020 path,
3021 daemon_incarnation,
3022 )));
3023 this
3024 }
3025
3026 pub fn with_forwarding(mut self, forwarding: Arc<ForwardingTable>) -> Self {
3027 self.forwarding = Some(forwarding);
3028 self
3029 }
3030
3031 pub fn with_handle(mut self, supervisor_handle: SupervisorHandle) -> Self {
3032 self.spawn_events = supervisor_handle.spawn_events.clone();
3033 self.supervisor_handle = Some(supervisor_handle);
3034 self
3035 }
3036
3037 pub fn with_health_config(mut self, health: HealthConfig) -> Self {
3038 self.health = health;
3039 self
3040 }
3041
3042 pub fn with_live_children_record(self, path: impl Into<PathBuf>) -> Self {
3046 self.child_roster.record_to(path.into());
3047 self
3048 }
3049
3050 #[cfg(target_os = "linux")]
3051 pub fn with_cgroup_placement(
3052 mut self,
3053 cgroup_placement: Option<subc_cgroup::Placement>,
3054 ) -> Self {
3055 self.cgroup_placement = cgroup_placement;
3056 self
3057 }
3058
3059 pub fn spawn(&self, spec: ModuleSpec) -> Result<SupervisedModule, SuperviseError> {
3065 validate_spec(&spec)?;
3066 self.establish_identity(&spec);
3067
3068 let runtime = self.runtime_config();
3069 let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::starting()));
3070 let activity_spawn = runtime.begin_activity_spawn(&spec.module_id);
3071 let spawned = spawn_child(
3072 &spec,
3073 runtime.connection_file_path.as_deref(),
3074 self.supervisor_handle.as_ref(),
3075 &runtime.stderr_ring,
3076 runtime.capture_logs_dir.as_deref(),
3077 &runtime.child_roster,
3078 #[cfg(target_os = "linux")]
3079 runtime.cgroup_placement.as_ref(),
3080 );
3081 #[cfg(test)]
3082 self.test_after_first_spawn.run(&spec.module_id);
3083 let child = match spawned {
3084 Ok(child) => child,
3085 Err(err) => {
3086 self.abandon_unrostered(&spec.module_id);
3089 return Err(err);
3090 }
3091 };
3092 self.mark_first_process_running(&spec, &runtime, &snapshot, &child)?;
3093 if let Some(spawn) = activity_spawn {
3094 spawn.complete(&snapshot);
3095 }
3096
3097 Ok(self.supervised_module(spec, runtime, snapshot, Some(child)))
3098 }
3099
3100 fn establish_identity(&self, spec: &ModuleSpec) {
3113 if let Some(supervisor_handle) = &self.supervisor_handle {
3114 supervisor_handle.apply_identity_configuration(spec);
3115 supervisor_handle.mark_configured(&spec.module_id);
3116 }
3117 }
3118
3119 fn abandon_unrostered(&self, module_id: &str) {
3122 if let Some(supervisor_handle) = &self.supervisor_handle {
3123 supervisor_handle.unmark_configured_unless_rostered(module_id);
3124 }
3125 }
3126
3127 fn mark_first_process_running(
3130 &self,
3131 spec: &ModuleSpec,
3132 runtime: &SupervisorRuntimeConfig,
3133 snapshot: &SharedSnapshot,
3134 child: &SupervisedChild,
3135 ) -> Result<(), SuperviseError> {
3136 if let Err(err) = set_running(snapshot, child, &spec.module_id, &runtime.spawn_events) {
3137 self.abandon_unrostered(&spec.module_id);
3138 return Err(err);
3139 }
3140 self.process_liveness
3141 .track(spec.module_id.clone(), Arc::clone(snapshot));
3142 Ok(())
3143 }
3144
3145 pub fn supervise_configured(
3151 &self,
3152 spec: ModuleSpec,
3153 enabled: bool,
3154 ) -> Result<SupervisedModule, SuperviseError> {
3155 self.supervise_configured_requested(
3156 spec,
3157 enabled,
3158 self.health.clone(),
3159 None,
3160 self.restart_policy,
3161 None,
3162 )
3163 }
3164
3165 pub fn supervise_configured_with_health(
3171 &self,
3172 spec: ModuleSpec,
3173 enabled: bool,
3174 health: HealthConfig,
3175 drain_timeout_ms: Option<u64>,
3176 restart_policy: RestartPolicy,
3177 ) -> Result<SupervisedModule, SuperviseError> {
3178 self.supervise_configured_requested(
3179 spec,
3180 enabled,
3181 health,
3182 drain_timeout_ms,
3183 restart_policy,
3184 None,
3185 )
3186 }
3187
3188 pub(crate) fn supervise_configured_requested(
3189 &self,
3190 spec: ModuleSpec,
3191 enabled: bool,
3192 health: HealthConfig,
3193 drain_timeout_ms: Option<u64>,
3194 restart_policy: RestartPolicy,
3195 requester: Option<Principal>,
3196 ) -> Result<SupervisedModule, SuperviseError> {
3197 let registered = self.register_configured_with_health(
3198 spec,
3199 enabled,
3200 health,
3201 drain_timeout_ms,
3202 restart_policy,
3203 )?;
3204 self.start_registered_configured(registered, requester)
3205 }
3206
3207 pub(crate) fn register_configured_with_health(
3212 &self,
3213 spec: ModuleSpec,
3214 enabled: bool,
3215 health: HealthConfig,
3216 drain_timeout_ms: Option<u64>,
3217 restart_policy: RestartPolicy,
3218 ) -> Result<RegisteredConfiguredModule, SuperviseError> {
3219 validate_spec(&spec)?;
3220 self.establish_identity(&spec);
3221
3222 let mut runtime = self.runtime_config();
3223 runtime.health = health;
3224 runtime.restart_policy = restart_policy;
3225 if let Some(ms) = drain_timeout_ms {
3226 runtime.drain_timeout = Duration::from_millis(ms);
3227 *runtime
3228 .effective_drain_timeout
3229 .lock()
3230 .unwrap_or_else(|poisoned| poisoned.into_inner()) = runtime.drain_timeout;
3231 }
3232 let snapshot = if enabled {
3233 SupervisorSnapshot::starting()
3234 } else {
3235 SupervisorSnapshot::disabled()
3236 };
3237 let snapshot = {
3238 let mut snapshot = snapshot;
3239 if enabled {
3240 if let Some((pid, reason)) = runtime.child_roster.previous_instance(&spec.module_id)
3241 {
3242 snapshot.waiting_for_previous_pid = Some(pid);
3243 snapshot.waiting_for_previous_detail = Some(reason.waiting_message(pid));
3244 }
3245 }
3246 snapshot
3247 };
3248 let (module, commands) =
3249 self.register_module(&spec, &runtime, Arc::new(Mutex::new(snapshot)));
3250 Ok(RegisteredConfiguredModule {
3251 spec,
3252 runtime,
3253 module,
3254 commands,
3255 })
3256 }
3257
3258 pub(crate) fn start_registered_configured(
3261 &self,
3262 registered: RegisteredConfiguredModule,
3263 requester: Option<Principal>,
3264 ) -> Result<SupervisedModule, SuperviseError> {
3265 let RegisteredConfiguredModule {
3266 spec,
3267 mut runtime,
3268 module,
3269 commands,
3270 } = registered;
3271 runtime.start_requester = requester;
3272 let snapshot = &module.inner.snapshot;
3273 let should_spawn = {
3274 let state = lock_snapshot(snapshot)?;
3275 state.enabled && state.waiting_for_previous_pid.is_none()
3276 };
3277 let child = if should_spawn {
3278 let activity_spawn = runtime.begin_activity_spawn(&spec.module_id);
3279 let spawned = spawn_child(
3280 &spec,
3281 runtime.connection_file_path.as_deref(),
3282 self.supervisor_handle.as_ref(),
3283 &runtime.stderr_ring,
3284 runtime.capture_logs_dir.as_deref(),
3285 &runtime.child_roster,
3286 #[cfg(target_os = "linux")]
3287 runtime.cgroup_placement.as_ref(),
3288 );
3289 #[cfg(test)]
3290 self.test_after_first_spawn.run(&spec.module_id);
3291 match spawned {
3292 Ok(child) => {
3293 self.mark_first_process_running(&spec, &runtime, snapshot, &child)?;
3294 if let Some(spawn) = activity_spawn {
3295 spawn.complete(snapshot);
3296 }
3297 Some(child)
3298 }
3299 Err(err) => {
3300 if runtime.health.critical {
3301 error!(
3302 module_id = %spec.module_id,
3303 program = %spec.program.display(),
3304 error = %err,
3305 "critical configured module failed to spawn; marking failed and alerting"
3306 );
3307 } else {
3308 error!(
3309 module_id = %spec.module_id,
3310 program = %spec.program.display(),
3311 error = %err,
3312 "configured module failed to spawn; marking failed and continuing"
3313 );
3314 }
3315 update_snapshot(snapshot, Some(&spec.module_id), |state| {
3316 state.state = ModuleState::Failed;
3317 clear_current_process_facts(state);
3318 })?;
3319 None
3320 }
3321 }
3322 } else {
3323 None
3324 };
3325 self.start_monitor(spec, runtime, &module, child, commands);
3326 Ok(module)
3327 }
3328
3329 fn runtime_config(&self) -> SupervisorRuntimeConfig {
3330 let effective_drain_timeout = Arc::new(Mutex::new(self.drain_timeout));
3331 SupervisorRuntimeConfig {
3332 start_requester: None,
3333 pending_activity_restart: false,
3334 provenance_probe: self.provenance_probe.clone(),
3335 scheduled_respawn: Arc::default(),
3336 deferred_reload_reply: Arc::default(),
3337 restart_policy: self.restart_policy,
3338 drain_timeout: self.drain_timeout,
3339 child_roster: self
3342 .child_roster
3343 .for_module(Arc::clone(&effective_drain_timeout)),
3344 effective_drain_timeout,
3345 default_drain_timeout: self.drain_timeout,
3346 health: self.health.clone(),
3347 connection_file_path: self.connection_file_path.clone(),
3348 capture_logs_dir: self.capture_logs_dir.clone(),
3349 forwarding: self.forwarding.clone(),
3350 supervisor_handle: self.supervisor_handle.clone(),
3351 stderr_ring: Arc::new(Mutex::new(StderrRing::new(StderrTailConfig::default()))),
3352 terminal_ring: Arc::new(Mutex::new(
3353 TerminalRing::new(
3354 TerminalRingConfig::default(),
3355 self.daemon_start_clock.started_at_ms(),
3356 )
3357 .with_start_clock(self.daemon_start_clock)
3358 .with_journal(self.terminal_journal.clone())
3359 .with_daemon_shutdown(self.child_roster.shutdown_flag()),
3360 )),
3361 spawn_events: self.spawn_events.clone(),
3362 #[cfg(target_os = "linux")]
3363 cgroup_placement: self.cgroup_placement.clone(),
3364 #[cfg(test)]
3365 test_seed_stale_facts_before_enable_spawn: false,
3366 #[cfg(test)]
3367 test_reload_exit_record_gate: None,
3368 }
3369 }
3370
3371 fn supervised_module(
3372 &self,
3373 spec: ModuleSpec,
3374 runtime: SupervisorRuntimeConfig,
3375 snapshot: SharedSnapshot,
3376 child: Option<SupervisedChild>,
3377 ) -> SupervisedModule {
3378 let (module, commands) = self.register_module(&spec, &runtime, snapshot);
3379 self.start_monitor(spec, runtime, &module, child, commands);
3380 module
3381 }
3382
3383 fn register_module(
3384 &self,
3385 spec: &ModuleSpec,
3386 runtime: &SupervisorRuntimeConfig,
3387 snapshot: SharedSnapshot,
3388 ) -> (SupervisedModule, mpsc::Receiver<SupervisorCommand>) {
3389 let configuration = Arc::new(Mutex::new(SupervisedConfiguration {
3390 spec: spec.clone(),
3391 health: runtime.health.clone(),
3392 }));
3393 let stderr_ring = Arc::clone(&runtime.stderr_ring);
3394 let terminal_ring = Arc::clone(&runtime.terminal_ring);
3395 let restart_policy = runtime.restart_policy;
3399 let effective_drain_timeout = Arc::clone(&runtime.effective_drain_timeout);
3400 let (tx, rx) = mpsc::channel(4);
3401 let module_id = spec.module_id.clone();
3402 let module = SupervisedModule {
3403 inner: Arc::new(SupervisedModuleInner {
3404 module_id: module_id.clone(),
3405 registry: Arc::clone(&self.registry),
3406 snapshot,
3407 configuration,
3408 stderr_ring,
3409 terminal_ring,
3410 commands: tx,
3411 monitor: Mutex::new(None),
3412 restart_policy,
3413 effective_drain_timeout,
3414 provenance_probe: self.provenance_probe.clone(),
3415 }),
3416 };
3417 if let Some(supervisor_handle) = &self.supervisor_handle {
3420 supervisor_handle.insert(module.clone());
3421 }
3422 (module, rx)
3423 }
3424
3425 fn start_monitor(
3426 &self,
3427 spec: ModuleSpec,
3428 runtime: SupervisorRuntimeConfig,
3429 module: &SupervisedModule,
3430 child: Option<SupervisedChild>,
3431 commands: mpsc::Receiver<SupervisorCommand>,
3432 ) {
3433 let monitor = tokio::spawn(supervise_loop(
3434 spec,
3435 runtime,
3436 Arc::clone(&self.registry),
3437 Arc::clone(&self.process_liveness),
3438 Arc::clone(&module.inner.snapshot),
3439 child,
3440 commands,
3441 ));
3442 *module
3443 .inner
3444 .monitor
3445 .lock()
3446 .unwrap_or_else(|p| p.into_inner()) = Some(monitor);
3447 }
3448}
3449
3450impl Default for Supervisor {
3451 fn default() -> Self {
3452 Self::new(Arc::new(Registry::default()), RestartPolicy::default())
3453 }
3454}
3455
3456#[derive(Clone)]
3458pub struct SupervisedModule {
3459 inner: Arc<SupervisedModuleInner>,
3460}
3461
3462struct SupervisedModuleInner {
3463 module_id: String,
3464 registry: Arc<Registry>,
3465 snapshot: SharedSnapshot,
3466 configuration: Arc<Mutex<SupervisedConfiguration>>,
3467 stderr_ring: Arc<Mutex<StderrRing>>,
3468 terminal_ring: Arc<Mutex<TerminalRing>>,
3469 commands: mpsc::Sender<SupervisorCommand>,
3470 monitor: Mutex<Option<JoinHandle<()>>>,
3471 restart_policy: RestartPolicy,
3475 effective_drain_timeout: Arc<Mutex<Duration>>,
3476 provenance_probe: ExecutableIdentityProbe,
3477}
3478
3479impl fmt::Debug for SupervisedModule {
3480 fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
3481 f.debug_struct("SupervisedModule")
3482 .field("module_id", &self.inner.module_id)
3483 .field("status", &self.status())
3484 .finish_non_exhaustive()
3485 }
3486}
3487
3488impl SupervisedModule {
3489 pub fn module_id(&self) -> &str {
3490 &self.inner.module_id
3491 }
3492
3493 #[cfg(test)]
3497 pub(crate) fn record_health_probe_failure_for_test(
3498 &self,
3499 detail: &str,
3500 ) -> Result<(), SuperviseError> {
3501 update_snapshot(&self.inner.snapshot, Some(&self.inner.module_id), |state| {
3502 state.health.consecutive_failures = state.health.consecutive_failures.saturating_add(1);
3503 state.health.detail = Some(detail.to_string());
3504 })
3505 }
3506
3507 pub fn state(&self) -> Result<ModuleState, SuperviseError> {
3508 Ok(lock_snapshot(&self.inner.snapshot)?.state)
3509 }
3510
3511 pub(crate) fn registration_end_reason(
3512 &self,
3513 ) -> Result<Option<RegistrationEndReason>, SuperviseError> {
3514 let snapshot = lock_snapshot(&self.inner.snapshot)?;
3515 Ok(match snapshot.state {
3516 ModuleState::Draining if snapshot.draining_to_replace => {
3517 Some(RegistrationEndReason::SupervisorRestart)
3518 }
3519 ModuleState::Draining => Some(RegistrationEndReason::SupervisorStop),
3520 ModuleState::Restarting if snapshot.draining_to_replace => {
3521 Some(RegistrationEndReason::SupervisorRestart)
3522 }
3523 _ => None,
3524 })
3525 }
3526
3527 pub fn stderr_tail(
3534 &self,
3535 max_lines: Option<usize>,
3536 max_bytes: Option<usize>,
3537 ) -> StderrTailSnapshot {
3538 self.inner
3539 .stderr_ring
3540 .lock()
3541 .unwrap_or_else(|poisoned| poisoned.into_inner())
3542 .snapshot(max_lines, max_bytes)
3543 }
3544
3545 pub fn terminal_history(&self) -> TerminalHistorySnapshot {
3550 self.inner
3551 .terminal_ring
3552 .lock()
3553 .unwrap_or_else(|poisoned| poisoned.into_inner())
3554 .snapshot()
3555 }
3556
3557 pub fn durable_terminal_history(&self) -> subc_control::TerminalHistory {
3562 durable_terminal_history_of(&self.inner.terminal_ring, &self.inner.module_id)
3563 }
3564
3565 pub(crate) async fn read_durable_terminal_history(
3570 &self,
3571 ) -> Result<subc_control::TerminalHistory, tokio::task::JoinError> {
3572 let terminal_ring = Arc::clone(&self.inner.terminal_ring);
3573 let module_id = self.inner.module_id.clone();
3574 tokio::task::spawn_blocking(move || durable_terminal_history_of(&terminal_ring, &module_id))
3575 .await
3576 }
3577
3578 pub fn status(&self) -> Result<ModuleStatus, SuperviseError> {
3579 self.status_with_snapshot_lock(&self.inner.snapshot, None)
3580 .map(|(status, _)| status)
3581 }
3582
3583 pub(crate) fn record_deliberate_severance(
3584 &self,
3585 identity: ProcessIdentity,
3586 ) -> Result<bool, SuperviseError> {
3587 let mut snapshot = lock_snapshot(&self.inner.snapshot)?;
3588 if snapshot.pid != Some(identity.pid)
3589 || snapshot.process_start_time != Some(identity.start_time)
3590 {
3591 return Ok(false);
3592 }
3593 snapshot.deliberate_severance = Some(identity);
3594 Ok(true)
3595 }
3596
3597 pub(crate) fn status_for_control(
3602 &self,
3603 caller: &'static str,
3604 ) -> Result<ModuleStatus, SuperviseError> {
3605 self.status_with_snapshot_lock(&self.inner.snapshot, Some(caller))
3606 .map(|(status, _)| status)
3607 }
3608
3609 fn status_with_snapshot_lock(
3610 &self,
3611 snapshot: &SharedSnapshot,
3612 caller: Option<&'static str>,
3613 ) -> Result<(ModuleStatus, Option<SpawnedFileIdentity>), SuperviseError> {
3614 let mut guard = match caller {
3615 Some(caller) => lock_snapshot_for_control(snapshot, &self.inner.module_id, caller)?,
3616 None => lock_snapshot(snapshot)?,
3617 };
3618 let restart_count =
3621 guard.crash_restarts_in_window(self.inner.restart_policy.window, Instant::now());
3622 let snapshot = guard.clone();
3623 drop(guard);
3624 let drain_timeout = *self.inner.effective_drain_timeout.lock().map_err(|_| {
3625 SuperviseError::StatePoisoned {
3626 module_id: Some(self.inner.module_id.clone()),
3627 }
3628 })?;
3629 let registration_active = self
3630 .inner
3631 .registry
3632 .get_module(&self.inner.module_id)
3633 .map_err(SuperviseError::Registry)?
3634 .is_some();
3635 let protocol = snapshot
3636 .spawned_protocol
3637 .unwrap_or(self.declared_protocol()?);
3638 let running_process =
3639 snapshot.enabled && snapshot.state == ModuleState::Running && snapshot.process_alive;
3640 let live = match protocol {
3646 ModuleProtocol::Subc => running_process && registration_active,
3647 ModuleProtocol::None => running_process,
3648 };
3649
3650 Ok((
3651 ModuleStatus {
3652 waiting_for_previous_pid: snapshot.waiting_for_previous_pid,
3653 waiting_for_previous_detail: snapshot.waiting_for_previous_detail.clone(),
3654 module_id: self.inner.module_id.clone(),
3655 launch_nonce_source: observed_launch_nonce_source(&snapshot),
3656 state: snapshot.state,
3657 enabled: snapshot.enabled,
3658 process_alive: snapshot.process_alive,
3659 registration_active,
3660 protocol,
3661 live,
3662 restart_count,
3663 lifetime_restarts: snapshot.lifetime_restarts,
3664 spawn_generation: snapshot.spawn_generation,
3665 max_restarts: self.inner.restart_policy.max_restarts,
3666 restart_window: self.inner.restart_policy.window,
3667 drain_timeout,
3668 restart_backoff: self.inner.restart_policy.backoff,
3669 restart_max_backoff: self.inner.restart_policy.max_backoff,
3670 pid: snapshot.reported_pid(),
3671 spawned_at_ms: snapshot.spawned_at_ms,
3672 spawned_from: snapshot.spawned_from,
3673 launch_spec_digest: snapshot.launch_spec_digest,
3674 process_start_time: snapshot.process_start_time,
3675 last_exit: snapshot.last_exit,
3676 health: snapshot.health,
3677 },
3678 snapshot.spawned_file_identity.clone(),
3679 ))
3680 }
3681
3682 #[cfg(test)]
3683 pub(crate) fn hold_snapshot_for_test(
3684 &self,
3685 acquired: std::sync::mpsc::Sender<()>,
3686 hold: Duration,
3687 ) -> std::thread::JoinHandle<()> {
3688 let snapshot = Arc::clone(&self.inner.snapshot);
3689 std::thread::spawn(move || {
3690 let _guard = snapshot.lock().expect("test snapshot lock is not poisoned");
3691 acquired
3692 .send(())
3693 .expect("test receiver waits for snapshot lock");
3694 std::thread::sleep(hold);
3695 })
3696 }
3697
3698 pub(crate) async fn status_and_running_image_agreement(
3704 &self,
3705 ) -> Result<(ModuleStatus, subc_control::RunningImageAgreement), SuperviseError> {
3706 let (status, identity) = self.status_with_snapshot_lock(&self.inner.snapshot, None)?;
3707 let image = self
3708 .inner
3709 .provenance_probe
3710 .observe(
3711 status.pid,
3712 status.spawned_from.as_deref(),
3713 identity,
3714 status.process_start_time,
3715 )
3716 .await;
3717 Ok((status, image))
3718 }
3719
3720 pub(crate) async fn running_image_agreement(&self) -> subc_control::RunningImageAgreement {
3721 let snapshot = match lock_snapshot(&self.inner.snapshot) {
3722 Ok(snapshot) => snapshot.clone(),
3723 Err(_) => {
3724 return subc_control::RunningImageAgreement::Unavailable {
3725 reason: subc_control::RunningImageUnavailableReason::NotRunning,
3726 };
3727 }
3728 };
3729 self.inner
3730 .provenance_probe
3731 .observe(
3732 snapshot.reported_pid(),
3733 snapshot.spawned_from.as_deref(),
3734 snapshot.spawned_file_identity.clone(),
3735 snapshot.process_start_time,
3736 )
3737 .await
3738 }
3739
3740 pub(crate) fn child_resource_usage(&self) -> subc_control::ChildResourceUsage {
3743 #[cfg(windows)]
3744 {
3745 let image = lock_snapshot(&self.inner.snapshot)
3746 .ok()
3747 .and_then(|snapshot| {
3748 snapshot
3749 .spawned_file_identity
3750 .as_ref()
3751 .and_then(|identity| identity.windows_image.clone())
3752 });
3753 match image {
3754 Some(image) => {
3755 crate::child_resources::read_windows_handle(image.process(), image.start_time())
3756 }
3757 None => subc_control::ChildResourceUsage::Unavailable {
3758 reason: subc_control::ChildResourceUnavailableReason::NotRunning,
3759 },
3760 }
3761 }
3762 #[cfg(not(windows))]
3763 {
3764 let (pid, start_time) = match lock_snapshot(&self.inner.snapshot) {
3765 Ok(snapshot) => (snapshot.reported_pid(), snapshot.process_start_time),
3766 Err(_) => {
3767 return subc_control::ChildResourceUsage::Unavailable {
3768 reason: subc_control::ChildResourceUnavailableReason::Unreadable,
3769 }
3770 }
3771 };
3772 crate::child_resources::read(pid, start_time)
3773 }
3774 }
3775
3776 pub(crate) fn will_recover_after_connection_loss(&self) -> Result<bool, SuperviseError> {
3777 let mut snapshot = lock_snapshot(&self.inner.snapshot)?;
3778 Ok(match snapshot.state {
3779 ModuleState::Restarting => true,
3780 ModuleState::Failed | ModuleState::Disabled => false,
3781 _ if snapshot.enabled && snapshot.respawn_pending => true,
3782 _ => daemon_will_restart(&mut snapshot, &self.inner.restart_policy, Instant::now()),
3783 })
3784 }
3785
3786 #[cfg(test)]
3787 pub(crate) fn is_warming(&self) -> Result<bool, SuperviseError> {
3788 self.is_warming_with_snapshot_lock(None)
3789 }
3790
3791 pub(crate) fn is_warming_for_control(
3792 &self,
3793 caller: &'static str,
3794 ) -> Result<bool, SuperviseError> {
3795 self.is_warming_with_snapshot_lock(Some(caller))
3796 }
3797
3798 fn is_warming_with_snapshot_lock(
3799 &self,
3800 caller: Option<&'static str>,
3801 ) -> Result<bool, SuperviseError> {
3802 let snapshot = match caller {
3803 Some(caller) => {
3804 lock_snapshot_for_control(&self.inner.snapshot, &self.inner.module_id, caller)?
3805 }
3806 None => lock_snapshot(&self.inner.snapshot)?,
3807 }
3808 .clone();
3809 Ok(matches!(
3810 snapshot.state,
3811 ModuleState::Starting | ModuleState::Running | ModuleState::Restarting
3812 ))
3813 }
3814
3815 pub async fn drain(&self) -> Result<(), SuperviseError> {
3817 self.stop().await
3818 }
3819
3820 pub(crate) async fn retire_requested(
3821 &self,
3822 requester: Option<Principal>,
3823 ) -> Result<(), SuperviseError> {
3824 match self.state()? {
3825 ModuleState::Stopped | ModuleState::Failed => return Ok(()),
3826 ModuleState::Starting
3827 | ModuleState::Running
3828 | ModuleState::Unresponsive
3829 | ModuleState::Restarting
3830 | ModuleState::Draining
3831 | ModuleState::Disabled => {}
3832 }
3833
3834 let (reply_tx, reply_rx) = oneshot::channel();
3835 self.inner
3836 .commands
3837 .send(SupervisorCommand::Retire {
3838 requester,
3839 reply: reply_tx,
3840 })
3841 .await
3842 .map_err(|_| SuperviseError::CommandClosed {
3843 module_id: self.inner.module_id.clone(),
3844 })?;
3845 reply_rx.await.map_err(|_| SuperviseError::CommandClosed {
3846 module_id: self.inner.module_id.clone(),
3847 })?
3848 }
3849
3850 pub async fn stop(&self) -> Result<(), SuperviseError> {
3851 self.stop_requested(None).await
3852 }
3853
3854 pub(crate) async fn stop_requested(
3855 &self,
3856 requester: Option<Principal>,
3857 ) -> Result<(), SuperviseError> {
3858 match self.state()? {
3859 ModuleState::Stopped | ModuleState::Failed => return Ok(()),
3860 ModuleState::Starting
3861 | ModuleState::Running
3862 | ModuleState::Unresponsive
3863 | ModuleState::Restarting
3864 | ModuleState::Draining
3865 | ModuleState::Disabled => {}
3866 }
3867
3868 let (reply_tx, reply_rx) = oneshot::channel();
3869 self.inner
3870 .commands
3871 .send(SupervisorCommand::Drain {
3872 requester,
3873 reply: reply_tx,
3874 })
3875 .await
3876 .map_err(|_| SuperviseError::CommandClosed {
3877 module_id: self.inner.module_id.clone(),
3878 })?;
3879 reply_rx.await.map_err(|_| SuperviseError::CommandClosed {
3880 module_id: self.inner.module_id.clone(),
3881 })?
3882 }
3883
3884 pub async fn restart(&self, drain_timeout_ms: Option<u64>) -> Result<(), SuperviseError> {
3885 self.restart_requested(drain_timeout_ms, None).await
3886 }
3887
3888 pub(crate) async fn restart_requested(
3889 &self,
3890 drain_timeout_ms: Option<u64>,
3891 requester: Option<Principal>,
3892 ) -> Result<(), SuperviseError> {
3893 let queued_at = Instant::now();
3894 let received_at_generation = lock_snapshot(&self.inner.snapshot)?.spawn_generation;
3895 let (reply_tx, reply_rx) = oneshot::channel();
3896 self.inner
3897 .commands
3898 .send(SupervisorCommand::Restart {
3899 requester,
3900 drain_timeout_ms,
3901 received_at_generation,
3902 queued_at,
3903 reply: reply_tx,
3904 })
3905 .await
3906 .map_err(|_| SuperviseError::CommandClosed {
3907 module_id: self.inner.module_id.clone(),
3908 })?;
3909 reply_rx.await.map_err(|_| SuperviseError::CommandClosed {
3910 module_id: self.inner.module_id.clone(),
3911 })?
3912 }
3913
3914 pub async fn swap(&self, ready_timeout: Option<Duration>) -> Result<(), SuperviseError> {
3919 self.swap_requested(ready_timeout, None).await
3920 }
3921
3922 pub(crate) async fn swap_requested(
3923 &self,
3924 ready_timeout: Option<Duration>,
3925 requester: Option<Principal>,
3926 ) -> Result<(), SuperviseError> {
3927 let (reply_tx, reply_rx) = oneshot::channel();
3928 self.inner
3929 .commands
3930 .send(SupervisorCommand::Swap {
3931 requester,
3932 ready_timeout,
3933 reply: reply_tx,
3934 })
3935 .await
3936 .map_err(|_| SuperviseError::CommandClosed {
3937 module_id: self.inner.module_id.clone(),
3938 })?;
3939 reply_rx.await.map_err(|_| SuperviseError::CommandClosed {
3940 module_id: self.inner.module_id.clone(),
3941 })?
3942 }
3943
3944 pub async fn reload(&self) -> Result<(), SuperviseError> {
3945 self.reload_requested(None).await
3946 }
3947
3948 pub(crate) async fn reload_requested(
3949 &self,
3950 requester: Option<Principal>,
3951 ) -> Result<(), SuperviseError> {
3952 let (reply_tx, reply_rx) = oneshot::channel();
3953 self.inner
3954 .commands
3955 .send(SupervisorCommand::Reload {
3956 requester,
3957 reply: reply_tx,
3958 })
3959 .await
3960 .map_err(|_| SuperviseError::CommandClosed {
3961 module_id: self.inner.module_id.clone(),
3962 })?;
3963 reply_rx.await.map_err(|_| SuperviseError::CommandClosed {
3964 module_id: self.inner.module_id.clone(),
3965 })?
3966 }
3967
3968 pub async fn set_enabled(&self, enabled: bool) -> Result<bool, SuperviseError> {
3969 self.set_enabled_requested(enabled, None).await
3970 }
3971
3972 pub(crate) async fn set_enabled_requested(
3973 &self,
3974 enabled: bool,
3975 requester: Option<Principal>,
3976 ) -> Result<bool, SuperviseError> {
3977 let (reply_tx, reply_rx) = oneshot::channel();
3978 self.inner
3979 .commands
3980 .send(SupervisorCommand::SetEnabled {
3981 requester,
3982 enabled,
3983 reply: reply_tx,
3984 })
3985 .await
3986 .map_err(|_| SuperviseError::CommandClosed {
3987 module_id: self.inner.module_id.clone(),
3988 })?;
3989 reply_rx.await.map_err(|_| SuperviseError::CommandClosed {
3990 module_id: self.inner.module_id.clone(),
3991 })?
3992 }
3993
3994 pub(crate) fn declared_protocol(&self) -> Result<ModuleProtocol, SuperviseError> {
3998 let configured = self
3999 .inner
4000 .configuration
4001 .lock()
4002 .map_err(|_| SuperviseError::StatePoisoned {
4003 module_id: Some(self.inner.module_id.clone()),
4004 })?
4005 .spec
4006 .protocol;
4007 let state = lock_snapshot(&self.inner.snapshot)?;
4008 Ok(state.spawned_protocol.unwrap_or(configured))
4009 }
4010
4011 pub(crate) fn configuration(&self) -> Result<(ModuleSpec, HealthConfig), SuperviseError> {
4012 let configuration =
4013 self.inner
4014 .configuration
4015 .lock()
4016 .map_err(|_| SuperviseError::StatePoisoned {
4017 module_id: Some(self.inner.module_id.clone()),
4018 })?;
4019 Ok((configuration.spec.clone(), configuration.health.clone()))
4020 }
4021
4022 #[cfg(any(test, feature = "test-support"))]
4026 pub async fn update_spec_for_test(&self, spec: ModuleSpec) -> Result<(), SuperviseError> {
4027 let (_, health) = self.configuration()?;
4028 let drain_timeout_ms = u64::try_from(
4029 self.inner
4030 .effective_drain_timeout
4031 .lock()
4032 .unwrap_or_else(|poisoned| poisoned.into_inner())
4033 .as_millis(),
4034 )
4035 .ok();
4036 self.update_configuration(spec, health, drain_timeout_ms)
4037 .await
4038 }
4039
4040 pub(crate) async fn update_configuration(
4041 &self,
4042 spec: ModuleSpec,
4043 health: HealthConfig,
4044 drain_timeout_ms: Option<u64>,
4045 ) -> Result<(), SuperviseError> {
4046 if spec.module_id != self.inner.module_id {
4047 return Err(SuperviseError::InvalidSpec {
4048 reason: "a supervised module's module_id cannot be changed".to_string(),
4049 });
4050 }
4051 validate_spec(&spec)?;
4052 let (reply_tx, reply_rx) = oneshot::channel();
4053 self.inner
4054 .commands
4055 .send(SupervisorCommand::UpdateConfiguration {
4056 spec: spec.clone(),
4057 health: health.clone(),
4058 drain_timeout_ms,
4059 reply: reply_tx,
4060 })
4061 .await
4062 .map_err(|_| SuperviseError::CommandClosed {
4063 module_id: self.inner.module_id.clone(),
4064 })?;
4065 reply_rx.await.map_err(|_| SuperviseError::CommandClosed {
4066 module_id: self.inner.module_id.clone(),
4067 })?;
4068 let mut configuration =
4069 self.inner
4070 .configuration
4071 .lock()
4072 .map_err(|_| SuperviseError::StatePoisoned {
4073 module_id: Some(self.inner.module_id.clone()),
4074 })?;
4075 configuration.spec = spec;
4076 configuration.health = health;
4077 Ok(())
4078 }
4079}
4080
4081impl Drop for SupervisedModuleInner {
4082 fn drop(&mut self) {
4083 let Ok(mut monitor) = self.monitor.lock() else {
4084 return;
4085 };
4086 if let Some(monitor) = monitor.as_ref().filter(|monitor| !monitor.is_finished()) {
4087 let _ = update_snapshot(&self.snapshot, Some(&self.module_id), |state| {
4088 state.state = ModuleState::Stopped;
4089 clear_current_process_facts(state);
4090 });
4091 monitor.abort();
4092 }
4093 let _ = monitor.take();
4094 }
4095}
4096
4097#[derive(Debug)]
4098enum SupervisorCommand {
4099 Drain {
4100 requester: Option<Principal>,
4101 reply: oneshot::Sender<Result<(), SuperviseError>>,
4102 },
4103 Retire {
4104 requester: Option<Principal>,
4105 reply: oneshot::Sender<Result<(), SuperviseError>>,
4106 },
4107 Restart {
4108 requester: Option<Principal>,
4109 drain_timeout_ms: Option<u64>,
4114 received_at_generation: u64,
4118 queued_at: Instant,
4121 reply: oneshot::Sender<Result<(), SuperviseError>>,
4122 },
4123 Reload {
4124 requester: Option<Principal>,
4125 reply: oneshot::Sender<Result<(), SuperviseError>>,
4126 },
4127 SetEnabled {
4128 requester: Option<Principal>,
4129 enabled: bool,
4130 reply: oneshot::Sender<Result<bool, SuperviseError>>,
4131 },
4132 UpdateConfiguration {
4133 spec: ModuleSpec,
4134 health: HealthConfig,
4135 drain_timeout_ms: Option<u64>,
4138 reply: oneshot::Sender<()>,
4139 },
4140 Swap {
4141 requester: Option<Principal>,
4142 ready_timeout: Option<Duration>,
4145 reply: oneshot::Sender<Result<(), SuperviseError>>,
4147 },
4148}
4149
4150#[derive(Debug)]
4151pub enum SuperviseError {
4152 InvalidSpec {
4153 reason: String,
4154 },
4155 Spawn {
4156 program: PathBuf,
4157 source: io::Error,
4158 cgroup_path: Option<PathBuf>,
4159 },
4160 Cgroup {
4161 module_id: String,
4162 source: io::Error,
4163 },
4164 LaunchNonce {
4167 reason: String,
4168 },
4169 Wait {
4170 module_id: String,
4171 source: io::Error,
4172 },
4173 Kill {
4174 module_id: String,
4175 source: io::Error,
4176 },
4177 Forwarding(ForwardingError),
4178 Registry(RegistryError),
4179 ReloadUnavailable {
4180 module_id: String,
4181 reason: String,
4182 },
4183 Disabled {
4188 module_id: String,
4189 },
4190 ReloadFailed {
4191 module_id: String,
4192 reason: String,
4193 },
4194 RegistrationStillActive {
4195 module_id: String,
4196 waited: Duration,
4197 },
4198 StatePoisoned {
4199 module_id: Option<String>,
4200 },
4201 CommandClosed {
4202 module_id: String,
4203 },
4204 SwapInProgress {
4208 module_id: String,
4209 },
4210 SwapRefused {
4212 module_id: String,
4213 reason: SwapRefusal,
4214 },
4215 SwapFailed {
4219 module_id: String,
4220 arm: SwapFailureArm,
4221 detail: String,
4222 candidate_exit: Option<ExitReport>,
4225 },
4226}
4227
4228#[derive(Debug, Clone, Copy, PartialEq, Eq)]
4230pub enum SwapRefusal {
4231 OverlapExclusive,
4233 NotRegistered,
4236 ProtocolNone,
4239 NotConfigured,
4242 AlreadySwapping,
4244}
4245
4246impl SwapRefusal {
4247 pub fn as_str(self) -> &'static str {
4248 match self {
4249 Self::OverlapExclusive => "overlap_exclusive",
4250 Self::NotRegistered => "not_registered",
4251 Self::ProtocolNone => "protocol_none",
4252 Self::NotConfigured => "not_configured",
4253 Self::AlreadySwapping => "already_swapping",
4254 }
4255 }
4256}
4257
4258#[derive(Debug, Clone, Copy, PartialEq, Eq)]
4261pub enum SwapFailureArm {
4262 SpawnFailed,
4264 NeverRegistered,
4266 NeverReady,
4268 CandidateExited,
4270 CandidateUnhealthy,
4272 Interrupted,
4276 CutoverLost,
4281}
4282
4283impl SwapFailureArm {
4284 pub fn as_str(self) -> &'static str {
4285 match self {
4286 Self::SpawnFailed => "spawn_failed",
4287 Self::NeverRegistered => "never_registered",
4288 Self::NeverReady => "never_ready",
4289 Self::CandidateExited => "candidate_exited",
4290 Self::CandidateUnhealthy => "candidate_unhealthy",
4291 Self::Interrupted => "interrupted",
4292 Self::CutoverLost => "cutover_lost",
4293 }
4294 }
4295}
4296
4297impl fmt::Display for SuperviseError {
4298 fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
4299 match self {
4300 Self::InvalidSpec { reason } => write!(f, "invalid module spec: {reason}"),
4301 Self::Spawn {
4302 program,
4303 source,
4304 cgroup_path: Some(cgroup_path),
4305 } => write!(
4306 f,
4307 "failed to place module in cgroup '{}' while spawning '{}': {source}",
4308 cgroup_path.display(),
4309 program.display()
4310 ),
4311 Self::Spawn {
4312 program,
4313 source,
4314 cgroup_path: None,
4315 } => write!(
4316 f,
4317 "failed to spawn module '{}': {source}",
4318 program.display()
4319 ),
4320 Self::Cgroup { module_id, source } => {
4321 write!(
4322 f,
4323 "failed to prepare cgroup for module '{module_id}': {source}"
4324 )
4325 }
4326 Self::LaunchNonce { reason } => {
4327 write!(
4328 f,
4329 "failed to generate reserved-module launch nonce: {reason}"
4330 )
4331 }
4332 Self::Wait { module_id, source } => {
4333 write!(f, "failed to wait for module '{module_id}': {source}")
4334 }
4335 Self::Kill { module_id, source } => {
4336 write!(f, "failed to kill module '{module_id}': {source}")
4337 }
4338 Self::Forwarding(err) => write!(f, "forwarding error: {err}"),
4339 Self::Registry(err) => write!(f, "registry error: {err}"),
4340 Self::ReloadUnavailable { module_id, reason } => {
4341 write!(f, "reload unavailable for module '{module_id}': {reason}")
4342 }
4343 Self::Disabled { module_id } => {
4344 write!(
4345 f,
4346 "module '{module_id}' is disabled; enable it before restart or reload"
4347 )
4348 }
4349 Self::ReloadFailed { module_id, reason } => {
4350 write!(f, "reload failed for module '{module_id}': {reason}")
4351 }
4352 Self::RegistrationStillActive { module_id, waited } => write!(
4353 f,
4354 "module '{module_id}' registration remained active after waiting {waited:?}"
4355 ),
4356 Self::StatePoisoned { module_id } => match module_id {
4357 Some(module_id) => {
4358 write!(f, "supervisor state for module '{module_id}' was poisoned")
4359 }
4360 None => write!(f, "supervisor state was poisoned"),
4361 },
4362 Self::CommandClosed { module_id } => {
4363 write!(
4364 f,
4365 "supervisor command channel for module '{module_id}' is closed"
4366 )
4367 }
4368 Self::SwapInProgress { module_id } => write!(
4369 f,
4370 "module '{module_id}' is being swapped; retry once the swap has cut over or failed, or stop the module to abort the swap"
4371 ),
4372 Self::SwapRefused { module_id, reason } => match reason {
4373 SwapRefusal::OverlapExclusive => write!(
4374 f,
4375 "module '{module_id}' is declared overlap: \"exclusive\" (the default): two processes of it must not run at once, so it cannot be swapped; use a plain restart, or declare overlap: \"safe\" in its config if it really tolerates a second process"
4376 ),
4377 SwapRefusal::NotRegistered => write!(
4378 f,
4379 "module '{module_id}' is not registered, so there is no serving process to keep while a replacement warms; use a plain restart"
4380 ),
4381 SwapRefusal::ProtocolNone => write!(
4382 f,
4383 "module '{module_id}' is protocol: \"none\" and never registers, so a swap could never see its replacement become ready; use a plain restart"
4384 ),
4385 SwapRefusal::NotConfigured => write!(
4386 f,
4387 "module '{module_id}' cannot be swapped: the supervisor was built without the forwarding table or shared handle a swap needs"
4388 ),
4389 SwapRefusal::AlreadySwapping => {
4390 write!(f, "module '{module_id}' is already being swapped")
4391 }
4392 },
4393 Self::SwapFailed {
4394 module_id,
4395 arm,
4396 detail,
4397 ..
4398 } => write!(
4399 f,
4400 "swap of module '{module_id}' failed ({}): {detail}; the running process was left serving",
4401 arm.as_str()
4402 ),
4403 }
4404 }
4405}
4406
4407impl Error for SuperviseError {
4408 fn source(&self) -> Option<&(dyn Error + 'static)> {
4409 match self {
4410 Self::Spawn { source, .. }
4411 | Self::Cgroup { source, .. }
4412 | Self::Wait { source, .. }
4413 | Self::Kill { source, .. } => Some(source),
4414 Self::Forwarding(err) => Some(err),
4415 Self::Registry(err) => Some(err),
4416 Self::LaunchNonce { .. }
4417 | Self::InvalidSpec { .. }
4418 | Self::ReloadUnavailable { .. }
4419 | Self::Disabled { .. }
4420 | Self::ReloadFailed { .. }
4421 | Self::RegistrationStillActive { .. }
4422 | Self::StatePoisoned { .. }
4423 | Self::CommandClosed { .. }
4424 | Self::SwapInProgress { .. }
4425 | Self::SwapRefused { .. }
4426 | Self::SwapFailed { .. } => None,
4427 }
4428 }
4429}
4430
4431pub(crate) fn validate_spec(spec: &ModuleSpec) -> Result<(), SuperviseError> {
4432 if spec.module_id.trim().is_empty() {
4433 return Err(SuperviseError::InvalidSpec {
4434 reason: "module_id must not be empty".to_string(),
4435 });
4436 }
4437
4438 Ok(())
4439}
4440
4441#[derive(Debug, Default)]
4442struct HealthProbeRuntime {
4443 configured_health: Option<HealthConfig>,
4444 registered_connection: Option<crate::ConnectionId>,
4445 advertised: bool,
4446 next_probe_at: Option<Instant>,
4447 probe_index: u64,
4448}
4449
4450fn running_protocol(spec: &ModuleSpec, snapshot: &SharedSnapshot) -> ModuleProtocol {
4451 lock_snapshot(snapshot)
4452 .ok()
4453 .and_then(|state| state.spawned_protocol)
4454 .unwrap_or(spec.protocol)
4455}
4456
4457impl HealthProbeRuntime {
4458 fn refresh_registration(
4459 &mut self,
4460 spec: &ModuleSpec,
4461 runtime: &SupervisorRuntimeConfig,
4462 registry: &Registry,
4463 snapshot: &SharedSnapshot,
4464 ) {
4465 if self.configured_health.as_ref() != Some(&runtime.health) {
4466 self.configured_health = Some(runtime.health.clone());
4467 self.next_probe_at = None;
4468 self.registered_connection = None;
4469 self.probe_index = 0;
4470 }
4471 if running_protocol(spec, snapshot) == ModuleProtocol::None {
4475 self.registered_connection = None;
4476 self.advertised = runtime.health.http.is_some();
4477 if !self.advertised {
4478 self.next_probe_at = None;
4479 let _ = update_snapshot(snapshot, Some(&spec.module_id), |state| {
4480 let unknown = ModuleHealthStatus::default();
4481 if state.health != unknown {
4482 state.health = unknown;
4483 }
4484 });
4485 } else if self.next_probe_at.is_none() {
4486 self.next_probe_at = Some(
4487 Instant::now()
4488 + jittered_health_delay(&spec.module_id, 0, runtime.health.cadence),
4489 );
4490 }
4491 return;
4492 }
4493
4494 let registration = match registry.get_module(&spec.module_id) {
4495 Ok(registration) => registration,
4496 Err(err) => {
4497 warn!(module_id = %spec.module_id, error = %err, "health prober could not read registry");
4498 self.advertised = false;
4499 self.next_probe_at = None;
4500 return;
4501 }
4502 };
4503
4504 let Some(registration) = registration else {
4505 self.registered_connection = None;
4506 self.advertised = false;
4507 self.next_probe_at = None;
4508 return;
4509 };
4510
4511 let advertised = registration
4512 .control_ops
4513 .iter()
4514 .any(|op| op == MODULE_CONTROL_OP_HEALTH_CHECK);
4515 if !advertised {
4516 self.registered_connection = Some(registration.connection_id);
4517 self.advertised = false;
4518 self.next_probe_at = None;
4519 let _ = update_snapshot(snapshot, Some(&spec.module_id), |state| {
4520 if state.health.status != SupervisorHealthStatus::Unknown
4521 || state.health.consecutive_failures != 0
4522 || state.health.last_probe_ms.is_some()
4523 || state.health.detail.is_some()
4524 || state.health.metrics.is_some()
4525 {
4526 state.health.status = SupervisorHealthStatus::Unknown;
4527 state.health.consecutive_failures = 0;
4528 state.health.last_probe_ms = None;
4529 state.health.detail = None;
4530 state.health.metrics = None;
4531 }
4532 });
4533 return;
4534 }
4535
4536 let reregistered = self.registered_connection != Some(registration.connection_id);
4537 self.registered_connection = Some(registration.connection_id);
4538 self.advertised = true;
4539 if reregistered || self.next_probe_at.is_none() {
4540 self.probe_index = 0;
4541 self.next_probe_at = Some(
4542 Instant::now() + jittered_health_delay(&spec.module_id, 0, runtime.health.cadence),
4543 );
4544 let _ = update_snapshot(snapshot, Some(&spec.module_id), |state| {
4545 state.health.status = SupervisorHealthStatus::Unknown;
4546 state.health.consecutive_failures = 0;
4547 state.health.detail = None;
4548 state.health.metrics = None;
4549 });
4550 }
4551 }
4552
4553 fn wake_after(&self) -> Option<Duration> {
4554 if !self.advertised {
4555 return None;
4556 }
4557 self.next_probe_at
4558 .map(|next| next.saturating_duration_since(Instant::now()))
4559 }
4560
4561 fn due(&self) -> bool {
4562 self.advertised
4563 && self
4564 .next_probe_at
4565 .is_some_and(|next| Instant::now() >= next)
4566 }
4567
4568 fn schedule_next(&mut self, spec: &ModuleSpec, cadence: Duration) {
4569 self.probe_index = self.probe_index.wrapping_add(1);
4570 self.next_probe_at = Some(
4571 Instant::now() + jittered_health_delay(&spec.module_id, self.probe_index, cadence),
4572 );
4573 }
4574}
4575
4576#[derive(Debug)]
4611enum HealthProbeEvidence {
4612 LaneDead,
4614 NoAnswer,
4616 BadAnswer,
4618 Misconfigured,
4620}
4621
4622#[derive(Debug)]
4623struct HealthProbeError {
4624 evidence: HealthProbeEvidence,
4625 message: String,
4626}
4627
4628impl HealthProbeError {
4629 fn lane_dead(message: impl Into<String>) -> Self {
4630 Self::with(HealthProbeEvidence::LaneDead, message)
4631 }
4632
4633 fn no_answer(message: impl Into<String>) -> Self {
4634 Self::with(HealthProbeEvidence::NoAnswer, message)
4635 }
4636
4637 fn bad_answer(message: impl Into<String>) -> Self {
4638 Self::with(HealthProbeEvidence::BadAnswer, message)
4639 }
4640
4641 fn misconfigured(message: impl Into<String>) -> Self {
4642 Self::with(HealthProbeEvidence::Misconfigured, message)
4643 }
4644
4645 fn with(evidence: HealthProbeEvidence, message: impl Into<String>) -> Self {
4646 Self {
4647 evidence,
4648 message: message.into(),
4649 }
4650 }
4651
4652 #[allow(dead_code)]
4666 fn is_proof_of_death(&self) -> bool {
4667 matches!(self.evidence, HealthProbeEvidence::LaneDead)
4668 }
4669
4670 fn label(&self) -> &'static str {
4678 match self.evidence {
4679 HealthProbeEvidence::LaneDead => "lane-dead",
4680 HealthProbeEvidence::NoAnswer => "no-answer",
4681 HealthProbeEvidence::BadAnswer => "bad-answer",
4682 HealthProbeEvidence::Misconfigured => "daemon-misconfigured",
4683 }
4684 }
4685}
4686
4687impl fmt::Display for HealthProbeError {
4688 fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
4689 f.write_str(&self.message)
4690 }
4691}
4692
4693async fn run_health_probe_cycle(
4694 spec: &ModuleSpec,
4695 runtime: &SupervisorRuntimeConfig,
4696 registry: &Registry,
4697 process_liveness: &SupervisorProcessLiveness,
4698 snapshot: &SharedSnapshot,
4699 child: &mut Option<SupervisedChild>,
4700) {
4701 let now_ms = unix_ms_now();
4702 let http = (running_protocol(spec, snapshot) == ModuleProtocol::None)
4703 .then_some(runtime.health.http.as_deref())
4704 .flatten();
4705 let result = match http {
4706 Some(url) => probe_http_health(url, runtime.health.deadline).await,
4707 None => probe_module_health(&spec.module_id, runtime, None).await,
4708 };
4709 match result {
4710 Ok(report) => {
4711 handle_health_report(
4712 spec,
4713 runtime,
4714 registry,
4715 process_liveness,
4716 snapshot,
4717 child,
4718 report,
4719 now_ms,
4720 )
4721 .await;
4722 }
4723 Err(err) => {
4724 if http.is_some() {
4725 let _ = update_snapshot(snapshot, Some(&spec.module_id), |state| {
4726 state.health.status = SupervisorHealthStatus::Failing;
4727 });
4728 }
4729 handle_health_probe_failure(
4730 spec,
4731 runtime,
4732 registry,
4733 process_liveness,
4734 snapshot,
4735 child,
4736 err,
4737 now_ms,
4738 )
4739 .await;
4740 }
4741 }
4742}
4743
4744pub(crate) struct HttpProbeTarget<'a> {
4745 address: std::net::SocketAddr,
4746 localhost: bool,
4747 authority: &'a str,
4748 path: String,
4749}
4750
4751pub(crate) fn parse_http_probe_url(url: &str) -> Result<HttpProbeTarget<'_>, String> {
4754 if url.bytes().any(|b| b <= b' ' || b == 127) || url.contains('#') {
4755 return Err("must not contain whitespace, controls, or a fragment".into());
4756 }
4757 let rest = url
4758 .strip_prefix("http://")
4759 .ok_or("must use plain http://")?;
4760 let split = rest.find(['/', '?']).unwrap_or(rest.len());
4761 let (authority, suffix) = rest.split_at(split);
4762 let (host, port) = if let Some(rest) = authority.strip_prefix("[::1]") {
4763 ("::1", rest)
4764 } else {
4765 let split = authority.find(':').unwrap_or(authority.len());
4766 authority.split_at(split)
4767 };
4768 let ip: std::net::IpAddr = match host {
4769 "127.0.0.1" | "localhost" => std::net::Ipv4Addr::LOCALHOST.into(),
4770 "::1" => std::net::Ipv6Addr::LOCALHOST.into(),
4771 _ => return Err("host must be 127.0.0.1, [::1], or localhost".into()),
4772 };
4773 let port = if port.is_empty() {
4774 80
4775 } else {
4776 port.strip_prefix(':')
4777 .and_then(|p| p.parse::<u16>().ok())
4778 .filter(|p| *p > 0)
4779 .ok_or("must have a valid nonzero TCP port")?
4780 };
4781 let path = if suffix.is_empty() {
4782 "/".into()
4783 } else if suffix.starts_with('?') {
4784 format!("/{suffix}")
4785 } else {
4786 suffix.into()
4787 };
4788 Ok(HttpProbeTarget {
4789 address: std::net::SocketAddr::new(ip, port),
4790 localhost: host == "localhost",
4791 authority,
4792 path,
4793 })
4794}
4795
4796async fn probe_http_health(
4797 url: &str,
4798 deadline: Duration,
4799) -> Result<HealthReport, HealthProbeError> {
4800 use tokio::io::{AsyncReadExt, AsyncWriteExt, BufReader};
4801 let target = parse_http_probe_url(url).map_err(HealthProbeError::misconfigured)?;
4802 let mut response_status = String::new();
4805 let mut body = Vec::new();
4806 let probe = async {
4807 let connection = match tokio::net::TcpStream::connect(target.address).await {
4810 Err(_) if target.localhost => {
4811 tokio::net::TcpStream::connect((
4812 std::net::Ipv6Addr::LOCALHOST,
4813 target.address.port(),
4814 ))
4815 .await
4816 }
4817 result => result,
4818 };
4819 let mut stream = connection.map_err(|error| {
4820 HealthProbeError::no_answer(format!("HTTP connect failed: {error}"))
4821 })?;
4822 stream.write_all(format!("GET {} HTTP/1.1\r\nHost: {}\r\nAccept-Encoding: identity\r\nConnection: close\r\n\r\n", target.path, target.authority).as_bytes()).await
4823 .map_err(|error| HealthProbeError::no_answer(format!("HTTP write failed: {error}")))?;
4824 let mut reader = BufReader::new(stream);
4825 let mut budget = 16 * 1024;
4826 let status = http_line(&mut reader, &mut budget).await?;
4827 let mut words = status.split_ascii_whitespace();
4828 let version = words.next();
4829 let code = words
4830 .next()
4831 .filter(|word| word.len() == 3)
4832 .and_then(|word| word.parse::<u16>().ok());
4833 if !matches!(version, Some("HTTP/1.1" | "HTTP/1.0"))
4834 || !code.is_some_and(|code| (100..600).contains(&code))
4835 {
4836 return Err(HealthProbeError::bad_answer(format!(
4837 "invalid HTTP status: {status}"
4838 )));
4839 }
4840 let code = code.expect("validated status code");
4841 response_status = status.clone();
4842 let mut length = None;
4843 let mut chunked = false;
4844 loop {
4845 let line = http_line(&mut reader, &mut budget).await?;
4846 if line.is_empty() {
4847 break;
4848 }
4849 if let Some((name, value)) = line.split_once(':') {
4850 if name.eq_ignore_ascii_case("content-length") {
4851 length = Some(value.trim().parse::<u64>().map_err(|_| {
4852 HealthProbeError::bad_answer("invalid HTTP Content-Length")
4853 })?);
4854 } else if name.eq_ignore_ascii_case("transfer-encoding") {
4855 chunked = value.trim().eq_ignore_ascii_case("chunked");
4856 }
4857 }
4858 }
4859 if chunked {
4860 while body.len() < 200 {
4861 let line = http_line(&mut reader, &mut budget).await?;
4862 let size = u64::from_str_radix(line.split(';').next().unwrap_or("").trim(), 16)
4863 .map_err(|_| HealthProbeError::bad_answer("invalid HTTP chunk size"))?;
4864 if size == 0 {
4865 break;
4866 }
4867 let count = size.min((200 - body.len()) as u64) as usize;
4868 let start = body.len();
4869 (&mut reader)
4870 .take(count as u64)
4871 .read_to_end(&mut body)
4872 .await
4873 .map_err(|error| {
4874 HealthProbeError::no_answer(format!("HTTP body read failed: {error}"))
4875 })?;
4876 if body.len() - start != count {
4877 return Err(HealthProbeError::bad_answer("truncated HTTP chunk"));
4878 }
4879 if size > count as u64 || body.len() == 200 {
4880 break;
4881 }
4882 if !http_line(&mut reader, &mut budget).await?.is_empty() {
4883 return Err(HealthProbeError::bad_answer("invalid HTTP chunk delimiter"));
4884 }
4885 }
4886 } else {
4887 reader
4888 .take(length.unwrap_or(200).min(200))
4889 .read_to_end(&mut body)
4890 .await
4891 .map_err(|error| {
4892 HealthProbeError::no_answer(format!("HTTP body read failed: {error}"))
4893 })?;
4894 }
4895 if (200..300).contains(&code) {
4896 Ok(HealthReport::ok())
4897 } else {
4898 Err(HealthProbeError::bad_answer(
4899 "HTTP health endpoint returned non-2xx",
4900 ))
4901 }
4902 };
4903 let mut result = timeout(deadline, probe).await.unwrap_or_else(|_| {
4904 Err(HealthProbeError::no_answer(format!(
4905 "HTTP probe timed out after {deadline:?}"
4906 )))
4907 });
4908 if let Err(error) = &mut result {
4909 if !response_status.is_empty() {
4910 error.message = format!(
4911 "{}; {response_status}: {}",
4912 error.message,
4913 String::from_utf8_lossy(&body)
4914 );
4915 }
4916 }
4917 result
4918}
4919
4920async fn http_line(
4921 reader: &mut tokio::io::BufReader<tokio::net::TcpStream>,
4922 remaining: &mut usize,
4923) -> Result<String, HealthProbeError> {
4924 use tokio::io::{AsyncBufReadExt, AsyncReadExt};
4925 let mut line = Vec::new();
4926 (&mut *reader)
4927 .take(*remaining as u64)
4928 .read_until(b'\n', &mut line)
4929 .await
4930 .map_err(|error| {
4931 HealthProbeError::no_answer(format!("HTTP header read failed: {error}"))
4932 })?;
4933 *remaining -= line.len();
4934 if !line.ends_with(b"\r\n") {
4935 return Err(HealthProbeError::bad_answer(
4936 "HTTP headers are incomplete or exceed 16 KiB",
4937 ));
4938 }
4939 line.truncate(line.len() - 2);
4940 String::from_utf8(line).map_err(|_| HealthProbeError::bad_answer("HTTP header is not UTF-8"))
4941}
4942
4943async fn probe_module_health(
4944 module_id: &str,
4945 runtime: &SupervisorRuntimeConfig,
4946 drain_deadline: Option<Instant>,
4947) -> Result<HealthReport, HealthProbeError> {
4948 let Some(forwarding) = runtime.forwarding.as_ref() else {
4949 return Err(HealthProbeError::misconfigured(
4950 "supervisor was not configured with a forwarding table",
4951 ));
4952 };
4953 let probe_started_at = Instant::now();
4954 let mut deadline = probe_started_at + runtime.health.deadline;
4955 if let Some(drain_deadline) = drain_deadline {
4956 deadline = deadline.min(drain_deadline);
4957 }
4958 let pending = if drain_deadline.is_some() {
4959 forwarding.begin_drain_health_probe_rpc_for(
4960 module_id,
4961 MODULE_CONTROL_OP_HEALTH_CHECK,
4962 probe_started_at,
4963 deadline,
4964 )
4965 } else {
4966 forwarding.begin_health_probe_rpc_for(
4967 module_id,
4968 MODULE_CONTROL_OP_HEALTH_CHECK,
4969 probe_started_at,
4970 deadline,
4971 )
4972 }
4973 .map_err(|err| {
4974 HealthProbeError::lane_dead(format!("failed to begin health.check RPC: {err}"))
4977 })?;
4978 await_health_probe(forwarding, pending, deadline, runtime.health.deadline).await
4979}
4980
4981async fn probe_endpoint_health(
4988 endpoint: crate::ModuleEndpointId,
4989 runtime: &SupervisorRuntimeConfig,
4990 deadline_cap: Option<Instant>,
4991) -> Result<HealthReport, HealthProbeError> {
4992 let Some(forwarding) = runtime.forwarding.as_ref() else {
4993 return Err(HealthProbeError::misconfigured(
4994 "supervisor was not configured with a forwarding table",
4995 ));
4996 };
4997 let probe_started_at = Instant::now();
4998 let mut deadline = probe_started_at + runtime.health.deadline;
4999 if let Some(cap) = deadline_cap {
5000 deadline = deadline.min(cap);
5001 }
5002 let pending = forwarding
5003 .begin_endpoint_health_probe_rpc_for(
5004 endpoint,
5005 MODULE_CONTROL_OP_HEALTH_CHECK,
5006 probe_started_at,
5007 deadline,
5008 )
5009 .map_err(|err| {
5010 HealthProbeError::lane_dead(format!("failed to begin health.check RPC: {err}"))
5011 })?;
5012 await_health_probe(forwarding, pending, deadline, runtime.health.deadline).await
5013}
5014
5015async fn await_health_probe(
5017 forwarding: &ForwardingTable,
5018 pending: PendingModuleControlRpc,
5019 deadline: Instant,
5020 probe_budget: Duration,
5021) -> Result<HealthReport, HealthProbeError> {
5022 let PendingModuleControlRpc {
5023 endpoint,
5024 module_sink,
5025 negotiated_ver,
5026 corr,
5027 receiver,
5028 } = pending;
5029 let body = serde_json::to_vec(&ModuleControlRequest::HealthCheck {}).map_err(|err| {
5030 HealthProbeError::misconfigured(format!("failed to encode health.check: {err}"))
5031 })?;
5032 let frame = Frame::build_with_version(
5033 negotiated_ver,
5034 FrameType::Request,
5035 control_flags(),
5036 0,
5037 0,
5038 corr,
5039 body,
5040 )
5041 .map_err(|err| {
5042 HealthProbeError::misconfigured(format!("failed to build health.check frame: {err}"))
5043 })?;
5044
5045 match timeout_at(deadline, module_sink.send(frame)).await {
5051 Ok(Ok(())) => {}
5052 Ok(Err(err)) => {
5053 let _ = forwarding.cancel_module_control_rpc(endpoint, corr);
5054 return Err(HealthProbeError::lane_dead(format!(
5057 "failed to send health.check: {err}"
5058 )));
5059 }
5060 Err(_elapsed) => {
5061 let _ = forwarding.cancel_module_control_rpc(endpoint, corr);
5062 return Err(HealthProbeError::no_answer(
5066 "health.check send timed out before enqueue (module egress full)",
5067 ));
5068 }
5069 }
5070
5071 match timeout_at(deadline, receiver).await {
5072 Ok(Ok(ModuleControlRpcOutcome::Response(response))) => {
5076 response.health_report().ok_or_else(|| {
5077 HealthProbeError::bad_answer("health.check RPC returned a non-health response")
5078 })
5079 }
5080 Ok(Ok(ModuleControlRpcOutcome::Rejected(body))) => Err(HealthProbeError::bad_answer(
5081 format!("health.check rejected: {}", body.message),
5082 )),
5083 Ok(Ok(ModuleControlRpcOutcome::ModuleGone(message))) => {
5084 Err(HealthProbeError::lane_dead(message))
5085 }
5086 Ok(Ok(ModuleControlRpcOutcome::MalformedResponse(message))) => {
5087 Err(HealthProbeError::bad_answer(message))
5088 }
5089 Ok(Ok(ModuleControlRpcOutcome::UnexpectedOp { expected, actual })) => {
5090 Err(HealthProbeError::bad_answer(format!(
5091 "expected module-control op '{expected}', got '{actual}'"
5092 )))
5093 }
5094 Ok(Ok(ModuleControlRpcOutcome::DeadlineElapsed)) => Err(HealthProbeError::bad_answer(
5098 "module answered health.check after its daemon deadline",
5099 )),
5100 Ok(Err(_)) => Err(HealthProbeError::misconfigured(
5101 "health.check waiter was canceled before the module responded",
5102 )),
5103 Err(_) => {
5104 let _ = forwarding.tombstone_health_probe_rpc(endpoint, corr);
5105 Err(HealthProbeError::no_answer(format!(
5106 "module did not answer health.check within {probe_budget:?}"
5107 )))
5108 }
5109 }
5110}
5111
5112#[allow(clippy::too_many_arguments)]
5113async fn handle_health_report(
5114 spec: &ModuleSpec,
5115 runtime: &SupervisorRuntimeConfig,
5116 registry: &Registry,
5117 process_liveness: &SupervisorProcessLiveness,
5118 snapshot: &SharedSnapshot,
5119 child: &mut Option<SupervisedChild>,
5120 report: HealthReport,
5121 now_ms: u64,
5122) {
5123 let status = supervisor_health_status(report.status);
5124 let detail = report.detail.clone();
5125 let metrics = truncate_health_metrics(report.metrics);
5126 let _ = update_snapshot(snapshot, Some(&spec.module_id), |state| {
5127 state.health.status = status;
5128 state.health.last_probe_ms = Some(now_ms);
5129 state.health.detail = detail.clone();
5130 state.health.metrics = metrics.clone();
5131 state.health.consecutive_failures = 0;
5132 });
5133
5134 let action = match report.status {
5135 HealthStatus::Ok => return,
5136 HealthStatus::Degraded => runtime.health.on_degraded,
5137 HealthStatus::Failing => runtime.health.on_failing,
5138 };
5139 apply_l3_health_action(
5140 spec,
5141 runtime,
5142 registry,
5143 process_liveness,
5144 snapshot,
5145 child,
5146 status,
5147 detail.as_deref(),
5148 action,
5149 now_ms,
5150 )
5151 .await;
5152}
5153
5154#[allow(clippy::too_many_arguments)]
5155async fn handle_health_probe_failure(
5156 spec: &ModuleSpec,
5157 runtime: &SupervisorRuntimeConfig,
5158 registry: &Registry,
5159 process_liveness: &SupervisorProcessLiveness,
5160 snapshot: &SharedSnapshot,
5161 child: &mut Option<SupervisedChild>,
5162 err: HealthProbeError,
5163 now_ms: u64,
5164) {
5165 let threshold = runtime.health.failure_threshold.max(1);
5166 let mut failures = 0;
5167 let detail = format!("[{}] {err}", err.label());
5172 let _ = update_snapshot(snapshot, Some(&spec.module_id), |state| {
5173 if state.spawned_protocol.unwrap_or(spec.protocol) == ModuleProtocol::Subc {
5176 state.health.status = SupervisorHealthStatus::Unknown;
5177 }
5178 state.health.last_probe_ms = Some(now_ms);
5179 state.health.consecutive_failures = state.health.consecutive_failures.saturating_add(1);
5180 state.health.detail = Some(detail.clone());
5181 state.health.metrics = None;
5182 failures = state.health.consecutive_failures;
5183 });
5184
5185 if failures < threshold {
5186 warn!(
5187 module_id = %spec.module_id,
5188 consecutive_failures = failures,
5189 threshold,
5190 evidence = err.label(),
5191 detail = %detail,
5192 "health.check probe failed"
5193 );
5194 return;
5195 }
5196
5197 let _ = update_snapshot(snapshot, Some(&spec.module_id), |state| {
5198 state.state = ModuleState::Unresponsive;
5199 state.health.status = SupervisorHealthStatus::Unresponsive;
5200 });
5201 if runtime.health.critical {
5205 error!(
5206 module_id = %spec.module_id,
5207 status = "unresponsive",
5208 evidence = err.label(),
5209 detail = %detail,
5210 "critical module health alert"
5211 );
5212 } else {
5213 warn!(
5214 module_id = %spec.module_id,
5215 status = "unresponsive",
5216 evidence = err.label(),
5217 detail = %detail,
5218 "module health threshold breached"
5219 );
5220 }
5221 if let Err(err) = health_restart_child(
5222 spec,
5223 runtime,
5224 registry,
5225 process_liveness,
5226 snapshot,
5227 child,
5228 SupervisorHealthStatus::Unresponsive,
5229 Some(&detail),
5230 now_ms,
5231 )
5232 .await
5233 {
5234 error!(module_id = %spec.module_id, error = %err, "health-triggered restart failed");
5235 }
5236}
5237
5238#[allow(clippy::too_many_arguments)]
5239async fn apply_l3_health_action(
5240 spec: &ModuleSpec,
5241 runtime: &SupervisorRuntimeConfig,
5242 registry: &Registry,
5243 process_liveness: &SupervisorProcessLiveness,
5244 snapshot: &SharedSnapshot,
5245 child: &mut Option<SupervisedChild>,
5246 status: SupervisorHealthStatus,
5247 detail: Option<&str>,
5248 action: HealthAction,
5249 now_ms: u64,
5250) {
5251 record_health_action(snapshot, &spec.module_id, action.to_string(), now_ms);
5252 match action {
5253 HealthAction::Report => {
5254 info!(
5255 module_id = %spec.module_id,
5256 status = ?status,
5257 detail,
5258 "module reported non-ok health"
5259 );
5260 }
5261 HealthAction::Alert => {
5262 error!(
5263 module_id = %spec.module_id,
5264 status = ?status,
5265 detail,
5266 "module health alert"
5267 );
5268 }
5269 HealthAction::Restart => {
5270 if let Err(err) = health_restart_child(
5271 spec,
5272 runtime,
5273 registry,
5274 process_liveness,
5275 snapshot,
5276 child,
5277 status,
5278 detail,
5279 now_ms,
5280 )
5281 .await
5282 {
5283 error!(module_id = %spec.module_id, error = %err, "health-triggered restart failed");
5284 }
5285 }
5286 }
5287}
5288
5289#[allow(clippy::too_many_arguments)]
5290async fn health_restart_child(
5291 spec: &ModuleSpec,
5292 runtime: &SupervisorRuntimeConfig,
5293 registry: &Registry,
5294 process_liveness: &SupervisorProcessLiveness,
5295 snapshot: &SharedSnapshot,
5296 child: &mut Option<SupervisedChild>,
5297 status: SupervisorHealthStatus,
5298 detail: Option<&str>,
5299 now_ms: u64,
5300) -> Result<(), SuperviseError> {
5301 let (enabled, schedule) = {
5302 let mut state = lock_snapshot(snapshot)?;
5303 let enabled = state.enabled;
5304 let schedule = if enabled {
5305 state.next_crash_restart(&runtime.restart_policy, Instant::now())
5306 } else {
5307 None
5308 };
5309 (enabled, schedule)
5310 };
5311
5312 if !enabled {
5313 return Err(SuperviseError::Disabled {
5314 module_id: spec.module_id.clone(),
5315 });
5316 }
5317
5318 if schedule.is_none() {
5319 record_health_action(snapshot, &spec.module_id, "failed".to_string(), now_ms);
5320 error!(
5321 module_id = %spec.module_id,
5322 status = ?status,
5323 detail,
5324 max_restarts = runtime.restart_policy.max_restarts,
5325 window_secs = runtime.restart_policy.window.as_secs(),
5326 reason = %runtime.restart_policy.budget_exhausted_detail(),
5327 "health restart budget exhausted; marking module failed"
5328 );
5329 let stop_notice = begin_forwarding_drain_if_configured(
5330 spec,
5331 runtime,
5332 registry,
5333 snapshot,
5334 Some(true),
5335 RouteCloseReason::Disable,
5336 )
5337 .await?;
5338 update_snapshot(snapshot, Some(&spec.module_id), |state| {
5339 state.drain_disposition_detail = Some(runtime.restart_policy.budget_exhausted_detail());
5340 })?;
5341 drain_optional_child(
5342 &spec.module_id,
5343 spec.protocol,
5344 stop_notice,
5345 registry,
5346 runtime.forwarding.as_deref(),
5347 snapshot,
5348 &runtime.terminal_ring,
5349 &runtime.spawn_events,
5350 child,
5351 runtime.drain_timeout,
5352 ModuleState::Failed,
5353 Some(true),
5354 )
5355 .await?;
5356 process_liveness.untrack_if_current(&spec.module_id, snapshot);
5357 return Ok(());
5358 }
5359
5360 let schedule = schedule.expect("a health restart must have a crash-restart schedule");
5361 let mut restart_count = 0;
5362 update_snapshot(snapshot, Some(&spec.module_id), |state| {
5363 restart_count = state.crash_restarts.len();
5364 state.state = ModuleState::Unresponsive;
5365 state.health.status = status;
5366 state.health.last_action = Some(HealthAction::Restart.to_string());
5367 state.health.last_action_ms = Some(now_ms);
5368 })?;
5369 warn!(
5370 module_id = %spec.module_id,
5371 status = ?status,
5372 detail,
5373 restart_count,
5374 restart_in_window = schedule.restart_in_window,
5375 delay_ms = schedule.delay.as_millis() as u64,
5376 "health-triggered module restart"
5377 );
5378
5379 let stop_notice = begin_forwarding_drain_if_configured(
5380 spec,
5381 runtime,
5382 registry,
5383 snapshot,
5384 Some(true),
5385 RouteCloseReason::Restart,
5386 )
5387 .await?;
5388 drain_optional_child(
5389 &spec.module_id,
5390 spec.protocol,
5391 stop_notice,
5392 registry,
5393 runtime.forwarding.as_deref(),
5394 snapshot,
5395 &runtime.terminal_ring,
5396 &runtime.spawn_events,
5397 child,
5398 runtime.drain_timeout,
5399 ModuleState::Restarting,
5400 Some(true),
5401 )
5402 .await?;
5403 schedule_respawn(
5404 runtime,
5405 snapshot,
5406 &spec.module_id,
5407 schedule.delay,
5408 RespawnKind::Spawn,
5409 )
5410}
5411
5412fn cancel_deferred_reload(runtime: &SupervisorRuntimeConfig, module_id: &str, reason: &str) {
5413 if let Some(reply) = runtime
5414 .deferred_reload_reply
5415 .lock()
5416 .unwrap_or_else(|p| p.into_inner())
5417 .take()
5418 {
5419 let _ = reply.send(Err(SuperviseError::ReloadFailed {
5420 module_id: module_id.to_string(),
5421 reason: reason.to_string(),
5422 }));
5423 }
5424}
5425
5426fn schedule_respawn(
5427 runtime: &SupervisorRuntimeConfig,
5428 snapshot: &SharedSnapshot,
5429 module_id: &str,
5430 delay: Duration,
5431 kind: RespawnKind,
5432) -> Result<(), SuperviseError> {
5433 cancel_deferred_reload(runtime, module_id, "respawn superseded by another restart");
5434 update_snapshot(snapshot, Some(module_id), |state| {
5435 state.respawn_pending = true
5436 })?;
5437 *runtime
5438 .scheduled_respawn
5439 .lock()
5440 .unwrap_or_else(|p| p.into_inner()) = Some(PendingRespawn {
5441 deadline: Instant::now() + delay,
5442 kind,
5443 });
5444 Ok(())
5445}
5446
5447fn record_health_action(snapshot: &SharedSnapshot, module_id: &str, action: String, now_ms: u64) {
5448 let _ = update_snapshot(snapshot, Some(module_id), |state| {
5449 state.health.last_action = Some(action);
5450 state.health.last_action_ms = Some(now_ms);
5451 });
5452}
5453
5454fn supervisor_health_status(status: HealthStatus) -> SupervisorHealthStatus {
5455 match status {
5456 HealthStatus::Ok => SupervisorHealthStatus::Ok,
5457 HealthStatus::Degraded => SupervisorHealthStatus::Degraded,
5458 HealthStatus::Failing => SupervisorHealthStatus::Failing,
5459 }
5460}
5461
5462fn truncate_health_metrics(metrics: Option<Value>) -> Option<Value> {
5474 let metrics = metrics?;
5475 match serde_json::to_vec(&metrics) {
5476 Ok(encoded) if encoded.len() > MAX_HEALTH_METRICS_BYTES => Some(serde_json::json!({
5477 "truncated": true,
5478 "original_bytes": encoded.len(),
5479 })),
5480 Ok(_) | Err(_) => Some(metrics),
5481 }
5482}
5483
5484fn jittered_health_delay(module_id: &str, probe_index: u64, cadence: Duration) -> Duration {
5490 if cadence.is_zero() {
5491 return Duration::ZERO;
5492 }
5493 let cadence_ms = cadence.as_millis() as u64;
5494 if cadence_ms == 0 {
5510 return cadence;
5511 }
5512 let jitter_span = (cadence_ms / 10).max(1);
5527 let hash = module_id.as_bytes().iter().fold(
5528 probe_index.wrapping_mul(0x9E37_79B9_7F4A_7C15),
5529 |acc, byte| {
5530 acc.wrapping_mul(1099511628211)
5531 .wrapping_add(u64::from(*byte))
5532 },
5533 );
5534 cadence + Duration::from_millis(hash % jitter_span)
5535}
5536
5537#[cfg(test)]
5538mod tests {
5539 use super::*;
5540
5541 #[test]
5542 fn readding_a_module_clears_its_rescan_removal_tombstone() {
5543 let handle = SupervisorHandle::new();
5544 let module_id = "readded-tombstone";
5545 handle.record_rescan_removal(module_id);
5546 assert!(handle.removal_tombstone_age_ms(module_id).is_some());
5547
5548 handle.apply_identity_configuration(&ModuleSpec {
5549 module_id: module_id.to_string(),
5550 program: PathBuf::from("/test/module"),
5551 args: Vec::new(),
5552 env: Vec::new(),
5553 reserved: false,
5554 reserved_prefixes: Vec::new(),
5555 protocol: ModuleProtocol::Subc,
5556 overlap: Default::default(),
5557 });
5558
5559 assert!(
5560 handle.removal_tombstone_age_ms(module_id).is_none(),
5561 "a re-added module must not retain a stale removal tombstone"
5562 );
5563 }
5564
5565 #[derive(Debug, PartialEq, Eq)]
5568 struct OwnerInSpawnWindow {
5569 module_id: String,
5570 configured: bool,
5571 on_roster: bool,
5572 admission_refusal: Option<&'static str>,
5573 }
5574
5575 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
5586 async fn a_new_module_is_configured_before_its_first_process_can_register() {
5587 use crate::scopes::ScopeTable;
5588 use subc_protocol::{error_codes, scope::ScopeSelector, Principal};
5589
5590 let dir = cortexkit_test_support::ScratchDir::new("configured-before-spawn");
5591 let stub = |module_id: &str, program: PathBuf| ModuleSpec {
5592 module_id: module_id.to_string(),
5593 program,
5594 args: Vec::new(),
5595 env: ["XDG_DATA_HOME", "XDG_RUNTIME_DIR", "XDG_CONFIG_HOME"]
5596 .into_iter()
5597 .map(|key| (key.to_string(), dir.path().display().to_string()))
5598 .chain([("FAKE_AFT_NEVER_CONNECT".to_string(), "1".to_string())])
5599 .collect(),
5600 reserved: false,
5601 reserved_prefixes: Vec::new(),
5602 protocol: ModuleProtocol::Subc,
5603 overlap: Default::default(),
5604 };
5605 let live = super::terminal_history_tests::fake_aft_stub_path();
5606 let missing = dir.path().join("definitely-missing-module");
5607
5608 let handle = SupervisorHandle::new();
5609 let mut supervisor =
5610 Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default())
5611 .with_handle(handle.clone());
5612 let observed = Arc::new(Mutex::new(Vec::<OwnerInSpawnWindow>::new()));
5613 let hook_handle = handle.clone();
5614 let hook_observed = Arc::clone(&observed);
5615 supervisor.test_after_first_spawn = AfterFirstSpawnHook(Some(Arc::new(move |module_id| {
5616 let configured = hook_handle.is_configured(module_id);
5619 let selector = ScopeSelector::new(
5620 Principal::Reserved {
5621 module_id: module_id.to_string(),
5622 },
5623 "s".to_string(),
5624 Some(1),
5625 );
5626 let carrier = Principal::Reserved {
5627 module_id: "carrier".to_string(),
5628 };
5629 let admission_refusal = match ScopeTable::new(Vec::<String>::new())
5630 .admit(&carrier, module_id, &selector, configured)
5631 {
5632 Ok(_) => None,
5633 Err(refusal) => Some(refusal.code),
5634 };
5635 hook_observed.lock().unwrap().push(OwnerInSpawnWindow {
5636 module_id: module_id.to_string(),
5637 configured,
5638 on_roster: hook_handle.get(module_id).is_some(),
5639 admission_refusal,
5640 });
5641 })));
5642
5643 let plain = supervisor.spawn(stub("plain", live.clone())).unwrap();
5644 let configured = supervisor
5645 .supervise_configured(stub("configured", live.clone()), true)
5646 .unwrap();
5647 let with_health = supervisor
5648 .supervise_configured_with_health(
5649 stub("with-health", live.clone()),
5650 true,
5651 HealthConfig::default(),
5652 None,
5653 RestartPolicy::default(),
5654 )
5655 .unwrap();
5656 let failed = supervisor
5659 .supervise_configured_with_health(
5660 stub("failed-spawn", missing.clone()),
5661 true,
5662 HealthConfig::default(),
5663 None,
5664 RestartPolicy::default(),
5665 )
5666 .unwrap();
5667 assert!(supervisor.spawn(stub("spawn-error", missing)).is_err());
5670
5671 let expected = [
5672 "plain",
5673 "configured",
5674 "with-health",
5675 "failed-spawn",
5676 "spawn-error",
5677 ]
5678 .into_iter()
5679 .map(|module_id| OwnerInSpawnWindow {
5680 module_id: module_id.to_string(),
5681 configured: true,
5682 on_roster: !matches!(module_id, "plain" | "spawn-error"),
5683 admission_refusal: Some(error_codes::SCOPE_NOT_SYNCED),
5684 })
5685 .collect::<Vec<_>>();
5686 assert_eq!(*observed.lock().unwrap(), expected);
5687
5688 for module_id in ["plain", "configured", "with-health", "failed-spawn"] {
5689 assert!(
5690 handle.get(module_id).is_some(),
5691 "{module_id} is on the roster"
5692 );
5693 assert!(
5694 handle.is_configured(module_id),
5695 "{module_id} stays configured"
5696 );
5697 }
5698 assert!(handle.get("spawn-error").is_none());
5699 assert!(
5700 !handle.is_configured("spawn-error"),
5701 "a plain spawn that failed must not leave its module marked configured"
5702 );
5703
5704 handle.retire("failed-spawn");
5706 assert!(!handle.is_configured("failed-spawn"));
5707
5708 for module in [plain, configured, with_health] {
5709 module.stop().await.unwrap();
5710 }
5711 drop(failed);
5712 }
5713
5714 fn stale_process_snapshot(state: ModuleState, enabled: bool) -> SharedSnapshot {
5715 let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::new(state, enabled)));
5716 update_snapshot(&snapshot, Some("stale-process-facts"), |snapshot| {
5717 snapshot.process_alive = true;
5718 snapshot.pid = Some(41);
5719 snapshot.spawned_at_ms = Some(42);
5720 snapshot.spawned_from = Some(PathBuf::from("/spawned/module"));
5721 snapshot.launch_spec_digest = Some(LaunchSpecDigest([0xA5; 32]));
5722 snapshot.spawned_file_identity = Some(SpawnedFileIdentity {
5723 device: 43,
5724 inode: 44,
5725 #[cfg(windows)]
5726 windows_image: None,
5727 });
5728 })
5729 .unwrap();
5730 snapshot
5731 }
5732
5733 fn assert_snapshot_process_facts_cleared(snapshot: &SharedSnapshot) {
5734 let snapshot = lock_snapshot(snapshot).unwrap();
5735 assert!(!snapshot.process_alive);
5736 assert_eq!(snapshot.pid, None);
5737 assert_eq!(snapshot.spawned_at_ms, None);
5738 assert_eq!(snapshot.spawned_from, None);
5739 assert_eq!(snapshot.launch_spec_digest, None);
5740 assert_eq!(snapshot.spawned_file_identity, None);
5741 }
5742
5743 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
5744 async fn failed_enable_spawn_clears_preexisting_current_process_facts() {
5745 let supervisor =
5746 Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default());
5747 let mut runtime = supervisor.runtime_config();
5748 runtime.test_seed_stale_facts_before_enable_spawn = true;
5749 let snapshot = stale_process_snapshot(ModuleState::Disabled, false);
5750 let mut child = None;
5751 let spec = ModuleSpec {
5752 module_id: "failed-enable-clears-facts".to_string(),
5753 program: PathBuf::from("/definitely/missing/failed-enable-module"),
5754 args: Vec::new(),
5755 env: Vec::new(),
5756 reserved: false,
5757 reserved_prefixes: Vec::new(),
5758 protocol: ModuleProtocol::Subc,
5759 overlap: Default::default(),
5760 };
5761
5762 let result = set_child_enabled(
5763 &spec,
5764 &runtime,
5765 &supervisor.registry,
5766 &supervisor.process_liveness,
5767 &snapshot,
5768 &mut child,
5769 true,
5770 )
5771 .await;
5772
5773 assert!(matches!(result, Err(SuperviseError::Spawn { .. })));
5774 assert_eq!(lock_snapshot(&snapshot).unwrap().state, ModuleState::Failed);
5775 assert_snapshot_process_facts_cleared(&snapshot);
5776 }
5777
5778 #[tokio::test]
5779 async fn start_revives_stranded_restarting_but_not_pending_backoff() {
5780 let supervisor =
5781 Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default());
5782 let runtime = supervisor.runtime_config();
5783 let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::new(
5784 ModuleState::Restarting,
5785 true,
5786 )));
5787 let spec = ModuleSpec {
5788 module_id: "start-stranded-restarting".to_string(),
5789 program: super::terminal_history_tests::fake_aft_stub_path(),
5790 args: Vec::new(),
5791 env: vec![("FAKE_AFT_NEVER_CONNECT".to_string(), "1".to_string())],
5792 reserved: false,
5793 reserved_prefixes: Vec::new(),
5794 protocol: ModuleProtocol::None,
5795 overlap: Default::default(),
5796 };
5797 let mut child = None;
5798 lock_snapshot(&snapshot).unwrap().respawn_pending = true;
5799 assert!(!super::set_child_enabled(
5800 &spec,
5801 &runtime,
5802 &Registry::default(),
5803 &supervisor.process_liveness,
5804 &snapshot,
5805 &mut child,
5806 true
5807 )
5808 .await
5809 .unwrap());
5810 assert!(child.is_none());
5811 lock_snapshot(&snapshot).unwrap().respawn_pending = false;
5812 assert!(super::set_child_enabled(
5813 &spec,
5814 &runtime,
5815 &Registry::default(),
5816 &supervisor.process_liveness,
5817 &snapshot,
5818 &mut child,
5819 true
5820 )
5821 .await
5822 .unwrap());
5823 assert_eq!(
5824 lock_snapshot(&snapshot).unwrap().state,
5825 ModuleState::Running
5826 );
5827 let mut child = child.unwrap();
5828 child.start_kill().unwrap();
5829 child.wait().await.unwrap();
5830 }
5831
5832 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
5833 async fn failed_reload_spawn_clears_current_process_facts() {
5834 let supervisor =
5835 Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default());
5836 let mut runtime = supervisor.runtime_config();
5837 runtime.restart_policy = RestartPolicy::new(0, Duration::ZERO);
5838 let snapshot = stale_process_snapshot(ModuleState::Running, true);
5839 let mut child = None;
5840 let spec = ModuleSpec {
5841 module_id: "failed-reload-clears-facts".to_string(),
5842 program: PathBuf::from("/unused/failed-reload-module"),
5843 args: Vec::new(),
5844 env: Vec::new(),
5845 reserved: false,
5846 reserved_prefixes: Vec::new(),
5847 protocol: ModuleProtocol::Subc,
5848 overlap: Default::default(),
5849 };
5850
5851 let result = handle_reload_spawn_failure(
5852 &spec,
5853 &runtime,
5854 &supervisor.process_liveness,
5855 &snapshot,
5856 &mut child,
5857 "forced reload spawn failure".to_string(),
5858 )
5859 .await;
5860
5861 assert!(matches!(result, Err(SuperviseError::ReloadFailed { .. })));
5862 assert_eq!(lock_snapshot(&snapshot).unwrap().state, ModuleState::Failed);
5863 assert_snapshot_process_facts_cleared(&snapshot);
5864 }
5865
5866 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
5867 async fn dropping_a_module_with_an_active_monitor_clears_current_process_facts() {
5868 let supervisor =
5869 Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default());
5870 let snapshot = stale_process_snapshot(ModuleState::Running, true);
5871 let module = supervisor.supervised_module(
5872 ModuleSpec {
5873 module_id: "drop-clears-facts".to_string(),
5874 program: PathBuf::from("/unused/drop-module"),
5875 args: Vec::new(),
5876 env: Vec::new(),
5877 reserved: false,
5878 reserved_prefixes: Vec::new(),
5879 protocol: ModuleProtocol::Subc,
5880 overlap: Default::default(),
5881 },
5882 supervisor.runtime_config(),
5883 Arc::clone(&snapshot),
5884 None,
5885 );
5886 assert!(!module
5887 .inner
5888 .monitor
5889 .lock()
5890 .unwrap()
5891 .as_ref()
5892 .unwrap()
5893 .is_finished());
5894
5895 drop(module);
5896
5897 assert_eq!(
5898 lock_snapshot(&snapshot).unwrap().state,
5899 ModuleState::Stopped
5900 );
5901 assert_snapshot_process_facts_cleared(&snapshot);
5902 }
5903
5904 #[cfg(unix)]
5905 #[tokio::test]
5906 async fn rescan_preserves_running_protocol_until_respawn() {
5907 let dir = cortexkit_test_support::ScratchDir::new("rescan-protocol");
5908 let initial = ModuleSpec {
5909 module_id: "rescan-protocol".into(),
5910 program: PathBuf::from("/bin/sleep"),
5911 args: vec!["60".into()],
5912 env: ["XDG_DATA_HOME", "XDG_RUNTIME_DIR", "XDG_CONFIG_HOME"]
5913 .into_iter()
5914 .map(|key| (key.into(), dir.path().to_string_lossy().into_owned()))
5915 .collect(),
5916 reserved: false,
5917 reserved_prefixes: vec![],
5918 protocol: ModuleProtocol::None,
5919 overlap: Default::default(),
5920 };
5921 let supervisor =
5922 Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default());
5923 let module = supervisor.spawn(initial.clone()).unwrap();
5924 assert!(module.status().unwrap().live);
5925 let mut next = initial;
5926 next.protocol = ModuleProtocol::Subc;
5927 module
5928 .update_configuration(next.clone(), HealthConfig::default(), None)
5929 .await
5930 .unwrap();
5931 assert!(
5932 module.status().unwrap().live,
5933 "rescan must not require HELLO from the old non-wire process"
5934 );
5935 let runtime = supervisor.runtime_config();
5936 let action = on_child_exit(
5937 &next,
5938 RestartPolicy::default(),
5939 &supervisor.registry,
5940 &module.inner.snapshot,
5941 &runtime.terminal_ring,
5942 &runtime.spawn_events,
5943 &runtime.child_roster,
5944 ExitReport {
5945 kind: ExitKind::Clean,
5946 code: Some(0),
5947 signal: None,
5948 at_ms: unix_ms_now(),
5949 },
5950 )
5951 .await;
5952 assert!(
5953 matches!(action, NextAction::Restart { .. }),
5954 "the old non-wire process's clean exit must restart"
5955 );
5956 module.drain().await.unwrap();
5957 }
5958
5959 #[cfg(unix)]
5960 #[tokio::test]
5961 async fn orphan_identity_matches_path_names_and_shebang_interpreters() {
5962 use std::os::unix::fs::PermissionsExt;
5963 let dir = cortexkit_test_support::ScratchDir::new("spawn-image-identity");
5964 let script = dir.join("module.sh");
5965 std::fs::write(&script, "#!/bin/sh\nwhile :; do sleep 0.1; done\n").unwrap();
5966 std::fs::set_permissions(&script, std::fs::Permissions::from_mode(0o700)).unwrap();
5967 let record_path = dir.join("live-children.json");
5968 let supervisor =
5969 Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default())
5970 .with_live_children_record(&record_path);
5971 for (program, args) in [
5972 (PathBuf::from("sleep"), vec!["60".into()]),
5973 (script, vec![]),
5974 ] {
5975 let spec = ModuleSpec {
5976 module_id: "image-identity".into(),
5977 program: program.clone(),
5978 args,
5979 env: ["XDG_DATA_HOME", "XDG_RUNTIME_DIR", "XDG_CONFIG_HOME"]
5980 .into_iter()
5981 .map(|key| (key.into(), dir.to_string_lossy().into_owned()))
5982 .collect(),
5983 reserved: false,
5984 reserved_prefixes: vec![],
5985 protocol: ModuleProtocol::None,
5986 overlap: Default::default(),
5987 };
5988 let module = supervisor.spawn(spec).unwrap();
5989 #[cfg(target_os = "macos")]
5990 {
5991 let deadline = Instant::now() + Duration::from_secs(5);
5994 while crate::live_children::read_record(&record_path)
5995 .unwrap()
5996 .iter()
5997 .all(|entry| entry.executable.is_none())
5998 {
5999 assert!(Instant::now() < deadline, "module image was not confirmed");
6000 tokio::time::sleep(Duration::from_millis(5)).await;
6001 }
6002 }
6003 let entry = crate::live_children::read_record(&record_path)
6004 .unwrap()
6005 .pop()
6006 .unwrap();
6007 let observed = subc_os::Process::open(entry.pid)
6008 .unwrap()
6009 .unwrap()
6010 .observe()
6011 .unwrap();
6012 let verdict = crate::live_children::identity_verdict(&entry, entry.pid, &observed);
6013 module.drain().await.unwrap();
6014 assert_eq!(
6015 verdict,
6016 crate::live_children::IdentityVerdict::Matches,
6017 "program {program:?}: recorded {entry:?}, observed {observed:?}"
6018 );
6019 }
6020 }
6021
6022 #[cfg(unix)]
6023 fn http_fixture(
6024 dir: &std::path::Path,
6025 url: &str,
6026 threshold: u32,
6027 ) -> crate::daemon_config::ConfiguredModule {
6028 let path = dir.join("subc.jsonc");
6029 std::fs::write(&path, serde_json::json!({"version": 1, "modules": {"http-process": {
6030 "program": "/bin/sleep", "args": ["60"], "protocol": "none",
6031 "env": {"XDG_DATA_HOME":dir,"XDG_RUNTIME_DIR":dir,"XDG_CONFIG_HOME":dir},
6032 "health": {"http":url,"cadence_ms":20,"deadline_ms":200,"failure_threshold":threshold},
6033 "restart": {"backoff_ms":1,"max_backoff_ms":1}, "drain_timeout_ms":10
6034 }}}).to_string()).unwrap();
6035 crate::daemon_config::load(&path)
6036 .unwrap()
6037 .unwrap()
6038 .modules
6039 .pop()
6040 .unwrap()
6041 }
6042
6043 #[cfg(unix)]
6044 async fn wait_http_health(module: &SupervisedModule, status: SupervisorHealthStatus) {
6045 timeout(Duration::from_secs(5), async {
6046 loop {
6047 if module.status().unwrap().health.status == status {
6048 break;
6049 }
6050 sleep(Duration::from_millis(5)).await;
6051 }
6052 })
6053 .await
6054 .unwrap_or_else(|_| {
6055 panic!(
6056 "expected {status:?}, got {:?}",
6057 module.status().unwrap().health
6058 )
6059 });
6060 }
6061
6062 #[cfg(unix)]
6063 #[tokio::test]
6064 async fn http_health_status_flips_ok_failing_ok() {
6065 use tokio::io::{AsyncReadExt, AsyncWriteExt};
6066 let dir = cortexkit_test_support::ScratchDir::new("http-status-flips");
6067 let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
6068 let status = Arc::new(std::sync::atomic::AtomicU16::new(200));
6069 let serving_status = status.clone();
6070 let url = format!("http://{}/healthz", listener.local_addr().unwrap());
6071 let server = tokio::spawn(async move {
6072 loop {
6073 let (mut stream, _) = listener.accept().await.unwrap();
6074 let mut request = [0u8; 2048];
6075 let count = stream.read(&mut request).await.unwrap();
6076 assert!(count > 0, "a probe must send an HTTP request");
6077 let code = serving_status.load(std::sync::atomic::Ordering::SeqCst);
6078 let body = if code == 200 {
6079 "ready"
6080 } else {
6081 "{\"status\":\"unavailable\",\"error\":\"scratch failure\"}"
6082 };
6083 let response = format!("HTTP/1.1 {code} scratch\r\nContent-Length: {}\r\nConnection: close\r\n\r\n{body}", body.len());
6084 let _ = stream.write_all(response.as_bytes()).await;
6085 }
6086 });
6087 let configured = http_fixture(&dir, &url, 1000);
6088 let module =
6089 Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default())
6090 .supervise_configured_with_health(
6091 configured.module_spec(),
6092 true,
6093 configured.health,
6094 configured.drain_timeout_ms,
6095 configured.restart,
6096 )
6097 .unwrap();
6098 wait_http_health(&module, SupervisorHealthStatus::Ok).await;
6099 status.store(503, std::sync::atomic::Ordering::SeqCst);
6100 wait_http_health(&module, SupervisorHealthStatus::Failing).await;
6101 assert!(module
6102 .status()
6103 .unwrap()
6104 .health
6105 .detail
6106 .unwrap()
6107 .contains("scratch failure"));
6108 status.store(200, std::sync::atomic::Ordering::SeqCst);
6109 wait_http_health(&module, SupervisorHealthStatus::Ok).await;
6110 assert_eq!(module.status().unwrap().health.consecutive_failures, 0);
6111 let before = module.status().unwrap();
6112 let (spec, mut health) = module.configuration().unwrap();
6113 health.http = None;
6114 module
6115 .update_configuration(spec.clone(), health.clone(), Some(10))
6116 .await
6117 .unwrap();
6118 wait_http_health(&module, SupervisorHealthStatus::Unknown).await;
6119 health.http = Some(url);
6120 module
6121 .update_configuration(spec, health, Some(10))
6122 .await
6123 .unwrap();
6124 wait_http_health(&module, SupervisorHealthStatus::Ok).await;
6125 assert_eq!(
6126 module.status().unwrap().pid,
6127 before.pid,
6128 "changing a probe must apply live, not restart its process"
6129 );
6130 let (spec, mut health) = module.configuration().unwrap();
6131 health.failure_threshold = 2;
6132 module
6133 .update_configuration(spec, health, Some(10))
6134 .await
6135 .unwrap();
6136 status.store(503, std::sync::atomic::Ordering::SeqCst);
6137 timeout(Duration::from_secs(5), async {
6138 while module.status().unwrap().spawn_generation == before.spawn_generation {
6139 sleep(Duration::from_millis(5)).await;
6140 }
6141 })
6142 .await
6143 .expect("sustained HTTP 503 responses must trigger the health restart policy");
6144 module.drain().await.unwrap();
6145 server.abort();
6146 }
6147
6148 #[cfg(unix)]
6149 #[tokio::test]
6150 async fn http_health_no_listener_restarts_after_consecutive_failures() {
6151 let dir = cortexkit_test_support::ScratchDir::new("http-refused");
6152 let unused = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
6153 let url = format!("http://{}/healthz", unused.local_addr().unwrap());
6154 drop(unused);
6155 let configured = http_fixture(&dir, &url, 2);
6156 let module =
6157 Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default())
6158 .supervise_configured_with_health(
6159 configured.module_spec(),
6160 true,
6161 configured.health,
6162 configured.drain_timeout_ms,
6163 configured.restart,
6164 )
6165 .unwrap();
6166 let before = module.status().unwrap().spawn_generation;
6167 timeout(Duration::from_secs(5), async {
6168 loop {
6169 let status = module.status().unwrap();
6170 if status.spawn_generation > before {
6171 assert!(status.lifetime_restarts > 0);
6172 break;
6173 }
6174 sleep(Duration::from_millis(5)).await;
6175 }
6176 })
6177 .await
6178 .expect("sustained HTTP refusal must trigger the health restart policy");
6179 module.drain().await.unwrap();
6180 }
6181
6182 #[cfg(unix)]
6183 #[tokio::test]
6184 async fn http_health_timeout_honours_deadline() {
6185 let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
6186 let url = format!("http://{}/healthz", listener.local_addr().unwrap());
6187 let server = tokio::spawn(async move {
6188 let _held = listener.accept().await.unwrap();
6189 std::future::pending::<()>().await;
6190 });
6191 let error = timeout(
6192 Duration::from_secs(1),
6193 probe_http_health(&url, Duration::from_millis(10)),
6194 )
6195 .await
6196 .expect("the probe must enforce its own deadline")
6197 .unwrap_err();
6198 server.abort();
6199 assert!(error.to_string().contains("timed out"));
6200 }
6201
6202 #[cfg(unix)]
6203 #[tokio::test]
6204 async fn http_health_localhost_reaches_an_ipv6_only_listener() {
6205 use tokio::io::{AsyncReadExt, AsyncWriteExt};
6206 let listener = tokio::net::TcpListener::bind("[::1]:0").await.unwrap();
6207 let url = format!(
6208 "http://localhost:{}/healthz",
6209 listener.local_addr().unwrap().port()
6210 );
6211 let server = tokio::spawn(async move {
6212 let (mut stream, _) = listener.accept().await.unwrap();
6213 let mut request = [0u8; 2048];
6214 assert!(stream.read(&mut request).await.unwrap() > 0);
6215 stream
6216 .write_all(b"HTTP/1.1 200 OK\r\nContent-Length: 2\r\n\r\nok")
6217 .await
6218 .unwrap();
6219 });
6220 assert_eq!(
6225 probe_http_health(&url, Duration::from_secs(10))
6226 .await
6227 .unwrap()
6228 .status,
6229 HealthStatus::Ok
6230 );
6231 server.await.unwrap();
6232 }
6233
6234 #[cfg(unix)]
6235 #[tokio::test]
6236 async fn http_health_timeout_keeps_partial_status_and_body() {
6237 use tokio::io::{AsyncReadExt, AsyncWriteExt};
6238 let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
6239 let url = format!("http://{}/healthz", listener.local_addr().unwrap());
6240 let server = tokio::spawn(async move {
6241 let (mut stream, _) = listener.accept().await.unwrap();
6242 let mut request = [0u8; 2048];
6243 assert!(stream.read(&mut request).await.unwrap() > 0);
6244 stream
6245 .write_all(
6246 b"HTTP/1.1 503 Unavailable\r\nContent-Length: 100\r\n\r\npartial diagnostic",
6247 )
6248 .await
6249 .unwrap();
6250 std::future::pending::<()>().await;
6251 });
6252 let error = probe_http_health(&url, Duration::from_secs(1))
6253 .await
6254 .unwrap_err()
6255 .to_string();
6256 server.abort();
6257 assert!(
6258 error.contains("timed out")
6259 && error.contains("503 Unavailable")
6260 && error.contains("partial diagnostic"),
6261 "{error}"
6262 );
6263 }
6264
6265 #[cfg(unix)]
6266 #[tokio::test]
6267 async fn http_health_diagnostic_body_is_bounded_to_200_bytes() {
6268 use tokio::io::{AsyncReadExt, AsyncWriteExt};
6269 let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
6270 let url = format!("http://{}/healthz", listener.local_addr().unwrap());
6271 let server = tokio::spawn(async move {
6272 let (mut stream, _) = listener.accept().await.unwrap();
6273 let mut request = [0u8; 2048];
6274 assert!(stream.read(&mut request).await.unwrap() > 0);
6275 let body = format!("{}not-in-diagnostic", "x".repeat(400));
6276 let response = format!(
6277 "HTTP/1.1 503 Unavailable\r\nContent-Length: {}\r\n\r\n{body}",
6278 body.len()
6279 );
6280 stream.write_all(response.as_bytes()).await.unwrap();
6281 });
6282 let error = probe_http_health(&url, Duration::from_secs(1))
6283 .await
6284 .unwrap_err()
6285 .to_string();
6286 server.await.unwrap();
6287 assert!(error.ends_with(&"x".repeat(200)), "{error}");
6288 assert_eq!(error.split(": ").last().unwrap().len(), 200);
6289 assert!(!error.contains("not-in-diagnostic"));
6290 }
6291
6292 #[cfg(unix)]
6293 #[tokio::test]
6294 async fn http_health_real_nats_server_monitoring() {
6295 if std::process::Command::new("nats-server")
6296 .arg("--version")
6297 .env("XDG_DATA_HOME", std::env::temp_dir())
6298 .env("XDG_RUNTIME_DIR", std::env::temp_dir())
6299 .env("XDG_CONFIG_HOME", std::env::temp_dir())
6300 .output()
6301 .is_err()
6302 {
6303 eprintln!(
6304 "skipped: http_health_real_nats_server_monitoring (nats-server is not on PATH)"
6305 );
6306 return;
6307 }
6308 let dir = cortexkit_test_support::ScratchDir::new("nats-http-monitoring");
6309 let monitor = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
6310 let port = monitor.local_addr().unwrap().port();
6311 let client = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
6312 let client_port = client.local_addr().unwrap().port();
6313 let config = dir.join("server.conf");
6314 std::fs::write(&config, format!("listen: 127.0.0.1:{client_port}\nhttp: 127.0.0.1:{port}\njetstream {{ store_dir: \"{}\" }}\n", dir.join("jetstream").display())).unwrap();
6315 let mut configured = http_fixture(&dir, &format!("http://127.0.0.1:{port}/healthz"), 1000);
6316 configured.program = PathBuf::from("nats-server");
6317 configured.args = vec!["-c".into(), config.to_string_lossy().into_owned()];
6318 drop(monitor);
6319 drop(client);
6320 let module =
6321 Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default())
6322 .supervise_configured_with_health(
6323 configured.module_spec(),
6324 true,
6325 configured.health,
6326 configured.drain_timeout_ms,
6327 configured.restart,
6328 )
6329 .unwrap();
6330 wait_http_health(&module, SupervisorHealthStatus::Ok).await;
6331 module.drain().await.unwrap();
6332 }
6333
6334 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
6335 async fn configuration_update_does_not_replace_captured_running_process_facts() {
6336 let supervisor =
6337 Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default());
6338 let snapshot = stale_process_snapshot(ModuleState::Running, true);
6339 let initial = ModuleSpec {
6340 module_id: "rescan-preserves-spawn-facts".to_string(),
6341 program: PathBuf::from("/spawned/module"),
6342 args: Vec::new(),
6343 env: Vec::new(),
6344 reserved: false,
6345 reserved_prefixes: Vec::new(),
6346 protocol: ModuleProtocol::Subc,
6347 overlap: Default::default(),
6348 };
6349 let module = supervisor.supervised_module(
6350 initial.clone(),
6351 supervisor.runtime_config(),
6352 snapshot,
6353 None,
6354 );
6355 let before = module.status().unwrap();
6356 let mut replacement = initial;
6357 replacement.program = PathBuf::from("/rescanned/replacement-module");
6358
6359 module
6360 .update_configuration(replacement, HealthConfig::default(), None)
6361 .await
6362 .unwrap();
6363
6364 let after = module.status().unwrap();
6365 assert_eq!(after.pid, before.pid);
6366 assert_eq!(after.spawned_at_ms, before.spawned_at_ms);
6367 assert_eq!(after.spawned_from, before.spawned_from);
6368 drop(module);
6369 }
6370}
6371
6372fn unix_ms_now() -> u64 {
6373 SystemTime::now()
6374 .duration_since(UNIX_EPOCH)
6375 .map(|duration| duration.as_millis().min(u128::from(u64::MAX)) as u64)
6376 .unwrap_or(0)
6377}
6378
6379async fn supervise_loop(
6380 mut spec: ModuleSpec,
6381 mut runtime: SupervisorRuntimeConfig,
6382 registry: Arc<Registry>,
6383 process_liveness: Arc<SupervisorProcessLiveness>,
6384 snapshot: SharedSnapshot,
6385 mut child: Option<SupervisedChild>,
6386 mut commands: mpsc::Receiver<SupervisorCommand>,
6387) {
6388 let mut health_probe = HealthProbeRuntime::default();
6389 let mut registration_changes = match registry.subscribe_module_changes(&spec.module_id) {
6393 Ok(changes) => Some(changes),
6394 Err(err) => {
6395 warn!(module_id = %spec.module_id, error = %err, "health prober could not subscribe to registry");
6398 None
6399 }
6400 };
6401 let mut pending_respawn: Option<PendingRespawn> = None;
6405 let mut requeued: VecDeque<SupervisorCommand> = VecDeque::new();
6408 let mut previous_instance_recheck = tokio::time::interval_at(
6409 Instant::now() + Duration::from_secs(5),
6410 Duration::from_secs(5),
6411 );
6412 previous_instance_recheck.set_missed_tick_behavior(tokio::time::MissedTickBehavior::Skip);
6413 loop {
6414 #[cfg(test)]
6415 {
6416 lock_snapshot(&snapshot).unwrap().actor_select = None;
6417 }
6418 #[cfg(target_os = "macos")]
6419 if let Some(active) = child.as_mut() {
6420 active.confirm_privacy_exec().await;
6421 }
6422 if let Some(scheduled) = runtime
6423 .scheduled_respawn
6424 .lock()
6425 .unwrap_or_else(|p| p.into_inner())
6426 .take()
6427 {
6428 pending_respawn = Some(scheduled);
6429 }
6430 if pending_respawn.is_some() && (child.is_some() || !respawn_still_pending(&snapshot)) {
6431 pending_respawn = None;
6432 cancel_deferred_reload(
6433 &runtime,
6434 &spec.module_id,
6435 "respawn cancelled by a supervisor command",
6436 );
6437 }
6438 if child.is_none()
6439 && pending_respawn.is_none()
6440 && lock_snapshot(&snapshot).is_ok_and(|state| state.waiting_for_previous_pid.is_none())
6441 {
6442 cancel_deferred_reload(
6443 &runtime,
6444 &spec.module_id,
6445 "respawn cancelled before a replacement was spawned",
6446 );
6447 let _ = update_snapshot(&snapshot, Some(&spec.module_id), |state| {
6448 state.respawn_pending = false;
6449 state.coalesced_restart_pending = false;
6450 if matches!(
6451 state.state,
6452 ModuleState::Restarting
6453 | ModuleState::Starting
6454 | ModuleState::Draining
6455 | ModuleState::Unresponsive
6456 ) {
6457 error!(module_id = %spec.module_id, state = ?state.state,
6458 "supervision operation ended without a child or pending respawn; marking failed so start can retry");
6459 state.state = ModuleState::Failed;
6460 clear_current_process_facts(state);
6461 }
6462 });
6463 }
6464 if let Some(command) = requeued.pop_front() {
6465 if !handle_supervisor_command(
6466 command,
6467 &mut spec,
6468 &mut runtime,
6469 ®istry,
6470 &process_liveness,
6471 &snapshot,
6472 &mut child,
6473 &mut commands,
6474 &mut requeued,
6475 )
6476 .await
6477 {
6478 return;
6479 }
6480 if child.is_some() || !respawn_still_pending(&snapshot) {
6481 pending_respawn = None;
6482 let _ = update_snapshot(&snapshot, Some(&spec.module_id), |state| {
6483 state.respawn_pending = false
6484 });
6485 }
6486 continue;
6487 }
6488 if child.is_some() {
6489 #[cfg(test)]
6490 {
6491 lock_snapshot(&snapshot).unwrap().actor_turns += 1;
6492 }
6493 health_probe.refresh_registration(&spec, &runtime, ®istry, &snapshot);
6494 let wake_after = health_probe.wake_after();
6495 let probe_sleep = async move {
6496 match wake_after {
6497 Some(delay) => sleep(delay).await,
6498 None => std::future::pending().await,
6499 }
6500 };
6501 tokio::pin!(probe_sleep);
6502 let wire_child = running_protocol(&spec, &snapshot) == ModuleProtocol::Subc;
6507 #[cfg(test)]
6508 {
6509 let registration_pending = wire_child
6510 && registration_changes
6511 .as_ref()
6512 .is_some_and(|changes| changes.has_changed().unwrap());
6513 let mut state = lock_snapshot(&snapshot).unwrap();
6514 state.actor_select = (!registration_pending).then_some(ActorSelectCheckpoint {
6515 generation: state.spawn_generation,
6516 turn: state.actor_turns,
6517 registered_connection: health_probe.registered_connection,
6518 next_probe_at: health_probe.next_probe_at,
6519 wake_after,
6520 });
6521 }
6522 let active_child = child.as_mut().expect("child checked above");
6523 tokio::select! {
6524 wait_result = active_child.wait() => {
6525 #[cfg(test)]
6526 { lock_snapshot(&snapshot).unwrap().actor_select = None; }
6527 let exit_report = match wait_result {
6536 Ok(status) => classify_reaped_child_exit(&snapshot, active_child, &status),
6537 Err(err) => {
6538 active_child.drain_stderr(&spec.module_id).await;
6539 fail_after_wait_error(
6540 &spec.module_id,
6541 &snapshot,
6542 &runtime.terminal_ring,
6543 &runtime.spawn_events,
6544 format!("failed to wait for supervised module: {err}"),
6545 );
6546 untrack_if_registration_released(
6547 &process_liveness,
6548 ®istry,
6549 &spec.module_id,
6550 &snapshot,
6551 );
6552 error!(module_id = %spec.module_id, error = %err, "failed to wait for supervised module");
6553 child = None;
6554 continue;
6555 }
6556 };
6557 active_child.drain_stderr(&spec.module_id).await;
6558
6559 let next = on_child_exit(
6560 &spec,
6561 runtime.restart_policy,
6562 ®istry,
6563 &snapshot,
6564 &runtime.terminal_ring,
6565 &runtime.spawn_events,
6566 &runtime.child_roster,
6567 exit_report,
6568 ).await;
6569 runtime.start_requester = None;
6570 runtime.pending_activity_restart = false;
6571 active_child.release_roster();
6574 match next {
6575 NextAction::Stop { registration_released } => {
6576 if registration_released {
6577 process_liveness.untrack_if_current(&spec.module_id, &snapshot);
6578 }
6579 child = None;
6580 }
6581 NextAction::Restart { schedule } => {
6582 let delay = schedule.map_or(
6583 runtime.restart_policy.delay_for_restart(0),
6584 |schedule| schedule.delay,
6585 );
6586 if let Some(schedule) = schedule {
6587 log_crash_respawn(&spec.module_id, schedule);
6588 }
6589 child = None;
6597 pending_respawn = Some(PendingRespawn { deadline: Instant::now() + delay, kind: RespawnKind::Spawn });
6598 let _ = update_snapshot(&snapshot, Some(&spec.module_id), |state| state.respawn_pending = true);
6599 }
6600 }
6601 }
6602 command = commands.recv() => {
6603 #[cfg(test)]
6604 { lock_snapshot(&snapshot).unwrap().actor_select = None; }
6605 let Some(command) = command else {
6606 return;
6607 };
6608 if !handle_supervisor_command(
6609 command,
6610 &mut spec,
6611 &mut runtime,
6612 ®istry,
6613 &process_liveness,
6614 &snapshot,
6615 &mut child,
6616 &mut commands,
6617 &mut requeued,
6618 ).await {
6619 return;
6620 }
6621 }
6622 _ = async {
6623 match registration_changes.as_mut() {
6624 Some(changes) => { let _ = changes.changed().await; }
6625 None => std::future::pending().await,
6626 }
6627 }, if wire_child => {
6628 #[cfg(test)]
6629 { lock_snapshot(&snapshot).unwrap().actor_select = None; }
6630 }
6631 _ = &mut probe_sleep => {
6632 #[cfg(test)]
6633 { lock_snapshot(&snapshot).unwrap().actor_select = None; }
6634 if health_probe.due() {
6635 run_health_probe_cycle(
6636 &spec,
6637 &runtime,
6638 ®istry,
6639 &process_liveness,
6640 &snapshot,
6641 &mut child,
6642 ).await;
6643 if child.is_none() {
6644 runtime.start_requester = None;
6645 runtime.pending_activity_restart = false;
6646 }
6647 if child.is_some() {
6648 health_probe.schedule_next(&spec, runtime.health.cadence);
6649 }
6650 }
6651 }
6652 }
6653 } else if let Some(pending) = pending_respawn {
6654 tokio::select! {
6655 _ = sleep_until(pending.deadline) => {
6656 pending_respawn = None;
6657 let _ = update_snapshot(&snapshot, Some(&spec.module_id), |state| state.respawn_pending = false);
6658 if !respawn_still_pending(&snapshot) {
6662 continue;
6663 }
6664 if runtime.child_roster.is_closed() {
6669 let _ = update_snapshot(&snapshot, Some(&spec.module_id), |state| {
6670 state.state = ModuleState::Stopped;
6671 });
6672 debug!(module_id = %spec.module_id, "crash respawn cancelled by daemon shutdown");
6673 continue;
6674 }
6675 if let Err(err) = release_dead_registration(
6676 ®istry,
6677 runtime.forwarding.as_deref(),
6678 &snapshot,
6679 &spec.module_id,
6680 ).await {
6681 fail_snapshot(&snapshot, Some(&spec.module_id), None);
6682 error!(module_id = %spec.module_id, error = %err, "registration did not release before restart");
6683 continue;
6684 }
6685
6686 if matches!(pending.kind, RespawnKind::Reload) {
6687 let reply = runtime.deferred_reload_reply.lock().unwrap_or_else(|p| p.into_inner()).take();
6688 let result = finish_reload_child(&spec, &runtime, ®istry, &process_liveness, &snapshot, &mut child).await;
6689 if result.is_ok() && runtime.pending_activity_restart {
6690 runtime.record_activity_transition("subc.module_restarted", &spec.module_id, runtime.start_requester.clone());
6691 }
6692 runtime.pending_activity_restart = false;
6693 if let Some(reply) = reply { let _ = reply.send(result); }
6694 continue;
6695 }
6696 process_liveness.track(spec.module_id.clone(), Arc::clone(&snapshot));
6697 match spawn_and_mark_running(&spec, &runtime, &snapshot) {
6698 Ok(next_child) => {
6699 child = Some(next_child);
6700 if runtime.pending_activity_restart {
6701 runtime.record_activity_transition("subc.module_restarted", &spec.module_id, runtime.start_requester.clone());
6702 runtime.pending_activity_restart = false;
6703 }
6704 debug!(module_id = %spec.module_id, "supervised module restarted after crash");
6705 }
6706 Err(err) => {
6707 fail_snapshot(&snapshot, Some(&spec.module_id), None);
6708 process_liveness.untrack_if_current(&spec.module_id, &snapshot);
6709 error!(module_id = %spec.module_id, error = %err, "failed to restart supervised module");
6710 }
6711 }
6712 }
6713 command = commands.recv() => {
6714 let Some(command) = command else {
6715 return;
6716 };
6717 if !handle_supervisor_command(
6718 command,
6719 &mut spec,
6720 &mut runtime,
6721 ®istry,
6722 &process_liveness,
6723 &snapshot,
6724 &mut child,
6725 &mut commands,
6726 &mut requeued,
6727 ).await {
6728 return;
6729 }
6730 if child.is_some() || !respawn_still_pending(&snapshot) {
6735 pending_respawn = None;
6736 let _ = update_snapshot(&snapshot, Some(&spec.module_id), |state| state.respawn_pending = false);
6737 }
6738 }
6739 }
6740 } else {
6741 let command = if lock_snapshot(&snapshot)
6742 .is_ok_and(|state| state.waiting_for_previous_pid.is_some())
6743 {
6744 tokio::select! {
6745 command = commands.recv() => command,
6746 _ = previous_instance_recheck.tick() => {
6747 if runtime.child_roster.previous_instance_pid(&spec.module_id).is_none() {
6748 let enabled = lock_snapshot(&snapshot).is_ok_and(|state| state.enabled);
6749 if enabled && !runtime.child_roster.is_closed() {
6750 process_liveness.track(spec.module_id.clone(), Arc::clone(&snapshot));
6751 match spawn_and_mark_running(&spec, &runtime, &snapshot) {
6752 Ok(next_child) => child = Some(next_child),
6753 Err(error) => {
6754 fail_snapshot(&snapshot, Some(&spec.module_id), None);
6755 error!(module_id = %spec.module_id, %error, "failed to start module after its previous instance exited");
6756 }
6757 }
6758 } else {
6759 let _ = update_snapshot(&snapshot, Some(&spec.module_id), |state| {
6760 state.waiting_for_previous_pid = None;
6761 state.waiting_for_previous_detail = None;
6762 });
6763 }
6764 }
6765 continue;
6766 }
6767 }
6768 } else {
6769 commands.recv().await
6770 };
6771 let Some(command) = command else {
6772 return;
6773 };
6774 if !handle_supervisor_command(
6775 command,
6776 &mut spec,
6777 &mut runtime,
6778 ®istry,
6779 &process_liveness,
6780 &snapshot,
6781 &mut child,
6782 &mut commands,
6783 &mut requeued,
6784 )
6785 .await
6786 {
6787 return;
6788 }
6789 }
6790 }
6791}
6792
6793fn log_crash_respawn(module_id: &str, schedule: CrashRestartSchedule) {
6794 info!(
6795 module_id,
6796 restart_in_window = schedule.restart_in_window,
6797 delay_ms = schedule.delay.as_millis() as u64,
6798 "respawning after crash"
6799 );
6800}
6801
6802fn respawn_still_pending(snapshot: &SharedSnapshot) -> bool {
6808 matches!(
6809 lock_snapshot(snapshot),
6810 Ok(state) if state.enabled && state.state == ModuleState::Restarting
6811 )
6812}
6813
6814enum NextAction {
6815 Stop {
6816 registration_released: bool,
6817 },
6818 Restart {
6819 schedule: Option<CrashRestartSchedule>,
6820 },
6821}
6822
6823#[allow(clippy::too_many_arguments)]
6824async fn handle_supervisor_command(
6825 command: SupervisorCommand,
6826 spec: &mut ModuleSpec,
6827 runtime: &mut SupervisorRuntimeConfig,
6828 registry: &Arc<Registry>,
6829 process_liveness: &SupervisorProcessLiveness,
6830 snapshot: &SharedSnapshot,
6831 child: &mut Option<SupervisedChild>,
6832 commands: &mut mpsc::Receiver<SupervisorCommand>,
6833 requeued: &mut VecDeque<SupervisorCommand>,
6834) -> bool {
6835 let command =
6836 if let Some((pid, reason)) = runtime.child_roster.previous_instance(&spec.module_id) {
6837 match command {
6838 SupervisorCommand::Restart { reply, .. }
6839 | SupervisorCommand::Reload { reply, .. }
6840 | SupervisorCommand::Swap { reply, .. } => {
6841 let _ = reply.send(Err(previous_instance_error(spec, pid, reason)));
6842 return true;
6843 }
6844 SupervisorCommand::SetEnabled {
6845 enabled: true,
6846 reply,
6847 ..
6848 } => {
6849 let _ = reply.send(Err(previous_instance_error(spec, pid, reason)));
6850 return true;
6851 }
6852 other => other,
6853 }
6854 } else {
6855 command
6856 };
6857 match command {
6858 SupervisorCommand::Drain { requester, reply } => {
6859 let result = drain_optional_child(
6862 &spec.module_id,
6863 spec.protocol,
6864 StopNotice::NotSent,
6865 registry,
6866 runtime.forwarding.as_deref(),
6867 snapshot,
6868 &runtime.terminal_ring,
6869 &runtime.spawn_events,
6870 child,
6871 runtime.drain_timeout,
6872 ModuleState::Stopped,
6873 None,
6874 )
6875 .await;
6876 let registration_released = result.is_ok();
6877 if registration_released {
6878 runtime.record_activity_transition(
6879 "subc.module_stopped",
6880 &spec.module_id,
6881 requester,
6882 );
6883 }
6884 let _ = reply.send(result);
6885 if registration_released {
6886 process_liveness.untrack_if_current(&spec.module_id, snapshot);
6887 }
6888 false
6889 }
6890 SupervisorCommand::Retire { requester, reply } => {
6891 let result = async {
6892 let stop_notice = begin_forwarding_drain_if_configured(
6893 spec,
6894 runtime,
6895 registry,
6896 snapshot,
6897 None,
6898 RouteCloseReason::Disable,
6899 )
6900 .await?;
6901 drain_optional_child(
6902 &spec.module_id,
6903 spec.protocol,
6904 stop_notice,
6905 registry,
6906 runtime.forwarding.as_deref(),
6907 snapshot,
6908 &runtime.terminal_ring,
6909 &runtime.spawn_events,
6910 child,
6911 runtime.drain_timeout,
6912 ModuleState::Stopped,
6913 None,
6914 )
6915 .await
6916 }
6917 .await;
6918 let registration_released = result.is_ok();
6919 if registration_released {
6920 runtime.record_activity_transition(
6921 "subc.module_stopped",
6922 &spec.module_id,
6923 requester,
6924 );
6925 }
6926 let _ = reply.send(result);
6927 if registration_released {
6928 process_liveness.untrack_if_current(&spec.module_id, snapshot);
6929 }
6930 false
6931 }
6932 SupervisorCommand::Restart {
6933 requester,
6934 drain_timeout_ms,
6935 received_at_generation,
6936 queued_at,
6937 reply,
6938 } => {
6939 info!(
6943 module_id = %spec.module_id,
6944 queued_ms = u64::try_from(queued_at.elapsed().as_millis()).unwrap_or(u64::MAX),
6945 "restart command dequeued"
6946 );
6947 let validation = match lock_snapshot(snapshot) {
6959 Ok(state) if !state.enabled => Err(SuperviseError::Disabled {
6960 module_id: spec.module_id.clone(),
6961 }),
6962 Ok(_) => Ok(()),
6963 Err(err) => Err(err),
6964 };
6965 let initiated = validation.is_ok();
6966 let _ = reply.send(validation);
6967 let satisfied_by_generation = if initiated
6981 && child
6982 .as_ref()
6983 .is_some_and(|child| child.spawned_at > queued_at)
6984 {
6985 lock_snapshot(snapshot).ok().and_then(|state| {
6986 (!state.configuration_updated_since_spawn).then_some(state.spawn_generation)
6987 })
6988 } else {
6989 None
6990 };
6991 let satisfied_by_pending = initiated
6992 && child.is_none()
6993 && lock_snapshot(snapshot).ok().is_some_and(|mut state| {
6994 let pending = state.respawn_pending && !state.configuration_updated_since_spawn;
6998 if pending {
6999 state.coalesced_restart_pending = true;
7000 }
7001 pending
7002 });
7003 if satisfied_by_pending {
7004 debug!(module_id = %spec.module_id, "restart coalesced into the pending replacement");
7005 } else if let Some(generation) = satisfied_by_generation {
7006 info!(
7007 module_id = %spec.module_id,
7008 received_at_generation,
7009 "restart already satisfied by generation {generation}; not restarting again"
7010 );
7011 } else if initiated {
7012 let drain_timeout = drain_timeout_ms
7015 .map(Duration::from_millis)
7016 .unwrap_or(runtime.drain_timeout);
7017 if let Err(err) = restart_child(
7018 spec,
7019 runtime,
7020 registry,
7021 process_liveness,
7022 snapshot,
7023 child,
7024 drain_timeout,
7025 )
7026 .await
7027 {
7028 warn!(
7029 module_id = %spec.module_id,
7030 error = %err,
7031 "operator restart failed after initiation ack; module state carries the outcome"
7032 );
7033 let _ = update_snapshot(snapshot, Some(&spec.module_id), |state| {
7034 state.state = ModuleState::Failed;
7035 clear_current_process_facts(state);
7036 });
7037 } else {
7038 runtime.start_requester = requester;
7039 runtime.pending_activity_restart = true;
7040 }
7041 }
7042 true
7043 }
7044 SupervisorCommand::Reload { requester, reply } => {
7045 let result =
7046 reload_child(spec, runtime, registry, process_liveness, snapshot, child).await;
7047 if result.is_ok() {
7048 runtime.start_requester = requester;
7049 runtime.pending_activity_restart = true;
7050 }
7051 if result.is_ok()
7052 && runtime
7053 .scheduled_respawn
7054 .lock()
7055 .unwrap_or_else(|p| p.into_inner())
7056 .as_ref()
7057 .is_some_and(|pending| matches!(pending.kind, RespawnKind::Reload))
7058 {
7059 *runtime
7060 .deferred_reload_reply
7061 .lock()
7062 .unwrap_or_else(|p| p.into_inner()) = Some(reply);
7063 } else {
7064 let _ = reply.send(result);
7065 }
7066 true
7067 }
7068 SupervisorCommand::SetEnabled {
7069 enabled,
7070 requester,
7071 reply,
7072 } => {
7073 let previous_requester = runtime.start_requester.clone();
7074 if enabled {
7075 runtime.start_requester = requester.clone();
7076 }
7077 let result = set_child_enabled(
7078 spec,
7079 runtime,
7080 registry,
7081 process_liveness,
7082 snapshot,
7083 child,
7084 enabled,
7085 )
7086 .await;
7087 if matches!(result, Ok(true)) {
7088 runtime.pending_activity_restart = false;
7089 if enabled {
7090 runtime.start_requester = requester;
7091 } else {
7092 runtime.start_requester = None;
7093 runtime.record_activity_transition(
7094 "subc.module_disabled",
7095 &spec.module_id,
7096 requester,
7097 );
7098 }
7099 } else {
7100 runtime.start_requester = previous_requester;
7101 }
7102 let _ = reply.send(result);
7103 true
7104 }
7105 SupervisorCommand::UpdateConfiguration {
7106 spec: next_spec,
7107 health,
7108 drain_timeout_ms,
7109 reply,
7110 } => {
7111 if let Some(handle) = &runtime.supervisor_handle {
7112 handle.apply_identity_configuration(&next_spec);
7113 }
7114 *spec = next_spec;
7115 let _ = update_snapshot(snapshot, Some(&spec.module_id), |state| {
7116 state.configuration_updated_since_spawn = true;
7117 });
7118 let health_changed = runtime.health != health;
7119 runtime.health = health;
7120 if health_changed {
7123 let _ = update_snapshot(snapshot, Some(&spec.module_id), |state| {
7124 state.health = ModuleHealthStatus::default();
7125 });
7126 }
7127 runtime.drain_timeout = drain_timeout_ms
7128 .map(Duration::from_millis)
7129 .unwrap_or(runtime.default_drain_timeout);
7130 *runtime
7131 .effective_drain_timeout
7132 .lock()
7133 .unwrap_or_else(|poisoned| poisoned.into_inner()) = runtime.drain_timeout;
7134 let _ = reply.send(());
7135 true
7136 }
7137 SupervisorCommand::Swap {
7138 requester,
7139 ready_timeout,
7140 reply,
7141 } => {
7142 let mut swap_runtime = runtime.clone();
7143 swap_runtime.start_requester = requester;
7144 let before = lock_snapshot(snapshot)
7145 .ok()
7146 .map(|state| state.spawn_generation);
7147 let end = swap::run_swap(
7148 spec,
7149 &swap_runtime,
7150 registry,
7151 process_liveness,
7152 snapshot,
7153 child,
7154 commands,
7155 ready_timeout.unwrap_or(DEFAULT_SWAP_READY_TIMEOUT),
7156 reply,
7157 )
7158 .await;
7159 if before
7160 != lock_snapshot(snapshot)
7161 .ok()
7162 .map(|state| state.spawn_generation)
7163 {
7164 runtime.start_requester = swap_runtime.start_requester;
7165 }
7166 requeued.extend(end.requeue);
7167 true
7168 }
7169 }
7170}
7171
7172async fn restart_child(
7173 spec: &ModuleSpec,
7174 runtime: &SupervisorRuntimeConfig,
7175 registry: &Registry,
7176 process_liveness: &SupervisorProcessLiveness,
7177 snapshot: &SharedSnapshot,
7178 child: &mut Option<SupervisedChild>,
7179 drain_timeout: Duration,
7180) -> Result<(), SuperviseError> {
7181 if !lock_snapshot(snapshot)?.enabled {
7183 return Err(SuperviseError::Disabled {
7184 module_id: spec.module_id.clone(),
7185 });
7186 }
7187 let stop_notice = begin_forwarding_drain_with_timeout(
7188 spec,
7189 runtime,
7190 registry,
7191 snapshot,
7192 None,
7193 RouteCloseReason::Restart,
7194 drain_timeout,
7195 )
7196 .await?;
7197
7198 if child.is_some() {
7199 drain_optional_child(
7200 &spec.module_id,
7201 spec.protocol,
7202 stop_notice,
7203 registry,
7204 runtime.forwarding.as_deref(),
7205 snapshot,
7206 &runtime.terminal_ring,
7207 &runtime.spawn_events,
7208 child,
7209 drain_timeout,
7210 ModuleState::Restarting,
7211 Some(true),
7212 )
7213 .await?;
7214 } else {
7215 update_snapshot(snapshot, Some(&spec.module_id), |state| {
7216 state.enabled = true;
7217 state.state = ModuleState::Restarting;
7218 clear_current_process_facts(state);
7219 })?;
7220 release_dead_registration(
7221 registry,
7222 runtime.forwarding.as_deref(),
7223 snapshot,
7224 &spec.module_id,
7225 )
7226 .await?;
7227 }
7228
7229 reset_restart_count(snapshot, &spec.module_id)?;
7230 process_liveness.track(spec.module_id.clone(), Arc::clone(snapshot));
7231 schedule_respawn(
7232 runtime,
7233 snapshot,
7234 &spec.module_id,
7235 runtime.restart_policy.backoff,
7236 RespawnKind::Spawn,
7237 )
7238}
7239
7240async fn reload_child(
7241 spec: &ModuleSpec,
7242 runtime: &SupervisorRuntimeConfig,
7243 registry: &Registry,
7244 process_liveness: &SupervisorProcessLiveness,
7245 snapshot: &SharedSnapshot,
7246 child: &mut Option<SupervisedChild>,
7247) -> Result<(), SuperviseError> {
7248 if !lock_snapshot(snapshot)?.enabled {
7250 return Err(SuperviseError::Disabled {
7251 module_id: spec.module_id.clone(),
7252 });
7253 }
7254 let stop_notice = begin_forwarding_drain(
7255 spec,
7256 runtime,
7257 registry,
7258 snapshot,
7259 Some(true),
7260 RouteCloseReason::Reload,
7261 )
7262 .await?;
7263
7264 if child.is_some() {
7265 drain_optional_child(
7266 &spec.module_id,
7267 spec.protocol,
7268 stop_notice,
7269 registry,
7270 runtime.forwarding.as_deref(),
7271 snapshot,
7272 &runtime.terminal_ring,
7273 &runtime.spawn_events,
7274 child,
7275 runtime.drain_timeout,
7276 ModuleState::Restarting,
7277 Some(true),
7278 )
7279 .await?;
7280 } else {
7281 update_snapshot(snapshot, Some(&spec.module_id), |state| {
7282 state.enabled = true;
7283 state.state = ModuleState::Restarting;
7284 clear_current_process_facts(state);
7285 })?;
7286 release_dead_registration(
7287 registry,
7288 runtime.forwarding.as_deref(),
7289 snapshot,
7290 &spec.module_id,
7291 )
7292 .await?;
7293 }
7294
7295 reset_restart_count(snapshot, &spec.module_id)?;
7296 process_liveness.track(spec.module_id.clone(), Arc::clone(snapshot));
7297 schedule_respawn(
7298 runtime,
7299 snapshot,
7300 &spec.module_id,
7301 runtime.restart_policy.backoff,
7302 RespawnKind::Reload,
7303 )
7304}
7305
7306async fn finish_reload_child(
7307 spec: &ModuleSpec,
7308 runtime: &SupervisorRuntimeConfig,
7309 registry: &Registry,
7310 process_liveness: &SupervisorProcessLiveness,
7311 snapshot: &SharedSnapshot,
7312 child: &mut Option<SupervisedChild>,
7313) -> Result<(), SuperviseError> {
7314 process_liveness.track(spec.module_id.clone(), Arc::clone(snapshot));
7315 let next_child = match spawn_and_mark_running(spec, runtime, snapshot) {
7316 Ok(next_child) => next_child,
7317 Err(err) => {
7318 return handle_reload_spawn_failure(
7319 spec,
7320 runtime,
7321 process_liveness,
7322 snapshot,
7323 child,
7324 format!("new child failed to spawn: {err}"),
7325 )
7326 .await;
7327 }
7328 };
7329 *child = Some(next_child);
7330
7331 let wait_outcome = {
7332 let active_child = child.as_mut().expect("new reload child was just stored");
7333 wait_for_registration_after_reload(
7334 registry,
7335 &spec.module_id,
7336 snapshot,
7337 active_child,
7338 REGISTRY_RELEASE_TIMEOUT,
7339 )
7340 .await?
7341 };
7342
7343 match wait_outcome {
7344 RegistrationWaitOutcome::Registered => {
7345 debug!(module_id = %spec.module_id, "supervised module reloaded and registered");
7346 Ok(())
7347 }
7348 RegistrationWaitOutcome::Exited(exit_report) => {
7349 if let Some(active_child) = child.as_mut() {
7350 active_child.drain_stderr(&spec.module_id).await;
7351 }
7352 let mut exited_child = child.take().expect("exited reload child is still stored");
7355 #[cfg(test)]
7356 if let Some(gate) = &runtime.test_reload_exit_record_gate {
7357 gate.reached.notify_one();
7358 gate.resume.notified().await;
7359 }
7360 let result = handle_reload_child_registration_failure(
7361 spec,
7362 runtime,
7363 registry,
7364 process_liveness,
7365 snapshot,
7366 child,
7367 ReloadRegistrationFailure {
7368 exit_report: registration_failure_exit_report(exit_report),
7369 reason: exited_child
7370 .spawn_failure
7371 .clone()
7372 .unwrap_or_else(|| "new child exited before registering".to_string()),
7373 },
7374 )
7375 .await;
7376 exited_child.release_roster();
7377 result
7378 }
7379 RegistrationWaitOutcome::TimedOut => {
7380 let mut timed_out_child = child
7381 .take()
7382 .expect("timed-out reload child is still running");
7383 timed_out_child
7384 .start_kill()
7385 .map_err(|source| SuperviseError::Kill {
7386 module_id: spec.module_id.clone(),
7387 source,
7388 })?;
7389 let status = timed_out_child
7390 .wait()
7391 .await
7392 .map_err(|source| SuperviseError::Wait {
7393 module_id: spec.module_id.clone(),
7394 source,
7395 })?;
7396 timed_out_child.drain_stderr(&spec.module_id).await;
7397 handle_reload_child_registration_failure(
7398 spec,
7399 runtime,
7400 registry,
7401 process_liveness,
7402 snapshot,
7403 child,
7404 ReloadRegistrationFailure {
7405 exit_report: registration_failure_exit_report(classify_reaped_child_exit(
7406 snapshot,
7407 &timed_out_child,
7408 &status,
7409 )),
7410 reason: format!(
7411 "new child did not register within {:?}",
7412 REGISTRY_RELEASE_TIMEOUT
7413 ),
7414 },
7415 )
7416 .await
7417 }
7418 }
7419}
7420
7421async fn set_child_enabled(
7422 spec: &ModuleSpec,
7423 runtime: &SupervisorRuntimeConfig,
7424 registry: &Registry,
7425 process_liveness: &SupervisorProcessLiveness,
7426 snapshot: &SharedSnapshot,
7427 child: &mut Option<SupervisedChild>,
7428 enabled: bool,
7429) -> Result<bool, SuperviseError> {
7430 let (current_enabled, current_state, respawn_pending) = {
7431 let state = lock_snapshot(snapshot)?;
7432 (state.enabled, state.state, state.respawn_pending)
7433 };
7434 let revive_terminal = enabled
7442 && current_enabled
7443 && child.is_none()
7444 && (matches!(current_state, ModuleState::Failed | ModuleState::Stopped)
7445 || (current_state == ModuleState::Restarting && !respawn_pending));
7446 if current_enabled == enabled && !revive_terminal {
7447 return Ok(false);
7448 }
7449
7450 if enabled {
7451 update_snapshot(snapshot, Some(&spec.module_id), |state| {
7452 state.enabled = true;
7453 state.state = ModuleState::Starting;
7454 clear_current_process_facts(state);
7455 })?;
7456 #[cfg(test)]
7457 if runtime.test_seed_stale_facts_before_enable_spawn {
7458 update_snapshot(snapshot, Some(&spec.module_id), |state| {
7459 state.process_alive = true;
7460 state.pid = Some(41);
7461 state.spawned_at_ms = Some(42);
7462 state.spawned_from = Some(PathBuf::from("/spawned/module"));
7463 state.spawned_file_identity = Some(SpawnedFileIdentity {
7464 device: 43,
7465 inode: 44,
7466 #[cfg(windows)]
7467 windows_image: None,
7468 });
7469 })?;
7470 }
7471 release_dead_registration(
7472 registry,
7473 runtime.forwarding.as_deref(),
7474 snapshot,
7475 &spec.module_id,
7476 )
7477 .await?;
7478 reset_restart_count(snapshot, &spec.module_id)?;
7479 process_liveness.track(spec.module_id.clone(), Arc::clone(snapshot));
7480 let next_child = match spawn_and_mark_running(spec, runtime, snapshot) {
7481 Ok(next_child) => next_child,
7482 Err(err) => {
7483 if let Err(state_err) = update_snapshot(snapshot, Some(&spec.module_id), |state| {
7484 state.state = ModuleState::Failed;
7485 clear_current_process_facts(state);
7486 }) {
7487 error!(module_id = %spec.module_id, error = %state_err, "failed to record enable spawn failure");
7488 }
7489 process_liveness.untrack_if_current(&spec.module_id, snapshot);
7490 return Err(err);
7491 }
7492 };
7493 *child = Some(next_child);
7494 debug!(module_id = %spec.module_id, "supervised module enabled");
7495 Ok(true)
7496 } else {
7497 let stop_notice = begin_forwarding_drain_if_configured(
7498 spec,
7499 runtime,
7500 registry,
7501 snapshot,
7502 Some(false),
7503 RouteCloseReason::Disable,
7504 )
7505 .await?;
7506 drain_optional_child(
7507 &spec.module_id,
7508 spec.protocol,
7509 stop_notice,
7510 registry,
7511 runtime.forwarding.as_deref(),
7512 snapshot,
7513 &runtime.terminal_ring,
7514 &runtime.spawn_events,
7515 child,
7516 runtime.drain_timeout,
7517 ModuleState::Disabled,
7518 Some(false),
7519 )
7520 .await?;
7521 debug!(module_id = %spec.module_id, "supervised module disabled");
7522 Ok(true)
7523 }
7524}
7525
7526#[allow(clippy::too_many_arguments)]
7527async fn on_child_exit(
7528 spec: &ModuleSpec,
7529 policy: RestartPolicy,
7530 registry: &Registry,
7531 snapshot: &SharedSnapshot,
7532 terminal_ring: &Arc<Mutex<TerminalRing>>,
7533 spawn_events: &SpawnEventFeed,
7534 roster: &ChildRoster,
7535 exit_report: ExitReport,
7536) -> NextAction {
7537 if roster.is_closed() {
7543 return on_child_exit_during_daemon_shutdown(
7544 spec,
7545 registry,
7546 snapshot,
7547 terminal_ring,
7548 spawn_events,
7549 exit_report,
7550 )
7551 .await;
7552 }
7553 let unrequested_clean_exit_of_protocol_none = exit_report.kind == ExitKind::Clean
7568 && running_protocol(spec, snapshot) == ModuleProtocol::None;
7569 match exit_report.kind {
7570 ExitKind::Clean if !unrequested_clean_exit_of_protocol_none => {
7571 info!(
7572 module_id = %spec.module_id,
7573 exit_code = ?exit_report.code,
7574 exit_signal = ?exit_report.signal,
7575 "supervised module exited cleanly"
7576 );
7577 if let Err(err) = publish_after_terminal_record(
7578 &spec.module_id,
7579 || {
7580 record_terminal(
7581 &spec.module_id,
7582 terminal_ring,
7583 spawn_events,
7584 &exit_report,
7585 TerminalDisposition::Stopped,
7586 )
7587 },
7588 || {
7589 update_snapshot(snapshot, Some(&spec.module_id), |state| {
7590 state.state = ModuleState::Stopped;
7591 clear_current_process_facts(state);
7592 state.last_exit = Some(exit_report.clone());
7593 })
7594 },
7595 ) {
7596 error!(module_id = %spec.module_id, error = %err, "failed to publish clean module exit");
7597 }
7598 let registration_released = match wait_for_registration_release(
7599 registry,
7600 &spec.module_id,
7601 REGISTRY_RELEASE_TIMEOUT,
7602 )
7603 .await
7604 {
7605 Ok(()) => true,
7606 Err(err) => {
7607 warn!(module_id = %spec.module_id, error = %err, "registration still active after clean exit");
7608 false
7609 }
7610 };
7611 NextAction::Stop {
7612 registration_released,
7613 }
7614 }
7615 ExitKind::Clean | ExitKind::Crash => {
7616 if unrequested_clean_exit_of_protocol_none {
7617 warn!(
7618 module_id = %spec.module_id,
7619 exit_code = ?exit_report.code,
7620 exit_signal = ?exit_report.signal,
7621 "protocol-none module exited cleanly without a stop request; handling it as a crash"
7622 );
7623 } else {
7624 warn!(
7625 module_id = %spec.module_id,
7626 exit_code = ?exit_report.code,
7627 exit_signal = ?exit_report.signal,
7628 "supervised module exited abnormally (crash)"
7629 );
7630 }
7631 let mut restart_schedule = None;
7632 let mut disposition = TerminalDisposition::Disabled;
7633 let mut final_state = ModuleState::Disabled;
7634 let mut disposition_detail = lock_snapshot(snapshot)
7638 .ok()
7639 .and_then(|mut state| state.spawn_failure.take());
7640 let now = Instant::now();
7641 if let Err(err) = update_snapshot(snapshot, Some(&spec.module_id), |state| {
7642 if state.enabled {
7643 if let Some(schedule) = state.next_crash_restart(&policy, now) {
7644 state.respawn_pending = true;
7650 final_state = ModuleState::Restarting;
7651 restart_schedule = Some(schedule);
7652 disposition = TerminalDisposition::Restarting;
7653 } else {
7654 final_state = ModuleState::Failed;
7655 disposition = TerminalDisposition::Failed;
7656 let budget = policy.budget_exhausted_detail();
7657 disposition_detail =
7658 Some(disposition_detail.take().map_or_else(
7659 || budget.clone(),
7660 |cause| format!("{cause}; {budget}"),
7661 ));
7662 }
7663 } else {
7664 disposition = TerminalDisposition::Disabled;
7665 }
7666 }) {
7667 error!(module_id = %spec.module_id, error = %err, "failed to record crashed module exit");
7668 return NextAction::Stop {
7669 registration_released: false,
7670 };
7671 }
7672 if disposition == TerminalDisposition::Failed {
7673 error!(
7678 module_id = %spec.module_id,
7679 max_restarts = policy.max_restarts,
7680 window_secs = policy.window.as_secs(),
7681 "module stopped: {}",
7682 policy.budget_exhausted_detail()
7683 );
7684 }
7685 if let Err(err) = publish_after_terminal_record(
7686 &spec.module_id,
7687 || {
7688 record_terminal_with_detail(
7689 &spec.module_id,
7690 terminal_ring,
7691 spawn_events,
7692 &exit_report,
7693 disposition,
7694 disposition_detail,
7695 )
7696 },
7697 || {
7698 update_snapshot(snapshot, Some(&spec.module_id), |state| {
7699 state.state = final_state;
7700 clear_current_process_facts(state);
7701 state.last_exit = Some(exit_report.clone());
7702 })
7703 },
7704 ) {
7705 error!(module_id = %spec.module_id, error = %err, "failed to publish crashed module exit");
7706 }
7707
7708 if let Some(schedule) = restart_schedule {
7709 NextAction::Restart {
7710 schedule: Some(schedule),
7711 }
7712 } else {
7713 let registration_released = match wait_for_registration_release(
7714 registry,
7715 &spec.module_id,
7716 REGISTRY_RELEASE_TIMEOUT,
7717 )
7718 .await
7719 {
7720 Ok(()) => true,
7721 Err(err) => {
7722 warn!(module_id = %spec.module_id, error = %err, "registration still active after failed module");
7723 false
7724 }
7725 };
7726 NextAction::Stop {
7727 registration_released,
7728 }
7729 }
7730 }
7731 ExitKind::DeliberateSeverance => {
7732 warn!(
7733 module_id = %spec.module_id,
7734 exit_code = ?exit_report.code,
7735 exit_signal = ?exit_report.signal,
7736 "supervised module exited after deliberate connection severance"
7737 );
7738 let should_restart = match lock_snapshot(snapshot) {
7739 Ok(state) => state.enabled,
7740 Err(err) => {
7741 error!(module_id = %spec.module_id, error = %err, "failed to read deliberately severed module state");
7742 return NextAction::Stop {
7743 registration_released: false,
7744 };
7745 }
7746 };
7747 let final_state = if should_restart {
7748 ModuleState::Restarting
7749 } else {
7750 ModuleState::Disabled
7751 };
7752 if let Err(err) = publish_after_terminal_record(
7753 &spec.module_id,
7754 || {
7755 record_terminal(
7756 &spec.module_id,
7757 terminal_ring,
7758 spawn_events,
7759 &exit_report,
7760 terminal_disposition(final_state),
7761 )
7762 },
7763 || {
7764 update_snapshot(snapshot, Some(&spec.module_id), |state| {
7765 state.state = final_state;
7766 clear_current_process_facts(state);
7767 state.last_exit = Some(exit_report.clone());
7768 state.lifetime_restarts += 1;
7769 })
7770 },
7771 ) {
7772 error!(module_id = %spec.module_id, error = %err, "failed to publish deliberately severed module exit");
7773 return NextAction::Stop {
7774 registration_released: false,
7775 };
7776 }
7777
7778 if should_restart {
7779 NextAction::Restart { schedule: None }
7780 } else {
7781 let registration_released = match wait_for_registration_release(
7782 registry,
7783 &spec.module_id,
7784 REGISTRY_RELEASE_TIMEOUT,
7785 )
7786 .await
7787 {
7788 Ok(()) => true,
7789 Err(err) => {
7790 warn!(module_id = %spec.module_id, error = %err, "registration still active after deliberately severed module exit");
7791 false
7792 }
7793 };
7794 NextAction::Stop {
7795 registration_released,
7796 }
7797 }
7798 }
7799 }
7800}
7801
7802async fn on_child_exit_during_daemon_shutdown(
7803 spec: &ModuleSpec,
7804 registry: &Registry,
7805 snapshot: &SharedSnapshot,
7806 terminal_ring: &Arc<Mutex<TerminalRing>>,
7807 spawn_events: &SpawnEventFeed,
7808 exit_report: ExitReport,
7809) -> NextAction {
7810 info!(
7811 module_id = %spec.module_id,
7812 exit_code = ?exit_report.code,
7813 exit_signal = ?exit_report.signal,
7814 exit_kind = ?exit_report.kind,
7815 "supervised module exited during daemon shutdown; not restarting it"
7816 );
7817 if let Err(err) = publish_after_terminal_record(
7818 &spec.module_id,
7819 || {
7820 record_terminal(
7821 &spec.module_id,
7822 terminal_ring,
7823 spawn_events,
7824 &exit_report,
7825 TerminalDisposition::DaemonShutdown,
7826 )
7827 },
7828 || {
7829 update_snapshot(snapshot, Some(&spec.module_id), |state| {
7830 state.state = ModuleState::Stopped;
7831 clear_current_process_facts(state);
7832 state.last_exit = Some(exit_report.clone());
7833 })
7834 },
7835 ) {
7836 error!(module_id = %spec.module_id, error = %err, "failed to publish module exit during daemon shutdown");
7837 }
7838 let registration_released =
7839 wait_for_registration_release(registry, &spec.module_id, REGISTRY_RELEASE_TIMEOUT)
7840 .await
7841 .is_ok();
7842 NextAction::Stop {
7843 registration_released,
7844 }
7845}
7846
7847fn publish_after_terminal_record<T>(
7852 module_id: &str,
7853 record: impl FnOnce(),
7854 publish: impl FnOnce() -> T,
7855) -> T {
7856 let recorded = std::panic::catch_unwind(std::panic::AssertUnwindSafe(record));
7857 let published = publish();
7858 if let Err(panic) = recorded {
7859 error!(
7860 module_id,
7861 "terminal recording panicked; published final state before resuming unwind"
7862 );
7863 std::panic::resume_unwind(panic);
7864 }
7865 published
7866}
7867
7868fn fail_after_wait_error(
7869 module_id: &str,
7870 snapshot: &SharedSnapshot,
7871 terminal_ring: &Arc<Mutex<TerminalRing>>,
7872 spawn_events: &SpawnEventFeed,
7873 detail: String,
7874) {
7875 let report = wait_error_exit_report();
7876 publish_after_terminal_record(
7877 module_id,
7878 || {
7879 record_terminal_with_detail(
7880 module_id,
7881 terminal_ring,
7882 spawn_events,
7883 &report,
7884 TerminalDisposition::Failed,
7885 Some(detail),
7886 )
7887 },
7888 || fail_snapshot(snapshot, Some(module_id), Some(report.clone())),
7889 );
7890}
7891
7892#[cfg(test)]
7893fn record_wait_error_terminal(
7894 module_id: &str,
7895 terminal_ring: &Arc<Mutex<TerminalRing>>,
7896 spawn_events: &SpawnEventFeed,
7897) {
7898 record_terminal(
7899 module_id,
7900 terminal_ring,
7901 spawn_events,
7902 &wait_error_exit_report(),
7903 TerminalDisposition::Failed,
7904 );
7905}
7906
7907fn record_terminal(
7908 module_id: &str,
7909 terminal_ring: &Arc<Mutex<TerminalRing>>,
7910 spawn_events: &SpawnEventFeed,
7911 exit_report: &ExitReport,
7912 disposition: TerminalDisposition,
7913) {
7914 record_terminal_with_detail(
7915 module_id,
7916 terminal_ring,
7917 spawn_events,
7918 exit_report,
7919 disposition,
7920 None,
7921 );
7922}
7923
7924fn durable_terminal_history_of(
7928 terminal_ring: &Mutex<TerminalRing>,
7929 module_id: &str,
7930) -> subc_control::TerminalHistory {
7931 let read = terminal_ring
7932 .lock()
7933 .unwrap_or_else(|p| p.into_inner())
7934 .capture_durable_history();
7935 read.read(module_id)
7936}
7937
7938fn record_terminal_with_detail(
7939 module_id: &str,
7940 terminal_ring: &Arc<Mutex<TerminalRing>>,
7941 spawn_events: &SpawnEventFeed,
7942 exit_report: &ExitReport,
7943 disposition: TerminalDisposition,
7944 disposition_detail: Option<String>,
7945) {
7946 #[cfg(test)]
7947 terminal_history_tests::before_terminal_record(spawn_events);
7948 spawn_events.emit_exited(module_id, exit_report.code, exit_report.signal);
7949 let record = TerminalRecord {
7950 exit_code: exit_report.code,
7951 exit_signal: exit_report.signal,
7952 at_ms: exit_report.at_ms,
7953 disposition,
7954 exit_kind: exit_report.kind.into(),
7955 disposition_detail,
7956 };
7957 terminal_ring
7958 .lock()
7959 .unwrap_or_else(|poisoned| poisoned.into_inner())
7960 .record_exit(module_id, record);
7961}
7962
7963fn untrack_if_registration_released(
7964 process_liveness: &SupervisorProcessLiveness,
7965 registry: &Registry,
7966 module_id: &str,
7967 snapshot: &SharedSnapshot,
7968) {
7969 match registry.get_module(module_id) {
7970 Ok(None) => process_liveness.untrack_if_current(module_id, snapshot),
7971 Ok(Some(_)) => {}
7972 Err(err) => {
7973 warn!(module_id, error = %err, "could not determine whether supervisor liveness can be untracked");
7974 }
7975 }
7976}
7977
7978#[cfg(test)]
7992fn apply_wire_spawn_args(
7993 command: &mut Command,
7994 spec: &ModuleSpec,
7995 connection_file_path: Option<&std::path::Path>,
7996 handle: Option<&SupervisorHandle>,
7997) -> Result<Option<NonceHandoff>, SuperviseError> {
7998 apply_wire_spawn_args_for_role(
7999 command,
8000 spec,
8001 connection_file_path,
8002 handle,
8003 SpawnRole::Plain,
8004 )
8005}
8006
8007#[cfg(unix)]
8012type NonceHandoff = subc_os::LaunchNonceHandoff;
8013#[cfg(windows)]
8014type NonceHandoff = subc_os::LaunchNoncePipeHandoff;
8015#[cfg(not(any(unix, windows)))]
8016type NonceHandoff = std::convert::Infallible;
8017
8018fn apply_wire_spawn_args_for_role(
8039 command: &mut Command,
8040 spec: &ModuleSpec,
8041 connection_file_path: Option<&std::path::Path>,
8042 handle: Option<&SupervisorHandle>,
8043 role: SpawnRole,
8044) -> Result<Option<NonceHandoff>, SuperviseError> {
8045 command.env(SUBC_MODULE_ID_ENV, &spec.module_id);
8046 command.env_remove(subc_os::LAUNCH_NONCE_FD_ENV);
8051 #[cfg(windows)]
8052 {
8053 command.env_remove(subc_os::LAUNCH_NONCE_PIPE_ENV);
8054 command.env_remove(subc_os::LAUNCH_NONCE_PIPE_FALLBACK_ENV);
8055 }
8056 command.env_remove(SUBC_LAUNCH_NONCE_ENV);
8058 if spec.protocol == ModuleProtocol::None {
8059 return Ok(None);
8060 }
8061 if let Some(connection_file_path) = connection_file_path {
8062 command.arg(SUBC_ARG).arg(connection_file_path);
8063 }
8064
8065 let nonce = generate_launch_nonce()?;
8069 if let Some(handle) = handle {
8070 match role {
8071 SpawnRole::Plain => {
8072 handle.set_spawn_nonce(&spec.module_id, nonce.clone());
8073 if spec.reserved {
8074 handle.set_reserved_nonce(&spec.module_id, nonce.clone());
8075 }
8076 }
8077 SpawnRole::SwapCandidate => handle.open_swap(&spec.module_id, nonce.clone()),
8078 }
8079 }
8080 #[cfg(unix)]
8081 let handoff = {
8082 let handoff =
8083 subc_os::LaunchNonceHandoff::new(&nonce).map_err(|source| SuperviseError::Spawn {
8084 program: spec.program.clone(),
8085 source,
8086 cgroup_path: None,
8087 })?;
8088 command.env(subc_os::LAUNCH_NONCE_FD_ENV, handoff.fd_env_value());
8089 Some(handoff)
8090 };
8091 #[cfg(windows)]
8092 let handoff = prepare_windows_nonce_pipe(command, spec, &nonce);
8093 #[cfg(not(any(unix, windows)))]
8094 let handoff = None;
8095 #[cfg(not(unix))]
8099 command.env(SUBC_LAUNCH_NONCE_ENV, nonce);
8100 Ok(handoff)
8101}
8102
8103fn apply_child_env(command: &mut Command, spec: &ModuleSpec) {
8104 command.env_remove(CK_LOG_ENV);
8105 command.env_remove(SUBC_SPAWN_ROLE_ENV);
8112 for (key, value) in &spec.env {
8113 if matches!(
8117 key.as_str(),
8118 CAPTURE_MAX_FILE_MB_ENV | CAPTURE_KEEP_ENV | CAPTURE_MAX_AGE_DAYS_ENV
8119 ) || key == SUBC_SPAWN_ROLE_ENV
8120 {
8121 continue;
8122 }
8123 command.env(key, value);
8124 }
8125}
8126
8127#[cfg(windows)]
8128fn prepare_windows_nonce_pipe(
8129 command: &mut Command,
8130 spec: &ModuleSpec,
8131 nonce: &str,
8132) -> Option<NonceHandoff> {
8133 match subc_os::LaunchNoncePipeHandoff::new(nonce) {
8134 Ok(handoff) => {
8135 command.env(subc_os::LAUNCH_NONCE_PIPE_ENV, handoff.name());
8136 command.env(subc_os::LAUNCH_NONCE_PIPE_FALLBACK_ENV, "env");
8141 Some(handoff)
8142 }
8143 Err(error) => {
8144 warn!(module_id = %spec.module_id, %error, "could not prepare launch nonce pipe; retaining the environment handoff");
8145 None
8146 }
8147 }
8148}
8149
8150#[cfg(windows)]
8151fn start_windows_nonce_pipe(
8152 handoff: Option<NonceHandoff>,
8153 pid: u32,
8154 spec: &ModuleSpec,
8155) -> Option<subc_os::LaunchNoncePipeDelivery> {
8156 handoff.and_then(|handoff| match handoff.serve(pid, REGISTRY_RELEASE_TIMEOUT) {
8160 Ok(delivery) => Some(delivery),
8161 Err(error) => {
8162 warn!(module_id = %spec.module_id, %error, "could not serve launch nonce pipe; retaining the environment handoff");
8163 None
8164 }
8165 })
8166}
8167
8168fn observed_launch_nonce_source(snapshot: &SupervisorSnapshot) -> Option<String> {
8169 #[cfg(windows)]
8170 if snapshot.process_alive && snapshot.spawned_protocol == Some(ModuleProtocol::Subc) {
8171 return Some(
8172 if snapshot
8173 .pipe_consumed
8174 .as_ref()
8175 .is_some_and(|consumed| consumed.get().is_some())
8176 {
8177 "pipe"
8178 } else {
8179 "env"
8180 }
8181 .to_owned(),
8182 );
8183 }
8184 let _ = snapshot;
8185 None
8186}
8187
8188#[derive(Debug, Clone, Copy, PartialEq, Eq)]
8191enum SpawnRole {
8192 Plain,
8193 SwapCandidate,
8194}
8195
8196fn apply_spawn_role(command: &mut Command, role: SpawnRole) {
8199 if role == SpawnRole::SwapCandidate {
8200 command.env(SUBC_SPAWN_ROLE_ENV, SPAWN_ROLE_SWAP_CANDIDATE);
8201 }
8202}
8203
8204fn previous_instance_error(
8205 spec: &ModuleSpec,
8206 pid: u32,
8207 reason: crate::live_children::RetainedReason,
8208) -> SuperviseError {
8209 SuperviseError::Spawn {
8210 program: spec.program.clone(),
8211 source: io::Error::new(io::ErrorKind::WouldBlock, reason.waiting_message(pid)),
8212 cgroup_path: None,
8213 }
8214}
8215
8216fn spawn_child(
8217 spec: &ModuleSpec,
8218 connection_file_path: Option<&std::path::Path>,
8219 handle: Option<&SupervisorHandle>,
8220 ring: &Arc<Mutex<StderrRing>>,
8221 capture_logs_dir: Option<&std::path::Path>,
8222 roster: &ChildRoster,
8223 #[cfg(target_os = "linux")] cgroup_placement: Option<&subc_cgroup::Placement>,
8224) -> Result<SupervisedChild, SuperviseError> {
8225 spawn_child_in_slot(
8226 spec,
8227 connection_file_path,
8228 handle,
8229 ring,
8230 capture_logs_dir,
8231 roster,
8232 #[cfg(target_os = "linux")]
8233 cgroup_placement,
8234 SpawnRole::Plain,
8235 false,
8236 )
8237}
8238
8239#[allow(clippy::too_many_arguments)]
8252fn spawn_child_in_slot(
8253 spec: &ModuleSpec,
8254 connection_file_path: Option<&std::path::Path>,
8255 handle: Option<&SupervisorHandle>,
8256 ring: &Arc<Mutex<StderrRing>>,
8257 capture_logs_dir: Option<&std::path::Path>,
8258 roster: &ChildRoster,
8259 #[cfg(target_os = "linux")] cgroup_placement: Option<&subc_cgroup::Placement>,
8260 role: SpawnRole,
8261 alternate_slot: bool,
8262) -> Result<SupervisedChild, SuperviseError> {
8263 if let Some((pid, reason)) = roster.previous_instance(&spec.module_id) {
8264 return Err(previous_instance_error(spec, pid, reason));
8265 }
8266 if roster.is_closed() {
8267 return Err(SuperviseError::Spawn {
8268 program: spec.program.clone(),
8269 source: io::Error::other("the daemon is shutting down; not starting a new process"),
8270 cgroup_path: None,
8271 });
8272 }
8273 #[cfg(target_os = "linux")]
8274 let cgroup_name = {
8275 if cgroup_placement.is_none() {
8280 swap::cgroup_name(&spec.module_id, alternate_slot)
8281 } else {
8282 let nonce = generate_launch_nonce()?;
8283 let suffix = u128::from_str_radix(&nonce[..32], 16).expect("hex launch nonce");
8284 let mut end = spec.module_id.len().min(64);
8287 while !spec.module_id.is_char_boundary(end) {
8288 end -= 1;
8289 }
8290 format!(
8291 "{}_{suffix}",
8292 swap::cgroup_name(&spec.module_id[..end], alternate_slot)
8293 )
8294 }
8295 };
8296 #[cfg(not(target_os = "linux"))]
8297 let _ = alternate_slot;
8298 #[cfg(target_os = "macos")]
8299 let (mut command, privacy_exec, exec_ack) = privacy_command(spec, roster)?;
8300 #[cfg(windows)]
8301 let image_capture = windows_spawn_capture(spec).map_err(|source| SuperviseError::Spawn {
8302 program: spec.program.clone(),
8303 source,
8304 cgroup_path: None,
8305 })?;
8306 #[cfg(windows)]
8307 let mut command = Command::new(image_capture.path());
8308 #[cfg(not(any(target_os = "macos", windows)))]
8309 let mut command = Command::new(&spec.program);
8310 command.args(&spec.args);
8311 apply_child_env(&mut command, spec);
8341 apply_spawn_role(&mut command, role);
8342 let nonce_handoff =
8343 apply_wire_spawn_args_for_role(&mut command, spec, connection_file_path, handle, role)?;
8344
8345 #[cfg(target_os = "linux")]
8346 let cgroup_path = cgroup_placement
8347 .map(|placement| placement.module_path(&cgroup_name))
8348 .transpose()
8349 .map_err(|source| SuperviseError::Cgroup {
8350 module_id: spec.module_id.clone(),
8351 source,
8352 })?;
8353 #[cfg(not(target_os = "linux"))]
8354 let cgroup_path: Option<PathBuf> = None;
8355 #[cfg(target_os = "linux")]
8356 if let Some(path) = &cgroup_path {
8357 if let Err(error) = apply_cgroup_placement(&mut command, spec, path) {
8358 if let Some(placement) = cgroup_placement {
8359 remove_module_cgroup(placement, &cgroup_name);
8360 }
8361 return Err(error);
8362 }
8363 }
8364
8365 let output_sink = if let Some(logs_dir) = capture_logs_dir {
8366 let path = logs_dir.join(format!("{}.stderr.log", spec.module_id));
8367 match ChildOutputSink::open(&path, capture_retention(spec)) {
8368 Ok(sink) => sink,
8369 Err(error) => {
8370 warn!(
8371 module_id = %spec.module_id,
8372 path = %path.display(),
8373 error = %error,
8374 "could not open child output capture file; forwarding to stderr"
8375 );
8376 ChildOutputSink::Stderr
8377 }
8378 }
8379 } else {
8380 ChildOutputSink::Stderr
8381 };
8382
8383 command.stdout(Stdio::piped());
8384 command.stderr(Stdio::piped());
8385 command.kill_on_drop(true);
8386 #[cfg(unix)]
8403 command.process_group(0);
8404 command.stdin(Stdio::null());
8405 #[cfg(unix)]
8409 if let Some(handoff) = nonce_handoff {
8410 handoff.install_last(command.as_std_mut());
8411 }
8412 #[cfg(not(any(unix, windows)))]
8413 let _ = nonce_handoff;
8414
8415 #[cfg(windows)]
8420 {
8421 subc_jobobject::ensure_root_job().map_err(|source| SuperviseError::Spawn {
8422 program: spec.program.clone(),
8423 source,
8424 cgroup_path: None,
8425 })?;
8426 subc_jobobject::suspend_on_create_async(&mut command);
8427 }
8428 let spawned_at = Instant::now();
8432 let mut child = match command.spawn() {
8433 Ok(child) => child,
8434 Err(source) => {
8435 #[cfg(target_os = "linux")]
8436 if let Some(placement) = cgroup_placement {
8437 remove_module_cgroup(placement, &cgroup_name);
8438 }
8439 return Err(SuperviseError::Spawn {
8440 program: spec.program.clone(),
8441 source,
8442 cgroup_path,
8443 });
8444 }
8445 };
8446 #[cfg(test)]
8447 spawn_record_tests::after_create(&child, spec);
8448 #[cfg(target_os = "macos")]
8453 drop(exec_ack);
8454
8455 #[cfg(windows)]
8460 let windows_image = match image_capture.bind_tokio(&child) {
8461 Ok(image) => Arc::new(image),
8462 Err(source) => {
8463 kill_and_reap_rejected_child(&mut child);
8464 return Err(SuperviseError::Spawn {
8465 program: spec.program.clone(),
8466 source,
8467 cgroup_path: None,
8468 });
8469 }
8470 };
8471 #[cfg(windows)]
8474 let job = contain_spawned_child(&mut child, spec)?;
8475 let spawned_at_ms = unix_ms_now();
8476 #[cfg(not(windows))]
8477 let spawned_from = spec.program.clone();
8478 #[cfg(windows)]
8479 let spawned_from = windows_image.path().to_path_buf();
8480 #[cfg(not(windows))]
8481 let spawned_file_identity = spawned_file_identity(&spawned_from);
8482 #[cfg(windows)]
8483 let spawned_file_identity = Some(SpawnedFileIdentity {
8484 device: 0,
8485 inode: 0,
8486 windows_image: Some(Arc::clone(&windows_image)),
8487 });
8488 let pid = child.id().ok_or_else(|| SuperviseError::Spawn {
8489 program: spec.program.clone(),
8490 source: io::Error::other("spawned child exposed no live pid"),
8491 cgroup_path: cgroup_path.clone(),
8492 })?;
8493 #[cfg(windows)]
8494 let nonce_delivery = start_windows_nonce_pipe(nonce_handoff, pid, spec);
8495 #[cfg(not(windows))]
8496 let process_start_time = crate::provenance::process_start_time(pid);
8497 #[cfg(windows)]
8498 let process_start_time = Some(windows_image.start_time());
8499 let process_identity = process_start_time.map(|start_time| ProcessIdentity { pid, start_time });
8500 #[cfg(all(test, target_os = "macos"))]
8501 privacy_exec_boundary_tests::before_image_sample(spec, pid);
8502 #[cfg(not(windows))]
8507 let recorded_image = observe_spawned_image(pid);
8508 #[cfg(target_os = "macos")]
8512 let recorded_image = if privacy_exec.is_some() {
8513 None
8514 } else {
8515 recorded_image
8516 };
8517 #[cfg(target_os = "linux")]
8518 let recorded_cgroup_name = cgroup_path.as_ref().map(|_| cgroup_name.clone());
8519 #[cfg(not(target_os = "linux"))]
8520 let recorded_cgroup_name = None;
8521 let roster_guard = roster.admit(
8522 spec.module_id.clone(),
8523 pid,
8524 spec.protocol,
8525 process_start_time,
8526 crate::child_roster::RecordedIdentity {
8527 #[cfg(not(windows))]
8528 start_time: recorded_image.map(|image| image.start_time),
8529 #[cfg(windows)]
8530 start_time: process_start_time,
8531 #[cfg(not(windows))]
8532 executable: recorded_image
8533 .and_then(|image| image.executable)
8534 .map(crate::live_children::ExecutableIdentity::from),
8535 #[cfg(windows)]
8536 executable: Some(crate::live_children::ExecutableIdentity::from(
8537 windows_image.identity(),
8538 )),
8539 cgroup_name: recorded_cgroup_name,
8540 #[cfg(target_os = "linux")]
8541 cgroup_placement: cgroup_placement.cloned(),
8542 },
8543 );
8544 let roster_guard = match roster_guard {
8545 Ok(guard) => guard,
8546 Err(source) => {
8547 #[cfg(windows)]
8548 if let Some(job) = &job {
8549 let _ = job.terminate();
8550 }
8551 #[cfg(target_os = "linux")]
8552 kill_module_cgroup(cgroup_placement, &cgroup_name);
8553 kill_and_reap_rejected_child(&mut child);
8554 #[cfg(target_os = "linux")]
8555 if let Some(placement) = cgroup_placement {
8556 remove_module_cgroup(placement, &cgroup_name);
8557 }
8558 return Err(SuperviseError::Spawn {
8559 program: spec.program.clone(),
8560 source: io::Error::other(format!(
8561 "could not durably record module child: {source}"
8562 )),
8563 cgroup_path,
8564 });
8565 }
8566 };
8567 if roster.is_closed() {
8576 #[cfg(target_os = "linux")]
8578 kill_module_cgroup(cgroup_placement, &cgroup_name);
8579 if let Err(error) = child.start_kill() {
8580 debug!(module_id = %spec.module_id, pid, %error, "kill of a process spawned during daemon shutdown failed; it may already have exited");
8581 }
8582 #[cfg(target_os = "linux")]
8583 if let Some(placement) = cgroup_placement {
8584 while matches!(child.try_wait(), Ok(None)) {
8588 std::thread::yield_now();
8589 }
8590 if matches!(
8591 subc_cgroup::kill_module(Some(placement), &cgroup_name),
8592 subc_cgroup::KillOutcome::Killed
8593 ) {
8594 if let Ok(path) = placement.module_path(&cgroup_name) {
8595 while std::fs::read_to_string(path.join("cgroup.events"))
8596 .ok()
8597 .is_some_and(|events| events.lines().any(|line| line == "populated 1"))
8598 {
8599 std::thread::yield_now();
8600 }
8601 }
8602 }
8603 remove_module_cgroup(placement, &cgroup_name);
8604 }
8605 drop(roster_guard);
8606 return Err(SuperviseError::Spawn {
8607 program: spec.program.clone(),
8608 source: io::Error::other(
8609 "the daemon began shutting down while this process was starting; ended it",
8610 ),
8611 cgroup_path,
8612 });
8613 }
8614
8615 #[cfg(windows)]
8616 if let Err(error) = resume_suspended_child(pid, spec) {
8617 kill_and_reap_rejected_child(&mut child);
8618 return Err(error);
8619 }
8620
8621 let stdout_pump = match child.stdout.take() {
8622 Some(stdout) => Some(tokio::spawn(pump_stdout_to(stdout, output_sink.clone()))),
8623 None => {
8624 warn!(
8625 module_id = %spec.module_id,
8626 "spawned child exposed no stdout pipe; file capture will be incomplete"
8627 );
8628 None
8629 }
8630 };
8631 let stderr_pump = match child.stderr.take() {
8632 Some(stderr) => {
8633 let generation = ring
8634 .lock()
8635 .unwrap_or_else(|poisoned| poisoned.into_inner())
8636 .begin_process();
8637 Some(StderrPump {
8638 task: tokio::spawn(pump_stderr_to(
8639 stderr,
8640 Arc::clone(ring),
8641 generation,
8642 output_sink,
8643 )),
8644 generation,
8645 })
8646 }
8647 None => {
8648 ring.lock()
8652 .unwrap_or_else(|poisoned| poisoned.into_inner())
8653 .mark_not_captured("stderr pipe was not available on spawn");
8654 warn!(
8655 module_id = %spec.module_id,
8656 "spawned child exposed no stderr pipe; tail will be unavailable"
8657 );
8658 None
8659 }
8660 };
8661
8662 let exec = Arc::new(crate::activity::ExecOutcome::new());
8663 #[cfg(target_os = "macos")]
8664 let pending_exec = privacy_exec.is_some();
8665 #[cfg(not(target_os = "macos"))]
8666 let pending_exec = false;
8667 if !pending_exec {
8668 exec.finish(crate::activity::ExecState::CommittedExec);
8669 }
8670 Ok(SupervisedChild {
8671 child,
8672 exec: crate::activity::ExecOwner(exec),
8673 #[cfg(windows)]
8674 nonce_delivery,
8675 protocol: spec.protocol,
8676 #[cfg(target_os = "linux")]
8677 module_id: cgroup_name,
8678 #[cfg(target_os = "linux")]
8679 cgroup_placement: cgroup_placement.cloned(),
8680 #[cfg(windows)]
8681 job,
8682 stdout_pump,
8683 stderr_pump,
8684 stderr_ring: Arc::clone(ring),
8685 spawned_at,
8686 spawned_at_ms,
8687 spawned_from,
8688 launch_spec_digest: LaunchSpecDigest::from_spec(spec),
8689 spawned_file_identity,
8690 process_start_time,
8691 process_identity,
8692 pid,
8693 roster_guard: Some(roster_guard),
8694 #[cfg(target_os = "macos")]
8695 privacy_exec,
8696 #[cfg(target_os = "macos")]
8697 report_ready: Arc::new(OnceLock::new()),
8698 spawn_failure: None,
8699 })
8700}
8701
8702#[cfg(test)]
8703mod spawn_record_tests {
8704 use super::*;
8705
8706 thread_local! {
8707 static LAST_PID: std::cell::Cell<Option<u32>> = const { std::cell::Cell::new(None) };
8708 }
8709
8710 pub(super) fn after_create(child: &Child, _spec: &ModuleSpec) {
8711 LAST_PID.set(child.id());
8712 #[cfg(windows)]
8713 if let Some((_, address)) = _spec
8714 .env
8715 .iter()
8716 .find(|(key, _)| key == "SUBC_TEST_AFTER_CREATE")
8717 {
8718 use std::io::{Read, Write};
8719 let mut stream = std::net::TcpStream::connect(address).unwrap();
8720 stream
8721 .write_all(&child.id().unwrap().to_ne_bytes())
8722 .unwrap();
8723 let mut release = [0];
8724 stream.read_exact(&mut release).unwrap();
8725 }
8726 }
8727
8728 #[cfg(windows)]
8729 pub(super) fn assignment_failpoint(spec: &ModuleSpec) -> io::Result<()> {
8730 if spec
8731 .env
8732 .iter()
8733 .any(|(key, _)| key == "SUBC_TEST_ASSIGNMENT_FAILURE")
8734 {
8735 Err(io::Error::other("injected module job assignment failure"))
8736 } else {
8737 Ok(())
8738 }
8739 }
8740
8741 fn spec() -> ModuleSpec {
8742 ModuleSpec {
8743 module_id: "spawn-record-worker".into(),
8744 #[cfg(unix)]
8745 program: "/bin/sleep".into(),
8746 #[cfg(windows)]
8747 program: std::env::current_exe().unwrap(),
8748 #[cfg(unix)]
8749 args: vec!["60".into()],
8750 #[cfg(windows)]
8751 args: vec![
8752 "--exact".into(),
8753 "supervise::spawn_record_tests::windows_store_touch_fixture".into(),
8754 "--nocapture".into(),
8755 ],
8756 env: vec![],
8757 reserved: false,
8758 reserved_prefixes: vec![],
8759 protocol: ModuleProtocol::None,
8760 overlap: Default::default(),
8761 }
8762 }
8763
8764 #[tokio::test]
8765 async fn record_sync_failure_fails_spawn_and_reaps_child() {
8766 let home = cortexkit_test_support::ScratchDir::new("record-sync-failure");
8767 let supervisor =
8768 Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default())
8769 .with_live_children_record(home.join("live-children.json"));
8770 crate::live_children::fail_next_file_sync();
8771 let result = supervisor.spawn(spec());
8772 let failed = result.is_err();
8773 if let Ok(module) = result {
8774 module.stop().await.unwrap();
8775 }
8776 assert!(failed, "record failure must fail spawn");
8777 let pid = LAST_PID.get().expect("the real OS child was created");
8778 assert!(
8779 subc_os::Process::open(pid).unwrap().is_none(),
8780 "child {pid} survived rejected admission or was not reaped"
8781 );
8782 }
8783
8784 #[cfg(target_os = "macos")]
8785 #[test]
8786 #[ignore = "manual on-device latency measurement"]
8787 fn durable_record_write_latency_50() {
8788 let home = cortexkit_test_support::ScratchDir::new("record-latency");
8789 let path = home.join("live-children.json");
8790 let children = vec![crate::live_children::LiveChild {
8791 module_id: "latency-worker".into(),
8792 pid: std::process::id(),
8793 protocol: ModuleProtocol::None,
8794 start_time: Some(1),
8795 executable: None,
8796 cgroup_name: None,
8797 }];
8798 let mut samples = Vec::new();
8799 for _ in 0..50 {
8800 let start = std::time::Instant::now();
8801 crate::live_children::write_record(&path, &children).unwrap();
8802 samples.push(start.elapsed());
8803 }
8804 samples.sort();
8805 eprintln!(
8806 "durable_record_write_latency_50: n=50 median={:?} p95={:?}",
8807 (samples[24] + samples[25]) / 2,
8808 samples[47]
8809 );
8810 }
8811
8812 #[cfg(windows)]
8813 #[test]
8814 fn windows_store_touch_fixture() {
8815 if let Some(path) = std::env::var_os("SUBC_TEST_STORE_TOUCH") {
8816 std::fs::write(path, b"module ran").unwrap();
8817 }
8818 }
8819
8820 #[cfg(windows)]
8821 #[tokio::test]
8822 async fn windows_daemon_fixture() {
8823 let Ok(address) = std::env::var("SUBC_TEST_DAEMON_BARRIER") else {
8824 return;
8825 };
8826 let mut spec = spec();
8827 spec.env.push(("SUBC_TEST_AFTER_CREATE".into(), address));
8828 let supervisor =
8829 Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default());
8830 let _module = supervisor.spawn(spec).unwrap();
8831 }
8832
8833 #[cfg(windows)]
8834 #[test]
8835 fn windows_daemon_death_before_module_job_assignment_leaves_no_child() {
8836 use std::io::Read;
8837 let listener = std::net::TcpListener::bind("127.0.0.1:0").unwrap();
8838 let mut command = std::process::Command::new(std::env::current_exe().unwrap());
8841 command
8842 .args([
8843 "--exact",
8844 "supervise::spawn_record_tests::windows_daemon_fixture",
8845 "--nocapture",
8846 ])
8847 .env(
8848 "SUBC_TEST_DAEMON_BARRIER",
8849 listener.local_addr().unwrap().to_string(),
8850 );
8851 let mut daemon = subc_jobobject::spawn_contained(&mut command).unwrap();
8852 listener.set_nonblocking(true).unwrap();
8853 let deadline = std::time::Instant::now() + Duration::from_secs(30);
8854 let mut stream = loop {
8855 if let Ok((stream, _)) = listener.accept() {
8856 break stream;
8857 }
8858 assert!(
8859 daemon.child.try_wait().unwrap().is_none(),
8860 "daemon failed before rendezvous"
8861 );
8862 assert!(
8863 std::time::Instant::now() < deadline,
8864 "daemon did not reach CreateProcess failpoint"
8865 );
8866 std::thread::yield_now();
8867 };
8868 stream
8869 .set_read_timeout(Some(Duration::from_secs(30)))
8870 .unwrap();
8871 let mut pid = [0; 4];
8872 stream.read_exact(&mut pid).unwrap();
8873 let pid = u32::from_ne_bytes(pid);
8874 assert!(subc_jobobject::process_exists(pid));
8875 daemon.child.kill().unwrap();
8878 daemon.child.wait().unwrap();
8879 let exited = subc_jobobject::wait_for_process_exit(pid, Duration::from_secs(10));
8880 if !exited {
8881 daemon.job.terminate().unwrap();
8882 }
8883 assert!(exited, "suspended child {pid} outlived hard-killed daemon");
8884 }
8885
8886 #[cfg(windows)]
8887 #[tokio::test]
8888 async fn windows_containment_failure_never_resumes() {
8889 let home = cortexkit_test_support::ScratchDir::new("assignment-refusal");
8890 let sentinel = home.join("store-touched");
8891 let mut spec = spec();
8892 spec.env
8893 .push(("SUBC_TEST_ASSIGNMENT_FAILURE".into(), "1".into()));
8894 spec.env.push((
8895 "SUBC_TEST_STORE_TOUCH".into(),
8896 sentinel.display().to_string(),
8897 ));
8898 let supervisor =
8899 Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default());
8900 let result = supervisor.spawn(spec);
8901 if let Ok(module) = result {
8904 let _ = tokio::time::timeout(Duration::from_secs(10), async {
8905 while !sentinel.exists() && module.state().unwrap() == ModuleState::Running {
8906 tokio::task::yield_now().await;
8907 }
8908 })
8909 .await;
8910 let _ = module.stop().await;
8911 panic!(
8912 "assignment failure resumed a module; store touched={}",
8913 sentinel.exists()
8914 );
8915 }
8916 assert!(
8917 !sentinel.exists(),
8918 "refused module executed store-touch sentinel"
8919 );
8920 assert!(!subc_jobobject::process_exists(LAST_PID.get().unwrap()));
8921 }
8922
8923 #[cfg(windows)]
8924 #[tokio::test]
8925 async fn windows_nested_daemon_fixture() {
8926 let Some(sentinel) = std::env::var_os("SUBC_TEST_NESTED_STORE") else {
8927 return;
8928 };
8929 let mut spec = spec();
8930 spec.env.push((
8931 "SUBC_TEST_STORE_TOUCH".into(),
8932 sentinel.to_str().unwrap().into(),
8933 ));
8934 let supervisor =
8935 Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default());
8936 let module = supervisor
8937 .spawn(spec)
8938 .expect("per-module job must nest below daemon and host jobs");
8939 tokio::time::timeout(Duration::from_secs(10), async {
8940 while !PathBuf::from(&sentinel).exists() {
8941 tokio::task::yield_now().await;
8942 }
8943 })
8944 .await
8945 .unwrap();
8946 module.stop().await.unwrap();
8947 }
8948
8949 #[cfg(windows)]
8950 #[test]
8951 fn windows_module_job_nests_under_host_job() {
8952 let home = cortexkit_test_support::ScratchDir::new("nested-jobs");
8953 let sentinel = home.join("nested-store-touched");
8954 let mut command = std::process::Command::new(std::env::current_exe().unwrap());
8955 command
8956 .args([
8957 "--exact",
8958 "supervise::spawn_record_tests::windows_nested_daemon_fixture",
8959 "--nocapture",
8960 ])
8961 .env("SUBC_TEST_NESTED_STORE", &sentinel);
8962 let mut daemon = subc_jobobject::spawn_contained(&mut command).unwrap();
8963 assert!(daemon.child.wait().unwrap().success());
8964 assert!(sentinel.exists(), "nested protocol:none module never ran");
8965 }
8966}
8967
8968#[cfg(target_os = "linux")]
8969pub(crate) fn kill_module_cgroup(placement: Option<&subc_cgroup::Placement>, module_id: &str) {
8970 use subc_cgroup::KillOutcome;
8971 match subc_cgroup::kill_module(placement, module_id) {
8972 KillOutcome::Killed => {}
8973 KillOutcome::NotPlaced | KillOutcome::Unsupported => {
8974 debug!(
8975 module_id,
8976 "cgroup tree kill unavailable; using direct-child kill"
8977 );
8978 }
8979 KillOutcome::IoError { path, error } => {
8980 warn!(module_id, path = %path.display(), %error, "cgroup tree kill failed; using direct-child kill");
8981 }
8982 }
8983}
8984
8985#[cfg(windows)]
8993fn contain_spawned_child(
8994 child: &mut Child,
8995 spec: &ModuleSpec,
8996) -> Result<Option<subc_jobobject::JobObject>, SuperviseError> {
8997 let assignment = || {
8998 let job = subc_jobobject::JobObject::new()?;
8999 #[cfg(test)]
9000 spawn_record_tests::assignment_failpoint(spec)?;
9001 job.assign(child)?;
9002 Ok::<_, io::Error>(job)
9003 };
9004 match assignment() {
9005 Ok(job) => Ok(Some(job)),
9006 Err(source) => {
9007 kill_and_reap_rejected_child(child);
9008 Err(SuperviseError::Spawn {
9009 program: spec.program.clone(),
9010 source: io::Error::other(format!(
9011 "Windows module job containment failed: {source}"
9012 )),
9013 cgroup_path: None,
9014 })
9015 }
9016 }
9017}
9018
9019fn kill_and_reap_rejected_child(child: &mut Child) {
9022 #[cfg(unix)]
9023 if let Some(pid) = child
9024 .id()
9025 .and_then(|pid| rustix::process::Pid::from_raw(pid as i32))
9026 {
9027 let _ = rustix::process::kill_process_group(pid, rustix::process::Signal::KILL);
9028 }
9029 let _ = child.start_kill();
9030 while matches!(child.try_wait(), Ok(None)) {
9031 std::thread::yield_now();
9032 }
9033}
9034
9035#[cfg(windows)]
9040fn resume_suspended_child(pid: u32, spec: &ModuleSpec) -> Result<(), SuperviseError> {
9041 if let Err(source) = subc_jobobject::resume_main_thread(pid) {
9042 let _ = std::process::Command::new("taskkill.exe")
9046 .args(["/PID", &pid.to_string(), "/T", "/F"])
9047 .stdin(Stdio::null())
9048 .stdout(Stdio::null())
9049 .stderr(Stdio::null())
9050 .status();
9051 return Err(SuperviseError::Spawn {
9052 program: spec.program.clone(),
9053 source,
9054 cgroup_path: None,
9055 });
9056 }
9057 Ok(())
9058}
9059
9060#[cfg(target_os = "linux")]
9061fn remove_module_cgroup(placement: &subc_cgroup::Placement, module_id: &str) {
9062 match placement.remove_module(module_id) {
9063 Ok(()) => debug!(module_id, "removed module cgroup after process exit"),
9064 Err(error) => warn!(
9065 module_id,
9066 error = %error,
9067 "could not remove module cgroup after process exit; continuing teardown"
9068 ),
9069 }
9070}
9071
9072#[cfg(target_os = "linux")]
9073async fn cleanup_reaped_cgroup(placement: &subc_cgroup::Placement, module_id: &str) {
9074 if matches!(
9078 subc_cgroup::kill_module(Some(placement), module_id),
9079 subc_cgroup::KillOutcome::Killed
9080 ) {
9081 if let Ok(path) = placement.module_path(module_id) {
9082 while std::fs::read_to_string(path.join("cgroup.events"))
9083 .ok()
9084 .is_some_and(|events| events.lines().any(|line| line == "populated 1"))
9085 {
9086 sleep(Duration::from_millis(1)).await;
9087 }
9088 }
9089 }
9090 remove_module_cgroup(placement, module_id);
9091}
9092
9093#[cfg(target_os = "linux")]
9094fn apply_cgroup_placement(
9095 command: &mut Command,
9096 spec: &ModuleSpec,
9097 path: &std::path::Path,
9098) -> Result<(), SuperviseError> {
9099 subc_cgroup::apply(command, path).map_err(|source| SuperviseError::Cgroup {
9100 module_id: spec.module_id.clone(),
9101 source,
9102 })
9103}
9104
9105fn capture_retention(spec: &ModuleSpec) -> Retention {
9106 let defaults = Retention::default();
9107 let value = |name: &str| {
9108 spec.env
9109 .iter()
9110 .rev()
9111 .find_map(|(key, value)| (key == name).then_some(value.as_str()))
9112 };
9113 Retention {
9114 max_file_mb: value(CAPTURE_MAX_FILE_MB_ENV)
9115 .and_then(|value| value.parse().ok())
9116 .unwrap_or(defaults.max_file_mb),
9117 keep: value(CAPTURE_KEEP_ENV)
9118 .and_then(|value| value.parse().ok())
9119 .unwrap_or(defaults.keep),
9120 max_age_days: value(CAPTURE_MAX_AGE_DAYS_ENV)
9121 .and_then(|value| value.parse().ok())
9122 .unwrap_or(defaults.max_age_days),
9123 }
9124}
9125
9126fn generate_launch_nonce() -> Result<String, SuperviseError> {
9129 let mut bytes = [0u8; 32];
9130 getrandom::getrandom(&mut bytes).map_err(|source| SuperviseError::LaunchNonce {
9131 reason: source.to_string(),
9132 })?;
9133 let mut hex = String::with_capacity(64);
9134 for b in bytes {
9135 use std::fmt::Write;
9136 let _ = write!(hex, "{b:02x}");
9137 }
9138 Ok(hex)
9139}
9140
9141fn constant_time_eq(a: &[u8], b: &[u8]) -> bool {
9144 if a.len() != b.len() {
9145 return false;
9146 }
9147 let mut diff = 0u8;
9148 for (x, y) in a.iter().zip(b.iter()) {
9149 diff |= x ^ y;
9150 }
9151 diff == 0
9152}
9153
9154#[cfg(not(windows))]
9158fn observe_spawned_image(pid: u32) -> Option<subc_os::Observation> {
9159 let process = subc_os::Process::open(pid).ok().flatten()?;
9160 #[cfg(target_os = "linux")]
9161 return sample_after_linux_exec(
9162 || {
9163 let stat = std::fs::read_to_string(format!("/proc/{pid}/stat")).ok()?;
9164 let committed = linux_exec_committed(&stat);
9165 #[cfg(test)]
9166 spawn_image_tests::report_exec_state(committed);
9167 committed
9168 },
9169 || process.observe(),
9170 );
9171 #[cfg(not(target_os = "linux"))]
9172 process.observe()
9173}
9174
9175#[cfg(target_os = "linux")]
9179fn linux_exec_committed(stat: &str) -> Option<bool> {
9180 const PF_FORKNOEXEC: u64 = 0x40;
9181 let flags = stat.rsplit_once(')')?.1.split_whitespace().nth(6)?;
9182 Some(flags.parse::<u64>().ok()? & PF_FORKNOEXEC == 0)
9183}
9184
9185#[cfg(target_os = "linux")]
9186fn sample_after_linux_exec(
9187 mut exec_committed: impl FnMut() -> Option<bool>,
9188 observe: impl FnOnce() -> Option<subc_os::Observation>,
9189) -> Option<subc_os::Observation> {
9190 let deadline = std::time::Instant::now() + Duration::from_secs(1);
9191 while !exec_committed()? {
9192 if std::time::Instant::now() >= deadline {
9196 return None;
9197 }
9198 std::thread::sleep(Duration::from_micros(50));
9199 }
9200 observe()
9201}
9202
9203#[cfg(all(test, target_os = "linux"))]
9204mod spawn_image_tests {
9205 use super::*;
9206 use std::cell::{Cell, RefCell};
9207
9208 enum ExecEvent {
9209 Uncommitted,
9210 Image(Option<subc_os::Observation>),
9211 }
9212
9213 thread_local! {
9214 static EXEC_EVENTS: RefCell<Option<std::sync::mpsc::Sender<ExecEvent>>> = const { RefCell::new(None) };
9215 }
9216
9217 pub(super) fn report_exec_state(committed: Option<bool>) {
9218 if committed == Some(false) {
9219 EXEC_EVENTS.with(|events| {
9220 if let Some(events) = events.borrow_mut().take() {
9221 let _ = events.send(ExecEvent::Uncommitted);
9222 }
9223 });
9224 }
9225 }
9226
9227 #[test]
9228 fn linux_spawn_observation_waits_for_a_paused_fork_to_exec() {
9229 use std::io::{Read, Write};
9230 use std::os::fd::AsRawFd;
9231
9232 let descriptor = std::fs::File::open("/dev/null").unwrap();
9233 let mut command = std::process::Command::new("/bin/sleep");
9234 command.arg("30");
9235 let (mut ready, mut release) =
9236 subc_os::fork_exec_test::pause_before_exec(&mut command, descriptor.as_raw_fd())
9237 .unwrap();
9238 let spawn = std::thread::spawn(move || command.spawn());
9239 let mut message = [0u8; 8];
9240 ready.read_exact(&mut message).unwrap();
9241 let pid = i32::from_ne_bytes(message[..4].try_into().unwrap()) as u32;
9242 let before = std::fs::read_to_string(format!("/proc/{pid}/stat"));
9243 let inherited = subc_os::Process::open(pid).unwrap().unwrap().observe();
9244 let (events, received) = std::sync::mpsc::channel();
9245 let observer = std::thread::spawn(move || {
9246 EXEC_EVENTS.with(|slot| *slot.borrow_mut() = Some(events.clone()));
9247 let image = observe_spawned_image(pid);
9248 let _ = events.send(ExecEvent::Image(image));
9249 });
9250 let first = received.recv_timeout(Duration::from_secs(5));
9254 release.write_all(&[1]).unwrap();
9255 let mut child = spawn.join().unwrap().unwrap();
9256 let after = match first {
9257 Ok(ExecEvent::Uncommitted) => match received.recv_timeout(Duration::from_secs(5)) {
9258 Ok(ExecEvent::Image(image)) => image,
9259 _ => None,
9260 },
9261 Ok(ExecEvent::Image(image)) => image,
9262 Err(_) => None,
9263 };
9264 observer.join().unwrap();
9265 child.kill().unwrap();
9266 child.wait().unwrap();
9267
9268 assert_eq!(linux_exec_committed(&before.unwrap()), Some(false));
9269 assert_eq!(
9270 inherited.unwrap().executable,
9271 subc_os::file_identity(&std::env::current_exe().unwrap())
9272 );
9273 assert_eq!(
9274 after.unwrap().executable,
9275 subc_os::file_identity(std::path::Path::new("/bin/sleep"))
9276 );
9277 }
9278
9279 #[test]
9280 fn spawned_image_is_not_sampled_until_linux_exec_commits() {
9281 let committed = Cell::new(false);
9282 let inherited = subc_os::Observation {
9283 start_time: 23,
9284 executable: Some(subc_os::FileIdentity {
9285 device: 29,
9286 inode: 31,
9287 }),
9288 };
9289 let module = subc_os::Observation {
9290 start_time: 23,
9291 executable: Some(subc_os::FileIdentity {
9292 device: 29,
9293 inode: 37,
9294 }),
9295 };
9296 let polls = Cell::new(0);
9297 let image = sample_after_linux_exec(
9298 || {
9299 polls.set(polls.get() + 1);
9300 committed.set(polls.get() == 2);
9303 Some(committed.get())
9304 },
9305 || Some(if committed.get() { module } else { inherited }),
9306 );
9307 assert_eq!(image, Some(module));
9308 assert_eq!(polls.get(), 2);
9309 }
9310
9311 #[test]
9312 fn linux_exec_flag_is_read_from_flags_after_the_last_parenthesis() {
9313 assert_eq!(
9314 linux_exec_committed("17 (a ) b) R 1 2 3 4 5 4194368"),
9315 Some(false)
9316 );
9317 assert_eq!(
9318 linux_exec_committed("17 (a ) b) R 1 2 3 4 5 4194304"),
9319 Some(true)
9320 );
9321 assert_eq!(linux_exec_committed("17 (a) R 1 2 3 4 5 invalid"), None);
9322 assert_eq!(linux_exec_committed("17 (a) R"), None);
9323 }
9324
9325 #[test]
9326 fn unreadable_exec_state_never_samples_an_inherited_image() {
9327 assert_eq!(
9328 sample_after_linux_exec(
9329 || None,
9330 || panic!("must not sample before exec confirmation")
9331 ),
9332 None
9333 );
9334 }
9335
9336 #[test]
9337 fn uncommitted_exec_times_out_without_sampling_an_inherited_image() {
9338 assert_eq!(
9339 sample_after_linux_exec(
9340 || Some(false),
9341 || panic!("must not sample after an exec timeout")
9342 ),
9343 None
9344 );
9345 }
9346}
9347
9348#[cfg(windows)]
9351fn windows_spawn_capture(spec: &ModuleSpec) -> io::Result<subc_os::ExecutableCapture> {
9352 let mut program = spec.program.clone();
9353 if program.is_absolute() || program.components().count() > 1 {
9354 match subc_os::ExecutableCapture::open(&program) {
9357 Ok(capture) => return Ok(capture),
9358 Err(error)
9359 if error.kind() == io::ErrorKind::NotFound && program.extension().is_none() => {}
9360 Err(error) => return Err(error),
9361 }
9362 program.set_extension("exe");
9363 return subc_os::ExecutableCapture::open(&program);
9364 }
9365 if program.extension().is_none() {
9366 program.set_extension("exe");
9367 }
9368 let custom_path = spec
9369 .env
9370 .iter()
9371 .rev()
9372 .find(|(key, _)| key.eq_ignore_ascii_case("PATH"));
9373 let mut directories = Vec::new();
9374 if let Some(parent) = std::env::current_exe()?.parent() {
9377 directories.push(parent.to_path_buf());
9378 }
9379 directories.push(std::env::current_dir()?);
9380 if let Some(windows) = std::env::var_os("SystemRoot") {
9381 let windows = PathBuf::from(windows);
9382 directories.push(windows.join("System32"));
9383 directories.push(windows.clone());
9384 }
9385 let path = custom_path
9386 .map(|(_, value)| std::ffi::OsString::from(value))
9387 .or_else(|| std::env::var_os("PATH"));
9388 if let Some(path) = path {
9389 directories.extend(std::env::split_paths(&path));
9390 }
9391 for directory in directories {
9392 match subc_os::ExecutableCapture::open(&directory.join(&program)) {
9393 Ok(capture) => return Ok(capture),
9394 Err(error) if error.kind() == io::ErrorKind::NotFound => continue,
9395 Err(error) => return Err(error),
9396 }
9397 }
9398 Err(io::Error::new(
9399 io::ErrorKind::NotFound,
9400 "resolved executable not found",
9401 ))
9402}
9403
9404fn spawn_and_mark_running(
9405 spec: &ModuleSpec,
9406 runtime: &SupervisorRuntimeConfig,
9407 snapshot: &SharedSnapshot,
9408) -> Result<SupervisedChild, SuperviseError> {
9409 let activity_spawn = runtime.begin_activity_spawn(&spec.module_id);
9410 let child = spawn_child(
9411 spec,
9412 runtime.connection_file_path.as_deref(),
9413 runtime.supervisor_handle.as_ref(),
9414 &runtime.stderr_ring,
9415 runtime.capture_logs_dir.as_deref(),
9416 &runtime.child_roster,
9417 #[cfg(target_os = "linux")]
9418 runtime.cgroup_placement.as_ref(),
9419 )?;
9420 set_running(snapshot, &child, &spec.module_id, &runtime.spawn_events)?;
9421 if let Some(spawn) = activity_spawn {
9422 spawn.complete(snapshot);
9423 }
9424 Ok(child)
9425}
9426
9427enum RegistrationWaitOutcome {
9428 Registered,
9429 Exited(ExitReport),
9430 TimedOut,
9431}
9432
9433struct ReloadRegistrationFailure {
9434 exit_report: ExitReport,
9435 reason: String,
9436}
9437
9438#[derive(Debug, Clone, Copy, PartialEq, Eq)]
9439enum BusyGaugeObservation {
9440 Quiescent,
9441 Busy,
9442 Omitted,
9443}
9444
9445fn busy_gauge_observation(metrics: Option<&Value>, gauges: &[String]) -> BusyGaugeObservation {
9446 let Some(metrics) = metrics.and_then(Value::as_object) else {
9447 return BusyGaugeObservation::Omitted;
9448 };
9449 let mut sum = 0u128;
9450 for gauge in gauges {
9451 let Some(value) = metrics.get(gauge) else {
9452 return BusyGaugeObservation::Omitted;
9453 };
9454 let Some(value) = value.as_u64() else {
9455 return BusyGaugeObservation::Busy;
9456 };
9457 sum = sum.saturating_add(u128::from(value));
9458 }
9459 if sum == 0 {
9460 BusyGaugeObservation::Quiescent
9461 } else {
9462 BusyGaugeObservation::Busy
9463 }
9464}
9465
9466fn declared_busy_gauges(
9467 registry: &Registry,
9468 module_id: &str,
9469) -> Result<Vec<String>, SuperviseError> {
9470 busy_gauges_of(
9471 registry
9472 .get_module(module_id)
9473 .map_err(SuperviseError::Registry)?,
9474 )
9475}
9476
9477fn declared_busy_gauges_for_connection(
9481 registry: &Registry,
9482 connection_id: ConnectionId,
9483) -> Result<Vec<String>, SuperviseError> {
9484 busy_gauges_of(
9485 registry
9486 .get_module_by_connection(connection_id)
9487 .map_err(SuperviseError::Registry)?,
9488 )
9489}
9490
9491fn busy_gauges_of(
9492 registration: Option<crate::registry::ModuleRegistration>,
9493) -> Result<Vec<String>, SuperviseError> {
9494 let Some(registration) = registration else {
9495 return Ok(Vec::new());
9496 };
9497 let Some(self_signals) = registration.manifest.self_signals else {
9498 return Ok(Vec::new());
9499 };
9500
9501 let mut gauges = Vec::new();
9502 for declaration in self_signals {
9503 if declaration.kind != SelfSignalKind::Busy {
9504 continue;
9505 }
9506 match declaration.anchored_to {
9507 SignalAnchor::HealthGauges { gauges: declared } if !declared.is_empty() => {
9508 gauges.extend(declared)
9509 }
9510 _ => {
9511 gauges.push(String::new());
9514 }
9515 }
9516 }
9517 Ok(gauges)
9518}
9519
9520async fn wait_for_forwarding_quiescence(
9527 forwarding: &ForwardingTable,
9528 module_id: &str,
9529 runtime: &SupervisorRuntimeConfig,
9530 endpoint: crate::ModuleEndpointId,
9531 deadline: Instant,
9532 busy_gauges: &[String],
9533 scope: DrainScope,
9534) -> Result<bool, SuperviseError> {
9535 let mut gauges_quiescent = busy_gauges.is_empty();
9536 let mut next_probe_at = Instant::now();
9537 let mut omission_counted = false;
9538
9539 loop {
9540 let now = Instant::now();
9541 if !busy_gauges.is_empty() && now >= next_probe_at && now < deadline {
9542 let report = match scope {
9543 DrainScope::Active => probe_module_health(module_id, runtime, Some(deadline)).await,
9544 DrainScope::Endpoint(endpoint) => {
9545 probe_endpoint_health(endpoint, runtime, Some(deadline)).await
9546 }
9547 };
9548 gauges_quiescent = match report {
9549 Ok(report) => match busy_gauge_observation(report.metrics.as_ref(), busy_gauges) {
9550 BusyGaugeObservation::Quiescent => true,
9551 BusyGaugeObservation::Busy => false,
9552 BusyGaugeObservation::Omitted => {
9553 if !omission_counted {
9554 forwarding
9555 .counters()
9556 .increment_drains_with_undeclared_gauge();
9557 omission_counted = true;
9558 }
9559 false
9560 }
9561 },
9562 Err(err) => {
9563 warn!(
9564 module_id,
9565 error = %err,
9566 "drain health.check did not produce declared busy gauges; treating module as busy"
9567 );
9568 false
9569 }
9570 };
9571 next_probe_at = Instant::now() + drain_busy_reprobe_interval();
9574 }
9575
9576 let in_flight = forwarding
9577 .endpoint_in_flight_count(endpoint)
9578 .map_err(SuperviseError::Forwarding)?;
9579 if in_flight == 0 && gauges_quiescent {
9580 return Ok(true);
9581 }
9582
9583 let now = Instant::now();
9584 if now >= deadline {
9585 return Ok(false);
9586 }
9587 let mut wait = deadline
9588 .saturating_duration_since(now)
9589 .min(REGISTRY_RELEASE_POLL);
9590 if !busy_gauges.is_empty() {
9591 wait = wait.min(next_probe_at.saturating_duration_since(now));
9592 }
9593 sleep(wait).await;
9594 }
9595}
9596
9597fn drain_busy_reprobe_interval() -> Duration {
9601 DRAIN_BUSY_REPROBE_INTERVAL.max(REGISTRY_RELEASE_POLL)
9602}
9603
9604fn drained_after_quiescence_wait(wait_result: &Result<bool, SuperviseError>) -> bool {
9612 match wait_result {
9613 Ok(drained) => *drained,
9614 Err(_) => false,
9615 }
9616}
9617
9618fn send_route_goodbyes(forwarding: &ForwardingTable, released_routes: Vec<GoodbyeTarget>) {
9619 for released in released_routes {
9620 let frame = match Frame::build_with_version(
9621 released.negotiated_ver,
9622 FrameType::Goodbye,
9623 control_flags(),
9624 released.channel,
9625 released.epoch,
9626 0,
9627 Vec::new(),
9628 ) {
9629 Ok(frame) => frame,
9630 Err(err) => {
9631 warn!(
9632 route_channel = released.channel,
9633 error = %err,
9634 "failed to build supervisor drain route GOODBYE frame"
9635 );
9636 continue;
9637 }
9638 };
9639 if !released.close_on_delivery_failure() {
9640 crate::forwarding::send_module_route_goodbye(
9641 &forwarding.counters(),
9642 &released.sink,
9643 frame,
9644 released.module_id.as_deref(),
9645 "supervisor drain",
9646 );
9647 continue;
9648 }
9649 if let Err(err) = released.sink.try_send(frame) {
9650 warn!(
9651 target_connection_id = released.connection_id.get(),
9652 route_channel = released.channel,
9653 error = %err,
9654 "supervisor drain route GOODBYE was not delivered to client; closing target connection"
9655 );
9656 let _ = forwarding.escalate_client_delivery_failure(
9657 released.connection_id,
9658 released.channel,
9659 released.epoch,
9660 CloseReason::new(
9661 "route_goodbye_delivery_failed",
9662 format!(
9663 "failed to enqueue supervisor drain route GOODBYE for channel {}: {err}",
9664 released.channel
9665 ),
9666 ),
9667 crate::forwarding::UndeliveredFrame {
9668 module_id: released.module_id.as_deref(),
9669 sink: &released.sink,
9670 },
9671 );
9672 }
9673 }
9674}
9675
9676fn send_module_draining(
9677 module_id: &str,
9678 reason: RouteCloseReason,
9679 deadline_ms: u64,
9680 target: &ModuleDrainTarget,
9681) {
9682 let body = match serde_json::to_vec(&ModuleControlCommand::Draining {
9683 reason,
9684 deadline_ms,
9685 }) {
9686 Ok(body) => body,
9687 Err(err) => {
9688 warn!(
9689 module_id,
9690 error = %err,
9691 "failed to encode module draining command"
9692 );
9693 return;
9694 }
9695 };
9696 let frame = match Frame::build_with_version(
9697 target.negotiated_ver,
9698 FrameType::Push,
9699 control_flags(),
9700 0,
9701 0,
9702 0,
9703 body,
9704 ) {
9705 Ok(frame) => frame,
9706 Err(err) => {
9707 warn!(
9708 module_id,
9709 error = %err,
9710 "failed to build module draining command frame"
9711 );
9712 return;
9713 }
9714 };
9715 if let Err(err) = target.sink.try_send(frame) {
9716 warn!(
9717 module_id,
9718 target_connection_id = target.endpoint.connection_id.get(),
9719 error = %err,
9720 "module draining command was not delivered to peer"
9721 );
9722 }
9723}
9724
9725fn module_goodbye_frame(module_id: &str, negotiated_ver: u8) -> Option<Frame> {
9727 match Frame::build_with_version(
9728 negotiated_ver,
9729 FrameType::Goodbye,
9730 control_flags(),
9731 0,
9732 0,
9733 0,
9734 Vec::new(),
9735 ) {
9736 Ok(frame) => Some(frame),
9737 Err(err) => {
9738 warn!(
9739 module_id,
9740 error = %err,
9741 "failed to build module GOODBYE frame"
9742 );
9743 None
9744 }
9745 }
9746}
9747
9748#[cfg(unix)]
9762async fn send_module_goodbyes_for_daemon_shutdown(
9763 forwarding: &Arc<ForwardingTable>,
9764 reason: &CloseReason,
9765 wait_for_flush: bool,
9766) {
9767 const GOODBYE_BUDGET: Duration = Duration::from_millis(500);
9768 let targets = match forwarding.module_connections() {
9769 Ok(targets) => targets,
9770 Err(err) => {
9771 warn!(error = %err, "could not list module connections for shutdown GOODBYE");
9772 return;
9773 }
9774 };
9775 let deadline = Instant::now() + GOODBYE_BUDGET;
9776 let mut sends = tokio::task::JoinSet::new();
9777 for target in targets {
9778 let Some(frame) = module_goodbye_frame(&target.module_id, target.negotiated_ver) else {
9779 continue;
9780 };
9781 if !wait_for_flush {
9782 if let Err(err) = target.sink.try_send(frame) {
9783 debug!(
9784 module_id = %target.module_id,
9785 error = %err,
9786 "shutdown module GOODBYE was not queued"
9787 );
9788 }
9789 continue;
9790 }
9791 let forwarding = Arc::clone(forwarding);
9792 let reason = reason.clone();
9793 sends.spawn(async move {
9794 match timeout_at(deadline, target.sink.send_flushed(frame)).await {
9795 Ok(Ok(())) => {}
9796 Ok(Err(err)) => debug!(
9797 module_id = %target.module_id,
9798 error = %err,
9799 "module connection closed before its shutdown GOODBYE was written"
9800 ),
9801 Err(_) => warn!(
9802 module_id = %target.module_id,
9803 budget = ?GOODBYE_BUDGET,
9804 "shutdown module GOODBYE was not written within its budget; closing anyway"
9805 ),
9806 }
9807 forwarding.request_connection_close(target.endpoint.connection_id, reason);
9808 });
9809 }
9810 while sends.join_next().await.is_some() {}
9812}
9813
9814fn send_module_goodbye(module_id: &str, forwarding: &ForwardingTable, target: &ModuleDrainTarget) {
9815 let Some(frame) = module_goodbye_frame(module_id, target.negotiated_ver) else {
9816 return;
9817 };
9818 if let Err(err) = target.sink.try_send(frame) {
9819 warn!(
9820 module_id,
9821 target_connection_id = target.endpoint.connection_id.get(),
9822 error = %err,
9823 "supervisor drain module GOODBYE was not delivered to peer; closing module connection"
9824 );
9825 forwarding.request_connection_close(
9826 target.endpoint.connection_id,
9827 CloseReason::new(
9828 "module_goodbye_delivery_failed",
9829 format!("failed to enqueue supervisor drain module GOODBYE for module '{module_id}': {err}"),
9830 ),
9831 );
9832 }
9833}
9834
9835#[derive(Clone, Copy)]
9836struct ForwardingDrainContext<'a> {
9837 spec: &'a ModuleSpec,
9838 runtime: &'a SupervisorRuntimeConfig,
9839 registry: &'a Registry,
9840 scope: DrainScope,
9841}
9842
9843#[derive(Debug, Clone, Copy, PartialEq, Eq)]
9845enum DrainScope {
9846 Active,
9849 Endpoint(crate::ModuleEndpointId),
9854}
9855
9856#[derive(Debug, Clone, Copy, PartialEq, Eq)]
9864enum StopNotice {
9865 SentOverConnection,
9868 NoConnection,
9872 NotSent,
9876}
9877
9878async fn begin_forwarding_drain(
9879 spec: &ModuleSpec,
9880 runtime: &SupervisorRuntimeConfig,
9881 registry: &Registry,
9882 snapshot: &SharedSnapshot,
9883 enabled: Option<bool>,
9884 reason: RouteCloseReason,
9885) -> Result<StopNotice, SuperviseError> {
9886 let Some(forwarding) = runtime.forwarding.as_ref() else {
9887 return Err(SuperviseError::ReloadUnavailable {
9888 module_id: spec.module_id.clone(),
9889 reason: "supervisor was not configured with a forwarding table".to_string(),
9890 });
9891 };
9892
9893 begin_forwarding_drain_with(
9894 forwarding,
9895 ForwardingDrainContext {
9896 spec,
9897 runtime,
9898 registry,
9899 scope: DrainScope::Active,
9900 },
9901 snapshot,
9902 enabled,
9903 reason,
9904 runtime.drain_timeout,
9905 )
9906 .await
9907}
9908
9909async fn begin_forwarding_drain_if_configured(
9910 spec: &ModuleSpec,
9911 runtime: &SupervisorRuntimeConfig,
9912 registry: &Registry,
9913 snapshot: &SharedSnapshot,
9914 enabled: Option<bool>,
9915 reason: RouteCloseReason,
9916) -> Result<StopNotice, SuperviseError> {
9917 begin_forwarding_drain_with_timeout(
9918 spec,
9919 runtime,
9920 registry,
9921 snapshot,
9922 enabled,
9923 reason,
9924 runtime.drain_timeout,
9925 )
9926 .await
9927}
9928
9929async fn begin_forwarding_drain_with_timeout(
9933 spec: &ModuleSpec,
9934 runtime: &SupervisorRuntimeConfig,
9935 registry: &Registry,
9936 snapshot: &SharedSnapshot,
9937 enabled: Option<bool>,
9938 reason: RouteCloseReason,
9939 drain_timeout: Duration,
9940) -> Result<StopNotice, SuperviseError> {
9941 let Some(forwarding) = runtime.forwarding.as_ref() else {
9942 return Ok(StopNotice::NotSent);
9943 };
9944
9945 begin_forwarding_drain_with(
9946 forwarding,
9947 ForwardingDrainContext {
9948 spec,
9949 runtime,
9950 registry,
9951 scope: DrainScope::Active,
9952 },
9953 snapshot,
9954 enabled,
9955 reason,
9956 drain_timeout,
9957 )
9958 .await
9959}
9960
9961async fn begin_forwarding_drain_with(
9962 forwarding: &ForwardingTable,
9963 context: ForwardingDrainContext<'_>,
9964 snapshot: &SharedSnapshot,
9965 enabled: Option<bool>,
9966 reason: RouteCloseReason,
9967 drain_timeout: Duration,
9968) -> Result<StopNotice, SuperviseError> {
9969 let ForwardingDrainContext {
9970 spec,
9971 runtime,
9972 registry,
9973 scope,
9974 } = context;
9975 debug_assert_ne!(reason, RouteCloseReason::Crash);
9976 let terminal = matches!(reason, RouteCloseReason::Disable);
9977 let drain_started_at = Instant::now();
9978 let drain_deadline = drain_started_at + drain_timeout;
9979 let deadline_ms =
9980 unix_ms_now().saturating_add(u64::try_from(drain_timeout.as_millis()).unwrap_or(u64::MAX));
9981 let busy_gauges = match scope {
9982 DrainScope::Active => declared_busy_gauges(registry, &spec.module_id)?,
9983 DrainScope::Endpoint(endpoint) => {
9984 declared_busy_gauges_for_connection(registry, endpoint.connection_id)?
9985 }
9986 };
9987
9988 let gate_started = Instant::now();
9991 let drain_target = match scope {
9992 DrainScope::Active => forwarding.begin_module_drain(&spec.module_id, reason),
9993 DrainScope::Endpoint(endpoint) => forwarding.begin_endpoint_drain(endpoint, reason),
9994 }
9995 .map_err(SuperviseError::Forwarding)?;
9996 info!(
10001 module_id = %spec.module_id,
10002 ?reason,
10003 gate_ms = u64::try_from(gate_started.elapsed().as_millis()).unwrap_or(u64::MAX),
10004 connected = drain_target.is_some(),
10005 "module drain began; route admission closed"
10006 );
10007 if scope == DrainScope::Active {
10008 update_snapshot(snapshot, Some(&spec.module_id), |state| {
10009 state.state = ModuleState::Draining;
10010 state.draining_to_replace =
10011 matches!(reason, RouteCloseReason::Restart | RouteCloseReason::Reload);
10012 if let Some(enabled) = enabled {
10013 state.enabled = enabled;
10014 }
10015 })?;
10016 }
10017
10018 let Some(target) = drain_target.as_ref() else {
10019 return Ok(StopNotice::NoConnection);
10023 };
10024 {
10025 send_module_draining(&spec.module_id, reason, deadline_ms, target);
10026 #[cfg(feature = "test-support")]
10027 forwarding.wait_before_route_closing_for_test().await;
10028 let routes = forwarding
10029 .endpoint_routes(target.endpoint)
10030 .map_err(SuperviseError::Forwarding)?;
10031 let routes_notified = routes.len();
10032 crate::control::send_route_control_pushes(
10033 forwarding,
10034 routes.clone(),
10035 ClientControlPush::RouteClosing {
10036 module_id: spec.module_id.clone(),
10037 channels: Vec::new(),
10038 reason,
10039 },
10040 );
10041 send_route_goodbyes(forwarding, target.abandoned_bindings.clone());
10042
10043 let wait_result = wait_for_forwarding_quiescence(
10049 forwarding,
10050 &spec.module_id,
10051 runtime,
10052 target.endpoint,
10053 drain_deadline,
10054 &busy_gauges,
10055 scope,
10056 )
10057 .await;
10058 let drained = drained_after_quiescence_wait(&wait_result);
10059 if let Err(err) = &wait_result {
10060 error!(
10061 module_id = %spec.module_id,
10062 ?reason,
10063 error = %err,
10064 "forwarding quiescence wait failed after route.closing; forcing route.closed(drained: false) so the client is not left waiting on an unfulfilled promise"
10065 );
10066 } else if !drained {
10067 let holdouts = forwarding
10073 .endpoint_drain_holdouts(target.endpoint)
10074 .unwrap_or_default();
10075 warn!(
10076 module_id = %spec.module_id,
10077 waited = ?drain_timeout,
10078 ?reason,
10079 held_requests = holdouts.requests,
10080 held_routes = holdouts.routes,
10081 total_routes = holdouts.total_routes,
10082 top_connections = ?holdouts.top_connections,
10083 held = %holdouts
10086 .held
10087 .iter()
10088 .map(|(channel, corr)| format!("{channel}:{corr}"))
10089 .collect::<Vec<_>>()
10090 .join(","),
10091 "route drain timed out before request quiescence; forcing teardown"
10092 );
10093 }
10094 crate::control::send_route_control_pushes(
10095 forwarding,
10096 routes,
10097 ClientControlPush::RouteClosed {
10098 module_id: spec.module_id.clone(),
10099 channels: Vec::new(),
10100 reason,
10101 drained,
10102 abandoned: target.abandoned_bindings.len() as u32,
10103 excluded_subscriptions: target.excluded_subscriptions,
10104 terminal: Some(terminal),
10105 },
10106 );
10107 wait_result?;
10108
10109 let released_routes = match forwarding.release_module_endpoint_routes(target.endpoint) {
10115 Ok(routes) => routes,
10116 Err(err) => {
10117 warn!(
10118 module_id = %spec.module_id,
10119 ?reason,
10120 error = %err,
10121 "failed to release module endpoint routes after route.closed; module GOODBYE will still be sent"
10122 );
10123 send_module_goodbye(&spec.module_id, forwarding, target);
10124 return Err(SuperviseError::Forwarding(err));
10125 }
10126 };
10127 let route_goodbye_count = released_routes.len();
10128 send_route_goodbyes(forwarding, released_routes);
10129 send_module_goodbye(&spec.module_id, forwarding, target);
10130
10131 info!(
10137 module_id = %spec.module_id,
10138 ?reason,
10139 routes_notified,
10140 route_goodbyes = route_goodbye_count,
10141 abandoned_reservations = target.abandoned_bindings.len(),
10142 excluded_subscriptions = target.excluded_subscriptions,
10143 drained,
10144 "module drain complete; consumers notified via route.closing/route.closed pushes and per-route GOODBYE frames"
10145 );
10146 }
10147
10148 Ok(StopNotice::SentOverConnection)
10149}
10150
10151async fn wait_for_registration_after_reload(
10154 registry: &Registry,
10155 module_id: &str,
10156 snapshot: &SharedSnapshot,
10157 child: &mut SupervisedChild,
10158 wait: Duration,
10159) -> Result<RegistrationWaitOutcome, SuperviseError> {
10160 wait_for_slot_registration(
10161 registry,
10162 crate::registry::RegistrationSlot::Active(module_id),
10163 module_id,
10164 snapshot,
10165 child,
10166 wait,
10167 )
10168 .await
10169}
10170
10171async fn wait_for_slot_registration(
10179 registry: &Registry,
10180 slot: crate::registry::RegistrationSlot<'_>,
10181 module_id: &str,
10182 snapshot: &SharedSnapshot,
10183 child: &mut SupervisedChild,
10184 wait: Duration,
10185) -> Result<RegistrationWaitOutcome, SuperviseError> {
10186 let deadline = Instant::now() + wait;
10187 loop {
10188 if registry
10189 .registration(slot)
10190 .map_err(SuperviseError::Registry)?
10191 .is_some()
10192 {
10193 return Ok(RegistrationWaitOutcome::Registered);
10194 }
10195
10196 let now = Instant::now();
10197 if now >= deadline {
10198 return Ok(RegistrationWaitOutcome::TimedOut);
10199 }
10200 let remaining = deadline.saturating_duration_since(now);
10201 let poll = remaining.min(REGISTRY_RELEASE_POLL);
10202
10203 tokio::select! {
10204 wait_result = child.wait() => {
10205 let status = wait_result.map_err(|source| SuperviseError::Wait {
10206 module_id: module_id.to_string(),
10207 source,
10208 })?;
10209 return Ok(RegistrationWaitOutcome::Exited(classify_reaped_child_exit(
10210 snapshot,
10211 child,
10212 &status,
10213 )));
10214 }
10215 _ = sleep(poll) => {}
10216 }
10217 }
10218}
10219
10220fn registration_failure_exit_report(mut exit_report: ExitReport) -> ExitReport {
10221 if exit_report.kind != ExitKind::DeliberateSeverance {
10224 exit_report.kind = ExitKind::Crash;
10225 }
10226 exit_report
10227}
10228
10229async fn handle_reload_child_registration_failure(
10230 spec: &ModuleSpec,
10231 runtime: &SupervisorRuntimeConfig,
10232 registry: &Registry,
10233 process_liveness: &SupervisorProcessLiveness,
10234 snapshot: &SharedSnapshot,
10235 _child: &mut Option<SupervisedChild>,
10236 failure: ReloadRegistrationFailure,
10237) -> Result<(), SuperviseError> {
10238 let ReloadRegistrationFailure {
10239 exit_report,
10240 reason,
10241 } = failure;
10242 match on_child_exit(
10243 spec,
10244 runtime.restart_policy,
10245 registry,
10246 snapshot,
10247 &runtime.terminal_ring,
10248 &runtime.spawn_events,
10249 &runtime.child_roster,
10250 exit_report,
10251 )
10252 .await
10253 {
10254 NextAction::Stop {
10255 registration_released,
10256 } => {
10257 if registration_released {
10258 process_liveness.untrack_if_current(&spec.module_id, snapshot);
10259 }
10260 }
10261 NextAction::Restart { schedule } => {
10262 let delay = schedule.map_or(runtime.restart_policy.delay_for_restart(0), |schedule| {
10263 schedule.delay
10264 });
10265 if let Some(schedule) = schedule {
10266 log_crash_respawn(&spec.module_id, schedule);
10267 }
10268 schedule_respawn(
10269 runtime,
10270 snapshot,
10271 &spec.module_id,
10272 delay,
10273 RespawnKind::Spawn,
10274 )?;
10275 }
10276 }
10277 Err(SuperviseError::ReloadFailed {
10278 module_id: spec.module_id.clone(),
10279 reason,
10280 })
10281}
10282
10283async fn handle_reload_spawn_failure(
10284 spec: &ModuleSpec,
10285 runtime: &SupervisorRuntimeConfig,
10286 process_liveness: &SupervisorProcessLiveness,
10287 snapshot: &SharedSnapshot,
10288 _child: &mut Option<SupervisedChild>,
10289 reason: String,
10290) -> Result<(), SuperviseError> {
10291 let now = Instant::now();
10292 let mut schedule = None;
10293 update_snapshot(snapshot, Some(&spec.module_id), |state| {
10294 clear_current_process_facts(state);
10295 if state.enabled {
10296 schedule = state.next_crash_restart(&runtime.restart_policy, now);
10297 state.state = if schedule.is_some() {
10298 ModuleState::Restarting
10299 } else {
10300 ModuleState::Failed
10301 };
10302 } else {
10303 state.state = ModuleState::Disabled;
10304 }
10305 })?;
10306 if let Some(schedule) = schedule {
10307 schedule_respawn(
10308 runtime,
10309 snapshot,
10310 &spec.module_id,
10311 schedule.delay,
10312 RespawnKind::Spawn,
10313 )?;
10314 } else {
10315 process_liveness.untrack_if_current(&spec.module_id, snapshot);
10316 }
10317 Err(SuperviseError::ReloadFailed {
10318 module_id: spec.module_id.clone(),
10319 reason,
10320 })
10321}
10322
10323fn control_flags() -> Flags {
10324 Flags::new(false, Priority::Passive, false)
10325}
10326
10327#[allow(clippy::too_many_arguments)]
10328async fn drain_optional_child(
10329 module_id: &str,
10330 protocol: ModuleProtocol,
10331 stop_notice: StopNotice,
10332 registry: &Registry,
10333 forwarding: Option<&ForwardingTable>,
10334 snapshot: &SharedSnapshot,
10335 terminal_ring: &Arc<Mutex<TerminalRing>>,
10336 spawn_events: &SpawnEventFeed,
10337 child: &mut Option<SupervisedChild>,
10338 drain_timeout: Duration,
10339 final_state: ModuleState,
10340 enabled: Option<bool>,
10341) -> Result<(), SuperviseError> {
10342 if let Some(child) = child.take() {
10343 drain_child_to_state(
10344 module_id,
10345 protocol,
10346 stop_notice,
10347 registry,
10348 forwarding,
10349 snapshot,
10350 terminal_ring,
10351 spawn_events,
10352 child,
10353 drain_timeout,
10354 final_state,
10355 enabled,
10356 )
10357 .await
10358 } else {
10359 update_snapshot(snapshot, Some(module_id), |state| {
10360 state.state = final_state;
10361 if let Some(enabled) = enabled {
10362 state.enabled = enabled;
10363 }
10364 clear_current_process_facts(state);
10365 })?;
10366 release_dead_registration(registry, forwarding, snapshot, module_id).await
10367 }
10368}
10369
10370#[allow(clippy::too_many_arguments)]
10371async fn drain_child_to_state(
10372 module_id: &str,
10373 _protocol: ModuleProtocol,
10374 stop_notice: StopNotice,
10375 registry: &Registry,
10376 forwarding: Option<&ForwardingTable>,
10377 snapshot: &SharedSnapshot,
10378 terminal_ring: &Arc<Mutex<TerminalRing>>,
10379 spawn_events: &SpawnEventFeed,
10380 mut child: SupervisedChild,
10381 drain_timeout: Duration,
10382 final_state: ModuleState,
10383 enabled: Option<bool>,
10384) -> Result<(), SuperviseError> {
10385 let protocol = child.protocol;
10386 update_snapshot(snapshot, Some(module_id), |state| {
10387 state.state = ModuleState::Draining;
10388 state.draining_to_replace = final_state == ModuleState::Restarting;
10389 if let Some(enabled) = enabled {
10390 state.enabled = enabled;
10391 }
10392 })?;
10393
10394 if stop_notice != StopNotice::SentOverConnection {
10405 if protocol == ModuleProtocol::Subc && stop_notice == StopNotice::NoConnection {
10406 info!(
10407 module_id,
10408 pid = child.pid,
10409 budget_ms = u64::try_from(drain_timeout.as_millis()).unwrap_or(u64::MAX),
10410 "module has no connection yet; requesting stop by signal"
10411 );
10412 }
10413 request_graceful_stop(module_id, &child);
10414 }
10415
10416 let exit_report = match timeout(drain_timeout, child.wait()).await {
10417 Ok(Ok(status)) => classify_reaped_child_exit(snapshot, &child, &status),
10418 Ok(Err(source)) => {
10419 fail_after_wait_error(
10420 module_id,
10421 snapshot,
10422 terminal_ring,
10423 spawn_events,
10424 format!("failed to wait during drain: {source}"),
10425 );
10426 return Err(SuperviseError::Wait {
10427 module_id: module_id.to_string(),
10428 source,
10429 });
10430 }
10431 Err(_) => {
10432 warn!(
10445 module_id,
10446 pid = child.pid,
10447 budget_ms = u64::try_from(drain_timeout.as_millis()).unwrap_or(u64::MAX),
10448 reason = ?final_state,
10449 ?stop_notice,
10450 "drain budget expired before the module exited; killing it"
10451 );
10452 child.start_kill().map_err(|source| {
10453 fail_after_wait_error(
10454 module_id,
10455 snapshot,
10456 terminal_ring,
10457 spawn_events,
10458 format!("failed to kill during drain: {source}"),
10459 );
10460 SuperviseError::Kill {
10461 module_id: module_id.to_string(),
10462 source,
10463 }
10464 })?;
10465 let status = child.wait().await.map_err(|source| {
10466 fail_after_wait_error(
10467 module_id,
10468 snapshot,
10469 terminal_ring,
10470 spawn_events,
10471 format!("failed to wait after drain kill: {source}"),
10472 );
10473 SuperviseError::Wait {
10474 module_id: module_id.to_string(),
10475 source,
10476 }
10477 })?;
10478 classify_reaped_child_exit(snapshot, &child, &status)
10479 }
10480 };
10481
10482 let detail = lock_snapshot(snapshot)?.drain_disposition_detail.take();
10483 publish_after_terminal_record(
10484 module_id,
10485 || {
10486 record_terminal_with_detail(
10487 module_id,
10488 terminal_ring,
10489 spawn_events,
10490 &exit_report,
10491 terminal_disposition(final_state),
10492 detail,
10493 )
10494 },
10495 || {
10496 update_snapshot(snapshot, Some(module_id), |state| {
10497 state.state = final_state;
10498 if let Some(enabled) = enabled {
10499 state.enabled = enabled;
10500 }
10501 clear_current_process_facts(state);
10502 state.last_exit = Some(exit_report.clone());
10503 if exit_report.kind == ExitKind::DeliberateSeverance {
10504 state.lifetime_restarts += 1;
10505 }
10506 })
10507 },
10508 )?;
10509 child.drain_stderr(module_id).await;
10510
10511 release_dead_registration(registry, forwarding, snapshot, module_id).await
10512}
10513
10514#[cfg(unix)]
10534fn request_graceful_stop(module_id: &str, child: &SupervisedChild) {
10535 let Some(pid) = child
10536 .id()
10537 .and_then(|pid| i32::try_from(pid).ok())
10538 .and_then(rustix::process::Pid::from_raw)
10539 else {
10540 debug!(
10541 module_id,
10542 "no pid to signal for teardown; falling through to the drain wait"
10543 );
10544 return;
10545 };
10546 match rustix::process::kill_process(pid, rustix::process::Signal::TERM) {
10547 Ok(()) => debug!(
10548 module_id,
10549 "sent SIGTERM to a module nothing else asked to stop"
10550 ),
10551 Err(err) => debug!(
10552 module_id,
10553 error = %err,
10554 "SIGTERM to module failed; the drain wait and kill still apply"
10555 ),
10556 }
10557}
10558
10559#[cfg(not(unix))]
10567fn request_graceful_stop(module_id: &str, _child: &SupervisedChild) {
10568 debug!(
10569 module_id,
10570 "no graceful stop signal exists on this platform; teardown of a module nothing asked to stop waits, then kills"
10571 );
10572}
10573
10574fn terminal_disposition(final_state: ModuleState) -> TerminalDisposition {
10575 match final_state {
10576 ModuleState::Stopped => TerminalDisposition::Stopped,
10577 ModuleState::Disabled => TerminalDisposition::Disabled,
10578 ModuleState::Restarting => TerminalDisposition::Restarting,
10579 ModuleState::Failed => TerminalDisposition::Failed,
10580 ModuleState::Starting
10581 | ModuleState::Running
10582 | ModuleState::Unresponsive
10583 | ModuleState::Draining => {
10584 unreachable!("terminal exits only finish in terminal or restarting states")
10585 }
10586 }
10587}
10588
10589async fn release_dead_registration(
10597 registry: &Registry,
10598 forwarding: Option<&ForwardingTable>,
10599 snapshot: &SharedSnapshot,
10600 module_id: &str,
10601) -> Result<(), SuperviseError> {
10602 let result = async {
10603 let registration = registry
10604 .get_module(module_id)
10605 .map_err(SuperviseError::Registry)?;
10606 match wait_for_registration_release(registry, module_id, REGISTRY_RELEASE_TIMEOUT).await {
10607 Ok(()) => return Ok(()),
10608 Err(SuperviseError::RegistrationStillActive { .. }) => {}
10609 Err(err) => return Err(err),
10610 }
10611 let pid = lock_snapshot(snapshot)?.reaped_pid;
10612 if let (Some(registration), Some(forwarding), Some(pid)) = (registration, forwarding, pid) {
10613 warn!(
10614 module_id,
10615 pid,
10616 connection_id = registration.connection_id.get(),
10617 "reaped module registration outlived release grace; closing dead connection"
10618 );
10619 forwarding.request_connection_close(
10620 registration.connection_id,
10621 CloseReason::new(
10622 "supervised_process_reaped",
10623 format!("module '{module_id}' pid {pid} exited"),
10624 ),
10625 );
10626 wait_for_slot_registration_release(
10627 registry,
10628 crate::registry::RegistrationSlot::Connection(registration.connection_id),
10629 REGISTRY_RELEASE_TIMEOUT,
10630 )
10631 .await?;
10632 }
10633 wait_for_registration_release(registry, module_id, Duration::ZERO).await
10634 }
10635 .await;
10636 if let Err(err) = &result {
10637 fail_snapshot(snapshot, Some(module_id), None);
10638 error!(module_id, error = %err, "registration release failed after child exit; module is failed and start can retry");
10639 }
10640 result
10641}
10642
10643async fn wait_for_registration_release(
10646 registry: &Registry,
10647 module_id: &str,
10648 wait: Duration,
10649) -> Result<(), SuperviseError> {
10650 wait_for_slot_registration_release(
10651 registry,
10652 crate::registry::RegistrationSlot::Active(module_id),
10653 wait,
10654 )
10655 .await
10656}
10657
10658async fn wait_for_slot_registration_release(
10666 registry: &Registry,
10667 slot: crate::registry::RegistrationSlot<'_>,
10668 wait: Duration,
10669) -> Result<(), SuperviseError> {
10670 let deadline = Instant::now() + wait;
10671 let mut release_events = registration_release_events().subscribe();
10672 let still_active = |registration: &crate::registry::ModuleRegistration| {
10673 SuperviseError::RegistrationStillActive {
10674 module_id: registration.manifest.module_id.clone(),
10675 waited: wait,
10676 }
10677 };
10678 loop {
10679 let _observed_generation = *release_events.borrow_and_update();
10680 let Some(registration) = registry
10681 .registration(slot)
10682 .map_err(SuperviseError::Registry)?
10683 else {
10684 return Ok(());
10685 };
10686
10687 let now = Instant::now();
10688 if now >= deadline {
10689 return Err(still_active(®istration));
10690 }
10691
10692 let remaining = deadline.saturating_duration_since(now);
10693 match timeout(remaining, release_events.changed()).await {
10694 Ok(Ok(())) | Ok(Err(_)) => {}
10695 Err(_) => return Err(still_active(®istration)),
10696 }
10697 }
10698}
10699
10700#[cfg(test)]
10701mod slot_registration_wait_tests {
10702 use super::*;
10703 use crate::registry::{ConnectionId, RegistrationSlot};
10704 use subc_protocol::manifest::ModuleManifest;
10705
10706 #[tokio::test]
10707 async fn enable_release_failure_is_failed_and_a_second_enable_retries() {
10708 let registry = Arc::new(Registry::default());
10709 let supervisor = Supervisor::new_for_test(Arc::clone(®istry), RestartPolicy::default());
10710 let runtime = supervisor.runtime_config();
10711 let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::disabled()));
10712 let spec = ModuleSpec {
10713 module_id: "enable-stale-registration".to_string(),
10714 program: PathBuf::from("/missing/enable-retry-test"),
10715 args: Vec::new(),
10716 env: Vec::new(),
10717 reserved: false,
10718 reserved_prefixes: Vec::new(),
10719 protocol: ModuleProtocol::Subc,
10720 overlap: Default::default(),
10721 };
10722 let connection = ConnectionId::new(90);
10723 registry
10724 .register_with_control_ops(
10725 ModuleManifest::builder(&spec.module_id, "0.1.0").build(),
10726 1,
10727 connection,
10728 Vec::new(),
10729 )
10730 .unwrap();
10731 let mut child = None;
10732 let err = set_child_enabled(
10733 &spec,
10734 &runtime,
10735 ®istry,
10736 &supervisor.process_liveness,
10737 &snapshot,
10738 &mut child,
10739 true,
10740 )
10741 .await
10742 .unwrap_err();
10743 assert!(matches!(
10744 err,
10745 SuperviseError::RegistrationStillActive { .. }
10746 ));
10747 assert_eq!(lock_snapshot(&snapshot).unwrap().state, ModuleState::Failed);
10748 assert!(child.is_none());
10749 registry.deregister_connection(connection).unwrap();
10750 let err = set_child_enabled(
10751 &spec,
10752 &runtime,
10753 ®istry,
10754 &supervisor.process_liveness,
10755 &snapshot,
10756 &mut child,
10757 true,
10758 )
10759 .await
10760 .unwrap_err();
10761 assert!(
10762 matches!(err, SuperviseError::Spawn { .. }),
10763 "second enable must attempt a spawn: {err}"
10764 );
10765 assert_eq!(lock_snapshot(&snapshot).unwrap().state, ModuleState::Failed);
10766 }
10767
10768 const INCUMBENT: u64 = 1;
10769 const CANDIDATE: u64 = 2;
10770
10771 fn swapped_registry() -> Arc<Registry> {
10772 let registry = Arc::new(Registry::default());
10773 let manifest = ModuleManifest::builder("m", "0.1.0").build();
10774 registry
10775 .register_with_control_ops(
10776 manifest.clone(),
10777 1,
10778 ConnectionId::new(INCUMBENT),
10779 Vec::new(),
10780 )
10781 .unwrap();
10782 registry
10783 .register_candidate_with_control_ops(
10784 manifest,
10785 1,
10786 ConnectionId::new(CANDIDATE),
10787 Vec::new(),
10788 )
10789 .unwrap();
10790 registry
10791 }
10792
10793 #[tokio::test]
10797 async fn incumbent_release_is_awaited_by_connection_not_by_module_id() {
10798 let registry = swapped_registry();
10799 registry.promote_candidate("m").unwrap().unwrap();
10800
10801 assert!(matches!(
10802 wait_for_registration_release(®istry, "m", Duration::from_millis(50)).await,
10803 Err(SuperviseError::RegistrationStillActive { .. })
10804 ));
10805
10806 assert!(matches!(
10808 wait_for_slot_registration_release(
10809 ®istry,
10810 RegistrationSlot::Connection(ConnectionId::new(INCUMBENT)),
10811 Duration::from_millis(50),
10812 )
10813 .await,
10814 Err(SuperviseError::RegistrationStillActive { .. })
10815 ));
10816
10817 let releaser = Arc::clone(®istry);
10818 let release = tokio::spawn(async move {
10819 sleep(Duration::from_millis(20)).await;
10820 releaser
10821 .deregister_connection(ConnectionId::new(INCUMBENT))
10822 .unwrap();
10823 notify_registration_release();
10824 });
10825 wait_for_slot_registration_release(
10826 ®istry,
10827 RegistrationSlot::Connection(ConnectionId::new(INCUMBENT)),
10828 Duration::from_secs(5),
10829 )
10830 .await
10831 .expect("the incumbent's own registration is released");
10832 release.await.unwrap();
10833 assert!(registry.get_module("m").unwrap().is_some());
10834 }
10835
10836 #[tokio::test]
10839 async fn candidate_slot_wait_ignores_the_incumbents_registration() {
10840 let registry = swapped_registry();
10841 assert!(matches!(
10842 wait_for_slot_registration_release(
10843 ®istry,
10844 RegistrationSlot::Candidate("m"),
10845 Duration::from_millis(50),
10846 )
10847 .await,
10848 Err(SuperviseError::RegistrationStillActive { .. })
10849 ));
10850 registry
10851 .deregister_connection(ConnectionId::new(CANDIDATE))
10852 .unwrap();
10853 wait_for_slot_registration_release(
10854 ®istry,
10855 RegistrationSlot::Candidate("m"),
10856 Duration::from_millis(50),
10857 )
10858 .await
10859 .expect("a candidate slot with no candidate is released");
10860 assert!(registry
10861 .registration(RegistrationSlot::Active("m"))
10862 .unwrap()
10863 .is_some());
10864 }
10865}
10866
10867fn classify_exit(status: &ExitStatus) -> ExitReport {
10868 ExitReport {
10869 kind: if status.success() {
10870 ExitKind::Clean
10871 } else {
10872 ExitKind::Crash
10873 },
10874 code: status.code(),
10875 signal: exit_signal(status),
10876 at_ms: unix_ms_now(),
10877 }
10878}
10879
10880fn wait_error_exit_report() -> ExitReport {
10886 ExitReport {
10887 kind: ExitKind::Crash,
10888 code: None,
10889 signal: None,
10890 at_ms: unix_ms_now(),
10891 }
10892}
10893
10894#[cfg(unix)]
10895fn exit_signal(status: &ExitStatus) -> Option<i32> {
10896 use std::os::unix::process::ExitStatusExt;
10897
10898 status.signal()
10899}
10900
10901#[cfg(not(unix))]
10902fn exit_signal(_status: &ExitStatus) -> Option<i32> {
10903 None
10904}
10905
10906fn reset_restart_count(snapshot: &SharedSnapshot, module_id: &str) -> Result<(), SuperviseError> {
10912 update_snapshot(snapshot, Some(module_id), |state| {
10913 state.clear_crash_restarts();
10914 })
10915}
10916
10917fn set_running(
10918 snapshot: &SharedSnapshot,
10919 child: &SupervisedChild,
10920 module_id: &str,
10921 spawn_events: &SpawnEventFeed,
10922) -> Result<(), SuperviseError> {
10923 let mut state = snapshot.lock().map_err(|_| SuperviseError::StatePoisoned {
10924 module_id: Some(module_id.to_string()),
10925 })?;
10926 state.spawn_generation = spawn_events.emit_spawned(module_id, child.pid, child.spawned_at_ms);
10927 if std::mem::take(&mut state.coalesced_restart_pending) {
10928 let generation = state.spawn_generation;
10929 info!(module_id, "restart already satisfied by generation {generation}; coalesced pending request completed");
10930 }
10931 state.drain_disposition_detail = None;
10932 state.spawn_failure = None;
10933 state.waiting_for_previous_pid = None;
10934 state.waiting_for_previous_detail = None;
10935 state.in_alternate_slot = false;
10938 state.configuration_updated_since_spawn = false;
10939 state.spawned_protocol = Some(child.protocol);
10940 state.state = ModuleState::Running;
10941 state.enabled = true;
10942 state.process_alive = true;
10943 state.pid = child.id();
10944 #[cfg(windows)]
10945 {
10946 state.pipe_consumed = child
10947 .nonce_delivery
10948 .as_ref()
10949 .map(|delivery| delivery.consumed());
10950 }
10951 #[cfg(target_os = "macos")]
10952 {
10953 state.report_ready = Some(Arc::clone(&child.report_ready));
10954 }
10955 state.spawned_at_ms = Some(child.spawned_at_ms);
10956 state.exec_outcome = Some(Arc::clone(&child.exec.0));
10957 state.spawned_from = Some(child.spawned_from.clone());
10958 state.launch_spec_digest = Some(child.launch_spec_digest.clone());
10959 state.spawned_file_identity = child.spawned_file_identity.clone();
10960 state.process_start_time = child.process_start_time;
10961 Ok(())
10962}
10963
10964fn clear_current_process_facts(state: &mut SupervisorSnapshot) {
10965 state.waiting_for_previous_pid = None;
10966 state.waiting_for_previous_detail = None;
10967 state.process_alive = false;
10968 state.spawned_protocol = None;
10969 state.pid = None;
10970 #[cfg(windows)]
10971 {
10972 state.pipe_consumed = None;
10973 }
10974 #[cfg(target_os = "macos")]
10975 {
10976 state.report_ready = None;
10977 }
10978 state.spawned_at_ms = None;
10979 state.exec_outcome = None;
10980 state.spawned_from = None;
10981 state.launch_spec_digest = None;
10982 state.spawned_file_identity = None;
10983 state.process_start_time = None;
10984 state.deliberate_severance = None;
10985}
10986
10987#[cfg(test)]
10988fn record_deliberate_severance(
10989 snapshot: &SharedSnapshot,
10990 identity: ProcessIdentity,
10991) -> Result<(), SuperviseError> {
10992 update_snapshot(snapshot, None, |state| {
10993 state.deliberate_severance = Some(identity);
10994 })
10995}
10996
10997fn apply_deliberate_severance_marker(
10998 snapshot: &SharedSnapshot,
10999 exited_identity: Option<ProcessIdentity>,
11000 mut exit_report: ExitReport,
11001) -> ExitReport {
11002 let marker = lock_snapshot(snapshot)
11003 .ok()
11004 .and_then(|mut state| state.deliberate_severance.take());
11005 if marker.is_some() && marker == exited_identity {
11006 exit_report.kind = ExitKind::DeliberateSeverance;
11007 }
11008 exit_report
11009}
11010
11011fn classify_reaped_child_exit(
11012 snapshot: &SharedSnapshot,
11013 child: &SupervisedChild,
11014 status: &ExitStatus,
11015) -> ExitReport {
11016 let _ = update_snapshot(snapshot, None, |state| {
11017 state.reaped_pid = Some(child.pid);
11018 state.spawn_failure = child.spawn_failure.clone();
11019 });
11020 apply_deliberate_severance_marker(snapshot, child.process_identity(), classify_exit(status))
11021}
11022
11023fn fail_snapshot(
11024 snapshot: &SharedSnapshot,
11025 module_id: Option<&str>,
11026 last_exit: Option<ExitReport>,
11027) {
11028 if let Err(err) = update_snapshot(snapshot, module_id, |state| {
11029 state.state = ModuleState::Failed;
11030 clear_current_process_facts(state);
11031 if let Some(last_exit) = last_exit {
11032 state.last_exit = Some(last_exit);
11033 }
11034 }) {
11035 error!(error = %err, "failed to mark supervisor state failed");
11036 }
11037}
11038
11039fn update_snapshot(
11040 snapshot: &SharedSnapshot,
11041 module_id: Option<&str>,
11042 update: impl FnOnce(&mut SupervisorSnapshot),
11043) -> Result<(), SuperviseError> {
11044 let mut state = snapshot.lock().map_err(|_| SuperviseError::StatePoisoned {
11045 module_id: module_id.map(ToOwned::to_owned),
11046 })?;
11047 update(&mut state);
11048 Ok(())
11049}
11050
11051const SLOW_SNAPSHOT_LOCK_THRESHOLD: Duration = Duration::from_millis(250);
11052
11053fn lock_snapshot_for_control<'a>(
11054 snapshot: &'a SharedSnapshot,
11055 module_id: &str,
11056 caller: &'static str,
11057) -> Result<std::sync::MutexGuard<'a, SupervisorSnapshot>, SuperviseError> {
11058 let started_at = Instant::now();
11059 let guard = lock_snapshot(snapshot)?;
11060 let waited = started_at.elapsed();
11061 if waited >= SLOW_SNAPSHOT_LOCK_THRESHOLD {
11062 warn!(
11063 module_id = %module_id,
11064 waited_ms = waited.as_millis() as u64,
11065 caller = %caller,
11066 "slow snapshot lock"
11067 );
11068 }
11069 Ok(guard)
11070}
11071
11072fn lock_snapshot(
11073 snapshot: &SharedSnapshot,
11074) -> Result<std::sync::MutexGuard<'_, SupervisorSnapshot>, SuperviseError> {
11075 snapshot
11076 .lock()
11077 .map_err(|_| SuperviseError::StatePoisoned { module_id: None })
11078}
11079
11080#[cfg(test)]
11081mod terminal_history_tests {
11082 use std::{
11083 path::PathBuf,
11084 sync::Arc,
11085 time::{Duration, Instant},
11086 };
11087
11088 use tokio::{sync::mpsc, time::sleep};
11089
11090 use super::{
11091 apply_deliberate_severance_marker, daemon_will_restart, drain_child_to_state,
11092 drained_after_quiescence_wait, handle_reload_spawn_failure, health_restart_child,
11093 lock_snapshot, on_child_exit, record_deliberate_severance, record_wait_error_terminal,
11094 reset_restart_count, spawn_and_mark_running, update_snapshot, wait_error_exit_report,
11095 ExitKind, ExitReport, ModuleProtocol, ModuleSpec, ModuleState, NextAction, ProcessIdentity,
11096 RestartPolicy, SpawnEventKind, StopNotice, SuperviseError, SupervisedConfiguration,
11097 SupervisedModule, SupervisedModuleInner, Supervisor, SupervisorHandle,
11098 SupervisorHealthStatus, SupervisorSnapshot,
11099 };
11100 use super::Instant as ClockInstant;
11105 use crate::{
11106 registry::Registry,
11107 terminal_ring::{TerminalRing, TerminalRingConfig},
11108 };
11109 use std::sync::Mutex;
11110 use subc_control::TerminalDisposition;
11111
11112 pub(super) fn fake_aft_stub_path() -> PathBuf {
11117 let mut path = std::env::current_exe().expect("current_exe available in tests");
11118 path.pop();
11119 path.pop();
11120 path.push(if cfg!(windows) {
11121 "fake-aft-stub.exe"
11122 } else {
11123 "fake-aft-stub"
11124 });
11125 assert!(
11126 path.exists(),
11127 "fake-aft-stub not built at {}: run `cargo test -p subc-core` (which builds \
11128 [[bin]] targets) rather than `cargo test -p subc-core --lib` (which does not)",
11129 path.display()
11130 );
11131 path
11132 }
11133
11134 #[test]
11135 fn reserved_never_spawned_refuses_every_hello() {
11136 let supervisor = SupervisorHandle::default();
11141 supervisor.apply_identity_configuration(&ModuleSpec {
11142 module_id: "never-spawned".to_string(),
11143 program: PathBuf::from("/usr/bin/false"),
11144 args: Vec::new(),
11145 env: Vec::new(),
11146 reserved: true,
11147 reserved_prefixes: Vec::new(),
11148 protocol: ModuleProtocol::Subc,
11149 overlap: Default::default(),
11150 });
11151 assert!(
11152 supervisor
11153 .reserved_hello_rejection("never-spawned", Some("any-forged-nonce"))
11154 .is_some(),
11155 "forged nonce must refuse on a reserved never-spawned id"
11156 );
11157 assert!(
11158 supervisor
11159 .reserved_hello_rejection("never-spawned", None)
11160 .is_some(),
11161 "absent nonce must refuse on a reserved never-spawned id"
11162 );
11163 supervisor.set_spawn_nonce("never-spawned", "minted".to_string());
11165 supervisor.apply_identity_configuration(&ModuleSpec {
11166 module_id: "never-spawned".to_string(),
11167 program: PathBuf::from("/usr/bin/false"),
11168 args: Vec::new(),
11169 env: Vec::new(),
11170 reserved: true,
11171 reserved_prefixes: Vec::new(),
11172 protocol: ModuleProtocol::Subc,
11173 overlap: Default::default(),
11174 });
11175 assert!(supervisor
11176 .reserved_hello_rejection("never-spawned", Some("minted"))
11177 .is_none());
11178 assert!(supervisor
11179 .reserved_hello_rejection("never-spawned", Some("forged"))
11180 .is_some());
11181 }
11182
11183 fn seed_crash_restarts(state: &mut SupervisorSnapshot, count: u32) {
11186 let now = ClockInstant::now();
11187 for _ in 0..count {
11188 state.crash_restarts.push_back(now);
11189 }
11190 }
11191
11192 fn age_oldest_crash_restart_out_of_window(state: &mut SupervisorSnapshot, window: Duration) {
11196 let aged = state
11197 .crash_restarts
11198 .front()
11199 .expect("a crash restart must be recorded before it can be aged")
11200 .checked_sub(window + Duration::from_secs(1))
11201 .expect("the test clock is far enough from its origin to age an instant");
11202 state.crash_restarts[0] = aged;
11203 }
11204
11205 fn snapshot_with_restarts(enabled: bool, count: u32) -> SupervisorSnapshot {
11206 let mut state = SupervisorSnapshot::new(ModuleState::Running, enabled);
11207 seed_crash_restarts(&mut state, count);
11208 state
11209 }
11210
11211 #[test]
11212 fn daemon_owned_recovery_predicate_uses_the_pre_increment_budget() {
11213 let policy = RestartPolicy::new(3, Duration::ZERO);
11214 let now = ClockInstant::now();
11215 assert!(daemon_will_restart(
11216 &mut snapshot_with_restarts(true, 2),
11217 &policy,
11218 now
11219 ));
11220 assert!(!daemon_will_restart(
11221 &mut snapshot_with_restarts(true, 3),
11222 &policy,
11223 now
11224 ));
11225 assert!(!daemon_will_restart(
11226 &mut snapshot_with_restarts(false, 0),
11227 &policy,
11228 now
11229 ));
11230 }
11231
11232 #[test]
11233 fn crash_restart_backoff_escalates_with_in_window_count() {
11234 let policy = RestartPolicy::new(4, Duration::from_millis(100))
11235 .with_max_backoff(Duration::from_secs(30));
11236 let now = ClockInstant::now();
11237 let mut state = SupervisorSnapshot::new(ModuleState::Running, true);
11238 let schedules = (0..4)
11239 .map(|_| {
11240 state
11241 .next_crash_restart(&policy, now)
11242 .expect("the test policy allows four crash restarts")
11243 })
11244 .collect::<Vec<_>>();
11245
11246 assert_eq!(
11247 schedules
11248 .iter()
11249 .map(|schedule| schedule.restart_in_window)
11250 .collect::<Vec<_>>(),
11251 vec![0, 1, 2, 3]
11252 );
11253 assert_eq!(
11254 schedules
11255 .iter()
11256 .map(|schedule| schedule.delay)
11257 .collect::<Vec<_>>(),
11258 vec![
11259 Duration::from_millis(100),
11260 Duration::from_secs(1),
11261 Duration::from_secs(10),
11262 Duration::from_secs(30),
11263 ]
11264 );
11265 }
11266
11267 #[test]
11268 fn crash_restart_backoff_resets_after_ring_clear() {
11269 let policy = RestartPolicy::new(3, Duration::from_millis(100));
11270 let now = ClockInstant::now();
11271 let mut state = SupervisorSnapshot::new(ModuleState::Running, true);
11272 assert_eq!(
11273 state.next_crash_restart(&policy, now).unwrap().delay,
11274 Duration::from_millis(100)
11275 );
11276 assert_eq!(
11277 state.next_crash_restart(&policy, now).unwrap().delay,
11278 Duration::from_secs(1)
11279 );
11280
11281 state.clear_crash_restarts();
11282 let schedule = state
11283 .next_crash_restart(&policy, now)
11284 .expect("a cleared ring must allow another restart");
11285 assert_eq!(schedule.restart_in_window, 0);
11286 assert_eq!(schedule.delay, Duration::from_millis(100));
11287 }
11288
11289 #[test]
11290 fn crash_restart_backoff_ignores_aged_restarts() {
11291 let policy = RestartPolicy::new(3, Duration::from_millis(100));
11292 let now = ClockInstant::now();
11293 let mut state = SupervisorSnapshot::new(ModuleState::Running, true);
11294 state
11295 .next_crash_restart(&policy, now)
11296 .expect("the first restart is allowed");
11297 state
11298 .next_crash_restart(&policy, now)
11299 .expect("the second restart is allowed");
11300 state.crash_restarts[0] = now
11301 .checked_sub(policy.window + Duration::from_secs(1))
11302 .expect("the fake clock can age a restart past the window");
11303
11304 let schedule = state
11305 .next_crash_restart(&policy, now)
11306 .expect("an aged restart must release its slot");
11307 assert_eq!(schedule.restart_in_window, 1);
11308 assert_eq!(schedule.delay, Duration::from_secs(1));
11309 assert_eq!(state.crash_restarts.len(), 2);
11310 }
11311
11312 #[test]
11316 fn a_budget_spent_before_the_window_no_longer_refuses() {
11317 let policy = RestartPolicy::new(3, Duration::ZERO);
11318 let mut state = snapshot_with_restarts(true, 3);
11319 let now = ClockInstant::now();
11320 assert!(!daemon_will_restart(&mut state, &policy, now));
11321
11322 assert!(daemon_will_restart(
11323 &mut state,
11324 &policy,
11325 now + policy.window + Duration::from_secs(1)
11326 ));
11327 assert!(
11328 state.crash_restarts.is_empty(),
11329 "reading the budget must drop the instants that left the window"
11330 );
11331 }
11332
11333 fn module_with_recovery_snapshot(
11334 state: ModuleState,
11335 enabled: bool,
11336 restart_count: u32,
11337 ) -> SupervisedModule {
11338 let registry = Arc::new(Registry::default());
11339 let supervisor =
11340 Supervisor::new_for_test(Arc::clone(®istry), RestartPolicy::new(3, Duration::ZERO));
11341 let runtime = supervisor.runtime_config();
11342 let spec = ModuleSpec {
11343 module_id: "recovery-snapshot".to_string(),
11344 program: fake_aft_stub_path(),
11345 args: Vec::new(),
11346 env: Vec::new(),
11347 reserved: false,
11348 reserved_prefixes: Vec::new(),
11349 protocol: ModuleProtocol::Subc,
11350 overlap: Default::default(),
11351 };
11352 let mut snapshot = SupervisorSnapshot::new(state, enabled);
11353 seed_crash_restarts(&mut snapshot, restart_count);
11354 let (commands, _rx) = mpsc::channel(4);
11357 SupervisedModule {
11358 inner: Arc::new(SupervisedModuleInner {
11359 module_id: spec.module_id.clone(),
11360 registry,
11361 snapshot: Arc::new(Mutex::new(snapshot)),
11362 configuration: Arc::new(Mutex::new(SupervisedConfiguration {
11363 spec,
11364 health: runtime.health,
11365 })),
11366 stderr_ring: runtime.stderr_ring,
11367 terminal_ring: runtime.terminal_ring,
11368 commands,
11369 monitor: Mutex::new(None),
11370 restart_policy: runtime.restart_policy,
11371 effective_drain_timeout: runtime.effective_drain_timeout,
11372 provenance_probe: supervisor.provenance_probe.clone(),
11373 }),
11374 }
11375 }
11376
11377 #[cfg(target_os = "linux")]
11378 #[tokio::test]
11379 async fn no_cgroup_placement_does_not_block_fake_aft_stub_spawn() {
11380 let supervisor =
11381 Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default())
11382 .with_cgroup_placement(None);
11383 let result = supervisor.spawn(ModuleSpec {
11384 module_id: "no-cgroup-placement".to_string(),
11385 program: fake_aft_stub_path(),
11386 args: Vec::new(),
11387 env: Vec::new(),
11388 reserved: false,
11389 reserved_prefixes: Vec::new(),
11390 protocol: ModuleProtocol::Subc,
11391 overlap: Default::default(),
11392 });
11393
11394 assert!(
11395 result.is_ok(),
11396 "no delegation must not turn an otherwise valid spawn into a failure: {result:?}"
11397 );
11398 }
11399
11400 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
11401 async fn undecided_snapshot_uses_shared_restart_predicate() {
11402 assert!(module_with_recovery_snapshot(ModuleState::Running, true, 2)
11403 .will_recover_after_connection_loss()
11404 .unwrap());
11405 assert!(
11406 !module_with_recovery_snapshot(ModuleState::Running, true, 3)
11407 .will_recover_after_connection_loss()
11408 .unwrap()
11409 );
11410 }
11411
11412 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
11413 async fn restarting_snapshot_at_exhausted_budget_is_non_terminal() {
11414 assert!(
11415 module_with_recovery_snapshot(ModuleState::Restarting, true, 3)
11416 .will_recover_after_connection_loss()
11417 .unwrap()
11418 );
11419 }
11420
11421 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
11422 async fn terminal_phase_snapshots_are_terminal_before_budget_exhaustion() {
11423 assert!(!module_with_recovery_snapshot(ModuleState::Failed, true, 0)
11424 .will_recover_after_connection_loss()
11425 .unwrap());
11426 assert!(
11427 !module_with_recovery_snapshot(ModuleState::Disabled, true, 0)
11428 .will_recover_after_connection_loss()
11429 .unwrap()
11430 );
11431 }
11432
11433 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
11434 async fn warming_snapshot_is_limited_to_startup_phases() {
11435 for state in [
11436 ModuleState::Starting,
11437 ModuleState::Running,
11438 ModuleState::Restarting,
11439 ] {
11440 assert!(
11441 module_with_recovery_snapshot(state, true, 0)
11442 .is_warming()
11443 .unwrap(),
11444 "{state:?} should be warming"
11445 );
11446 }
11447 for state in [
11448 ModuleState::Unresponsive,
11449 ModuleState::Draining,
11450 ModuleState::Stopped,
11451 ModuleState::Failed,
11452 ModuleState::Disabled,
11453 ] {
11454 assert!(
11455 !module_with_recovery_snapshot(state, true, 0)
11456 .is_warming()
11457 .unwrap(),
11458 "{state:?} should not be warming"
11459 );
11460 }
11461 }
11462
11463 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
11464 async fn terminal_history_survives_respawn_and_keeps_both_crashes_in_order() {
11465 let registry = Arc::new(Registry::default());
11466 let supervisor =
11467 Supervisor::new_for_test(Arc::clone(®istry), RestartPolicy::new(1, Duration::ZERO));
11468 let module = supervisor
11469 .spawn(ModuleSpec {
11470 module_id: "terminal-history".to_string(),
11471 program: fake_aft_stub_path(),
11472 args: Vec::new(),
11473 env: vec![("FAKE_AFT_EXIT_CODE".to_string(), "23".to_string())],
11474 reserved: false,
11475 reserved_prefixes: Vec::new(),
11476 protocol: ModuleProtocol::Subc,
11477 overlap: Default::default(),
11478 })
11479 .unwrap();
11480
11481 let deadline = Instant::now() + Duration::from_secs(5);
11482 loop {
11483 let history = module.terminal_history();
11484 if history.entries.len() == 2 {
11485 assert_eq!(module.status().unwrap().state, ModuleState::Failed);
11486 assert_eq!(history.dropped, 0);
11487 assert_eq!(
11488 history
11489 .entries
11490 .iter()
11491 .map(|entry| entry.exit_code)
11492 .collect::<Vec<_>>(),
11493 vec![Some(23), Some(23)]
11494 );
11495 assert!(history.entries[0].at_ms <= history.entries[1].at_ms);
11496 return;
11497 }
11498 assert!(
11499 Instant::now() < deadline,
11500 "module did not retain two terminal exits: {history:?}"
11501 );
11502 sleep(Duration::from_millis(10)).await;
11503 }
11504 }
11505
11506 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
11510 async fn disable_during_crash_backoff_cancels_pending_respawn() {
11511 let backoff = Duration::from_secs(2);
11512 let supervisor = Supervisor::new_for_test(
11513 Arc::new(Registry::default()),
11514 RestartPolicy::new(10, backoff),
11515 );
11516 let module = supervisor
11517 .spawn(ModuleSpec {
11518 module_id: "disable-during-backoff".to_string(),
11519 program: fake_aft_stub_path(),
11520 args: Vec::new(),
11521 env: vec![("FAKE_AFT_EXIT_CODE".to_string(), "23".to_string())],
11522 reserved: false,
11523 reserved_prefixes: Vec::new(),
11524 protocol: ModuleProtocol::Subc,
11525 overlap: Default::default(),
11526 })
11527 .unwrap();
11528
11529 let deadline = Instant::now() + Duration::from_secs(5);
11531 loop {
11532 if module.status().unwrap().state == ModuleState::Restarting {
11533 break;
11534 }
11535 assert!(
11536 Instant::now() < deadline,
11537 "module never entered the crash backoff"
11538 );
11539 sleep(Duration::from_millis(10)).await;
11540 }
11541
11542 let started = Instant::now();
11543 module.set_enabled(false).await.unwrap();
11544 let waited = started.elapsed();
11545
11546 assert!(
11547 waited < backoff / 2,
11548 "disable waited {waited:?} behind the {backoff:?} crash backoff; the operator command must preempt the pending respawn"
11549 );
11550 assert_eq!(module.status().unwrap().state, ModuleState::Disabled);
11551
11552 sleep(backoff + Duration::from_millis(500)).await;
11554 let status = module.status().unwrap();
11555 assert_eq!(status.state, ModuleState::Disabled);
11556 assert_eq!(
11557 status.spawn_generation, 1,
11558 "module respawned after the operator disabled it"
11559 );
11560 }
11561
11562 #[cfg(unix)]
11565 fn protocol_none_sigterm_exits_clean_spec(
11566 module_id: &str,
11567 dir: &std::path::Path,
11568 ) -> (ModuleSpec, PathBuf, PathBuf) {
11569 let ready = dir.join("ready");
11570 let marker = dir.join("sigterm");
11571 let spec = ModuleSpec {
11572 module_id: module_id.to_string(),
11573 program: fake_aft_stub_path(),
11574 args: Vec::new(),
11575 env: vec![
11576 ("FAKE_AFT_NEVER_CONNECT".to_string(), "1".to_string()),
11577 (
11578 "FAKE_AFT_SIGTERM_MARKER_PATH".to_string(),
11579 marker.display().to_string(),
11580 ),
11581 (
11582 "FAKE_AFT_NEVER_CONNECT_READY_PATH".to_string(),
11583 ready.display().to_string(),
11584 ),
11585 ],
11586 reserved: false,
11587 reserved_prefixes: Vec::new(),
11588 protocol: ModuleProtocol::None,
11589 overlap: Default::default(),
11590 };
11591 (spec, ready, marker)
11592 }
11593
11594 #[cfg(unix)]
11598 async fn wait_for_file(path: &std::path::Path) {
11599 let deadline = Instant::now() + Duration::from_secs(10);
11600 while !path.exists() {
11601 assert!(
11602 Instant::now() < deadline,
11603 "{} never appeared",
11604 path.display()
11605 );
11606 sleep(Duration::from_millis(10)).await;
11607 }
11608 }
11609
11610 #[cfg(unix)]
11614 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
11615 async fn protocol_none_unrequested_clean_exit_restarts_as_a_crash() {
11616 let dir = cortexkit_test_support::ScratchDir::new("none-unrequested-clean-exit");
11617 let (spec, ready, marker) =
11618 protocol_none_sigterm_exits_clean_spec("none-unrequested-clean-exit", dir.path());
11619 let supervisor = Supervisor::new_for_test(
11620 Arc::new(Registry::default()),
11621 RestartPolicy::new(3, Duration::ZERO),
11622 );
11623 let module = supervisor.spawn(spec).unwrap();
11624 wait_for_file(&ready).await;
11625 let deadline = Instant::now() + Duration::from_secs(10);
11629 let first_pid = loop {
11630 let status = module.status().unwrap();
11631 if status.state == ModuleState::Running {
11632 if let Some(pid) = status.pid {
11633 break pid;
11634 }
11635 }
11636 assert!(
11637 Instant::now() < deadline,
11638 "a running module must report its pid after exec confirmation: {status:?}"
11639 );
11640 sleep(Duration::from_millis(10)).await;
11641 };
11642
11643 rustix::process::kill_process(
11644 rustix::process::Pid::from_raw(i32::try_from(first_pid).unwrap()).unwrap(),
11645 rustix::process::Signal::TERM,
11646 )
11647 .unwrap();
11648
11649 let deadline = Instant::now() + Duration::from_secs(10);
11650 let respawned = loop {
11651 let status = module.status().unwrap();
11652 if status.state == ModuleState::Running
11653 && status.pid.is_some_and(|pid| pid != first_pid)
11654 {
11655 break status;
11656 }
11657 assert!(
11658 Instant::now() < deadline,
11659 "protocol-none module was not respawned after an unrequested clean exit: {status:?}"
11660 );
11661 sleep(Duration::from_millis(10)).await;
11662 };
11663 assert_eq!(respawned.spawn_generation, 2);
11664 assert!(
11665 marker.exists(),
11666 "the child must have exited through its SIGTERM handler (exit 0), or this proves nothing about clean exits"
11667 );
11668
11669 let history = module.terminal_history();
11670 assert_eq!(history.entries.len(), 1, "{history:?}");
11671 let entry = &history.entries[0];
11672 assert_eq!(entry.exit_code, Some(0));
11673 assert_eq!(entry.exit_kind, subc_control::TerminalExitKind::Clean);
11674 assert_eq!(entry.disposition, TerminalDisposition::Restarting);
11675
11676 module.stop().await.unwrap();
11677 }
11678
11679 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
11683 async fn protocol_none_repeated_clean_exits_exhaust_the_restart_budget() {
11684 let supervisor = Supervisor::new_for_test(
11685 Arc::new(Registry::default()),
11686 RestartPolicy::new(1, Duration::ZERO),
11687 );
11688 let module = supervisor
11689 .spawn(ModuleSpec {
11690 module_id: "none-clean-exit-budget".to_string(),
11691 program: fake_aft_stub_path(),
11692 args: Vec::new(),
11693 env: vec![("FAKE_AFT_EXIT_CODE".to_string(), "0".to_string())],
11694 reserved: false,
11695 reserved_prefixes: Vec::new(),
11696 protocol: ModuleProtocol::None,
11697 overlap: Default::default(),
11698 })
11699 .unwrap();
11700
11701 let deadline = Instant::now() + Duration::from_secs(10);
11704 loop {
11705 let status = module.status().unwrap();
11706 if status.state == ModuleState::Failed {
11707 break;
11708 }
11709 assert!(
11710 Instant::now() < deadline,
11711 "module never exhausted its budget: {status:?} {:?}",
11712 module.terminal_history()
11713 );
11714 sleep(Duration::from_millis(10)).await;
11715 }
11716 let history = module.terminal_history();
11717 assert_eq!(
11718 history
11719 .entries
11720 .iter()
11721 .map(|entry| (entry.exit_code, entry.disposition.clone()))
11722 .collect::<Vec<_>>(),
11723 vec![
11724 (Some(0), TerminalDisposition::Restarting),
11725 (Some(0), TerminalDisposition::Failed),
11726 ]
11727 );
11728 let detail = history.entries[1]
11729 .disposition_detail
11730 .as_deref()
11731 .expect("a budget failure names the budget");
11732 assert!(detail.contains("max_restarts=1"), "{detail}");
11733 assert_eq!(module.status().unwrap().spawn_generation, 2);
11734 }
11735
11736 #[test]
11737 fn restart_budget_failure_is_published_after_its_terminal_record() {
11738 use std::io::{Read, Write};
11739 let supervisor = Supervisor::new_for_test(
11740 Arc::new(Registry::default()),
11741 RestartPolicy::new(0, Duration::ZERO),
11742 );
11743 let runtime = supervisor.runtime_config();
11744 let spec = ModuleSpec {
11745 protocol: ModuleProtocol::None,
11746 ..windowed_crash_spec("budget-publication-order")
11747 };
11748 let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::new(
11749 ModuleState::Running,
11750 true,
11751 )));
11752 lock_snapshot(&snapshot).unwrap().process_alive = true;
11753 let reaped_snapshot = snapshot.clone();
11754 let mut barrier = terminal_record_barrier(&runtime.spawn_events);
11755 let history = runtime.terminal_ring.clone();
11756 let reap = std::thread::spawn(move || {
11757 tokio::runtime::Builder::new_current_thread()
11758 .enable_all()
11759 .build()
11760 .unwrap()
11761 .block_on(on_child_exit(
11762 &spec,
11763 runtime.restart_policy,
11764 &supervisor.registry,
11765 &reaped_snapshot,
11766 &runtime.terminal_ring,
11767 &runtime.spawn_events,
11768 &runtime.child_roster,
11769 ExitReport {
11770 kind: ExitKind::Clean,
11771 code: Some(0),
11772 signal: None,
11773 at_ms: 1,
11774 },
11775 ))
11776 });
11777 let mut reached = [0];
11778 barrier.read_exact(&mut reached).unwrap();
11779 assert_eq!(&reached, b"R");
11780 let (before_state, before_alive) = {
11781 let state = lock_snapshot(&snapshot).unwrap();
11782 (state.state, state.process_alive)
11783 };
11784 let before_history = history.lock().unwrap().snapshot();
11785 barrier.write_all(b"X").unwrap();
11786 assert!(matches!(reap.join().unwrap(), NextAction::Stop { .. }));
11787 assert!(before_history.entries.is_empty());
11788 assert_ne!(
11789 before_state,
11790 ModuleState::Failed,
11791 "Failed was visible before its terminal record could be written"
11792 );
11793 assert!(before_alive);
11794 assert_eq!(lock_snapshot(&snapshot).unwrap().state, ModuleState::Failed);
11795 let history = history.lock().unwrap().snapshot();
11796 assert_eq!(history.entries.len(), 1);
11797 assert_eq!(history.entries[0].exit_code, Some(0));
11798 assert_eq!(history.entries[0].disposition, TerminalDisposition::Failed);
11799 }
11800
11801 #[test]
11802 fn crash_restart_at_budget_limit_remains_recoverable_while_recording() {
11803 use std::io::{Read, Write};
11804
11805 let supervisor = Supervisor::new_for_test(
11806 Arc::new(Registry::default()),
11807 RestartPolicy::new(3, Duration::ZERO),
11808 );
11809 let mut runtime = supervisor.runtime_config();
11810 let module = module_with_recovery_snapshot(ModuleState::Running, true, 2);
11811 let snapshot = module.inner.snapshot.clone();
11812 lock_snapshot(&snapshot).unwrap().process_alive = true;
11813 runtime.terminal_ring = module.inner.terminal_ring.clone();
11814 let spec = windowed_crash_spec(module.module_id());
11815 let mut barrier = terminal_record_barrier(&runtime.spawn_events);
11816 let reaped_snapshot = snapshot.clone();
11817 let reap = std::thread::spawn(move || {
11818 tokio::runtime::Builder::new_current_thread()
11819 .enable_all()
11820 .build()
11821 .unwrap()
11822 .block_on(on_child_exit(
11823 &spec,
11824 runtime.restart_policy,
11825 &supervisor.registry,
11826 &reaped_snapshot,
11827 &runtime.terminal_ring,
11828 &runtime.spawn_events,
11829 &runtime.child_roster,
11830 ExitReport {
11831 kind: ExitKind::Crash,
11832 code: Some(2),
11833 signal: None,
11834 at_ms: 1,
11835 },
11836 ))
11837 });
11838
11839 let mut reached = [0];
11843 barrier.read_exact(&mut reached).unwrap();
11844 assert_eq!(&reached, b"R");
11845 let before_status = module.status().unwrap();
11846 let before_history = module.terminal_history();
11847 let before_recovers = module.will_recover_after_connection_loss().unwrap();
11848 barrier.write_all(b"X").unwrap();
11849 assert!(matches!(reap.join().unwrap(), NextAction::Restart { .. }));
11850
11851 assert_eq!(before_status.state, ModuleState::Running);
11852 assert!(before_status.process_alive);
11853 assert_eq!(before_status.restart_count, 3);
11854 assert!(before_history.entries.is_empty());
11855 assert!(
11856 before_recovers,
11857 "connection cleanup would report a terminal close for a scheduled crash restart"
11858 );
11859 let restarting = module.status().unwrap();
11860 assert_eq!(restarting.state, ModuleState::Restarting);
11861 assert_eq!(restarting.restart_count, 3);
11862 assert!(module.will_recover_after_connection_loss().unwrap());
11863 let history = module.terminal_history();
11864 assert_eq!(history.entries.len(), 1);
11865 assert_eq!(
11866 history.entries[0].disposition,
11867 TerminalDisposition::Restarting
11868 );
11869
11870 update_snapshot(&snapshot, Some(module.module_id()), |state| {
11873 state.state = ModuleState::Running;
11874 state.respawn_pending = false;
11875 })
11876 .unwrap();
11877 assert!(!module.will_recover_after_connection_loss().unwrap());
11878 update_snapshot(&snapshot, Some(module.module_id()), |state| {
11879 state.enabled = false;
11880 state.respawn_pending = true;
11881 })
11882 .unwrap();
11883 assert!(!module.will_recover_after_connection_loss().unwrap());
11884 }
11885
11886 fn terminal_record_barrier(events: &super::SpawnEventFeed) -> std::net::TcpStream {
11887 use std::net::{TcpListener, TcpStream};
11888 let listener = TcpListener::bind("127.0.0.1:0").unwrap();
11889 let writer = TcpStream::connect(listener.local_addr().unwrap()).unwrap();
11890 let (barrier, _) = listener.accept().unwrap();
11891 for stream in [&writer, &barrier] {
11892 stream
11893 .set_read_timeout(Some(Duration::from_secs(10)))
11894 .unwrap();
11895 stream
11896 .set_write_timeout(Some(Duration::from_secs(10)))
11897 .unwrap();
11898 }
11899 *events.1.lock().unwrap() = Some(writer);
11900 barrier
11901 }
11902
11903 pub(super) fn before_terminal_record(events: &super::SpawnEventFeed) {
11904 use std::io::{Read, Write};
11905 let barrier = events.1.lock().unwrap().take();
11906 if let Some(mut barrier) = barrier {
11907 barrier.write_all(b"R").unwrap();
11908 let mut release = [0];
11909 barrier.read_exact(&mut release).unwrap();
11910 assert_eq!(&release, b"X");
11911 }
11912 }
11913
11914 #[test]
11915 fn health_restart_budget_failure_is_published_after_its_terminal_record() {
11916 use std::io::{Read, Write};
11917
11918 let supervisor = Supervisor::new_for_test(
11919 Arc::new(Registry::default()),
11920 RestartPolicy::new(0, Duration::ZERO),
11921 );
11922 let mut runtime = supervisor.runtime_config();
11923 let module = module_with_recovery_snapshot(ModuleState::Running, true, 0);
11924 let snapshot = module.inner.snapshot.clone();
11925 runtime.terminal_ring = module.inner.terminal_ring.clone();
11926 let spec = ModuleSpec {
11927 module_id: module.module_id().into(),
11928 program: fake_aft_stub_path(),
11929 protocol: ModuleProtocol::None,
11930 env: vec![("FAKE_AFT_EXIT_CODE".into(), "17".into())],
11931 ..windowed_crash_spec(module.module_id())
11932 };
11933 let mut barrier = terminal_record_barrier(&runtime.spawn_events);
11934 let reap = std::thread::spawn(move || {
11935 tokio::runtime::Builder::new_current_thread()
11936 .enable_all()
11937 .build()
11938 .unwrap()
11939 .block_on(async {
11940 let mut child =
11941 Some(spawn_and_mark_running(&spec, &runtime, &snapshot).unwrap());
11942 health_restart_child(
11943 &spec,
11944 &runtime,
11945 &supervisor.registry,
11946 &supervisor.process_liveness,
11947 &snapshot,
11948 &mut child,
11949 SupervisorHealthStatus::Failing,
11950 Some("health probe did not answer"),
11951 1,
11952 )
11953 .await
11954 .unwrap();
11955 assert!(child.is_none());
11956 });
11957 });
11958 let mut reached = [0];
11963 barrier.read_exact(&mut reached).unwrap();
11964 assert_eq!(&reached, b"R");
11965 let before_status = module.status().unwrap();
11966 let before_history = module.terminal_history();
11967 barrier.write_all(b"X").unwrap();
11968 reap.join().unwrap();
11969 assert!(before_history.entries.is_empty());
11970 assert_ne!(
11971 before_status.state,
11972 ModuleState::Failed,
11973 "Failed was visible before its terminal record could be written"
11974 );
11975 assert!(
11976 before_status.process_alive,
11977 "process death was visible before its terminal record could be written"
11978 );
11979 let failed = module.status().unwrap();
11980 assert_eq!(failed.state, ModuleState::Failed);
11981 assert!(!failed.process_alive);
11982 let history = module.terminal_history();
11983 assert_eq!(history.entries.len(), 1);
11984 assert_eq!(history.entries[0].disposition, TerminalDisposition::Failed);
11985 assert_eq!(
11986 history.entries[0].disposition_detail.as_deref(),
11987 Some("crash budget exhausted: max_restarts=0 within window_secs=600")
11988 );
11989 }
11990
11991 #[test]
11992 fn observed_exits_publish_process_death_after_terminal_history() {
11993 use std::io::{Read, Write};
11994 for (kind, enabled, shutdown, wait_error, expected_state) in [
11995 (ExitKind::Clean, true, false, false, ModuleState::Stopped),
11996 (ExitKind::Crash, true, false, false, ModuleState::Restarting),
11997 (ExitKind::Crash, false, false, false, ModuleState::Disabled),
11998 (
11999 ExitKind::DeliberateSeverance,
12000 true,
12001 false,
12002 false,
12003 ModuleState::Restarting,
12004 ),
12005 #[cfg(unix)]
12006 (ExitKind::Crash, true, true, false, ModuleState::Stopped),
12007 (ExitKind::Crash, true, false, true, ModuleState::Failed),
12008 ] {
12009 let supervisor = Supervisor::new_for_test(
12010 Arc::new(Registry::default()),
12011 RestartPolicy::new(1, Duration::ZERO),
12012 );
12013 let runtime = supervisor.runtime_config();
12014 #[cfg(unix)]
12015 if shutdown {
12016 runtime.child_roster.close();
12017 }
12018 #[cfg(not(unix))]
12019 let _ = shutdown;
12020 let spec = windowed_crash_spec("exit-publication-order");
12021 let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::new(
12022 ModuleState::Running,
12023 enabled,
12024 )));
12025 lock_snapshot(&snapshot).unwrap().process_alive = true;
12026 let reaped_snapshot = snapshot.clone();
12027 let history = runtime.terminal_ring.clone();
12028 let mut barrier = terminal_record_barrier(&runtime.spawn_events);
12029 let reap = std::thread::spawn(move || {
12030 if wait_error {
12031 super::fail_after_wait_error(
12032 &spec.module_id,
12033 &reaped_snapshot,
12034 &runtime.terminal_ring,
12035 &runtime.spawn_events,
12036 "wait failed".into(),
12037 );
12038 } else {
12039 tokio::runtime::Builder::new_current_thread()
12040 .enable_all()
12041 .build()
12042 .unwrap()
12043 .block_on(on_child_exit(
12044 &spec,
12045 runtime.restart_policy,
12046 &supervisor.registry,
12047 &reaped_snapshot,
12048 &runtime.terminal_ring,
12049 &runtime.spawn_events,
12050 &runtime.child_roster,
12051 ExitReport {
12052 kind,
12053 code: Some(0),
12054 signal: None,
12055 at_ms: 1,
12056 },
12057 ));
12058 }
12059 });
12060 let mut reached = [0];
12061 barrier.read_exact(&mut reached).unwrap();
12062 assert_eq!(&reached, b"R");
12063 let before_alive = lock_snapshot(&snapshot).unwrap().process_alive;
12064 let before_history = history.lock().unwrap().snapshot();
12065 barrier.write_all(b"X").unwrap();
12066 reap.join().unwrap();
12067 assert!(
12068 before_alive,
12069 "{kind:?}: death was published before recording"
12070 );
12071 assert!(before_history.entries.is_empty());
12072 assert_eq!(lock_snapshot(&snapshot).unwrap().state, expected_state);
12073 assert!(!lock_snapshot(&snapshot).unwrap().process_alive);
12074 assert_eq!(history.lock().unwrap().snapshot().entries.len(), 1);
12075 }
12076 }
12077
12078 #[tokio::test]
12079 async fn health_restart_budget_failure_remains_failed_when_journal_append_fails() {
12080 let dir = cortexkit_test_support::ScratchDir::new("health-budget-journal-failure");
12081 let path = dir.join("terminals.jsonl");
12082 std::fs::create_dir(&path).unwrap();
12083 let supervisor = Supervisor::new_for_test(
12084 Arc::new(Registry::default()),
12085 RestartPolicy::new(0, Duration::ZERO),
12086 )
12087 .with_terminal_journal(path, "health-budget-journal-failure".into());
12088 let runtime = supervisor.runtime_config();
12089 let spec = ModuleSpec {
12090 program: fake_aft_stub_path(),
12091 protocol: ModuleProtocol::None,
12092 env: vec![("FAKE_AFT_EXIT_CODE".into(), "17".into())],
12093 ..windowed_crash_spec("health-budget-journal-failure")
12094 };
12095 let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::starting()));
12096 let mut child = Some(spawn_and_mark_running(&spec, &runtime, &snapshot).unwrap());
12097 health_restart_child(
12098 &spec,
12099 &runtime,
12100 &supervisor.registry,
12101 &supervisor.process_liveness,
12102 &snapshot,
12103 &mut child,
12104 SupervisorHealthStatus::Failing,
12105 None,
12106 1,
12107 )
12108 .await
12109 .unwrap();
12110 assert_eq!(lock_snapshot(&snapshot).unwrap().state, ModuleState::Failed);
12111 assert!(!lock_snapshot(&snapshot).unwrap().process_alive);
12112 let history = runtime
12113 .terminal_ring
12114 .lock()
12115 .unwrap()
12116 .durable_history(&spec.module_id);
12117 assert!(history.journal_write_failures > 0);
12118 assert_eq!(history.entries.len(), 1);
12119 assert_eq!(history.entries[0].disposition, TerminalDisposition::Failed);
12120 }
12121
12122 #[test]
12123 fn health_restart_budget_failure_remains_failed_when_exit_recording_panics() {
12124 let supervisor = Supervisor::new_for_test(
12125 Arc::new(Registry::default()),
12126 RestartPolicy::new(0, Duration::ZERO),
12127 );
12128 let runtime = supervisor.runtime_config();
12129 let spec = ModuleSpec {
12130 program: fake_aft_stub_path(),
12131 protocol: ModuleProtocol::None,
12132 env: vec![("FAKE_AFT_EXIT_CODE".into(), "17".into())],
12133 ..windowed_crash_spec("health-budget-recording-panic")
12134 };
12135 let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::starting()));
12136 let result = std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| {
12137 tokio::runtime::Builder::new_current_thread()
12138 .enable_all()
12139 .build()
12140 .unwrap()
12141 .block_on(async {
12142 let mut child =
12143 Some(spawn_and_mark_running(&spec, &runtime, &snapshot).unwrap());
12144 runtime.spawn_events.0.lock().unwrap().seq = u64::MAX;
12145 health_restart_child(
12146 &spec,
12147 &runtime,
12148 &supervisor.registry,
12149 &supervisor.process_liveness,
12150 &snapshot,
12151 &mut child,
12152 SupervisorHealthStatus::Failing,
12153 None,
12154 1,
12155 )
12156 .await
12157 .unwrap();
12158 });
12159 }));
12160 let panic = result.expect_err("recording must still unwind");
12161 assert_eq!(
12162 panic.downcast_ref::<String>().map(String::as_str),
12163 Some("spawn event sequence exhausted")
12164 );
12165 let state = lock_snapshot(&snapshot).unwrap();
12166 assert_eq!(state.state, ModuleState::Failed);
12167 assert!(!state.process_alive);
12168 assert!(state.last_exit.is_some());
12169 }
12170
12171 #[tokio::test]
12172 async fn restart_budget_failure_remains_failed_when_journal_append_fails() {
12173 let dir = cortexkit_test_support::ScratchDir::new("budget-journal-failure");
12174 let path = dir.join("terminals.jsonl");
12175 std::fs::create_dir(&path).unwrap();
12176 let supervisor = Supervisor::new_for_test(
12177 Arc::new(Registry::default()),
12178 RestartPolicy::new(0, Duration::ZERO),
12179 )
12180 .with_terminal_journal(path, "budget-journal-failure".into());
12181 let runtime = supervisor.runtime_config();
12182 let spec = windowed_crash_spec("budget-journal-failure");
12183 let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::starting()));
12184 assert!(matches!(
12185 on_child_exit(
12186 &spec,
12187 runtime.restart_policy,
12188 &supervisor.registry,
12189 &snapshot,
12190 &runtime.terminal_ring,
12191 &runtime.spawn_events,
12192 &runtime.child_roster,
12193 crash_exit_report(1),
12194 )
12195 .await,
12196 NextAction::Stop { .. }
12197 ));
12198 assert_eq!(lock_snapshot(&snapshot).unwrap().state, ModuleState::Failed);
12199 let history = runtime
12200 .terminal_ring
12201 .lock()
12202 .unwrap()
12203 .durable_history(&spec.module_id);
12204 assert!(history.journal_write_failures > 0);
12205 assert_eq!(history.entries.len(), 1);
12206 assert_eq!(history.entries[0].disposition, TerminalDisposition::Failed);
12207 }
12208
12209 #[test]
12210 fn restart_budget_failure_remains_failed_when_exit_recording_panics() {
12211 let supervisor = Supervisor::new_for_test(
12212 Arc::new(Registry::default()),
12213 RestartPolicy::new(0, Duration::ZERO),
12214 );
12215 let runtime = supervisor.runtime_config();
12216 let spec = windowed_crash_spec("budget-recording-panic");
12217 let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::starting()));
12218 runtime.spawn_events.emit_spawned(&spec.module_id, 1, 1);
12219 runtime.spawn_events.0.lock().unwrap().seq = u64::MAX;
12222 let result = std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| {
12223 tokio::runtime::Builder::new_current_thread()
12224 .enable_all()
12225 .build()
12226 .unwrap()
12227 .block_on(on_child_exit(
12228 &spec,
12229 runtime.restart_policy,
12230 &supervisor.registry,
12231 &snapshot,
12232 &runtime.terminal_ring,
12233 &runtime.spawn_events,
12234 &runtime.child_roster,
12235 crash_exit_report(1),
12236 ))
12237 }));
12238 let panic = result.err().expect("recording must still unwind");
12239 assert_eq!(
12240 panic.downcast_ref::<String>().map(String::as_str),
12241 Some("spawn event sequence exhausted")
12242 );
12243 assert_eq!(lock_snapshot(&snapshot).unwrap().state, ModuleState::Failed);
12244 }
12245
12246 #[cfg(unix)]
12249 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
12250 async fn protocol_none_requested_stop_and_disable_do_not_respawn() {
12251 for disable in [false, true] {
12252 let label = if disable {
12253 "none-requested-disable"
12254 } else {
12255 "none-requested-stop"
12256 };
12257 let dir = cortexkit_test_support::ScratchDir::new(label);
12258 let (spec, ready, marker) = protocol_none_sigterm_exits_clean_spec(label, dir.path());
12259 let supervisor = Supervisor::new_for_test(
12260 Arc::new(Registry::default()),
12261 RestartPolicy::new(3, Duration::ZERO),
12262 );
12263 let module = supervisor.spawn(spec).unwrap();
12264 wait_for_file(&ready).await;
12265
12266 if disable {
12267 module.set_enabled(false).await.unwrap();
12268 } else {
12269 module.stop().await.unwrap();
12270 }
12271 assert!(
12272 marker.exists(),
12273 "{label}: the child must have left through its SIGTERM handler with exit 0"
12274 );
12275
12276 sleep(Duration::from_millis(500)).await;
12279 let status = module.status().unwrap();
12280 let expected = if disable {
12281 ModuleState::Disabled
12282 } else {
12283 ModuleState::Stopped
12284 };
12285 assert_eq!(status.state, expected, "{label}");
12286 assert_eq!(
12287 status.spawn_generation, 1,
12288 "{label}: respawned after a requested stop"
12289 );
12290 let history = module.terminal_history();
12291 assert_eq!(history.entries.len(), 1, "{label}: {history:?}");
12292 assert_eq!(history.entries[0].exit_code, Some(0), "{label}");
12293 assert_ne!(
12294 history.entries[0].disposition,
12295 TerminalDisposition::Restarting,
12296 "{label}"
12297 );
12298 }
12299 }
12300
12301 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
12304 async fn subc_wire_clean_exit_is_still_a_stop() {
12305 let supervisor = Supervisor::new_for_test(
12306 Arc::new(Registry::default()),
12307 RestartPolicy::new(3, Duration::ZERO),
12308 );
12309 let module = supervisor
12310 .spawn(ModuleSpec {
12311 module_id: "wire-clean-exit".to_string(),
12312 program: fake_aft_stub_path(),
12313 args: Vec::new(),
12314 env: vec![("FAKE_AFT_EXIT_CODE".to_string(), "0".to_string())],
12315 reserved: false,
12316 reserved_prefixes: Vec::new(),
12317 protocol: ModuleProtocol::Subc,
12318 overlap: Default::default(),
12319 })
12320 .unwrap();
12321
12322 let deadline = Instant::now() + Duration::from_secs(10);
12323 while module.terminal_history().entries.is_empty() {
12324 assert!(Instant::now() < deadline, "module never exited");
12325 sleep(Duration::from_millis(10)).await;
12326 }
12327 sleep(Duration::from_millis(500)).await;
12329 let status = module.status().unwrap();
12330 assert_eq!(status.state, ModuleState::Stopped);
12331 assert_eq!(status.spawn_generation, 1);
12332 let history = module.terminal_history();
12333 assert_eq!(history.entries.len(), 1, "{history:?}");
12334 assert_eq!(history.entries[0].exit_code, Some(0));
12335 assert_eq!(history.entries[0].disposition, TerminalDisposition::Stopped);
12336 }
12337
12338 #[cfg(unix)]
12339 #[tokio::test]
12340 async fn reload_exit_keeps_roster_until_terminal_is_recorded_during_shutdown() {
12341 let dir = cortexkit_test_support::ScratchDir::new("reload-roster-terminal-order");
12342 let record = dir.join("live-children.json");
12343 let supervisor = Supervisor::new_for_test(
12344 Arc::new(Registry::default()),
12345 RestartPolicy::new(0, Duration::ZERO),
12346 );
12347 let mut runtime = supervisor.runtime_config();
12348 runtime.child_roster.record_to(record.clone());
12349 let gate = Arc::new(super::ReloadExitRecordGate::default());
12350 runtime.test_reload_exit_record_gate = Some(Arc::clone(&gate));
12351 let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::starting()));
12352 let spec = ModuleSpec {
12353 module_id: "reload-exit-roster".into(),
12354 program: fake_aft_stub_path(),
12355 args: Vec::new(),
12356 env: vec![("FAKE_AFT_EXIT_CODE".into(), "23".into())],
12357 reserved: false,
12358 reserved_prefixes: Vec::new(),
12359 protocol: ModuleProtocol::Subc,
12360 overlap: Default::default(),
12361 };
12362 let mut child = None;
12363 let reload = super::finish_reload_child(
12364 &spec,
12365 &runtime,
12366 &supervisor.registry,
12367 &supervisor.process_liveness,
12368 &snapshot,
12369 &mut child,
12370 );
12371 tokio::pin!(reload);
12372 tokio::select! {
12373 result = &mut reload => panic!("reload missed the before-record gate: {result:?}"),
12374 _ = gate.reached.notified() => {}
12375 }
12376 assert!(runtime
12377 .terminal_ring
12378 .lock()
12379 .unwrap()
12380 .snapshot()
12381 .entries
12382 .is_empty());
12383 assert_eq!(
12384 crate::live_children::read_record(&record).unwrap().len(),
12385 1,
12386 "shutdown must still wait for the reaped child until its terminal record exists"
12387 );
12388 runtime.child_roster.close();
12389 gate.resume.notify_one();
12390 assert!(reload.await.is_err());
12391 assert!(crate::live_children::read_record(&record)
12392 .unwrap()
12393 .is_empty());
12394 let history = runtime.terminal_ring.lock().unwrap().snapshot();
12395 assert_eq!(history.entries.len(), 1);
12396 assert_eq!(
12397 history.entries[0].disposition,
12398 TerminalDisposition::DaemonShutdown
12399 );
12400 }
12401
12402 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
12406 async fn every_restart_increment_path_advances_lifetime_count() {
12407 let supervisor = Supervisor::new_for_test(
12408 Arc::new(Registry::default()),
12409 RestartPolicy::new(1, Duration::ZERO),
12410 );
12411 let runtime = supervisor.runtime_config();
12412 let spec = ModuleSpec {
12413 module_id: "lifetime-increment-path".to_string(),
12414 program: PathBuf::from("/unused/lifetime-increment-path"),
12415 args: Vec::new(),
12416 env: Vec::new(),
12417 reserved: false,
12418 reserved_prefixes: Vec::new(),
12419 protocol: ModuleProtocol::Subc,
12420 overlap: Default::default(),
12421 };
12422
12423 let crash_snapshot = Arc::new(Mutex::new(SupervisorSnapshot::starting()));
12424 assert!(matches!(
12425 on_child_exit(
12426 &spec,
12427 runtime.restart_policy,
12428 &supervisor.registry,
12429 &crash_snapshot,
12430 &runtime.terminal_ring,
12431 &runtime.spawn_events,
12432 &runtime.child_roster,
12433 ExitReport {
12434 kind: ExitKind::Crash,
12435 code: Some(1),
12436 signal: None,
12437 at_ms: 1,
12438 },
12439 )
12440 .await,
12441 NextAction::Restart { schedule: _ }
12442 ));
12443 let (crash_restarts, crash_lifetime) = {
12444 let state = lock_snapshot(&crash_snapshot).unwrap();
12445 (state.crash_restarts.len(), state.lifetime_restarts)
12446 };
12447 assert_eq!(crash_restarts, 1);
12448 assert_eq!(crash_lifetime, 1);
12449
12450 let health_snapshot = Arc::new(Mutex::new(SupervisorSnapshot::starting()));
12451 let mut health_child = None;
12452 assert!(matches!(
12453 health_restart_child(
12454 &spec,
12455 &runtime,
12456 &supervisor.registry,
12457 &supervisor.process_liveness,
12458 &health_snapshot,
12459 &mut health_child,
12460 SupervisorHealthStatus::Failing,
12461 None,
12462 2,
12463 )
12464 .await,
12465 Ok(())
12466 ));
12467 assert!(health_child.is_none());
12468 assert!(lock_snapshot(&health_snapshot).unwrap().respawn_pending);
12469 assert!(runtime.scheduled_respawn.lock().unwrap().take().is_some());
12470 let (health_restarts, health_lifetime) = {
12471 let state = lock_snapshot(&health_snapshot).unwrap();
12472 (state.crash_restarts.len(), state.lifetime_restarts)
12473 };
12474 assert_eq!(health_restarts, 1);
12475 assert_eq!(health_lifetime, 1);
12476
12477 let reload_snapshot = Arc::new(Mutex::new(SupervisorSnapshot::starting()));
12478 let mut reload_child = None;
12479 assert!(matches!(
12480 handle_reload_spawn_failure(
12481 &spec,
12482 &runtime,
12483 &supervisor.process_liveness,
12484 &reload_snapshot,
12485 &mut reload_child,
12486 "forced reload spawn failure".to_string(),
12487 )
12488 .await,
12489 Err(SuperviseError::ReloadFailed { .. })
12490 ));
12491 let (reload_restarts, reload_lifetime) = {
12492 let state = lock_snapshot(&reload_snapshot).unwrap();
12493 (state.crash_restarts.len(), state.lifetime_restarts)
12494 };
12495 assert_eq!(reload_restarts, 1);
12496 assert_eq!(reload_lifetime, 1);
12497 }
12498
12499 #[tokio::test]
12500 async fn deliberately_severed_live_child_records_lifetime_without_spending_restart_budget() {
12501 let supervisor = Supervisor::new_for_test(
12502 Arc::new(Registry::default()),
12503 RestartPolicy::new(3, Duration::ZERO),
12504 );
12505 let runtime = supervisor.runtime_config();
12506 let spec = ModuleSpec {
12507 module_id: "deliberately-severed".to_string(),
12508 program: PathBuf::from("/unused/deliberately-severed"),
12509 args: Vec::new(),
12510 env: Vec::new(),
12511 reserved: false,
12512 reserved_prefixes: Vec::new(),
12513 protocol: ModuleProtocol::Subc,
12514 overlap: Default::default(),
12515 };
12516 let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::starting()));
12517 let process = ProcessIdentity {
12518 pid: 41,
12519 start_time: 101,
12520 };
12521 record_deliberate_severance(&snapshot, process).unwrap();
12522 let exit_report = apply_deliberate_severance_marker(
12523 &snapshot,
12524 Some(process),
12525 ExitReport {
12526 kind: ExitKind::Crash,
12527 code: Some(1),
12528 signal: None,
12529 at_ms: 1,
12530 },
12531 );
12532 assert_eq!(exit_report.kind, ExitKind::DeliberateSeverance);
12533
12534 assert!(matches!(
12535 on_child_exit(
12536 &spec,
12537 runtime.restart_policy,
12538 &supervisor.registry,
12539 &snapshot,
12540 &runtime.terminal_ring,
12541 &runtime.spawn_events,
12542 &runtime.child_roster,
12543 exit_report,
12544 )
12545 .await,
12546 NextAction::Restart { schedule: _ }
12547 ));
12548 let state = lock_snapshot(&snapshot).unwrap();
12549 assert_eq!(state.lifetime_restarts, 1);
12550 assert_eq!(state.crash_restarts.len(), 0);
12551 }
12552
12553 #[tokio::test]
12554 async fn genuine_crash_spends_restart_budget_and_records_lifetime() {
12555 let supervisor = Supervisor::new_for_test(
12556 Arc::new(Registry::default()),
12557 RestartPolicy::new(3, Duration::ZERO),
12558 );
12559 let runtime = supervisor.runtime_config();
12560 let spec = ModuleSpec {
12561 module_id: "genuine-crash".to_string(),
12562 program: PathBuf::from("/unused/genuine-crash"),
12563 args: Vec::new(),
12564 env: Vec::new(),
12565 reserved: false,
12566 reserved_prefixes: Vec::new(),
12567 protocol: ModuleProtocol::Subc,
12568 overlap: Default::default(),
12569 };
12570 let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::starting()));
12571
12572 assert!(matches!(
12573 on_child_exit(
12574 &spec,
12575 runtime.restart_policy,
12576 &supervisor.registry,
12577 &snapshot,
12578 &runtime.terminal_ring,
12579 &runtime.spawn_events,
12580 &runtime.child_roster,
12581 ExitReport {
12582 kind: ExitKind::Crash,
12583 code: Some(1),
12584 signal: None,
12585 at_ms: 1,
12586 },
12587 )
12588 .await,
12589 NextAction::Restart { schedule: _ }
12590 ));
12591 let state = lock_snapshot(&snapshot).unwrap();
12592 assert_eq!(state.lifetime_restarts, 1);
12593 assert_eq!(state.crash_restarts.len(), 1);
12594 }
12595
12596 fn crash_exit_report(at_ms: u64) -> ExitReport {
12597 ExitReport {
12598 kind: ExitKind::Crash,
12599 code: Some(1),
12600 signal: None,
12601 at_ms,
12602 }
12603 }
12604
12605 fn windowed_crash_spec(module_id: &str) -> ModuleSpec {
12606 ModuleSpec {
12607 module_id: module_id.to_string(),
12608 program: PathBuf::from("/unused").join(module_id),
12609 args: Vec::new(),
12610 env: Vec::new(),
12611 reserved: false,
12612 reserved_prefixes: Vec::new(),
12613 protocol: ModuleProtocol::Subc,
12614 overlap: Default::default(),
12615 }
12616 }
12617
12618 #[tokio::test]
12624 async fn three_crashes_inside_the_window_stop_the_module_and_name_the_window() {
12625 let (logs, _guard) = crate::router::test_log::log_capture(tracing::Level::ERROR);
12626 let supervisor = Supervisor::new_for_test(
12627 Arc::new(Registry::default()),
12628 RestartPolicy::new(2, Duration::ZERO),
12629 );
12630 let runtime = supervisor.runtime_config();
12631 let spec = windowed_crash_spec("crash-loop-in-window");
12632 let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::starting()));
12633
12634 for attempt in 1..=2 {
12635 assert!(
12636 matches!(
12637 on_child_exit(
12638 &spec,
12639 runtime.restart_policy,
12640 &supervisor.registry,
12641 &snapshot,
12642 &runtime.terminal_ring,
12643 &runtime.spawn_events,
12644 &runtime.child_roster,
12645 crash_exit_report(attempt),
12646 )
12647 .await,
12648 NextAction::Restart { schedule: _ }
12649 ),
12650 "crash {attempt} is inside the budget and must respawn"
12651 );
12652 }
12653
12654 assert!(matches!(
12655 on_child_exit(
12656 &spec,
12657 runtime.restart_policy,
12658 &supervisor.registry,
12659 &snapshot,
12660 &runtime.terminal_ring,
12661 &runtime.spawn_events,
12662 &runtime.child_roster,
12663 crash_exit_report(3),
12664 )
12665 .await,
12666 NextAction::Stop { .. }
12667 ));
12668
12669 {
12670 let state = lock_snapshot(&snapshot).unwrap();
12671 assert_eq!(state.state, ModuleState::Failed);
12672 assert_eq!(state.crash_restarts.len(), 2);
12673 assert_eq!(state.lifetime_restarts, 2);
12674 }
12675
12676 let history = runtime
12677 .terminal_ring
12678 .lock()
12679 .expect("terminal ring is not poisoned")
12680 .snapshot();
12681 let last = history
12682 .entries
12683 .last()
12684 .expect("the refused crash is retained");
12685 assert_eq!(last.disposition, TerminalDisposition::Failed);
12686 assert_eq!(
12687 last.disposition_detail.as_deref(),
12688 Some("crash budget exhausted: max_restarts=2 within window_secs=600")
12689 );
12690
12691 let captured = crate::router::test_log::captured_logs(&logs);
12692 assert!(
12693 captured.contains("crash budget exhausted: max_restarts=2 within window_secs=600"),
12694 "the stop must be logged with its window: {captured}"
12695 );
12696 }
12697
12698 #[tokio::test]
12706 async fn a_crash_older_than_the_window_frees_its_slot_for_a_later_crash() {
12707 let supervisor = Supervisor::new_for_test(
12708 Arc::new(Registry::default()),
12709 RestartPolicy::new(2, Duration::ZERO),
12710 );
12711 let runtime = supervisor.runtime_config();
12712 let spec = windowed_crash_spec("crash-across-windows");
12713 let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::starting()));
12714
12715 for attempt in 1..=2 {
12716 assert!(matches!(
12717 on_child_exit(
12718 &spec,
12719 runtime.restart_policy,
12720 &supervisor.registry,
12721 &snapshot,
12722 &runtime.terminal_ring,
12723 &runtime.spawn_events,
12724 &runtime.child_roster,
12725 crash_exit_report(attempt),
12726 )
12727 .await,
12728 NextAction::Restart { schedule: _ }
12729 ));
12730 }
12731
12732 update_snapshot(&snapshot, Some(&spec.module_id), |state| {
12735 age_oldest_crash_restart_out_of_window(state, runtime.restart_policy.window);
12736 })
12737 .unwrap();
12738
12739 assert!(
12740 matches!(
12741 on_child_exit(
12742 &spec,
12743 runtime.restart_policy,
12744 &supervisor.registry,
12745 &snapshot,
12746 &runtime.terminal_ring,
12747 &runtime.spawn_events,
12748 &runtime.child_roster,
12749 crash_exit_report(3),
12750 )
12751 .await,
12752 NextAction::Restart { schedule: _ }
12753 ),
12754 "a crash older than the window must not hold a budget slot"
12755 );
12756
12757 let state = lock_snapshot(&snapshot).unwrap();
12758 assert_eq!(state.state, ModuleState::Restarting);
12759 assert_eq!(
12760 state.crash_restarts.len(),
12761 2,
12762 "the aged instant is dropped and the new one takes its place"
12763 );
12764 assert_eq!(
12765 state.lifetime_restarts, 3,
12766 "the ledger counts every restart, including the ones the window forgot"
12767 );
12768 }
12769
12770 #[tokio::test]
12775 async fn an_operator_restart_clears_the_ring_and_leaves_the_ledger_alone() {
12776 let supervisor = Supervisor::new_for_test(
12777 Arc::new(Registry::default()),
12778 RestartPolicy::new(2, Duration::ZERO),
12779 );
12780 let runtime = supervisor.runtime_config();
12781 let spec = windowed_crash_spec("operator-cleared-budget");
12782 let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::starting()));
12783
12784 for attempt in 1..=2 {
12785 assert!(matches!(
12786 on_child_exit(
12787 &spec,
12788 runtime.restart_policy,
12789 &supervisor.registry,
12790 &snapshot,
12791 &runtime.terminal_ring,
12792 &runtime.spawn_events,
12793 &runtime.child_roster,
12794 crash_exit_report(attempt),
12795 )
12796 .await,
12797 NextAction::Restart { schedule: _ }
12798 ));
12799 }
12800
12801 reset_restart_count(&snapshot, &spec.module_id).unwrap();
12802 {
12803 let state = lock_snapshot(&snapshot).unwrap();
12804 assert!(
12805 state.crash_restarts.is_empty(),
12806 "an operator restart returns the full budget"
12807 );
12808 assert_eq!(
12809 state.lifetime_restarts, 2,
12810 "clearing the budget must not unmake the crashes"
12811 );
12812 }
12813
12814 assert!(
12815 matches!(
12816 on_child_exit(
12817 &spec,
12818 runtime.restart_policy,
12819 &supervisor.registry,
12820 &snapshot,
12821 &runtime.terminal_ring,
12822 &runtime.spawn_events,
12823 &runtime.child_roster,
12824 crash_exit_report(3),
12825 )
12826 .await,
12827 NextAction::Restart { schedule: _ }
12828 ),
12829 "the cleared budget must be spendable again"
12830 );
12831 let state = lock_snapshot(&snapshot).unwrap();
12832 assert_eq!(state.crash_restarts.len(), 1);
12833 assert_eq!(state.lifetime_restarts, 3);
12834 }
12835
12836 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
12837 async fn severance_marker_for_a_dead_child_does_not_label_its_successor() {
12838 let severed = ProcessIdentity {
12839 pid: 41,
12840 start_time: 101,
12841 };
12842 let successor = ProcessIdentity {
12843 pid: 41,
12844 start_time: 202,
12845 };
12846 let module = module_with_recovery_snapshot(ModuleState::Running, true, 0);
12847 update_snapshot(&module.inner.snapshot, Some("recovery-snapshot"), |state| {
12848 state.pid = Some(successor.pid);
12849 state.process_start_time = Some(successor.start_time);
12850 })
12851 .unwrap();
12852 assert!(!module.record_deliberate_severance(severed).unwrap());
12853
12854 let exit_report = apply_deliberate_severance_marker(
12855 &module.inner.snapshot,
12856 Some(successor),
12857 ExitReport {
12858 kind: ExitKind::Crash,
12859 code: Some(1),
12860 signal: None,
12861 at_ms: 1,
12862 },
12863 );
12864
12865 assert_eq!(exit_report.kind, ExitKind::Crash);
12866 }
12867
12868 #[tokio::test]
12869 async fn drain_reap_marks_deliberate_severance_and_records_lifetime_without_budget() {
12870 let registry = Registry::default();
12871 let supervisor = Supervisor::new_for_test(
12872 Arc::new(Registry::default()),
12873 RestartPolicy::new(3, Duration::ZERO),
12874 );
12875 let runtime = supervisor.runtime_config();
12876 let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::starting()));
12877 let spec = ModuleSpec {
12878 module_id: "drain-deliberate-severance".to_string(),
12879 program: fake_aft_stub_path(),
12880 args: Vec::new(),
12881 env: vec![("FAKE_AFT_EXIT_CODE".to_string(), "23".to_string())],
12882 reserved: false,
12883 reserved_prefixes: Vec::new(),
12884 protocol: ModuleProtocol::Subc,
12885 overlap: Default::default(),
12886 };
12887 let mut child = spawn_and_mark_running(&spec, &runtime, &snapshot).unwrap();
12888 let process = ProcessIdentity {
12889 pid: 41,
12890 start_time: 101,
12891 };
12892 child.process_identity = Some(process);
12893 update_snapshot(&snapshot, Some(&spec.module_id), |state| {
12894 state.pid = Some(process.pid);
12895 state.process_start_time = Some(process.start_time);
12896 })
12897 .unwrap();
12898 record_deliberate_severance(&snapshot, process).unwrap();
12899
12900 drain_child_to_state(
12901 &spec.module_id,
12902 spec.protocol,
12903 StopNotice::SentOverConnection,
12906 ®istry,
12907 None,
12908 &snapshot,
12909 &runtime.terminal_ring,
12910 &runtime.spawn_events,
12911 child,
12912 Duration::from_secs(1),
12913 ModuleState::Stopped,
12914 Some(false),
12915 )
12916 .await
12917 .unwrap();
12918
12919 let state = lock_snapshot(&snapshot).unwrap();
12920 assert_eq!(
12921 state.last_exit.as_ref().map(|exit| exit.kind),
12922 Some(ExitKind::DeliberateSeverance)
12923 );
12924 assert_eq!(state.lifetime_restarts, 1);
12925 assert_eq!(state.crash_restarts.len(), 0);
12926 drop(state);
12927 let history = runtime.terminal_ring.lock().unwrap().snapshot();
12928 assert_eq!(
12929 history.entries[0].exit_kind,
12930 subc_control::TerminalExitKind::DeliberateSeverance
12931 );
12932 }
12933
12934 #[tokio::test]
12935 async fn ordinary_drain_reap_does_not_record_a_lifetime_restart() {
12936 let registry = Registry::default();
12937 let supervisor = Supervisor::new_for_test(
12938 Arc::new(Registry::default()),
12939 RestartPolicy::new(3, Duration::ZERO),
12940 );
12941 let runtime = supervisor.runtime_config();
12942 let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::starting()));
12943 let spec = ModuleSpec {
12944 module_id: "ordinary-drain".to_string(),
12945 program: fake_aft_stub_path(),
12946 args: Vec::new(),
12947 env: vec![("FAKE_AFT_EXIT_CODE".to_string(), "23".to_string())],
12948 reserved: false,
12949 reserved_prefixes: Vec::new(),
12950 protocol: ModuleProtocol::Subc,
12951 overlap: Default::default(),
12952 };
12953 let child = spawn_and_mark_running(&spec, &runtime, &snapshot).unwrap();
12954
12955 drain_child_to_state(
12956 &spec.module_id,
12957 spec.protocol,
12958 StopNotice::SentOverConnection,
12961 ®istry,
12962 None,
12963 &snapshot,
12964 &runtime.terminal_ring,
12965 &runtime.spawn_events,
12966 child,
12967 Duration::from_secs(1),
12968 ModuleState::Stopped,
12969 Some(false),
12970 )
12971 .await
12972 .unwrap();
12973
12974 let state = lock_snapshot(&snapshot).unwrap();
12975 assert_eq!(
12976 state.last_exit.as_ref().map(|exit| exit.kind),
12977 Some(ExitKind::Crash)
12978 );
12979 assert_eq!(state.lifetime_restarts, 0);
12980 assert_eq!(state.crash_restarts.len(), 0);
12981 }
12982
12983 #[test]
12984 fn fatal_connection_teardown_cannot_arm_a_marker_for_a_surviving_process() {
12985 assert!(!include_str!("server.rs")
12991 .contains("router.record_deliberate_connection_severance(ctx.connection_id)"));
12992 }
12993
12994 #[test]
13001 fn drained_after_quiescence_wait_passes_ok_through_and_forces_false_on_err() {
13002 assert!(drained_after_quiescence_wait(&Ok(true)));
13003 assert!(!drained_after_quiescence_wait(&Ok(false)));
13004 assert!(!drained_after_quiescence_wait(&Err(
13005 SuperviseError::StatePoisoned { module_id: None }
13006 )));
13007 }
13008
13009 #[test]
13018 fn wait_error_exit_report_records_a_failed_terminal_with_no_code_or_signal() {
13019 let ring = Arc::new(Mutex::new(TerminalRing::new(
13020 TerminalRingConfig::default(),
13021 0,
13022 )));
13023 record_wait_error_terminal("wait-error", &ring, &super::SpawnEventFeed::default());
13024
13025 let snapshot = ring.lock().unwrap().snapshot();
13026 assert_eq!(snapshot.entries.len(), 1);
13027 let entry = &snapshot.entries[0];
13028 assert_eq!(entry.exit_code, None);
13029 assert_eq!(entry.exit_signal, None);
13030 assert_eq!(entry.disposition, TerminalDisposition::Failed);
13031 }
13032
13033 #[test]
13034 fn wait_error_exit_path_preserves_spawn_event_density() {
13035 let feed = super::SpawnEventFeed::default();
13036 feed.configure_incarnation("wait-error-density".to_string());
13037 feed.emit_spawned("wait-error", 41, 1);
13038 let ring = Arc::new(Mutex::new(TerminalRing::new(
13039 TerminalRingConfig::default(),
13040 0,
13041 )));
13042
13043 record_wait_error_terminal("wait-error", &ring, &feed);
13044 feed.emit_spawned("after-wait-error", 42, 2);
13045
13046 let state = feed.0.lock().unwrap();
13047 let sequences = state
13048 .events
13049 .iter()
13050 .map(|event| event.cursor.seq)
13051 .collect::<Vec<_>>();
13052 assert_eq!(sequences, vec![1, 2, 3]);
13053 assert_eq!(state.events[1].kind, SpawnEventKind::Exited);
13054 assert_eq!(state.events[1].exit_code, None);
13055 assert_eq!(state.events[1].exit_signal, None);
13056 }
13057
13058 #[test]
13062 fn wait_error_exit_report_is_classified_as_a_crash() {
13063 assert_eq!(wait_error_exit_report().kind, ExitKind::Crash);
13064 }
13065}
13066
13067#[cfg(all(test, unix))]
13068mod restart_ordering_tests {
13069 use super::*;
13070 use std::io::Write;
13071 use tokio::io::AsyncReadExt;
13072
13073 #[test]
13074 #[ignore = "child process fixture"]
13075 fn serving_child() {
13076 let Ok(address_file) = std::env::var("RESTART_TEST_ADDRESS_FILE") else {
13077 return;
13078 };
13079 let listener = std::net::TcpListener::bind("127.0.0.1:0").unwrap();
13080 std::fs::write(address_file, listener.local_addr().unwrap().to_string()).unwrap();
13081 for stream in listener.incoming() {
13082 writeln!(stream.unwrap(), "{}", std::process::id()).unwrap();
13083 }
13084 }
13085
13086 async fn restart_before_publication(requests: usize) {
13087 let home = cortexkit_test_support::ScratchDir::new("restart-before-publication");
13088 let address_file = home.path().join("address");
13089 let spec = ModuleSpec {
13090 module_id: "restart-serving-child".into(),
13091 program: std::env::current_exe().unwrap(),
13092 args: vec![
13093 "--exact".into(),
13094 "supervise::restart_ordering_tests::serving_child".into(),
13095 "--ignored".into(),
13096 "--nocapture".into(),
13097 ],
13098 env: vec![(
13099 "RESTART_TEST_ADDRESS_FILE".into(),
13100 address_file.display().to_string(),
13101 )],
13102 reserved: false,
13103 reserved_prefixes: vec![],
13104 protocol: ModuleProtocol::None,
13105 overlap: Default::default(),
13106 };
13107 let mut supervisor =
13108 Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default());
13109 let (spawned_tx, spawned_rx) = oneshot::channel();
13110 let spawned_tx = Mutex::new(Some(spawned_tx));
13111 let (release_tx, release_rx) = std::sync::mpsc::channel();
13112 let release_rx = Mutex::new(release_rx);
13113 supervisor.test_after_first_spawn = AfterFirstSpawnHook(Some(Arc::new(move |_| {
13116 spawned_tx.lock().unwrap().take().unwrap().send(()).unwrap();
13117 release_rx
13118 .lock()
13119 .unwrap()
13120 .recv_timeout(Duration::from_secs(20))
13121 .unwrap();
13122 })));
13123 let registered = supervisor
13124 .register_configured_with_health(
13125 spec.clone(),
13126 true,
13127 HealthConfig::default(),
13128 None,
13129 RestartPolicy::new(3, Duration::ZERO),
13130 )
13131 .unwrap();
13132 let module = registered.module.clone();
13133 let startup = tokio::task::spawn_blocking(move || {
13134 supervisor.start_registered_configured(registered, None)
13135 });
13136 spawned_rx.await.unwrap();
13137 let old_pid = tokio::time::timeout(Duration::from_secs(10), async {
13138 let address = loop {
13139 if let Ok(address) = std::fs::read_to_string(&address_file) {
13140 if !address.is_empty() {
13141 break address;
13142 }
13143 }
13144 tokio::time::sleep(Duration::from_millis(1)).await;
13145 };
13146 let mut stream = tokio::net::TcpStream::connect(address).await.unwrap();
13147 let mut reply = String::new();
13148 stream.read_to_string(&mut reply).await.unwrap();
13149 reply.trim().parse::<u32>().unwrap()
13150 })
13151 .await
13152 .expect("child must serve before generation publication");
13153 assert_eq!(
13154 lock_snapshot(&module.inner.snapshot)
13155 .unwrap()
13156 .spawn_generation,
13157 0
13158 );
13159 let mut replies = Vec::new();
13160 for _ in 0..requests {
13161 let mut restart = Box::pin(module.restart(Some(0)));
13162 std::future::poll_fn(|cx| {
13165 assert!(std::future::Future::poll(restart.as_mut(), cx).is_pending());
13166 std::task::Poll::Ready(())
13167 })
13168 .await;
13169 replies.push(restart);
13170 }
13171 release_tx.send(()).unwrap();
13172 startup.await.unwrap().unwrap();
13173 for reply in replies {
13174 reply.await.unwrap();
13175 }
13176 let replacement = tokio::time::timeout(Duration::from_secs(10), async {
13179 loop {
13180 if lock_snapshot(&module.inner.snapshot)
13181 .unwrap()
13182 .spawn_generation
13183 > 1
13184 {
13185 break;
13186 }
13187 tokio::time::sleep(Duration::from_millis(1)).await;
13188 }
13189 })
13190 .await;
13191 module.update_spec_for_test(spec).await.unwrap();
13195 let (generation, pid) = {
13196 let state = lock_snapshot(&module.inner.snapshot).unwrap();
13197 (state.spawn_generation, state.pid)
13198 };
13199 module.stop().await.unwrap();
13200 replacement.expect("restart must publish a replacement for the already-serving child");
13201 assert_ne!(
13202 pid,
13203 Some(old_pid),
13204 "a serving child predating the restart must be replaced"
13205 );
13206 assert_eq!(generation, 2, "queued restarts must share one replacement");
13207 }
13208
13209 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
13210 async fn restart_received_after_serving_before_generation_publication_replaces_child() {
13211 restart_before_publication(1).await;
13212 }
13213
13214 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
13215 async fn restart_queued_behind_restart_does_not_kill_its_replacement() {
13216 restart_before_publication(2).await;
13217 }
13218
13219 #[tokio::test]
13220 async fn restart_queued_before_replacement_spawn_keeps_the_new_child() {
13221 let supervisor =
13222 Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default());
13223 let registered = supervisor
13224 .register_configured_with_health(
13225 ModuleSpec {
13226 module_id: "queued-before-replacement".into(),
13227 program: "/bin/sh".into(),
13228 args: vec!["-c".into(), "exec sleep 600".into()],
13229 env: vec![],
13230 reserved: false,
13231 reserved_prefixes: vec![],
13232 protocol: ModuleProtocol::None,
13233 overlap: Default::default(),
13234 },
13235 true,
13236 HealthConfig::default(),
13237 None,
13238 RestartPolicy::default(),
13239 )
13240 .unwrap();
13241 let RegisteredConfiguredModule {
13242 mut spec,
13243 mut runtime,
13244 module,
13245 mut commands,
13246 } = registered;
13247 let snapshot = &module.inner.snapshot;
13248 let mut restart = Box::pin(module.restart(Some(0)));
13249 std::future::poll_fn(|cx| {
13250 assert!(std::future::Future::poll(restart.as_mut(), cx).is_pending());
13251 std::task::Poll::Ready(())
13252 })
13253 .await;
13254 let mut child = Some(spawn_and_mark_running(&spec, &runtime, snapshot).unwrap());
13257 let replacement_pid = child.as_ref().unwrap().id();
13258 let command = commands.recv().await.unwrap();
13259 assert!(
13260 handle_supervisor_command(
13261 command,
13262 &mut spec,
13263 &mut runtime,
13264 &supervisor.registry,
13265 &supervisor.process_liveness,
13266 snapshot,
13267 &mut child,
13268 &mut commands,
13269 &mut VecDeque::new(),
13270 )
13271 .await
13272 );
13273 restart.await.unwrap();
13274 let kept_pid = child.as_ref().and_then(SupervisedChild::id);
13275 let pending = lock_snapshot(snapshot).unwrap().respawn_pending;
13276 if let Some(mut child) = child {
13277 child.start_kill().unwrap();
13278 child.wait().await.unwrap();
13279 }
13280 assert_eq!(
13281 kept_pid, replacement_pid,
13282 "a newer replacement must survive"
13283 );
13284 assert!(
13285 !pending,
13286 "a satisfied restart must not schedule another spawn"
13287 );
13288 }
13289}
13290
13291#[cfg(all(test, unix))]
13292mod health_event_tests {
13293 use super::*;
13294 use cortexkit_test_support::ScratchDir;
13295 use subc_protocol::{manifest::Concurrency, session::ModuleControlResponse};
13296
13297 struct Actor {
13298 module: SupervisedModule,
13299 registry: Arc<Registry>,
13300 forwarding: Arc<ForwardingTable>,
13301 spec: ModuleSpec,
13302 health: HealthConfig,
13303 rx: mpsc::Receiver<crate::router::OutboundFrame>,
13304 _home: ScratchDir,
13305 }
13306
13307 async fn wait_for<T>(reason: &str, mut observe: impl FnMut() -> Option<T>) -> T {
13308 let deadline = std::time::Instant::now() + Duration::from_secs(10);
13312 let tick = Instant::now();
13313 loop {
13314 if let Some(value) = observe() {
13315 assert_eq!(Instant::now(), tick, "{reason} must not wait for a timer");
13316 return value;
13317 }
13318 assert!(
13319 std::time::Instant::now() < deadline,
13320 "timed out waiting for {reason}"
13321 );
13322 tokio::task::yield_now().await;
13323 }
13324 }
13325
13326 impl Actor {
13327 async fn start(protocol: ModuleProtocol, cadence: Duration) -> Self {
13328 let home = ScratchDir::new("health-events");
13329 let registry = Arc::new(Registry::default());
13330 let forwarding = Arc::new(ForwardingTable::default());
13331 let health = HealthConfig {
13332 cadence,
13333 deadline: Duration::from_secs(3600),
13334 ..HealthConfig::default()
13335 };
13336 let supervisor =
13337 Supervisor::new_for_test(registry.clone(), RestartPolicy::new(3, Duration::ZERO))
13338 .with_forwarding(forwarding.clone())
13339 .with_health_config(health.clone());
13340 let spec = ModuleSpec {
13341 module_id: "event-health-child".into(),
13342 program: PathBuf::from("/bin/sh"),
13343 args: vec!["-c".into(), "exec sleep 600".into()],
13346 env: ["XDG_DATA_HOME", "XDG_RUNTIME_DIR", "XDG_CONFIG_HOME"]
13347 .into_iter()
13348 .map(|key| (key.into(), home.path().display().to_string()))
13349 .collect(),
13350 reserved: false,
13351 reserved_prefixes: vec![],
13352 protocol,
13353 overlap: Default::default(),
13354 };
13355 let module = supervisor.spawn(spec.clone()).unwrap();
13356 let (_, rx) = mpsc::channel(8);
13357 let actor = Self {
13358 module,
13359 registry,
13360 forwarding,
13361 spec,
13362 health,
13363 rx,
13364 _home: home,
13365 };
13366 actor
13367 .wait_for_select(|checkpoint| checkpoint.generation > 0)
13368 .await;
13369 actor
13370 }
13371
13372 fn turns(&self) -> u64 {
13373 lock_snapshot(&self.module.inner.snapshot)
13374 .unwrap()
13375 .actor_turns
13376 }
13377
13378 async fn wait_for_select(
13379 &self,
13380 expected: impl Fn(&ActorSelectCheckpoint) -> bool,
13381 ) -> ActorSelectCheckpoint {
13382 wait_for("actor select checkpoint", || {
13383 let state = lock_snapshot(&self.module.inner.snapshot).unwrap();
13384 state.actor_select.filter(|checkpoint| {
13385 checkpoint.generation == state.spawn_generation
13386 && state.state == ModuleState::Running
13387 && state.process_alive
13388 && state.reported_pid().is_some()
13389 && expected(checkpoint)
13390 })
13391 })
13392 .await
13393 }
13394
13395 fn hello(&mut self, connection: u64, health: bool) {
13396 let manifest =
13397 subc_protocol::manifest::ModuleManifest::builder(&self.spec.module_id, "1.0.0")
13398 .protocol_ver(subc_protocol::PROTOCOL_VERSION)
13399 .build();
13400 let connection = ConnectionId::new(connection);
13401 let (tx, rx) = mpsc::channel(8);
13402 self.rx = rx;
13403 self.forwarding
13404 .register_module_connection(
13405 connection,
13406 self.spec.module_id.clone(),
13407 subc_protocol::PROTOCOL_VERSION,
13408 Concurrency::ModuleManaged,
13409 FrameSink::new(tx),
13410 )
13411 .unwrap();
13412 self.registry
13413 .register_with_control_ops(
13414 manifest,
13415 subc_protocol::PROTOCOL_VERSION,
13416 connection,
13417 if health {
13418 vec![MODULE_CONTROL_OP_HEALTH_CHECK.into()]
13419 } else {
13420 vec![]
13421 },
13422 )
13423 .unwrap();
13424 }
13425
13426 async fn probe(&mut self) -> crate::router::OutboundFrame {
13427 let frame = wait_for("outbound health probe", || match self.rx.try_recv() {
13428 Ok(frame) => Some(frame),
13429 Err(mpsc::error::TryRecvError::Empty) => None,
13430 Err(mpsc::error::TryRecvError::Disconnected) => panic!("health peer disconnected"),
13431 })
13432 .await;
13433 let body: Value = serde_json::from_slice(&frame.body).unwrap();
13434 assert_eq!(body["op"], MODULE_CONTROL_OP_HEALTH_CHECK);
13435 frame
13436 }
13437
13438 fn no_probe(&mut self) {
13439 assert!(matches!(
13440 self.rx.try_recv(),
13441 Err(mpsc::error::TryRecvError::Empty)
13442 ));
13443 }
13444
13445 fn answer(&self, connection: u64, frame: crate::router::OutboundFrame) {
13446 self.forwarding
13447 .complete_module_control_rpc(
13448 ConnectionId::new(connection),
13449 frame.header.corr,
13450 Some(MODULE_CONTROL_OP_HEALTH_CHECK),
13451 ModuleControlRpcOutcome::Response(ModuleControlResponse::HealthCheck {
13452 status: HealthStatus::Ok,
13453 detail: None,
13454 metrics: None,
13455 }),
13456 )
13457 .unwrap();
13458 }
13459 }
13460
13461 #[tokio::test(start_paused = true)]
13462 async fn no_health_children_park_for_sixty_seconds() {
13463 let none = Actor::start(ModuleProtocol::None, Duration::from_secs(30)).await;
13464 let unregistered = Actor::start(ModuleProtocol::Subc, Duration::from_secs(30)).await;
13465 let mut unadvertised = Actor::start(ModuleProtocol::Subc, Duration::from_secs(30)).await;
13466 unadvertised.hello(1, false);
13467 for _ in 0..6000 {
13472 tokio::time::advance(Duration::from_millis(10)).await;
13473 tokio::task::yield_now().await;
13474 }
13475 for actor in [&none, &unregistered, &unadvertised] {
13476 assert!(
13477 actor.turns() <= 5,
13478 "no-health actor woke {} times",
13479 actor.turns()
13480 );
13481 assert_eq!(actor.module.state().unwrap(), ModuleState::Running);
13482 }
13483 }
13484
13485 #[tokio::test(start_paused = true)]
13486 async fn late_hello_starts_a_due_probe_in_the_notification_tick() {
13487 let mut actor = Actor::start(ModuleProtocol::Subc, Duration::ZERO).await;
13491 let tick = Instant::now();
13492 actor.hello(1, true);
13493 actor.probe().await;
13494 assert_eq!(Instant::now(), tick, "HELLO must not wait for a poll");
13495 }
13496
13497 #[tokio::test(start_paused = true)]
13498 async fn hello_keeps_the_jittered_cadence_and_catalog_updates_do_not_reset_it() {
13499 let mut actor = Actor::start(ModuleProtocol::Subc, Duration::from_secs(30)).await;
13500 let hello_tick = Instant::now();
13501 actor.hello(1, true);
13502 let armed = actor
13503 .wait_for_select(|checkpoint| {
13504 checkpoint.registered_connection == Some(ConnectionId::new(1))
13505 && checkpoint.next_probe_at.is_some()
13506 })
13507 .await;
13508 let delay = jittered_health_delay(&actor.spec.module_id, 0, actor.health.cadence);
13509 assert!((Duration::from_secs(30)..Duration::from_secs(33)).contains(&delay));
13510 let first_deadline = hello_tick + delay;
13511 assert_eq!(armed.next_probe_at, Some(first_deadline));
13512 tokio::time::advance(Duration::from_secs(29)).await;
13513 let turns = actor.turns();
13514 actor
13515 .registry
13516 .replace_catalog_for_connection(ConnectionId::new(1), vec![], None, Some(true))
13517 .unwrap();
13518 let updated = actor
13519 .wait_for_select(|checkpoint| checkpoint.turn > turns)
13520 .await;
13521 assert_eq!(
13522 updated.next_probe_at,
13523 Some(first_deadline),
13524 "catalog update must not reset cadence"
13525 );
13526 actor.no_probe();
13527 tokio::time::advance(first_deadline - Instant::now()).await;
13528 let frame = actor.probe().await;
13529 actor.answer(1, frame);
13530 let next = jittered_health_delay(&actor.spec.module_id, 1, actor.health.cadence);
13531 let next_deadline = Instant::now() + next;
13532 let rearmed = actor
13533 .wait_for_select(|checkpoint| {
13534 checkpoint
13535 .next_probe_at
13536 .is_some_and(|deadline| deadline > first_deadline)
13537 })
13538 .await;
13539 assert_eq!(rearmed.next_probe_at, Some(next_deadline));
13540 tokio::time::advance(next - Duration::from_nanos(1)).await;
13541 actor.no_probe();
13542 tokio::time::advance(Duration::from_nanos(1)).await;
13543 actor.probe().await;
13544 }
13545
13546 #[tokio::test(start_paused = true)]
13547 async fn disconnect_disarms_and_reconnect_rearms_health() {
13548 let mut actor = Actor::start(ModuleProtocol::Subc, Duration::from_secs(30)).await;
13549 actor.hello(1, true);
13550 actor
13551 .wait_for_select(|checkpoint| {
13552 checkpoint.registered_connection == Some(ConnectionId::new(1))
13553 && checkpoint.next_probe_at.is_some()
13554 })
13555 .await;
13556 let turns = actor.turns();
13557 let tick = Instant::now();
13558 actor
13559 .registry
13560 .deregister_connection(ConnectionId::new(1))
13561 .unwrap();
13562 let disarmed = actor
13563 .wait_for_select(|checkpoint| {
13564 checkpoint.turn > turns && checkpoint.registered_connection.is_none()
13565 })
13566 .await;
13567 assert_eq!(disarmed.next_probe_at, None);
13568 assert_eq!(disarmed.wake_after, None);
13569 assert_eq!(
13570 Instant::now(),
13571 tick,
13572 "disconnect must disarm in the notification tick"
13573 );
13574 tokio::time::advance(Duration::from_secs(60)).await;
13575 actor.no_probe();
13576 let reconnect_tick = Instant::now();
13577 actor.hello(2, true);
13578 let rearmed = actor
13579 .wait_for_select(|checkpoint| {
13580 checkpoint.registered_connection == Some(ConnectionId::new(2))
13581 && checkpoint.next_probe_at.is_some()
13582 })
13583 .await;
13584 let delay = jittered_health_delay(&actor.spec.module_id, 0, actor.health.cadence);
13585 assert_eq!(rearmed.next_probe_at, Some(reconnect_tick + delay));
13586 tokio::time::advance(delay).await;
13587 actor.probe().await;
13588 }
13589
13590 #[tokio::test(start_paused = true)]
13591 async fn reregistration_without_health_disarms_the_old_schedule() {
13592 let mut actor = Actor::start(ModuleProtocol::Subc, Duration::from_secs(30)).await;
13593 actor.hello(1, true);
13594 actor
13595 .wait_for_select(|checkpoint| {
13596 checkpoint.registered_connection == Some(ConnectionId::new(1))
13597 && checkpoint.next_probe_at.is_some()
13598 })
13599 .await;
13600 actor
13601 .registry
13602 .deregister_connection(ConnectionId::new(1))
13603 .unwrap();
13604 actor.hello(2, false);
13605 let disarmed = actor
13606 .wait_for_select(|checkpoint| {
13607 checkpoint.registered_connection == Some(ConnectionId::new(2))
13608 })
13609 .await;
13610 assert_eq!(disarmed.next_probe_at, None);
13611 assert_eq!(disarmed.wake_after, None);
13612 tokio::time::advance(Duration::from_secs(60)).await;
13613 actor.no_probe();
13614 }
13615
13616 #[tokio::test(start_paused = true)]
13617 async fn restart_waits_for_its_own_hello_before_rearming_health() {
13618 let mut actor = Actor::start(ModuleProtocol::Subc, Duration::from_secs(30)).await;
13619 actor.hello(1, true);
13620 let armed = actor
13621 .wait_for_select(|checkpoint| {
13622 checkpoint.registered_connection == Some(ConnectionId::new(1))
13623 && checkpoint.next_probe_at.is_some()
13624 })
13625 .await;
13626 actor
13630 .registry
13631 .deregister_connection(ConnectionId::new(1))
13632 .unwrap();
13633 actor.module.restart(Some(0)).await.unwrap();
13634 let parked = actor
13635 .wait_for_select(|checkpoint| {
13636 checkpoint.generation > armed.generation
13637 && checkpoint.registered_connection.is_none()
13638 && checkpoint.next_probe_at.is_none()
13639 })
13640 .await;
13641 assert_eq!(parked.wake_after, None);
13642 actor.rx.close();
13643 while actor.rx.try_recv().is_ok() {}
13646 let turns = actor.turns();
13647 tokio::time::advance(Duration::from_secs(60)).await;
13648 let after = actor.turns();
13649 eprintln!(
13650 "restart park turns: before={turns} after={after} delta={}",
13651 after - turns
13652 );
13653 assert_eq!(after, turns, "replacement must park before HELLO");
13654 let hello_tick = Instant::now();
13655 actor.hello(2, true);
13656 let rearmed = actor
13657 .wait_for_select(|checkpoint| {
13658 checkpoint.registered_connection == Some(ConnectionId::new(2))
13659 && checkpoint.next_probe_at.is_some()
13660 })
13661 .await;
13662 let delay = jittered_health_delay(&actor.spec.module_id, 0, actor.health.cadence);
13663 assert_eq!(rearmed.next_probe_at, Some(hello_tick + delay));
13664 tokio::time::advance(delay).await;
13665 actor.probe().await;
13666 }
13667
13668 #[tokio::test(start_paused = true)]
13669 async fn rescan_adds_and_removes_http_health_without_polling() {
13670 use tokio::{
13671 io::{AsyncReadExt, AsyncWriteExt},
13672 net::TcpListener,
13673 };
13674 let mut actor = Actor::start(ModuleProtocol::None, Duration::from_secs(30)).await;
13675 let listener = TcpListener::bind("127.0.0.1:0").await.unwrap();
13676 actor.health.http = Some(format!("http://{}/health", listener.local_addr().unwrap()));
13677 actor.health.cadence = Duration::from_millis(1);
13679 let (started_tx, mut started_rx) = mpsc::channel(8);
13680 let peer = tokio::spawn(async move {
13681 loop {
13682 let (mut socket, _) = listener.accept().await.unwrap();
13683 let mut request = [0; 1024];
13684 assert!(socket.read(&mut request).await.unwrap() > 0);
13685 started_tx.send(()).await.unwrap();
13686 socket
13687 .write_all(b"HTTP/1.1 200 OK\r\nContent-Length: 0\r\n\r\n")
13688 .await
13689 .unwrap();
13690 }
13691 });
13692 let tick = Instant::now();
13693 let turns = actor.turns();
13694 actor
13695 .module
13696 .update_configuration(actor.spec.clone(), actor.health.clone(), None)
13697 .await
13698 .unwrap();
13699 let armed = actor
13700 .wait_for_select(|checkpoint| {
13701 checkpoint.turn > turns && checkpoint.next_probe_at.is_some()
13702 })
13703 .await;
13704 assert_eq!(armed.next_probe_at, Some(tick + Duration::from_millis(1)));
13705 assert_eq!(Instant::now(), tick);
13706 tokio::time::advance(Duration::from_millis(1)).await;
13707 let mut started = false;
13708 wait_for("successful HTTP probe", || {
13709 started |= started_rx.try_recv().is_ok();
13710 (started && actor.module.status().unwrap().health.status == SupervisorHealthStatus::Ok)
13711 .then_some(())
13712 })
13713 .await;
13714 assert!(started, "added HTTP check must start at its first deadline");
13715 assert_eq!(
13716 actor.module.status().unwrap().health.status,
13717 SupervisorHealthStatus::Ok
13718 );
13719 assert_eq!(Instant::now(), tick + Duration::from_millis(1));
13720 actor.health.http = None;
13721 let turns = actor.turns();
13722 actor
13723 .module
13724 .update_configuration(actor.spec.clone(), actor.health.clone(), None)
13725 .await
13726 .unwrap();
13727 let parked = actor
13728 .wait_for_select(|checkpoint| {
13729 checkpoint.turn > turns && checkpoint.next_probe_at.is_none()
13730 })
13731 .await;
13732 assert_eq!(parked.wake_after, None);
13733 let turns = actor.turns();
13734 tokio::time::advance(Duration::from_secs(60)).await;
13735 assert_eq!(
13736 actor.turns(),
13737 turns,
13738 "removed HTTP check must park the actor"
13739 );
13740 assert!(
13741 started_rx.try_recv().is_err(),
13742 "removed HTTP check must stay disarmed"
13743 );
13744 assert_eq!(
13745 actor.module.status().unwrap().health,
13746 ModuleHealthStatus::default()
13747 );
13748 peer.abort();
13749 }
13750}
13751
13752#[cfg(test)]
13753mod drain_busy_reprobe_tests {
13754 use super::*;
13758 use std::sync::Mutex as StdMutex;
13759 use subc_protocol::{manifest::Concurrency, session::ModuleControlResponse};
13760
13761 const MODULE_ID: &str = "drain-reprobe-module";
13762 const CONNECTION: u64 = 7;
13763 const GAUGE: &str = "runs_in_flight";
13764 const CADENCE: Duration = Duration::from_secs(30);
13767 const CEILING: Duration = Duration::from_secs(30);
13768
13769 #[derive(Clone, Copy)]
13771 enum Answer {
13772 BusyUntil { until: Instant, latency: Duration },
13775 Silent,
13777 }
13778
13779 struct Drain {
13780 forwarding: Arc<ForwardingTable>,
13781 runtime: SupervisorRuntimeConfig,
13782 endpoint: crate::ModuleEndpointId,
13783 probes: Arc<StdMutex<Vec<Instant>>>,
13785 responder: tokio::task::JoinHandle<()>,
13786 }
13787
13788 impl Drain {
13789 fn start(answer: Answer) -> Self {
13790 let registry = Arc::new(Registry::default());
13791 let forwarding = Arc::new(ForwardingTable::default());
13792 let health = HealthConfig {
13793 cadence: CADENCE,
13794 ..HealthConfig::default()
13795 };
13796 let runtime = Supervisor::new_for_test(registry, RestartPolicy::new(3, Duration::ZERO))
13797 .with_forwarding(forwarding.clone())
13798 .with_health_config(health)
13799 .runtime_config();
13800 let (tx, mut rx) = mpsc::channel::<crate::router::OutboundFrame>(8);
13801 let endpoint = forwarding
13802 .register_module_connection(
13803 ConnectionId::new(CONNECTION),
13804 MODULE_ID.to_string(),
13805 subc_protocol::PROTOCOL_VERSION,
13806 Concurrency::ModuleManaged,
13807 FrameSink::new(tx),
13808 )
13809 .unwrap();
13810 let probes = Arc::new(StdMutex::new(Vec::new()));
13811 let responder = tokio::spawn({
13812 let forwarding = forwarding.clone();
13813 let probes = probes.clone();
13814 async move {
13815 while let Some(frame) = rx.recv().await {
13816 let body: Value = serde_json::from_slice(&frame.body).unwrap();
13817 assert_eq!(body["op"], MODULE_CONTROL_OP_HEALTH_CHECK);
13818 probes.lock().unwrap().push(Instant::now());
13819 let Answer::BusyUntil { until, latency } = answer else {
13820 continue;
13821 };
13822 sleep(latency).await;
13823 let busy = u64::from(Instant::now() < until);
13824 let _ = forwarding.complete_module_control_rpc(
13827 ConnectionId::new(CONNECTION),
13828 frame.header.corr,
13829 Some(MODULE_CONTROL_OP_HEALTH_CHECK),
13830 ModuleControlRpcOutcome::Response(ModuleControlResponse::HealthCheck {
13831 status: HealthStatus::Ok,
13832 detail: None,
13833 metrics: Some(serde_json::json!({ GAUGE: busy })),
13834 }),
13835 );
13836 }
13837 }
13838 });
13839 Self {
13840 forwarding,
13841 runtime,
13842 endpoint,
13843 probes,
13844 responder,
13845 }
13846 }
13847
13848 async fn run(&self, ceiling: Duration) -> (bool, Duration) {
13851 let started = Instant::now();
13852 let drained = wait_for_forwarding_quiescence(
13853 &self.forwarding,
13854 MODULE_ID,
13855 &self.runtime,
13856 self.endpoint,
13857 started + ceiling,
13858 &[GAUGE.to_string()],
13859 DrainScope::Active,
13860 )
13861 .await
13862 .unwrap();
13863 (drained, started.elapsed())
13864 }
13865
13866 fn probes(&self) -> Vec<Instant> {
13867 self.probes.lock().unwrap().clone()
13868 }
13869 }
13870
13871 impl Drop for Drain {
13872 fn drop(&mut self) {
13873 self.responder.abort();
13874 }
13875 }
13876
13877 #[tokio::test(start_paused = true)]
13878 async fn busy_at_first_probe_then_quiet_drains_long_before_the_ceiling() {
13879 let drain = Drain::start(Answer::BusyUntil {
13880 until: Instant::now() + Duration::from_secs(1),
13881 latency: Duration::ZERO,
13882 });
13883 let (drained, elapsed) = drain.run(CEILING).await;
13884 assert!(
13885 drained,
13886 "gauges that read zero during the drain must let it finish drained, \
13887 not wait out the {CADENCE:?} health cadence (elapsed {elapsed:?})"
13888 );
13889 assert!(
13890 elapsed < Duration::from_secs(3),
13891 "drain took {elapsed:?} after the gauges went quiet at 1s"
13892 );
13893 let probes = drain.probes();
13894 assert!(
13895 probes.len() >= 2,
13896 "the module was busy at the first probe, so a quiet verdict needs a second: {probes:?}"
13897 );
13898 }
13899
13900 #[tokio::test(start_paused = true)]
13901 async fn busy_throughout_ends_undrained_at_the_ceiling_with_bounded_probes() {
13902 let started = Instant::now();
13903 let drain = Drain::start(Answer::BusyUntil {
13904 until: started + Duration::from_secs(3600),
13905 latency: Duration::ZERO,
13906 });
13907 let (drained, elapsed) = drain.run(CEILING).await;
13908 assert!(!drained, "a module busy throughout must not drain");
13909 assert!(
13910 elapsed >= CEILING && elapsed < CEILING + drain_busy_reprobe_interval(),
13911 "a busy module is held until the ceiling and no longer: {elapsed:?}"
13912 );
13913 let probes = drain.probes();
13916 let max_probes = (CEILING.as_millis() / drain_busy_reprobe_interval().as_millis()) + 1;
13917 assert!(
13918 probes.len() as u128 <= max_probes,
13919 "{} probes in a {CEILING:?} drain exceeds the {max_probes} the interval allows",
13920 probes.len()
13921 );
13922 for pair in probes.windows(2) {
13923 assert!(
13924 pair[1] - pair[0] >= drain_busy_reprobe_interval(),
13925 "consecutive probes {:?} apart",
13926 pair[1] - pair[0]
13927 );
13928 }
13929 }
13930
13931 #[tokio::test(start_paused = true)]
13932 async fn a_slow_probe_is_never_overlapped_by_the_next() {
13933 let latency = Duration::from_secs(1);
13936 let drain = Drain::start(Answer::BusyUntil {
13937 until: Instant::now() + Duration::from_secs(3600),
13938 latency,
13939 });
13940 let (drained, _) = drain.run(Duration::from_secs(5)).await;
13941 assert!(!drained);
13942 let probes = drain.probes();
13943 assert!(probes.len() >= 2, "the drain must re-probe: {probes:?}");
13944 for pair in probes.windows(2) {
13945 assert!(
13946 pair[1] - pair[0] >= latency + drain_busy_reprobe_interval(),
13947 "a probe started {:?} after the previous one, before it could finish",
13948 pair[1] - pair[0]
13949 );
13950 }
13951 }
13952
13953 #[tokio::test(start_paused = true)]
13954 async fn an_unanswered_probe_cannot_hold_the_drain_past_its_ceiling() {
13955 let drain = Drain::start(Answer::Silent);
13956 let ceiling = Duration::from_secs(2);
13957 let (drained, elapsed) = drain.run(ceiling).await;
13958 assert!(!drained, "an unanswered probe counts as busy");
13959 assert!(
13960 elapsed >= ceiling && elapsed < ceiling + drain_busy_reprobe_interval(),
13961 "the probe deadline is capped at the drain ceiling: {elapsed:?}"
13962 );
13963 assert_eq!(
13964 drain.probes().len(),
13965 1,
13966 "the unanswered probe holds the only slot until the ceiling"
13967 );
13968 }
13969}
13970
13971#[cfg(test)]
13972mod health_evidence_tests {
13973 use super::{HealthProbeError, HealthProbeEvidence};
13974 use std::collections::HashSet;
13975
13976 #[test]
13984 fn only_a_dead_lane_is_proof_of_death() {
13985 assert!(HealthProbeError::lane_dead("gone").is_proof_of_death());
13986 assert!(!HealthProbeError::no_answer("timed out").is_proof_of_death());
13990 assert!(!HealthProbeError::bad_answer("garbage").is_proof_of_death());
13991 assert!(!HealthProbeError::misconfigured("no table").is_proof_of_death());
13992 }
13993
13994 #[test]
14000 fn every_evidence_class_has_a_distinct_label() {
14001 let labels = [
14002 HealthProbeError::lane_dead("").label(),
14003 HealthProbeError::no_answer("").label(),
14004 HealthProbeError::bad_answer("").label(),
14005 HealthProbeError::misconfigured("").label(),
14006 ];
14007 let unique: HashSet<_> = labels.iter().collect();
14008 assert_eq!(unique.len(), labels.len(), "labels collided: {labels:?}");
14009 }
14010
14011 #[test]
14017 fn classification_preserves_the_original_message() {
14018 let err = HealthProbeError::no_answer("module did not answer within 5s");
14019 assert_eq!(err.to_string(), "module did not answer within 5s");
14020 assert!(matches!(err.evidence, HealthProbeEvidence::NoAnswer));
14021 }
14022}
14023
14024#[cfg(test)]
14025mod health_tombstone_tests {
14026 use std::{path::PathBuf, sync::Arc, time::Duration};
14027
14028 use subc_protocol::{
14029 manifest::Concurrency,
14030 session::{HealthStatus, ModuleControlResponse},
14031 };
14032 use tokio::sync::mpsc;
14033
14034 use super::{
14035 probe_module_health, HealthAction, HealthConfig, HealthProbeEvidence, ModuleProtocol,
14036 ModuleSpec, RestartPolicy, Supervisor, SupervisorRuntimeConfig,
14037 };
14038 use crate::{
14039 control::ControlHandler,
14040 forwarding::{ForwardingTable, ModuleControlRpcCompletion, ModuleControlRpcOutcome},
14041 registry::{ConnectionId, Registry},
14042 router::FrameSink,
14043 };
14044
14045 struct ProbeHarness {
14046 spec: ModuleSpec,
14047 runtime: SupervisorRuntimeConfig,
14048 forwarding: Arc<ForwardingTable>,
14049 module_connection: ConnectionId,
14050 module_rx: mpsc::Receiver<crate::router::OutboundFrame>,
14051 handler: ControlHandler,
14052 module: super::SupervisedModule,
14053 }
14054
14055 fn probe_harness() -> ProbeHarness {
14056 let registry = Arc::new(Registry::default());
14057 let forwarding = Arc::new(ForwardingTable::default());
14058 let supervisor_handle = super::SupervisorHandle::new();
14059 let health = HealthConfig {
14060 http: None,
14061 cadence: Duration::from_secs(30),
14062 deadline: Duration::from_secs(5),
14063 failure_threshold: 3,
14064 on_degraded: HealthAction::Report,
14065 on_failing: HealthAction::Report,
14066 critical: false,
14067 };
14068 let supervisor = Supervisor::new_for_test(Arc::clone(®istry), RestartPolicy::default())
14069 .with_forwarding(Arc::clone(&forwarding))
14070 .with_handle(supervisor_handle.clone())
14071 .with_health_config(health);
14072 let spec = ModuleSpec {
14073 module_id: "late-health-module".to_string(),
14074 program: PathBuf::from("disabled-module"),
14075 args: Vec::new(),
14076 env: Vec::new(),
14077 reserved: false,
14078 reserved_prefixes: Vec::new(),
14079 protocol: ModuleProtocol::Subc,
14080 overlap: Default::default(),
14081 };
14082 let module = supervisor
14083 .supervise_configured(spec.clone(), false)
14084 .unwrap();
14085 let runtime = supervisor.runtime_config();
14086 let handler = ControlHandler::with_forwarding(registry, Arc::clone(&forwarding))
14087 .with_supervisor(supervisor_handle);
14088 let module_connection = ConnectionId::new(700);
14089 let (module_tx, module_rx) = mpsc::channel(8);
14090 forwarding
14091 .register_module_connection(
14092 module_connection,
14093 spec.module_id.clone(),
14094 subc_protocol::PROTOCOL_VERSION,
14095 Concurrency::ModuleManaged,
14096 FrameSink::new(module_tx),
14097 )
14098 .unwrap();
14099
14100 ProbeHarness {
14101 spec,
14102 runtime,
14103 forwarding,
14104 module_connection,
14105 module_rx,
14106 handler,
14107 module,
14108 }
14109 }
14110
14111 async fn finish_after(
14112 harness: &mut ProbeHarness,
14113 stall: Duration,
14114 ) -> ModuleControlRpcCompletion {
14115 assert!(stall > harness.runtime.health.deadline);
14116 let deadline = harness.runtime.health.deadline;
14117 let probe = probe_module_health(&harness.spec.module_id, &harness.runtime, None);
14118 let answer = async {
14119 let frame = harness.module_rx.recv().await.expect("health.check frame");
14120 tokio::time::advance(deadline).await;
14121 tokio::task::yield_now().await;
14122 tokio::time::advance(stall - deadline).await;
14123 harness
14124 .forwarding
14125 .complete_module_control_rpc(
14126 harness.module_connection,
14127 frame.header.corr,
14128 Some("health.check"),
14129 ModuleControlRpcOutcome::Response(ModuleControlResponse::HealthCheck {
14130 status: HealthStatus::Ok,
14131 detail: None,
14132 metrics: None,
14133 }),
14134 )
14135 .unwrap()
14136 };
14137 let (probe_result, completion) = tokio::join!(probe, answer);
14138 let err = probe_result.expect_err("probe must miss its deadline");
14139 assert!(matches!(err.evidence, HealthProbeEvidence::NoAnswer));
14140 completion
14141 }
14142
14143 async fn time_out_without_answer(harness: &mut ProbeHarness) {
14144 let deadline = harness.runtime.health.deadline;
14145 let probe = probe_module_health(&harness.spec.module_id, &harness.runtime, None);
14146 let exhaust_deadline = async {
14147 let _frame = harness.module_rx.recv().await.expect("health.check frame");
14148 tokio::time::advance(deadline).await;
14149 tokio::task::yield_now().await;
14150 };
14151 let (probe_result, ()) = tokio::join!(probe, exhaust_deadline);
14152 let err = probe_result.expect_err("probe must miss its deadline");
14153 assert!(matches!(err.evidence, HealthProbeEvidence::NoAnswer));
14154 }
14155
14156 async fn run_probe_cycle(harness: &mut ProbeHarness, answer: bool) {
14157 let registry = Arc::clone(&harness.module.inner.registry);
14158 let snapshot = Arc::clone(&harness.module.inner.snapshot);
14159 let process_liveness = super::SupervisorProcessLiveness::default();
14160 let mut child = None;
14161 let cycle = super::run_health_probe_cycle(
14162 &harness.spec,
14163 &harness.runtime,
14164 ®istry,
14165 &process_liveness,
14166 &snapshot,
14167 &mut child,
14168 );
14169 let peer = async {
14170 let frame = harness.module_rx.recv().await.expect("health.check frame");
14171 if answer {
14172 harness
14173 .forwarding
14174 .complete_module_control_rpc(
14175 harness.module_connection,
14176 frame.header.corr,
14177 Some("health.check"),
14178 ModuleControlRpcOutcome::Response(ModuleControlResponse::HealthCheck {
14179 status: HealthStatus::Ok,
14180 detail: None,
14181 metrics: Some(serde_json::json!({"ready": true})),
14182 }),
14183 )
14184 .unwrap();
14185 } else {
14186 tokio::time::advance(harness.runtime.health.deadline).await;
14187 tokio::task::yield_now().await;
14188 }
14189 };
14190 tokio::join!(cycle, peer);
14191 }
14192
14193 #[tokio::test(start_paused = true)]
14194 async fn unanswered_probe_is_unknown_until_threshold_and_ok_report_recovers() {
14195 let mut harness = probe_harness();
14196 let monitor = harness.module.inner.monitor.lock().unwrap().take().unwrap();
14200 monitor.abort();
14201 let _ = monitor.await;
14202 super::update_snapshot(&harness.module.inner.snapshot, None, |state| {
14203 state.enabled = true;
14204 state.state = super::ModuleState::Running;
14205 state.process_alive = true;
14206 })
14207 .unwrap();
14208
14209 run_probe_cycle(&mut harness, true).await;
14210 assert_eq!(
14211 harness.module.status().unwrap().health.status,
14212 super::SupervisorHealthStatus::Ok
14213 );
14214
14215 for failures in 1..harness.runtime.health.failure_threshold {
14216 run_probe_cycle(&mut harness, false).await;
14217 let status = harness.module.status().unwrap();
14218 assert_eq!(status.health.status, super::SupervisorHealthStatus::Unknown);
14219 assert_eq!(status.health.consecutive_failures, failures);
14220 assert!(status.health.last_probe_ms.is_some());
14221 assert!(status.health.detail.unwrap().starts_with("[no-answer]"));
14222 assert!(status.health.metrics.is_none());
14223 assert_eq!(status.state, super::ModuleState::Running);
14224 assert!(status.process_alive);
14225 assert_eq!(status.restart_count, 0);
14226 assert_eq!(status.lifetime_restarts, 0);
14227 assert!(status.health.last_action.is_none());
14228 assert!(harness.runtime.scheduled_respawn.lock().unwrap().is_none());
14229 }
14230
14231 run_probe_cycle(&mut harness, true).await;
14232 let recovered = harness.module.status().unwrap();
14233 assert_eq!(recovered.health.status, super::SupervisorHealthStatus::Ok);
14234 assert_eq!(recovered.health.consecutive_failures, 0);
14235 assert!(recovered.health.detail.is_none());
14236 assert_eq!(
14237 recovered.health.metrics,
14238 Some(serde_json::json!({"ready": true}))
14239 );
14240 assert_eq!(recovered.lifetime_restarts, 0);
14241
14242 for failures in 1..=harness.runtime.health.failure_threshold {
14243 run_probe_cycle(&mut harness, false).await;
14244 let status = harness.module.status().unwrap();
14245 assert_eq!(status.health.consecutive_failures, failures);
14246 if failures < harness.runtime.health.failure_threshold {
14247 assert_eq!(status.health.status, super::SupervisorHealthStatus::Unknown);
14248 assert_eq!(status.state, super::ModuleState::Running);
14249 assert_eq!(status.lifetime_restarts, 0);
14250 assert!(harness.runtime.scheduled_respawn.lock().unwrap().is_none());
14251 } else {
14252 assert_eq!(
14253 status.health.status,
14254 super::SupervisorHealthStatus::Unresponsive
14255 );
14256 assert_eq!(status.state, super::ModuleState::Restarting);
14257 assert_eq!(status.restart_count, 1);
14258 assert_eq!(status.lifetime_restarts, 1);
14259 assert_eq!(status.health.last_action.as_deref(), Some("restart"));
14260 assert!(harness.runtime.scheduled_respawn.lock().unwrap().is_some());
14261 }
14262 }
14263 }
14264
14265 #[tokio::test(start_paused = true)]
14266 async fn late_health_answers_record_start_anchored_latency_for_two_stalls() {
14267 let mut harness = probe_harness();
14268
14269 let first = finish_after(&mut harness, Duration::from_secs(8)).await;
14270 let first_latency = match &first {
14271 ModuleControlRpcCompletion::LateHealthAnswer { latency, .. } => *latency,
14272 other => panic!("late answer was not retained: {other:?}"),
14273 };
14274 assert!(harness.handler.observe_module_control_completion(first));
14275
14276 let second = finish_after(&mut harness, Duration::from_secs(11)).await;
14277 let second_latency = match &second {
14278 ModuleControlRpcCompletion::LateHealthAnswer { latency, .. } => *latency,
14279 other => panic!("late answer was not retained: {other:?}"),
14280 };
14281 assert!(harness.handler.observe_module_control_completion(second));
14282
14283 assert_eq!(first_latency, Duration::from_secs(8));
14284 assert_eq!(
14285 second_latency - first_latency,
14286 Duration::from_secs(3),
14287 "latency must grow linearly with the additional stall"
14288 );
14289 let health = harness.module.status().unwrap().health;
14290 assert_eq!(health.late_answer_count, 2);
14291 assert_eq!(health.last_late_answer_latency_ms, Some(11_000));
14292 }
14293
14294 #[tokio::test(start_paused = true)]
14302 async fn late_answer_clears_the_consecutive_failure_streak() {
14303 let mut harness = probe_harness();
14304
14305 time_out_without_answer(&mut harness).await;
14307 harness
14308 .module
14309 .record_health_probe_failure_for_test("[no-answer] test miss")
14310 .unwrap();
14311 assert_eq!(
14312 harness.module.status().unwrap().health.consecutive_failures,
14313 1,
14314 "precondition: the miss must be on the streak before the late answer"
14315 );
14316
14317 let late = finish_after(&mut harness, Duration::from_secs(9)).await;
14319 assert!(matches!(
14320 late,
14321 ModuleControlRpcCompletion::LateHealthAnswer { .. }
14322 ));
14323 assert!(harness.handler.observe_module_control_completion(late));
14324
14325 let health = harness.module.status().unwrap().health;
14326 assert_eq!(
14327 health.consecutive_failures, 0,
14328 "a late answer is an answer: the streak must reset"
14329 );
14330 assert_eq!(health.late_answer_count, 1);
14331 }
14332
14333 #[tokio::test(start_paused = true)]
14334 async fn repeated_serial_probe_cycles_keep_one_tombstone_per_endpoint() {
14335 let mut harness = probe_harness();
14336
14337 for _ in 0..20 {
14338 time_out_without_answer(&mut harness).await;
14339 assert_eq!(
14340 harness.forwarding.health_probe_tombstone_count().unwrap(),
14341 1
14342 );
14343 }
14344 }
14345}
14346
14347#[cfg(test)]
14348mod child_env_tests {
14349 #[cfg(windows)]
14350 use super::{
14351 clear_current_process_facts, observed_launch_nonce_source, ModuleState, SupervisorSnapshot,
14352 };
14353 #[cfg(windows)]
14354 use std::sync::{Arc, OnceLock};
14355 #[cfg(windows)]
14356 #[test]
14357 fn windows_wire_spawn_prepares_pipe_and_keeps_phase_one_env() {
14358 let spec = spec(Vec::new());
14359 let mut command = Command::new("unused");
14360 let handoff = apply_wire_spawn_args(&mut command, &spec, None, None)
14361 .unwrap()
14362 .unwrap();
14363 let name = command
14364 .as_std()
14365 .get_envs()
14366 .find(|(key, _)| *key == OsStr::new(subc_os::launch_nonce::LAUNCH_NONCE_PIPE_ENV))
14367 .and_then(|(_, value)| value)
14368 .unwrap();
14369 assert_eq!(name, OsStr::new(handoff.name()));
14370 assert!(handoff.name().starts_with(r"\\.\pipe\subc-launch-"));
14371 assert!(command.as_std().get_envs().any(|(key, value)| key
14372 == OsStr::new(subc_os::LAUNCH_NONCE_PIPE_FALLBACK_ENV)
14373 && value == Some(OsStr::new("env"))));
14374 assert!(command
14375 .as_std()
14376 .get_envs()
14377 .any(|(key, value)| key == OsStr::new(SUBC_LAUNCH_NONCE_ENV) && value.is_some()));
14378 }
14379
14380 #[cfg(windows)]
14381 #[test]
14382 fn windows_source_reporting_tracks_consumption_and_resets_per_spawn() {
14383 let mut snapshot = SupervisorSnapshot::new(ModuleState::Running, true);
14384 snapshot.process_alive = true;
14385 snapshot.spawned_protocol = Some(ModuleProtocol::Subc);
14386 assert_eq!(
14387 observed_launch_nonce_source(&snapshot).as_deref(),
14388 Some("env")
14389 );
14390 let consumed = Arc::new(OnceLock::new());
14391 snapshot.pipe_consumed = Some(consumed.clone());
14392 assert_eq!(
14393 observed_launch_nonce_source(&snapshot).as_deref(),
14394 Some("env")
14395 );
14396 consumed.set(()).unwrap();
14397 assert_eq!(
14398 observed_launch_nonce_source(&snapshot).as_deref(),
14399 Some("pipe")
14400 );
14401 snapshot.pipe_consumed = Some(Arc::new(OnceLock::new()));
14402 assert_eq!(
14403 observed_launch_nonce_source(&snapshot).as_deref(),
14404 Some("env")
14405 );
14406 clear_current_process_facts(&mut snapshot);
14407 assert_eq!(observed_launch_nonce_source(&snapshot), None);
14408 }
14409 use super::{
14410 apply_child_env, apply_spawn_role, apply_wire_spawn_args, LaunchSpecDigest, ModuleProtocol,
14411 ModuleSpec, SpawnRole, SupervisorHandle, SPAWN_ROLE_SWAP_CANDIDATE, SUBC_ARG,
14412 SUBC_LAUNCH_NONCE_ENV, SUBC_MODULE_ID_ENV, SUBC_SPAWN_ROLE_ENV,
14413 };
14414 use std::{ffi::OsStr, path::PathBuf};
14415 use tokio::process::Command;
14416
14417 fn spec(env: Vec<(String, String)>) -> ModuleSpec {
14418 ModuleSpec {
14419 module_id: "env-plan".to_string(),
14420 program: PathBuf::from("/nonexistent"),
14421 args: Vec::new(),
14422 env,
14423 reserved: false,
14424 reserved_prefixes: Vec::new(),
14425 protocol: ModuleProtocol::Subc,
14426 overlap: Default::default(),
14427 }
14428 }
14429
14430 #[test]
14431 fn configured_launch_spec_digest_is_canonical_and_excludes_injected_inputs() {
14432 let base = spec(vec![("A".into(), "B=C".into()), ("Z".into(), "v".into())]);
14433 let digest = LaunchSpecDigest::from_spec(&base);
14434 let mut reordered = base.clone();
14435 reordered.env.reverse();
14436 reordered.module_id = "different-id".into();
14437 reordered.reserved = true;
14438 reordered.overlap = Default::default();
14439 assert_eq!(digest, LaunchSpecDigest::from_spec(&reordered));
14440 let mut delimiter_collision = base.clone();
14441 delimiter_collision.env[0] = ("A=B".into(), "C".into());
14442 assert_ne!(digest, LaunchSpecDigest::from_spec(&delimiter_collision));
14443 let mut changed = base.clone();
14444 changed.protocol = ModuleProtocol::None;
14445 assert_ne!(digest, LaunchSpecDigest::from_spec(&changed));
14446 changed = base.clone();
14447 changed.args.push("--flag".into());
14448 assert_ne!(digest, LaunchSpecDigest::from_spec(&changed));
14449 changed = base.clone();
14450 changed.program = PathBuf::from("/other");
14451 assert_ne!(digest, LaunchSpecDigest::from_spec(&changed));
14452 changed = base.clone();
14453 changed.env[1].1 = "other".into();
14454 assert_ne!(digest, LaunchSpecDigest::from_spec(&changed));
14455 assert!(!format!("{digest:?}").contains("B=C"));
14456 assert!(!format!("{digest:?}").contains("[u8"));
14457 }
14458
14459 #[cfg(unix)]
14460 #[test]
14461 fn native_program_bytes_do_not_collapse_through_lossy_utf8() {
14462 use std::os::unix::ffi::OsStringExt;
14463 let mut left = spec(Vec::new());
14464 left.program = std::ffi::OsString::from_vec(vec![b'/', 0xff]).into();
14465 let mut right = left.clone();
14466 right.program = std::ffi::OsString::from_vec(vec![b'/', 0xfe]).into();
14467 assert_eq!(
14468 left.program.to_string_lossy(),
14469 right.program.to_string_lossy()
14470 );
14471 assert_ne!(
14472 LaunchSpecDigest::from_spec(&left),
14473 LaunchSpecDigest::from_spec(&right)
14474 );
14475 }
14476
14477 #[test]
14491 fn ambient_ck_log_is_removed_and_a_configured_one_survives() {
14492 let mut command = Command::new("/nonexistent");
14493 apply_child_env(&mut command, &spec(Vec::new()));
14494 let removed = command
14495 .as_std()
14496 .get_envs()
14497 .any(|(key, value)| key == OsStr::new("CK_LOG") && value.is_none());
14498 assert!(
14499 removed,
14500 "ambient CK_LOG must be explicitly removed for an unconfigured module"
14501 );
14502
14503 let mut configured = Command::new("/nonexistent");
14504 apply_child_env(
14505 &mut configured,
14506 &spec(vec![("CK_LOG".to_string(), "debug".to_string())]),
14507 );
14508 let effective = configured
14509 .as_std()
14510 .get_envs()
14511 .filter(|(key, _)| *key == OsStr::new("CK_LOG"))
14512 .last()
14513 .map(|(_, value)| value.map(|v| v.to_string_lossy().into_owned()));
14514 assert_eq!(
14515 effective,
14516 Some(Some("debug".to_string())),
14517 "a module's configured CK_LOG must survive the ambient removal"
14518 );
14519 }
14520
14521 #[test]
14530 fn protocol_none_spawn_carries_no_subc_argument_and_no_nonce() {
14531 let connection_file = std::path::Path::new("/run/subc-connection.json");
14532 let handle = SupervisorHandle::new();
14533
14534 let mut none_spec = spec(Vec::new());
14535 none_spec.protocol = ModuleProtocol::None;
14536 let mut none = Command::new("/nonexistent");
14537 let none_handoff =
14538 apply_wire_spawn_args(&mut none, &none_spec, Some(connection_file), Some(&handle))
14539 .expect("protocol-none spawn args apply");
14540 assert!(
14541 none_handoff.is_none(),
14542 "protocol:none spawn must not receive a nonce descriptor"
14543 );
14544 #[cfg(windows)]
14545 assert!(!none.as_std().get_envs().any(|(key, value)| key
14546 == OsStr::new(subc_os::LAUNCH_NONCE_PIPE_FALLBACK_ENV)
14547 && value.is_some()));
14548 assert!(
14549 !none.as_std().get_envs().any(|(key, value)| key
14550 == OsStr::new(subc_os::LAUNCH_NONCE_FD_ENV)
14551 && value.is_some()),
14552 "protocol:none spawn must not name a nonce descriptor"
14553 );
14554 let none_args: Vec<String> = none
14555 .as_std()
14556 .get_args()
14557 .map(|a| a.to_string_lossy().into_owned())
14558 .collect();
14559 assert!(
14560 !none_args.iter().any(|a| a == SUBC_ARG),
14561 "protocol:none argv must not carry --subc; got {none_args:?}"
14562 );
14563 let none_has_nonce = none
14564 .as_std()
14565 .get_envs()
14566 .any(|(key, value)| key == OsStr::new(SUBC_LAUNCH_NONCE_ENV) && value.is_some());
14567 assert!(
14568 !none_has_nonce,
14569 "protocol:none spawn must not receive a launch nonce"
14570 );
14571 let none_has_module_id = none
14572 .as_std()
14573 .get_envs()
14574 .any(|(key, value)| key == OsStr::new(SUBC_MODULE_ID_ENV) && value.is_some());
14575 assert!(
14576 none_has_module_id,
14577 "SUBC_MODULE_ID is inert and stays on every path"
14578 );
14579 assert!(
14580 handle.spawn_nonce(&none_spec.module_id).is_none(),
14581 "no nonce record for a process that will never present one"
14582 );
14583
14584 let wire_spec = spec(Vec::new());
14586 let mut wire = Command::new("/nonexistent");
14587 let wire_handoff =
14588 apply_wire_spawn_args(&mut wire, &wire_spec, Some(connection_file), Some(&handle))
14589 .expect("subc-wire spawn args apply");
14590 let wire_fd_env = wire
14591 .as_std()
14592 .get_envs()
14593 .find(|(key, _)| *key == OsStr::new(subc_os::LAUNCH_NONCE_FD_ENV))
14594 .and_then(|(_, value)| value.map(|v| v.to_string_lossy().into_owned()));
14595 #[cfg(unix)]
14596 assert_eq!(
14597 wire_fd_env,
14598 Some(wire_handoff.expect("a descriptor handoff").fd_env_value()),
14599 "a subc-wire spawn names the pipe it will receive at descriptor 3"
14600 );
14601 #[cfg(windows)]
14602 assert!(wire_handoff.is_some() && wire_fd_env.is_none());
14603 #[cfg(not(any(unix, windows)))]
14604 assert!(wire_handoff.is_none() && wire_fd_env.is_none());
14605 let wire_args: Vec<String> = wire
14606 .as_std()
14607 .get_args()
14608 .map(|a| a.to_string_lossy().into_owned())
14609 .collect();
14610 assert_eq!(
14611 wire_args,
14612 vec![
14613 SUBC_ARG.to_string(),
14614 connection_file.to_string_lossy().into_owned()
14615 ],
14616 "a subc-wire spawn still carries --subc <path>"
14617 );
14618 assert_eq!(
14619 wire.as_std()
14620 .get_envs()
14621 .any(|(key, value)| key == OsStr::new(SUBC_LAUNCH_NONCE_ENV) && value.is_some()),
14622 !cfg!(unix),
14623 "only Windows supplies the environment nonce"
14624 );
14625 assert!(handle.spawn_nonce(&wire_spec.module_id).is_some());
14626 }
14627
14628 #[test]
14638 fn plain_spawn_removes_the_spawn_role_even_when_the_spec_sets_it() {
14639 let role = |command: &Command| {
14640 command
14641 .as_std()
14642 .get_envs()
14643 .filter(|(key, _)| *key == OsStr::new(SUBC_SPAWN_ROLE_ENV))
14644 .last()
14645 .map(|(_, value)| value.map(|v| v.to_string_lossy().into_owned()))
14646 };
14647 let forged = spec(vec![(
14648 SUBC_SPAWN_ROLE_ENV.to_string(),
14649 SPAWN_ROLE_SWAP_CANDIDATE.to_string(),
14650 )]);
14651
14652 let mut plain = Command::new("/nonexistent");
14653 apply_child_env(&mut plain, &forged);
14654 apply_spawn_role(&mut plain, SpawnRole::Plain);
14655 assert_eq!(
14656 role(&plain),
14657 Some(None),
14658 "a plain spawn must remove SUBC_SPAWN_ROLE, whatever the spec says"
14659 );
14660
14661 let mut candidate = Command::new("/nonexistent");
14662 apply_child_env(&mut candidate, &spec(Vec::new()));
14663 apply_spawn_role(&mut candidate, SpawnRole::SwapCandidate);
14664 assert_eq!(
14665 role(&candidate),
14666 Some(Some(SPAWN_ROLE_SWAP_CANDIDATE.to_string()))
14667 );
14668 }
14669
14670 #[test]
14676 fn daemon_private_capture_keys_are_not_passed_to_the_child() {
14677 let mut command = Command::new("/nonexistent");
14678 apply_child_env(
14679 &mut command,
14680 &spec(vec![
14681 (super::CAPTURE_KEEP_ENV.to_string(), "5".to_string()),
14682 ("KEPT".to_string(), "yes".to_string()),
14683 ]),
14684 );
14685 let keys: Vec<String> = command
14686 .as_std()
14687 .get_envs()
14688 .filter(|(_, value)| value.is_some())
14689 .map(|(key, _)| key.to_string_lossy().into_owned())
14690 .collect();
14691 assert!(keys.contains(&"KEPT".to_string()), "got {keys:?}");
14692 assert!(
14693 !keys.contains(&super::CAPTURE_KEEP_ENV.to_string()),
14694 "daemon-private capture key leaked to the child: {keys:?}"
14695 );
14696 }
14697}
14698
14699#[cfg(test)]
14700mod jitter_tests {
14701 use super::jittered_health_delay;
14702 use std::{collections::HashSet, time::Duration};
14703
14704 const FLEET: [&str; 14] = [
14713 "aft",
14714 "alfonso-core",
14715 "magic-context",
14716 "broca",
14717 "thalamus",
14718 "quota",
14719 "engram",
14720 "plexus",
14721 "cerebellum",
14722 "astrocyte",
14723 "synapse",
14724 "subc-mcp",
14725 "cortexkit-credentials",
14726 "subc-federation",
14727 ];
14728
14729 #[test]
14737 fn probe_delays_disperse_across_the_fleet() {
14738 let cadence = Duration::from_secs(30);
14739 let delays: HashSet<Duration> = FLEET
14740 .iter()
14741 .map(|id| jittered_health_delay(id, 0, cadence))
14742 .collect();
14743 assert_eq!(
14744 delays.len(),
14745 FLEET.len(),
14746 "every supervised module must land on its own probe offset"
14747 );
14748 }
14749
14750 #[test]
14756 fn jitter_only_delays_and_stays_within_one_tenth_of_cadence() {
14757 let cadence = Duration::from_secs(30);
14758 let span = cadence / 10;
14759 for id in FLEET {
14760 for probe_index in 0..8 {
14761 let delay = jittered_health_delay(id, probe_index, cadence);
14762 assert!(
14763 delay >= cadence,
14764 "{id}#{probe_index}: jitter must not shorten the cadence"
14765 );
14766 assert!(
14767 delay < cadence + span,
14768 "{id}#{probe_index}: jitter must stay inside one tenth of the cadence"
14769 );
14770 }
14771 }
14772 }
14773
14774 #[test]
14780 fn a_module_offset_is_stable_across_restarts() {
14781 let cadence = Duration::from_secs(30);
14782 for id in FLEET {
14783 assert_eq!(
14784 jittered_health_delay(id, 0, cadence),
14785 jittered_health_delay(id, 0, cadence),
14786 "{id}: the same module and probe index must produce the same offset"
14787 );
14788 }
14789 }
14790
14791 #[test]
14793 fn zero_cadence_yields_zero_delay() {
14794 assert_eq!(
14795 jittered_health_delay("aft", 0, Duration::ZERO),
14796 Duration::ZERO
14797 );
14798 }
14799}
14800
14801#[cfg(all(test, target_os = "linux"))]
14802mod cgroup_placement_tests {
14803 use super::{
14804 apply_cgroup_placement, remove_module_cgroup, LaunchSpecDigest, ModuleProtocol, ModuleSpec,
14805 SuperviseError, SupervisedChild,
14806 };
14807 use crate::stderr_tail::{StderrRing, StderrTailConfig};
14808 use cortexkit_test_support::ScratchDir;
14809 use std::{
14810 fs, io,
14811 path::{Path, PathBuf},
14812 sync::{Arc, Mutex},
14813 };
14814 use tokio::process::Command;
14815
14816 #[tokio::test]
14817 async fn unique_spawn_cgroups_clean_up_across_restarts_and_shutdown() {
14818 use super::*;
14819 let dir = ScratchDir::new("unique-spawn-cgroups");
14820 let root = PathBuf::from(format!(
14821 "/sys/fs/cgroup/subc-unique-test-{}-{}",
14822 std::process::id(),
14823 unix_ms_now()
14824 ));
14825 if let Err(error) = fs::create_dir(&root) {
14826 assert!(
14827 std::env::var_os("SUBC_REQUIRE_CGROUP_TEST").is_none(),
14828 "required cgroup test cannot execute: {error}"
14829 );
14830 eprintln!("SKIP unique_spawn_cgroups_clean_up_across_restarts_and_shutdown: {error}");
14831 return;
14832 }
14833 let placement = subc_cgroup::prepare_at(&root).unwrap().unwrap();
14834 let group_count = || {
14835 fs::read_dir(root.join("subc-modules"))
14836 .unwrap()
14837 .map(|entry| entry.unwrap().file_type().unwrap())
14838 .filter(|kind| kind.is_dir())
14839 .count()
14840 };
14841 let supervisor =
14842 Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default())
14843 .with_cgroup_placement(Some(placement.clone()));
14844 let runtime = supervisor.runtime_config();
14845 let mut spec = ModuleSpec {
14846 module_id: "unique-spawn".into(),
14847 program: PathBuf::from("/bin/sleep"),
14848 args: vec!["60".into()],
14849 env: ["XDG_DATA_HOME", "XDG_RUNTIME_DIR", "XDG_CONFIG_HOME"]
14850 .into_iter()
14851 .map(|key| (key.into(), dir.to_string_lossy().into_owned()))
14852 .collect(),
14853 reserved: false,
14854 reserved_prefixes: vec![],
14855 protocol: ModuleProtocol::None,
14856 overlap: Default::default(),
14857 };
14858 let spawn = |spec: &ModuleSpec| {
14859 spawn_child(
14860 spec,
14861 None,
14862 None,
14863 &runtime.stderr_ring,
14864 None,
14865 &runtime.child_roster,
14866 Some(&placement),
14867 )
14868 .unwrap()
14869 };
14870 let mut live = spawn(&spec);
14871 for _ in 0..3 {
14872 let next = spawn(&spec);
14874 assert_ne!(live.module_id, next.module_id);
14875 live.start_kill().unwrap();
14876 live.wait().await.unwrap();
14877 live = next;
14878 assert!(
14879 live.child.try_wait().unwrap().is_none(),
14880 "retiring the old slot must not kill the replacement"
14881 );
14882 assert_eq!(
14883 group_count(),
14884 1,
14885 "only the live spawn's cgroup should remain"
14886 );
14887 }
14888 supervisor.begin_daemon_shutdown();
14889 let reap = tokio::spawn(async move {
14890 live.wait().await.unwrap();
14891 });
14892 supervisor
14893 .end_children_for_daemon_shutdown(false, std::future::pending())
14894 .await;
14895 reap.await.unwrap();
14896 assert_eq!(group_count(), 0);
14897 spec.program = PathBuf::from("/bin/true");
14899 spec.args.clear();
14900 let fresh_roster = ChildRoster::default();
14901 let mut short = spawn_child(
14902 &spec,
14903 None,
14904 None,
14905 &runtime.stderr_ring,
14906 None,
14907 &fresh_roster,
14908 Some(&placement),
14909 )
14910 .unwrap();
14911 short.wait().await.unwrap();
14912 assert_eq!(group_count(), 0);
14913 spec.module_id = "_".repeat(255);
14914 let mut long_id = spawn_child(
14915 &spec,
14916 None,
14917 None,
14918 &runtime.stderr_ring,
14919 None,
14920 &fresh_roster,
14921 Some(&placement),
14922 )
14923 .unwrap();
14924 long_id.wait().await.unwrap();
14925 assert_eq!(
14926 group_count(),
14927 0,
14928 "valid long module IDs must not exceed cgroup NAME_MAX"
14929 );
14930 fs::remove_dir(root.join("subc-modules")).unwrap();
14931 fs::remove_dir(root).unwrap();
14932 eprintln!("EXECUTED unique_spawn_cgroups_clean_up_across_restarts_and_shutdown");
14933 }
14934
14935 #[test]
14936 fn failed_parent_cgroup_open_is_a_cgroup_supervision_error() {
14937 let path = Path::new("/definitely-missing-subc-cgroup");
14938 let mut command = Command::new("true");
14939 let error = apply_cgroup_placement(
14940 &mut command,
14941 &ModuleSpec {
14942 module_id: "broken-cgroup".to_string(),
14943 program: PathBuf::from("true"),
14944 args: Vec::new(),
14945 env: Vec::new(),
14946 reserved: false,
14947 reserved_prefixes: Vec::new(),
14948 protocol: ModuleProtocol::Subc,
14949 overlap: Default::default(),
14950 },
14951 path,
14952 )
14953 .expect_err("a parent cgroup open failure must reject the supervised spawn");
14954 let reason = error.to_string();
14955
14956 assert!(
14957 matches!(error, SuperviseError::Cgroup { .. }),
14958 "parent cgroup open must be reported as a cgroup supervision error: {reason}"
14959 );
14960 assert!(
14961 reason.contains("/definitely-missing-subc-cgroup/cgroup.procs"),
14962 "parent cgroup open failure must name cgroup.procs: {reason}"
14963 );
14964 }
14965
14966 #[tokio::test]
14967 async fn reaping_a_child_removes_its_empty_module_cgroup() {
14968 let root = ScratchDir::new("supervisor-reap-cgroup");
14969 fs::write(root.join("cgroup.procs"), b"").expect("write scratch cgroup marker");
14970 let placement = subc_cgroup::prepare_at(&root)
14971 .expect("prepare scratch cgroup root")
14972 .expect("scratch root has a cgroup.procs marker");
14973 let module_id = "reaped-module";
14974 let module = placement
14975 .module_path(module_id)
14976 .expect("create scratch module cgroup");
14977 let child = Command::new("true")
14978 .env("XDG_DATA_HOME", root.path())
14979 .env("XDG_RUNTIME_DIR", root.path())
14980 .env("XDG_CONFIG_HOME", root.path())
14981 .spawn()
14982 .expect("spawn short-lived child");
14983 let pid = child.id().expect("spawned child has pid");
14984 let mut child = SupervisedChild {
14985 child,
14986 exec: crate::activity::ExecOwner(Arc::new(crate::activity::ExecOutcome::new())),
14987 protocol: ModuleProtocol::Subc,
14988 module_id: module_id.to_string(),
14989 cgroup_placement: Some(placement),
14990 stdout_pump: None,
14991 stderr_pump: None,
14992 stderr_ring: Arc::new(Mutex::new(StderrRing::new(StderrTailConfig::default()))),
14993 spawned_at: tokio::time::Instant::now(),
14994 spawned_at_ms: 0,
14995 spawned_from: PathBuf::from("true"),
14996 launch_spec_digest: LaunchSpecDigest([0; 32]),
14997 spawned_file_identity: None,
14998 process_start_time: None,
14999 process_identity: None,
15000 pid,
15001 roster_guard: None,
15002 #[cfg(target_os = "macos")]
15003 privacy_exec: None,
15004 spawn_failure: None,
15005 };
15006
15007 child.wait().await.expect("reap short-lived child");
15008
15009 assert!(
15010 !module.exists(),
15011 "reaping the supervised child must remove its empty cgroup"
15012 );
15013 }
15014
15015 #[test]
15016 fn non_empty_cgroup_removal_is_reported_without_blocking_teardown() {
15017 let root = ScratchDir::new("supervisor-non-empty-cgroup");
15018 fs::write(root.join("cgroup.procs"), b"").expect("write scratch cgroup marker");
15019 let placement = subc_cgroup::prepare_at(&root)
15020 .expect("prepare scratch cgroup root")
15021 .expect("scratch root has a cgroup.procs marker");
15022 let module = placement
15023 .module_path("surviving-module")
15024 .expect("create scratch module cgroup");
15025 fs::write(module.join("surviving-process"), b"still present")
15026 .expect("make scratch cgroup non-empty");
15027 let (logs, _guard) = crate::router::test_log::log_capture(tracing::Level::WARN);
15028
15029 remove_module_cgroup(&placement, "surviving-module");
15030
15031 let logs = crate::router::test_log::captured_logs(&logs);
15032 assert!(
15033 module.exists(),
15034 "failed removal must leave the cgroup intact"
15035 );
15036 assert!(
15037 logs.contains("could not remove module cgroup after process exit; continuing teardown")
15038 && logs.contains("surviving-module"),
15039 "best-effort removal must report the failure without returning it: {logs}"
15040 );
15041 }
15042
15043 #[test]
15044 fn cgroup_pre_exec_spawn_failure_names_the_cgroup_path() {
15045 let cgroup_path = PathBuf::from("/sys/fs/cgroup/subc-modules/broken-module");
15046 let reason = SuperviseError::Spawn {
15047 program: PathBuf::from("/bin/true"),
15048 source: io::Error::from_raw_os_error(13),
15049 cgroup_path: Some(cgroup_path.clone()),
15050 }
15051 .to_string();
15052
15053 assert!(
15054 reason.contains(&cgroup_path.display().to_string()),
15055 "a pre_exec spawn failure must name the cgroup path: {reason}"
15056 );
15057 }
15058}
15059
15060#[cfg(test)]
15061mod spawn_subscriber_lag_tests {
15062 use super::*;
15063
15064 #[tokio::test]
15069 async fn lagged_spawn_subscriber_receives_a_terminal_lagged_error_after_its_queued_frames() {
15070 let feed = SpawnEventFeed::default();
15071 feed.configure_incarnation("lag-incarnation".to_string());
15072 let (tx, mut rx) = mpsc::channel(1);
15075 feed.subscribe(ConnectionId::new(1), 7, 1, None, FrameSink::new(tx))
15076 .expect("subscribe");
15077 let emitted = SPAWN_SUBSCRIBER_BUFFER + 16;
15078 for index in 0..emitted {
15079 feed.emit_spawned(&format!("lag-module-{index}"), 1000, 0);
15080 tokio::task::yield_now().await;
15083 }
15084 assert_eq!(
15085 feed.subscriber_count(),
15086 0,
15087 "the lagged subscriber must be removed"
15088 );
15089
15090 let mut data = Vec::new();
15091 let mut last = None;
15092 loop {
15093 let next = tokio::time::timeout(Duration::from_secs(5), rx.recv())
15094 .await
15095 .expect("the forwarder must finish once the subscriber is dropped");
15096 let Some(outbound) = next else { break };
15097 let frame = outbound.frame;
15098 if frame.header.ty == FrameType::StreamData {
15099 assert!(last.is_none(), "no data may follow the terminal frame");
15100 let event: SpawnEvent = serde_json::from_slice(&frame.body).unwrap();
15101 data.push(event.cursor.seq);
15102 } else {
15103 assert!(last.is_none(), "exactly one terminal frame");
15104 last = Some(frame);
15105 }
15106 }
15107 assert!(!data.is_empty(), "queued frames drain before the terminal");
15108 for pair in data.windows(2) {
15109 assert_eq!(
15110 pair[1],
15111 pair[0] + 1,
15112 "queued frames arrive dense and in order"
15113 );
15114 }
15115 let terminal = last.expect("a lagged subscriber must receive a terminal frame");
15116 assert_eq!(terminal.header.ty, FrameType::Error);
15117 assert_eq!(terminal.header.corr, 7);
15118 let body: subc_protocol::ErrorBody = serde_json::from_slice(&terminal.body).unwrap();
15119 assert_eq!(body.code, SPAWN_SUBSCRIBER_LAGGED_CODE);
15120 let detail = body.detail.expect("lagged error carries detail");
15121 assert_eq!(
15122 detail["first_undelivered_cursor"]["seq"],
15123 data.last().unwrap() + 1,
15124 "the named cursor is the first event the subscriber did not receive"
15125 );
15126 assert_eq!(
15127 detail["first_undelivered_cursor"]["daemon_incarnation"],
15128 "lag-incarnation"
15129 );
15130 }
15131}
15132
15133#[cfg(test)]
15134mod terminal_history_read_concurrency_tests {
15135 use super::*;
15136 use crate::terminal_journal::read_pause;
15137 use cortexkit_test_support::ScratchDir;
15138 use std::sync::mpsc as std_mpsc;
15139
15140 fn journaled_ring(
15141 journal: &Arc<crate::terminal_journal::TerminalJournal>,
15142 ) -> Arc<Mutex<TerminalRing>> {
15143 Arc::new(Mutex::new(
15144 TerminalRing::new(TerminalRingConfig::default(), 1)
15145 .with_journal(Some(Arc::clone(journal))),
15146 ))
15147 }
15148
15149 fn crash(at_ms: u64) -> ExitReport {
15150 ExitReport {
15151 kind: ExitKind::Crash,
15152 code: Some(1),
15153 signal: None,
15154 at_ms,
15155 }
15156 }
15157
15158 fn record_within(
15161 module_id: &'static str,
15162 ring: &Arc<Mutex<TerminalRing>>,
15163 at_ms: u64,
15164 bound: Duration,
15165 ) -> bool {
15166 let ring = Arc::clone(ring);
15167 let (done, done_rx) = std_mpsc::channel();
15168 std::thread::spawn(move || {
15169 record_terminal(
15170 module_id,
15171 &ring,
15172 &SpawnEventFeed::default(),
15173 &crash(at_ms),
15174 TerminalDisposition::Restarting,
15175 );
15176 let _ = done.send(());
15177 });
15178 done_rx.recv_timeout(bound).is_ok()
15179 }
15180
15181 #[test]
15186 fn exits_recorded_during_a_paused_history_read_are_not_blocked_or_half_merged() {
15187 let dir = ScratchDir::new("terminal-history-concurrent-read");
15188 let path = dir.join("terminals.jsonl");
15189 let journal = Arc::new(crate::terminal_journal::TerminalJournal::open(
15190 path.clone(),
15191 "daemon".into(),
15192 ));
15193 let reader_ring = journaled_ring(&journal);
15194 let other_ring = journaled_ring(&journal);
15195 assert!(record_within(
15196 "reader-module",
15197 &reader_ring,
15198 10,
15199 Duration::from_secs(5)
15200 ));
15201
15202 let (started, release) = read_pause::install(&path);
15203 let reading = {
15204 let ring = Arc::clone(&reader_ring);
15205 std::thread::spawn(move || durable_terminal_history_of(&ring, "reader-module"))
15206 };
15207 started
15208 .recv_timeout(Duration::from_secs(5))
15209 .expect("the history read reached its pause");
15210
15211 let bound = Duration::from_secs(1);
15212 assert!(
15213 record_within("other-module", &other_ring, 20, bound),
15214 "another module's exit waited on a history read (journal writer held)"
15215 );
15216 assert!(
15217 record_within("reader-module", &reader_ring, 30, bound),
15218 "the read module's own exit waited on its history read (ring held)"
15219 );
15220
15221 drop(release);
15222 let paused = reading.join().unwrap();
15223 assert_eq!(
15224 paused.entries.iter().map(|e| e.at_ms).collect::<Vec<_>>(),
15225 vec![10],
15226 "an exit recorded after the read began lands in neither half of it"
15227 );
15228 assert_eq!(paused.journal_skipped_lines, 0);
15229 assert_eq!(paused.journal_read_errors, 0);
15230
15231 let after = durable_terminal_history_of(&reader_ring, "reader-module");
15232 assert_eq!(
15233 after.entries.iter().map(|e| e.at_ms).collect::<Vec<_>>(),
15234 vec![10, 30],
15235 "the next read merges ring and journal with no duplicate"
15236 );
15237 assert_eq!(after.journal_skipped_lines, 0);
15238 }
15239}
15240
15241#[cfg(test)]
15246mod stderr_settle_tests {
15247 use std::{
15248 future::Future,
15249 io,
15250 pin::Pin,
15251 sync::{Arc, Mutex},
15252 task::{Context, Poll},
15253 time::Duration,
15254 };
15255
15256 use tokio::{
15257 io::{AsyncRead, ReadBuf},
15258 sync::oneshot,
15259 time::Instant,
15260 };
15261
15262 use super::{settle_stderr_pump, settle_stdout_pump, StderrPump};
15263 use crate::stderr_tail::{
15264 pump_stderr_to, pump_stdout_to, split_capture_stamp, untimed, CaptureState,
15265 ChildOutputSink, OutputSink, StderrRing, StderrTailConfig, TailEntry,
15266 };
15267
15268 const BOUND: Duration = Duration::from_millis(250);
15269
15270 struct HeldReader {
15274 before: Option<Vec<u8>>,
15275 gate: Option<oneshot::Receiver<()>>,
15276 after: io::Cursor<Vec<u8>>,
15277 }
15278
15279 impl AsyncRead for HeldReader {
15280 fn poll_read(
15281 mut self: Pin<&mut Self>,
15282 cx: &mut Context<'_>,
15283 buf: &mut ReadBuf<'_>,
15284 ) -> Poll<io::Result<()>> {
15285 if let Some(bytes) = self.before.take() {
15286 buf.put_slice(&bytes);
15287 return Poll::Ready(Ok(()));
15288 }
15289 if let Some(gate) = self.gate.as_mut() {
15290 match Pin::new(gate).poll(cx) {
15291 Poll::Pending => return Poll::Pending,
15292 Poll::Ready(_) => self.gate = None,
15293 }
15294 }
15295 Pin::new(&mut self.after).poll_read(cx, buf)
15296 }
15297 }
15298
15299 struct DiscardSink;
15300
15301 impl OutputSink for DiscardSink {
15302 fn write_line(&mut self, _line: &[u8]) {}
15303 }
15304
15305 fn line(text: &str) -> TailEntry {
15306 TailEntry::Line {
15307 text: text.to_string(),
15308 truncated: false,
15309 at_ms: None,
15310 }
15311 }
15312
15313 fn lock(ring: &Arc<Mutex<StderrRing>>) -> std::sync::MutexGuard<'_, StderrRing> {
15314 ring.lock().unwrap()
15315 }
15316
15317 fn held_pump(
15321 ring: &Arc<Mutex<StderrRing>>,
15322 before: &str,
15323 after: &str,
15324 ) -> (StderrPump, oneshot::Sender<()>) {
15325 let generation = lock(ring).begin_process();
15326 let (release, gate) = oneshot::channel();
15327 let reader = HeldReader {
15328 before: Some(before.as_bytes().to_vec()),
15329 gate: Some(gate),
15330 after: io::Cursor::new(after.as_bytes().to_vec()),
15331 };
15332 let task = tokio::spawn(pump_stderr_to(
15333 reader,
15334 Arc::clone(ring),
15335 generation,
15336 DiscardSink,
15337 ));
15338 (StderrPump { task, generation }, release)
15339 }
15340
15341 async fn wait_until(ring: &Arc<Mutex<StderrRing>>, done: impl Fn(&StderrRing) -> bool) {
15342 for _ in 0..1000 {
15343 if done(&lock(ring)) {
15344 return;
15345 }
15346 tokio::time::sleep(Duration::from_millis(1)).await;
15347 }
15348 panic!(
15349 "ring never reached the expected state: {:?}",
15350 lock(ring).snapshot(None, None)
15351 );
15352 }
15353
15354 #[tokio::test(start_paused = true)]
15355 async fn a_crash_line_the_reader_had_not_reached_by_the_bound_is_kept_before_the_restart() {
15356 let ring = Arc::new(Mutex::new(StderrRing::new(StderrTailConfig::default())));
15357 let (pump, release) = held_pump(&ring, "booting\n", "config error: missing storage\n");
15358
15359 settle_stderr_pump("crasher", &ring, pump, BOUND).await;
15360 let before_release = lock(&ring).snapshot(None, None);
15361 assert!(
15362 matches!(before_release.capture, CaptureState::Incomplete { .. }),
15363 "a reader that has not reached EOF cannot claim a whole tail: {before_release:?}"
15364 );
15365
15366 let next = lock(&ring).begin_process();
15369 lock(&ring).push_line_from(next, "next process booting");
15370 release.send(()).unwrap();
15371 wait_until(&ring, |ring| {
15372 ring.snapshot(None, None).capture == CaptureState::Captured
15373 })
15374 .await;
15375
15376 assert_eq!(
15377 untimed(lock(&ring).snapshot(None, None).entries),
15378 vec![
15379 line("booting"),
15380 line("config error: missing storage"),
15381 TailEntry::ProcessStart,
15382 line("next process booting"),
15383 ],
15384 "the crash's last line must survive a slow reader and stay in the crashed process's section"
15385 );
15386 }
15387
15388 #[tokio::test(start_paused = true)]
15389 async fn a_pipe_held_open_by_a_descendant_reads_incomplete_without_delaying_the_restart_past_the_bound(
15390 ) {
15391 let ring = Arc::new(Mutex::new(StderrRing::new(StderrTailConfig::default())));
15392 let (pump, _held) = held_pump(&ring, "parent exiting\n", "");
15395
15396 let started = Instant::now();
15397 settle_stderr_pump("orphaning", &ring, pump, BOUND).await;
15398 assert_eq!(
15399 started.elapsed(),
15400 BOUND,
15401 "the restart must wait exactly the bound for a pipe that stays open, no longer"
15402 );
15403
15404 let next = lock(&ring).begin_process();
15405 lock(&ring).push_line_from(next, "next process booting");
15406 tokio::time::sleep(Duration::from_secs(60)).await;
15407
15408 let snapshot = lock(&ring).snapshot(None, None);
15409 match &snapshot.capture {
15410 CaptureState::Incomplete { reason } => assert!(
15411 reason.contains("had not reached EOF") && reason.contains("250ms"),
15412 "the reason must say what is missing and after how long: {reason}"
15413 ),
15414 other => panic!("expected Incomplete while the pipe is held open, got {other:?}"),
15415 }
15416 assert_eq!(
15417 untimed(snapshot.entries),
15418 vec![
15419 line("parent exiting"),
15420 TailEntry::ProcessStart,
15421 line("next process booting"),
15422 ]
15423 );
15424 }
15425
15426 #[tokio::test(start_paused = true)]
15427 async fn a_reader_that_reaches_eof_within_the_bound_leaves_the_tail_captured() {
15428 let ring = Arc::new(Mutex::new(StderrRing::new(StderrTailConfig::default())));
15429 let (pump, release) = held_pump(&ring, "one\n", "two\n");
15430 release.send(()).unwrap();
15431
15432 settle_stderr_pump("clean", &ring, pump, BOUND).await;
15433
15434 let snapshot = lock(&ring).snapshot(None, None);
15435 assert_eq!(snapshot.capture, CaptureState::Captured);
15436 assert_eq!(untimed(snapshot.entries), vec![line("one"), line("two")]);
15437 }
15438
15439 #[tokio::test(start_paused = true)]
15440 async fn stdout_buffered_past_the_exit_drain_bound_reaches_the_capture_file() {
15441 const BURST: usize = 150;
15442 let capture = cortexkit_test_support::ScratchDir::new("stdout-exit-drain");
15443 let path = capture.join("two-pipe.stderr.log");
15444 let sink = ChildOutputSink::open(&path, cortexkit_log::Retention::default()).unwrap();
15445 let ring = Arc::new(Mutex::new(StderrRing::new(StderrTailConfig::default())));
15446 let generation = lock(&ring).begin_process();
15447 let stdout = (0..BURST)
15448 .map(|i| format!("out-{i:04}-{}\n", "o".repeat(64)))
15449 .collect::<String>();
15450 let stderr = (0..BURST)
15451 .map(|i| format!("err-{i:04}-{}\n", "e".repeat(64)))
15452 .collect::<String>();
15453 let first_chunk = stdout.find('\n').unwrap() + 1 + "out-0001-".len();
15458 let (release, gate) = oneshot::channel();
15459 let reader = HeldReader {
15460 before: Some(stdout.as_bytes()[..first_chunk].to_vec()),
15461 gate: Some(gate),
15462 after: io::Cursor::new(stdout.as_bytes()[first_chunk..stdout.len() - 1].to_vec()),
15463 };
15464 let (finished, completion) = oneshot::channel();
15465 let stdout_sink = sink.clone();
15466 let pump = tokio::spawn(async move {
15467 pump_stdout_to(reader, stdout_sink).await;
15468 finished.send(()).unwrap();
15469 });
15470 pump_stderr_to(io::Cursor::new(stderr.into_bytes()), ring, generation, sink).await;
15471
15472 let started = Instant::now();
15473 settle_stdout_pump("two-pipe", pump, BOUND).await;
15474 assert_eq!(
15475 started.elapsed(),
15476 BOUND,
15477 "a late reader must not hold up restart"
15478 );
15479 assert_eq!(
15480 std::fs::read_to_string(&path).unwrap().lines().count(),
15481 BURST + 1
15482 );
15483
15484 release
15485 .send(())
15486 .expect("stdout reader must survive the exit drain bound");
15487 tokio::time::timeout(Duration::from_secs(1), completion)
15488 .await
15489 .expect("released stdout reader must reach EOF")
15490 .expect("stdout capture must finish at EOF, not be cancelled");
15491 let contents = std::fs::read_to_string(&path).unwrap();
15492 let mut lines = contents
15493 .lines()
15494 .map(|line| split_capture_stamp(line).expect("capture stamp").1)
15495 .collect::<Vec<_>>();
15496 let mut expected = (0..BURST)
15497 .flat_map(|i| {
15498 [
15499 format!("out-{i:04}-{}", "o".repeat(64)),
15500 format!("err-{i:04}-{}", "e".repeat(64)),
15501 ]
15502 })
15503 .collect::<Vec<_>>();
15504 lines.sort_unstable();
15505 expected.sort_unstable();
15506 assert_eq!(
15507 lines, expected,
15508 "both pipes must retain every complete line exactly once"
15509 );
15510 }
15511}
15512
15513#[cfg(all(test, windows))]
15527mod job_containment_tests {
15528 use super::*;
15529 use cortexkit_test_support::ScratchDir;
15530 use std::{
15531 path::{Path, PathBuf},
15532 sync::{Arc, Mutex},
15533 time::{Duration, Instant},
15534 };
15535
15536 fn stub_path() -> PathBuf {
15542 let mut path = std::env::current_exe().expect("current_exe available in tests");
15543 path.pop();
15544 path.pop();
15545 path.push("fake-aft-stub.exe");
15546 assert!(
15547 path.exists(),
15548 "fake-aft-stub not built at {}: run `cargo test -p subc-core` (which builds \
15549 [[bin]] targets) rather than `cargo test -p subc-core --lib` (which does not)",
15550 path.display()
15551 );
15552 path
15553 }
15554
15555 fn read_grandchild_pid(path: &Path) -> u32 {
15557 let deadline = Instant::now() + Duration::from_secs(10);
15558 loop {
15559 if let Ok(contents) = std::fs::read_to_string(path) {
15560 if let Ok(pid) = contents.trim().parse() {
15561 return pid;
15562 }
15563 }
15564 assert!(
15565 Instant::now() < deadline,
15566 "the stub never recorded a grandchild pid at {}",
15567 path.display()
15568 );
15569 std::thread::sleep(Duration::from_millis(10));
15570 }
15571 }
15572
15573 struct Fixture {
15576 _dir: ScratchDir,
15577 module_id: String,
15578 grandchild: u32,
15579 child: Option<SupervisedChild>,
15580 registry: Arc<Registry>,
15581 snapshot: Arc<Mutex<SupervisorSnapshot>>,
15582 terminal_ring: Arc<Mutex<TerminalRing>>,
15583 spawn_events: SpawnEventFeed,
15584 }
15585
15586 fn fixture(label: &str, module_id: &str) -> Fixture {
15587 let dir = ScratchDir::new(label);
15588 let pid_file = dir.join("grandchild.pid");
15589 let supervisor = Supervisor::new_for_test(
15590 Arc::new(Registry::default()),
15591 RestartPolicy::new(3, Duration::ZERO),
15592 );
15593 let runtime = supervisor.runtime_config();
15594 let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::starting()));
15595 let spec = ModuleSpec {
15596 module_id: module_id.to_string(),
15597 program: stub_path(),
15598 args: Vec::new(),
15602 env: vec![
15603 ("FAKE_AFT_NEVER_CONNECT".to_string(), "1".to_string()),
15604 (
15605 "FAKE_AFT_GRANDCHILD_PID_FILE".to_string(),
15606 pid_file.display().to_string(),
15607 ),
15608 ],
15609 reserved: false,
15610 reserved_prefixes: Vec::new(),
15611 protocol: ModuleProtocol::Subc,
15612 overlap: Default::default(),
15613 };
15614 let child = spawn_and_mark_running(&spec, &runtime, &snapshot)
15615 .expect("spawn the supervised fixture");
15616 let grandchild = read_grandchild_pid(&pid_file);
15617 Fixture {
15618 _dir: dir,
15619 module_id: module_id.to_string(),
15620 grandchild,
15621 child: Some(child),
15622 registry: Arc::new(Registry::default()),
15623 snapshot,
15624 terminal_ring: Arc::clone(&runtime.terminal_ring),
15625 spawn_events: SpawnEventFeed::default(),
15626 }
15627 }
15628
15629 impl Fixture {
15630 async fn drain(&mut self) {
15632 let child = self
15633 .child
15634 .take()
15635 .expect("the fixture child is still present");
15636 drain_child_to_state(
15637 &self.module_id,
15638 ModuleProtocol::Subc,
15639 StopNotice::NotSent,
15642 &self.registry,
15643 None,
15644 &self.snapshot,
15645 &self.terminal_ring,
15646 &self.spawn_events,
15647 child,
15648 Duration::from_millis(500),
15649 ModuleState::Stopped,
15650 Some(false),
15651 )
15652 .await
15653 .expect("drain the supervised fixture");
15654 }
15655 }
15656
15657 #[tokio::test]
15663 async fn teardown_reaps_the_grandchild() {
15664 let mut fixture = fixture("teardown-grandchild", "tree-teardown");
15665 let grandchild = fixture.grandchild;
15666
15667 assert!(
15668 subc_jobobject::process_exists(grandchild),
15669 "grandchild {grandchild} must be alive before teardown, or this proves nothing"
15670 );
15671
15672 fixture.drain().await;
15673
15674 assert!(
15675 subc_jobobject::wait_for_process_exit(grandchild, Duration::from_secs(10)),
15676 "grandchild {grandchild} outlived module teardown: the tree was not contained"
15677 );
15678 }
15679
15680 #[test]
15693 fn an_uncontained_grandchild_survives_a_direct_child_kill() {
15694 let dir = ScratchDir::new("teardown-uncontained");
15695 let pid_file = dir.join("grandchild.pid");
15696 let mut child = std::process::Command::new(stub_path())
15697 .env("FAKE_AFT_NEVER_CONNECT", "1")
15698 .env(
15699 "FAKE_AFT_GRANDCHILD_PID_FILE",
15700 pid_file.display().to_string(),
15701 )
15702 .stdin(std::process::Stdio::null())
15703 .stdout(std::process::Stdio::null())
15704 .stderr(std::process::Stdio::null())
15705 .spawn()
15706 .expect("spawn the uncontained fixture");
15707 let grandchild = read_grandchild_pid(&pid_file);
15708
15709 child.kill().expect("kill the direct child");
15711 let _ = child.wait();
15712
15713 assert!(
15714 subc_jobobject::process_exists(grandchild),
15715 "grandchild {grandchild} died with the direct child, so this control no longer \
15716 distinguishes contained from uncontained teardown and the regression test is \
15717 passing vacuously"
15718 );
15719
15720 kill_tree(grandchild);
15723 }
15724
15725 #[tokio::test]
15734 async fn dropping_containment_reaps_the_grandchild() {
15735 let mut fixture = fixture("drop-containment", "tree-drop");
15736 let grandchild = fixture.grandchild;
15737
15738 assert!(subc_jobobject::process_exists(grandchild));
15739
15740 fixture.child.as_mut().expect("child present").job = None;
15742
15743 assert!(
15744 subc_jobobject::wait_for_process_exit(grandchild, Duration::from_secs(10)),
15745 "grandchild {grandchild} survived the containment handle closing, so a daemon \
15746 crash would leave the tree behind"
15747 );
15748 }
15749
15750 fn kill_tree(pid: u32) {
15752 let _ = std::process::Command::new("taskkill.exe")
15753 .args(["/PID", &pid.to_string(), "/T", "/F"])
15754 .stdin(std::process::Stdio::null())
15755 .stdout(std::process::Stdio::null())
15756 .stderr(std::process::Stdio::null())
15757 .status();
15758 assert!(
15759 subc_jobobject::wait_for_process_exit(pid, Duration::from_secs(10)),
15760 "could not clean up grandchild {pid}"
15761 );
15762 }
15763}
15764
15765#[cfg(test)]
15766mod privacy_trampoline_configuration_tests {
15767 #[tokio::test]
15768 #[cfg_attr(not(target_os = "macos"), ignore = "requires macOS privacy trampoline")]
15769 async fn macos_spawn_without_a_configured_trampoline_refuses_by_name() {
15770 #[cfg(target_os = "macos")]
15771 {
15772 let supervisor = super::Supervisor::new(
15773 std::sync::Arc::new(crate::Registry::default()),
15774 super::RestartPolicy::default(),
15775 );
15776 let error = supervisor.spawn(spec()).unwrap_err();
15777 assert!(
15778 error
15779 .to_string()
15780 .contains("no privacy trampoline configured"),
15781 "{error}"
15782 );
15783 }
15784 }
15785
15786 #[tokio::test]
15787 #[cfg_attr(not(target_os = "macos"), ignore = "requires macOS privacy trampoline")]
15788 async fn macos_wrong_trampoline_is_rejected_by_the_startup_probe() {
15789 #[cfg(target_os = "macos")]
15790 {
15791 let supervisor = super::Supervisor::new(
15792 std::sync::Arc::new(crate::Registry::default()),
15793 super::RestartPolicy::default(),
15794 )
15795 .with_privacy_trampoline(std::env::current_exe().unwrap());
15796 let error = supervisor.spawn(spec()).unwrap_err();
15797 assert!(
15798 error
15799 .to_string()
15800 .contains("binary does not implement the privacy trampoline protocol"),
15801 "{error}"
15802 );
15803 }
15804 }
15805
15806 #[cfg(target_os = "macos")]
15807 fn spec() -> super::ModuleSpec {
15808 super::ModuleSpec {
15809 module_id: "privacy-configuration".into(),
15810 program: "/bin/sleep".into(),
15811 args: vec!["30".into()],
15812 env: vec![],
15813 reserved: false,
15814 reserved_prefixes: vec![],
15815 protocol: subc_control::ModuleProtocol::None,
15816 overlap: super::ModuleOverlap::Exclusive,
15817 }
15818 }
15819}
15820
15821#[cfg(test)]
15822mod privacy_exec_boundary_tests {
15823 #[cfg(target_os = "macos")]
15824 use super::*;
15825 #[cfg(target_os = "macos")]
15826 use std::{
15827 io::{Read, Write},
15828 net::{TcpListener, TcpStream},
15829 };
15830
15831 #[cfg(target_os = "macos")]
15834 pub(super) fn before_image_sample(spec: &ModuleSpec, pid: u32) {
15835 if let Some((_, path)) = spec
15836 .env
15837 .iter()
15838 .find(|(key, _)| key == "SUBC_TEST_PRIVACY_SAMPLE_BARRIER")
15839 {
15840 let mut barrier = TcpStream::connect(path).unwrap();
15841 barrier
15842 .set_read_timeout(Some(Duration::from_secs(30)))
15843 .unwrap();
15844 barrier.write_all(&pid.to_ne_bytes()).unwrap();
15845 let mut release = [0];
15846 barrier.read_exact(&mut release).unwrap();
15847 assert_eq!(&release, b"X");
15848 }
15849 }
15850
15851 #[cfg(target_os = "macos")]
15852 fn spec(program: &str, args: &[&str]) -> ModuleSpec {
15853 ModuleSpec {
15854 module_id: "privacy-boundary".into(),
15855 program: program.into(),
15856 args: args.iter().map(|arg| (*arg).into()).collect(),
15857 env: vec![],
15858 reserved: false,
15859 reserved_prefixes: vec![],
15860 protocol: ModuleProtocol::None,
15861 overlap: ModuleOverlap::Exclusive,
15862 }
15863 }
15864
15865 #[cfg(target_os = "macos")]
15866 async fn accept(listener: TcpListener) -> TcpStream {
15867 let listener = tokio::net::TcpListener::from_std({
15869 listener.set_nonblocking(true).unwrap();
15870 listener
15871 })
15872 .unwrap();
15873 let (stream, _) = tokio::time::timeout(Duration::from_secs(30), listener.accept())
15874 .await
15875 .unwrap()
15876 .unwrap();
15877 let stream = stream.into_std().unwrap();
15878 stream.set_nonblocking(false).unwrap();
15879 stream
15880 .set_read_timeout(Some(Duration::from_secs(30)))
15881 .unwrap();
15882 stream
15883 }
15884
15885 #[tokio::test]
15886 #[cfg_attr(not(target_os = "macos"), ignore = "requires macOS privacy trampoline")]
15887 async fn macos_roster_withholds_a_nonnull_trampoline_image_until_exec_confirmation() {
15888 #[cfg(target_os = "macos")]
15889 {
15890 let root = cortexkit_test_support::ScratchDir::new("privacy-roster-barrier");
15891 let exec_listener = TcpListener::bind("127.0.0.1:0").unwrap();
15894 let sample_listener = TcpListener::bind("127.0.0.1:0").unwrap();
15895 let record = root.join("live-children.json");
15896 let supervisor =
15897 Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default())
15898 .with_live_children_record(&record);
15899 let runtime = supervisor.runtime_config();
15900 let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::starting()));
15901 let mut spec = spec("/bin/sleep", &["30"]);
15902 spec.env = vec![
15903 (
15904 "SUBC_TEST_PRIVACY_EXEC_BARRIER".into(),
15905 exec_listener.local_addr().unwrap().to_string(),
15906 ),
15907 (
15908 "SUBC_TEST_PRIVACY_SAMPLE_BARRIER".into(),
15909 sample_listener.local_addr().unwrap().to_string(),
15910 ),
15911 ];
15912 let spawn = tokio::task::spawn_blocking(move || {
15913 spawn_and_mark_running(&spec, &runtime, &snapshot).unwrap()
15914 });
15915 let mut sample = accept(sample_listener).await;
15916 let mut pid = [0; 4];
15917 sample.read_exact(&mut pid).unwrap();
15918 let pid = u32::from_ne_bytes(pid);
15919 let mut exec = accept(exec_listener).await;
15920 let mut ready = [0];
15921 exec.read_exact(&mut ready).unwrap();
15922 assert_eq!(&ready, b"R");
15923 let trampoline = subc_os::file_identity(&test_privacy_trampoline()).unwrap();
15926 assert_eq!(
15927 observe_spawned_image(pid).unwrap().executable,
15928 Some(trampoline)
15929 );
15930 sample.write_all(b"X").unwrap();
15931 let mut child = spawn.await.unwrap();
15932 let early = crate::live_children::read_record(&record).unwrap();
15933 assert_eq!(early.len(), 1);
15934 assert_eq!(early[0].pid, pid);
15935 assert_eq!(
15936 early[0].executable, None,
15937 "unconfirmed trampoline image entered the roster"
15938 );
15939 assert!(child.report_ready.get().is_none());
15940 child.privacy_exec.as_mut().unwrap().deadline =
15943 tokio::time::Instant::now() + Duration::from_secs(30);
15944 exec.write_all(b"X").unwrap();
15945 child.confirm_privacy_exec().await;
15946 assert_eq!(child.spawn_failure, None);
15947 assert!(child.report_ready.get().is_some());
15948 let confirmed = crate::live_children::read_record(&record).unwrap();
15949 let module = subc_os::file_identity(std::path::Path::new("/bin/sleep")).unwrap();
15950 assert_ne!(module, trampoline);
15951 assert_eq!(confirmed[0].executable, Some(module.into()));
15952 child.start_kill().unwrap();
15953 child.wait().await.unwrap();
15954 child.release_roster();
15955 }
15956 }
15957
15958 #[tokio::test]
15959 #[cfg_attr(not(target_os = "macos"), ignore = "requires macOS privacy trampoline")]
15960 async fn macos_already_exited_121_in_try_wait_is_a_module_exit_not_a_trampoline_refusal() {
15961 #[cfg(target_os = "macos")]
15962 {
15963 let registry = Arc::new(Registry::default());
15964 let policy = RestartPolicy::new(0, Duration::ZERO);
15965 let supervisor = Supervisor::new_for_test(Arc::clone(®istry), policy);
15966 let runtime = supervisor.runtime_config();
15967 let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::starting()));
15968 let spec = spec("/bin/sh", &["-c", "exit 121"]);
15969 let mut child = spawn_and_mark_running(&spec, &runtime, &snapshot).unwrap();
15973 let pid = child.pid;
15974 tokio::task::spawn_blocking(move || {
15975 subc_os::fork_exec_test::wait_for_child_exit_without_reaping(pid)
15976 })
15977 .await
15978 .unwrap()
15979 .unwrap();
15980 child.privacy_exec.as_mut().unwrap().deadline =
15981 tokio::time::Instant::now() + Duration::from_secs(30);
15982 let status = child.wait().await.unwrap();
15983 assert_eq!(status.code(), Some(121));
15984 assert!(child.privacy_exec.is_none());
15985 assert!(
15986 child.report_ready.get().is_none(),
15987 "an exited module must not publish a live pid"
15988 );
15989 let report = classify_reaped_child_exit(&snapshot, &child, &status);
15990 on_child_exit(
15991 &spec,
15992 policy,
15993 ®istry,
15994 &snapshot,
15995 &runtime.terminal_ring,
15996 &runtime.spawn_events,
15997 &runtime.child_roster,
15998 report,
15999 )
16000 .await;
16001 let state = lock_snapshot(&snapshot).unwrap();
16002 assert_eq!(state.state, ModuleState::Failed);
16003 assert_eq!(state.last_exit.as_ref().unwrap().code, Some(121));
16004 assert_eq!(state.reported_pid(), None);
16005 drop(state);
16006 let history = runtime.terminal_ring.lock().unwrap().snapshot();
16007 assert_eq!(history.entries.len(), 1);
16008 let terminal = &history.entries[0];
16009 assert_eq!(terminal.exit_code, Some(121));
16010 assert_eq!(terminal.exit_kind, subc_control::TerminalExitKind::Crash);
16011 assert_eq!(terminal.disposition, TerminalDisposition::Failed);
16012 assert_eq!(
16013 terminal.disposition_detail.as_deref(),
16014 Some(policy.budget_exhausted_detail().as_str()),
16015 "module exit 121 was classified as a trampoline refusal: {terminal:?}"
16016 );
16017 assert_eq!(child.spawn_failure, None);
16018 child.release_roster();
16019 }
16020 }
16021}
16022
16023#[cfg(all(test, unix))]
16027mod launch_nonce_descriptor_tests {
16028 use super::{ChildRoster, ModuleProtocol, ModuleSpec, SupervisorHandle};
16029 use crate::stderr_tail::{StderrRing, StderrTailConfig};
16030 use cortexkit_test_support::ScratchDir;
16031 use std::{
16032 path::PathBuf,
16033 sync::{Arc, Mutex},
16034 time::{Duration, Instant},
16035 };
16036
16037 async fn probe(role: super::SpawnRole) {
16038 let scratch = ScratchDir::new("launch-nonce-descriptor");
16039 let fd_copy = scratch.join("from-descriptor");
16040 let env_copy = scratch.join("environment");
16041 let script = format!(
16042 "env > '{env}.tmp' && mv '{env}.tmp' '{env}'; cat <&3 > '{fd}.tmp' && mv '{fd}.tmp' '{fd}'; sleep 30",
16043 fd = fd_copy.display(), env = env_copy.display(),
16044 );
16045 let xdg = |name: &str| (name.to_string(), scratch.join(name).display().to_string());
16046 let spec = ModuleSpec {
16047 module_id: "nonce-descriptor-probe".to_string(),
16048 program: PathBuf::from("/bin/sh"),
16049 args: vec!["-c".to_string(), script],
16050 env: vec![
16051 xdg("XDG_DATA_HOME"),
16052 xdg("XDG_RUNTIME_DIR"),
16053 xdg("XDG_CONFIG_HOME"),
16054 ("SUBC_LAUNCH_NONCE".to_string(), "must-not-leak".to_string()),
16055 ],
16056 reserved: true,
16057 reserved_prefixes: Vec::new(),
16058 protocol: ModuleProtocol::Subc,
16059 overlap: Default::default(),
16060 };
16061 let handle = SupervisorHandle::new();
16062 let ring = Arc::new(Mutex::new(StderrRing::new(StderrTailConfig::default())));
16063 let roster = ChildRoster::default();
16064 #[cfg(target_os = "macos")]
16065 {
16066 let path = super::test_privacy_trampoline();
16067 roster.set_privacy_trampoline(super::probe_privacy_trampoline(&path).map(|()| path));
16068 }
16069 let child = super::spawn_child_in_slot(
16070 &spec,
16071 None,
16072 Some(&handle),
16073 &ring,
16074 None,
16075 &roster,
16076 #[cfg(target_os = "linux")]
16077 None,
16078 role,
16079 matches!(role, super::SpawnRole::SwapCandidate),
16080 )
16081 .expect("spawn probe");
16082 let deadline = Instant::now() + Duration::from_secs(10);
16083 while !(fd_copy.exists() && env_copy.exists()) {
16084 assert!(Instant::now() < deadline, "probe never wrote its copies");
16085 tokio::time::sleep(Duration::from_millis(20)).await;
16086 }
16087 let nonce = std::fs::read_to_string(fd_copy).unwrap();
16088 assert!(!nonce.is_empty());
16089 let environment = std::fs::read_to_string(env_copy).unwrap();
16090 assert!(environment
16091 .lines()
16092 .any(|line| line.starts_with("SUBC_LAUNCH_NONCE_FD=3:")));
16093 let copy = environment
16094 .lines()
16095 .find_map(|line| line.strip_prefix("SUBC_LAUNCH_NONCE="));
16096 assert_eq!(
16097 copy, None,
16098 "Unix children must never receive the environment nonce"
16099 );
16100 if matches!(role, super::SpawnRole::Plain) {
16101 assert_eq!(
16102 handle.spawn_nonce(&spec.module_id).as_deref(),
16103 Some(nonce.as_str())
16104 );
16105 }
16106 drop(child);
16107 }
16108
16109 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
16110 async fn a_spawned_module_receives_its_nonce_only_on_descriptor_3() {
16111 probe(super::SpawnRole::Plain).await;
16112 }
16113
16114 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
16115 async fn swap_candidate_receives_its_nonce_only_on_descriptor_3() {
16116 probe(super::SpawnRole::SwapCandidate).await;
16117 }
16118}
16119
16120#[cfg(all(test, target_os = "linux"))]
16121mod cgroup_containment_tests {
16122 use super::*;
16123 use cortexkit_test_support::ScratchDir;
16124
16125 fn running(pid: u32) -> bool {
16126 std::fs::read_to_string(format!("/proc/{pid}/stat"))
16128 .ok()
16129 .and_then(|stat| {
16130 stat.rsplit_once(") ")
16131 .map(|(_, rest)| rest.starts_with('Z'))
16132 })
16133 .is_some_and(|zombie| !zombie)
16134 }
16135
16136 #[tokio::test]
16137 async fn linux_teardown_reaps_the_grandchild() {
16138 teardown_tree("linux_teardown_reaps_the_grandchild", false).await;
16139 }
16140
16141 #[tokio::test]
16142 async fn linux_shutdown_straggler_reaps_the_grandchild() {
16143 teardown_tree("linux_shutdown_straggler_reaps_the_grandchild", true).await;
16144 }
16145
16146 async fn teardown_tree(test_name: &str, shutdown: bool) {
16147 let dir = ScratchDir::new(test_name);
16148 let root = PathBuf::from(format!(
16149 "/sys/fs/cgroup/subc-tree-test-{test_name}-{}-{}",
16150 std::process::id(),
16151 unix_ms_now()
16152 ));
16153 if let Err(error) = std::fs::create_dir(&root) {
16154 assert!(
16155 std::env::var_os("SUBC_REQUIRE_CGROUP_TEST").is_none(),
16156 "required cgroup test cannot execute: {error}"
16157 );
16158 eprintln!(
16159 "SKIP {test_name}: no writable delegated cgroup at {}: {error}",
16160 root.display()
16161 );
16162 return;
16163 }
16164 let placement = subc_cgroup::prepare_at(&root)
16165 .expect("prepare isolated kernel cgroup")
16166 .expect("isolated cgroup is delegated");
16167 let module_id = "tree-teardown";
16168 let module = placement
16169 .module_path(module_id)
16170 .expect("create isolated module cgroup");
16171 if !module.join("cgroup.kill").exists() {
16172 std::fs::remove_dir(&module).unwrap();
16173 std::fs::remove_dir(root.join("subc-modules")).unwrap();
16174 std::fs::remove_dir(&root).unwrap();
16175 assert!(
16176 std::env::var_os("SUBC_REQUIRE_CGROUP_TEST").is_none(),
16177 "required cgroup.kill interface unavailable"
16178 );
16179 eprintln!("SKIP {test_name}: cgroup.kill unavailable (kernel < 5.14)");
16180 return;
16181 }
16182 let supervisor = Supervisor::new_for_test(
16183 Arc::new(Registry::default()),
16184 RestartPolicy::new(3, Duration::ZERO),
16185 )
16186 .with_cgroup_placement(Some(placement));
16187 let mut runtime = supervisor.runtime_config();
16188 runtime.child_roster = runtime
16189 .child_roster
16190 .for_module(Arc::new(Mutex::new(Duration::from_millis(100))));
16191 let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::starting()));
16192 let pid_file = dir.join("grandchild.pid");
16193 let spec = ModuleSpec {
16194 module_id: module_id.to_string(),
16195 program: PathBuf::from("/bin/sh"),
16196 args: vec![
16197 "-c".into(),
16198 "trap '' TERM; sleep 600 & echo $! > \"$1\"; wait".into(),
16199 "fixture".into(),
16200 pid_file.display().to_string(),
16201 ],
16202 env: ["XDG_DATA_HOME", "XDG_RUNTIME_DIR", "XDG_CONFIG_HOME"]
16203 .into_iter()
16204 .map(|key| (key.to_string(), dir.display().to_string()))
16205 .collect(),
16206 reserved: false,
16207 reserved_prefixes: Vec::new(),
16208 protocol: ModuleProtocol::None,
16209 overlap: Default::default(),
16210 };
16211 let child =
16212 spawn_and_mark_running(&spec, &runtime, &snapshot).expect("spawn supervised tree");
16213 let deadline = tokio::time::Instant::now() + Duration::from_secs(10);
16214 let grandchild: u32 = loop {
16215 if let Ok(contents) = std::fs::read_to_string(&pid_file) {
16216 if let Ok(pid) = contents.trim().parse() {
16217 break pid;
16218 }
16219 }
16220 assert!(
16221 tokio::time::Instant::now() < deadline,
16222 "grandchild pid was not recorded"
16223 );
16224 tokio::time::sleep(Duration::from_millis(10)).await;
16225 };
16226 assert!(
16227 running(grandchild),
16228 "grandchild must be alive before teardown"
16229 );
16230 if shutdown {
16231 let mut child = child;
16232 crate::child_roster::end_children_for_daemon_shutdown(
16233 &runtime.child_roster,
16234 false,
16235 std::future::pending(),
16236 )
16237 .await;
16238 child.wait().await.expect("reap shutdown straggler");
16239 } else {
16240 drain_child_to_state(
16241 module_id,
16242 ModuleProtocol::None,
16243 StopNotice::NotSent,
16244 &Registry::default(),
16245 None,
16246 &snapshot,
16247 &runtime.terminal_ring,
16248 &SpawnEventFeed::default(),
16249 child,
16250 Duration::from_millis(100),
16251 ModuleState::Stopped,
16252 Some(false),
16253 )
16254 .await
16255 .expect("real supervisor teardown");
16256 }
16257 let deadline = tokio::time::Instant::now() + Duration::from_secs(5);
16258 while running(grandchild) && tokio::time::Instant::now() < deadline {
16259 tokio::time::sleep(Duration::from_millis(10)).await;
16260 }
16261 let survived = running(grandchild);
16262 if survived {
16264 let pid = rustix::process::Pid::from_raw(grandchild as i32).unwrap();
16265 let _ = rustix::process::kill_process(pid, rustix::process::Signal::KILL);
16266 tokio::time::sleep(Duration::from_millis(100)).await;
16267 }
16268 if module.exists() {
16269 std::fs::remove_dir(&module).expect("remove empty module cgroup");
16270 }
16271 std::fs::remove_dir(root.join("subc-modules")).unwrap();
16272 std::fs::remove_dir(&root).unwrap();
16273 assert!(
16274 !survived,
16275 "grandchild {grandchild} outlived module teardown"
16276 );
16277 eprintln!("EXECUTED {test_name}: grandchild {grandchild} killed in isolated cgroup");
16278 }
16279}