1use std::{
2 collections::{HashMap, HashSet, VecDeque},
3 error::Error,
4 fmt, io,
5 path::PathBuf,
6 process::{ExitStatus, Stdio},
7 sync::{Arc, Mutex, OnceLock},
8 time::{Duration, SystemTime, UNIX_EPOCH},
9};
10
11use cortexkit_log::Retention;
12use serde_json::Value;
13use sha2::{Digest, Sha256};
14use subc_control::{
15 ClientControlPush, LiveSpawn, ModuleProtocol, RouteCloseReason, SpawnCursor, SpawnEvent,
16 SpawnEventKind, SpawnSnapshot, SupervisorHealthStatus, TerminalDisposition, TerminalExitKind,
17};
18use subc_protocol::{
19 manifest::{SelfSignalKind, SignalAnchor},
20 session::{
21 HealthReport, HealthStatus, ModuleControlCommand, ModuleControlRequest,
22 MODULE_CONTROL_OP_HEALTH_CHECK,
23 },
24 Flags, FrameType, Principal, Priority, SUBC_LAUNCH_NONCE_ENV, SUBC_MODULE_ID_ENV,
25};
26use tokio::{
27 process::{Child, Command},
28 sync::{mpsc, oneshot, watch, Mutex as AsyncMutex},
29 task::JoinHandle,
30 time::{sleep, sleep_until, timeout, timeout_at, Instant},
31};
32use tracing::{debug, error, info, warn};
33
34use crate::{
35 child_roster::ChildRoster,
36 daemon_config::{
37 CAPTURE_KEEP_ENV, CAPTURE_MAX_AGE_DAYS_ENV, CAPTURE_MAX_FILE_MB_ENV, CK_LOG_ENV,
38 },
39 forwarding::{
40 CloseReason, ForwardingError, ForwardingTable, GoodbyeTarget, ModuleControlRpcOutcome,
41 ModuleDrainTarget, PendingModuleControlRpc,
42 },
43 provenance::{ExecutableIdentityProbe, SpawnedFileIdentity},
44 registry::{ConnectionId, RegistrationEndReason, RegistryError},
45 stderr_tail::{
46 pump_stderr_to, pump_stdout_to, ChildOutputSink, StderrRing, StderrTailConfig,
47 StderrTailSnapshot,
48 },
49 terminal_ring::{TerminalHistorySnapshot, TerminalRecord, TerminalRing, TerminalRingConfig},
50 Frame, FrameSink, Registry,
51};
52
53#[cfg(not(windows))]
54use crate::provenance::spawned_file_identity;
55
56#[path = "supervise_swap.rs"]
57mod swap;
58
59#[path = "supervise_activity.rs"]
60mod activity;
61
62#[cfg(unix)]
63#[path = "supervise/admission.rs"]
64mod admission;
65#[cfg(all(test, windows))]
66mod carried_job_tests;
67pub(crate) mod containment;
68#[cfg(all(test, unix))]
69mod containment_tests;
70mod slow_retry;
71use slow_retry::{FailedRetry, FailureContext};
72
73#[cfg(all(test, unix))]
74mod slow_retry_tests;
75
76pub const SUBC_ARG: &str = "--subc";
82
83const DEFAULT_MAX_RESTARTS: u32 = 3;
84const DEFAULT_BACKOFF: Duration = Duration::from_millis(100);
85const DEFAULT_MAX_BACKOFF: Duration = Duration::from_secs(30);
86const DEFAULT_RESTART_WINDOW: Duration = Duration::from_secs(600);
90pub const DEFAULT_DRAIN_TIMEOUT: Duration = Duration::from_secs(30);
101pub(crate) const DEFAULT_REGISTER_TIMEOUT: Duration = Duration::from_secs(120);
102const REGISTRY_RELEASE_TIMEOUT: Duration = Duration::from_secs(1);
103const REGISTRY_RELEASE_POLL: Duration = Duration::from_millis(10);
104const DRAIN_BUSY_REPROBE_INTERVAL: Duration = Duration::from_millis(250);
116const STDERR_PUMP_DRAIN_TIMEOUT: Duration = Duration::from_millis(250);
139pub const SPAWN_EVENT_RING_CAPACITY: usize = 4096;
141const SPAWN_SUBSCRIBER_BUFFER: usize = SPAWN_EVENT_RING_CAPACITY + 1;
142pub(crate) const SPAWN_SUBSCRIBER_LAGGED_CODE: &str = "spawn_subscriber_lagged";
147
148#[cfg(target_os = "macos")]
149struct PrivacyExec {
150 reader: tokio::io::unix::AsyncFd<std::io::PipeReader>,
151 deadline: tokio::time::Instant,
152 expected: Option<subc_os::FileIdentity>,
153 trampoline: Option<subc_os::FileIdentity>,
154 script: bool,
155 module_id: String,
156 #[cfg(test)]
157 confirmation_pause: Option<(String, Option<tokio::net::TcpStream>)>,
158}
159
160#[cfg(all(test, target_os = "macos"))]
161pub(crate) fn test_privacy_trampoline() -> PathBuf {
162 let path = std::env::current_exe()
165 .unwrap()
166 .parent()
167 .unwrap()
168 .parent()
169 .unwrap()
170 .join("privacy-trampoline-fixture");
171 assert!(
175 path.exists(),
176 "privacy-trampoline-fixture not built at {}: run `cargo build -p subc-daemon \
177 --bins --features test-support` or `cargo test -p subc-daemon` first",
178 path.display()
179 );
180 path
181}
182
183#[cfg(target_os = "macos")]
184fn probe_privacy_trampoline(path: &std::path::Path) -> Result<(), String> {
185 use std::io::Read;
186 let mut probe = std::process::Command::new(path)
187 .args(["__disclaim-exec", "--probe"])
188 .stdin(Stdio::null())
189 .stdout(Stdio::piped())
190 .stderr(Stdio::piped())
191 .spawn()
192 .map_err(|error| {
193 format!(
194 "privacy trampoline probe failed for {}: {error}",
195 path.display()
196 )
197 })?;
198 let deadline = std::time::Instant::now() + Duration::from_secs(5);
199 let status = loop {
200 match probe.try_wait() {
201 Ok(Some(status)) => break status,
202 Ok(None) if std::time::Instant::now() < deadline => {
203 std::thread::sleep(Duration::from_millis(5))
204 }
205 result => {
206 let _ = probe.kill();
207 let _ = probe.wait();
208 return Err(format!(
209 "privacy trampoline probe failed or timed out for {}: {result:?}",
210 path.display()
211 ));
212 }
213 }
214 };
215 let mut answer = String::new();
216 if let Some(stdout) = probe.stdout.take() {
217 let _ = stdout.take(256).read_to_string(&mut answer);
218 }
219 if status.success() && answer.trim() == subc_os::privacy_identity::TRAMPOLINE_PROBE {
220 return Ok(());
221 }
222 let mut diagnostic = String::new();
223 if let Some(stderr) = probe.stderr.take() {
224 let _ = stderr.take(1024).read_to_string(&mut diagnostic);
225 }
226 let cause = diagnostic
227 .trim()
228 .strip_prefix("ck-subc: own privacy identity refused: ")
229 .unwrap_or("binary does not implement the privacy trampoline protocol");
230 Err(format!(
231 "{cause}: probe of {} exited {status}",
232 path.display()
233 ))
234}
235
236#[cfg(target_os = "macos")]
237fn privacy_command(
238 spec: &ModuleSpec,
239 roster: &ChildRoster,
240) -> Result<
241 (
242 Command,
243 Option<PrivacyExec>,
244 subc_os::privacy_identity::ExecAcknowledgement,
245 ),
246 SuperviseError,
247> {
248 let failure = |cause: String| {
249 warn!(module_id = %spec.module_id, %cause, "privacy identity trampoline refused module spawn");
250 SuperviseError::Spawn {
251 program: spec.program.clone(),
252 source: io::Error::other(cause),
253 cgroup_path: None,
254 }
255 };
256 let trampoline = roster.privacy_trampoline().map_err(failure)?;
257 let program = if spec.program.components().count() == 1 && !spec.program.is_absolute() {
262 let path = spec
263 .env
264 .iter()
265 .find(|(key, _)| key == "PATH")
266 .map(|(_, value)| std::ffi::OsString::from(value))
267 .or_else(|| std::env::var_os("PATH"))
268 .unwrap_or_else(|| "/usr/bin:/bin".into());
269 std::env::split_paths(&path)
270 .map(|dir| dir.join(&spec.program))
271 .find(|path| path.is_file())
272 .unwrap_or_else(|| spec.program.clone())
273 } else {
274 spec.program.clone()
275 };
276 let expected = subc_os::file_identity(&program);
277 let trampoline_image = subc_os::file_identity(&trampoline);
278 let script = {
279 use std::io::Read;
280 let mut prefix = [0u8; 2];
281 std::fs::File::open(&program)
282 .is_ok_and(|mut file| file.read_exact(&mut prefix).is_ok() && &prefix == b"#!")
283 };
284 if expected.is_none() || expected == subc_os::file_identity(&trampoline) {
285 return Err(failure(
286 "privacy identity module executable is missing or is the trampoline itself".to_string(),
287 ));
288 }
289 let (reader, ack) = subc_os::privacy_identity::ExecAcknowledgement::pipe()
290 .map_err(|error| failure(error.to_string()))?;
291 let reader =
292 tokio::io::unix::AsyncFd::new(reader).map_err(|error| failure(error.to_string()))?;
293 let mut command = Command::new(&trampoline);
294 command
295 .arg("__disclaim-exec")
296 .arg(ack.fd().to_string())
297 .arg(&program);
298 Ok((
299 command,
300 Some(PrivacyExec {
301 reader,
302 deadline: tokio::time::Instant::now() + Duration::from_secs(5),
303 expected,
304 trampoline: trampoline_image,
305 script,
306 module_id: spec.module_id.clone(),
307 #[cfg(test)]
308 confirmation_pause: spec
309 .env
310 .iter()
311 .find(|(key, _)| key == "SUBC_TEST_BEFORE_EXEC_CONFIRM")
312 .map(|(_, address)| (address.clone(), None)),
313 }),
314 ack,
315 ))
316}
317
318struct SupervisedChild {
319 subtree_target: crate::subtree_resources::Target,
320 child: containment::ProcessChild,
321 exec: crate::activity::ExecOwner,
322 #[cfg(windows)]
323 nonce_delivery: Option<subc_os::LaunchNoncePipeDelivery>,
324 #[cfg(target_os = "macos")]
325 privacy_exec: Option<PrivacyExec>,
326 #[cfg(target_os = "macos")]
333 report_ready: Arc<OnceLock<()>>,
334 spawn_failure: Option<String>,
337 protocol: ModuleProtocol,
342 #[cfg(target_os = "linux")]
349 module_id: String,
350 #[cfg(target_os = "linux")]
351 cgroup_placement: Option<subc_cgroup::Placement>,
352 #[cfg(windows)]
375 job: Option<Arc<Mutex<subc_jobobject::JobObject>>>,
376 stdout_pump: Option<JoinHandle<()>>,
377 stderr_pump: Option<StderrPump>,
378 stderr_ring: Arc<Mutex<StderrRing>>,
379 spawned_at: Instant,
382 registration_deadline: Option<Instant>,
385 registration_watch: Option<RegistrationWatch>,
386 spawned_at_ms: u64,
387 spawned_from: PathBuf,
388 launch_spec_digest: LaunchSpecDigest,
389 spawned_file_identity: Option<SpawnedFileIdentity>,
390 process_start_time: Option<u64>,
391 process_identity: Option<ProcessIdentity>,
392 pid: u32,
393 roster_guard: Option<crate::child_roster::RosterGuard>,
396}
397
398impl SupervisedChild {
399 fn registered_since_spawn(&self) -> bool {
400 self.registration_watch
401 .as_ref()
402 .is_some_and(|watch| *watch.changes.borrow() != watch.baseline)
403 }
404
405 fn id(&self) -> Option<u32> {
406 Some(self.pid)
407 }
408
409 fn process_identity(&self) -> Option<ProcessIdentity> {
410 self.process_identity
411 }
412
413 async fn wait(&mut self) -> io::Result<ExitStatus> {
414 #[cfg(target_os = "macos")]
415 self.confirm_privacy_exec().await;
416 let result = self.child.wait().await;
424 #[cfg(target_os = "linux")]
425 if result.is_ok() && self.child.container().is_none() {
426 if let Some(placement) = self.cgroup_placement.as_ref() {
427 cleanup_reaped_cgroup(placement, &self.module_id).await;
431 self.cgroup_placement = None;
432 }
433 }
434 result
435 }
436
437 async fn finish_exit(&mut self, carry: bool) -> io::Result<()> {
438 if self.child.container().is_some() {
439 if !carry {
440 self.child.terminate_container().await;
441 }
442 } else {
443 #[cfg(target_os = "linux")]
444 if let Some(placement) = self.cgroup_placement.as_ref() {
445 cleanup_reaped_cgroup(placement, &self.module_id).await;
446 self.cgroup_placement = None;
447 }
448 }
449 self.child.reap().await?;
450 self.release_roster();
451 Ok(())
452 }
453
454 #[cfg(target_os = "macos")]
455 async fn confirm_privacy_exec(&mut self) {
456 let Some(pending) = &mut self.privacy_exec else {
457 return;
458 };
459 let result = tokio::time::timeout_at(pending.deadline, async {
460 let mut record = Vec::new();
461 loop {
462 let mut ready = pending.reader.readable().await?;
463 let read = ready.try_io(|reader| {
464 use std::io::Read;
465 let mut reader = reader.get_ref();
466 let mut buffer = [0u8; 256];
467 reader.read(&mut buffer).map(|count| (count, buffer))
468 });
469 match read {
470 Ok(Ok((0, _))) => return Ok::<_, io::Error>(record),
471 Ok(Ok((count, buffer))) => {
472 if record.len() + count > 1024 {
473 return Err(io::Error::other(
474 "privacy exec refusal record is too long",
475 ));
476 }
477 record.extend_from_slice(&buffer[..count]);
478 }
479 Ok(Err(error)) => return Err(error),
480 Err(_) => continue,
481 }
482 }
483 })
484 .await;
485 #[cfg(test)]
486 let result = if let Some((address, stream)) = &mut pending.confirmation_pause {
487 use tokio::io::{AsyncReadExt, AsyncWriteExt};
488 if stream.is_none() {
489 *stream = Some(
490 tokio::net::TcpStream::connect(address.as_str())
491 .await
492 .unwrap(),
493 );
494 stream.as_mut().unwrap().write_all(b"R").await.unwrap();
495 stream
496 .as_mut()
497 .unwrap()
498 .write_all(&self.pid.to_ne_bytes())
499 .await
500 .unwrap();
501 }
502 let mut reply = [0];
503 stream
504 .as_mut()
505 .unwrap()
506 .read_exact(&mut reply)
507 .await
508 .unwrap();
509 pending.confirmation_pause = None;
510 if &reply == b"F" {
511 Ok(Err(io::Error::other("injected late exec refusal")))
512 } else {
513 assert_eq!(&reply, b"X");
514 result
515 }
516 } else {
517 result
518 };
519 let pending = self.privacy_exec.as_ref().expect("pending exec");
522 let cause = match result {
523 Err(_) => Some("privacy identity trampoline did not exec within 5s".to_string()),
524 Ok(Err(error)) => Some(format!(
525 "privacy identity exec acknowledgement failed: {error}"
526 )),
527 Ok(Ok(record)) if !record.is_empty() => Some(
528 std::str::from_utf8(&record)
529 .ok()
530 .and_then(|record| {
531 record
532 .trim()
533 .strip_prefix(subc_os::privacy_identity::EXEC_REFUSAL_TAG)
534 })
535 .filter(|cause| !cause.is_empty())
536 .unwrap_or("invalid privacy exec refusal record")
537 .to_string(),
538 ),
539 Ok(Ok(_)) => match self.child.try_wait() {
540 Ok(Some(_status)) => None,
544 Err(error) => Some(format!("privacy identity trampoline wait failed: {error}")),
545 Ok(None) => {
546 let image = observe_spawned_image(self.pid);
547 if let Some(image) = image.filter(|image| {
548 image.executable.is_some()
549 && image.executable != pending.trampoline
550 && (image.executable == pending.expected || pending.script)
551 }) {
552 if let Err(error) = self
553 .roster_guard
554 .as_ref()
555 .map(|guard| guard.confirm_executable(image))
556 .transpose()
557 {
558 Some(format!(
559 "could not durably record confirmed module image: {error}"
560 ))
561 } else {
562 let _ = self.report_ready.set(());
563 info!(module_id = %pending.module_id, pid = self.pid,
564 "module spawned with own privacy identity (responsibility disclaimed)");
565 None
566 }
567 } else if image.is_none()
568 || image.is_some_and(|image| image.executable.is_none())
569 {
570 match tokio::time::timeout_at(pending.deadline, self.child.wait()).await {
577 Ok(Ok(_status)) => None,
578 Ok(Err(error)) => Some(format!("privacy identity trampoline wait failed: {error}")),
579 Err(_) => Some("privacy identity module executable remained unreadable until the 5s exec deadline".to_string()),
580 }
581 } else {
582 Some("privacy identity trampoline executable mismatch: expected the module program, not ck-subc".to_string())
583 }
584 }
585 },
586 };
587 let pending = self.privacy_exec.take().expect("pending exec");
588 self.exec.0.finish(if cause.is_some() {
591 crate::activity::ExecState::Refused
592 } else {
593 crate::activity::ExecState::CommittedExec
594 });
595 if let Some(cause) = cause {
596 warn!(module_id = %pending.module_id, pid = self.pid, %cause, "privacy identity trampoline refused module spawn");
597 self.spawn_failure = Some(cause);
598 if let Some(container) = self.child.container() {
601 container.hard_kill();
602 } else if let Some(pid) = rustix::process::Pid::from_raw(self.pid as i32) {
603 let _ = rustix::process::kill_process_group(pid, rustix::process::Signal::KILL);
604 }
605 let _ = self.child.start_kill();
606 }
607 }
608
609 fn release_roster(&mut self) {
613 self.roster_guard = None;
614 }
615
616 fn start_kill(&mut self) -> io::Result<()> {
629 if let Some(container) = self.child.container() {
630 container.hard_kill();
631 return self.kill_leader();
632 }
633 #[cfg(windows)]
634 if let Some(job) = &self.job {
635 if let Err(error) = job.lock().unwrap_or_else(|p| p.into_inner()).terminate() {
636 debug!(
637 error = %error,
638 "job termination failed; the direct-child kill still owns the outcome"
639 );
640 }
641 }
642 #[cfg(target_os = "linux")]
643 kill_module_cgroup(self.cgroup_placement.as_ref(), &self.module_id);
644 self.child.start_kill()
645 }
646
647 fn kill_leader(&mut self) -> io::Result<()> {
648 self.child.start_kill()
649 }
650
651 async fn drain_stderr(&mut self, module_id: &str) {
652 if let Some(pump) = self.stdout_pump.take() {
653 settle_stdout_pump(module_id, pump, STDERR_PUMP_DRAIN_TIMEOUT).await;
654 }
655
656 let Some(pump) = self.stderr_pump.take() else {
657 return;
658 };
659 settle_stderr_pump(
660 module_id,
661 &self.stderr_ring,
662 pump,
663 STDERR_PUMP_DRAIN_TIMEOUT,
664 )
665 .await;
666 }
667}
668
669async fn settle_stdout_pump(module_id: &str, mut pump: JoinHandle<()>, bound: Duration) {
673 match timeout(bound, &mut pump).await {
674 Ok(Ok(())) => {}
675 Ok(Err(error)) => {
676 warn!(module_id, error = %error, "stdout pump ended unexpectedly");
677 }
678 Err(_) => {
679 drop(pump);
680 warn!(
681 module_id,
682 waited = ?bound,
683 "stdout pipe of the exited process has not drained; its reader keeps running without delaying the restart"
684 );
685 }
686 }
687}
688
689struct StderrPump {
692 task: JoinHandle<()>,
693 generation: u64,
694}
695
696async fn settle_stderr_pump(
702 module_id: &str,
703 ring: &Arc<Mutex<StderrRing>>,
704 pump: StderrPump,
705 bound: Duration,
706) {
707 let lock = || ring.lock().unwrap_or_else(|poisoned| poisoned.into_inner());
708 let StderrPump {
709 mut task,
710 generation,
711 } = pump;
712 lock().retire_pump(generation);
713 match timeout(bound, &mut task).await {
714 Ok(Ok(())) => {}
715 Ok(Err(err)) => {
716 let mut ring = lock();
717 ring.mark_incomplete_from(generation, format!("stderr pump ended unexpectedly: {err}"));
718 ring.finish_pump(generation);
719 warn!(module_id, error = %err, "stderr pump ended before clean EOF");
720 }
721 Err(_) => {
722 drop(task);
724 lock().mark_pump_late(
725 generation,
726 format!(
727 "stderr of the exited process had not reached EOF {bound:?} after it was \
728 retired (a descendant may still hold the pipe open); lines it still \
729 writes are kept in that process's section"
730 ),
731 );
732 warn!(
733 module_id,
734 waited = ?bound,
735 "stderr pipe of the exited process is still open; its reader keeps running without delaying the restart"
736 );
737 }
738 }
739}
740
741fn registration_release_events() -> &'static watch::Sender<u64> {
742 static EVENTS: OnceLock<watch::Sender<u64>> = OnceLock::new();
743 EVENTS.get_or_init(|| {
744 let (sender, _receiver) = watch::channel(0);
745 sender
746 })
747}
748
749pub(crate) fn notify_registration_release() {
750 let events = registration_release_events();
751 let next_generation = (*events.borrow()).wrapping_add(1);
752 events.send_replace(next_generation);
753}
754
755#[derive(Debug, Clone, PartialEq, Eq)]
757pub struct ModuleSpec {
758 pub module_id: String,
759 pub program: PathBuf,
760 pub args: Vec<String>,
761 pub env: Vec<(String, String)>,
762 pub reserved: bool,
767 pub reserved_prefixes: Vec<String>,
772 pub protocol: ModuleProtocol,
791 pub overlap: ModuleOverlap,
796 pub carry_containment: bool,
800}
801
802#[derive(Clone, PartialEq, Eq)]
805pub(crate) struct LaunchSpecDigest([u8; 32]);
806
807impl fmt::Debug for LaunchSpecDigest {
808 fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
809 f.write_str("LaunchSpecDigest([redacted])")
810 }
811}
812
813impl LaunchSpecDigest {
814 pub(crate) fn from_spec(spec: &ModuleSpec) -> Self {
815 fn field(hash: &mut Sha256, bytes: &[u8]) {
816 hash.update((bytes.len() as u64).to_le_bytes());
817 hash.update(bytes);
818 }
819 let mut hash = Sha256::new();
820 if let Some(program) = spec.program.to_str() {
821 field(&mut hash, b"utf8-program");
822 field(&mut hash, program.as_bytes());
823 } else {
824 #[cfg(unix)]
827 {
828 use std::os::unix::ffi::OsStrExt;
829 field(&mut hash, b"unix-program");
830 field(&mut hash, spec.program.as_os_str().as_bytes());
831 }
832 #[cfg(windows)]
833 {
834 use std::os::windows::ffi::OsStrExt;
835 field(&mut hash, b"windows-program");
836 let bytes = spec
837 .program
838 .as_os_str()
839 .encode_wide()
840 .flat_map(u16::to_le_bytes)
841 .collect::<Vec<_>>();
842 field(&mut hash, &bytes);
843 }
844 #[cfg(not(any(unix, windows)))]
845 field(&mut hash, spec.program.to_string_lossy().as_bytes());
846 }
847 hash.update((spec.args.len() as u64).to_le_bytes());
848 for arg in &spec.args {
849 field(&mut hash, arg.as_bytes());
850 }
851 let mut env = spec.env.iter().collect::<Vec<_>>();
852 env.sort_unstable();
853 hash.update((env.len() as u64).to_le_bytes());
854 for (key, value) in env {
855 field(&mut hash, key.as_bytes());
856 field(&mut hash, value.as_bytes());
857 }
858 field(
859 &mut hash,
860 match spec.protocol {
861 ModuleProtocol::Subc => b"subc",
862 ModuleProtocol::None => b"none",
863 },
864 );
865 Self(hash.finalize().into())
866 }
867}
868
869#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)]
876pub enum ModuleOverlap {
877 #[default]
879 Exclusive,
880 Safe,
892}
893
894impl ModuleOverlap {
895 pub fn as_str(self) -> &'static str {
896 match self {
897 Self::Exclusive => "exclusive",
898 Self::Safe => "safe",
899 }
900 }
901}
902
903pub const SUBC_SPAWN_ROLE_ENV: &str = "SUBC_SPAWN_ROLE";
913pub const SPAWN_ROLE_SWAP_CANDIDATE: &str = "swap_candidate";
915pub const DEFAULT_SWAP_READY_TIMEOUT: Duration = Duration::from_secs(100);
920
921#[derive(Debug, Clone, Copy, PartialEq, Eq)]
939pub struct RestartPolicy {
940 pub max_restarts: u32,
941 pub backoff: Duration,
944 pub max_backoff: Duration,
946 pub window: Duration,
950}
951
952impl RestartPolicy {
953 pub fn new(max_restarts: u32, backoff: Duration) -> Self {
957 Self {
958 max_restarts,
959 backoff,
960 max_backoff: DEFAULT_MAX_BACKOFF,
961 window: DEFAULT_RESTART_WINDOW,
962 }
963 }
964
965 pub fn with_max_backoff(mut self, max_backoff: Duration) -> Self {
966 self.max_backoff = max_backoff;
967 self
968 }
969
970 pub fn with_window(mut self, window: Duration) -> Self {
971 self.window = window;
972 self
973 }
974
975 fn delay_for_restart(&self, restart_in_window: u32) -> Duration {
980 if self.backoff.is_zero() || self.max_backoff.is_zero() {
981 return Duration::ZERO;
982 }
983
984 let mut delay = self.backoff;
985 for _ in 0..restart_in_window {
986 if delay >= self.max_backoff {
987 return self.max_backoff;
988 }
989 delay = delay
990 .checked_mul(10)
991 .unwrap_or(self.max_backoff)
992 .min(self.max_backoff);
993 }
994 delay.min(self.max_backoff)
995 }
996
997 fn budget_exhausted_detail(&self) -> String {
1002 format!(
1003 "crash budget exhausted: max_restarts={} within window_secs={}",
1004 self.max_restarts,
1005 self.window.as_secs()
1006 )
1007 }
1008}
1009
1010impl Default for RestartPolicy {
1011 fn default() -> Self {
1012 Self {
1013 max_restarts: DEFAULT_MAX_RESTARTS,
1014 backoff: DEFAULT_BACKOFF,
1015 max_backoff: DEFAULT_MAX_BACKOFF,
1016 window: DEFAULT_RESTART_WINDOW,
1017 }
1018 }
1019}
1020
1021#[derive(Debug, Clone, Copy, PartialEq, Eq)]
1022struct CrashRestartSchedule {
1023 restart_in_window: u32,
1024 delay: Duration,
1025}
1026
1027fn daemon_will_restart(
1034 state: &mut SupervisorSnapshot,
1035 policy: &RestartPolicy,
1036 now: Instant,
1037) -> bool {
1038 state.enabled && state.crash_restarts_in_window(policy.window, now) < policy.max_restarts
1039}
1040
1041const DEFAULT_HEALTH_CADENCE: Duration = Duration::from_secs(30);
1042const DEFAULT_HEALTH_DEADLINE: Duration = Duration::from_secs(5);
1043const DEFAULT_HEALTH_FAILURE_THRESHOLD: u32 = 3;
1044const MAX_HEALTH_METRICS_BYTES: usize = 16 * 1024;
1045
1046#[derive(Debug, Clone, Copy, PartialEq, Eq)]
1047pub enum HealthAction {
1048 Report,
1049 Restart,
1050 Alert,
1051}
1052
1053impl fmt::Display for HealthAction {
1054 fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
1055 f.write_str(match self {
1056 Self::Report => "report",
1057 Self::Restart => "restart",
1058 Self::Alert => "alert",
1059 })
1060 }
1061}
1062
1063#[derive(Debug, Clone, PartialEq, Eq)]
1064pub struct HealthConfig {
1065 pub http: Option<String>,
1069 pub cadence: Duration,
1070 pub deadline: Duration,
1071 pub failure_threshold: u32,
1072 pub on_degraded: HealthAction,
1073 pub on_failing: HealthAction,
1074 pub critical: bool,
1075}
1076
1077impl Default for HealthConfig {
1078 fn default() -> Self {
1079 Self {
1080 http: None,
1081 cadence: DEFAULT_HEALTH_CADENCE,
1082 deadline: DEFAULT_HEALTH_DEADLINE,
1083 failure_threshold: DEFAULT_HEALTH_FAILURE_THRESHOLD,
1084 on_degraded: HealthAction::Report,
1085 on_failing: HealthAction::Report,
1086 critical: false,
1087 }
1088 }
1089}
1090
1091#[derive(Debug, Clone, PartialEq)]
1109pub struct ModuleHealthStatus {
1110 pub status: SupervisorHealthStatus,
1111 pub last_probe_ms: Option<u64>,
1112 pub detail: Option<String>,
1113 pub metrics: Option<Value>,
1114 pub consecutive_failures: u32,
1115 pub late_answer_count: u64,
1118 pub last_late_answer_latency_ms: Option<u64>,
1120 pub last_action: Option<String>,
1121 pub last_action_ms: Option<u64>,
1125}
1126
1127impl Default for ModuleHealthStatus {
1128 fn default() -> Self {
1129 Self {
1130 status: SupervisorHealthStatus::Unknown,
1131 last_probe_ms: None,
1132 detail: None,
1133 metrics: None,
1134 consecutive_failures: 0,
1135 late_answer_count: 0,
1136 last_late_answer_latency_ms: None,
1137 last_action: None,
1138 last_action_ms: None,
1139 }
1140 }
1141}
1142
1143#[derive(Debug, Clone, Copy, PartialEq, Eq)]
1145pub enum ModuleState {
1146 Starting,
1147 Running,
1148 Unresponsive,
1149 Restarting,
1150 Draining,
1151 Stopped,
1152 Failed,
1153 Disabled,
1154}
1155
1156impl fmt::Display for ModuleState {
1157 fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
1158 f.write_str(match self {
1159 Self::Starting => "starting",
1160 Self::Running => "running",
1161 Self::Unresponsive => "unresponsive",
1162 Self::Restarting => "restarting",
1163 Self::Draining => "draining",
1164 Self::Stopped => "stopped",
1165 Self::Failed => "failed",
1166 Self::Disabled => "disabled",
1167 })
1168 }
1169}
1170
1171#[derive(Debug, Clone, Copy, PartialEq, Eq)]
1173pub enum ExitKind {
1174 Clean,
1175 Crash,
1176 DeliberateSeverance,
1177}
1178
1179impl From<ExitKind> for TerminalExitKind {
1180 fn from(kind: ExitKind) -> Self {
1181 match kind {
1182 ExitKind::Clean => Self::Clean,
1183 ExitKind::Crash => Self::Crash,
1184 ExitKind::DeliberateSeverance => Self::DeliberateSeverance,
1185 }
1186 }
1187}
1188
1189#[derive(Debug, Clone, Copy, PartialEq, Eq)]
1192pub(crate) struct ProcessIdentity {
1193 pub(crate) pid: u32,
1194 pub(crate) start_time: u64,
1195}
1196
1197#[derive(Debug, Clone, PartialEq, Eq)]
1199pub struct ExitReport {
1200 pub kind: ExitKind,
1201 pub code: Option<i32>,
1202 pub signal: Option<i32>,
1203 pub at_ms: u64,
1204}
1205
1206#[derive(Debug, Clone, PartialEq)]
1209pub struct ModuleStatus {
1210 pub module_id: String,
1211 pub waiting_for_previous_pid: Option<u32>,
1213 pub waiting_for_previous_detail: Option<String>,
1215 pub launch_nonce_source: Option<String>,
1220 pub state: ModuleState,
1221 pub enabled: bool,
1222 pub process_alive: bool,
1223 pub registration_active: bool,
1224 pub protocol: ModuleProtocol,
1229 pub live: bool,
1240 pub restart_count: u32,
1244 pub lifetime_restarts: u32,
1248 pub spawn_generation: u64,
1249 pub max_restarts: u32,
1254 pub restart_window: Duration,
1258 pub drain_timeout: Duration,
1262 pub restart_backoff: Duration,
1263 pub restart_max_backoff: Duration,
1264 pub pid: Option<u32>,
1269 pub spawned_at_ms: Option<u64>,
1270 pub spawned_from: Option<PathBuf>,
1271 pub(crate) launch_spec_digest: Option<LaunchSpecDigest>,
1272 pub process_start_time: Option<u64>,
1273 pub last_exit: Option<ExitReport>,
1274 pub health: ModuleHealthStatus,
1275}
1276
1277#[cfg(test)]
1278#[derive(Debug, Clone, Copy, PartialEq)]
1279struct ActorSelectCheckpoint {
1280 generation: u64,
1281 turn: u64,
1282 registered_connection: Option<ConnectionId>,
1283 next_probe_at: Option<Instant>,
1284 wake_after: Option<Duration>,
1285}
1286
1287#[derive(Debug, Clone, PartialEq)]
1288struct SupervisorSnapshot {
1289 subtree_target: crate::subtree_resources::Target,
1290 failed_retry: FailedRetry,
1291 exec_outcome: Option<Arc<crate::activity::ExecOutcome>>,
1292 #[cfg(windows)]
1293 pipe_consumed: Option<Arc<OnceLock<()>>>,
1294 #[cfg(test)]
1296 actor_turns: u64,
1297 #[cfg(test)]
1300 actor_select: Option<ActorSelectCheckpoint>,
1301 state: ModuleState,
1302 enabled: bool,
1303 process_alive: bool,
1304 spawned_protocol: Option<ModuleProtocol>,
1305 spawn_failure: Option<String>,
1306 waiting_for_previous_pid: Option<u32>,
1307 waiting_for_previous_detail: Option<String>,
1308 crash_restarts: VecDeque<Instant>,
1314 lifetime_restarts: u32,
1315 spawn_generation: u64,
1324 pid: Option<u32>,
1325 #[cfg(target_os = "macos")]
1326 report_ready: Option<Arc<OnceLock<()>>>,
1327 reaped_pid: Option<u32>,
1329 respawn_pending: bool,
1331 coalesced_restart_pending: bool,
1333 spawned_at_ms: Option<u64>,
1334 spawned_from: Option<PathBuf>,
1335 launch_spec_digest: Option<LaunchSpecDigest>,
1336 spawned_file_identity: Option<SpawnedFileIdentity>,
1337 process_start_time: Option<u64>,
1338 deliberate_severance: Option<ProcessIdentity>,
1339 last_exit: Option<ExitReport>,
1340 drain_disposition_detail: Option<String>,
1342 health: ModuleHealthStatus,
1343 in_alternate_slot: bool,
1348 draining_to_replace: bool,
1355 configuration_updated_since_spawn: bool,
1361}
1362
1363impl SupervisorSnapshot {
1364 fn reported_pid(&self) -> Option<u32> {
1369 #[cfg(target_os = "macos")]
1370 if self
1371 .report_ready
1372 .as_ref()
1373 .is_some_and(|ready| ready.get().is_none())
1374 {
1375 return None;
1376 }
1377 self.pid
1378 }
1379
1380 fn starting() -> Self {
1381 Self::new(ModuleState::Starting, true)
1382 }
1383
1384 fn disabled() -> Self {
1385 Self::new(ModuleState::Disabled, false)
1386 }
1387
1388 fn crash_restarts_in_window(&mut self, window: Duration, now: Instant) -> u32 {
1392 while let Some(oldest) = self.crash_restarts.front() {
1393 if now.duration_since(*oldest) > window {
1394 self.crash_restarts.pop_front();
1395 } else {
1396 break;
1397 }
1398 }
1399 u32::try_from(self.crash_restarts.len()).unwrap_or(u32::MAX)
1400 }
1401
1402 fn record_crash_restart(&mut self, policy: &RestartPolicy, now: Instant) {
1408 self.crash_restarts.push_back(now);
1409 while self.crash_restarts.len() > policy.max_restarts as usize {
1410 self.crash_restarts.pop_front();
1411 }
1412 self.lifetime_restarts += 1;
1413 }
1414
1415 fn next_crash_restart(
1419 &mut self,
1420 policy: &RestartPolicy,
1421 now: Instant,
1422 ) -> Option<CrashRestartSchedule> {
1423 let restart_in_window = self.crash_restarts_in_window(policy.window, now);
1424 if restart_in_window >= policy.max_restarts {
1425 return None;
1426 }
1427 self.record_crash_restart(policy, now);
1428 Some(CrashRestartSchedule {
1429 restart_in_window,
1430 delay: policy.delay_for_restart(restart_in_window),
1431 })
1432 }
1433
1434 fn clear_crash_restarts(&mut self) {
1439 self.crash_restarts.clear();
1440 }
1441
1442 fn new(state: ModuleState, enabled: bool) -> Self {
1443 Self {
1444 subtree_target: crate::subtree_resources::Target::Unavailable("not_running"),
1445 failed_retry: FailedRetry::default(),
1446 exec_outcome: None,
1447 #[cfg(test)]
1448 actor_turns: 0,
1449 #[cfg(test)]
1450 actor_select: None,
1451 state,
1452 enabled,
1453 process_alive: false,
1454 #[cfg(windows)]
1455 pipe_consumed: None,
1456 spawned_protocol: None,
1457 spawn_failure: None,
1458 waiting_for_previous_pid: None,
1459 waiting_for_previous_detail: None,
1460 crash_restarts: VecDeque::new(),
1461 lifetime_restarts: 0,
1462 spawn_generation: 0,
1463 pid: None,
1464 #[cfg(target_os = "macos")]
1465 report_ready: None,
1466 reaped_pid: None,
1467 respawn_pending: false,
1468 coalesced_restart_pending: false,
1469 spawned_at_ms: None,
1470 spawned_from: None,
1471 launch_spec_digest: None,
1472 spawned_file_identity: None,
1473 process_start_time: None,
1474 deliberate_severance: None,
1475 last_exit: None,
1476 drain_disposition_detail: None,
1477 health: ModuleHealthStatus::default(),
1478 in_alternate_slot: false,
1479 draining_to_replace: false,
1480 configuration_updated_since_spawn: false,
1481 }
1482 }
1483}
1484
1485type SharedSnapshot = Arc<Mutex<SupervisorSnapshot>>;
1486
1487type SpawnSubscriberKey = (ConnectionId, u64);
1488
1489#[derive(Debug)]
1490struct SpawnSubscriber {
1491 version: u8,
1492 frames: mpsc::Sender<Frame>,
1493 lagged: Option<oneshot::Sender<SpawnCursor>>,
1497}
1498
1499#[derive(Debug)]
1500struct SpawnEventState {
1501 daemon_incarnation: String,
1502 seq: u64,
1503 capacity: usize,
1504 live: HashMap<String, LiveSpawn>,
1505 generations: HashMap<String, u64>,
1506 events: VecDeque<SpawnEvent>,
1507 subscribers: HashMap<SpawnSubscriberKey, SpawnSubscriber>,
1508}
1509
1510impl Default for SpawnEventState {
1511 fn default() -> Self {
1512 Self {
1513 daemon_incarnation: "unconfigured".to_string(),
1514 seq: 0,
1515 capacity: SPAWN_EVENT_RING_CAPACITY,
1516 live: HashMap::new(),
1517 generations: HashMap::new(),
1518 events: VecDeque::new(),
1519 subscribers: HashMap::new(),
1520 }
1521 }
1522}
1523
1524#[derive(Debug, Clone, Default)]
1525struct SpawnEventFeed(
1526 Arc<Mutex<SpawnEventState>>,
1527 #[cfg(test)] Arc<Mutex<Option<std::net::TcpStream>>>,
1528);
1529
1530#[derive(Debug, Clone, PartialEq, Eq)]
1531pub(crate) enum SpawnSubscribeRefusal {
1532 ForeignIncarnation { current: String },
1533 TooOld { oldest: SpawnCursor },
1534 Frame(String),
1535}
1536
1537impl SpawnEventFeed {
1538 fn configure_incarnation(&self, daemon_incarnation: String) {
1539 let mut state = self.0.lock().unwrap_or_else(|p| p.into_inner());
1540 state.daemon_incarnation = daemon_incarnation;
1541 state.seq = 0;
1542 state.live.clear();
1543 state.generations.clear();
1544 state.events.clear();
1545 state.subscribers.clear();
1546 }
1547
1548 fn cursor(state: &SpawnEventState) -> SpawnCursor {
1549 SpawnCursor {
1550 daemon_incarnation: state.daemon_incarnation.clone(),
1551 seq: state.seq,
1552 }
1553 }
1554
1555 fn snapshot(&self) -> SpawnSnapshot {
1556 let state = self.0.lock().unwrap_or_else(|p| p.into_inner());
1557 let mut live = state.live.values().cloned().collect::<Vec<_>>();
1558 live.sort_by(|left, right| left.module_id.cmp(&right.module_id));
1559 SpawnSnapshot {
1560 cursor: Self::cursor(&state),
1561 ring_bound: state.capacity as u64,
1562 live,
1563 }
1564 }
1565
1566 fn emit_spawned(&self, module_id: &str, pid: u32, spawned_at_ms: u64) -> u64 {
1567 let mut state = self.0.lock().unwrap_or_else(|p| p.into_inner());
1568 let generation = state
1569 .generations
1570 .get(module_id)
1571 .copied()
1572 .unwrap_or(0)
1573 .checked_add(1)
1574 .expect("spawn generation exhausted");
1575 state.generations.insert(module_id.to_string(), generation);
1576 let live = LiveSpawn {
1577 module_id: module_id.to_string(),
1578 spawn_generation: generation,
1579 pid,
1580 spawned_at_ms,
1581 };
1582 state.live.insert(module_id.to_string(), live);
1583 Self::emit_locked(
1584 &mut state,
1585 SpawnEventKind::Spawned,
1586 module_id.to_string(),
1587 generation,
1588 pid,
1589 None,
1590 None,
1591 );
1592 generation
1593 }
1594
1595 fn emit_exited(&self, module_id: &str, exit_code: Option<i32>, exit_signal: Option<i32>) {
1596 let mut state = self.0.lock().unwrap_or_else(|p| p.into_inner());
1597 let Some(live) = state.live.remove(module_id) else {
1598 warn!(
1599 module_id,
1600 "terminal record had no live spawn event identity"
1601 );
1602 return;
1603 };
1604 Self::emit_locked(
1605 &mut state,
1606 SpawnEventKind::Exited,
1607 module_id.to_string(),
1608 live.spawn_generation,
1609 live.pid,
1610 exit_code,
1611 exit_signal,
1612 );
1613 }
1614
1615 fn emit_superseded_exited(
1622 &self,
1623 module_id: &str,
1624 spawn_generation: u64,
1625 pid: u32,
1626 exit_code: Option<i32>,
1627 exit_signal: Option<i32>,
1628 ) {
1629 let mut state = self.0.lock().unwrap_or_else(|p| p.into_inner());
1630 if state
1631 .live
1632 .get(module_id)
1633 .is_some_and(|live| live.spawn_generation == spawn_generation)
1634 {
1635 state.live.remove(module_id);
1636 }
1637 Self::emit_locked(
1638 &mut state,
1639 SpawnEventKind::Exited,
1640 module_id.to_string(),
1641 spawn_generation,
1642 pid,
1643 exit_code,
1644 exit_signal,
1645 );
1646 }
1647
1648 #[allow(clippy::too_many_arguments)]
1649 fn emit_locked(
1650 state: &mut SpawnEventState,
1651 kind: SpawnEventKind,
1652 module_id: String,
1653 spawn_generation: u64,
1654 pid: u32,
1655 exit_code: Option<i32>,
1656 exit_signal: Option<i32>,
1657 ) {
1658 state.seq = state
1659 .seq
1660 .checked_add(1)
1661 .expect("spawn event sequence exhausted");
1662 let event = SpawnEvent {
1663 cursor: Self::cursor(state),
1664 kind,
1665 module_id,
1666 spawn_generation,
1667 pid,
1668 exit_code,
1669 exit_signal,
1670 };
1671 state.events.push_back(event.clone());
1672 while state.events.len() > state.capacity {
1673 state.events.pop_front();
1674 }
1675 let body = match serde_json::to_vec(&event) {
1676 Ok(body) => body,
1677 Err(error) => {
1678 error!(%error, "failed to serialize supervisor spawn event");
1679 return;
1680 }
1681 };
1682 state.subscribers.retain(|(connection_id, corr), subscriber| {
1683 let frame = Frame::build_with_version(
1684 subscriber.version,
1685 FrameType::StreamData,
1686 control_flags(),
1687 0,
1688 0,
1689 *corr,
1690 body.clone(),
1691 );
1692 match frame {
1693 Ok(frame) => {
1694 if subscriber.frames.try_send(frame).is_ok() {
1695 true
1696 } else {
1697 warn!(connection_id = connection_id.get(), corr, "dropping lagged supervisor spawn subscriber");
1698 if let Some(lagged) = subscriber.lagged.take() {
1699 let _ = lagged.send(event.cursor.clone());
1700 }
1701 false
1702 }
1703 }
1704 Err(error) => {
1705 warn!(connection_id = connection_id.get(), corr, %error, "dropping supervisor spawn subscriber after frame build failure");
1706 false
1707 }
1708 }
1709 });
1710 }
1711
1712 fn subscribe(
1713 &self,
1714 connection_id: ConnectionId,
1715 corr: u64,
1716 version: u8,
1717 since: Option<SpawnCursor>,
1718 sink: FrameSink,
1719 ) -> Result<(), SpawnSubscribeRefusal> {
1720 let (frames, mut receiver) = mpsc::channel(SPAWN_SUBSCRIBER_BUFFER);
1721 let (lagged, mut lagged_rx) = oneshot::channel::<SpawnCursor>();
1722 {
1723 let mut state = self.0.lock().unwrap_or_else(|p| p.into_inner());
1724 let replay = if let Some(since) = since {
1725 if since.daemon_incarnation != state.daemon_incarnation {
1726 return Err(SpawnSubscribeRefusal::ForeignIncarnation {
1727 current: state.daemon_incarnation.clone(),
1728 });
1729 }
1730 if let Some(oldest) = state.events.front().map(|event| event.cursor.clone()) {
1731 if since.seq < oldest.seq.saturating_sub(1) {
1732 return Err(SpawnSubscribeRefusal::TooOld { oldest });
1733 }
1734 }
1735 state
1736 .events
1737 .iter()
1738 .filter(|event| event.cursor.seq > since.seq)
1739 .cloned()
1740 .collect::<Vec<_>>()
1741 } else {
1742 Vec::new()
1743 };
1744 for event in replay {
1745 let body = serde_json::to_vec(&event)
1746 .map_err(|error| SpawnSubscribeRefusal::Frame(error.to_string()))?;
1747 let frame = Frame::build_with_version(
1748 version,
1749 FrameType::StreamData,
1750 control_flags(),
1751 0,
1752 0,
1753 corr,
1754 body,
1755 )
1756 .map_err(|error| SpawnSubscribeRefusal::Frame(error.to_string()))?;
1757 frames
1758 .try_send(frame)
1759 .map_err(|error| SpawnSubscribeRefusal::Frame(error.to_string()))?;
1760 }
1761 state.subscribers.insert(
1762 (connection_id, corr),
1763 SpawnSubscriber {
1764 version,
1765 frames,
1766 lagged: Some(lagged),
1767 },
1768 );
1769 }
1770 tokio::spawn(async move {
1781 while let Some(frame) = receiver.recv().await {
1782 if sink.send(frame).await.is_err() {
1783 return;
1784 }
1785 }
1786 let Ok(first_undelivered) = lagged_rx.try_recv() else {
1787 return;
1788 };
1789 match spawn_subscriber_lagged_frame(version, corr, first_undelivered) {
1790 Ok(frame) => {
1791 let _ = sink.send(frame).await;
1792 }
1793 Err(error) => {
1794 error!(%error, corr, "failed to build lagged spawn subscriber terminal frame");
1795 }
1796 }
1797 });
1798 Ok(())
1799 }
1800
1801 fn cancel(&self, connection_id: ConnectionId, corr: u64) -> bool {
1802 let Some(subscriber) = self
1803 .0
1804 .lock()
1805 .unwrap_or_else(|p| p.into_inner())
1806 .subscribers
1807 .remove(&(connection_id, corr))
1808 else {
1809 return false;
1810 };
1811 if let Ok(frame) = Frame::build_with_version(
1812 subscriber.version,
1813 FrameType::StreamEnd,
1814 control_flags(),
1815 0,
1816 0,
1817 corr,
1818 Vec::new(),
1819 ) {
1820 tokio::spawn(async move {
1821 let _ = subscriber.frames.send(frame).await;
1822 });
1823 }
1824 true
1825 }
1826
1827 fn remove_connection(&self, connection_id: ConnectionId) {
1828 self.0
1829 .lock()
1830 .unwrap_or_else(|p| p.into_inner())
1831 .subscribers
1832 .retain(|(subscriber_connection, _), _| *subscriber_connection != connection_id);
1833 }
1834
1835 #[cfg(any(test, feature = "test-support"))]
1836 fn set_capacity(&self, capacity: usize) {
1837 self.0.lock().unwrap_or_else(|p| p.into_inner()).capacity = capacity;
1838 }
1839
1840 #[cfg(any(test, feature = "test-support"))]
1841 fn subscriber_count(&self) -> usize {
1842 self.0
1843 .lock()
1844 .unwrap_or_else(|p| p.into_inner())
1845 .subscribers
1846 .len()
1847 }
1848}
1849
1850fn spawn_subscriber_lagged_frame(
1853 version: u8,
1854 corr: u64,
1855 first_undelivered: SpawnCursor,
1856) -> Result<Frame, String> {
1857 let body = serde_json::to_vec(&subc_protocol::ErrorBody {
1859 code: SPAWN_SUBSCRIBER_LAGGED_CODE.to_string(),
1860 message: "spawn subscriber fell behind and was dropped; resubscribe from the last cursor received"
1861 .to_string(),
1862 detail: Some(serde_json::json!({
1863 "first_undelivered_cursor": first_undelivered
1864 })),
1865 })
1866 .map_err(|error| error.to_string())?;
1867 Frame::build_with_version(
1868 version,
1869 FrameType::Error,
1870 crate::router::daemon_error_flags(),
1871 0,
1872 0,
1873 corr,
1874 body,
1875 )
1876 .map_err(|error| error.to_string())
1877}
1878
1879pub trait ModuleProcessLiveness: Send + Sync {
1880 fn process_live(&self, module_id: &str) -> Option<bool>;
1881
1882 fn process_replacing(&self, _module_id: &str) -> bool {
1888 false
1889 }
1890}
1891
1892#[derive(Debug, Clone, Default)]
1894pub struct SupervisorProcessLiveness {
1895 snapshots: Arc<Mutex<HashMap<String, SharedSnapshot>>>,
1896}
1897
1898impl SupervisorProcessLiveness {
1899 pub fn new() -> Self {
1900 Self::default()
1901 }
1902
1903 fn track(&self, module_id: String, snapshot: SharedSnapshot) {
1904 let mut snapshots = self
1905 .snapshots
1906 .lock()
1907 .unwrap_or_else(|poisoned| poisoned.into_inner());
1908 snapshots.insert(module_id, snapshot);
1909 }
1910
1911 fn untrack_if_current(&self, module_id: &str, snapshot: &SharedSnapshot) {
1912 let mut snapshots = self
1913 .snapshots
1914 .lock()
1915 .unwrap_or_else(|poisoned| poisoned.into_inner());
1916 let is_current = snapshots
1917 .get(module_id)
1918 .map(|tracked| Arc::ptr_eq(tracked, snapshot))
1919 .unwrap_or(false);
1920 if is_current {
1921 snapshots.remove(module_id);
1922 }
1923 }
1924}
1925
1926impl ModuleProcessLiveness for SupervisorProcessLiveness {
1927 fn process_live(&self, module_id: &str) -> Option<bool> {
1928 let snapshot = {
1929 let snapshots = self
1930 .snapshots
1931 .lock()
1932 .unwrap_or_else(|poisoned| poisoned.into_inner());
1933 snapshots.get(module_id).cloned()
1934 }?;
1935 let snapshot = snapshot
1936 .lock()
1937 .unwrap_or_else(|poisoned| poisoned.into_inner());
1938 Some(snapshot.state == ModuleState::Running && snapshot.process_alive)
1939 }
1940
1941 fn process_replacing(&self, module_id: &str) -> bool {
1942 let Some(snapshot) = self
1943 .snapshots
1944 .lock()
1945 .unwrap_or_else(|poisoned| poisoned.into_inner())
1946 .get(module_id)
1947 .cloned()
1948 else {
1949 return false;
1950 };
1951 let snapshot = snapshot
1952 .lock()
1953 .unwrap_or_else(|poisoned| poisoned.into_inner());
1954 snapshot.enabled
1955 && match snapshot.state {
1956 ModuleState::Restarting => true,
1957 ModuleState::Draining => snapshot.draining_to_replace,
1958 ModuleState::Starting
1959 | ModuleState::Running
1960 | ModuleState::Unresponsive
1961 | ModuleState::Stopped
1962 | ModuleState::Failed
1963 | ModuleState::Disabled => false,
1964 }
1965 }
1966}
1967
1968#[cfg(test)]
1969#[derive(Debug, Default)]
1970struct ReloadExitRecordGate {
1971 reached: tokio::sync::Notify,
1972 resume: tokio::sync::Notify,
1973}
1974
1975#[derive(Debug, Clone, Copy)]
1976enum RespawnKind {
1977 Spawn,
1978 Reload,
1979}
1980
1981#[derive(Debug, Clone, Copy)]
1982struct PendingRespawn {
1983 deadline: Instant,
1984 kind: RespawnKind,
1985 daemon_initiated: bool,
1986}
1987
1988type ReloadReply = oneshot::Sender<Result<(), SuperviseError>>;
1989
1990#[derive(Debug, Clone)]
1991struct SupervisorRuntimeConfig {
1992 registry: Arc<Registry>,
1993 recovery_is_daemon: bool,
1995 start_requester: Option<Principal>,
1996 pending_activity_restart: bool,
1997 provenance_probe: ExecutableIdentityProbe,
1998 scheduled_respawn: Arc<Mutex<Option<PendingRespawn>>>,
2000 deferred_reload_reply: Arc<Mutex<Option<ReloadReply>>>,
2002 restart_policy: RestartPolicy,
2003 register_timeout: Duration,
2004 drain_timeout: Duration,
2007 effective_drain_timeout: Arc<Mutex<Duration>>,
2010 default_drain_timeout: Duration,
2013 health: HealthConfig,
2014 connection_file_path: Option<PathBuf>,
2015 capture_logs_dir: Option<PathBuf>,
2016 forwarding: Option<Arc<ForwardingTable>>,
2017 supervisor_handle: Option<SupervisorHandle>,
2020 stderr_ring: Arc<Mutex<StderrRing>>,
2027 terminal_ring: Arc<Mutex<TerminalRing>>,
2028 spawn_events: SpawnEventFeed,
2029 child_roster: ChildRoster,
2030 #[cfg(target_os = "linux")]
2031 cgroup_placement: Option<subc_cgroup::Placement>,
2032 #[cfg(test)]
2033 test_seed_stale_facts_before_enable_spawn: bool,
2034 #[cfg(test)]
2035 test_reload_exit_record_gate: Option<Arc<ReloadExitRecordGate>>,
2036}
2037
2038impl SupervisorRuntimeConfig {
2039 fn failure_context(&self, daemon_initiated: bool) -> FailureContext {
2040 FailureContext {
2041 daemon_initiated,
2042 shutdown: self.child_roster.shutdown_flag(),
2043 }
2044 }
2045}
2046
2047#[derive(Debug, Clone, PartialEq, Eq)]
2048struct SupervisedConfiguration {
2049 spec: ModuleSpec,
2050 health: HealthConfig,
2051}
2052
2053#[derive(Debug, Clone, Default)]
2059pub struct SupervisorHandle {
2060 restart_intent: Arc<Mutex<DaemonRestartIntent>>,
2061 activity: Arc<Mutex<Option<crate::activity::ActivityLedger>>>,
2062 activity_images: Arc<Mutex<HashMap<String, activity::ActivityImage>>>,
2063 modules: Arc<Mutex<HashMap<String, SupervisedModule>>>,
2064 configured_ids: Arc<Mutex<HashSet<String>>>,
2076 spawn_events: SpawnEventFeed,
2077 reserved_nonces: Arc<Mutex<HashMap<String, Option<String>>>>,
2088 removal_tombstones: Arc<Mutex<HashMap<String, u64>>>,
2094 spawn_nonces: Arc<Mutex<HashMap<String, String>>>,
2098 reserved_prefix_owners: Arc<Mutex<HashMap<String, String>>>,
2106 swaps: Arc<Mutex<HashMap<String, OpenSwap>>>,
2112 promotion_observer: PromotionObserverSlot,
2114 operation_lock: Arc<AsyncMutex<()>>,
2118}
2119
2120pub(crate) trait SwapPromotionObserver: Send + Sync {
2129 fn swap_promoted(&self, registration: &crate::registry::ModuleRegistration);
2130}
2131
2132#[derive(Clone, Default)]
2136struct PromotionObserverSlot(Arc<Mutex<Option<std::sync::Weak<dyn SwapPromotionObserver>>>>);
2137
2138impl fmt::Debug for PromotionObserverSlot {
2139 fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
2140 f.write_str("PromotionObserverSlot")
2141 }
2142}
2143
2144#[derive(Debug, Clone)]
2146struct OpenSwap {
2147 candidate_nonce: String,
2150 incumbent_nonce: Option<String>,
2155 candidate_admitted: bool,
2159}
2160
2161#[derive(Debug, Clone, Copy, PartialEq, Eq)]
2164pub(crate) enum SwapHelloAdmission {
2165 NotSwapping,
2168 Candidate,
2170 Refused,
2173}
2174
2175#[derive(Debug, Clone, PartialEq, Eq)]
2176pub(crate) enum ReservedHelloRejection {
2177 Exact {
2178 module_id: String,
2179 },
2180 Prefix {
2181 prefix: String,
2182 owner_module_id: String,
2183 },
2184}
2185
2186#[derive(Debug)]
2187struct DaemonRestartIntent {
2188 expires: Option<Duration>,
2189 window: Duration,
2190}
2191
2192impl Default for DaemonRestartIntent {
2193 fn default() -> Self {
2194 Self {
2195 expires: None,
2196 window: Duration::from_secs(120),
2197 }
2198 }
2199}
2200
2201impl SupervisorHandle {
2202 pub(crate) fn announce_daemon_restart(&self) -> u64 {
2203 let mut intent = self
2204 .restart_intent
2205 .lock()
2206 .unwrap_or_else(|p| p.into_inner());
2207 intent.expires = Some(subc_uptime::suspend_inclusive_now() + intent.window);
2208 unix_ms_now().saturating_add(intent.window.as_millis() as u64)
2209 }
2210
2211 #[cfg(feature = "test-support")]
2212 pub(crate) fn set_shutdown_intent_window(&self, window: Duration) {
2213 self.restart_intent
2214 .lock()
2215 .unwrap_or_else(|p| p.into_inner())
2216 .window = window;
2217 }
2218
2219 #[cfg(unix)]
2220 fn take_daemon_shutdown_reason(&self) -> RouteCloseReason {
2221 let expiry = self
2222 .restart_intent
2223 .lock()
2224 .unwrap_or_else(|p| p.into_inner())
2225 .expires
2226 .take();
2227 if expiry.is_some_and(|expiry| subc_uptime::suspend_inclusive_now() < expiry) {
2228 RouteCloseReason::Restart
2229 } else {
2230 RouteCloseReason::Disable
2231 }
2232 }
2233
2234 pub fn new() -> Self {
2235 Self::default()
2236 }
2237
2238 pub(crate) fn spawn_snapshot(&self) -> SpawnSnapshot {
2239 self.spawn_events.snapshot()
2240 }
2241
2242 pub(crate) fn subscribe_spawns(
2243 &self,
2244 connection_id: ConnectionId,
2245 corr: u64,
2246 version: u8,
2247 since: Option<SpawnCursor>,
2248 sink: FrameSink,
2249 ) -> Result<(), SpawnSubscribeRefusal> {
2250 self.spawn_events
2251 .subscribe(connection_id, corr, version, since, sink)
2252 }
2253
2254 pub(crate) fn cancel_spawn_subscription(&self, connection_id: ConnectionId, corr: u64) -> bool {
2255 self.spawn_events.cancel(connection_id, corr)
2256 }
2257
2258 pub(crate) fn remove_spawn_subscribers(&self, connection_id: ConnectionId) {
2259 self.spawn_events.remove_connection(connection_id);
2260 }
2261
2262 #[cfg(any(test, feature = "test-support"))]
2263 pub fn set_spawn_event_capacity_for_test(&self, capacity: usize) {
2264 assert!(capacity > 0, "spawn event capacity must be non-zero");
2265 self.spawn_events.set_capacity(capacity);
2266 }
2267
2268 #[cfg(any(test, feature = "test-support"))]
2269 pub fn spawn_subscriber_count_for_test(&self) -> usize {
2270 self.spawn_events.subscriber_count()
2271 }
2272
2273 pub fn set_spawn_nonce(&self, module_id: &str, nonce: String) {
2276 self.spawn_nonces
2277 .lock()
2278 .unwrap_or_else(|poisoned| poisoned.into_inner())
2279 .insert(module_id.to_string(), nonce);
2280 }
2281
2282 pub fn set_reserved_nonce(&self, module_id: &str, nonce: String) {
2285 self.reserved_nonces
2286 .lock()
2287 .unwrap_or_else(|poisoned| poisoned.into_inner())
2288 .insert(module_id.to_string(), Some(nonce));
2289 }
2290
2291 pub fn set_reserved_prefixes(&self, owner_module_id: &str, prefixes: &[String]) {
2293 let mut owners = self
2294 .reserved_prefix_owners
2295 .lock()
2296 .unwrap_or_else(|poisoned| poisoned.into_inner());
2297 owners.retain(|_, owner| owner != owner_module_id);
2298 for prefix in prefixes {
2299 owners.insert(prefix.clone(), owner_module_id.to_string());
2300 }
2301 }
2302
2303 #[cfg(test)]
2305 pub(crate) fn spawn_nonce(&self, module_id: &str) -> Option<String> {
2306 self.spawn_nonces
2307 .lock()
2308 .unwrap_or_else(|poisoned| poisoned.into_inner())
2309 .get(module_id)
2310 .cloned()
2311 }
2312
2313 fn apply_identity_configuration(&self, spec: &ModuleSpec) {
2314 self.set_reserved_prefixes(&spec.module_id, &spec.reserved_prefixes);
2315 let spawn_nonce = self
2316 .spawn_nonces
2317 .lock()
2318 .unwrap_or_else(|poisoned| poisoned.into_inner())
2319 .get(&spec.module_id)
2320 .cloned();
2321 let mut reserved_nonces = self
2322 .reserved_nonces
2323 .lock()
2324 .unwrap_or_else(|poisoned| poisoned.into_inner());
2325 if spec.reserved {
2326 reserved_nonces.insert(spec.module_id.clone(), spawn_nonce);
2331 }
2332 drop(reserved_nonces);
2333 self.removal_tombstones
2337 .lock()
2338 .unwrap_or_else(|poisoned| poisoned.into_inner())
2339 .remove(&spec.module_id);
2340 }
2341
2342 pub fn reserved_hello_authorized(&self, module_id: &str, presented: Option<&str>) -> bool {
2347 self.reserved_hello_verification(module_id, presented)
2348 .is_ok()
2349 }
2350
2351 pub(crate) fn reserved_hello_verification(
2354 &self,
2355 module_id: &str,
2356 presented: Option<&str>,
2357 ) -> Result<bool, ReservedHelloRejection> {
2358 let nonces = self
2359 .reserved_nonces
2360 .lock()
2361 .unwrap_or_else(|poisoned| poisoned.into_inner());
2362 if let Some(expected) = nonces.get(module_id) {
2363 let authorized = match expected {
2367 Some(expected) => {
2368 presented.is_some_and(|p| constant_time_eq(expected.as_bytes(), p.as_bytes()))
2369 }
2370 None => false,
2371 };
2372 if authorized {
2373 return Ok(true);
2374 }
2375 return Err(ReservedHelloRejection::Exact {
2376 module_id: module_id.to_string(),
2377 });
2378 }
2379 drop(nonces);
2380
2381 let matched_prefix = self
2382 .reserved_prefix_owners
2383 .lock()
2384 .unwrap_or_else(|poisoned| poisoned.into_inner())
2385 .iter()
2386 .filter(|(prefix, _)| module_id.starts_with(prefix.as_str()))
2387 .max_by_key(|(prefix, _)| prefix.len())
2388 .map(|(prefix, owner)| (prefix.clone(), owner.clone()));
2389 let Some((prefix, owner_module_id)) = matched_prefix else {
2390 return Ok(false);
2391 };
2392
2393 let authorized = presented.is_some_and(|presented| {
2394 self.spawn_nonces
2395 .lock()
2396 .unwrap_or_else(|poisoned| poisoned.into_inner())
2397 .get(&owner_module_id)
2398 .is_some_and(|expected| constant_time_eq(expected.as_bytes(), presented.as_bytes()))
2399 || self.swap_nonce_matches(&owner_module_id, presented)
2402 });
2403 if authorized {
2404 Ok(true)
2405 } else {
2406 Err(ReservedHelloRejection::Prefix {
2407 prefix,
2408 owner_module_id,
2409 })
2410 }
2411 }
2412
2413 pub fn spawned_consumer_authorized(&self, module_id: &str, presented: &str) -> bool {
2418 if presented.is_empty() {
2419 return false;
2420 }
2421 let nonces = self
2422 .spawn_nonces
2423 .lock()
2424 .unwrap_or_else(|poisoned| poisoned.into_inner());
2425 let current = nonces
2426 .get(module_id)
2427 .is_some_and(|expected| constant_time_eq(expected.as_bytes(), presented.as_bytes()));
2428 drop(nonces);
2429 current || self.swap_nonce_matches(module_id, presented)
2434 }
2435
2436 fn swap_nonce_matches(&self, module_id: &str, presented: &str) -> bool {
2438 let swaps = self
2439 .swaps
2440 .lock()
2441 .unwrap_or_else(|poisoned| poisoned.into_inner());
2442 swaps.get(module_id).is_some_and(|swap| {
2443 constant_time_eq(swap.candidate_nonce.as_bytes(), presented.as_bytes())
2444 || swap.incumbent_nonce.as_deref().is_some_and(|incumbent| {
2445 constant_time_eq(incumbent.as_bytes(), presented.as_bytes())
2446 })
2447 })
2448 }
2449
2450 pub(crate) fn open_swap(&self, module_id: &str, candidate_nonce: String) {
2453 let incumbent_nonce = self
2454 .spawn_nonces
2455 .lock()
2456 .unwrap_or_else(|poisoned| poisoned.into_inner())
2457 .get(module_id)
2458 .cloned();
2459 self.swaps
2460 .lock()
2461 .unwrap_or_else(|poisoned| poisoned.into_inner())
2462 .insert(
2463 module_id.to_string(),
2464 OpenSwap {
2465 candidate_nonce,
2466 incumbent_nonce,
2467 candidate_admitted: false,
2468 },
2469 );
2470 }
2471
2472 pub(crate) fn close_swap(&self, module_id: &str) {
2475 self.swaps
2476 .lock()
2477 .unwrap_or_else(|poisoned| poisoned.into_inner())
2478 .remove(module_id);
2479 }
2480
2481 pub(crate) fn set_swap_promotion_observer(
2484 &self,
2485 observer: std::sync::Weak<dyn SwapPromotionObserver>,
2486 ) {
2487 *self
2488 .promotion_observer
2489 .0
2490 .lock()
2491 .unwrap_or_else(|poisoned| poisoned.into_inner()) = Some(observer);
2492 }
2493
2494 fn notify_swap_promoted(&self, registration: &crate::registry::ModuleRegistration) {
2497 let observer = self
2498 .promotion_observer
2499 .0
2500 .lock()
2501 .unwrap_or_else(|poisoned| poisoned.into_inner())
2502 .as_ref()
2503 .and_then(std::sync::Weak::upgrade);
2504 if let Some(observer) = observer {
2505 observer.swap_promoted(registration);
2506 }
2507 }
2508
2509 pub(crate) fn swap_open(&self, module_id: &str) -> bool {
2511 self.swaps
2512 .lock()
2513 .unwrap_or_else(|poisoned| poisoned.into_inner())
2514 .contains_key(module_id)
2515 }
2516
2517 fn promote_swap_nonce(&self, module_id: &str, reserved: bool) {
2522 let candidate_nonce = self
2523 .swaps
2524 .lock()
2525 .unwrap_or_else(|poisoned| poisoned.into_inner())
2526 .get(module_id)
2527 .map(|swap| swap.candidate_nonce.clone());
2528 let Some(nonce) = candidate_nonce else {
2529 return;
2530 };
2531 self.set_spawn_nonce(module_id, nonce.clone());
2532 if reserved {
2533 self.set_reserved_nonce(module_id, nonce);
2534 }
2535 }
2536
2537 pub(crate) fn swap_hello_admission(
2552 &self,
2553 module_id: &str,
2554 presented: Option<&str>,
2555 ) -> SwapHelloAdmission {
2556 let swaps = self
2557 .swaps
2558 .lock()
2559 .unwrap_or_else(|poisoned| poisoned.into_inner());
2560 let Some(swap) = swaps.get(module_id) else {
2561 return SwapHelloAdmission::NotSwapping;
2562 };
2563 let Some(presented) = presented else {
2564 return SwapHelloAdmission::Refused;
2565 };
2566 if constant_time_eq(swap.candidate_nonce.as_bytes(), presented.as_bytes()) {
2567 return if swap.candidate_admitted {
2568 SwapHelloAdmission::Refused
2569 } else {
2570 SwapHelloAdmission::Candidate
2571 };
2572 }
2573 if swap
2574 .incumbent_nonce
2575 .as_deref()
2576 .is_some_and(|incumbent| constant_time_eq(incumbent.as_bytes(), presented.as_bytes()))
2577 {
2578 return SwapHelloAdmission::NotSwapping;
2579 }
2580 SwapHelloAdmission::Refused
2581 }
2582
2583 pub(crate) fn mark_swap_candidate_admitted(&self, module_id: &str) {
2586 if let Some(swap) = self
2587 .swaps
2588 .lock()
2589 .unwrap_or_else(|poisoned| poisoned.into_inner())
2590 .get_mut(module_id)
2591 {
2592 swap.candidate_admitted = true;
2593 }
2594 }
2595
2596 pub fn spawn_launch_nonce_for(&self, module_id: &str) -> Option<String> {
2598 self.spawn_nonces
2599 .lock()
2600 .unwrap_or_else(|poisoned| poisoned.into_inner())
2601 .get(module_id)
2602 .cloned()
2603 }
2604
2605 pub fn reserved_launch_nonce_for(&self, module_id: &str) -> Option<String> {
2607 self.reserved_nonces
2608 .lock()
2609 .unwrap_or_else(|poisoned| poisoned.into_inner())
2610 .get(module_id)
2611 .cloned()
2612 .flatten()
2613 }
2614
2615 pub fn insert(&self, module: SupervisedModule) -> Option<SupervisedModule> {
2616 self.mark_configured(module.module_id());
2620 let mut modules = self
2621 .modules
2622 .lock()
2623 .unwrap_or_else(|poisoned| poisoned.into_inner());
2624 modules.insert(module.module_id().to_string(), module)
2625 }
2626
2627 fn mark_configured(&self, module_id: &str) {
2631 self.configured_ids
2632 .lock()
2633 .unwrap_or_else(|poisoned| poisoned.into_inner())
2634 .insert(module_id.to_string());
2635 }
2636
2637 fn unmark_configured_unless_rostered(&self, module_id: &str) {
2641 let modules = self
2642 .modules
2643 .lock()
2644 .unwrap_or_else(|poisoned| poisoned.into_inner());
2645 if !modules.contains_key(module_id) {
2646 self.configured_ids
2647 .lock()
2648 .unwrap_or_else(|poisoned| poisoned.into_inner())
2649 .remove(module_id);
2650 }
2651 }
2652
2653 pub(crate) fn is_configured(&self, module_id: &str) -> bool {
2661 self.configured_ids
2662 .lock()
2663 .unwrap_or_else(|poisoned| poisoned.into_inner())
2664 .contains(module_id)
2665 }
2666
2667 pub fn get(&self, module_id: &str) -> Option<SupervisedModule> {
2668 let modules = self
2669 .modules
2670 .lock()
2671 .unwrap_or_else(|poisoned| poisoned.into_inner());
2672 modules.get(module_id).cloned()
2673 }
2674
2675 pub(crate) fn record_late_health_answer(
2676 &self,
2677 module_id: &str,
2678 latency_ms: u64,
2679 ) -> Result<bool, SuperviseError> {
2680 let Some(module) = self.get(module_id) else {
2681 return Ok(false);
2682 };
2683 update_snapshot(&module.inner.snapshot, Some(module_id), |state| {
2684 state.health.late_answer_count = state.health.late_answer_count.saturating_add(1);
2685 state.health.last_late_answer_latency_ms = Some(latency_ms);
2686 state.health.consecutive_failures = 0;
2694 })?;
2695 Ok(true)
2696 }
2697
2698 pub fn record_deliberate_severance(&self, module_id: &str) -> Result<bool, SuperviseError> {
2704 let Some(module) = self.get(module_id) else {
2705 return Ok(false);
2706 };
2707 let snapshot = lock_snapshot(&module.inner.snapshot)?;
2708 let Some((pid, start_time)) = snapshot.pid.zip(snapshot.process_start_time) else {
2709 return Ok(false);
2710 };
2711 drop(snapshot);
2712 module.record_deliberate_severance(ProcessIdentity { pid, start_time })
2713 }
2714
2715 pub fn list(&self) -> Vec<SupervisedModule> {
2716 let modules = self
2717 .modules
2718 .lock()
2719 .unwrap_or_else(|poisoned| poisoned.into_inner());
2720 let mut modules = modules.values().cloned().collect::<Vec<_>>();
2721 modules.sort_by(|left, right| left.module_id().cmp(right.module_id()));
2722 modules
2723 }
2724
2725 pub(crate) fn retire(&self, module_id: &str) -> Option<SupervisedModule> {
2726 self.activity_images
2727 .lock()
2728 .unwrap_or_else(|p| p.into_inner())
2729 .remove(module_id);
2730 self.spawn_nonces
2731 .lock()
2732 .unwrap_or_else(|poisoned| poisoned.into_inner())
2733 .remove(module_id);
2734 self.close_swap(module_id);
2735 let mut reserved_nonces = self
2736 .reserved_nonces
2737 .lock()
2738 .unwrap_or_else(|poisoned| poisoned.into_inner());
2739 if reserved_nonces.contains_key(module_id) {
2740 reserved_nonces.insert(module_id.to_string(), None);
2743 }
2744 drop(reserved_nonces);
2745 self.reserved_prefix_owners
2746 .lock()
2747 .unwrap_or_else(|poisoned| poisoned.into_inner())
2748 .retain(|_, owner| owner != module_id);
2749 let removed = self
2750 .modules
2751 .lock()
2752 .unwrap_or_else(|poisoned| poisoned.into_inner())
2753 .remove(module_id);
2754 self.configured_ids
2755 .lock()
2756 .unwrap_or_else(|poisoned| poisoned.into_inner())
2757 .remove(module_id);
2758 removed
2759 }
2760
2761 pub(crate) fn record_rescan_removal(&self, module_id: &str) {
2764 self.removal_tombstones
2765 .lock()
2766 .unwrap_or_else(|poisoned| poisoned.into_inner())
2767 .insert(module_id.to_string(), unix_ms_now());
2768 }
2769
2770 pub(crate) fn removal_tombstone_age_ms(&self, module_id: &str) -> Option<u64> {
2772 self.removal_tombstones
2773 .lock()
2774 .unwrap_or_else(|poisoned| poisoned.into_inner())
2775 .get(module_id)
2776 .copied()
2777 .map(|removed_at_ms| unix_ms_now().saturating_sub(removed_at_ms))
2778 }
2779
2780 pub(crate) fn release_retained_reserved_gate(&self, module_id: &str) -> bool {
2785 if self.get(module_id).is_some() {
2786 return false;
2787 }
2788 let mut reserved_nonces = self
2789 .reserved_nonces
2790 .lock()
2791 .unwrap_or_else(|poisoned| poisoned.into_inner());
2792 if !matches!(reserved_nonces.get(module_id), Some(None)) {
2793 return false;
2794 }
2795 reserved_nonces.remove(module_id);
2796 true
2797 }
2798
2799 pub(crate) fn operation_lock(&self) -> Arc<AsyncMutex<()>> {
2800 Arc::clone(&self.operation_lock)
2801 }
2802}
2803
2804#[derive(Debug, Clone)]
2806pub struct Supervisor {
2807 registry: Arc<Registry>,
2808 restart_policy: RestartPolicy,
2809 drain_timeout: Duration,
2810 connection_file_path: Option<PathBuf>,
2811 capture_logs_dir: Option<PathBuf>,
2812 forwarding: Option<Arc<ForwardingTable>>,
2813 process_liveness: Arc<SupervisorProcessLiveness>,
2814 supervisor_handle: Option<SupervisorHandle>,
2815 health: HealthConfig,
2816 daemon_start_clock: crate::clock::StartClock,
2817 terminal_journal: Option<Arc<crate::terminal_journal::TerminalJournal>>,
2818 spawn_events: SpawnEventFeed,
2819 provenance_probe: ExecutableIdentityProbe,
2820 child_roster: ChildRoster,
2823 #[cfg(target_os = "linux")]
2824 cgroup_placement: Option<subc_cgroup::Placement>,
2825 #[cfg(test)]
2826 test_after_first_spawn: AfterFirstSpawnHook,
2827}
2828
2829pub(crate) struct RegisteredConfiguredModule {
2833 spec: ModuleSpec,
2834 runtime: SupervisorRuntimeConfig,
2835 module: SupervisedModule,
2836 commands: mpsc::Receiver<SupervisorCommand>,
2837}
2838
2839#[cfg(test)]
2845#[derive(Clone, Default)]
2846struct AfterFirstSpawnHook(Option<AfterFirstSpawnFn>);
2847
2848#[cfg(test)]
2849type AfterFirstSpawnFn = Arc<dyn Fn(&str) + Send + Sync>;
2850
2851#[cfg(test)]
2852impl fmt::Debug for AfterFirstSpawnHook {
2853 fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
2854 f.write_str("AfterFirstSpawnHook")
2855 }
2856}
2857
2858#[cfg(test)]
2859impl AfterFirstSpawnHook {
2860 fn run(&self, module_id: &str) {
2861 if let Some(hook) = &self.0 {
2862 hook(module_id);
2863 }
2864 }
2865}
2866
2867impl Supervisor {
2868 #[cfg(test)]
2869 pub(crate) fn new_for_test(registry: Arc<Registry>, policy: RestartPolicy) -> Self {
2870 let supervisor = Self::new(registry, policy);
2871 #[cfg(target_os = "macos")]
2872 let supervisor = supervisor.with_privacy_trampoline(test_privacy_trampoline());
2873 supervisor
2874 }
2875 pub fn with_privacy_trampoline(self, path: impl Into<PathBuf>) -> Self {
2880 let path = path.into();
2881 #[cfg(target_os = "macos")]
2882 {
2883 let result = probe_privacy_trampoline(&path).map(|()| path);
2884 if let Err(cause) = &result {
2885 error!(%cause, "macOS privacy identity unavailable; supervised launches will refuse");
2886 }
2887 self.child_roster.set_privacy_trampoline(result);
2888 }
2889 #[cfg(not(target_os = "macos"))]
2890 let _ = path;
2891 self
2892 }
2893 pub(crate) fn daemon_shutdown_flag(&self) -> crate::child_roster::DaemonShutdownFlag {
2896 self.child_roster.shutdown_flag()
2897 }
2898
2899 #[cfg(unix)]
2910 pub(crate) fn begin_daemon_shutdown(&self) {
2911 self.child_roster.close();
2912 if let Some(journal) = &self.terminal_journal {
2913 journal.stamp_shutdown();
2914 }
2915 }
2916
2917 #[cfg(unix)]
2922 pub(crate) async fn drain_for_daemon_shutdown(&self) -> Result<(), SuperviseError> {
2923 const NOTICE_BUDGET: Duration = Duration::from_millis(500);
2924 const DRAIN_BUDGET: Duration = Duration::from_secs(2);
2925 let Some(forwarding) = &self.forwarding else {
2926 return Ok(());
2927 };
2928 let notice_started = Instant::now();
2929 let modules = forwarding
2930 .begin_daemon_drain()
2931 .map_err(SuperviseError::Forwarding)?;
2932 let deadline_ms =
2933 unix_ms_now().saturating_add((NOTICE_BUDGET + DRAIN_BUDGET).as_millis() as u64);
2934 let shutdown_reason = self
2935 .supervisor_handle
2936 .as_ref()
2937 .map(SupervisorHandle::take_daemon_shutdown_reason)
2938 .unwrap_or(RouteCloseReason::Disable);
2939 let mut modules_told = 0;
2940 let mut notices = tokio::task::JoinSet::new();
2941 let mut drains = Vec::new();
2942 for connection in forwarding
2943 .module_connections()
2944 .map_err(SuperviseError::Forwarding)?
2945 {
2946 let target = ModuleDrainTarget {
2947 endpoint: connection.endpoint,
2948 sink: connection.sink,
2949 negotiated_ver: connection.negotiated_ver,
2950 abandoned_bindings: Vec::new(),
2951 excluded_subscriptions: 0,
2952 };
2953 notices.spawn(async move {
2954 (
2955 true,
2956 send_module_draining(
2957 &connection.module_id,
2958 shutdown_reason,
2959 deadline_ms,
2960 &target,
2961 true,
2962 )
2963 .await,
2964 )
2965 });
2966 }
2967 for (module_id, endpoint) in modules {
2968 let Some(target) = forwarding
2969 .begin_endpoint_drain(endpoint, RouteCloseReason::Restart)
2970 .map_err(SuperviseError::Forwarding)?
2971 else {
2972 continue;
2973 };
2974 let routes = forwarding
2975 .endpoint_routes(target.endpoint)
2976 .map_err(SuperviseError::Forwarding)?;
2977 let mut recipients = Vec::new();
2980 let mut clients: Vec<(crate::forwarding::GoodbyeTarget, Vec<u16>)> = Vec::new();
2981 for route in routes {
2982 let client = route.goodbye_target;
2983 if let Some((_, channels)) = clients
2984 .iter_mut()
2985 .find(|(existing, _)| existing.connection_id == client.connection_id)
2986 {
2987 channels.push(client.channel);
2988 } else {
2989 let channel = client.channel;
2990 clients.push((client, vec![channel]));
2991 }
2992 }
2993 for (client, mut channels) in clients {
2994 channels.sort_unstable();
2995 channels.dedup();
2996 let closing = serde_json::to_vec(&ClientControlPush::RouteClosing {
2997 module_id: module_id.clone(),
2998 channels,
2999 reason: RouteCloseReason::Restart,
3000 })
3001 .expect("route closing serializes");
3002 recipients.push((client.sink, client.negotiated_ver, closing));
3003 }
3004 for (sink, version, body) in recipients {
3005 notices.spawn(async move {
3006 let frame = Frame::build_with_version(
3007 version,
3008 FrameType::Push,
3009 control_flags(),
3010 0,
3011 0,
3012 0,
3013 body,
3014 )
3015 .expect("bounded lifecycle notice frame builds");
3016 (false, sink.send_flushed(frame).await.is_ok())
3017 });
3018 }
3019 let gauges = declared_busy_gauges(&self.registry, &module_id)?;
3020 drains.push((module_id, target.endpoint, gauges));
3021 }
3022 let notice_deadline = Instant::now() + NOTICE_BUDGET;
3025 while let Ok(Some(result)) = timeout_at(notice_deadline, notices.join_next()).await {
3026 match result {
3027 Ok((true, true)) => modules_told += 1,
3028 Ok((false, true)) => {}
3029 result => warn!(?result, "daemon shutdown notice delivery failed"),
3030 }
3031 }
3032 notices.abort_all();
3033 info!(
3034 ?shutdown_reason,
3035 modules_told, "daemon shutdown module notices delivered"
3036 );
3037 let notice_elapsed = notice_started.elapsed();
3038 let drain_started = Instant::now();
3039 let deadline = drain_started + DRAIN_BUDGET;
3040 let mut waits = tokio::task::JoinSet::new();
3041 for (module_id, endpoint, gauges) in drains {
3042 let forwarding = Arc::clone(forwarding);
3043 let mut runtime = self.runtime_config();
3044 runtime.health.cadence = Duration::from_millis(100);
3045 waits.spawn(async move {
3046 let wait_result = wait_for_forwarding_quiescence(
3047 &forwarding,
3048 &module_id,
3049 &runtime,
3050 endpoint,
3051 deadline,
3052 &gauges,
3053 DrainScope::Endpoint(endpoint),
3054 &mut DrainGaugeTrace::new(&gauges),
3055 )
3056 .await;
3057 forwarding
3061 .record_daemon_drain_result(
3062 endpoint,
3063 drained_after_quiescence_wait(&wait_result),
3064 )
3065 .map_err(SuperviseError::Forwarding)?;
3066 wait_result
3067 });
3068 }
3069 while let Ok(Some(result)) = timeout_at(deadline, waits.join_next()).await {
3070 if !matches!(result, Ok(Ok(true))) {
3071 warn!(?result, "daemon shutdown drain did not reach quiescence");
3072 }
3073 }
3074 info!(
3077 notice_elapsed_ms = notice_elapsed.as_millis() as u64,
3078 drain_elapsed_ms = drain_started.elapsed().as_millis() as u64,
3079 "daemon shutdown forwarding drain complete"
3080 );
3081 Ok(())
3082 }
3083
3084 #[cfg(unix)]
3096 pub(crate) async fn end_children_for_daemon_shutdown(
3097 &self,
3098 already_escalated: bool,
3099 escalate: impl std::future::Future<Output = ()>,
3100 ) {
3101 tokio::pin!(escalate);
3102 let mut escalated = already_escalated;
3103 if let Some(forwarding) = &self.forwarding {
3104 let reason = CloseReason::new(
3105 "daemon_shutdown",
3106 "the daemon is exiting after its shutdown notice and drain",
3107 );
3108 if escalated {
3109 send_module_goodbyes_for_daemon_shutdown(forwarding, &reason, false).await;
3112 } else {
3113 tokio::select! {
3114 biased;
3115 _ = escalate.as_mut() => {
3116 info!("second SIGTERM: abandoning module GOODBYE delivery");
3117 escalated = true;
3118 }
3119 _ = send_module_goodbyes_for_daemon_shutdown(forwarding, &reason, true) => {}
3120 }
3121 }
3122 let closed = forwarding.close_all_connections(&reason);
3123 debug!(closed, "closed established connections for daemon shutdown");
3124 }
3125 let escalated_here = escalated && !already_escalated;
3129 let remaining_escalate = async move {
3130 if escalated_here {
3131 std::future::pending::<()>().await;
3132 } else {
3133 escalate.await;
3134 }
3135 };
3136 crate::child_roster::end_children_for_daemon_shutdown(
3137 &self.child_roster,
3138 escalated,
3139 remaining_escalate,
3140 )
3141 .await;
3142 }
3143
3144 pub fn new(registry: Arc<Registry>, restart_policy: RestartPolicy) -> Self {
3145 Self {
3146 registry,
3147 restart_policy,
3148 drain_timeout: DEFAULT_DRAIN_TIMEOUT,
3149 connection_file_path: None,
3150 capture_logs_dir: None,
3151 forwarding: None,
3152 process_liveness: Arc::new(SupervisorProcessLiveness::default()),
3153 supervisor_handle: None,
3154 health: HealthConfig::default(),
3155 daemon_start_clock: crate::clock::StartClock::capture(),
3156 terminal_journal: None,
3157 spawn_events: SpawnEventFeed::default(),
3158 provenance_probe: ExecutableIdentityProbe::default(),
3159 child_roster: ChildRoster::default(),
3160 #[cfg(target_os = "linux")]
3161 cgroup_placement: None,
3162 #[cfg(test)]
3163 test_after_first_spawn: AfterFirstSpawnHook::default(),
3164 }
3165 }
3166
3167 pub fn with_drain_timeout(mut self, drain_timeout: Duration) -> Self {
3168 self.drain_timeout = drain_timeout;
3169 self
3170 }
3171
3172 pub(crate) fn default_drain_timeout(&self) -> Duration {
3174 self.drain_timeout
3175 }
3176
3177 pub fn with_process_liveness(
3178 mut self,
3179 process_liveness: Arc<SupervisorProcessLiveness>,
3180 ) -> Self {
3181 self.process_liveness = process_liveness;
3182 self
3183 }
3184
3185 pub fn with_connection_file_path(mut self, connection_file_path: impl Into<PathBuf>) -> Self {
3186 self.connection_file_path = Some(connection_file_path.into());
3187 self
3188 }
3189
3190 pub fn with_capture_logs_dir(mut self, logs_dir: impl Into<PathBuf>) -> Self {
3192 self.capture_logs_dir = Some(logs_dir.into());
3193 self
3194 }
3195
3196 pub fn with_daemon_incarnation(self, daemon_incarnation: String) -> Self {
3199 self.spawn_events.configure_incarnation(daemon_incarnation);
3203 self
3204 }
3205
3206 pub fn with_terminal_journal(self, path: PathBuf, daemon_incarnation: String) -> Self {
3209 let mut this = self.with_daemon_incarnation(daemon_incarnation.clone());
3210 this.terminal_journal = Some(Arc::new(crate::terminal_journal::TerminalJournal::open(
3211 path,
3212 daemon_incarnation,
3213 )));
3214 this
3215 }
3216
3217 pub fn with_forwarding(mut self, forwarding: Arc<ForwardingTable>) -> Self {
3218 self.child_roster.counters = forwarding.counters();
3219 self.forwarding = Some(forwarding);
3220 self
3221 }
3222
3223 pub fn with_handle(mut self, supervisor_handle: SupervisorHandle) -> Self {
3224 self.spawn_events = supervisor_handle.spawn_events.clone();
3225 self.supervisor_handle = Some(supervisor_handle);
3226 self
3227 }
3228
3229 pub fn with_health_config(mut self, health: HealthConfig) -> Self {
3230 self.health = health;
3231 self
3232 }
3233
3234 pub fn with_live_children_record(self, path: impl Into<PathBuf>) -> Self {
3238 self.child_roster.record_to(path.into());
3239 self
3240 }
3241
3242 #[cfg(target_os = "linux")]
3243 pub fn with_cgroup_placement(
3244 mut self,
3245 cgroup_placement: Option<subc_cgroup::Placement>,
3246 ) -> Self {
3247 self.child_roster
3248 .set_cgroup_placement(cgroup_placement.clone());
3249 self.cgroup_placement = cgroup_placement;
3250 self
3251 }
3252
3253 pub fn spawn(&self, spec: ModuleSpec) -> Result<SupervisedModule, SuperviseError> {
3259 validate_spec(&spec)?;
3260 self.establish_identity(&spec);
3261
3262 let runtime = self.runtime_config();
3263 let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::starting()));
3264 let activity_spawn = runtime.begin_activity_spawn(&spec.module_id);
3265 let registration_watch = watch_registration(&self.registry, &spec);
3266 let spawned = spawn_child(
3267 &spec,
3268 runtime.connection_file_path.as_deref(),
3269 self.supervisor_handle.as_ref(),
3270 &runtime.stderr_ring,
3271 runtime.capture_logs_dir.as_deref(),
3272 &runtime.child_roster,
3273 #[cfg(target_os = "linux")]
3274 runtime.cgroup_placement.as_ref(),
3275 );
3276 #[cfg(test)]
3277 self.test_after_first_spawn.run(&spec.module_id);
3278 let mut child = match spawned {
3279 Ok(child) => child,
3280 Err(err) => {
3281 self.abandon_unrostered(&spec.module_id);
3284 return Err(err);
3285 }
3286 };
3287 arm_registration_deadline(&mut child, runtime.register_timeout, registration_watch);
3288 self.mark_first_process_running(&spec, &runtime, &snapshot, &mut child)?;
3289 if let Some(spawn) = activity_spawn {
3290 spawn.complete(&snapshot);
3291 }
3292
3293 Ok(self.supervised_module(spec, runtime, snapshot, Some(child)))
3294 }
3295
3296 fn establish_identity(&self, spec: &ModuleSpec) {
3309 if let Some(supervisor_handle) = &self.supervisor_handle {
3310 supervisor_handle.apply_identity_configuration(spec);
3311 supervisor_handle.mark_configured(&spec.module_id);
3312 }
3313 }
3314
3315 fn abandon_unrostered(&self, module_id: &str) {
3318 if let Some(supervisor_handle) = &self.supervisor_handle {
3319 supervisor_handle.unmark_configured_unless_rostered(module_id);
3320 }
3321 }
3322
3323 fn mark_first_process_running(
3326 &self,
3327 spec: &ModuleSpec,
3328 runtime: &SupervisorRuntimeConfig,
3329 snapshot: &SharedSnapshot,
3330 child: &mut SupervisedChild,
3331 ) -> Result<(), SuperviseError> {
3332 if let Err(err) = set_running(snapshot, child, &spec.module_id, &runtime.spawn_events) {
3333 containment::terminate_blocking(&runtime.child_roster.containment);
3334 self.abandon_unrostered(&spec.module_id);
3335 return Err(err);
3336 }
3337 self.process_liveness
3338 .track(spec.module_id.clone(), Arc::clone(snapshot));
3339 Ok(())
3340 }
3341
3342 pub fn supervise_configured(
3348 &self,
3349 spec: ModuleSpec,
3350 enabled: bool,
3351 ) -> Result<SupervisedModule, SuperviseError> {
3352 self.supervise_configured_requested(
3353 spec,
3354 enabled,
3355 self.health.clone(),
3356 None,
3357 self.restart_policy,
3358 DEFAULT_REGISTER_TIMEOUT,
3359 None,
3360 )
3361 }
3362
3363 pub fn supervise_configured_with_health(
3369 &self,
3370 spec: ModuleSpec,
3371 enabled: bool,
3372 health: HealthConfig,
3373 drain_timeout_ms: Option<u64>,
3374 restart_policy: RestartPolicy,
3375 ) -> Result<SupervisedModule, SuperviseError> {
3376 self.supervise_configured_requested(
3377 spec,
3378 enabled,
3379 health,
3380 drain_timeout_ms,
3381 restart_policy,
3382 DEFAULT_REGISTER_TIMEOUT,
3383 None,
3384 )
3385 }
3386
3387 #[allow(clippy::too_many_arguments)]
3388 pub(crate) fn supervise_configured_requested(
3389 &self,
3390 spec: ModuleSpec,
3391 enabled: bool,
3392 health: HealthConfig,
3393 drain_timeout_ms: Option<u64>,
3394 restart_policy: RestartPolicy,
3395 register_timeout: Duration,
3396 requester: Option<Principal>,
3397 ) -> Result<SupervisedModule, SuperviseError> {
3398 let registered = self.register_configured_with_health(
3399 spec,
3400 enabled,
3401 health,
3402 drain_timeout_ms,
3403 restart_policy,
3404 register_timeout,
3405 )?;
3406 let daemon_initiated = requester.is_none();
3407 self.start_registered_configured(registered, requester, daemon_initiated)
3408 }
3409
3410 pub(crate) fn register_configured_with_health(
3415 &self,
3416 spec: ModuleSpec,
3417 enabled: bool,
3418 health: HealthConfig,
3419 drain_timeout_ms: Option<u64>,
3420 restart_policy: RestartPolicy,
3421 register_timeout: Duration,
3422 ) -> Result<RegisteredConfiguredModule, SuperviseError> {
3423 validate_spec(&spec)?;
3424 self.establish_identity(&spec);
3425
3426 let mut runtime = self.runtime_config();
3427 runtime.health = health;
3428 runtime.restart_policy = restart_policy;
3429 runtime.register_timeout = register_timeout;
3430 if let Some(ms) = drain_timeout_ms {
3431 runtime.drain_timeout = Duration::from_millis(ms);
3432 *runtime
3433 .effective_drain_timeout
3434 .lock()
3435 .unwrap_or_else(|poisoned| poisoned.into_inner()) = runtime.drain_timeout;
3436 }
3437 let snapshot = if enabled {
3438 SupervisorSnapshot::starting()
3439 } else {
3440 SupervisorSnapshot::disabled()
3441 };
3442 let snapshot = {
3443 let mut snapshot = snapshot;
3444 if enabled {
3445 if let Some((pid, reason)) = runtime.child_roster.previous_instance(&spec.module_id)
3446 {
3447 snapshot.waiting_for_previous_pid = Some(pid);
3448 snapshot.waiting_for_previous_detail = Some(reason.waiting_message(pid));
3449 }
3450 }
3451 snapshot
3452 };
3453 let (module, commands) =
3454 self.register_module(&spec, &runtime, Arc::new(Mutex::new(snapshot)));
3455 Ok(RegisteredConfiguredModule {
3456 spec,
3457 runtime,
3458 module,
3459 commands,
3460 })
3461 }
3462
3463 pub(crate) fn start_registered_configured(
3466 &self,
3467 registered: RegisteredConfiguredModule,
3468 requester: Option<Principal>,
3469 daemon_initiated: bool,
3470 ) -> Result<SupervisedModule, SuperviseError> {
3471 let RegisteredConfiguredModule {
3472 spec,
3473 mut runtime,
3474 module,
3475 commands,
3476 } = registered;
3477 runtime.start_requester = requester;
3478 runtime.recovery_is_daemon = daemon_initiated;
3479 let snapshot = &module.inner.snapshot;
3480 let should_spawn = {
3481 let state = lock_snapshot(snapshot)?;
3482 state.enabled && state.waiting_for_previous_pid.is_none()
3483 };
3484 let child = if should_spawn {
3485 let activity_spawn = runtime.begin_activity_spawn(&spec.module_id);
3486 let registration_watch = watch_registration(&self.registry, &spec);
3487 let spawned = spawn_child(
3488 &spec,
3489 runtime.connection_file_path.as_deref(),
3490 self.supervisor_handle.as_ref(),
3491 &runtime.stderr_ring,
3492 runtime.capture_logs_dir.as_deref(),
3493 &runtime.child_roster,
3494 #[cfg(target_os = "linux")]
3495 runtime.cgroup_placement.as_ref(),
3496 );
3497 #[cfg(test)]
3498 self.test_after_first_spawn.run(&spec.module_id);
3499 match spawned {
3500 Ok(mut child) => {
3501 arm_registration_deadline(
3502 &mut child,
3503 runtime.register_timeout,
3504 registration_watch,
3505 );
3506 self.mark_first_process_running(&spec, &runtime, snapshot, &mut child)?;
3507 if let Some(spawn) = activity_spawn {
3508 spawn.complete(snapshot);
3509 }
3510 Some(child)
3511 }
3512 Err(err) => {
3513 if runtime.health.critical {
3514 error!(
3515 module_id = %spec.module_id,
3516 program = %spec.program.display(),
3517 error = %err,
3518 "critical configured module failed to spawn; marking failed and alerting"
3519 );
3520 } else {
3521 error!(
3522 module_id = %spec.module_id,
3523 program = %spec.program.display(),
3524 error = %err,
3525 "configured module failed to spawn; marking failed and continuing"
3526 );
3527 }
3528 fail_snapshot(
3529 snapshot,
3530 Some(&spec.module_id),
3531 None,
3532 &runtime.failure_context(daemon_initiated),
3533 );
3534 None
3535 }
3536 }
3537 } else {
3538 None
3539 };
3540 self.start_monitor(spec, runtime, &module, child, commands);
3541 Ok(module)
3542 }
3543
3544 fn runtime_config(&self) -> SupervisorRuntimeConfig {
3545 let effective_drain_timeout = Arc::new(Mutex::new(self.drain_timeout));
3546 SupervisorRuntimeConfig {
3547 registry: self.registry.clone(),
3548 recovery_is_daemon: true,
3549 start_requester: None,
3550 pending_activity_restart: false,
3551 provenance_probe: self.provenance_probe.clone(),
3552 scheduled_respawn: Arc::default(),
3553 deferred_reload_reply: Arc::default(),
3554 restart_policy: self.restart_policy,
3555 register_timeout: DEFAULT_REGISTER_TIMEOUT,
3556 drain_timeout: self.drain_timeout,
3557 child_roster: self
3560 .child_roster
3561 .for_module(Arc::clone(&effective_drain_timeout)),
3562 effective_drain_timeout,
3563 default_drain_timeout: self.drain_timeout,
3564 health: self.health.clone(),
3565 connection_file_path: self.connection_file_path.clone(),
3566 capture_logs_dir: self.capture_logs_dir.clone(),
3567 forwarding: self.forwarding.clone(),
3568 supervisor_handle: self.supervisor_handle.clone(),
3569 stderr_ring: Arc::new(Mutex::new(StderrRing::new(StderrTailConfig::default()))),
3570 terminal_ring: Arc::new(Mutex::new(
3571 TerminalRing::new(
3572 TerminalRingConfig::default(),
3573 self.daemon_start_clock.started_at_ms(),
3574 )
3575 .with_start_clock(self.daemon_start_clock)
3576 .with_journal(self.terminal_journal.clone())
3577 .with_daemon_shutdown(self.child_roster.shutdown_flag()),
3578 )),
3579 spawn_events: self.spawn_events.clone(),
3580 #[cfg(target_os = "linux")]
3581 cgroup_placement: self.cgroup_placement.clone(),
3582 #[cfg(test)]
3583 test_seed_stale_facts_before_enable_spawn: false,
3584 #[cfg(test)]
3585 test_reload_exit_record_gate: None,
3586 }
3587 }
3588
3589 fn supervised_module(
3590 &self,
3591 spec: ModuleSpec,
3592 runtime: SupervisorRuntimeConfig,
3593 snapshot: SharedSnapshot,
3594 child: Option<SupervisedChild>,
3595 ) -> SupervisedModule {
3596 let (module, commands) = self.register_module(&spec, &runtime, snapshot);
3597 self.start_monitor(spec, runtime, &module, child, commands);
3598 module
3599 }
3600
3601 fn register_module(
3602 &self,
3603 spec: &ModuleSpec,
3604 runtime: &SupervisorRuntimeConfig,
3605 snapshot: SharedSnapshot,
3606 ) -> (SupervisedModule, mpsc::Receiver<SupervisorCommand>) {
3607 let configuration = Arc::new(Mutex::new(SupervisedConfiguration {
3608 spec: spec.clone(),
3609 health: runtime.health.clone(),
3610 }));
3611 let stderr_ring = Arc::clone(&runtime.stderr_ring);
3612 let terminal_ring = Arc::clone(&runtime.terminal_ring);
3613 let restart_policy = runtime.restart_policy;
3617 let effective_drain_timeout = Arc::clone(&runtime.effective_drain_timeout);
3618 let (tx, rx) = mpsc::channel(4);
3619 let module_id = spec.module_id.clone();
3620 let module = SupervisedModule {
3621 inner: Arc::new(SupervisedModuleInner {
3622 module_id: module_id.clone(),
3623 registry: Arc::clone(&self.registry),
3624 snapshot,
3625 configuration,
3626 stderr_ring,
3627 terminal_ring,
3628 commands: tx,
3629 monitor: Mutex::new(None),
3630 restart_policy,
3631 effective_drain_timeout,
3632 provenance_probe: self.provenance_probe.clone(),
3633 retained: runtime.child_roster.retained_view(),
3634 }),
3635 };
3636 if let Some(supervisor_handle) = &self.supervisor_handle {
3639 supervisor_handle.insert(module.clone());
3640 }
3641 (module, rx)
3642 }
3643
3644 fn start_monitor(
3645 &self,
3646 spec: ModuleSpec,
3647 runtime: SupervisorRuntimeConfig,
3648 module: &SupervisedModule,
3649 child: Option<SupervisedChild>,
3650 commands: mpsc::Receiver<SupervisorCommand>,
3651 ) {
3652 let monitor = tokio::spawn(supervise_loop(
3653 spec,
3654 runtime,
3655 Arc::clone(&self.registry),
3656 Arc::clone(&self.process_liveness),
3657 Arc::clone(&module.inner.snapshot),
3658 child,
3659 commands,
3660 ));
3661 *module
3662 .inner
3663 .monitor
3664 .lock()
3665 .unwrap_or_else(|p| p.into_inner()) = Some(monitor);
3666 }
3667}
3668
3669impl Default for Supervisor {
3670 fn default() -> Self {
3671 Self::new(Arc::new(Registry::default()), RestartPolicy::default())
3672 }
3673}
3674
3675#[derive(Clone)]
3677pub struct SupervisedModule {
3678 inner: Arc<SupervisedModuleInner>,
3679}
3680
3681struct SupervisedModuleInner {
3682 retained: crate::child_roster::RetainedView,
3683 module_id: String,
3684 registry: Arc<Registry>,
3685 snapshot: SharedSnapshot,
3686 configuration: Arc<Mutex<SupervisedConfiguration>>,
3687 stderr_ring: Arc<Mutex<StderrRing>>,
3688 terminal_ring: Arc<Mutex<TerminalRing>>,
3689 commands: mpsc::Sender<SupervisorCommand>,
3690 monitor: Mutex<Option<JoinHandle<()>>>,
3691 restart_policy: RestartPolicy,
3695 effective_drain_timeout: Arc<Mutex<Duration>>,
3696 provenance_probe: ExecutableIdentityProbe,
3697}
3698
3699impl fmt::Debug for SupervisedModule {
3700 fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
3701 f.debug_struct("SupervisedModule")
3702 .field("module_id", &self.inner.module_id)
3703 .field("status", &self.status())
3704 .finish_non_exhaustive()
3705 }
3706}
3707
3708impl SupervisedModule {
3709 pub fn module_id(&self) -> &str {
3710 &self.inner.module_id
3711 }
3712
3713 #[cfg(test)]
3717 pub(crate) fn record_health_probe_failure_for_test(
3718 &self,
3719 detail: &str,
3720 ) -> Result<(), SuperviseError> {
3721 update_snapshot(&self.inner.snapshot, Some(&self.inner.module_id), |state| {
3722 state.health.consecutive_failures = state.health.consecutive_failures.saturating_add(1);
3723 state.health.detail = Some(detail.to_string());
3724 })
3725 }
3726
3727 pub fn state(&self) -> Result<ModuleState, SuperviseError> {
3728 Ok(lock_snapshot(&self.inner.snapshot)?.state)
3729 }
3730
3731 pub(crate) fn registration_end_reason(
3732 &self,
3733 ) -> Result<Option<RegistrationEndReason>, SuperviseError> {
3734 let snapshot = lock_snapshot(&self.inner.snapshot)?;
3735 Ok(match snapshot.state {
3736 ModuleState::Draining if snapshot.draining_to_replace => {
3737 Some(RegistrationEndReason::SupervisorRestart)
3738 }
3739 ModuleState::Draining => Some(RegistrationEndReason::SupervisorStop),
3740 ModuleState::Restarting if snapshot.draining_to_replace => {
3741 Some(RegistrationEndReason::SupervisorRestart)
3742 }
3743 _ => None,
3744 })
3745 }
3746
3747 pub fn stderr_tail(
3754 &self,
3755 max_lines: Option<usize>,
3756 max_bytes: Option<usize>,
3757 ) -> StderrTailSnapshot {
3758 self.inner
3759 .stderr_ring
3760 .lock()
3761 .unwrap_or_else(|poisoned| poisoned.into_inner())
3762 .snapshot(max_lines, max_bytes)
3763 }
3764
3765 pub fn terminal_history(&self) -> TerminalHistorySnapshot {
3770 self.inner
3771 .terminal_ring
3772 .lock()
3773 .unwrap_or_else(|poisoned| poisoned.into_inner())
3774 .snapshot()
3775 }
3776
3777 pub fn durable_terminal_history(&self) -> subc_control::TerminalHistory {
3782 durable_terminal_history_of(&self.inner.terminal_ring, &self.inner.module_id)
3783 }
3784
3785 pub(crate) async fn read_durable_terminal_history(
3790 &self,
3791 ) -> Result<subc_control::TerminalHistory, tokio::task::JoinError> {
3792 let terminal_ring = Arc::clone(&self.inner.terminal_ring);
3793 let module_id = self.inner.module_id.clone();
3794 tokio::task::spawn_blocking(move || durable_terminal_history_of(&terminal_ring, &module_id))
3795 .await
3796 }
3797
3798 pub fn status(&self) -> Result<ModuleStatus, SuperviseError> {
3799 self.status_with_snapshot_lock(&self.inner.snapshot, None)
3800 .map(|(status, _)| status)
3801 }
3802
3803 pub(crate) fn next_retry_at_ms(&self) -> Result<Option<u64>, SuperviseError> {
3804 let state = lock_snapshot(&self.inner.snapshot)?;
3805 Ok(state
3806 .failed_retry
3807 .pending
3808 .filter(|_| state.state == ModuleState::Failed && state.enabled)
3809 .map(|retry| {
3810 unix_ms_now().saturating_add(
3811 u64::try_from(
3812 retry
3813 .deadline
3814 .saturating_duration_since(Instant::now())
3815 .as_millis(),
3816 )
3817 .unwrap_or(u64::MAX),
3818 )
3819 }))
3820 }
3821
3822 pub(crate) fn record_deliberate_severance(
3823 &self,
3824 identity: ProcessIdentity,
3825 ) -> Result<bool, SuperviseError> {
3826 let mut snapshot = lock_snapshot(&self.inner.snapshot)?;
3827 if snapshot.pid != Some(identity.pid)
3828 || snapshot.process_start_time != Some(identity.start_time)
3829 {
3830 return Ok(false);
3831 }
3832 snapshot.deliberate_severance = Some(identity);
3833 Ok(true)
3834 }
3835
3836 pub(crate) fn status_for_control(
3841 &self,
3842 caller: &'static str,
3843 ) -> Result<ModuleStatus, SuperviseError> {
3844 self.status_with_snapshot_lock(&self.inner.snapshot, Some(caller))
3845 .map(|(status, _)| status)
3846 }
3847
3848 fn status_with_snapshot_lock(
3849 &self,
3850 snapshot: &SharedSnapshot,
3851 caller: Option<&'static str>,
3852 ) -> Result<(ModuleStatus, Option<SpawnedFileIdentity>), SuperviseError> {
3853 let mut guard = match caller {
3854 Some(caller) => lock_snapshot_for_control(snapshot, &self.inner.module_id, caller)?,
3855 None => lock_snapshot(snapshot)?,
3856 };
3857 let restart_count =
3860 guard.crash_restarts_in_window(self.inner.restart_policy.window, Instant::now());
3861 let snapshot = guard.clone();
3862 drop(guard);
3863 let drain_timeout = *self.inner.effective_drain_timeout.lock().map_err(|_| {
3864 SuperviseError::StatePoisoned {
3865 module_id: Some(self.inner.module_id.clone()),
3866 }
3867 })?;
3868 let registration_active = self
3869 .inner
3870 .registry
3871 .get_module(&self.inner.module_id)
3872 .map_err(SuperviseError::Registry)?
3873 .is_some();
3874 let protocol = snapshot
3875 .spawned_protocol
3876 .unwrap_or(self.declared_protocol()?);
3877 let running_process =
3878 snapshot.enabled && snapshot.state == ModuleState::Running && snapshot.process_alive;
3879 let live = match protocol {
3885 ModuleProtocol::Subc => running_process && registration_active,
3886 ModuleProtocol::None => running_process,
3887 };
3888 let retained = self.inner.retained.waiting_detail(&self.inner.module_id);
3891 let carry_configured = self
3892 .inner
3893 .configuration
3894 .lock()
3895 .map_err(|_| SuperviseError::StatePoisoned {
3896 module_id: Some(self.inner.module_id.clone()),
3897 })?
3898 .spec
3899 .carry_containment;
3900 let recovery_block = carries_containment(
3901 carry_configured,
3902 self.inner
3903 .retained
3904 .has_carried_container(&self.inner.module_id),
3905 )
3906 .then(|| self.inner.retained.recovery_block())
3907 .flatten();
3908 let launch_nonce_source = observed_launch_nonce_source(&snapshot);
3909 let pid = snapshot.reported_pid();
3910 let mut health = snapshot.health;
3911 if let Some(detail) = recovery_block
3912 .as_ref()
3913 .or_else(|| retained.as_ref().map(|(_, detail)| detail))
3914 {
3915 health.detail = Some(match health.detail {
3916 Some(previous) => format!("{previous}; {detail}"),
3917 None => detail.clone(),
3918 });
3919 }
3920
3921 Ok((
3922 ModuleStatus {
3923 waiting_for_previous_pid: retained
3924 .as_ref()
3925 .map(|(pid, _)| *pid)
3926 .filter(|pid| *pid > 0)
3927 .or(snapshot.waiting_for_previous_pid),
3928 waiting_for_previous_detail: recovery_block
3929 .or_else(|| retained.map(|(_, detail)| detail))
3930 .or(snapshot.waiting_for_previous_detail),
3931 module_id: self.inner.module_id.clone(),
3932 launch_nonce_source,
3933 state: snapshot.state,
3934 enabled: snapshot.enabled,
3935 process_alive: snapshot.process_alive,
3936 registration_active,
3937 protocol,
3938 live,
3939 restart_count,
3940 lifetime_restarts: snapshot.lifetime_restarts,
3941 spawn_generation: snapshot.spawn_generation,
3942 max_restarts: self.inner.restart_policy.max_restarts,
3943 restart_window: self.inner.restart_policy.window,
3944 drain_timeout,
3945 restart_backoff: self.inner.restart_policy.backoff,
3946 restart_max_backoff: self.inner.restart_policy.max_backoff,
3947 pid,
3948 spawned_at_ms: snapshot.spawned_at_ms,
3949 spawned_from: snapshot.spawned_from,
3950 launch_spec_digest: snapshot.launch_spec_digest,
3951 process_start_time: snapshot.process_start_time,
3952 last_exit: snapshot.last_exit,
3953 health,
3954 },
3955 snapshot.spawned_file_identity.clone(),
3956 ))
3957 }
3958
3959 #[cfg(test)]
3960 pub(crate) fn hold_snapshot_for_test(
3961 &self,
3962 acquired: std::sync::mpsc::Sender<()>,
3963 hold: Duration,
3964 ) -> std::thread::JoinHandle<()> {
3965 let snapshot = Arc::clone(&self.inner.snapshot);
3966 std::thread::spawn(move || {
3967 let _guard = snapshot.lock().expect("test snapshot lock is not poisoned");
3968 acquired
3969 .send(())
3970 .expect("test receiver waits for snapshot lock");
3971 std::thread::sleep(hold);
3972 })
3973 }
3974
3975 pub(crate) async fn status_and_running_image_agreement(
3981 &self,
3982 ) -> Result<(ModuleStatus, subc_control::RunningImageAgreement), SuperviseError> {
3983 let (status, identity) = self.status_with_snapshot_lock(&self.inner.snapshot, None)?;
3984 let image = self
3985 .inner
3986 .provenance_probe
3987 .observe(
3988 status.pid,
3989 status.spawned_from.as_deref(),
3990 identity,
3991 status.process_start_time,
3992 )
3993 .await;
3994 Ok((status, image))
3995 }
3996
3997 pub(crate) async fn running_image_agreement(&self) -> subc_control::RunningImageAgreement {
3998 let snapshot = match lock_snapshot(&self.inner.snapshot) {
3999 Ok(snapshot) => snapshot.clone(),
4000 Err(_) => {
4001 return subc_control::RunningImageAgreement::Unavailable {
4002 reason: subc_control::RunningImageUnavailableReason::NotRunning,
4003 };
4004 }
4005 };
4006 self.inner
4007 .provenance_probe
4008 .observe(
4009 snapshot.reported_pid(),
4010 snapshot.spawned_from.as_deref(),
4011 snapshot.spawned_file_identity.clone(),
4012 snapshot.process_start_time,
4013 )
4014 .await
4015 }
4016
4017 pub(crate) fn child_resource_usage(&self) -> subc_control::ChildResourceUsage {
4020 #[cfg(windows)]
4021 {
4022 let image = lock_snapshot(&self.inner.snapshot)
4023 .ok()
4024 .and_then(|snapshot| {
4025 snapshot
4026 .spawned_file_identity
4027 .as_ref()
4028 .and_then(|identity| identity.windows_image.clone())
4029 });
4030 match image {
4031 Some(image) => {
4032 crate::child_resources::read_windows_handle(image.process(), image.start_time())
4033 }
4034 None => subc_control::ChildResourceUsage::Unavailable {
4035 reason: subc_control::ChildResourceUnavailableReason::NotRunning,
4036 },
4037 }
4038 }
4039 #[cfg(not(windows))]
4040 {
4041 let (pid, start_time) = match lock_snapshot(&self.inner.snapshot) {
4042 Ok(snapshot) => (snapshot.reported_pid(), snapshot.process_start_time),
4043 Err(_) => {
4044 return subc_control::ChildResourceUsage::Unavailable {
4045 reason: subc_control::ChildResourceUnavailableReason::Unreadable,
4046 }
4047 }
4048 };
4049 crate::child_resources::read(pid, start_time)
4050 }
4051 }
4052
4053 pub(crate) fn subtree_target(&self) -> crate::subtree_resources::Target {
4054 self.inner
4055 .snapshot
4056 .try_lock()
4057 .map(|snapshot| snapshot.subtree_target.clone())
4058 .unwrap_or(crate::subtree_resources::Target::Unavailable("read_failed"))
4059 }
4060
4061 #[cfg(test)]
4062 pub(crate) fn set_subtree_target_for_test(&self, target: crate::subtree_resources::Target) {
4063 self.inner.snapshot.lock().unwrap().subtree_target = target;
4064 }
4065
4066 pub(crate) fn will_recover_after_connection_loss(&self) -> Result<bool, SuperviseError> {
4067 let mut snapshot = lock_snapshot(&self.inner.snapshot)?;
4068 Ok(match snapshot.state {
4069 ModuleState::Restarting => true,
4070 ModuleState::Failed | ModuleState::Disabled => false,
4071 _ if snapshot.enabled && snapshot.respawn_pending => true,
4072 _ => daemon_will_restart(&mut snapshot, &self.inner.restart_policy, Instant::now()),
4073 })
4074 }
4075
4076 #[cfg(test)]
4077 pub(crate) fn is_warming(&self) -> Result<bool, SuperviseError> {
4078 self.is_warming_with_snapshot_lock(None)
4079 }
4080
4081 pub(crate) fn is_warming_for_control(
4082 &self,
4083 caller: &'static str,
4084 ) -> Result<bool, SuperviseError> {
4085 self.is_warming_with_snapshot_lock(Some(caller))
4086 }
4087
4088 fn is_warming_with_snapshot_lock(
4089 &self,
4090 caller: Option<&'static str>,
4091 ) -> Result<bool, SuperviseError> {
4092 let snapshot = match caller {
4093 Some(caller) => {
4094 lock_snapshot_for_control(&self.inner.snapshot, &self.inner.module_id, caller)?
4095 }
4096 None => lock_snapshot(&self.inner.snapshot)?,
4097 }
4098 .clone();
4099 Ok(matches!(
4100 snapshot.state,
4101 ModuleState::Starting | ModuleState::Running | ModuleState::Restarting
4102 ))
4103 }
4104
4105 pub async fn drain(&self) -> Result<(), SuperviseError> {
4107 self.stop().await
4108 }
4109
4110 pub(crate) async fn retire_requested(
4111 &self,
4112 requester: Option<Principal>,
4113 ) -> Result<(), SuperviseError> {
4114 match self.state()? {
4115 ModuleState::Stopped => return Ok(()),
4116 ModuleState::Failed
4117 if lock_snapshot(&self.inner.snapshot)?
4118 .failed_retry
4119 .pending
4120 .is_none() =>
4121 {
4122 return Ok(())
4123 }
4124 ModuleState::Failed
4125 | ModuleState::Starting
4126 | ModuleState::Running
4127 | ModuleState::Unresponsive
4128 | ModuleState::Restarting
4129 | ModuleState::Draining
4130 | ModuleState::Disabled => {}
4131 }
4132
4133 let (reply_tx, reply_rx) = oneshot::channel();
4134 self.inner
4135 .commands
4136 .send(SupervisorCommand::Retire {
4137 requester,
4138 reply: reply_tx,
4139 })
4140 .await
4141 .map_err(|_| SuperviseError::CommandClosed {
4142 module_id: self.inner.module_id.clone(),
4143 })?;
4144 reply_rx.await.map_err(|_| SuperviseError::CommandClosed {
4145 module_id: self.inner.module_id.clone(),
4146 })?
4147 }
4148
4149 pub async fn stop(&self) -> Result<(), SuperviseError> {
4150 self.stop_requested(None).await
4151 }
4152
4153 pub(crate) async fn stop_requested(
4154 &self,
4155 requester: Option<Principal>,
4156 ) -> Result<(), SuperviseError> {
4157 match self.state()? {
4158 ModuleState::Stopped => return Ok(()),
4159 ModuleState::Failed
4160 if lock_snapshot(&self.inner.snapshot)?
4161 .failed_retry
4162 .pending
4163 .is_none() =>
4164 {
4165 return Ok(())
4166 }
4167 ModuleState::Failed
4168 | ModuleState::Starting
4169 | ModuleState::Running
4170 | ModuleState::Unresponsive
4171 | ModuleState::Restarting
4172 | ModuleState::Draining
4173 | ModuleState::Disabled => {}
4174 }
4175
4176 let (reply_tx, reply_rx) = oneshot::channel();
4177 self.inner
4178 .commands
4179 .send(SupervisorCommand::Drain {
4180 requester,
4181 reply: reply_tx,
4182 })
4183 .await
4184 .map_err(|_| SuperviseError::CommandClosed {
4185 module_id: self.inner.module_id.clone(),
4186 })?;
4187 reply_rx.await.map_err(|_| SuperviseError::CommandClosed {
4188 module_id: self.inner.module_id.clone(),
4189 })?
4190 }
4191
4192 pub async fn restart(&self, drain_timeout_ms: Option<u64>) -> Result<(), SuperviseError> {
4193 self.restart_requested(drain_timeout_ms, None).await
4194 }
4195
4196 pub(crate) async fn restart_requested(
4197 &self,
4198 drain_timeout_ms: Option<u64>,
4199 requester: Option<Principal>,
4200 ) -> Result<(), SuperviseError> {
4201 let queued_at = Instant::now();
4202 let received_at_generation = lock_snapshot(&self.inner.snapshot)?.spawn_generation;
4203 let (reply_tx, reply_rx) = oneshot::channel();
4204 self.inner
4205 .commands
4206 .send(SupervisorCommand::Restart {
4207 requester,
4208 drain_timeout_ms,
4209 received_at_generation,
4210 queued_at,
4211 reply: reply_tx,
4212 })
4213 .await
4214 .map_err(|_| SuperviseError::CommandClosed {
4215 module_id: self.inner.module_id.clone(),
4216 })?;
4217 reply_rx.await.map_err(|_| SuperviseError::CommandClosed {
4218 module_id: self.inner.module_id.clone(),
4219 })?
4220 }
4221
4222 pub async fn swap(&self, ready_timeout: Option<Duration>) -> Result<(), SuperviseError> {
4227 self.swap_requested(ready_timeout, None).await
4228 }
4229
4230 pub(crate) async fn swap_requested(
4231 &self,
4232 ready_timeout: Option<Duration>,
4233 requester: Option<Principal>,
4234 ) -> Result<(), SuperviseError> {
4235 let (reply_tx, reply_rx) = oneshot::channel();
4236 self.inner
4237 .commands
4238 .send(SupervisorCommand::Swap {
4239 requester,
4240 ready_timeout,
4241 reply: reply_tx,
4242 })
4243 .await
4244 .map_err(|_| SuperviseError::CommandClosed {
4245 module_id: self.inner.module_id.clone(),
4246 })?;
4247 reply_rx.await.map_err(|_| SuperviseError::CommandClosed {
4248 module_id: self.inner.module_id.clone(),
4249 })?
4250 }
4251
4252 pub async fn reload(&self) -> Result<(), SuperviseError> {
4253 self.reload_requested(None).await
4254 }
4255
4256 pub(crate) async fn reload_requested(
4257 &self,
4258 requester: Option<Principal>,
4259 ) -> Result<(), SuperviseError> {
4260 let (reply_tx, reply_rx) = oneshot::channel();
4261 self.inner
4262 .commands
4263 .send(SupervisorCommand::Reload {
4264 requester,
4265 reply: reply_tx,
4266 })
4267 .await
4268 .map_err(|_| SuperviseError::CommandClosed {
4269 module_id: self.inner.module_id.clone(),
4270 })?;
4271 reply_rx.await.map_err(|_| SuperviseError::CommandClosed {
4272 module_id: self.inner.module_id.clone(),
4273 })?
4274 }
4275
4276 pub async fn set_enabled(&self, enabled: bool) -> Result<bool, SuperviseError> {
4277 self.set_enabled_requested(enabled, None).await
4278 }
4279
4280 pub(crate) async fn set_enabled_requested(
4281 &self,
4282 enabled: bool,
4283 requester: Option<Principal>,
4284 ) -> Result<bool, SuperviseError> {
4285 let (reply_tx, reply_rx) = oneshot::channel();
4286 self.inner
4287 .commands
4288 .send(SupervisorCommand::SetEnabled {
4289 requester,
4290 enabled,
4291 reply: reply_tx,
4292 })
4293 .await
4294 .map_err(|_| SuperviseError::CommandClosed {
4295 module_id: self.inner.module_id.clone(),
4296 })?;
4297 reply_rx.await.map_err(|_| SuperviseError::CommandClosed {
4298 module_id: self.inner.module_id.clone(),
4299 })?
4300 }
4301
4302 pub(crate) fn declared_protocol(&self) -> Result<ModuleProtocol, SuperviseError> {
4306 let configured = self
4307 .inner
4308 .configuration
4309 .lock()
4310 .map_err(|_| SuperviseError::StatePoisoned {
4311 module_id: Some(self.inner.module_id.clone()),
4312 })?
4313 .spec
4314 .protocol;
4315 let state = lock_snapshot(&self.inner.snapshot)?;
4316 Ok(state.spawned_protocol.unwrap_or(configured))
4317 }
4318
4319 pub(crate) fn configuration(&self) -> Result<(ModuleSpec, HealthConfig), SuperviseError> {
4320 let configuration =
4321 self.inner
4322 .configuration
4323 .lock()
4324 .map_err(|_| SuperviseError::StatePoisoned {
4325 module_id: Some(self.inner.module_id.clone()),
4326 })?;
4327 Ok((configuration.spec.clone(), configuration.health.clone()))
4328 }
4329
4330 #[cfg(any(test, feature = "test-support"))]
4334 pub async fn update_spec_for_test(&self, spec: ModuleSpec) -> Result<(), SuperviseError> {
4335 let (_, health) = self.configuration()?;
4336 let drain_timeout_ms = u64::try_from(
4337 self.inner
4338 .effective_drain_timeout
4339 .lock()
4340 .unwrap_or_else(|poisoned| poisoned.into_inner())
4341 .as_millis(),
4342 )
4343 .ok();
4344 self.update_configuration(spec, health, drain_timeout_ms)
4345 .await
4346 }
4347
4348 pub(crate) async fn update_configuration(
4349 &self,
4350 spec: ModuleSpec,
4351 health: HealthConfig,
4352 drain_timeout_ms: Option<u64>,
4353 ) -> Result<(), SuperviseError> {
4354 if spec.module_id != self.inner.module_id {
4355 return Err(SuperviseError::InvalidSpec {
4356 reason: "a supervised module's module_id cannot be changed".to_string(),
4357 });
4358 }
4359 validate_spec(&spec)?;
4360 let (reply_tx, reply_rx) = oneshot::channel();
4361 self.inner
4362 .commands
4363 .send(SupervisorCommand::UpdateConfiguration {
4364 spec: spec.clone(),
4365 health: health.clone(),
4366 drain_timeout_ms,
4367 reply: reply_tx,
4368 })
4369 .await
4370 .map_err(|_| SuperviseError::CommandClosed {
4371 module_id: self.inner.module_id.clone(),
4372 })?;
4373 reply_rx.await.map_err(|_| SuperviseError::CommandClosed {
4374 module_id: self.inner.module_id.clone(),
4375 })?;
4376 let mut configuration =
4377 self.inner
4378 .configuration
4379 .lock()
4380 .map_err(|_| SuperviseError::StatePoisoned {
4381 module_id: Some(self.inner.module_id.clone()),
4382 })?;
4383 configuration.spec = spec;
4384 configuration.health = health;
4385 Ok(())
4386 }
4387}
4388
4389impl Drop for SupervisedModuleInner {
4390 fn drop(&mut self) {
4391 let Ok(mut monitor) = self.monitor.lock() else {
4392 return;
4393 };
4394 if let Some(monitor) = monitor.as_ref().filter(|monitor| !monitor.is_finished()) {
4395 let _ = update_snapshot(&self.snapshot, Some(&self.module_id), |state| {
4396 state.state = ModuleState::Stopped;
4397 clear_current_process_facts(state);
4398 });
4399 monitor.abort();
4400 }
4401 let _ = monitor.take();
4402 }
4403}
4404
4405#[derive(Debug)]
4406enum SupervisorCommand {
4407 Drain {
4408 requester: Option<Principal>,
4409 reply: oneshot::Sender<Result<(), SuperviseError>>,
4410 },
4411 Retire {
4412 requester: Option<Principal>,
4413 reply: oneshot::Sender<Result<(), SuperviseError>>,
4414 },
4415 Restart {
4416 requester: Option<Principal>,
4417 drain_timeout_ms: Option<u64>,
4422 received_at_generation: u64,
4426 queued_at: Instant,
4429 reply: oneshot::Sender<Result<(), SuperviseError>>,
4430 },
4431 Reload {
4432 requester: Option<Principal>,
4433 reply: oneshot::Sender<Result<(), SuperviseError>>,
4434 },
4435 SetEnabled {
4436 requester: Option<Principal>,
4437 enabled: bool,
4438 reply: oneshot::Sender<Result<bool, SuperviseError>>,
4439 },
4440 UpdateConfiguration {
4441 spec: ModuleSpec,
4442 health: HealthConfig,
4443 drain_timeout_ms: Option<u64>,
4446 reply: oneshot::Sender<()>,
4447 },
4448 Swap {
4449 requester: Option<Principal>,
4450 ready_timeout: Option<Duration>,
4453 reply: oneshot::Sender<Result<(), SuperviseError>>,
4455 },
4456}
4457
4458#[derive(Debug)]
4459pub enum SuperviseError {
4460 InvalidSpec {
4461 reason: String,
4462 },
4463 Spawn {
4464 program: PathBuf,
4465 source: io::Error,
4466 cgroup_path: Option<PathBuf>,
4467 },
4468 Cgroup {
4469 module_id: String,
4470 source: io::Error,
4471 },
4472 LaunchNonce {
4475 reason: String,
4476 },
4477 Wait {
4478 module_id: String,
4479 source: io::Error,
4480 },
4481 Kill {
4482 module_id: String,
4483 source: io::Error,
4484 },
4485 Forwarding(ForwardingError),
4486 Registry(RegistryError),
4487 ReloadUnavailable {
4488 module_id: String,
4489 reason: String,
4490 },
4491 Disabled {
4496 module_id: String,
4497 },
4498 ReloadFailed {
4499 module_id: String,
4500 reason: String,
4501 },
4502 RegistrationStillActive {
4503 module_id: String,
4504 waited: Duration,
4505 },
4506 StatePoisoned {
4507 module_id: Option<String>,
4508 },
4509 CommandClosed {
4510 module_id: String,
4511 },
4512 SwapInProgress {
4516 module_id: String,
4517 },
4518 SwapRefused {
4520 module_id: String,
4521 reason: SwapRefusal,
4522 },
4523 SwapFailed {
4527 module_id: String,
4528 arm: SwapFailureArm,
4529 detail: String,
4530 candidate_exit: Option<ExitReport>,
4533 },
4534}
4535
4536#[derive(Debug, Clone, Copy, PartialEq, Eq)]
4538pub enum SwapRefusal {
4539 OverlapExclusive,
4541 NotRegistered,
4544 ProtocolNone,
4547 NotConfigured,
4550 AlreadySwapping,
4552}
4553
4554impl SwapRefusal {
4555 pub fn as_str(self) -> &'static str {
4556 match self {
4557 Self::OverlapExclusive => "overlap_exclusive",
4558 Self::NotRegistered => "not_registered",
4559 Self::ProtocolNone => "protocol_none",
4560 Self::NotConfigured => "not_configured",
4561 Self::AlreadySwapping => "already_swapping",
4562 }
4563 }
4564}
4565
4566#[derive(Debug, Clone, Copy, PartialEq, Eq)]
4569pub enum SwapFailureArm {
4570 SpawnFailed,
4572 NeverRegistered,
4574 NeverReady,
4576 CandidateExited,
4578 CandidateUnhealthy,
4580 Interrupted,
4584 CutoverLost,
4589}
4590
4591impl SwapFailureArm {
4592 pub fn as_str(self) -> &'static str {
4593 match self {
4594 Self::SpawnFailed => "spawn_failed",
4595 Self::NeverRegistered => "never_registered",
4596 Self::NeverReady => "never_ready",
4597 Self::CandidateExited => "candidate_exited",
4598 Self::CandidateUnhealthy => "candidate_unhealthy",
4599 Self::Interrupted => "interrupted",
4600 Self::CutoverLost => "cutover_lost",
4601 }
4602 }
4603}
4604
4605impl fmt::Display for SuperviseError {
4606 fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
4607 match self {
4608 Self::InvalidSpec { reason } => write!(f, "invalid module spec: {reason}"),
4609 Self::Spawn {
4610 program,
4611 source,
4612 cgroup_path: Some(cgroup_path),
4613 } => write!(
4614 f,
4615 "failed to place module in cgroup '{}' while spawning '{}': {source}",
4616 cgroup_path.display(),
4617 program.display()
4618 ),
4619 Self::Spawn {
4620 program,
4621 source,
4622 cgroup_path: None,
4623 } => write!(
4624 f,
4625 "failed to spawn module '{}': {source}",
4626 program.display()
4627 ),
4628 Self::Cgroup { module_id, source } => {
4629 write!(
4630 f,
4631 "failed to prepare cgroup for module '{module_id}': {source}"
4632 )
4633 }
4634 Self::LaunchNonce { reason } => {
4635 write!(
4636 f,
4637 "failed to generate reserved-module launch nonce: {reason}"
4638 )
4639 }
4640 Self::Wait { module_id, source } => {
4641 write!(f, "failed to wait for module '{module_id}': {source}")
4642 }
4643 Self::Kill { module_id, source } => {
4644 write!(f, "failed to kill module '{module_id}': {source}")
4645 }
4646 Self::Forwarding(err) => write!(f, "forwarding error: {err}"),
4647 Self::Registry(err) => write!(f, "registry error: {err}"),
4648 Self::ReloadUnavailable { module_id, reason } => {
4649 write!(f, "reload unavailable for module '{module_id}': {reason}")
4650 }
4651 Self::Disabled { module_id } => {
4652 write!(
4653 f,
4654 "module '{module_id}' is disabled; enable it before restart or reload"
4655 )
4656 }
4657 Self::ReloadFailed { module_id, reason } => {
4658 write!(f, "reload failed for module '{module_id}': {reason}")
4659 }
4660 Self::RegistrationStillActive { module_id, waited } => write!(
4661 f,
4662 "module '{module_id}' registration remained active after waiting {waited:?}"
4663 ),
4664 Self::StatePoisoned { module_id } => match module_id {
4665 Some(module_id) => {
4666 write!(f, "supervisor state for module '{module_id}' was poisoned")
4667 }
4668 None => write!(f, "supervisor state was poisoned"),
4669 },
4670 Self::CommandClosed { module_id } => {
4671 write!(
4672 f,
4673 "supervisor command channel for module '{module_id}' is closed"
4674 )
4675 }
4676 Self::SwapInProgress { module_id } => write!(
4677 f,
4678 "module '{module_id}' is being swapped; retry once the swap has cut over or failed, or stop the module to abort the swap"
4679 ),
4680 Self::SwapRefused { module_id, reason } => match reason {
4681 SwapRefusal::OverlapExclusive => write!(
4682 f,
4683 "module '{module_id}' is declared overlap: \"exclusive\" (the default): two processes of it must not run at once, so it cannot be swapped; use a plain restart, or declare overlap: \"safe\" in its config if it really tolerates a second process"
4684 ),
4685 SwapRefusal::NotRegistered => write!(
4686 f,
4687 "module '{module_id}' is not registered, so there is no serving process to keep while a replacement warms; use a plain restart"
4688 ),
4689 SwapRefusal::ProtocolNone => write!(
4690 f,
4691 "module '{module_id}' is protocol: \"none\" and never registers, so a swap could never see its replacement become ready; use a plain restart"
4692 ),
4693 SwapRefusal::NotConfigured => write!(
4694 f,
4695 "module '{module_id}' cannot be swapped: the supervisor was built without the forwarding table or shared handle a swap needs"
4696 ),
4697 SwapRefusal::AlreadySwapping => {
4698 write!(f, "module '{module_id}' is already being swapped")
4699 }
4700 },
4701 Self::SwapFailed {
4702 module_id,
4703 arm,
4704 detail,
4705 ..
4706 } => write!(
4707 f,
4708 "swap of module '{module_id}' failed ({}): {detail}; the running process was left serving",
4709 arm.as_str()
4710 ),
4711 }
4712 }
4713}
4714
4715impl Error for SuperviseError {
4716 fn source(&self) -> Option<&(dyn Error + 'static)> {
4717 match self {
4718 Self::Spawn { source, .. }
4719 | Self::Cgroup { source, .. }
4720 | Self::Wait { source, .. }
4721 | Self::Kill { source, .. } => Some(source),
4722 Self::Forwarding(err) => Some(err),
4723 Self::Registry(err) => Some(err),
4724 Self::LaunchNonce { .. }
4725 | Self::InvalidSpec { .. }
4726 | Self::ReloadUnavailable { .. }
4727 | Self::Disabled { .. }
4728 | Self::ReloadFailed { .. }
4729 | Self::RegistrationStillActive { .. }
4730 | Self::StatePoisoned { .. }
4731 | Self::CommandClosed { .. }
4732 | Self::SwapInProgress { .. }
4733 | Self::SwapRefused { .. }
4734 | Self::SwapFailed { .. } => None,
4735 }
4736 }
4737}
4738
4739pub(crate) fn validate_spec(spec: &ModuleSpec) -> Result<(), SuperviseError> {
4740 if spec.carry_containment && spec.overlap == ModuleOverlap::Safe {
4741 return Err(SuperviseError::InvalidSpec {
4742 reason: "carry_containment cannot be combined with overlap: safe".to_owned(),
4743 });
4744 }
4745 if spec.module_id.trim().is_empty() {
4746 return Err(SuperviseError::InvalidSpec {
4747 reason: "module_id must not be empty".to_string(),
4748 });
4749 }
4750
4751 Ok(())
4752}
4753
4754#[derive(Debug, Default)]
4755struct HealthProbeRuntime {
4756 spawn_generation: Option<u64>,
4757 configured_health: Option<HealthConfig>,
4758 registered_connection: Option<crate::ConnectionId>,
4759 advertised: bool,
4760 next_probe_at: Option<Instant>,
4761 probe_index: u64,
4762}
4763
4764fn running_protocol(spec: &ModuleSpec, snapshot: &SharedSnapshot) -> ModuleProtocol {
4765 lock_snapshot(snapshot)
4766 .ok()
4767 .and_then(|state| state.spawned_protocol)
4768 .unwrap_or(spec.protocol)
4769}
4770
4771impl HealthProbeRuntime {
4772 fn refresh_registration(
4773 &mut self,
4774 spec: &ModuleSpec,
4775 runtime: &SupervisorRuntimeConfig,
4776 registry: &Registry,
4777 snapshot: &SharedSnapshot,
4778 ) {
4779 let generation = lock_snapshot(snapshot)
4780 .ok()
4781 .map(|state| state.spawn_generation);
4782 if self.spawn_generation != generation {
4783 *self = Self {
4787 spawn_generation: generation,
4788 ..Self::default()
4789 };
4790 }
4791 if self.configured_health.as_ref() != Some(&runtime.health) {
4792 self.configured_health = Some(runtime.health.clone());
4793 self.next_probe_at = None;
4794 self.registered_connection = None;
4795 self.probe_index = 0;
4796 }
4797 if running_protocol(spec, snapshot) == ModuleProtocol::None {
4801 self.registered_connection = None;
4802 self.advertised = runtime.health.http.is_some();
4803 if !self.advertised {
4804 self.next_probe_at = None;
4805 let _ = update_snapshot(snapshot, Some(&spec.module_id), |state| {
4806 let unknown = ModuleHealthStatus::default();
4807 if state.health != unknown {
4808 state.health = unknown;
4809 }
4810 });
4811 } else if self.next_probe_at.is_none() {
4812 self.next_probe_at = Some(
4813 Instant::now()
4814 + jittered_health_delay(&spec.module_id, 0, runtime.health.cadence),
4815 );
4816 }
4817 return;
4818 }
4819
4820 let registration = match registry.get_module(&spec.module_id) {
4821 Ok(registration) => registration,
4822 Err(err) => {
4823 warn!(module_id = %spec.module_id, error = %err, "health prober could not read registry");
4824 self.advertised = false;
4825 self.next_probe_at = None;
4826 return;
4827 }
4828 };
4829
4830 let Some(registration) = registration else {
4831 self.registered_connection = None;
4832 self.advertised = false;
4833 self.next_probe_at = None;
4834 return;
4835 };
4836
4837 let advertised = registration
4838 .control_ops
4839 .iter()
4840 .any(|op| op == MODULE_CONTROL_OP_HEALTH_CHECK);
4841 if !advertised {
4842 self.registered_connection = Some(registration.connection_id);
4843 self.advertised = false;
4844 self.next_probe_at = None;
4845 let _ = update_snapshot(snapshot, Some(&spec.module_id), |state| {
4846 if state.health.status != SupervisorHealthStatus::Unknown
4847 || state.health.consecutive_failures != 0
4848 || state.health.last_probe_ms.is_some()
4849 || state.health.detail.is_some()
4850 || state.health.metrics.is_some()
4851 {
4852 state.health.status = SupervisorHealthStatus::Unknown;
4853 state.health.consecutive_failures = 0;
4854 state.health.last_probe_ms = None;
4855 state.health.detail = None;
4856 state.health.metrics = None;
4857 }
4858 });
4859 return;
4860 }
4861
4862 let reregistered = self.registered_connection != Some(registration.connection_id);
4863 self.registered_connection = Some(registration.connection_id);
4864 self.advertised = true;
4865 if reregistered || self.next_probe_at.is_none() {
4866 self.probe_index = 0;
4867 self.next_probe_at = Some(
4868 Instant::now() + jittered_health_delay(&spec.module_id, 0, runtime.health.cadence),
4869 );
4870 let _ = update_snapshot(snapshot, Some(&spec.module_id), |state| {
4871 state.health.status = SupervisorHealthStatus::Unknown;
4872 state.health.consecutive_failures = 0;
4873 state.health.detail = None;
4874 state.health.metrics = None;
4875 });
4876 }
4877 }
4878
4879 fn wake_at(&self) -> Option<Instant> {
4880 if !self.advertised {
4881 return None;
4882 }
4883 self.next_probe_at
4884 }
4885
4886 fn due(&self) -> bool {
4887 self.advertised
4888 && self
4889 .next_probe_at
4890 .is_some_and(|next| Instant::now() >= next)
4891 }
4892
4893 fn schedule_next(&mut self, spec: &ModuleSpec, cadence: Duration) {
4894 self.probe_index = self.probe_index.wrapping_add(1);
4895 self.next_probe_at = Some(
4896 Instant::now() + jittered_health_delay(&spec.module_id, self.probe_index, cadence),
4897 );
4898 }
4899}
4900
4901#[derive(Debug)]
4936enum HealthProbeEvidence {
4937 LaneDead,
4939 NoAnswer,
4941 BadAnswer,
4943 Misconfigured,
4945}
4946
4947#[derive(Debug)]
4948struct HealthProbeError {
4949 evidence: HealthProbeEvidence,
4950 message: String,
4951}
4952
4953impl HealthProbeError {
4954 fn lane_dead(message: impl Into<String>) -> Self {
4955 Self::with(HealthProbeEvidence::LaneDead, message)
4956 }
4957
4958 fn no_answer(message: impl Into<String>) -> Self {
4959 Self::with(HealthProbeEvidence::NoAnswer, message)
4960 }
4961
4962 fn bad_answer(message: impl Into<String>) -> Self {
4963 Self::with(HealthProbeEvidence::BadAnswer, message)
4964 }
4965
4966 fn misconfigured(message: impl Into<String>) -> Self {
4967 Self::with(HealthProbeEvidence::Misconfigured, message)
4968 }
4969
4970 fn with(evidence: HealthProbeEvidence, message: impl Into<String>) -> Self {
4971 Self {
4972 evidence,
4973 message: message.into(),
4974 }
4975 }
4976
4977 #[allow(dead_code)]
4991 fn is_proof_of_death(&self) -> bool {
4992 matches!(self.evidence, HealthProbeEvidence::LaneDead)
4993 }
4994
4995 fn label(&self) -> &'static str {
5003 match self.evidence {
5004 HealthProbeEvidence::LaneDead => "lane-dead",
5005 HealthProbeEvidence::NoAnswer => "no-answer",
5006 HealthProbeEvidence::BadAnswer => "bad-answer",
5007 HealthProbeEvidence::Misconfigured => "daemon-misconfigured",
5008 }
5009 }
5010}
5011
5012impl fmt::Display for HealthProbeError {
5013 fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
5014 f.write_str(&self.message)
5015 }
5016}
5017
5018async fn run_health_probe_cycle(
5019 spec: &ModuleSpec,
5020 runtime: &SupervisorRuntimeConfig,
5021 registry: &Registry,
5022 process_liveness: &SupervisorProcessLiveness,
5023 snapshot: &SharedSnapshot,
5024 child: &mut Option<SupervisedChild>,
5025) {
5026 let now_ms = unix_ms_now();
5027 let http = (running_protocol(spec, snapshot) == ModuleProtocol::None)
5028 .then_some(runtime.health.http.as_deref())
5029 .flatten();
5030 let result = match http {
5031 Some(url) => probe_http_health(url, runtime.health.deadline).await,
5032 None => probe_module_health(&spec.module_id, runtime, None).await,
5033 };
5034 match result {
5035 Ok(report) => {
5036 handle_health_report(
5037 spec,
5038 runtime,
5039 registry,
5040 process_liveness,
5041 snapshot,
5042 child,
5043 report,
5044 now_ms,
5045 )
5046 .await;
5047 }
5048 Err(err) => {
5049 if http.is_some() {
5050 let _ = update_snapshot(snapshot, Some(&spec.module_id), |state| {
5051 state.health.status = SupervisorHealthStatus::Failing;
5052 });
5053 }
5054 handle_health_probe_failure(
5055 spec,
5056 runtime,
5057 registry,
5058 process_liveness,
5059 snapshot,
5060 child,
5061 err,
5062 now_ms,
5063 )
5064 .await;
5065 }
5066 }
5067}
5068
5069pub(crate) struct HttpProbeTarget<'a> {
5070 address: std::net::SocketAddr,
5071 localhost: bool,
5072 authority: &'a str,
5073 path: String,
5074}
5075
5076pub(crate) fn parse_http_probe_url(url: &str) -> Result<HttpProbeTarget<'_>, String> {
5079 if url.bytes().any(|b| b <= b' ' || b == 127) || url.contains('#') {
5080 return Err("must not contain whitespace, controls, or a fragment".into());
5081 }
5082 let rest = url
5083 .strip_prefix("http://")
5084 .ok_or("must use plain http://")?;
5085 let split = rest.find(['/', '?']).unwrap_or(rest.len());
5086 let (authority, suffix) = rest.split_at(split);
5087 let (host, port) = if let Some(rest) = authority.strip_prefix("[::1]") {
5088 ("::1", rest)
5089 } else {
5090 let split = authority.find(':').unwrap_or(authority.len());
5091 authority.split_at(split)
5092 };
5093 let ip: std::net::IpAddr = match host {
5094 "127.0.0.1" | "localhost" => std::net::Ipv4Addr::LOCALHOST.into(),
5095 "::1" => std::net::Ipv6Addr::LOCALHOST.into(),
5096 _ => return Err("host must be 127.0.0.1, [::1], or localhost".into()),
5097 };
5098 let port = if port.is_empty() {
5099 80
5100 } else {
5101 port.strip_prefix(':')
5102 .and_then(|p| p.parse::<u16>().ok())
5103 .filter(|p| *p > 0)
5104 .ok_or("must have a valid nonzero TCP port")?
5105 };
5106 let path = if suffix.is_empty() {
5107 "/".into()
5108 } else if suffix.starts_with('?') {
5109 format!("/{suffix}")
5110 } else {
5111 suffix.into()
5112 };
5113 Ok(HttpProbeTarget {
5114 address: std::net::SocketAddr::new(ip, port),
5115 localhost: host == "localhost",
5116 authority,
5117 path,
5118 })
5119}
5120
5121async fn probe_http_health(
5122 url: &str,
5123 deadline: Duration,
5124) -> Result<HealthReport, HealthProbeError> {
5125 use tokio::io::{AsyncReadExt, AsyncWriteExt, BufReader};
5126 let target = parse_http_probe_url(url).map_err(HealthProbeError::misconfigured)?;
5127 let mut response_status = String::new();
5130 let mut body = Vec::new();
5131 let probe = async {
5132 let connection = match tokio::net::TcpStream::connect(target.address).await {
5135 Err(_) if target.localhost => {
5136 tokio::net::TcpStream::connect((
5137 std::net::Ipv6Addr::LOCALHOST,
5138 target.address.port(),
5139 ))
5140 .await
5141 }
5142 result => result,
5143 };
5144 let mut stream = connection.map_err(|error| {
5145 HealthProbeError::no_answer(format!("HTTP connect failed: {error}"))
5146 })?;
5147 stream.write_all(format!("GET {} HTTP/1.1\r\nHost: {}\r\nAccept-Encoding: identity\r\nConnection: close\r\n\r\n", target.path, target.authority).as_bytes()).await
5148 .map_err(|error| HealthProbeError::no_answer(format!("HTTP write failed: {error}")))?;
5149 let mut reader = BufReader::new(stream);
5150 let mut budget = 16 * 1024;
5151 let status = http_line(&mut reader, &mut budget).await?;
5152 let mut words = status.split_ascii_whitespace();
5153 let version = words.next();
5154 let code = words
5155 .next()
5156 .filter(|word| word.len() == 3)
5157 .and_then(|word| word.parse::<u16>().ok());
5158 if !matches!(version, Some("HTTP/1.1" | "HTTP/1.0"))
5159 || !code.is_some_and(|code| (100..600).contains(&code))
5160 {
5161 return Err(HealthProbeError::bad_answer(format!(
5162 "invalid HTTP status: {status}"
5163 )));
5164 }
5165 let code = code.expect("validated status code");
5166 response_status = status.clone();
5167 let mut length = None;
5168 let mut chunked = false;
5169 loop {
5170 let line = http_line(&mut reader, &mut budget).await?;
5171 if line.is_empty() {
5172 break;
5173 }
5174 if let Some((name, value)) = line.split_once(':') {
5175 if name.eq_ignore_ascii_case("content-length") {
5176 length = Some(value.trim().parse::<u64>().map_err(|_| {
5177 HealthProbeError::bad_answer("invalid HTTP Content-Length")
5178 })?);
5179 } else if name.eq_ignore_ascii_case("transfer-encoding") {
5180 chunked = value.trim().eq_ignore_ascii_case("chunked");
5181 }
5182 }
5183 }
5184 if chunked {
5185 while body.len() < 200 {
5186 let line = http_line(&mut reader, &mut budget).await?;
5187 let size = u64::from_str_radix(line.split(';').next().unwrap_or("").trim(), 16)
5188 .map_err(|_| HealthProbeError::bad_answer("invalid HTTP chunk size"))?;
5189 if size == 0 {
5190 break;
5191 }
5192 let count = size.min((200 - body.len()) as u64) as usize;
5193 let start = body.len();
5194 (&mut reader)
5195 .take(count as u64)
5196 .read_to_end(&mut body)
5197 .await
5198 .map_err(|error| {
5199 HealthProbeError::no_answer(format!("HTTP body read failed: {error}"))
5200 })?;
5201 if body.len() - start != count {
5202 return Err(HealthProbeError::bad_answer("truncated HTTP chunk"));
5203 }
5204 if size > count as u64 || body.len() == 200 {
5205 break;
5206 }
5207 if !http_line(&mut reader, &mut budget).await?.is_empty() {
5208 return Err(HealthProbeError::bad_answer("invalid HTTP chunk delimiter"));
5209 }
5210 }
5211 } else {
5212 reader
5213 .take(length.unwrap_or(200).min(200))
5214 .read_to_end(&mut body)
5215 .await
5216 .map_err(|error| {
5217 HealthProbeError::no_answer(format!("HTTP body read failed: {error}"))
5218 })?;
5219 }
5220 if (200..300).contains(&code) {
5221 Ok(HealthReport::ok())
5222 } else {
5223 Err(HealthProbeError::bad_answer(
5224 "HTTP health endpoint returned non-2xx",
5225 ))
5226 }
5227 };
5228 let mut result = timeout(deadline, probe).await.unwrap_or_else(|_| {
5229 Err(HealthProbeError::no_answer(format!(
5230 "HTTP probe timed out after {deadline:?}"
5231 )))
5232 });
5233 if let Err(error) = &mut result {
5234 if !response_status.is_empty() {
5235 error.message = format!(
5236 "{}; {response_status}: {}",
5237 error.message,
5238 String::from_utf8_lossy(&body)
5239 );
5240 }
5241 }
5242 result
5243}
5244
5245async fn http_line(
5246 reader: &mut tokio::io::BufReader<tokio::net::TcpStream>,
5247 remaining: &mut usize,
5248) -> Result<String, HealthProbeError> {
5249 use tokio::io::{AsyncBufReadExt, AsyncReadExt};
5250 let mut line = Vec::new();
5251 (&mut *reader)
5252 .take(*remaining as u64)
5253 .read_until(b'\n', &mut line)
5254 .await
5255 .map_err(|error| {
5256 HealthProbeError::no_answer(format!("HTTP header read failed: {error}"))
5257 })?;
5258 *remaining -= line.len();
5259 if !line.ends_with(b"\r\n") {
5260 return Err(HealthProbeError::bad_answer(
5261 "HTTP headers are incomplete or exceed 16 KiB",
5262 ));
5263 }
5264 line.truncate(line.len() - 2);
5265 String::from_utf8(line).map_err(|_| HealthProbeError::bad_answer("HTTP header is not UTF-8"))
5266}
5267
5268async fn probe_module_health(
5269 module_id: &str,
5270 runtime: &SupervisorRuntimeConfig,
5271 drain_deadline: Option<Instant>,
5272) -> Result<HealthReport, HealthProbeError> {
5273 let Some(forwarding) = runtime.forwarding.as_ref() else {
5274 return Err(HealthProbeError::misconfigured(
5275 "supervisor was not configured with a forwarding table",
5276 ));
5277 };
5278 let probe_started_at = Instant::now();
5279 let mut deadline = probe_started_at + runtime.health.deadline;
5280 if let Some(drain_deadline) = drain_deadline {
5281 deadline = deadline.min(drain_deadline);
5282 }
5283 let pending = if drain_deadline.is_some() {
5284 forwarding.begin_drain_health_probe_rpc_for(
5285 module_id,
5286 MODULE_CONTROL_OP_HEALTH_CHECK,
5287 probe_started_at,
5288 deadline,
5289 )
5290 } else {
5291 forwarding.begin_health_probe_rpc_for(
5292 module_id,
5293 MODULE_CONTROL_OP_HEALTH_CHECK,
5294 probe_started_at,
5295 deadline,
5296 )
5297 }
5298 .map_err(|err| {
5299 HealthProbeError::lane_dead(format!("failed to begin health.check RPC: {err}"))
5302 })?;
5303 await_health_probe(forwarding, pending, deadline, runtime.health.deadline).await
5304}
5305
5306async fn probe_endpoint_health(
5313 endpoint: crate::ModuleEndpointId,
5314 runtime: &SupervisorRuntimeConfig,
5315 deadline_cap: Option<Instant>,
5316) -> Result<HealthReport, HealthProbeError> {
5317 let Some(forwarding) = runtime.forwarding.as_ref() else {
5318 return Err(HealthProbeError::misconfigured(
5319 "supervisor was not configured with a forwarding table",
5320 ));
5321 };
5322 let probe_started_at = Instant::now();
5323 let mut deadline = probe_started_at + runtime.health.deadline;
5324 if let Some(cap) = deadline_cap {
5325 deadline = deadline.min(cap);
5326 }
5327 let pending = forwarding
5328 .begin_endpoint_health_probe_rpc_for(
5329 endpoint,
5330 MODULE_CONTROL_OP_HEALTH_CHECK,
5331 probe_started_at,
5332 deadline,
5333 )
5334 .map_err(|err| {
5335 HealthProbeError::lane_dead(format!("failed to begin health.check RPC: {err}"))
5336 })?;
5337 await_health_probe(forwarding, pending, deadline, runtime.health.deadline).await
5338}
5339
5340async fn await_health_probe(
5342 forwarding: &ForwardingTable,
5343 pending: PendingModuleControlRpc,
5344 deadline: Instant,
5345 probe_budget: Duration,
5346) -> Result<HealthReport, HealthProbeError> {
5347 let PendingModuleControlRpc {
5348 endpoint,
5349 module_sink,
5350 negotiated_ver,
5351 corr,
5352 receiver,
5353 } = pending;
5354 let body = serde_json::to_vec(&ModuleControlRequest::HealthCheck {}).map_err(|err| {
5355 HealthProbeError::misconfigured(format!("failed to encode health.check: {err}"))
5356 })?;
5357 let frame = Frame::build_with_version(
5358 negotiated_ver,
5359 FrameType::Request,
5360 control_flags(),
5361 0,
5362 0,
5363 corr,
5364 body,
5365 )
5366 .map_err(|err| {
5367 HealthProbeError::misconfigured(format!("failed to build health.check frame: {err}"))
5368 })?;
5369
5370 match timeout_at(deadline, module_sink.send(frame)).await {
5376 Ok(Ok(())) => {}
5377 Ok(Err(err)) => {
5378 let _ = forwarding.cancel_module_control_rpc(endpoint, corr);
5379 return Err(HealthProbeError::lane_dead(format!(
5382 "failed to send health.check: {err}"
5383 )));
5384 }
5385 Err(_elapsed) => {
5386 let _ = forwarding.cancel_module_control_rpc(endpoint, corr);
5387 return Err(HealthProbeError::no_answer(
5391 "health.check send timed out before enqueue (module egress full)",
5392 ));
5393 }
5394 }
5395
5396 match timeout_at(deadline, receiver).await {
5397 Ok(Ok(ModuleControlRpcOutcome::Response(response))) => {
5401 response.health_report().ok_or_else(|| {
5402 HealthProbeError::bad_answer("health.check RPC returned a non-health response")
5403 })
5404 }
5405 Ok(Ok(ModuleControlRpcOutcome::Rejected(body))) => Err(HealthProbeError::bad_answer(
5406 format!("health.check rejected: {}", body.message),
5407 )),
5408 Ok(Ok(ModuleControlRpcOutcome::ModuleGone(message))) => {
5409 Err(HealthProbeError::lane_dead(message))
5410 }
5411 Ok(Ok(ModuleControlRpcOutcome::MalformedResponse(message))) => {
5412 Err(HealthProbeError::bad_answer(message))
5413 }
5414 Ok(Ok(ModuleControlRpcOutcome::UnexpectedOp { expected, actual })) => {
5415 Err(HealthProbeError::bad_answer(format!(
5416 "expected module-control op '{expected}', got '{actual}'"
5417 )))
5418 }
5419 Ok(Ok(ModuleControlRpcOutcome::DeadlineElapsed)) => Err(HealthProbeError::bad_answer(
5423 "module answered health.check after its daemon deadline",
5424 )),
5425 Ok(Err(_)) => Err(HealthProbeError::misconfigured(
5426 "health.check waiter was canceled before the module responded",
5427 )),
5428 Err(_) => {
5429 let _ = forwarding.tombstone_health_probe_rpc(endpoint, corr);
5430 Err(HealthProbeError::no_answer(format!(
5431 "module did not answer health.check within {probe_budget:?}"
5432 )))
5433 }
5434 }
5435}
5436
5437#[allow(clippy::too_many_arguments)]
5438async fn handle_health_report(
5439 spec: &ModuleSpec,
5440 runtime: &SupervisorRuntimeConfig,
5441 registry: &Registry,
5442 process_liveness: &SupervisorProcessLiveness,
5443 snapshot: &SharedSnapshot,
5444 child: &mut Option<SupervisedChild>,
5445 report: HealthReport,
5446 now_ms: u64,
5447) {
5448 let status = supervisor_health_status(report.status);
5449 let detail = report.detail.clone();
5450 let metrics = truncate_health_metrics(report.metrics);
5451 let _ = update_snapshot(snapshot, Some(&spec.module_id), |state| {
5452 state.health.status = status;
5453 state.health.last_probe_ms = Some(now_ms);
5454 state.health.detail = detail.clone();
5455 state.health.metrics = metrics.clone();
5456 state.health.consecutive_failures = 0;
5457 state.failed_retry.observe_health(
5458 report.status == HealthStatus::Ok,
5459 runtime.restart_policy.window,
5460 );
5461 });
5462
5463 let action = match report.status {
5464 HealthStatus::Ok => return,
5465 HealthStatus::Degraded => runtime.health.on_degraded,
5466 HealthStatus::Failing => runtime.health.on_failing,
5467 };
5468 apply_l3_health_action(
5469 spec,
5470 runtime,
5471 registry,
5472 process_liveness,
5473 snapshot,
5474 child,
5475 status,
5476 detail.as_deref(),
5477 action,
5478 now_ms,
5479 )
5480 .await;
5481}
5482
5483#[allow(clippy::too_many_arguments)]
5484async fn handle_health_probe_failure(
5485 spec: &ModuleSpec,
5486 runtime: &SupervisorRuntimeConfig,
5487 registry: &Registry,
5488 process_liveness: &SupervisorProcessLiveness,
5489 snapshot: &SharedSnapshot,
5490 child: &mut Option<SupervisedChild>,
5491 err: HealthProbeError,
5492 now_ms: u64,
5493) {
5494 let threshold = runtime.health.failure_threshold.max(1);
5495 let mut failures = 0;
5496 let detail = format!("[{}] {err}", err.label());
5501 let _ = update_snapshot(snapshot, Some(&spec.module_id), |state| {
5502 state
5503 .failed_retry
5504 .observe_health(false, runtime.restart_policy.window);
5505 if state.spawned_protocol.unwrap_or(spec.protocol) == ModuleProtocol::Subc {
5508 state.health.status = SupervisorHealthStatus::Unknown;
5509 }
5510 state.health.last_probe_ms = Some(now_ms);
5511 state.health.consecutive_failures = state.health.consecutive_failures.saturating_add(1);
5512 state.health.detail = Some(detail.clone());
5513 state.health.metrics = None;
5514 failures = state.health.consecutive_failures;
5515 });
5516
5517 if failures < threshold {
5518 warn!(
5519 module_id = %spec.module_id,
5520 consecutive_failures = failures,
5521 threshold,
5522 evidence = err.label(),
5523 detail = %detail,
5524 "health.check probe failed"
5525 );
5526 return;
5527 }
5528
5529 let _ = update_snapshot(snapshot, Some(&spec.module_id), |state| {
5530 state.state = ModuleState::Unresponsive;
5531 state.health.status = SupervisorHealthStatus::Unresponsive;
5532 });
5533 if runtime.health.critical {
5537 error!(
5538 module_id = %spec.module_id,
5539 status = "unresponsive",
5540 evidence = err.label(),
5541 detail = %detail,
5542 "critical module health alert"
5543 );
5544 } else {
5545 warn!(
5546 module_id = %spec.module_id,
5547 status = "unresponsive",
5548 evidence = err.label(),
5549 detail = %detail,
5550 "module health threshold breached"
5551 );
5552 }
5553 if let Err(err) = health_restart_child(
5554 spec,
5555 runtime,
5556 registry,
5557 process_liveness,
5558 snapshot,
5559 child,
5560 SupervisorHealthStatus::Unresponsive,
5561 Some(&detail),
5562 now_ms,
5563 )
5564 .await
5565 {
5566 error!(module_id = %spec.module_id, error = %err, "health-triggered restart failed");
5567 }
5568}
5569
5570#[allow(clippy::too_many_arguments)]
5571async fn apply_l3_health_action(
5572 spec: &ModuleSpec,
5573 runtime: &SupervisorRuntimeConfig,
5574 registry: &Registry,
5575 process_liveness: &SupervisorProcessLiveness,
5576 snapshot: &SharedSnapshot,
5577 child: &mut Option<SupervisedChild>,
5578 status: SupervisorHealthStatus,
5579 detail: Option<&str>,
5580 action: HealthAction,
5581 now_ms: u64,
5582) {
5583 record_health_action(snapshot, &spec.module_id, action.to_string(), now_ms);
5584 match action {
5585 HealthAction::Report => {
5586 info!(
5587 module_id = %spec.module_id,
5588 status = ?status,
5589 detail,
5590 "module reported non-ok health"
5591 );
5592 }
5593 HealthAction::Alert => {
5594 error!(
5595 module_id = %spec.module_id,
5596 status = ?status,
5597 detail,
5598 "module health alert"
5599 );
5600 }
5601 HealthAction::Restart => {
5602 if let Err(err) = health_restart_child(
5603 spec,
5604 runtime,
5605 registry,
5606 process_liveness,
5607 snapshot,
5608 child,
5609 status,
5610 detail,
5611 now_ms,
5612 )
5613 .await
5614 {
5615 error!(module_id = %spec.module_id, error = %err, "health-triggered restart failed");
5616 }
5617 }
5618 }
5619}
5620
5621#[allow(clippy::too_many_arguments)]
5622async fn health_restart_child(
5623 spec: &ModuleSpec,
5624 runtime: &SupervisorRuntimeConfig,
5625 registry: &Registry,
5626 process_liveness: &SupervisorProcessLiveness,
5627 snapshot: &SharedSnapshot,
5628 child: &mut Option<SupervisedChild>,
5629 status: SupervisorHealthStatus,
5630 detail: Option<&str>,
5631 now_ms: u64,
5632) -> Result<(), SuperviseError> {
5633 let (enabled, schedule) = {
5634 let mut state = lock_snapshot(snapshot)?;
5635 let enabled = state.enabled;
5636 let schedule = if enabled {
5637 state.next_crash_restart(&runtime.restart_policy, Instant::now())
5638 } else {
5639 None
5640 };
5641 (enabled, schedule)
5642 };
5643
5644 if !enabled {
5645 return Err(SuperviseError::Disabled {
5646 module_id: spec.module_id.clone(),
5647 });
5648 }
5649
5650 if schedule.is_none() {
5651 record_health_action(snapshot, &spec.module_id, "failed".to_string(), now_ms);
5652 error!(
5653 module_id = %spec.module_id,
5654 status = ?status,
5655 detail,
5656 max_restarts = runtime.restart_policy.max_restarts,
5657 window_secs = runtime.restart_policy.window.as_secs(),
5658 reason = %runtime.restart_policy.budget_exhausted_detail(),
5659 "health restart budget exhausted; marking module failed"
5660 );
5661 let stop_notice = begin_forwarding_drain_if_configured(
5662 spec,
5663 runtime,
5664 registry,
5665 snapshot,
5666 Some(true),
5667 RouteCloseReason::Disable,
5668 )
5669 .await?;
5670 update_snapshot(snapshot, Some(&spec.module_id), |state| {
5671 state.drain_disposition_detail = Some(runtime.restart_policy.budget_exhausted_detail());
5672 })?;
5673 let result = drain_optional_child(
5674 &spec.module_id,
5675 spec.protocol,
5676 stop_notice,
5677 registry,
5678 runtime.forwarding.as_deref(),
5679 snapshot,
5680 &runtime.terminal_ring,
5681 &runtime.spawn_events,
5682 child,
5683 runtime.drain_timeout,
5684 ModuleState::Failed,
5685 Some(true),
5686 &runtime.failure_context(true),
5687 )
5688 .await;
5689 update_snapshot(snapshot, Some(&spec.module_id), |state| {
5690 if state.state == ModuleState::Failed && !runtime.child_roster.is_closed() {
5691 state.failed_retry.schedule(&spec.module_id);
5692 }
5693 })?;
5694 result?;
5695 process_liveness.untrack_if_current(&spec.module_id, snapshot);
5696 return Ok(());
5697 }
5698
5699 let schedule = schedule.expect("a health restart must have a crash-restart schedule");
5700 let mut restart_count = 0;
5701 update_snapshot(snapshot, Some(&spec.module_id), |state| {
5702 restart_count = state.crash_restarts.len();
5703 state.state = ModuleState::Unresponsive;
5704 state.health.status = status;
5705 state.health.last_action = Some(HealthAction::Restart.to_string());
5706 state.health.last_action_ms = Some(now_ms);
5707 })?;
5708 warn!(
5709 module_id = %spec.module_id,
5710 status = ?status,
5711 detail,
5712 restart_count,
5713 restart_in_window = schedule.restart_in_window,
5714 delay_ms = schedule.delay.as_millis() as u64,
5715 "health-triggered module restart"
5716 );
5717
5718 let stop_notice = begin_forwarding_drain_if_configured(
5719 spec,
5720 runtime,
5721 registry,
5722 snapshot,
5723 Some(true),
5724 RouteCloseReason::Restart,
5725 )
5726 .await?;
5727 drain_optional_child(
5728 &spec.module_id,
5729 spec.protocol,
5730 stop_notice,
5731 registry,
5732 runtime.forwarding.as_deref(),
5733 snapshot,
5734 &runtime.terminal_ring,
5735 &runtime.spawn_events,
5736 child,
5737 runtime.drain_timeout,
5738 ModuleState::Restarting,
5739 Some(true),
5740 &runtime.failure_context(true),
5741 )
5742 .await?;
5743 schedule_respawn(
5744 runtime,
5745 snapshot,
5746 &spec.module_id,
5747 schedule.delay,
5748 RespawnKind::Spawn,
5749 true,
5750 )
5751}
5752
5753fn cancel_deferred_reload(runtime: &SupervisorRuntimeConfig, module_id: &str, reason: &str) {
5754 if let Some(reply) = runtime
5755 .deferred_reload_reply
5756 .lock()
5757 .unwrap_or_else(|p| p.into_inner())
5758 .take()
5759 {
5760 let _ = reply.send(Err(SuperviseError::ReloadFailed {
5761 module_id: module_id.to_string(),
5762 reason: reason.to_string(),
5763 }));
5764 }
5765}
5766
5767fn schedule_respawn(
5768 runtime: &SupervisorRuntimeConfig,
5769 snapshot: &SharedSnapshot,
5770 module_id: &str,
5771 delay: Duration,
5772 kind: RespawnKind,
5773 daemon_initiated: bool,
5774) -> Result<(), SuperviseError> {
5775 cancel_deferred_reload(runtime, module_id, "respawn superseded by another restart");
5776 update_snapshot(snapshot, Some(module_id), |state| {
5777 state.respawn_pending = true
5778 })?;
5779 *runtime
5780 .scheduled_respawn
5781 .lock()
5782 .unwrap_or_else(|p| p.into_inner()) = Some(PendingRespawn {
5783 deadline: Instant::now() + delay,
5784 kind,
5785 daemon_initiated,
5786 });
5787 Ok(())
5788}
5789
5790fn record_health_action(snapshot: &SharedSnapshot, module_id: &str, action: String, now_ms: u64) {
5791 let _ = update_snapshot(snapshot, Some(module_id), |state| {
5792 state.health.last_action = Some(action);
5793 state.health.last_action_ms = Some(now_ms);
5794 });
5795}
5796
5797fn supervisor_health_status(status: HealthStatus) -> SupervisorHealthStatus {
5798 match status {
5799 HealthStatus::Ok => SupervisorHealthStatus::Ok,
5800 HealthStatus::Degraded => SupervisorHealthStatus::Degraded,
5801 HealthStatus::Failing => SupervisorHealthStatus::Failing,
5802 }
5803}
5804
5805fn truncate_health_metrics(metrics: Option<Value>) -> Option<Value> {
5817 let metrics = metrics?;
5818 match serde_json::to_vec(&metrics) {
5819 Ok(encoded) if encoded.len() > MAX_HEALTH_METRICS_BYTES => Some(serde_json::json!({
5820 "truncated": true,
5821 "original_bytes": encoded.len(),
5822 })),
5823 Ok(_) | Err(_) => Some(metrics),
5824 }
5825}
5826
5827fn jittered_health_delay(module_id: &str, probe_index: u64, cadence: Duration) -> Duration {
5833 if cadence.is_zero() {
5834 return Duration::ZERO;
5835 }
5836 let cadence_ms = cadence.as_millis() as u64;
5837 if cadence_ms == 0 {
5853 return cadence;
5854 }
5855 let jitter_span = (cadence_ms / 10).max(1);
5870 let hash = module_id.as_bytes().iter().fold(
5871 probe_index.wrapping_mul(0x9E37_79B9_7F4A_7C15),
5872 |acc, byte| {
5873 acc.wrapping_mul(1099511628211)
5874 .wrapping_add(u64::from(*byte))
5875 },
5876 );
5877 cadence + Duration::from_millis(hash % jitter_span)
5878}
5879
5880#[cfg(test)]
5881mod tests {
5882 use super::*;
5883
5884 #[test]
5885 fn readding_a_module_clears_its_rescan_removal_tombstone() {
5886 let handle = SupervisorHandle::new();
5887 let module_id = "readded-tombstone";
5888 handle.record_rescan_removal(module_id);
5889 assert!(handle.removal_tombstone_age_ms(module_id).is_some());
5890
5891 handle.apply_identity_configuration(&ModuleSpec {
5892 module_id: module_id.to_string(),
5893 program: PathBuf::from("/test/module"),
5894 args: Vec::new(),
5895 env: Vec::new(),
5896 reserved: false,
5897 reserved_prefixes: Vec::new(),
5898 protocol: ModuleProtocol::Subc,
5899 overlap: Default::default(),
5900 carry_containment: false,
5901 });
5902
5903 assert!(
5904 handle.removal_tombstone_age_ms(module_id).is_none(),
5905 "a re-added module must not retain a stale removal tombstone"
5906 );
5907 }
5908
5909 #[derive(Debug, PartialEq, Eq)]
5912 struct OwnerInSpawnWindow {
5913 module_id: String,
5914 configured: bool,
5915 on_roster: bool,
5916 admission_refusal: Option<&'static str>,
5917 }
5918
5919 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
5930 async fn a_new_module_is_configured_before_its_first_process_can_register() {
5931 use crate::scopes::ScopeTable;
5932 use subc_protocol::{error_codes, scope::ScopeSelector, Principal};
5933
5934 let dir = cortexkit_test_support::ScratchDir::new("configured-before-spawn");
5935 let stub = |module_id: &str, program: PathBuf| ModuleSpec {
5936 module_id: module_id.to_string(),
5937 program,
5938 args: Vec::new(),
5939 env: ["XDG_DATA_HOME", "XDG_RUNTIME_DIR", "XDG_CONFIG_HOME"]
5940 .into_iter()
5941 .map(|key| (key.to_string(), dir.path().display().to_string()))
5942 .chain([("FAKE_AFT_NEVER_CONNECT".to_string(), "1".to_string())])
5943 .collect(),
5944 reserved: false,
5945 reserved_prefixes: Vec::new(),
5946 protocol: ModuleProtocol::Subc,
5947 overlap: Default::default(),
5948 carry_containment: false,
5949 };
5950 let live = super::terminal_history_tests::fake_aft_stub_path();
5951 let missing = dir.path().join("definitely-missing-module");
5952
5953 let handle = SupervisorHandle::new();
5954 let mut supervisor =
5955 Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default())
5956 .with_handle(handle.clone());
5957 let observed = Arc::new(Mutex::new(Vec::<OwnerInSpawnWindow>::new()));
5958 let hook_handle = handle.clone();
5959 let hook_observed = Arc::clone(&observed);
5960 supervisor.test_after_first_spawn = AfterFirstSpawnHook(Some(Arc::new(move |module_id| {
5961 let configured = hook_handle.is_configured(module_id);
5964 let selector = ScopeSelector::new(
5965 Principal::Reserved {
5966 module_id: module_id.to_string(),
5967 },
5968 "s".to_string(),
5969 Some(1),
5970 );
5971 let carrier = Principal::Reserved {
5972 module_id: "carrier".to_string(),
5973 };
5974 let admission_refusal = match ScopeTable::new(Vec::<String>::new())
5975 .admit(&carrier, module_id, &selector, configured)
5976 {
5977 Ok(_) => None,
5978 Err(refusal) => Some(refusal.code),
5979 };
5980 hook_observed.lock().unwrap().push(OwnerInSpawnWindow {
5981 module_id: module_id.to_string(),
5982 configured,
5983 on_roster: hook_handle.get(module_id).is_some(),
5984 admission_refusal,
5985 });
5986 })));
5987
5988 let plain = supervisor.spawn(stub("plain", live.clone())).unwrap();
5989 let configured = supervisor
5990 .supervise_configured(stub("configured", live.clone()), true)
5991 .unwrap();
5992 let with_health = supervisor
5993 .supervise_configured_with_health(
5994 stub("with-health", live.clone()),
5995 true,
5996 HealthConfig::default(),
5997 None,
5998 RestartPolicy::default(),
5999 )
6000 .unwrap();
6001 let failed = supervisor
6004 .supervise_configured_with_health(
6005 stub("failed-spawn", missing.clone()),
6006 true,
6007 HealthConfig::default(),
6008 None,
6009 RestartPolicy::default(),
6010 )
6011 .unwrap();
6012 assert!(supervisor.spawn(stub("spawn-error", missing)).is_err());
6015
6016 let expected = [
6017 "plain",
6018 "configured",
6019 "with-health",
6020 "failed-spawn",
6021 "spawn-error",
6022 ]
6023 .into_iter()
6024 .map(|module_id| OwnerInSpawnWindow {
6025 module_id: module_id.to_string(),
6026 configured: true,
6027 on_roster: !matches!(module_id, "plain" | "spawn-error"),
6028 admission_refusal: Some(error_codes::SCOPE_NOT_SYNCED),
6029 })
6030 .collect::<Vec<_>>();
6031 assert_eq!(*observed.lock().unwrap(), expected);
6032
6033 for module_id in ["plain", "configured", "with-health", "failed-spawn"] {
6034 assert!(
6035 handle.get(module_id).is_some(),
6036 "{module_id} is on the roster"
6037 );
6038 assert!(
6039 handle.is_configured(module_id),
6040 "{module_id} stays configured"
6041 );
6042 }
6043 assert!(handle.get("spawn-error").is_none());
6044 assert!(
6045 !handle.is_configured("spawn-error"),
6046 "a plain spawn that failed must not leave its module marked configured"
6047 );
6048
6049 handle.retire("failed-spawn");
6051 assert!(!handle.is_configured("failed-spawn"));
6052
6053 for module in [plain, configured, with_health] {
6054 module.stop().await.unwrap();
6055 }
6056 drop(failed);
6057 }
6058
6059 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
6066 async fn a_configured_module_reads_starting_while_its_first_process_already_runs() {
6067 let dir = cortexkit_test_support::ScratchDir::new("starting-while-spawned");
6068 let spec = ModuleSpec {
6069 module_id: "spawned-not-admitted".to_string(),
6070 program: super::terminal_history_tests::fake_aft_stub_path(),
6071 args: Vec::new(),
6072 env: ["XDG_DATA_HOME", "XDG_RUNTIME_DIR", "XDG_CONFIG_HOME"]
6073 .into_iter()
6074 .map(|key| (key.to_string(), dir.path().display().to_string()))
6075 .chain([("FAKE_AFT_NEVER_CONNECT".to_string(), "1".to_string())])
6076 .collect(),
6077 reserved: false,
6078 reserved_prefixes: Vec::new(),
6079 protocol: ModuleProtocol::Subc,
6080 overlap: Default::default(),
6081 carry_containment: false,
6082 };
6083 let handle = SupervisorHandle::new();
6084 let mut supervisor =
6085 Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default())
6086 .with_handle(handle.clone());
6087 let observed = Arc::new(Mutex::new(None));
6088 let hook_handle = handle.clone();
6089 let hook_observed = Arc::clone(&observed);
6090 supervisor.test_after_first_spawn = AfterFirstSpawnHook(Some(Arc::new(move |module_id| {
6091 let state = hook_handle
6092 .get(module_id)
6093 .expect("a configured module is on the roster before it is spawned")
6094 .state()
6095 .unwrap();
6096 *hook_observed.lock().unwrap() = Some(state);
6097 })));
6098
6099 let module = supervisor.supervise_configured(spec, true).unwrap();
6100 assert_eq!(
6101 *observed.lock().unwrap(),
6102 Some(ModuleState::Starting),
6103 "between the spawn and its admission the supervisor still reports starting"
6104 );
6105 assert_eq!(module.state().unwrap(), ModuleState::Running);
6106 module.stop().await.unwrap();
6107 }
6108
6109 fn stale_process_snapshot(state: ModuleState, enabled: bool) -> SharedSnapshot {
6110 let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::new(state, enabled)));
6111 update_snapshot(&snapshot, Some("stale-process-facts"), |snapshot| {
6112 snapshot.process_alive = true;
6113 snapshot.pid = Some(41);
6114 snapshot.spawned_at_ms = Some(42);
6115 snapshot.spawned_from = Some(PathBuf::from("/spawned/module"));
6116 snapshot.launch_spec_digest = Some(LaunchSpecDigest([0xA5; 32]));
6117 snapshot.spawned_file_identity = Some(SpawnedFileIdentity {
6118 device: 43,
6119 inode: 44,
6120 #[cfg(windows)]
6121 windows_image: None,
6122 });
6123 })
6124 .unwrap();
6125 snapshot
6126 }
6127
6128 fn assert_snapshot_process_facts_cleared(snapshot: &SharedSnapshot) {
6129 let snapshot = lock_snapshot(snapshot).unwrap();
6130 assert!(!snapshot.process_alive);
6131 assert_eq!(snapshot.pid, None);
6132 assert_eq!(snapshot.spawned_at_ms, None);
6133 assert_eq!(snapshot.spawned_from, None);
6134 assert_eq!(snapshot.launch_spec_digest, None);
6135 assert_eq!(snapshot.spawned_file_identity, None);
6136 }
6137
6138 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
6139 async fn failed_enable_spawn_clears_preexisting_current_process_facts() {
6140 let supervisor =
6141 Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default());
6142 let mut runtime = supervisor.runtime_config();
6143 runtime.test_seed_stale_facts_before_enable_spawn = true;
6144 let snapshot = stale_process_snapshot(ModuleState::Disabled, false);
6145 let mut child = None;
6146 let spec = ModuleSpec {
6147 module_id: "failed-enable-clears-facts".to_string(),
6148 program: PathBuf::from("/definitely/missing/failed-enable-module"),
6149 args: Vec::new(),
6150 env: Vec::new(),
6151 reserved: false,
6152 reserved_prefixes: Vec::new(),
6153 protocol: ModuleProtocol::Subc,
6154 overlap: Default::default(),
6155 carry_containment: false,
6156 };
6157
6158 let result = set_child_enabled(
6159 &spec,
6160 &runtime,
6161 &supervisor.registry,
6162 &supervisor.process_liveness,
6163 &snapshot,
6164 &mut child,
6165 true,
6166 &runtime.failure_context(false),
6167 )
6168 .await;
6169
6170 assert!(matches!(result, Err(SuperviseError::Spawn { .. })));
6171 assert_eq!(lock_snapshot(&snapshot).unwrap().state, ModuleState::Failed);
6172 assert_snapshot_process_facts_cleared(&snapshot);
6173 }
6174
6175 #[tokio::test]
6176 async fn start_revives_stranded_restarting_but_not_pending_backoff() {
6177 let supervisor =
6178 Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default());
6179 let runtime = supervisor.runtime_config();
6180 let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::new(
6181 ModuleState::Restarting,
6182 true,
6183 )));
6184 let spec = ModuleSpec {
6185 module_id: "start-stranded-restarting".to_string(),
6186 program: super::terminal_history_tests::fake_aft_stub_path(),
6187 args: Vec::new(),
6188 env: vec![("FAKE_AFT_NEVER_CONNECT".to_string(), "1".to_string())],
6189 reserved: false,
6190 reserved_prefixes: Vec::new(),
6191 protocol: ModuleProtocol::None,
6192 overlap: Default::default(),
6193 carry_containment: false,
6194 };
6195 let mut child = None;
6196 lock_snapshot(&snapshot).unwrap().respawn_pending = true;
6197 assert!(!super::set_child_enabled(
6198 &spec,
6199 &runtime,
6200 &Registry::default(),
6201 &supervisor.process_liveness,
6202 &snapshot,
6203 &mut child,
6204 true,
6205 &runtime.failure_context(false),
6206 )
6207 .await
6208 .unwrap());
6209 assert!(child.is_none());
6210 lock_snapshot(&snapshot).unwrap().respawn_pending = false;
6211 assert!(super::set_child_enabled(
6212 &spec,
6213 &runtime,
6214 &Registry::default(),
6215 &supervisor.process_liveness,
6216 &snapshot,
6217 &mut child,
6218 true,
6219 &runtime.failure_context(false),
6220 )
6221 .await
6222 .unwrap());
6223 assert_eq!(
6224 lock_snapshot(&snapshot).unwrap().state,
6225 ModuleState::Running
6226 );
6227 let mut child = child.unwrap();
6228 child.start_kill().unwrap();
6229 child.wait().await.unwrap();
6230 }
6231
6232 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
6233 async fn failed_reload_spawn_clears_current_process_facts() {
6234 let supervisor =
6235 Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default());
6236 let mut runtime = supervisor.runtime_config();
6237 runtime.restart_policy = RestartPolicy::new(0, Duration::ZERO);
6238 let snapshot = stale_process_snapshot(ModuleState::Running, true);
6239 let mut child = None;
6240 let spec = ModuleSpec {
6241 module_id: "failed-reload-clears-facts".to_string(),
6242 program: PathBuf::from("/unused/failed-reload-module"),
6243 args: Vec::new(),
6244 env: Vec::new(),
6245 reserved: false,
6246 reserved_prefixes: Vec::new(),
6247 protocol: ModuleProtocol::Subc,
6248 overlap: Default::default(),
6249 carry_containment: false,
6250 };
6251
6252 let result = handle_reload_spawn_failure(
6253 &spec,
6254 &runtime,
6255 &supervisor.process_liveness,
6256 &snapshot,
6257 &mut child,
6258 "forced reload spawn failure".to_string(),
6259 )
6260 .await;
6261
6262 assert!(matches!(result, Err(SuperviseError::ReloadFailed { .. })));
6263 assert_eq!(lock_snapshot(&snapshot).unwrap().state, ModuleState::Failed);
6264 assert_snapshot_process_facts_cleared(&snapshot);
6265 }
6266
6267 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
6268 async fn dropping_a_module_with_an_active_monitor_clears_current_process_facts() {
6269 let supervisor =
6270 Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default());
6271 let snapshot = stale_process_snapshot(ModuleState::Running, true);
6272 let module = supervisor.supervised_module(
6273 ModuleSpec {
6274 module_id: "drop-clears-facts".to_string(),
6275 program: PathBuf::from("/unused/drop-module"),
6276 args: Vec::new(),
6277 env: Vec::new(),
6278 reserved: false,
6279 reserved_prefixes: Vec::new(),
6280 protocol: ModuleProtocol::Subc,
6281 overlap: Default::default(),
6282 carry_containment: false,
6283 },
6284 supervisor.runtime_config(),
6285 Arc::clone(&snapshot),
6286 None,
6287 );
6288 assert!(!module
6289 .inner
6290 .monitor
6291 .lock()
6292 .unwrap()
6293 .as_ref()
6294 .unwrap()
6295 .is_finished());
6296
6297 drop(module);
6298
6299 assert_eq!(
6300 lock_snapshot(&snapshot).unwrap().state,
6301 ModuleState::Stopped
6302 );
6303 assert_snapshot_process_facts_cleared(&snapshot);
6304 }
6305
6306 #[cfg(unix)]
6307 #[tokio::test]
6308 async fn rescan_preserves_running_protocol_until_respawn() {
6309 let dir = cortexkit_test_support::ScratchDir::new("rescan-protocol");
6310 let initial = ModuleSpec {
6311 module_id: "rescan-protocol".into(),
6312 program: PathBuf::from("/bin/sleep"),
6313 args: vec!["60".into()],
6314 env: ["XDG_DATA_HOME", "XDG_RUNTIME_DIR", "XDG_CONFIG_HOME"]
6315 .into_iter()
6316 .map(|key| (key.into(), dir.path().to_string_lossy().into_owned()))
6317 .collect(),
6318 reserved: false,
6319 reserved_prefixes: vec![],
6320 protocol: ModuleProtocol::None,
6321 overlap: Default::default(),
6322 carry_containment: false,
6323 };
6324 let supervisor =
6325 Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default());
6326 let module = supervisor.spawn(initial.clone()).unwrap();
6327 assert!(module.status().unwrap().live);
6328 let mut next = initial;
6329 next.protocol = ModuleProtocol::Subc;
6330 module
6331 .update_configuration(next.clone(), HealthConfig::default(), None)
6332 .await
6333 .unwrap();
6334 assert!(
6335 module.status().unwrap().live,
6336 "rescan must not require HELLO from the old non-wire process"
6337 );
6338 let runtime = supervisor.runtime_config();
6339 let action = on_child_exit(
6340 &next,
6341 RestartPolicy::default(),
6342 &supervisor.registry,
6343 &module.inner.snapshot,
6344 &runtime.terminal_ring,
6345 &runtime.spawn_events,
6346 &runtime.child_roster,
6347 ExitReport {
6348 kind: ExitKind::Clean,
6349 code: Some(0),
6350 signal: None,
6351 at_ms: unix_ms_now(),
6352 },
6353 true,
6354 )
6355 .await;
6356 assert!(
6357 matches!(action, NextAction::Restart { .. }),
6358 "the old non-wire process's clean exit must restart"
6359 );
6360 module.drain().await.unwrap();
6361 }
6362
6363 #[cfg(unix)]
6364 #[tokio::test]
6365 async fn orphan_identity_matches_path_names_and_shebang_interpreters() {
6366 use std::os::unix::fs::PermissionsExt;
6367 let dir = cortexkit_test_support::ScratchDir::new("spawn-image-identity");
6368 let script = dir.join("module.sh");
6369 std::fs::write(&script, "#!/bin/sh\nwhile :; do sleep 0.1; done\n").unwrap();
6370 std::fs::set_permissions(&script, std::fs::Permissions::from_mode(0o700)).unwrap();
6371 let record_path = dir.join("live-children.json");
6372 let supervisor =
6373 Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default())
6374 .with_live_children_record(&record_path);
6375 for (program, args) in [
6376 (PathBuf::from("sleep"), vec!["60".into()]),
6377 (script, vec![]),
6378 ] {
6379 let spec = ModuleSpec {
6380 module_id: "image-identity".into(),
6381 program: program.clone(),
6382 args,
6383 env: ["XDG_DATA_HOME", "XDG_RUNTIME_DIR", "XDG_CONFIG_HOME"]
6384 .into_iter()
6385 .map(|key| (key.into(), dir.to_string_lossy().into_owned()))
6386 .collect(),
6387 reserved: false,
6388 reserved_prefixes: vec![],
6389 protocol: ModuleProtocol::None,
6390 overlap: Default::default(),
6391 carry_containment: false,
6392 };
6393 let module = supervisor.spawn(spec).unwrap();
6394 #[cfg(target_os = "macos")]
6395 {
6396 let deadline = Instant::now() + Duration::from_secs(5);
6399 while crate::live_children::read_record(&record_path)
6400 .unwrap()
6401 .iter()
6402 .all(|entry| entry.executable.is_none())
6403 {
6404 assert!(Instant::now() < deadline, "module image was not confirmed");
6405 tokio::time::sleep(Duration::from_millis(5)).await;
6406 }
6407 }
6408 let entry = crate::live_children::read_record(&record_path)
6409 .unwrap()
6410 .pop()
6411 .unwrap();
6412 let observed = subc_os::Process::open(entry.pid)
6413 .unwrap()
6414 .unwrap()
6415 .observe()
6416 .unwrap();
6417 let verdict = crate::live_children::identity_verdict(&entry, entry.pid, &observed);
6418 module.drain().await.unwrap();
6419 assert_eq!(
6420 verdict,
6421 crate::live_children::IdentityVerdict::Matches,
6422 "program {program:?}: recorded {entry:?}, observed {observed:?}"
6423 );
6424 }
6425 }
6426
6427 #[cfg(unix)]
6428 fn http_fixture(
6429 dir: &std::path::Path,
6430 url: &str,
6431 threshold: u32,
6432 ) -> crate::daemon_config::ConfiguredModule {
6433 let path = dir.join("subc.jsonc");
6434 std::fs::write(&path, serde_json::json!({"version": 1, "modules": {"http-process": {
6435 "program": "/bin/sleep", "args": ["60"], "protocol": "none",
6436 "env": {"XDG_DATA_HOME":dir,"XDG_RUNTIME_DIR":dir,"XDG_CONFIG_HOME":dir},
6437 "health": {"http":url,"cadence_ms":20,"deadline_ms":200,"failure_threshold":threshold},
6438 "restart": {"backoff_ms":1,"max_backoff_ms":1}, "drain_timeout_ms":10
6439 }}}).to_string()).unwrap();
6440 crate::daemon_config::load(&path)
6441 .unwrap()
6442 .unwrap()
6443 .modules
6444 .pop()
6445 .unwrap()
6446 }
6447
6448 #[cfg(unix)]
6449 async fn wait_http_health(module: &SupervisedModule, status: SupervisorHealthStatus) {
6450 timeout(Duration::from_secs(5), async {
6451 loop {
6452 if module.status().unwrap().health.status == status {
6453 break;
6454 }
6455 sleep(Duration::from_millis(5)).await;
6456 }
6457 })
6458 .await
6459 .unwrap_or_else(|_| {
6460 panic!(
6461 "expected {status:?}, got {:?}",
6462 module.status().unwrap().health
6463 )
6464 });
6465 }
6466
6467 #[cfg(unix)]
6468 #[tokio::test]
6469 async fn http_health_status_flips_ok_failing_ok() {
6470 use tokio::io::{AsyncReadExt, AsyncWriteExt};
6471 let dir = cortexkit_test_support::ScratchDir::new("http-status-flips");
6472 let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
6473 let status = Arc::new(std::sync::atomic::AtomicU16::new(200));
6474 let serving_status = status.clone();
6475 let url = format!("http://{}/healthz", listener.local_addr().unwrap());
6476 let server = tokio::spawn(async move {
6477 loop {
6478 let (mut stream, _) = listener.accept().await.unwrap();
6479 let mut request = [0u8; 2048];
6480 let count = stream.read(&mut request).await.unwrap();
6481 assert!(count > 0, "a probe must send an HTTP request");
6482 let code = serving_status.load(std::sync::atomic::Ordering::SeqCst);
6483 let body = if code == 200 {
6484 "ready"
6485 } else {
6486 "{\"status\":\"unavailable\",\"error\":\"scratch failure\"}"
6487 };
6488 let response = format!("HTTP/1.1 {code} scratch\r\nContent-Length: {}\r\nConnection: close\r\n\r\n{body}", body.len());
6489 let _ = stream.write_all(response.as_bytes()).await;
6490 }
6491 });
6492 let configured = http_fixture(&dir, &url, 1000);
6493 let module =
6494 Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default())
6495 .supervise_configured_with_health(
6496 configured.module_spec(),
6497 true,
6498 configured.health,
6499 configured.drain_timeout_ms,
6500 configured.restart,
6501 )
6502 .unwrap();
6503 wait_http_health(&module, SupervisorHealthStatus::Ok).await;
6504 status.store(503, std::sync::atomic::Ordering::SeqCst);
6505 wait_http_health(&module, SupervisorHealthStatus::Failing).await;
6506 assert!(module
6507 .status()
6508 .unwrap()
6509 .health
6510 .detail
6511 .unwrap()
6512 .contains("scratch failure"));
6513 status.store(200, std::sync::atomic::Ordering::SeqCst);
6514 wait_http_health(&module, SupervisorHealthStatus::Ok).await;
6515 assert_eq!(module.status().unwrap().health.consecutive_failures, 0);
6516 let before = module.status().unwrap();
6517 let (spec, mut health) = module.configuration().unwrap();
6518 health.http = None;
6519 module
6520 .update_configuration(spec.clone(), health.clone(), Some(10))
6521 .await
6522 .unwrap();
6523 wait_http_health(&module, SupervisorHealthStatus::Unknown).await;
6524 health.http = Some(url);
6525 module
6526 .update_configuration(spec, health, Some(10))
6527 .await
6528 .unwrap();
6529 wait_http_health(&module, SupervisorHealthStatus::Ok).await;
6530 assert_eq!(
6531 module.status().unwrap().pid,
6532 before.pid,
6533 "changing a probe must apply live, not restart its process"
6534 );
6535 let (spec, mut health) = module.configuration().unwrap();
6536 health.failure_threshold = 2;
6537 module
6538 .update_configuration(spec, health, Some(10))
6539 .await
6540 .unwrap();
6541 status.store(503, std::sync::atomic::Ordering::SeqCst);
6542 timeout(Duration::from_secs(5), async {
6543 while module.status().unwrap().spawn_generation == before.spawn_generation {
6544 sleep(Duration::from_millis(5)).await;
6545 }
6546 })
6547 .await
6548 .expect("sustained HTTP 503 responses must trigger the health restart policy");
6549 module.drain().await.unwrap();
6550 server.abort();
6551 }
6552
6553 #[cfg(unix)]
6554 #[tokio::test]
6555 async fn http_health_replacement_recovers_after_one_restart() {
6556 use tokio::io::{AsyncReadExt, AsyncWriteExt};
6557 let dir = cortexkit_test_support::ScratchDir::new("http-replacement-recovers");
6558 let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
6559 let url = format!("http://{}/healthz", listener.local_addr().unwrap());
6560 let mut configured = http_fixture(&dir, &url, 2);
6561 configured.restart.max_restarts = 2;
6562 let module =
6563 Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default())
6564 .supervise_configured_with_health(
6565 configured.module_spec(),
6566 true,
6567 configured.health,
6568 configured.drain_timeout_ms,
6569 configured.restart,
6570 )
6571 .unwrap();
6572 let first_generation = module.status().unwrap().spawn_generation;
6573 let observed = module.clone();
6574 let (healthy_tx, mut healthy_rx) = mpsc::channel(8);
6575 let server = tokio::spawn(async move {
6576 let mut generation = first_generation;
6577 let mut probes = 0;
6578 loop {
6579 let (mut stream, _) = listener.accept().await.unwrap();
6580 let mut request = [0; 2048];
6581 assert!(stream.read(&mut request).await.unwrap() > 0);
6582 let current = observed.status().unwrap().spawn_generation;
6583 if current != generation {
6584 generation = current;
6585 probes = 0;
6586 }
6587 probes += 1;
6588 let healthy = generation != first_generation && probes > 1;
6591 let code = if healthy { 200 } else { 503 };
6592 let response = format!(
6593 "HTTP/1.1 {code} scratch\r\nContent-Length: 0\r\nConnection: close\r\n\r\n"
6594 );
6595 let _ = stream.write_all(response.as_bytes()).await;
6596 if healthy {
6597 healthy_tx.send(()).await.unwrap();
6598 }
6599 }
6600 });
6601 timeout(Duration::from_secs(5), async {
6602 let mut healthy_probes = 0;
6603 loop {
6604 if module.state().unwrap() == ModuleState::Failed {
6605 break;
6606 }
6607 while healthy_rx.try_recv().is_ok() {
6608 healthy_probes += 1;
6609 }
6610 if healthy_probes >= 3
6611 && module.status().unwrap().health.status == SupervisorHealthStatus::Ok
6612 {
6613 break;
6614 }
6615 sleep(Duration::from_millis(5)).await;
6616 }
6617 })
6618 .await
6619 .expect("the replacement must settle healthy without exhausting its restart budget");
6620 let status = module.status().unwrap();
6621 module.drain().await.unwrap();
6622 server.abort();
6623 assert_eq!(status.state, ModuleState::Running);
6624 assert_eq!(status.health.status, SupervisorHealthStatus::Ok);
6625 assert_eq!(status.health.consecutive_failures, 0);
6626 assert_eq!(status.lifetime_restarts, 1);
6627 assert_eq!(status.restart_count, 1);
6628 }
6629
6630 #[cfg(unix)]
6631 #[tokio::test]
6632 async fn http_health_sustained_failure_exhausts_restart_budget() {
6633 use std::collections::BTreeMap;
6634 use tokio::io::{AsyncReadExt, AsyncWriteExt};
6635 let dir = cortexkit_test_support::ScratchDir::new("http-health-budget");
6636 let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
6637 let url = format!("http://{}/healthz", listener.local_addr().unwrap());
6638 let mut configured = http_fixture(&dir, &url, 2);
6639 configured.restart.max_restarts = 2;
6640 let module =
6641 Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default())
6642 .supervise_configured_with_health(
6643 configured.module_spec(),
6644 true,
6645 configured.health,
6646 configured.drain_timeout_ms,
6647 configured.restart,
6648 )
6649 .unwrap();
6650 let observed = module.clone();
6651 let probes = Arc::new(Mutex::new(BTreeMap::<u64, u32>::new()));
6652 let served = probes.clone();
6653 let server = tokio::spawn(async move {
6654 loop {
6655 let (mut stream, _) = listener.accept().await.unwrap();
6656 let mut request = [0; 2048];
6657 assert!(stream.read(&mut request).await.unwrap() > 0);
6658 *served
6659 .lock()
6660 .unwrap()
6661 .entry(observed.status().unwrap().spawn_generation)
6662 .or_default() += 1;
6663 let _ = stream
6664 .write_all(b"HTTP/1.1 503 Unavailable\r\nContent-Length: 0\r\nConnection: close\r\n\r\n")
6665 .await;
6666 }
6667 });
6668 timeout(Duration::from_secs(5), async {
6669 while module.state().unwrap() != ModuleState::Failed {
6670 sleep(Duration::from_millis(5)).await;
6671 }
6672 })
6673 .await
6674 .expect("sustained failures must still exhaust the health restart budget");
6675 let status = module.status().unwrap();
6676 server.abort();
6677 assert_eq!(status.restart_count, 2);
6678 assert_eq!(status.lifetime_restarts, 2);
6679 assert!(!status.process_alive);
6680 assert_eq!(
6681 probes.lock().unwrap().values().copied().collect::<Vec<_>>(),
6682 vec![2, 2, 2]
6683 );
6684 }
6685
6686 #[cfg(unix)]
6687 #[tokio::test]
6688 async fn spawn_clears_the_previous_process_health() {
6689 let dir = cortexkit_test_support::ScratchDir::new("spawn-fresh-health");
6690 let mut configured = http_fixture(&dir, "http://127.0.0.1:1/healthz", 2);
6691 configured.health.cadence = Duration::from_secs(3600);
6692 let module =
6693 Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default())
6694 .supervise_configured_with_health(
6695 configured.module_spec(),
6696 false,
6697 configured.health,
6698 configured.drain_timeout_ms,
6699 configured.restart,
6700 )
6701 .unwrap();
6702 update_snapshot(&module.inner.snapshot, Some(module.module_id()), |state| {
6703 state.health.status = SupervisorHealthStatus::Unresponsive;
6704 state.health.consecutive_failures = 3;
6705 state.health.detail = Some("old process refused connections".into());
6706 state.health.metrics = Some(serde_json::json!({"old": true}));
6707 state.health.last_probe_ms = Some(123);
6708 state.health.last_action = Some("restart".into());
6709 state.health.last_action_ms = Some(124);
6710 })
6711 .unwrap();
6712 module.set_enabled(true).await.unwrap();
6713 let health = module.status().unwrap().health;
6714 module.drain().await.unwrap();
6715 assert_eq!(health.consecutive_failures, 0);
6716 assert_eq!(health.status, SupervisorHealthStatus::Unknown);
6717 assert_eq!(health.detail, None);
6718 assert_eq!(health.metrics, None);
6719 assert_eq!(health.last_probe_ms, None);
6720 assert_eq!(health.last_action.as_deref(), Some("restart"));
6721 assert_eq!(health.last_action_ms, Some(124));
6722 }
6723
6724 #[cfg(unix)]
6725 #[tokio::test]
6726 async fn http_health_no_listener_restarts_after_consecutive_failures() {
6727 let dir = cortexkit_test_support::ScratchDir::new("http-refused");
6728 let unused = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
6729 let url = format!("http://{}/healthz", unused.local_addr().unwrap());
6730 drop(unused);
6731 let configured = http_fixture(&dir, &url, 2);
6732 let module =
6733 Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default())
6734 .supervise_configured_with_health(
6735 configured.module_spec(),
6736 true,
6737 configured.health,
6738 configured.drain_timeout_ms,
6739 configured.restart,
6740 )
6741 .unwrap();
6742 let before = module.status().unwrap().spawn_generation;
6743 timeout(Duration::from_secs(5), async {
6744 loop {
6745 let status = module.status().unwrap();
6746 if status.spawn_generation > before {
6747 assert!(status.lifetime_restarts > 0);
6748 break;
6749 }
6750 sleep(Duration::from_millis(5)).await;
6751 }
6752 })
6753 .await
6754 .expect("sustained HTTP refusal must trigger the health restart policy");
6755 module.drain().await.unwrap();
6756 }
6757
6758 #[cfg(unix)]
6759 #[tokio::test]
6760 async fn http_health_timeout_honours_deadline() {
6761 let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
6762 let url = format!("http://{}/healthz", listener.local_addr().unwrap());
6763 let server = tokio::spawn(async move {
6764 let _held = listener.accept().await.unwrap();
6765 std::future::pending::<()>().await;
6766 });
6767 let error = timeout(
6768 Duration::from_secs(1),
6769 probe_http_health(&url, Duration::from_millis(10)),
6770 )
6771 .await
6772 .expect("the probe must enforce its own deadline")
6773 .unwrap_err();
6774 server.abort();
6775 assert!(error.to_string().contains("timed out"));
6776 }
6777
6778 #[cfg(unix)]
6779 #[tokio::test]
6780 async fn http_health_localhost_reaches_an_ipv6_only_listener() {
6781 use tokio::io::{AsyncReadExt, AsyncWriteExt};
6782 let listener = tokio::net::TcpListener::bind("[::1]:0").await.unwrap();
6783 let url = format!(
6784 "http://localhost:{}/healthz",
6785 listener.local_addr().unwrap().port()
6786 );
6787 let server = tokio::spawn(async move {
6788 let (mut stream, _) = listener.accept().await.unwrap();
6789 let mut request = [0u8; 2048];
6790 assert!(stream.read(&mut request).await.unwrap() > 0);
6791 stream
6792 .write_all(b"HTTP/1.1 200 OK\r\nContent-Length: 2\r\n\r\nok")
6793 .await
6794 .unwrap();
6795 });
6796 assert_eq!(
6801 probe_http_health(&url, Duration::from_secs(10))
6802 .await
6803 .unwrap()
6804 .status,
6805 HealthStatus::Ok
6806 );
6807 server.await.unwrap();
6808 }
6809
6810 #[cfg(unix)]
6811 #[tokio::test]
6812 async fn http_health_timeout_keeps_partial_status_and_body() {
6813 use tokio::io::{AsyncReadExt, AsyncWriteExt};
6814 let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
6815 let url = format!("http://{}/healthz", listener.local_addr().unwrap());
6816 let server = tokio::spawn(async move {
6817 let (mut stream, _) = listener.accept().await.unwrap();
6818 let mut request = [0u8; 2048];
6819 assert!(stream.read(&mut request).await.unwrap() > 0);
6820 stream
6821 .write_all(
6822 b"HTTP/1.1 503 Unavailable\r\nContent-Length: 100\r\n\r\npartial diagnostic",
6823 )
6824 .await
6825 .unwrap();
6826 std::future::pending::<()>().await;
6827 });
6828 let error = probe_http_health(&url, Duration::from_secs(1))
6829 .await
6830 .unwrap_err()
6831 .to_string();
6832 server.abort();
6833 assert!(
6834 error.contains("timed out")
6835 && error.contains("503 Unavailable")
6836 && error.contains("partial diagnostic"),
6837 "{error}"
6838 );
6839 }
6840
6841 #[cfg(unix)]
6842 #[tokio::test]
6843 async fn http_health_diagnostic_body_is_bounded_to_200_bytes() {
6844 use tokio::io::{AsyncReadExt, AsyncWriteExt};
6845 let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
6846 let url = format!("http://{}/healthz", listener.local_addr().unwrap());
6847 let server = tokio::spawn(async move {
6848 let (mut stream, _) = listener.accept().await.unwrap();
6849 let mut request = [0u8; 2048];
6850 assert!(stream.read(&mut request).await.unwrap() > 0);
6851 let body = format!("{}not-in-diagnostic", "x".repeat(400));
6852 let response = format!(
6853 "HTTP/1.1 503 Unavailable\r\nContent-Length: {}\r\n\r\n{body}",
6854 body.len()
6855 );
6856 stream.write_all(response.as_bytes()).await.unwrap();
6857 });
6858 let error = probe_http_health(&url, Duration::from_secs(1))
6859 .await
6860 .unwrap_err()
6861 .to_string();
6862 server.await.unwrap();
6863 assert!(error.ends_with(&"x".repeat(200)), "{error}");
6864 assert_eq!(error.split(": ").last().unwrap().len(), 200);
6865 assert!(!error.contains("not-in-diagnostic"));
6866 }
6867
6868 #[cfg(unix)]
6869 #[tokio::test]
6870 async fn http_health_real_nats_server_monitoring() {
6871 if std::process::Command::new("nats-server")
6872 .arg("--version")
6873 .env("XDG_DATA_HOME", std::env::temp_dir())
6874 .env("XDG_RUNTIME_DIR", std::env::temp_dir())
6875 .env("XDG_CONFIG_HOME", std::env::temp_dir())
6876 .output()
6877 .is_err()
6878 {
6879 eprintln!(
6880 "skipped: http_health_real_nats_server_monitoring (nats-server is not on PATH)"
6881 );
6882 return;
6883 }
6884 let dir = cortexkit_test_support::ScratchDir::new("nats-http-monitoring");
6885 let monitor = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
6886 let port = monitor.local_addr().unwrap().port();
6887 let client = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
6888 let client_port = client.local_addr().unwrap().port();
6889 let config = dir.join("server.conf");
6890 std::fs::write(&config, format!("listen: 127.0.0.1:{client_port}\nhttp: 127.0.0.1:{port}\njetstream {{ store_dir: \"{}\" }}\n", dir.join("jetstream").display())).unwrap();
6891 let mut configured = http_fixture(&dir, &format!("http://127.0.0.1:{port}/healthz"), 1000);
6892 configured.program = PathBuf::from("nats-server");
6893 configured.args = vec!["-c".into(), config.to_string_lossy().into_owned()];
6894 drop(monitor);
6895 drop(client);
6896 let module =
6897 Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default())
6898 .supervise_configured_with_health(
6899 configured.module_spec(),
6900 true,
6901 configured.health,
6902 configured.drain_timeout_ms,
6903 configured.restart,
6904 )
6905 .unwrap();
6906 wait_http_health(&module, SupervisorHealthStatus::Ok).await;
6907 module.drain().await.unwrap();
6908 }
6909
6910 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
6911 async fn configuration_update_does_not_replace_captured_running_process_facts() {
6912 let supervisor =
6913 Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default());
6914 let snapshot = stale_process_snapshot(ModuleState::Running, true);
6915 let initial = ModuleSpec {
6916 module_id: "rescan-preserves-spawn-facts".to_string(),
6917 program: PathBuf::from("/spawned/module"),
6918 args: Vec::new(),
6919 env: Vec::new(),
6920 reserved: false,
6921 reserved_prefixes: Vec::new(),
6922 protocol: ModuleProtocol::Subc,
6923 overlap: Default::default(),
6924 carry_containment: false,
6925 };
6926 let module = supervisor.supervised_module(
6927 initial.clone(),
6928 supervisor.runtime_config(),
6929 snapshot,
6930 None,
6931 );
6932 let before = module.status().unwrap();
6933 let mut replacement = initial;
6934 replacement.program = PathBuf::from("/rescanned/replacement-module");
6935
6936 module
6937 .update_configuration(replacement, HealthConfig::default(), None)
6938 .await
6939 .unwrap();
6940
6941 let after = module.status().unwrap();
6942 assert_eq!(after.pid, before.pid);
6943 assert_eq!(after.spawned_at_ms, before.spawned_at_ms);
6944 assert_eq!(after.spawned_from, before.spawned_from);
6945 drop(module);
6946 }
6947}
6948
6949fn unix_ms_now() -> u64 {
6950 SystemTime::now()
6951 .duration_since(UNIX_EPOCH)
6952 .map(|duration| duration.as_millis().min(u128::from(u64::MAX)) as u64)
6953 .unwrap_or(0)
6954}
6955
6956async fn supervise_loop(
6957 mut spec: ModuleSpec,
6958 mut runtime: SupervisorRuntimeConfig,
6959 registry: Arc<Registry>,
6960 process_liveness: Arc<SupervisorProcessLiveness>,
6961 snapshot: SharedSnapshot,
6962 mut child: Option<SupervisedChild>,
6963 mut commands: mpsc::Receiver<SupervisorCommand>,
6964) {
6965 let mut health_probe = HealthProbeRuntime::default();
6966 let mut registration_changes = match registry.subscribe_module_changes(&spec.module_id) {
6970 Ok(changes) => Some(changes),
6971 Err(err) => {
6972 warn!(module_id = %spec.module_id, error = %err, "health prober could not subscribe to registry");
6975 None
6976 }
6977 };
6978 let mut pending_respawn: Option<PendingRespawn> = None;
6982 let mut requeued: VecDeque<SupervisorCommand> = VecDeque::new();
6985 let mut previous_instance_recheck = tokio::time::interval_at(
6986 Instant::now() + Duration::from_secs(5),
6987 Duration::from_secs(5),
6988 );
6989 previous_instance_recheck.set_missed_tick_behavior(tokio::time::MissedTickBehavior::Skip);
6990 loop {
6991 #[cfg(test)]
6992 {
6993 lock_snapshot(&snapshot).unwrap().actor_select = None;
6994 }
6995 #[cfg(target_os = "macos")]
6996 if let Some(active) = child.as_mut() {
6997 active.confirm_privacy_exec().await;
6998 }
6999 if let Some(scheduled) = runtime
7000 .scheduled_respawn
7001 .lock()
7002 .unwrap_or_else(|p| p.into_inner())
7003 .take()
7004 {
7005 runtime.recovery_is_daemon = scheduled.daemon_initiated;
7006 pending_respawn = Some(scheduled);
7007 }
7008 if pending_respawn.is_some() && (child.is_some() || !respawn_still_pending(&snapshot)) {
7009 pending_respawn = None;
7010 cancel_deferred_reload(
7011 &runtime,
7012 &spec.module_id,
7013 "respawn cancelled by a supervisor command",
7014 );
7015 }
7016 if child.is_none()
7017 && pending_respawn.is_none()
7018 && lock_snapshot(&snapshot).is_ok_and(|state| state.waiting_for_previous_pid.is_none())
7019 {
7020 cancel_deferred_reload(
7021 &runtime,
7022 &spec.module_id,
7023 "respawn cancelled before a replacement was spawned",
7024 );
7025 let mut stranded = false;
7026 let _ = update_snapshot(&snapshot, Some(&spec.module_id), |state| {
7027 state.respawn_pending = false;
7028 state.coalesced_restart_pending = false;
7029 if matches!(
7030 state.state,
7031 ModuleState::Restarting
7032 | ModuleState::Starting
7033 | ModuleState::Draining
7034 | ModuleState::Unresponsive
7035 ) {
7036 error!(module_id = %spec.module_id, state = ?state.state,
7037 "supervision operation ended without a child or pending respawn; marking failed so start can retry");
7038 stranded = true;
7039 }
7040 });
7041 if stranded {
7042 fail_snapshot(
7043 &snapshot,
7044 Some(&spec.module_id),
7045 None,
7046 &runtime.failure_context(runtime.recovery_is_daemon),
7047 );
7048 }
7049 }
7050 if child.is_none()
7051 && (runtime.child_roster.is_closed()
7052 || pending_respawn.is_none()
7053 || !lock_snapshot(&snapshot)
7054 .is_ok_and(|state| state.enabled && state.state == ModuleState::Restarting))
7055 {
7056 containment::terminate(&runtime.child_roster.containment).await;
7057 }
7058 if let Some(command) = requeued.pop_front() {
7059 if !handle_supervisor_command(
7060 command,
7061 &mut spec,
7062 &mut runtime,
7063 ®istry,
7064 &process_liveness,
7065 &snapshot,
7066 &mut child,
7067 &mut commands,
7068 &mut requeued,
7069 )
7070 .await
7071 {
7072 containment::terminate(&runtime.child_roster.containment).await;
7073 return;
7074 }
7075 if child.is_some() || !respawn_still_pending(&snapshot) {
7076 pending_respawn = None;
7077 let _ = update_snapshot(&snapshot, Some(&spec.module_id), |state| {
7078 state.respawn_pending = false
7079 });
7080 }
7081 continue;
7082 }
7083 if child.is_some() {
7084 runtime.recovery_is_daemon = true;
7085 #[cfg(test)]
7086 {
7087 lock_snapshot(&snapshot).unwrap().actor_turns += 1;
7088 }
7089 health_probe.refresh_registration(&spec, &runtime, ®istry, &snapshot);
7090 let active = child.as_mut().expect("child checked above");
7091 if runtime.child_roster.is_closed() || active.registered_since_spawn() {
7092 active.registration_deadline = None;
7093 }
7094 let registration_deadline = active.registration_deadline;
7095 let no_checks = runtime.health.http.is_none()
7096 && (running_protocol(&spec, &snapshot) == ModuleProtocol::None
7097 || (health_probe.registered_connection.is_some() && !health_probe.advertised));
7098 let uptime_reset = lock_snapshot(&snapshot).ok().and_then(|mut state| {
7099 state
7100 .failed_retry
7101 .uptime_reset_at(no_checks, runtime.restart_policy.window)
7102 });
7103 let wake_at = match (health_probe.wake_at(), uptime_reset) {
7104 (probe, Some(reset)) => Some(probe.map_or(reset, |probe| probe.min(reset))),
7105 (probe, None) => probe,
7106 };
7107 let wake_at = match (wake_at, registration_deadline) {
7108 (Some(wake), Some(register)) => Some(wake.min(register)),
7109 (wake, register) => wake.or(register),
7110 };
7111 #[cfg(test)]
7112 let wake_after = wake_at.map(|at| at.saturating_duration_since(Instant::now()));
7113 let probe_sleep = supervisor_sleep(wake_at);
7114 tokio::pin!(probe_sleep);
7115 let wire_child = running_protocol(&spec, &snapshot) == ModuleProtocol::Subc;
7120 #[cfg(test)]
7121 {
7122 let registration_pending = wire_child
7123 && registration_changes
7124 .as_ref()
7125 .is_some_and(|changes| changes.has_changed().unwrap());
7126 let mut state = lock_snapshot(&snapshot).unwrap();
7127 state.actor_select = (!registration_pending).then_some(ActorSelectCheckpoint {
7128 generation: state.spawn_generation,
7129 turn: state.actor_turns,
7130 registered_connection: health_probe.registered_connection,
7131 next_probe_at: health_probe.next_probe_at,
7132 wake_after,
7133 });
7134 }
7135 let active_child = child.as_mut().expect("child checked above");
7136 tokio::select! {
7137 wait_result = active_child.wait() => {
7138 if let Ok(mut state) = lock_snapshot(&snapshot) {
7139 state.failed_retry.uptime_reset_at(no_checks, runtime.restart_policy.window);
7140 }
7141 #[cfg(test)]
7142 { lock_snapshot(&snapshot).unwrap().actor_select = None; }
7143 let exit_report = match wait_result {
7152 Ok(status) => classify_reaped_child_exit(&snapshot, active_child, &status),
7153 Err(err) => {
7154 active_child.drain_stderr(&spec.module_id).await;
7155 fail_after_wait_error(
7156 &spec.module_id,
7157 &snapshot,
7158 &runtime.terminal_ring,
7159 &runtime.spawn_events,
7160 format!("failed to wait for supervised module: {err}"),
7161 &runtime.failure_context(true),
7162 );
7163 untrack_if_registration_released(
7164 &process_liveness,
7165 ®istry,
7166 &spec.module_id,
7167 &snapshot,
7168 );
7169 error!(module_id = %spec.module_id, error = %err, "failed to wait for supervised module");
7170 child = None;
7171 continue;
7172 }
7173 };
7174 active_child.drain_stderr(&spec.module_id).await;
7175
7176 let next = on_child_exit(
7177 &spec,
7178 runtime.restart_policy,
7179 ®istry,
7180 &snapshot,
7181 &runtime.terminal_ring,
7182 &runtime.spawn_events,
7183 &runtime.child_roster,
7184 exit_report,
7185 true,
7186 ).await;
7187 runtime.start_requester = None;
7188 runtime.pending_activity_restart = false;
7189 let carry = matches!(next, NextAction::Restart { .. })
7192 && active_child.spawn_failure.is_none()
7193 && !runtime.child_roster.is_closed();
7194 if let Err(error) = active_child.finish_exit(carry).await {
7195 containment::terminate(&runtime.child_roster.containment).await;
7196 fail_after_wait_error(&spec.module_id, &snapshot, &runtime.terminal_ring,
7197 &runtime.spawn_events, format!("failed to reap classified exit: {error}"),
7198 &runtime.failure_context(true));
7199 child = None;
7200 continue;
7201 }
7202 match next {
7203 NextAction::Stop { registration_released } => {
7204 if registration_released {
7205 process_liveness.untrack_if_current(&spec.module_id, &snapshot);
7206 }
7207 child = None;
7208 }
7209 NextAction::Restart { schedule } => {
7210 let delay = schedule.map_or(
7211 runtime.restart_policy.delay_for_restart(0),
7212 |schedule| schedule.delay,
7213 );
7214 if let Some(schedule) = schedule {
7215 log_crash_respawn(&spec.module_id, schedule);
7216 }
7217 child = None;
7225 pending_respawn = Some(PendingRespawn { deadline: Instant::now() + delay, kind: RespawnKind::Spawn, daemon_initiated: true });
7226 let _ = update_snapshot(&snapshot, Some(&spec.module_id), |state| state.respawn_pending = true);
7227 }
7228 }
7229 }
7230 command = commands.recv() => {
7231 #[cfg(test)]
7232 { lock_snapshot(&snapshot).unwrap().actor_select = None; }
7233 let Some(command) = command else {
7234 containment::terminate(&runtime.child_roster.containment).await;
7235 return;
7236 };
7237 if !handle_supervisor_command(
7238 command,
7239 &mut spec,
7240 &mut runtime,
7241 ®istry,
7242 &process_liveness,
7243 &snapshot,
7244 &mut child,
7245 &mut commands,
7246 &mut requeued,
7247 ).await {
7248 containment::terminate(&runtime.child_roster.containment).await;
7249 return;
7250 }
7251 }
7252 _ = async {
7253 match registration_changes.as_mut() {
7254 Some(changes) => { let _ = changes.changed().await; }
7255 None => std::future::pending().await,
7256 }
7257 }, if wire_child => {
7258 #[cfg(test)]
7259 { lock_snapshot(&snapshot).unwrap().actor_select = None; }
7260 }
7261 _ = &mut probe_sleep => {
7262 #[cfg(test)]
7263 { lock_snapshot(&snapshot).unwrap().actor_select = None; }
7264 if registration_deadline.is_some_and(|deadline| Instant::now() >= deadline) {
7265 active_child.registration_deadline = None;
7268 if !runtime.child_roster.is_closed()
7269 && !active_child.registered_since_spawn()
7270 && active_child.child.try_wait().is_ok_and(|status| status.is_none())
7271 {
7272 warn!(module_id = %spec.module_id, pid = active_child.pid,
7273 wait_secs = runtime.register_timeout.as_secs(),
7274 "supervised module registration_timeout: first HELLO deadline expired; stopping child");
7275 active_child.spawn_failure = Some("registration_timeout".into());
7276 if let Err(error) = stop_unregistered_child(active_child).await {
7277 error!(module_id = %spec.module_id, %error, "failed to stop unregistered child");
7278 fail_after_wait_error(&spec.module_id, &snapshot, &runtime.terminal_ring,
7279 &runtime.spawn_events, format!("registration_timeout: {error}"),
7280 &runtime.failure_context(true));
7281 child = None;
7282 }
7283 continue;
7286 }
7287 }
7288 if health_probe.due() {
7289 run_health_probe_cycle(
7290 &spec,
7291 &runtime,
7292 ®istry,
7293 &process_liveness,
7294 &snapshot,
7295 &mut child,
7296 ).await;
7297 if child.is_none() {
7298 runtime.start_requester = None;
7299 runtime.pending_activity_restart = false;
7300 }
7301 if child.is_some() {
7302 health_probe.schedule_next(&spec, runtime.health.cadence);
7303 }
7304 }
7305 }
7306 }
7307 } else if let Some(pending) = pending_respawn {
7308 tokio::select! {
7309 _ = runtime.child_roster.closed() => {
7310 pending_respawn = None;
7311 let _ = update_snapshot(&snapshot, Some(&spec.module_id), |state| {
7312 state.state = ModuleState::Stopped;
7313 state.respawn_pending = false;
7314 });
7315 containment::terminate(&runtime.child_roster.containment).await;
7316 }
7317 _ = sleep_until(pending.deadline) => {
7318 pending_respawn = None;
7319 let _ = update_snapshot(&snapshot, Some(&spec.module_id), |state| state.respawn_pending = false);
7320 if !respawn_still_pending(&snapshot) {
7324 continue;
7325 }
7326 if runtime.child_roster.is_closed() {
7331 let _ = update_snapshot(&snapshot, Some(&spec.module_id), |state| {
7332 state.state = ModuleState::Stopped;
7333 });
7334 debug!(module_id = %spec.module_id, "crash respawn cancelled by daemon shutdown");
7335 continue;
7336 }
7337 if let Err(err) = release_dead_registration(
7338 ®istry,
7339 runtime.forwarding.as_deref(),
7340 &snapshot,
7341 &spec.module_id,
7342 &runtime.failure_context(runtime.recovery_is_daemon),
7343 ).await {
7344 error!(module_id = %spec.module_id, error = %err, "registration did not release before restart");
7345 continue;
7346 }
7347
7348 if matches!(pending.kind, RespawnKind::Reload) {
7349 let reply = runtime.deferred_reload_reply.lock().unwrap_or_else(|p| p.into_inner()).take();
7350 let result = finish_reload_child(&spec, &runtime, ®istry, &process_liveness, &snapshot, &mut child).await;
7351 if result.is_ok() && runtime.pending_activity_restart {
7352 runtime.record_activity_transition("subc.module_restarted", &spec.module_id, runtime.start_requester.clone());
7353 }
7354 runtime.pending_activity_restart = false;
7355 if let Some(reply) = reply { let _ = reply.send(result); }
7356 continue;
7357 }
7358 process_liveness.track(spec.module_id.clone(), Arc::clone(&snapshot));
7359 match spawn_and_mark_running(&spec, &runtime, &snapshot) {
7360 Ok(next_child) => {
7361 child = Some(next_child);
7362 if runtime.pending_activity_restart {
7363 runtime.record_activity_transition("subc.module_restarted", &spec.module_id, runtime.start_requester.clone());
7364 runtime.pending_activity_restart = false;
7365 }
7366 debug!(module_id = %spec.module_id, "supervised module restarted after crash");
7367 }
7368 Err(err) => {
7369 fail_snapshot(&snapshot, Some(&spec.module_id), None, &runtime.failure_context(runtime.recovery_is_daemon));
7370 process_liveness.untrack_if_current(&spec.module_id, &snapshot);
7371 error!(module_id = %spec.module_id, error = %err, "failed to restart supervised module");
7372 }
7373 }
7374 }
7375 command = commands.recv() => {
7376 let Some(command) = command else {
7377 containment::terminate(&runtime.child_roster.containment).await;
7378 return;
7379 };
7380 if !handle_supervisor_command(
7381 command,
7382 &mut spec,
7383 &mut runtime,
7384 ®istry,
7385 &process_liveness,
7386 &snapshot,
7387 &mut child,
7388 &mut commands,
7389 &mut requeued,
7390 ).await {
7391 containment::terminate(&runtime.child_roster.containment).await;
7392 return;
7393 }
7394 if child.is_some() || !respawn_still_pending(&snapshot) {
7399 pending_respawn = None;
7400 let _ = update_snapshot(&snapshot, Some(&spec.module_id), |state| state.respawn_pending = false);
7401 }
7402 }
7403 }
7404 } else {
7405 let slow_retry = lock_snapshot(&snapshot).ok().and_then(|state| {
7406 (state.enabled && state.state == ModuleState::Failed)
7407 .then_some(state.failed_retry.pending)
7408 .flatten()
7409 });
7410 let command = if lock_snapshot(&snapshot)
7411 .is_ok_and(|state| state.waiting_for_previous_pid.is_some())
7412 {
7413 tokio::select! {
7414 command = commands.recv() => command,
7415 _ = previous_instance_recheck.tick() => {
7416 if runtime.child_roster.previous_instance_pid(&spec.module_id).is_none() {
7417 let enabled = lock_snapshot(&snapshot).is_ok_and(|state| state.enabled);
7418 if enabled && !runtime.child_roster.is_closed() {
7419 process_liveness.track(spec.module_id.clone(), Arc::clone(&snapshot));
7420 match spawn_and_mark_running(&spec, &runtime, &snapshot) {
7421 Ok(next_child) => child = Some(next_child),
7422 Err(error) => {
7423 fail_snapshot(&snapshot, Some(&spec.module_id), None, &runtime.failure_context(true));
7424 error!(module_id = %spec.module_id, %error, "failed to start module after its previous instance exited");
7425 }
7426 }
7427 } else {
7428 let _ = update_snapshot(&snapshot, Some(&spec.module_id), |state| {
7429 state.waiting_for_previous_pid = None;
7430 state.waiting_for_previous_detail = None;
7431 });
7432 }
7433 }
7434 continue;
7435 }
7436 }
7437 } else {
7438 tokio::select! {
7439 biased;
7440 command = commands.recv() => command,
7441 _ = async {
7442 match slow_retry {
7443 Some(retry) => sleep_until(retry.deadline).await,
7444 None => std::future::pending().await,
7445 }
7446 } => {
7447 let retry = slow_retry.expect("only a scheduled slow retry can wake");
7448 let _ = update_snapshot(&snapshot, Some(&spec.module_id), |state| {
7449 state.failed_retry.pending = None;
7450 });
7451 if runtime.child_roster.is_closed() {
7452 continue;
7453 }
7454 info!(module_id = %spec.module_id, attempt = retry.attempt,
7455 delay_secs = retry.delay.as_secs(), "starting failed module retry");
7456 if let Err(error) = set_child_enabled(
7457 &spec, &runtime, ®istry, &process_liveness, &snapshot, &mut child, true,
7458 &runtime.failure_context(true),
7459 ).await {
7460 error!(module_id = %spec.module_id, %error, "failed module retry could not start");
7461 }
7462 continue;
7463 }
7464 }
7465 };
7466 let Some(command) = command else {
7467 containment::terminate(&runtime.child_roster.containment).await;
7468 return;
7469 };
7470 if !handle_supervisor_command(
7471 command,
7472 &mut spec,
7473 &mut runtime,
7474 ®istry,
7475 &process_liveness,
7476 &snapshot,
7477 &mut child,
7478 &mut commands,
7479 &mut requeued,
7480 )
7481 .await
7482 {
7483 containment::terminate(&runtime.child_roster.containment).await;
7484 return;
7485 }
7486 }
7487 }
7488}
7489
7490async fn supervisor_sleep(wake_at: Option<Instant>) {
7493 match wake_at {
7494 Some(deadline) => sleep_until(deadline).await,
7495 None => std::future::pending().await,
7496 }
7497}
7498
7499fn log_crash_respawn(module_id: &str, schedule: CrashRestartSchedule) {
7500 info!(
7501 module_id,
7502 restart_in_window = schedule.restart_in_window,
7503 delay_ms = schedule.delay.as_millis() as u64,
7504 "respawning after crash"
7505 );
7506}
7507
7508fn respawn_still_pending(snapshot: &SharedSnapshot) -> bool {
7514 matches!(
7515 lock_snapshot(snapshot),
7516 Ok(state) if state.enabled && state.state == ModuleState::Restarting
7517 )
7518}
7519
7520enum NextAction {
7521 Stop {
7522 registration_released: bool,
7523 },
7524 Restart {
7525 schedule: Option<CrashRestartSchedule>,
7526 },
7527}
7528
7529#[allow(clippy::too_many_arguments)]
7530async fn handle_supervisor_command(
7531 command: SupervisorCommand,
7532 spec: &mut ModuleSpec,
7533 runtime: &mut SupervisorRuntimeConfig,
7534 registry: &Arc<Registry>,
7535 process_liveness: &SupervisorProcessLiveness,
7536 snapshot: &SharedSnapshot,
7537 child: &mut Option<SupervisedChild>,
7538 commands: &mut mpsc::Receiver<SupervisorCommand>,
7539 requeued: &mut VecDeque<SupervisorCommand>,
7540) -> bool {
7541 if !matches!(command, SupervisorCommand::UpdateConfiguration { .. }) {
7542 runtime.recovery_is_daemon = false;
7543 }
7544 if matches!(
7545 command,
7546 SupervisorCommand::Drain { .. }
7547 | SupervisorCommand::Retire { .. }
7548 | SupervisorCommand::SetEnabled { .. }
7549 | SupervisorCommand::Restart { .. }
7550 ) {
7551 let _ = update_snapshot(snapshot, Some(&spec.module_id), |state| {
7552 state.failed_retry.cancel()
7553 });
7554 }
7555 let blocked = recovery_block_error(spec, &runtime.child_roster).or_else(|| {
7556 runtime
7557 .child_roster
7558 .previous_instance(&spec.module_id)
7559 .map(|(pid, reason)| previous_instance_error(spec, pid, reason))
7560 });
7561 let command = if let Some(error) = blocked {
7562 match command {
7563 SupervisorCommand::Restart { reply, .. }
7564 | SupervisorCommand::Reload { reply, .. }
7565 | SupervisorCommand::Swap { reply, .. } => {
7566 let _ = reply.send(Err(error));
7567 return true;
7568 }
7569 SupervisorCommand::SetEnabled {
7570 enabled: true,
7571 reply,
7572 ..
7573 } => {
7574 let _ = reply.send(Err(error));
7575 return true;
7576 }
7577 other => other,
7578 }
7579 } else {
7580 command
7581 };
7582 match command {
7583 SupervisorCommand::Drain { requester, reply } => {
7584 let result = drain_optional_child(
7587 &spec.module_id,
7588 spec.protocol,
7589 StopNotice::NotSent,
7590 registry,
7591 runtime.forwarding.as_deref(),
7592 snapshot,
7593 &runtime.terminal_ring,
7594 &runtime.spawn_events,
7595 child,
7596 runtime.drain_timeout,
7597 ModuleState::Stopped,
7598 None,
7599 &FailureContext::operator(),
7600 )
7601 .await;
7602 containment::terminate(&runtime.child_roster.containment).await;
7603 let registration_released = result.is_ok();
7604 if registration_released {
7605 runtime.record_activity_transition(
7606 "subc.module_stopped",
7607 &spec.module_id,
7608 requester,
7609 );
7610 }
7611 let _ = reply.send(result);
7612 if registration_released {
7613 process_liveness.untrack_if_current(&spec.module_id, snapshot);
7614 }
7615 false
7616 }
7617 SupervisorCommand::Retire { requester, reply } => {
7618 let result = async {
7619 let stop_notice = begin_forwarding_drain_if_configured(
7620 spec,
7621 runtime,
7622 registry,
7623 snapshot,
7624 None,
7625 RouteCloseReason::Disable,
7626 )
7627 .await?;
7628 drain_optional_child(
7629 &spec.module_id,
7630 spec.protocol,
7631 stop_notice,
7632 registry,
7633 runtime.forwarding.as_deref(),
7634 snapshot,
7635 &runtime.terminal_ring,
7636 &runtime.spawn_events,
7637 child,
7638 runtime.drain_timeout,
7639 ModuleState::Stopped,
7640 None,
7641 &FailureContext::operator(),
7642 )
7643 .await
7644 }
7645 .await;
7646 containment::terminate(&runtime.child_roster.containment).await;
7647 let registration_released = result.is_ok();
7648 if registration_released {
7649 runtime.record_activity_transition(
7650 "subc.module_stopped",
7651 &spec.module_id,
7652 requester,
7653 );
7654 }
7655 let _ = reply.send(result);
7656 if registration_released {
7657 process_liveness.untrack_if_current(&spec.module_id, snapshot);
7658 }
7659 false
7660 }
7661 SupervisorCommand::Restart {
7662 requester,
7663 drain_timeout_ms,
7664 received_at_generation,
7665 queued_at,
7666 reply,
7667 } => {
7668 info!(
7672 module_id = %spec.module_id,
7673 queued_ms = u64::try_from(queued_at.elapsed().as_millis()).unwrap_or(u64::MAX),
7674 "restart command dequeued"
7675 );
7676 let validation = match lock_snapshot(snapshot) {
7688 Ok(state) if !state.enabled => Err(SuperviseError::Disabled {
7689 module_id: spec.module_id.clone(),
7690 }),
7691 Ok(_) => Ok(()),
7692 Err(err) => Err(err),
7693 };
7694 let initiated = validation.is_ok();
7695 let _ = reply.send(validation);
7696 let satisfied_by_generation = if initiated
7710 && child
7711 .as_ref()
7712 .is_some_and(|child| child.spawned_at > queued_at)
7713 {
7714 lock_snapshot(snapshot).ok().and_then(|state| {
7715 (!state.configuration_updated_since_spawn).then_some(state.spawn_generation)
7716 })
7717 } else {
7718 None
7719 };
7720 let satisfied_by_pending = initiated
7721 && child.is_none()
7722 && lock_snapshot(snapshot).ok().is_some_and(|mut state| {
7723 let pending = state.respawn_pending && !state.configuration_updated_since_spawn;
7727 if pending {
7728 state.coalesced_restart_pending = true;
7729 }
7730 pending
7731 });
7732 if satisfied_by_pending {
7733 debug!(module_id = %spec.module_id, "restart coalesced into the pending replacement");
7734 } else if let Some(generation) = satisfied_by_generation {
7735 info!(
7736 module_id = %spec.module_id,
7737 received_at_generation,
7738 "restart already satisfied by generation {generation}; not restarting again"
7739 );
7740 } else if initiated {
7741 let drain_timeout = drain_timeout_ms
7744 .map(Duration::from_millis)
7745 .unwrap_or(runtime.drain_timeout);
7746 if let Err(err) = restart_child(
7747 spec,
7748 runtime,
7749 registry,
7750 process_liveness,
7751 snapshot,
7752 child,
7753 drain_timeout,
7754 )
7755 .await
7756 {
7757 warn!(
7758 module_id = %spec.module_id,
7759 error = %err,
7760 "operator restart failed after initiation ack; module state carries the outcome"
7761 );
7762 let _ = update_snapshot(snapshot, Some(&spec.module_id), |state| {
7763 state.state = ModuleState::Failed;
7764 clear_current_process_facts(state);
7765 });
7766 } else {
7767 runtime.start_requester = requester;
7768 runtime.pending_activity_restart = true;
7769 }
7770 }
7771 true
7772 }
7773 SupervisorCommand::Reload { requester, reply } => {
7774 let result =
7775 reload_child(spec, runtime, registry, process_liveness, snapshot, child).await;
7776 if result.is_ok() {
7777 runtime.start_requester = requester;
7778 runtime.pending_activity_restart = true;
7779 }
7780 if result.is_ok()
7781 && runtime
7782 .scheduled_respawn
7783 .lock()
7784 .unwrap_or_else(|p| p.into_inner())
7785 .as_ref()
7786 .is_some_and(|pending| matches!(pending.kind, RespawnKind::Reload))
7787 {
7788 *runtime
7789 .deferred_reload_reply
7790 .lock()
7791 .unwrap_or_else(|p| p.into_inner()) = Some(reply);
7792 } else {
7793 let _ = reply.send(result);
7794 }
7795 true
7796 }
7797 SupervisorCommand::SetEnabled {
7798 enabled,
7799 requester,
7800 reply,
7801 } => {
7802 let previous_requester = runtime.start_requester.clone();
7803 if enabled {
7804 runtime.start_requester = requester.clone();
7805 }
7806 let result = set_child_enabled(
7807 spec,
7808 runtime,
7809 registry,
7810 process_liveness,
7811 snapshot,
7812 child,
7813 enabled,
7814 &runtime.failure_context(false),
7815 )
7816 .await;
7817 if !enabled {
7818 containment::terminate(&runtime.child_roster.containment).await;
7819 }
7820 if matches!(result, Ok(true)) {
7821 runtime.pending_activity_restart = false;
7822 if enabled {
7823 runtime.start_requester = requester;
7824 } else {
7825 runtime.start_requester = None;
7826 runtime.record_activity_transition(
7827 "subc.module_disabled",
7828 &spec.module_id,
7829 requester,
7830 );
7831 }
7832 } else {
7833 runtime.start_requester = previous_requester;
7834 }
7835 let _ = reply.send(result);
7836 true
7837 }
7838 SupervisorCommand::UpdateConfiguration {
7839 spec: next_spec,
7840 health,
7841 drain_timeout_ms,
7842 reply,
7843 } => {
7844 if let Some(handle) = &runtime.supervisor_handle {
7845 handle.apply_identity_configuration(&next_spec);
7846 }
7847 *spec = next_spec;
7848 let _ = update_snapshot(snapshot, Some(&spec.module_id), |state| {
7849 state.configuration_updated_since_spawn = true;
7850 });
7851 let health_changed = runtime.health != health;
7852 runtime.health = health;
7853 if health_changed {
7856 let _ = update_snapshot(snapshot, Some(&spec.module_id), |state| {
7857 state.health = ModuleHealthStatus::default();
7858 });
7859 }
7860 runtime.drain_timeout = drain_timeout_ms
7861 .map(Duration::from_millis)
7862 .unwrap_or(runtime.default_drain_timeout);
7863 *runtime
7864 .effective_drain_timeout
7865 .lock()
7866 .unwrap_or_else(|poisoned| poisoned.into_inner()) = runtime.drain_timeout;
7867 let _ = reply.send(());
7868 true
7869 }
7870 SupervisorCommand::Swap {
7871 requester,
7872 ready_timeout,
7873 reply,
7874 } => {
7875 if spec.carry_containment
7876 || containment::current(&runtime.child_roster.containment).is_some()
7877 {
7878 let _ = reply.send(Err(SuperviseError::InvalidSpec {
7879 reason: "a carried container cannot overlap a swap candidate".to_owned(),
7880 }));
7881 return true;
7882 }
7883 let mut swap_runtime = runtime.clone();
7884 swap_runtime.start_requester = requester;
7885 let before = lock_snapshot(snapshot)
7886 .ok()
7887 .map(|state| state.spawn_generation);
7888 let end = swap::run_swap(
7889 spec,
7890 &swap_runtime,
7891 registry,
7892 process_liveness,
7893 snapshot,
7894 child,
7895 commands,
7896 ready_timeout.unwrap_or(DEFAULT_SWAP_READY_TIMEOUT),
7897 reply,
7898 )
7899 .await;
7900 if before
7901 != lock_snapshot(snapshot)
7902 .ok()
7903 .map(|state| state.spawn_generation)
7904 {
7905 runtime.start_requester = swap_runtime.start_requester;
7906 }
7907 requeued.extend(end.requeue);
7908 true
7909 }
7910 }
7911}
7912
7913async fn restart_child(
7914 spec: &ModuleSpec,
7915 runtime: &SupervisorRuntimeConfig,
7916 registry: &Registry,
7917 process_liveness: &SupervisorProcessLiveness,
7918 snapshot: &SharedSnapshot,
7919 child: &mut Option<SupervisedChild>,
7920 drain_timeout: Duration,
7921) -> Result<(), SuperviseError> {
7922 if !lock_snapshot(snapshot)?.enabled {
7924 return Err(SuperviseError::Disabled {
7925 module_id: spec.module_id.clone(),
7926 });
7927 }
7928 let stop_notice = begin_forwarding_drain_with_timeout(
7929 spec,
7930 runtime,
7931 registry,
7932 snapshot,
7933 None,
7934 RouteCloseReason::Restart,
7935 drain_timeout,
7936 )
7937 .await?;
7938
7939 if child.is_some() {
7940 drain_optional_child(
7941 &spec.module_id,
7942 spec.protocol,
7943 stop_notice,
7944 registry,
7945 runtime.forwarding.as_deref(),
7946 snapshot,
7947 &runtime.terminal_ring,
7948 &runtime.spawn_events,
7949 child,
7950 drain_timeout,
7951 ModuleState::Restarting,
7952 Some(true),
7953 &FailureContext::operator(),
7954 )
7955 .await?;
7956 } else {
7957 update_snapshot(snapshot, Some(&spec.module_id), |state| {
7958 state.enabled = true;
7959 state.state = ModuleState::Restarting;
7960 clear_current_process_facts(state);
7961 })?;
7962 release_dead_registration(
7963 registry,
7964 runtime.forwarding.as_deref(),
7965 snapshot,
7966 &spec.module_id,
7967 &runtime.failure_context(false),
7968 )
7969 .await?;
7970 }
7971
7972 reset_restart_count(snapshot, &spec.module_id)?;
7973 process_liveness.track(spec.module_id.clone(), Arc::clone(snapshot));
7974 schedule_respawn(
7975 runtime,
7976 snapshot,
7977 &spec.module_id,
7978 runtime.restart_policy.backoff,
7979 RespawnKind::Spawn,
7980 false,
7981 )
7982}
7983
7984async fn reload_child(
7985 spec: &ModuleSpec,
7986 runtime: &SupervisorRuntimeConfig,
7987 registry: &Registry,
7988 process_liveness: &SupervisorProcessLiveness,
7989 snapshot: &SharedSnapshot,
7990 child: &mut Option<SupervisedChild>,
7991) -> Result<(), SuperviseError> {
7992 if !lock_snapshot(snapshot)?.enabled {
7994 return Err(SuperviseError::Disabled {
7995 module_id: spec.module_id.clone(),
7996 });
7997 }
7998 let stop_notice = begin_forwarding_drain(
7999 spec,
8000 runtime,
8001 registry,
8002 snapshot,
8003 Some(true),
8004 RouteCloseReason::Reload,
8005 )
8006 .await?;
8007
8008 if child.is_some() {
8009 drain_optional_child(
8010 &spec.module_id,
8011 spec.protocol,
8012 stop_notice,
8013 registry,
8014 runtime.forwarding.as_deref(),
8015 snapshot,
8016 &runtime.terminal_ring,
8017 &runtime.spawn_events,
8018 child,
8019 runtime.drain_timeout,
8020 ModuleState::Restarting,
8021 Some(true),
8022 &FailureContext::operator(),
8023 )
8024 .await?;
8025 } else {
8026 update_snapshot(snapshot, Some(&spec.module_id), |state| {
8027 state.enabled = true;
8028 state.state = ModuleState::Restarting;
8029 clear_current_process_facts(state);
8030 })?;
8031 release_dead_registration(
8032 registry,
8033 runtime.forwarding.as_deref(),
8034 snapshot,
8035 &spec.module_id,
8036 &runtime.failure_context(false),
8037 )
8038 .await?;
8039 }
8040
8041 reset_restart_count(snapshot, &spec.module_id)?;
8042 process_liveness.track(spec.module_id.clone(), Arc::clone(snapshot));
8043 schedule_respawn(
8044 runtime,
8045 snapshot,
8046 &spec.module_id,
8047 runtime.restart_policy.backoff,
8048 RespawnKind::Reload,
8049 false,
8050 )
8051}
8052
8053async fn finish_reload_child(
8054 spec: &ModuleSpec,
8055 runtime: &SupervisorRuntimeConfig,
8056 registry: &Registry,
8057 process_liveness: &SupervisorProcessLiveness,
8058 snapshot: &SharedSnapshot,
8059 child: &mut Option<SupervisedChild>,
8060) -> Result<(), SuperviseError> {
8061 process_liveness.track(spec.module_id.clone(), Arc::clone(snapshot));
8062 let next_child = match spawn_and_mark_running(spec, runtime, snapshot) {
8063 Ok(next_child) => next_child,
8064 Err(err) => {
8065 return handle_reload_spawn_failure(
8066 spec,
8067 runtime,
8068 process_liveness,
8069 snapshot,
8070 child,
8071 format!("new child failed to spawn: {err}"),
8072 )
8073 .await;
8074 }
8075 };
8076 *child = Some(next_child);
8077
8078 let wait_outcome = {
8079 let active_child = child.as_mut().expect("new reload child was just stored");
8080 wait_for_registration_after_reload(
8081 registry,
8082 &spec.module_id,
8083 snapshot,
8084 active_child,
8085 REGISTRY_RELEASE_TIMEOUT,
8086 )
8087 .await?
8088 };
8089
8090 match wait_outcome {
8091 RegistrationWaitOutcome::Registered => {
8092 child
8093 .as_mut()
8094 .expect("registered reload child")
8095 .registration_deadline = None;
8096 debug!(module_id = %spec.module_id, "supervised module reloaded and registered");
8097 Ok(())
8098 }
8099 RegistrationWaitOutcome::Exited(exit_report) => {
8100 if let Some(active_child) = child.as_mut() {
8101 active_child.drain_stderr(&spec.module_id).await;
8102 }
8103 let mut exited_child = child.take().expect("exited reload child is still stored");
8106 #[cfg(test)]
8107 if let Some(gate) = &runtime.test_reload_exit_record_gate {
8108 gate.reached.notify_one();
8109 gate.resume.notified().await;
8110 }
8111 let result = handle_reload_child_registration_failure(
8112 spec,
8113 runtime,
8114 registry,
8115 process_liveness,
8116 snapshot,
8117 child,
8118 ReloadRegistrationFailure {
8119 exit_report: registration_failure_exit_report(exit_report),
8120 reason: exited_child
8121 .spawn_failure
8122 .clone()
8123 .unwrap_or_else(|| "new child exited before registering".to_string()),
8124 },
8125 )
8126 .await;
8127 exited_child
8128 .finish_exit(false)
8129 .await
8130 .map_err(|source| SuperviseError::Wait {
8131 module_id: spec.module_id.clone(),
8132 source,
8133 })?;
8134 result
8135 }
8136 RegistrationWaitOutcome::TimedOut => {
8137 let mut timed_out_child = child
8138 .take()
8139 .expect("timed-out reload child is still running");
8140 timed_out_child
8141 .start_kill()
8142 .map_err(|source| SuperviseError::Kill {
8143 module_id: spec.module_id.clone(),
8144 source,
8145 })?;
8146 let status = timed_out_child
8147 .wait()
8148 .await
8149 .map_err(|source| SuperviseError::Wait {
8150 module_id: spec.module_id.clone(),
8151 source,
8152 })?;
8153 timed_out_child.drain_stderr(&spec.module_id).await;
8154 let result = handle_reload_child_registration_failure(
8155 spec,
8156 runtime,
8157 registry,
8158 process_liveness,
8159 snapshot,
8160 child,
8161 ReloadRegistrationFailure {
8162 exit_report: registration_failure_exit_report(classify_reaped_child_exit(
8163 snapshot,
8164 &timed_out_child,
8165 &status,
8166 )),
8167 reason: format!(
8168 "new child did not register within {:?}",
8169 REGISTRY_RELEASE_TIMEOUT
8170 ),
8171 },
8172 )
8173 .await;
8174 timed_out_child
8175 .finish_exit(false)
8176 .await
8177 .map_err(|source| SuperviseError::Wait {
8178 module_id: spec.module_id.clone(),
8179 source,
8180 })?;
8181 result
8182 }
8183 }
8184}
8185
8186#[allow(clippy::too_many_arguments)]
8187async fn set_child_enabled(
8188 spec: &ModuleSpec,
8189 runtime: &SupervisorRuntimeConfig,
8190 registry: &Registry,
8191 process_liveness: &SupervisorProcessLiveness,
8192 snapshot: &SharedSnapshot,
8193 child: &mut Option<SupervisedChild>,
8194 enabled: bool,
8195 failure: &FailureContext,
8196) -> Result<bool, SuperviseError> {
8197 let (current_enabled, current_state, respawn_pending) = {
8198 let state = lock_snapshot(snapshot)?;
8199 (state.enabled, state.state, state.respawn_pending)
8200 };
8201 let revive_terminal = enabled
8209 && current_enabled
8210 && child.is_none()
8211 && (matches!(current_state, ModuleState::Failed | ModuleState::Stopped)
8212 || (current_state == ModuleState::Restarting && !respawn_pending));
8213 if current_enabled == enabled && !revive_terminal {
8214 return Ok(false);
8215 }
8216
8217 if enabled {
8218 update_snapshot(snapshot, Some(&spec.module_id), |state| {
8219 state.enabled = true;
8220 state.state = ModuleState::Starting;
8221 clear_current_process_facts(state);
8222 })?;
8223 #[cfg(test)]
8224 if runtime.test_seed_stale_facts_before_enable_spawn {
8225 update_snapshot(snapshot, Some(&spec.module_id), |state| {
8226 state.process_alive = true;
8227 state.pid = Some(41);
8228 state.spawned_at_ms = Some(42);
8229 state.spawned_from = Some(PathBuf::from("/spawned/module"));
8230 state.spawned_file_identity = Some(SpawnedFileIdentity {
8231 device: 43,
8232 inode: 44,
8233 #[cfg(windows)]
8234 windows_image: None,
8235 });
8236 })?;
8237 }
8238 release_dead_registration(
8239 registry,
8240 runtime.forwarding.as_deref(),
8241 snapshot,
8242 &spec.module_id,
8243 failure,
8244 )
8245 .await?;
8246 reset_restart_count(snapshot, &spec.module_id)?;
8247 process_liveness.track(spec.module_id.clone(), Arc::clone(snapshot));
8248 let next_child = match spawn_and_mark_running(spec, runtime, snapshot) {
8249 Ok(next_child) => next_child,
8250 Err(err) => {
8251 fail_snapshot(snapshot, Some(&spec.module_id), None, failure);
8252 process_liveness.untrack_if_current(&spec.module_id, snapshot);
8253 return Err(err);
8254 }
8255 };
8256 *child = Some(next_child);
8257 debug!(module_id = %spec.module_id, "supervised module enabled");
8258 Ok(true)
8259 } else {
8260 let stop_notice = begin_forwarding_drain_if_configured(
8261 spec,
8262 runtime,
8263 registry,
8264 snapshot,
8265 Some(false),
8266 RouteCloseReason::Disable,
8267 )
8268 .await?;
8269 drain_optional_child(
8270 &spec.module_id,
8271 spec.protocol,
8272 stop_notice,
8273 registry,
8274 runtime.forwarding.as_deref(),
8275 snapshot,
8276 &runtime.terminal_ring,
8277 &runtime.spawn_events,
8278 child,
8279 runtime.drain_timeout,
8280 ModuleState::Disabled,
8281 Some(false),
8282 failure,
8283 )
8284 .await?;
8285 debug!(module_id = %spec.module_id, "supervised module disabled");
8286 Ok(true)
8287 }
8288}
8289
8290#[allow(clippy::too_many_arguments)]
8291async fn on_child_exit(
8292 spec: &ModuleSpec,
8293 policy: RestartPolicy,
8294 registry: &Registry,
8295 snapshot: &SharedSnapshot,
8296 terminal_ring: &Arc<Mutex<TerminalRing>>,
8297 spawn_events: &SpawnEventFeed,
8298 roster: &ChildRoster,
8299 exit_report: ExitReport,
8300 daemon_initiated: bool,
8301) -> NextAction {
8302 if roster.is_closed() {
8308 return on_child_exit_during_daemon_shutdown(
8309 spec,
8310 registry,
8311 snapshot,
8312 terminal_ring,
8313 spawn_events,
8314 exit_report,
8315 )
8316 .await;
8317 }
8318 let unrequested_clean_exit_of_protocol_none = exit_report.kind == ExitKind::Clean
8334 && running_protocol(spec, snapshot) == ModuleProtocol::None;
8335 match exit_report.kind {
8336 ExitKind::Clean if !unrequested_clean_exit_of_protocol_none => {
8337 info!(
8338 module_id = %spec.module_id,
8339 exit_code = ?exit_report.code,
8340 exit_signal = ?exit_report.signal,
8341 "supervised module exited cleanly"
8342 );
8343 if let Err(err) = publish_after_terminal_record(
8344 &spec.module_id,
8345 || {
8346 record_terminal(
8347 &spec.module_id,
8348 terminal_ring,
8349 spawn_events,
8350 &exit_report,
8351 TerminalDisposition::Stopped,
8352 )
8353 },
8354 || {
8355 update_snapshot(snapshot, Some(&spec.module_id), |state| {
8356 state.state = ModuleState::Stopped;
8357 clear_current_process_facts(state);
8358 state.last_exit = Some(exit_report.clone());
8359 })
8360 },
8361 ) {
8362 error!(module_id = %spec.module_id, error = %err, "failed to publish clean module exit");
8363 }
8364 let registration_released = match wait_for_registration_release(
8365 registry,
8366 &spec.module_id,
8367 REGISTRY_RELEASE_TIMEOUT,
8368 )
8369 .await
8370 {
8371 Ok(()) => true,
8372 Err(err) => {
8373 warn!(module_id = %spec.module_id, error = %err, "registration still active after clean exit");
8374 false
8375 }
8376 };
8377 NextAction::Stop {
8378 registration_released,
8379 }
8380 }
8381 ExitKind::Clean | ExitKind::Crash => {
8382 if unrequested_clean_exit_of_protocol_none {
8383 warn!(
8384 module_id = %spec.module_id,
8385 exit_code = ?exit_report.code,
8386 exit_signal = ?exit_report.signal,
8387 "protocol-none module exited cleanly without a stop request; handling it as a crash"
8388 );
8389 } else {
8390 warn!(
8391 module_id = %spec.module_id,
8392 exit_code = ?exit_report.code,
8393 exit_signal = ?exit_report.signal,
8394 "supervised module exited abnormally (crash)"
8395 );
8396 }
8397 let mut restart_schedule = None;
8398 let mut disposition = TerminalDisposition::Disabled;
8399 let mut final_state = ModuleState::Disabled;
8400 let mut disposition_detail = lock_snapshot(snapshot)
8404 .ok()
8405 .and_then(|mut state| state.spawn_failure.take());
8406 let now = Instant::now();
8407 if let Err(err) = update_snapshot(snapshot, Some(&spec.module_id), |state| {
8408 if state.enabled {
8409 if let Some(schedule) = state.next_crash_restart(&policy, now) {
8410 state.respawn_pending = true;
8416 final_state = ModuleState::Restarting;
8417 restart_schedule = Some(schedule);
8418 disposition = TerminalDisposition::Restarting;
8419 } else {
8420 final_state = ModuleState::Failed;
8421 disposition = TerminalDisposition::Failed;
8422 let budget = policy.budget_exhausted_detail();
8423 disposition_detail =
8424 Some(disposition_detail.take().map_or_else(
8425 || budget.clone(),
8426 |cause| format!("{cause}; {budget}"),
8427 ));
8428 }
8429 } else {
8430 disposition = TerminalDisposition::Disabled;
8431 }
8432 }) {
8433 error!(module_id = %spec.module_id, error = %err, "failed to record crashed module exit");
8434 return NextAction::Stop {
8435 registration_released: false,
8436 };
8437 }
8438 if disposition == TerminalDisposition::Failed {
8439 error!(
8444 module_id = %spec.module_id,
8445 max_restarts = policy.max_restarts,
8446 window_secs = policy.window.as_secs(),
8447 "module stopped: {}",
8448 policy.budget_exhausted_detail()
8449 );
8450 }
8451 if let Err(err) = publish_after_terminal_record(
8452 &spec.module_id,
8453 || {
8454 record_terminal_with_detail(
8455 &spec.module_id,
8456 terminal_ring,
8457 spawn_events,
8458 &exit_report,
8459 disposition,
8460 disposition_detail,
8461 )
8462 },
8463 || {
8464 update_snapshot(snapshot, Some(&spec.module_id), |state| {
8465 state.state = final_state;
8466 clear_current_process_facts(state);
8467 if final_state == ModuleState::Failed
8468 && daemon_initiated
8469 && !roster.is_closed()
8470 {
8471 state.failed_retry.schedule(&spec.module_id);
8472 }
8473 state.last_exit = Some(exit_report.clone());
8474 })
8475 },
8476 ) {
8477 error!(module_id = %spec.module_id, error = %err, "failed to publish crashed module exit");
8478 }
8479
8480 if let Some(schedule) = restart_schedule {
8481 NextAction::Restart {
8482 schedule: Some(schedule),
8483 }
8484 } else {
8485 let registration_released = match wait_for_registration_release(
8486 registry,
8487 &spec.module_id,
8488 REGISTRY_RELEASE_TIMEOUT,
8489 )
8490 .await
8491 {
8492 Ok(()) => true,
8493 Err(err) => {
8494 warn!(module_id = %spec.module_id, error = %err, "registration still active after failed module");
8495 false
8496 }
8497 };
8498 NextAction::Stop {
8499 registration_released,
8500 }
8501 }
8502 }
8503 ExitKind::DeliberateSeverance => {
8504 warn!(
8505 module_id = %spec.module_id,
8506 exit_code = ?exit_report.code,
8507 exit_signal = ?exit_report.signal,
8508 "supervised module exited after deliberate connection severance"
8509 );
8510 let should_restart = match lock_snapshot(snapshot) {
8511 Ok(state) => state.enabled,
8512 Err(err) => {
8513 error!(module_id = %spec.module_id, error = %err, "failed to read deliberately severed module state");
8514 return NextAction::Stop {
8515 registration_released: false,
8516 };
8517 }
8518 };
8519 let final_state = if should_restart {
8520 ModuleState::Restarting
8521 } else {
8522 ModuleState::Disabled
8523 };
8524 if let Err(err) = publish_after_terminal_record(
8525 &spec.module_id,
8526 || {
8527 record_terminal(
8528 &spec.module_id,
8529 terminal_ring,
8530 spawn_events,
8531 &exit_report,
8532 terminal_disposition(final_state),
8533 )
8534 },
8535 || {
8536 update_snapshot(snapshot, Some(&spec.module_id), |state| {
8537 state.state = final_state;
8538 clear_current_process_facts(state);
8539 state.last_exit = Some(exit_report.clone());
8540 state.lifetime_restarts += 1;
8541 })
8542 },
8543 ) {
8544 error!(module_id = %spec.module_id, error = %err, "failed to publish deliberately severed module exit");
8545 return NextAction::Stop {
8546 registration_released: false,
8547 };
8548 }
8549
8550 if should_restart {
8551 NextAction::Restart { schedule: None }
8552 } else {
8553 let registration_released = match wait_for_registration_release(
8554 registry,
8555 &spec.module_id,
8556 REGISTRY_RELEASE_TIMEOUT,
8557 )
8558 .await
8559 {
8560 Ok(()) => true,
8561 Err(err) => {
8562 warn!(module_id = %spec.module_id, error = %err, "registration still active after deliberately severed module exit");
8563 false
8564 }
8565 };
8566 NextAction::Stop {
8567 registration_released,
8568 }
8569 }
8570 }
8571 }
8572}
8573
8574async fn on_child_exit_during_daemon_shutdown(
8575 spec: &ModuleSpec,
8576 registry: &Registry,
8577 snapshot: &SharedSnapshot,
8578 terminal_ring: &Arc<Mutex<TerminalRing>>,
8579 spawn_events: &SpawnEventFeed,
8580 exit_report: ExitReport,
8581) -> NextAction {
8582 info!(
8583 module_id = %spec.module_id,
8584 exit_code = ?exit_report.code,
8585 exit_signal = ?exit_report.signal,
8586 exit_kind = ?exit_report.kind,
8587 "supervised module exited during daemon shutdown; not restarting it"
8588 );
8589 if let Err(err) = publish_after_terminal_record(
8590 &spec.module_id,
8591 || {
8592 record_terminal(
8593 &spec.module_id,
8594 terminal_ring,
8595 spawn_events,
8596 &exit_report,
8597 TerminalDisposition::DaemonShutdown,
8598 )
8599 },
8600 || {
8601 update_snapshot(snapshot, Some(&spec.module_id), |state| {
8602 state.state = ModuleState::Stopped;
8603 clear_current_process_facts(state);
8604 state.last_exit = Some(exit_report.clone());
8605 })
8606 },
8607 ) {
8608 error!(module_id = %spec.module_id, error = %err, "failed to publish module exit during daemon shutdown");
8609 }
8610 let registration_released =
8611 wait_for_registration_release(registry, &spec.module_id, REGISTRY_RELEASE_TIMEOUT)
8612 .await
8613 .is_ok();
8614 NextAction::Stop {
8615 registration_released,
8616 }
8617}
8618
8619fn publish_after_terminal_record<T>(
8624 module_id: &str,
8625 record: impl FnOnce(),
8626 publish: impl FnOnce() -> T,
8627) -> T {
8628 let recorded = std::panic::catch_unwind(std::panic::AssertUnwindSafe(record));
8629 let published = publish();
8630 if let Err(panic) = recorded {
8631 error!(
8632 module_id,
8633 "terminal recording panicked; published final state before resuming unwind"
8634 );
8635 std::panic::resume_unwind(panic);
8636 }
8637 published
8638}
8639
8640fn fail_after_wait_error(
8641 module_id: &str,
8642 snapshot: &SharedSnapshot,
8643 terminal_ring: &Arc<Mutex<TerminalRing>>,
8644 spawn_events: &SpawnEventFeed,
8645 detail: String,
8646 failure: &FailureContext,
8647) {
8648 let report = wait_error_exit_report();
8649 publish_after_terminal_record(
8650 module_id,
8651 || {
8652 record_terminal_with_detail(
8653 module_id,
8654 terminal_ring,
8655 spawn_events,
8656 &report,
8657 TerminalDisposition::Failed,
8658 Some(detail),
8659 )
8660 },
8661 || fail_snapshot(snapshot, Some(module_id), Some(report.clone()), failure),
8662 );
8663}
8664
8665#[cfg(test)]
8666fn record_wait_error_terminal(
8667 module_id: &str,
8668 terminal_ring: &Arc<Mutex<TerminalRing>>,
8669 spawn_events: &SpawnEventFeed,
8670) {
8671 record_terminal(
8672 module_id,
8673 terminal_ring,
8674 spawn_events,
8675 &wait_error_exit_report(),
8676 TerminalDisposition::Failed,
8677 );
8678}
8679
8680fn record_terminal(
8681 module_id: &str,
8682 terminal_ring: &Arc<Mutex<TerminalRing>>,
8683 spawn_events: &SpawnEventFeed,
8684 exit_report: &ExitReport,
8685 disposition: TerminalDisposition,
8686) {
8687 record_terminal_with_detail(
8688 module_id,
8689 terminal_ring,
8690 spawn_events,
8691 exit_report,
8692 disposition,
8693 None,
8694 );
8695}
8696
8697fn durable_terminal_history_of(
8701 terminal_ring: &Mutex<TerminalRing>,
8702 module_id: &str,
8703) -> subc_control::TerminalHistory {
8704 let read = terminal_ring
8705 .lock()
8706 .unwrap_or_else(|p| p.into_inner())
8707 .capture_durable_history();
8708 read.read(module_id)
8709}
8710
8711fn record_terminal_with_detail(
8712 module_id: &str,
8713 terminal_ring: &Arc<Mutex<TerminalRing>>,
8714 spawn_events: &SpawnEventFeed,
8715 exit_report: &ExitReport,
8716 disposition: TerminalDisposition,
8717 disposition_detail: Option<String>,
8718) {
8719 #[cfg(test)]
8720 terminal_history_tests::before_terminal_record(spawn_events);
8721 spawn_events.emit_exited(module_id, exit_report.code, exit_report.signal);
8722 let record = TerminalRecord {
8723 exit_code: exit_report.code,
8724 exit_signal: exit_report.signal,
8725 at_ms: exit_report.at_ms,
8726 disposition,
8727 exit_kind: exit_report.kind.into(),
8728 disposition_detail,
8729 };
8730 terminal_ring
8731 .lock()
8732 .unwrap_or_else(|poisoned| poisoned.into_inner())
8733 .record_exit(module_id, record);
8734}
8735
8736fn untrack_if_registration_released(
8737 process_liveness: &SupervisorProcessLiveness,
8738 registry: &Registry,
8739 module_id: &str,
8740 snapshot: &SharedSnapshot,
8741) {
8742 match registry.get_module(module_id) {
8743 Ok(None) => process_liveness.untrack_if_current(module_id, snapshot),
8744 Ok(Some(_)) => {}
8745 Err(err) => {
8746 warn!(module_id, error = %err, "could not determine whether supervisor liveness can be untracked");
8747 }
8748 }
8749}
8750
8751#[cfg(test)]
8765fn apply_wire_spawn_args(
8766 command: &mut Command,
8767 spec: &ModuleSpec,
8768 connection_file_path: Option<&std::path::Path>,
8769 handle: Option<&SupervisorHandle>,
8770) -> Result<Option<NonceHandoff>, SuperviseError> {
8771 apply_wire_spawn_args_for_role(
8772 command,
8773 spec,
8774 connection_file_path,
8775 handle,
8776 SpawnRole::Plain,
8777 )
8778}
8779
8780#[cfg(unix)]
8785type NonceHandoff = subc_os::LaunchNonceHandoff;
8786#[cfg(windows)]
8787type NonceHandoff = subc_os::LaunchNoncePipeHandoff;
8788#[cfg(not(any(unix, windows)))]
8789type NonceHandoff = std::convert::Infallible;
8790
8791fn apply_wire_spawn_args_for_role(
8812 command: &mut Command,
8813 spec: &ModuleSpec,
8814 connection_file_path: Option<&std::path::Path>,
8815 handle: Option<&SupervisorHandle>,
8816 role: SpawnRole,
8817) -> Result<Option<NonceHandoff>, SuperviseError> {
8818 command.env(SUBC_MODULE_ID_ENV, &spec.module_id);
8819 command.env_remove(subc_os::LAUNCH_NONCE_FD_ENV);
8824 #[cfg(windows)]
8825 {
8826 command.env_remove(subc_os::LAUNCH_NONCE_PIPE_ENV);
8827 command.env_remove(subc_os::LAUNCH_NONCE_PIPE_FALLBACK_ENV);
8828 }
8829 command.env_remove(SUBC_LAUNCH_NONCE_ENV);
8831 if spec.protocol == ModuleProtocol::None {
8832 return Ok(None);
8833 }
8834 if let Some(connection_file_path) = connection_file_path {
8835 command.arg(SUBC_ARG).arg(connection_file_path);
8836 }
8837
8838 let nonce = generate_launch_nonce()?;
8842 if let Some(handle) = handle {
8843 match role {
8844 SpawnRole::Plain => {
8845 handle.set_spawn_nonce(&spec.module_id, nonce.clone());
8846 if spec.reserved {
8847 handle.set_reserved_nonce(&spec.module_id, nonce.clone());
8848 }
8849 }
8850 SpawnRole::SwapCandidate => handle.open_swap(&spec.module_id, nonce.clone()),
8851 }
8852 }
8853 #[cfg(unix)]
8854 let handoff = {
8855 let handoff =
8856 subc_os::LaunchNonceHandoff::new(&nonce).map_err(|source| SuperviseError::Spawn {
8857 program: spec.program.clone(),
8858 source,
8859 cgroup_path: None,
8860 })?;
8861 command.env(subc_os::LAUNCH_NONCE_FD_ENV, handoff.fd_env_value());
8862 Some(handoff)
8863 };
8864 #[cfg(windows)]
8865 let handoff = prepare_windows_nonce_pipe(command, spec, &nonce);
8866 #[cfg(not(any(unix, windows)))]
8867 let handoff = None;
8868 #[cfg(not(unix))]
8872 command.env(SUBC_LAUNCH_NONCE_ENV, nonce);
8873 Ok(handoff)
8874}
8875
8876fn apply_child_env(command: &mut Command, spec: &ModuleSpec) {
8877 command.env_remove(CK_LOG_ENV);
8878 command.env_remove(SUBC_SPAWN_ROLE_ENV);
8885 for (key, value) in &spec.env {
8886 if matches!(
8890 key.as_str(),
8891 CAPTURE_MAX_FILE_MB_ENV | CAPTURE_KEEP_ENV | CAPTURE_MAX_AGE_DAYS_ENV
8892 ) || key == SUBC_SPAWN_ROLE_ENV
8893 {
8894 continue;
8895 }
8896 command.env(key, value);
8897 }
8898}
8899
8900#[cfg(windows)]
8901fn prepare_windows_nonce_pipe(
8902 command: &mut Command,
8903 spec: &ModuleSpec,
8904 nonce: &str,
8905) -> Option<NonceHandoff> {
8906 match subc_os::LaunchNoncePipeHandoff::new(nonce) {
8907 Ok(handoff) => {
8908 command.env(subc_os::LAUNCH_NONCE_PIPE_ENV, handoff.name());
8909 command.env(subc_os::LAUNCH_NONCE_PIPE_FALLBACK_ENV, "env");
8914 Some(handoff)
8915 }
8916 Err(error) => {
8917 warn!(module_id = %spec.module_id, %error, "could not prepare launch nonce pipe; retaining the environment handoff");
8918 None
8919 }
8920 }
8921}
8922
8923#[cfg(windows)]
8924fn start_windows_nonce_pipe(
8925 handoff: Option<NonceHandoff>,
8926 pid: u32,
8927 spec: &ModuleSpec,
8928) -> Option<subc_os::LaunchNoncePipeDelivery> {
8929 handoff.and_then(|handoff| match handoff.serve(pid, REGISTRY_RELEASE_TIMEOUT) {
8933 Ok(delivery) => Some(delivery),
8934 Err(error) => {
8935 warn!(module_id = %spec.module_id, %error, "could not serve launch nonce pipe; retaining the environment handoff");
8936 None
8937 }
8938 })
8939}
8940
8941fn observed_launch_nonce_source(snapshot: &SupervisorSnapshot) -> Option<String> {
8942 #[cfg(windows)]
8943 if snapshot.process_alive && snapshot.spawned_protocol == Some(ModuleProtocol::Subc) {
8944 return Some(
8945 if snapshot
8946 .pipe_consumed
8947 .as_ref()
8948 .is_some_and(|consumed| consumed.get().is_some())
8949 {
8950 "pipe"
8951 } else {
8952 "env"
8953 }
8954 .to_owned(),
8955 );
8956 }
8957 let _ = snapshot;
8958 None
8959}
8960
8961#[derive(Debug, Clone, Copy, PartialEq, Eq)]
8964enum SpawnRole {
8965 Plain,
8966 SwapCandidate,
8967}
8968
8969fn apply_spawn_role(command: &mut Command, role: SpawnRole) {
8972 if role == SpawnRole::SwapCandidate {
8973 command.env(SUBC_SPAWN_ROLE_ENV, SPAWN_ROLE_SWAP_CANDIDATE);
8974 }
8975}
8976
8977fn previous_instance_error(
8978 spec: &ModuleSpec,
8979 pid: u32,
8980 reason: crate::live_children::RetainedReason,
8981) -> SuperviseError {
8982 SuperviseError::Spawn {
8983 program: spec.program.clone(),
8984 source: io::Error::new(io::ErrorKind::WouldBlock, reason.waiting_message(pid)),
8985 cgroup_path: None,
8986 }
8987}
8988
8989fn carries_containment(configured: bool, has_container: bool) -> bool {
8990 configured || has_container
8991}
8992
8993fn recovery_block_error(spec: &ModuleSpec, roster: &ChildRoster) -> Option<SuperviseError> {
8994 if !carries_containment(
8995 spec.carry_containment,
8996 containment::current(&roster.containment).is_some(),
8997 ) {
8998 return None;
8999 }
9000 Some(SuperviseError::Spawn {
9001 program: spec.program.clone(),
9002 source: io::Error::new(io::ErrorKind::WouldBlock, roster.recovery_block()?),
9003 cgroup_path: None,
9004 })
9005}
9006
9007fn spawn_child(
9008 spec: &ModuleSpec,
9009 connection_file_path: Option<&std::path::Path>,
9010 handle: Option<&SupervisorHandle>,
9011 ring: &Arc<Mutex<StderrRing>>,
9012 capture_logs_dir: Option<&std::path::Path>,
9013 roster: &ChildRoster,
9014 #[cfg(target_os = "linux")] cgroup_placement: Option<&subc_cgroup::Placement>,
9015) -> Result<SupervisedChild, SuperviseError> {
9016 #[cfg(all(test, unix))]
9017 if let Some(source) = containment_tests::join_failure(spec, roster) {
9018 containment::terminate_blocking(&roster.containment);
9019 return Err(SuperviseError::Spawn {
9020 program: spec.program.clone(),
9021 source,
9022 cgroup_path: None,
9023 });
9024 }
9025 let result = spawn_child_in_slot(
9026 spec,
9027 connection_file_path,
9028 handle,
9029 ring,
9030 capture_logs_dir,
9031 roster,
9032 #[cfg(target_os = "linux")]
9033 cgroup_placement,
9034 SpawnRole::Plain,
9035 false,
9036 );
9037 if result.is_err() {
9038 if roster.admission_pending(&spec.module_id) {
9039 containment::terminate_deferred(&roster.containment);
9040 } else {
9041 containment::terminate_blocking(&roster.containment);
9042 }
9043 }
9044 result
9045}
9046
9047#[allow(clippy::too_many_arguments)]
9060fn spawn_child_in_slot(
9061 spec: &ModuleSpec,
9062 connection_file_path: Option<&std::path::Path>,
9063 handle: Option<&SupervisorHandle>,
9064 ring: &Arc<Mutex<StderrRing>>,
9065 capture_logs_dir: Option<&std::path::Path>,
9066 roster: &ChildRoster,
9067 #[cfg(target_os = "linux")] cgroup_placement: Option<&subc_cgroup::Placement>,
9068 role: SpawnRole,
9069 alternate_slot: bool,
9070) -> Result<SupervisedChild, SuperviseError> {
9071 if let Some(error) = recovery_block_error(spec, roster) {
9072 return Err(error);
9073 }
9074 if let Some((pid, reason)) = roster.previous_instance(&spec.module_id) {
9075 return Err(previous_instance_error(spec, pid, reason));
9076 }
9077 if roster.is_closed() {
9078 return Err(SuperviseError::Spawn {
9079 program: spec.program.clone(),
9080 source: io::Error::other("the daemon is shutting down; not starting a new process"),
9081 cgroup_path: None,
9082 });
9083 }
9084 let carried = containment::current(&roster.containment);
9085 let carrying = carries_containment(spec.carry_containment, carried.is_some());
9086 if role != SpawnRole::Plain && carrying {
9087 return Err(SuperviseError::InvalidSpec {
9088 reason: "a carried container cannot overlap a swap candidate".to_owned(),
9089 });
9090 }
9091 #[cfg(unix)]
9092 let launch_identity = if carrying && carried.is_none() {
9093 Some(
9094 containment::LaunchIdentity::capture().map_err(|source| SuperviseError::Spawn {
9095 program: spec.program.clone(),
9096 source,
9097 cgroup_path: None,
9098 })?,
9099 )
9100 } else {
9101 None
9102 };
9103 #[cfg(target_os = "linux")]
9104 let cgroup_placement = match &carried {
9105 Some(container) => container.cgroup().map(|(placement, _)| placement),
9106 None => cgroup_placement,
9107 };
9108 #[cfg(unix)]
9109 let group_target = if let Some(container) = carried
9110 .as_ref()
9111 .filter(|container| container.group().is_some())
9112 {
9113 containment::GroupTarget::Pinned(Arc::clone(container))
9114 } else {
9115 #[cfg(target_os = "linux")]
9116 let cgroup_container = carrying && cgroup_placement.is_some();
9117 #[cfg(not(target_os = "linux"))]
9118 let cgroup_container = false;
9119 if cgroup_container {
9120 containment::GroupTarget::NoGroup
9121 } else {
9122 containment::GroupTarget::OwnLeader
9123 }
9124 };
9125 #[cfg(target_os = "linux")]
9126 let cgroup_name = {
9127 if let Some((_, name)) = carried.as_ref().and_then(|container| container.cgroup()) {
9132 name.to_owned()
9133 } else if cgroup_placement.is_none() {
9134 swap::cgroup_name(&spec.module_id, alternate_slot)
9135 } else {
9136 let nonce = generate_launch_nonce()?;
9137 let suffix = u128::from_str_radix(&nonce[..32], 16).expect("hex launch nonce");
9138 let mut end = spec.module_id.len().min(64);
9141 while !spec.module_id.is_char_boundary(end) {
9142 end -= 1;
9143 }
9144 format!(
9145 "{}_{suffix}",
9146 swap::cgroup_name(&spec.module_id[..end], alternate_slot)
9147 )
9148 }
9149 };
9150 #[cfg(not(target_os = "linux"))]
9151 let _ = alternate_slot;
9152 #[cfg(target_os = "macos")]
9153 let (mut command, privacy_exec, exec_ack) = privacy_command(spec, roster)?;
9154 #[cfg(windows)]
9155 let image_capture = windows_spawn_capture(spec).map_err(|source| SuperviseError::Spawn {
9156 program: spec.program.clone(),
9157 source,
9158 cgroup_path: None,
9159 })?;
9160 #[cfg(windows)]
9161 let mut command = Command::new(image_capture.path());
9162 #[cfg(not(any(target_os = "macos", windows)))]
9163 let mut command = Command::new(&spec.program);
9164 command.args(&spec.args);
9165 apply_child_env(&mut command, spec);
9195 apply_spawn_role(&mut command, role);
9196 let nonce_handoff =
9197 apply_wire_spawn_args_for_role(&mut command, spec, connection_file_path, handle, role)?;
9198
9199 #[cfg(target_os = "linux")]
9200 let cgroup_path = cgroup_placement
9201 .map(|placement| {
9202 if let Some(container) = &carried {
9203 let path = container
9204 .cgroup_path()
9205 .ok_or_else(|| io::Error::other("carried container has no cgroup path"))?;
9206 placement.existing_module_path(path)?.ok_or_else(|| {
9207 io::Error::new(
9208 io::ErrorKind::NotFound,
9209 "carried cgroup disappeared before replacement",
9210 )
9211 })
9212 } else {
9213 if carrying {
9214 containment::prepare_cgroup_path(placement, &cgroup_name)
9215 } else {
9216 placement.module_path(&cgroup_name)
9217 }
9218 }
9219 })
9220 .transpose()
9221 .map_err(|source| SuperviseError::Cgroup {
9222 module_id: spec.module_id.clone(),
9223 source,
9224 })?;
9225 #[cfg(not(target_os = "linux"))]
9226 let cgroup_path: Option<PathBuf> = None;
9227 #[cfg(unix)]
9228 let mut admission_gate = if carrying {
9229 Some(
9230 admission::Gate::install(&mut command, spec, roster).map_err(|source| {
9231 SuperviseError::Spawn {
9232 program: spec.program.clone(),
9233 source,
9234 cgroup_path: cgroup_path.clone(),
9235 }
9236 })?,
9237 )
9238 } else {
9239 None
9240 };
9241 #[cfg(target_os = "macos")]
9242 exec_ack.install(command.as_std_mut());
9243 #[cfg(target_os = "linux")]
9244 if let Some(path) = &cgroup_path {
9245 if let Err(error) = apply_cgroup_placement(&mut command, spec, path) {
9246 if let Some(placement) = cgroup_placement.filter(|_| carried.is_none()) {
9247 remove_module_cgroup(placement, &cgroup_name);
9248 }
9249 return Err(error);
9250 }
9251 }
9252
9253 let output_sink = if let Some(logs_dir) = capture_logs_dir {
9254 let path = logs_dir.join(format!("{}.stderr.log", spec.module_id));
9255 match ChildOutputSink::open(&path, capture_retention(spec)) {
9256 Ok(sink) => sink,
9257 Err(error) => {
9258 warn!(
9259 module_id = %spec.module_id,
9260 path = %path.display(),
9261 error = %error,
9262 "could not open child output capture file; forwarding to stderr"
9263 );
9264 ChildOutputSink::Stderr
9265 }
9266 }
9267 } else {
9268 ChildOutputSink::Stderr
9269 };
9270
9271 command.stdout(Stdio::piped());
9272 command.stderr(Stdio::piped());
9273 command.kill_on_drop(!carrying);
9274 #[cfg(unix)]
9291 command.process_group(
9292 carried
9293 .as_ref()
9294 .map(|container| container.verified_join_group())
9295 .transpose()
9296 .map_err(|source| SuperviseError::Spawn {
9297 program: spec.program.clone(),
9298 source,
9299 cgroup_path: cgroup_path.clone(),
9300 })?
9301 .flatten()
9302 .unwrap_or(0) as i32,
9303 );
9304 command.stdin(Stdio::null());
9305 #[cfg(unix)]
9306 if let Some(gate) = &mut admission_gate {
9307 gate.install_placement(&mut command);
9308 }
9309 #[cfg(unix)]
9313 if let Some(handoff) = nonce_handoff {
9314 handoff.install_last(command.as_std_mut());
9315 }
9316 #[cfg(not(any(unix, windows)))]
9317 let _ = nonce_handoff;
9318
9319 #[cfg(windows)]
9324 {
9325 subc_jobobject::ensure_root_job().map_err(|source| SuperviseError::Spawn {
9326 program: spec.program.clone(),
9327 source,
9328 cgroup_path: None,
9329 })?;
9330 subc_jobobject::suspend_on_create_async(&mut command);
9331 }
9332 let spawned_at = Instant::now();
9336 #[cfg(all(test, unix))]
9337 containment_tests::record_spawn_attempt(spec, &mut command);
9338 #[cfg(unix)]
9339 let spawned = match admission_gate {
9340 Some(gate) => gate
9341 .spawn(
9342 command,
9343 spec,
9344 roster,
9345 launch_identity,
9346 #[cfg(target_os = "linux")]
9347 cgroup_placement
9348 .zip(cgroup_path.as_ref())
9349 .map(|(placement, path)| {
9350 (placement.clone(), cgroup_name.clone(), path.clone())
9351 }),
9352 )
9353 .map(|(child, admitted)| (child, Some(admitted))),
9354 None => command.spawn().map(|child| (child, None)),
9355 };
9356 #[cfg(not(unix))]
9357 let spawned = command.spawn();
9358 let spawned = match spawned {
9359 Ok(child) => child,
9360 Err(source) => {
9361 if carried
9362 .as_ref()
9363 .and_then(|container| container.group())
9364 .is_some()
9365 {
9366 containment::terminate_blocking(&roster.containment);
9367 }
9368 #[cfg(target_os = "linux")]
9369 if let Some(placement) = cgroup_placement.filter(|_| carried.is_none()) {
9370 remove_module_cgroup(placement, &cgroup_name);
9371 }
9372 return Err(SuperviseError::Spawn {
9373 program: spec.program.clone(),
9374 source,
9375 cgroup_path,
9376 });
9377 }
9378 };
9379 #[cfg(unix)]
9380 let (mut child, mut admitted) = spawned;
9381 #[cfg(unix)]
9382 let group_target = admitted
9383 .as_ref()
9384 .map(|attempt| attempt.group_target())
9385 .unwrap_or(group_target);
9386 #[cfg(not(unix))]
9387 let mut child = spawned;
9388 #[cfg(test)]
9389 spawn_record_tests::after_create(&child, spec);
9390 #[cfg(target_os = "macos")]
9395 drop(exec_ack);
9396
9397 #[cfg(windows)]
9402 let windows_image = match image_capture.bind_tokio(&child) {
9403 Ok(image) => Arc::new(image),
9404 Err(source) => {
9405 kill_and_reap_rejected_child(&mut child);
9406 return Err(SuperviseError::Spawn {
9407 program: spec.program.clone(),
9408 source,
9409 cgroup_path: None,
9410 });
9411 }
9412 };
9413 #[cfg(windows)]
9416 let job = contain_spawned_child(&mut child, spec, carried.as_ref())?;
9417 let spawned_at_ms = unix_ms_now();
9418 #[cfg(not(windows))]
9419 let spawned_from = spec.program.clone();
9420 #[cfg(windows)]
9421 let spawned_from = windows_image.path().to_path_buf();
9422 #[cfg(not(windows))]
9423 let spawned_file_identity = spawned_file_identity(&spawned_from);
9424 #[cfg(windows)]
9425 let spawned_file_identity = Some(SpawnedFileIdentity {
9426 device: 0,
9427 inode: 0,
9428 windows_image: Some(Arc::clone(&windows_image)),
9429 });
9430 let pid = child.id().ok_or_else(|| SuperviseError::Spawn {
9431 program: spec.program.clone(),
9432 source: io::Error::other("spawned child exposed no live pid"),
9433 cgroup_path: cgroup_path.clone(),
9434 })?;
9435 #[cfg(windows)]
9436 let nonce_delivery = start_windows_nonce_pipe(nonce_handoff, pid, spec);
9437 #[cfg(not(windows))]
9438 let process_start_time = admitted
9439 .as_ref()
9440 .map(|admitted| admitted.birth)
9441 .or_else(|| crate::provenance::process_start_time(pid));
9442 #[cfg(windows)]
9443 let process_start_time = Some(windows_image.start_time());
9444 let process_identity = process_start_time.map(|start_time| ProcessIdentity { pid, start_time });
9445 #[cfg(all(test, target_os = "macos"))]
9446 privacy_exec_boundary_tests::before_image_sample(spec, pid);
9447 #[cfg(not(windows))]
9452 let recorded_image = observe_spawned_image(pid);
9453 #[cfg(target_os = "macos")]
9457 let recorded_image = if privacy_exec.is_some() {
9458 None
9459 } else {
9460 recorded_image
9461 };
9462 #[cfg(target_os = "linux")]
9463 let recorded_cgroup_name = cgroup_path.as_ref().map(|_| cgroup_name.clone());
9464 #[cfg(not(target_os = "linux"))]
9465 let recorded_cgroup_name = None;
9466 #[cfg(unix)]
9467 let preadmitted = admitted
9468 .as_mut()
9469 .and_then(|admitted| admitted.leader.take());
9470 #[cfg(not(unix))]
9471 let preadmitted: Option<crate::child_roster::RosterGuard> = None;
9472 let roster_guard = if let Some(guard) = preadmitted {
9473 Ok(guard)
9474 } else {
9475 roster.admit(
9476 spec.module_id.clone(),
9477 pid,
9478 spec.protocol,
9479 process_start_time,
9480 crate::child_roster::RecordedIdentity {
9481 container: None,
9482 #[cfg(unix)]
9483 carried: None,
9484 #[cfg(not(windows))]
9485 start_time: recorded_image.map(|image| image.start_time),
9486 #[cfg(windows)]
9487 start_time: process_start_time,
9488 #[cfg(not(windows))]
9489 executable: recorded_image
9490 .and_then(|image| image.executable)
9491 .map(crate::live_children::ExecutableIdentity::from),
9492 #[cfg(windows)]
9493 executable: Some(crate::live_children::ExecutableIdentity::from(
9494 windows_image.identity(),
9495 )),
9496 cgroup_name: recorded_cgroup_name,
9497 #[cfg(target_os = "linux")]
9498 cgroup_placement: cgroup_placement.cloned(),
9499 },
9500 )
9501 };
9502 let roster_guard = match roster_guard {
9503 Ok(guard) => guard,
9504 Err(failure) => {
9505 return failure.with_cleanup(|source| {
9506 #[cfg(windows)]
9507 if let Some(job) = &job {
9508 let _ = job.lock().unwrap_or_else(|p| p.into_inner()).terminate();
9509 }
9510 #[cfg(target_os = "linux")]
9511 kill_module_cgroup(cgroup_placement, &cgroup_name);
9512 #[cfg(unix)]
9513 let rejected_group = group_target.clone();
9514 kill_and_reap_rejected_child(
9515 &mut child,
9516 #[cfg(unix)]
9517 rejected_group,
9518 );
9519 #[cfg(target_os = "linux")]
9520 if let Some(placement) = cgroup_placement.filter(|_| carried.is_none()) {
9521 remove_module_cgroup(placement, &cgroup_name);
9522 }
9523 Err(SuperviseError::Spawn {
9524 program: spec.program.clone(),
9525 source: io::Error::other(format!(
9526 "could not durably record module child: {source}"
9527 )),
9528 cgroup_path,
9529 })
9530 });
9531 }
9532 };
9533 #[cfg(unix)]
9534 if admitted.is_some() {
9535 if let Some(image) = recorded_image {
9536 if let Err(source) = roster_guard.confirm_executable(image) {
9537 kill_and_reap_rejected_child(&mut child, group_target.clone());
9538 return Err(SuperviseError::Spawn {
9539 program: spec.program.clone(),
9540 source,
9541 cgroup_path,
9542 });
9543 }
9544 }
9545 }
9546 if roster.is_closed() {
9551 if carrying {
9552 #[cfg(unix)]
9553 let rejected_group = admitted
9554 .as_ref()
9555 .map(|admitted| admitted.rejection_target())
9556 .unwrap_or_else(|| group_target.clone());
9557 #[cfg(windows)]
9558 if let Some(job) = &job {
9559 let _ = job.lock().unwrap_or_else(|p| p.into_inner()).terminate();
9560 }
9561 kill_and_reap_rejected_child(
9562 &mut child,
9563 #[cfg(unix)]
9564 rejected_group,
9565 );
9566 #[cfg(all(test, unix))]
9567 let _ =
9568 roster.admission_event_for_test(&spec.module_id, "after_rejected_child", Some(pid));
9569 drop(roster_guard);
9570 return Err(SuperviseError::Spawn {
9571 program: spec.program.clone(),
9572 source: io::Error::other(
9573 "the daemon began shutting down during carried admission; ended the container",
9574 ),
9575 cgroup_path,
9576 });
9577 }
9578 #[cfg(target_os = "linux")]
9580 kill_module_cgroup(cgroup_placement, &cgroup_name);
9581 if let Err(error) = child.start_kill() {
9582 debug!(module_id = %spec.module_id, pid, %error, "kill of a process spawned during daemon shutdown failed; it may already have exited");
9583 }
9584 #[cfg(target_os = "linux")]
9585 if let Some(placement) = cgroup_placement.filter(|_| carried.is_none()) {
9586 while matches!(child.try_wait(), Ok(None)) {
9590 std::thread::yield_now();
9591 }
9592 if matches!(
9593 subc_cgroup::kill_module(Some(placement), &cgroup_name),
9594 subc_cgroup::KillOutcome::Killed
9595 ) {
9596 if let Ok(path) = placement.module_path(&cgroup_name) {
9597 while std::fs::read_to_string(path.join("cgroup.events"))
9598 .ok()
9599 .is_some_and(|events| events.lines().any(|line| line == "populated 1"))
9600 {
9601 std::thread::yield_now();
9602 }
9603 }
9604 }
9605 remove_module_cgroup(placement, &cgroup_name);
9606 }
9607 drop(roster_guard);
9608 return Err(SuperviseError::Spawn {
9609 program: spec.program.clone(),
9610 source: io::Error::other(
9611 "the daemon began shutting down while this process was starting; ended it",
9612 ),
9613 cgroup_path,
9614 });
9615 }
9616
9617 let stdout_pump = match child.stdout.take() {
9618 Some(stdout) => Some(tokio::spawn(pump_stdout_to(stdout, output_sink.clone()))),
9619 None => {
9620 warn!(
9621 module_id = %spec.module_id,
9622 "spawned child exposed no stdout pipe; file capture will be incomplete"
9623 );
9624 None
9625 }
9626 };
9627 let stderr_pump = match child.stderr.take() {
9628 Some(stderr) => {
9629 let generation = ring
9630 .lock()
9631 .unwrap_or_else(|poisoned| poisoned.into_inner())
9632 .begin_process();
9633 Some(StderrPump {
9634 task: tokio::spawn(pump_stderr_to(
9635 stderr,
9636 Arc::clone(ring),
9637 generation,
9638 output_sink,
9639 )),
9640 generation,
9641 })
9642 }
9643 None => {
9644 ring.lock()
9648 .unwrap_or_else(|poisoned| poisoned.into_inner())
9649 .mark_not_captured("stderr pipe was not available on spawn");
9650 warn!(
9651 module_id = %spec.module_id,
9652 "spawned child exposed no stderr pipe; tail will be unavailable"
9653 );
9654 None
9655 }
9656 };
9657
9658 let exec = Arc::new(crate::activity::ExecOutcome::new());
9659 #[cfg(target_os = "macos")]
9660 let pending_exec = privacy_exec.is_some();
9661 #[cfg(not(target_os = "macos"))]
9662 let pending_exec = false;
9663 if !pending_exec {
9664 exec.finish(crate::activity::ExecState::CommittedExec);
9665 }
9666 #[cfg(windows)]
9667 let subtree_target = job
9668 .as_ref()
9669 .map(|job| crate::subtree_resources::Target::Job(Arc::downgrade(job)))
9670 .unwrap_or(crate::subtree_resources::Target::Unavailable("no_job"));
9671 #[cfg(target_os = "linux")]
9672 let subtree_target = cgroup_path
9673 .map(crate::subtree_resources::Target::Cgroup)
9674 .unwrap_or(crate::subtree_resources::Target::Unavailable("no_cgroup"));
9675 #[cfg(not(any(target_os = "linux", windows)))]
9676 let subtree_target = crate::subtree_resources::Target::Unavailable("unsupported_platform");
9677 let mut supervised = SupervisedChild {
9678 subtree_target,
9679 child: containment::ProcessChild::plain(child, pid),
9680 exec: crate::activity::ExecOwner(exec),
9681 #[cfg(windows)]
9682 nonce_delivery,
9683 protocol: spec.protocol,
9684 #[cfg(target_os = "linux")]
9685 module_id: cgroup_name,
9686 #[cfg(target_os = "linux")]
9687 cgroup_placement: cgroup_placement.cloned(),
9688 #[cfg(windows)]
9689 job,
9690 stdout_pump,
9691 stderr_pump,
9692 stderr_ring: Arc::clone(ring),
9693 spawned_at,
9694 registration_deadline: None,
9695 registration_watch: None,
9696 spawned_at_ms,
9697 spawned_from,
9698 launch_spec_digest: LaunchSpecDigest::from_spec(spec),
9699 spawned_file_identity,
9700 process_start_time,
9701 process_identity,
9702 pid,
9703 roster_guard: Some(roster_guard),
9704 #[cfg(target_os = "macos")]
9705 privacy_exec,
9706 #[cfg(target_os = "macos")]
9707 report_ready: Arc::new(OnceLock::new()),
9708 spawn_failure: None,
9709 };
9710 if let Err(error) = containment::adopt(&mut supervised, spec, roster) {
9711 supervised.child.reject(
9712 #[cfg(unix)]
9713 group_target,
9714 );
9715 #[cfg(all(test, unix))]
9716 let _ = roster.admission_event_for_test(&spec.module_id, "after_rejected_child", Some(pid));
9717 return Err(error);
9718 }
9719 #[cfg(unix)]
9720 if let Some(admitted) = admitted.take() {
9721 admitted.disarm();
9722 }
9723 #[cfg(windows)]
9724 if let Err(error) = resume_suspended_child(pid, spec) {
9725 containment::terminate_blocking(&roster.containment);
9726 let _ = supervised.child.start_kill();
9727 return Err(error);
9728 }
9729 Ok(supervised)
9730}
9731
9732#[cfg(test)]
9733mod spawn_record_tests {
9734 use super::*;
9735
9736 thread_local! {
9737 static LAST_PID: std::cell::Cell<Option<u32>> = const { std::cell::Cell::new(None) };
9738 }
9739
9740 pub(super) fn after_create(child: &Child, _spec: &ModuleSpec) {
9741 LAST_PID.set(child.id());
9742 #[cfg(windows)]
9743 if let Some((_, address)) = _spec
9744 .env
9745 .iter()
9746 .find(|(key, _)| key == "SUBC_TEST_AFTER_CREATE")
9747 {
9748 use std::io::{Read, Write};
9749 let mut stream = std::net::TcpStream::connect(address).unwrap();
9750 stream
9751 .write_all(&child.id().unwrap().to_ne_bytes())
9752 .unwrap();
9753 let mut release = [0];
9754 stream.read_exact(&mut release).unwrap();
9755 }
9756 }
9757
9758 #[cfg(windows)]
9759 pub(super) fn assignment_failpoint(spec: &ModuleSpec) -> io::Result<()> {
9760 if spec
9761 .env
9762 .iter()
9763 .any(|(key, _)| key == "SUBC_TEST_ASSIGNMENT_FAILURE")
9764 {
9765 Err(io::Error::other("injected module job assignment failure"))
9766 } else {
9767 Ok(())
9768 }
9769 }
9770
9771 fn spec() -> ModuleSpec {
9772 ModuleSpec {
9773 module_id: "spawn-record-worker".into(),
9774 #[cfg(unix)]
9775 program: "/bin/sleep".into(),
9776 #[cfg(windows)]
9777 program: std::env::current_exe().unwrap(),
9778 #[cfg(unix)]
9779 args: vec!["60".into()],
9780 #[cfg(windows)]
9781 args: vec![
9782 "--exact".into(),
9783 "supervise::spawn_record_tests::windows_store_touch_fixture".into(),
9784 "--nocapture".into(),
9785 ],
9786 env: vec![],
9787 reserved: false,
9788 reserved_prefixes: vec![],
9789 protocol: ModuleProtocol::None,
9790 overlap: Default::default(),
9791 carry_containment: false,
9792 }
9793 }
9794
9795 #[tokio::test]
9796 async fn record_sync_failure_fails_spawn_and_reaps_child() {
9797 let home = cortexkit_test_support::ScratchDir::new("record-sync-failure");
9798 let supervisor =
9799 Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default())
9800 .with_live_children_record(home.join("live-children.json"));
9801 crate::live_children::fail_next_file_sync();
9802 let result = supervisor.spawn(spec());
9803 let failed = result.is_err();
9804 if let Ok(module) = result {
9805 module.stop().await.unwrap();
9806 }
9807 assert!(failed, "record failure must fail spawn");
9808 let pid = LAST_PID.get().expect("the real OS child was created");
9809 assert!(
9810 subc_os::Process::open(pid).unwrap().is_none(),
9811 "child {pid} survived rejected admission or was not reaped"
9812 );
9813 }
9814
9815 #[cfg(target_os = "macos")]
9816 #[test]
9817 #[ignore = "manual on-device latency measurement"]
9818 fn durable_record_write_latency_50() {
9819 let home = cortexkit_test_support::ScratchDir::new("record-latency");
9820 let path = home.join("live-children.json");
9821 let children = vec![crate::live_children::LiveChild {
9822 module_id: "latency-worker".into(),
9823 pid: std::process::id(),
9824 protocol: ModuleProtocol::None,
9825 start_time: Some(1),
9826 executable: None,
9827 cgroup_name: None,
9828 container: None,
9829 }];
9830 let mut samples = Vec::new();
9831 for _ in 0..50 {
9832 let start = std::time::Instant::now();
9833 crate::live_children::write_record(&path, &children).unwrap();
9834 samples.push(start.elapsed());
9835 }
9836 samples.sort();
9837 eprintln!(
9838 "durable_record_write_latency_50: n=50 median={:?} p95={:?}",
9839 (samples[24] + samples[25]) / 2,
9840 samples[47]
9841 );
9842 }
9843
9844 #[cfg(windows)]
9845 #[test]
9846 fn windows_store_touch_fixture() {
9847 if let Some(path) = std::env::var_os("SUBC_TEST_STORE_TOUCH") {
9848 std::fs::write(path, b"module ran").unwrap();
9849 }
9850 }
9851
9852 #[cfg(windows)]
9853 #[tokio::test]
9854 async fn windows_daemon_fixture() {
9855 let Ok(address) = std::env::var("SUBC_TEST_DAEMON_BARRIER") else {
9856 return;
9857 };
9858 let mut spec = spec();
9859 spec.env.push(("SUBC_TEST_AFTER_CREATE".into(), address));
9860 let supervisor =
9861 Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default());
9862 let _module = supervisor.spawn(spec).unwrap();
9863 }
9864
9865 #[cfg(windows)]
9866 #[test]
9867 fn windows_daemon_death_before_module_job_assignment_leaves_no_child() {
9868 use std::io::Read;
9869 let listener = std::net::TcpListener::bind("127.0.0.1:0").unwrap();
9870 let mut command = std::process::Command::new(std::env::current_exe().unwrap());
9873 command
9874 .args([
9875 "--exact",
9876 "supervise::spawn_record_tests::windows_daemon_fixture",
9877 "--nocapture",
9878 ])
9879 .env(
9880 "SUBC_TEST_DAEMON_BARRIER",
9881 listener.local_addr().unwrap().to_string(),
9882 );
9883 let mut daemon = subc_jobobject::spawn_contained(&mut command).unwrap();
9884 listener.set_nonblocking(true).unwrap();
9885 let deadline = std::time::Instant::now() + Duration::from_secs(30);
9886 let mut stream = loop {
9887 if let Ok((stream, _)) = listener.accept() {
9888 break stream;
9889 }
9890 assert!(
9891 daemon.child.try_wait().unwrap().is_none(),
9892 "daemon failed before rendezvous"
9893 );
9894 assert!(
9895 std::time::Instant::now() < deadline,
9896 "daemon did not reach CreateProcess failpoint"
9897 );
9898 std::thread::yield_now();
9899 };
9900 stream
9901 .set_read_timeout(Some(Duration::from_secs(30)))
9902 .unwrap();
9903 let mut pid = [0; 4];
9904 stream.read_exact(&mut pid).unwrap();
9905 let pid = u32::from_ne_bytes(pid);
9906 assert!(subc_jobobject::process_exists(pid));
9907 daemon.child.kill().unwrap();
9910 daemon.child.wait().unwrap();
9911 let exited = subc_jobobject::wait_for_process_exit(pid, Duration::from_secs(10));
9912 if !exited {
9913 daemon.job.terminate().unwrap();
9914 }
9915 assert!(exited, "suspended child {pid} outlived hard-killed daemon");
9916 }
9917
9918 #[cfg(windows)]
9919 #[tokio::test]
9920 async fn windows_containment_failure_never_resumes() {
9921 let home = cortexkit_test_support::ScratchDir::new("assignment-refusal");
9922 let sentinel = home.join("store-touched");
9923 let mut spec = spec();
9924 spec.env
9925 .push(("SUBC_TEST_ASSIGNMENT_FAILURE".into(), "1".into()));
9926 spec.env.push((
9927 "SUBC_TEST_STORE_TOUCH".into(),
9928 sentinel.display().to_string(),
9929 ));
9930 let supervisor =
9931 Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default());
9932 let result = supervisor.spawn(spec);
9933 if let Ok(module) = result {
9936 let _ = tokio::time::timeout(Duration::from_secs(10), async {
9937 while !sentinel.exists() && module.state().unwrap() == ModuleState::Running {
9938 tokio::task::yield_now().await;
9939 }
9940 })
9941 .await;
9942 let _ = module.stop().await;
9943 panic!(
9944 "assignment failure resumed a module; store touched={}",
9945 sentinel.exists()
9946 );
9947 }
9948 assert!(
9949 !sentinel.exists(),
9950 "refused module executed store-touch sentinel"
9951 );
9952 assert!(!subc_jobobject::process_exists(LAST_PID.get().unwrap()));
9953 }
9954
9955 #[cfg(windows)]
9956 #[tokio::test]
9957 async fn windows_nested_daemon_fixture() {
9958 let Some(sentinel) = std::env::var_os("SUBC_TEST_NESTED_STORE") else {
9959 return;
9960 };
9961 let mut spec = spec();
9962 spec.env.push((
9963 "SUBC_TEST_STORE_TOUCH".into(),
9964 sentinel.to_str().unwrap().into(),
9965 ));
9966 let supervisor =
9967 Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default());
9968 let module = supervisor
9969 .spawn(spec)
9970 .expect("per-module job must nest below daemon and host jobs");
9971 tokio::time::timeout(Duration::from_secs(10), async {
9972 while !PathBuf::from(&sentinel).exists() {
9973 tokio::task::yield_now().await;
9974 }
9975 })
9976 .await
9977 .unwrap();
9978 module.stop().await.unwrap();
9979 }
9980
9981 #[cfg(windows)]
9982 #[test]
9983 fn windows_module_job_nests_under_host_job() {
9984 let home = cortexkit_test_support::ScratchDir::new("nested-jobs");
9985 let sentinel = home.join("nested-store-touched");
9986 let mut command = std::process::Command::new(std::env::current_exe().unwrap());
9987 command
9988 .args([
9989 "--exact",
9990 "supervise::spawn_record_tests::windows_nested_daemon_fixture",
9991 "--nocapture",
9992 ])
9993 .env("SUBC_TEST_NESTED_STORE", &sentinel);
9994 let mut daemon = subc_jobobject::spawn_contained(&mut command).unwrap();
9995 assert!(daemon.child.wait().unwrap().success());
9996 assert!(sentinel.exists(), "nested protocol:none module never ran");
9997 }
9998}
9999
10000#[cfg(target_os = "linux")]
10001pub(crate) fn kill_module_cgroup(placement: Option<&subc_cgroup::Placement>, module_id: &str) {
10002 use subc_cgroup::KillOutcome;
10003 match subc_cgroup::kill_module(placement, module_id) {
10004 KillOutcome::Killed => {}
10005 KillOutcome::NotPlaced | KillOutcome::Unsupported => {
10006 debug!(
10007 module_id,
10008 "cgroup tree kill unavailable; using direct-child kill"
10009 );
10010 }
10011 KillOutcome::IoError { path, error } => {
10012 warn!(module_id, path = %path.display(), %error, "cgroup tree kill failed; using direct-child kill");
10013 }
10014 }
10015}
10016
10017#[cfg(windows)]
10025fn contain_spawned_child(
10026 child: &mut Child,
10027 spec: &ModuleSpec,
10028 carried: Option<&Arc<containment::CarriedContainment>>,
10029) -> Result<Option<Arc<Mutex<subc_jobobject::JobObject>>>, SuperviseError> {
10030 let assignment = || {
10031 let job = match carried {
10032 Some(container) => container.job(),
10033 None => Arc::new(Mutex::new(subc_jobobject::JobObject::new()?)),
10034 };
10035 #[cfg(test)]
10036 spawn_record_tests::assignment_failpoint(spec)?;
10037 job.lock()
10038 .unwrap_or_else(|p| p.into_inner())
10039 .assign(child)?;
10040 Ok::<_, io::Error>(job)
10041 };
10042 match assignment() {
10043 Ok(job) => Ok(Some(job)),
10044 Err(source) => {
10045 kill_and_reap_rejected_child(child);
10046 Err(SuperviseError::Spawn {
10047 program: spec.program.clone(),
10048 source: io::Error::other(format!(
10049 "Windows module job containment failed: {source}"
10050 )),
10051 cgroup_path: None,
10052 })
10053 }
10054 }
10055}
10056
10057fn kill_and_reap_rejected_child(child: &mut Child, #[cfg(unix)] group: containment::GroupTarget) {
10060 #[cfg(unix)]
10061 group.signal_rejected(child);
10062 let _ = child.start_kill();
10063 while matches!(child.try_wait(), Ok(None)) {
10064 std::thread::yield_now();
10065 }
10066}
10067
10068#[cfg(windows)]
10073fn resume_suspended_child(pid: u32, spec: &ModuleSpec) -> Result<(), SuperviseError> {
10074 if let Err(source) = subc_jobobject::resume_main_thread(pid) {
10075 let _ = std::process::Command::new("taskkill.exe")
10079 .args(["/PID", &pid.to_string(), "/T", "/F"])
10080 .stdin(Stdio::null())
10081 .stdout(Stdio::null())
10082 .stderr(Stdio::null())
10083 .status();
10084 return Err(SuperviseError::Spawn {
10085 program: spec.program.clone(),
10086 source,
10087 cgroup_path: None,
10088 });
10089 }
10090 Ok(())
10091}
10092
10093#[cfg(target_os = "linux")]
10094fn remove_module_cgroup(placement: &subc_cgroup::Placement, module_id: &str) {
10095 match placement.remove_module(module_id) {
10096 Ok(()) => debug!(module_id, "removed module cgroup after process exit"),
10097 Err(error) => warn!(
10098 module_id,
10099 error = %error,
10100 "could not remove module cgroup after process exit; continuing teardown"
10101 ),
10102 }
10103}
10104
10105#[cfg(target_os = "linux")]
10106const REAPED_CGROUP_TERM_GRACE: Duration = Duration::from_secs(2);
10107#[cfg(target_os = "linux")]
10108const REAPED_CGROUP_FREEZE_TIMEOUT: Duration = Duration::from_millis(100);
10109
10110#[cfg(target_os = "linux")]
10111async fn cleanup_reaped_cgroup(placement: &subc_cgroup::Placement, module_id: &str) {
10112 let path = placement.module_path(module_id).ok();
10113 if path.as_ref().is_some_and(|path| {
10114 std::fs::read_to_string(path.join("cgroup.events"))
10115 .ok()
10116 .is_some_and(|events| events.lines().any(|line| line == "populated 0"))
10117 }) {
10118 remove_module_cgroup(placement, module_id);
10119 return;
10120 }
10121
10122 if terminate_reaped_cgroup(placement, module_id).await {
10125 if let Some(path) = &path {
10126 let deadline = Instant::now() + REAPED_CGROUP_TERM_GRACE;
10127 while cgroup_is_populated(path) && Instant::now() < deadline {
10128 sleep_until((Instant::now() + Duration::from_millis(1)).min(deadline)).await;
10129 }
10130 }
10131 }
10132
10133 if matches!(
10137 subc_cgroup::kill_module(Some(placement), module_id),
10138 subc_cgroup::KillOutcome::Killed
10139 ) {
10140 if let Some(path) = &path {
10141 while cgroup_is_populated(path) {
10142 sleep(Duration::from_millis(1)).await;
10143 }
10144 }
10145 }
10146 remove_module_cgroup(placement, module_id);
10147}
10148
10149#[cfg(target_os = "linux")]
10150fn cgroup_is_populated(path: &std::path::Path) -> bool {
10151 std::fs::read_to_string(path.join("cgroup.events"))
10152 .ok()
10153 .is_some_and(|events| events.lines().any(|line| line == "populated 1"))
10154}
10155
10156#[cfg(target_os = "linux")]
10157async fn terminate_reaped_cgroup(placement: &subc_cgroup::Placement, module_id: &str) -> bool {
10158 let frozen =
10159 match subc_cgroup::freeze_module(Some(placement), module_id, REAPED_CGROUP_FREEZE_TIMEOUT)
10160 .await
10161 {
10162 subc_cgroup::FreezeOutcome::Frozen(frozen) => frozen,
10163 outcome => {
10164 tracing::debug!(
10165 module_id,
10166 ?outcome,
10167 "cgroup freeze failed; skipping SIGTERM grace"
10168 );
10169 return false;
10170 }
10171 };
10172 let listed = match frozen.processes() {
10173 subc_cgroup::ProcsOutcome::Listed(pids) => {
10174 for pid in pids.into_iter().filter(|pid| *pid > 1) {
10175 if let Some(pid) = rustix::process::Pid::from_raw(pid) {
10176 if let Err(error) =
10177 rustix::process::kill_process(pid, rustix::process::Signal::TERM)
10178 {
10179 tracing::debug!(module_id, %pid, %error, "leftover cgroup SIGTERM failed");
10180 }
10181 }
10182 }
10183 true
10184 }
10185 outcome => {
10186 tracing::debug!(
10187 module_id,
10188 ?outcome,
10189 "cgroup process listing failed; skipping SIGTERM grace"
10190 );
10191 false
10192 }
10193 };
10194 match frozen.thaw() {
10195 subc_cgroup::ThawOutcome::Thawed => listed,
10196 outcome => {
10197 tracing::debug!(
10198 module_id,
10199 ?outcome,
10200 "cgroup thaw failed; proceeding to cgroup.kill"
10201 );
10202 false
10203 }
10204 }
10205}
10206
10207#[cfg(target_os = "linux")]
10208fn apply_cgroup_placement(
10209 command: &mut Command,
10210 spec: &ModuleSpec,
10211 path: &std::path::Path,
10212) -> Result<(), SuperviseError> {
10213 subc_cgroup::apply(command, path).map_err(|source| SuperviseError::Cgroup {
10214 module_id: spec.module_id.clone(),
10215 source,
10216 })
10217}
10218
10219fn capture_retention(spec: &ModuleSpec) -> Retention {
10220 let defaults = Retention::default();
10221 let value = |name: &str| {
10222 spec.env
10223 .iter()
10224 .rev()
10225 .find_map(|(key, value)| (key == name).then_some(value.as_str()))
10226 };
10227 Retention {
10228 max_file_mb: value(CAPTURE_MAX_FILE_MB_ENV)
10229 .and_then(|value| value.parse().ok())
10230 .unwrap_or(defaults.max_file_mb),
10231 keep: value(CAPTURE_KEEP_ENV)
10232 .and_then(|value| value.parse().ok())
10233 .unwrap_or(defaults.keep),
10234 max_age_days: value(CAPTURE_MAX_AGE_DAYS_ENV)
10235 .and_then(|value| value.parse().ok())
10236 .unwrap_or(defaults.max_age_days),
10237 }
10238}
10239
10240fn generate_launch_nonce() -> Result<String, SuperviseError> {
10243 let mut bytes = [0u8; 32];
10244 getrandom::getrandom(&mut bytes).map_err(|source| SuperviseError::LaunchNonce {
10245 reason: source.to_string(),
10246 })?;
10247 let mut hex = String::with_capacity(64);
10248 for b in bytes {
10249 use std::fmt::Write;
10250 let _ = write!(hex, "{b:02x}");
10251 }
10252 Ok(hex)
10253}
10254
10255fn constant_time_eq(a: &[u8], b: &[u8]) -> bool {
10258 if a.len() != b.len() {
10259 return false;
10260 }
10261 let mut diff = 0u8;
10262 for (x, y) in a.iter().zip(b.iter()) {
10263 diff |= x ^ y;
10264 }
10265 diff == 0
10266}
10267
10268#[cfg(not(windows))]
10272fn observe_spawned_image(pid: u32) -> Option<subc_os::Observation> {
10273 let process = subc_os::Process::open(pid).ok().flatten()?;
10274 #[cfg(target_os = "linux")]
10275 return sample_after_linux_exec(
10276 || {
10277 let stat = std::fs::read_to_string(format!("/proc/{pid}/stat")).ok()?;
10278 let committed = linux_exec_committed(&stat);
10279 #[cfg(test)]
10280 spawn_image_tests::report_exec_state(committed);
10281 committed
10282 },
10283 || process.observe(),
10284 );
10285 #[cfg(not(target_os = "linux"))]
10286 process.observe()
10287}
10288
10289#[cfg(target_os = "linux")]
10293fn linux_exec_committed(stat: &str) -> Option<bool> {
10294 const PF_FORKNOEXEC: u64 = 0x40;
10295 let flags = stat.rsplit_once(')')?.1.split_whitespace().nth(6)?;
10296 Some(flags.parse::<u64>().ok()? & PF_FORKNOEXEC == 0)
10297}
10298
10299#[cfg(target_os = "linux")]
10300fn sample_after_linux_exec(
10301 mut exec_committed: impl FnMut() -> Option<bool>,
10302 observe: impl FnOnce() -> Option<subc_os::Observation>,
10303) -> Option<subc_os::Observation> {
10304 let deadline = std::time::Instant::now() + Duration::from_secs(1);
10305 while !exec_committed()? {
10306 if std::time::Instant::now() >= deadline {
10310 return None;
10311 }
10312 std::thread::sleep(Duration::from_micros(50));
10313 }
10314 observe()
10315}
10316
10317#[cfg(all(test, target_os = "linux"))]
10318mod spawn_image_tests {
10319 use super::*;
10320 use std::cell::{Cell, RefCell};
10321
10322 enum ExecEvent {
10323 Uncommitted,
10324 Image(Option<subc_os::Observation>),
10325 }
10326
10327 thread_local! {
10328 static EXEC_EVENTS: RefCell<Option<std::sync::mpsc::Sender<ExecEvent>>> = const { RefCell::new(None) };
10329 }
10330
10331 pub(super) fn report_exec_state(committed: Option<bool>) {
10332 if committed == Some(false) {
10333 EXEC_EVENTS.with(|events| {
10334 if let Some(events) = events.borrow_mut().take() {
10335 let _ = events.send(ExecEvent::Uncommitted);
10336 }
10337 });
10338 }
10339 }
10340
10341 #[test]
10342 fn linux_spawn_observation_waits_for_a_paused_fork_to_exec() {
10343 use std::io::{Read, Write};
10344 use std::os::fd::AsRawFd;
10345
10346 let descriptor = std::fs::File::open("/dev/null").unwrap();
10347 let mut command = std::process::Command::new("/bin/sleep");
10348 command.arg("30");
10349 let (mut ready, mut release) =
10350 subc_os::fork_exec_test::pause_before_exec(&mut command, descriptor.as_raw_fd())
10351 .unwrap();
10352 let spawn = std::thread::spawn(move || command.spawn());
10353 let mut message = [0u8; 8];
10354 ready.read_exact(&mut message).unwrap();
10355 let pid = i32::from_ne_bytes(message[..4].try_into().unwrap()) as u32;
10356 let before = std::fs::read_to_string(format!("/proc/{pid}/stat"));
10357 let inherited = subc_os::Process::open(pid).unwrap().unwrap().observe();
10358 let (events, received) = std::sync::mpsc::channel();
10359 let observer = std::thread::spawn(move || {
10360 EXEC_EVENTS.with(|slot| *slot.borrow_mut() = Some(events.clone()));
10361 let image = observe_spawned_image(pid);
10362 let _ = events.send(ExecEvent::Image(image));
10363 });
10364 let first = received.recv_timeout(Duration::from_secs(5));
10368 release.write_all(&[1]).unwrap();
10369 let mut child = spawn.join().unwrap().unwrap();
10370 let after = match first {
10371 Ok(ExecEvent::Uncommitted) => match received.recv_timeout(Duration::from_secs(5)) {
10372 Ok(ExecEvent::Image(image)) => image,
10373 _ => None,
10374 },
10375 Ok(ExecEvent::Image(image)) => image,
10376 Err(_) => None,
10377 };
10378 observer.join().unwrap();
10379 child.kill().unwrap();
10380 child.wait().unwrap();
10381
10382 assert_eq!(linux_exec_committed(&before.unwrap()), Some(false));
10383 assert_eq!(
10384 inherited.unwrap().executable,
10385 subc_os::file_identity(&std::env::current_exe().unwrap())
10386 );
10387 assert_eq!(
10388 after.unwrap().executable,
10389 subc_os::file_identity(std::path::Path::new("/bin/sleep"))
10390 );
10391 }
10392
10393 #[test]
10394 fn spawned_image_is_not_sampled_until_linux_exec_commits() {
10395 let committed = Cell::new(false);
10396 let inherited = subc_os::Observation {
10397 start_time: 23,
10398 executable: Some(subc_os::FileIdentity {
10399 device: 29,
10400 inode: 31,
10401 }),
10402 };
10403 let module = subc_os::Observation {
10404 start_time: 23,
10405 executable: Some(subc_os::FileIdentity {
10406 device: 29,
10407 inode: 37,
10408 }),
10409 };
10410 let polls = Cell::new(0);
10411 let image = sample_after_linux_exec(
10412 || {
10413 polls.set(polls.get() + 1);
10414 committed.set(polls.get() == 2);
10417 Some(committed.get())
10418 },
10419 || Some(if committed.get() { module } else { inherited }),
10420 );
10421 assert_eq!(image, Some(module));
10422 assert_eq!(polls.get(), 2);
10423 }
10424
10425 #[test]
10426 fn linux_exec_flag_is_read_from_flags_after_the_last_parenthesis() {
10427 assert_eq!(
10428 linux_exec_committed("17 (a ) b) R 1 2 3 4 5 4194368"),
10429 Some(false)
10430 );
10431 assert_eq!(
10432 linux_exec_committed("17 (a ) b) R 1 2 3 4 5 4194304"),
10433 Some(true)
10434 );
10435 assert_eq!(linux_exec_committed("17 (a) R 1 2 3 4 5 invalid"), None);
10436 assert_eq!(linux_exec_committed("17 (a) R"), None);
10437 }
10438
10439 #[test]
10440 fn unreadable_exec_state_never_samples_an_inherited_image() {
10441 assert_eq!(
10442 sample_after_linux_exec(
10443 || None,
10444 || panic!("must not sample before exec confirmation")
10445 ),
10446 None
10447 );
10448 }
10449
10450 #[test]
10451 fn uncommitted_exec_times_out_without_sampling_an_inherited_image() {
10452 assert_eq!(
10453 sample_after_linux_exec(
10454 || Some(false),
10455 || panic!("must not sample after an exec timeout")
10456 ),
10457 None
10458 );
10459 }
10460}
10461
10462#[cfg(windows)]
10465fn windows_spawn_capture(spec: &ModuleSpec) -> io::Result<subc_os::ExecutableCapture> {
10466 let mut program = spec.program.clone();
10467 if program.is_absolute() || program.components().count() > 1 {
10468 match subc_os::ExecutableCapture::open(&program) {
10471 Ok(capture) => return Ok(capture),
10472 Err(error)
10473 if error.kind() == io::ErrorKind::NotFound && program.extension().is_none() => {}
10474 Err(error) => return Err(error),
10475 }
10476 program.set_extension("exe");
10477 return subc_os::ExecutableCapture::open(&program);
10478 }
10479 if program.extension().is_none() {
10480 program.set_extension("exe");
10481 }
10482 let custom_path = spec
10483 .env
10484 .iter()
10485 .rev()
10486 .find(|(key, _)| key.eq_ignore_ascii_case("PATH"));
10487 let mut directories = Vec::new();
10488 if let Some(parent) = std::env::current_exe()?.parent() {
10491 directories.push(parent.to_path_buf());
10492 }
10493 directories.push(std::env::current_dir()?);
10494 if let Some(windows) = std::env::var_os("SystemRoot") {
10495 let windows = PathBuf::from(windows);
10496 directories.push(windows.join("System32"));
10497 directories.push(windows.clone());
10498 }
10499 let path = custom_path
10500 .map(|(_, value)| std::ffi::OsString::from(value))
10501 .or_else(|| std::env::var_os("PATH"));
10502 if let Some(path) = path {
10503 directories.extend(std::env::split_paths(&path));
10504 }
10505 for directory in directories {
10506 match subc_os::ExecutableCapture::open(&directory.join(&program)) {
10507 Ok(capture) => return Ok(capture),
10508 Err(error) if error.kind() == io::ErrorKind::NotFound => continue,
10509 Err(error) => return Err(error),
10510 }
10511 }
10512 Err(io::Error::new(
10513 io::ErrorKind::NotFound,
10514 "resolved executable not found",
10515 ))
10516}
10517
10518fn spawn_and_mark_running(
10519 spec: &ModuleSpec,
10520 runtime: &SupervisorRuntimeConfig,
10521 snapshot: &SharedSnapshot,
10522) -> Result<SupervisedChild, SuperviseError> {
10523 let activity_spawn = runtime.begin_activity_spawn(&spec.module_id);
10524 let registration_watch = watch_registration(&runtime.registry, spec);
10525 #[cfg(unix)]
10526 let joining_group = containment::current(&runtime.child_roster.containment)
10527 .is_some_and(|container| container.group().is_some());
10528 let spawn = || {
10529 spawn_child(
10530 spec,
10531 runtime.connection_file_path.as_deref(),
10532 runtime.supervisor_handle.as_ref(),
10533 &runtime.stderr_ring,
10534 runtime.capture_logs_dir.as_deref(),
10535 &runtime.child_roster,
10536 #[cfg(target_os = "linux")]
10537 runtime.cgroup_placement.as_ref(),
10538 )
10539 };
10540 let result = spawn();
10541 #[cfg(unix)]
10542 let result = if result.is_err()
10543 && joining_group
10544 && !runtime.child_roster.is_closed()
10545 && lock_snapshot(snapshot).is_ok_and(|state| state.enabled)
10546 && runtime
10547 .child_roster
10548 .previous_instance(&spec.module_id)
10549 .is_none()
10550 {
10551 warn!(module_id = %spec.module_id, "carried process-group spawn failed; trying one fresh container");
10555 spawn()
10556 } else {
10557 result
10558 };
10559 let child = result?;
10560 let mut child = child;
10561 arm_registration_deadline(&mut child, runtime.register_timeout, registration_watch);
10562 if let Err(error) = set_running(snapshot, &child, &spec.module_id, &runtime.spawn_events) {
10563 containment::terminate_blocking(&runtime.child_roster.containment);
10564 return Err(error);
10565 }
10566 if let Some(spawn) = activity_spawn {
10567 spawn.complete(snapshot);
10568 }
10569 Ok(child)
10570}
10571
10572struct RegistrationWatch {
10573 changes: watch::Receiver<u64>,
10574 baseline: u64,
10575}
10576
10577fn watch_registration(registry: &Registry, spec: &ModuleSpec) -> Option<RegistrationWatch> {
10581 if spec.protocol != ModuleProtocol::Subc {
10582 return None;
10583 }
10584 let changes = registry.subscribe_module_changes(&spec.module_id).ok()?;
10585 let baseline = *changes.borrow();
10586 Some(RegistrationWatch { changes, baseline })
10587}
10588
10589fn arm_registration_deadline(
10590 child: &mut SupervisedChild,
10591 budget: Duration,
10592 registration_watch: Option<RegistrationWatch>,
10593) {
10594 child.registration_deadline =
10595 (child.protocol == ModuleProtocol::Subc).then_some(child.spawned_at + budget);
10596 child.registration_watch = registration_watch;
10597}
10598
10599async fn stop_unregistered_child(child: &mut SupervisedChild) -> io::Result<()> {
10600 if let Some(container) = child.child.container() {
10601 container.stop_members().await;
10602 child.kill_leader()?;
10603 child.wait().await?;
10604 return Ok(());
10605 }
10606 #[cfg(unix)]
10607 {
10608 let group = unreaped_registration_group(child)?;
10611 let _ = rustix::process::kill_process_group(group, rustix::process::Signal::TERM);
10612 sleep(crate::child_roster::TERM_TO_KILL).await;
10617 let group = unreaped_registration_group(child)?;
10618 let _ = rustix::process::kill_process_group(group, rustix::process::Signal::KILL);
10619 }
10620 child.start_kill()?;
10623 child.wait().await?;
10624 Ok(())
10625}
10626
10627#[cfg(unix)]
10628fn unreaped_registration_group(child: &SupervisedChild) -> io::Result<rustix::process::Pid> {
10629 if let Some(container) = child.child.container() {
10630 return container
10631 .group()
10632 .and_then(|pid| rustix::process::Pid::from_raw(pid as i32))
10633 .ok_or_else(|| io::Error::other("a carried cgroup has no signalable process group"));
10634 }
10635 let group = i32::try_from(child.pid)
10638 .ok()
10639 .filter(|pid| *pid > 1)
10640 .and_then(rustix::process::Pid::from_raw)
10641 .ok_or_else(|| {
10642 io::Error::new(
10643 io::ErrorKind::InvalidInput,
10644 "invalid supervised process group",
10645 )
10646 })?;
10647 if child.child.id() != Some(child.pid) {
10648 return Err(io::Error::new(
10649 io::ErrorKind::InvalidInput,
10650 "refusing process-group signal after supervised child was reaped",
10651 ));
10652 }
10653 Ok(group)
10654}
10655
10656enum RegistrationWaitOutcome {
10657 Registered,
10658 Exited(ExitReport),
10659 TimedOut,
10660}
10661
10662struct ReloadRegistrationFailure {
10663 exit_report: ExitReport,
10664 reason: String,
10665}
10666
10667#[derive(Debug, Clone, Copy, PartialEq, Eq)]
10668enum BusyGaugeObservation {
10669 Quiescent,
10670 Busy,
10671 Omitted,
10672}
10673
10674#[derive(Debug, Clone, PartialEq, Eq)]
10676enum GaugeVerdict {
10677 Zero,
10679 NonZero(u64),
10681 NotACount(String),
10685 Missing,
10687 ProbeFailed(&'static str),
10690}
10691
10692impl GaugeVerdict {
10693 fn reads_quiet(&self) -> bool {
10694 matches!(self, Self::Zero)
10695 }
10696}
10697
10698impl fmt::Display for GaugeVerdict {
10699 fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
10700 match self {
10701 Self::Zero => f.write_str("zero"),
10702 Self::NonZero(value) => write!(f, "non_zero({value})"),
10703 Self::NotACount(value) => write!(f, "not_a_count({value})"),
10704 Self::Missing => f.write_str("missing_from_health_body"),
10705 Self::ProbeFailed(kind) => write!(f, "probe_failed({kind})"),
10706 }
10707 }
10708}
10709
10710const GAUGE_VALUE_TEXT_LIMIT: usize = 32;
10713
10714fn gauge_verdicts(metrics: Option<&Value>, gauges: &[String]) -> Vec<GaugeVerdict> {
10716 let metrics = metrics.and_then(Value::as_object);
10717 gauges
10718 .iter()
10719 .map(
10720 |gauge| match metrics.and_then(|metrics| metrics.get(gauge)) {
10721 None => GaugeVerdict::Missing,
10722 Some(value) => match value.as_u64() {
10723 Some(0) => GaugeVerdict::Zero,
10724 Some(count) => GaugeVerdict::NonZero(count),
10725 None => {
10726 let mut text = value.to_string();
10727 if text.len() > GAUGE_VALUE_TEXT_LIMIT {
10728 let mut end = GAUGE_VALUE_TEXT_LIMIT;
10729 while !text.is_char_boundary(end) {
10730 end -= 1;
10731 }
10732 text.truncate(end);
10733 text.push_str("...");
10734 }
10735 GaugeVerdict::NotACount(text)
10736 }
10737 },
10738 },
10739 )
10740 .collect()
10741}
10742
10743fn busy_gauge_observation(verdicts: &[GaugeVerdict]) -> BusyGaugeObservation {
10746 if verdicts
10747 .iter()
10748 .any(|verdict| matches!(verdict, GaugeVerdict::Missing))
10749 {
10750 BusyGaugeObservation::Omitted
10751 } else if verdicts.iter().all(GaugeVerdict::reads_quiet) {
10752 BusyGaugeObservation::Quiescent
10753 } else {
10754 BusyGaugeObservation::Busy
10755 }
10756}
10757
10758#[derive(Debug, Clone, Default, PartialEq, Eq)]
10763struct GaugeTrace {
10764 latest: Option<(GaugeVerdict, Duration)>,
10765 last_value: Option<(GaugeVerdict, Duration)>,
10766}
10767
10768#[derive(Debug, Clone, Default, PartialEq, Eq)]
10774struct DrainGaugeTrace {
10775 probes: u32,
10777 gauges: Vec<(String, GaugeTrace)>,
10779}
10780
10781impl DrainGaugeTrace {
10782 fn new(gauges: &[String]) -> Self {
10783 Self {
10784 probes: 0,
10785 gauges: gauges
10786 .iter()
10787 .map(|gauge| (gauge.clone(), GaugeTrace::default()))
10788 .collect(),
10789 }
10790 }
10791
10792 fn record(&mut self, verdicts: &[GaugeVerdict], at: Duration) {
10794 self.probes = self.probes.saturating_add(1);
10795 for ((_, trace), verdict) in self.gauges.iter_mut().zip(verdicts) {
10796 trace.latest = Some((verdict.clone(), at));
10797 if matches!(
10798 verdict,
10799 GaugeVerdict::Zero | GaugeVerdict::NonZero(_) | GaugeVerdict::NotACount(_)
10800 ) {
10801 trace.last_value = Some((verdict.clone(), at));
10802 }
10803 }
10804 }
10805}
10806
10807impl fmt::Display for DrainGaugeTrace {
10812 fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
10813 if self.gauges.is_empty() {
10814 return f.write_str("none declared");
10815 }
10816 for (index, (name, trace)) in self.gauges.iter().enumerate() {
10817 if index > 0 {
10818 f.write_str(" | ")?;
10819 }
10820 write!(f, "{name}: ")?;
10821 match &trace.latest {
10822 Some((verdict, at)) => write!(f, "{verdict} @{}ms", at.as_millis())?,
10823 None => f.write_str("never probed")?,
10824 }
10825 match &trace.last_value {
10826 Some((value, at)) => write!(f, "; last value {value} @{}ms", at.as_millis())?,
10827 None => f.write_str("; no value ever read")?,
10828 }
10829 }
10830 Ok(())
10831 }
10832}
10833
10834pub(crate) fn declared_busy_gauges(
10835 registry: &Registry,
10836 module_id: &str,
10837) -> Result<Vec<String>, SuperviseError> {
10838 busy_gauges_of(
10839 registry
10840 .get_module(module_id)
10841 .map_err(SuperviseError::Registry)?,
10842 )
10843}
10844
10845fn declared_busy_gauges_for_connection(
10849 registry: &Registry,
10850 connection_id: ConnectionId,
10851) -> Result<Vec<String>, SuperviseError> {
10852 busy_gauges_of(
10853 registry
10854 .get_module_by_connection(connection_id)
10855 .map_err(SuperviseError::Registry)?,
10856 )
10857}
10858
10859fn busy_gauges_of(
10860 registration: Option<crate::registry::ModuleRegistration>,
10861) -> Result<Vec<String>, SuperviseError> {
10862 let Some(registration) = registration else {
10863 return Ok(Vec::new());
10864 };
10865 let Some(self_signals) = registration.manifest.self_signals else {
10866 return Ok(Vec::new());
10867 };
10868
10869 let mut gauges = Vec::new();
10870 for declaration in self_signals {
10871 if declaration.kind != SelfSignalKind::Busy {
10872 continue;
10873 }
10874 match declaration.anchored_to {
10875 SignalAnchor::HealthGauges { gauges: declared } if !declared.is_empty() => {
10876 gauges.extend(declared)
10877 }
10878 _ => {
10879 gauges.push(String::new());
10882 }
10883 }
10884 }
10885 Ok(gauges)
10886}
10887
10888#[allow(clippy::too_many_arguments)]
10897async fn wait_for_forwarding_quiescence(
10898 forwarding: &ForwardingTable,
10899 module_id: &str,
10900 runtime: &SupervisorRuntimeConfig,
10901 endpoint: crate::ModuleEndpointId,
10902 deadline: Instant,
10903 busy_gauges: &[String],
10904 scope: DrainScope,
10905 trace: &mut DrainGaugeTrace,
10906) -> Result<bool, SuperviseError> {
10907 let wait_started = Instant::now();
10908 let mut gauges_quiescent = busy_gauges.is_empty();
10909 let mut next_probe_at = wait_started;
10910 let mut omission_reported = false;
10911
10912 loop {
10913 let now = Instant::now();
10914 if !busy_gauges.is_empty() && now >= next_probe_at && now < deadline {
10915 let report = match scope {
10916 DrainScope::Active => probe_module_health(module_id, runtime, Some(deadline)).await,
10917 DrainScope::Endpoint(endpoint) => {
10918 probe_endpoint_health(endpoint, runtime, Some(deadline)).await
10919 }
10920 };
10921 let verdicts = match &report {
10922 Ok(report) => gauge_verdicts(report.metrics.as_ref(), busy_gauges),
10923 Err(err) => vec![GaugeVerdict::ProbeFailed(err.label()); busy_gauges.len()],
10924 };
10925 trace.record(&verdicts, wait_started.elapsed());
10926 gauges_quiescent = match report {
10927 Ok(_) => match busy_gauge_observation(&verdicts) {
10928 BusyGaugeObservation::Quiescent => true,
10929 BusyGaugeObservation::Busy => false,
10930 BusyGaugeObservation::Omitted => {
10931 if !omission_reported {
10934 forwarding
10935 .counters()
10936 .increment_drains_with_undeclared_gauge();
10937 let missing = busy_gauges
10938 .iter()
10939 .zip(&verdicts)
10940 .filter(|(_, verdict)| matches!(verdict, GaugeVerdict::Missing))
10941 .map(|(gauge, _)| gauge.as_str())
10942 .collect::<Vec<_>>()
10943 .join(",");
10944 warn!(
10945 module_id,
10946 missing_gauges = %missing,
10947 "drain health.check answer lacks a declared busy gauge; the drain treats the module as busy until it reports it"
10948 );
10949 omission_reported = true;
10950 }
10951 false
10952 }
10953 },
10954 Err(err) => {
10955 warn!(
10956 module_id,
10957 error = %err,
10958 "drain health.check did not produce declared busy gauges; treating module as busy"
10959 );
10960 false
10961 }
10962 };
10963 next_probe_at = Instant::now() + drain_busy_reprobe_interval();
10966 }
10967
10968 let in_flight = forwarding
10969 .endpoint_in_flight_count(endpoint)
10970 .map_err(SuperviseError::Forwarding)?;
10971 if in_flight == 0 && gauges_quiescent {
10972 return Ok(true);
10973 }
10974
10975 let now = Instant::now();
10976 if now >= deadline {
10977 return Ok(false);
10978 }
10979 let mut wait = deadline
10980 .saturating_duration_since(now)
10981 .min(REGISTRY_RELEASE_POLL);
10982 if !busy_gauges.is_empty() {
10983 wait = wait.min(next_probe_at.saturating_duration_since(now));
10984 }
10985 sleep(wait).await;
10986 }
10987}
10988
10989fn drain_busy_reprobe_interval() -> Duration {
10993 DRAIN_BUSY_REPROBE_INTERVAL.max(REGISTRY_RELEASE_POLL)
10994}
10995
10996fn warn_drain_timed_out(
11011 module_id: &str,
11012 drain_timeout: Duration,
11013 reason: RouteCloseReason,
11014 holdouts: &crate::forwarding::DrainHoldouts,
11015 gauges: &DrainGaugeTrace,
11016) {
11017 warn!(
11018 module_id = %module_id,
11019 waited = ?drain_timeout,
11020 ?reason,
11021 held_requests = holdouts.requests,
11022 held_routes = holdouts.routes,
11023 total_routes = holdouts.total_routes,
11024 top_connections = ?holdouts.top_connections,
11025 held = %holdouts
11028 .held
11029 .iter()
11030 .map(|(channel, corr)| format!("{channel}:{corr}"))
11031 .collect::<Vec<_>>()
11032 .join(","),
11033 gauge_probes = gauges.probes,
11034 busy_gauges = %gauges,
11035 "route drain timed out before request quiescence; forcing teardown"
11036 );
11037}
11038
11039fn drained_after_quiescence_wait(wait_result: &Result<bool, SuperviseError>) -> bool {
11047 match wait_result {
11048 Ok(drained) => *drained,
11049 Err(_) => false,
11050 }
11051}
11052
11053fn send_route_goodbyes(forwarding: &ForwardingTable, released_routes: Vec<GoodbyeTarget>) {
11054 for released in released_routes {
11055 let frame = match Frame::build_with_version(
11056 released.negotiated_ver,
11057 FrameType::Goodbye,
11058 control_flags(),
11059 released.channel,
11060 released.epoch,
11061 0,
11062 Vec::new(),
11063 ) {
11064 Ok(frame) => frame,
11065 Err(err) => {
11066 warn!(
11067 route_channel = released.channel,
11068 error = %err,
11069 "failed to build supervisor drain route GOODBYE frame"
11070 );
11071 continue;
11072 }
11073 };
11074 if !released.close_on_delivery_failure() {
11075 crate::forwarding::send_module_route_goodbye(
11076 &forwarding.counters(),
11077 &released.sink,
11078 frame,
11079 released.module_id.as_deref(),
11080 "supervisor drain",
11081 );
11082 continue;
11083 }
11084 if let Err(err) = released.sink.try_send(frame) {
11085 warn!(
11086 target_connection_id = released.connection_id.get(),
11087 route_channel = released.channel,
11088 error = %err,
11089 "supervisor drain route GOODBYE was not delivered to client; closing target connection"
11090 );
11091 let _ = forwarding.escalate_client_delivery_failure(
11092 released.connection_id,
11093 released.channel,
11094 released.epoch,
11095 CloseReason::new(
11096 "route_goodbye_delivery_failed",
11097 format!(
11098 "failed to enqueue supervisor drain route GOODBYE for channel {}: {err}",
11099 released.channel
11100 ),
11101 ),
11102 crate::forwarding::UndeliveredFrame {
11103 module_id: released.module_id.as_deref(),
11104 sink: &released.sink,
11105 },
11106 );
11107 }
11108 }
11109}
11110
11111async fn send_module_draining(
11112 module_id: &str,
11113 reason: RouteCloseReason,
11114 deadline_ms: u64,
11115 target: &ModuleDrainTarget,
11116 _wait_for_flush: bool,
11117) -> bool {
11118 let body = match serde_json::to_vec(&ModuleControlCommand::Draining {
11119 reason,
11120 deadline_ms,
11121 }) {
11122 Ok(body) => body,
11123 Err(err) => {
11124 warn!(
11125 module_id,
11126 error = %err,
11127 "failed to encode module draining command"
11128 );
11129 return false;
11130 }
11131 };
11132 let frame = match Frame::build_with_version(
11133 target.negotiated_ver,
11134 FrameType::Push,
11135 control_flags(),
11136 0,
11137 0,
11138 0,
11139 body,
11140 ) {
11141 Ok(frame) => frame,
11142 Err(err) => {
11143 warn!(
11144 module_id,
11145 error = %err,
11146 "failed to build module draining command frame"
11147 );
11148 return false;
11149 }
11150 };
11151 #[cfg(unix)]
11152 if _wait_for_flush {
11153 return target.sink.send_flushed(frame).await.is_ok();
11154 }
11155 if let Err(err) = target.sink.try_send(frame) {
11156 warn!(
11157 module_id,
11158 target_connection_id = target.endpoint.connection_id.get(),
11159 error = %err,
11160 "module draining command was not delivered to peer"
11161 );
11162 return false;
11163 }
11164 true
11165}
11166
11167fn module_goodbye_frame(module_id: &str, negotiated_ver: u8) -> Option<Frame> {
11169 match Frame::build_with_version(
11170 negotiated_ver,
11171 FrameType::Goodbye,
11172 control_flags(),
11173 0,
11174 0,
11175 0,
11176 Vec::new(),
11177 ) {
11178 Ok(frame) => Some(frame),
11179 Err(err) => {
11180 warn!(
11181 module_id,
11182 error = %err,
11183 "failed to build module GOODBYE frame"
11184 );
11185 None
11186 }
11187 }
11188}
11189
11190#[cfg(unix)]
11204async fn send_module_goodbyes_for_daemon_shutdown(
11205 forwarding: &Arc<ForwardingTable>,
11206 reason: &CloseReason,
11207 wait_for_flush: bool,
11208) {
11209 const GOODBYE_BUDGET: Duration = Duration::from_millis(500);
11210 let targets = match forwarding.module_connections() {
11211 Ok(targets) => targets,
11212 Err(err) => {
11213 warn!(error = %err, "could not list module connections for shutdown GOODBYE");
11214 return;
11215 }
11216 };
11217 let deadline = Instant::now() + GOODBYE_BUDGET;
11218 let mut sends = tokio::task::JoinSet::new();
11219 for target in targets {
11220 let Some(frame) = module_goodbye_frame(&target.module_id, target.negotiated_ver) else {
11221 continue;
11222 };
11223 if !wait_for_flush {
11224 if let Err(err) = target.sink.try_send(frame) {
11225 debug!(
11226 module_id = %target.module_id,
11227 error = %err,
11228 "shutdown module GOODBYE was not queued"
11229 );
11230 }
11231 continue;
11232 }
11233 let forwarding = Arc::clone(forwarding);
11234 let reason = reason.clone();
11235 sends.spawn(async move {
11236 match timeout_at(deadline, target.sink.send_flushed(frame)).await {
11237 Ok(Ok(())) => {}
11238 Ok(Err(err)) => debug!(
11239 module_id = %target.module_id,
11240 error = %err,
11241 "module connection closed before its shutdown GOODBYE was written"
11242 ),
11243 Err(_) => warn!(
11244 module_id = %target.module_id,
11245 budget = ?GOODBYE_BUDGET,
11246 "shutdown module GOODBYE was not written within its budget; closing anyway"
11247 ),
11248 }
11249 forwarding.request_connection_close(target.endpoint.connection_id, reason);
11250 });
11251 }
11252 while sends.join_next().await.is_some() {}
11254}
11255
11256fn send_module_goodbye(module_id: &str, forwarding: &ForwardingTable, target: &ModuleDrainTarget) {
11257 let Some(frame) = module_goodbye_frame(module_id, target.negotiated_ver) else {
11258 return;
11259 };
11260 if let Err(err) = target.sink.try_send(frame) {
11261 warn!(
11262 module_id,
11263 target_connection_id = target.endpoint.connection_id.get(),
11264 error = %err,
11265 "supervisor drain module GOODBYE was not delivered to peer; closing module connection"
11266 );
11267 forwarding.request_connection_close(
11268 target.endpoint.connection_id,
11269 CloseReason::new(
11270 "module_goodbye_delivery_failed",
11271 format!("failed to enqueue supervisor drain module GOODBYE for module '{module_id}': {err}"),
11272 ),
11273 );
11274 }
11275}
11276
11277#[derive(Clone, Copy)]
11278struct ForwardingDrainContext<'a> {
11279 spec: &'a ModuleSpec,
11280 runtime: &'a SupervisorRuntimeConfig,
11281 registry: &'a Registry,
11282 scope: DrainScope,
11283}
11284
11285#[derive(Debug, Clone, Copy, PartialEq, Eq)]
11287enum DrainScope {
11288 Active,
11291 Endpoint(crate::ModuleEndpointId),
11296}
11297
11298#[derive(Debug, Clone, Copy, PartialEq, Eq)]
11306enum StopNotice {
11307 SentOverConnection,
11310 NoConnection,
11314 NotSent,
11318}
11319
11320async fn begin_forwarding_drain(
11321 spec: &ModuleSpec,
11322 runtime: &SupervisorRuntimeConfig,
11323 registry: &Registry,
11324 snapshot: &SharedSnapshot,
11325 enabled: Option<bool>,
11326 reason: RouteCloseReason,
11327) -> Result<StopNotice, SuperviseError> {
11328 let Some(forwarding) = runtime.forwarding.as_ref() else {
11329 return Err(SuperviseError::ReloadUnavailable {
11330 module_id: spec.module_id.clone(),
11331 reason: "supervisor was not configured with a forwarding table".to_string(),
11332 });
11333 };
11334
11335 begin_forwarding_drain_with(
11336 forwarding,
11337 ForwardingDrainContext {
11338 spec,
11339 runtime,
11340 registry,
11341 scope: DrainScope::Active,
11342 },
11343 snapshot,
11344 enabled,
11345 reason,
11346 runtime.drain_timeout,
11347 )
11348 .await
11349}
11350
11351async fn begin_forwarding_drain_if_configured(
11352 spec: &ModuleSpec,
11353 runtime: &SupervisorRuntimeConfig,
11354 registry: &Registry,
11355 snapshot: &SharedSnapshot,
11356 enabled: Option<bool>,
11357 reason: RouteCloseReason,
11358) -> Result<StopNotice, SuperviseError> {
11359 begin_forwarding_drain_with_timeout(
11360 spec,
11361 runtime,
11362 registry,
11363 snapshot,
11364 enabled,
11365 reason,
11366 runtime.drain_timeout,
11367 )
11368 .await
11369}
11370
11371async fn begin_forwarding_drain_with_timeout(
11375 spec: &ModuleSpec,
11376 runtime: &SupervisorRuntimeConfig,
11377 registry: &Registry,
11378 snapshot: &SharedSnapshot,
11379 enabled: Option<bool>,
11380 reason: RouteCloseReason,
11381 drain_timeout: Duration,
11382) -> Result<StopNotice, SuperviseError> {
11383 let Some(forwarding) = runtime.forwarding.as_ref() else {
11384 return Ok(StopNotice::NotSent);
11385 };
11386
11387 begin_forwarding_drain_with(
11388 forwarding,
11389 ForwardingDrainContext {
11390 spec,
11391 runtime,
11392 registry,
11393 scope: DrainScope::Active,
11394 },
11395 snapshot,
11396 enabled,
11397 reason,
11398 drain_timeout,
11399 )
11400 .await
11401}
11402
11403async fn begin_forwarding_drain_with(
11404 forwarding: &ForwardingTable,
11405 context: ForwardingDrainContext<'_>,
11406 snapshot: &SharedSnapshot,
11407 enabled: Option<bool>,
11408 reason: RouteCloseReason,
11409 drain_timeout: Duration,
11410) -> Result<StopNotice, SuperviseError> {
11411 let ForwardingDrainContext {
11412 spec,
11413 runtime,
11414 registry,
11415 scope,
11416 } = context;
11417 debug_assert_ne!(reason, RouteCloseReason::Crash);
11418 let terminal = matches!(reason, RouteCloseReason::Disable);
11419 let drain_started_at = Instant::now();
11420 let drain_deadline = drain_started_at + drain_timeout;
11421 let deadline_ms =
11422 unix_ms_now().saturating_add(u64::try_from(drain_timeout.as_millis()).unwrap_or(u64::MAX));
11423 let busy_gauges = match scope {
11424 DrainScope::Active => declared_busy_gauges(registry, &spec.module_id)?,
11425 DrainScope::Endpoint(endpoint) => {
11426 declared_busy_gauges_for_connection(registry, endpoint.connection_id)?
11427 }
11428 };
11429
11430 let gate_started = Instant::now();
11433 let drain_target = match scope {
11434 DrainScope::Active => forwarding.begin_module_drain(&spec.module_id, reason),
11435 DrainScope::Endpoint(endpoint) => forwarding.begin_endpoint_drain(endpoint, reason),
11436 }
11437 .map_err(SuperviseError::Forwarding)?;
11438 info!(
11443 module_id = %spec.module_id,
11444 ?reason,
11445 gate_ms = u64::try_from(gate_started.elapsed().as_millis()).unwrap_or(u64::MAX),
11446 connected = drain_target.is_some(),
11447 "module drain began; route admission closed"
11448 );
11449 if scope == DrainScope::Active {
11450 update_snapshot(snapshot, Some(&spec.module_id), |state| {
11451 state.state = ModuleState::Draining;
11452 state.draining_to_replace =
11453 matches!(reason, RouteCloseReason::Restart | RouteCloseReason::Reload);
11454 if let Some(enabled) = enabled {
11455 state.enabled = enabled;
11456 }
11457 })?;
11458 }
11459
11460 let Some(target) = drain_target.as_ref() else {
11461 return Ok(StopNotice::NoConnection);
11465 };
11466 {
11467 send_module_draining(&spec.module_id, reason, deadline_ms, target, false).await;
11468 #[cfg(feature = "test-support")]
11469 forwarding.wait_before_route_closing_for_test().await;
11470 let routes = forwarding
11471 .endpoint_routes(target.endpoint)
11472 .map_err(SuperviseError::Forwarding)?;
11473 let routes_notified = routes.len();
11474 crate::control::send_route_control_pushes(
11475 forwarding,
11476 routes.clone(),
11477 ClientControlPush::RouteClosing {
11478 module_id: spec.module_id.clone(),
11479 channels: Vec::new(),
11480 reason,
11481 },
11482 );
11483 send_route_goodbyes(forwarding, target.abandoned_bindings.clone());
11484
11485 let mut gauge_trace = DrainGaugeTrace::new(&busy_gauges);
11491 let wait_result = wait_for_forwarding_quiescence(
11492 forwarding,
11493 &spec.module_id,
11494 runtime,
11495 target.endpoint,
11496 drain_deadline,
11497 &busy_gauges,
11498 scope,
11499 &mut gauge_trace,
11500 )
11501 .await;
11502 let drained = drained_after_quiescence_wait(&wait_result);
11503 if let Err(err) = &wait_result {
11504 error!(
11505 module_id = %spec.module_id,
11506 ?reason,
11507 error = %err,
11508 "forwarding quiescence wait failed after route.closing; forcing route.closed(drained: false) so the client is not left waiting on an unfulfilled promise"
11509 );
11510 } else if !drained {
11511 let holdouts = forwarding
11513 .endpoint_drain_holdouts(target.endpoint)
11514 .unwrap_or_default();
11515 warn_drain_timed_out(
11516 &spec.module_id,
11517 drain_timeout,
11518 reason,
11519 &holdouts,
11520 &gauge_trace,
11521 );
11522 }
11523 crate::control::send_route_control_pushes(
11524 forwarding,
11525 routes,
11526 ClientControlPush::RouteClosed {
11527 module_id: spec.module_id.clone(),
11528 channels: Vec::new(),
11529 reason,
11530 drained,
11531 abandoned: target.abandoned_bindings.len() as u32,
11532 excluded_subscriptions: target.excluded_subscriptions,
11533 terminal: Some(terminal),
11534 },
11535 );
11536 wait_result?;
11537
11538 let released_routes = match forwarding.release_module_endpoint_routes(target.endpoint) {
11544 Ok(routes) => routes,
11545 Err(err) => {
11546 warn!(
11547 module_id = %spec.module_id,
11548 ?reason,
11549 error = %err,
11550 "failed to release module endpoint routes after route.closed; module GOODBYE will still be sent"
11551 );
11552 send_module_goodbye(&spec.module_id, forwarding, target);
11553 return Err(SuperviseError::Forwarding(err));
11554 }
11555 };
11556 let route_goodbye_count = released_routes.len();
11557 send_route_goodbyes(forwarding, released_routes);
11558 send_module_goodbye(&spec.module_id, forwarding, target);
11559
11560 info!(
11566 module_id = %spec.module_id,
11567 ?reason,
11568 routes_notified,
11569 route_goodbyes = route_goodbye_count,
11570 abandoned_reservations = target.abandoned_bindings.len(),
11571 excluded_subscriptions = target.excluded_subscriptions,
11572 drained,
11573 "module drain complete; consumers notified via route.closing/route.closed pushes and per-route GOODBYE frames"
11574 );
11575 }
11576
11577 Ok(StopNotice::SentOverConnection)
11578}
11579
11580async fn wait_for_registration_after_reload(
11583 registry: &Registry,
11584 module_id: &str,
11585 snapshot: &SharedSnapshot,
11586 child: &mut SupervisedChild,
11587 wait: Duration,
11588) -> Result<RegistrationWaitOutcome, SuperviseError> {
11589 wait_for_slot_registration(
11590 registry,
11591 crate::registry::RegistrationSlot::Active(module_id),
11592 module_id,
11593 snapshot,
11594 child,
11595 wait,
11596 )
11597 .await
11598}
11599
11600async fn wait_for_slot_registration(
11608 registry: &Registry,
11609 slot: crate::registry::RegistrationSlot<'_>,
11610 module_id: &str,
11611 snapshot: &SharedSnapshot,
11612 child: &mut SupervisedChild,
11613 wait: Duration,
11614) -> Result<RegistrationWaitOutcome, SuperviseError> {
11615 let deadline = Instant::now() + wait;
11616 loop {
11617 if registry
11618 .registration(slot)
11619 .map_err(SuperviseError::Registry)?
11620 .is_some()
11621 {
11622 return Ok(RegistrationWaitOutcome::Registered);
11623 }
11624
11625 let now = Instant::now();
11626 if now >= deadline {
11627 return Ok(RegistrationWaitOutcome::TimedOut);
11628 }
11629 let remaining = deadline.saturating_duration_since(now);
11630 let poll = remaining.min(REGISTRY_RELEASE_POLL);
11631
11632 tokio::select! {
11633 wait_result = child.wait() => {
11634 let status = wait_result.map_err(|source| SuperviseError::Wait {
11635 module_id: module_id.to_string(),
11636 source,
11637 })?;
11638 return Ok(RegistrationWaitOutcome::Exited(classify_reaped_child_exit(
11639 snapshot,
11640 child,
11641 &status,
11642 )));
11643 }
11644 _ = sleep(poll) => {}
11645 }
11646 }
11647}
11648
11649fn registration_failure_exit_report(mut exit_report: ExitReport) -> ExitReport {
11650 if exit_report.kind != ExitKind::DeliberateSeverance {
11654 exit_report.kind = ExitKind::Crash;
11655 }
11656 exit_report
11657}
11658
11659async fn handle_reload_child_registration_failure(
11660 spec: &ModuleSpec,
11661 runtime: &SupervisorRuntimeConfig,
11662 registry: &Registry,
11663 process_liveness: &SupervisorProcessLiveness,
11664 snapshot: &SharedSnapshot,
11665 _child: &mut Option<SupervisedChild>,
11666 failure: ReloadRegistrationFailure,
11667) -> Result<(), SuperviseError> {
11668 let ReloadRegistrationFailure {
11669 exit_report,
11670 reason,
11671 } = failure;
11672 match on_child_exit(
11673 spec,
11674 runtime.restart_policy,
11675 registry,
11676 snapshot,
11677 &runtime.terminal_ring,
11678 &runtime.spawn_events,
11679 &runtime.child_roster,
11680 exit_report,
11681 runtime.recovery_is_daemon,
11682 )
11683 .await
11684 {
11685 NextAction::Stop {
11686 registration_released,
11687 } => {
11688 if registration_released {
11689 process_liveness.untrack_if_current(&spec.module_id, snapshot);
11690 }
11691 }
11692 NextAction::Restart { schedule } => {
11693 let delay = schedule.map_or(runtime.restart_policy.delay_for_restart(0), |schedule| {
11694 schedule.delay
11695 });
11696 if let Some(schedule) = schedule {
11697 log_crash_respawn(&spec.module_id, schedule);
11698 }
11699 schedule_respawn(
11700 runtime,
11701 snapshot,
11702 &spec.module_id,
11703 delay,
11704 RespawnKind::Spawn,
11705 true,
11706 )?;
11707 }
11708 }
11709 Err(SuperviseError::ReloadFailed {
11710 module_id: spec.module_id.clone(),
11711 reason,
11712 })
11713}
11714
11715async fn handle_reload_spawn_failure(
11716 spec: &ModuleSpec,
11717 runtime: &SupervisorRuntimeConfig,
11718 process_liveness: &SupervisorProcessLiveness,
11719 snapshot: &SharedSnapshot,
11720 _child: &mut Option<SupervisedChild>,
11721 reason: String,
11722) -> Result<(), SuperviseError> {
11723 let now = Instant::now();
11724 let mut schedule = None;
11725 update_snapshot(snapshot, Some(&spec.module_id), |state| {
11726 clear_current_process_facts(state);
11727 if state.enabled {
11728 schedule = state.next_crash_restart(&runtime.restart_policy, now);
11729 state.state = if schedule.is_some() {
11730 ModuleState::Restarting
11731 } else {
11732 if runtime
11733 .failure_context(runtime.recovery_is_daemon)
11734 .may_retry()
11735 {
11736 state.failed_retry.schedule(&spec.module_id);
11737 }
11738 ModuleState::Failed
11739 };
11740 } else {
11741 state.state = ModuleState::Disabled;
11742 }
11743 })?;
11744 if let Some(schedule) = schedule {
11745 schedule_respawn(
11746 runtime,
11747 snapshot,
11748 &spec.module_id,
11749 schedule.delay,
11750 RespawnKind::Spawn,
11751 true,
11752 )?;
11753 } else {
11754 process_liveness.untrack_if_current(&spec.module_id, snapshot);
11755 }
11756 Err(SuperviseError::ReloadFailed {
11757 module_id: spec.module_id.clone(),
11758 reason,
11759 })
11760}
11761
11762fn control_flags() -> Flags {
11763 Flags::new(false, Priority::Passive, false)
11764}
11765
11766#[allow(clippy::too_many_arguments)]
11767async fn drain_optional_child(
11768 module_id: &str,
11769 protocol: ModuleProtocol,
11770 stop_notice: StopNotice,
11771 registry: &Registry,
11772 forwarding: Option<&ForwardingTable>,
11773 snapshot: &SharedSnapshot,
11774 terminal_ring: &Arc<Mutex<TerminalRing>>,
11775 spawn_events: &SpawnEventFeed,
11776 child: &mut Option<SupervisedChild>,
11777 drain_timeout: Duration,
11778 final_state: ModuleState,
11779 enabled: Option<bool>,
11780 failure: &FailureContext,
11781) -> Result<(), SuperviseError> {
11782 if let Some(child) = child.take() {
11783 drain_child_to_state(
11784 module_id,
11785 protocol,
11786 stop_notice,
11787 registry,
11788 forwarding,
11789 snapshot,
11790 terminal_ring,
11791 spawn_events,
11792 child,
11793 drain_timeout,
11794 final_state,
11795 enabled,
11796 failure,
11797 )
11798 .await
11799 } else {
11800 update_snapshot(snapshot, Some(module_id), |state| {
11801 state.state = final_state;
11802 if final_state == ModuleState::Failed {
11803 state.failed_retry.failed_at = Some(Instant::now());
11804 }
11805 if let Some(enabled) = enabled {
11806 state.enabled = enabled;
11807 }
11808 clear_current_process_facts(state);
11809 })?;
11810 release_dead_registration(registry, forwarding, snapshot, module_id, failure).await
11811 }
11812}
11813
11814#[allow(clippy::too_many_arguments)]
11815async fn drain_child_to_state(
11816 module_id: &str,
11817 _protocol: ModuleProtocol,
11818 stop_notice: StopNotice,
11819 registry: &Registry,
11820 forwarding: Option<&ForwardingTable>,
11821 snapshot: &SharedSnapshot,
11822 terminal_ring: &Arc<Mutex<TerminalRing>>,
11823 spawn_events: &SpawnEventFeed,
11824 mut child: SupervisedChild,
11825 drain_timeout: Duration,
11826 final_state: ModuleState,
11827 enabled: Option<bool>,
11828 failure: &FailureContext,
11829) -> Result<(), SuperviseError> {
11830 let protocol = child.protocol;
11831 update_snapshot(snapshot, Some(module_id), |state| {
11832 state.state = ModuleState::Draining;
11833 state.draining_to_replace = final_state == ModuleState::Restarting;
11834 if let Some(enabled) = enabled {
11835 state.enabled = enabled;
11836 }
11837 })?;
11838
11839 if stop_notice != StopNotice::SentOverConnection {
11850 if protocol == ModuleProtocol::Subc && stop_notice == StopNotice::NoConnection {
11851 info!(
11852 module_id,
11853 pid = child.pid,
11854 budget_ms = u64::try_from(drain_timeout.as_millis()).unwrap_or(u64::MAX),
11855 "module has no connection yet; requesting stop by signal"
11856 );
11857 }
11858 request_graceful_stop(module_id, &child);
11859 }
11860
11861 let exit_report = match timeout(drain_timeout, child.wait()).await {
11862 Ok(Ok(status)) => classify_reaped_child_exit(snapshot, &child, &status),
11863 Ok(Err(source)) => {
11864 fail_after_wait_error(
11865 module_id,
11866 snapshot,
11867 terminal_ring,
11868 spawn_events,
11869 format!("failed to wait during drain: {source}"),
11870 failure,
11871 );
11872 return Err(SuperviseError::Wait {
11873 module_id: module_id.to_string(),
11874 source,
11875 });
11876 }
11877 Err(_) => {
11878 warn!(
11891 module_id,
11892 pid = child.pid,
11893 budget_ms = u64::try_from(drain_timeout.as_millis()).unwrap_or(u64::MAX),
11894 reason = ?final_state,
11895 ?stop_notice,
11896 "drain budget expired before the module exited; killing it"
11897 );
11898 let kill =
11899 if final_state == ModuleState::Restarting && child.child.container().is_some() {
11900 child.kill_leader()
11901 } else {
11902 child.start_kill()
11903 };
11904 kill.map_err(|source| {
11905 fail_after_wait_error(
11906 module_id,
11907 snapshot,
11908 terminal_ring,
11909 spawn_events,
11910 format!("failed to kill during drain: {source}"),
11911 failure,
11912 );
11913 SuperviseError::Kill {
11914 module_id: module_id.to_string(),
11915 source,
11916 }
11917 })?;
11918 let status = child.wait().await.map_err(|source| {
11919 fail_after_wait_error(
11920 module_id,
11921 snapshot,
11922 terminal_ring,
11923 spawn_events,
11924 format!("failed to wait after drain kill: {source}"),
11925 failure,
11926 );
11927 SuperviseError::Wait {
11928 module_id: module_id.to_string(),
11929 source,
11930 }
11931 })?;
11932 classify_reaped_child_exit(snapshot, &child, &status)
11933 }
11934 };
11935
11936 let detail = lock_snapshot(snapshot)?.drain_disposition_detail.take();
11937 publish_after_terminal_record(
11938 module_id,
11939 || {
11940 record_terminal_with_detail(
11941 module_id,
11942 terminal_ring,
11943 spawn_events,
11944 &exit_report,
11945 terminal_disposition(final_state),
11946 detail,
11947 )
11948 },
11949 || {
11950 update_snapshot(snapshot, Some(module_id), |state| {
11951 state.state = final_state;
11952 if final_state == ModuleState::Failed {
11953 state.failed_retry.failed_at = Some(Instant::now());
11954 }
11955 if let Some(enabled) = enabled {
11956 state.enabled = enabled;
11957 }
11958 clear_current_process_facts(state);
11959 state.last_exit = Some(exit_report.clone());
11960 if exit_report.kind == ExitKind::DeliberateSeverance {
11961 state.lifetime_restarts += 1;
11962 }
11963 })
11964 },
11965 )?;
11966 child.drain_stderr(module_id).await;
11967
11968 let released =
11969 release_dead_registration(registry, forwarding, snapshot, module_id, failure).await;
11970 child
11971 .finish_exit(
11972 released.is_ok()
11973 && final_state == ModuleState::Restarting
11974 && child.spawn_failure.is_none(),
11975 )
11976 .await
11977 .map_err(|source| SuperviseError::Wait {
11978 module_id: module_id.to_owned(),
11979 source,
11980 })?;
11981 released
11982}
11983
11984#[cfg(unix)]
12004fn request_graceful_stop(module_id: &str, child: &SupervisedChild) {
12005 let Some(pid) = child
12006 .id()
12007 .and_then(|pid| i32::try_from(pid).ok())
12008 .and_then(rustix::process::Pid::from_raw)
12009 else {
12010 debug!(
12011 module_id,
12012 "no pid to signal for teardown; falling through to the drain wait"
12013 );
12014 return;
12015 };
12016 match rustix::process::kill_process(pid, rustix::process::Signal::TERM) {
12017 Ok(()) => debug!(
12018 module_id,
12019 "sent SIGTERM to a module nothing else asked to stop"
12020 ),
12021 Err(err) => debug!(
12022 module_id,
12023 error = %err,
12024 "SIGTERM to module failed; the drain wait and kill still apply"
12025 ),
12026 }
12027}
12028
12029#[cfg(not(unix))]
12037fn request_graceful_stop(module_id: &str, _child: &SupervisedChild) {
12038 debug!(
12039 module_id,
12040 "no graceful stop signal exists on this platform; teardown of a module nothing asked to stop waits, then kills"
12041 );
12042}
12043
12044fn terminal_disposition(final_state: ModuleState) -> TerminalDisposition {
12045 match final_state {
12046 ModuleState::Stopped => TerminalDisposition::Stopped,
12047 ModuleState::Disabled => TerminalDisposition::Disabled,
12048 ModuleState::Restarting => TerminalDisposition::Restarting,
12049 ModuleState::Failed => TerminalDisposition::Failed,
12050 ModuleState::Starting
12051 | ModuleState::Running
12052 | ModuleState::Unresponsive
12053 | ModuleState::Draining => {
12054 unreachable!("terminal exits only finish in terminal or restarting states")
12055 }
12056 }
12057}
12058
12059async fn release_dead_registration(
12067 registry: &Registry,
12068 forwarding: Option<&ForwardingTable>,
12069 snapshot: &SharedSnapshot,
12070 module_id: &str,
12071 failure: &FailureContext,
12072) -> Result<(), SuperviseError> {
12073 let result = async {
12074 let registration = registry
12075 .get_module(module_id)
12076 .map_err(SuperviseError::Registry)?;
12077 match wait_for_registration_release(registry, module_id, REGISTRY_RELEASE_TIMEOUT).await {
12078 Ok(()) => return Ok(()),
12079 Err(SuperviseError::RegistrationStillActive { .. }) => {}
12080 Err(err) => return Err(err),
12081 }
12082 let pid = lock_snapshot(snapshot)?.reaped_pid;
12083 if let (Some(registration), Some(forwarding), Some(pid)) = (registration, forwarding, pid) {
12084 warn!(
12085 module_id,
12086 pid,
12087 connection_id = registration.connection_id.get(),
12088 "reaped module registration outlived release grace; closing dead connection"
12089 );
12090 forwarding.request_connection_close(
12091 registration.connection_id,
12092 CloseReason::new(
12093 "supervised_process_reaped",
12094 format!("module '{module_id}' pid {pid} exited"),
12095 ),
12096 );
12097 wait_for_slot_registration_release(
12098 registry,
12099 crate::registry::RegistrationSlot::Connection(registration.connection_id),
12100 REGISTRY_RELEASE_TIMEOUT,
12101 )
12102 .await?;
12103 }
12104 wait_for_registration_release(registry, module_id, Duration::ZERO).await
12105 }
12106 .await;
12107 if let Err(err) = &result {
12108 fail_snapshot(snapshot, Some(module_id), None, failure);
12109 error!(module_id, error = %err, "registration release failed after child exit; module is failed and start can retry");
12110 }
12111 result
12112}
12113
12114async fn wait_for_registration_release(
12117 registry: &Registry,
12118 module_id: &str,
12119 wait: Duration,
12120) -> Result<(), SuperviseError> {
12121 wait_for_slot_registration_release(
12122 registry,
12123 crate::registry::RegistrationSlot::Active(module_id),
12124 wait,
12125 )
12126 .await
12127}
12128
12129async fn wait_for_slot_registration_release(
12137 registry: &Registry,
12138 slot: crate::registry::RegistrationSlot<'_>,
12139 wait: Duration,
12140) -> Result<(), SuperviseError> {
12141 let deadline = Instant::now() + wait;
12142 let mut release_events = registration_release_events().subscribe();
12143 let still_active = |registration: &crate::registry::ModuleRegistration| {
12144 SuperviseError::RegistrationStillActive {
12145 module_id: registration.manifest.module_id.clone(),
12146 waited: wait,
12147 }
12148 };
12149 loop {
12150 let _observed_generation = *release_events.borrow_and_update();
12151 let Some(registration) = registry
12152 .registration(slot)
12153 .map_err(SuperviseError::Registry)?
12154 else {
12155 return Ok(());
12156 };
12157
12158 let now = Instant::now();
12159 if now >= deadline {
12160 return Err(still_active(®istration));
12161 }
12162
12163 let remaining = deadline.saturating_duration_since(now);
12164 match timeout(remaining, release_events.changed()).await {
12165 Ok(Ok(())) | Ok(Err(_)) => {}
12166 Err(_) => return Err(still_active(®istration)),
12167 }
12168 }
12169}
12170
12171#[cfg(test)]
12172mod slot_registration_wait_tests {
12173 use super::*;
12174 use crate::registry::{ConnectionId, RegistrationSlot};
12175 use subc_protocol::manifest::ModuleManifest;
12176
12177 #[tokio::test]
12178 async fn enable_release_failure_is_failed_and_a_second_enable_retries() {
12179 let registry = Arc::new(Registry::default());
12180 let supervisor = Supervisor::new_for_test(Arc::clone(®istry), RestartPolicy::default());
12181 let runtime = supervisor.runtime_config();
12182 let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::disabled()));
12183 let spec = ModuleSpec {
12184 module_id: "enable-stale-registration".to_string(),
12185 program: PathBuf::from("/missing/enable-retry-test"),
12186 args: Vec::new(),
12187 env: Vec::new(),
12188 reserved: false,
12189 reserved_prefixes: Vec::new(),
12190 protocol: ModuleProtocol::Subc,
12191 overlap: Default::default(),
12192 carry_containment: false,
12193 };
12194 let connection = ConnectionId::new(90);
12195 registry
12196 .register_with_control_ops(
12197 ModuleManifest::builder(&spec.module_id, "0.1.0").build(),
12198 1,
12199 connection,
12200 Vec::new(),
12201 )
12202 .unwrap();
12203 let mut child = None;
12204 let err = set_child_enabled(
12205 &spec,
12206 &runtime,
12207 ®istry,
12208 &supervisor.process_liveness,
12209 &snapshot,
12210 &mut child,
12211 true,
12212 &runtime.failure_context(false),
12213 )
12214 .await
12215 .unwrap_err();
12216 assert!(matches!(
12217 err,
12218 SuperviseError::RegistrationStillActive { .. }
12219 ));
12220 assert_eq!(lock_snapshot(&snapshot).unwrap().state, ModuleState::Failed);
12221 assert!(child.is_none());
12222 registry.deregister_connection(connection).unwrap();
12223 let err = set_child_enabled(
12224 &spec,
12225 &runtime,
12226 ®istry,
12227 &supervisor.process_liveness,
12228 &snapshot,
12229 &mut child,
12230 true,
12231 &runtime.failure_context(false),
12232 )
12233 .await
12234 .unwrap_err();
12235 assert!(
12236 matches!(err, SuperviseError::Spawn { .. }),
12237 "second enable must attempt a spawn: {err}"
12238 );
12239 assert_eq!(lock_snapshot(&snapshot).unwrap().state, ModuleState::Failed);
12240 }
12241
12242 const INCUMBENT: u64 = 1;
12243 const CANDIDATE: u64 = 2;
12244
12245 fn swapped_registry() -> Arc<Registry> {
12246 let registry = Arc::new(Registry::default());
12247 let manifest = ModuleManifest::builder("m", "0.1.0").build();
12248 registry
12249 .register_with_control_ops(
12250 manifest.clone(),
12251 1,
12252 ConnectionId::new(INCUMBENT),
12253 Vec::new(),
12254 )
12255 .unwrap();
12256 registry
12257 .register_candidate_with_control_ops(
12258 manifest,
12259 1,
12260 ConnectionId::new(CANDIDATE),
12261 Vec::new(),
12262 )
12263 .unwrap();
12264 registry
12265 }
12266
12267 #[tokio::test]
12271 async fn incumbent_release_is_awaited_by_connection_not_by_module_id() {
12272 let registry = swapped_registry();
12273 registry.promote_candidate("m").unwrap().unwrap();
12274
12275 assert!(matches!(
12276 wait_for_registration_release(®istry, "m", Duration::from_millis(50)).await,
12277 Err(SuperviseError::RegistrationStillActive { .. })
12278 ));
12279
12280 assert!(matches!(
12282 wait_for_slot_registration_release(
12283 ®istry,
12284 RegistrationSlot::Connection(ConnectionId::new(INCUMBENT)),
12285 Duration::from_millis(50),
12286 )
12287 .await,
12288 Err(SuperviseError::RegistrationStillActive { .. })
12289 ));
12290
12291 let releaser = Arc::clone(®istry);
12292 let release = tokio::spawn(async move {
12293 sleep(Duration::from_millis(20)).await;
12294 releaser
12295 .deregister_connection(ConnectionId::new(INCUMBENT))
12296 .unwrap();
12297 notify_registration_release();
12298 });
12299 wait_for_slot_registration_release(
12300 ®istry,
12301 RegistrationSlot::Connection(ConnectionId::new(INCUMBENT)),
12302 Duration::from_secs(5),
12303 )
12304 .await
12305 .expect("the incumbent's own registration is released");
12306 release.await.unwrap();
12307 assert!(registry.get_module("m").unwrap().is_some());
12308 }
12309
12310 #[tokio::test]
12313 async fn candidate_slot_wait_ignores_the_incumbents_registration() {
12314 let registry = swapped_registry();
12315 assert!(matches!(
12316 wait_for_slot_registration_release(
12317 ®istry,
12318 RegistrationSlot::Candidate("m"),
12319 Duration::from_millis(50),
12320 )
12321 .await,
12322 Err(SuperviseError::RegistrationStillActive { .. })
12323 ));
12324 registry
12325 .deregister_connection(ConnectionId::new(CANDIDATE))
12326 .unwrap();
12327 wait_for_slot_registration_release(
12328 ®istry,
12329 RegistrationSlot::Candidate("m"),
12330 Duration::from_millis(50),
12331 )
12332 .await
12333 .expect("a candidate slot with no candidate is released");
12334 assert!(registry
12335 .registration(RegistrationSlot::Active("m"))
12336 .unwrap()
12337 .is_some());
12338 }
12339}
12340
12341fn classify_exit(status: &ExitStatus) -> ExitReport {
12342 ExitReport {
12343 kind: if status.success() {
12344 ExitKind::Clean
12345 } else {
12346 ExitKind::Crash
12347 },
12348 code: status.code(),
12349 signal: exit_signal(status),
12350 at_ms: unix_ms_now(),
12351 }
12352}
12353
12354fn wait_error_exit_report() -> ExitReport {
12360 ExitReport {
12361 kind: ExitKind::Crash,
12362 code: None,
12363 signal: None,
12364 at_ms: unix_ms_now(),
12365 }
12366}
12367
12368#[cfg(unix)]
12369fn exit_signal(status: &ExitStatus) -> Option<i32> {
12370 use std::os::unix::process::ExitStatusExt;
12371
12372 status.signal()
12373}
12374
12375#[cfg(not(unix))]
12376fn exit_signal(_status: &ExitStatus) -> Option<i32> {
12377 None
12378}
12379
12380fn reset_restart_count(snapshot: &SharedSnapshot, module_id: &str) -> Result<(), SuperviseError> {
12386 update_snapshot(snapshot, Some(module_id), |state| {
12387 state.clear_crash_restarts();
12388 })
12389}
12390
12391fn set_running(
12392 snapshot: &SharedSnapshot,
12393 child: &SupervisedChild,
12394 module_id: &str,
12395 spawn_events: &SpawnEventFeed,
12396) -> Result<(), SuperviseError> {
12397 let mut state = snapshot.lock().map_err(|_| SuperviseError::StatePoisoned {
12398 module_id: Some(module_id.to_string()),
12399 })?;
12400 state.spawn_generation = spawn_events.emit_spawned(module_id, child.pid, child.spawned_at_ms);
12401 if std::mem::take(&mut state.coalesced_restart_pending) {
12402 let generation = state.spawn_generation;
12403 info!(module_id, "restart already satisfied by generation {generation}; coalesced pending request completed");
12404 }
12405 state.drain_disposition_detail = None;
12406 state.spawn_failure = None;
12407 state.waiting_for_previous_pid = None;
12408 state.waiting_for_previous_detail = None;
12409 state.in_alternate_slot = false;
12412 state.configuration_updated_since_spawn = false;
12413 state.spawned_protocol = Some(child.protocol);
12414 state.state = ModuleState::Running;
12415 state.enabled = true;
12416 state.process_alive = true;
12417 state.pid = child.id();
12418 state.subtree_target = child.subtree_target.clone();
12419 state.health.status = SupervisorHealthStatus::Unknown;
12422 state.health.consecutive_failures = 0;
12423 state.health.detail = None;
12424 state.health.metrics = None;
12425 state.health.last_probe_ms = None;
12426 state.failed_retry.healthy_since = None;
12427 state.failed_retry.failed_at = None;
12428 state.failed_retry.running_since = Some(child.spawned_at);
12429 #[cfg(windows)]
12430 {
12431 state.pipe_consumed = child
12432 .nonce_delivery
12433 .as_ref()
12434 .map(|delivery| delivery.consumed());
12435 }
12436 #[cfg(target_os = "macos")]
12437 {
12438 state.report_ready = Some(Arc::clone(&child.report_ready));
12439 }
12440 state.spawned_at_ms = Some(child.spawned_at_ms);
12441 state.exec_outcome = Some(Arc::clone(&child.exec.0));
12442 state.spawned_from = Some(child.spawned_from.clone());
12443 state.launch_spec_digest = Some(child.launch_spec_digest.clone());
12444 state.spawned_file_identity = child.spawned_file_identity.clone();
12445 state.process_start_time = child.process_start_time;
12446 Ok(())
12447}
12448
12449fn clear_current_process_facts(state: &mut SupervisorSnapshot) {
12450 state.subtree_target = crate::subtree_resources::Target::Unavailable("not_running");
12451 state.failed_retry.healthy_since = None;
12452 state.failed_retry.running_since = None;
12453 state.waiting_for_previous_pid = None;
12454 state.waiting_for_previous_detail = None;
12455 state.process_alive = false;
12456 state.spawned_protocol = None;
12457 state.pid = None;
12458 #[cfg(windows)]
12459 {
12460 state.pipe_consumed = None;
12461 }
12462 #[cfg(target_os = "macos")]
12463 {
12464 state.report_ready = None;
12465 }
12466 state.spawned_at_ms = None;
12467 state.exec_outcome = None;
12468 state.spawned_from = None;
12469 state.launch_spec_digest = None;
12470 state.spawned_file_identity = None;
12471 state.process_start_time = None;
12472 state.deliberate_severance = None;
12473}
12474
12475#[cfg(test)]
12476fn record_deliberate_severance(
12477 snapshot: &SharedSnapshot,
12478 identity: ProcessIdentity,
12479) -> Result<(), SuperviseError> {
12480 update_snapshot(snapshot, None, |state| {
12481 state.deliberate_severance = Some(identity);
12482 })
12483}
12484
12485fn apply_deliberate_severance_marker(
12486 snapshot: &SharedSnapshot,
12487 exited_identity: Option<ProcessIdentity>,
12488 mut exit_report: ExitReport,
12489) -> ExitReport {
12490 let marker = lock_snapshot(snapshot)
12491 .ok()
12492 .and_then(|mut state| state.deliberate_severance.take());
12493 if marker.is_some() && marker == exited_identity {
12494 exit_report.kind = ExitKind::DeliberateSeverance;
12495 }
12496 exit_report
12497}
12498
12499fn classify_reaped_child_exit(
12500 snapshot: &SharedSnapshot,
12501 child: &SupervisedChild,
12502 status: &ExitStatus,
12503) -> ExitReport {
12504 let _ = update_snapshot(snapshot, None, |state| {
12505 state.reaped_pid = Some(child.pid);
12506 state.spawn_failure = child.spawn_failure.clone();
12507 });
12508 if child.spawn_failure.as_deref() == Some("registration_timeout") {
12509 registration_failure_exit_report(classify_exit(status))
12510 } else {
12511 apply_deliberate_severance_marker(snapshot, child.process_identity(), classify_exit(status))
12512 }
12513}
12514
12515fn fail_snapshot(
12516 snapshot: &SharedSnapshot,
12517 module_id: Option<&str>,
12518 last_exit: Option<ExitReport>,
12519 failure: &FailureContext,
12520) {
12521 if let Err(err) = update_snapshot(snapshot, module_id, |state| {
12522 state.state = ModuleState::Failed;
12523 clear_current_process_facts(state);
12524 if failure.may_retry() && state.enabled {
12525 state.failed_retry.schedule(module_id.unwrap_or("unknown"));
12526 } else if failure.shutdown.is_set() {
12527 state.failed_retry.cancel();
12528 }
12529 if let Some(last_exit) = last_exit {
12530 state.last_exit = Some(last_exit);
12531 }
12532 }) {
12533 error!(error = %err, "failed to mark supervisor state failed");
12534 }
12535}
12536
12537fn update_snapshot(
12538 snapshot: &SharedSnapshot,
12539 module_id: Option<&str>,
12540 update: impl FnOnce(&mut SupervisorSnapshot),
12541) -> Result<(), SuperviseError> {
12542 let mut state = snapshot.lock().map_err(|_| SuperviseError::StatePoisoned {
12543 module_id: module_id.map(ToOwned::to_owned),
12544 })?;
12545 update(&mut state);
12546 Ok(())
12547}
12548
12549const SLOW_SNAPSHOT_LOCK_THRESHOLD: Duration = Duration::from_millis(250);
12550
12551fn lock_snapshot_for_control<'a>(
12552 snapshot: &'a SharedSnapshot,
12553 module_id: &str,
12554 caller: &'static str,
12555) -> Result<std::sync::MutexGuard<'a, SupervisorSnapshot>, SuperviseError> {
12556 let started_at = Instant::now();
12557 let guard = lock_snapshot(snapshot)?;
12558 let waited = started_at.elapsed();
12559 if waited >= SLOW_SNAPSHOT_LOCK_THRESHOLD {
12560 warn!(
12561 module_id = %module_id,
12562 waited_ms = waited.as_millis() as u64,
12563 caller = %caller,
12564 "slow snapshot lock"
12565 );
12566 }
12567 Ok(guard)
12568}
12569
12570fn lock_snapshot(
12571 snapshot: &SharedSnapshot,
12572) -> Result<std::sync::MutexGuard<'_, SupervisorSnapshot>, SuperviseError> {
12573 snapshot
12574 .lock()
12575 .map_err(|_| SuperviseError::StatePoisoned { module_id: None })
12576}
12577
12578#[cfg(test)]
12579mod terminal_history_tests {
12580 use std::{
12581 path::PathBuf,
12582 sync::Arc,
12583 time::{Duration, Instant},
12584 };
12585
12586 use tokio::{sync::mpsc, time::sleep};
12587
12588 use super::{
12589 apply_deliberate_severance_marker, daemon_will_restart, drain_child_to_state,
12590 drained_after_quiescence_wait, handle_reload_spawn_failure, health_restart_child,
12591 lock_snapshot, on_child_exit, record_deliberate_severance, record_wait_error_terminal,
12592 reset_restart_count, spawn_and_mark_running, update_snapshot, wait_error_exit_report,
12593 ExitKind, ExitReport, ModuleProtocol, ModuleSpec, ModuleState, NextAction, ProcessIdentity,
12594 RestartPolicy, SpawnEventKind, StopNotice, SuperviseError, SupervisedConfiguration,
12595 SupervisedModule, SupervisedModuleInner, Supervisor, SupervisorHandle,
12596 SupervisorHealthStatus, SupervisorSnapshot,
12597 };
12598 use super::Instant as ClockInstant;
12603 use crate::{
12604 registry::Registry,
12605 terminal_ring::{TerminalRing, TerminalRingConfig},
12606 };
12607 use std::sync::Mutex;
12608 use subc_control::TerminalDisposition;
12609
12610 pub(super) fn fake_aft_stub_path() -> PathBuf {
12615 let mut path = std::env::current_exe().expect("current_exe available in tests");
12616 path.pop();
12617 path.pop();
12618 path.push(if cfg!(windows) {
12619 "fake-aft-stub.exe"
12620 } else {
12621 "fake-aft-stub"
12622 });
12623 assert!(
12624 path.exists(),
12625 "fake-aft-stub not built at {}: run `cargo test -p subc-core` (which builds \
12626 [[bin]] targets) rather than `cargo test -p subc-core --lib` (which does not)",
12627 path.display()
12628 );
12629 path
12630 }
12631
12632 #[test]
12633 fn reserved_never_spawned_refuses_every_hello() {
12634 let supervisor = SupervisorHandle::default();
12639 supervisor.apply_identity_configuration(&ModuleSpec {
12640 module_id: "never-spawned".to_string(),
12641 program: PathBuf::from("/usr/bin/false"),
12642 args: Vec::new(),
12643 env: Vec::new(),
12644 reserved: true,
12645 reserved_prefixes: Vec::new(),
12646 protocol: ModuleProtocol::Subc,
12647 overlap: Default::default(),
12648 carry_containment: false,
12649 });
12650 assert!(
12651 supervisor
12652 .reserved_hello_verification("never-spawned", Some("any-forged-nonce"))
12653 .is_err(),
12654 "forged nonce must refuse on a reserved never-spawned id"
12655 );
12656 assert!(
12657 supervisor
12658 .reserved_hello_verification("never-spawned", None)
12659 .is_err(),
12660 "absent nonce must refuse on a reserved never-spawned id"
12661 );
12662 supervisor.set_spawn_nonce("never-spawned", "minted".to_string());
12664 supervisor.apply_identity_configuration(&ModuleSpec {
12665 module_id: "never-spawned".to_string(),
12666 program: PathBuf::from("/usr/bin/false"),
12667 args: Vec::new(),
12668 env: Vec::new(),
12669 reserved: true,
12670 reserved_prefixes: Vec::new(),
12671 protocol: ModuleProtocol::Subc,
12672 overlap: Default::default(),
12673 carry_containment: false,
12674 });
12675 assert!(supervisor
12676 .reserved_hello_verification("never-spawned", Some("minted"))
12677 .is_ok());
12678 assert!(supervisor
12679 .reserved_hello_verification("never-spawned", Some("forged"))
12680 .is_err());
12681 }
12682
12683 fn seed_crash_restarts(state: &mut SupervisorSnapshot, count: u32) {
12686 let now = ClockInstant::now();
12687 for _ in 0..count {
12688 state.crash_restarts.push_back(now);
12689 }
12690 }
12691
12692 fn age_oldest_crash_restart_out_of_window(state: &mut SupervisorSnapshot, window: Duration) {
12696 let aged = state
12697 .crash_restarts
12698 .front()
12699 .expect("a crash restart must be recorded before it can be aged")
12700 .checked_sub(window + Duration::from_secs(1))
12701 .expect("the test clock is far enough from its origin to age an instant");
12702 state.crash_restarts[0] = aged;
12703 }
12704
12705 fn snapshot_with_restarts(enabled: bool, count: u32) -> SupervisorSnapshot {
12706 let mut state = SupervisorSnapshot::new(ModuleState::Running, enabled);
12707 seed_crash_restarts(&mut state, count);
12708 state
12709 }
12710
12711 #[test]
12712 fn daemon_owned_recovery_predicate_uses_the_pre_increment_budget() {
12713 let policy = RestartPolicy::new(3, Duration::ZERO);
12714 let now = ClockInstant::now();
12715 assert!(daemon_will_restart(
12716 &mut snapshot_with_restarts(true, 2),
12717 &policy,
12718 now
12719 ));
12720 assert!(!daemon_will_restart(
12721 &mut snapshot_with_restarts(true, 3),
12722 &policy,
12723 now
12724 ));
12725 assert!(!daemon_will_restart(
12726 &mut snapshot_with_restarts(false, 0),
12727 &policy,
12728 now
12729 ));
12730 }
12731
12732 #[test]
12733 fn crash_restart_backoff_escalates_with_in_window_count() {
12734 let policy = RestartPolicy::new(4, Duration::from_millis(100))
12735 .with_max_backoff(Duration::from_secs(30));
12736 let now = ClockInstant::now();
12737 let mut state = SupervisorSnapshot::new(ModuleState::Running, true);
12738 let schedules = (0..4)
12739 .map(|_| {
12740 state
12741 .next_crash_restart(&policy, now)
12742 .expect("the test policy allows four crash restarts")
12743 })
12744 .collect::<Vec<_>>();
12745
12746 assert_eq!(
12747 schedules
12748 .iter()
12749 .map(|schedule| schedule.restart_in_window)
12750 .collect::<Vec<_>>(),
12751 vec![0, 1, 2, 3]
12752 );
12753 assert_eq!(
12754 schedules
12755 .iter()
12756 .map(|schedule| schedule.delay)
12757 .collect::<Vec<_>>(),
12758 vec![
12759 Duration::from_millis(100),
12760 Duration::from_secs(1),
12761 Duration::from_secs(10),
12762 Duration::from_secs(30),
12763 ]
12764 );
12765 }
12766
12767 #[test]
12768 fn crash_restart_backoff_resets_after_ring_clear() {
12769 let policy = RestartPolicy::new(3, Duration::from_millis(100));
12770 let now = ClockInstant::now();
12771 let mut state = SupervisorSnapshot::new(ModuleState::Running, true);
12772 assert_eq!(
12773 state.next_crash_restart(&policy, now).unwrap().delay,
12774 Duration::from_millis(100)
12775 );
12776 assert_eq!(
12777 state.next_crash_restart(&policy, now).unwrap().delay,
12778 Duration::from_secs(1)
12779 );
12780
12781 state.clear_crash_restarts();
12782 let schedule = state
12783 .next_crash_restart(&policy, now)
12784 .expect("a cleared ring must allow another restart");
12785 assert_eq!(schedule.restart_in_window, 0);
12786 assert_eq!(schedule.delay, Duration::from_millis(100));
12787 }
12788
12789 #[test]
12790 fn crash_restart_backoff_ignores_aged_restarts() {
12791 let policy = RestartPolicy::new(3, Duration::from_millis(100));
12792 let now = ClockInstant::now();
12793 let mut state = SupervisorSnapshot::new(ModuleState::Running, true);
12794 state
12795 .next_crash_restart(&policy, now)
12796 .expect("the first restart is allowed");
12797 state
12798 .next_crash_restart(&policy, now)
12799 .expect("the second restart is allowed");
12800 state.crash_restarts[0] = now
12801 .checked_sub(policy.window + Duration::from_secs(1))
12802 .expect("the fake clock can age a restart past the window");
12803
12804 let schedule = state
12805 .next_crash_restart(&policy, now)
12806 .expect("an aged restart must release its slot");
12807 assert_eq!(schedule.restart_in_window, 1);
12808 assert_eq!(schedule.delay, Duration::from_secs(1));
12809 assert_eq!(state.crash_restarts.len(), 2);
12810 }
12811
12812 #[test]
12816 fn a_budget_spent_before_the_window_no_longer_refuses() {
12817 let policy = RestartPolicy::new(3, Duration::ZERO);
12818 let mut state = snapshot_with_restarts(true, 3);
12819 let now = ClockInstant::now();
12820 assert!(!daemon_will_restart(&mut state, &policy, now));
12821
12822 assert!(daemon_will_restart(
12823 &mut state,
12824 &policy,
12825 now + policy.window + Duration::from_secs(1)
12826 ));
12827 assert!(
12828 state.crash_restarts.is_empty(),
12829 "reading the budget must drop the instants that left the window"
12830 );
12831 }
12832
12833 fn module_with_recovery_snapshot(
12834 state: ModuleState,
12835 enabled: bool,
12836 restart_count: u32,
12837 ) -> SupervisedModule {
12838 let registry = Arc::new(Registry::default());
12839 let supervisor =
12840 Supervisor::new_for_test(Arc::clone(®istry), RestartPolicy::new(3, Duration::ZERO));
12841 let runtime = supervisor.runtime_config();
12842 let spec = ModuleSpec {
12843 module_id: "recovery-snapshot".to_string(),
12844 program: fake_aft_stub_path(),
12845 args: Vec::new(),
12846 env: Vec::new(),
12847 reserved: false,
12848 reserved_prefixes: Vec::new(),
12849 protocol: ModuleProtocol::Subc,
12850 overlap: Default::default(),
12851 carry_containment: false,
12852 };
12853 let mut snapshot = SupervisorSnapshot::new(state, enabled);
12854 seed_crash_restarts(&mut snapshot, restart_count);
12855 let (commands, _rx) = mpsc::channel(4);
12858 SupervisedModule {
12859 inner: Arc::new(SupervisedModuleInner {
12860 module_id: spec.module_id.clone(),
12861 registry,
12862 snapshot: Arc::new(Mutex::new(snapshot)),
12863 configuration: Arc::new(Mutex::new(SupervisedConfiguration {
12864 spec,
12865 health: runtime.health,
12866 })),
12867 stderr_ring: runtime.stderr_ring,
12868 terminal_ring: runtime.terminal_ring,
12869 commands,
12870 monitor: Mutex::new(None),
12871 restart_policy: runtime.restart_policy,
12872 effective_drain_timeout: runtime.effective_drain_timeout,
12873 provenance_probe: supervisor.provenance_probe.clone(),
12874 retained: runtime.child_roster.retained_view(),
12875 }),
12876 }
12877 }
12878
12879 #[cfg(target_os = "linux")]
12880 #[tokio::test]
12881 async fn no_cgroup_placement_does_not_block_fake_aft_stub_spawn() {
12882 let supervisor =
12883 Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default())
12884 .with_cgroup_placement(None);
12885 let result = supervisor.spawn(ModuleSpec {
12886 module_id: "no-cgroup-placement".to_string(),
12887 program: fake_aft_stub_path(),
12888 args: Vec::new(),
12889 env: Vec::new(),
12890 reserved: false,
12891 reserved_prefixes: Vec::new(),
12892 protocol: ModuleProtocol::Subc,
12893 overlap: Default::default(),
12894 carry_containment: false,
12895 });
12896
12897 assert!(
12898 result.is_ok(),
12899 "no delegation must not turn an otherwise valid spawn into a failure: {result:?}"
12900 );
12901 }
12902
12903 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
12904 async fn undecided_snapshot_uses_shared_restart_predicate() {
12905 assert!(module_with_recovery_snapshot(ModuleState::Running, true, 2)
12906 .will_recover_after_connection_loss()
12907 .unwrap());
12908 assert!(
12909 !module_with_recovery_snapshot(ModuleState::Running, true, 3)
12910 .will_recover_after_connection_loss()
12911 .unwrap()
12912 );
12913 }
12914
12915 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
12916 async fn restarting_snapshot_at_exhausted_budget_is_non_terminal() {
12917 assert!(
12918 module_with_recovery_snapshot(ModuleState::Restarting, true, 3)
12919 .will_recover_after_connection_loss()
12920 .unwrap()
12921 );
12922 }
12923
12924 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
12925 async fn terminal_phase_snapshots_are_terminal_before_budget_exhaustion() {
12926 assert!(!module_with_recovery_snapshot(ModuleState::Failed, true, 0)
12927 .will_recover_after_connection_loss()
12928 .unwrap());
12929 assert!(
12930 !module_with_recovery_snapshot(ModuleState::Disabled, true, 0)
12931 .will_recover_after_connection_loss()
12932 .unwrap()
12933 );
12934 }
12935
12936 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
12937 async fn warming_snapshot_is_limited_to_startup_phases() {
12938 for state in [
12939 ModuleState::Starting,
12940 ModuleState::Running,
12941 ModuleState::Restarting,
12942 ] {
12943 assert!(
12944 module_with_recovery_snapshot(state, true, 0)
12945 .is_warming()
12946 .unwrap(),
12947 "{state:?} should be warming"
12948 );
12949 }
12950 for state in [
12951 ModuleState::Unresponsive,
12952 ModuleState::Draining,
12953 ModuleState::Stopped,
12954 ModuleState::Failed,
12955 ModuleState::Disabled,
12956 ] {
12957 assert!(
12958 !module_with_recovery_snapshot(state, true, 0)
12959 .is_warming()
12960 .unwrap(),
12961 "{state:?} should not be warming"
12962 );
12963 }
12964 }
12965
12966 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
12967 async fn terminal_history_survives_respawn_and_keeps_both_crashes_in_order() {
12968 let registry = Arc::new(Registry::default());
12969 let supervisor =
12970 Supervisor::new_for_test(Arc::clone(®istry), RestartPolicy::new(1, Duration::ZERO));
12971 let module = supervisor
12972 .spawn(ModuleSpec {
12973 module_id: "terminal-history".to_string(),
12974 program: fake_aft_stub_path(),
12975 args: Vec::new(),
12976 env: vec![("FAKE_AFT_EXIT_CODE".to_string(), "23".to_string())],
12977 reserved: false,
12978 reserved_prefixes: Vec::new(),
12979 protocol: ModuleProtocol::Subc,
12980 overlap: Default::default(),
12981 carry_containment: false,
12982 })
12983 .unwrap();
12984
12985 let deadline = Instant::now() + Duration::from_secs(5);
12986 loop {
12987 let history = module.terminal_history();
12988 if history.entries.len() == 2 {
12989 assert_eq!(module.status().unwrap().state, ModuleState::Failed);
12990 assert_eq!(history.dropped, 0);
12991 assert_eq!(
12992 history
12993 .entries
12994 .iter()
12995 .map(|entry| entry.exit_code)
12996 .collect::<Vec<_>>(),
12997 vec![Some(23), Some(23)]
12998 );
12999 assert!(history.entries[0].at_ms <= history.entries[1].at_ms);
13000 return;
13001 }
13002 assert!(
13003 Instant::now() < deadline,
13004 "module did not retain two terminal exits: {history:?}"
13005 );
13006 sleep(Duration::from_millis(10)).await;
13007 }
13008 }
13009
13010 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
13014 async fn disable_during_crash_backoff_cancels_pending_respawn() {
13015 let backoff = Duration::from_secs(2);
13016 let supervisor = Supervisor::new_for_test(
13017 Arc::new(Registry::default()),
13018 RestartPolicy::new(10, backoff),
13019 );
13020 let module = supervisor
13021 .spawn(ModuleSpec {
13022 module_id: "disable-during-backoff".to_string(),
13023 program: fake_aft_stub_path(),
13024 args: Vec::new(),
13025 env: vec![("FAKE_AFT_EXIT_CODE".to_string(), "23".to_string())],
13026 reserved: false,
13027 reserved_prefixes: Vec::new(),
13028 protocol: ModuleProtocol::Subc,
13029 overlap: Default::default(),
13030 carry_containment: false,
13031 })
13032 .unwrap();
13033
13034 let deadline = Instant::now() + Duration::from_secs(5);
13036 loop {
13037 if module.status().unwrap().state == ModuleState::Restarting {
13038 break;
13039 }
13040 assert!(
13041 Instant::now() < deadline,
13042 "module never entered the crash backoff"
13043 );
13044 sleep(Duration::from_millis(10)).await;
13045 }
13046
13047 let started = Instant::now();
13048 module.set_enabled(false).await.unwrap();
13049 let waited = started.elapsed();
13050
13051 assert!(
13052 waited < backoff / 2,
13053 "disable waited {waited:?} behind the {backoff:?} crash backoff; the operator command must preempt the pending respawn"
13054 );
13055 assert_eq!(module.status().unwrap().state, ModuleState::Disabled);
13056
13057 sleep(backoff + Duration::from_millis(500)).await;
13059 let status = module.status().unwrap();
13060 assert_eq!(status.state, ModuleState::Disabled);
13061 assert_eq!(
13062 status.spawn_generation, 1,
13063 "module respawned after the operator disabled it"
13064 );
13065 }
13066
13067 #[cfg(unix)]
13070 fn protocol_none_sigterm_exits_clean_spec(
13071 module_id: &str,
13072 dir: &std::path::Path,
13073 ) -> (ModuleSpec, PathBuf, PathBuf) {
13074 let ready = dir.join("ready");
13075 let marker = dir.join("sigterm");
13076 let spec = ModuleSpec {
13077 module_id: module_id.to_string(),
13078 program: fake_aft_stub_path(),
13079 args: Vec::new(),
13080 env: vec![
13081 ("FAKE_AFT_NEVER_CONNECT".to_string(), "1".to_string()),
13082 (
13083 "FAKE_AFT_SIGTERM_MARKER_PATH".to_string(),
13084 marker.display().to_string(),
13085 ),
13086 (
13087 "FAKE_AFT_NEVER_CONNECT_READY_PATH".to_string(),
13088 ready.display().to_string(),
13089 ),
13090 ],
13091 reserved: false,
13092 reserved_prefixes: Vec::new(),
13093 protocol: ModuleProtocol::None,
13094 overlap: Default::default(),
13095 carry_containment: false,
13096 };
13097 (spec, ready, marker)
13098 }
13099
13100 #[cfg(unix)]
13104 async fn wait_for_file(path: &std::path::Path) {
13105 let deadline = Instant::now() + Duration::from_secs(10);
13106 while !path.exists() {
13107 assert!(
13108 Instant::now() < deadline,
13109 "{} never appeared",
13110 path.display()
13111 );
13112 sleep(Duration::from_millis(10)).await;
13113 }
13114 }
13115
13116 #[cfg(unix)]
13120 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
13121 async fn protocol_none_unrequested_clean_exit_restarts_as_a_crash() {
13122 let dir = cortexkit_test_support::ScratchDir::new("none-unrequested-clean-exit");
13123 let (spec, ready, marker) =
13124 protocol_none_sigterm_exits_clean_spec("none-unrequested-clean-exit", dir.path());
13125 let supervisor = Supervisor::new_for_test(
13126 Arc::new(Registry::default()),
13127 RestartPolicy::new(3, Duration::ZERO),
13128 );
13129 let module = supervisor.spawn(spec).unwrap();
13130 wait_for_file(&ready).await;
13131 let deadline = Instant::now() + Duration::from_secs(10);
13135 let first_pid = loop {
13136 let status = module.status().unwrap();
13137 if status.state == ModuleState::Running {
13138 if let Some(pid) = status.pid {
13139 break pid;
13140 }
13141 }
13142 assert!(
13143 Instant::now() < deadline,
13144 "a running module must report its pid after exec confirmation: {status:?}"
13145 );
13146 sleep(Duration::from_millis(10)).await;
13147 };
13148
13149 rustix::process::kill_process(
13150 rustix::process::Pid::from_raw(i32::try_from(first_pid).unwrap()).unwrap(),
13151 rustix::process::Signal::TERM,
13152 )
13153 .unwrap();
13154
13155 let deadline = Instant::now() + Duration::from_secs(10);
13156 let respawned = loop {
13157 let status = module.status().unwrap();
13158 if status.state == ModuleState::Running
13159 && status.pid.is_some_and(|pid| pid != first_pid)
13160 {
13161 break status;
13162 }
13163 assert!(
13164 Instant::now() < deadline,
13165 "protocol-none module was not respawned after an unrequested clean exit: {status:?}"
13166 );
13167 sleep(Duration::from_millis(10)).await;
13168 };
13169 assert_eq!(respawned.spawn_generation, 2);
13170 assert!(
13171 marker.exists(),
13172 "the child must have exited through its SIGTERM handler (exit 0), or this proves nothing about clean exits"
13173 );
13174
13175 let history = module.terminal_history();
13176 assert_eq!(history.entries.len(), 1, "{history:?}");
13177 let entry = &history.entries[0];
13178 assert_eq!(entry.exit_code, Some(0));
13179 assert_eq!(entry.exit_kind, subc_control::TerminalExitKind::Clean);
13180 assert_eq!(entry.disposition, TerminalDisposition::Restarting);
13181
13182 module.stop().await.unwrap();
13183 }
13184
13185 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
13189 async fn protocol_none_repeated_clean_exits_exhaust_the_restart_budget() {
13190 let supervisor = Supervisor::new_for_test(
13191 Arc::new(Registry::default()),
13192 RestartPolicy::new(1, Duration::ZERO),
13193 );
13194 let module = supervisor
13195 .spawn(ModuleSpec {
13196 module_id: "none-clean-exit-budget".to_string(),
13197 program: fake_aft_stub_path(),
13198 args: Vec::new(),
13199 env: vec![("FAKE_AFT_EXIT_CODE".to_string(), "0".to_string())],
13200 reserved: false,
13201 reserved_prefixes: Vec::new(),
13202 protocol: ModuleProtocol::None,
13203 overlap: Default::default(),
13204 carry_containment: false,
13205 })
13206 .unwrap();
13207
13208 let deadline = Instant::now() + Duration::from_secs(10);
13211 loop {
13212 let status = module.status().unwrap();
13213 if status.state == ModuleState::Failed {
13214 break;
13215 }
13216 assert!(
13217 Instant::now() < deadline,
13218 "module never exhausted its budget: {status:?} {:?}",
13219 module.terminal_history()
13220 );
13221 sleep(Duration::from_millis(10)).await;
13222 }
13223 let history = module.terminal_history();
13224 assert_eq!(
13225 history
13226 .entries
13227 .iter()
13228 .map(|entry| (entry.exit_code, entry.disposition.clone()))
13229 .collect::<Vec<_>>(),
13230 vec![
13231 (Some(0), TerminalDisposition::Restarting),
13232 (Some(0), TerminalDisposition::Failed),
13233 ]
13234 );
13235 let detail = history.entries[1]
13236 .disposition_detail
13237 .as_deref()
13238 .expect("a budget failure names the budget");
13239 assert!(detail.contains("max_restarts=1"), "{detail}");
13240 assert_eq!(module.status().unwrap().spawn_generation, 2);
13241 }
13242
13243 #[test]
13244 fn restart_budget_failure_is_published_after_its_terminal_record() {
13245 use std::io::{Read, Write};
13246 let supervisor = Supervisor::new_for_test(
13247 Arc::new(Registry::default()),
13248 RestartPolicy::new(0, Duration::ZERO),
13249 );
13250 let runtime = supervisor.runtime_config();
13251 let spec = ModuleSpec {
13252 protocol: ModuleProtocol::None,
13253 ..windowed_crash_spec("budget-publication-order")
13254 };
13255 let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::new(
13256 ModuleState::Running,
13257 true,
13258 )));
13259 lock_snapshot(&snapshot).unwrap().process_alive = true;
13260 let reaped_snapshot = snapshot.clone();
13261 let mut barrier = terminal_record_barrier(&runtime.spawn_events);
13262 let history = runtime.terminal_ring.clone();
13263 let reap = std::thread::spawn(move || {
13264 tokio::runtime::Builder::new_current_thread()
13265 .enable_all()
13266 .build()
13267 .unwrap()
13268 .block_on(on_child_exit(
13269 &spec,
13270 runtime.restart_policy,
13271 &supervisor.registry,
13272 &reaped_snapshot,
13273 &runtime.terminal_ring,
13274 &runtime.spawn_events,
13275 &runtime.child_roster,
13276 ExitReport {
13277 kind: ExitKind::Clean,
13278 code: Some(0),
13279 signal: None,
13280 at_ms: 1,
13281 },
13282 true,
13283 ))
13284 });
13285 let mut reached = [0];
13286 barrier.read_exact(&mut reached).unwrap();
13287 assert_eq!(&reached, b"R");
13288 let (before_state, before_alive) = {
13289 let state = lock_snapshot(&snapshot).unwrap();
13290 (state.state, state.process_alive)
13291 };
13292 let before_history = history.lock().unwrap().snapshot();
13293 barrier.write_all(b"X").unwrap();
13294 assert!(matches!(reap.join().unwrap(), NextAction::Stop { .. }));
13295 assert!(before_history.entries.is_empty());
13296 assert_ne!(
13297 before_state,
13298 ModuleState::Failed,
13299 "Failed was visible before its terminal record could be written"
13300 );
13301 assert!(before_alive);
13302 assert_eq!(lock_snapshot(&snapshot).unwrap().state, ModuleState::Failed);
13303 let history = history.lock().unwrap().snapshot();
13304 assert_eq!(history.entries.len(), 1);
13305 assert_eq!(history.entries[0].exit_code, Some(0));
13306 assert_eq!(history.entries[0].disposition, TerminalDisposition::Failed);
13307 }
13308
13309 #[test]
13310 fn crash_restart_at_budget_limit_remains_recoverable_while_recording() {
13311 use std::io::{Read, Write};
13312
13313 let supervisor = Supervisor::new_for_test(
13314 Arc::new(Registry::default()),
13315 RestartPolicy::new(3, Duration::ZERO),
13316 );
13317 let mut runtime = supervisor.runtime_config();
13318 let module = module_with_recovery_snapshot(ModuleState::Running, true, 2);
13319 let snapshot = module.inner.snapshot.clone();
13320 lock_snapshot(&snapshot).unwrap().process_alive = true;
13321 runtime.terminal_ring = module.inner.terminal_ring.clone();
13322 let spec = windowed_crash_spec(module.module_id());
13323 let mut barrier = terminal_record_barrier(&runtime.spawn_events);
13324 let reaped_snapshot = snapshot.clone();
13325 let reap = std::thread::spawn(move || {
13326 tokio::runtime::Builder::new_current_thread()
13327 .enable_all()
13328 .build()
13329 .unwrap()
13330 .block_on(on_child_exit(
13331 &spec,
13332 runtime.restart_policy,
13333 &supervisor.registry,
13334 &reaped_snapshot,
13335 &runtime.terminal_ring,
13336 &runtime.spawn_events,
13337 &runtime.child_roster,
13338 ExitReport {
13339 kind: ExitKind::Crash,
13340 code: Some(2),
13341 signal: None,
13342 at_ms: 1,
13343 },
13344 true,
13345 ))
13346 });
13347
13348 let mut reached = [0];
13352 barrier.read_exact(&mut reached).unwrap();
13353 assert_eq!(&reached, b"R");
13354 let before_status = module.status().unwrap();
13355 let before_history = module.terminal_history();
13356 let before_recovers = module.will_recover_after_connection_loss().unwrap();
13357 barrier.write_all(b"X").unwrap();
13358 assert!(matches!(reap.join().unwrap(), NextAction::Restart { .. }));
13359
13360 assert_eq!(before_status.state, ModuleState::Running);
13361 assert!(before_status.process_alive);
13362 assert_eq!(before_status.restart_count, 3);
13363 assert!(before_history.entries.is_empty());
13364 assert!(
13365 before_recovers,
13366 "connection cleanup would report a terminal close for a scheduled crash restart"
13367 );
13368 let restarting = module.status().unwrap();
13369 assert_eq!(restarting.state, ModuleState::Restarting);
13370 assert_eq!(restarting.restart_count, 3);
13371 assert!(module.will_recover_after_connection_loss().unwrap());
13372 let history = module.terminal_history();
13373 assert_eq!(history.entries.len(), 1);
13374 assert_eq!(
13375 history.entries[0].disposition,
13376 TerminalDisposition::Restarting
13377 );
13378
13379 update_snapshot(&snapshot, Some(module.module_id()), |state| {
13382 state.state = ModuleState::Running;
13383 state.respawn_pending = false;
13384 })
13385 .unwrap();
13386 assert!(!module.will_recover_after_connection_loss().unwrap());
13387 update_snapshot(&snapshot, Some(module.module_id()), |state| {
13388 state.enabled = false;
13389 state.respawn_pending = true;
13390 })
13391 .unwrap();
13392 assert!(!module.will_recover_after_connection_loss().unwrap());
13393 }
13394
13395 fn terminal_record_barrier(events: &super::SpawnEventFeed) -> std::net::TcpStream {
13396 use std::net::{TcpListener, TcpStream};
13397 let listener = TcpListener::bind("127.0.0.1:0").unwrap();
13398 let writer = TcpStream::connect(listener.local_addr().unwrap()).unwrap();
13399 let (barrier, _) = listener.accept().unwrap();
13400 for stream in [&writer, &barrier] {
13401 stream
13402 .set_read_timeout(Some(Duration::from_secs(10)))
13403 .unwrap();
13404 stream
13405 .set_write_timeout(Some(Duration::from_secs(10)))
13406 .unwrap();
13407 }
13408 *events.1.lock().unwrap() = Some(writer);
13409 barrier
13410 }
13411
13412 pub(super) fn before_terminal_record(events: &super::SpawnEventFeed) {
13413 use std::io::{Read, Write};
13414 let barrier = events.1.lock().unwrap().take();
13415 if let Some(mut barrier) = barrier {
13416 barrier.write_all(b"R").unwrap();
13417 let mut release = [0];
13418 barrier.read_exact(&mut release).unwrap();
13419 assert_eq!(&release, b"X");
13420 }
13421 }
13422
13423 #[test]
13424 fn health_restart_budget_failure_is_published_after_its_terminal_record() {
13425 use std::io::{Read, Write};
13426
13427 let supervisor = Supervisor::new_for_test(
13428 Arc::new(Registry::default()),
13429 RestartPolicy::new(0, Duration::ZERO),
13430 );
13431 let mut runtime = supervisor.runtime_config();
13432 let module = module_with_recovery_snapshot(ModuleState::Running, true, 0);
13433 let snapshot = module.inner.snapshot.clone();
13434 runtime.terminal_ring = module.inner.terminal_ring.clone();
13435 let spec = ModuleSpec {
13436 module_id: module.module_id().into(),
13437 program: fake_aft_stub_path(),
13438 protocol: ModuleProtocol::None,
13439 env: vec![("FAKE_AFT_EXIT_CODE".into(), "17".into())],
13440 ..windowed_crash_spec(module.module_id())
13441 };
13442 let mut barrier = terminal_record_barrier(&runtime.spawn_events);
13443 let reap = std::thread::spawn(move || {
13444 tokio::runtime::Builder::new_current_thread()
13445 .enable_all()
13446 .build()
13447 .unwrap()
13448 .block_on(async {
13449 let mut child =
13450 Some(spawn_and_mark_running(&spec, &runtime, &snapshot).unwrap());
13451 health_restart_child(
13452 &spec,
13453 &runtime,
13454 &supervisor.registry,
13455 &supervisor.process_liveness,
13456 &snapshot,
13457 &mut child,
13458 SupervisorHealthStatus::Failing,
13459 Some("health probe did not answer"),
13460 1,
13461 )
13462 .await
13463 .unwrap();
13464 assert!(child.is_none());
13465 });
13466 });
13467 let mut reached = [0];
13472 barrier.read_exact(&mut reached).unwrap();
13473 assert_eq!(&reached, b"R");
13474 let before_status = module.status().unwrap();
13475 let before_history = module.terminal_history();
13476 barrier.write_all(b"X").unwrap();
13477 reap.join().unwrap();
13478 assert!(before_history.entries.is_empty());
13479 assert_ne!(
13480 before_status.state,
13481 ModuleState::Failed,
13482 "Failed was visible before its terminal record could be written"
13483 );
13484 assert!(
13485 before_status.process_alive,
13486 "process death was visible before its terminal record could be written"
13487 );
13488 let failed = module.status().unwrap();
13489 assert_eq!(failed.state, ModuleState::Failed);
13490 assert!(!failed.process_alive);
13491 let history = module.terminal_history();
13492 assert_eq!(history.entries.len(), 1);
13493 assert_eq!(history.entries[0].disposition, TerminalDisposition::Failed);
13494 assert_eq!(
13495 history.entries[0].disposition_detail.as_deref(),
13496 Some("crash budget exhausted: max_restarts=0 within window_secs=600")
13497 );
13498 }
13499
13500 #[test]
13501 fn observed_exits_publish_process_death_after_terminal_history() {
13502 use std::io::{Read, Write};
13503 for (kind, enabled, shutdown, wait_error, expected_state) in [
13504 (ExitKind::Clean, true, false, false, ModuleState::Stopped),
13505 (ExitKind::Crash, true, false, false, ModuleState::Restarting),
13506 (ExitKind::Crash, false, false, false, ModuleState::Disabled),
13507 (
13508 ExitKind::DeliberateSeverance,
13509 true,
13510 false,
13511 false,
13512 ModuleState::Restarting,
13513 ),
13514 #[cfg(unix)]
13515 (ExitKind::Crash, true, true, false, ModuleState::Stopped),
13516 (ExitKind::Crash, true, false, true, ModuleState::Failed),
13517 ] {
13518 let supervisor = Supervisor::new_for_test(
13519 Arc::new(Registry::default()),
13520 RestartPolicy::new(1, Duration::ZERO),
13521 );
13522 let runtime = supervisor.runtime_config();
13523 #[cfg(unix)]
13524 if shutdown {
13525 runtime.child_roster.close();
13526 }
13527 #[cfg(not(unix))]
13528 let _ = shutdown;
13529 let spec = windowed_crash_spec("exit-publication-order");
13530 let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::new(
13531 ModuleState::Running,
13532 enabled,
13533 )));
13534 lock_snapshot(&snapshot).unwrap().process_alive = true;
13535 let reaped_snapshot = snapshot.clone();
13536 let history = runtime.terminal_ring.clone();
13537 let mut barrier = terminal_record_barrier(&runtime.spawn_events);
13538 let reap = std::thread::spawn(move || {
13539 if wait_error {
13540 super::fail_after_wait_error(
13541 &spec.module_id,
13542 &reaped_snapshot,
13543 &runtime.terminal_ring,
13544 &runtime.spawn_events,
13545 "wait failed".into(),
13546 &runtime.failure_context(true),
13547 );
13548 } else {
13549 tokio::runtime::Builder::new_current_thread()
13550 .enable_all()
13551 .build()
13552 .unwrap()
13553 .block_on(on_child_exit(
13554 &spec,
13555 runtime.restart_policy,
13556 &supervisor.registry,
13557 &reaped_snapshot,
13558 &runtime.terminal_ring,
13559 &runtime.spawn_events,
13560 &runtime.child_roster,
13561 ExitReport {
13562 kind,
13563 code: Some(0),
13564 signal: None,
13565 at_ms: 1,
13566 },
13567 true,
13568 ));
13569 }
13570 });
13571 let mut reached = [0];
13572 barrier.read_exact(&mut reached).unwrap();
13573 assert_eq!(&reached, b"R");
13574 let before_alive = lock_snapshot(&snapshot).unwrap().process_alive;
13575 let before_history = history.lock().unwrap().snapshot();
13576 barrier.write_all(b"X").unwrap();
13577 reap.join().unwrap();
13578 assert!(
13579 before_alive,
13580 "{kind:?}: death was published before recording"
13581 );
13582 assert!(before_history.entries.is_empty());
13583 assert_eq!(lock_snapshot(&snapshot).unwrap().state, expected_state);
13584 assert!(!lock_snapshot(&snapshot).unwrap().process_alive);
13585 assert_eq!(history.lock().unwrap().snapshot().entries.len(), 1);
13586 }
13587 }
13588
13589 #[tokio::test]
13590 async fn health_restart_budget_failure_remains_failed_when_journal_append_fails() {
13591 let dir = cortexkit_test_support::ScratchDir::new("health-budget-journal-failure");
13592 let path = dir.join("terminals.jsonl");
13593 std::fs::create_dir(&path).unwrap();
13594 let supervisor = Supervisor::new_for_test(
13595 Arc::new(Registry::default()),
13596 RestartPolicy::new(0, Duration::ZERO),
13597 )
13598 .with_terminal_journal(path, "health-budget-journal-failure".into());
13599 let runtime = supervisor.runtime_config();
13600 let spec = ModuleSpec {
13601 program: fake_aft_stub_path(),
13602 protocol: ModuleProtocol::None,
13603 env: vec![("FAKE_AFT_EXIT_CODE".into(), "17".into())],
13604 ..windowed_crash_spec("health-budget-journal-failure")
13605 };
13606 let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::starting()));
13607 let mut child = Some(spawn_and_mark_running(&spec, &runtime, &snapshot).unwrap());
13608 health_restart_child(
13609 &spec,
13610 &runtime,
13611 &supervisor.registry,
13612 &supervisor.process_liveness,
13613 &snapshot,
13614 &mut child,
13615 SupervisorHealthStatus::Failing,
13616 None,
13617 1,
13618 )
13619 .await
13620 .unwrap();
13621 assert_eq!(lock_snapshot(&snapshot).unwrap().state, ModuleState::Failed);
13622 assert!(!lock_snapshot(&snapshot).unwrap().process_alive);
13623 let history = runtime
13624 .terminal_ring
13625 .lock()
13626 .unwrap()
13627 .durable_history(&spec.module_id);
13628 assert!(history.journal_write_failures > 0);
13629 assert_eq!(history.entries.len(), 1);
13630 assert_eq!(history.entries[0].disposition, TerminalDisposition::Failed);
13631 }
13632
13633 #[test]
13634 fn health_restart_budget_failure_remains_failed_when_exit_recording_panics() {
13635 let supervisor = Supervisor::new_for_test(
13636 Arc::new(Registry::default()),
13637 RestartPolicy::new(0, Duration::ZERO),
13638 );
13639 let runtime = supervisor.runtime_config();
13640 let spec = ModuleSpec {
13641 program: fake_aft_stub_path(),
13642 protocol: ModuleProtocol::None,
13643 env: vec![("FAKE_AFT_EXIT_CODE".into(), "17".into())],
13644 ..windowed_crash_spec("health-budget-recording-panic")
13645 };
13646 let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::starting()));
13647 let result = std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| {
13648 tokio::runtime::Builder::new_current_thread()
13649 .enable_all()
13650 .build()
13651 .unwrap()
13652 .block_on(async {
13653 let mut child =
13654 Some(spawn_and_mark_running(&spec, &runtime, &snapshot).unwrap());
13655 runtime.spawn_events.0.lock().unwrap().seq = u64::MAX;
13656 health_restart_child(
13657 &spec,
13658 &runtime,
13659 &supervisor.registry,
13660 &supervisor.process_liveness,
13661 &snapshot,
13662 &mut child,
13663 SupervisorHealthStatus::Failing,
13664 None,
13665 1,
13666 )
13667 .await
13668 .unwrap();
13669 });
13670 }));
13671 let panic = result.expect_err("recording must still unwind");
13672 assert_eq!(
13673 panic.downcast_ref::<String>().map(String::as_str),
13674 Some("spawn event sequence exhausted")
13675 );
13676 let state = lock_snapshot(&snapshot).unwrap();
13677 assert_eq!(state.state, ModuleState::Failed);
13678 assert!(!state.process_alive);
13679 assert!(state.last_exit.is_some());
13680 }
13681
13682 #[tokio::test]
13683 async fn restart_budget_failure_remains_failed_when_journal_append_fails() {
13684 let dir = cortexkit_test_support::ScratchDir::new("budget-journal-failure");
13685 let path = dir.join("terminals.jsonl");
13686 std::fs::create_dir(&path).unwrap();
13687 let supervisor = Supervisor::new_for_test(
13688 Arc::new(Registry::default()),
13689 RestartPolicy::new(0, Duration::ZERO),
13690 )
13691 .with_terminal_journal(path, "budget-journal-failure".into());
13692 let runtime = supervisor.runtime_config();
13693 let spec = windowed_crash_spec("budget-journal-failure");
13694 let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::starting()));
13695 assert!(matches!(
13696 on_child_exit(
13697 &spec,
13698 runtime.restart_policy,
13699 &supervisor.registry,
13700 &snapshot,
13701 &runtime.terminal_ring,
13702 &runtime.spawn_events,
13703 &runtime.child_roster,
13704 crash_exit_report(1),
13705 true,
13706 )
13707 .await,
13708 NextAction::Stop { .. }
13709 ));
13710 assert_eq!(lock_snapshot(&snapshot).unwrap().state, ModuleState::Failed);
13711 let history = runtime
13712 .terminal_ring
13713 .lock()
13714 .unwrap()
13715 .durable_history(&spec.module_id);
13716 assert!(history.journal_write_failures > 0);
13717 assert_eq!(history.entries.len(), 1);
13718 assert_eq!(history.entries[0].disposition, TerminalDisposition::Failed);
13719 }
13720
13721 #[test]
13722 fn restart_budget_failure_remains_failed_when_exit_recording_panics() {
13723 let supervisor = Supervisor::new_for_test(
13724 Arc::new(Registry::default()),
13725 RestartPolicy::new(0, Duration::ZERO),
13726 );
13727 let runtime = supervisor.runtime_config();
13728 let spec = windowed_crash_spec("budget-recording-panic");
13729 let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::starting()));
13730 runtime.spawn_events.emit_spawned(&spec.module_id, 1, 1);
13731 runtime.spawn_events.0.lock().unwrap().seq = u64::MAX;
13734 let result = std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| {
13735 tokio::runtime::Builder::new_current_thread()
13736 .enable_all()
13737 .build()
13738 .unwrap()
13739 .block_on(on_child_exit(
13740 &spec,
13741 runtime.restart_policy,
13742 &supervisor.registry,
13743 &snapshot,
13744 &runtime.terminal_ring,
13745 &runtime.spawn_events,
13746 &runtime.child_roster,
13747 crash_exit_report(1),
13748 true,
13749 ))
13750 }));
13751 let panic = result.err().expect("recording must still unwind");
13752 assert_eq!(
13753 panic.downcast_ref::<String>().map(String::as_str),
13754 Some("spawn event sequence exhausted")
13755 );
13756 assert_eq!(lock_snapshot(&snapshot).unwrap().state, ModuleState::Failed);
13757 }
13758
13759 #[cfg(unix)]
13762 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
13763 async fn protocol_none_requested_stop_and_disable_do_not_respawn() {
13764 for disable in [false, true] {
13765 let label = if disable {
13766 "none-requested-disable"
13767 } else {
13768 "none-requested-stop"
13769 };
13770 let dir = cortexkit_test_support::ScratchDir::new(label);
13771 let (spec, ready, marker) = protocol_none_sigterm_exits_clean_spec(label, dir.path());
13772 let supervisor = Supervisor::new_for_test(
13773 Arc::new(Registry::default()),
13774 RestartPolicy::new(3, Duration::ZERO),
13775 );
13776 let module = supervisor.spawn(spec).unwrap();
13777 wait_for_file(&ready).await;
13778
13779 if disable {
13780 module.set_enabled(false).await.unwrap();
13781 } else {
13782 module.stop().await.unwrap();
13783 }
13784 assert!(
13785 marker.exists(),
13786 "{label}: the child must have left through its SIGTERM handler with exit 0"
13787 );
13788
13789 sleep(Duration::from_millis(500)).await;
13792 let status = module.status().unwrap();
13793 let expected = if disable {
13794 ModuleState::Disabled
13795 } else {
13796 ModuleState::Stopped
13797 };
13798 assert_eq!(status.state, expected, "{label}");
13799 assert_eq!(
13800 status.spawn_generation, 1,
13801 "{label}: respawned after a requested stop"
13802 );
13803 let history = module.terminal_history();
13804 assert_eq!(history.entries.len(), 1, "{label}: {history:?}");
13805 assert_eq!(history.entries[0].exit_code, Some(0), "{label}");
13806 assert_ne!(
13807 history.entries[0].disposition,
13808 TerminalDisposition::Restarting,
13809 "{label}"
13810 );
13811 }
13812 }
13813
13814 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
13817 async fn subc_wire_clean_exit_is_still_a_stop() {
13818 let supervisor = Supervisor::new_for_test(
13819 Arc::new(Registry::default()),
13820 RestartPolicy::new(3, Duration::ZERO),
13821 );
13822 let module = supervisor
13823 .spawn(ModuleSpec {
13824 module_id: "wire-clean-exit".to_string(),
13825 program: fake_aft_stub_path(),
13826 args: Vec::new(),
13827 env: vec![("FAKE_AFT_EXIT_CODE".to_string(), "0".to_string())],
13828 reserved: false,
13829 reserved_prefixes: Vec::new(),
13830 protocol: ModuleProtocol::Subc,
13831 overlap: Default::default(),
13832 carry_containment: false,
13833 })
13834 .unwrap();
13835
13836 let deadline = Instant::now() + Duration::from_secs(10);
13837 while module.terminal_history().entries.is_empty() {
13838 assert!(Instant::now() < deadline, "module never exited");
13839 sleep(Duration::from_millis(10)).await;
13840 }
13841 sleep(Duration::from_millis(500)).await;
13843 let status = module.status().unwrap();
13844 assert_eq!(status.state, ModuleState::Stopped);
13845 assert_eq!(status.spawn_generation, 1);
13846 let history = module.terminal_history();
13847 assert_eq!(history.entries.len(), 1, "{history:?}");
13848 assert_eq!(history.entries[0].exit_code, Some(0));
13849 assert_eq!(history.entries[0].disposition, TerminalDisposition::Stopped);
13850 }
13851
13852 #[cfg(unix)]
13853 #[tokio::test]
13854 async fn reload_exit_keeps_roster_until_terminal_is_recorded_during_shutdown() {
13855 let dir = cortexkit_test_support::ScratchDir::new("reload-roster-terminal-order");
13856 let record = dir.join("live-children.json");
13857 let supervisor = Supervisor::new_for_test(
13858 Arc::new(Registry::default()),
13859 RestartPolicy::new(0, Duration::ZERO),
13860 );
13861 let mut runtime = supervisor.runtime_config();
13862 runtime.child_roster.record_to(record.clone());
13863 let gate = Arc::new(super::ReloadExitRecordGate::default());
13864 runtime.test_reload_exit_record_gate = Some(Arc::clone(&gate));
13865 let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::starting()));
13866 let spec = ModuleSpec {
13867 module_id: "reload-exit-roster".into(),
13868 program: fake_aft_stub_path(),
13869 args: Vec::new(),
13870 env: vec![("FAKE_AFT_EXIT_CODE".into(), "23".into())],
13871 reserved: false,
13872 reserved_prefixes: Vec::new(),
13873 protocol: ModuleProtocol::Subc,
13874 overlap: Default::default(),
13875 carry_containment: false,
13876 };
13877 let mut child = None;
13878 let reload = super::finish_reload_child(
13879 &spec,
13880 &runtime,
13881 &supervisor.registry,
13882 &supervisor.process_liveness,
13883 &snapshot,
13884 &mut child,
13885 );
13886 tokio::pin!(reload);
13887 tokio::select! {
13888 result = &mut reload => panic!("reload missed the before-record gate: {result:?}"),
13889 _ = gate.reached.notified() => {}
13890 }
13891 assert!(runtime
13892 .terminal_ring
13893 .lock()
13894 .unwrap()
13895 .snapshot()
13896 .entries
13897 .is_empty());
13898 assert_eq!(
13899 crate::live_children::read_record(&record).unwrap().len(),
13900 1,
13901 "shutdown must still wait for the reaped child until its terminal record exists"
13902 );
13903 runtime.child_roster.close();
13904 gate.resume.notify_one();
13905 assert!(reload.await.is_err());
13906 assert!(crate::live_children::read_record(&record)
13907 .unwrap()
13908 .is_empty());
13909 let history = runtime.terminal_ring.lock().unwrap().snapshot();
13910 assert_eq!(history.entries.len(), 1);
13911 assert_eq!(
13912 history.entries[0].disposition,
13913 TerminalDisposition::DaemonShutdown
13914 );
13915 }
13916
13917 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
13921 async fn every_restart_increment_path_advances_lifetime_count() {
13922 let supervisor = Supervisor::new_for_test(
13923 Arc::new(Registry::default()),
13924 RestartPolicy::new(1, Duration::ZERO),
13925 );
13926 let runtime = supervisor.runtime_config();
13927 let spec = ModuleSpec {
13928 module_id: "lifetime-increment-path".to_string(),
13929 program: PathBuf::from("/unused/lifetime-increment-path"),
13930 args: Vec::new(),
13931 env: Vec::new(),
13932 reserved: false,
13933 reserved_prefixes: Vec::new(),
13934 protocol: ModuleProtocol::Subc,
13935 overlap: Default::default(),
13936 carry_containment: false,
13937 };
13938
13939 let crash_snapshot = Arc::new(Mutex::new(SupervisorSnapshot::starting()));
13940 assert!(matches!(
13941 on_child_exit(
13942 &spec,
13943 runtime.restart_policy,
13944 &supervisor.registry,
13945 &crash_snapshot,
13946 &runtime.terminal_ring,
13947 &runtime.spawn_events,
13948 &runtime.child_roster,
13949 ExitReport {
13950 kind: ExitKind::Crash,
13951 code: Some(1),
13952 signal: None,
13953 at_ms: 1,
13954 },
13955 true,
13956 )
13957 .await,
13958 NextAction::Restart { schedule: _ }
13959 ));
13960 let (crash_restarts, crash_lifetime) = {
13961 let state = lock_snapshot(&crash_snapshot).unwrap();
13962 (state.crash_restarts.len(), state.lifetime_restarts)
13963 };
13964 assert_eq!(crash_restarts, 1);
13965 assert_eq!(crash_lifetime, 1);
13966
13967 let health_snapshot = Arc::new(Mutex::new(SupervisorSnapshot::starting()));
13968 let mut health_child = None;
13969 assert!(matches!(
13970 health_restart_child(
13971 &spec,
13972 &runtime,
13973 &supervisor.registry,
13974 &supervisor.process_liveness,
13975 &health_snapshot,
13976 &mut health_child,
13977 SupervisorHealthStatus::Failing,
13978 None,
13979 2,
13980 )
13981 .await,
13982 Ok(())
13983 ));
13984 assert!(health_child.is_none());
13985 assert!(lock_snapshot(&health_snapshot).unwrap().respawn_pending);
13986 assert!(runtime.scheduled_respawn.lock().unwrap().take().is_some());
13987 let (health_restarts, health_lifetime) = {
13988 let state = lock_snapshot(&health_snapshot).unwrap();
13989 (state.crash_restarts.len(), state.lifetime_restarts)
13990 };
13991 assert_eq!(health_restarts, 1);
13992 assert_eq!(health_lifetime, 1);
13993
13994 let reload_snapshot = Arc::new(Mutex::new(SupervisorSnapshot::starting()));
13995 let mut reload_child = None;
13996 assert!(matches!(
13997 handle_reload_spawn_failure(
13998 &spec,
13999 &runtime,
14000 &supervisor.process_liveness,
14001 &reload_snapshot,
14002 &mut reload_child,
14003 "forced reload spawn failure".to_string(),
14004 )
14005 .await,
14006 Err(SuperviseError::ReloadFailed { .. })
14007 ));
14008 let (reload_restarts, reload_lifetime) = {
14009 let state = lock_snapshot(&reload_snapshot).unwrap();
14010 (state.crash_restarts.len(), state.lifetime_restarts)
14011 };
14012 assert_eq!(reload_restarts, 1);
14013 assert_eq!(reload_lifetime, 1);
14014 }
14015
14016 #[tokio::test]
14017 async fn deliberately_severed_live_child_records_lifetime_without_spending_restart_budget() {
14018 let supervisor = Supervisor::new_for_test(
14019 Arc::new(Registry::default()),
14020 RestartPolicy::new(3, Duration::ZERO),
14021 );
14022 let runtime = supervisor.runtime_config();
14023 let spec = ModuleSpec {
14024 module_id: "deliberately-severed".to_string(),
14025 program: PathBuf::from("/unused/deliberately-severed"),
14026 args: Vec::new(),
14027 env: Vec::new(),
14028 reserved: false,
14029 reserved_prefixes: Vec::new(),
14030 protocol: ModuleProtocol::Subc,
14031 overlap: Default::default(),
14032 carry_containment: false,
14033 };
14034 let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::starting()));
14035 let process = ProcessIdentity {
14036 pid: 41,
14037 start_time: 101,
14038 };
14039 record_deliberate_severance(&snapshot, process).unwrap();
14040 let exit_report = apply_deliberate_severance_marker(
14041 &snapshot,
14042 Some(process),
14043 ExitReport {
14044 kind: ExitKind::Crash,
14045 code: Some(1),
14046 signal: None,
14047 at_ms: 1,
14048 },
14049 );
14050 assert_eq!(exit_report.kind, ExitKind::DeliberateSeverance);
14051
14052 assert!(matches!(
14053 on_child_exit(
14054 &spec,
14055 runtime.restart_policy,
14056 &supervisor.registry,
14057 &snapshot,
14058 &runtime.terminal_ring,
14059 &runtime.spawn_events,
14060 &runtime.child_roster,
14061 exit_report,
14062 true,
14063 )
14064 .await,
14065 NextAction::Restart { schedule: _ }
14066 ));
14067 let state = lock_snapshot(&snapshot).unwrap();
14068 assert_eq!(state.lifetime_restarts, 1);
14069 assert_eq!(state.crash_restarts.len(), 0);
14070 }
14071
14072 #[tokio::test]
14073 async fn genuine_crash_spends_restart_budget_and_records_lifetime() {
14074 let supervisor = Supervisor::new_for_test(
14075 Arc::new(Registry::default()),
14076 RestartPolicy::new(3, Duration::ZERO),
14077 );
14078 let runtime = supervisor.runtime_config();
14079 let spec = ModuleSpec {
14080 module_id: "genuine-crash".to_string(),
14081 program: PathBuf::from("/unused/genuine-crash"),
14082 args: Vec::new(),
14083 env: Vec::new(),
14084 reserved: false,
14085 reserved_prefixes: Vec::new(),
14086 protocol: ModuleProtocol::Subc,
14087 overlap: Default::default(),
14088 carry_containment: false,
14089 };
14090 let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::starting()));
14091
14092 assert!(matches!(
14093 on_child_exit(
14094 &spec,
14095 runtime.restart_policy,
14096 &supervisor.registry,
14097 &snapshot,
14098 &runtime.terminal_ring,
14099 &runtime.spawn_events,
14100 &runtime.child_roster,
14101 ExitReport {
14102 kind: ExitKind::Crash,
14103 code: Some(1),
14104 signal: None,
14105 at_ms: 1,
14106 },
14107 true,
14108 )
14109 .await,
14110 NextAction::Restart { schedule: _ }
14111 ));
14112 let state = lock_snapshot(&snapshot).unwrap();
14113 assert_eq!(state.lifetime_restarts, 1);
14114 assert_eq!(state.crash_restarts.len(), 1);
14115 }
14116
14117 fn crash_exit_report(at_ms: u64) -> ExitReport {
14118 ExitReport {
14119 kind: ExitKind::Crash,
14120 code: Some(1),
14121 signal: None,
14122 at_ms,
14123 }
14124 }
14125
14126 fn windowed_crash_spec(module_id: &str) -> ModuleSpec {
14127 ModuleSpec {
14128 module_id: module_id.to_string(),
14129 program: PathBuf::from("/unused").join(module_id),
14130 args: Vec::new(),
14131 env: Vec::new(),
14132 reserved: false,
14133 reserved_prefixes: Vec::new(),
14134 protocol: ModuleProtocol::Subc,
14135 overlap: Default::default(),
14136 carry_containment: false,
14137 }
14138 }
14139
14140 #[tokio::test]
14146 async fn three_crashes_inside_the_window_stop_the_module_and_name_the_window() {
14147 let (logs, _guard) = crate::router::test_log::log_capture(tracing::Level::ERROR);
14148 let supervisor = Supervisor::new_for_test(
14149 Arc::new(Registry::default()),
14150 RestartPolicy::new(2, Duration::ZERO),
14151 );
14152 let runtime = supervisor.runtime_config();
14153 let spec = windowed_crash_spec("crash-loop-in-window");
14154 let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::starting()));
14155
14156 for attempt in 1..=2 {
14157 assert!(
14158 matches!(
14159 on_child_exit(
14160 &spec,
14161 runtime.restart_policy,
14162 &supervisor.registry,
14163 &snapshot,
14164 &runtime.terminal_ring,
14165 &runtime.spawn_events,
14166 &runtime.child_roster,
14167 crash_exit_report(attempt),
14168 true,
14169 )
14170 .await,
14171 NextAction::Restart { schedule: _ }
14172 ),
14173 "crash {attempt} is inside the budget and must respawn"
14174 );
14175 }
14176
14177 assert!(matches!(
14178 on_child_exit(
14179 &spec,
14180 runtime.restart_policy,
14181 &supervisor.registry,
14182 &snapshot,
14183 &runtime.terminal_ring,
14184 &runtime.spawn_events,
14185 &runtime.child_roster,
14186 crash_exit_report(3),
14187 true,
14188 )
14189 .await,
14190 NextAction::Stop { .. }
14191 ));
14192
14193 {
14194 let state = lock_snapshot(&snapshot).unwrap();
14195 assert_eq!(state.state, ModuleState::Failed);
14196 assert_eq!(state.crash_restarts.len(), 2);
14197 assert_eq!(state.lifetime_restarts, 2);
14198 }
14199
14200 let history = runtime
14201 .terminal_ring
14202 .lock()
14203 .expect("terminal ring is not poisoned")
14204 .snapshot();
14205 let last = history
14206 .entries
14207 .last()
14208 .expect("the refused crash is retained");
14209 assert_eq!(last.disposition, TerminalDisposition::Failed);
14210 assert_eq!(
14211 last.disposition_detail.as_deref(),
14212 Some("crash budget exhausted: max_restarts=2 within window_secs=600")
14213 );
14214
14215 let captured = crate::router::test_log::captured_logs(&logs);
14216 assert!(
14217 captured.contains("crash budget exhausted: max_restarts=2 within window_secs=600"),
14218 "the stop must be logged with its window: {captured}"
14219 );
14220 }
14221
14222 #[tokio::test]
14230 async fn a_crash_older_than_the_window_frees_its_slot_for_a_later_crash() {
14231 let supervisor = Supervisor::new_for_test(
14232 Arc::new(Registry::default()),
14233 RestartPolicy::new(2, Duration::ZERO),
14234 );
14235 let runtime = supervisor.runtime_config();
14236 let spec = windowed_crash_spec("crash-across-windows");
14237 let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::starting()));
14238
14239 for attempt in 1..=2 {
14240 assert!(matches!(
14241 on_child_exit(
14242 &spec,
14243 runtime.restart_policy,
14244 &supervisor.registry,
14245 &snapshot,
14246 &runtime.terminal_ring,
14247 &runtime.spawn_events,
14248 &runtime.child_roster,
14249 crash_exit_report(attempt),
14250 true,
14251 )
14252 .await,
14253 NextAction::Restart { schedule: _ }
14254 ));
14255 }
14256
14257 update_snapshot(&snapshot, Some(&spec.module_id), |state| {
14260 age_oldest_crash_restart_out_of_window(state, runtime.restart_policy.window);
14261 })
14262 .unwrap();
14263
14264 assert!(
14265 matches!(
14266 on_child_exit(
14267 &spec,
14268 runtime.restart_policy,
14269 &supervisor.registry,
14270 &snapshot,
14271 &runtime.terminal_ring,
14272 &runtime.spawn_events,
14273 &runtime.child_roster,
14274 crash_exit_report(3),
14275 true,
14276 )
14277 .await,
14278 NextAction::Restart { schedule: _ }
14279 ),
14280 "a crash older than the window must not hold a budget slot"
14281 );
14282
14283 let state = lock_snapshot(&snapshot).unwrap();
14284 assert_eq!(state.state, ModuleState::Restarting);
14285 assert_eq!(
14286 state.crash_restarts.len(),
14287 2,
14288 "the aged instant is dropped and the new one takes its place"
14289 );
14290 assert_eq!(
14291 state.lifetime_restarts, 3,
14292 "the ledger counts every restart, including the ones the window forgot"
14293 );
14294 }
14295
14296 #[tokio::test]
14301 async fn an_operator_restart_clears_the_ring_and_leaves_the_ledger_alone() {
14302 let supervisor = Supervisor::new_for_test(
14303 Arc::new(Registry::default()),
14304 RestartPolicy::new(2, Duration::ZERO),
14305 );
14306 let runtime = supervisor.runtime_config();
14307 let spec = windowed_crash_spec("operator-cleared-budget");
14308 let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::starting()));
14309
14310 for attempt in 1..=2 {
14311 assert!(matches!(
14312 on_child_exit(
14313 &spec,
14314 runtime.restart_policy,
14315 &supervisor.registry,
14316 &snapshot,
14317 &runtime.terminal_ring,
14318 &runtime.spawn_events,
14319 &runtime.child_roster,
14320 crash_exit_report(attempt),
14321 true,
14322 )
14323 .await,
14324 NextAction::Restart { schedule: _ }
14325 ));
14326 }
14327
14328 reset_restart_count(&snapshot, &spec.module_id).unwrap();
14329 {
14330 let state = lock_snapshot(&snapshot).unwrap();
14331 assert!(
14332 state.crash_restarts.is_empty(),
14333 "an operator restart returns the full budget"
14334 );
14335 assert_eq!(
14336 state.lifetime_restarts, 2,
14337 "clearing the budget must not unmake the crashes"
14338 );
14339 }
14340
14341 assert!(
14342 matches!(
14343 on_child_exit(
14344 &spec,
14345 runtime.restart_policy,
14346 &supervisor.registry,
14347 &snapshot,
14348 &runtime.terminal_ring,
14349 &runtime.spawn_events,
14350 &runtime.child_roster,
14351 crash_exit_report(3),
14352 true,
14353 )
14354 .await,
14355 NextAction::Restart { schedule: _ }
14356 ),
14357 "the cleared budget must be spendable again"
14358 );
14359 let state = lock_snapshot(&snapshot).unwrap();
14360 assert_eq!(state.crash_restarts.len(), 1);
14361 assert_eq!(state.lifetime_restarts, 3);
14362 }
14363
14364 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
14365 async fn severance_marker_for_a_dead_child_does_not_label_its_successor() {
14366 let severed = ProcessIdentity {
14367 pid: 41,
14368 start_time: 101,
14369 };
14370 let successor = ProcessIdentity {
14371 pid: 41,
14372 start_time: 202,
14373 };
14374 let module = module_with_recovery_snapshot(ModuleState::Running, true, 0);
14375 update_snapshot(&module.inner.snapshot, Some("recovery-snapshot"), |state| {
14376 state.pid = Some(successor.pid);
14377 state.process_start_time = Some(successor.start_time);
14378 })
14379 .unwrap();
14380 assert!(!module.record_deliberate_severance(severed).unwrap());
14381
14382 let exit_report = apply_deliberate_severance_marker(
14383 &module.inner.snapshot,
14384 Some(successor),
14385 ExitReport {
14386 kind: ExitKind::Crash,
14387 code: Some(1),
14388 signal: None,
14389 at_ms: 1,
14390 },
14391 );
14392
14393 assert_eq!(exit_report.kind, ExitKind::Crash);
14394 }
14395
14396 #[tokio::test]
14397 async fn drain_reap_marks_deliberate_severance_and_records_lifetime_without_budget() {
14398 let registry = Registry::default();
14399 let supervisor = Supervisor::new_for_test(
14400 Arc::new(Registry::default()),
14401 RestartPolicy::new(3, Duration::ZERO),
14402 );
14403 let runtime = supervisor.runtime_config();
14404 let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::starting()));
14405 let spec = ModuleSpec {
14406 module_id: "drain-deliberate-severance".to_string(),
14407 program: fake_aft_stub_path(),
14408 args: Vec::new(),
14409 env: vec![("FAKE_AFT_EXIT_CODE".to_string(), "23".to_string())],
14410 reserved: false,
14411 reserved_prefixes: Vec::new(),
14412 protocol: ModuleProtocol::Subc,
14413 overlap: Default::default(),
14414 carry_containment: false,
14415 };
14416 let mut child = spawn_and_mark_running(&spec, &runtime, &snapshot).unwrap();
14417 let process = ProcessIdentity {
14418 pid: 41,
14419 start_time: 101,
14420 };
14421 child.process_identity = Some(process);
14422 update_snapshot(&snapshot, Some(&spec.module_id), |state| {
14423 state.pid = Some(process.pid);
14424 state.process_start_time = Some(process.start_time);
14425 })
14426 .unwrap();
14427 record_deliberate_severance(&snapshot, process).unwrap();
14428
14429 drain_child_to_state(
14430 &spec.module_id,
14431 spec.protocol,
14432 StopNotice::SentOverConnection,
14435 ®istry,
14436 None,
14437 &snapshot,
14438 &runtime.terminal_ring,
14439 &runtime.spawn_events,
14440 child,
14441 Duration::from_secs(1),
14442 ModuleState::Stopped,
14443 Some(false),
14444 &super::FailureContext::operator(),
14445 )
14446 .await
14447 .unwrap();
14448
14449 let state = lock_snapshot(&snapshot).unwrap();
14450 assert_eq!(
14451 state.last_exit.as_ref().map(|exit| exit.kind),
14452 Some(ExitKind::DeliberateSeverance)
14453 );
14454 assert_eq!(state.lifetime_restarts, 1);
14455 assert_eq!(state.crash_restarts.len(), 0);
14456 drop(state);
14457 let history = runtime.terminal_ring.lock().unwrap().snapshot();
14458 assert_eq!(
14459 history.entries[0].exit_kind,
14460 subc_control::TerminalExitKind::DeliberateSeverance
14461 );
14462 }
14463
14464 #[tokio::test]
14465 async fn ordinary_drain_reap_does_not_record_a_lifetime_restart() {
14466 let registry = Registry::default();
14467 let supervisor = Supervisor::new_for_test(
14468 Arc::new(Registry::default()),
14469 RestartPolicy::new(3, Duration::ZERO),
14470 );
14471 let runtime = supervisor.runtime_config();
14472 let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::starting()));
14473 let spec = ModuleSpec {
14474 module_id: "ordinary-drain".to_string(),
14475 program: fake_aft_stub_path(),
14476 args: Vec::new(),
14477 env: vec![("FAKE_AFT_EXIT_CODE".to_string(), "23".to_string())],
14478 reserved: false,
14479 reserved_prefixes: Vec::new(),
14480 protocol: ModuleProtocol::Subc,
14481 overlap: Default::default(),
14482 carry_containment: false,
14483 };
14484 let child = spawn_and_mark_running(&spec, &runtime, &snapshot).unwrap();
14485
14486 drain_child_to_state(
14487 &spec.module_id,
14488 spec.protocol,
14489 StopNotice::SentOverConnection,
14492 ®istry,
14493 None,
14494 &snapshot,
14495 &runtime.terminal_ring,
14496 &runtime.spawn_events,
14497 child,
14498 Duration::from_secs(1),
14499 ModuleState::Stopped,
14500 Some(false),
14501 &super::FailureContext::operator(),
14502 )
14503 .await
14504 .unwrap();
14505
14506 let state = lock_snapshot(&snapshot).unwrap();
14507 assert_eq!(
14508 state.last_exit.as_ref().map(|exit| exit.kind),
14509 Some(ExitKind::Crash)
14510 );
14511 assert_eq!(state.lifetime_restarts, 0);
14512 assert_eq!(state.crash_restarts.len(), 0);
14513 }
14514
14515 #[test]
14516 fn fatal_connection_teardown_cannot_arm_a_marker_for_a_surviving_process() {
14517 assert!(!include_str!("server.rs")
14523 .contains("router.record_deliberate_connection_severance(ctx.connection_id)"));
14524 }
14525
14526 #[test]
14533 fn drained_after_quiescence_wait_passes_ok_through_and_forces_false_on_err() {
14534 assert!(drained_after_quiescence_wait(&Ok(true)));
14535 assert!(!drained_after_quiescence_wait(&Ok(false)));
14536 assert!(!drained_after_quiescence_wait(&Err(
14537 SuperviseError::StatePoisoned { module_id: None }
14538 )));
14539 }
14540
14541 #[test]
14550 fn wait_error_exit_report_records_a_failed_terminal_with_no_code_or_signal() {
14551 let ring = Arc::new(Mutex::new(TerminalRing::new(
14552 TerminalRingConfig::default(),
14553 0,
14554 )));
14555 record_wait_error_terminal("wait-error", &ring, &super::SpawnEventFeed::default());
14556
14557 let snapshot = ring.lock().unwrap().snapshot();
14558 assert_eq!(snapshot.entries.len(), 1);
14559 let entry = &snapshot.entries[0];
14560 assert_eq!(entry.exit_code, None);
14561 assert_eq!(entry.exit_signal, None);
14562 assert_eq!(entry.disposition, TerminalDisposition::Failed);
14563 }
14564
14565 #[test]
14566 fn wait_error_exit_path_preserves_spawn_event_density() {
14567 let feed = super::SpawnEventFeed::default();
14568 feed.configure_incarnation("wait-error-density".to_string());
14569 feed.emit_spawned("wait-error", 41, 1);
14570 let ring = Arc::new(Mutex::new(TerminalRing::new(
14571 TerminalRingConfig::default(),
14572 0,
14573 )));
14574
14575 record_wait_error_terminal("wait-error", &ring, &feed);
14576 feed.emit_spawned("after-wait-error", 42, 2);
14577
14578 let state = feed.0.lock().unwrap();
14579 let sequences = state
14580 .events
14581 .iter()
14582 .map(|event| event.cursor.seq)
14583 .collect::<Vec<_>>();
14584 assert_eq!(sequences, vec![1, 2, 3]);
14585 assert_eq!(state.events[1].kind, SpawnEventKind::Exited);
14586 assert_eq!(state.events[1].exit_code, None);
14587 assert_eq!(state.events[1].exit_signal, None);
14588 }
14589
14590 #[test]
14594 fn wait_error_exit_report_is_classified_as_a_crash() {
14595 assert_eq!(wait_error_exit_report().kind, ExitKind::Crash);
14596 }
14597}
14598
14599#[cfg(all(test, unix))]
14600mod restart_ordering_tests {
14601 use super::*;
14602 use std::io::Write;
14603 use tokio::io::AsyncReadExt;
14604
14605 #[test]
14606 #[ignore = "child process fixture"]
14607 fn serving_child() {
14608 let Ok(address_file) = std::env::var("RESTART_TEST_ADDRESS_FILE") else {
14609 return;
14610 };
14611 let listener = std::net::TcpListener::bind("127.0.0.1:0").unwrap();
14612 std::fs::write(address_file, listener.local_addr().unwrap().to_string()).unwrap();
14613 for stream in listener.incoming() {
14614 writeln!(stream.unwrap(), "{}", std::process::id()).unwrap();
14615 }
14616 }
14617
14618 async fn restart_before_publication(requests: usize) {
14619 let home = cortexkit_test_support::ScratchDir::new("restart-before-publication");
14620 let address_file = home.path().join("address");
14621 let spec = ModuleSpec {
14622 module_id: "restart-serving-child".into(),
14623 program: std::env::current_exe().unwrap(),
14624 args: vec![
14625 "--exact".into(),
14626 "supervise::restart_ordering_tests::serving_child".into(),
14627 "--ignored".into(),
14628 "--nocapture".into(),
14629 ],
14630 env: vec![(
14631 "RESTART_TEST_ADDRESS_FILE".into(),
14632 address_file.display().to_string(),
14633 )],
14634 reserved: false,
14635 reserved_prefixes: vec![],
14636 protocol: ModuleProtocol::None,
14637 overlap: Default::default(),
14638 carry_containment: false,
14639 };
14640 let mut supervisor =
14641 Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default());
14642 let (spawned_tx, spawned_rx) = oneshot::channel();
14643 let spawned_tx = Mutex::new(Some(spawned_tx));
14644 let (release_tx, release_rx) = std::sync::mpsc::channel();
14645 let release_rx = Mutex::new(release_rx);
14646 supervisor.test_after_first_spawn = AfterFirstSpawnHook(Some(Arc::new(move |_| {
14649 spawned_tx.lock().unwrap().take().unwrap().send(()).unwrap();
14650 release_rx
14651 .lock()
14652 .unwrap()
14653 .recv_timeout(Duration::from_secs(20))
14654 .unwrap();
14655 })));
14656 let registered = supervisor
14657 .register_configured_with_health(
14658 spec.clone(),
14659 true,
14660 HealthConfig::default(),
14661 None,
14662 RestartPolicy::new(3, Duration::ZERO),
14663 DEFAULT_REGISTER_TIMEOUT,
14664 )
14665 .unwrap();
14666 let module = registered.module.clone();
14667 let startup = tokio::task::spawn_blocking(move || {
14668 supervisor.start_registered_configured(registered, None, true)
14669 });
14670 spawned_rx.await.unwrap();
14671 let old_pid = tokio::time::timeout(Duration::from_secs(10), async {
14672 let address = loop {
14673 if let Ok(address) = std::fs::read_to_string(&address_file) {
14674 if !address.is_empty() {
14675 break address;
14676 }
14677 }
14678 tokio::time::sleep(Duration::from_millis(1)).await;
14679 };
14680 let mut stream = tokio::net::TcpStream::connect(address).await.unwrap();
14681 let mut reply = String::new();
14682 stream.read_to_string(&mut reply).await.unwrap();
14683 reply.trim().parse::<u32>().unwrap()
14684 })
14685 .await
14686 .expect("child must serve before generation publication");
14687 assert_eq!(
14688 lock_snapshot(&module.inner.snapshot)
14689 .unwrap()
14690 .spawn_generation,
14691 0
14692 );
14693 let mut replies = Vec::new();
14694 for _ in 0..requests {
14695 let mut restart = Box::pin(module.restart(Some(0)));
14696 std::future::poll_fn(|cx| {
14699 assert!(std::future::Future::poll(restart.as_mut(), cx).is_pending());
14700 std::task::Poll::Ready(())
14701 })
14702 .await;
14703 replies.push(restart);
14704 }
14705 release_tx.send(()).unwrap();
14706 startup.await.unwrap().unwrap();
14707 for reply in replies {
14708 reply.await.unwrap();
14709 }
14710 let replacement = tokio::time::timeout(Duration::from_secs(10), async {
14713 loop {
14714 if lock_snapshot(&module.inner.snapshot)
14715 .unwrap()
14716 .spawn_generation
14717 > 1
14718 {
14719 break;
14720 }
14721 tokio::time::sleep(Duration::from_millis(1)).await;
14722 }
14723 })
14724 .await;
14725 module.update_spec_for_test(spec).await.unwrap();
14729 let (generation, pid) = {
14730 let state = lock_snapshot(&module.inner.snapshot).unwrap();
14731 (state.spawn_generation, state.pid)
14732 };
14733 module.stop().await.unwrap();
14734 replacement.expect("restart must publish a replacement for the already-serving child");
14735 assert_ne!(
14736 pid,
14737 Some(old_pid),
14738 "a serving child predating the restart must be replaced"
14739 );
14740 assert_eq!(generation, 2, "queued restarts must share one replacement");
14741 }
14742
14743 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
14744 async fn restart_received_after_serving_before_generation_publication_replaces_child() {
14745 restart_before_publication(1).await;
14746 }
14747
14748 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
14749 async fn restart_queued_behind_restart_does_not_kill_its_replacement() {
14750 restart_before_publication(2).await;
14751 }
14752
14753 #[tokio::test]
14754 async fn restart_queued_before_replacement_spawn_keeps_the_new_child() {
14755 let supervisor =
14756 Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default());
14757 let registered = supervisor
14758 .register_configured_with_health(
14759 ModuleSpec {
14760 module_id: "queued-before-replacement".into(),
14761 program: "/bin/sleep".into(),
14766 args: vec!["600".into()],
14767 env: vec![],
14768 reserved: false,
14769 reserved_prefixes: vec![],
14770 protocol: ModuleProtocol::None,
14771 overlap: Default::default(),
14772 carry_containment: false,
14773 },
14774 true,
14775 HealthConfig::default(),
14776 None,
14777 RestartPolicy::default(),
14778 DEFAULT_REGISTER_TIMEOUT,
14779 )
14780 .unwrap();
14781 let RegisteredConfiguredModule {
14782 mut spec,
14783 mut runtime,
14784 module,
14785 mut commands,
14786 } = registered;
14787 let snapshot = &module.inner.snapshot;
14788 let mut restart = Box::pin(module.restart(Some(0)));
14789 std::future::poll_fn(|cx| {
14790 assert!(std::future::Future::poll(restart.as_mut(), cx).is_pending());
14791 std::task::Poll::Ready(())
14792 })
14793 .await;
14794 let mut child = Some(spawn_and_mark_running(&spec, &runtime, snapshot).unwrap());
14797 let replacement_pid = child.as_ref().unwrap().id();
14798 let command = commands.recv().await.unwrap();
14799 assert!(
14800 handle_supervisor_command(
14801 command,
14802 &mut spec,
14803 &mut runtime,
14804 &supervisor.registry,
14805 &supervisor.process_liveness,
14806 snapshot,
14807 &mut child,
14808 &mut commands,
14809 &mut VecDeque::new(),
14810 )
14811 .await
14812 );
14813 restart.await.unwrap();
14814 let kept_pid = child.as_ref().and_then(SupervisedChild::id);
14815 let pending = lock_snapshot(snapshot).unwrap().respawn_pending;
14816 if let Some(mut child) = child {
14817 child.start_kill().unwrap();
14818 child.wait().await.unwrap();
14819 }
14820 assert_eq!(
14821 kept_pid, replacement_pid,
14822 "a newer replacement must survive"
14823 );
14824 assert!(
14825 !pending,
14826 "a satisfied restart must not schedule another spawn"
14827 );
14828 }
14829}
14830
14831#[cfg(all(test, unix))]
14832mod health_event_tests {
14833 use super::*;
14834 use cortexkit_test_support::ScratchDir;
14835 use subc_protocol::{manifest::Concurrency, session::ModuleControlResponse};
14836
14837 struct Actor {
14838 module: SupervisedModule,
14839 registry: Arc<Registry>,
14840 forwarding: Arc<ForwardingTable>,
14841 spec: ModuleSpec,
14842 health: HealthConfig,
14843 rx: mpsc::Receiver<crate::router::OutboundFrame>,
14844 roster: ChildRoster,
14845 _home: ScratchDir,
14846 }
14847
14848 async fn wait_for<T>(reason: &str, mut observe: impl FnMut() -> Option<T>) -> T {
14849 let deadline = std::time::Instant::now() + Duration::from_secs(10);
14853 let tick = Instant::now();
14854 loop {
14855 if let Some(value) = observe() {
14856 assert_eq!(Instant::now(), tick, "{reason} must not wait for a timer");
14857 return value;
14858 }
14859 assert!(
14860 std::time::Instant::now() < deadline,
14861 "timed out waiting for {reason}"
14862 );
14863 tokio::task::yield_now().await;
14864 }
14865 }
14866
14867 impl Actor {
14868 async fn start(protocol: ModuleProtocol, cadence: Duration) -> Self {
14869 Self::start_with_policy(
14870 protocol,
14871 cadence,
14872 DEFAULT_REGISTER_TIMEOUT,
14873 RestartPolicy::new(3, Duration::ZERO),
14874 )
14875 .await
14876 }
14877
14878 async fn start_with_policy(
14879 protocol: ModuleProtocol,
14880 cadence: Duration,
14881 register_timeout: Duration,
14882 policy: RestartPolicy,
14883 ) -> Self {
14884 let home = ScratchDir::new("health-events");
14885 let registry = Arc::new(Registry::default());
14886 let forwarding = Arc::new(ForwardingTable::default());
14887 let health = HealthConfig {
14888 cadence,
14889 deadline: Duration::from_secs(3600),
14890 ..HealthConfig::default()
14891 };
14892 let supervisor = Supervisor::new_for_test(registry.clone(), policy)
14893 .with_forwarding(forwarding.clone())
14894 .with_health_config(health.clone());
14895 let spec = ModuleSpec {
14896 module_id: "event-health-child".into(),
14897 program: PathBuf::from("/bin/sleep"),
14905 args: vec!["600".into()],
14908 env: ["XDG_DATA_HOME", "XDG_RUNTIME_DIR", "XDG_CONFIG_HOME"]
14909 .into_iter()
14910 .map(|key| (key.into(), home.path().display().to_string()))
14911 .collect(),
14912 reserved: false,
14913 reserved_prefixes: vec![],
14914 protocol,
14915 overlap: Default::default(),
14916 carry_containment: false,
14917 };
14918 let module = supervisor
14919 .supervise_configured_requested(
14920 spec.clone(),
14921 true,
14922 health.clone(),
14923 Some(0),
14924 policy,
14925 register_timeout,
14926 None,
14927 )
14928 .unwrap();
14929 let (_, rx) = mpsc::channel(8);
14930 let actor = Self {
14931 module,
14932 registry,
14933 forwarding,
14934 spec,
14935 health,
14936 rx,
14937 roster: supervisor.child_roster.clone(),
14938 _home: home,
14939 };
14940 actor
14941 .wait_for_select(|checkpoint| checkpoint.generation > 0)
14942 .await;
14943 actor
14944 }
14945
14946 async fn expire_registration_wait(&self, remaining: Duration) {
14947 tokio::time::advance(remaining).await;
14948 wait_for("registration timeout TERM grace", || {
14949 lock_snapshot(&self.module.inner.snapshot)
14950 .unwrap()
14951 .actor_select
14952 .is_none()
14953 .then_some(())
14954 })
14955 .await;
14956 tokio::time::advance(crate::child_roster::TERM_TO_KILL).await;
14957 }
14958
14959 fn turns(&self) -> u64 {
14960 lock_snapshot(&self.module.inner.snapshot)
14961 .unwrap()
14962 .actor_turns
14963 }
14964
14965 async fn wait_for_select(
14966 &self,
14967 expected: impl Fn(&ActorSelectCheckpoint) -> bool,
14968 ) -> ActorSelectCheckpoint {
14969 wait_for("actor select checkpoint", || {
14970 let state = lock_snapshot(&self.module.inner.snapshot).unwrap();
14971 state.actor_select.filter(|checkpoint| {
14972 checkpoint.generation == state.spawn_generation
14973 && state.state == ModuleState::Running
14974 && state.process_alive
14975 && state.reported_pid().is_some()
14976 && expected(checkpoint)
14977 })
14978 })
14979 .await
14980 }
14981
14982 fn hello(&mut self, connection: u64, health: bool) {
14983 let manifest =
14984 subc_protocol::manifest::ModuleManifest::builder(&self.spec.module_id, "1.0.0")
14985 .protocol_ver(subc_protocol::PROTOCOL_VERSION)
14986 .build();
14987 let connection = ConnectionId::new(connection);
14988 let (tx, rx) = mpsc::channel(8);
14989 self.rx = rx;
14990 self.forwarding
14991 .register_module_connection(
14992 connection,
14993 self.spec.module_id.clone(),
14994 subc_protocol::PROTOCOL_VERSION,
14995 Concurrency::ModuleManaged,
14996 FrameSink::new(tx),
14997 )
14998 .unwrap();
14999 self.registry
15000 .register_with_control_ops(
15001 manifest,
15002 subc_protocol::PROTOCOL_VERSION,
15003 connection,
15004 if health {
15005 vec![MODULE_CONTROL_OP_HEALTH_CHECK.into()]
15006 } else {
15007 vec![]
15008 },
15009 )
15010 .unwrap();
15011 }
15012
15013 async fn probe(&mut self) -> crate::router::OutboundFrame {
15014 let frame = wait_for("outbound health probe", || match self.rx.try_recv() {
15015 Ok(frame) => Some(frame),
15016 Err(mpsc::error::TryRecvError::Empty) => None,
15017 Err(mpsc::error::TryRecvError::Disconnected) => panic!("health peer disconnected"),
15018 })
15019 .await;
15020 let body: Value = serde_json::from_slice(&frame.body).unwrap();
15021 assert_eq!(body["op"], MODULE_CONTROL_OP_HEALTH_CHECK);
15022 frame
15023 }
15024
15025 fn no_probe(&mut self) {
15026 assert!(matches!(
15027 self.rx.try_recv(),
15028 Err(mpsc::error::TryRecvError::Empty)
15029 ));
15030 }
15031
15032 fn answer(&self, connection: u64, frame: crate::router::OutboundFrame) {
15033 self.forwarding
15034 .complete_module_control_rpc(
15035 ConnectionId::new(connection),
15036 frame.header.corr,
15037 Some(MODULE_CONTROL_OP_HEALTH_CHECK),
15038 ModuleControlRpcOutcome::Response(ModuleControlResponse::HealthCheck {
15039 status: HealthStatus::Ok,
15040 detail: None,
15041 metrics: None,
15042 }),
15043 )
15044 .unwrap();
15045 }
15046 }
15047
15048 #[tokio::test(start_paused = true)]
15049 async fn registration_timeout_kills_at_120_seconds_and_respawns() {
15050 let actor = Actor::start(ModuleProtocol::Subc, Duration::from_secs(30)).await;
15051 let initial = actor.module.status().unwrap();
15052 tokio::time::advance(Duration::from_secs(119)).await;
15053 tokio::task::yield_now().await;
15054 assert_eq!(actor.module.status().unwrap().pid, initial.pid);
15055 assert!(actor.module.terminal_history().entries.is_empty());
15056 actor.expire_registration_wait(Duration::from_secs(1)).await;
15057 actor
15058 .wait_for_select(|at| at.generation > initial.spawn_generation)
15059 .await;
15060 let status = actor.module.status().unwrap();
15061 assert_ne!(status.pid, initial.pid);
15062 assert!(!cortexkit_test_support::process_alive(
15063 initial.pid.unwrap() as i32
15064 ));
15065 assert_eq!(status.restart_count, 1);
15066 assert_eq!(status.last_exit.unwrap().kind, ExitKind::Crash);
15067 let entries = actor.module.terminal_history().entries;
15068 assert_eq!(entries.len(), 1);
15069 assert_eq!(
15070 entries[0].disposition_detail.as_deref(),
15071 Some("registration_timeout")
15072 );
15073 actor.module.stop().await.unwrap();
15074 }
15075
15076 #[tokio::test(start_paused = true)]
15077 async fn registration_timeout_is_cancelled_by_hello_at_119_seconds() {
15078 let mut actor = Actor::start(ModuleProtocol::Subc, Duration::from_secs(30)).await;
15079 let pid = actor.module.status().unwrap().pid;
15080 tokio::time::advance(Duration::from_secs(119)).await;
15081 actor.hello(1, false);
15082 let registered = actor
15083 .wait_for_select(|at| at.registered_connection.is_some())
15084 .await;
15085 assert_eq!(
15086 registered.wake_after, None,
15087 "HELLO must cancel the deadline"
15088 );
15089 tokio::time::advance(Duration::from_secs(300)).await;
15090 tokio::task::yield_now().await;
15091 assert_eq!(actor.module.status().unwrap().pid, pid);
15092 assert!(actor.module.terminal_history().entries.is_empty());
15093 actor
15095 .registry
15096 .deregister_connection(ConnectionId::new(1))
15097 .unwrap();
15098 actor
15099 .wait_for_select(|at| at.registered_connection.is_none())
15100 .await;
15101 tokio::time::advance(Duration::from_secs(300)).await;
15102 tokio::task::yield_now().await;
15103 assert_eq!(actor.module.status().unwrap().pid, pid);
15104 assert!(actor.module.terminal_history().entries.is_empty());
15105 actor.module.stop().await.unwrap();
15106
15107 let mut transient = Actor::start(ModuleProtocol::Subc, Duration::from_secs(30)).await;
15108 let pid = transient.module.status().unwrap().pid;
15109 tokio::time::advance(Duration::from_secs(119)).await;
15110 let turns = transient.turns();
15111 transient.hello(1, false);
15114 transient
15115 .registry
15116 .deregister_connection(ConnectionId::new(1))
15117 .unwrap();
15118 let parked = transient.wait_for_select(|at| at.turn > turns).await;
15119 assert_eq!(
15120 parked.wake_after, None,
15121 "transient HELLO must cancel the deadline"
15122 );
15123 tokio::time::advance(Duration::from_secs(300)).await;
15124 tokio::task::yield_now().await;
15125 assert_eq!(transient.module.status().unwrap().pid, pid);
15126 assert!(transient.module.terminal_history().entries.is_empty());
15127 transient.module.stop().await.unwrap();
15128 }
15129
15130 #[tokio::test(start_paused = true)]
15131 async fn registration_timeouts_exhaust_budget_and_schedule_slow_retry() {
15132 let actor = Actor::start_with_policy(
15133 ModuleProtocol::Subc,
15134 Duration::from_secs(30),
15135 Duration::from_secs(10),
15136 RestartPolicy::new(1, Duration::ZERO),
15137 )
15138 .await;
15139 let generation = actor.module.status().unwrap().spawn_generation;
15140 actor
15141 .expire_registration_wait(Duration::from_secs(10))
15142 .await;
15143 actor.wait_for_select(|at| at.generation > generation).await;
15144 actor
15145 .expire_registration_wait(Duration::from_secs(10))
15146 .await;
15147 let retry = wait_for("failed retry schedule", || {
15148 let state = lock_snapshot(&actor.module.inner.snapshot).unwrap();
15149 (state.state == ModuleState::Failed)
15150 .then_some(state.failed_retry.pending)
15151 .flatten()
15152 })
15153 .await;
15154 assert_eq!(retry.delay, Duration::from_secs(600));
15155 assert_eq!(retry.attempt, 1);
15156 let entries = actor.module.terminal_history().entries;
15157 assert_eq!(entries.len(), 2);
15158 assert_eq!(entries[1].disposition, TerminalDisposition::Failed);
15159 assert!(entries[1]
15160 .disposition_detail
15161 .as_deref()
15162 .unwrap()
15163 .contains("registration_timeout"));
15164 tokio::time::advance(Duration::from_secs(599)).await;
15165 tokio::task::yield_now().await;
15166 assert_eq!(actor.module.state().unwrap(), ModuleState::Failed);
15167 tokio::time::advance(Duration::from_secs(1)).await;
15168 actor
15169 .wait_for_select(|at| at.generation > generation + 1)
15170 .await;
15171 actor
15173 .expire_registration_wait(Duration::from_secs(10))
15174 .await;
15175 wait_for("slow retry registration timeout", || {
15176 (actor.module.terminal_history().entries.len() == 3).then_some(())
15177 })
15178 .await;
15179 actor.module.stop().await.unwrap();
15180 }
15181
15182 #[tokio::test(start_paused = true)]
15183 async fn registration_timeout_honours_ten_second_override() {
15184 let actor = Actor::start_with_policy(
15185 ModuleProtocol::Subc,
15186 Duration::from_secs(30),
15187 Duration::from_secs(10),
15188 RestartPolicy::new(3, Duration::ZERO),
15189 )
15190 .await;
15191 let generation = actor.module.status().unwrap().spawn_generation;
15192 tokio::time::advance(Duration::from_secs(9)).await;
15193 tokio::task::yield_now().await;
15194 assert!(actor.module.terminal_history().entries.is_empty());
15195 actor.expire_registration_wait(Duration::from_secs(1)).await;
15196 actor.wait_for_select(|at| at.generation > generation).await;
15197 assert_eq!(actor.module.terminal_history().entries.len(), 1);
15198 actor.module.stop().await.unwrap();
15199 }
15200
15201 #[tokio::test]
15202 async fn registration_timeout_kills_term_ignoring_helper_before_reaping_leader() {
15203 use std::os::unix::process::{CommandExt, ExitStatusExt};
15204
15205 struct Helper(std::process::Child);
15206 impl Drop for Helper {
15207 fn drop(&mut self) {
15208 if matches!(self.0.try_wait(), Ok(None)) {
15211 let _ = self.0.kill();
15212 let _ = self.0.wait();
15213 }
15214 }
15215 }
15216
15217 let home = ScratchDir::new("registration-timeout-helper");
15218 let supervisor = Supervisor::new_for_test(
15219 Arc::new(Registry::default()),
15220 RestartPolicy::new(0, Duration::ZERO),
15221 );
15222 let spec = ModuleSpec {
15223 module_id: "registration-timeout-group".into(),
15224 program: "/bin/sleep".into(),
15225 args: vec!["120".into()],
15226 env: ["XDG_DATA_HOME", "XDG_RUNTIME_DIR", "XDG_CONFIG_HOME"]
15227 .into_iter()
15228 .map(|key| (key.into(), home.path().display().to_string()))
15229 .collect(),
15230 reserved: false,
15231 reserved_prefixes: vec![],
15232 protocol: ModuleProtocol::Subc,
15233 overlap: Default::default(),
15234 carry_containment: false,
15235 };
15236 let module = supervisor
15237 .supervise_configured_requested(
15238 spec,
15239 true,
15240 HealthConfig::default(),
15241 None,
15242 RestartPolicy::new(0, Duration::ZERO),
15243 Duration::from_secs(10),
15244 None,
15245 )
15246 .unwrap();
15247 let leader = timeout(Duration::from_secs(5), async {
15248 loop {
15249 if let Some(pid) = module.status().unwrap().pid {
15250 break pid;
15251 }
15252 sleep(Duration::from_millis(5)).await;
15253 }
15254 })
15255 .await
15256 .expect("supervised group leader must be reported before the deadline");
15257 assert!(leader > 1);
15258 let ready = home.join("helper.ready");
15259 let mut helper = Helper(
15262 std::process::Command::new("/bin/sh")
15263 .args([
15264 "-c",
15265 "trap '' TERM; : > \"$1\"; exec /bin/sleep 120",
15266 "helper",
15267 ])
15268 .arg(&ready)
15269 .process_group(i32::try_from(leader).unwrap())
15270 .stdout(std::process::Stdio::null())
15271 .stderr(std::process::Stdio::null())
15272 .spawn()
15273 .unwrap(),
15274 );
15275 timeout(Duration::from_secs(5), async {
15276 while !ready.exists() {
15277 assert!(helper.0.try_wait().unwrap().is_none());
15278 sleep(Duration::from_millis(5)).await;
15279 }
15280 })
15281 .await
15282 .expect("helper must install TERM ignore before the registration deadline");
15283 let helper_pid =
15284 rustix::process::Pid::from_raw(i32::try_from(helper.0.id()).unwrap()).unwrap();
15285 rustix::process::kill_process(helper_pid, rustix::process::Signal::TERM).unwrap();
15286 sleep(Duration::from_millis(20)).await;
15287 assert!(
15288 helper.0.try_wait().unwrap().is_none(),
15289 "helper must ignore TERM"
15290 );
15291
15292 timeout(Duration::from_secs(20), async {
15293 while module.state().unwrap() != ModuleState::Failed {
15294 sleep(Duration::from_millis(5)).await;
15295 }
15296 })
15297 .await
15298 .expect("registration timeout must stop and reap the group leader");
15299 let helper_exit = timeout(Duration::from_secs(3), async {
15300 loop {
15301 if let Some(status) = helper.0.try_wait().unwrap() {
15302 break status;
15303 }
15304 sleep(Duration::from_millis(5)).await;
15305 }
15306 })
15307 .await
15308 .expect("TERM-ignoring helper survived the registration timeout's group KILL");
15309 assert_eq!(helper_exit.signal(), Some(9));
15310 assert!(!cortexkit_test_support::process_alive(
15311 i32::try_from(leader).unwrap()
15312 ));
15313 assert_eq!(
15314 module.status().unwrap().last_exit.unwrap().kind,
15315 ExitKind::Crash
15316 );
15317 assert!(module.terminal_history().entries[0]
15318 .disposition_detail
15319 .as_deref()
15320 .unwrap()
15321 .contains("registration_timeout"));
15322 module.stop().await.unwrap();
15323 }
15324
15325 #[tokio::test(start_paused = true)]
15326 async fn registration_timeout_does_not_count_an_earlier_exit() {
15327 let actor = Actor::start_with_policy(
15328 ModuleProtocol::Subc,
15329 Duration::from_secs(30),
15330 DEFAULT_REGISTER_TIMEOUT,
15331 RestartPolicy::new(0, Duration::ZERO),
15332 )
15333 .await;
15334 let pid = actor.module.status().unwrap().pid.unwrap();
15335 rustix::process::kill_process(
15336 rustix::process::Pid::from_raw(pid as i32).unwrap(),
15337 rustix::process::Signal::TERM,
15338 )
15339 .unwrap();
15340 wait_for("early exit", || {
15341 (actor.module.state().unwrap() == ModuleState::Failed).then_some(())
15342 })
15343 .await;
15344 tokio::time::advance(Duration::from_secs(121)).await;
15345 tokio::task::yield_now().await;
15346 let entries = actor.module.terminal_history().entries;
15347 assert_eq!(entries.len(), 1);
15348 assert!(!entries[0]
15349 .disposition_detail
15350 .as_deref()
15351 .unwrap()
15352 .contains("registration_timeout"));
15353 assert_eq!(actor.module.status().unwrap().lifetime_restarts, 0);
15354 actor.module.stop().await.unwrap();
15355 }
15356
15357 #[tokio::test(start_paused = true)]
15358 async fn registration_timeout_is_cancelled_by_daemon_shutdown() {
15359 let actor = Actor::start(ModuleProtocol::Subc, Duration::from_secs(30)).await;
15360 let pid = actor.module.status().unwrap().pid;
15361 tokio::time::advance(Duration::from_secs(119)).await;
15362 actor.roster.close();
15363 let turns = actor.turns();
15364 tokio::time::advance(Duration::from_secs(300)).await;
15365 let parked = actor.wait_for_select(|at| at.turn > turns).await;
15366 assert_eq!(parked.wake_after, None);
15367 assert_eq!(actor.module.status().unwrap().pid, pid);
15368 assert!(actor.module.terminal_history().entries.is_empty());
15369 actor.module.stop().await.unwrap();
15370 let status = actor.module.status().unwrap();
15371 assert_eq!(status.restart_count, 0);
15372 assert_eq!(
15373 actor.module.terminal_history().entries[0].disposition,
15374 TerminalDisposition::DaemonShutdown
15375 );
15376 }
15377
15378 #[tokio::test(start_paused = true)]
15379 async fn no_health_children_park_for_sixty_seconds() {
15380 let none = Actor::start(ModuleProtocol::None, Duration::from_secs(30)).await;
15381 let unregistered = Actor::start(ModuleProtocol::Subc, Duration::from_secs(30)).await;
15382 let mut unadvertised = Actor::start(ModuleProtocol::Subc, Duration::from_secs(30)).await;
15383 unadvertised.hello(1, false);
15384 for _ in 0..6000 {
15389 tokio::time::advance(Duration::from_millis(10)).await;
15390 tokio::task::yield_now().await;
15391 }
15392 for actor in [&none, &unregistered, &unadvertised] {
15393 assert!(
15394 actor.turns() <= 5,
15395 "no-health actor woke {} times",
15396 actor.turns()
15397 );
15398 assert_eq!(actor.module.state().unwrap(), ModuleState::Running);
15399 }
15400 }
15401
15402 #[tokio::test(start_paused = true)]
15403 async fn late_hello_starts_a_due_probe_in_the_notification_tick() {
15404 let mut actor = Actor::start(ModuleProtocol::Subc, Duration::ZERO).await;
15408 let tick = Instant::now();
15409 actor.hello(1, true);
15410 actor.probe().await;
15411 assert_eq!(Instant::now(), tick, "HELLO must not wait for a poll");
15412 }
15413
15414 #[tokio::test(start_paused = true)]
15415 async fn hello_keeps_the_jittered_cadence_and_catalog_updates_do_not_reset_it() {
15416 let mut actor = Actor::start(ModuleProtocol::Subc, Duration::from_secs(30)).await;
15417 let hello_tick = Instant::now();
15418 actor.hello(1, true);
15419 let armed = actor
15420 .wait_for_select(|checkpoint| {
15421 checkpoint.registered_connection == Some(ConnectionId::new(1))
15422 && checkpoint.next_probe_at.is_some()
15423 })
15424 .await;
15425 let delay = jittered_health_delay(&actor.spec.module_id, 0, actor.health.cadence);
15426 assert!((Duration::from_secs(30)..Duration::from_secs(33)).contains(&delay));
15427 let first_deadline = hello_tick + delay;
15428 assert_eq!(armed.next_probe_at, Some(first_deadline));
15429 tokio::time::advance(Duration::from_secs(29)).await;
15430 let turns = actor.turns();
15431 actor
15432 .registry
15433 .replace_catalog_for_connection(ConnectionId::new(1), vec![], None, Some(true))
15434 .unwrap();
15435 let updated = actor
15436 .wait_for_select(|checkpoint| checkpoint.turn > turns)
15437 .await;
15438 assert_eq!(
15439 updated.next_probe_at,
15440 Some(first_deadline),
15441 "catalog update must not reset cadence"
15442 );
15443 actor.no_probe();
15444 tokio::time::advance(first_deadline - Instant::now()).await;
15445 let frame = actor.probe().await;
15446 actor.answer(1, frame);
15447 let next = jittered_health_delay(&actor.spec.module_id, 1, actor.health.cadence);
15448 let next_deadline = Instant::now() + next;
15449 let rearmed = actor
15450 .wait_for_select(|checkpoint| {
15451 checkpoint
15452 .next_probe_at
15453 .is_some_and(|deadline| deadline > first_deadline)
15454 })
15455 .await;
15456 assert_eq!(rearmed.next_probe_at, Some(next_deadline));
15457 tokio::time::advance(next - Duration::from_nanos(1)).await;
15458 actor.no_probe();
15459 tokio::time::advance(Duration::from_nanos(1)).await;
15460 actor.probe().await;
15461 }
15462
15463 #[tokio::test(start_paused = true)]
15464 async fn disconnect_disarms_and_reconnect_rearms_health() {
15465 let mut actor = Actor::start(ModuleProtocol::Subc, Duration::from_secs(30)).await;
15466 actor.hello(1, true);
15467 actor
15468 .wait_for_select(|checkpoint| {
15469 checkpoint.registered_connection == Some(ConnectionId::new(1))
15470 && checkpoint.next_probe_at.is_some()
15471 })
15472 .await;
15473 let turns = actor.turns();
15474 let tick = Instant::now();
15475 actor
15476 .registry
15477 .deregister_connection(ConnectionId::new(1))
15478 .unwrap();
15479 let disarmed = actor
15480 .wait_for_select(|checkpoint| {
15481 checkpoint.turn > turns && checkpoint.registered_connection.is_none()
15482 })
15483 .await;
15484 assert_eq!(disarmed.next_probe_at, None);
15485 assert_eq!(disarmed.wake_after, None);
15486 assert_eq!(
15487 Instant::now(),
15488 tick,
15489 "disconnect must disarm in the notification tick"
15490 );
15491 tokio::time::advance(Duration::from_secs(60)).await;
15492 actor.no_probe();
15493 let reconnect_tick = Instant::now();
15494 actor.hello(2, true);
15495 let rearmed = actor
15496 .wait_for_select(|checkpoint| {
15497 checkpoint.registered_connection == Some(ConnectionId::new(2))
15498 && checkpoint.next_probe_at.is_some()
15499 })
15500 .await;
15501 let delay = jittered_health_delay(&actor.spec.module_id, 0, actor.health.cadence);
15502 assert_eq!(rearmed.next_probe_at, Some(reconnect_tick + delay));
15503 tokio::time::advance(delay).await;
15504 actor.probe().await;
15505 }
15506
15507 #[tokio::test(start_paused = true)]
15508 async fn reregistration_without_health_disarms_the_old_schedule() {
15509 let mut actor = Actor::start(ModuleProtocol::Subc, Duration::from_secs(30)).await;
15510 actor.hello(1, true);
15511 actor
15512 .wait_for_select(|checkpoint| {
15513 checkpoint.registered_connection == Some(ConnectionId::new(1))
15514 && checkpoint.next_probe_at.is_some()
15515 })
15516 .await;
15517 actor
15518 .registry
15519 .deregister_connection(ConnectionId::new(1))
15520 .unwrap();
15521 actor.hello(2, false);
15522 let disarmed = actor
15523 .wait_for_select(|checkpoint| {
15524 checkpoint.registered_connection == Some(ConnectionId::new(2))
15525 })
15526 .await;
15527 assert_eq!(disarmed.next_probe_at, None);
15528 assert_eq!(disarmed.wake_after, None);
15529 tokio::time::advance(Duration::from_secs(60)).await;
15530 actor.no_probe();
15531 }
15532
15533 #[tokio::test(start_paused = true)]
15534 async fn restart_waits_for_its_own_hello_before_rearming_health() {
15535 let mut actor = Actor::start(ModuleProtocol::Subc, Duration::from_secs(30)).await;
15536 actor.hello(1, true);
15537 let armed = actor
15538 .wait_for_select(|checkpoint| {
15539 checkpoint.registered_connection == Some(ConnectionId::new(1))
15540 && checkpoint.next_probe_at.is_some()
15541 })
15542 .await;
15543 actor
15547 .registry
15548 .deregister_connection(ConnectionId::new(1))
15549 .unwrap();
15550 actor.module.restart(Some(0)).await.unwrap();
15551 let parked = actor
15552 .wait_for_select(|checkpoint| {
15553 checkpoint.generation > armed.generation
15554 && checkpoint.registered_connection.is_none()
15555 && checkpoint.next_probe_at.is_none()
15556 })
15557 .await;
15558 assert_eq!(parked.wake_after, Some(DEFAULT_REGISTER_TIMEOUT));
15559 actor.rx.close();
15560 while actor.rx.try_recv().is_ok() {}
15563 let turns = actor.turns();
15564 tokio::time::advance(Duration::from_secs(60)).await;
15565 let after = actor.turns();
15566 eprintln!(
15567 "restart park turns: before={turns} after={after} delta={}",
15568 after - turns
15569 );
15570 assert_eq!(after, turns, "replacement must park before HELLO");
15571 let hello_tick = Instant::now();
15572 actor.hello(2, true);
15573 let rearmed = actor
15574 .wait_for_select(|checkpoint| {
15575 checkpoint.registered_connection == Some(ConnectionId::new(2))
15576 && checkpoint.next_probe_at.is_some()
15577 })
15578 .await;
15579 let delay = jittered_health_delay(&actor.spec.module_id, 0, actor.health.cadence);
15580 assert_eq!(rearmed.next_probe_at, Some(hello_tick + delay));
15581 tokio::time::advance(delay).await;
15582 actor.probe().await;
15583 }
15584
15585 #[tokio::test(start_paused = true)]
15586 async fn http_probe_waits_a_full_cadence_for_each_spawn() {
15587 use tokio::{
15588 io::{AsyncReadExt, AsyncWriteExt},
15589 net::TcpListener,
15590 };
15591 let mut actor = Actor::start(ModuleProtocol::None, Duration::from_secs(30)).await;
15592 let listener = TcpListener::bind("127.0.0.1:0").await.unwrap();
15593 actor.health.http = Some(format!("http://{}/healthz", listener.local_addr().unwrap()));
15594 let (started_tx, mut started_rx) = mpsc::channel(8);
15595 let peer = tokio::spawn(async move {
15596 loop {
15597 let (mut socket, _) = listener.accept().await.unwrap();
15598 let mut request = [0; 1024];
15599 assert!(socket.read(&mut request).await.unwrap() > 0);
15600 started_tx.send(Instant::now()).await.unwrap();
15601 socket
15602 .write_all(b"HTTP/1.1 200 OK\r\nContent-Length: 0\r\n\r\n")
15603 .await
15604 .unwrap();
15605 }
15606 });
15607 actor
15608 .module
15609 .update_configuration(actor.spec.clone(), actor.health.clone(), None)
15610 .await
15611 .unwrap();
15612 let first_tick = Instant::now();
15613 let first = actor
15614 .wait_for_select(|checkpoint| checkpoint.next_probe_at.is_some())
15615 .await;
15616 assert!(first.next_probe_at.unwrap() >= first_tick + actor.health.cadence);
15617 assert!(started_rx.try_recv().is_err());
15618
15619 tokio::time::advance(
15622 first.next_probe_at.unwrap() - Instant::now() - Duration::from_secs(1),
15623 )
15624 .await;
15625 actor.module.restart(Some(0)).await.unwrap();
15626 let spawn_tick = Instant::now();
15627 let replacement = actor
15628 .wait_for_select(|checkpoint| checkpoint.generation > first.generation)
15629 .await;
15630 let deadline = replacement.next_probe_at.unwrap();
15631 assert!(
15632 deadline >= spawn_tick + actor.health.cadence,
15633 "a replacement must not inherit its predecessor's probe deadline"
15634 );
15635 tokio::time::advance(actor.health.cadence - Duration::from_nanos(1)).await;
15636 assert!(
15637 started_rx.try_recv().is_err(),
15638 "no HTTP probe during startup grace"
15639 );
15640 tokio::time::advance(deadline - Instant::now()).await;
15641 let landed = wait_for("first replacement HTTP probe", || {
15642 started_rx.try_recv().ok()
15643 })
15644 .await;
15645 assert!(landed >= spawn_tick + actor.health.cadence);
15646 wait_for("healthy replacement", || {
15647 (actor.module.status().unwrap().health.status == SupervisorHealthStatus::Ok)
15648 .then_some(())
15649 })
15650 .await;
15651 peer.abort();
15652 }
15653
15654 #[tokio::test(start_paused = true)]
15655 async fn rescan_adds_and_removes_http_health_without_polling() {
15656 use tokio::{
15657 io::{AsyncReadExt, AsyncWriteExt},
15658 net::TcpListener,
15659 };
15660 let mut actor = Actor::start(ModuleProtocol::None, Duration::from_secs(30)).await;
15661 let listener = TcpListener::bind("127.0.0.1:0").await.unwrap();
15662 actor.health.http = Some(format!("http://{}/health", listener.local_addr().unwrap()));
15663 actor.health.cadence = Duration::from_millis(1);
15665 let (started_tx, mut started_rx) = mpsc::channel(8);
15666 let peer = tokio::spawn(async move {
15667 loop {
15668 let (mut socket, _) = listener.accept().await.unwrap();
15669 let mut request = [0; 1024];
15670 assert!(socket.read(&mut request).await.unwrap() > 0);
15671 started_tx.send(()).await.unwrap();
15672 socket
15673 .write_all(b"HTTP/1.1 200 OK\r\nContent-Length: 0\r\n\r\n")
15674 .await
15675 .unwrap();
15676 }
15677 });
15678 let tick = Instant::now();
15679 let turns = actor.turns();
15680 actor
15681 .module
15682 .update_configuration(actor.spec.clone(), actor.health.clone(), None)
15683 .await
15684 .unwrap();
15685 let armed = actor
15686 .wait_for_select(|checkpoint| {
15687 checkpoint.turn > turns && checkpoint.next_probe_at.is_some()
15688 })
15689 .await;
15690 assert_eq!(armed.next_probe_at, Some(tick + Duration::from_millis(1)));
15691 assert_eq!(Instant::now(), tick);
15692 tokio::time::advance(Duration::from_millis(1)).await;
15693 let mut started = false;
15694 wait_for("successful HTTP probe", || {
15695 started |= started_rx.try_recv().is_ok();
15696 (started && actor.module.status().unwrap().health.status == SupervisorHealthStatus::Ok)
15697 .then_some(())
15698 })
15699 .await;
15700 assert!(started, "added HTTP check must start at its first deadline");
15701 assert_eq!(
15702 actor.module.status().unwrap().health.status,
15703 SupervisorHealthStatus::Ok
15704 );
15705 assert_eq!(Instant::now(), tick + Duration::from_millis(1));
15706 actor.health.http = None;
15707 let turns = actor.turns();
15708 actor
15709 .module
15710 .update_configuration(actor.spec.clone(), actor.health.clone(), None)
15711 .await
15712 .unwrap();
15713 let parked = actor
15714 .wait_for_select(|checkpoint| {
15715 checkpoint.turn > turns && checkpoint.next_probe_at.is_none()
15716 })
15717 .await;
15718 assert_eq!(parked.wake_after, None);
15719 let turns = actor.turns();
15720 tokio::time::advance(Duration::from_secs(60)).await;
15721 assert_eq!(
15722 actor.turns(),
15723 turns,
15724 "removed HTTP check must park the actor"
15725 );
15726 assert!(
15727 started_rx.try_recv().is_err(),
15728 "removed HTTP check must stay disarmed"
15729 );
15730 assert_eq!(
15731 actor.module.status().unwrap().health,
15732 ModuleHealthStatus::default()
15733 );
15734 peer.abort();
15735 }
15736}
15737
15738#[cfg(test)]
15739mod drain_busy_reprobe_tests {
15740 use super::*;
15744 use std::collections::BTreeMap;
15745 use std::sync::Mutex as StdMutex;
15746 use subc_protocol::{manifest::Concurrency, session::ModuleControlResponse};
15747
15748 const MODULE_ID: &str = "drain-reprobe-module";
15749 const CONNECTION: u64 = 7;
15750 const GAUGE: &str = "runs_in_flight";
15751 const CADENCE: Duration = Duration::from_secs(30);
15754 const CEILING: Duration = Duration::from_secs(30);
15755
15756 #[derive(Clone, Copy)]
15758 enum Answer {
15759 BusyUntil { until: Instant, latency: Duration },
15762 OmitsGauge,
15764 Silent,
15766 }
15767
15768 struct Drain {
15769 forwarding: Arc<ForwardingTable>,
15770 runtime: SupervisorRuntimeConfig,
15771 endpoint: crate::ModuleEndpointId,
15772 probes: Arc<StdMutex<Vec<Instant>>>,
15774 responder: tokio::task::JoinHandle<()>,
15775 }
15776
15777 impl Drain {
15778 fn start(answer: Answer) -> Self {
15779 let registry = Arc::new(Registry::default());
15780 let forwarding = Arc::new(ForwardingTable::default());
15781 let health = HealthConfig {
15782 cadence: CADENCE,
15783 ..HealthConfig::default()
15784 };
15785 let runtime = Supervisor::new_for_test(registry, RestartPolicy::new(3, Duration::ZERO))
15786 .with_forwarding(forwarding.clone())
15787 .with_health_config(health)
15788 .runtime_config();
15789 let (tx, mut rx) = mpsc::channel::<crate::router::OutboundFrame>(8);
15790 let endpoint = forwarding
15791 .register_module_connection(
15792 ConnectionId::new(CONNECTION),
15793 MODULE_ID.to_string(),
15794 subc_protocol::PROTOCOL_VERSION,
15795 Concurrency::ModuleManaged,
15796 FrameSink::new(tx),
15797 )
15798 .unwrap();
15799 let probes = Arc::new(StdMutex::new(Vec::new()));
15800 let responder = tokio::spawn({
15801 let forwarding = forwarding.clone();
15802 let probes = probes.clone();
15803 async move {
15804 while let Some(frame) = rx.recv().await {
15805 let body: Value = serde_json::from_slice(&frame.body).unwrap();
15806 assert_eq!(body["op"], MODULE_CONTROL_OP_HEALTH_CHECK);
15807 probes.lock().unwrap().push(Instant::now());
15808 let metrics = match answer {
15809 Answer::BusyUntil { until, latency } => {
15810 sleep(latency).await;
15811 let busy = u64::from(Instant::now() < until);
15812 serde_json::json!({ GAUGE: busy })
15813 }
15814 Answer::OmitsGauge => serde_json::json!({ "queue_depth": 0 }),
15815 Answer::Silent => continue,
15816 };
15817 let _ = forwarding.complete_module_control_rpc(
15820 ConnectionId::new(CONNECTION),
15821 frame.header.corr,
15822 Some(MODULE_CONTROL_OP_HEALTH_CHECK),
15823 ModuleControlRpcOutcome::Response(ModuleControlResponse::HealthCheck {
15824 status: HealthStatus::Ok,
15825 detail: None,
15826 metrics: Some(metrics),
15827 }),
15828 );
15829 }
15830 }
15831 });
15832 Self {
15833 forwarding,
15834 runtime,
15835 endpoint,
15836 probes,
15837 responder,
15838 }
15839 }
15840
15841 async fn run(&self, ceiling: Duration) -> (bool, Duration) {
15844 let (drained, elapsed, _) = self.run_traced(ceiling).await;
15845 (drained, elapsed)
15846 }
15847
15848 async fn run_traced(&self, ceiling: Duration) -> (bool, Duration, DrainGaugeTrace) {
15850 let started = Instant::now();
15851 let mut trace = DrainGaugeTrace::new(&[GAUGE.to_string()]);
15852 let drained = wait_for_forwarding_quiescence(
15853 &self.forwarding,
15854 MODULE_ID,
15855 &self.runtime,
15856 self.endpoint,
15857 started + ceiling,
15858 &[GAUGE.to_string()],
15859 DrainScope::Active,
15860 &mut trace,
15861 )
15862 .await
15863 .unwrap();
15864 (drained, started.elapsed(), trace)
15865 }
15866
15867 fn probes(&self) -> Vec<Instant> {
15868 self.probes.lock().unwrap().clone()
15869 }
15870 }
15871
15872 impl Drop for Drain {
15873 fn drop(&mut self) {
15874 self.responder.abort();
15875 }
15876 }
15877
15878 #[tokio::test(start_paused = true)]
15879 async fn busy_at_first_probe_then_quiet_drains_long_before_the_ceiling() {
15880 let drain = Drain::start(Answer::BusyUntil {
15881 until: Instant::now() + Duration::from_secs(1),
15882 latency: Duration::ZERO,
15883 });
15884 let (drained, elapsed) = drain.run(CEILING).await;
15885 assert!(
15886 drained,
15887 "gauges that read zero during the drain must let it finish drained, \
15888 not wait out the {CADENCE:?} health cadence (elapsed {elapsed:?})"
15889 );
15890 assert!(
15891 elapsed < Duration::from_secs(3),
15892 "drain took {elapsed:?} after the gauges went quiet at 1s"
15893 );
15894 let probes = drain.probes();
15895 assert!(
15896 probes.len() >= 2,
15897 "the module was busy at the first probe, so a quiet verdict needs a second: {probes:?}"
15898 );
15899 }
15900
15901 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
15906 async fn an_instant_busy_first_answer_does_not_stop_later_probes_in_real_time() {
15907 let drain = Drain::start(Answer::BusyUntil {
15908 until: Instant::now() + Duration::from_millis(600),
15909 latency: Duration::ZERO,
15910 });
15911 let (drained, elapsed, trace) = drain.run_traced(CEILING).await;
15912 assert!(drained, "the gauge fell at 600ms: {trace}");
15913 assert!(
15914 elapsed < Duration::from_secs(3),
15915 "drain took {elapsed:?} after the gauge fell at 600ms: {trace}"
15916 );
15917 let probes = drain.probes();
15918 assert!(
15919 probes.len() >= 2 && trace.probes as usize == probes.len(),
15920 "the drain must re-probe after the first busy answer: {probes:?}, {trace}"
15921 );
15922 }
15923
15924 #[tokio::test(start_paused = true)]
15925 async fn busy_throughout_ends_undrained_at_the_ceiling_with_bounded_probes() {
15926 let started = Instant::now();
15927 let drain = Drain::start(Answer::BusyUntil {
15928 until: started + Duration::from_secs(3600),
15929 latency: Duration::ZERO,
15930 });
15931 let (drained, elapsed) = drain.run(CEILING).await;
15932 assert!(!drained, "a module busy throughout must not drain");
15933 assert!(
15934 elapsed >= CEILING && elapsed < CEILING + drain_busy_reprobe_interval(),
15935 "a busy module is held until the ceiling and no longer: {elapsed:?}"
15936 );
15937 let probes = drain.probes();
15940 let max_probes = (CEILING.as_millis() / drain_busy_reprobe_interval().as_millis()) + 1;
15941 assert!(
15942 probes.len() as u128 <= max_probes,
15943 "{} probes in a {CEILING:?} drain exceeds the {max_probes} the interval allows",
15944 probes.len()
15945 );
15946 for pair in probes.windows(2) {
15947 assert!(
15948 pair[1] - pair[0] >= drain_busy_reprobe_interval(),
15949 "consecutive probes {:?} apart",
15950 pair[1] - pair[0]
15951 );
15952 }
15953 }
15954
15955 #[tokio::test(start_paused = true)]
15956 async fn a_slow_probe_is_never_overlapped_by_the_next() {
15957 let latency = Duration::from_secs(1);
15960 let drain = Drain::start(Answer::BusyUntil {
15961 until: Instant::now() + Duration::from_secs(3600),
15962 latency,
15963 });
15964 let (drained, _) = drain.run(Duration::from_secs(5)).await;
15965 assert!(!drained);
15966 let probes = drain.probes();
15967 assert!(probes.len() >= 2, "the drain must re-probe: {probes:?}");
15968 for pair in probes.windows(2) {
15969 assert!(
15970 pair[1] - pair[0] >= latency + drain_busy_reprobe_interval(),
15971 "a probe started {:?} after the previous one, before it could finish",
15972 pair[1] - pair[0]
15973 );
15974 }
15975 }
15976
15977 #[tokio::test(start_paused = true)]
15978 async fn an_unanswered_probe_cannot_hold_the_drain_past_its_ceiling() {
15979 let drain = Drain::start(Answer::Silent);
15980 let ceiling = Duration::from_secs(2);
15981 let (drained, elapsed) = drain.run(ceiling).await;
15982 assert!(!drained, "an unanswered probe counts as busy");
15983 assert!(
15984 elapsed >= ceiling && elapsed < ceiling + drain_busy_reprobe_interval(),
15985 "the probe deadline is capped at the drain ceiling: {elapsed:?}"
15986 );
15987 assert_eq!(
15988 drain.probes().len(),
15989 1,
15990 "the unanswered probe holds the only slot until the ceiling"
15991 );
15992 }
15993
15994 #[derive(Clone, Default)]
15996 struct LogCapture {
15997 events: Arc<StdMutex<Vec<BTreeMap<String, String>>>>,
15998 }
15999
16000 impl<S: tracing::Subscriber> tracing_subscriber::Layer<S> for LogCapture {
16001 fn on_event(
16002 &self,
16003 event: &tracing::Event<'_>,
16004 _context: tracing_subscriber::layer::Context<'_, S>,
16005 ) {
16006 struct Fields(BTreeMap<String, String>);
16007 impl tracing::field::Visit for Fields {
16008 fn record_debug(&mut self, field: &tracing::field::Field, value: &dyn fmt::Debug) {
16009 self.0
16010 .insert(field.name().to_string(), format!("{value:?}"));
16011 }
16012 }
16013 let mut fields = Fields(BTreeMap::new());
16014 event.record(&mut fields);
16015 self.events.lock().unwrap().push(fields.0);
16016 }
16017 }
16018
16019 impl LogCapture {
16020 fn install(&self) -> tracing::subscriber::DefaultGuard {
16021 use tracing_subscriber::layer::SubscriberExt;
16022 tracing::subscriber::set_default(tracing_subscriber::registry().with(self.clone()))
16023 }
16024
16025 fn with_message(&self, needle: &str) -> Vec<BTreeMap<String, String>> {
16026 self.events
16027 .lock()
16028 .unwrap()
16029 .iter()
16030 .filter(|fields| fields.get("message").is_some_and(|m| m.contains(needle)))
16031 .cloned()
16032 .collect()
16033 }
16034 }
16035
16036 #[tokio::test(start_paused = true)]
16041 async fn a_missing_declared_gauge_is_named_on_the_drain_timeout_line() {
16042 let capture = LogCapture::default();
16043 let _guard = capture.install();
16044 let drain = Drain::start(Answer::OmitsGauge);
16045 let (drained, _, trace) = drain.run_traced(Duration::from_secs(2)).await;
16046 assert!(!drained, "a gauge never reported cannot read quiet");
16047 assert!(trace.probes >= 2, "the drain must re-probe: {trace:?}");
16048
16049 let missing = capture.with_message("lacks a declared busy gauge");
16050 assert_eq!(missing.len(), 1, "one warning per drain: {missing:?}");
16051 assert_eq!(
16052 missing[0].get("missing_gauges").map(String::as_str),
16053 Some(GAUGE)
16054 );
16055
16056 warn_drain_timed_out(
16057 MODULE_ID,
16058 Duration::from_secs(2),
16059 RouteCloseReason::Restart,
16060 &Default::default(),
16061 &trace,
16062 );
16063 let timeouts = capture.with_message("route drain timed out");
16064 assert_eq!(timeouts.len(), 1, "{timeouts:?}");
16065 let gauges = &timeouts[0]["busy_gauges"];
16066 assert!(
16067 gauges.starts_with(&format!("{GAUGE}: missing_from_health_body @")),
16068 "the timeout line must name the gauge and why it read busy: {gauges}"
16069 );
16070 assert!(
16071 gauges.ends_with("; no value ever read"),
16072 "no value was ever read for the gauge: {gauges}"
16073 );
16074 assert_eq!(timeouts[0]["gauge_probes"], trace.probes.to_string());
16075 }
16076
16077 #[tokio::test(start_paused = true)]
16081 async fn the_drain_trace_names_a_busy_value_and_an_unanswered_probe() {
16082 let busy = Drain::start(Answer::BusyUntil {
16083 until: Instant::now() + Duration::from_secs(3600),
16084 latency: Duration::ZERO,
16085 });
16086 let (_, _, trace) = busy.run_traced(Duration::from_secs(1)).await;
16087 let rendered = trace.to_string();
16088 assert!(
16089 rendered.starts_with(&format!("{GAUGE}: non_zero(1) @"))
16090 && rendered.contains("; last value non_zero(1) @"),
16091 "{rendered}"
16092 );
16093
16094 let silent = Drain::start(Answer::Silent);
16095 let (_, _, trace) = silent.run_traced(Duration::from_secs(1)).await;
16096 assert_eq!(
16099 trace.to_string(),
16100 format!("{GAUGE}: probe_failed(no-answer) @1000ms; no value ever read")
16101 );
16102 assert_eq!(trace.probes, 1);
16103 }
16104
16105 #[test]
16107 fn a_gauge_that_is_not_a_count_is_busy_and_shown_as_sent() {
16108 let gauges = [GAUGE.to_string()];
16109 let verdicts = gauge_verdicts(Some(&serde_json::json!({ GAUGE: 0.0 })), &gauges);
16110 assert_eq!(verdicts, vec![GaugeVerdict::NotACount("0.0".to_string())]);
16111 assert_eq!(
16112 busy_gauge_observation(&verdicts),
16113 BusyGaugeObservation::Busy
16114 );
16115 let zero = gauge_verdicts(Some(&serde_json::json!({ GAUGE: 0 })), &gauges);
16116 assert_eq!(
16117 busy_gauge_observation(&zero),
16118 BusyGaugeObservation::Quiescent
16119 );
16120 let none = gauge_verdicts(None, &gauges);
16121 assert_eq!(busy_gauge_observation(&none), BusyGaugeObservation::Omitted);
16122 }
16123}
16124
16125#[cfg(test)]
16126mod health_evidence_tests {
16127 use super::{HealthProbeError, HealthProbeEvidence};
16128 use std::collections::HashSet;
16129
16130 #[test]
16138 fn only_a_dead_lane_is_proof_of_death() {
16139 assert!(HealthProbeError::lane_dead("gone").is_proof_of_death());
16140 assert!(!HealthProbeError::no_answer("timed out").is_proof_of_death());
16144 assert!(!HealthProbeError::bad_answer("garbage").is_proof_of_death());
16145 assert!(!HealthProbeError::misconfigured("no table").is_proof_of_death());
16146 }
16147
16148 #[test]
16154 fn every_evidence_class_has_a_distinct_label() {
16155 let labels = [
16156 HealthProbeError::lane_dead("").label(),
16157 HealthProbeError::no_answer("").label(),
16158 HealthProbeError::bad_answer("").label(),
16159 HealthProbeError::misconfigured("").label(),
16160 ];
16161 let unique: HashSet<_> = labels.iter().collect();
16162 assert_eq!(unique.len(), labels.len(), "labels collided: {labels:?}");
16163 }
16164
16165 #[test]
16171 fn classification_preserves_the_original_message() {
16172 let err = HealthProbeError::no_answer("module did not answer within 5s");
16173 assert_eq!(err.to_string(), "module did not answer within 5s");
16174 assert!(matches!(err.evidence, HealthProbeEvidence::NoAnswer));
16175 }
16176}
16177
16178#[cfg(test)]
16179mod health_tombstone_tests {
16180 use std::{path::PathBuf, sync::Arc, time::Duration};
16181
16182 use subc_protocol::{
16183 manifest::Concurrency,
16184 session::{HealthStatus, ModuleControlResponse},
16185 };
16186 use tokio::sync::mpsc;
16187
16188 use super::{
16189 probe_module_health, HealthAction, HealthConfig, HealthProbeEvidence, ModuleProtocol,
16190 ModuleSpec, RestartPolicy, Supervisor, SupervisorRuntimeConfig,
16191 };
16192 use crate::{
16193 control::ControlHandler,
16194 forwarding::{ForwardingTable, ModuleControlRpcCompletion, ModuleControlRpcOutcome},
16195 registry::{ConnectionId, Registry},
16196 router::FrameSink,
16197 };
16198
16199 struct ProbeHarness {
16200 spec: ModuleSpec,
16201 runtime: SupervisorRuntimeConfig,
16202 forwarding: Arc<ForwardingTable>,
16203 module_connection: ConnectionId,
16204 module_rx: mpsc::Receiver<crate::router::OutboundFrame>,
16205 handler: ControlHandler,
16206 module: super::SupervisedModule,
16207 }
16208
16209 fn probe_harness() -> ProbeHarness {
16210 let registry = Arc::new(Registry::default());
16211 let forwarding = Arc::new(ForwardingTable::default());
16212 let supervisor_handle = super::SupervisorHandle::new();
16213 let health = HealthConfig {
16214 http: None,
16215 cadence: Duration::from_secs(30),
16216 deadline: Duration::from_secs(5),
16217 failure_threshold: 3,
16218 on_degraded: HealthAction::Report,
16219 on_failing: HealthAction::Report,
16220 critical: false,
16221 };
16222 let supervisor = Supervisor::new_for_test(Arc::clone(®istry), RestartPolicy::default())
16223 .with_forwarding(Arc::clone(&forwarding))
16224 .with_handle(supervisor_handle.clone())
16225 .with_health_config(health);
16226 let spec = ModuleSpec {
16227 module_id: "late-health-module".to_string(),
16228 program: PathBuf::from("disabled-module"),
16229 args: Vec::new(),
16230 env: Vec::new(),
16231 reserved: false,
16232 reserved_prefixes: Vec::new(),
16233 protocol: ModuleProtocol::Subc,
16234 overlap: Default::default(),
16235 carry_containment: false,
16236 };
16237 let module = supervisor
16238 .supervise_configured(spec.clone(), false)
16239 .unwrap();
16240 let runtime = supervisor.runtime_config();
16241 let handler = ControlHandler::with_forwarding(registry, Arc::clone(&forwarding))
16242 .with_supervisor(supervisor_handle);
16243 let module_connection = ConnectionId::new(700);
16244 let (module_tx, module_rx) = mpsc::channel(8);
16245 forwarding
16246 .register_module_connection(
16247 module_connection,
16248 spec.module_id.clone(),
16249 subc_protocol::PROTOCOL_VERSION,
16250 Concurrency::ModuleManaged,
16251 FrameSink::new(module_tx),
16252 )
16253 .unwrap();
16254
16255 ProbeHarness {
16256 spec,
16257 runtime,
16258 forwarding,
16259 module_connection,
16260 module_rx,
16261 handler,
16262 module,
16263 }
16264 }
16265
16266 async fn finish_after(
16267 harness: &mut ProbeHarness,
16268 stall: Duration,
16269 ) -> ModuleControlRpcCompletion {
16270 assert!(stall > harness.runtime.health.deadline);
16271 let deadline = harness.runtime.health.deadline;
16272 let probe = probe_module_health(&harness.spec.module_id, &harness.runtime, None);
16273 let answer = async {
16274 let frame = harness.module_rx.recv().await.expect("health.check frame");
16275 tokio::time::advance(deadline).await;
16276 tokio::task::yield_now().await;
16277 tokio::time::advance(stall - deadline).await;
16278 harness
16279 .forwarding
16280 .complete_module_control_rpc(
16281 harness.module_connection,
16282 frame.header.corr,
16283 Some("health.check"),
16284 ModuleControlRpcOutcome::Response(ModuleControlResponse::HealthCheck {
16285 status: HealthStatus::Ok,
16286 detail: None,
16287 metrics: None,
16288 }),
16289 )
16290 .unwrap()
16291 };
16292 let (probe_result, completion) = tokio::join!(probe, answer);
16293 let err = probe_result.expect_err("probe must miss its deadline");
16294 assert!(matches!(err.evidence, HealthProbeEvidence::NoAnswer));
16295 completion
16296 }
16297
16298 async fn time_out_without_answer(harness: &mut ProbeHarness) {
16299 let deadline = harness.runtime.health.deadline;
16300 let probe = probe_module_health(&harness.spec.module_id, &harness.runtime, None);
16301 let exhaust_deadline = async {
16302 let _frame = harness.module_rx.recv().await.expect("health.check frame");
16303 tokio::time::advance(deadline).await;
16304 tokio::task::yield_now().await;
16305 };
16306 let (probe_result, ()) = tokio::join!(probe, exhaust_deadline);
16307 let err = probe_result.expect_err("probe must miss its deadline");
16308 assert!(matches!(err.evidence, HealthProbeEvidence::NoAnswer));
16309 }
16310
16311 async fn run_probe_cycle(harness: &mut ProbeHarness, answer: bool) {
16312 let registry = Arc::clone(&harness.module.inner.registry);
16313 let snapshot = Arc::clone(&harness.module.inner.snapshot);
16314 let process_liveness = super::SupervisorProcessLiveness::default();
16315 let mut child = None;
16316 let cycle = super::run_health_probe_cycle(
16317 &harness.spec,
16318 &harness.runtime,
16319 ®istry,
16320 &process_liveness,
16321 &snapshot,
16322 &mut child,
16323 );
16324 let peer = async {
16325 let frame = harness.module_rx.recv().await.expect("health.check frame");
16326 if answer {
16327 harness
16328 .forwarding
16329 .complete_module_control_rpc(
16330 harness.module_connection,
16331 frame.header.corr,
16332 Some("health.check"),
16333 ModuleControlRpcOutcome::Response(ModuleControlResponse::HealthCheck {
16334 status: HealthStatus::Ok,
16335 detail: None,
16336 metrics: Some(serde_json::json!({"ready": true})),
16337 }),
16338 )
16339 .unwrap();
16340 } else {
16341 tokio::time::advance(harness.runtime.health.deadline).await;
16342 tokio::task::yield_now().await;
16343 }
16344 };
16345 tokio::join!(cycle, peer);
16346 }
16347
16348 #[tokio::test(start_paused = true)]
16349 async fn unanswered_probe_is_unknown_until_threshold_and_ok_report_recovers() {
16350 let mut harness = probe_harness();
16351 let monitor = harness.module.inner.monitor.lock().unwrap().take().unwrap();
16355 monitor.abort();
16356 let _ = monitor.await;
16357 super::update_snapshot(&harness.module.inner.snapshot, None, |state| {
16358 state.enabled = true;
16359 state.state = super::ModuleState::Running;
16360 state.process_alive = true;
16361 })
16362 .unwrap();
16363
16364 run_probe_cycle(&mut harness, true).await;
16365 assert_eq!(
16366 harness.module.status().unwrap().health.status,
16367 super::SupervisorHealthStatus::Ok
16368 );
16369
16370 for failures in 1..harness.runtime.health.failure_threshold {
16371 run_probe_cycle(&mut harness, false).await;
16372 let status = harness.module.status().unwrap();
16373 assert_eq!(status.health.status, super::SupervisorHealthStatus::Unknown);
16374 assert_eq!(status.health.consecutive_failures, failures);
16375 assert!(status.health.last_probe_ms.is_some());
16376 assert!(status.health.detail.unwrap().starts_with("[no-answer]"));
16377 assert!(status.health.metrics.is_none());
16378 assert_eq!(status.state, super::ModuleState::Running);
16379 assert!(status.process_alive);
16380 assert_eq!(status.restart_count, 0);
16381 assert_eq!(status.lifetime_restarts, 0);
16382 assert!(status.health.last_action.is_none());
16383 assert!(harness.runtime.scheduled_respawn.lock().unwrap().is_none());
16384 }
16385
16386 run_probe_cycle(&mut harness, true).await;
16387 let recovered = harness.module.status().unwrap();
16388 assert_eq!(recovered.health.status, super::SupervisorHealthStatus::Ok);
16389 assert_eq!(recovered.health.consecutive_failures, 0);
16390 assert!(recovered.health.detail.is_none());
16391 assert_eq!(
16392 recovered.health.metrics,
16393 Some(serde_json::json!({"ready": true}))
16394 );
16395 assert_eq!(recovered.lifetime_restarts, 0);
16396
16397 for failures in 1..=harness.runtime.health.failure_threshold {
16398 run_probe_cycle(&mut harness, false).await;
16399 let status = harness.module.status().unwrap();
16400 assert_eq!(status.health.consecutive_failures, failures);
16401 if failures < harness.runtime.health.failure_threshold {
16402 assert_eq!(status.health.status, super::SupervisorHealthStatus::Unknown);
16403 assert_eq!(status.state, super::ModuleState::Running);
16404 assert_eq!(status.lifetime_restarts, 0);
16405 assert!(harness.runtime.scheduled_respawn.lock().unwrap().is_none());
16406 } else {
16407 assert_eq!(
16408 status.health.status,
16409 super::SupervisorHealthStatus::Unresponsive
16410 );
16411 assert_eq!(status.state, super::ModuleState::Restarting);
16412 assert_eq!(status.restart_count, 1);
16413 assert_eq!(status.lifetime_restarts, 1);
16414 assert_eq!(status.health.last_action.as_deref(), Some("restart"));
16415 assert!(harness.runtime.scheduled_respawn.lock().unwrap().is_some());
16416 }
16417 }
16418 }
16419
16420 #[tokio::test(start_paused = true)]
16421 async fn late_health_answers_record_start_anchored_latency_for_two_stalls() {
16422 let mut harness = probe_harness();
16423
16424 let first = finish_after(&mut harness, Duration::from_secs(8)).await;
16425 let first_latency = match &first {
16426 ModuleControlRpcCompletion::LateHealthAnswer { latency, .. } => *latency,
16427 other => panic!("late answer was not retained: {other:?}"),
16428 };
16429 assert!(harness.handler.observe_module_control_completion(first));
16430
16431 let second = finish_after(&mut harness, Duration::from_secs(11)).await;
16432 let second_latency = match &second {
16433 ModuleControlRpcCompletion::LateHealthAnswer { latency, .. } => *latency,
16434 other => panic!("late answer was not retained: {other:?}"),
16435 };
16436 assert!(harness.handler.observe_module_control_completion(second));
16437
16438 assert_eq!(first_latency, Duration::from_secs(8));
16439 assert_eq!(
16440 second_latency - first_latency,
16441 Duration::from_secs(3),
16442 "latency must grow linearly with the additional stall"
16443 );
16444 let health = harness.module.status().unwrap().health;
16445 assert_eq!(health.late_answer_count, 2);
16446 assert_eq!(health.last_late_answer_latency_ms, Some(11_000));
16447 }
16448
16449 #[tokio::test(start_paused = true)]
16457 async fn late_answer_clears_the_consecutive_failure_streak() {
16458 let mut harness = probe_harness();
16459
16460 time_out_without_answer(&mut harness).await;
16462 harness
16463 .module
16464 .record_health_probe_failure_for_test("[no-answer] test miss")
16465 .unwrap();
16466 assert_eq!(
16467 harness.module.status().unwrap().health.consecutive_failures,
16468 1,
16469 "precondition: the miss must be on the streak before the late answer"
16470 );
16471
16472 let late = finish_after(&mut harness, Duration::from_secs(9)).await;
16474 assert!(matches!(
16475 late,
16476 ModuleControlRpcCompletion::LateHealthAnswer { .. }
16477 ));
16478 assert!(harness.handler.observe_module_control_completion(late));
16479
16480 let health = harness.module.status().unwrap().health;
16481 assert_eq!(
16482 health.consecutive_failures, 0,
16483 "a late answer is an answer: the streak must reset"
16484 );
16485 assert_eq!(health.late_answer_count, 1);
16486 }
16487
16488 #[tokio::test(start_paused = true)]
16489 async fn repeated_serial_probe_cycles_keep_one_tombstone_per_endpoint() {
16490 let mut harness = probe_harness();
16491
16492 for _ in 0..20 {
16493 time_out_without_answer(&mut harness).await;
16494 assert_eq!(
16495 harness.forwarding.health_probe_tombstone_count().unwrap(),
16496 1
16497 );
16498 }
16499 }
16500}
16501
16502#[cfg(test)]
16503mod child_env_tests {
16504 #[cfg(windows)]
16505 use super::{
16506 clear_current_process_facts, observed_launch_nonce_source, ModuleState, SupervisorSnapshot,
16507 };
16508 #[cfg(windows)]
16509 use std::sync::{Arc, OnceLock};
16510 #[cfg(windows)]
16511 #[test]
16512 fn windows_wire_spawn_prepares_pipe_and_keeps_phase_one_env() {
16513 let spec = spec(Vec::new());
16514 let mut command = Command::new("unused");
16515 let handoff = apply_wire_spawn_args(&mut command, &spec, None, None)
16516 .unwrap()
16517 .unwrap();
16518 let name = command
16519 .as_std()
16520 .get_envs()
16521 .find(|(key, _)| *key == OsStr::new(subc_os::launch_nonce::LAUNCH_NONCE_PIPE_ENV))
16522 .and_then(|(_, value)| value)
16523 .unwrap();
16524 assert_eq!(name, OsStr::new(handoff.name()));
16525 assert!(handoff.name().starts_with(r"\\.\pipe\subc-launch-"));
16526 assert!(command.as_std().get_envs().any(|(key, value)| key
16527 == OsStr::new(subc_os::LAUNCH_NONCE_PIPE_FALLBACK_ENV)
16528 && value == Some(OsStr::new("env"))));
16529 assert!(command
16530 .as_std()
16531 .get_envs()
16532 .any(|(key, value)| key == OsStr::new(SUBC_LAUNCH_NONCE_ENV) && value.is_some()));
16533 }
16534
16535 #[cfg(windows)]
16536 #[test]
16537 fn windows_source_reporting_tracks_consumption_and_resets_per_spawn() {
16538 let mut snapshot = SupervisorSnapshot::new(ModuleState::Running, true);
16539 snapshot.process_alive = true;
16540 snapshot.spawned_protocol = Some(ModuleProtocol::Subc);
16541 assert_eq!(
16542 observed_launch_nonce_source(&snapshot).as_deref(),
16543 Some("env")
16544 );
16545 let consumed = Arc::new(OnceLock::new());
16546 snapshot.pipe_consumed = Some(consumed.clone());
16547 assert_eq!(
16548 observed_launch_nonce_source(&snapshot).as_deref(),
16549 Some("env")
16550 );
16551 consumed.set(()).unwrap();
16552 assert_eq!(
16553 observed_launch_nonce_source(&snapshot).as_deref(),
16554 Some("pipe")
16555 );
16556 snapshot.pipe_consumed = Some(Arc::new(OnceLock::new()));
16557 assert_eq!(
16558 observed_launch_nonce_source(&snapshot).as_deref(),
16559 Some("env")
16560 );
16561 clear_current_process_facts(&mut snapshot);
16562 assert_eq!(observed_launch_nonce_source(&snapshot), None);
16563 }
16564 use super::{
16565 apply_child_env, apply_spawn_role, apply_wire_spawn_args, LaunchSpecDigest, ModuleProtocol,
16566 ModuleSpec, SpawnRole, SupervisorHandle, SPAWN_ROLE_SWAP_CANDIDATE, SUBC_ARG,
16567 SUBC_LAUNCH_NONCE_ENV, SUBC_MODULE_ID_ENV, SUBC_SPAWN_ROLE_ENV,
16568 };
16569 use std::{ffi::OsStr, path::PathBuf};
16570 use tokio::process::Command;
16571
16572 fn spec(env: Vec<(String, String)>) -> ModuleSpec {
16573 ModuleSpec {
16574 module_id: "env-plan".to_string(),
16575 program: PathBuf::from("/nonexistent"),
16576 args: Vec::new(),
16577 env,
16578 reserved: false,
16579 reserved_prefixes: Vec::new(),
16580 protocol: ModuleProtocol::Subc,
16581 overlap: Default::default(),
16582 carry_containment: false,
16583 }
16584 }
16585
16586 #[test]
16587 fn configured_launch_spec_digest_is_canonical_and_excludes_injected_inputs() {
16588 let base = spec(vec![("A".into(), "B=C".into()), ("Z".into(), "v".into())]);
16589 let digest = LaunchSpecDigest::from_spec(&base);
16590 let mut reordered = base.clone();
16591 reordered.env.reverse();
16592 reordered.module_id = "different-id".into();
16593 reordered.reserved = true;
16594 reordered.overlap = Default::default();
16595 assert_eq!(digest, LaunchSpecDigest::from_spec(&reordered));
16596 let mut delimiter_collision = base.clone();
16597 delimiter_collision.env[0] = ("A=B".into(), "C".into());
16598 assert_ne!(digest, LaunchSpecDigest::from_spec(&delimiter_collision));
16599 let mut changed = base.clone();
16600 changed.protocol = ModuleProtocol::None;
16601 assert_ne!(digest, LaunchSpecDigest::from_spec(&changed));
16602 changed = base.clone();
16603 changed.args.push("--flag".into());
16604 assert_ne!(digest, LaunchSpecDigest::from_spec(&changed));
16605 changed = base.clone();
16606 changed.program = PathBuf::from("/other");
16607 assert_ne!(digest, LaunchSpecDigest::from_spec(&changed));
16608 changed = base.clone();
16609 changed.env[1].1 = "other".into();
16610 assert_ne!(digest, LaunchSpecDigest::from_spec(&changed));
16611 assert!(!format!("{digest:?}").contains("B=C"));
16612 assert!(!format!("{digest:?}").contains("[u8"));
16613 }
16614
16615 #[cfg(unix)]
16616 #[test]
16617 fn native_program_bytes_do_not_collapse_through_lossy_utf8() {
16618 use std::os::unix::ffi::OsStringExt;
16619 let mut left = spec(Vec::new());
16620 left.program = std::ffi::OsString::from_vec(vec![b'/', 0xff]).into();
16621 let mut right = left.clone();
16622 right.program = std::ffi::OsString::from_vec(vec![b'/', 0xfe]).into();
16623 assert_eq!(
16624 left.program.to_string_lossy(),
16625 right.program.to_string_lossy()
16626 );
16627 assert_ne!(
16628 LaunchSpecDigest::from_spec(&left),
16629 LaunchSpecDigest::from_spec(&right)
16630 );
16631 }
16632
16633 #[test]
16647 fn ambient_ck_log_is_removed_and_a_configured_one_survives() {
16648 let mut command = Command::new("/nonexistent");
16649 apply_child_env(&mut command, &spec(Vec::new()));
16650 let removed = command
16651 .as_std()
16652 .get_envs()
16653 .any(|(key, value)| key == OsStr::new("CK_LOG") && value.is_none());
16654 assert!(
16655 removed,
16656 "ambient CK_LOG must be explicitly removed for an unconfigured module"
16657 );
16658
16659 let mut configured = Command::new("/nonexistent");
16660 apply_child_env(
16661 &mut configured,
16662 &spec(vec![("CK_LOG".to_string(), "debug".to_string())]),
16663 );
16664 let effective = configured
16665 .as_std()
16666 .get_envs()
16667 .filter(|(key, _)| *key == OsStr::new("CK_LOG"))
16668 .last()
16669 .map(|(_, value)| value.map(|v| v.to_string_lossy().into_owned()));
16670 assert_eq!(
16671 effective,
16672 Some(Some("debug".to_string())),
16673 "a module's configured CK_LOG must survive the ambient removal"
16674 );
16675 }
16676
16677 #[test]
16686 fn protocol_none_spawn_carries_no_subc_argument_and_no_nonce() {
16687 let connection_file = std::path::Path::new("/run/subc-connection.json");
16688 let handle = SupervisorHandle::new();
16689
16690 let mut none_spec = spec(Vec::new());
16691 none_spec.protocol = ModuleProtocol::None;
16692 let mut none = Command::new("/nonexistent");
16693 let none_handoff =
16694 apply_wire_spawn_args(&mut none, &none_spec, Some(connection_file), Some(&handle))
16695 .expect("protocol-none spawn args apply");
16696 assert!(
16697 none_handoff.is_none(),
16698 "protocol:none spawn must not receive a nonce descriptor"
16699 );
16700 #[cfg(windows)]
16701 assert!(!none.as_std().get_envs().any(|(key, value)| key
16702 == OsStr::new(subc_os::LAUNCH_NONCE_PIPE_FALLBACK_ENV)
16703 && value.is_some()));
16704 assert!(
16705 !none.as_std().get_envs().any(|(key, value)| key
16706 == OsStr::new(subc_os::LAUNCH_NONCE_FD_ENV)
16707 && value.is_some()),
16708 "protocol:none spawn must not name a nonce descriptor"
16709 );
16710 let none_args: Vec<String> = none
16711 .as_std()
16712 .get_args()
16713 .map(|a| a.to_string_lossy().into_owned())
16714 .collect();
16715 assert!(
16716 !none_args.iter().any(|a| a == SUBC_ARG),
16717 "protocol:none argv must not carry --subc; got {none_args:?}"
16718 );
16719 let none_has_nonce = none
16720 .as_std()
16721 .get_envs()
16722 .any(|(key, value)| key == OsStr::new(SUBC_LAUNCH_NONCE_ENV) && value.is_some());
16723 assert!(
16724 !none_has_nonce,
16725 "protocol:none spawn must not receive a launch nonce"
16726 );
16727 let none_has_module_id = none
16728 .as_std()
16729 .get_envs()
16730 .any(|(key, value)| key == OsStr::new(SUBC_MODULE_ID_ENV) && value.is_some());
16731 assert!(
16732 none_has_module_id,
16733 "SUBC_MODULE_ID is inert and stays on every path"
16734 );
16735 assert!(
16736 handle.spawn_nonce(&none_spec.module_id).is_none(),
16737 "no nonce record for a process that will never present one"
16738 );
16739
16740 let wire_spec = spec(Vec::new());
16742 let mut wire = Command::new("/nonexistent");
16743 let wire_handoff =
16744 apply_wire_spawn_args(&mut wire, &wire_spec, Some(connection_file), Some(&handle))
16745 .expect("subc-wire spawn args apply");
16746 let wire_fd_env = wire
16747 .as_std()
16748 .get_envs()
16749 .find(|(key, _)| *key == OsStr::new(subc_os::LAUNCH_NONCE_FD_ENV))
16750 .and_then(|(_, value)| value.map(|v| v.to_string_lossy().into_owned()));
16751 #[cfg(unix)]
16752 assert_eq!(
16753 wire_fd_env,
16754 Some(wire_handoff.expect("a descriptor handoff").fd_env_value()),
16755 "a subc-wire spawn names the pipe it will receive at descriptor 3"
16756 );
16757 #[cfg(windows)]
16758 assert!(wire_handoff.is_some() && wire_fd_env.is_none());
16759 #[cfg(not(any(unix, windows)))]
16760 assert!(wire_handoff.is_none() && wire_fd_env.is_none());
16761 let wire_args: Vec<String> = wire
16762 .as_std()
16763 .get_args()
16764 .map(|a| a.to_string_lossy().into_owned())
16765 .collect();
16766 assert_eq!(
16767 wire_args,
16768 vec![
16769 SUBC_ARG.to_string(),
16770 connection_file.to_string_lossy().into_owned()
16771 ],
16772 "a subc-wire spawn still carries --subc <path>"
16773 );
16774 assert_eq!(
16775 wire.as_std()
16776 .get_envs()
16777 .any(|(key, value)| key == OsStr::new(SUBC_LAUNCH_NONCE_ENV) && value.is_some()),
16778 !cfg!(unix),
16779 "only Windows supplies the environment nonce"
16780 );
16781 assert!(handle.spawn_nonce(&wire_spec.module_id).is_some());
16782 }
16783
16784 #[test]
16794 fn plain_spawn_removes_the_spawn_role_even_when_the_spec_sets_it() {
16795 let role = |command: &Command| {
16796 command
16797 .as_std()
16798 .get_envs()
16799 .filter(|(key, _)| *key == OsStr::new(SUBC_SPAWN_ROLE_ENV))
16800 .last()
16801 .map(|(_, value)| value.map(|v| v.to_string_lossy().into_owned()))
16802 };
16803 let forged = spec(vec![(
16804 SUBC_SPAWN_ROLE_ENV.to_string(),
16805 SPAWN_ROLE_SWAP_CANDIDATE.to_string(),
16806 )]);
16807
16808 let mut plain = Command::new("/nonexistent");
16809 apply_child_env(&mut plain, &forged);
16810 apply_spawn_role(&mut plain, SpawnRole::Plain);
16811 assert_eq!(
16812 role(&plain),
16813 Some(None),
16814 "a plain spawn must remove SUBC_SPAWN_ROLE, whatever the spec says"
16815 );
16816
16817 let mut candidate = Command::new("/nonexistent");
16818 apply_child_env(&mut candidate, &spec(Vec::new()));
16819 apply_spawn_role(&mut candidate, SpawnRole::SwapCandidate);
16820 assert_eq!(
16821 role(&candidate),
16822 Some(Some(SPAWN_ROLE_SWAP_CANDIDATE.to_string()))
16823 );
16824 }
16825
16826 #[test]
16832 fn daemon_private_capture_keys_are_not_passed_to_the_child() {
16833 let mut command = Command::new("/nonexistent");
16834 apply_child_env(
16835 &mut command,
16836 &spec(vec![
16837 (super::CAPTURE_KEEP_ENV.to_string(), "5".to_string()),
16838 ("KEPT".to_string(), "yes".to_string()),
16839 ]),
16840 );
16841 let keys: Vec<String> = command
16842 .as_std()
16843 .get_envs()
16844 .filter(|(_, value)| value.is_some())
16845 .map(|(key, _)| key.to_string_lossy().into_owned())
16846 .collect();
16847 assert!(keys.contains(&"KEPT".to_string()), "got {keys:?}");
16848 assert!(
16849 !keys.contains(&super::CAPTURE_KEEP_ENV.to_string()),
16850 "daemon-private capture key leaked to the child: {keys:?}"
16851 );
16852 }
16853}
16854
16855#[cfg(test)]
16856mod jitter_tests {
16857 use super::jittered_health_delay;
16858 use std::{collections::HashSet, time::Duration};
16859
16860 const FLEET: [&str; 14] = [
16869 "aft",
16870 "alfonso-core",
16871 "magic-context",
16872 "broca",
16873 "thalamus",
16874 "quota",
16875 "engram",
16876 "plexus",
16877 "cerebellum",
16878 "astrocyte",
16879 "synapse",
16880 "subc-mcp",
16881 "cortexkit-credentials",
16882 "subc-federation",
16883 ];
16884
16885 #[test]
16893 fn probe_delays_disperse_across_the_fleet() {
16894 let cadence = Duration::from_secs(30);
16895 let delays: HashSet<Duration> = FLEET
16896 .iter()
16897 .map(|id| jittered_health_delay(id, 0, cadence))
16898 .collect();
16899 assert_eq!(
16900 delays.len(),
16901 FLEET.len(),
16902 "every supervised module must land on its own probe offset"
16903 );
16904 }
16905
16906 #[test]
16912 fn jitter_only_delays_and_stays_within_one_tenth_of_cadence() {
16913 let cadence = Duration::from_secs(30);
16914 let span = cadence / 10;
16915 for id in FLEET {
16916 for probe_index in 0..8 {
16917 let delay = jittered_health_delay(id, probe_index, cadence);
16918 assert!(
16919 delay >= cadence,
16920 "{id}#{probe_index}: jitter must not shorten the cadence"
16921 );
16922 assert!(
16923 delay < cadence + span,
16924 "{id}#{probe_index}: jitter must stay inside one tenth of the cadence"
16925 );
16926 }
16927 }
16928 }
16929
16930 #[test]
16936 fn a_module_offset_is_stable_across_restarts() {
16937 let cadence = Duration::from_secs(30);
16938 for id in FLEET {
16939 assert_eq!(
16940 jittered_health_delay(id, 0, cadence),
16941 jittered_health_delay(id, 0, cadence),
16942 "{id}: the same module and probe index must produce the same offset"
16943 );
16944 }
16945 }
16946
16947 #[test]
16949 fn zero_cadence_yields_zero_delay() {
16950 assert_eq!(
16951 jittered_health_delay("aft", 0, Duration::ZERO),
16952 Duration::ZERO
16953 );
16954 }
16955}
16956
16957#[cfg(all(test, target_os = "linux"))]
16958mod cgroup_placement_tests {
16959 use super::{
16960 apply_cgroup_placement, remove_module_cgroup, LaunchSpecDigest, ModuleProtocol, ModuleSpec,
16961 SuperviseError, SupervisedChild,
16962 };
16963 use crate::stderr_tail::{StderrRing, StderrTailConfig};
16964 use cortexkit_test_support::ScratchDir;
16965 use std::{
16966 fs, io,
16967 path::{Path, PathBuf},
16968 sync::{Arc, Mutex},
16969 };
16970 use tokio::process::Command;
16971
16972 #[tokio::test]
16973 async fn unique_spawn_cgroups_clean_up_across_restarts_and_shutdown() {
16974 use super::*;
16975 let dir = ScratchDir::new("unique-spawn-cgroups");
16976 let root = PathBuf::from(format!(
16977 "/sys/fs/cgroup/subc-unique-test-{}-{}",
16978 std::process::id(),
16979 unix_ms_now()
16980 ));
16981 if let Err(error) = fs::create_dir(&root) {
16982 assert!(
16983 std::env::var_os("SUBC_REQUIRE_CGROUP_TEST").is_none(),
16984 "required cgroup test cannot execute: {error}"
16985 );
16986 eprintln!("SKIP unique_spawn_cgroups_clean_up_across_restarts_and_shutdown: {error}");
16987 return;
16988 }
16989 let placement = subc_cgroup::prepare_at(&root).unwrap().unwrap();
16990 let group_count = || {
16991 fs::read_dir(root.join("subc-modules"))
16992 .unwrap()
16993 .map(|entry| entry.unwrap().file_type().unwrap())
16994 .filter(|kind| kind.is_dir())
16995 .count()
16996 };
16997 let supervisor =
16998 Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default())
16999 .with_cgroup_placement(Some(placement.clone()));
17000 let runtime = supervisor.runtime_config();
17001 let mut spec = ModuleSpec {
17002 module_id: "unique-spawn".into(),
17003 program: PathBuf::from("/bin/sleep"),
17004 args: vec!["60".into()],
17005 env: ["XDG_DATA_HOME", "XDG_RUNTIME_DIR", "XDG_CONFIG_HOME"]
17006 .into_iter()
17007 .map(|key| (key.into(), dir.to_string_lossy().into_owned()))
17008 .collect(),
17009 reserved: false,
17010 reserved_prefixes: vec![],
17011 protocol: ModuleProtocol::None,
17012 overlap: Default::default(),
17013 carry_containment: false,
17014 };
17015 let spawn = |spec: &ModuleSpec| {
17016 spawn_child(
17017 spec,
17018 None,
17019 None,
17020 &runtime.stderr_ring,
17021 None,
17022 &runtime.child_roster,
17023 Some(&placement),
17024 )
17025 .unwrap()
17026 };
17027 let mut live = spawn(&spec);
17028 for _ in 0..3 {
17029 let next = spawn(&spec);
17031 assert_ne!(live.module_id, next.module_id);
17032 live.start_kill().unwrap();
17033 live.wait().await.unwrap();
17034 live = next;
17035 assert!(
17036 live.child.try_wait().unwrap().is_none(),
17037 "retiring the old slot must not kill the replacement"
17038 );
17039 assert_eq!(
17040 group_count(),
17041 1,
17042 "only the live spawn's cgroup should remain"
17043 );
17044 }
17045 supervisor.begin_daemon_shutdown();
17046 let reap = tokio::spawn(async move {
17047 live.wait().await.unwrap();
17048 });
17049 supervisor
17050 .end_children_for_daemon_shutdown(false, std::future::pending())
17051 .await;
17052 reap.await.unwrap();
17053 assert_eq!(group_count(), 0);
17054 spec.program = PathBuf::from("/bin/true");
17056 spec.args.clear();
17057 let fresh_roster = ChildRoster::default();
17058 let mut short = spawn_child(
17059 &spec,
17060 None,
17061 None,
17062 &runtime.stderr_ring,
17063 None,
17064 &fresh_roster,
17065 Some(&placement),
17066 )
17067 .unwrap();
17068 short.wait().await.unwrap();
17069 assert_eq!(group_count(), 0);
17070 spec.module_id = "_".repeat(255);
17071 let mut long_id = spawn_child(
17072 &spec,
17073 None,
17074 None,
17075 &runtime.stderr_ring,
17076 None,
17077 &fresh_roster,
17078 Some(&placement),
17079 )
17080 .unwrap();
17081 long_id.wait().await.unwrap();
17082 assert_eq!(
17083 group_count(),
17084 0,
17085 "valid long module IDs must not exceed cgroup NAME_MAX"
17086 );
17087 fs::remove_dir(root.join("subc-modules")).unwrap();
17088 fs::remove_dir(root).unwrap();
17089 eprintln!("EXECUTED unique_spawn_cgroups_clean_up_across_restarts_and_shutdown");
17090 }
17091
17092 #[test]
17093 fn failed_parent_cgroup_open_is_a_cgroup_supervision_error() {
17094 let path = Path::new("/definitely-missing-subc-cgroup");
17095 let mut command = Command::new("true");
17096 let error = apply_cgroup_placement(
17097 &mut command,
17098 &ModuleSpec {
17099 module_id: "broken-cgroup".to_string(),
17100 program: PathBuf::from("true"),
17101 args: Vec::new(),
17102 env: Vec::new(),
17103 reserved: false,
17104 reserved_prefixes: Vec::new(),
17105 protocol: ModuleProtocol::Subc,
17106 overlap: Default::default(),
17107 carry_containment: false,
17108 },
17109 path,
17110 )
17111 .expect_err("a parent cgroup open failure must reject the supervised spawn");
17112 let reason = error.to_string();
17113
17114 assert!(
17115 matches!(error, SuperviseError::Cgroup { .. }),
17116 "parent cgroup open must be reported as a cgroup supervision error: {reason}"
17117 );
17118 assert!(
17119 reason.contains("/definitely-missing-subc-cgroup/cgroup.procs"),
17120 "parent cgroup open failure must name cgroup.procs: {reason}"
17121 );
17122 }
17123
17124 #[tokio::test]
17125 async fn reaping_a_child_removes_its_empty_module_cgroup() {
17126 let root = ScratchDir::new("supervisor-reap-cgroup");
17127 fs::write(root.join("cgroup.procs"), b"").expect("write scratch cgroup marker");
17128 let placement = subc_cgroup::prepare_at(&root)
17129 .expect("prepare scratch cgroup root")
17130 .expect("scratch root has a cgroup.procs marker");
17131 let module_id = "reaped-module";
17132 let module = placement
17133 .module_path(module_id)
17134 .expect("create scratch module cgroup");
17135 let child = Command::new("true")
17136 .env("XDG_DATA_HOME", root.path())
17137 .env("XDG_RUNTIME_DIR", root.path())
17138 .env("XDG_CONFIG_HOME", root.path())
17139 .spawn()
17140 .expect("spawn short-lived child");
17141 let pid = child.id().expect("spawned child has pid");
17142 let mut child = SupervisedChild {
17143 subtree_target: crate::subtree_resources::Target::Unavailable("not_running"),
17144 child: crate::supervise::containment::ProcessChild::plain(child, pid),
17145 exec: crate::activity::ExecOwner(Arc::new(crate::activity::ExecOutcome::new())),
17146 protocol: ModuleProtocol::Subc,
17147 module_id: module_id.to_string(),
17148 cgroup_placement: Some(placement),
17149 stdout_pump: None,
17150 stderr_pump: None,
17151 stderr_ring: Arc::new(Mutex::new(StderrRing::new(StderrTailConfig::default()))),
17152 spawned_at: tokio::time::Instant::now(),
17153 registration_deadline: None,
17154 registration_watch: None,
17155 spawned_at_ms: 0,
17156 spawned_from: PathBuf::from("true"),
17157 launch_spec_digest: LaunchSpecDigest([0; 32]),
17158 spawned_file_identity: None,
17159 process_start_time: None,
17160 process_identity: None,
17161 pid,
17162 roster_guard: None,
17163 #[cfg(target_os = "macos")]
17164 privacy_exec: None,
17165 spawn_failure: None,
17166 };
17167
17168 child.wait().await.expect("reap short-lived child");
17169
17170 assert!(
17171 !module.exists(),
17172 "waiting for a non-carried child must remove its empty cgroup before returning"
17173 );
17174 child.finish_exit(false).await.unwrap();
17175 }
17176
17177 #[test]
17178 fn non_empty_cgroup_removal_is_reported_without_blocking_teardown() {
17179 let root = ScratchDir::new("supervisor-non-empty-cgroup");
17180 fs::write(root.join("cgroup.procs"), b"").expect("write scratch cgroup marker");
17181 let placement = subc_cgroup::prepare_at(&root)
17182 .expect("prepare scratch cgroup root")
17183 .expect("scratch root has a cgroup.procs marker");
17184 let module = placement
17185 .module_path("surviving-module")
17186 .expect("create scratch module cgroup");
17187 fs::write(module.join("surviving-process"), b"still present")
17188 .expect("make scratch cgroup non-empty");
17189 let (logs, _guard) = crate::router::test_log::log_capture(tracing::Level::WARN);
17190
17191 remove_module_cgroup(&placement, "surviving-module");
17192
17193 let logs = crate::router::test_log::captured_logs(&logs);
17194 assert!(
17195 module.exists(),
17196 "failed removal must leave the cgroup intact"
17197 );
17198 assert!(
17199 logs.contains("could not remove module cgroup after process exit; continuing teardown")
17200 && logs.contains("surviving-module"),
17201 "best-effort removal must report the failure without returning it: {logs}"
17202 );
17203 }
17204
17205 #[test]
17206 fn cgroup_pre_exec_spawn_failure_names_the_cgroup_path() {
17207 let cgroup_path = PathBuf::from("/sys/fs/cgroup/subc-modules/broken-module");
17208 let reason = SuperviseError::Spawn {
17209 program: PathBuf::from("/bin/true"),
17210 source: io::Error::from_raw_os_error(13),
17211 cgroup_path: Some(cgroup_path.clone()),
17212 }
17213 .to_string();
17214
17215 assert!(
17216 reason.contains(&cgroup_path.display().to_string()),
17217 "a pre_exec spawn failure must name the cgroup path: {reason}"
17218 );
17219 }
17220}
17221
17222#[cfg(test)]
17223mod spawn_subscriber_lag_tests {
17224 use super::*;
17225
17226 #[tokio::test]
17231 async fn lagged_spawn_subscriber_receives_a_terminal_lagged_error_after_its_queued_frames() {
17232 let feed = SpawnEventFeed::default();
17233 feed.configure_incarnation("lag-incarnation".to_string());
17234 let (tx, mut rx) = mpsc::channel(1);
17237 feed.subscribe(ConnectionId::new(1), 7, 1, None, FrameSink::new(tx))
17238 .expect("subscribe");
17239 let emitted = SPAWN_SUBSCRIBER_BUFFER + 16;
17240 for index in 0..emitted {
17241 feed.emit_spawned(&format!("lag-module-{index}"), 1000, 0);
17242 tokio::task::yield_now().await;
17245 }
17246 assert_eq!(
17247 feed.subscriber_count(),
17248 0,
17249 "the lagged subscriber must be removed"
17250 );
17251
17252 let mut data = Vec::new();
17253 let mut last = None;
17254 loop {
17255 let next = tokio::time::timeout(Duration::from_secs(5), rx.recv())
17256 .await
17257 .expect("the forwarder must finish once the subscriber is dropped");
17258 let Some(outbound) = next else { break };
17259 let frame = outbound.frame;
17260 if frame.header.ty == FrameType::StreamData {
17261 assert!(last.is_none(), "no data may follow the terminal frame");
17262 let event: SpawnEvent = serde_json::from_slice(&frame.body).unwrap();
17263 data.push(event.cursor.seq);
17264 } else {
17265 assert!(last.is_none(), "exactly one terminal frame");
17266 last = Some(frame);
17267 }
17268 }
17269 assert!(!data.is_empty(), "queued frames drain before the terminal");
17270 for pair in data.windows(2) {
17271 assert_eq!(
17272 pair[1],
17273 pair[0] + 1,
17274 "queued frames arrive dense and in order"
17275 );
17276 }
17277 let terminal = last.expect("a lagged subscriber must receive a terminal frame");
17278 assert_eq!(terminal.header.ty, FrameType::Error);
17279 assert_eq!(terminal.header.corr, 7);
17280 let body: subc_protocol::ErrorBody = serde_json::from_slice(&terminal.body).unwrap();
17281 assert_eq!(body.code, SPAWN_SUBSCRIBER_LAGGED_CODE);
17282 let detail = body.detail.expect("lagged error carries detail");
17283 assert_eq!(
17284 detail["first_undelivered_cursor"]["seq"],
17285 data.last().unwrap() + 1,
17286 "the named cursor is the first event the subscriber did not receive"
17287 );
17288 assert_eq!(
17289 detail["first_undelivered_cursor"]["daemon_incarnation"],
17290 "lag-incarnation"
17291 );
17292 }
17293}
17294
17295#[cfg(test)]
17296mod terminal_history_read_concurrency_tests {
17297 use super::*;
17298 use crate::terminal_journal::read_pause;
17299 use cortexkit_test_support::ScratchDir;
17300 use std::sync::mpsc as std_mpsc;
17301
17302 fn journaled_ring(
17303 journal: &Arc<crate::terminal_journal::TerminalJournal>,
17304 ) -> Arc<Mutex<TerminalRing>> {
17305 Arc::new(Mutex::new(
17306 TerminalRing::new(TerminalRingConfig::default(), 1)
17307 .with_journal(Some(Arc::clone(journal))),
17308 ))
17309 }
17310
17311 fn crash(at_ms: u64) -> ExitReport {
17312 ExitReport {
17313 kind: ExitKind::Crash,
17314 code: Some(1),
17315 signal: None,
17316 at_ms,
17317 }
17318 }
17319
17320 fn record_within(
17323 module_id: &'static str,
17324 ring: &Arc<Mutex<TerminalRing>>,
17325 at_ms: u64,
17326 bound: Duration,
17327 ) -> bool {
17328 let ring = Arc::clone(ring);
17329 let (done, done_rx) = std_mpsc::channel();
17330 std::thread::spawn(move || {
17331 record_terminal(
17332 module_id,
17333 &ring,
17334 &SpawnEventFeed::default(),
17335 &crash(at_ms),
17336 TerminalDisposition::Restarting,
17337 );
17338 let _ = done.send(());
17339 });
17340 done_rx.recv_timeout(bound).is_ok()
17341 }
17342
17343 #[test]
17348 fn exits_recorded_during_a_paused_history_read_are_not_blocked_or_half_merged() {
17349 let dir = ScratchDir::new("terminal-history-concurrent-read");
17350 let path = dir.join("terminals.jsonl");
17351 let journal = Arc::new(crate::terminal_journal::TerminalJournal::open(
17352 path.clone(),
17353 "daemon".into(),
17354 ));
17355 let reader_ring = journaled_ring(&journal);
17356 let other_ring = journaled_ring(&journal);
17357 assert!(record_within(
17358 "reader-module",
17359 &reader_ring,
17360 10,
17361 Duration::from_secs(5)
17362 ));
17363
17364 let (started, release) = read_pause::install(&path);
17365 let reading = {
17366 let ring = Arc::clone(&reader_ring);
17367 std::thread::spawn(move || durable_terminal_history_of(&ring, "reader-module"))
17368 };
17369 started
17370 .recv_timeout(Duration::from_secs(5))
17371 .expect("the history read reached its pause");
17372
17373 let bound = Duration::from_secs(1);
17374 assert!(
17375 record_within("other-module", &other_ring, 20, bound),
17376 "another module's exit waited on a history read (journal writer held)"
17377 );
17378 assert!(
17379 record_within("reader-module", &reader_ring, 30, bound),
17380 "the read module's own exit waited on its history read (ring held)"
17381 );
17382
17383 drop(release);
17384 let paused = reading.join().unwrap();
17385 assert_eq!(
17386 paused.entries.iter().map(|e| e.at_ms).collect::<Vec<_>>(),
17387 vec![10],
17388 "an exit recorded after the read began lands in neither half of it"
17389 );
17390 assert_eq!(paused.journal_skipped_lines, 0);
17391 assert_eq!(paused.journal_read_errors, 0);
17392
17393 let after = durable_terminal_history_of(&reader_ring, "reader-module");
17394 assert_eq!(
17395 after.entries.iter().map(|e| e.at_ms).collect::<Vec<_>>(),
17396 vec![10, 30],
17397 "the next read merges ring and journal with no duplicate"
17398 );
17399 assert_eq!(after.journal_skipped_lines, 0);
17400 }
17401}
17402
17403#[cfg(test)]
17408mod stderr_settle_tests {
17409 use std::{
17410 future::Future,
17411 io,
17412 pin::Pin,
17413 sync::{Arc, Mutex},
17414 task::{Context, Poll},
17415 time::Duration,
17416 };
17417
17418 use tokio::{
17419 io::{AsyncRead, ReadBuf},
17420 sync::oneshot,
17421 time::Instant,
17422 };
17423
17424 use super::{settle_stderr_pump, settle_stdout_pump, StderrPump};
17425 use crate::stderr_tail::{
17426 pump_stderr_to, pump_stdout_to, split_capture_stamp, untimed, CaptureState,
17427 ChildOutputSink, OutputSink, StderrRing, StderrTailConfig, TailEntry,
17428 };
17429
17430 const BOUND: Duration = Duration::from_millis(250);
17431
17432 struct HeldReader {
17436 before: Option<Vec<u8>>,
17437 gate: Option<oneshot::Receiver<()>>,
17438 after: io::Cursor<Vec<u8>>,
17439 }
17440
17441 impl AsyncRead for HeldReader {
17442 fn poll_read(
17443 mut self: Pin<&mut Self>,
17444 cx: &mut Context<'_>,
17445 buf: &mut ReadBuf<'_>,
17446 ) -> Poll<io::Result<()>> {
17447 if let Some(bytes) = self.before.take() {
17448 buf.put_slice(&bytes);
17449 return Poll::Ready(Ok(()));
17450 }
17451 if let Some(gate) = self.gate.as_mut() {
17452 match Pin::new(gate).poll(cx) {
17453 Poll::Pending => return Poll::Pending,
17454 Poll::Ready(_) => self.gate = None,
17455 }
17456 }
17457 Pin::new(&mut self.after).poll_read(cx, buf)
17458 }
17459 }
17460
17461 struct DiscardSink;
17462
17463 impl OutputSink for DiscardSink {
17464 fn write_line(&mut self, _line: &[u8]) {}
17465 }
17466
17467 fn line(text: &str) -> TailEntry {
17468 TailEntry::Line {
17469 text: text.to_string(),
17470 truncated: false,
17471 at_ms: None,
17472 }
17473 }
17474
17475 fn lock(ring: &Arc<Mutex<StderrRing>>) -> std::sync::MutexGuard<'_, StderrRing> {
17476 ring.lock().unwrap()
17477 }
17478
17479 fn held_pump(
17483 ring: &Arc<Mutex<StderrRing>>,
17484 before: &str,
17485 after: &str,
17486 ) -> (StderrPump, oneshot::Sender<()>) {
17487 let generation = lock(ring).begin_process();
17488 let (release, gate) = oneshot::channel();
17489 let reader = HeldReader {
17490 before: Some(before.as_bytes().to_vec()),
17491 gate: Some(gate),
17492 after: io::Cursor::new(after.as_bytes().to_vec()),
17493 };
17494 let task = tokio::spawn(pump_stderr_to(
17495 reader,
17496 Arc::clone(ring),
17497 generation,
17498 DiscardSink,
17499 ));
17500 (StderrPump { task, generation }, release)
17501 }
17502
17503 async fn wait_until(ring: &Arc<Mutex<StderrRing>>, done: impl Fn(&StderrRing) -> bool) {
17504 for _ in 0..1000 {
17505 if done(&lock(ring)) {
17506 return;
17507 }
17508 tokio::time::sleep(Duration::from_millis(1)).await;
17509 }
17510 panic!(
17511 "ring never reached the expected state: {:?}",
17512 lock(ring).snapshot(None, None)
17513 );
17514 }
17515
17516 #[tokio::test(start_paused = true)]
17517 async fn a_crash_line_the_reader_had_not_reached_by_the_bound_is_kept_before_the_restart() {
17518 let ring = Arc::new(Mutex::new(StderrRing::new(StderrTailConfig::default())));
17519 let (pump, release) = held_pump(&ring, "booting\n", "config error: missing storage\n");
17520
17521 settle_stderr_pump("crasher", &ring, pump, BOUND).await;
17522 let before_release = lock(&ring).snapshot(None, None);
17523 assert!(
17524 matches!(before_release.capture, CaptureState::Incomplete { .. }),
17525 "a reader that has not reached EOF cannot claim a whole tail: {before_release:?}"
17526 );
17527
17528 let next = lock(&ring).begin_process();
17531 lock(&ring).push_line_from(next, "next process booting");
17532 release.send(()).unwrap();
17533 wait_until(&ring, |ring| {
17534 ring.snapshot(None, None).capture == CaptureState::Captured
17535 })
17536 .await;
17537
17538 assert_eq!(
17539 untimed(lock(&ring).snapshot(None, None).entries),
17540 vec![
17541 line("booting"),
17542 line("config error: missing storage"),
17543 TailEntry::ProcessStart,
17544 line("next process booting"),
17545 ],
17546 "the crash's last line must survive a slow reader and stay in the crashed process's section"
17547 );
17548 }
17549
17550 #[tokio::test(start_paused = true)]
17551 async fn a_pipe_held_open_by_a_descendant_reads_incomplete_without_delaying_the_restart_past_the_bound(
17552 ) {
17553 let ring = Arc::new(Mutex::new(StderrRing::new(StderrTailConfig::default())));
17554 let (pump, _held) = held_pump(&ring, "parent exiting\n", "");
17557
17558 let started = Instant::now();
17559 settle_stderr_pump("orphaning", &ring, pump, BOUND).await;
17560 assert_eq!(
17561 started.elapsed(),
17562 BOUND,
17563 "the restart must wait exactly the bound for a pipe that stays open, no longer"
17564 );
17565
17566 let next = lock(&ring).begin_process();
17567 lock(&ring).push_line_from(next, "next process booting");
17568 tokio::time::sleep(Duration::from_secs(60)).await;
17569
17570 let snapshot = lock(&ring).snapshot(None, None);
17571 match &snapshot.capture {
17572 CaptureState::Incomplete { reason } => assert!(
17573 reason.contains("had not reached EOF") && reason.contains("250ms"),
17574 "the reason must say what is missing and after how long: {reason}"
17575 ),
17576 other => panic!("expected Incomplete while the pipe is held open, got {other:?}"),
17577 }
17578 assert_eq!(
17579 untimed(snapshot.entries),
17580 vec![
17581 line("parent exiting"),
17582 TailEntry::ProcessStart,
17583 line("next process booting"),
17584 ]
17585 );
17586 }
17587
17588 #[tokio::test(start_paused = true)]
17589 async fn a_reader_that_reaches_eof_within_the_bound_leaves_the_tail_captured() {
17590 let ring = Arc::new(Mutex::new(StderrRing::new(StderrTailConfig::default())));
17591 let (pump, release) = held_pump(&ring, "one\n", "two\n");
17592 release.send(()).unwrap();
17593
17594 settle_stderr_pump("clean", &ring, pump, BOUND).await;
17595
17596 let snapshot = lock(&ring).snapshot(None, None);
17597 assert_eq!(snapshot.capture, CaptureState::Captured);
17598 assert_eq!(untimed(snapshot.entries), vec![line("one"), line("two")]);
17599 }
17600
17601 #[tokio::test(start_paused = true)]
17602 async fn stdout_buffered_past_the_exit_drain_bound_reaches_the_capture_file() {
17603 const BURST: usize = 150;
17604 let capture = cortexkit_test_support::ScratchDir::new("stdout-exit-drain");
17605 let path = capture.join("two-pipe.stderr.log");
17606 let sink = ChildOutputSink::open(&path, cortexkit_log::Retention::default()).unwrap();
17607 let ring = Arc::new(Mutex::new(StderrRing::new(StderrTailConfig::default())));
17608 let generation = lock(&ring).begin_process();
17609 let stdout = (0..BURST)
17610 .map(|i| format!("out-{i:04}-{}\n", "o".repeat(64)))
17611 .collect::<String>();
17612 let stderr = (0..BURST)
17613 .map(|i| format!("err-{i:04}-{}\n", "e".repeat(64)))
17614 .collect::<String>();
17615 let first_chunk = stdout.find('\n').unwrap() + 1 + "out-0001-".len();
17620 let (release, gate) = oneshot::channel();
17621 let reader = HeldReader {
17622 before: Some(stdout.as_bytes()[..first_chunk].to_vec()),
17623 gate: Some(gate),
17624 after: io::Cursor::new(stdout.as_bytes()[first_chunk..stdout.len() - 1].to_vec()),
17625 };
17626 let (finished, completion) = oneshot::channel();
17627 let stdout_sink = sink.clone();
17628 let pump = tokio::spawn(async move {
17629 pump_stdout_to(reader, stdout_sink).await;
17630 finished.send(()).unwrap();
17631 });
17632 pump_stderr_to(io::Cursor::new(stderr.into_bytes()), ring, generation, sink).await;
17633
17634 let started = Instant::now();
17635 settle_stdout_pump("two-pipe", pump, BOUND).await;
17636 assert_eq!(
17637 started.elapsed(),
17638 BOUND,
17639 "a late reader must not hold up restart"
17640 );
17641 assert_eq!(
17642 std::fs::read_to_string(&path).unwrap().lines().count(),
17643 BURST + 1
17644 );
17645
17646 release
17647 .send(())
17648 .expect("stdout reader must survive the exit drain bound");
17649 tokio::time::timeout(Duration::from_secs(1), completion)
17650 .await
17651 .expect("released stdout reader must reach EOF")
17652 .expect("stdout capture must finish at EOF, not be cancelled");
17653 let contents = std::fs::read_to_string(&path).unwrap();
17654 let mut lines = contents
17655 .lines()
17656 .map(|line| split_capture_stamp(line).expect("capture stamp").1)
17657 .collect::<Vec<_>>();
17658 let mut expected = (0..BURST)
17659 .flat_map(|i| {
17660 [
17661 format!("out-{i:04}-{}", "o".repeat(64)),
17662 format!("err-{i:04}-{}", "e".repeat(64)),
17663 ]
17664 })
17665 .collect::<Vec<_>>();
17666 lines.sort_unstable();
17667 expected.sort_unstable();
17668 assert_eq!(
17669 lines, expected,
17670 "both pipes must retain every complete line exactly once"
17671 );
17672 }
17673}
17674
17675#[cfg(all(test, windows))]
17689mod job_containment_tests {
17690 use super::*;
17691 use cortexkit_test_support::ScratchDir;
17692 use std::{
17693 path::{Path, PathBuf},
17694 sync::{Arc, Mutex},
17695 time::{Duration, Instant},
17696 };
17697
17698 fn stub_path() -> PathBuf {
17704 let mut path = std::env::current_exe().expect("current_exe available in tests");
17705 path.pop();
17706 path.pop();
17707 path.push("fake-aft-stub.exe");
17708 assert!(
17709 path.exists(),
17710 "fake-aft-stub not built at {}: run `cargo test -p subc-core` (which builds \
17711 [[bin]] targets) rather than `cargo test -p subc-core --lib` (which does not)",
17712 path.display()
17713 );
17714 path
17715 }
17716
17717 fn read_grandchild_pid(path: &Path) -> u32 {
17719 let deadline = Instant::now() + Duration::from_secs(10);
17720 loop {
17721 if let Ok(contents) = std::fs::read_to_string(path) {
17722 if let Ok(pid) = contents.trim().parse() {
17723 return pid;
17724 }
17725 }
17726 assert!(
17727 Instant::now() < deadline,
17728 "the stub never recorded a grandchild pid at {}",
17729 path.display()
17730 );
17731 std::thread::sleep(Duration::from_millis(10));
17732 }
17733 }
17734
17735 struct Fixture {
17738 _dir: ScratchDir,
17739 module_id: String,
17740 grandchild: u32,
17741 child: Option<SupervisedChild>,
17742 registry: Arc<Registry>,
17743 snapshot: Arc<Mutex<SupervisorSnapshot>>,
17744 terminal_ring: Arc<Mutex<TerminalRing>>,
17745 spawn_events: SpawnEventFeed,
17746 }
17747
17748 fn fixture(label: &str, module_id: &str) -> Fixture {
17749 let dir = ScratchDir::new(label);
17750 let pid_file = dir.join("grandchild.pid");
17751 let supervisor = Supervisor::new_for_test(
17752 Arc::new(Registry::default()),
17753 RestartPolicy::new(3, Duration::ZERO),
17754 );
17755 let runtime = supervisor.runtime_config();
17756 let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::starting()));
17757 let spec = ModuleSpec {
17758 module_id: module_id.to_string(),
17759 program: stub_path(),
17760 args: Vec::new(),
17764 env: vec![
17765 ("FAKE_AFT_NEVER_CONNECT".to_string(), "1".to_string()),
17766 (
17767 "FAKE_AFT_GRANDCHILD_PID_FILE".to_string(),
17768 pid_file.display().to_string(),
17769 ),
17770 ],
17771 reserved: false,
17772 reserved_prefixes: Vec::new(),
17773 protocol: ModuleProtocol::Subc,
17774 overlap: Default::default(),
17775 carry_containment: false,
17776 };
17777 let child = spawn_and_mark_running(&spec, &runtime, &snapshot)
17778 .expect("spawn the supervised fixture");
17779 let grandchild = read_grandchild_pid(&pid_file);
17780 Fixture {
17781 _dir: dir,
17782 module_id: module_id.to_string(),
17783 grandchild,
17784 child: Some(child),
17785 registry: Arc::new(Registry::default()),
17786 snapshot,
17787 terminal_ring: Arc::clone(&runtime.terminal_ring),
17788 spawn_events: SpawnEventFeed::default(),
17789 }
17790 }
17791
17792 impl Fixture {
17793 async fn drain(&mut self) {
17795 let child = self
17796 .child
17797 .take()
17798 .expect("the fixture child is still present");
17799 drain_child_to_state(
17800 &self.module_id,
17801 ModuleProtocol::Subc,
17802 StopNotice::NotSent,
17805 &self.registry,
17806 None,
17807 &self.snapshot,
17808 &self.terminal_ring,
17809 &self.spawn_events,
17810 child,
17811 Duration::from_millis(500),
17812 ModuleState::Stopped,
17813 Some(false),
17814 &FailureContext::operator(),
17815 )
17816 .await
17817 .expect("drain the supervised fixture");
17818 }
17819 }
17820
17821 #[tokio::test]
17827 async fn teardown_reaps_the_grandchild() {
17828 let mut fixture = fixture("teardown-grandchild", "tree-teardown");
17829 let grandchild = fixture.grandchild;
17830
17831 assert!(
17832 subc_jobobject::process_exists(grandchild),
17833 "grandchild {grandchild} must be alive before teardown, or this proves nothing"
17834 );
17835
17836 fixture.drain().await;
17837
17838 assert!(
17839 subc_jobobject::wait_for_process_exit(grandchild, Duration::from_secs(10)),
17840 "grandchild {grandchild} outlived module teardown: the tree was not contained"
17841 );
17842 }
17843
17844 #[test]
17857 fn an_uncontained_grandchild_survives_a_direct_child_kill() {
17858 let dir = ScratchDir::new("teardown-uncontained");
17859 let pid_file = dir.join("grandchild.pid");
17860 let mut child = std::process::Command::new(stub_path())
17861 .env("FAKE_AFT_NEVER_CONNECT", "1")
17862 .env(
17863 "FAKE_AFT_GRANDCHILD_PID_FILE",
17864 pid_file.display().to_string(),
17865 )
17866 .stdin(std::process::Stdio::null())
17867 .stdout(std::process::Stdio::null())
17868 .stderr(std::process::Stdio::null())
17869 .spawn()
17870 .expect("spawn the uncontained fixture");
17871 let grandchild = read_grandchild_pid(&pid_file);
17872
17873 child.kill().expect("kill the direct child");
17875 let _ = child.wait();
17876
17877 assert!(
17878 subc_jobobject::process_exists(grandchild),
17879 "grandchild {grandchild} died with the direct child, so this control no longer \
17880 distinguishes contained from uncontained teardown and the regression test is \
17881 passing vacuously"
17882 );
17883
17884 kill_tree(grandchild);
17887 }
17888
17889 #[tokio::test]
17898 async fn dropping_containment_reaps_the_grandchild() {
17899 let mut fixture = fixture("drop-containment", "tree-drop");
17900 let grandchild = fixture.grandchild;
17901
17902 assert!(subc_jobobject::process_exists(grandchild));
17903
17904 fixture.child.as_mut().expect("child present").job = None;
17906
17907 assert!(
17908 subc_jobobject::wait_for_process_exit(grandchild, Duration::from_secs(10)),
17909 "grandchild {grandchild} survived the containment handle closing, so a daemon \
17910 crash would leave the tree behind"
17911 );
17912 }
17913
17914 fn kill_tree(pid: u32) {
17916 let _ = std::process::Command::new("taskkill.exe")
17917 .args(["/PID", &pid.to_string(), "/T", "/F"])
17918 .stdin(std::process::Stdio::null())
17919 .stdout(std::process::Stdio::null())
17920 .stderr(std::process::Stdio::null())
17921 .status();
17922 assert!(
17923 subc_jobobject::wait_for_process_exit(pid, Duration::from_secs(10)),
17924 "could not clean up grandchild {pid}"
17925 );
17926 }
17927}
17928
17929#[cfg(test)]
17930mod privacy_trampoline_configuration_tests {
17931 #[tokio::test]
17932 #[cfg_attr(not(target_os = "macos"), ignore = "requires macOS privacy trampoline")]
17933 async fn macos_spawn_without_a_configured_trampoline_refuses_by_name() {
17934 #[cfg(target_os = "macos")]
17935 {
17936 let supervisor = super::Supervisor::new(
17937 std::sync::Arc::new(crate::Registry::default()),
17938 super::RestartPolicy::default(),
17939 );
17940 let error = supervisor.spawn(spec()).unwrap_err();
17941 assert!(
17942 error
17943 .to_string()
17944 .contains("no privacy trampoline configured"),
17945 "{error}"
17946 );
17947 }
17948 }
17949
17950 #[tokio::test]
17951 #[cfg_attr(not(target_os = "macos"), ignore = "requires macOS privacy trampoline")]
17952 async fn macos_wrong_trampoline_is_rejected_by_the_startup_probe() {
17953 #[cfg(target_os = "macos")]
17954 {
17955 let supervisor = super::Supervisor::new(
17956 std::sync::Arc::new(crate::Registry::default()),
17957 super::RestartPolicy::default(),
17958 )
17959 .with_privacy_trampoline(std::env::current_exe().unwrap());
17960 let error = supervisor.spawn(spec()).unwrap_err();
17961 assert!(
17962 error
17963 .to_string()
17964 .contains("binary does not implement the privacy trampoline protocol"),
17965 "{error}"
17966 );
17967 }
17968 }
17969
17970 #[cfg(target_os = "macos")]
17971 fn spec() -> super::ModuleSpec {
17972 super::ModuleSpec {
17973 module_id: "privacy-configuration".into(),
17974 program: "/bin/sleep".into(),
17975 args: vec!["30".into()],
17976 env: vec![],
17977 reserved: false,
17978 reserved_prefixes: vec![],
17979 protocol: subc_control::ModuleProtocol::None,
17980 overlap: super::ModuleOverlap::Exclusive,
17981 carry_containment: false,
17982 }
17983 }
17984}
17985
17986#[cfg(test)]
17987mod privacy_exec_boundary_tests {
17988 #[cfg(target_os = "macos")]
17989 use super::*;
17990 #[cfg(target_os = "macos")]
17991 use std::{
17992 io::{Read, Write},
17993 net::{TcpListener, TcpStream},
17994 };
17995
17996 #[cfg(target_os = "macos")]
17999 pub(super) fn before_image_sample(spec: &ModuleSpec, pid: u32) {
18000 if let Some((_, path)) = spec
18001 .env
18002 .iter()
18003 .find(|(key, _)| key == "SUBC_TEST_PRIVACY_SAMPLE_BARRIER")
18004 {
18005 let mut barrier = TcpStream::connect(path).unwrap();
18006 barrier
18007 .set_read_timeout(Some(Duration::from_secs(30)))
18008 .unwrap();
18009 barrier.write_all(&pid.to_ne_bytes()).unwrap();
18010 let mut release = [0];
18011 barrier.read_exact(&mut release).unwrap();
18012 assert_eq!(&release, b"X");
18013 }
18014 }
18015
18016 #[cfg(target_os = "macos")]
18017 fn spec(program: &str, args: &[&str]) -> ModuleSpec {
18018 ModuleSpec {
18019 module_id: "privacy-boundary".into(),
18020 program: program.into(),
18021 args: args.iter().map(|arg| (*arg).into()).collect(),
18022 env: vec![],
18023 reserved: false,
18024 reserved_prefixes: vec![],
18025 protocol: ModuleProtocol::None,
18026 overlap: ModuleOverlap::Exclusive,
18027 carry_containment: false,
18028 }
18029 }
18030
18031 #[cfg(target_os = "macos")]
18032 async fn accept(listener: TcpListener) -> TcpStream {
18033 let listener = tokio::net::TcpListener::from_std({
18035 listener.set_nonblocking(true).unwrap();
18036 listener
18037 })
18038 .unwrap();
18039 let (stream, _) = tokio::time::timeout(Duration::from_secs(30), listener.accept())
18040 .await
18041 .unwrap()
18042 .unwrap();
18043 let stream = stream.into_std().unwrap();
18044 stream.set_nonblocking(false).unwrap();
18045 stream
18046 .set_read_timeout(Some(Duration::from_secs(30)))
18047 .unwrap();
18048 stream
18049 }
18050
18051 #[tokio::test]
18052 #[cfg_attr(not(target_os = "macos"), ignore = "requires macOS privacy trampoline")]
18053 async fn macos_roster_withholds_a_nonnull_trampoline_image_until_exec_confirmation() {
18054 #[cfg(target_os = "macos")]
18055 {
18056 let root = cortexkit_test_support::ScratchDir::new("privacy-roster-barrier");
18057 let exec_listener = TcpListener::bind("127.0.0.1:0").unwrap();
18060 let sample_listener = TcpListener::bind("127.0.0.1:0").unwrap();
18061 let record = root.join("live-children.json");
18062 let supervisor =
18063 Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default())
18064 .with_live_children_record(&record);
18065 let runtime = supervisor.runtime_config();
18066 let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::starting()));
18067 let mut spec = spec("/bin/sleep", &["30"]);
18068 spec.env = vec![
18069 (
18070 "SUBC_TEST_PRIVACY_EXEC_BARRIER".into(),
18071 exec_listener.local_addr().unwrap().to_string(),
18072 ),
18073 (
18074 "SUBC_TEST_PRIVACY_SAMPLE_BARRIER".into(),
18075 sample_listener.local_addr().unwrap().to_string(),
18076 ),
18077 ];
18078 let spawn = tokio::task::spawn_blocking(move || {
18079 spawn_and_mark_running(&spec, &runtime, &snapshot).unwrap()
18080 });
18081 let mut sample = accept(sample_listener).await;
18082 let mut pid = [0; 4];
18083 sample.read_exact(&mut pid).unwrap();
18084 let pid = u32::from_ne_bytes(pid);
18085 let mut exec = accept(exec_listener).await;
18086 let mut ready = [0];
18087 exec.read_exact(&mut ready).unwrap();
18088 assert_eq!(&ready, b"R");
18089 let trampoline = subc_os::file_identity(&test_privacy_trampoline()).unwrap();
18092 assert_eq!(
18093 observe_spawned_image(pid).unwrap().executable,
18094 Some(trampoline)
18095 );
18096 sample.write_all(b"X").unwrap();
18097 let mut child = spawn.await.unwrap();
18098 let early = crate::live_children::read_record(&record).unwrap();
18099 assert_eq!(early.len(), 1);
18100 assert_eq!(early[0].pid, pid);
18101 assert_eq!(
18102 early[0].executable, None,
18103 "unconfirmed trampoline image entered the roster"
18104 );
18105 assert!(child.report_ready.get().is_none());
18106 child.privacy_exec.as_mut().unwrap().deadline =
18109 tokio::time::Instant::now() + Duration::from_secs(30);
18110 exec.write_all(b"X").unwrap();
18111 child.confirm_privacy_exec().await;
18112 assert_eq!(child.spawn_failure, None);
18113 assert!(child.report_ready.get().is_some());
18114 let confirmed = crate::live_children::read_record(&record).unwrap();
18115 let module = subc_os::file_identity(std::path::Path::new("/bin/sleep")).unwrap();
18116 assert_ne!(module, trampoline);
18117 assert_eq!(confirmed[0].executable, Some(module.into()));
18118 child.start_kill().unwrap();
18119 child.wait().await.unwrap();
18120 child.release_roster();
18121 }
18122 }
18123
18124 #[tokio::test]
18125 #[cfg_attr(not(target_os = "macos"), ignore = "requires macOS privacy trampoline")]
18126 async fn macos_already_exited_121_in_try_wait_is_a_module_exit_not_a_trampoline_refusal() {
18127 #[cfg(target_os = "macos")]
18128 {
18129 let registry = Arc::new(Registry::default());
18130 let policy = RestartPolicy::new(0, Duration::ZERO);
18131 let supervisor = Supervisor::new_for_test(Arc::clone(®istry), policy);
18132 let runtime = supervisor.runtime_config();
18133 let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::starting()));
18134 let spec = spec("/bin/sh", &["-c", "exit 121"]);
18135 let mut child = spawn_and_mark_running(&spec, &runtime, &snapshot).unwrap();
18139 let pid = child.pid;
18140 tokio::task::spawn_blocking(move || {
18141 subc_os::fork_exec_test::wait_for_child_exit_without_reaping(pid)
18142 })
18143 .await
18144 .unwrap()
18145 .unwrap();
18146 child.privacy_exec.as_mut().unwrap().deadline =
18147 tokio::time::Instant::now() + Duration::from_secs(30);
18148 let status = child.wait().await.unwrap();
18149 assert_eq!(status.code(), Some(121));
18150 assert!(child.privacy_exec.is_none());
18151 assert!(
18152 child.report_ready.get().is_none(),
18153 "an exited module must not publish a live pid"
18154 );
18155 let report = classify_reaped_child_exit(&snapshot, &child, &status);
18156 on_child_exit(
18157 &spec,
18158 policy,
18159 ®istry,
18160 &snapshot,
18161 &runtime.terminal_ring,
18162 &runtime.spawn_events,
18163 &runtime.child_roster,
18164 report,
18165 true,
18166 )
18167 .await;
18168 let state = lock_snapshot(&snapshot).unwrap();
18169 assert_eq!(state.state, ModuleState::Failed);
18170 assert_eq!(state.last_exit.as_ref().unwrap().code, Some(121));
18171 assert_eq!(state.reported_pid(), None);
18172 drop(state);
18173 let history = runtime.terminal_ring.lock().unwrap().snapshot();
18174 assert_eq!(history.entries.len(), 1);
18175 let terminal = &history.entries[0];
18176 assert_eq!(terminal.exit_code, Some(121));
18177 assert_eq!(terminal.exit_kind, subc_control::TerminalExitKind::Crash);
18178 assert_eq!(terminal.disposition, TerminalDisposition::Failed);
18179 assert_eq!(
18180 terminal.disposition_detail.as_deref(),
18181 Some(policy.budget_exhausted_detail().as_str()),
18182 "module exit 121 was classified as a trampoline refusal: {terminal:?}"
18183 );
18184 assert_eq!(child.spawn_failure, None);
18185 child.release_roster();
18186 }
18187 }
18188}
18189
18190#[cfg(all(test, unix))]
18194mod launch_nonce_descriptor_tests {
18195 use super::{ChildRoster, ModuleProtocol, ModuleSpec, SupervisorHandle};
18196 use crate::stderr_tail::{StderrRing, StderrTailConfig};
18197 use cortexkit_test_support::ScratchDir;
18198 use std::{
18199 path::PathBuf,
18200 sync::{Arc, Mutex},
18201 time::{Duration, Instant},
18202 };
18203
18204 async fn probe(role: super::SpawnRole) {
18205 let scratch = ScratchDir::new("launch-nonce-descriptor");
18206 let fd_copy = scratch.join("from-descriptor");
18207 let env_copy = scratch.join("environment");
18208 let script = format!(
18209 "env > '{env}.tmp' && mv '{env}.tmp' '{env}'; cat <&3 > '{fd}.tmp' && mv '{fd}.tmp' '{fd}'; sleep 30",
18210 fd = fd_copy.display(), env = env_copy.display(),
18211 );
18212 let xdg = |name: &str| (name.to_string(), scratch.join(name).display().to_string());
18213 let spec = ModuleSpec {
18214 module_id: "nonce-descriptor-probe".to_string(),
18215 program: PathBuf::from("/bin/sh"),
18216 args: vec!["-c".to_string(), script],
18217 env: vec![
18218 xdg("XDG_DATA_HOME"),
18219 xdg("XDG_RUNTIME_DIR"),
18220 xdg("XDG_CONFIG_HOME"),
18221 ("SUBC_LAUNCH_NONCE".to_string(), "must-not-leak".to_string()),
18222 ],
18223 reserved: true,
18224 reserved_prefixes: Vec::new(),
18225 protocol: ModuleProtocol::Subc,
18226 overlap: Default::default(),
18227 carry_containment: false,
18228 };
18229 let handle = SupervisorHandle::new();
18230 let ring = Arc::new(Mutex::new(StderrRing::new(StderrTailConfig::default())));
18231 let roster = ChildRoster::default();
18232 #[cfg(target_os = "macos")]
18233 {
18234 let path = super::test_privacy_trampoline();
18235 roster.set_privacy_trampoline(super::probe_privacy_trampoline(&path).map(|()| path));
18236 }
18237 let child = super::spawn_child_in_slot(
18238 &spec,
18239 None,
18240 Some(&handle),
18241 &ring,
18242 None,
18243 &roster,
18244 #[cfg(target_os = "linux")]
18245 None,
18246 role,
18247 matches!(role, super::SpawnRole::SwapCandidate),
18248 )
18249 .expect("spawn probe");
18250 let deadline = Instant::now() + Duration::from_secs(10);
18251 while !(fd_copy.exists() && env_copy.exists()) {
18252 assert!(Instant::now() < deadline, "probe never wrote its copies");
18253 tokio::time::sleep(Duration::from_millis(20)).await;
18254 }
18255 let nonce = std::fs::read_to_string(fd_copy).unwrap();
18256 assert!(!nonce.is_empty());
18257 let environment = std::fs::read_to_string(env_copy).unwrap();
18258 assert!(environment
18259 .lines()
18260 .any(|line| line.starts_with("SUBC_LAUNCH_NONCE_FD=3:")));
18261 let copy = environment
18262 .lines()
18263 .find_map(|line| line.strip_prefix("SUBC_LAUNCH_NONCE="));
18264 assert_eq!(
18265 copy, None,
18266 "Unix children must never receive the environment nonce"
18267 );
18268 if matches!(role, super::SpawnRole::Plain) {
18269 assert_eq!(
18270 handle.spawn_nonce(&spec.module_id).as_deref(),
18271 Some(nonce.as_str())
18272 );
18273 }
18274 drop(child);
18275 }
18276
18277 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
18278 async fn a_spawned_module_receives_its_nonce_only_on_descriptor_3() {
18279 probe(super::SpawnRole::Plain).await;
18280 }
18281
18282 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
18283 async fn swap_candidate_receives_its_nonce_only_on_descriptor_3() {
18284 probe(super::SpawnRole::SwapCandidate).await;
18285 }
18286}
18287
18288#[cfg(all(test, target_os = "linux"))]
18289#[path = "supervise/cgroup_reap_tests.rs"]
18290mod cgroup_reap_tests;
18291
18292#[cfg(all(test, target_os = "linux"))]
18293mod cgroup_containment_tests {
18294 use super::*;
18295 use cortexkit_test_support::ScratchDir;
18296
18297 #[tokio::test]
18298 async fn linux_subtree_memory_covers_module_children() {
18299 let dir = ScratchDir::new("subtree-memory");
18300 let root = PathBuf::from(format!(
18301 "/sys/fs/cgroup/subc-memory-test-{}-{}",
18302 std::process::id(),
18303 unix_ms_now()
18304 ));
18305 if let Err(error) = std::fs::create_dir(&root) {
18306 assert!(
18307 std::env::var_os("SUBC_REQUIRE_CGROUP_TEST").is_none(),
18308 "required cgroup test cannot execute: {error}"
18309 );
18310 eprintln!("SKIP linux_subtree_memory_covers_module_children: {error}");
18311 return;
18312 }
18313 let placement = subc_cgroup::prepare_at(&root)
18314 .unwrap()
18315 .expect("delegated cgroup");
18316 let supervisor = Supervisor::new_for_test(
18317 Arc::new(Registry::default()),
18318 RestartPolicy::new(3, Duration::ZERO),
18319 )
18320 .with_cgroup_placement(Some(placement));
18321 let ready = dir.join("ready");
18322 let module = supervisor
18323 .spawn(ModuleSpec {
18324 module_id: "memory-tree".into(),
18325 program: PathBuf::from("/bin/sh"),
18326 args: vec![
18327 "-c".into(),
18328 "sleep 600 & sleep 600 & sleep 600 & echo ready > \"$1\"; wait".into(),
18329 "fixture".into(),
18330 ready.display().to_string(),
18331 ],
18332 env: ["XDG_DATA_HOME", "XDG_RUNTIME_DIR", "XDG_CONFIG_HOME"]
18333 .into_iter()
18334 .map(|key| (key.into(), dir.display().to_string()))
18335 .collect(),
18336 reserved: false,
18337 reserved_prefixes: Vec::new(),
18338 protocol: ModuleProtocol::None,
18339 overlap: Default::default(),
18340 carry_containment: false,
18341 })
18342 .unwrap();
18343 let deadline = tokio::time::Instant::now() + Duration::from_secs(10);
18344 while !ready.exists() {
18345 assert!(
18346 tokio::time::Instant::now() < deadline,
18347 "children never started"
18348 );
18349 tokio::time::sleep(Duration::from_millis(10)).await;
18350 }
18351 let readings = crate::subtree_resources::read(vec![module.subtree_target()]).await;
18352 let reading = &readings[0];
18353 assert!(
18354 reading.process_count.is_some_and(|count| count >= 4),
18355 "{reading:?}"
18356 );
18357 assert!(
18358 reading.memory_bytes.is_some_and(|bytes| bytes > 0),
18359 "{reading:?}"
18360 );
18361 module.stop().await.unwrap();
18362 drop(module);
18363 std::fs::remove_dir(root.join("subc-modules")).unwrap();
18364 std::fs::remove_dir(&root).unwrap();
18365 eprintln!("EXECUTED linux_subtree_memory_covers_module_children: {reading:?}");
18366 }
18367
18368 fn running(pid: u32) -> bool {
18369 std::fs::read_to_string(format!("/proc/{pid}/stat"))
18371 .ok()
18372 .and_then(|stat| {
18373 stat.rsplit_once(") ")
18374 .map(|(_, rest)| rest.starts_with('Z'))
18375 })
18376 .is_some_and(|zombie| !zombie)
18377 }
18378
18379 #[tokio::test]
18380 async fn linux_teardown_reaps_the_grandchild() {
18381 teardown_tree("linux_teardown_reaps_the_grandchild", false).await;
18382 }
18383
18384 #[tokio::test]
18385 async fn linux_shutdown_straggler_reaps_the_grandchild() {
18386 teardown_tree("linux_shutdown_straggler_reaps_the_grandchild", true).await;
18387 }
18388
18389 async fn teardown_tree(test_name: &str, shutdown: bool) {
18390 let dir = ScratchDir::new(test_name);
18391 let root = PathBuf::from(format!(
18392 "/sys/fs/cgroup/subc-tree-test-{test_name}-{}-{}",
18393 std::process::id(),
18394 unix_ms_now()
18395 ));
18396 if let Err(error) = std::fs::create_dir(&root) {
18397 assert!(
18398 std::env::var_os("SUBC_REQUIRE_CGROUP_TEST").is_none(),
18399 "required cgroup test cannot execute: {error}"
18400 );
18401 eprintln!(
18402 "SKIP {test_name}: no writable delegated cgroup at {}: {error}",
18403 root.display()
18404 );
18405 return;
18406 }
18407 let placement = subc_cgroup::prepare_at(&root)
18408 .expect("prepare isolated kernel cgroup")
18409 .expect("isolated cgroup is delegated");
18410 let module_id = "tree-teardown";
18411 let module = placement
18412 .module_path(module_id)
18413 .expect("create isolated module cgroup");
18414 if !module.join("cgroup.kill").exists() {
18415 std::fs::remove_dir(&module).unwrap();
18416 std::fs::remove_dir(root.join("subc-modules")).unwrap();
18417 std::fs::remove_dir(&root).unwrap();
18418 assert!(
18419 std::env::var_os("SUBC_REQUIRE_CGROUP_TEST").is_none(),
18420 "required cgroup.kill interface unavailable"
18421 );
18422 eprintln!("SKIP {test_name}: cgroup.kill unavailable (kernel < 5.14)");
18423 return;
18424 }
18425 let supervisor = Supervisor::new_for_test(
18426 Arc::new(Registry::default()),
18427 RestartPolicy::new(3, Duration::ZERO),
18428 )
18429 .with_cgroup_placement(Some(placement));
18430 let mut runtime = supervisor.runtime_config();
18431 runtime.child_roster = runtime
18432 .child_roster
18433 .for_module(Arc::new(Mutex::new(Duration::from_millis(100))));
18434 let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::starting()));
18435 let pid_file = dir.join("grandchild.pid");
18436 let spec = ModuleSpec {
18437 module_id: module_id.to_string(),
18438 program: PathBuf::from("/bin/sh"),
18439 args: vec![
18440 "-c".into(),
18441 "trap '' TERM; sleep 600 & echo $! > \"$1\"; wait".into(),
18442 "fixture".into(),
18443 pid_file.display().to_string(),
18444 ],
18445 env: ["XDG_DATA_HOME", "XDG_RUNTIME_DIR", "XDG_CONFIG_HOME"]
18446 .into_iter()
18447 .map(|key| (key.to_string(), dir.display().to_string()))
18448 .collect(),
18449 reserved: false,
18450 reserved_prefixes: Vec::new(),
18451 protocol: ModuleProtocol::None,
18452 overlap: Default::default(),
18453 carry_containment: false,
18454 };
18455 let child =
18456 spawn_and_mark_running(&spec, &runtime, &snapshot).expect("spawn supervised tree");
18457 let deadline = tokio::time::Instant::now() + Duration::from_secs(10);
18458 let grandchild: u32 = loop {
18459 if let Ok(contents) = std::fs::read_to_string(&pid_file) {
18460 if let Ok(pid) = contents.trim().parse() {
18461 break pid;
18462 }
18463 }
18464 assert!(
18465 tokio::time::Instant::now() < deadline,
18466 "grandchild pid was not recorded"
18467 );
18468 tokio::time::sleep(Duration::from_millis(10)).await;
18469 };
18470 assert!(
18471 running(grandchild),
18472 "grandchild must be alive before teardown"
18473 );
18474 if shutdown {
18475 let mut child = child;
18476 crate::child_roster::end_children_for_daemon_shutdown(
18477 &runtime.child_roster,
18478 false,
18479 std::future::pending(),
18480 )
18481 .await;
18482 child.wait().await.expect("reap shutdown straggler");
18483 } else {
18484 drain_child_to_state(
18485 module_id,
18486 ModuleProtocol::None,
18487 StopNotice::NotSent,
18488 &Registry::default(),
18489 None,
18490 &snapshot,
18491 &runtime.terminal_ring,
18492 &SpawnEventFeed::default(),
18493 child,
18494 Duration::from_millis(100),
18495 ModuleState::Stopped,
18496 Some(false),
18497 &FailureContext::operator(),
18498 )
18499 .await
18500 .expect("real supervisor teardown");
18501 }
18502 let deadline = tokio::time::Instant::now() + Duration::from_secs(5);
18503 while running(grandchild) && tokio::time::Instant::now() < deadline {
18504 tokio::time::sleep(Duration::from_millis(10)).await;
18505 }
18506 let survived = running(grandchild);
18507 if survived {
18509 let pid = rustix::process::Pid::from_raw(grandchild as i32).unwrap();
18510 let _ = rustix::process::kill_process(pid, rustix::process::Signal::KILL);
18511 tokio::time::sleep(Duration::from_millis(100)).await;
18512 }
18513 if module.exists() {
18514 std::fs::remove_dir(&module).expect("remove empty module cgroup");
18515 }
18516 std::fs::remove_dir(root.join("subc-modules")).unwrap();
18517 std::fs::remove_dir(&root).unwrap();
18518 assert!(
18519 !survived,
18520 "grandchild {grandchild} outlived module teardown"
18521 );
18522 eprintln!("EXECUTED {test_name}: grandchild {grandchild} killed in isolated cgroup");
18523 }
18524}
18525
18526#[cfg(all(test, unix))]
18529mod record_write_stall_tests {
18530 use super::*;
18531 use std::sync::atomic::{AtomicBool, Ordering};
18532 use subc_transport::{authenticate_client, ConnectionInfo, Endpoint, SCHEMA_VERSION};
18533
18534 const MODULE_AUTH_DEADLINE: Duration = Duration::from_secs(2);
18537 const HELD_RECORD_WRITE: Duration = Duration::from_secs(3);
18541
18542 fn sleeping_spec() -> ModuleSpec {
18554 ModuleSpec {
18555 module_id: "record-write-stall".into(),
18556 program: "/bin/sleep".into(),
18557 args: vec!["600".into()],
18558 env: vec![],
18559 reserved: false,
18560 reserved_prefixes: vec![],
18561 protocol: ModuleProtocol::None,
18562 overlap: Default::default(),
18563 carry_containment: false,
18564 }
18565 }
18566
18567 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
18568 async fn module_auth_completes_while_the_restart_waits_on_the_record_write() {
18569 let home = cortexkit_test_support::ScratchDir::new("record-write-stall");
18570 let supervisor =
18571 Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default())
18572 .with_live_children_record(home.join("live-children.json"));
18573 let module = supervisor.spawn(sleeping_spec()).unwrap();
18574 let old_pid = tokio::time::timeout(Duration::from_secs(10), async {
18579 loop {
18580 if let Some(pid) = lock_snapshot(&module.inner.snapshot)
18581 .unwrap()
18582 .reported_pid()
18583 {
18584 return pid;
18585 }
18586 tokio::time::sleep(Duration::from_millis(10)).await;
18587 }
18588 })
18589 .await
18590 .expect("the first process was admitted as the module");
18591
18592 let key = vec![0x42; 32];
18595 let daemon_id = [0x24; 16];
18596 let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
18597 let port = listener.local_addr().unwrap().port();
18598 let server = tokio::spawn(crate::server::serve_listener(
18599 listener,
18600 Arc::new(crate::router::Router::with_default_self_handler()),
18601 crate::server::ServerAuth::with_limits(
18602 key.clone(),
18603 daemon_id,
18604 "test",
18605 crate::server::DEFAULT_AUTH_DEADLINE,
18606 4,
18607 ),
18608 ));
18609
18610 let (held_tx, held_rx) = std::sync::mpsc::channel();
18613 let armed = AtomicBool::new(true);
18614 supervisor
18615 .child_roster
18616 .set_record_write_hook_for_test(move |live| {
18617 let admitting_replacement = live.values().any(|entry| entry.pid != old_pid);
18618 if admitting_replacement && armed.swap(false, Ordering::SeqCst) {
18619 let _ = held_tx.send(());
18620 std::thread::sleep(HELD_RECORD_WRITE);
18621 }
18622 });
18623
18624 let module_process = std::thread::spawn(move || {
18627 held_rx
18628 .recv_timeout(Duration::from_secs(30))
18629 .expect("the restart reached the replacement's record write");
18630 let conn = ConnectionInfo {
18631 schema: SCHEMA_VERSION,
18632 wire_version: None,
18633 endpoints: vec![Endpoint {
18634 host: "127.0.0.1".to_owned(),
18635 port,
18636 }],
18637 key,
18638 daemon_id,
18639 pid: std::process::id(),
18640 daemon_ver: "test".to_owned(),
18641 };
18642 tokio::runtime::Builder::new_current_thread()
18643 .enable_all()
18644 .build()
18645 .unwrap()
18646 .block_on(async {
18647 let started = Instant::now();
18648 let mut stream = tokio::net::TcpStream::connect(("127.0.0.1", port))
18649 .await
18650 .unwrap();
18651 let result =
18652 authenticate_client(&mut stream, &conn, MODULE_AUTH_DEADLINE).await;
18653 (result, started.elapsed())
18654 })
18655 });
18656
18657 module.restart(Some(0)).await.unwrap();
18658 let (auth, elapsed) = tokio::task::spawn_blocking(move || module_process.join().unwrap())
18659 .await
18660 .unwrap();
18661 assert!(
18662 auth.is_ok(),
18663 "the daemon must answer a module's handshake while a restart waits on the \
18664 live-children record write; got {auth:?} after {elapsed:?}"
18665 );
18666
18667 tokio::time::timeout(Duration::from_secs(20), async {
18668 while lock_snapshot(&module.inner.snapshot)
18669 .unwrap()
18670 .reported_pid()
18671 .is_none_or(|pid| pid == old_pid)
18672 {
18673 tokio::time::sleep(Duration::from_millis(10)).await;
18674 }
18675 })
18676 .await
18677 .expect("the restart published its replacement");
18678 module.stop().await.unwrap();
18679 server.abort();
18680 }
18681}
18682
18683#[cfg(all(test, unix))]
18684mod retained_status_tests {
18685 use super::*;
18686
18687 #[tokio::test]
18688 async fn cached_container_block_is_visible_in_status_and_health_without_rechecking() {
18689 use crate::live_children::{ContainerIdentity, RetainedReason};
18690 for (reason, carried, expected) in [
18691 (
18692 RetainedReason::ContainerSurvivors(vec![4242]),
18693 true,
18694 Some("waiting: container survivors (pids [4242])".to_owned()),
18695 ),
18696 (
18697 RetainedReason::ContainerUnverifiable("member inspection denied".into()),
18698 true,
18699 Some(
18700 "waiting: container survivors could not be verified: member inspection denied"
18701 .to_owned(),
18702 ),
18703 ),
18704 (RetainedReason::Survived, false, None),
18705 ] {
18706 let supervisor =
18707 Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default());
18708 let spec = ModuleSpec {
18709 module_id: "retained-status-fixture".into(),
18710 program: PathBuf::from("/not-spawned"),
18711 args: Vec::new(),
18712 env: Vec::new(),
18713 reserved: false,
18714 reserved_prefixes: Vec::new(),
18715 protocol: ModuleProtocol::None,
18716 overlap: ModuleOverlap::Exclusive,
18717 carry_containment: carried,
18718 };
18719 let module = supervisor
18720 .supervise_configured(spec.clone(), false)
18721 .unwrap();
18722 let guard = supervisor
18726 .child_roster
18727 .admit(
18728 spec.module_id,
18729 0,
18730 ModuleProtocol::None,
18731 None,
18732 crate::child_roster::RecordedIdentity {
18733 container: carried.then_some(ContainerIdentity {
18734 pgid: Some(0),
18735 created_start_time: Some(0),
18736 cgroup_path: None,
18737 ..ContainerIdentity::test_metadata()
18738 }),
18739 ..crate::child_roster::RecordedIdentity::default()
18740 },
18741 )
18742 .unwrap();
18743 guard.retain(reason);
18744 let status = module.status().unwrap();
18745 assert_eq!(status.waiting_for_previous_detail, expected.clone());
18746 assert_eq!(status.health.detail, expected);
18747 assert!(!status.process_alive);
18748 }
18749 }
18750}