1mod resource_launch;
2pub(crate) use resource_launch::{ResourceCommitError, ResourceLaunch};
3
4use std::{
5 collections::{HashMap, HashSet, VecDeque},
6 error::Error,
7 fmt, io,
8 path::PathBuf,
9 process::{ExitStatus, Stdio},
10 sync::{Arc, Mutex, OnceLock},
11 time::{Duration, SystemTime, UNIX_EPOCH},
12};
13
14use cortexkit_log::Retention;
15use serde_json::Value;
16use sha2::{Digest, Sha256};
17use subc_control::{
18 ClientControlPush, LiveSpawn, ModuleProtocol, RouteCloseReason, SpawnCursor, SpawnEvent,
19 SpawnEventKind, SpawnSnapshot, SupervisorHealthStatus, TerminalDisposition, TerminalExitKind,
20};
21use subc_protocol::{
22 manifest::{SelfSignalKind, SignalAnchor},
23 session::{
24 HealthReport, HealthStatus, ModuleControlCommand, ModuleControlRequest,
25 MODULE_CONTROL_OP_HEALTH_CHECK,
26 },
27 Flags, FrameType, Principal, Priority, SUBC_LAUNCH_NONCE_ENV, SUBC_MODULE_ID_ENV,
28};
29use tokio::{
30 process::{Child, Command},
31 sync::{mpsc, oneshot, watch, Mutex as AsyncMutex},
32 task::JoinHandle,
33 time::{sleep, sleep_until, timeout, timeout_at, Instant},
34};
35use tracing::{debug, error, info, warn};
36
37use crate::{
38 child_roster::ChildRoster,
39 daemon_config::{
40 CAPTURE_KEEP_ENV, CAPTURE_MAX_AGE_DAYS_ENV, CAPTURE_MAX_FILE_MB_ENV, CK_LOG_ENV,
41 },
42 forwarding::{
43 CloseReason, ForwardingError, ForwardingTable, GoodbyeTarget, ModuleControlRpcOutcome,
44 ModuleDrainTarget, PendingModuleControlRpc,
45 },
46 provenance::{ExecutableIdentityProbe, SpawnedFileIdentity},
47 registry::{ConnectionId, RegistrationEndReason, RegistryError},
48 stderr_tail::{
49 pump_stderr_to, pump_stdout_to, ChildOutputSink, StderrRing, StderrTailConfig,
50 StderrTailSnapshot,
51 },
52 terminal_ring::{TerminalHistorySnapshot, TerminalRecord, TerminalRing, TerminalRingConfig},
53 Frame, FrameSink, Registry,
54};
55
56#[cfg(not(windows))]
57use crate::provenance::spawned_file_identity;
58
59#[path = "supervise_swap.rs"]
60mod swap;
61
62#[path = "supervise_activity.rs"]
63mod activity;
64
65#[cfg(unix)]
66#[path = "supervise/admission.rs"]
67mod admission;
68#[cfg(all(test, windows))]
69mod carried_job_tests;
70pub(crate) mod containment;
71#[cfg(all(test, unix))]
72mod containment_tests;
73mod slow_retry;
74use slow_retry::{FailedRetry, FailureContext};
75
76#[cfg(all(test, unix))]
77mod slow_retry_tests;
78
79pub const SUBC_ARG: &str = "--subc";
85
86const DEFAULT_MAX_RESTARTS: u32 = 3;
87const DEFAULT_BACKOFF: Duration = Duration::from_millis(100);
88const DEFAULT_MAX_BACKOFF: Duration = Duration::from_secs(30);
89const DEFAULT_RESTART_WINDOW: Duration = Duration::from_secs(600);
93pub const DEFAULT_DRAIN_TIMEOUT: Duration = Duration::from_secs(30);
104pub(crate) const DEFAULT_REGISTER_TIMEOUT: Duration = Duration::from_secs(120);
105const REGISTRY_RELEASE_TIMEOUT: Duration = Duration::from_secs(1);
106const REGISTRY_RELEASE_POLL: Duration = Duration::from_millis(10);
107const DRAIN_BUSY_REPROBE_INTERVAL: Duration = Duration::from_millis(250);
119const STDERR_PUMP_DRAIN_TIMEOUT: Duration = Duration::from_millis(250);
142pub const SPAWN_EVENT_RING_CAPACITY: usize = 4096;
144const SPAWN_SUBSCRIBER_BUFFER: usize = SPAWN_EVENT_RING_CAPACITY + 1;
145pub(crate) const SPAWN_SUBSCRIBER_LAGGED_CODE: &str = "spawn_subscriber_lagged";
150
151#[cfg(target_os = "macos")]
152struct PrivacyExec {
153 reader: tokio::io::unix::AsyncFd<std::io::PipeReader>,
154 deadline: tokio::time::Instant,
155 expected: Option<subc_os::FileIdentity>,
156 trampoline: Option<subc_os::FileIdentity>,
157 script: bool,
158 module_id: String,
159 #[cfg(test)]
160 confirmation_pause: Option<(String, Option<tokio::net::TcpStream>)>,
161}
162
163#[cfg(all(test, target_os = "macos"))]
164pub(crate) fn test_privacy_trampoline() -> PathBuf {
165 let path = std::env::current_exe()
168 .unwrap()
169 .parent()
170 .unwrap()
171 .parent()
172 .unwrap()
173 .join("privacy-trampoline-fixture");
174 assert!(
178 path.exists(),
179 "privacy-trampoline-fixture not built at {}: run `cargo build -p subc-daemon \
180 --bins --features test-support` or `cargo test -p subc-daemon` first",
181 path.display()
182 );
183 path
184}
185
186#[cfg(target_os = "macos")]
187fn probe_privacy_trampoline(path: &std::path::Path) -> Result<(), String> {
188 use std::io::Read;
189 let mut probe = std::process::Command::new(path)
190 .args(["__disclaim-exec", "--probe"])
191 .stdin(Stdio::null())
192 .stdout(Stdio::piped())
193 .stderr(Stdio::piped())
194 .spawn()
195 .map_err(|error| {
196 format!(
197 "privacy trampoline probe failed for {}: {error}",
198 path.display()
199 )
200 })?;
201 let deadline = std::time::Instant::now() + Duration::from_secs(5);
202 let status = loop {
203 match probe.try_wait() {
204 Ok(Some(status)) => break status,
205 Ok(None) if std::time::Instant::now() < deadline => {
206 std::thread::sleep(Duration::from_millis(5))
207 }
208 result => {
209 let _ = probe.kill();
210 let _ = probe.wait();
211 return Err(format!(
212 "privacy trampoline probe failed or timed out for {}: {result:?}",
213 path.display()
214 ));
215 }
216 }
217 };
218 let mut answer = String::new();
219 if let Some(stdout) = probe.stdout.take() {
220 let _ = stdout.take(256).read_to_string(&mut answer);
221 }
222 if status.success() && answer.trim() == subc_os::privacy_identity::TRAMPOLINE_PROBE {
223 return Ok(());
224 }
225 let mut diagnostic = String::new();
226 if let Some(stderr) = probe.stderr.take() {
227 let _ = stderr.take(1024).read_to_string(&mut diagnostic);
228 }
229 let cause = diagnostic
230 .trim()
231 .strip_prefix("ck-subc: own privacy identity refused: ")
232 .unwrap_or("binary does not implement the privacy trampoline protocol");
233 Err(format!(
234 "{cause}: probe of {} exited {status}",
235 path.display()
236 ))
237}
238
239#[cfg(target_os = "macos")]
240fn privacy_command(
241 spec: &ModuleSpec,
242 roster: &ChildRoster,
243) -> Result<
244 (
245 Command,
246 Option<PrivacyExec>,
247 subc_os::privacy_identity::ExecAcknowledgement,
248 ),
249 SuperviseError,
250> {
251 let failure = |cause: String| {
252 warn!(module_id = %spec.module_id, %cause, "privacy identity trampoline refused module spawn");
253 SuperviseError::Spawn {
254 program: spec.program.clone(),
255 source: io::Error::other(cause),
256 cgroup_path: None,
257 }
258 };
259 let trampoline = roster.privacy_trampoline().map_err(failure)?;
260 let program = if spec.program.components().count() == 1 && !spec.program.is_absolute() {
265 let path = spec
266 .env
267 .iter()
268 .find(|(key, _)| key == "PATH")
269 .map(|(_, value)| std::ffi::OsString::from(value))
270 .or_else(|| std::env::var_os("PATH"))
271 .unwrap_or_else(|| "/usr/bin:/bin".into());
272 std::env::split_paths(&path)
273 .map(|dir| dir.join(&spec.program))
274 .find(|path| path.is_file())
275 .unwrap_or_else(|| spec.program.clone())
276 } else {
277 spec.program.clone()
278 };
279 let expected = subc_os::file_identity(&program);
280 let trampoline_image = subc_os::file_identity(&trampoline);
281 let script = {
282 use std::io::Read;
283 let mut prefix = [0u8; 2];
284 std::fs::File::open(&program)
285 .is_ok_and(|mut file| file.read_exact(&mut prefix).is_ok() && &prefix == b"#!")
286 };
287 if expected.is_none() || expected == subc_os::file_identity(&trampoline) {
288 return Err(failure(
289 "privacy identity module executable is missing or is the trampoline itself".to_string(),
290 ));
291 }
292 let (reader, ack) = subc_os::privacy_identity::ExecAcknowledgement::pipe()
293 .map_err(|error| failure(error.to_string()))?;
294 let reader =
295 tokio::io::unix::AsyncFd::new(reader).map_err(|error| failure(error.to_string()))?;
296 let mut command = Command::new(&trampoline);
297 command
298 .arg("__disclaim-exec")
299 .arg(ack.fd().to_string())
300 .arg(&program);
301 Ok((
302 command,
303 Some(PrivacyExec {
304 reader,
305 deadline: tokio::time::Instant::now() + Duration::from_secs(5),
306 expected,
307 trampoline: trampoline_image,
308 script,
309 module_id: spec.module_id.clone(),
310 #[cfg(test)]
311 confirmation_pause: spec
312 .env
313 .iter()
314 .find(|(key, _)| key == "SUBC_TEST_BEFORE_EXEC_CONFIRM")
315 .map(|(_, address)| (address.clone(), None)),
316 }),
317 ack,
318 ))
319}
320
321struct SupervisedChild {
322 subtree_target: crate::subtree_resources::Target,
323 child: containment::ProcessChild,
324 exec: crate::activity::ExecOwner,
325 #[cfg(windows)]
326 nonce_delivery: Option<subc_os::LaunchNoncePipeDelivery>,
327 #[cfg(target_os = "macos")]
328 privacy_exec: Option<PrivacyExec>,
329 #[cfg(target_os = "macos")]
336 report_ready: Arc<OnceLock<()>>,
337 spawn_failure: Option<String>,
340 protocol: ModuleProtocol,
345 #[cfg(target_os = "linux")]
352 module_id: String,
353 #[cfg(target_os = "linux")]
354 cgroup_placement: Option<subc_cgroup::Placement>,
355 #[cfg(windows)]
378 job: Option<Arc<Mutex<subc_jobobject::JobObject>>>,
379 stdout_pump: Option<JoinHandle<()>>,
380 stderr_pump: Option<StderrPump>,
381 stderr_ring: Arc<Mutex<StderrRing>>,
382 spawned_at: Instant,
385 registration_deadline: Option<Instant>,
388 registration_watch: Option<RegistrationWatch>,
389 spawned_at_ms: u64,
390 spawned_from: PathBuf,
391 launch_spec_digest: LaunchSpecDigest,
392 spawned_file_identity: Option<SpawnedFileIdentity>,
393 process_start_time: Option<u64>,
394 process_identity: Option<ProcessIdentity>,
395 pid: u32,
396 roster_guard: Option<crate::child_roster::RosterGuard>,
399}
400
401impl SupervisedChild {
402 fn registered_since_spawn(&self) -> bool {
403 self.registration_watch
404 .as_ref()
405 .is_some_and(|watch| *watch.changes.borrow() != watch.baseline)
406 }
407
408 fn id(&self) -> Option<u32> {
409 Some(self.pid)
410 }
411
412 fn process_identity(&self) -> Option<ProcessIdentity> {
413 self.process_identity
414 }
415
416 async fn wait(&mut self) -> io::Result<ExitStatus> {
417 #[cfg(target_os = "macos")]
418 self.confirm_privacy_exec().await;
419 let result = self.child.wait().await;
427 #[cfg(target_os = "linux")]
428 if result.is_ok() && self.child.container().is_none() {
429 if let Some(placement) = self.cgroup_placement.as_ref() {
430 cleanup_reaped_cgroup(placement, &self.module_id).await;
434 self.cgroup_placement = None;
435 }
436 }
437 result
438 }
439
440 async fn finish_exit(&mut self, carry: bool) -> io::Result<()> {
441 if self.child.container().is_some() {
442 if !carry {
443 self.child.terminate_container().await;
444 }
445 } else {
446 #[cfg(target_os = "linux")]
447 if let Some(placement) = self.cgroup_placement.as_ref() {
448 cleanup_reaped_cgroup(placement, &self.module_id).await;
449 self.cgroup_placement = None;
450 }
451 }
452 self.child.reap().await?;
453 self.release_roster();
454 Ok(())
455 }
456
457 #[cfg(target_os = "macos")]
458 async fn confirm_privacy_exec(&mut self) {
459 let Some(pending) = &mut self.privacy_exec else {
460 return;
461 };
462 let result = tokio::time::timeout_at(pending.deadline, async {
463 let mut record = Vec::new();
464 loop {
465 let mut ready = pending.reader.readable().await?;
466 let read = ready.try_io(|reader| {
467 use std::io::Read;
468 let mut reader = reader.get_ref();
469 let mut buffer = [0u8; 256];
470 reader.read(&mut buffer).map(|count| (count, buffer))
471 });
472 match read {
473 Ok(Ok((0, _))) => return Ok::<_, io::Error>(record),
474 Ok(Ok((count, buffer))) => {
475 if record.len() + count > 1024 {
476 return Err(io::Error::other(
477 "privacy exec refusal record is too long",
478 ));
479 }
480 record.extend_from_slice(&buffer[..count]);
481 }
482 Ok(Err(error)) => return Err(error),
483 Err(_) => continue,
484 }
485 }
486 })
487 .await;
488 #[cfg(test)]
489 let result = if let Some((address, stream)) = &mut pending.confirmation_pause {
490 use tokio::io::{AsyncReadExt, AsyncWriteExt};
491 if stream.is_none() {
492 *stream = Some(
493 tokio::net::TcpStream::connect(address.as_str())
494 .await
495 .unwrap(),
496 );
497 stream.as_mut().unwrap().write_all(b"R").await.unwrap();
498 stream
499 .as_mut()
500 .unwrap()
501 .write_all(&self.pid.to_ne_bytes())
502 .await
503 .unwrap();
504 }
505 let mut reply = [0];
506 stream
507 .as_mut()
508 .unwrap()
509 .read_exact(&mut reply)
510 .await
511 .unwrap();
512 pending.confirmation_pause = None;
513 if &reply == b"F" {
514 Ok(Err(io::Error::other("injected late exec refusal")))
515 } else {
516 assert_eq!(&reply, b"X");
517 result
518 }
519 } else {
520 result
521 };
522 let pending = self.privacy_exec.as_ref().expect("pending exec");
525 let cause = match result {
526 Err(_) => Some("privacy identity trampoline did not exec within 5s".to_string()),
527 Ok(Err(error)) => Some(format!(
528 "privacy identity exec acknowledgement failed: {error}"
529 )),
530 Ok(Ok(record)) if !record.is_empty() => Some(
531 std::str::from_utf8(&record)
532 .ok()
533 .and_then(|record| {
534 record
535 .trim()
536 .strip_prefix(subc_os::privacy_identity::EXEC_REFUSAL_TAG)
537 })
538 .filter(|cause| !cause.is_empty())
539 .unwrap_or("invalid privacy exec refusal record")
540 .to_string(),
541 ),
542 Ok(Ok(_)) => match self.child.try_wait() {
543 Ok(Some(_status)) => None,
547 Err(error) => Some(format!("privacy identity trampoline wait failed: {error}")),
548 Ok(None) => {
549 let image = observe_spawned_image(self.pid);
550 if let Some(image) = image.filter(|image| {
551 image.executable.is_some()
552 && image.executable != pending.trampoline
553 && (image.executable == pending.expected || pending.script)
554 }) {
555 if let Err(error) = self
556 .roster_guard
557 .as_ref()
558 .map(|guard| guard.confirm_executable(image))
559 .transpose()
560 {
561 Some(format!(
562 "could not durably record confirmed module image: {error}"
563 ))
564 } else {
565 let _ = self.report_ready.set(());
566 info!(module_id = %pending.module_id, pid = self.pid,
567 "module spawned with own privacy identity (responsibility disclaimed)");
568 None
569 }
570 } else if image.is_none()
571 || image.is_some_and(|image| image.executable.is_none())
572 {
573 match tokio::time::timeout_at(pending.deadline, self.child.wait()).await {
580 Ok(Ok(_status)) => None,
581 Ok(Err(error)) => Some(format!("privacy identity trampoline wait failed: {error}")),
582 Err(_) => Some("privacy identity module executable remained unreadable until the 5s exec deadline".to_string()),
583 }
584 } else {
585 Some("privacy identity trampoline executable mismatch: expected the module program, not ck-subc".to_string())
586 }
587 }
588 },
589 };
590 let pending = self.privacy_exec.take().expect("pending exec");
591 self.exec.0.finish(if cause.is_some() {
594 crate::activity::ExecState::Refused
595 } else {
596 crate::activity::ExecState::CommittedExec
597 });
598 if let Some(cause) = cause {
599 warn!(module_id = %pending.module_id, pid = self.pid, %cause, "privacy identity trampoline refused module spawn");
600 self.spawn_failure = Some(cause);
601 if let Some(container) = self.child.container() {
604 container.hard_kill();
605 } else if let Some(pid) = rustix::process::Pid::from_raw(self.pid as i32) {
606 let _ = rustix::process::kill_process_group(pid, rustix::process::Signal::KILL);
607 }
608 let _ = self.child.start_kill();
609 }
610 }
611
612 fn release_roster(&mut self) {
616 self.roster_guard = None;
617 }
618
619 fn start_kill(&mut self) -> io::Result<()> {
632 if let Some(container) = self.child.container() {
633 container.hard_kill();
634 return self.kill_leader();
635 }
636 #[cfg(windows)]
637 if let Some(job) = &self.job {
638 if let Err(error) = job.lock().unwrap_or_else(|p| p.into_inner()).terminate() {
639 debug!(
640 error = %error,
641 "job termination failed; the direct-child kill still owns the outcome"
642 );
643 }
644 }
645 #[cfg(target_os = "linux")]
646 kill_module_cgroup(self.cgroup_placement.as_ref(), &self.module_id);
647 self.child.start_kill()
648 }
649
650 fn kill_leader(&mut self) -> io::Result<()> {
651 self.child.start_kill()
652 }
653
654 async fn drain_stderr(&mut self, module_id: &str) {
655 if let Some(pump) = self.stdout_pump.take() {
656 settle_stdout_pump(module_id, pump, STDERR_PUMP_DRAIN_TIMEOUT).await;
657 }
658
659 let Some(pump) = self.stderr_pump.take() else {
660 return;
661 };
662 settle_stderr_pump(
663 module_id,
664 &self.stderr_ring,
665 pump,
666 STDERR_PUMP_DRAIN_TIMEOUT,
667 )
668 .await;
669 }
670}
671
672async fn settle_stdout_pump(module_id: &str, mut pump: JoinHandle<()>, bound: Duration) {
676 match timeout(bound, &mut pump).await {
677 Ok(Ok(())) => {}
678 Ok(Err(error)) => {
679 warn!(module_id, error = %error, "stdout pump ended unexpectedly");
680 }
681 Err(_) => {
682 drop(pump);
683 warn!(
684 module_id,
685 waited = ?bound,
686 "stdout pipe of the exited process has not drained; its reader keeps running without delaying the restart"
687 );
688 }
689 }
690}
691
692struct StderrPump {
695 task: JoinHandle<()>,
696 generation: u64,
697}
698
699async fn settle_stderr_pump(
705 module_id: &str,
706 ring: &Arc<Mutex<StderrRing>>,
707 pump: StderrPump,
708 bound: Duration,
709) {
710 let lock = || ring.lock().unwrap_or_else(|poisoned| poisoned.into_inner());
711 let StderrPump {
712 mut task,
713 generation,
714 } = pump;
715 lock().retire_pump(generation);
716 match timeout(bound, &mut task).await {
717 Ok(Ok(())) => {}
718 Ok(Err(err)) => {
719 let mut ring = lock();
720 ring.mark_incomplete_from(generation, format!("stderr pump ended unexpectedly: {err}"));
721 ring.finish_pump(generation);
722 warn!(module_id, error = %err, "stderr pump ended before clean EOF");
723 }
724 Err(_) => {
725 drop(task);
727 lock().mark_pump_late(
728 generation,
729 format!(
730 "stderr of the exited process had not reached EOF {bound:?} after it was \
731 retired (a descendant may still hold the pipe open); lines it still \
732 writes are kept in that process's section"
733 ),
734 );
735 warn!(
736 module_id,
737 waited = ?bound,
738 "stderr pipe of the exited process is still open; its reader keeps running without delaying the restart"
739 );
740 }
741 }
742}
743
744fn registration_release_events() -> &'static watch::Sender<u64> {
745 static EVENTS: OnceLock<watch::Sender<u64>> = OnceLock::new();
746 EVENTS.get_or_init(|| {
747 let (sender, _receiver) = watch::channel(0);
748 sender
749 })
750}
751
752pub(crate) fn notify_registration_release() {
753 let events = registration_release_events();
754 let next_generation = (*events.borrow()).wrapping_add(1);
755 events.send_replace(next_generation);
756}
757
758#[derive(Debug, Clone, PartialEq, Eq)]
760pub struct ModuleSpec {
761 pub module_id: String,
762 pub program: PathBuf,
763 pub args: Vec<String>,
764 pub env: Vec<(String, String)>,
765 pub reserved: bool,
770 pub reserved_prefixes: Vec<String>,
775 pub protocol: ModuleProtocol,
794 pub overlap: ModuleOverlap,
799 pub carry_containment: bool,
803}
804
805#[derive(Clone, PartialEq, Eq)]
808pub(crate) struct LaunchSpecDigest([u8; 32]);
809
810impl fmt::Debug for LaunchSpecDigest {
811 fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
812 f.write_str("LaunchSpecDigest([redacted])")
813 }
814}
815
816impl LaunchSpecDigest {
817 pub(crate) fn from_spec(spec: &ModuleSpec) -> Self {
818 fn field(hash: &mut Sha256, bytes: &[u8]) {
819 hash.update((bytes.len() as u64).to_le_bytes());
820 hash.update(bytes);
821 }
822 let mut hash = Sha256::new();
823 if let Some(program) = spec.program.to_str() {
824 field(&mut hash, b"utf8-program");
825 field(&mut hash, program.as_bytes());
826 } else {
827 #[cfg(unix)]
830 {
831 use std::os::unix::ffi::OsStrExt;
832 field(&mut hash, b"unix-program");
833 field(&mut hash, spec.program.as_os_str().as_bytes());
834 }
835 #[cfg(windows)]
836 {
837 use std::os::windows::ffi::OsStrExt;
838 field(&mut hash, b"windows-program");
839 let bytes = spec
840 .program
841 .as_os_str()
842 .encode_wide()
843 .flat_map(u16::to_le_bytes)
844 .collect::<Vec<_>>();
845 field(&mut hash, &bytes);
846 }
847 #[cfg(not(any(unix, windows)))]
848 field(&mut hash, spec.program.to_string_lossy().as_bytes());
849 }
850 hash.update((spec.args.len() as u64).to_le_bytes());
851 for arg in &spec.args {
852 field(&mut hash, arg.as_bytes());
853 }
854 let mut env = spec.env.iter().collect::<Vec<_>>();
855 env.sort_unstable();
856 hash.update((env.len() as u64).to_le_bytes());
857 for (key, value) in env {
858 field(&mut hash, key.as_bytes());
859 field(&mut hash, value.as_bytes());
860 }
861 field(
862 &mut hash,
863 match spec.protocol {
864 ModuleProtocol::Subc => b"subc",
865 ModuleProtocol::None => b"none",
866 },
867 );
868 Self(hash.finalize().into())
869 }
870}
871
872#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)]
879pub enum ModuleOverlap {
880 #[default]
882 Exclusive,
883 Safe,
895}
896
897impl ModuleOverlap {
898 pub fn as_str(self) -> &'static str {
899 match self {
900 Self::Exclusive => "exclusive",
901 Self::Safe => "safe",
902 }
903 }
904}
905
906pub const SUBC_SPAWN_ROLE_ENV: &str = "SUBC_SPAWN_ROLE";
916pub const SPAWN_ROLE_SWAP_CANDIDATE: &str = "swap_candidate";
918pub const DEFAULT_SWAP_READY_TIMEOUT: Duration = Duration::from_secs(100);
923
924#[derive(Debug, Clone, Copy, PartialEq, Eq)]
942pub struct RestartPolicy {
943 pub max_restarts: u32,
944 pub backoff: Duration,
947 pub max_backoff: Duration,
949 pub window: Duration,
953}
954
955impl RestartPolicy {
956 pub fn new(max_restarts: u32, backoff: Duration) -> Self {
960 Self {
961 max_restarts,
962 backoff,
963 max_backoff: DEFAULT_MAX_BACKOFF,
964 window: DEFAULT_RESTART_WINDOW,
965 }
966 }
967
968 pub fn with_max_backoff(mut self, max_backoff: Duration) -> Self {
969 self.max_backoff = max_backoff;
970 self
971 }
972
973 pub fn with_window(mut self, window: Duration) -> Self {
974 self.window = window;
975 self
976 }
977
978 fn delay_for_restart(&self, restart_in_window: u32) -> Duration {
983 if self.backoff.is_zero() || self.max_backoff.is_zero() {
984 return Duration::ZERO;
985 }
986
987 let mut delay = self.backoff;
988 for _ in 0..restart_in_window {
989 if delay >= self.max_backoff {
990 return self.max_backoff;
991 }
992 delay = delay
993 .checked_mul(10)
994 .unwrap_or(self.max_backoff)
995 .min(self.max_backoff);
996 }
997 delay.min(self.max_backoff)
998 }
999
1000 fn budget_exhausted_detail(&self) -> String {
1005 format!(
1006 "crash budget exhausted: max_restarts={} within window_secs={}",
1007 self.max_restarts,
1008 self.window.as_secs()
1009 )
1010 }
1011}
1012
1013impl Default for RestartPolicy {
1014 fn default() -> Self {
1015 Self {
1016 max_restarts: DEFAULT_MAX_RESTARTS,
1017 backoff: DEFAULT_BACKOFF,
1018 max_backoff: DEFAULT_MAX_BACKOFF,
1019 window: DEFAULT_RESTART_WINDOW,
1020 }
1021 }
1022}
1023
1024#[derive(Debug, Clone, Copy, PartialEq, Eq)]
1025struct CrashRestartSchedule {
1026 restart_in_window: u32,
1027 delay: Duration,
1028}
1029
1030fn daemon_will_restart(
1037 state: &mut SupervisorSnapshot,
1038 policy: &RestartPolicy,
1039 now: Instant,
1040) -> bool {
1041 state.enabled && state.crash_restarts_in_window(policy.window, now) < policy.max_restarts
1042}
1043
1044const DEFAULT_HEALTH_CADENCE: Duration = Duration::from_secs(30);
1045const DEFAULT_HEALTH_DEADLINE: Duration = Duration::from_secs(5);
1046const DEFAULT_HEALTH_FAILURE_THRESHOLD: u32 = 3;
1047const MAX_HEALTH_METRICS_BYTES: usize = 16 * 1024;
1048
1049#[derive(Debug, Clone, Copy, PartialEq, Eq)]
1050pub enum HealthAction {
1051 Report,
1052 Restart,
1053 Alert,
1054}
1055
1056impl fmt::Display for HealthAction {
1057 fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
1058 f.write_str(match self {
1059 Self::Report => "report",
1060 Self::Restart => "restart",
1061 Self::Alert => "alert",
1062 })
1063 }
1064}
1065
1066#[derive(Debug, Clone, PartialEq, Eq)]
1067pub struct HealthConfig {
1068 pub http: Option<String>,
1072 pub cadence: Duration,
1073 pub deadline: Duration,
1074 pub failure_threshold: u32,
1075 pub on_degraded: HealthAction,
1076 pub on_failing: HealthAction,
1077 pub critical: bool,
1078}
1079
1080impl Default for HealthConfig {
1081 fn default() -> Self {
1082 Self {
1083 http: None,
1084 cadence: DEFAULT_HEALTH_CADENCE,
1085 deadline: DEFAULT_HEALTH_DEADLINE,
1086 failure_threshold: DEFAULT_HEALTH_FAILURE_THRESHOLD,
1087 on_degraded: HealthAction::Report,
1088 on_failing: HealthAction::Report,
1089 critical: false,
1090 }
1091 }
1092}
1093
1094#[derive(Debug, Clone, PartialEq)]
1112pub struct ModuleHealthStatus {
1113 pub status: SupervisorHealthStatus,
1114 pub last_probe_ms: Option<u64>,
1115 pub detail: Option<String>,
1116 pub metrics: Option<Value>,
1117 pub consecutive_failures: u32,
1118 pub late_answer_count: u64,
1121 pub last_late_answer_latency_ms: Option<u64>,
1123 pub last_action: Option<String>,
1124 pub last_action_ms: Option<u64>,
1128}
1129
1130impl Default for ModuleHealthStatus {
1131 fn default() -> Self {
1132 Self {
1133 status: SupervisorHealthStatus::Unknown,
1134 last_probe_ms: None,
1135 detail: None,
1136 metrics: None,
1137 consecutive_failures: 0,
1138 late_answer_count: 0,
1139 last_late_answer_latency_ms: None,
1140 last_action: None,
1141 last_action_ms: None,
1142 }
1143 }
1144}
1145
1146#[derive(Debug, Clone, Copy, PartialEq, Eq)]
1148pub enum ModuleState {
1149 Starting,
1150 Running,
1151 Unresponsive,
1152 Restarting,
1153 Draining,
1154 Stopped,
1155 Failed,
1156 Disabled,
1157}
1158
1159impl fmt::Display for ModuleState {
1160 fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
1161 f.write_str(match self {
1162 Self::Starting => "starting",
1163 Self::Running => "running",
1164 Self::Unresponsive => "unresponsive",
1165 Self::Restarting => "restarting",
1166 Self::Draining => "draining",
1167 Self::Stopped => "stopped",
1168 Self::Failed => "failed",
1169 Self::Disabled => "disabled",
1170 })
1171 }
1172}
1173
1174#[derive(Debug, Clone, Copy, PartialEq, Eq)]
1176pub enum ExitKind {
1177 Clean,
1178 Crash,
1179 DeliberateSeverance,
1180}
1181
1182impl From<ExitKind> for TerminalExitKind {
1183 fn from(kind: ExitKind) -> Self {
1184 match kind {
1185 ExitKind::Clean => Self::Clean,
1186 ExitKind::Crash => Self::Crash,
1187 ExitKind::DeliberateSeverance => Self::DeliberateSeverance,
1188 }
1189 }
1190}
1191
1192#[derive(Debug, Clone, Copy, PartialEq, Eq)]
1195pub(crate) struct ProcessIdentity {
1196 pub(crate) pid: u32,
1197 pub(crate) start_time: u64,
1198}
1199
1200#[derive(Debug, Clone, PartialEq, Eq)]
1202pub struct ExitReport {
1203 pub kind: ExitKind,
1204 pub code: Option<i32>,
1205 pub signal: Option<i32>,
1206 pub at_ms: u64,
1207}
1208
1209#[derive(Debug, Clone, PartialEq)]
1212pub struct ModuleStatus {
1213 pub module_id: String,
1214 pub waiting_for_previous_pid: Option<u32>,
1216 pub waiting_for_previous_detail: Option<String>,
1218 pub launch_nonce_source: Option<String>,
1223 pub state: ModuleState,
1224 pub enabled: bool,
1225 pub process_alive: bool,
1226 pub registration_active: bool,
1227 pub protocol: ModuleProtocol,
1232 pub live: bool,
1243 pub restart_count: u32,
1247 pub lifetime_restarts: u32,
1251 pub spawn_generation: u64,
1252 pub max_restarts: u32,
1257 pub restart_window: Duration,
1261 pub drain_timeout: Duration,
1265 pub restart_backoff: Duration,
1266 pub restart_max_backoff: Duration,
1267 pub pid: Option<u32>,
1272 pub spawned_at_ms: Option<u64>,
1273 pub spawned_from: Option<PathBuf>,
1274 pub(crate) launch_spec_digest: Option<LaunchSpecDigest>,
1275 pub process_start_time: Option<u64>,
1276 pub last_exit: Option<ExitReport>,
1277 pub health: ModuleHealthStatus,
1278}
1279
1280#[cfg(test)]
1281#[derive(Debug, Clone, Copy, PartialEq)]
1282struct ActorSelectCheckpoint {
1283 generation: u64,
1284 turn: u64,
1285 registered_connection: Option<ConnectionId>,
1286 next_probe_at: Option<Instant>,
1287 wake_after: Option<Duration>,
1288}
1289
1290#[derive(Debug, Clone, PartialEq)]
1291struct SupervisorSnapshot {
1292 subtree_target: crate::subtree_resources::Target,
1293 failed_retry: FailedRetry,
1294 exec_outcome: Option<Arc<crate::activity::ExecOutcome>>,
1295 #[cfg(windows)]
1296 pipe_consumed: Option<Arc<OnceLock<()>>>,
1297 #[cfg(test)]
1299 actor_turns: u64,
1300 #[cfg(test)]
1303 actor_select: Option<ActorSelectCheckpoint>,
1304 state: ModuleState,
1305 enabled: bool,
1306 process_alive: bool,
1307 spawned_protocol: Option<ModuleProtocol>,
1308 spawn_failure: Option<String>,
1309 waiting_for_previous_pid: Option<u32>,
1310 waiting_for_previous_detail: Option<String>,
1311 crash_restarts: VecDeque<Instant>,
1317 lifetime_restarts: u32,
1318 spawn_generation: u64,
1327 pid: Option<u32>,
1328 #[cfg(target_os = "macos")]
1329 report_ready: Option<Arc<OnceLock<()>>>,
1330 reaped_pid: Option<u32>,
1332 respawn_pending: bool,
1334 coalesced_restart_pending: bool,
1336 spawned_at_ms: Option<u64>,
1337 spawned_from: Option<PathBuf>,
1338 launch_spec_digest: Option<LaunchSpecDigest>,
1339 spawned_file_identity: Option<SpawnedFileIdentity>,
1340 process_start_time: Option<u64>,
1341 deliberate_severance: Option<ProcessIdentity>,
1342 last_exit: Option<ExitReport>,
1343 drain_disposition_detail: Option<String>,
1345 health: ModuleHealthStatus,
1346 in_alternate_slot: bool,
1351 draining_to_replace: bool,
1358 configuration_updated_since_spawn: bool,
1364}
1365
1366impl SupervisorSnapshot {
1367 fn reported_pid(&self) -> Option<u32> {
1372 #[cfg(target_os = "macos")]
1373 if self
1374 .report_ready
1375 .as_ref()
1376 .is_some_and(|ready| ready.get().is_none())
1377 {
1378 return None;
1379 }
1380 self.pid
1381 }
1382
1383 fn starting() -> Self {
1384 Self::new(ModuleState::Starting, true)
1385 }
1386
1387 fn disabled() -> Self {
1388 Self::new(ModuleState::Disabled, false)
1389 }
1390
1391 fn crash_restarts_in_window(&mut self, window: Duration, now: Instant) -> u32 {
1395 while let Some(oldest) = self.crash_restarts.front() {
1396 if now.duration_since(*oldest) > window {
1397 self.crash_restarts.pop_front();
1398 } else {
1399 break;
1400 }
1401 }
1402 u32::try_from(self.crash_restarts.len()).unwrap_or(u32::MAX)
1403 }
1404
1405 fn record_crash_restart(&mut self, policy: &RestartPolicy, now: Instant) {
1411 self.crash_restarts.push_back(now);
1412 while self.crash_restarts.len() > policy.max_restarts as usize {
1413 self.crash_restarts.pop_front();
1414 }
1415 self.lifetime_restarts += 1;
1416 }
1417
1418 fn next_crash_restart(
1422 &mut self,
1423 policy: &RestartPolicy,
1424 now: Instant,
1425 ) -> Option<CrashRestartSchedule> {
1426 let restart_in_window = self.crash_restarts_in_window(policy.window, now);
1427 if restart_in_window >= policy.max_restarts {
1428 return None;
1429 }
1430 self.record_crash_restart(policy, now);
1431 Some(CrashRestartSchedule {
1432 restart_in_window,
1433 delay: policy.delay_for_restart(restart_in_window),
1434 })
1435 }
1436
1437 fn clear_crash_restarts(&mut self) {
1442 self.crash_restarts.clear();
1443 }
1444
1445 fn new(state: ModuleState, enabled: bool) -> Self {
1446 Self {
1447 subtree_target: crate::subtree_resources::Target::Unavailable("not_running"),
1448 failed_retry: FailedRetry::default(),
1449 exec_outcome: None,
1450 #[cfg(test)]
1451 actor_turns: 0,
1452 #[cfg(test)]
1453 actor_select: None,
1454 state,
1455 enabled,
1456 process_alive: false,
1457 #[cfg(windows)]
1458 pipe_consumed: None,
1459 spawned_protocol: None,
1460 spawn_failure: None,
1461 waiting_for_previous_pid: None,
1462 waiting_for_previous_detail: None,
1463 crash_restarts: VecDeque::new(),
1464 lifetime_restarts: 0,
1465 spawn_generation: 0,
1466 pid: None,
1467 #[cfg(target_os = "macos")]
1468 report_ready: None,
1469 reaped_pid: None,
1470 respawn_pending: false,
1471 coalesced_restart_pending: false,
1472 spawned_at_ms: None,
1473 spawned_from: None,
1474 launch_spec_digest: None,
1475 spawned_file_identity: None,
1476 process_start_time: None,
1477 deliberate_severance: None,
1478 last_exit: None,
1479 drain_disposition_detail: None,
1480 health: ModuleHealthStatus::default(),
1481 in_alternate_slot: false,
1482 draining_to_replace: false,
1483 configuration_updated_since_spawn: false,
1484 }
1485 }
1486}
1487
1488type SharedSnapshot = Arc<Mutex<SupervisorSnapshot>>;
1489
1490type SpawnSubscriberKey = (ConnectionId, u64);
1491
1492#[derive(Debug)]
1493struct SpawnSubscriber {
1494 version: u8,
1495 frames: mpsc::Sender<Frame>,
1496 lagged: Option<oneshot::Sender<SpawnCursor>>,
1500}
1501
1502#[derive(Debug)]
1503struct SpawnEventState {
1504 daemon_incarnation: String,
1505 seq: u64,
1506 capacity: usize,
1507 live: HashMap<String, LiveSpawn>,
1508 generations: HashMap<String, u64>,
1509 events: VecDeque<SpawnEvent>,
1510 subscribers: HashMap<SpawnSubscriberKey, SpawnSubscriber>,
1511}
1512
1513impl Default for SpawnEventState {
1514 fn default() -> Self {
1515 Self {
1516 daemon_incarnation: "unconfigured".to_string(),
1517 seq: 0,
1518 capacity: SPAWN_EVENT_RING_CAPACITY,
1519 live: HashMap::new(),
1520 generations: HashMap::new(),
1521 events: VecDeque::new(),
1522 subscribers: HashMap::new(),
1523 }
1524 }
1525}
1526
1527#[derive(Debug, Clone, Default)]
1528struct SpawnEventFeed(
1529 Arc<Mutex<SpawnEventState>>,
1530 #[cfg(test)] Arc<Mutex<Option<std::net::TcpStream>>>,
1531);
1532
1533#[derive(Debug, Clone, PartialEq, Eq)]
1534pub(crate) enum SpawnSubscribeRefusal {
1535 ForeignIncarnation { current: String },
1536 TooOld { oldest: SpawnCursor },
1537 Frame(String),
1538}
1539
1540impl SpawnEventFeed {
1541 fn configure_incarnation(&self, daemon_incarnation: String) {
1542 let mut state = self.0.lock().unwrap_or_else(|p| p.into_inner());
1543 state.daemon_incarnation = daemon_incarnation;
1544 state.seq = 0;
1545 state.live.clear();
1546 state.generations.clear();
1547 state.events.clear();
1548 state.subscribers.clear();
1549 }
1550
1551 fn cursor(state: &SpawnEventState) -> SpawnCursor {
1552 SpawnCursor {
1553 daemon_incarnation: state.daemon_incarnation.clone(),
1554 seq: state.seq,
1555 }
1556 }
1557
1558 fn snapshot(&self) -> SpawnSnapshot {
1559 let state = self.0.lock().unwrap_or_else(|p| p.into_inner());
1560 let mut live = state.live.values().cloned().collect::<Vec<_>>();
1561 live.sort_by(|left, right| left.module_id.cmp(&right.module_id));
1562 SpawnSnapshot {
1563 cursor: Self::cursor(&state),
1564 ring_bound: state.capacity as u64,
1565 live,
1566 }
1567 }
1568
1569 fn emit_spawned(&self, module_id: &str, pid: u32, spawned_at_ms: u64) -> u64 {
1570 let mut state = self.0.lock().unwrap_or_else(|p| p.into_inner());
1571 let generation = state
1572 .generations
1573 .get(module_id)
1574 .copied()
1575 .unwrap_or(0)
1576 .checked_add(1)
1577 .expect("spawn generation exhausted");
1578 state.generations.insert(module_id.to_string(), generation);
1579 let live = LiveSpawn {
1580 module_id: module_id.to_string(),
1581 spawn_generation: generation,
1582 pid,
1583 spawned_at_ms,
1584 };
1585 state.live.insert(module_id.to_string(), live);
1586 Self::emit_locked(
1587 &mut state,
1588 SpawnEventKind::Spawned,
1589 module_id.to_string(),
1590 generation,
1591 pid,
1592 None,
1593 None,
1594 );
1595 generation
1596 }
1597
1598 fn emit_exited(&self, module_id: &str, exit_code: Option<i32>, exit_signal: Option<i32>) {
1599 let mut state = self.0.lock().unwrap_or_else(|p| p.into_inner());
1600 let Some(live) = state.live.remove(module_id) else {
1601 warn!(
1602 module_id,
1603 "terminal record had no live spawn event identity"
1604 );
1605 return;
1606 };
1607 Self::emit_locked(
1608 &mut state,
1609 SpawnEventKind::Exited,
1610 module_id.to_string(),
1611 live.spawn_generation,
1612 live.pid,
1613 exit_code,
1614 exit_signal,
1615 );
1616 }
1617
1618 fn emit_superseded_exited(
1625 &self,
1626 module_id: &str,
1627 spawn_generation: u64,
1628 pid: u32,
1629 exit_code: Option<i32>,
1630 exit_signal: Option<i32>,
1631 ) {
1632 let mut state = self.0.lock().unwrap_or_else(|p| p.into_inner());
1633 if state
1634 .live
1635 .get(module_id)
1636 .is_some_and(|live| live.spawn_generation == spawn_generation)
1637 {
1638 state.live.remove(module_id);
1639 }
1640 Self::emit_locked(
1641 &mut state,
1642 SpawnEventKind::Exited,
1643 module_id.to_string(),
1644 spawn_generation,
1645 pid,
1646 exit_code,
1647 exit_signal,
1648 );
1649 }
1650
1651 #[allow(clippy::too_many_arguments)]
1652 fn emit_locked(
1653 state: &mut SpawnEventState,
1654 kind: SpawnEventKind,
1655 module_id: String,
1656 spawn_generation: u64,
1657 pid: u32,
1658 exit_code: Option<i32>,
1659 exit_signal: Option<i32>,
1660 ) {
1661 state.seq = state
1662 .seq
1663 .checked_add(1)
1664 .expect("spawn event sequence exhausted");
1665 let event = SpawnEvent {
1666 cursor: Self::cursor(state),
1667 kind,
1668 module_id,
1669 spawn_generation,
1670 pid,
1671 exit_code,
1672 exit_signal,
1673 };
1674 state.events.push_back(event.clone());
1675 while state.events.len() > state.capacity {
1676 state.events.pop_front();
1677 }
1678 let body = match serde_json::to_vec(&event) {
1679 Ok(body) => body,
1680 Err(error) => {
1681 error!(%error, "failed to serialize supervisor spawn event");
1682 return;
1683 }
1684 };
1685 state.subscribers.retain(|(connection_id, corr), subscriber| {
1686 let frame = Frame::build_with_version(
1687 subscriber.version,
1688 FrameType::StreamData,
1689 control_flags(),
1690 0,
1691 0,
1692 *corr,
1693 body.clone(),
1694 );
1695 match frame {
1696 Ok(frame) => {
1697 if subscriber.frames.try_send(frame).is_ok() {
1698 true
1699 } else {
1700 warn!(connection_id = connection_id.get(), corr, "dropping lagged supervisor spawn subscriber");
1701 if let Some(lagged) = subscriber.lagged.take() {
1702 let _ = lagged.send(event.cursor.clone());
1703 }
1704 false
1705 }
1706 }
1707 Err(error) => {
1708 warn!(connection_id = connection_id.get(), corr, %error, "dropping supervisor spawn subscriber after frame build failure");
1709 false
1710 }
1711 }
1712 });
1713 }
1714
1715 fn subscribe(
1716 &self,
1717 connection_id: ConnectionId,
1718 corr: u64,
1719 version: u8,
1720 since: Option<SpawnCursor>,
1721 sink: FrameSink,
1722 ) -> Result<(), SpawnSubscribeRefusal> {
1723 let (frames, mut receiver) = mpsc::channel(SPAWN_SUBSCRIBER_BUFFER);
1724 let (lagged, mut lagged_rx) = oneshot::channel::<SpawnCursor>();
1725 {
1726 let mut state = self.0.lock().unwrap_or_else(|p| p.into_inner());
1727 let replay = if let Some(since) = since {
1728 if since.daemon_incarnation != state.daemon_incarnation {
1729 return Err(SpawnSubscribeRefusal::ForeignIncarnation {
1730 current: state.daemon_incarnation.clone(),
1731 });
1732 }
1733 if let Some(oldest) = state.events.front().map(|event| event.cursor.clone()) {
1734 if since.seq < oldest.seq.saturating_sub(1) {
1735 return Err(SpawnSubscribeRefusal::TooOld { oldest });
1736 }
1737 }
1738 state
1739 .events
1740 .iter()
1741 .filter(|event| event.cursor.seq > since.seq)
1742 .cloned()
1743 .collect::<Vec<_>>()
1744 } else {
1745 Vec::new()
1746 };
1747 for event in replay {
1748 let body = serde_json::to_vec(&event)
1749 .map_err(|error| SpawnSubscribeRefusal::Frame(error.to_string()))?;
1750 let frame = Frame::build_with_version(
1751 version,
1752 FrameType::StreamData,
1753 control_flags(),
1754 0,
1755 0,
1756 corr,
1757 body,
1758 )
1759 .map_err(|error| SpawnSubscribeRefusal::Frame(error.to_string()))?;
1760 frames
1761 .try_send(frame)
1762 .map_err(|error| SpawnSubscribeRefusal::Frame(error.to_string()))?;
1763 }
1764 state.subscribers.insert(
1765 (connection_id, corr),
1766 SpawnSubscriber {
1767 version,
1768 frames,
1769 lagged: Some(lagged),
1770 },
1771 );
1772 }
1773 tokio::spawn(async move {
1784 while let Some(frame) = receiver.recv().await {
1785 if sink.send(frame).await.is_err() {
1786 return;
1787 }
1788 }
1789 let Ok(first_undelivered) = lagged_rx.try_recv() else {
1790 return;
1791 };
1792 match spawn_subscriber_lagged_frame(version, corr, first_undelivered) {
1793 Ok(frame) => {
1794 let _ = sink.send(frame).await;
1795 }
1796 Err(error) => {
1797 error!(%error, corr, "failed to build lagged spawn subscriber terminal frame");
1798 }
1799 }
1800 });
1801 Ok(())
1802 }
1803
1804 fn cancel(&self, connection_id: ConnectionId, corr: u64) -> bool {
1805 let Some(subscriber) = self
1806 .0
1807 .lock()
1808 .unwrap_or_else(|p| p.into_inner())
1809 .subscribers
1810 .remove(&(connection_id, corr))
1811 else {
1812 return false;
1813 };
1814 if let Ok(frame) = Frame::build_with_version(
1815 subscriber.version,
1816 FrameType::StreamEnd,
1817 control_flags(),
1818 0,
1819 0,
1820 corr,
1821 Vec::new(),
1822 ) {
1823 tokio::spawn(async move {
1824 let _ = subscriber.frames.send(frame).await;
1825 });
1826 }
1827 true
1828 }
1829
1830 fn remove_connection(&self, connection_id: ConnectionId) {
1831 self.0
1832 .lock()
1833 .unwrap_or_else(|p| p.into_inner())
1834 .subscribers
1835 .retain(|(subscriber_connection, _), _| *subscriber_connection != connection_id);
1836 }
1837
1838 #[cfg(any(test, feature = "test-support"))]
1839 fn set_capacity(&self, capacity: usize) {
1840 self.0.lock().unwrap_or_else(|p| p.into_inner()).capacity = capacity;
1841 }
1842
1843 #[cfg(any(test, feature = "test-support"))]
1844 fn subscriber_count(&self) -> usize {
1845 self.0
1846 .lock()
1847 .unwrap_or_else(|p| p.into_inner())
1848 .subscribers
1849 .len()
1850 }
1851}
1852
1853fn spawn_subscriber_lagged_frame(
1856 version: u8,
1857 corr: u64,
1858 first_undelivered: SpawnCursor,
1859) -> Result<Frame, String> {
1860 let body = serde_json::to_vec(&subc_protocol::ErrorBody {
1862 code: SPAWN_SUBSCRIBER_LAGGED_CODE.to_string(),
1863 message: "spawn subscriber fell behind and was dropped; resubscribe from the last cursor received"
1864 .to_string(),
1865 detail: Some(serde_json::json!({
1866 "first_undelivered_cursor": first_undelivered
1867 })),
1868 })
1869 .map_err(|error| error.to_string())?;
1870 Frame::build_with_version(
1871 version,
1872 FrameType::Error,
1873 crate::router::daemon_error_flags(),
1874 0,
1875 0,
1876 corr,
1877 body,
1878 )
1879 .map_err(|error| error.to_string())
1880}
1881
1882pub trait ModuleProcessLiveness: Send + Sync {
1883 fn process_live(&self, module_id: &str) -> Option<bool>;
1884
1885 fn process_replacing(&self, _module_id: &str) -> bool {
1891 false
1892 }
1893}
1894
1895#[derive(Debug, Clone, Default)]
1897pub struct SupervisorProcessLiveness {
1898 snapshots: Arc<Mutex<HashMap<String, SharedSnapshot>>>,
1899}
1900
1901impl SupervisorProcessLiveness {
1902 pub fn new() -> Self {
1903 Self::default()
1904 }
1905
1906 fn track(&self, module_id: String, snapshot: SharedSnapshot) {
1907 let mut snapshots = self
1908 .snapshots
1909 .lock()
1910 .unwrap_or_else(|poisoned| poisoned.into_inner());
1911 snapshots.insert(module_id, snapshot);
1912 }
1913
1914 fn untrack_if_current(&self, module_id: &str, snapshot: &SharedSnapshot) {
1915 let mut snapshots = self
1916 .snapshots
1917 .lock()
1918 .unwrap_or_else(|poisoned| poisoned.into_inner());
1919 let is_current = snapshots
1920 .get(module_id)
1921 .map(|tracked| Arc::ptr_eq(tracked, snapshot))
1922 .unwrap_or(false);
1923 if is_current {
1924 snapshots.remove(module_id);
1925 }
1926 }
1927}
1928
1929impl ModuleProcessLiveness for SupervisorProcessLiveness {
1930 fn process_live(&self, module_id: &str) -> Option<bool> {
1931 let snapshot = {
1932 let snapshots = self
1933 .snapshots
1934 .lock()
1935 .unwrap_or_else(|poisoned| poisoned.into_inner());
1936 snapshots.get(module_id).cloned()
1937 }?;
1938 let snapshot = snapshot
1939 .lock()
1940 .unwrap_or_else(|poisoned| poisoned.into_inner());
1941 Some(snapshot.state == ModuleState::Running && snapshot.process_alive)
1942 }
1943
1944 fn process_replacing(&self, module_id: &str) -> bool {
1945 let Some(snapshot) = self
1946 .snapshots
1947 .lock()
1948 .unwrap_or_else(|poisoned| poisoned.into_inner())
1949 .get(module_id)
1950 .cloned()
1951 else {
1952 return false;
1953 };
1954 let snapshot = snapshot
1955 .lock()
1956 .unwrap_or_else(|poisoned| poisoned.into_inner());
1957 snapshot.enabled
1958 && match snapshot.state {
1959 ModuleState::Restarting => true,
1960 ModuleState::Draining => snapshot.draining_to_replace,
1961 ModuleState::Starting
1962 | ModuleState::Running
1963 | ModuleState::Unresponsive
1964 | ModuleState::Stopped
1965 | ModuleState::Failed
1966 | ModuleState::Disabled => false,
1967 }
1968 }
1969}
1970
1971#[cfg(test)]
1972#[derive(Debug, Default)]
1973struct ReloadExitRecordGate {
1974 reached: tokio::sync::Notify,
1975 resume: tokio::sync::Notify,
1976}
1977
1978#[derive(Debug, Clone, Copy)]
1979enum RespawnKind {
1980 Spawn,
1981 Reload,
1982}
1983
1984#[derive(Debug, Clone, Copy)]
1985struct PendingRespawn {
1986 deadline: Instant,
1987 kind: RespawnKind,
1988 daemon_initiated: bool,
1989}
1990
1991type ReloadReply = oneshot::Sender<Result<(), SuperviseError>>;
1992
1993#[derive(Debug, Clone)]
1994struct SupervisorRuntimeConfig {
1995 registry: Arc<Registry>,
1996 recovery_is_daemon: bool,
1998 start_requester: Option<Principal>,
1999 pending_activity_restart: bool,
2000 provenance_probe: ExecutableIdentityProbe,
2001 scheduled_respawn: Arc<Mutex<Option<PendingRespawn>>>,
2003 deferred_reload_reply: Arc<Mutex<Option<ReloadReply>>>,
2005 restart_policy: RestartPolicy,
2006 register_timeout: Duration,
2007 drain_timeout: Duration,
2010 effective_drain_timeout: Arc<Mutex<Duration>>,
2013 default_drain_timeout: Duration,
2016 health: HealthConfig,
2017 connection_file_path: Option<PathBuf>,
2018 capture_logs_dir: Option<PathBuf>,
2019 forwarding: Option<Arc<ForwardingTable>>,
2020 supervisor_handle: Option<SupervisorHandle>,
2023 stderr_ring: Arc<Mutex<StderrRing>>,
2030 terminal_ring: Arc<Mutex<TerminalRing>>,
2031 spawn_events: SpawnEventFeed,
2032 child_roster: ChildRoster,
2033 #[cfg(target_os = "linux")]
2034 cgroup_placement: Option<subc_cgroup::Placement>,
2035 #[cfg(test)]
2036 test_seed_stale_facts_before_enable_spawn: bool,
2037 #[cfg(test)]
2038 test_reload_exit_record_gate: Option<Arc<ReloadExitRecordGate>>,
2039}
2040
2041impl SupervisorRuntimeConfig {
2042 fn failure_context(&self, daemon_initiated: bool) -> FailureContext {
2043 FailureContext {
2044 daemon_initiated,
2045 shutdown: self.child_roster.shutdown_flag(),
2046 }
2047 }
2048}
2049
2050#[derive(Debug, Clone, PartialEq, Eq)]
2051struct SupervisedConfiguration {
2052 spec: ModuleSpec,
2053 health: HealthConfig,
2054}
2055
2056#[derive(Debug, Clone, Default)]
2062pub struct SupervisorHandle {
2063 restart_intent: Arc<Mutex<DaemonRestartIntent>>,
2064 activity: Arc<Mutex<Option<crate::activity::ActivityLedger>>>,
2065 activity_images: Arc<Mutex<HashMap<String, activity::ActivityImage>>>,
2066 modules: Arc<Mutex<HashMap<String, SupervisedModule>>>,
2067 configured_ids: Arc<Mutex<HashSet<String>>>,
2079 spawn_events: SpawnEventFeed,
2080 reserved_nonces: Arc<Mutex<HashMap<String, Option<String>>>>,
2091 removal_tombstones: Arc<Mutex<HashMap<String, u64>>>,
2097 spawn_nonces: Arc<Mutex<HashMap<String, String>>>,
2101 reserved_prefix_owners: Arc<Mutex<HashMap<String, String>>>,
2109 swaps: Arc<Mutex<HashMap<String, OpenSwap>>>,
2115 promotion_observer: PromotionObserverSlot,
2117 operation_lock: Arc<AsyncMutex<()>>,
2121}
2122
2123pub(crate) trait SwapPromotionObserver: Send + Sync {
2132 fn swap_promoted(&self, registration: &crate::registry::ModuleRegistration);
2133}
2134
2135#[derive(Clone, Default)]
2139struct PromotionObserverSlot(Arc<Mutex<Option<std::sync::Weak<dyn SwapPromotionObserver>>>>);
2140
2141impl fmt::Debug for PromotionObserverSlot {
2142 fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
2143 f.write_str("PromotionObserverSlot")
2144 }
2145}
2146
2147#[derive(Debug, Clone)]
2149struct OpenSwap {
2150 candidate_nonce: String,
2153 incumbent_nonce: Option<String>,
2158 candidate_admitted: bool,
2162}
2163
2164#[derive(Debug, Clone, Copy, PartialEq, Eq)]
2167pub(crate) enum SwapHelloAdmission {
2168 NotSwapping,
2171 Candidate,
2173 Refused,
2176}
2177
2178#[derive(Debug, Clone, PartialEq, Eq)]
2179pub(crate) enum ReservedHelloRejection {
2180 Exact {
2181 module_id: String,
2182 },
2183 Prefix {
2184 prefix: String,
2185 owner_module_id: String,
2186 },
2187}
2188
2189#[derive(Debug)]
2190struct DaemonRestartIntent {
2191 expires: Option<Duration>,
2192 window: Duration,
2193}
2194
2195impl Default for DaemonRestartIntent {
2196 fn default() -> Self {
2197 Self {
2198 expires: None,
2199 window: Duration::from_secs(120),
2200 }
2201 }
2202}
2203
2204impl SupervisorHandle {
2205 pub(crate) fn announce_daemon_restart(&self) -> u64 {
2206 let mut intent = self
2207 .restart_intent
2208 .lock()
2209 .unwrap_or_else(|p| p.into_inner());
2210 intent.expires = Some(subc_uptime::suspend_inclusive_now() + intent.window);
2211 unix_ms_now().saturating_add(intent.window.as_millis() as u64)
2212 }
2213
2214 #[cfg(feature = "test-support")]
2215 pub(crate) fn set_shutdown_intent_window(&self, window: Duration) {
2216 self.restart_intent
2217 .lock()
2218 .unwrap_or_else(|p| p.into_inner())
2219 .window = window;
2220 }
2221
2222 #[cfg(unix)]
2223 fn take_daemon_shutdown_reason(&self) -> RouteCloseReason {
2224 let expiry = self
2225 .restart_intent
2226 .lock()
2227 .unwrap_or_else(|p| p.into_inner())
2228 .expires
2229 .take();
2230 if expiry.is_some_and(|expiry| subc_uptime::suspend_inclusive_now() < expiry) {
2231 RouteCloseReason::Restart
2232 } else {
2233 RouteCloseReason::Disable
2234 }
2235 }
2236
2237 pub fn new() -> Self {
2238 Self::default()
2239 }
2240
2241 pub(crate) fn spawn_snapshot(&self) -> SpawnSnapshot {
2242 self.spawn_events.snapshot()
2243 }
2244
2245 pub(crate) fn subscribe_spawns(
2246 &self,
2247 connection_id: ConnectionId,
2248 corr: u64,
2249 version: u8,
2250 since: Option<SpawnCursor>,
2251 sink: FrameSink,
2252 ) -> Result<(), SpawnSubscribeRefusal> {
2253 self.spawn_events
2254 .subscribe(connection_id, corr, version, since, sink)
2255 }
2256
2257 pub(crate) fn cancel_spawn_subscription(&self, connection_id: ConnectionId, corr: u64) -> bool {
2258 self.spawn_events.cancel(connection_id, corr)
2259 }
2260
2261 pub(crate) fn remove_spawn_subscribers(&self, connection_id: ConnectionId) {
2262 self.spawn_events.remove_connection(connection_id);
2263 }
2264
2265 #[cfg(any(test, feature = "test-support"))]
2266 pub fn set_spawn_event_capacity_for_test(&self, capacity: usize) {
2267 assert!(capacity > 0, "spawn event capacity must be non-zero");
2268 self.spawn_events.set_capacity(capacity);
2269 }
2270
2271 #[cfg(any(test, feature = "test-support"))]
2272 pub fn spawn_subscriber_count_for_test(&self) -> usize {
2273 self.spawn_events.subscriber_count()
2274 }
2275
2276 pub fn set_spawn_nonce(&self, module_id: &str, nonce: String) {
2279 self.spawn_nonces
2280 .lock()
2281 .unwrap_or_else(|poisoned| poisoned.into_inner())
2282 .insert(module_id.to_string(), nonce);
2283 }
2284
2285 pub fn set_reserved_nonce(&self, module_id: &str, nonce: String) {
2288 self.reserved_nonces
2289 .lock()
2290 .unwrap_or_else(|poisoned| poisoned.into_inner())
2291 .insert(module_id.to_string(), Some(nonce));
2292 }
2293
2294 pub fn set_reserved_prefixes(&self, owner_module_id: &str, prefixes: &[String]) {
2296 let mut owners = self
2297 .reserved_prefix_owners
2298 .lock()
2299 .unwrap_or_else(|poisoned| poisoned.into_inner());
2300 owners.retain(|_, owner| owner != owner_module_id);
2301 for prefix in prefixes {
2302 owners.insert(prefix.clone(), owner_module_id.to_string());
2303 }
2304 }
2305
2306 #[cfg(test)]
2308 pub(crate) fn spawn_nonce(&self, module_id: &str) -> Option<String> {
2309 self.spawn_nonces
2310 .lock()
2311 .unwrap_or_else(|poisoned| poisoned.into_inner())
2312 .get(module_id)
2313 .cloned()
2314 }
2315
2316 fn apply_identity_configuration(&self, spec: &ModuleSpec) {
2317 self.set_reserved_prefixes(&spec.module_id, &spec.reserved_prefixes);
2318 let spawn_nonce = self
2319 .spawn_nonces
2320 .lock()
2321 .unwrap_or_else(|poisoned| poisoned.into_inner())
2322 .get(&spec.module_id)
2323 .cloned();
2324 let mut reserved_nonces = self
2325 .reserved_nonces
2326 .lock()
2327 .unwrap_or_else(|poisoned| poisoned.into_inner());
2328 if spec.reserved {
2329 reserved_nonces.insert(spec.module_id.clone(), spawn_nonce);
2334 }
2335 drop(reserved_nonces);
2336 self.removal_tombstones
2340 .lock()
2341 .unwrap_or_else(|poisoned| poisoned.into_inner())
2342 .remove(&spec.module_id);
2343 }
2344
2345 pub fn reserved_hello_authorized(&self, module_id: &str, presented: Option<&str>) -> bool {
2350 self.reserved_hello_verification(module_id, presented)
2351 .is_ok()
2352 }
2353
2354 pub(crate) fn reserved_hello_verification(
2357 &self,
2358 module_id: &str,
2359 presented: Option<&str>,
2360 ) -> Result<bool, ReservedHelloRejection> {
2361 let nonces = self
2362 .reserved_nonces
2363 .lock()
2364 .unwrap_or_else(|poisoned| poisoned.into_inner());
2365 if let Some(expected) = nonces.get(module_id) {
2366 let authorized = match expected {
2370 Some(expected) => {
2371 presented.is_some_and(|p| constant_time_eq(expected.as_bytes(), p.as_bytes()))
2372 }
2373 None => false,
2374 };
2375 if authorized {
2376 return Ok(true);
2377 }
2378 return Err(ReservedHelloRejection::Exact {
2379 module_id: module_id.to_string(),
2380 });
2381 }
2382 drop(nonces);
2383
2384 let matched_prefix = self
2385 .reserved_prefix_owners
2386 .lock()
2387 .unwrap_or_else(|poisoned| poisoned.into_inner())
2388 .iter()
2389 .filter(|(prefix, _)| module_id.starts_with(prefix.as_str()))
2390 .max_by_key(|(prefix, _)| prefix.len())
2391 .map(|(prefix, owner)| (prefix.clone(), owner.clone()));
2392 let Some((prefix, owner_module_id)) = matched_prefix else {
2393 return Ok(false);
2394 };
2395
2396 let authorized = presented.is_some_and(|presented| {
2397 self.spawn_nonces
2398 .lock()
2399 .unwrap_or_else(|poisoned| poisoned.into_inner())
2400 .get(&owner_module_id)
2401 .is_some_and(|expected| constant_time_eq(expected.as_bytes(), presented.as_bytes()))
2402 || self.swap_nonce_matches(&owner_module_id, presented)
2405 });
2406 if authorized {
2407 Ok(true)
2408 } else {
2409 Err(ReservedHelloRejection::Prefix {
2410 prefix,
2411 owner_module_id,
2412 })
2413 }
2414 }
2415
2416 pub fn spawned_consumer_authorized(&self, module_id: &str, presented: &str) -> bool {
2421 if presented.is_empty() {
2422 return false;
2423 }
2424 let nonces = self
2425 .spawn_nonces
2426 .lock()
2427 .unwrap_or_else(|poisoned| poisoned.into_inner());
2428 let current = nonces
2429 .get(module_id)
2430 .is_some_and(|expected| constant_time_eq(expected.as_bytes(), presented.as_bytes()));
2431 drop(nonces);
2432 current || self.swap_nonce_matches(module_id, presented)
2437 }
2438
2439 fn swap_nonce_matches(&self, module_id: &str, presented: &str) -> bool {
2441 let swaps = self
2442 .swaps
2443 .lock()
2444 .unwrap_or_else(|poisoned| poisoned.into_inner());
2445 swaps.get(module_id).is_some_and(|swap| {
2446 constant_time_eq(swap.candidate_nonce.as_bytes(), presented.as_bytes())
2447 || swap.incumbent_nonce.as_deref().is_some_and(|incumbent| {
2448 constant_time_eq(incumbent.as_bytes(), presented.as_bytes())
2449 })
2450 })
2451 }
2452
2453 pub(crate) fn open_swap(&self, module_id: &str, candidate_nonce: String) {
2456 let incumbent_nonce = self
2457 .spawn_nonces
2458 .lock()
2459 .unwrap_or_else(|poisoned| poisoned.into_inner())
2460 .get(module_id)
2461 .cloned();
2462 self.swaps
2463 .lock()
2464 .unwrap_or_else(|poisoned| poisoned.into_inner())
2465 .insert(
2466 module_id.to_string(),
2467 OpenSwap {
2468 candidate_nonce,
2469 incumbent_nonce,
2470 candidate_admitted: false,
2471 },
2472 );
2473 }
2474
2475 pub(crate) fn close_swap(&self, module_id: &str) {
2478 self.swaps
2479 .lock()
2480 .unwrap_or_else(|poisoned| poisoned.into_inner())
2481 .remove(module_id);
2482 }
2483
2484 pub(crate) fn set_swap_promotion_observer(
2487 &self,
2488 observer: std::sync::Weak<dyn SwapPromotionObserver>,
2489 ) {
2490 *self
2491 .promotion_observer
2492 .0
2493 .lock()
2494 .unwrap_or_else(|poisoned| poisoned.into_inner()) = Some(observer);
2495 }
2496
2497 fn notify_swap_promoted(&self, registration: &crate::registry::ModuleRegistration) {
2500 let observer = self
2501 .promotion_observer
2502 .0
2503 .lock()
2504 .unwrap_or_else(|poisoned| poisoned.into_inner())
2505 .as_ref()
2506 .and_then(std::sync::Weak::upgrade);
2507 if let Some(observer) = observer {
2508 observer.swap_promoted(registration);
2509 }
2510 }
2511
2512 pub(crate) fn swap_open(&self, module_id: &str) -> bool {
2514 self.swaps
2515 .lock()
2516 .unwrap_or_else(|poisoned| poisoned.into_inner())
2517 .contains_key(module_id)
2518 }
2519
2520 fn promote_swap_nonce(&self, module_id: &str, reserved: bool) {
2525 let candidate_nonce = self
2526 .swaps
2527 .lock()
2528 .unwrap_or_else(|poisoned| poisoned.into_inner())
2529 .get(module_id)
2530 .map(|swap| swap.candidate_nonce.clone());
2531 let Some(nonce) = candidate_nonce else {
2532 return;
2533 };
2534 self.set_spawn_nonce(module_id, nonce.clone());
2535 if reserved {
2536 self.set_reserved_nonce(module_id, nonce);
2537 }
2538 }
2539
2540 pub(crate) fn swap_hello_admission(
2555 &self,
2556 module_id: &str,
2557 presented: Option<&str>,
2558 ) -> SwapHelloAdmission {
2559 let swaps = self
2560 .swaps
2561 .lock()
2562 .unwrap_or_else(|poisoned| poisoned.into_inner());
2563 let Some(swap) = swaps.get(module_id) else {
2564 return SwapHelloAdmission::NotSwapping;
2565 };
2566 let Some(presented) = presented else {
2567 return SwapHelloAdmission::Refused;
2568 };
2569 if constant_time_eq(swap.candidate_nonce.as_bytes(), presented.as_bytes()) {
2570 return if swap.candidate_admitted {
2571 SwapHelloAdmission::Refused
2572 } else {
2573 SwapHelloAdmission::Candidate
2574 };
2575 }
2576 if swap
2577 .incumbent_nonce
2578 .as_deref()
2579 .is_some_and(|incumbent| constant_time_eq(incumbent.as_bytes(), presented.as_bytes()))
2580 {
2581 return SwapHelloAdmission::NotSwapping;
2582 }
2583 SwapHelloAdmission::Refused
2584 }
2585
2586 pub(crate) fn mark_swap_candidate_admitted(&self, module_id: &str) {
2589 if let Some(swap) = self
2590 .swaps
2591 .lock()
2592 .unwrap_or_else(|poisoned| poisoned.into_inner())
2593 .get_mut(module_id)
2594 {
2595 swap.candidate_admitted = true;
2596 }
2597 }
2598
2599 pub fn spawn_launch_nonce_for(&self, module_id: &str) -> Option<String> {
2601 self.spawn_nonces
2602 .lock()
2603 .unwrap_or_else(|poisoned| poisoned.into_inner())
2604 .get(module_id)
2605 .cloned()
2606 }
2607
2608 pub fn reserved_launch_nonce_for(&self, module_id: &str) -> Option<String> {
2610 self.reserved_nonces
2611 .lock()
2612 .unwrap_or_else(|poisoned| poisoned.into_inner())
2613 .get(module_id)
2614 .cloned()
2615 .flatten()
2616 }
2617
2618 pub fn insert(&self, module: SupervisedModule) -> Option<SupervisedModule> {
2619 self.mark_configured(module.module_id());
2623 let mut modules = self
2624 .modules
2625 .lock()
2626 .unwrap_or_else(|poisoned| poisoned.into_inner());
2627 modules.insert(module.module_id().to_string(), module)
2628 }
2629
2630 fn mark_configured(&self, module_id: &str) {
2634 self.configured_ids
2635 .lock()
2636 .unwrap_or_else(|poisoned| poisoned.into_inner())
2637 .insert(module_id.to_string());
2638 }
2639
2640 fn unmark_configured_unless_rostered(&self, module_id: &str) {
2644 let modules = self
2645 .modules
2646 .lock()
2647 .unwrap_or_else(|poisoned| poisoned.into_inner());
2648 if !modules.contains_key(module_id) {
2649 self.configured_ids
2650 .lock()
2651 .unwrap_or_else(|poisoned| poisoned.into_inner())
2652 .remove(module_id);
2653 }
2654 }
2655
2656 pub(crate) fn is_configured(&self, module_id: &str) -> bool {
2664 self.configured_ids
2665 .lock()
2666 .unwrap_or_else(|poisoned| poisoned.into_inner())
2667 .contains(module_id)
2668 }
2669
2670 pub fn get(&self, module_id: &str) -> Option<SupervisedModule> {
2671 let modules = self
2672 .modules
2673 .lock()
2674 .unwrap_or_else(|poisoned| poisoned.into_inner());
2675 modules.get(module_id).cloned()
2676 }
2677
2678 pub(crate) fn record_late_health_answer(
2679 &self,
2680 module_id: &str,
2681 latency_ms: u64,
2682 ) -> Result<bool, SuperviseError> {
2683 let Some(module) = self.get(module_id) else {
2684 return Ok(false);
2685 };
2686 update_snapshot(&module.inner.snapshot, Some(module_id), |state| {
2687 state.health.late_answer_count = state.health.late_answer_count.saturating_add(1);
2688 state.health.last_late_answer_latency_ms = Some(latency_ms);
2689 state.health.consecutive_failures = 0;
2697 })?;
2698 Ok(true)
2699 }
2700
2701 pub fn record_deliberate_severance(&self, module_id: &str) -> Result<bool, SuperviseError> {
2707 let Some(module) = self.get(module_id) else {
2708 return Ok(false);
2709 };
2710 let snapshot = lock_snapshot(&module.inner.snapshot)?;
2711 let Some((pid, start_time)) = snapshot.pid.zip(snapshot.process_start_time) else {
2712 return Ok(false);
2713 };
2714 drop(snapshot);
2715 module.record_deliberate_severance(ProcessIdentity { pid, start_time })
2716 }
2717
2718 pub fn list(&self) -> Vec<SupervisedModule> {
2719 let modules = self
2720 .modules
2721 .lock()
2722 .unwrap_or_else(|poisoned| poisoned.into_inner());
2723 let mut modules = modules.values().cloned().collect::<Vec<_>>();
2724 modules.sort_by(|left, right| left.module_id().cmp(right.module_id()));
2725 modules
2726 }
2727
2728 pub(crate) fn retire(&self, module_id: &str) -> Option<SupervisedModule> {
2729 self.activity_images
2730 .lock()
2731 .unwrap_or_else(|p| p.into_inner())
2732 .remove(module_id);
2733 self.spawn_nonces
2734 .lock()
2735 .unwrap_or_else(|poisoned| poisoned.into_inner())
2736 .remove(module_id);
2737 self.close_swap(module_id);
2738 let mut reserved_nonces = self
2739 .reserved_nonces
2740 .lock()
2741 .unwrap_or_else(|poisoned| poisoned.into_inner());
2742 if reserved_nonces.contains_key(module_id) {
2743 reserved_nonces.insert(module_id.to_string(), None);
2746 }
2747 drop(reserved_nonces);
2748 self.reserved_prefix_owners
2749 .lock()
2750 .unwrap_or_else(|poisoned| poisoned.into_inner())
2751 .retain(|_, owner| owner != module_id);
2752 let removed = self
2753 .modules
2754 .lock()
2755 .unwrap_or_else(|poisoned| poisoned.into_inner())
2756 .remove(module_id);
2757 self.configured_ids
2758 .lock()
2759 .unwrap_or_else(|poisoned| poisoned.into_inner())
2760 .remove(module_id);
2761 removed
2762 }
2763
2764 pub(crate) fn record_rescan_removal(&self, module_id: &str) {
2767 self.removal_tombstones
2768 .lock()
2769 .unwrap_or_else(|poisoned| poisoned.into_inner())
2770 .insert(module_id.to_string(), unix_ms_now());
2771 }
2772
2773 pub(crate) fn removal_tombstone_age_ms(&self, module_id: &str) -> Option<u64> {
2775 self.removal_tombstones
2776 .lock()
2777 .unwrap_or_else(|poisoned| poisoned.into_inner())
2778 .get(module_id)
2779 .copied()
2780 .map(|removed_at_ms| unix_ms_now().saturating_sub(removed_at_ms))
2781 }
2782
2783 pub(crate) fn release_retained_reserved_gate(&self, module_id: &str) -> bool {
2788 if self.get(module_id).is_some() {
2789 return false;
2790 }
2791 let mut reserved_nonces = self
2792 .reserved_nonces
2793 .lock()
2794 .unwrap_or_else(|poisoned| poisoned.into_inner());
2795 if !matches!(reserved_nonces.get(module_id), Some(None)) {
2796 return false;
2797 }
2798 reserved_nonces.remove(module_id);
2799 true
2800 }
2801
2802 pub(crate) fn operation_lock(&self) -> Arc<AsyncMutex<()>> {
2803 Arc::clone(&self.operation_lock)
2804 }
2805}
2806
2807#[derive(Debug, Clone)]
2809pub struct Supervisor {
2810 registry: Arc<Registry>,
2811 restart_policy: RestartPolicy,
2812 drain_timeout: Duration,
2813 connection_file_path: Option<PathBuf>,
2814 capture_logs_dir: Option<PathBuf>,
2815 forwarding: Option<Arc<ForwardingTable>>,
2816 process_liveness: Arc<SupervisorProcessLiveness>,
2817 supervisor_handle: Option<SupervisorHandle>,
2818 health: HealthConfig,
2819 daemon_start_clock: crate::clock::StartClock,
2820 terminal_journal: Option<Arc<crate::terminal_journal::TerminalJournal>>,
2821 spawn_events: SpawnEventFeed,
2822 provenance_probe: ExecutableIdentityProbe,
2823 child_roster: ChildRoster,
2826 #[cfg(target_os = "linux")]
2827 cgroup_placement: Option<subc_cgroup::Placement>,
2828 #[cfg(test)]
2829 test_after_first_spawn: AfterFirstSpawnHook,
2830}
2831
2832pub(crate) struct RegisteredConfiguredModule {
2836 spec: ModuleSpec,
2837 runtime: SupervisorRuntimeConfig,
2838 module: SupervisedModule,
2839 commands: mpsc::Receiver<SupervisorCommand>,
2840}
2841
2842#[cfg(test)]
2848#[derive(Clone, Default)]
2849struct AfterFirstSpawnHook(Option<AfterFirstSpawnFn>);
2850
2851#[cfg(test)]
2852type AfterFirstSpawnFn = Arc<dyn Fn(&str) + Send + Sync>;
2853
2854#[cfg(test)]
2855impl fmt::Debug for AfterFirstSpawnHook {
2856 fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
2857 f.write_str("AfterFirstSpawnHook")
2858 }
2859}
2860
2861#[cfg(test)]
2862impl AfterFirstSpawnHook {
2863 fn run(&self, module_id: &str) {
2864 if let Some(hook) = &self.0 {
2865 hook(module_id);
2866 }
2867 }
2868}
2869
2870impl Supervisor {
2871 #[cfg(test)]
2872 pub(crate) fn new_for_test(registry: Arc<Registry>, policy: RestartPolicy) -> Self {
2873 let supervisor = Self::new(registry, policy);
2874 #[cfg(target_os = "macos")]
2875 let supervisor = supervisor.with_privacy_trampoline(test_privacy_trampoline());
2876 supervisor
2877 }
2878 pub fn with_privacy_trampoline(self, path: impl Into<PathBuf>) -> Self {
2883 let path = path.into();
2884 #[cfg(target_os = "macos")]
2885 {
2886 let result = probe_privacy_trampoline(&path).map(|()| path);
2887 if let Err(cause) = &result {
2888 error!(%cause, "macOS privacy identity unavailable; supervised launches will refuse");
2889 }
2890 self.child_roster.set_privacy_trampoline(result);
2891 }
2892 #[cfg(not(target_os = "macos"))]
2893 let _ = path;
2894 self
2895 }
2896 pub(crate) fn daemon_shutdown_flag(&self) -> crate::child_roster::DaemonShutdownFlag {
2899 self.child_roster.shutdown_flag()
2900 }
2901
2902 #[cfg(unix)]
2913 pub(crate) fn begin_daemon_shutdown(&self) {
2914 self.child_roster.close();
2915 if let Some(journal) = &self.terminal_journal {
2916 journal.stamp_shutdown();
2917 }
2918 }
2919
2920 #[cfg(unix)]
2925 pub(crate) async fn drain_for_daemon_shutdown(&self) -> Result<(), SuperviseError> {
2926 const NOTICE_BUDGET: Duration = Duration::from_millis(500);
2927 const DRAIN_BUDGET: Duration = Duration::from_secs(2);
2928 let Some(forwarding) = &self.forwarding else {
2929 return Ok(());
2930 };
2931 let notice_started = Instant::now();
2932 let modules = forwarding
2933 .begin_daemon_drain()
2934 .map_err(SuperviseError::Forwarding)?;
2935 let deadline_ms =
2936 unix_ms_now().saturating_add((NOTICE_BUDGET + DRAIN_BUDGET).as_millis() as u64);
2937 let shutdown_reason = self
2938 .supervisor_handle
2939 .as_ref()
2940 .map(SupervisorHandle::take_daemon_shutdown_reason)
2941 .unwrap_or(RouteCloseReason::Disable);
2942 let mut modules_told = 0;
2943 let mut notices = tokio::task::JoinSet::new();
2944 let mut drains = Vec::new();
2945 for connection in forwarding
2946 .module_connections()
2947 .map_err(SuperviseError::Forwarding)?
2948 {
2949 let target = ModuleDrainTarget {
2950 endpoint: connection.endpoint,
2951 sink: connection.sink,
2952 negotiated_ver: connection.negotiated_ver,
2953 abandoned_bindings: Vec::new(),
2954 excluded_subscriptions: 0,
2955 };
2956 notices.spawn(async move {
2957 (
2958 true,
2959 send_module_draining(
2960 &connection.module_id,
2961 shutdown_reason,
2962 deadline_ms,
2963 &target,
2964 true,
2965 )
2966 .await,
2967 )
2968 });
2969 }
2970 for (module_id, endpoint) in modules {
2971 let Some(target) = forwarding
2972 .begin_endpoint_drain(endpoint, RouteCloseReason::Restart)
2973 .map_err(SuperviseError::Forwarding)?
2974 else {
2975 continue;
2976 };
2977 let routes = forwarding
2978 .endpoint_routes(target.endpoint)
2979 .map_err(SuperviseError::Forwarding)?;
2980 let mut recipients = Vec::new();
2983 let mut clients: Vec<(crate::forwarding::GoodbyeTarget, Vec<u16>)> = Vec::new();
2984 for route in routes {
2985 let client = route.goodbye_target;
2986 if let Some((_, channels)) = clients
2987 .iter_mut()
2988 .find(|(existing, _)| existing.connection_id == client.connection_id)
2989 {
2990 channels.push(client.channel);
2991 } else {
2992 let channel = client.channel;
2993 clients.push((client, vec![channel]));
2994 }
2995 }
2996 for (client, mut channels) in clients {
2997 channels.sort_unstable();
2998 channels.dedup();
2999 let closing = serde_json::to_vec(&ClientControlPush::RouteClosing {
3000 module_id: module_id.clone(),
3001 channels,
3002 reason: RouteCloseReason::Restart,
3003 })
3004 .expect("route closing serializes");
3005 recipients.push((client.sink, client.negotiated_ver, closing));
3006 }
3007 for (sink, version, body) in recipients {
3008 notices.spawn(async move {
3009 let frame = Frame::build_with_version(
3010 version,
3011 FrameType::Push,
3012 control_flags(),
3013 0,
3014 0,
3015 0,
3016 body,
3017 )
3018 .expect("bounded lifecycle notice frame builds");
3019 (false, sink.send_flushed(frame).await.is_ok())
3020 });
3021 }
3022 let gauges = declared_busy_gauges(&self.registry, &module_id)?;
3023 drains.push((module_id, target.endpoint, gauges));
3024 }
3025 let notice_deadline = Instant::now() + NOTICE_BUDGET;
3028 while let Ok(Some(result)) = timeout_at(notice_deadline, notices.join_next()).await {
3029 match result {
3030 Ok((true, true)) => modules_told += 1,
3031 Ok((false, true)) => {}
3032 result => warn!(?result, "daemon shutdown notice delivery failed"),
3033 }
3034 }
3035 notices.abort_all();
3036 info!(
3037 ?shutdown_reason,
3038 modules_told, "daemon shutdown module notices delivered"
3039 );
3040 let notice_elapsed = notice_started.elapsed();
3041 let drain_started = Instant::now();
3042 let deadline = drain_started + DRAIN_BUDGET;
3043 let mut waits = tokio::task::JoinSet::new();
3044 for (module_id, endpoint, gauges) in drains {
3045 let forwarding = Arc::clone(forwarding);
3046 let mut runtime = self.runtime_config();
3047 runtime.health.cadence = Duration::from_millis(100);
3048 waits.spawn(async move {
3049 let wait_result = wait_for_forwarding_quiescence(
3050 &forwarding,
3051 &module_id,
3052 &runtime,
3053 endpoint,
3054 deadline,
3055 &gauges,
3056 DrainScope::Endpoint(endpoint),
3057 &mut DrainGaugeTrace::new(&gauges),
3058 )
3059 .await;
3060 forwarding
3064 .record_daemon_drain_result(
3065 endpoint,
3066 drained_after_quiescence_wait(&wait_result),
3067 )
3068 .map_err(SuperviseError::Forwarding)?;
3069 wait_result
3070 });
3071 }
3072 while let Ok(Some(result)) = timeout_at(deadline, waits.join_next()).await {
3073 if !matches!(result, Ok(Ok(true))) {
3074 warn!(?result, "daemon shutdown drain did not reach quiescence");
3075 }
3076 }
3077 info!(
3080 notice_elapsed_ms = notice_elapsed.as_millis() as u64,
3081 drain_elapsed_ms = drain_started.elapsed().as_millis() as u64,
3082 "daemon shutdown forwarding drain complete"
3083 );
3084 Ok(())
3085 }
3086
3087 #[cfg(unix)]
3099 pub(crate) async fn end_children_for_daemon_shutdown(
3100 &self,
3101 already_escalated: bool,
3102 escalate: impl std::future::Future<Output = ()>,
3103 ) {
3104 tokio::pin!(escalate);
3105 let mut escalated = already_escalated;
3106 if let Some(forwarding) = &self.forwarding {
3107 let reason = CloseReason::new(
3108 "daemon_shutdown",
3109 "the daemon is exiting after its shutdown notice and drain",
3110 );
3111 if escalated {
3112 send_module_goodbyes_for_daemon_shutdown(forwarding, &reason, false).await;
3115 } else {
3116 tokio::select! {
3117 biased;
3118 _ = escalate.as_mut() => {
3119 info!("second SIGTERM: abandoning module GOODBYE delivery");
3120 escalated = true;
3121 }
3122 _ = send_module_goodbyes_for_daemon_shutdown(forwarding, &reason, true) => {}
3123 }
3124 }
3125 let closed = forwarding.close_all_connections(&reason);
3126 debug!(closed, "closed established connections for daemon shutdown");
3127 }
3128 let escalated_here = escalated && !already_escalated;
3132 let remaining_escalate = async move {
3133 if escalated_here {
3134 std::future::pending::<()>().await;
3135 } else {
3136 escalate.await;
3137 }
3138 };
3139 crate::child_roster::end_children_for_daemon_shutdown(
3140 &self.child_roster,
3141 escalated,
3142 remaining_escalate,
3143 )
3144 .await;
3145 }
3146
3147 pub fn new(registry: Arc<Registry>, restart_policy: RestartPolicy) -> Self {
3148 Self {
3149 registry,
3150 restart_policy,
3151 drain_timeout: DEFAULT_DRAIN_TIMEOUT,
3152 connection_file_path: None,
3153 capture_logs_dir: None,
3154 forwarding: None,
3155 process_liveness: Arc::new(SupervisorProcessLiveness::default()),
3156 supervisor_handle: None,
3157 health: HealthConfig::default(),
3158 daemon_start_clock: crate::clock::StartClock::capture(),
3159 terminal_journal: None,
3160 spawn_events: SpawnEventFeed::default(),
3161 provenance_probe: ExecutableIdentityProbe::default(),
3162 child_roster: ChildRoster::default(),
3163 #[cfg(target_os = "linux")]
3164 cgroup_placement: None,
3165 #[cfg(test)]
3166 test_after_first_spawn: AfterFirstSpawnHook::default(),
3167 }
3168 }
3169
3170 pub fn with_drain_timeout(mut self, drain_timeout: Duration) -> Self {
3171 self.drain_timeout = drain_timeout;
3172 self
3173 }
3174
3175 pub(crate) fn default_drain_timeout(&self) -> Duration {
3177 self.drain_timeout
3178 }
3179
3180 pub fn with_process_liveness(
3181 mut self,
3182 process_liveness: Arc<SupervisorProcessLiveness>,
3183 ) -> Self {
3184 self.process_liveness = process_liveness;
3185 self
3186 }
3187
3188 pub fn with_connection_file_path(mut self, connection_file_path: impl Into<PathBuf>) -> Self {
3189 self.connection_file_path = Some(connection_file_path.into());
3190 self
3191 }
3192
3193 pub fn with_capture_logs_dir(mut self, logs_dir: impl Into<PathBuf>) -> Self {
3195 self.capture_logs_dir = Some(logs_dir.into());
3196 self
3197 }
3198
3199 pub fn with_daemon_incarnation(self, daemon_incarnation: String) -> Self {
3202 self.spawn_events.configure_incarnation(daemon_incarnation);
3206 self
3207 }
3208
3209 pub fn with_terminal_journal(self, path: PathBuf, daemon_incarnation: String) -> Self {
3212 let mut this = self.with_daemon_incarnation(daemon_incarnation.clone());
3213 this.terminal_journal = Some(Arc::new(crate::terminal_journal::TerminalJournal::open(
3214 path,
3215 daemon_incarnation,
3216 )));
3217 this
3218 }
3219
3220 pub fn with_forwarding(mut self, forwarding: Arc<ForwardingTable>) -> Self {
3221 self.child_roster.counters = forwarding.counters();
3222 self.forwarding = Some(forwarding);
3223 self
3224 }
3225
3226 pub fn with_handle(mut self, supervisor_handle: SupervisorHandle) -> Self {
3227 self.spawn_events = supervisor_handle.spawn_events.clone();
3228 self.supervisor_handle = Some(supervisor_handle);
3229 self
3230 }
3231
3232 pub fn with_health_config(mut self, health: HealthConfig) -> Self {
3233 self.health = health;
3234 self
3235 }
3236
3237 pub fn with_live_children_record(self, path: impl Into<PathBuf>) -> Self {
3241 self.child_roster.record_to(path.into());
3242 self
3243 }
3244
3245 #[cfg(target_os = "linux")]
3246 pub fn with_cgroup_placement(
3247 mut self,
3248 cgroup_placement: Option<subc_cgroup::Placement>,
3249 ) -> Self {
3250 self.child_roster
3251 .set_cgroup_placement(cgroup_placement.clone());
3252 self.cgroup_placement = cgroup_placement;
3253 self
3254 }
3255
3256 pub fn spawn(&self, spec: ModuleSpec) -> Result<SupervisedModule, SuperviseError> {
3262 validate_spec(&spec)?;
3263 self.establish_identity(&spec);
3264
3265 let runtime = self.runtime_config();
3266 let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::starting()));
3267 let activity_spawn = runtime.begin_activity_spawn(&spec.module_id);
3268 let registration_watch = watch_registration(&self.registry, &spec);
3269 let spawned = spawn_child(
3270 &spec,
3271 runtime.connection_file_path.as_deref(),
3272 self.supervisor_handle.as_ref(),
3273 &runtime.stderr_ring,
3274 runtime.capture_logs_dir.as_deref(),
3275 &runtime.child_roster,
3276 #[cfg(target_os = "linux")]
3277 runtime.cgroup_placement.as_ref(),
3278 );
3279 #[cfg(test)]
3280 self.test_after_first_spawn.run(&spec.module_id);
3281 let mut child = match spawned {
3282 Ok(child) => child,
3283 Err(err) => {
3284 self.abandon_unrostered(&spec.module_id);
3287 return Err(err);
3288 }
3289 };
3290 arm_registration_deadline(&mut child, runtime.register_timeout, registration_watch);
3291 self.mark_first_process_running(&spec, &runtime, &snapshot, &mut child)?;
3292 if let Some(spawn) = activity_spawn {
3293 spawn.complete(&snapshot);
3294 }
3295
3296 Ok(self.supervised_module(spec, runtime, snapshot, Some(child)))
3297 }
3298
3299 fn establish_identity(&self, spec: &ModuleSpec) {
3312 if let Some(supervisor_handle) = &self.supervisor_handle {
3313 supervisor_handle.apply_identity_configuration(spec);
3314 supervisor_handle.mark_configured(&spec.module_id);
3315 }
3316 }
3317
3318 fn abandon_unrostered(&self, module_id: &str) {
3321 if let Some(supervisor_handle) = &self.supervisor_handle {
3322 supervisor_handle.unmark_configured_unless_rostered(module_id);
3323 }
3324 }
3325
3326 fn mark_first_process_running(
3329 &self,
3330 spec: &ModuleSpec,
3331 runtime: &SupervisorRuntimeConfig,
3332 snapshot: &SharedSnapshot,
3333 child: &mut SupervisedChild,
3334 ) -> Result<(), SuperviseError> {
3335 if let Err(err) = set_running(snapshot, child, &spec.module_id, &runtime.spawn_events) {
3336 containment::terminate_blocking(&runtime.child_roster.containment);
3337 self.abandon_unrostered(&spec.module_id);
3338 return Err(err);
3339 }
3340 self.process_liveness
3341 .track(spec.module_id.clone(), Arc::clone(snapshot));
3342 Ok(())
3343 }
3344
3345 pub fn supervise_configured(
3351 &self,
3352 spec: ModuleSpec,
3353 enabled: bool,
3354 ) -> Result<SupervisedModule, SuperviseError> {
3355 self.supervise_configured_requested(
3356 spec,
3357 enabled,
3358 self.health.clone(),
3359 None,
3360 self.restart_policy,
3361 DEFAULT_REGISTER_TIMEOUT,
3362 None,
3363 )
3364 }
3365
3366 pub fn supervise_configured_with_health(
3372 &self,
3373 spec: ModuleSpec,
3374 enabled: bool,
3375 health: HealthConfig,
3376 drain_timeout_ms: Option<u64>,
3377 restart_policy: RestartPolicy,
3378 ) -> Result<SupervisedModule, SuperviseError> {
3379 self.supervise_configured_requested(
3380 spec,
3381 enabled,
3382 health,
3383 drain_timeout_ms,
3384 restart_policy,
3385 DEFAULT_REGISTER_TIMEOUT,
3386 None,
3387 )
3388 }
3389
3390 #[allow(clippy::too_many_arguments)]
3391 pub(crate) fn supervise_configured_requested(
3392 &self,
3393 spec: ModuleSpec,
3394 enabled: bool,
3395 health: HealthConfig,
3396 drain_timeout_ms: Option<u64>,
3397 restart_policy: RestartPolicy,
3398 register_timeout: Duration,
3399 requester: Option<Principal>,
3400 ) -> Result<SupervisedModule, SuperviseError> {
3401 let registered = self.register_configured_with_health(
3402 spec,
3403 enabled,
3404 health,
3405 drain_timeout_ms,
3406 restart_policy,
3407 register_timeout,
3408 )?;
3409 let daemon_initiated = requester.is_none();
3410 self.start_registered_configured(registered, requester, daemon_initiated)
3411 }
3412
3413 pub(crate) fn register_configured_with_health(
3418 &self,
3419 spec: ModuleSpec,
3420 enabled: bool,
3421 health: HealthConfig,
3422 drain_timeout_ms: Option<u64>,
3423 restart_policy: RestartPolicy,
3424 register_timeout: Duration,
3425 ) -> Result<RegisteredConfiguredModule, SuperviseError> {
3426 validate_spec(&spec)?;
3427 self.establish_identity(&spec);
3428
3429 let mut runtime = self.runtime_config();
3430 runtime.health = health;
3431 runtime.restart_policy = restart_policy;
3432 runtime.register_timeout = register_timeout;
3433 if let Some(ms) = drain_timeout_ms {
3434 runtime.drain_timeout = Duration::from_millis(ms);
3435 *runtime
3436 .effective_drain_timeout
3437 .lock()
3438 .unwrap_or_else(|poisoned| poisoned.into_inner()) = runtime.drain_timeout;
3439 }
3440 let snapshot = if enabled {
3441 SupervisorSnapshot::starting()
3442 } else {
3443 SupervisorSnapshot::disabled()
3444 };
3445 let snapshot = {
3446 let mut snapshot = snapshot;
3447 if enabled {
3448 if let Some((pid, reason)) = runtime.child_roster.previous_instance(&spec.module_id)
3449 {
3450 snapshot.waiting_for_previous_pid = Some(pid);
3451 snapshot.waiting_for_previous_detail = Some(reason.waiting_message(pid));
3452 }
3453 }
3454 snapshot
3455 };
3456 let (module, commands) =
3457 self.register_module(&spec, &runtime, Arc::new(Mutex::new(snapshot)));
3458 Ok(RegisteredConfiguredModule {
3459 spec,
3460 runtime,
3461 module,
3462 commands,
3463 })
3464 }
3465
3466 pub(crate) fn start_registered_configured(
3469 &self,
3470 registered: RegisteredConfiguredModule,
3471 requester: Option<Principal>,
3472 daemon_initiated: bool,
3473 ) -> Result<SupervisedModule, SuperviseError> {
3474 let RegisteredConfiguredModule {
3475 spec,
3476 mut runtime,
3477 module,
3478 commands,
3479 } = registered;
3480 runtime.start_requester = requester;
3481 runtime.recovery_is_daemon = daemon_initiated;
3482 let snapshot = &module.inner.snapshot;
3483 let should_spawn = {
3484 let state = lock_snapshot(snapshot)?;
3485 state.enabled && state.waiting_for_previous_pid.is_none()
3486 };
3487 let child = if should_spawn {
3488 let activity_spawn = runtime.begin_activity_spawn(&spec.module_id);
3489 let registration_watch = watch_registration(&self.registry, &spec);
3490 let spawned = spawn_child(
3491 &spec,
3492 runtime.connection_file_path.as_deref(),
3493 self.supervisor_handle.as_ref(),
3494 &runtime.stderr_ring,
3495 runtime.capture_logs_dir.as_deref(),
3496 &runtime.child_roster,
3497 #[cfg(target_os = "linux")]
3498 runtime.cgroup_placement.as_ref(),
3499 );
3500 #[cfg(test)]
3501 self.test_after_first_spawn.run(&spec.module_id);
3502 match spawned {
3503 Ok(mut child) => {
3504 arm_registration_deadline(
3505 &mut child,
3506 runtime.register_timeout,
3507 registration_watch,
3508 );
3509 self.mark_first_process_running(&spec, &runtime, snapshot, &mut child)?;
3510 if let Some(spawn) = activity_spawn {
3511 spawn.complete(snapshot);
3512 }
3513 Some(child)
3514 }
3515 Err(err) => {
3516 if runtime.health.critical {
3517 error!(
3518 module_id = %spec.module_id,
3519 program = %spec.program.display(),
3520 error = %err,
3521 "critical configured module failed to spawn; marking failed and alerting"
3522 );
3523 } else {
3524 error!(
3525 module_id = %spec.module_id,
3526 program = %spec.program.display(),
3527 error = %err,
3528 "configured module failed to spawn; marking failed and continuing"
3529 );
3530 }
3531 fail_snapshot(
3532 snapshot,
3533 Some(&spec.module_id),
3534 None,
3535 &runtime.failure_context(daemon_initiated),
3536 );
3537 None
3538 }
3539 }
3540 } else {
3541 None
3542 };
3543 self.start_monitor(spec, runtime, &module, child, commands);
3544 Ok(module)
3545 }
3546
3547 fn runtime_config(&self) -> SupervisorRuntimeConfig {
3548 let effective_drain_timeout = Arc::new(Mutex::new(self.drain_timeout));
3549 SupervisorRuntimeConfig {
3550 registry: self.registry.clone(),
3551 recovery_is_daemon: true,
3552 start_requester: None,
3553 pending_activity_restart: false,
3554 provenance_probe: self.provenance_probe.clone(),
3555 scheduled_respawn: Arc::default(),
3556 deferred_reload_reply: Arc::default(),
3557 restart_policy: self.restart_policy,
3558 register_timeout: DEFAULT_REGISTER_TIMEOUT,
3559 drain_timeout: self.drain_timeout,
3560 child_roster: self
3563 .child_roster
3564 .for_module(Arc::clone(&effective_drain_timeout)),
3565 effective_drain_timeout,
3566 default_drain_timeout: self.drain_timeout,
3567 health: self.health.clone(),
3568 connection_file_path: self.connection_file_path.clone(),
3569 capture_logs_dir: self.capture_logs_dir.clone(),
3570 forwarding: self.forwarding.clone(),
3571 supervisor_handle: self.supervisor_handle.clone(),
3572 stderr_ring: Arc::new(Mutex::new(StderrRing::new(StderrTailConfig::default()))),
3573 terminal_ring: Arc::new(Mutex::new(
3574 TerminalRing::new(
3575 TerminalRingConfig::default(),
3576 self.daemon_start_clock.started_at_ms(),
3577 )
3578 .with_start_clock(self.daemon_start_clock)
3579 .with_journal(self.terminal_journal.clone())
3580 .with_daemon_shutdown(self.child_roster.shutdown_flag()),
3581 )),
3582 spawn_events: self.spawn_events.clone(),
3583 #[cfg(target_os = "linux")]
3584 cgroup_placement: self.cgroup_placement.clone(),
3585 #[cfg(test)]
3586 test_seed_stale_facts_before_enable_spawn: false,
3587 #[cfg(test)]
3588 test_reload_exit_record_gate: None,
3589 }
3590 }
3591
3592 fn supervised_module(
3593 &self,
3594 spec: ModuleSpec,
3595 runtime: SupervisorRuntimeConfig,
3596 snapshot: SharedSnapshot,
3597 child: Option<SupervisedChild>,
3598 ) -> SupervisedModule {
3599 let (module, commands) = self.register_module(&spec, &runtime, snapshot);
3600 self.start_monitor(spec, runtime, &module, child, commands);
3601 module
3602 }
3603
3604 fn register_module(
3605 &self,
3606 spec: &ModuleSpec,
3607 runtime: &SupervisorRuntimeConfig,
3608 snapshot: SharedSnapshot,
3609 ) -> (SupervisedModule, mpsc::Receiver<SupervisorCommand>) {
3610 let configuration = Arc::new(Mutex::new(SupervisedConfiguration {
3611 spec: spec.clone(),
3612 health: runtime.health.clone(),
3613 }));
3614 let stderr_ring = Arc::clone(&runtime.stderr_ring);
3615 let terminal_ring = Arc::clone(&runtime.terminal_ring);
3616 let restart_policy = runtime.restart_policy;
3620 let effective_drain_timeout = Arc::clone(&runtime.effective_drain_timeout);
3621 let (tx, rx) = mpsc::channel(4);
3622 let module_id = spec.module_id.clone();
3623 let module = SupervisedModule {
3624 inner: Arc::new(SupervisedModuleInner {
3625 module_id: module_id.clone(),
3626 registry: Arc::clone(&self.registry),
3627 snapshot,
3628 configuration,
3629 stderr_ring,
3630 terminal_ring,
3631 commands: tx,
3632 monitor: Mutex::new(None),
3633 restart_policy,
3634 effective_drain_timeout,
3635 provenance_probe: self.provenance_probe.clone(),
3636 retained: runtime.child_roster.retained_view(),
3637 }),
3638 };
3639 if let Some(supervisor_handle) = &self.supervisor_handle {
3642 supervisor_handle.insert(module.clone());
3643 }
3644 (module, rx)
3645 }
3646
3647 fn start_monitor(
3648 &self,
3649 spec: ModuleSpec,
3650 runtime: SupervisorRuntimeConfig,
3651 module: &SupervisedModule,
3652 child: Option<SupervisedChild>,
3653 commands: mpsc::Receiver<SupervisorCommand>,
3654 ) {
3655 let monitor = tokio::spawn(supervise_loop(
3656 spec,
3657 runtime,
3658 Arc::clone(&self.registry),
3659 Arc::clone(&self.process_liveness),
3660 Arc::clone(&module.inner.snapshot),
3661 child,
3662 commands,
3663 ));
3664 *module
3665 .inner
3666 .monitor
3667 .lock()
3668 .unwrap_or_else(|p| p.into_inner()) = Some(monitor);
3669 }
3670}
3671
3672impl Default for Supervisor {
3673 fn default() -> Self {
3674 Self::new(Arc::new(Registry::default()), RestartPolicy::default())
3675 }
3676}
3677
3678#[derive(Clone)]
3680pub struct SupervisedModule {
3681 inner: Arc<SupervisedModuleInner>,
3682}
3683
3684struct SupervisedModuleInner {
3685 retained: crate::child_roster::RetainedView,
3686 module_id: String,
3687 registry: Arc<Registry>,
3688 snapshot: SharedSnapshot,
3689 configuration: Arc<Mutex<SupervisedConfiguration>>,
3690 stderr_ring: Arc<Mutex<StderrRing>>,
3691 terminal_ring: Arc<Mutex<TerminalRing>>,
3692 commands: mpsc::Sender<SupervisorCommand>,
3693 monitor: Mutex<Option<JoinHandle<()>>>,
3694 restart_policy: RestartPolicy,
3698 effective_drain_timeout: Arc<Mutex<Duration>>,
3699 provenance_probe: ExecutableIdentityProbe,
3700}
3701
3702impl fmt::Debug for SupervisedModule {
3703 fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
3704 f.debug_struct("SupervisedModule")
3705 .field("module_id", &self.inner.module_id)
3706 .field("status", &self.status())
3707 .finish_non_exhaustive()
3708 }
3709}
3710
3711impl SupervisedModule {
3712 pub fn module_id(&self) -> &str {
3713 &self.inner.module_id
3714 }
3715
3716 #[cfg(test)]
3720 pub(crate) fn record_health_probe_failure_for_test(
3721 &self,
3722 detail: &str,
3723 ) -> Result<(), SuperviseError> {
3724 update_snapshot(&self.inner.snapshot, Some(&self.inner.module_id), |state| {
3725 state.health.consecutive_failures = state.health.consecutive_failures.saturating_add(1);
3726 state.health.detail = Some(detail.to_string());
3727 })
3728 }
3729
3730 pub fn state(&self) -> Result<ModuleState, SuperviseError> {
3731 Ok(lock_snapshot(&self.inner.snapshot)?.state)
3732 }
3733
3734 pub(crate) fn registration_end_reason(
3735 &self,
3736 ) -> Result<Option<RegistrationEndReason>, SuperviseError> {
3737 let snapshot = lock_snapshot(&self.inner.snapshot)?;
3738 Ok(match snapshot.state {
3739 ModuleState::Draining if snapshot.draining_to_replace => {
3740 Some(RegistrationEndReason::SupervisorRestart)
3741 }
3742 ModuleState::Draining => Some(RegistrationEndReason::SupervisorStop),
3743 ModuleState::Restarting if snapshot.draining_to_replace => {
3744 Some(RegistrationEndReason::SupervisorRestart)
3745 }
3746 _ => None,
3747 })
3748 }
3749
3750 pub fn stderr_tail(
3757 &self,
3758 max_lines: Option<usize>,
3759 max_bytes: Option<usize>,
3760 ) -> StderrTailSnapshot {
3761 self.inner
3762 .stderr_ring
3763 .lock()
3764 .unwrap_or_else(|poisoned| poisoned.into_inner())
3765 .snapshot(max_lines, max_bytes)
3766 }
3767
3768 pub fn terminal_history(&self) -> TerminalHistorySnapshot {
3773 self.inner
3774 .terminal_ring
3775 .lock()
3776 .unwrap_or_else(|poisoned| poisoned.into_inner())
3777 .snapshot()
3778 }
3779
3780 pub fn durable_terminal_history(&self) -> subc_control::TerminalHistory {
3785 durable_terminal_history_of(&self.inner.terminal_ring, &self.inner.module_id)
3786 }
3787
3788 pub(crate) async fn read_durable_terminal_history(
3793 &self,
3794 ) -> Result<subc_control::TerminalHistory, tokio::task::JoinError> {
3795 let terminal_ring = Arc::clone(&self.inner.terminal_ring);
3796 let module_id = self.inner.module_id.clone();
3797 tokio::task::spawn_blocking(move || durable_terminal_history_of(&terminal_ring, &module_id))
3798 .await
3799 }
3800
3801 pub fn status(&self) -> Result<ModuleStatus, SuperviseError> {
3802 self.status_with_snapshot_lock(&self.inner.snapshot, None)
3803 .map(|(status, _)| status)
3804 }
3805
3806 pub(crate) fn next_retry_at_ms(&self) -> Result<Option<u64>, SuperviseError> {
3807 let state = lock_snapshot(&self.inner.snapshot)?;
3808 Ok(state
3809 .failed_retry
3810 .pending
3811 .filter(|_| state.state == ModuleState::Failed && state.enabled)
3812 .map(|retry| {
3813 unix_ms_now().saturating_add(
3814 u64::try_from(
3815 retry
3816 .deadline
3817 .saturating_duration_since(Instant::now())
3818 .as_millis(),
3819 )
3820 .unwrap_or(u64::MAX),
3821 )
3822 }))
3823 }
3824
3825 pub(crate) fn record_deliberate_severance(
3826 &self,
3827 identity: ProcessIdentity,
3828 ) -> Result<bool, SuperviseError> {
3829 let mut snapshot = lock_snapshot(&self.inner.snapshot)?;
3830 if snapshot.pid != Some(identity.pid)
3831 || snapshot.process_start_time != Some(identity.start_time)
3832 {
3833 return Ok(false);
3834 }
3835 snapshot.deliberate_severance = Some(identity);
3836 Ok(true)
3837 }
3838
3839 pub(crate) fn status_for_control(
3844 &self,
3845 caller: &'static str,
3846 ) -> Result<ModuleStatus, SuperviseError> {
3847 self.status_with_snapshot_lock(&self.inner.snapshot, Some(caller))
3848 .map(|(status, _)| status)
3849 }
3850
3851 fn status_with_snapshot_lock(
3852 &self,
3853 snapshot: &SharedSnapshot,
3854 caller: Option<&'static str>,
3855 ) -> Result<(ModuleStatus, Option<SpawnedFileIdentity>), SuperviseError> {
3856 let mut guard = match caller {
3857 Some(caller) => lock_snapshot_for_control(snapshot, &self.inner.module_id, caller)?,
3858 None => lock_snapshot(snapshot)?,
3859 };
3860 let restart_count =
3863 guard.crash_restarts_in_window(self.inner.restart_policy.window, Instant::now());
3864 let snapshot = guard.clone();
3865 drop(guard);
3866 let drain_timeout = *self.inner.effective_drain_timeout.lock().map_err(|_| {
3867 SuperviseError::StatePoisoned {
3868 module_id: Some(self.inner.module_id.clone()),
3869 }
3870 })?;
3871 let registration_active = self
3872 .inner
3873 .registry
3874 .get_module(&self.inner.module_id)
3875 .map_err(SuperviseError::Registry)?
3876 .is_some();
3877 let protocol = snapshot
3878 .spawned_protocol
3879 .unwrap_or(self.declared_protocol()?);
3880 let running_process =
3881 snapshot.enabled && snapshot.state == ModuleState::Running && snapshot.process_alive;
3882 let live = match protocol {
3888 ModuleProtocol::Subc => running_process && registration_active,
3889 ModuleProtocol::None => running_process,
3890 };
3891 let retained = self.inner.retained.waiting_detail(&self.inner.module_id);
3894 let carry_configured = self
3895 .inner
3896 .configuration
3897 .lock()
3898 .map_err(|_| SuperviseError::StatePoisoned {
3899 module_id: Some(self.inner.module_id.clone()),
3900 })?
3901 .spec
3902 .carry_containment;
3903 let recovery_block = carries_containment(
3904 carry_configured,
3905 self.inner
3906 .retained
3907 .has_carried_container(&self.inner.module_id),
3908 )
3909 .then(|| self.inner.retained.recovery_block())
3910 .flatten();
3911 let launch_nonce_source = observed_launch_nonce_source(&snapshot);
3912 let pid = snapshot.reported_pid();
3913 let mut health = snapshot.health;
3914 if let Some(detail) = recovery_block
3915 .as_ref()
3916 .or_else(|| retained.as_ref().map(|(_, detail)| detail))
3917 {
3918 health.detail = Some(match health.detail {
3919 Some(previous) => format!("{previous}; {detail}"),
3920 None => detail.clone(),
3921 });
3922 }
3923
3924 Ok((
3925 ModuleStatus {
3926 waiting_for_previous_pid: retained
3927 .as_ref()
3928 .map(|(pid, _)| *pid)
3929 .filter(|pid| *pid > 0)
3930 .or(snapshot.waiting_for_previous_pid),
3931 waiting_for_previous_detail: recovery_block
3932 .or_else(|| retained.map(|(_, detail)| detail))
3933 .or(snapshot.waiting_for_previous_detail),
3934 module_id: self.inner.module_id.clone(),
3935 launch_nonce_source,
3936 state: snapshot.state,
3937 enabled: snapshot.enabled,
3938 process_alive: snapshot.process_alive,
3939 registration_active,
3940 protocol,
3941 live,
3942 restart_count,
3943 lifetime_restarts: snapshot.lifetime_restarts,
3944 spawn_generation: snapshot.spawn_generation,
3945 max_restarts: self.inner.restart_policy.max_restarts,
3946 restart_window: self.inner.restart_policy.window,
3947 drain_timeout,
3948 restart_backoff: self.inner.restart_policy.backoff,
3949 restart_max_backoff: self.inner.restart_policy.max_backoff,
3950 pid,
3951 spawned_at_ms: snapshot.spawned_at_ms,
3952 spawned_from: snapshot.spawned_from,
3953 launch_spec_digest: snapshot.launch_spec_digest,
3954 process_start_time: snapshot.process_start_time,
3955 last_exit: snapshot.last_exit,
3956 health,
3957 },
3958 snapshot.spawned_file_identity.clone(),
3959 ))
3960 }
3961
3962 #[cfg(test)]
3963 pub(crate) fn hold_snapshot_for_test(
3964 &self,
3965 acquired: std::sync::mpsc::Sender<()>,
3966 hold: Duration,
3967 ) -> std::thread::JoinHandle<()> {
3968 let snapshot = Arc::clone(&self.inner.snapshot);
3969 std::thread::spawn(move || {
3970 let _guard = snapshot.lock().expect("test snapshot lock is not poisoned");
3971 acquired
3972 .send(())
3973 .expect("test receiver waits for snapshot lock");
3974 std::thread::sleep(hold);
3975 })
3976 }
3977
3978 pub(crate) async fn status_and_running_image_agreement(
3984 &self,
3985 ) -> Result<(ModuleStatus, subc_control::RunningImageAgreement), SuperviseError> {
3986 let (status, identity) = self.status_with_snapshot_lock(&self.inner.snapshot, None)?;
3987 let image = self
3988 .inner
3989 .provenance_probe
3990 .observe(
3991 status.pid,
3992 status.spawned_from.as_deref(),
3993 identity,
3994 status.process_start_time,
3995 )
3996 .await;
3997 Ok((status, image))
3998 }
3999
4000 pub(crate) async fn running_image_agreement(&self) -> subc_control::RunningImageAgreement {
4001 let snapshot = match lock_snapshot(&self.inner.snapshot) {
4002 Ok(snapshot) => snapshot.clone(),
4003 Err(_) => {
4004 return subc_control::RunningImageAgreement::Unavailable {
4005 reason: subc_control::RunningImageUnavailableReason::NotRunning,
4006 };
4007 }
4008 };
4009 self.inner
4010 .provenance_probe
4011 .observe(
4012 snapshot.reported_pid(),
4013 snapshot.spawned_from.as_deref(),
4014 snapshot.spawned_file_identity.clone(),
4015 snapshot.process_start_time,
4016 )
4017 .await
4018 }
4019
4020 pub(crate) fn child_resource_usage(&self) -> subc_control::ChildResourceUsage {
4023 #[cfg(windows)]
4024 {
4025 let image = lock_snapshot(&self.inner.snapshot)
4026 .ok()
4027 .and_then(|snapshot| {
4028 snapshot
4029 .spawned_file_identity
4030 .as_ref()
4031 .and_then(|identity| identity.windows_image.clone())
4032 });
4033 match image {
4034 Some(image) => {
4035 crate::child_resources::read_windows_handle(image.process(), image.start_time())
4036 }
4037 None => subc_control::ChildResourceUsage::Unavailable {
4038 reason: subc_control::ChildResourceUnavailableReason::NotRunning,
4039 },
4040 }
4041 }
4042 #[cfg(not(windows))]
4043 {
4044 let (pid, start_time) = match lock_snapshot(&self.inner.snapshot) {
4045 Ok(snapshot) => (snapshot.reported_pid(), snapshot.process_start_time),
4046 Err(_) => {
4047 return subc_control::ChildResourceUsage::Unavailable {
4048 reason: subc_control::ChildResourceUnavailableReason::Unreadable,
4049 }
4050 }
4051 };
4052 crate::child_resources::read(pid, start_time)
4053 }
4054 }
4055
4056 #[cfg(all(test, any(target_os = "linux", windows)))]
4057 pub(crate) fn subtree_target(&self) -> crate::subtree_resources::Target {
4058 self.inner
4059 .snapshot
4060 .try_lock()
4061 .map(|snapshot| snapshot.subtree_target.clone())
4062 .unwrap_or(crate::subtree_resources::Target::Unavailable("read_failed"))
4063 }
4064
4065 #[cfg(test)]
4066 pub(crate) fn set_subtree_target_for_test(&self, target: crate::subtree_resources::Target) {
4067 self.inner.snapshot.lock().unwrap().subtree_target = target;
4068 }
4069
4070 pub(crate) fn will_recover_after_connection_loss(&self) -> Result<bool, SuperviseError> {
4071 let mut snapshot = lock_snapshot(&self.inner.snapshot)?;
4072 Ok(match snapshot.state {
4073 ModuleState::Restarting => true,
4074 ModuleState::Failed | ModuleState::Disabled => false,
4075 _ if snapshot.enabled && snapshot.respawn_pending => true,
4076 _ => daemon_will_restart(&mut snapshot, &self.inner.restart_policy, Instant::now()),
4077 })
4078 }
4079
4080 #[cfg(test)]
4081 pub(crate) fn is_warming(&self) -> Result<bool, SuperviseError> {
4082 self.is_warming_with_snapshot_lock(None)
4083 }
4084
4085 pub(crate) fn is_warming_for_control(
4086 &self,
4087 caller: &'static str,
4088 ) -> Result<bool, SuperviseError> {
4089 self.is_warming_with_snapshot_lock(Some(caller))
4090 }
4091
4092 fn is_warming_with_snapshot_lock(
4093 &self,
4094 caller: Option<&'static str>,
4095 ) -> Result<bool, SuperviseError> {
4096 let snapshot = match caller {
4097 Some(caller) => {
4098 lock_snapshot_for_control(&self.inner.snapshot, &self.inner.module_id, caller)?
4099 }
4100 None => lock_snapshot(&self.inner.snapshot)?,
4101 }
4102 .clone();
4103 Ok(matches!(
4104 snapshot.state,
4105 ModuleState::Starting | ModuleState::Running | ModuleState::Restarting
4106 ))
4107 }
4108
4109 pub async fn drain(&self) -> Result<(), SuperviseError> {
4111 self.stop().await
4112 }
4113
4114 pub(crate) async fn retire_requested(
4115 &self,
4116 requester: Option<Principal>,
4117 ) -> Result<(), SuperviseError> {
4118 match self.state()? {
4119 ModuleState::Stopped => return Ok(()),
4120 ModuleState::Failed
4121 if lock_snapshot(&self.inner.snapshot)?
4122 .failed_retry
4123 .pending
4124 .is_none() =>
4125 {
4126 return Ok(())
4127 }
4128 ModuleState::Failed
4129 | ModuleState::Starting
4130 | ModuleState::Running
4131 | ModuleState::Unresponsive
4132 | ModuleState::Restarting
4133 | ModuleState::Draining
4134 | ModuleState::Disabled => {}
4135 }
4136
4137 let (reply_tx, reply_rx) = oneshot::channel();
4138 self.inner
4139 .commands
4140 .send(SupervisorCommand::Retire {
4141 requester,
4142 reply: reply_tx,
4143 })
4144 .await
4145 .map_err(|_| SuperviseError::CommandClosed {
4146 module_id: self.inner.module_id.clone(),
4147 })?;
4148 reply_rx.await.map_err(|_| SuperviseError::CommandClosed {
4149 module_id: self.inner.module_id.clone(),
4150 })?
4151 }
4152
4153 pub async fn stop(&self) -> Result<(), SuperviseError> {
4154 self.stop_requested(None).await
4155 }
4156
4157 pub(crate) async fn stop_requested(
4158 &self,
4159 requester: Option<Principal>,
4160 ) -> Result<(), SuperviseError> {
4161 match self.state()? {
4162 ModuleState::Stopped => return Ok(()),
4163 ModuleState::Failed
4164 if lock_snapshot(&self.inner.snapshot)?
4165 .failed_retry
4166 .pending
4167 .is_none() =>
4168 {
4169 return Ok(())
4170 }
4171 ModuleState::Failed
4172 | ModuleState::Starting
4173 | ModuleState::Running
4174 | ModuleState::Unresponsive
4175 | ModuleState::Restarting
4176 | ModuleState::Draining
4177 | ModuleState::Disabled => {}
4178 }
4179
4180 let (reply_tx, reply_rx) = oneshot::channel();
4181 self.inner
4182 .commands
4183 .send(SupervisorCommand::Drain {
4184 requester,
4185 reply: reply_tx,
4186 })
4187 .await
4188 .map_err(|_| SuperviseError::CommandClosed {
4189 module_id: self.inner.module_id.clone(),
4190 })?;
4191 reply_rx.await.map_err(|_| SuperviseError::CommandClosed {
4192 module_id: self.inner.module_id.clone(),
4193 })?
4194 }
4195
4196 pub async fn restart(&self, drain_timeout_ms: Option<u64>) -> Result<(), SuperviseError> {
4197 self.restart_requested(drain_timeout_ms, None).await
4198 }
4199
4200 pub(crate) async fn restart_requested(
4201 &self,
4202 drain_timeout_ms: Option<u64>,
4203 requester: Option<Principal>,
4204 ) -> Result<(), SuperviseError> {
4205 let queued_at = Instant::now();
4206 let received_at_generation = lock_snapshot(&self.inner.snapshot)?.spawn_generation;
4207 let (reply_tx, reply_rx) = oneshot::channel();
4208 self.inner
4209 .commands
4210 .send(SupervisorCommand::Restart {
4211 requester,
4212 drain_timeout_ms,
4213 received_at_generation,
4214 queued_at,
4215 reply: reply_tx,
4216 })
4217 .await
4218 .map_err(|_| SuperviseError::CommandClosed {
4219 module_id: self.inner.module_id.clone(),
4220 })?;
4221 reply_rx.await.map_err(|_| SuperviseError::CommandClosed {
4222 module_id: self.inner.module_id.clone(),
4223 })?
4224 }
4225
4226 pub async fn swap(&self, ready_timeout: Option<Duration>) -> Result<(), SuperviseError> {
4231 self.swap_requested(ready_timeout, None).await
4232 }
4233
4234 pub(crate) async fn swap_requested(
4235 &self,
4236 ready_timeout: Option<Duration>,
4237 requester: Option<Principal>,
4238 ) -> Result<(), SuperviseError> {
4239 let (reply_tx, reply_rx) = oneshot::channel();
4240 self.inner
4241 .commands
4242 .send(SupervisorCommand::Swap {
4243 requester,
4244 ready_timeout,
4245 reply: reply_tx,
4246 })
4247 .await
4248 .map_err(|_| SuperviseError::CommandClosed {
4249 module_id: self.inner.module_id.clone(),
4250 })?;
4251 reply_rx.await.map_err(|_| SuperviseError::CommandClosed {
4252 module_id: self.inner.module_id.clone(),
4253 })?
4254 }
4255
4256 pub async fn reload(&self) -> Result<(), SuperviseError> {
4257 self.reload_requested(None).await
4258 }
4259
4260 pub(crate) async fn reload_requested(
4261 &self,
4262 requester: Option<Principal>,
4263 ) -> Result<(), SuperviseError> {
4264 let (reply_tx, reply_rx) = oneshot::channel();
4265 self.inner
4266 .commands
4267 .send(SupervisorCommand::Reload {
4268 requester,
4269 reply: reply_tx,
4270 })
4271 .await
4272 .map_err(|_| SuperviseError::CommandClosed {
4273 module_id: self.inner.module_id.clone(),
4274 })?;
4275 reply_rx.await.map_err(|_| SuperviseError::CommandClosed {
4276 module_id: self.inner.module_id.clone(),
4277 })?
4278 }
4279
4280 pub async fn set_enabled(&self, enabled: bool) -> Result<bool, SuperviseError> {
4281 self.set_enabled_requested(enabled, None).await
4282 }
4283
4284 pub(crate) async fn set_enabled_requested(
4285 &self,
4286 enabled: bool,
4287 requester: Option<Principal>,
4288 ) -> Result<bool, SuperviseError> {
4289 let (reply_tx, reply_rx) = oneshot::channel();
4290 self.inner
4291 .commands
4292 .send(SupervisorCommand::SetEnabled {
4293 requester,
4294 enabled,
4295 reply: reply_tx,
4296 })
4297 .await
4298 .map_err(|_| SuperviseError::CommandClosed {
4299 module_id: self.inner.module_id.clone(),
4300 })?;
4301 reply_rx.await.map_err(|_| SuperviseError::CommandClosed {
4302 module_id: self.inner.module_id.clone(),
4303 })?
4304 }
4305
4306 pub(crate) fn declared_protocol(&self) -> Result<ModuleProtocol, SuperviseError> {
4310 let configured = self
4311 .inner
4312 .configuration
4313 .lock()
4314 .map_err(|_| SuperviseError::StatePoisoned {
4315 module_id: Some(self.inner.module_id.clone()),
4316 })?
4317 .spec
4318 .protocol;
4319 let state = lock_snapshot(&self.inner.snapshot)?;
4320 Ok(state.spawned_protocol.unwrap_or(configured))
4321 }
4322
4323 pub(crate) fn configuration(&self) -> Result<(ModuleSpec, HealthConfig), SuperviseError> {
4324 let configuration =
4325 self.inner
4326 .configuration
4327 .lock()
4328 .map_err(|_| SuperviseError::StatePoisoned {
4329 module_id: Some(self.inner.module_id.clone()),
4330 })?;
4331 Ok((configuration.spec.clone(), configuration.health.clone()))
4332 }
4333
4334 #[cfg(any(test, feature = "test-support"))]
4338 pub async fn update_spec_for_test(&self, spec: ModuleSpec) -> Result<(), SuperviseError> {
4339 let (_, health) = self.configuration()?;
4340 let drain_timeout_ms = u64::try_from(
4341 self.inner
4342 .effective_drain_timeout
4343 .lock()
4344 .unwrap_or_else(|poisoned| poisoned.into_inner())
4345 .as_millis(),
4346 )
4347 .ok();
4348 self.update_configuration(spec, health, drain_timeout_ms)
4349 .await
4350 }
4351
4352 pub(crate) async fn update_configuration(
4353 &self,
4354 spec: ModuleSpec,
4355 health: HealthConfig,
4356 drain_timeout_ms: Option<u64>,
4357 ) -> Result<(), SuperviseError> {
4358 if spec.module_id != self.inner.module_id {
4359 return Err(SuperviseError::InvalidSpec {
4360 reason: "a supervised module's module_id cannot be changed".to_string(),
4361 });
4362 }
4363 validate_spec(&spec)?;
4364 let (reply_tx, reply_rx) = oneshot::channel();
4365 self.inner
4366 .commands
4367 .send(SupervisorCommand::UpdateConfiguration {
4368 spec: spec.clone(),
4369 health: health.clone(),
4370 drain_timeout_ms,
4371 reply: reply_tx,
4372 })
4373 .await
4374 .map_err(|_| SuperviseError::CommandClosed {
4375 module_id: self.inner.module_id.clone(),
4376 })?;
4377 reply_rx.await.map_err(|_| SuperviseError::CommandClosed {
4378 module_id: self.inner.module_id.clone(),
4379 })?;
4380 let mut configuration =
4381 self.inner
4382 .configuration
4383 .lock()
4384 .map_err(|_| SuperviseError::StatePoisoned {
4385 module_id: Some(self.inner.module_id.clone()),
4386 })?;
4387 configuration.spec = spec;
4388 configuration.health = health;
4389 Ok(())
4390 }
4391}
4392
4393impl Drop for SupervisedModuleInner {
4394 fn drop(&mut self) {
4395 let Ok(mut monitor) = self.monitor.lock() else {
4396 return;
4397 };
4398 if let Some(monitor) = monitor.as_ref().filter(|monitor| !monitor.is_finished()) {
4399 let _ = update_snapshot(&self.snapshot, Some(&self.module_id), |state| {
4400 state.state = ModuleState::Stopped;
4401 clear_current_process_facts(state);
4402 });
4403 monitor.abort();
4404 }
4405 let _ = monitor.take();
4406 }
4407}
4408
4409#[derive(Debug)]
4410enum SupervisorCommand {
4411 Drain {
4412 requester: Option<Principal>,
4413 reply: oneshot::Sender<Result<(), SuperviseError>>,
4414 },
4415 Retire {
4416 requester: Option<Principal>,
4417 reply: oneshot::Sender<Result<(), SuperviseError>>,
4418 },
4419 Restart {
4420 requester: Option<Principal>,
4421 drain_timeout_ms: Option<u64>,
4426 received_at_generation: u64,
4430 queued_at: Instant,
4433 reply: oneshot::Sender<Result<(), SuperviseError>>,
4434 },
4435 Reload {
4436 requester: Option<Principal>,
4437 reply: oneshot::Sender<Result<(), SuperviseError>>,
4438 },
4439 SetEnabled {
4440 requester: Option<Principal>,
4441 enabled: bool,
4442 reply: oneshot::Sender<Result<bool, SuperviseError>>,
4443 },
4444 UpdateConfiguration {
4445 spec: ModuleSpec,
4446 health: HealthConfig,
4447 drain_timeout_ms: Option<u64>,
4450 reply: oneshot::Sender<()>,
4451 },
4452 Swap {
4453 requester: Option<Principal>,
4454 ready_timeout: Option<Duration>,
4457 reply: oneshot::Sender<Result<(), SuperviseError>>,
4459 },
4460}
4461
4462#[derive(Debug)]
4463pub enum SuperviseError {
4464 InvalidSpec {
4465 reason: String,
4466 },
4467 Spawn {
4468 program: PathBuf,
4469 source: io::Error,
4470 cgroup_path: Option<PathBuf>,
4471 },
4472 Cgroup {
4473 module_id: String,
4474 source: io::Error,
4475 },
4476 LaunchNonce {
4479 reason: String,
4480 },
4481 Wait {
4482 module_id: String,
4483 source: io::Error,
4484 },
4485 Kill {
4486 module_id: String,
4487 source: io::Error,
4488 },
4489 Forwarding(ForwardingError),
4490 Registry(RegistryError),
4491 ReloadUnavailable {
4492 module_id: String,
4493 reason: String,
4494 },
4495 Disabled {
4500 module_id: String,
4501 },
4502 ReloadFailed {
4503 module_id: String,
4504 reason: String,
4505 },
4506 RegistrationStillActive {
4507 module_id: String,
4508 waited: Duration,
4509 },
4510 StatePoisoned {
4511 module_id: Option<String>,
4512 },
4513 CommandClosed {
4514 module_id: String,
4515 },
4516 SwapInProgress {
4520 module_id: String,
4521 },
4522 SwapRefused {
4524 module_id: String,
4525 reason: SwapRefusal,
4526 },
4527 SwapFailed {
4531 module_id: String,
4532 arm: SwapFailureArm,
4533 detail: String,
4534 candidate_exit: Option<ExitReport>,
4537 },
4538}
4539
4540#[derive(Debug, Clone, Copy, PartialEq, Eq)]
4542pub enum SwapRefusal {
4543 OverlapExclusive,
4545 NotRegistered,
4548 ProtocolNone,
4551 NotConfigured,
4554 AlreadySwapping,
4556}
4557
4558impl SwapRefusal {
4559 pub fn as_str(self) -> &'static str {
4560 match self {
4561 Self::OverlapExclusive => "overlap_exclusive",
4562 Self::NotRegistered => "not_registered",
4563 Self::ProtocolNone => "protocol_none",
4564 Self::NotConfigured => "not_configured",
4565 Self::AlreadySwapping => "already_swapping",
4566 }
4567 }
4568}
4569
4570#[derive(Debug, Clone, Copy, PartialEq, Eq)]
4573pub enum SwapFailureArm {
4574 SpawnFailed,
4576 NeverRegistered,
4578 NeverReady,
4580 CandidateExited,
4582 CandidateUnhealthy,
4584 Interrupted,
4588 CutoverLost,
4593}
4594
4595impl SwapFailureArm {
4596 pub fn as_str(self) -> &'static str {
4597 match self {
4598 Self::SpawnFailed => "spawn_failed",
4599 Self::NeverRegistered => "never_registered",
4600 Self::NeverReady => "never_ready",
4601 Self::CandidateExited => "candidate_exited",
4602 Self::CandidateUnhealthy => "candidate_unhealthy",
4603 Self::Interrupted => "interrupted",
4604 Self::CutoverLost => "cutover_lost",
4605 }
4606 }
4607}
4608
4609impl fmt::Display for SuperviseError {
4610 fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
4611 match self {
4612 Self::InvalidSpec { reason } => write!(f, "invalid module spec: {reason}"),
4613 Self::Spawn {
4614 program,
4615 source,
4616 cgroup_path: Some(cgroup_path),
4617 } => write!(
4618 f,
4619 "failed to place module in cgroup '{}' while spawning '{}': {source}",
4620 cgroup_path.display(),
4621 program.display()
4622 ),
4623 Self::Spawn {
4624 program,
4625 source,
4626 cgroup_path: None,
4627 } => write!(
4628 f,
4629 "failed to spawn module '{}': {source}",
4630 program.display()
4631 ),
4632 Self::Cgroup { module_id, source } => {
4633 write!(
4634 f,
4635 "failed to prepare cgroup for module '{module_id}': {source}"
4636 )
4637 }
4638 Self::LaunchNonce { reason } => {
4639 write!(
4640 f,
4641 "failed to generate reserved-module launch nonce: {reason}"
4642 )
4643 }
4644 Self::Wait { module_id, source } => {
4645 write!(f, "failed to wait for module '{module_id}': {source}")
4646 }
4647 Self::Kill { module_id, source } => {
4648 write!(f, "failed to kill module '{module_id}': {source}")
4649 }
4650 Self::Forwarding(err) => write!(f, "forwarding error: {err}"),
4651 Self::Registry(err) => write!(f, "registry error: {err}"),
4652 Self::ReloadUnavailable { module_id, reason } => {
4653 write!(f, "reload unavailable for module '{module_id}': {reason}")
4654 }
4655 Self::Disabled { module_id } => {
4656 write!(
4657 f,
4658 "module '{module_id}' is disabled; enable it before restart or reload"
4659 )
4660 }
4661 Self::ReloadFailed { module_id, reason } => {
4662 write!(f, "reload failed for module '{module_id}': {reason}")
4663 }
4664 Self::RegistrationStillActive { module_id, waited } => write!(
4665 f,
4666 "module '{module_id}' registration remained active after waiting {waited:?}"
4667 ),
4668 Self::StatePoisoned { module_id } => match module_id {
4669 Some(module_id) => {
4670 write!(f, "supervisor state for module '{module_id}' was poisoned")
4671 }
4672 None => write!(f, "supervisor state was poisoned"),
4673 },
4674 Self::CommandClosed { module_id } => {
4675 write!(
4676 f,
4677 "supervisor command channel for module '{module_id}' is closed"
4678 )
4679 }
4680 Self::SwapInProgress { module_id } => write!(
4681 f,
4682 "module '{module_id}' is being swapped; retry once the swap has cut over or failed, or stop the module to abort the swap"
4683 ),
4684 Self::SwapRefused { module_id, reason } => match reason {
4685 SwapRefusal::OverlapExclusive => write!(
4686 f,
4687 "module '{module_id}' is declared overlap: \"exclusive\" (the default): two processes of it must not run at once, so it cannot be swapped; use a plain restart, or declare overlap: \"safe\" in its config if it really tolerates a second process"
4688 ),
4689 SwapRefusal::NotRegistered => write!(
4690 f,
4691 "module '{module_id}' is not registered, so there is no serving process to keep while a replacement warms; use a plain restart"
4692 ),
4693 SwapRefusal::ProtocolNone => write!(
4694 f,
4695 "module '{module_id}' is protocol: \"none\" and never registers, so a swap could never see its replacement become ready; use a plain restart"
4696 ),
4697 SwapRefusal::NotConfigured => write!(
4698 f,
4699 "module '{module_id}' cannot be swapped: the supervisor was built without the forwarding table or shared handle a swap needs"
4700 ),
4701 SwapRefusal::AlreadySwapping => {
4702 write!(f, "module '{module_id}' is already being swapped")
4703 }
4704 },
4705 Self::SwapFailed {
4706 module_id,
4707 arm,
4708 detail,
4709 ..
4710 } => write!(
4711 f,
4712 "swap of module '{module_id}' failed ({}): {detail}; the running process was left serving",
4713 arm.as_str()
4714 ),
4715 }
4716 }
4717}
4718
4719impl Error for SuperviseError {
4720 fn source(&self) -> Option<&(dyn Error + 'static)> {
4721 match self {
4722 Self::Spawn { source, .. }
4723 | Self::Cgroup { source, .. }
4724 | Self::Wait { source, .. }
4725 | Self::Kill { source, .. } => Some(source),
4726 Self::Forwarding(err) => Some(err),
4727 Self::Registry(err) => Some(err),
4728 Self::LaunchNonce { .. }
4729 | Self::InvalidSpec { .. }
4730 | Self::ReloadUnavailable { .. }
4731 | Self::Disabled { .. }
4732 | Self::ReloadFailed { .. }
4733 | Self::RegistrationStillActive { .. }
4734 | Self::StatePoisoned { .. }
4735 | Self::CommandClosed { .. }
4736 | Self::SwapInProgress { .. }
4737 | Self::SwapRefused { .. }
4738 | Self::SwapFailed { .. } => None,
4739 }
4740 }
4741}
4742
4743pub(crate) fn validate_spec(spec: &ModuleSpec) -> Result<(), SuperviseError> {
4744 if spec.carry_containment && spec.overlap == ModuleOverlap::Safe {
4745 return Err(SuperviseError::InvalidSpec {
4746 reason: "carry_containment cannot be combined with overlap: safe".to_owned(),
4747 });
4748 }
4749 if spec.module_id.trim().is_empty() {
4750 return Err(SuperviseError::InvalidSpec {
4751 reason: "module_id must not be empty".to_string(),
4752 });
4753 }
4754
4755 Ok(())
4756}
4757
4758#[derive(Debug, Default)]
4759struct HealthProbeRuntime {
4760 spawn_generation: Option<u64>,
4761 configured_health: Option<HealthConfig>,
4762 registered_connection: Option<crate::ConnectionId>,
4763 advertised: bool,
4764 next_probe_at: Option<Instant>,
4765 probe_index: u64,
4766}
4767
4768fn running_protocol(spec: &ModuleSpec, snapshot: &SharedSnapshot) -> ModuleProtocol {
4769 lock_snapshot(snapshot)
4770 .ok()
4771 .and_then(|state| state.spawned_protocol)
4772 .unwrap_or(spec.protocol)
4773}
4774
4775impl HealthProbeRuntime {
4776 fn refresh_registration(
4777 &mut self,
4778 spec: &ModuleSpec,
4779 runtime: &SupervisorRuntimeConfig,
4780 registry: &Registry,
4781 snapshot: &SharedSnapshot,
4782 ) {
4783 let generation = lock_snapshot(snapshot)
4784 .ok()
4785 .map(|state| state.spawn_generation);
4786 if self.spawn_generation != generation {
4787 *self = Self {
4791 spawn_generation: generation,
4792 ..Self::default()
4793 };
4794 }
4795 if self.configured_health.as_ref() != Some(&runtime.health) {
4796 self.configured_health = Some(runtime.health.clone());
4797 self.next_probe_at = None;
4798 self.registered_connection = None;
4799 self.probe_index = 0;
4800 }
4801 if running_protocol(spec, snapshot) == ModuleProtocol::None {
4805 self.registered_connection = None;
4806 self.advertised = runtime.health.http.is_some();
4807 if !self.advertised {
4808 self.next_probe_at = None;
4809 let _ = update_snapshot(snapshot, Some(&spec.module_id), |state| {
4810 let unknown = ModuleHealthStatus::default();
4811 if state.health != unknown {
4812 state.health = unknown;
4813 }
4814 });
4815 } else if self.next_probe_at.is_none() {
4816 self.next_probe_at = Some(
4817 Instant::now()
4818 + jittered_health_delay(&spec.module_id, 0, runtime.health.cadence),
4819 );
4820 }
4821 return;
4822 }
4823
4824 let registration = match registry.get_module(&spec.module_id) {
4825 Ok(registration) => registration,
4826 Err(err) => {
4827 warn!(module_id = %spec.module_id, error = %err, "health prober could not read registry");
4828 self.advertised = false;
4829 self.next_probe_at = None;
4830 return;
4831 }
4832 };
4833
4834 let Some(registration) = registration else {
4835 self.registered_connection = None;
4836 self.advertised = false;
4837 self.next_probe_at = None;
4838 return;
4839 };
4840
4841 let advertised = registration
4842 .control_ops
4843 .iter()
4844 .any(|op| op == MODULE_CONTROL_OP_HEALTH_CHECK);
4845 if !advertised {
4846 self.registered_connection = Some(registration.connection_id);
4847 self.advertised = false;
4848 self.next_probe_at = None;
4849 let _ = update_snapshot(snapshot, Some(&spec.module_id), |state| {
4850 if state.health.status != SupervisorHealthStatus::Unknown
4851 || state.health.consecutive_failures != 0
4852 || state.health.last_probe_ms.is_some()
4853 || state.health.detail.is_some()
4854 || state.health.metrics.is_some()
4855 {
4856 state.health.status = SupervisorHealthStatus::Unknown;
4857 state.health.consecutive_failures = 0;
4858 state.health.last_probe_ms = None;
4859 state.health.detail = None;
4860 state.health.metrics = None;
4861 }
4862 });
4863 return;
4864 }
4865
4866 let reregistered = self.registered_connection != Some(registration.connection_id);
4867 self.registered_connection = Some(registration.connection_id);
4868 self.advertised = true;
4869 if reregistered || self.next_probe_at.is_none() {
4870 self.probe_index = 0;
4871 self.next_probe_at = Some(
4872 Instant::now() + jittered_health_delay(&spec.module_id, 0, runtime.health.cadence),
4873 );
4874 let _ = update_snapshot(snapshot, Some(&spec.module_id), |state| {
4875 state.health.status = SupervisorHealthStatus::Unknown;
4876 state.health.consecutive_failures = 0;
4877 state.health.detail = None;
4878 state.health.metrics = None;
4879 });
4880 }
4881 }
4882
4883 fn wake_at(&self) -> Option<Instant> {
4884 if !self.advertised {
4885 return None;
4886 }
4887 self.next_probe_at
4888 }
4889
4890 fn due(&self) -> bool {
4891 self.advertised
4892 && self
4893 .next_probe_at
4894 .is_some_and(|next| Instant::now() >= next)
4895 }
4896
4897 fn schedule_next(&mut self, spec: &ModuleSpec, cadence: Duration) {
4898 self.probe_index = self.probe_index.wrapping_add(1);
4899 self.next_probe_at = Some(
4900 Instant::now() + jittered_health_delay(&spec.module_id, self.probe_index, cadence),
4901 );
4902 }
4903}
4904
4905#[derive(Debug)]
4940enum HealthProbeEvidence {
4941 LaneDead,
4943 NoAnswer,
4945 BadAnswer,
4947 Misconfigured,
4949}
4950
4951#[derive(Debug)]
4952struct HealthProbeError {
4953 evidence: HealthProbeEvidence,
4954 message: String,
4955}
4956
4957impl HealthProbeError {
4958 fn lane_dead(message: impl Into<String>) -> Self {
4959 Self::with(HealthProbeEvidence::LaneDead, message)
4960 }
4961
4962 fn no_answer(message: impl Into<String>) -> Self {
4963 Self::with(HealthProbeEvidence::NoAnswer, message)
4964 }
4965
4966 fn bad_answer(message: impl Into<String>) -> Self {
4967 Self::with(HealthProbeEvidence::BadAnswer, message)
4968 }
4969
4970 fn misconfigured(message: impl Into<String>) -> Self {
4971 Self::with(HealthProbeEvidence::Misconfigured, message)
4972 }
4973
4974 fn with(evidence: HealthProbeEvidence, message: impl Into<String>) -> Self {
4975 Self {
4976 evidence,
4977 message: message.into(),
4978 }
4979 }
4980
4981 #[allow(dead_code)]
4995 fn is_proof_of_death(&self) -> bool {
4996 matches!(self.evidence, HealthProbeEvidence::LaneDead)
4997 }
4998
4999 fn label(&self) -> &'static str {
5007 match self.evidence {
5008 HealthProbeEvidence::LaneDead => "lane-dead",
5009 HealthProbeEvidence::NoAnswer => "no-answer",
5010 HealthProbeEvidence::BadAnswer => "bad-answer",
5011 HealthProbeEvidence::Misconfigured => "daemon-misconfigured",
5012 }
5013 }
5014}
5015
5016impl fmt::Display for HealthProbeError {
5017 fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
5018 f.write_str(&self.message)
5019 }
5020}
5021
5022async fn run_health_probe_cycle(
5023 spec: &ModuleSpec,
5024 runtime: &SupervisorRuntimeConfig,
5025 registry: &Registry,
5026 process_liveness: &SupervisorProcessLiveness,
5027 snapshot: &SharedSnapshot,
5028 child: &mut Option<SupervisedChild>,
5029) {
5030 let now_ms = unix_ms_now();
5031 let http = (running_protocol(spec, snapshot) == ModuleProtocol::None)
5032 .then_some(runtime.health.http.as_deref())
5033 .flatten();
5034 let result = match http {
5035 Some(url) => probe_http_health(url, runtime.health.deadline).await,
5036 None => probe_module_health(&spec.module_id, runtime, None).await,
5037 };
5038 match result {
5039 Ok(report) => {
5040 handle_health_report(
5041 spec,
5042 runtime,
5043 registry,
5044 process_liveness,
5045 snapshot,
5046 child,
5047 report,
5048 now_ms,
5049 )
5050 .await;
5051 }
5052 Err(err) => {
5053 if http.is_some() {
5054 let _ = update_snapshot(snapshot, Some(&spec.module_id), |state| {
5055 state.health.status = SupervisorHealthStatus::Failing;
5056 });
5057 }
5058 handle_health_probe_failure(
5059 spec,
5060 runtime,
5061 registry,
5062 process_liveness,
5063 snapshot,
5064 child,
5065 err,
5066 now_ms,
5067 )
5068 .await;
5069 }
5070 }
5071}
5072
5073pub(crate) struct HttpProbeTarget<'a> {
5074 address: std::net::SocketAddr,
5075 localhost: bool,
5076 authority: &'a str,
5077 path: String,
5078}
5079
5080pub(crate) fn parse_http_probe_url(url: &str) -> Result<HttpProbeTarget<'_>, String> {
5083 if url.bytes().any(|b| b <= b' ' || b == 127) || url.contains('#') {
5084 return Err("must not contain whitespace, controls, or a fragment".into());
5085 }
5086 let rest = url
5087 .strip_prefix("http://")
5088 .ok_or("must use plain http://")?;
5089 let split = rest.find(['/', '?']).unwrap_or(rest.len());
5090 let (authority, suffix) = rest.split_at(split);
5091 let (host, port) = if let Some(rest) = authority.strip_prefix("[::1]") {
5092 ("::1", rest)
5093 } else {
5094 let split = authority.find(':').unwrap_or(authority.len());
5095 authority.split_at(split)
5096 };
5097 let ip: std::net::IpAddr = match host {
5098 "127.0.0.1" | "localhost" => std::net::Ipv4Addr::LOCALHOST.into(),
5099 "::1" => std::net::Ipv6Addr::LOCALHOST.into(),
5100 _ => return Err("host must be 127.0.0.1, [::1], or localhost".into()),
5101 };
5102 let port = if port.is_empty() {
5103 80
5104 } else {
5105 port.strip_prefix(':')
5106 .and_then(|p| p.parse::<u16>().ok())
5107 .filter(|p| *p > 0)
5108 .ok_or("must have a valid nonzero TCP port")?
5109 };
5110 let path = if suffix.is_empty() {
5111 "/".into()
5112 } else if suffix.starts_with('?') {
5113 format!("/{suffix}")
5114 } else {
5115 suffix.into()
5116 };
5117 Ok(HttpProbeTarget {
5118 address: std::net::SocketAddr::new(ip, port),
5119 localhost: host == "localhost",
5120 authority,
5121 path,
5122 })
5123}
5124
5125async fn probe_http_health(
5126 url: &str,
5127 deadline: Duration,
5128) -> Result<HealthReport, HealthProbeError> {
5129 use tokio::io::{AsyncReadExt, AsyncWriteExt, BufReader};
5130 let target = parse_http_probe_url(url).map_err(HealthProbeError::misconfigured)?;
5131 let mut response_status = String::new();
5134 let mut body = Vec::new();
5135 let probe = async {
5136 let connection = match tokio::net::TcpStream::connect(target.address).await {
5139 Err(_) if target.localhost => {
5140 tokio::net::TcpStream::connect((
5141 std::net::Ipv6Addr::LOCALHOST,
5142 target.address.port(),
5143 ))
5144 .await
5145 }
5146 result => result,
5147 };
5148 let mut stream = connection.map_err(|error| {
5149 HealthProbeError::no_answer(format!("HTTP connect failed: {error}"))
5150 })?;
5151 stream.write_all(format!("GET {} HTTP/1.1\r\nHost: {}\r\nAccept-Encoding: identity\r\nConnection: close\r\n\r\n", target.path, target.authority).as_bytes()).await
5152 .map_err(|error| HealthProbeError::no_answer(format!("HTTP write failed: {error}")))?;
5153 let mut reader = BufReader::new(stream);
5154 let mut budget = 16 * 1024;
5155 let status = http_line(&mut reader, &mut budget).await?;
5156 let mut words = status.split_ascii_whitespace();
5157 let version = words.next();
5158 let code = words
5159 .next()
5160 .filter(|word| word.len() == 3)
5161 .and_then(|word| word.parse::<u16>().ok());
5162 if !matches!(version, Some("HTTP/1.1" | "HTTP/1.0"))
5163 || !code.is_some_and(|code| (100..600).contains(&code))
5164 {
5165 return Err(HealthProbeError::bad_answer(format!(
5166 "invalid HTTP status: {status}"
5167 )));
5168 }
5169 let code = code.expect("validated status code");
5170 response_status = status.clone();
5171 let mut length = None;
5172 let mut chunked = false;
5173 loop {
5174 let line = http_line(&mut reader, &mut budget).await?;
5175 if line.is_empty() {
5176 break;
5177 }
5178 if let Some((name, value)) = line.split_once(':') {
5179 if name.eq_ignore_ascii_case("content-length") {
5180 length = Some(value.trim().parse::<u64>().map_err(|_| {
5181 HealthProbeError::bad_answer("invalid HTTP Content-Length")
5182 })?);
5183 } else if name.eq_ignore_ascii_case("transfer-encoding") {
5184 chunked = value.trim().eq_ignore_ascii_case("chunked");
5185 }
5186 }
5187 }
5188 if chunked {
5189 while body.len() < 200 {
5190 let line = http_line(&mut reader, &mut budget).await?;
5191 let size = u64::from_str_radix(line.split(';').next().unwrap_or("").trim(), 16)
5192 .map_err(|_| HealthProbeError::bad_answer("invalid HTTP chunk size"))?;
5193 if size == 0 {
5194 break;
5195 }
5196 let count = size.min((200 - body.len()) as u64) as usize;
5197 let start = body.len();
5198 (&mut reader)
5199 .take(count as u64)
5200 .read_to_end(&mut body)
5201 .await
5202 .map_err(|error| {
5203 HealthProbeError::no_answer(format!("HTTP body read failed: {error}"))
5204 })?;
5205 if body.len() - start != count {
5206 return Err(HealthProbeError::bad_answer("truncated HTTP chunk"));
5207 }
5208 if size > count as u64 || body.len() == 200 {
5209 break;
5210 }
5211 if !http_line(&mut reader, &mut budget).await?.is_empty() {
5212 return Err(HealthProbeError::bad_answer("invalid HTTP chunk delimiter"));
5213 }
5214 }
5215 } else {
5216 reader
5217 .take(length.unwrap_or(200).min(200))
5218 .read_to_end(&mut body)
5219 .await
5220 .map_err(|error| {
5221 HealthProbeError::no_answer(format!("HTTP body read failed: {error}"))
5222 })?;
5223 }
5224 if (200..300).contains(&code) {
5225 Ok(HealthReport::ok())
5226 } else {
5227 Err(HealthProbeError::bad_answer(
5228 "HTTP health endpoint returned non-2xx",
5229 ))
5230 }
5231 };
5232 let mut result = timeout(deadline, probe).await.unwrap_or_else(|_| {
5233 Err(HealthProbeError::no_answer(format!(
5234 "HTTP probe timed out after {deadline:?}"
5235 )))
5236 });
5237 if let Err(error) = &mut result {
5238 if !response_status.is_empty() {
5239 error.message = format!(
5240 "{}; {response_status}: {}",
5241 error.message,
5242 String::from_utf8_lossy(&body)
5243 );
5244 }
5245 }
5246 result
5247}
5248
5249async fn http_line(
5250 reader: &mut tokio::io::BufReader<tokio::net::TcpStream>,
5251 remaining: &mut usize,
5252) -> Result<String, HealthProbeError> {
5253 use tokio::io::{AsyncBufReadExt, AsyncReadExt};
5254 let mut line = Vec::new();
5255 (&mut *reader)
5256 .take(*remaining as u64)
5257 .read_until(b'\n', &mut line)
5258 .await
5259 .map_err(|error| {
5260 HealthProbeError::no_answer(format!("HTTP header read failed: {error}"))
5261 })?;
5262 *remaining -= line.len();
5263 if !line.ends_with(b"\r\n") {
5264 return Err(HealthProbeError::bad_answer(
5265 "HTTP headers are incomplete or exceed 16 KiB",
5266 ));
5267 }
5268 line.truncate(line.len() - 2);
5269 String::from_utf8(line).map_err(|_| HealthProbeError::bad_answer("HTTP header is not UTF-8"))
5270}
5271
5272async fn probe_module_health(
5273 module_id: &str,
5274 runtime: &SupervisorRuntimeConfig,
5275 drain_deadline: Option<Instant>,
5276) -> Result<HealthReport, HealthProbeError> {
5277 let Some(forwarding) = runtime.forwarding.as_ref() else {
5278 return Err(HealthProbeError::misconfigured(
5279 "supervisor was not configured with a forwarding table",
5280 ));
5281 };
5282 let probe_started_at = Instant::now();
5283 let mut deadline = probe_started_at + runtime.health.deadline;
5284 if let Some(drain_deadline) = drain_deadline {
5285 deadline = deadline.min(drain_deadline);
5286 }
5287 let pending = if drain_deadline.is_some() {
5288 forwarding.begin_drain_health_probe_rpc_for(
5289 module_id,
5290 MODULE_CONTROL_OP_HEALTH_CHECK,
5291 probe_started_at,
5292 deadline,
5293 )
5294 } else {
5295 forwarding.begin_health_probe_rpc_for(
5296 module_id,
5297 MODULE_CONTROL_OP_HEALTH_CHECK,
5298 probe_started_at,
5299 deadline,
5300 )
5301 }
5302 .map_err(|err| {
5303 HealthProbeError::lane_dead(format!("failed to begin health.check RPC: {err}"))
5306 })?;
5307 await_health_probe(forwarding, pending, deadline, runtime.health.deadline).await
5308}
5309
5310async fn probe_endpoint_health(
5317 endpoint: crate::ModuleEndpointId,
5318 runtime: &SupervisorRuntimeConfig,
5319 deadline_cap: Option<Instant>,
5320) -> Result<HealthReport, HealthProbeError> {
5321 let Some(forwarding) = runtime.forwarding.as_ref() else {
5322 return Err(HealthProbeError::misconfigured(
5323 "supervisor was not configured with a forwarding table",
5324 ));
5325 };
5326 let probe_started_at = Instant::now();
5327 let mut deadline = probe_started_at + runtime.health.deadline;
5328 if let Some(cap) = deadline_cap {
5329 deadline = deadline.min(cap);
5330 }
5331 let pending = forwarding
5332 .begin_endpoint_health_probe_rpc_for(
5333 endpoint,
5334 MODULE_CONTROL_OP_HEALTH_CHECK,
5335 probe_started_at,
5336 deadline,
5337 )
5338 .map_err(|err| {
5339 HealthProbeError::lane_dead(format!("failed to begin health.check RPC: {err}"))
5340 })?;
5341 await_health_probe(forwarding, pending, deadline, runtime.health.deadline).await
5342}
5343
5344async fn await_health_probe(
5346 forwarding: &ForwardingTable,
5347 pending: PendingModuleControlRpc,
5348 deadline: Instant,
5349 probe_budget: Duration,
5350) -> Result<HealthReport, HealthProbeError> {
5351 let PendingModuleControlRpc {
5352 endpoint,
5353 module_sink,
5354 negotiated_ver,
5355 corr,
5356 receiver,
5357 } = pending;
5358 let body = serde_json::to_vec(&ModuleControlRequest::HealthCheck {}).map_err(|err| {
5359 HealthProbeError::misconfigured(format!("failed to encode health.check: {err}"))
5360 })?;
5361 let frame = Frame::build_with_version(
5362 negotiated_ver,
5363 FrameType::Request,
5364 control_flags(),
5365 0,
5366 0,
5367 corr,
5368 body,
5369 )
5370 .map_err(|err| {
5371 HealthProbeError::misconfigured(format!("failed to build health.check frame: {err}"))
5372 })?;
5373
5374 match timeout_at(deadline, module_sink.send(frame)).await {
5380 Ok(Ok(())) => {}
5381 Ok(Err(err)) => {
5382 let _ = forwarding.cancel_module_control_rpc(endpoint, corr);
5383 return Err(HealthProbeError::lane_dead(format!(
5386 "failed to send health.check: {err}"
5387 )));
5388 }
5389 Err(_elapsed) => {
5390 let _ = forwarding.cancel_module_control_rpc(endpoint, corr);
5391 return Err(HealthProbeError::no_answer(
5395 "health.check send timed out before enqueue (module egress full)",
5396 ));
5397 }
5398 }
5399
5400 match timeout_at(deadline, receiver).await {
5401 Ok(Ok(ModuleControlRpcOutcome::Response(response))) => {
5405 response.health_report().ok_or_else(|| {
5406 HealthProbeError::bad_answer("health.check RPC returned a non-health response")
5407 })
5408 }
5409 Ok(Ok(ModuleControlRpcOutcome::Rejected(body))) => Err(HealthProbeError::bad_answer(
5410 format!("health.check rejected: {}", body.message),
5411 )),
5412 Ok(Ok(ModuleControlRpcOutcome::ModuleGone(message))) => {
5413 Err(HealthProbeError::lane_dead(message))
5414 }
5415 Ok(Ok(ModuleControlRpcOutcome::MalformedResponse(message))) => {
5416 Err(HealthProbeError::bad_answer(message))
5417 }
5418 Ok(Ok(ModuleControlRpcOutcome::UnexpectedOp { expected, actual })) => {
5419 Err(HealthProbeError::bad_answer(format!(
5420 "expected module-control op '{expected}', got '{actual}'"
5421 )))
5422 }
5423 Ok(Ok(ModuleControlRpcOutcome::DeadlineElapsed)) => Err(HealthProbeError::bad_answer(
5427 "module answered health.check after its daemon deadline",
5428 )),
5429 Ok(Err(_)) => Err(HealthProbeError::misconfigured(
5430 "health.check waiter was canceled before the module responded",
5431 )),
5432 Err(_) => {
5433 let _ = forwarding.tombstone_health_probe_rpc(endpoint, corr);
5434 Err(HealthProbeError::no_answer(format!(
5435 "module did not answer health.check within {probe_budget:?}"
5436 )))
5437 }
5438 }
5439}
5440
5441#[allow(clippy::too_many_arguments)]
5442async fn handle_health_report(
5443 spec: &ModuleSpec,
5444 runtime: &SupervisorRuntimeConfig,
5445 registry: &Registry,
5446 process_liveness: &SupervisorProcessLiveness,
5447 snapshot: &SharedSnapshot,
5448 child: &mut Option<SupervisedChild>,
5449 report: HealthReport,
5450 now_ms: u64,
5451) {
5452 let status = supervisor_health_status(report.status);
5453 let detail = report.detail.clone();
5454 let metrics = truncate_health_metrics(report.metrics);
5455 let _ = update_snapshot(snapshot, Some(&spec.module_id), |state| {
5456 state.health.status = status;
5457 state.health.last_probe_ms = Some(now_ms);
5458 state.health.detail = detail.clone();
5459 state.health.metrics = metrics.clone();
5460 state.health.consecutive_failures = 0;
5461 state.failed_retry.observe_health(
5462 report.status == HealthStatus::Ok,
5463 runtime.restart_policy.window,
5464 );
5465 });
5466
5467 let action = match report.status {
5468 HealthStatus::Ok => return,
5469 HealthStatus::Degraded => runtime.health.on_degraded,
5470 HealthStatus::Failing => runtime.health.on_failing,
5471 };
5472 apply_l3_health_action(
5473 spec,
5474 runtime,
5475 registry,
5476 process_liveness,
5477 snapshot,
5478 child,
5479 status,
5480 detail.as_deref(),
5481 action,
5482 now_ms,
5483 )
5484 .await;
5485}
5486
5487#[allow(clippy::too_many_arguments)]
5488async fn handle_health_probe_failure(
5489 spec: &ModuleSpec,
5490 runtime: &SupervisorRuntimeConfig,
5491 registry: &Registry,
5492 process_liveness: &SupervisorProcessLiveness,
5493 snapshot: &SharedSnapshot,
5494 child: &mut Option<SupervisedChild>,
5495 err: HealthProbeError,
5496 now_ms: u64,
5497) {
5498 let threshold = runtime.health.failure_threshold.max(1);
5499 let mut failures = 0;
5500 let detail = format!("[{}] {err}", err.label());
5505 let _ = update_snapshot(snapshot, Some(&spec.module_id), |state| {
5506 state
5507 .failed_retry
5508 .observe_health(false, runtime.restart_policy.window);
5509 if state.spawned_protocol.unwrap_or(spec.protocol) == ModuleProtocol::Subc {
5512 state.health.status = SupervisorHealthStatus::Unknown;
5513 }
5514 state.health.last_probe_ms = Some(now_ms);
5515 state.health.consecutive_failures = state.health.consecutive_failures.saturating_add(1);
5516 state.health.detail = Some(detail.clone());
5517 state.health.metrics = None;
5518 failures = state.health.consecutive_failures;
5519 });
5520
5521 if failures < threshold {
5522 warn!(
5523 module_id = %spec.module_id,
5524 consecutive_failures = failures,
5525 threshold,
5526 evidence = err.label(),
5527 detail = %detail,
5528 "health.check probe failed"
5529 );
5530 return;
5531 }
5532
5533 let _ = update_snapshot(snapshot, Some(&spec.module_id), |state| {
5534 state.state = ModuleState::Unresponsive;
5535 state.health.status = SupervisorHealthStatus::Unresponsive;
5536 });
5537 if runtime.health.critical {
5541 error!(
5542 module_id = %spec.module_id,
5543 status = "unresponsive",
5544 evidence = err.label(),
5545 detail = %detail,
5546 "critical module health alert"
5547 );
5548 } else {
5549 warn!(
5550 module_id = %spec.module_id,
5551 status = "unresponsive",
5552 evidence = err.label(),
5553 detail = %detail,
5554 "module health threshold breached"
5555 );
5556 }
5557 if let Err(err) = health_restart_child(
5558 spec,
5559 runtime,
5560 registry,
5561 process_liveness,
5562 snapshot,
5563 child,
5564 SupervisorHealthStatus::Unresponsive,
5565 Some(&detail),
5566 now_ms,
5567 )
5568 .await
5569 {
5570 error!(module_id = %spec.module_id, error = %err, "health-triggered restart failed");
5571 }
5572}
5573
5574#[allow(clippy::too_many_arguments)]
5575async fn apply_l3_health_action(
5576 spec: &ModuleSpec,
5577 runtime: &SupervisorRuntimeConfig,
5578 registry: &Registry,
5579 process_liveness: &SupervisorProcessLiveness,
5580 snapshot: &SharedSnapshot,
5581 child: &mut Option<SupervisedChild>,
5582 status: SupervisorHealthStatus,
5583 detail: Option<&str>,
5584 action: HealthAction,
5585 now_ms: u64,
5586) {
5587 record_health_action(snapshot, &spec.module_id, action.to_string(), now_ms);
5588 match action {
5589 HealthAction::Report => {
5590 info!(
5591 module_id = %spec.module_id,
5592 status = ?status,
5593 detail,
5594 "module reported non-ok health"
5595 );
5596 }
5597 HealthAction::Alert => {
5598 error!(
5599 module_id = %spec.module_id,
5600 status = ?status,
5601 detail,
5602 "module health alert"
5603 );
5604 }
5605 HealthAction::Restart => {
5606 if let Err(err) = health_restart_child(
5607 spec,
5608 runtime,
5609 registry,
5610 process_liveness,
5611 snapshot,
5612 child,
5613 status,
5614 detail,
5615 now_ms,
5616 )
5617 .await
5618 {
5619 error!(module_id = %spec.module_id, error = %err, "health-triggered restart failed");
5620 }
5621 }
5622 }
5623}
5624
5625#[allow(clippy::too_many_arguments)]
5626async fn health_restart_child(
5627 spec: &ModuleSpec,
5628 runtime: &SupervisorRuntimeConfig,
5629 registry: &Registry,
5630 process_liveness: &SupervisorProcessLiveness,
5631 snapshot: &SharedSnapshot,
5632 child: &mut Option<SupervisedChild>,
5633 status: SupervisorHealthStatus,
5634 detail: Option<&str>,
5635 now_ms: u64,
5636) -> Result<(), SuperviseError> {
5637 let (enabled, schedule) = {
5638 let mut state = lock_snapshot(snapshot)?;
5639 let enabled = state.enabled;
5640 let schedule = if enabled {
5641 state.next_crash_restart(&runtime.restart_policy, Instant::now())
5642 } else {
5643 None
5644 };
5645 (enabled, schedule)
5646 };
5647
5648 if !enabled {
5649 return Err(SuperviseError::Disabled {
5650 module_id: spec.module_id.clone(),
5651 });
5652 }
5653
5654 if schedule.is_none() {
5655 record_health_action(snapshot, &spec.module_id, "failed".to_string(), now_ms);
5656 error!(
5657 module_id = %spec.module_id,
5658 status = ?status,
5659 detail,
5660 max_restarts = runtime.restart_policy.max_restarts,
5661 window_secs = runtime.restart_policy.window.as_secs(),
5662 reason = %runtime.restart_policy.budget_exhausted_detail(),
5663 "health restart budget exhausted; marking module failed"
5664 );
5665 let stop_notice = begin_forwarding_drain_if_configured(
5666 spec,
5667 runtime,
5668 registry,
5669 snapshot,
5670 Some(true),
5671 RouteCloseReason::Disable,
5672 )
5673 .await?;
5674 update_snapshot(snapshot, Some(&spec.module_id), |state| {
5675 state.drain_disposition_detail = Some(runtime.restart_policy.budget_exhausted_detail());
5676 })?;
5677 let result = drain_optional_child(
5678 &spec.module_id,
5679 spec.protocol,
5680 stop_notice,
5681 registry,
5682 runtime.forwarding.as_deref(),
5683 snapshot,
5684 &runtime.terminal_ring,
5685 &runtime.spawn_events,
5686 child,
5687 runtime.drain_timeout,
5688 ModuleState::Failed,
5689 Some(true),
5690 &runtime.failure_context(true),
5691 )
5692 .await;
5693 update_snapshot(snapshot, Some(&spec.module_id), |state| {
5694 if state.state == ModuleState::Failed && !runtime.child_roster.is_closed() {
5695 state.failed_retry.schedule(&spec.module_id);
5696 }
5697 })?;
5698 result?;
5699 process_liveness.untrack_if_current(&spec.module_id, snapshot);
5700 return Ok(());
5701 }
5702
5703 let schedule = schedule.expect("a health restart must have a crash-restart schedule");
5704 let mut restart_count = 0;
5705 update_snapshot(snapshot, Some(&spec.module_id), |state| {
5706 restart_count = state.crash_restarts.len();
5707 state.state = ModuleState::Unresponsive;
5708 state.health.status = status;
5709 state.health.last_action = Some(HealthAction::Restart.to_string());
5710 state.health.last_action_ms = Some(now_ms);
5711 })?;
5712 warn!(
5713 module_id = %spec.module_id,
5714 status = ?status,
5715 detail,
5716 restart_count,
5717 restart_in_window = schedule.restart_in_window,
5718 delay_ms = schedule.delay.as_millis() as u64,
5719 "health-triggered module restart"
5720 );
5721
5722 let stop_notice = begin_forwarding_drain_if_configured(
5723 spec,
5724 runtime,
5725 registry,
5726 snapshot,
5727 Some(true),
5728 RouteCloseReason::Restart,
5729 )
5730 .await?;
5731 drain_optional_child(
5732 &spec.module_id,
5733 spec.protocol,
5734 stop_notice,
5735 registry,
5736 runtime.forwarding.as_deref(),
5737 snapshot,
5738 &runtime.terminal_ring,
5739 &runtime.spawn_events,
5740 child,
5741 runtime.drain_timeout,
5742 ModuleState::Restarting,
5743 Some(true),
5744 &runtime.failure_context(true),
5745 )
5746 .await?;
5747 schedule_respawn(
5748 runtime,
5749 snapshot,
5750 &spec.module_id,
5751 schedule.delay,
5752 RespawnKind::Spawn,
5753 true,
5754 )
5755}
5756
5757fn cancel_deferred_reload(runtime: &SupervisorRuntimeConfig, module_id: &str, reason: &str) {
5758 if let Some(reply) = runtime
5759 .deferred_reload_reply
5760 .lock()
5761 .unwrap_or_else(|p| p.into_inner())
5762 .take()
5763 {
5764 let _ = reply.send(Err(SuperviseError::ReloadFailed {
5765 module_id: module_id.to_string(),
5766 reason: reason.to_string(),
5767 }));
5768 }
5769}
5770
5771fn schedule_respawn(
5772 runtime: &SupervisorRuntimeConfig,
5773 snapshot: &SharedSnapshot,
5774 module_id: &str,
5775 delay: Duration,
5776 kind: RespawnKind,
5777 daemon_initiated: bool,
5778) -> Result<(), SuperviseError> {
5779 cancel_deferred_reload(runtime, module_id, "respawn superseded by another restart");
5780 update_snapshot(snapshot, Some(module_id), |state| {
5781 state.respawn_pending = true
5782 })?;
5783 *runtime
5784 .scheduled_respawn
5785 .lock()
5786 .unwrap_or_else(|p| p.into_inner()) = Some(PendingRespawn {
5787 deadline: Instant::now() + delay,
5788 kind,
5789 daemon_initiated,
5790 });
5791 Ok(())
5792}
5793
5794fn record_health_action(snapshot: &SharedSnapshot, module_id: &str, action: String, now_ms: u64) {
5795 let _ = update_snapshot(snapshot, Some(module_id), |state| {
5796 state.health.last_action = Some(action);
5797 state.health.last_action_ms = Some(now_ms);
5798 });
5799}
5800
5801fn supervisor_health_status(status: HealthStatus) -> SupervisorHealthStatus {
5802 match status {
5803 HealthStatus::Ok => SupervisorHealthStatus::Ok,
5804 HealthStatus::Degraded => SupervisorHealthStatus::Degraded,
5805 HealthStatus::Failing => SupervisorHealthStatus::Failing,
5806 }
5807}
5808
5809fn truncate_health_metrics(metrics: Option<Value>) -> Option<Value> {
5821 let metrics = metrics?;
5822 match serde_json::to_vec(&metrics) {
5823 Ok(encoded) if encoded.len() > MAX_HEALTH_METRICS_BYTES => Some(serde_json::json!({
5824 "truncated": true,
5825 "original_bytes": encoded.len(),
5826 })),
5827 Ok(_) | Err(_) => Some(metrics),
5828 }
5829}
5830
5831fn jittered_health_delay(module_id: &str, probe_index: u64, cadence: Duration) -> Duration {
5837 if cadence.is_zero() {
5838 return Duration::ZERO;
5839 }
5840 let cadence_ms = cadence.as_millis() as u64;
5841 if cadence_ms == 0 {
5857 return cadence;
5858 }
5859 let jitter_span = (cadence_ms / 10).max(1);
5874 let hash = module_id.as_bytes().iter().fold(
5875 probe_index.wrapping_mul(0x9E37_79B9_7F4A_7C15),
5876 |acc, byte| {
5877 acc.wrapping_mul(1099511628211)
5878 .wrapping_add(u64::from(*byte))
5879 },
5880 );
5881 cadence + Duration::from_millis(hash % jitter_span)
5882}
5883
5884#[cfg(test)]
5885mod tests {
5886 use super::*;
5887
5888 #[test]
5889 fn readding_a_module_clears_its_rescan_removal_tombstone() {
5890 let handle = SupervisorHandle::new();
5891 let module_id = "readded-tombstone";
5892 handle.record_rescan_removal(module_id);
5893 assert!(handle.removal_tombstone_age_ms(module_id).is_some());
5894
5895 handle.apply_identity_configuration(&ModuleSpec {
5896 module_id: module_id.to_string(),
5897 program: PathBuf::from("/test/module"),
5898 args: Vec::new(),
5899 env: Vec::new(),
5900 reserved: false,
5901 reserved_prefixes: Vec::new(),
5902 protocol: ModuleProtocol::Subc,
5903 overlap: Default::default(),
5904 carry_containment: false,
5905 });
5906
5907 assert!(
5908 handle.removal_tombstone_age_ms(module_id).is_none(),
5909 "a re-added module must not retain a stale removal tombstone"
5910 );
5911 }
5912
5913 #[derive(Debug, PartialEq, Eq)]
5916 struct OwnerInSpawnWindow {
5917 module_id: String,
5918 configured: bool,
5919 on_roster: bool,
5920 admission_refusal: Option<&'static str>,
5921 }
5922
5923 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
5934 async fn a_new_module_is_configured_before_its_first_process_can_register() {
5935 use crate::scopes::ScopeTable;
5936 use subc_protocol::{error_codes, scope::ScopeSelector, Principal};
5937
5938 let dir = cortexkit_test_support::ScratchDir::new("configured-before-spawn");
5939 let stub = |module_id: &str, program: PathBuf| ModuleSpec {
5940 module_id: module_id.to_string(),
5941 program,
5942 args: Vec::new(),
5943 env: ["XDG_DATA_HOME", "XDG_RUNTIME_DIR", "XDG_CONFIG_HOME"]
5944 .into_iter()
5945 .map(|key| (key.to_string(), dir.path().display().to_string()))
5946 .chain([("FAKE_AFT_NEVER_CONNECT".to_string(), "1".to_string())])
5947 .collect(),
5948 reserved: false,
5949 reserved_prefixes: Vec::new(),
5950 protocol: ModuleProtocol::Subc,
5951 overlap: Default::default(),
5952 carry_containment: false,
5953 };
5954 let live = super::terminal_history_tests::fake_aft_stub_path();
5955 let missing = dir.path().join("definitely-missing-module");
5956
5957 let handle = SupervisorHandle::new();
5958 let mut supervisor =
5959 Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default())
5960 .with_handle(handle.clone());
5961 let observed = Arc::new(Mutex::new(Vec::<OwnerInSpawnWindow>::new()));
5962 let hook_handle = handle.clone();
5963 let hook_observed = Arc::clone(&observed);
5964 supervisor.test_after_first_spawn = AfterFirstSpawnHook(Some(Arc::new(move |module_id| {
5965 let configured = hook_handle.is_configured(module_id);
5968 let selector = ScopeSelector::new(
5969 Principal::Reserved {
5970 module_id: module_id.to_string(),
5971 },
5972 "s".to_string(),
5973 Some(1),
5974 );
5975 let carrier = Principal::Reserved {
5976 module_id: "carrier".to_string(),
5977 };
5978 let admission_refusal = match ScopeTable::new(Vec::<String>::new())
5979 .admit(&carrier, module_id, &selector, configured)
5980 {
5981 Ok(_) => None,
5982 Err(refusal) => Some(refusal.code),
5983 };
5984 hook_observed.lock().unwrap().push(OwnerInSpawnWindow {
5985 module_id: module_id.to_string(),
5986 configured,
5987 on_roster: hook_handle.get(module_id).is_some(),
5988 admission_refusal,
5989 });
5990 })));
5991
5992 let plain = supervisor.spawn(stub("plain", live.clone())).unwrap();
5993 let configured = supervisor
5994 .supervise_configured(stub("configured", live.clone()), true)
5995 .unwrap();
5996 let with_health = supervisor
5997 .supervise_configured_with_health(
5998 stub("with-health", live.clone()),
5999 true,
6000 HealthConfig::default(),
6001 None,
6002 RestartPolicy::default(),
6003 )
6004 .unwrap();
6005 let failed = supervisor
6008 .supervise_configured_with_health(
6009 stub("failed-spawn", missing.clone()),
6010 true,
6011 HealthConfig::default(),
6012 None,
6013 RestartPolicy::default(),
6014 )
6015 .unwrap();
6016 assert!(supervisor.spawn(stub("spawn-error", missing)).is_err());
6019
6020 let expected = [
6021 "plain",
6022 "configured",
6023 "with-health",
6024 "failed-spawn",
6025 "spawn-error",
6026 ]
6027 .into_iter()
6028 .map(|module_id| OwnerInSpawnWindow {
6029 module_id: module_id.to_string(),
6030 configured: true,
6031 on_roster: !matches!(module_id, "plain" | "spawn-error"),
6032 admission_refusal: Some(error_codes::SCOPE_NOT_SYNCED),
6033 })
6034 .collect::<Vec<_>>();
6035 assert_eq!(*observed.lock().unwrap(), expected);
6036
6037 for module_id in ["plain", "configured", "with-health", "failed-spawn"] {
6038 assert!(
6039 handle.get(module_id).is_some(),
6040 "{module_id} is on the roster"
6041 );
6042 assert!(
6043 handle.is_configured(module_id),
6044 "{module_id} stays configured"
6045 );
6046 }
6047 assert!(handle.get("spawn-error").is_none());
6048 assert!(
6049 !handle.is_configured("spawn-error"),
6050 "a plain spawn that failed must not leave its module marked configured"
6051 );
6052
6053 handle.retire("failed-spawn");
6055 assert!(!handle.is_configured("failed-spawn"));
6056
6057 for module in [plain, configured, with_health] {
6058 module.stop().await.unwrap();
6059 }
6060 drop(failed);
6061 }
6062
6063 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
6070 async fn a_configured_module_reads_starting_while_its_first_process_already_runs() {
6071 let dir = cortexkit_test_support::ScratchDir::new("starting-while-spawned");
6072 let spec = ModuleSpec {
6073 module_id: "spawned-not-admitted".to_string(),
6074 program: super::terminal_history_tests::fake_aft_stub_path(),
6075 args: Vec::new(),
6076 env: ["XDG_DATA_HOME", "XDG_RUNTIME_DIR", "XDG_CONFIG_HOME"]
6077 .into_iter()
6078 .map(|key| (key.to_string(), dir.path().display().to_string()))
6079 .chain([("FAKE_AFT_NEVER_CONNECT".to_string(), "1".to_string())])
6080 .collect(),
6081 reserved: false,
6082 reserved_prefixes: Vec::new(),
6083 protocol: ModuleProtocol::Subc,
6084 overlap: Default::default(),
6085 carry_containment: false,
6086 };
6087 let handle = SupervisorHandle::new();
6088 let mut supervisor =
6089 Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default())
6090 .with_handle(handle.clone());
6091 let observed = Arc::new(Mutex::new(None));
6092 let hook_handle = handle.clone();
6093 let hook_observed = Arc::clone(&observed);
6094 supervisor.test_after_first_spawn = AfterFirstSpawnHook(Some(Arc::new(move |module_id| {
6095 let state = hook_handle
6096 .get(module_id)
6097 .expect("a configured module is on the roster before it is spawned")
6098 .state()
6099 .unwrap();
6100 *hook_observed.lock().unwrap() = Some(state);
6101 })));
6102
6103 let module = supervisor.supervise_configured(spec, true).unwrap();
6104 assert_eq!(
6105 *observed.lock().unwrap(),
6106 Some(ModuleState::Starting),
6107 "between the spawn and its admission the supervisor still reports starting"
6108 );
6109 assert_eq!(module.state().unwrap(), ModuleState::Running);
6110 module.stop().await.unwrap();
6111 }
6112
6113 fn stale_process_snapshot(state: ModuleState, enabled: bool) -> SharedSnapshot {
6114 let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::new(state, enabled)));
6115 update_snapshot(&snapshot, Some("stale-process-facts"), |snapshot| {
6116 snapshot.process_alive = true;
6117 snapshot.pid = Some(41);
6118 snapshot.spawned_at_ms = Some(42);
6119 snapshot.spawned_from = Some(PathBuf::from("/spawned/module"));
6120 snapshot.launch_spec_digest = Some(LaunchSpecDigest([0xA5; 32]));
6121 snapshot.spawned_file_identity = Some(SpawnedFileIdentity {
6122 device: 43,
6123 inode: 44,
6124 #[cfg(windows)]
6125 windows_image: None,
6126 });
6127 })
6128 .unwrap();
6129 snapshot
6130 }
6131
6132 fn assert_snapshot_process_facts_cleared(snapshot: &SharedSnapshot) {
6133 let snapshot = lock_snapshot(snapshot).unwrap();
6134 assert!(!snapshot.process_alive);
6135 assert_eq!(snapshot.pid, None);
6136 assert_eq!(snapshot.spawned_at_ms, None);
6137 assert_eq!(snapshot.spawned_from, None);
6138 assert_eq!(snapshot.launch_spec_digest, None);
6139 assert_eq!(snapshot.spawned_file_identity, None);
6140 }
6141
6142 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
6143 async fn failed_enable_spawn_clears_preexisting_current_process_facts() {
6144 let supervisor =
6145 Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default());
6146 let mut runtime = supervisor.runtime_config();
6147 runtime.test_seed_stale_facts_before_enable_spawn = true;
6148 let snapshot = stale_process_snapshot(ModuleState::Disabled, false);
6149 let mut child = None;
6150 let spec = ModuleSpec {
6151 module_id: "failed-enable-clears-facts".to_string(),
6152 program: PathBuf::from("/definitely/missing/failed-enable-module"),
6153 args: Vec::new(),
6154 env: Vec::new(),
6155 reserved: false,
6156 reserved_prefixes: Vec::new(),
6157 protocol: ModuleProtocol::Subc,
6158 overlap: Default::default(),
6159 carry_containment: false,
6160 };
6161
6162 let result = set_child_enabled(
6163 &spec,
6164 &runtime,
6165 &supervisor.registry,
6166 &supervisor.process_liveness,
6167 &snapshot,
6168 &mut child,
6169 true,
6170 &runtime.failure_context(false),
6171 )
6172 .await;
6173
6174 assert!(matches!(result, Err(SuperviseError::Spawn { .. })));
6175 assert_eq!(lock_snapshot(&snapshot).unwrap().state, ModuleState::Failed);
6176 assert_snapshot_process_facts_cleared(&snapshot);
6177 }
6178
6179 #[tokio::test]
6180 async fn start_revives_stranded_restarting_but_not_pending_backoff() {
6181 let supervisor =
6182 Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default());
6183 let runtime = supervisor.runtime_config();
6184 let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::new(
6185 ModuleState::Restarting,
6186 true,
6187 )));
6188 let spec = ModuleSpec {
6189 module_id: "start-stranded-restarting".to_string(),
6190 program: super::terminal_history_tests::fake_aft_stub_path(),
6191 args: Vec::new(),
6192 env: vec![("FAKE_AFT_NEVER_CONNECT".to_string(), "1".to_string())],
6193 reserved: false,
6194 reserved_prefixes: Vec::new(),
6195 protocol: ModuleProtocol::None,
6196 overlap: Default::default(),
6197 carry_containment: false,
6198 };
6199 let mut child = None;
6200 lock_snapshot(&snapshot).unwrap().respawn_pending = true;
6201 assert!(!super::set_child_enabled(
6202 &spec,
6203 &runtime,
6204 &Registry::default(),
6205 &supervisor.process_liveness,
6206 &snapshot,
6207 &mut child,
6208 true,
6209 &runtime.failure_context(false),
6210 )
6211 .await
6212 .unwrap());
6213 assert!(child.is_none());
6214 lock_snapshot(&snapshot).unwrap().respawn_pending = false;
6215 assert!(super::set_child_enabled(
6216 &spec,
6217 &runtime,
6218 &Registry::default(),
6219 &supervisor.process_liveness,
6220 &snapshot,
6221 &mut child,
6222 true,
6223 &runtime.failure_context(false),
6224 )
6225 .await
6226 .unwrap());
6227 assert_eq!(
6228 lock_snapshot(&snapshot).unwrap().state,
6229 ModuleState::Running
6230 );
6231 let mut child = child.unwrap();
6232 child.start_kill().unwrap();
6233 child.wait().await.unwrap();
6234 }
6235
6236 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
6237 async fn failed_reload_spawn_clears_current_process_facts() {
6238 let supervisor =
6239 Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default());
6240 let mut runtime = supervisor.runtime_config();
6241 runtime.restart_policy = RestartPolicy::new(0, Duration::ZERO);
6242 let snapshot = stale_process_snapshot(ModuleState::Running, true);
6243 let mut child = None;
6244 let spec = ModuleSpec {
6245 module_id: "failed-reload-clears-facts".to_string(),
6246 program: PathBuf::from("/unused/failed-reload-module"),
6247 args: Vec::new(),
6248 env: Vec::new(),
6249 reserved: false,
6250 reserved_prefixes: Vec::new(),
6251 protocol: ModuleProtocol::Subc,
6252 overlap: Default::default(),
6253 carry_containment: false,
6254 };
6255
6256 let result = handle_reload_spawn_failure(
6257 &spec,
6258 &runtime,
6259 &supervisor.process_liveness,
6260 &snapshot,
6261 &mut child,
6262 "forced reload spawn failure".to_string(),
6263 )
6264 .await;
6265
6266 assert!(matches!(result, Err(SuperviseError::ReloadFailed { .. })));
6267 assert_eq!(lock_snapshot(&snapshot).unwrap().state, ModuleState::Failed);
6268 assert_snapshot_process_facts_cleared(&snapshot);
6269 }
6270
6271 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
6272 async fn dropping_a_module_with_an_active_monitor_clears_current_process_facts() {
6273 let supervisor =
6274 Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default());
6275 let snapshot = stale_process_snapshot(ModuleState::Running, true);
6276 let module = supervisor.supervised_module(
6277 ModuleSpec {
6278 module_id: "drop-clears-facts".to_string(),
6279 program: PathBuf::from("/unused/drop-module"),
6280 args: Vec::new(),
6281 env: Vec::new(),
6282 reserved: false,
6283 reserved_prefixes: Vec::new(),
6284 protocol: ModuleProtocol::Subc,
6285 overlap: Default::default(),
6286 carry_containment: false,
6287 },
6288 supervisor.runtime_config(),
6289 Arc::clone(&snapshot),
6290 None,
6291 );
6292 assert!(!module
6293 .inner
6294 .monitor
6295 .lock()
6296 .unwrap()
6297 .as_ref()
6298 .unwrap()
6299 .is_finished());
6300
6301 drop(module);
6302
6303 assert_eq!(
6304 lock_snapshot(&snapshot).unwrap().state,
6305 ModuleState::Stopped
6306 );
6307 assert_snapshot_process_facts_cleared(&snapshot);
6308 }
6309
6310 #[cfg(unix)]
6311 #[tokio::test]
6312 async fn rescan_preserves_running_protocol_until_respawn() {
6313 let dir = cortexkit_test_support::ScratchDir::new("rescan-protocol");
6314 let initial = ModuleSpec {
6315 module_id: "rescan-protocol".into(),
6316 program: PathBuf::from("/bin/sleep"),
6317 args: vec!["60".into()],
6318 env: ["XDG_DATA_HOME", "XDG_RUNTIME_DIR", "XDG_CONFIG_HOME"]
6319 .into_iter()
6320 .map(|key| (key.into(), dir.path().to_string_lossy().into_owned()))
6321 .collect(),
6322 reserved: false,
6323 reserved_prefixes: vec![],
6324 protocol: ModuleProtocol::None,
6325 overlap: Default::default(),
6326 carry_containment: false,
6327 };
6328 let supervisor =
6329 Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default());
6330 let module = supervisor.spawn(initial.clone()).unwrap();
6331 assert!(module.status().unwrap().live);
6332 let mut next = initial;
6333 next.protocol = ModuleProtocol::Subc;
6334 module
6335 .update_configuration(next.clone(), HealthConfig::default(), None)
6336 .await
6337 .unwrap();
6338 assert!(
6339 module.status().unwrap().live,
6340 "rescan must not require HELLO from the old non-wire process"
6341 );
6342 let runtime = supervisor.runtime_config();
6343 let action = on_child_exit(
6344 &next,
6345 RestartPolicy::default(),
6346 &supervisor.registry,
6347 &module.inner.snapshot,
6348 &runtime.terminal_ring,
6349 &runtime.spawn_events,
6350 &runtime.child_roster,
6351 ExitReport {
6352 kind: ExitKind::Clean,
6353 code: Some(0),
6354 signal: None,
6355 at_ms: unix_ms_now(),
6356 },
6357 true,
6358 )
6359 .await;
6360 assert!(
6361 matches!(action, NextAction::Restart { .. }),
6362 "the old non-wire process's clean exit must restart"
6363 );
6364 module.drain().await.unwrap();
6365 }
6366
6367 #[cfg(unix)]
6368 #[tokio::test]
6369 async fn orphan_identity_matches_path_names_and_shebang_interpreters() {
6370 use std::os::unix::fs::PermissionsExt;
6371 let dir = cortexkit_test_support::ScratchDir::new("spawn-image-identity");
6372 let script = dir.join("module.sh");
6373 std::fs::write(&script, "#!/bin/sh\nwhile :; do sleep 0.1; done\n").unwrap();
6374 std::fs::set_permissions(&script, std::fs::Permissions::from_mode(0o700)).unwrap();
6375 let record_path = dir.join("live-children.json");
6376 let supervisor =
6377 Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default())
6378 .with_live_children_record(&record_path);
6379 for (program, args) in [
6380 (PathBuf::from("sleep"), vec!["60".into()]),
6381 (script, vec![]),
6382 ] {
6383 let spec = ModuleSpec {
6384 module_id: "image-identity".into(),
6385 program: program.clone(),
6386 args,
6387 env: ["XDG_DATA_HOME", "XDG_RUNTIME_DIR", "XDG_CONFIG_HOME"]
6388 .into_iter()
6389 .map(|key| (key.into(), dir.to_string_lossy().into_owned()))
6390 .collect(),
6391 reserved: false,
6392 reserved_prefixes: vec![],
6393 protocol: ModuleProtocol::None,
6394 overlap: Default::default(),
6395 carry_containment: false,
6396 };
6397 let module = supervisor.spawn(spec).unwrap();
6398 #[cfg(target_os = "macos")]
6399 {
6400 let deadline = Instant::now() + Duration::from_secs(5);
6403 while crate::live_children::read_record(&record_path)
6404 .unwrap()
6405 .iter()
6406 .all(|entry| entry.executable.is_none())
6407 {
6408 assert!(Instant::now() < deadline, "module image was not confirmed");
6409 tokio::time::sleep(Duration::from_millis(5)).await;
6410 }
6411 }
6412 let entry = crate::live_children::read_record(&record_path)
6413 .unwrap()
6414 .pop()
6415 .unwrap();
6416 let observed = subc_os::Process::open(entry.pid)
6417 .unwrap()
6418 .unwrap()
6419 .observe()
6420 .unwrap();
6421 let verdict = crate::live_children::identity_verdict(&entry, entry.pid, &observed);
6422 module.drain().await.unwrap();
6423 assert_eq!(
6424 verdict,
6425 crate::live_children::IdentityVerdict::Matches,
6426 "program {program:?}: recorded {entry:?}, observed {observed:?}"
6427 );
6428 }
6429 }
6430
6431 #[cfg(unix)]
6432 fn http_fixture(
6433 dir: &std::path::Path,
6434 url: &str,
6435 threshold: u32,
6436 ) -> crate::daemon_config::ConfiguredModule {
6437 let path = dir.join("subc.jsonc");
6438 std::fs::write(&path, serde_json::json!({"version": 1, "modules": {"http-process": {
6439 "program": "/bin/sleep", "args": ["60"], "protocol": "none",
6440 "env": {"XDG_DATA_HOME":dir,"XDG_RUNTIME_DIR":dir,"XDG_CONFIG_HOME":dir},
6441 "health": {"http":url,"cadence_ms":20,"deadline_ms":200,"failure_threshold":threshold},
6442 "restart": {"backoff_ms":1,"max_backoff_ms":1}, "drain_timeout_ms":10
6443 }}}).to_string()).unwrap();
6444 crate::daemon_config::load(&path)
6445 .unwrap()
6446 .unwrap()
6447 .modules
6448 .pop()
6449 .unwrap()
6450 }
6451
6452 #[cfg(unix)]
6453 async fn wait_http_health(module: &SupervisedModule, status: SupervisorHealthStatus) {
6454 timeout(Duration::from_secs(5), async {
6455 loop {
6456 if module.status().unwrap().health.status == status {
6457 break;
6458 }
6459 sleep(Duration::from_millis(5)).await;
6460 }
6461 })
6462 .await
6463 .unwrap_or_else(|_| {
6464 panic!(
6465 "expected {status:?}, got {:?}",
6466 module.status().unwrap().health
6467 )
6468 });
6469 }
6470
6471 #[cfg(unix)]
6472 #[tokio::test]
6473 async fn http_health_status_flips_ok_failing_ok() {
6474 use tokio::io::{AsyncReadExt, AsyncWriteExt};
6475 let dir = cortexkit_test_support::ScratchDir::new("http-status-flips");
6476 let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
6477 let status = Arc::new(std::sync::atomic::AtomicU16::new(200));
6478 let serving_status = status.clone();
6479 let url = format!("http://{}/healthz", listener.local_addr().unwrap());
6480 let server = tokio::spawn(async move {
6481 loop {
6482 let (mut stream, _) = listener.accept().await.unwrap();
6483 let mut request = [0u8; 2048];
6484 let count = stream.read(&mut request).await.unwrap();
6485 assert!(count > 0, "a probe must send an HTTP request");
6486 let code = serving_status.load(std::sync::atomic::Ordering::SeqCst);
6487 let body = if code == 200 {
6488 "ready"
6489 } else {
6490 "{\"status\":\"unavailable\",\"error\":\"scratch failure\"}"
6491 };
6492 let response = format!("HTTP/1.1 {code} scratch\r\nContent-Length: {}\r\nConnection: close\r\n\r\n{body}", body.len());
6493 let _ = stream.write_all(response.as_bytes()).await;
6494 }
6495 });
6496 let configured = http_fixture(&dir, &url, 1000);
6497 let module =
6498 Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default())
6499 .supervise_configured_with_health(
6500 configured.module_spec(),
6501 true,
6502 configured.health,
6503 configured.drain_timeout_ms,
6504 configured.restart,
6505 )
6506 .unwrap();
6507 wait_http_health(&module, SupervisorHealthStatus::Ok).await;
6508 status.store(503, std::sync::atomic::Ordering::SeqCst);
6509 wait_http_health(&module, SupervisorHealthStatus::Failing).await;
6510 assert!(module
6511 .status()
6512 .unwrap()
6513 .health
6514 .detail
6515 .unwrap()
6516 .contains("scratch failure"));
6517 status.store(200, std::sync::atomic::Ordering::SeqCst);
6518 wait_http_health(&module, SupervisorHealthStatus::Ok).await;
6519 assert_eq!(module.status().unwrap().health.consecutive_failures, 0);
6520 let before = module.status().unwrap();
6521 let (spec, mut health) = module.configuration().unwrap();
6522 health.http = None;
6523 module
6524 .update_configuration(spec.clone(), health.clone(), Some(10))
6525 .await
6526 .unwrap();
6527 wait_http_health(&module, SupervisorHealthStatus::Unknown).await;
6528 health.http = Some(url);
6529 module
6530 .update_configuration(spec, health, Some(10))
6531 .await
6532 .unwrap();
6533 wait_http_health(&module, SupervisorHealthStatus::Ok).await;
6534 assert_eq!(
6535 module.status().unwrap().pid,
6536 before.pid,
6537 "changing a probe must apply live, not restart its process"
6538 );
6539 let (spec, mut health) = module.configuration().unwrap();
6540 health.failure_threshold = 2;
6541 module
6542 .update_configuration(spec, health, Some(10))
6543 .await
6544 .unwrap();
6545 status.store(503, std::sync::atomic::Ordering::SeqCst);
6546 timeout(Duration::from_secs(5), async {
6547 while module.status().unwrap().spawn_generation == before.spawn_generation {
6548 sleep(Duration::from_millis(5)).await;
6549 }
6550 })
6551 .await
6552 .expect("sustained HTTP 503 responses must trigger the health restart policy");
6553 module.drain().await.unwrap();
6554 server.abort();
6555 }
6556
6557 #[cfg(unix)]
6558 #[tokio::test]
6559 async fn http_health_replacement_recovers_after_one_restart() {
6560 use tokio::io::{AsyncReadExt, AsyncWriteExt};
6561 let dir = cortexkit_test_support::ScratchDir::new("http-replacement-recovers");
6562 let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
6563 let url = format!("http://{}/healthz", listener.local_addr().unwrap());
6564 let mut configured = http_fixture(&dir, &url, 2);
6565 configured.restart.max_restarts = 2;
6566 let module =
6567 Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default())
6568 .supervise_configured_with_health(
6569 configured.module_spec(),
6570 true,
6571 configured.health,
6572 configured.drain_timeout_ms,
6573 configured.restart,
6574 )
6575 .unwrap();
6576 let first_generation = module.status().unwrap().spawn_generation;
6577 let observed = module.clone();
6578 let (healthy_tx, mut healthy_rx) = mpsc::channel(8);
6579 let server = tokio::spawn(async move {
6580 let mut generation = first_generation;
6581 let mut probes = 0;
6582 loop {
6583 let (mut stream, _) = listener.accept().await.unwrap();
6584 let mut request = [0; 2048];
6585 assert!(stream.read(&mut request).await.unwrap() > 0);
6586 let current = observed.status().unwrap().spawn_generation;
6587 if current != generation {
6588 generation = current;
6589 probes = 0;
6590 }
6591 probes += 1;
6592 let healthy = generation != first_generation && probes > 1;
6595 let code = if healthy { 200 } else { 503 };
6596 let response = format!(
6597 "HTTP/1.1 {code} scratch\r\nContent-Length: 0\r\nConnection: close\r\n\r\n"
6598 );
6599 let _ = stream.write_all(response.as_bytes()).await;
6600 if healthy {
6601 healthy_tx.send(()).await.unwrap();
6602 }
6603 }
6604 });
6605 timeout(Duration::from_secs(5), async {
6606 let mut healthy_probes = 0;
6607 loop {
6608 if module.state().unwrap() == ModuleState::Failed {
6609 break;
6610 }
6611 while healthy_rx.try_recv().is_ok() {
6612 healthy_probes += 1;
6613 }
6614 if healthy_probes >= 3
6615 && module.status().unwrap().health.status == SupervisorHealthStatus::Ok
6616 {
6617 break;
6618 }
6619 sleep(Duration::from_millis(5)).await;
6620 }
6621 })
6622 .await
6623 .expect("the replacement must settle healthy without exhausting its restart budget");
6624 let status = module.status().unwrap();
6625 module.drain().await.unwrap();
6626 server.abort();
6627 assert_eq!(status.state, ModuleState::Running);
6628 assert_eq!(status.health.status, SupervisorHealthStatus::Ok);
6629 assert_eq!(status.health.consecutive_failures, 0);
6630 assert_eq!(status.lifetime_restarts, 1);
6631 assert_eq!(status.restart_count, 1);
6632 }
6633
6634 #[cfg(unix)]
6635 #[tokio::test]
6636 async fn http_health_sustained_failure_exhausts_restart_budget() {
6637 use std::collections::BTreeMap;
6638 use tokio::io::{AsyncReadExt, AsyncWriteExt};
6639 let dir = cortexkit_test_support::ScratchDir::new("http-health-budget");
6640 let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
6641 let url = format!("http://{}/healthz", listener.local_addr().unwrap());
6642 let mut configured = http_fixture(&dir, &url, 2);
6643 configured.restart.max_restarts = 2;
6644 let module =
6645 Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default())
6646 .supervise_configured_with_health(
6647 configured.module_spec(),
6648 true,
6649 configured.health,
6650 configured.drain_timeout_ms,
6651 configured.restart,
6652 )
6653 .unwrap();
6654 let observed = module.clone();
6655 let probes = Arc::new(Mutex::new(BTreeMap::<u64, u32>::new()));
6656 let served = probes.clone();
6657 let server = tokio::spawn(async move {
6658 loop {
6659 let (mut stream, _) = listener.accept().await.unwrap();
6660 let mut request = [0; 2048];
6661 assert!(stream.read(&mut request).await.unwrap() > 0);
6662 *served
6663 .lock()
6664 .unwrap()
6665 .entry(observed.status().unwrap().spawn_generation)
6666 .or_default() += 1;
6667 let _ = stream
6668 .write_all(b"HTTP/1.1 503 Unavailable\r\nContent-Length: 0\r\nConnection: close\r\n\r\n")
6669 .await;
6670 }
6671 });
6672 timeout(Duration::from_secs(5), async {
6673 while module.state().unwrap() != ModuleState::Failed {
6674 sleep(Duration::from_millis(5)).await;
6675 }
6676 })
6677 .await
6678 .expect("sustained failures must still exhaust the health restart budget");
6679 let status = module.status().unwrap();
6680 server.abort();
6681 assert_eq!(status.restart_count, 2);
6682 assert_eq!(status.lifetime_restarts, 2);
6683 assert!(!status.process_alive);
6684 assert_eq!(
6685 probes.lock().unwrap().values().copied().collect::<Vec<_>>(),
6686 vec![2, 2, 2]
6687 );
6688 }
6689
6690 #[cfg(unix)]
6691 #[tokio::test]
6692 async fn spawn_clears_the_previous_process_health() {
6693 let dir = cortexkit_test_support::ScratchDir::new("spawn-fresh-health");
6694 let mut configured = http_fixture(&dir, "http://127.0.0.1:1/healthz", 2);
6695 configured.health.cadence = Duration::from_secs(3600);
6696 let module =
6697 Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default())
6698 .supervise_configured_with_health(
6699 configured.module_spec(),
6700 false,
6701 configured.health,
6702 configured.drain_timeout_ms,
6703 configured.restart,
6704 )
6705 .unwrap();
6706 update_snapshot(&module.inner.snapshot, Some(module.module_id()), |state| {
6707 state.health.status = SupervisorHealthStatus::Unresponsive;
6708 state.health.consecutive_failures = 3;
6709 state.health.detail = Some("old process refused connections".into());
6710 state.health.metrics = Some(serde_json::json!({"old": true}));
6711 state.health.last_probe_ms = Some(123);
6712 state.health.last_action = Some("restart".into());
6713 state.health.last_action_ms = Some(124);
6714 })
6715 .unwrap();
6716 module.set_enabled(true).await.unwrap();
6717 let health = module.status().unwrap().health;
6718 module.drain().await.unwrap();
6719 assert_eq!(health.consecutive_failures, 0);
6720 assert_eq!(health.status, SupervisorHealthStatus::Unknown);
6721 assert_eq!(health.detail, None);
6722 assert_eq!(health.metrics, None);
6723 assert_eq!(health.last_probe_ms, None);
6724 assert_eq!(health.last_action.as_deref(), Some("restart"));
6725 assert_eq!(health.last_action_ms, Some(124));
6726 }
6727
6728 #[cfg(unix)]
6729 #[tokio::test]
6730 async fn http_health_no_listener_restarts_after_consecutive_failures() {
6731 let dir = cortexkit_test_support::ScratchDir::new("http-refused");
6732 let unused = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
6733 let url = format!("http://{}/healthz", unused.local_addr().unwrap());
6734 drop(unused);
6735 let configured = http_fixture(&dir, &url, 2);
6736 let module =
6737 Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default())
6738 .supervise_configured_with_health(
6739 configured.module_spec(),
6740 true,
6741 configured.health,
6742 configured.drain_timeout_ms,
6743 configured.restart,
6744 )
6745 .unwrap();
6746 let before = module.status().unwrap().spawn_generation;
6747 timeout(Duration::from_secs(5), async {
6748 loop {
6749 let status = module.status().unwrap();
6750 if status.spawn_generation > before {
6751 assert!(status.lifetime_restarts > 0);
6752 break;
6753 }
6754 sleep(Duration::from_millis(5)).await;
6755 }
6756 })
6757 .await
6758 .expect("sustained HTTP refusal must trigger the health restart policy");
6759 module.drain().await.unwrap();
6760 }
6761
6762 #[cfg(unix)]
6763 #[tokio::test]
6764 async fn http_health_timeout_honours_deadline() {
6765 let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
6766 let url = format!("http://{}/healthz", listener.local_addr().unwrap());
6767 let server = tokio::spawn(async move {
6768 let _held = listener.accept().await.unwrap();
6769 std::future::pending::<()>().await;
6770 });
6771 let error = timeout(
6772 Duration::from_secs(1),
6773 probe_http_health(&url, Duration::from_millis(10)),
6774 )
6775 .await
6776 .expect("the probe must enforce its own deadline")
6777 .unwrap_err();
6778 server.abort();
6779 assert!(error.to_string().contains("timed out"));
6780 }
6781
6782 #[cfg(unix)]
6783 #[tokio::test]
6784 async fn http_health_localhost_reaches_an_ipv6_only_listener() {
6785 use tokio::io::{AsyncReadExt, AsyncWriteExt};
6786 let listener = tokio::net::TcpListener::bind("[::1]:0").await.unwrap();
6787 let url = format!(
6788 "http://localhost:{}/healthz",
6789 listener.local_addr().unwrap().port()
6790 );
6791 let server = tokio::spawn(async move {
6792 let (mut stream, _) = listener.accept().await.unwrap();
6793 let mut request = [0u8; 2048];
6794 assert!(stream.read(&mut request).await.unwrap() > 0);
6795 stream
6796 .write_all(b"HTTP/1.1 200 OK\r\nContent-Length: 2\r\n\r\nok")
6797 .await
6798 .unwrap();
6799 });
6800 assert_eq!(
6805 probe_http_health(&url, Duration::from_secs(10))
6806 .await
6807 .unwrap()
6808 .status,
6809 HealthStatus::Ok
6810 );
6811 server.await.unwrap();
6812 }
6813
6814 #[cfg(unix)]
6815 #[tokio::test]
6816 async fn http_health_timeout_keeps_partial_status_and_body() {
6817 use tokio::io::{AsyncReadExt, AsyncWriteExt};
6818 let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
6819 let url = format!("http://{}/healthz", listener.local_addr().unwrap());
6820 let server = tokio::spawn(async move {
6821 let (mut stream, _) = listener.accept().await.unwrap();
6822 let mut request = [0u8; 2048];
6823 assert!(stream.read(&mut request).await.unwrap() > 0);
6824 stream
6825 .write_all(
6826 b"HTTP/1.1 503 Unavailable\r\nContent-Length: 100\r\n\r\npartial diagnostic",
6827 )
6828 .await
6829 .unwrap();
6830 std::future::pending::<()>().await;
6831 });
6832 let error = probe_http_health(&url, Duration::from_secs(1))
6833 .await
6834 .unwrap_err()
6835 .to_string();
6836 server.abort();
6837 assert!(
6838 error.contains("timed out")
6839 && error.contains("503 Unavailable")
6840 && error.contains("partial diagnostic"),
6841 "{error}"
6842 );
6843 }
6844
6845 #[cfg(unix)]
6846 #[tokio::test]
6847 async fn http_health_diagnostic_body_is_bounded_to_200_bytes() {
6848 use tokio::io::{AsyncReadExt, AsyncWriteExt};
6849 let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
6850 let url = format!("http://{}/healthz", listener.local_addr().unwrap());
6851 let server = tokio::spawn(async move {
6852 let (mut stream, _) = listener.accept().await.unwrap();
6853 let mut request = [0u8; 2048];
6854 assert!(stream.read(&mut request).await.unwrap() > 0);
6855 let body = format!("{}not-in-diagnostic", "x".repeat(400));
6856 let response = format!(
6857 "HTTP/1.1 503 Unavailable\r\nContent-Length: {}\r\n\r\n{body}",
6858 body.len()
6859 );
6860 stream.write_all(response.as_bytes()).await.unwrap();
6861 });
6862 let error = probe_http_health(&url, Duration::from_secs(1))
6863 .await
6864 .unwrap_err()
6865 .to_string();
6866 server.await.unwrap();
6867 assert!(error.ends_with(&"x".repeat(200)), "{error}");
6868 assert_eq!(error.split(": ").last().unwrap().len(), 200);
6869 assert!(!error.contains("not-in-diagnostic"));
6870 }
6871
6872 #[cfg(unix)]
6873 #[tokio::test]
6874 async fn http_health_real_nats_server_monitoring() {
6875 if std::process::Command::new("nats-server")
6876 .arg("--version")
6877 .env("XDG_DATA_HOME", std::env::temp_dir())
6878 .env("XDG_RUNTIME_DIR", std::env::temp_dir())
6879 .env("XDG_CONFIG_HOME", std::env::temp_dir())
6880 .output()
6881 .is_err()
6882 {
6883 eprintln!(
6884 "skipped: http_health_real_nats_server_monitoring (nats-server is not on PATH)"
6885 );
6886 return;
6887 }
6888 let dir = cortexkit_test_support::ScratchDir::new("nats-http-monitoring");
6889 let monitor = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
6890 let port = monitor.local_addr().unwrap().port();
6891 let client = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
6892 let client_port = client.local_addr().unwrap().port();
6893 let config = dir.join("server.conf");
6894 std::fs::write(&config, format!("listen: 127.0.0.1:{client_port}\nhttp: 127.0.0.1:{port}\njetstream {{ store_dir: \"{}\" }}\n", dir.join("jetstream").display())).unwrap();
6895 let mut configured = http_fixture(&dir, &format!("http://127.0.0.1:{port}/healthz"), 1000);
6896 configured.program = PathBuf::from("nats-server");
6897 configured.args = vec!["-c".into(), config.to_string_lossy().into_owned()];
6898 drop(monitor);
6899 drop(client);
6900 let module =
6901 Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default())
6902 .supervise_configured_with_health(
6903 configured.module_spec(),
6904 true,
6905 configured.health,
6906 configured.drain_timeout_ms,
6907 configured.restart,
6908 )
6909 .unwrap();
6910 wait_http_health(&module, SupervisorHealthStatus::Ok).await;
6911 module.drain().await.unwrap();
6912 }
6913
6914 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
6915 async fn configuration_update_does_not_replace_captured_running_process_facts() {
6916 let supervisor =
6917 Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default());
6918 let snapshot = stale_process_snapshot(ModuleState::Running, true);
6919 let initial = ModuleSpec {
6920 module_id: "rescan-preserves-spawn-facts".to_string(),
6921 program: PathBuf::from("/spawned/module"),
6922 args: Vec::new(),
6923 env: Vec::new(),
6924 reserved: false,
6925 reserved_prefixes: Vec::new(),
6926 protocol: ModuleProtocol::Subc,
6927 overlap: Default::default(),
6928 carry_containment: false,
6929 };
6930 let module = supervisor.supervised_module(
6931 initial.clone(),
6932 supervisor.runtime_config(),
6933 snapshot,
6934 None,
6935 );
6936 let before = module.status().unwrap();
6937 let mut replacement = initial;
6938 replacement.program = PathBuf::from("/rescanned/replacement-module");
6939
6940 module
6941 .update_configuration(replacement, HealthConfig::default(), None)
6942 .await
6943 .unwrap();
6944
6945 let after = module.status().unwrap();
6946 assert_eq!(after.pid, before.pid);
6947 assert_eq!(after.spawned_at_ms, before.spawned_at_ms);
6948 assert_eq!(after.spawned_from, before.spawned_from);
6949 drop(module);
6950 }
6951}
6952
6953fn unix_ms_now() -> u64 {
6954 SystemTime::now()
6955 .duration_since(UNIX_EPOCH)
6956 .map(|duration| duration.as_millis().min(u128::from(u64::MAX)) as u64)
6957 .unwrap_or(0)
6958}
6959
6960async fn supervise_loop(
6961 mut spec: ModuleSpec,
6962 mut runtime: SupervisorRuntimeConfig,
6963 registry: Arc<Registry>,
6964 process_liveness: Arc<SupervisorProcessLiveness>,
6965 snapshot: SharedSnapshot,
6966 mut child: Option<SupervisedChild>,
6967 mut commands: mpsc::Receiver<SupervisorCommand>,
6968) {
6969 let mut health_probe = HealthProbeRuntime::default();
6970 let mut registration_changes = match registry.subscribe_module_changes(&spec.module_id) {
6974 Ok(changes) => Some(changes),
6975 Err(err) => {
6976 warn!(module_id = %spec.module_id, error = %err, "health prober could not subscribe to registry");
6979 None
6980 }
6981 };
6982 let mut pending_respawn: Option<PendingRespawn> = None;
6986 let mut requeued: VecDeque<SupervisorCommand> = VecDeque::new();
6989 let mut previous_instance_recheck = tokio::time::interval_at(
6990 Instant::now() + Duration::from_secs(5),
6991 Duration::from_secs(5),
6992 );
6993 previous_instance_recheck.set_missed_tick_behavior(tokio::time::MissedTickBehavior::Skip);
6994 loop {
6995 #[cfg(test)]
6996 {
6997 lock_snapshot(&snapshot).unwrap().actor_select = None;
6998 }
6999 #[cfg(target_os = "macos")]
7000 if let Some(active) = child.as_mut() {
7001 active.confirm_privacy_exec().await;
7002 }
7003 if let Some(scheduled) = runtime
7004 .scheduled_respawn
7005 .lock()
7006 .unwrap_or_else(|p| p.into_inner())
7007 .take()
7008 {
7009 runtime.recovery_is_daemon = scheduled.daemon_initiated;
7010 pending_respawn = Some(scheduled);
7011 }
7012 if pending_respawn.is_some() && (child.is_some() || !respawn_still_pending(&snapshot)) {
7013 pending_respawn = None;
7014 cancel_deferred_reload(
7015 &runtime,
7016 &spec.module_id,
7017 "respawn cancelled by a supervisor command",
7018 );
7019 }
7020 if child.is_none()
7021 && pending_respawn.is_none()
7022 && lock_snapshot(&snapshot).is_ok_and(|state| state.waiting_for_previous_pid.is_none())
7023 {
7024 cancel_deferred_reload(
7025 &runtime,
7026 &spec.module_id,
7027 "respawn cancelled before a replacement was spawned",
7028 );
7029 let mut stranded = false;
7030 let _ = update_snapshot(&snapshot, Some(&spec.module_id), |state| {
7031 state.respawn_pending = false;
7032 state.coalesced_restart_pending = false;
7033 if matches!(
7034 state.state,
7035 ModuleState::Restarting
7036 | ModuleState::Starting
7037 | ModuleState::Draining
7038 | ModuleState::Unresponsive
7039 ) {
7040 error!(module_id = %spec.module_id, state = ?state.state,
7041 "supervision operation ended without a child or pending respawn; marking failed so start can retry");
7042 stranded = true;
7043 }
7044 });
7045 if stranded {
7046 fail_snapshot(
7047 &snapshot,
7048 Some(&spec.module_id),
7049 None,
7050 &runtime.failure_context(runtime.recovery_is_daemon),
7051 );
7052 }
7053 }
7054 if child.is_none()
7055 && (runtime.child_roster.is_closed()
7056 || pending_respawn.is_none()
7057 || !lock_snapshot(&snapshot)
7058 .is_ok_and(|state| state.enabled && state.state == ModuleState::Restarting))
7059 {
7060 containment::terminate(&runtime.child_roster.containment).await;
7061 }
7062 if let Some(command) = requeued.pop_front() {
7063 if !handle_supervisor_command(
7064 command,
7065 &mut spec,
7066 &mut runtime,
7067 ®istry,
7068 &process_liveness,
7069 &snapshot,
7070 &mut child,
7071 &mut commands,
7072 &mut requeued,
7073 )
7074 .await
7075 {
7076 containment::terminate(&runtime.child_roster.containment).await;
7077 return;
7078 }
7079 if child.is_some() || !respawn_still_pending(&snapshot) {
7080 pending_respawn = None;
7081 let _ = update_snapshot(&snapshot, Some(&spec.module_id), |state| {
7082 state.respawn_pending = false
7083 });
7084 }
7085 continue;
7086 }
7087 if child.is_some() {
7088 runtime.recovery_is_daemon = true;
7089 #[cfg(test)]
7090 {
7091 lock_snapshot(&snapshot).unwrap().actor_turns += 1;
7092 }
7093 health_probe.refresh_registration(&spec, &runtime, ®istry, &snapshot);
7094 let active = child.as_mut().expect("child checked above");
7095 if runtime.child_roster.is_closed() || active.registered_since_spawn() {
7096 active.registration_deadline = None;
7097 }
7098 let registration_deadline = active.registration_deadline;
7099 let no_checks = runtime.health.http.is_none()
7100 && (running_protocol(&spec, &snapshot) == ModuleProtocol::None
7101 || (health_probe.registered_connection.is_some() && !health_probe.advertised));
7102 let uptime_reset = lock_snapshot(&snapshot).ok().and_then(|mut state| {
7103 state
7104 .failed_retry
7105 .uptime_reset_at(no_checks, runtime.restart_policy.window)
7106 });
7107 let wake_at = match (health_probe.wake_at(), uptime_reset) {
7108 (probe, Some(reset)) => Some(probe.map_or(reset, |probe| probe.min(reset))),
7109 (probe, None) => probe,
7110 };
7111 let wake_at = match (wake_at, registration_deadline) {
7112 (Some(wake), Some(register)) => Some(wake.min(register)),
7113 (wake, register) => wake.or(register),
7114 };
7115 #[cfg(test)]
7116 let wake_after = wake_at.map(|at| at.saturating_duration_since(Instant::now()));
7117 let probe_sleep = supervisor_sleep(wake_at);
7118 tokio::pin!(probe_sleep);
7119 let wire_child = running_protocol(&spec, &snapshot) == ModuleProtocol::Subc;
7124 #[cfg(test)]
7125 {
7126 let registration_pending = wire_child
7127 && registration_changes
7128 .as_ref()
7129 .is_some_and(|changes| changes.has_changed().unwrap());
7130 let mut state = lock_snapshot(&snapshot).unwrap();
7131 state.actor_select = (!registration_pending).then_some(ActorSelectCheckpoint {
7132 generation: state.spawn_generation,
7133 turn: state.actor_turns,
7134 registered_connection: health_probe.registered_connection,
7135 next_probe_at: health_probe.next_probe_at,
7136 wake_after,
7137 });
7138 }
7139 let active_child = child.as_mut().expect("child checked above");
7140 tokio::select! {
7141 wait_result = active_child.wait() => {
7142 if let Ok(mut state) = lock_snapshot(&snapshot) {
7143 state.failed_retry.uptime_reset_at(no_checks, runtime.restart_policy.window);
7144 }
7145 #[cfg(test)]
7146 { lock_snapshot(&snapshot).unwrap().actor_select = None; }
7147 let exit_report = match wait_result {
7156 Ok(status) => classify_reaped_child_exit(&snapshot, active_child, &status),
7157 Err(err) => {
7158 active_child.drain_stderr(&spec.module_id).await;
7159 fail_after_wait_error(
7160 &spec.module_id,
7161 &snapshot,
7162 &runtime.terminal_ring,
7163 &runtime.spawn_events,
7164 format!("failed to wait for supervised module: {err}"),
7165 &runtime.failure_context(true),
7166 );
7167 untrack_if_registration_released(
7168 &process_liveness,
7169 ®istry,
7170 &spec.module_id,
7171 &snapshot,
7172 );
7173 error!(module_id = %spec.module_id, error = %err, "failed to wait for supervised module");
7174 child = None;
7175 continue;
7176 }
7177 };
7178 active_child.drain_stderr(&spec.module_id).await;
7179
7180 let next = on_child_exit(
7181 &spec,
7182 runtime.restart_policy,
7183 ®istry,
7184 &snapshot,
7185 &runtime.terminal_ring,
7186 &runtime.spawn_events,
7187 &runtime.child_roster,
7188 exit_report,
7189 true,
7190 ).await;
7191 runtime.start_requester = None;
7192 runtime.pending_activity_restart = false;
7193 let carry = matches!(next, NextAction::Restart { .. })
7196 && active_child.spawn_failure.is_none()
7197 && !runtime.child_roster.is_closed();
7198 if let Err(error) = active_child.finish_exit(carry).await {
7199 containment::terminate(&runtime.child_roster.containment).await;
7200 fail_after_wait_error(&spec.module_id, &snapshot, &runtime.terminal_ring,
7201 &runtime.spawn_events, format!("failed to reap classified exit: {error}"),
7202 &runtime.failure_context(true));
7203 child = None;
7204 continue;
7205 }
7206 match next {
7207 NextAction::Stop { registration_released } => {
7208 if registration_released {
7209 process_liveness.untrack_if_current(&spec.module_id, &snapshot);
7210 }
7211 child = None;
7212 }
7213 NextAction::Restart { schedule } => {
7214 let delay = schedule.map_or(
7215 runtime.restart_policy.delay_for_restart(0),
7216 |schedule| schedule.delay,
7217 );
7218 if let Some(schedule) = schedule {
7219 log_crash_respawn(&spec.module_id, schedule);
7220 }
7221 child = None;
7229 pending_respawn = Some(PendingRespawn { deadline: Instant::now() + delay, kind: RespawnKind::Spawn, daemon_initiated: true });
7230 let _ = update_snapshot(&snapshot, Some(&spec.module_id), |state| state.respawn_pending = true);
7231 }
7232 }
7233 }
7234 command = commands.recv() => {
7235 #[cfg(test)]
7236 { lock_snapshot(&snapshot).unwrap().actor_select = None; }
7237 let Some(command) = command else {
7238 containment::terminate(&runtime.child_roster.containment).await;
7239 return;
7240 };
7241 if !handle_supervisor_command(
7242 command,
7243 &mut spec,
7244 &mut runtime,
7245 ®istry,
7246 &process_liveness,
7247 &snapshot,
7248 &mut child,
7249 &mut commands,
7250 &mut requeued,
7251 ).await {
7252 containment::terminate(&runtime.child_roster.containment).await;
7253 return;
7254 }
7255 }
7256 _ = async {
7257 match registration_changes.as_mut() {
7258 Some(changes) => { let _ = changes.changed().await; }
7259 None => std::future::pending().await,
7260 }
7261 }, if wire_child => {
7262 #[cfg(test)]
7263 { lock_snapshot(&snapshot).unwrap().actor_select = None; }
7264 }
7265 _ = &mut probe_sleep => {
7266 #[cfg(test)]
7267 { lock_snapshot(&snapshot).unwrap().actor_select = None; }
7268 if registration_deadline.is_some_and(|deadline| Instant::now() >= deadline) {
7269 active_child.registration_deadline = None;
7272 if !runtime.child_roster.is_closed()
7273 && !active_child.registered_since_spawn()
7274 && active_child.child.try_wait().is_ok_and(|status| status.is_none())
7275 {
7276 warn!(module_id = %spec.module_id, pid = active_child.pid,
7277 wait_secs = runtime.register_timeout.as_secs(),
7278 "supervised module registration_timeout: first HELLO deadline expired; stopping child");
7279 active_child.spawn_failure = Some("registration_timeout".into());
7280 if let Err(error) = stop_unregistered_child(active_child).await {
7281 error!(module_id = %spec.module_id, %error, "failed to stop unregistered child");
7282 fail_after_wait_error(&spec.module_id, &snapshot, &runtime.terminal_ring,
7283 &runtime.spawn_events, format!("registration_timeout: {error}"),
7284 &runtime.failure_context(true));
7285 child = None;
7286 }
7287 continue;
7290 }
7291 }
7292 if health_probe.due() {
7293 run_health_probe_cycle(
7294 &spec,
7295 &runtime,
7296 ®istry,
7297 &process_liveness,
7298 &snapshot,
7299 &mut child,
7300 ).await;
7301 if child.is_none() {
7302 runtime.start_requester = None;
7303 runtime.pending_activity_restart = false;
7304 }
7305 if child.is_some() {
7306 health_probe.schedule_next(&spec, runtime.health.cadence);
7307 }
7308 }
7309 }
7310 }
7311 } else if let Some(pending) = pending_respawn {
7312 tokio::select! {
7313 _ = runtime.child_roster.closed() => {
7314 pending_respawn = None;
7315 let _ = update_snapshot(&snapshot, Some(&spec.module_id), |state| {
7316 state.state = ModuleState::Stopped;
7317 state.respawn_pending = false;
7318 });
7319 containment::terminate(&runtime.child_roster.containment).await;
7320 }
7321 _ = sleep_until(pending.deadline) => {
7322 pending_respawn = None;
7323 let _ = update_snapshot(&snapshot, Some(&spec.module_id), |state| state.respawn_pending = false);
7324 if !respawn_still_pending(&snapshot) {
7328 continue;
7329 }
7330 if runtime.child_roster.is_closed() {
7335 let _ = update_snapshot(&snapshot, Some(&spec.module_id), |state| {
7336 state.state = ModuleState::Stopped;
7337 });
7338 debug!(module_id = %spec.module_id, "crash respawn cancelled by daemon shutdown");
7339 continue;
7340 }
7341 if let Err(err) = release_dead_registration(
7342 ®istry,
7343 runtime.forwarding.as_deref(),
7344 &snapshot,
7345 &spec.module_id,
7346 &runtime.failure_context(runtime.recovery_is_daemon),
7347 ).await {
7348 error!(module_id = %spec.module_id, error = %err, "registration did not release before restart");
7349 continue;
7350 }
7351
7352 if matches!(pending.kind, RespawnKind::Reload) {
7353 let reply = runtime.deferred_reload_reply.lock().unwrap_or_else(|p| p.into_inner()).take();
7354 let result = finish_reload_child(&spec, &runtime, ®istry, &process_liveness, &snapshot, &mut child).await;
7355 if result.is_ok() && runtime.pending_activity_restart {
7356 runtime.record_activity_transition("subc.module_restarted", &spec.module_id, runtime.start_requester.clone());
7357 }
7358 runtime.pending_activity_restart = false;
7359 if let Some(reply) = reply { let _ = reply.send(result); }
7360 continue;
7361 }
7362 process_liveness.track(spec.module_id.clone(), Arc::clone(&snapshot));
7363 match spawn_and_mark_running(&spec, &runtime, &snapshot) {
7364 Ok(next_child) => {
7365 child = Some(next_child);
7366 if runtime.pending_activity_restart {
7367 runtime.record_activity_transition("subc.module_restarted", &spec.module_id, runtime.start_requester.clone());
7368 runtime.pending_activity_restart = false;
7369 }
7370 debug!(module_id = %spec.module_id, "supervised module restarted after crash");
7371 }
7372 Err(err) => {
7373 fail_snapshot(&snapshot, Some(&spec.module_id), None, &runtime.failure_context(runtime.recovery_is_daemon));
7374 process_liveness.untrack_if_current(&spec.module_id, &snapshot);
7375 error!(module_id = %spec.module_id, error = %err, "failed to restart supervised module");
7376 }
7377 }
7378 }
7379 command = commands.recv() => {
7380 let Some(command) = command else {
7381 containment::terminate(&runtime.child_roster.containment).await;
7382 return;
7383 };
7384 if !handle_supervisor_command(
7385 command,
7386 &mut spec,
7387 &mut runtime,
7388 ®istry,
7389 &process_liveness,
7390 &snapshot,
7391 &mut child,
7392 &mut commands,
7393 &mut requeued,
7394 ).await {
7395 containment::terminate(&runtime.child_roster.containment).await;
7396 return;
7397 }
7398 if child.is_some() || !respawn_still_pending(&snapshot) {
7403 pending_respawn = None;
7404 let _ = update_snapshot(&snapshot, Some(&spec.module_id), |state| state.respawn_pending = false);
7405 }
7406 }
7407 }
7408 } else {
7409 let slow_retry = lock_snapshot(&snapshot).ok().and_then(|state| {
7410 (state.enabled && state.state == ModuleState::Failed)
7411 .then_some(state.failed_retry.pending)
7412 .flatten()
7413 });
7414 let command = if lock_snapshot(&snapshot)
7415 .is_ok_and(|state| state.waiting_for_previous_pid.is_some())
7416 {
7417 tokio::select! {
7418 command = commands.recv() => command,
7419 _ = previous_instance_recheck.tick() => {
7420 if runtime.child_roster.previous_instance_pid(&spec.module_id).is_none() {
7421 let enabled = lock_snapshot(&snapshot).is_ok_and(|state| state.enabled);
7422 if enabled && !runtime.child_roster.is_closed() {
7423 process_liveness.track(spec.module_id.clone(), Arc::clone(&snapshot));
7424 match spawn_and_mark_running(&spec, &runtime, &snapshot) {
7425 Ok(next_child) => child = Some(next_child),
7426 Err(error) => {
7427 fail_snapshot(&snapshot, Some(&spec.module_id), None, &runtime.failure_context(true));
7428 error!(module_id = %spec.module_id, %error, "failed to start module after its previous instance exited");
7429 }
7430 }
7431 } else {
7432 let _ = update_snapshot(&snapshot, Some(&spec.module_id), |state| {
7433 state.waiting_for_previous_pid = None;
7434 state.waiting_for_previous_detail = None;
7435 });
7436 }
7437 }
7438 continue;
7439 }
7440 }
7441 } else {
7442 tokio::select! {
7443 biased;
7444 command = commands.recv() => command,
7445 _ = async {
7446 match slow_retry {
7447 Some(retry) => sleep_until(retry.deadline).await,
7448 None => std::future::pending().await,
7449 }
7450 } => {
7451 let retry = slow_retry.expect("only a scheduled slow retry can wake");
7452 let _ = update_snapshot(&snapshot, Some(&spec.module_id), |state| {
7453 state.failed_retry.pending = None;
7454 });
7455 if runtime.child_roster.is_closed() {
7456 continue;
7457 }
7458 info!(module_id = %spec.module_id, attempt = retry.attempt,
7459 delay_secs = retry.delay.as_secs(), "starting failed module retry");
7460 if let Err(error) = set_child_enabled(
7461 &spec, &runtime, ®istry, &process_liveness, &snapshot, &mut child, true,
7462 &runtime.failure_context(true),
7463 ).await {
7464 error!(module_id = %spec.module_id, %error, "failed module retry could not start");
7465 }
7466 continue;
7467 }
7468 }
7469 };
7470 let Some(command) = command else {
7471 containment::terminate(&runtime.child_roster.containment).await;
7472 return;
7473 };
7474 if !handle_supervisor_command(
7475 command,
7476 &mut spec,
7477 &mut runtime,
7478 ®istry,
7479 &process_liveness,
7480 &snapshot,
7481 &mut child,
7482 &mut commands,
7483 &mut requeued,
7484 )
7485 .await
7486 {
7487 containment::terminate(&runtime.child_roster.containment).await;
7488 return;
7489 }
7490 }
7491 }
7492}
7493
7494async fn supervisor_sleep(wake_at: Option<Instant>) {
7497 match wake_at {
7498 Some(deadline) => sleep_until(deadline).await,
7499 None => std::future::pending().await,
7500 }
7501}
7502
7503fn log_crash_respawn(module_id: &str, schedule: CrashRestartSchedule) {
7504 info!(
7505 module_id,
7506 restart_in_window = schedule.restart_in_window,
7507 delay_ms = schedule.delay.as_millis() as u64,
7508 "respawning after crash"
7509 );
7510}
7511
7512fn respawn_still_pending(snapshot: &SharedSnapshot) -> bool {
7518 matches!(
7519 lock_snapshot(snapshot),
7520 Ok(state) if state.enabled && state.state == ModuleState::Restarting
7521 )
7522}
7523
7524enum NextAction {
7525 Stop {
7526 registration_released: bool,
7527 },
7528 Restart {
7529 schedule: Option<CrashRestartSchedule>,
7530 },
7531}
7532
7533#[allow(clippy::too_many_arguments)]
7534async fn handle_supervisor_command(
7535 command: SupervisorCommand,
7536 spec: &mut ModuleSpec,
7537 runtime: &mut SupervisorRuntimeConfig,
7538 registry: &Arc<Registry>,
7539 process_liveness: &SupervisorProcessLiveness,
7540 snapshot: &SharedSnapshot,
7541 child: &mut Option<SupervisedChild>,
7542 commands: &mut mpsc::Receiver<SupervisorCommand>,
7543 requeued: &mut VecDeque<SupervisorCommand>,
7544) -> bool {
7545 if !matches!(command, SupervisorCommand::UpdateConfiguration { .. }) {
7546 runtime.recovery_is_daemon = false;
7547 }
7548 if matches!(
7549 command,
7550 SupervisorCommand::Drain { .. }
7551 | SupervisorCommand::Retire { .. }
7552 | SupervisorCommand::SetEnabled { .. }
7553 | SupervisorCommand::Restart { .. }
7554 ) {
7555 let _ = update_snapshot(snapshot, Some(&spec.module_id), |state| {
7556 state.failed_retry.cancel()
7557 });
7558 }
7559 let blocked = recovery_block_error(spec, &runtime.child_roster).or_else(|| {
7560 runtime
7561 .child_roster
7562 .previous_instance(&spec.module_id)
7563 .map(|(pid, reason)| previous_instance_error(spec, pid, reason))
7564 });
7565 let command = if let Some(error) = blocked {
7566 match command {
7567 SupervisorCommand::Restart { reply, .. }
7568 | SupervisorCommand::Reload { reply, .. }
7569 | SupervisorCommand::Swap { reply, .. } => {
7570 let _ = reply.send(Err(error));
7571 return true;
7572 }
7573 SupervisorCommand::SetEnabled {
7574 enabled: true,
7575 reply,
7576 ..
7577 } => {
7578 let _ = reply.send(Err(error));
7579 return true;
7580 }
7581 other => other,
7582 }
7583 } else {
7584 command
7585 };
7586 match command {
7587 SupervisorCommand::Drain { requester, reply } => {
7588 let result = drain_optional_child(
7591 &spec.module_id,
7592 spec.protocol,
7593 StopNotice::NotSent,
7594 registry,
7595 runtime.forwarding.as_deref(),
7596 snapshot,
7597 &runtime.terminal_ring,
7598 &runtime.spawn_events,
7599 child,
7600 runtime.drain_timeout,
7601 ModuleState::Stopped,
7602 None,
7603 &FailureContext::operator(),
7604 )
7605 .await;
7606 containment::terminate(&runtime.child_roster.containment).await;
7607 let registration_released = result.is_ok();
7608 if registration_released {
7609 runtime.record_activity_transition(
7610 "subc.module_stopped",
7611 &spec.module_id,
7612 requester,
7613 );
7614 }
7615 let _ = reply.send(result);
7616 if registration_released {
7617 process_liveness.untrack_if_current(&spec.module_id, snapshot);
7618 }
7619 false
7620 }
7621 SupervisorCommand::Retire { requester, reply } => {
7622 let result = async {
7623 let stop_notice = begin_forwarding_drain_if_configured(
7624 spec,
7625 runtime,
7626 registry,
7627 snapshot,
7628 None,
7629 RouteCloseReason::Disable,
7630 )
7631 .await?;
7632 drain_optional_child(
7633 &spec.module_id,
7634 spec.protocol,
7635 stop_notice,
7636 registry,
7637 runtime.forwarding.as_deref(),
7638 snapshot,
7639 &runtime.terminal_ring,
7640 &runtime.spawn_events,
7641 child,
7642 runtime.drain_timeout,
7643 ModuleState::Stopped,
7644 None,
7645 &FailureContext::operator(),
7646 )
7647 .await
7648 }
7649 .await;
7650 containment::terminate(&runtime.child_roster.containment).await;
7651 let registration_released = result.is_ok();
7652 if registration_released {
7653 runtime.record_activity_transition(
7654 "subc.module_stopped",
7655 &spec.module_id,
7656 requester,
7657 );
7658 }
7659 let _ = reply.send(result);
7660 if registration_released {
7661 process_liveness.untrack_if_current(&spec.module_id, snapshot);
7662 }
7663 false
7664 }
7665 SupervisorCommand::Restart {
7666 requester,
7667 drain_timeout_ms,
7668 received_at_generation,
7669 queued_at,
7670 reply,
7671 } => {
7672 info!(
7676 module_id = %spec.module_id,
7677 queued_ms = u64::try_from(queued_at.elapsed().as_millis()).unwrap_or(u64::MAX),
7678 "restart command dequeued"
7679 );
7680 let validation = match lock_snapshot(snapshot) {
7692 Ok(state) if !state.enabled => Err(SuperviseError::Disabled {
7693 module_id: spec.module_id.clone(),
7694 }),
7695 Ok(_) => Ok(()),
7696 Err(err) => Err(err),
7697 };
7698 let initiated = validation.is_ok();
7699 let _ = reply.send(validation);
7700 let satisfied_by_generation = if initiated
7714 && child
7715 .as_ref()
7716 .is_some_and(|child| child.spawned_at > queued_at)
7717 {
7718 lock_snapshot(snapshot).ok().and_then(|state| {
7719 (!state.configuration_updated_since_spawn).then_some(state.spawn_generation)
7720 })
7721 } else {
7722 None
7723 };
7724 let satisfied_by_pending = initiated
7725 && child.is_none()
7726 && lock_snapshot(snapshot).ok().is_some_and(|mut state| {
7727 let pending = state.respawn_pending && !state.configuration_updated_since_spawn;
7731 if pending {
7732 state.coalesced_restart_pending = true;
7733 }
7734 pending
7735 });
7736 if satisfied_by_pending {
7737 debug!(module_id = %spec.module_id, "restart coalesced into the pending replacement");
7738 } else if let Some(generation) = satisfied_by_generation {
7739 info!(
7740 module_id = %spec.module_id,
7741 received_at_generation,
7742 "restart already satisfied by generation {generation}; not restarting again"
7743 );
7744 } else if initiated {
7745 let drain_timeout = drain_timeout_ms
7748 .map(Duration::from_millis)
7749 .unwrap_or(runtime.drain_timeout);
7750 if let Err(err) = restart_child(
7751 spec,
7752 runtime,
7753 registry,
7754 process_liveness,
7755 snapshot,
7756 child,
7757 drain_timeout,
7758 )
7759 .await
7760 {
7761 warn!(
7762 module_id = %spec.module_id,
7763 error = %err,
7764 "operator restart failed after initiation ack; module state carries the outcome"
7765 );
7766 let _ = update_snapshot(snapshot, Some(&spec.module_id), |state| {
7767 state.state = ModuleState::Failed;
7768 clear_current_process_facts(state);
7769 });
7770 } else {
7771 runtime.start_requester = requester;
7772 runtime.pending_activity_restart = true;
7773 }
7774 }
7775 true
7776 }
7777 SupervisorCommand::Reload { requester, reply } => {
7778 let result =
7779 reload_child(spec, runtime, registry, process_liveness, snapshot, child).await;
7780 if result.is_ok() {
7781 runtime.start_requester = requester;
7782 runtime.pending_activity_restart = true;
7783 }
7784 if result.is_ok()
7785 && runtime
7786 .scheduled_respawn
7787 .lock()
7788 .unwrap_or_else(|p| p.into_inner())
7789 .as_ref()
7790 .is_some_and(|pending| matches!(pending.kind, RespawnKind::Reload))
7791 {
7792 *runtime
7793 .deferred_reload_reply
7794 .lock()
7795 .unwrap_or_else(|p| p.into_inner()) = Some(reply);
7796 } else {
7797 let _ = reply.send(result);
7798 }
7799 true
7800 }
7801 SupervisorCommand::SetEnabled {
7802 enabled,
7803 requester,
7804 reply,
7805 } => {
7806 let previous_requester = runtime.start_requester.clone();
7807 if enabled {
7808 runtime.start_requester = requester.clone();
7809 }
7810 let result = set_child_enabled(
7811 spec,
7812 runtime,
7813 registry,
7814 process_liveness,
7815 snapshot,
7816 child,
7817 enabled,
7818 &runtime.failure_context(false),
7819 )
7820 .await;
7821 if !enabled {
7822 containment::terminate(&runtime.child_roster.containment).await;
7823 }
7824 if matches!(result, Ok(true)) {
7825 runtime.pending_activity_restart = false;
7826 if enabled {
7827 runtime.start_requester = requester;
7828 } else {
7829 runtime.start_requester = None;
7830 runtime.record_activity_transition(
7831 "subc.module_disabled",
7832 &spec.module_id,
7833 requester,
7834 );
7835 }
7836 } else {
7837 runtime.start_requester = previous_requester;
7838 }
7839 let _ = reply.send(result);
7840 true
7841 }
7842 SupervisorCommand::UpdateConfiguration {
7843 spec: next_spec,
7844 health,
7845 drain_timeout_ms,
7846 reply,
7847 } => {
7848 if let Some(handle) = &runtime.supervisor_handle {
7849 handle.apply_identity_configuration(&next_spec);
7850 }
7851 *spec = next_spec;
7852 let _ = update_snapshot(snapshot, Some(&spec.module_id), |state| {
7853 state.configuration_updated_since_spawn = true;
7854 });
7855 let health_changed = runtime.health != health;
7856 runtime.health = health;
7857 if health_changed {
7860 let _ = update_snapshot(snapshot, Some(&spec.module_id), |state| {
7861 state.health = ModuleHealthStatus::default();
7862 });
7863 }
7864 runtime.drain_timeout = drain_timeout_ms
7865 .map(Duration::from_millis)
7866 .unwrap_or(runtime.default_drain_timeout);
7867 *runtime
7868 .effective_drain_timeout
7869 .lock()
7870 .unwrap_or_else(|poisoned| poisoned.into_inner()) = runtime.drain_timeout;
7871 let _ = reply.send(());
7872 true
7873 }
7874 SupervisorCommand::Swap {
7875 requester,
7876 ready_timeout,
7877 reply,
7878 } => {
7879 if spec.carry_containment
7880 || containment::current(&runtime.child_roster.containment).is_some()
7881 {
7882 let _ = reply.send(Err(SuperviseError::InvalidSpec {
7883 reason: "a carried container cannot overlap a swap candidate".to_owned(),
7884 }));
7885 return true;
7886 }
7887 let mut swap_runtime = runtime.clone();
7888 swap_runtime.start_requester = requester;
7889 let before = lock_snapshot(snapshot)
7890 .ok()
7891 .map(|state| state.spawn_generation);
7892 let end = swap::run_swap(
7893 spec,
7894 &swap_runtime,
7895 registry,
7896 process_liveness,
7897 snapshot,
7898 child,
7899 commands,
7900 ready_timeout.unwrap_or(DEFAULT_SWAP_READY_TIMEOUT),
7901 reply,
7902 )
7903 .await;
7904 if before
7905 != lock_snapshot(snapshot)
7906 .ok()
7907 .map(|state| state.spawn_generation)
7908 {
7909 runtime.start_requester = swap_runtime.start_requester;
7910 }
7911 requeued.extend(end.requeue);
7912 true
7913 }
7914 }
7915}
7916
7917async fn restart_child(
7918 spec: &ModuleSpec,
7919 runtime: &SupervisorRuntimeConfig,
7920 registry: &Registry,
7921 process_liveness: &SupervisorProcessLiveness,
7922 snapshot: &SharedSnapshot,
7923 child: &mut Option<SupervisedChild>,
7924 drain_timeout: Duration,
7925) -> Result<(), SuperviseError> {
7926 if !lock_snapshot(snapshot)?.enabled {
7928 return Err(SuperviseError::Disabled {
7929 module_id: spec.module_id.clone(),
7930 });
7931 }
7932 let stop_notice = begin_forwarding_drain_with_timeout(
7933 spec,
7934 runtime,
7935 registry,
7936 snapshot,
7937 None,
7938 RouteCloseReason::Restart,
7939 drain_timeout,
7940 )
7941 .await?;
7942
7943 if child.is_some() {
7944 drain_optional_child(
7945 &spec.module_id,
7946 spec.protocol,
7947 stop_notice,
7948 registry,
7949 runtime.forwarding.as_deref(),
7950 snapshot,
7951 &runtime.terminal_ring,
7952 &runtime.spawn_events,
7953 child,
7954 drain_timeout,
7955 ModuleState::Restarting,
7956 Some(true),
7957 &FailureContext::operator(),
7958 )
7959 .await?;
7960 } else {
7961 update_snapshot(snapshot, Some(&spec.module_id), |state| {
7962 state.enabled = true;
7963 state.state = ModuleState::Restarting;
7964 clear_current_process_facts(state);
7965 })?;
7966 release_dead_registration(
7967 registry,
7968 runtime.forwarding.as_deref(),
7969 snapshot,
7970 &spec.module_id,
7971 &runtime.failure_context(false),
7972 )
7973 .await?;
7974 }
7975
7976 reset_restart_count(snapshot, &spec.module_id)?;
7977 process_liveness.track(spec.module_id.clone(), Arc::clone(snapshot));
7978 schedule_respawn(
7979 runtime,
7980 snapshot,
7981 &spec.module_id,
7982 runtime.restart_policy.backoff,
7983 RespawnKind::Spawn,
7984 false,
7985 )
7986}
7987
7988async fn reload_child(
7989 spec: &ModuleSpec,
7990 runtime: &SupervisorRuntimeConfig,
7991 registry: &Registry,
7992 process_liveness: &SupervisorProcessLiveness,
7993 snapshot: &SharedSnapshot,
7994 child: &mut Option<SupervisedChild>,
7995) -> Result<(), SuperviseError> {
7996 if !lock_snapshot(snapshot)?.enabled {
7998 return Err(SuperviseError::Disabled {
7999 module_id: spec.module_id.clone(),
8000 });
8001 }
8002 let stop_notice = begin_forwarding_drain(
8003 spec,
8004 runtime,
8005 registry,
8006 snapshot,
8007 Some(true),
8008 RouteCloseReason::Reload,
8009 )
8010 .await?;
8011
8012 if child.is_some() {
8013 drain_optional_child(
8014 &spec.module_id,
8015 spec.protocol,
8016 stop_notice,
8017 registry,
8018 runtime.forwarding.as_deref(),
8019 snapshot,
8020 &runtime.terminal_ring,
8021 &runtime.spawn_events,
8022 child,
8023 runtime.drain_timeout,
8024 ModuleState::Restarting,
8025 Some(true),
8026 &FailureContext::operator(),
8027 )
8028 .await?;
8029 } else {
8030 update_snapshot(snapshot, Some(&spec.module_id), |state| {
8031 state.enabled = true;
8032 state.state = ModuleState::Restarting;
8033 clear_current_process_facts(state);
8034 })?;
8035 release_dead_registration(
8036 registry,
8037 runtime.forwarding.as_deref(),
8038 snapshot,
8039 &spec.module_id,
8040 &runtime.failure_context(false),
8041 )
8042 .await?;
8043 }
8044
8045 reset_restart_count(snapshot, &spec.module_id)?;
8046 process_liveness.track(spec.module_id.clone(), Arc::clone(snapshot));
8047 schedule_respawn(
8048 runtime,
8049 snapshot,
8050 &spec.module_id,
8051 runtime.restart_policy.backoff,
8052 RespawnKind::Reload,
8053 false,
8054 )
8055}
8056
8057async fn finish_reload_child(
8058 spec: &ModuleSpec,
8059 runtime: &SupervisorRuntimeConfig,
8060 registry: &Registry,
8061 process_liveness: &SupervisorProcessLiveness,
8062 snapshot: &SharedSnapshot,
8063 child: &mut Option<SupervisedChild>,
8064) -> Result<(), SuperviseError> {
8065 process_liveness.track(spec.module_id.clone(), Arc::clone(snapshot));
8066 let next_child = match spawn_and_mark_running(spec, runtime, snapshot) {
8067 Ok(next_child) => next_child,
8068 Err(err) => {
8069 return handle_reload_spawn_failure(
8070 spec,
8071 runtime,
8072 process_liveness,
8073 snapshot,
8074 child,
8075 format!("new child failed to spawn: {err}"),
8076 )
8077 .await;
8078 }
8079 };
8080 *child = Some(next_child);
8081
8082 let wait_outcome = {
8083 let active_child = child.as_mut().expect("new reload child was just stored");
8084 wait_for_registration_after_reload(
8085 registry,
8086 &spec.module_id,
8087 snapshot,
8088 active_child,
8089 REGISTRY_RELEASE_TIMEOUT,
8090 )
8091 .await?
8092 };
8093
8094 match wait_outcome {
8095 RegistrationWaitOutcome::Registered => {
8096 child
8097 .as_mut()
8098 .expect("registered reload child")
8099 .registration_deadline = None;
8100 debug!(module_id = %spec.module_id, "supervised module reloaded and registered");
8101 Ok(())
8102 }
8103 RegistrationWaitOutcome::Exited(exit_report) => {
8104 if let Some(active_child) = child.as_mut() {
8105 active_child.drain_stderr(&spec.module_id).await;
8106 }
8107 let mut exited_child = child.take().expect("exited reload child is still stored");
8110 #[cfg(test)]
8111 if let Some(gate) = &runtime.test_reload_exit_record_gate {
8112 gate.reached.notify_one();
8113 gate.resume.notified().await;
8114 }
8115 let result = handle_reload_child_registration_failure(
8116 spec,
8117 runtime,
8118 registry,
8119 process_liveness,
8120 snapshot,
8121 child,
8122 ReloadRegistrationFailure {
8123 exit_report: registration_failure_exit_report(exit_report),
8124 reason: exited_child
8125 .spawn_failure
8126 .clone()
8127 .unwrap_or_else(|| "new child exited before registering".to_string()),
8128 },
8129 )
8130 .await;
8131 exited_child
8132 .finish_exit(false)
8133 .await
8134 .map_err(|source| SuperviseError::Wait {
8135 module_id: spec.module_id.clone(),
8136 source,
8137 })?;
8138 result
8139 }
8140 RegistrationWaitOutcome::TimedOut => {
8141 let mut timed_out_child = child
8142 .take()
8143 .expect("timed-out reload child is still running");
8144 timed_out_child
8145 .start_kill()
8146 .map_err(|source| SuperviseError::Kill {
8147 module_id: spec.module_id.clone(),
8148 source,
8149 })?;
8150 let status = timed_out_child
8151 .wait()
8152 .await
8153 .map_err(|source| SuperviseError::Wait {
8154 module_id: spec.module_id.clone(),
8155 source,
8156 })?;
8157 timed_out_child.drain_stderr(&spec.module_id).await;
8158 let result = handle_reload_child_registration_failure(
8159 spec,
8160 runtime,
8161 registry,
8162 process_liveness,
8163 snapshot,
8164 child,
8165 ReloadRegistrationFailure {
8166 exit_report: registration_failure_exit_report(classify_reaped_child_exit(
8167 snapshot,
8168 &timed_out_child,
8169 &status,
8170 )),
8171 reason: format!(
8172 "new child did not register within {:?}",
8173 REGISTRY_RELEASE_TIMEOUT
8174 ),
8175 },
8176 )
8177 .await;
8178 timed_out_child
8179 .finish_exit(false)
8180 .await
8181 .map_err(|source| SuperviseError::Wait {
8182 module_id: spec.module_id.clone(),
8183 source,
8184 })?;
8185 result
8186 }
8187 }
8188}
8189
8190#[allow(clippy::too_many_arguments)]
8191async fn set_child_enabled(
8192 spec: &ModuleSpec,
8193 runtime: &SupervisorRuntimeConfig,
8194 registry: &Registry,
8195 process_liveness: &SupervisorProcessLiveness,
8196 snapshot: &SharedSnapshot,
8197 child: &mut Option<SupervisedChild>,
8198 enabled: bool,
8199 failure: &FailureContext,
8200) -> Result<bool, SuperviseError> {
8201 let (current_enabled, current_state, respawn_pending) = {
8202 let state = lock_snapshot(snapshot)?;
8203 (state.enabled, state.state, state.respawn_pending)
8204 };
8205 let revive_terminal = enabled
8213 && current_enabled
8214 && child.is_none()
8215 && (matches!(current_state, ModuleState::Failed | ModuleState::Stopped)
8216 || (current_state == ModuleState::Restarting && !respawn_pending));
8217 if current_enabled == enabled && !revive_terminal {
8218 return Ok(false);
8219 }
8220
8221 if enabled {
8222 update_snapshot(snapshot, Some(&spec.module_id), |state| {
8223 state.enabled = true;
8224 state.state = ModuleState::Starting;
8225 clear_current_process_facts(state);
8226 })?;
8227 #[cfg(test)]
8228 if runtime.test_seed_stale_facts_before_enable_spawn {
8229 update_snapshot(snapshot, Some(&spec.module_id), |state| {
8230 state.process_alive = true;
8231 state.pid = Some(41);
8232 state.spawned_at_ms = Some(42);
8233 state.spawned_from = Some(PathBuf::from("/spawned/module"));
8234 state.spawned_file_identity = Some(SpawnedFileIdentity {
8235 device: 43,
8236 inode: 44,
8237 #[cfg(windows)]
8238 windows_image: None,
8239 });
8240 })?;
8241 }
8242 release_dead_registration(
8243 registry,
8244 runtime.forwarding.as_deref(),
8245 snapshot,
8246 &spec.module_id,
8247 failure,
8248 )
8249 .await?;
8250 reset_restart_count(snapshot, &spec.module_id)?;
8251 process_liveness.track(spec.module_id.clone(), Arc::clone(snapshot));
8252 let next_child = match spawn_and_mark_running(spec, runtime, snapshot) {
8253 Ok(next_child) => next_child,
8254 Err(err) => {
8255 fail_snapshot(snapshot, Some(&spec.module_id), None, failure);
8256 process_liveness.untrack_if_current(&spec.module_id, snapshot);
8257 return Err(err);
8258 }
8259 };
8260 *child = Some(next_child);
8261 debug!(module_id = %spec.module_id, "supervised module enabled");
8262 Ok(true)
8263 } else {
8264 let stop_notice = begin_forwarding_drain_if_configured(
8265 spec,
8266 runtime,
8267 registry,
8268 snapshot,
8269 Some(false),
8270 RouteCloseReason::Disable,
8271 )
8272 .await?;
8273 drain_optional_child(
8274 &spec.module_id,
8275 spec.protocol,
8276 stop_notice,
8277 registry,
8278 runtime.forwarding.as_deref(),
8279 snapshot,
8280 &runtime.terminal_ring,
8281 &runtime.spawn_events,
8282 child,
8283 runtime.drain_timeout,
8284 ModuleState::Disabled,
8285 Some(false),
8286 failure,
8287 )
8288 .await?;
8289 debug!(module_id = %spec.module_id, "supervised module disabled");
8290 Ok(true)
8291 }
8292}
8293
8294#[allow(clippy::too_many_arguments)]
8295async fn on_child_exit(
8296 spec: &ModuleSpec,
8297 policy: RestartPolicy,
8298 registry: &Registry,
8299 snapshot: &SharedSnapshot,
8300 terminal_ring: &Arc<Mutex<TerminalRing>>,
8301 spawn_events: &SpawnEventFeed,
8302 roster: &ChildRoster,
8303 exit_report: ExitReport,
8304 daemon_initiated: bool,
8305) -> NextAction {
8306 if roster.is_closed() {
8312 return on_child_exit_during_daemon_shutdown(
8313 spec,
8314 registry,
8315 snapshot,
8316 terminal_ring,
8317 spawn_events,
8318 exit_report,
8319 )
8320 .await;
8321 }
8322 let unrequested_clean_exit_of_protocol_none = exit_report.kind == ExitKind::Clean
8338 && running_protocol(spec, snapshot) == ModuleProtocol::None;
8339 match exit_report.kind {
8340 ExitKind::Clean if !unrequested_clean_exit_of_protocol_none => {
8341 info!(
8342 module_id = %spec.module_id,
8343 exit_code = ?exit_report.code,
8344 exit_signal = ?exit_report.signal,
8345 "supervised module exited cleanly"
8346 );
8347 if let Err(err) = publish_after_terminal_record(
8348 &spec.module_id,
8349 || {
8350 record_terminal(
8351 &spec.module_id,
8352 terminal_ring,
8353 spawn_events,
8354 &exit_report,
8355 TerminalDisposition::Stopped,
8356 )
8357 },
8358 || {
8359 update_snapshot(snapshot, Some(&spec.module_id), |state| {
8360 state.state = ModuleState::Stopped;
8361 clear_current_process_facts(state);
8362 state.last_exit = Some(exit_report.clone());
8363 })
8364 },
8365 ) {
8366 error!(module_id = %spec.module_id, error = %err, "failed to publish clean module exit");
8367 }
8368 let registration_released = match wait_for_registration_release(
8369 registry,
8370 &spec.module_id,
8371 REGISTRY_RELEASE_TIMEOUT,
8372 )
8373 .await
8374 {
8375 Ok(()) => true,
8376 Err(err) => {
8377 warn!(module_id = %spec.module_id, error = %err, "registration still active after clean exit");
8378 false
8379 }
8380 };
8381 NextAction::Stop {
8382 registration_released,
8383 }
8384 }
8385 ExitKind::Clean | ExitKind::Crash => {
8386 if unrequested_clean_exit_of_protocol_none {
8387 warn!(
8388 module_id = %spec.module_id,
8389 exit_code = ?exit_report.code,
8390 exit_signal = ?exit_report.signal,
8391 "protocol-none module exited cleanly without a stop request; handling it as a crash"
8392 );
8393 } else {
8394 warn!(
8395 module_id = %spec.module_id,
8396 exit_code = ?exit_report.code,
8397 exit_signal = ?exit_report.signal,
8398 "supervised module exited abnormally (crash)"
8399 );
8400 }
8401 let mut restart_schedule = None;
8402 let mut disposition = TerminalDisposition::Disabled;
8403 let mut final_state = ModuleState::Disabled;
8404 let mut disposition_detail = lock_snapshot(snapshot)
8408 .ok()
8409 .and_then(|mut state| state.spawn_failure.take());
8410 let now = Instant::now();
8411 if let Err(err) = update_snapshot(snapshot, Some(&spec.module_id), |state| {
8412 if state.enabled {
8413 if let Some(schedule) = state.next_crash_restart(&policy, now) {
8414 state.respawn_pending = true;
8420 final_state = ModuleState::Restarting;
8421 restart_schedule = Some(schedule);
8422 disposition = TerminalDisposition::Restarting;
8423 } else {
8424 final_state = ModuleState::Failed;
8425 disposition = TerminalDisposition::Failed;
8426 let budget = policy.budget_exhausted_detail();
8427 disposition_detail =
8428 Some(disposition_detail.take().map_or_else(
8429 || budget.clone(),
8430 |cause| format!("{cause}; {budget}"),
8431 ));
8432 }
8433 } else {
8434 disposition = TerminalDisposition::Disabled;
8435 }
8436 }) {
8437 error!(module_id = %spec.module_id, error = %err, "failed to record crashed module exit");
8438 return NextAction::Stop {
8439 registration_released: false,
8440 };
8441 }
8442 if disposition == TerminalDisposition::Failed {
8443 error!(
8448 module_id = %spec.module_id,
8449 max_restarts = policy.max_restarts,
8450 window_secs = policy.window.as_secs(),
8451 "module stopped: {}",
8452 policy.budget_exhausted_detail()
8453 );
8454 }
8455 if let Err(err) = publish_after_terminal_record(
8456 &spec.module_id,
8457 || {
8458 record_terminal_with_detail(
8459 &spec.module_id,
8460 terminal_ring,
8461 spawn_events,
8462 &exit_report,
8463 disposition,
8464 disposition_detail,
8465 )
8466 },
8467 || {
8468 update_snapshot(snapshot, Some(&spec.module_id), |state| {
8469 state.state = final_state;
8470 clear_current_process_facts(state);
8471 if final_state == ModuleState::Failed
8472 && daemon_initiated
8473 && !roster.is_closed()
8474 {
8475 state.failed_retry.schedule(&spec.module_id);
8476 }
8477 state.last_exit = Some(exit_report.clone());
8478 })
8479 },
8480 ) {
8481 error!(module_id = %spec.module_id, error = %err, "failed to publish crashed module exit");
8482 }
8483
8484 if let Some(schedule) = restart_schedule {
8485 NextAction::Restart {
8486 schedule: Some(schedule),
8487 }
8488 } else {
8489 let registration_released = match wait_for_registration_release(
8490 registry,
8491 &spec.module_id,
8492 REGISTRY_RELEASE_TIMEOUT,
8493 )
8494 .await
8495 {
8496 Ok(()) => true,
8497 Err(err) => {
8498 warn!(module_id = %spec.module_id, error = %err, "registration still active after failed module");
8499 false
8500 }
8501 };
8502 NextAction::Stop {
8503 registration_released,
8504 }
8505 }
8506 }
8507 ExitKind::DeliberateSeverance => {
8508 warn!(
8509 module_id = %spec.module_id,
8510 exit_code = ?exit_report.code,
8511 exit_signal = ?exit_report.signal,
8512 "supervised module exited after deliberate connection severance"
8513 );
8514 let should_restart = match lock_snapshot(snapshot) {
8515 Ok(state) => state.enabled,
8516 Err(err) => {
8517 error!(module_id = %spec.module_id, error = %err, "failed to read deliberately severed module state");
8518 return NextAction::Stop {
8519 registration_released: false,
8520 };
8521 }
8522 };
8523 let final_state = if should_restart {
8524 ModuleState::Restarting
8525 } else {
8526 ModuleState::Disabled
8527 };
8528 if let Err(err) = publish_after_terminal_record(
8529 &spec.module_id,
8530 || {
8531 record_terminal(
8532 &spec.module_id,
8533 terminal_ring,
8534 spawn_events,
8535 &exit_report,
8536 terminal_disposition(final_state),
8537 )
8538 },
8539 || {
8540 update_snapshot(snapshot, Some(&spec.module_id), |state| {
8541 state.state = final_state;
8542 clear_current_process_facts(state);
8543 state.last_exit = Some(exit_report.clone());
8544 state.lifetime_restarts += 1;
8545 })
8546 },
8547 ) {
8548 error!(module_id = %spec.module_id, error = %err, "failed to publish deliberately severed module exit");
8549 return NextAction::Stop {
8550 registration_released: false,
8551 };
8552 }
8553
8554 if should_restart {
8555 NextAction::Restart { schedule: None }
8556 } else {
8557 let registration_released = match wait_for_registration_release(
8558 registry,
8559 &spec.module_id,
8560 REGISTRY_RELEASE_TIMEOUT,
8561 )
8562 .await
8563 {
8564 Ok(()) => true,
8565 Err(err) => {
8566 warn!(module_id = %spec.module_id, error = %err, "registration still active after deliberately severed module exit");
8567 false
8568 }
8569 };
8570 NextAction::Stop {
8571 registration_released,
8572 }
8573 }
8574 }
8575 }
8576}
8577
8578async fn on_child_exit_during_daemon_shutdown(
8579 spec: &ModuleSpec,
8580 registry: &Registry,
8581 snapshot: &SharedSnapshot,
8582 terminal_ring: &Arc<Mutex<TerminalRing>>,
8583 spawn_events: &SpawnEventFeed,
8584 exit_report: ExitReport,
8585) -> NextAction {
8586 info!(
8587 module_id = %spec.module_id,
8588 exit_code = ?exit_report.code,
8589 exit_signal = ?exit_report.signal,
8590 exit_kind = ?exit_report.kind,
8591 "supervised module exited during daemon shutdown; not restarting it"
8592 );
8593 if let Err(err) = publish_after_terminal_record(
8594 &spec.module_id,
8595 || {
8596 record_terminal(
8597 &spec.module_id,
8598 terminal_ring,
8599 spawn_events,
8600 &exit_report,
8601 TerminalDisposition::DaemonShutdown,
8602 )
8603 },
8604 || {
8605 update_snapshot(snapshot, Some(&spec.module_id), |state| {
8606 state.state = ModuleState::Stopped;
8607 clear_current_process_facts(state);
8608 state.last_exit = Some(exit_report.clone());
8609 })
8610 },
8611 ) {
8612 error!(module_id = %spec.module_id, error = %err, "failed to publish module exit during daemon shutdown");
8613 }
8614 let registration_released =
8615 wait_for_registration_release(registry, &spec.module_id, REGISTRY_RELEASE_TIMEOUT)
8616 .await
8617 .is_ok();
8618 NextAction::Stop {
8619 registration_released,
8620 }
8621}
8622
8623fn publish_after_terminal_record<T>(
8628 module_id: &str,
8629 record: impl FnOnce(),
8630 publish: impl FnOnce() -> T,
8631) -> T {
8632 let recorded = std::panic::catch_unwind(std::panic::AssertUnwindSafe(record));
8633 let published = publish();
8634 if let Err(panic) = recorded {
8635 error!(
8636 module_id,
8637 "terminal recording panicked; published final state before resuming unwind"
8638 );
8639 std::panic::resume_unwind(panic);
8640 }
8641 published
8642}
8643
8644fn fail_after_wait_error(
8645 module_id: &str,
8646 snapshot: &SharedSnapshot,
8647 terminal_ring: &Arc<Mutex<TerminalRing>>,
8648 spawn_events: &SpawnEventFeed,
8649 detail: String,
8650 failure: &FailureContext,
8651) {
8652 let report = wait_error_exit_report();
8653 publish_after_terminal_record(
8654 module_id,
8655 || {
8656 record_terminal_with_detail(
8657 module_id,
8658 terminal_ring,
8659 spawn_events,
8660 &report,
8661 TerminalDisposition::Failed,
8662 Some(detail),
8663 )
8664 },
8665 || fail_snapshot(snapshot, Some(module_id), Some(report.clone()), failure),
8666 );
8667}
8668
8669#[cfg(test)]
8670fn record_wait_error_terminal(
8671 module_id: &str,
8672 terminal_ring: &Arc<Mutex<TerminalRing>>,
8673 spawn_events: &SpawnEventFeed,
8674) {
8675 record_terminal(
8676 module_id,
8677 terminal_ring,
8678 spawn_events,
8679 &wait_error_exit_report(),
8680 TerminalDisposition::Failed,
8681 );
8682}
8683
8684fn record_terminal(
8685 module_id: &str,
8686 terminal_ring: &Arc<Mutex<TerminalRing>>,
8687 spawn_events: &SpawnEventFeed,
8688 exit_report: &ExitReport,
8689 disposition: TerminalDisposition,
8690) {
8691 record_terminal_with_detail(
8692 module_id,
8693 terminal_ring,
8694 spawn_events,
8695 exit_report,
8696 disposition,
8697 None,
8698 );
8699}
8700
8701fn durable_terminal_history_of(
8705 terminal_ring: &Mutex<TerminalRing>,
8706 module_id: &str,
8707) -> subc_control::TerminalHistory {
8708 let read = terminal_ring
8709 .lock()
8710 .unwrap_or_else(|p| p.into_inner())
8711 .capture_durable_history();
8712 read.read(module_id)
8713}
8714
8715fn record_terminal_with_detail(
8716 module_id: &str,
8717 terminal_ring: &Arc<Mutex<TerminalRing>>,
8718 spawn_events: &SpawnEventFeed,
8719 exit_report: &ExitReport,
8720 disposition: TerminalDisposition,
8721 disposition_detail: Option<String>,
8722) {
8723 #[cfg(test)]
8724 terminal_history_tests::before_terminal_record(spawn_events);
8725 spawn_events.emit_exited(module_id, exit_report.code, exit_report.signal);
8726 let record = TerminalRecord {
8727 exit_code: exit_report.code,
8728 exit_signal: exit_report.signal,
8729 at_ms: exit_report.at_ms,
8730 disposition,
8731 exit_kind: exit_report.kind.into(),
8732 disposition_detail,
8733 };
8734 terminal_ring
8735 .lock()
8736 .unwrap_or_else(|poisoned| poisoned.into_inner())
8737 .record_exit(module_id, record);
8738}
8739
8740fn untrack_if_registration_released(
8741 process_liveness: &SupervisorProcessLiveness,
8742 registry: &Registry,
8743 module_id: &str,
8744 snapshot: &SharedSnapshot,
8745) {
8746 match registry.get_module(module_id) {
8747 Ok(None) => process_liveness.untrack_if_current(module_id, snapshot),
8748 Ok(Some(_)) => {}
8749 Err(err) => {
8750 warn!(module_id, error = %err, "could not determine whether supervisor liveness can be untracked");
8751 }
8752 }
8753}
8754
8755#[cfg(test)]
8769fn apply_wire_spawn_args(
8770 command: &mut Command,
8771 spec: &ModuleSpec,
8772 connection_file_path: Option<&std::path::Path>,
8773 handle: Option<&SupervisorHandle>,
8774) -> Result<Option<NonceHandoff>, SuperviseError> {
8775 apply_wire_spawn_args_for_role(
8776 command,
8777 spec,
8778 connection_file_path,
8779 handle,
8780 SpawnRole::Plain,
8781 )
8782}
8783
8784#[cfg(unix)]
8789type NonceHandoff = subc_os::LaunchNonceHandoff;
8790#[cfg(windows)]
8791type NonceHandoff = subc_os::LaunchNoncePipeHandoff;
8792#[cfg(not(any(unix, windows)))]
8793type NonceHandoff = std::convert::Infallible;
8794
8795fn apply_wire_spawn_args_for_role(
8816 command: &mut Command,
8817 spec: &ModuleSpec,
8818 connection_file_path: Option<&std::path::Path>,
8819 handle: Option<&SupervisorHandle>,
8820 role: SpawnRole,
8821) -> Result<Option<NonceHandoff>, SuperviseError> {
8822 command.env(SUBC_MODULE_ID_ENV, &spec.module_id);
8823 command.env_remove(subc_os::LAUNCH_NONCE_FD_ENV);
8828 #[cfg(windows)]
8829 {
8830 command.env_remove(subc_os::LAUNCH_NONCE_PIPE_ENV);
8831 command.env_remove(subc_os::LAUNCH_NONCE_PIPE_FALLBACK_ENV);
8832 }
8833 command.env_remove(SUBC_LAUNCH_NONCE_ENV);
8835 if spec.protocol == ModuleProtocol::None {
8836 return Ok(None);
8837 }
8838 if let Some(connection_file_path) = connection_file_path {
8839 command.arg(SUBC_ARG).arg(connection_file_path);
8840 }
8841
8842 let nonce = generate_launch_nonce()?;
8846 if let Some(handle) = handle {
8847 match role {
8848 SpawnRole::Plain => {
8849 handle.set_spawn_nonce(&spec.module_id, nonce.clone());
8850 if spec.reserved {
8851 handle.set_reserved_nonce(&spec.module_id, nonce.clone());
8852 }
8853 }
8854 SpawnRole::SwapCandidate => handle.open_swap(&spec.module_id, nonce.clone()),
8855 }
8856 }
8857 #[cfg(unix)]
8858 let handoff = {
8859 let handoff =
8860 subc_os::LaunchNonceHandoff::new(&nonce).map_err(|source| SuperviseError::Spawn {
8861 program: spec.program.clone(),
8862 source,
8863 cgroup_path: None,
8864 })?;
8865 command.env(subc_os::LAUNCH_NONCE_FD_ENV, handoff.fd_env_value());
8866 Some(handoff)
8867 };
8868 #[cfg(windows)]
8869 let handoff = prepare_windows_nonce_pipe(command, spec, &nonce);
8870 #[cfg(not(any(unix, windows)))]
8871 let handoff = None;
8872 #[cfg(not(unix))]
8876 command.env(SUBC_LAUNCH_NONCE_ENV, nonce);
8877 Ok(handoff)
8878}
8879
8880fn apply_child_env(command: &mut Command, spec: &ModuleSpec) {
8881 command.env_remove(CK_LOG_ENV);
8882 command.env_remove(SUBC_SPAWN_ROLE_ENV);
8889 for (key, value) in &spec.env {
8890 if matches!(
8894 key.as_str(),
8895 CAPTURE_MAX_FILE_MB_ENV | CAPTURE_KEEP_ENV | CAPTURE_MAX_AGE_DAYS_ENV
8896 ) || key == SUBC_SPAWN_ROLE_ENV
8897 {
8898 continue;
8899 }
8900 command.env(key, value);
8901 }
8902}
8903
8904#[cfg(windows)]
8905fn prepare_windows_nonce_pipe(
8906 command: &mut Command,
8907 spec: &ModuleSpec,
8908 nonce: &str,
8909) -> Option<NonceHandoff> {
8910 match subc_os::LaunchNoncePipeHandoff::new(nonce) {
8911 Ok(handoff) => {
8912 command.env(subc_os::LAUNCH_NONCE_PIPE_ENV, handoff.name());
8913 command.env(subc_os::LAUNCH_NONCE_PIPE_FALLBACK_ENV, "env");
8918 Some(handoff)
8919 }
8920 Err(error) => {
8921 warn!(module_id = %spec.module_id, %error, "could not prepare launch nonce pipe; retaining the environment handoff");
8922 None
8923 }
8924 }
8925}
8926
8927#[cfg(windows)]
8928fn start_windows_nonce_pipe(
8929 handoff: Option<NonceHandoff>,
8930 pid: u32,
8931 spec: &ModuleSpec,
8932) -> Option<subc_os::LaunchNoncePipeDelivery> {
8933 handoff.and_then(|handoff| match handoff.serve(pid, REGISTRY_RELEASE_TIMEOUT) {
8937 Ok(delivery) => Some(delivery),
8938 Err(error) => {
8939 warn!(module_id = %spec.module_id, %error, "could not serve launch nonce pipe; retaining the environment handoff");
8940 None
8941 }
8942 })
8943}
8944
8945fn observed_launch_nonce_source(snapshot: &SupervisorSnapshot) -> Option<String> {
8946 #[cfg(windows)]
8947 if snapshot.process_alive && snapshot.spawned_protocol == Some(ModuleProtocol::Subc) {
8948 return Some(
8949 if snapshot
8950 .pipe_consumed
8951 .as_ref()
8952 .is_some_and(|consumed| consumed.get().is_some())
8953 {
8954 "pipe"
8955 } else {
8956 "env"
8957 }
8958 .to_owned(),
8959 );
8960 }
8961 let _ = snapshot;
8962 None
8963}
8964
8965#[derive(Debug, Clone, Copy, PartialEq, Eq)]
8968enum SpawnRole {
8969 Plain,
8970 SwapCandidate,
8971}
8972
8973fn apply_spawn_role(command: &mut Command, role: SpawnRole) {
8976 if role == SpawnRole::SwapCandidate {
8977 command.env(SUBC_SPAWN_ROLE_ENV, SPAWN_ROLE_SWAP_CANDIDATE);
8978 }
8979}
8980
8981fn previous_instance_error(
8982 spec: &ModuleSpec,
8983 pid: u32,
8984 reason: crate::live_children::RetainedReason,
8985) -> SuperviseError {
8986 SuperviseError::Spawn {
8987 program: spec.program.clone(),
8988 source: io::Error::new(io::ErrorKind::WouldBlock, reason.waiting_message(pid)),
8989 cgroup_path: None,
8990 }
8991}
8992
8993fn carries_containment(configured: bool, has_container: bool) -> bool {
8994 configured || has_container
8995}
8996
8997fn recovery_block_error(spec: &ModuleSpec, roster: &ChildRoster) -> Option<SuperviseError> {
8998 if !carries_containment(
8999 spec.carry_containment,
9000 containment::current(&roster.containment).is_some(),
9001 ) {
9002 return None;
9003 }
9004 Some(SuperviseError::Spawn {
9005 program: spec.program.clone(),
9006 source: io::Error::new(io::ErrorKind::WouldBlock, roster.recovery_block()?),
9007 cgroup_path: None,
9008 })
9009}
9010
9011fn spawn_child(
9012 spec: &ModuleSpec,
9013 connection_file_path: Option<&std::path::Path>,
9014 handle: Option<&SupervisorHandle>,
9015 ring: &Arc<Mutex<StderrRing>>,
9016 capture_logs_dir: Option<&std::path::Path>,
9017 roster: &ChildRoster,
9018 #[cfg(target_os = "linux")] cgroup_placement: Option<&subc_cgroup::Placement>,
9019) -> Result<SupervisedChild, SuperviseError> {
9020 #[cfg(all(test, unix))]
9021 if let Some(source) = containment_tests::join_failure(spec, roster) {
9022 containment::terminate_blocking(&roster.containment);
9023 return Err(SuperviseError::Spawn {
9024 program: spec.program.clone(),
9025 source,
9026 cgroup_path: None,
9027 });
9028 }
9029 let result = spawn_child_in_slot(
9030 spec,
9031 connection_file_path,
9032 handle,
9033 ring,
9034 capture_logs_dir,
9035 roster,
9036 #[cfg(target_os = "linux")]
9037 cgroup_placement,
9038 SpawnRole::Plain,
9039 false,
9040 );
9041 if result.is_err() {
9042 if roster.admission_pending(&spec.module_id) {
9043 containment::terminate_deferred(&roster.containment);
9044 } else {
9045 containment::terminate_blocking(&roster.containment);
9046 }
9047 }
9048 result
9049}
9050
9051#[allow(clippy::too_many_arguments)]
9064fn spawn_child_in_slot(
9065 spec: &ModuleSpec,
9066 connection_file_path: Option<&std::path::Path>,
9067 handle: Option<&SupervisorHandle>,
9068 ring: &Arc<Mutex<StderrRing>>,
9069 capture_logs_dir: Option<&std::path::Path>,
9070 roster: &ChildRoster,
9071 #[cfg(target_os = "linux")] cgroup_placement: Option<&subc_cgroup::Placement>,
9072 role: SpawnRole,
9073 alternate_slot: bool,
9074) -> Result<SupervisedChild, SuperviseError> {
9075 if let Some(error) = recovery_block_error(spec, roster) {
9076 return Err(error);
9077 }
9078 if let Some((pid, reason)) = roster.previous_instance(&spec.module_id) {
9079 return Err(previous_instance_error(spec, pid, reason));
9080 }
9081 if roster.is_closed() {
9082 return Err(SuperviseError::Spawn {
9083 program: spec.program.clone(),
9084 source: io::Error::other("the daemon is shutting down; not starting a new process"),
9085 cgroup_path: None,
9086 });
9087 }
9088 let carried = containment::current(&roster.containment);
9089 let carrying = carries_containment(spec.carry_containment, carried.is_some());
9090 if role != SpawnRole::Plain && carrying {
9091 return Err(SuperviseError::InvalidSpec {
9092 reason: "a carried container cannot overlap a swap candidate".to_owned(),
9093 });
9094 }
9095 #[cfg(unix)]
9096 let launch_identity = if carrying && carried.is_none() {
9097 Some(
9098 containment::LaunchIdentity::capture().map_err(|source| SuperviseError::Spawn {
9099 program: spec.program.clone(),
9100 source,
9101 cgroup_path: None,
9102 })?,
9103 )
9104 } else {
9105 None
9106 };
9107 #[cfg(target_os = "linux")]
9108 let cgroup_placement = match &carried {
9109 Some(container) => container.cgroup().map(|(placement, _)| placement),
9110 None => cgroup_placement,
9111 };
9112 #[cfg(unix)]
9113 let group_target = if let Some(container) = carried
9114 .as_ref()
9115 .filter(|container| container.group().is_some())
9116 {
9117 containment::GroupTarget::Pinned(Arc::clone(container))
9118 } else {
9119 #[cfg(target_os = "linux")]
9120 let cgroup_container = carrying && cgroup_placement.is_some();
9121 #[cfg(not(target_os = "linux"))]
9122 let cgroup_container = false;
9123 if cgroup_container {
9124 containment::GroupTarget::NoGroup
9125 } else {
9126 containment::GroupTarget::OwnLeader
9127 }
9128 };
9129 #[cfg(target_os = "linux")]
9130 let cgroup_name = {
9131 if let Some((_, name)) = carried.as_ref().and_then(|container| container.cgroup()) {
9136 name.to_owned()
9137 } else if cgroup_placement.is_none() {
9138 swap::cgroup_name(&spec.module_id, alternate_slot)
9139 } else {
9140 let nonce = generate_launch_nonce()?;
9141 let suffix = u128::from_str_radix(&nonce[..32], 16).expect("hex launch nonce");
9142 let mut end = spec.module_id.len().min(64);
9145 while !spec.module_id.is_char_boundary(end) {
9146 end -= 1;
9147 }
9148 format!(
9149 "{}_{suffix}",
9150 swap::cgroup_name(&spec.module_id[..end], alternate_slot)
9151 )
9152 }
9153 };
9154 #[cfg(not(target_os = "linux"))]
9155 let _ = alternate_slot;
9156 #[cfg(target_os = "macos")]
9157 let (mut command, privacy_exec, exec_ack) = privacy_command(spec, roster)?;
9158 #[cfg(windows)]
9159 let image_capture = windows_spawn_capture(spec).map_err(|source| SuperviseError::Spawn {
9160 program: spec.program.clone(),
9161 source,
9162 cgroup_path: None,
9163 })?;
9164 #[cfg(windows)]
9165 let mut command = Command::new(image_capture.path());
9166 #[cfg(not(any(target_os = "macos", windows)))]
9167 let mut command = Command::new(&spec.program);
9168 command.args(&spec.args);
9169 apply_child_env(&mut command, spec);
9199 apply_spawn_role(&mut command, role);
9200 let nonce_handoff =
9201 apply_wire_spawn_args_for_role(&mut command, spec, connection_file_path, handle, role)?;
9202
9203 #[cfg(target_os = "linux")]
9204 let cgroup_path = cgroup_placement
9205 .map(|placement| {
9206 if let Some(container) = &carried {
9207 let path = container
9208 .cgroup_path()
9209 .ok_or_else(|| io::Error::other("carried container has no cgroup path"))?;
9210 placement.existing_module_path(path)?.ok_or_else(|| {
9211 io::Error::new(
9212 io::ErrorKind::NotFound,
9213 "carried cgroup disappeared before replacement",
9214 )
9215 })
9216 } else {
9217 if carrying {
9218 containment::prepare_cgroup_path(placement, &cgroup_name)
9219 } else {
9220 placement.module_path(&cgroup_name)
9221 }
9222 }
9223 })
9224 .transpose()
9225 .map_err(|source| SuperviseError::Cgroup {
9226 module_id: spec.module_id.clone(),
9227 source,
9228 })?;
9229 #[cfg(not(target_os = "linux"))]
9230 let cgroup_path: Option<PathBuf> = None;
9231 #[cfg(unix)]
9232 let mut admission_gate = if carrying {
9233 Some(
9234 admission::Gate::install(&mut command, spec, roster).map_err(|source| {
9235 SuperviseError::Spawn {
9236 program: spec.program.clone(),
9237 source,
9238 cgroup_path: cgroup_path.clone(),
9239 }
9240 })?,
9241 )
9242 } else {
9243 None
9244 };
9245 #[cfg(target_os = "macos")]
9246 exec_ack.install(command.as_std_mut());
9247 #[cfg(target_os = "linux")]
9248 if let Some(path) = &cgroup_path {
9249 if let Err(error) = apply_cgroup_placement(&mut command, spec, path) {
9250 if let Some(placement) = cgroup_placement.filter(|_| carried.is_none()) {
9251 remove_module_cgroup(placement, &cgroup_name);
9252 }
9253 return Err(error);
9254 }
9255 }
9256
9257 let output_sink = if let Some(logs_dir) = capture_logs_dir {
9258 let path = logs_dir.join(format!("{}.stderr.log", spec.module_id));
9259 match ChildOutputSink::open(&path, capture_retention(spec)) {
9260 Ok(sink) => sink,
9261 Err(error) => {
9262 warn!(
9263 module_id = %spec.module_id,
9264 path = %path.display(),
9265 error = %error,
9266 "could not open child output capture file; forwarding to stderr"
9267 );
9268 ChildOutputSink::Stderr
9269 }
9270 }
9271 } else {
9272 ChildOutputSink::Stderr
9273 };
9274
9275 command.stdout(Stdio::piped());
9276 command.stderr(Stdio::piped());
9277 command.kill_on_drop(!carrying);
9278 #[cfg(unix)]
9295 command.process_group(
9296 carried
9297 .as_ref()
9298 .map(|container| container.verified_join_group())
9299 .transpose()
9300 .map_err(|source| SuperviseError::Spawn {
9301 program: spec.program.clone(),
9302 source,
9303 cgroup_path: cgroup_path.clone(),
9304 })?
9305 .flatten()
9306 .unwrap_or(0) as i32,
9307 );
9308 command.stdin(Stdio::null());
9309 #[cfg(unix)]
9310 if let Some(gate) = &mut admission_gate {
9311 gate.install_placement(&mut command);
9312 }
9313 #[cfg(unix)]
9317 if let Some(handoff) = nonce_handoff {
9318 handoff.install_last(command.as_std_mut());
9319 }
9320 #[cfg(not(any(unix, windows)))]
9321 let _ = nonce_handoff;
9322
9323 #[cfg(windows)]
9328 {
9329 subc_jobobject::ensure_root_job().map_err(|source| SuperviseError::Spawn {
9330 program: spec.program.clone(),
9331 source,
9332 cgroup_path: None,
9333 })?;
9334 subc_jobobject::suspend_on_create_async(&mut command);
9335 }
9336 let spawned_at = Instant::now();
9340 #[cfg(all(test, unix))]
9341 containment_tests::record_spawn_attempt(spec, &mut command);
9342 #[cfg(unix)]
9343 let spawned = match admission_gate {
9344 Some(gate) => gate
9345 .spawn(
9346 command,
9347 spec,
9348 roster,
9349 launch_identity,
9350 #[cfg(target_os = "linux")]
9351 cgroup_placement
9352 .zip(cgroup_path.as_ref())
9353 .map(|(placement, path)| {
9354 (placement.clone(), cgroup_name.clone(), path.clone())
9355 }),
9356 )
9357 .map(|(child, admitted)| (child, Some(admitted))),
9358 None => command.spawn().map(|child| (child, None)),
9359 };
9360 #[cfg(not(unix))]
9361 let spawned = command.spawn();
9362 let spawned = match spawned {
9363 Ok(child) => child,
9364 Err(source) => {
9365 if carried
9366 .as_ref()
9367 .and_then(|container| container.group())
9368 .is_some()
9369 {
9370 containment::terminate_blocking(&roster.containment);
9371 }
9372 #[cfg(target_os = "linux")]
9373 if let Some(placement) = cgroup_placement.filter(|_| carried.is_none()) {
9374 remove_module_cgroup(placement, &cgroup_name);
9375 }
9376 return Err(SuperviseError::Spawn {
9377 program: spec.program.clone(),
9378 source,
9379 cgroup_path,
9380 });
9381 }
9382 };
9383 #[cfg(unix)]
9384 let (mut child, mut admitted) = spawned;
9385 #[cfg(unix)]
9386 let group_target = admitted
9387 .as_ref()
9388 .map(|attempt| attempt.group_target())
9389 .unwrap_or(group_target);
9390 #[cfg(not(unix))]
9391 let mut child = spawned;
9392 #[cfg(test)]
9393 spawn_record_tests::after_create(&child, spec);
9394 #[cfg(target_os = "macos")]
9399 drop(exec_ack);
9400
9401 #[cfg(windows)]
9406 let windows_image = match image_capture.bind_tokio(&child) {
9407 Ok(image) => Arc::new(image),
9408 Err(source) => {
9409 kill_and_reap_rejected_child(&mut child);
9410 return Err(SuperviseError::Spawn {
9411 program: spec.program.clone(),
9412 source,
9413 cgroup_path: None,
9414 });
9415 }
9416 };
9417 #[cfg(windows)]
9420 let job = contain_spawned_child(&mut child, spec, carried.as_ref())?;
9421 let spawned_at_ms = unix_ms_now();
9422 #[cfg(not(windows))]
9423 let spawned_from = spec.program.clone();
9424 #[cfg(windows)]
9425 let spawned_from = windows_image.path().to_path_buf();
9426 #[cfg(not(windows))]
9427 let spawned_file_identity = spawned_file_identity(&spawned_from);
9428 #[cfg(windows)]
9429 let spawned_file_identity = Some(SpawnedFileIdentity {
9430 device: 0,
9431 inode: 0,
9432 windows_image: Some(Arc::clone(&windows_image)),
9433 });
9434 let pid = child.id().ok_or_else(|| SuperviseError::Spawn {
9435 program: spec.program.clone(),
9436 source: io::Error::other("spawned child exposed no live pid"),
9437 cgroup_path: cgroup_path.clone(),
9438 })?;
9439 #[cfg(windows)]
9440 let nonce_delivery = start_windows_nonce_pipe(nonce_handoff, pid, spec);
9441 #[cfg(not(windows))]
9442 let process_start_time = admitted
9443 .as_ref()
9444 .map(|admitted| admitted.birth)
9445 .or_else(|| crate::provenance::process_start_time(pid));
9446 #[cfg(windows)]
9447 let process_start_time = Some(windows_image.start_time());
9448 let process_identity = process_start_time.map(|start_time| ProcessIdentity { pid, start_time });
9449 #[cfg(all(test, target_os = "macos"))]
9450 privacy_exec_boundary_tests::before_image_sample(spec, pid);
9451 #[cfg(not(windows))]
9456 let recorded_image = observe_spawned_image(pid);
9457 #[cfg(target_os = "macos")]
9461 let recorded_image = if privacy_exec.is_some() {
9462 None
9463 } else {
9464 recorded_image
9465 };
9466 #[cfg(target_os = "linux")]
9467 let recorded_cgroup_name = cgroup_path.as_ref().map(|_| cgroup_name.clone());
9468 #[cfg(not(target_os = "linux"))]
9469 let recorded_cgroup_name = None;
9470 #[cfg(unix)]
9471 let preadmitted = admitted
9472 .as_mut()
9473 .and_then(|admitted| admitted.leader.take());
9474 #[cfg(not(unix))]
9475 let preadmitted: Option<crate::child_roster::RosterGuard> = None;
9476 let roster_guard = if let Some(guard) = preadmitted {
9477 Ok(guard)
9478 } else {
9479 roster.admit(
9480 spec.module_id.clone(),
9481 pid,
9482 spec.protocol,
9483 process_start_time,
9484 crate::child_roster::RecordedIdentity {
9485 container: None,
9486 #[cfg(unix)]
9487 carried: None,
9488 #[cfg(not(windows))]
9489 start_time: recorded_image.map(|image| image.start_time),
9490 #[cfg(windows)]
9491 start_time: process_start_time,
9492 #[cfg(not(windows))]
9493 executable: recorded_image
9494 .and_then(|image| image.executable)
9495 .map(crate::live_children::ExecutableIdentity::from),
9496 #[cfg(windows)]
9497 executable: Some(crate::live_children::ExecutableIdentity::from(
9498 windows_image.identity(),
9499 )),
9500 cgroup_name: recorded_cgroup_name,
9501 #[cfg(target_os = "linux")]
9502 cgroup_placement: cgroup_placement.cloned(),
9503 },
9504 )
9505 };
9506 let roster_guard = match roster_guard {
9507 Ok(guard) => guard,
9508 Err(failure) => {
9509 return failure.with_cleanup(|source| {
9510 #[cfg(windows)]
9511 if let Some(job) = &job {
9512 let _ = job.lock().unwrap_or_else(|p| p.into_inner()).terminate();
9513 }
9514 #[cfg(target_os = "linux")]
9515 kill_module_cgroup(cgroup_placement, &cgroup_name);
9516 #[cfg(unix)]
9517 let rejected_group = group_target.clone();
9518 kill_and_reap_rejected_child(
9519 &mut child,
9520 #[cfg(unix)]
9521 rejected_group,
9522 );
9523 #[cfg(target_os = "linux")]
9524 if let Some(placement) = cgroup_placement.filter(|_| carried.is_none()) {
9525 remove_module_cgroup(placement, &cgroup_name);
9526 }
9527 Err(SuperviseError::Spawn {
9528 program: spec.program.clone(),
9529 source: io::Error::other(format!(
9530 "could not durably record module child: {source}"
9531 )),
9532 cgroup_path,
9533 })
9534 });
9535 }
9536 };
9537 #[cfg(unix)]
9538 if admitted.is_some() {
9539 if let Some(image) = recorded_image {
9540 if let Err(source) = roster_guard.confirm_executable(image) {
9541 kill_and_reap_rejected_child(&mut child, group_target.clone());
9542 return Err(SuperviseError::Spawn {
9543 program: spec.program.clone(),
9544 source,
9545 cgroup_path,
9546 });
9547 }
9548 }
9549 }
9550 if roster.is_closed() {
9555 if carrying {
9556 #[cfg(unix)]
9557 let rejected_group = admitted
9558 .as_ref()
9559 .map(|admitted| admitted.rejection_target())
9560 .unwrap_or_else(|| group_target.clone());
9561 #[cfg(windows)]
9562 if let Some(job) = &job {
9563 let _ = job.lock().unwrap_or_else(|p| p.into_inner()).terminate();
9564 }
9565 kill_and_reap_rejected_child(
9566 &mut child,
9567 #[cfg(unix)]
9568 rejected_group,
9569 );
9570 #[cfg(all(test, unix))]
9571 let _ =
9572 roster.admission_event_for_test(&spec.module_id, "after_rejected_child", Some(pid));
9573 drop(roster_guard);
9574 return Err(SuperviseError::Spawn {
9575 program: spec.program.clone(),
9576 source: io::Error::other(
9577 "the daemon began shutting down during carried admission; ended the container",
9578 ),
9579 cgroup_path,
9580 });
9581 }
9582 #[cfg(target_os = "linux")]
9584 kill_module_cgroup(cgroup_placement, &cgroup_name);
9585 if let Err(error) = child.start_kill() {
9586 debug!(module_id = %spec.module_id, pid, %error, "kill of a process spawned during daemon shutdown failed; it may already have exited");
9587 }
9588 #[cfg(target_os = "linux")]
9589 if let Some(placement) = cgroup_placement.filter(|_| carried.is_none()) {
9590 while matches!(child.try_wait(), Ok(None)) {
9594 std::thread::yield_now();
9595 }
9596 if matches!(
9597 subc_cgroup::kill_module(Some(placement), &cgroup_name),
9598 subc_cgroup::KillOutcome::Killed
9599 ) {
9600 if let Ok(path) = placement.module_path(&cgroup_name) {
9601 while std::fs::read_to_string(path.join("cgroup.events"))
9602 .ok()
9603 .is_some_and(|events| events.lines().any(|line| line == "populated 1"))
9604 {
9605 std::thread::yield_now();
9606 }
9607 }
9608 }
9609 remove_module_cgroup(placement, &cgroup_name);
9610 }
9611 drop(roster_guard);
9612 return Err(SuperviseError::Spawn {
9613 program: spec.program.clone(),
9614 source: io::Error::other(
9615 "the daemon began shutting down while this process was starting; ended it",
9616 ),
9617 cgroup_path,
9618 });
9619 }
9620
9621 let stdout_pump = match child.stdout.take() {
9622 Some(stdout) => Some(tokio::spawn(pump_stdout_to(stdout, output_sink.clone()))),
9623 None => {
9624 warn!(
9625 module_id = %spec.module_id,
9626 "spawned child exposed no stdout pipe; file capture will be incomplete"
9627 );
9628 None
9629 }
9630 };
9631 let stderr_pump = match child.stderr.take() {
9632 Some(stderr) => {
9633 let generation = ring
9634 .lock()
9635 .unwrap_or_else(|poisoned| poisoned.into_inner())
9636 .begin_process();
9637 Some(StderrPump {
9638 task: tokio::spawn(pump_stderr_to(
9639 stderr,
9640 Arc::clone(ring),
9641 generation,
9642 output_sink,
9643 )),
9644 generation,
9645 })
9646 }
9647 None => {
9648 ring.lock()
9652 .unwrap_or_else(|poisoned| poisoned.into_inner())
9653 .mark_not_captured("stderr pipe was not available on spawn");
9654 warn!(
9655 module_id = %spec.module_id,
9656 "spawned child exposed no stderr pipe; tail will be unavailable"
9657 );
9658 None
9659 }
9660 };
9661
9662 let exec = Arc::new(crate::activity::ExecOutcome::new());
9663 #[cfg(target_os = "macos")]
9664 let pending_exec = privacy_exec.is_some();
9665 #[cfg(not(target_os = "macos"))]
9666 let pending_exec = false;
9667 if !pending_exec {
9668 exec.finish(crate::activity::ExecState::CommittedExec);
9669 }
9670 #[cfg(windows)]
9671 let subtree_target = job
9672 .as_ref()
9673 .map(|job| crate::subtree_resources::Target::Job(Arc::downgrade(job)))
9674 .unwrap_or(crate::subtree_resources::Target::Unavailable("no_job"));
9675 #[cfg(target_os = "linux")]
9676 let subtree_target = cgroup_path
9677 .map(crate::subtree_resources::Target::Cgroup)
9678 .unwrap_or(crate::subtree_resources::Target::Unavailable("no_cgroup"));
9679 #[cfg(not(any(target_os = "linux", windows)))]
9680 let subtree_target = crate::subtree_resources::Target::Unavailable("unsupported_platform");
9681 let mut supervised = SupervisedChild {
9682 subtree_target,
9683 child: containment::ProcessChild::plain(child, pid),
9684 exec: crate::activity::ExecOwner(exec),
9685 #[cfg(windows)]
9686 nonce_delivery,
9687 protocol: spec.protocol,
9688 #[cfg(target_os = "linux")]
9689 module_id: cgroup_name,
9690 #[cfg(target_os = "linux")]
9691 cgroup_placement: cgroup_placement.cloned(),
9692 #[cfg(windows)]
9693 job,
9694 stdout_pump,
9695 stderr_pump,
9696 stderr_ring: Arc::clone(ring),
9697 spawned_at,
9698 registration_deadline: None,
9699 registration_watch: None,
9700 spawned_at_ms,
9701 spawned_from,
9702 launch_spec_digest: LaunchSpecDigest::from_spec(spec),
9703 spawned_file_identity,
9704 process_start_time,
9705 process_identity,
9706 pid,
9707 roster_guard: Some(roster_guard),
9708 #[cfg(target_os = "macos")]
9709 privacy_exec,
9710 #[cfg(target_os = "macos")]
9711 report_ready: Arc::new(OnceLock::new()),
9712 spawn_failure: None,
9713 };
9714 if let Err(error) = containment::adopt(&mut supervised, spec, roster) {
9715 supervised.child.reject(
9716 #[cfg(unix)]
9717 group_target,
9718 );
9719 #[cfg(all(test, unix))]
9720 let _ = roster.admission_event_for_test(&spec.module_id, "after_rejected_child", Some(pid));
9721 return Err(error);
9722 }
9723 #[cfg(unix)]
9724 if let Some(admitted) = admitted.take() {
9725 admitted.disarm();
9726 }
9727 #[cfg(windows)]
9728 if let Err(error) = resume_suspended_child(pid, spec) {
9729 containment::terminate_blocking(&roster.containment);
9730 let _ = supervised.child.start_kill();
9731 return Err(error);
9732 }
9733 Ok(supervised)
9734}
9735
9736#[cfg(test)]
9737mod spawn_record_tests {
9738 use super::*;
9739
9740 thread_local! {
9741 static LAST_PID: std::cell::Cell<Option<u32>> = const { std::cell::Cell::new(None) };
9742 }
9743
9744 pub(super) fn after_create(child: &Child, _spec: &ModuleSpec) {
9745 LAST_PID.set(child.id());
9746 #[cfg(windows)]
9747 if let Some((_, address)) = _spec
9748 .env
9749 .iter()
9750 .find(|(key, _)| key == "SUBC_TEST_AFTER_CREATE")
9751 {
9752 use std::io::{Read, Write};
9753 let mut stream = std::net::TcpStream::connect(address).unwrap();
9754 stream
9755 .write_all(&child.id().unwrap().to_ne_bytes())
9756 .unwrap();
9757 let mut release = [0];
9758 stream.read_exact(&mut release).unwrap();
9759 }
9760 }
9761
9762 #[cfg(windows)]
9763 pub(super) fn assignment_failpoint(spec: &ModuleSpec) -> io::Result<()> {
9764 if spec
9765 .env
9766 .iter()
9767 .any(|(key, _)| key == "SUBC_TEST_ASSIGNMENT_FAILURE")
9768 {
9769 Err(io::Error::other("injected module job assignment failure"))
9770 } else {
9771 Ok(())
9772 }
9773 }
9774
9775 fn spec() -> ModuleSpec {
9776 ModuleSpec {
9777 module_id: "spawn-record-worker".into(),
9778 #[cfg(unix)]
9779 program: "/bin/sleep".into(),
9780 #[cfg(windows)]
9781 program: std::env::current_exe().unwrap(),
9782 #[cfg(unix)]
9783 args: vec!["60".into()],
9784 #[cfg(windows)]
9785 args: vec![
9786 "--exact".into(),
9787 "supervise::spawn_record_tests::windows_store_touch_fixture".into(),
9788 "--nocapture".into(),
9789 ],
9790 env: vec![],
9791 reserved: false,
9792 reserved_prefixes: vec![],
9793 protocol: ModuleProtocol::None,
9794 overlap: Default::default(),
9795 carry_containment: false,
9796 }
9797 }
9798
9799 #[tokio::test]
9800 async fn record_sync_failure_fails_spawn_and_reaps_child() {
9801 let home = cortexkit_test_support::ScratchDir::new("record-sync-failure");
9802 let supervisor =
9803 Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default())
9804 .with_live_children_record(home.join("live-children.json"));
9805 crate::live_children::fail_next_file_sync();
9806 let result = supervisor.spawn(spec());
9807 let failed = result.is_err();
9808 if let Ok(module) = result {
9809 module.stop().await.unwrap();
9810 }
9811 assert!(failed, "record failure must fail spawn");
9812 let pid = LAST_PID.get().expect("the real OS child was created");
9813 assert!(
9814 subc_os::Process::open(pid).unwrap().is_none(),
9815 "child {pid} survived rejected admission or was not reaped"
9816 );
9817 }
9818
9819 #[cfg(target_os = "macos")]
9820 #[test]
9821 #[ignore = "manual on-device latency measurement"]
9822 fn durable_record_write_latency_50() {
9823 let home = cortexkit_test_support::ScratchDir::new("record-latency");
9824 let path = home.join("live-children.json");
9825 let children = vec![crate::live_children::LiveChild {
9826 module_id: "latency-worker".into(),
9827 pid: std::process::id(),
9828 protocol: ModuleProtocol::None,
9829 start_time: Some(1),
9830 executable: None,
9831 cgroup_name: None,
9832 container: None,
9833 }];
9834 let mut samples = Vec::new();
9835 for _ in 0..50 {
9836 let start = std::time::Instant::now();
9837 crate::live_children::write_record(&path, &children).unwrap();
9838 samples.push(start.elapsed());
9839 }
9840 samples.sort();
9841 eprintln!(
9842 "durable_record_write_latency_50: n=50 median={:?} p95={:?}",
9843 (samples[24] + samples[25]) / 2,
9844 samples[47]
9845 );
9846 }
9847
9848 #[cfg(windows)]
9849 #[test]
9850 fn windows_store_touch_fixture() {
9851 if let Some(path) = std::env::var_os("SUBC_TEST_STORE_TOUCH") {
9852 std::fs::write(path, b"module ran").unwrap();
9853 }
9854 }
9855
9856 #[cfg(windows)]
9857 #[tokio::test]
9858 async fn windows_daemon_fixture() {
9859 let Ok(address) = std::env::var("SUBC_TEST_DAEMON_BARRIER") else {
9860 return;
9861 };
9862 let mut spec = spec();
9863 spec.env.push(("SUBC_TEST_AFTER_CREATE".into(), address));
9864 let supervisor =
9865 Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default());
9866 let _module = supervisor.spawn(spec).unwrap();
9867 }
9868
9869 #[cfg(windows)]
9870 #[test]
9871 fn windows_daemon_death_before_module_job_assignment_leaves_no_child() {
9872 use std::io::Read;
9873 let listener = std::net::TcpListener::bind("127.0.0.1:0").unwrap();
9874 let mut command = std::process::Command::new(std::env::current_exe().unwrap());
9877 command
9878 .args([
9879 "--exact",
9880 "supervise::spawn_record_tests::windows_daemon_fixture",
9881 "--nocapture",
9882 ])
9883 .env(
9884 "SUBC_TEST_DAEMON_BARRIER",
9885 listener.local_addr().unwrap().to_string(),
9886 );
9887 let mut daemon = subc_jobobject::spawn_contained(&mut command).unwrap();
9888 listener.set_nonblocking(true).unwrap();
9889 let deadline = std::time::Instant::now() + Duration::from_secs(30);
9890 let mut stream = loop {
9891 if let Ok((stream, _)) = listener.accept() {
9892 break stream;
9893 }
9894 assert!(
9895 daemon.child.try_wait().unwrap().is_none(),
9896 "daemon failed before rendezvous"
9897 );
9898 assert!(
9899 std::time::Instant::now() < deadline,
9900 "daemon did not reach CreateProcess failpoint"
9901 );
9902 std::thread::yield_now();
9903 };
9904 stream
9905 .set_read_timeout(Some(Duration::from_secs(30)))
9906 .unwrap();
9907 let mut pid = [0; 4];
9908 stream.read_exact(&mut pid).unwrap();
9909 let pid = u32::from_ne_bytes(pid);
9910 assert!(subc_jobobject::process_exists(pid));
9911 daemon.child.kill().unwrap();
9914 daemon.child.wait().unwrap();
9915 let exited = subc_jobobject::wait_for_process_exit(pid, Duration::from_secs(10));
9916 if !exited {
9917 daemon.job.terminate().unwrap();
9918 }
9919 assert!(exited, "suspended child {pid} outlived hard-killed daemon");
9920 }
9921
9922 #[cfg(windows)]
9923 #[tokio::test]
9924 async fn windows_containment_failure_never_resumes() {
9925 let home = cortexkit_test_support::ScratchDir::new("assignment-refusal");
9926 let sentinel = home.join("store-touched");
9927 let mut spec = spec();
9928 spec.env
9929 .push(("SUBC_TEST_ASSIGNMENT_FAILURE".into(), "1".into()));
9930 spec.env.push((
9931 "SUBC_TEST_STORE_TOUCH".into(),
9932 sentinel.display().to_string(),
9933 ));
9934 let supervisor =
9935 Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default());
9936 let result = supervisor.spawn(spec);
9937 if let Ok(module) = result {
9940 let _ = tokio::time::timeout(Duration::from_secs(10), async {
9941 while !sentinel.exists() && module.state().unwrap() == ModuleState::Running {
9942 tokio::task::yield_now().await;
9943 }
9944 })
9945 .await;
9946 let _ = module.stop().await;
9947 panic!(
9948 "assignment failure resumed a module; store touched={}",
9949 sentinel.exists()
9950 );
9951 }
9952 assert!(
9953 !sentinel.exists(),
9954 "refused module executed store-touch sentinel"
9955 );
9956 assert!(!subc_jobobject::process_exists(LAST_PID.get().unwrap()));
9957 }
9958
9959 #[cfg(windows)]
9960 #[tokio::test]
9961 async fn windows_nested_daemon_fixture() {
9962 let Some(sentinel) = std::env::var_os("SUBC_TEST_NESTED_STORE") else {
9963 return;
9964 };
9965 let mut spec = spec();
9966 spec.env.push((
9967 "SUBC_TEST_STORE_TOUCH".into(),
9968 sentinel.to_str().unwrap().into(),
9969 ));
9970 let supervisor =
9971 Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default());
9972 let module = supervisor
9973 .spawn(spec)
9974 .expect("per-module job must nest below daemon and host jobs");
9975 tokio::time::timeout(Duration::from_secs(10), async {
9976 while !PathBuf::from(&sentinel).exists() {
9977 tokio::task::yield_now().await;
9978 }
9979 })
9980 .await
9981 .unwrap();
9982 module.stop().await.unwrap();
9983 }
9984
9985 #[cfg(windows)]
9986 #[test]
9987 fn windows_module_job_nests_under_host_job() {
9988 let home = cortexkit_test_support::ScratchDir::new("nested-jobs");
9989 let sentinel = home.join("nested-store-touched");
9990 let mut command = std::process::Command::new(std::env::current_exe().unwrap());
9991 command
9992 .args([
9993 "--exact",
9994 "supervise::spawn_record_tests::windows_nested_daemon_fixture",
9995 "--nocapture",
9996 ])
9997 .env("SUBC_TEST_NESTED_STORE", &sentinel);
9998 let mut daemon = subc_jobobject::spawn_contained(&mut command).unwrap();
9999 assert!(daemon.child.wait().unwrap().success());
10000 assert!(sentinel.exists(), "nested protocol:none module never ran");
10001 }
10002}
10003
10004#[cfg(target_os = "linux")]
10005pub(crate) fn kill_module_cgroup(placement: Option<&subc_cgroup::Placement>, module_id: &str) {
10006 use subc_cgroup::KillOutcome;
10007 match subc_cgroup::kill_module(placement, module_id) {
10008 KillOutcome::Killed => {}
10009 KillOutcome::NotPlaced | KillOutcome::Unsupported => {
10010 debug!(
10011 module_id,
10012 "cgroup tree kill unavailable; using direct-child kill"
10013 );
10014 }
10015 KillOutcome::IoError { path, error } => {
10016 warn!(module_id, path = %path.display(), %error, "cgroup tree kill failed; using direct-child kill");
10017 }
10018 }
10019}
10020
10021#[cfg(windows)]
10029fn contain_spawned_child(
10030 child: &mut Child,
10031 spec: &ModuleSpec,
10032 carried: Option<&Arc<containment::CarriedContainment>>,
10033) -> Result<Option<Arc<Mutex<subc_jobobject::JobObject>>>, SuperviseError> {
10034 let assignment = || {
10035 let job = match carried {
10036 Some(container) => container.job(),
10037 None => Arc::new(Mutex::new(subc_jobobject::JobObject::new()?)),
10038 };
10039 #[cfg(test)]
10040 spawn_record_tests::assignment_failpoint(spec)?;
10041 job.lock()
10042 .unwrap_or_else(|p| p.into_inner())
10043 .assign(child)?;
10044 Ok::<_, io::Error>(job)
10045 };
10046 match assignment() {
10047 Ok(job) => Ok(Some(job)),
10048 Err(source) => {
10049 kill_and_reap_rejected_child(child);
10050 Err(SuperviseError::Spawn {
10051 program: spec.program.clone(),
10052 source: io::Error::other(format!(
10053 "Windows module job containment failed: {source}"
10054 )),
10055 cgroup_path: None,
10056 })
10057 }
10058 }
10059}
10060
10061fn kill_and_reap_rejected_child(child: &mut Child, #[cfg(unix)] group: containment::GroupTarget) {
10064 #[cfg(unix)]
10065 group.signal_rejected(child);
10066 let _ = child.start_kill();
10067 while matches!(child.try_wait(), Ok(None)) {
10068 std::thread::yield_now();
10069 }
10070}
10071
10072#[cfg(windows)]
10077fn resume_suspended_child(pid: u32, spec: &ModuleSpec) -> Result<(), SuperviseError> {
10078 if let Err(source) = subc_jobobject::resume_main_thread(pid) {
10079 let _ = std::process::Command::new("taskkill.exe")
10083 .args(["/PID", &pid.to_string(), "/T", "/F"])
10084 .stdin(Stdio::null())
10085 .stdout(Stdio::null())
10086 .stderr(Stdio::null())
10087 .status();
10088 return Err(SuperviseError::Spawn {
10089 program: spec.program.clone(),
10090 source,
10091 cgroup_path: None,
10092 });
10093 }
10094 Ok(())
10095}
10096
10097#[cfg(target_os = "linux")]
10098fn remove_module_cgroup(placement: &subc_cgroup::Placement, module_id: &str) {
10099 match placement.remove_module(module_id) {
10100 Ok(()) => debug!(module_id, "removed module cgroup after process exit"),
10101 Err(error) => warn!(
10102 module_id,
10103 error = %error,
10104 "could not remove module cgroup after process exit; continuing teardown"
10105 ),
10106 }
10107}
10108
10109#[cfg(target_os = "linux")]
10110const REAPED_CGROUP_TERM_GRACE: Duration = Duration::from_secs(2);
10111#[cfg(target_os = "linux")]
10112const REAPED_CGROUP_FREEZE_TIMEOUT: Duration = Duration::from_millis(100);
10113
10114#[cfg(target_os = "linux")]
10115async fn cleanup_reaped_cgroup(placement: &subc_cgroup::Placement, module_id: &str) {
10116 let path = placement.module_path(module_id).ok();
10117 if path.as_ref().is_some_and(|path| {
10118 std::fs::read_to_string(path.join("cgroup.events"))
10119 .ok()
10120 .is_some_and(|events| events.lines().any(|line| line == "populated 0"))
10121 }) {
10122 remove_module_cgroup(placement, module_id);
10123 return;
10124 }
10125
10126 if terminate_reaped_cgroup(placement, module_id).await {
10129 if let Some(path) = &path {
10130 let deadline = Instant::now() + REAPED_CGROUP_TERM_GRACE;
10131 while cgroup_is_populated(path) && Instant::now() < deadline {
10132 sleep_until((Instant::now() + Duration::from_millis(1)).min(deadline)).await;
10133 }
10134 }
10135 }
10136
10137 if matches!(
10141 subc_cgroup::kill_module(Some(placement), module_id),
10142 subc_cgroup::KillOutcome::Killed
10143 ) {
10144 if let Some(path) = &path {
10145 while cgroup_is_populated(path) {
10146 sleep(Duration::from_millis(1)).await;
10147 }
10148 }
10149 }
10150 remove_module_cgroup(placement, module_id);
10151}
10152
10153#[cfg(target_os = "linux")]
10154fn cgroup_is_populated(path: &std::path::Path) -> bool {
10155 std::fs::read_to_string(path.join("cgroup.events"))
10156 .ok()
10157 .is_some_and(|events| events.lines().any(|line| line == "populated 1"))
10158}
10159
10160#[cfg(target_os = "linux")]
10161async fn terminate_reaped_cgroup(placement: &subc_cgroup::Placement, module_id: &str) -> bool {
10162 let frozen =
10163 match subc_cgroup::freeze_module(Some(placement), module_id, REAPED_CGROUP_FREEZE_TIMEOUT)
10164 .await
10165 {
10166 subc_cgroup::FreezeOutcome::Frozen(frozen) => frozen,
10167 outcome => {
10168 tracing::debug!(
10169 module_id,
10170 ?outcome,
10171 "cgroup freeze failed; skipping SIGTERM grace"
10172 );
10173 return false;
10174 }
10175 };
10176 let listed = match frozen.processes() {
10177 subc_cgroup::ProcsOutcome::Listed(pids) => {
10178 for pid in pids.into_iter().filter(|pid| *pid > 1) {
10179 if let Some(pid) = rustix::process::Pid::from_raw(pid) {
10180 if let Err(error) =
10181 rustix::process::kill_process(pid, rustix::process::Signal::TERM)
10182 {
10183 tracing::debug!(module_id, %pid, %error, "leftover cgroup SIGTERM failed");
10184 }
10185 }
10186 }
10187 true
10188 }
10189 outcome => {
10190 tracing::debug!(
10191 module_id,
10192 ?outcome,
10193 "cgroup process listing failed; skipping SIGTERM grace"
10194 );
10195 false
10196 }
10197 };
10198 match frozen.thaw() {
10199 subc_cgroup::ThawOutcome::Thawed => listed,
10200 outcome => {
10201 tracing::debug!(
10202 module_id,
10203 ?outcome,
10204 "cgroup thaw failed; proceeding to cgroup.kill"
10205 );
10206 false
10207 }
10208 }
10209}
10210
10211#[cfg(target_os = "linux")]
10212fn apply_cgroup_placement(
10213 command: &mut Command,
10214 spec: &ModuleSpec,
10215 path: &std::path::Path,
10216) -> Result<(), SuperviseError> {
10217 subc_cgroup::apply(command, path).map_err(|source| SuperviseError::Cgroup {
10218 module_id: spec.module_id.clone(),
10219 source,
10220 })
10221}
10222
10223fn capture_retention(spec: &ModuleSpec) -> Retention {
10224 let defaults = Retention::default();
10225 let value = |name: &str| {
10226 spec.env
10227 .iter()
10228 .rev()
10229 .find_map(|(key, value)| (key == name).then_some(value.as_str()))
10230 };
10231 Retention {
10232 max_file_mb: value(CAPTURE_MAX_FILE_MB_ENV)
10233 .and_then(|value| value.parse().ok())
10234 .unwrap_or(defaults.max_file_mb),
10235 keep: value(CAPTURE_KEEP_ENV)
10236 .and_then(|value| value.parse().ok())
10237 .unwrap_or(defaults.keep),
10238 max_age_days: value(CAPTURE_MAX_AGE_DAYS_ENV)
10239 .and_then(|value| value.parse().ok())
10240 .unwrap_or(defaults.max_age_days),
10241 }
10242}
10243
10244fn generate_launch_nonce() -> Result<String, SuperviseError> {
10247 let mut bytes = [0u8; 32];
10248 getrandom::getrandom(&mut bytes).map_err(|source| SuperviseError::LaunchNonce {
10249 reason: source.to_string(),
10250 })?;
10251 let mut hex = String::with_capacity(64);
10252 for b in bytes {
10253 use std::fmt::Write;
10254 let _ = write!(hex, "{b:02x}");
10255 }
10256 Ok(hex)
10257}
10258
10259fn constant_time_eq(a: &[u8], b: &[u8]) -> bool {
10262 if a.len() != b.len() {
10263 return false;
10264 }
10265 let mut diff = 0u8;
10266 for (x, y) in a.iter().zip(b.iter()) {
10267 diff |= x ^ y;
10268 }
10269 diff == 0
10270}
10271
10272#[cfg(not(windows))]
10276fn observe_spawned_image(pid: u32) -> Option<subc_os::Observation> {
10277 let process = subc_os::Process::open(pid).ok().flatten()?;
10278 #[cfg(target_os = "linux")]
10279 return sample_after_linux_exec(
10280 || {
10281 let stat = std::fs::read_to_string(format!("/proc/{pid}/stat")).ok()?;
10282 let committed = linux_exec_committed(&stat);
10283 #[cfg(test)]
10284 spawn_image_tests::report_exec_state(committed);
10285 committed
10286 },
10287 || process.observe(),
10288 );
10289 #[cfg(not(target_os = "linux"))]
10290 process.observe()
10291}
10292
10293#[cfg(target_os = "linux")]
10297fn linux_exec_committed(stat: &str) -> Option<bool> {
10298 const PF_FORKNOEXEC: u64 = 0x40;
10299 let flags = stat.rsplit_once(')')?.1.split_whitespace().nth(6)?;
10300 Some(flags.parse::<u64>().ok()? & PF_FORKNOEXEC == 0)
10301}
10302
10303#[cfg(target_os = "linux")]
10304fn sample_after_linux_exec(
10305 mut exec_committed: impl FnMut() -> Option<bool>,
10306 observe: impl FnOnce() -> Option<subc_os::Observation>,
10307) -> Option<subc_os::Observation> {
10308 let deadline = std::time::Instant::now() + Duration::from_secs(1);
10309 while !exec_committed()? {
10310 if std::time::Instant::now() >= deadline {
10314 return None;
10315 }
10316 std::thread::sleep(Duration::from_micros(50));
10317 }
10318 observe()
10319}
10320
10321#[cfg(all(test, target_os = "linux"))]
10322mod spawn_image_tests {
10323 use super::*;
10324 use std::cell::{Cell, RefCell};
10325
10326 enum ExecEvent {
10327 Uncommitted,
10328 Image(Option<subc_os::Observation>),
10329 }
10330
10331 thread_local! {
10332 static EXEC_EVENTS: RefCell<Option<std::sync::mpsc::Sender<ExecEvent>>> = const { RefCell::new(None) };
10333 }
10334
10335 pub(super) fn report_exec_state(committed: Option<bool>) {
10336 if committed == Some(false) {
10337 EXEC_EVENTS.with(|events| {
10338 if let Some(events) = events.borrow_mut().take() {
10339 let _ = events.send(ExecEvent::Uncommitted);
10340 }
10341 });
10342 }
10343 }
10344
10345 #[test]
10346 fn linux_spawn_observation_waits_for_a_paused_fork_to_exec() {
10347 use std::io::{Read, Write};
10348 use std::os::fd::AsRawFd;
10349
10350 let descriptor = std::fs::File::open("/dev/null").unwrap();
10351 let mut command = std::process::Command::new("/bin/sleep");
10352 command.arg("30");
10353 let (mut ready, mut release) =
10354 subc_os::fork_exec_test::pause_before_exec(&mut command, descriptor.as_raw_fd())
10355 .unwrap();
10356 let spawn = std::thread::spawn(move || command.spawn());
10357 let mut message = [0u8; 8];
10358 ready.read_exact(&mut message).unwrap();
10359 let pid = i32::from_ne_bytes(message[..4].try_into().unwrap()) as u32;
10360 let before = std::fs::read_to_string(format!("/proc/{pid}/stat"));
10361 let inherited = subc_os::Process::open(pid).unwrap().unwrap().observe();
10362 let (events, received) = std::sync::mpsc::channel();
10363 let observer = std::thread::spawn(move || {
10364 EXEC_EVENTS.with(|slot| *slot.borrow_mut() = Some(events.clone()));
10365 let image = observe_spawned_image(pid);
10366 let _ = events.send(ExecEvent::Image(image));
10367 });
10368 let first = received.recv_timeout(Duration::from_secs(5));
10372 release.write_all(&[1]).unwrap();
10373 let mut child = spawn.join().unwrap().unwrap();
10374 let after = match first {
10375 Ok(ExecEvent::Uncommitted) => match received.recv_timeout(Duration::from_secs(5)) {
10376 Ok(ExecEvent::Image(image)) => image,
10377 _ => None,
10378 },
10379 Ok(ExecEvent::Image(image)) => image,
10380 Err(_) => None,
10381 };
10382 observer.join().unwrap();
10383 child.kill().unwrap();
10384 child.wait().unwrap();
10385
10386 assert_eq!(linux_exec_committed(&before.unwrap()), Some(false));
10387 assert_eq!(
10388 inherited.unwrap().executable,
10389 subc_os::file_identity(&std::env::current_exe().unwrap())
10390 );
10391 assert_eq!(
10392 after.unwrap().executable,
10393 subc_os::file_identity(std::path::Path::new("/bin/sleep"))
10394 );
10395 }
10396
10397 #[test]
10398 fn spawned_image_is_not_sampled_until_linux_exec_commits() {
10399 let committed = Cell::new(false);
10400 let inherited = subc_os::Observation {
10401 start_time: 23,
10402 executable: Some(subc_os::FileIdentity {
10403 device: 29,
10404 inode: 31,
10405 }),
10406 };
10407 let module = subc_os::Observation {
10408 start_time: 23,
10409 executable: Some(subc_os::FileIdentity {
10410 device: 29,
10411 inode: 37,
10412 }),
10413 };
10414 let polls = Cell::new(0);
10415 let image = sample_after_linux_exec(
10416 || {
10417 polls.set(polls.get() + 1);
10418 committed.set(polls.get() == 2);
10421 Some(committed.get())
10422 },
10423 || Some(if committed.get() { module } else { inherited }),
10424 );
10425 assert_eq!(image, Some(module));
10426 assert_eq!(polls.get(), 2);
10427 }
10428
10429 #[test]
10430 fn linux_exec_flag_is_read_from_flags_after_the_last_parenthesis() {
10431 assert_eq!(
10432 linux_exec_committed("17 (a ) b) R 1 2 3 4 5 4194368"),
10433 Some(false)
10434 );
10435 assert_eq!(
10436 linux_exec_committed("17 (a ) b) R 1 2 3 4 5 4194304"),
10437 Some(true)
10438 );
10439 assert_eq!(linux_exec_committed("17 (a) R 1 2 3 4 5 invalid"), None);
10440 assert_eq!(linux_exec_committed("17 (a) R"), None);
10441 }
10442
10443 #[test]
10444 fn unreadable_exec_state_never_samples_an_inherited_image() {
10445 assert_eq!(
10446 sample_after_linux_exec(
10447 || None,
10448 || panic!("must not sample before exec confirmation")
10449 ),
10450 None
10451 );
10452 }
10453
10454 #[test]
10455 fn uncommitted_exec_times_out_without_sampling_an_inherited_image() {
10456 assert_eq!(
10457 sample_after_linux_exec(
10458 || Some(false),
10459 || panic!("must not sample after an exec timeout")
10460 ),
10461 None
10462 );
10463 }
10464}
10465
10466#[cfg(windows)]
10469fn windows_spawn_capture(spec: &ModuleSpec) -> io::Result<subc_os::ExecutableCapture> {
10470 let mut program = spec.program.clone();
10471 if program.is_absolute() || program.components().count() > 1 {
10472 match subc_os::ExecutableCapture::open(&program) {
10475 Ok(capture) => return Ok(capture),
10476 Err(error)
10477 if error.kind() == io::ErrorKind::NotFound && program.extension().is_none() => {}
10478 Err(error) => return Err(error),
10479 }
10480 program.set_extension("exe");
10481 return subc_os::ExecutableCapture::open(&program);
10482 }
10483 if program.extension().is_none() {
10484 program.set_extension("exe");
10485 }
10486 let custom_path = spec
10487 .env
10488 .iter()
10489 .rev()
10490 .find(|(key, _)| key.eq_ignore_ascii_case("PATH"));
10491 let mut directories = Vec::new();
10492 if let Some(parent) = std::env::current_exe()?.parent() {
10495 directories.push(parent.to_path_buf());
10496 }
10497 directories.push(std::env::current_dir()?);
10498 if let Some(windows) = std::env::var_os("SystemRoot") {
10499 let windows = PathBuf::from(windows);
10500 directories.push(windows.join("System32"));
10501 directories.push(windows.clone());
10502 }
10503 let path = custom_path
10504 .map(|(_, value)| std::ffi::OsString::from(value))
10505 .or_else(|| std::env::var_os("PATH"));
10506 if let Some(path) = path {
10507 directories.extend(std::env::split_paths(&path));
10508 }
10509 for directory in directories {
10510 match subc_os::ExecutableCapture::open(&directory.join(&program)) {
10511 Ok(capture) => return Ok(capture),
10512 Err(error) if error.kind() == io::ErrorKind::NotFound => continue,
10513 Err(error) => return Err(error),
10514 }
10515 }
10516 Err(io::Error::new(
10517 io::ErrorKind::NotFound,
10518 "resolved executable not found",
10519 ))
10520}
10521
10522fn spawn_and_mark_running(
10523 spec: &ModuleSpec,
10524 runtime: &SupervisorRuntimeConfig,
10525 snapshot: &SharedSnapshot,
10526) -> Result<SupervisedChild, SuperviseError> {
10527 let activity_spawn = runtime.begin_activity_spawn(&spec.module_id);
10528 let registration_watch = watch_registration(&runtime.registry, spec);
10529 #[cfg(unix)]
10530 let joining_group = containment::current(&runtime.child_roster.containment)
10531 .is_some_and(|container| container.group().is_some());
10532 let spawn = || {
10533 spawn_child(
10534 spec,
10535 runtime.connection_file_path.as_deref(),
10536 runtime.supervisor_handle.as_ref(),
10537 &runtime.stderr_ring,
10538 runtime.capture_logs_dir.as_deref(),
10539 &runtime.child_roster,
10540 #[cfg(target_os = "linux")]
10541 runtime.cgroup_placement.as_ref(),
10542 )
10543 };
10544 let result = spawn();
10545 #[cfg(unix)]
10546 let result = if result.is_err()
10547 && joining_group
10548 && !runtime.child_roster.is_closed()
10549 && lock_snapshot(snapshot).is_ok_and(|state| state.enabled)
10550 && runtime
10551 .child_roster
10552 .previous_instance(&spec.module_id)
10553 .is_none()
10554 {
10555 warn!(module_id = %spec.module_id, "carried process-group spawn failed; trying one fresh container");
10559 spawn()
10560 } else {
10561 result
10562 };
10563 let child = result?;
10564 let mut child = child;
10565 arm_registration_deadline(&mut child, runtime.register_timeout, registration_watch);
10566 if let Err(error) = set_running(snapshot, &child, &spec.module_id, &runtime.spawn_events) {
10567 containment::terminate_blocking(&runtime.child_roster.containment);
10568 return Err(error);
10569 }
10570 if let Some(spawn) = activity_spawn {
10571 spawn.complete(snapshot);
10572 }
10573 Ok(child)
10574}
10575
10576struct RegistrationWatch {
10577 changes: watch::Receiver<u64>,
10578 baseline: u64,
10579}
10580
10581fn watch_registration(registry: &Registry, spec: &ModuleSpec) -> Option<RegistrationWatch> {
10585 if spec.protocol != ModuleProtocol::Subc {
10586 return None;
10587 }
10588 let changes = registry.subscribe_module_changes(&spec.module_id).ok()?;
10589 let baseline = *changes.borrow();
10590 Some(RegistrationWatch { changes, baseline })
10591}
10592
10593fn arm_registration_deadline(
10594 child: &mut SupervisedChild,
10595 budget: Duration,
10596 registration_watch: Option<RegistrationWatch>,
10597) {
10598 child.registration_deadline =
10599 (child.protocol == ModuleProtocol::Subc).then_some(child.spawned_at + budget);
10600 child.registration_watch = registration_watch;
10601}
10602
10603async fn stop_unregistered_child(child: &mut SupervisedChild) -> io::Result<()> {
10604 if let Some(container) = child.child.container() {
10605 container.stop_members().await;
10606 child.kill_leader()?;
10607 child.wait().await?;
10608 return Ok(());
10609 }
10610 #[cfg(unix)]
10611 {
10612 let group = unreaped_registration_group(child)?;
10615 let _ = rustix::process::kill_process_group(group, rustix::process::Signal::TERM);
10616 sleep(crate::child_roster::TERM_TO_KILL).await;
10621 let group = unreaped_registration_group(child)?;
10622 let _ = rustix::process::kill_process_group(group, rustix::process::Signal::KILL);
10623 }
10624 child.start_kill()?;
10627 child.wait().await?;
10628 Ok(())
10629}
10630
10631#[cfg(unix)]
10632fn unreaped_registration_group(child: &SupervisedChild) -> io::Result<rustix::process::Pid> {
10633 if let Some(container) = child.child.container() {
10634 return container
10635 .group()
10636 .and_then(|pid| rustix::process::Pid::from_raw(pid as i32))
10637 .ok_or_else(|| io::Error::other("a carried cgroup has no signalable process group"));
10638 }
10639 let group = i32::try_from(child.pid)
10642 .ok()
10643 .filter(|pid| *pid > 1)
10644 .and_then(rustix::process::Pid::from_raw)
10645 .ok_or_else(|| {
10646 io::Error::new(
10647 io::ErrorKind::InvalidInput,
10648 "invalid supervised process group",
10649 )
10650 })?;
10651 if child.child.id() != Some(child.pid) {
10652 return Err(io::Error::new(
10653 io::ErrorKind::InvalidInput,
10654 "refusing process-group signal after supervised child was reaped",
10655 ));
10656 }
10657 Ok(group)
10658}
10659
10660enum RegistrationWaitOutcome {
10661 Registered,
10662 Exited(ExitReport),
10663 TimedOut,
10664}
10665
10666struct ReloadRegistrationFailure {
10667 exit_report: ExitReport,
10668 reason: String,
10669}
10670
10671#[derive(Debug, Clone, Copy, PartialEq, Eq)]
10672enum BusyGaugeObservation {
10673 Quiescent,
10674 Busy,
10675 Omitted,
10676}
10677
10678#[derive(Debug, Clone, PartialEq, Eq)]
10680enum GaugeVerdict {
10681 Zero,
10683 NonZero(u64),
10685 NotACount(String),
10689 Missing,
10691 ProbeFailed(&'static str),
10694}
10695
10696impl GaugeVerdict {
10697 fn reads_quiet(&self) -> bool {
10698 matches!(self, Self::Zero)
10699 }
10700}
10701
10702impl fmt::Display for GaugeVerdict {
10703 fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
10704 match self {
10705 Self::Zero => f.write_str("zero"),
10706 Self::NonZero(value) => write!(f, "non_zero({value})"),
10707 Self::NotACount(value) => write!(f, "not_a_count({value})"),
10708 Self::Missing => f.write_str("missing_from_health_body"),
10709 Self::ProbeFailed(kind) => write!(f, "probe_failed({kind})"),
10710 }
10711 }
10712}
10713
10714const GAUGE_VALUE_TEXT_LIMIT: usize = 32;
10717
10718fn gauge_verdicts(metrics: Option<&Value>, gauges: &[String]) -> Vec<GaugeVerdict> {
10720 let metrics = metrics.and_then(Value::as_object);
10721 gauges
10722 .iter()
10723 .map(
10724 |gauge| match metrics.and_then(|metrics| metrics.get(gauge)) {
10725 None => GaugeVerdict::Missing,
10726 Some(value) => match value.as_u64() {
10727 Some(0) => GaugeVerdict::Zero,
10728 Some(count) => GaugeVerdict::NonZero(count),
10729 None => {
10730 let mut text = value.to_string();
10731 if text.len() > GAUGE_VALUE_TEXT_LIMIT {
10732 let mut end = GAUGE_VALUE_TEXT_LIMIT;
10733 while !text.is_char_boundary(end) {
10734 end -= 1;
10735 }
10736 text.truncate(end);
10737 text.push_str("...");
10738 }
10739 GaugeVerdict::NotACount(text)
10740 }
10741 },
10742 },
10743 )
10744 .collect()
10745}
10746
10747fn busy_gauge_observation(verdicts: &[GaugeVerdict]) -> BusyGaugeObservation {
10750 if verdicts
10751 .iter()
10752 .any(|verdict| matches!(verdict, GaugeVerdict::Missing))
10753 {
10754 BusyGaugeObservation::Omitted
10755 } else if verdicts.iter().all(GaugeVerdict::reads_quiet) {
10756 BusyGaugeObservation::Quiescent
10757 } else {
10758 BusyGaugeObservation::Busy
10759 }
10760}
10761
10762#[derive(Debug, Clone, Default, PartialEq, Eq)]
10767struct GaugeTrace {
10768 latest: Option<(GaugeVerdict, Duration)>,
10769 last_value: Option<(GaugeVerdict, Duration)>,
10770}
10771
10772#[derive(Debug, Clone, Default, PartialEq, Eq)]
10778struct DrainGaugeTrace {
10779 probes: u32,
10781 gauges: Vec<(String, GaugeTrace)>,
10783}
10784
10785impl DrainGaugeTrace {
10786 fn new(gauges: &[String]) -> Self {
10787 Self {
10788 probes: 0,
10789 gauges: gauges
10790 .iter()
10791 .map(|gauge| (gauge.clone(), GaugeTrace::default()))
10792 .collect(),
10793 }
10794 }
10795
10796 fn record(&mut self, verdicts: &[GaugeVerdict], at: Duration) {
10798 self.probes = self.probes.saturating_add(1);
10799 for ((_, trace), verdict) in self.gauges.iter_mut().zip(verdicts) {
10800 trace.latest = Some((verdict.clone(), at));
10801 if matches!(
10802 verdict,
10803 GaugeVerdict::Zero | GaugeVerdict::NonZero(_) | GaugeVerdict::NotACount(_)
10804 ) {
10805 trace.last_value = Some((verdict.clone(), at));
10806 }
10807 }
10808 }
10809}
10810
10811impl fmt::Display for DrainGaugeTrace {
10816 fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
10817 if self.gauges.is_empty() {
10818 return f.write_str("none declared");
10819 }
10820 for (index, (name, trace)) in self.gauges.iter().enumerate() {
10821 if index > 0 {
10822 f.write_str(" | ")?;
10823 }
10824 write!(f, "{name}: ")?;
10825 match &trace.latest {
10826 Some((verdict, at)) => write!(f, "{verdict} @{}ms", at.as_millis())?,
10827 None => f.write_str("never probed")?,
10828 }
10829 match &trace.last_value {
10830 Some((value, at)) => write!(f, "; last value {value} @{}ms", at.as_millis())?,
10831 None => f.write_str("; no value ever read")?,
10832 }
10833 }
10834 Ok(())
10835 }
10836}
10837
10838pub(crate) fn declared_busy_gauges(
10839 registry: &Registry,
10840 module_id: &str,
10841) -> Result<Vec<String>, SuperviseError> {
10842 busy_gauges_of(
10843 registry
10844 .get_module(module_id)
10845 .map_err(SuperviseError::Registry)?,
10846 )
10847}
10848
10849fn declared_busy_gauges_for_connection(
10853 registry: &Registry,
10854 connection_id: ConnectionId,
10855) -> Result<Vec<String>, SuperviseError> {
10856 busy_gauges_of(
10857 registry
10858 .get_module_by_connection(connection_id)
10859 .map_err(SuperviseError::Registry)?,
10860 )
10861}
10862
10863fn busy_gauges_of(
10864 registration: Option<crate::registry::ModuleRegistration>,
10865) -> Result<Vec<String>, SuperviseError> {
10866 let Some(registration) = registration else {
10867 return Ok(Vec::new());
10868 };
10869 let Some(self_signals) = registration.manifest.self_signals else {
10870 return Ok(Vec::new());
10871 };
10872
10873 let mut gauges = Vec::new();
10874 for declaration in self_signals {
10875 if declaration.kind != SelfSignalKind::Busy {
10876 continue;
10877 }
10878 match declaration.anchored_to {
10879 SignalAnchor::HealthGauges { gauges: declared } if !declared.is_empty() => {
10880 gauges.extend(declared)
10881 }
10882 _ => {
10883 gauges.push(String::new());
10886 }
10887 }
10888 }
10889 Ok(gauges)
10890}
10891
10892#[allow(clippy::too_many_arguments)]
10901async fn wait_for_forwarding_quiescence(
10902 forwarding: &ForwardingTable,
10903 module_id: &str,
10904 runtime: &SupervisorRuntimeConfig,
10905 endpoint: crate::ModuleEndpointId,
10906 deadline: Instant,
10907 busy_gauges: &[String],
10908 scope: DrainScope,
10909 trace: &mut DrainGaugeTrace,
10910) -> Result<bool, SuperviseError> {
10911 let wait_started = Instant::now();
10912 let mut gauges_quiescent = busy_gauges.is_empty();
10913 let mut next_probe_at = wait_started;
10914 let mut omission_reported = false;
10915
10916 loop {
10917 let now = Instant::now();
10918 if !busy_gauges.is_empty() && now >= next_probe_at && now < deadline {
10919 let report = match scope {
10920 DrainScope::Active => probe_module_health(module_id, runtime, Some(deadline)).await,
10921 DrainScope::Endpoint(endpoint) => {
10922 probe_endpoint_health(endpoint, runtime, Some(deadline)).await
10923 }
10924 };
10925 let verdicts = match &report {
10926 Ok(report) => gauge_verdicts(report.metrics.as_ref(), busy_gauges),
10927 Err(err) => vec![GaugeVerdict::ProbeFailed(err.label()); busy_gauges.len()],
10928 };
10929 trace.record(&verdicts, wait_started.elapsed());
10930 gauges_quiescent = match report {
10931 Ok(_) => match busy_gauge_observation(&verdicts) {
10932 BusyGaugeObservation::Quiescent => true,
10933 BusyGaugeObservation::Busy => false,
10934 BusyGaugeObservation::Omitted => {
10935 if !omission_reported {
10938 forwarding
10939 .counters()
10940 .increment_drains_with_undeclared_gauge();
10941 let missing = busy_gauges
10942 .iter()
10943 .zip(&verdicts)
10944 .filter(|(_, verdict)| matches!(verdict, GaugeVerdict::Missing))
10945 .map(|(gauge, _)| gauge.as_str())
10946 .collect::<Vec<_>>()
10947 .join(",");
10948 warn!(
10949 module_id,
10950 missing_gauges = %missing,
10951 "drain health.check answer lacks a declared busy gauge; the drain treats the module as busy until it reports it"
10952 );
10953 omission_reported = true;
10954 }
10955 false
10956 }
10957 },
10958 Err(err) => {
10959 warn!(
10960 module_id,
10961 error = %err,
10962 "drain health.check did not produce declared busy gauges; treating module as busy"
10963 );
10964 false
10965 }
10966 };
10967 next_probe_at = Instant::now() + drain_busy_reprobe_interval();
10970 }
10971
10972 let in_flight = forwarding
10973 .endpoint_in_flight_count(endpoint)
10974 .map_err(SuperviseError::Forwarding)?;
10975 if in_flight == 0 && gauges_quiescent {
10976 return Ok(true);
10977 }
10978
10979 let now = Instant::now();
10980 if now >= deadline {
10981 return Ok(false);
10982 }
10983 let mut wait = deadline
10984 .saturating_duration_since(now)
10985 .min(REGISTRY_RELEASE_POLL);
10986 if !busy_gauges.is_empty() {
10987 wait = wait.min(next_probe_at.saturating_duration_since(now));
10988 }
10989 sleep(wait).await;
10990 }
10991}
10992
10993fn drain_busy_reprobe_interval() -> Duration {
10997 DRAIN_BUSY_REPROBE_INTERVAL.max(REGISTRY_RELEASE_POLL)
10998}
10999
11000fn warn_drain_timed_out(
11015 module_id: &str,
11016 drain_timeout: Duration,
11017 reason: RouteCloseReason,
11018 holdouts: &crate::forwarding::DrainHoldouts,
11019 gauges: &DrainGaugeTrace,
11020) {
11021 warn!(
11022 module_id = %module_id,
11023 waited = ?drain_timeout,
11024 ?reason,
11025 held_requests = holdouts.requests,
11026 held_routes = holdouts.routes,
11027 total_routes = holdouts.total_routes,
11028 top_connections = ?holdouts.top_connections,
11029 held = %holdouts
11032 .held
11033 .iter()
11034 .map(|(channel, corr)| format!("{channel}:{corr}"))
11035 .collect::<Vec<_>>()
11036 .join(","),
11037 gauge_probes = gauges.probes,
11038 busy_gauges = %gauges,
11039 "route drain timed out before request quiescence; forcing teardown"
11040 );
11041}
11042
11043fn drained_after_quiescence_wait(wait_result: &Result<bool, SuperviseError>) -> bool {
11051 match wait_result {
11052 Ok(drained) => *drained,
11053 Err(_) => false,
11054 }
11055}
11056
11057fn send_route_goodbyes(forwarding: &ForwardingTable, released_routes: Vec<GoodbyeTarget>) {
11058 for released in released_routes {
11059 let frame = match Frame::build_with_version(
11060 released.negotiated_ver,
11061 FrameType::Goodbye,
11062 control_flags(),
11063 released.channel,
11064 released.epoch,
11065 0,
11066 Vec::new(),
11067 ) {
11068 Ok(frame) => frame,
11069 Err(err) => {
11070 warn!(
11071 route_channel = released.channel,
11072 error = %err,
11073 "failed to build supervisor drain route GOODBYE frame"
11074 );
11075 continue;
11076 }
11077 };
11078 if !released.close_on_delivery_failure() {
11079 crate::forwarding::send_module_route_goodbye(
11080 &forwarding.counters(),
11081 &released.sink,
11082 frame,
11083 released.module_id.as_deref(),
11084 "supervisor drain",
11085 );
11086 continue;
11087 }
11088 if let Err(err) = released.sink.try_send(frame) {
11089 warn!(
11090 target_connection_id = released.connection_id.get(),
11091 route_channel = released.channel,
11092 error = %err,
11093 "supervisor drain route GOODBYE was not delivered to client; closing target connection"
11094 );
11095 let _ = forwarding.escalate_client_delivery_failure(
11096 released.connection_id,
11097 released.channel,
11098 released.epoch,
11099 CloseReason::new(
11100 "route_goodbye_delivery_failed",
11101 format!(
11102 "failed to enqueue supervisor drain route GOODBYE for channel {}: {err}",
11103 released.channel
11104 ),
11105 ),
11106 crate::forwarding::UndeliveredFrame {
11107 module_id: released.module_id.as_deref(),
11108 sink: &released.sink,
11109 },
11110 );
11111 }
11112 }
11113}
11114
11115async fn send_module_draining(
11116 module_id: &str,
11117 reason: RouteCloseReason,
11118 deadline_ms: u64,
11119 target: &ModuleDrainTarget,
11120 _wait_for_flush: bool,
11121) -> bool {
11122 let body = match serde_json::to_vec(&ModuleControlCommand::Draining {
11123 reason,
11124 deadline_ms,
11125 }) {
11126 Ok(body) => body,
11127 Err(err) => {
11128 warn!(
11129 module_id,
11130 error = %err,
11131 "failed to encode module draining command"
11132 );
11133 return false;
11134 }
11135 };
11136 let frame = match Frame::build_with_version(
11137 target.negotiated_ver,
11138 FrameType::Push,
11139 control_flags(),
11140 0,
11141 0,
11142 0,
11143 body,
11144 ) {
11145 Ok(frame) => frame,
11146 Err(err) => {
11147 warn!(
11148 module_id,
11149 error = %err,
11150 "failed to build module draining command frame"
11151 );
11152 return false;
11153 }
11154 };
11155 #[cfg(unix)]
11156 if _wait_for_flush {
11157 return target.sink.send_flushed(frame).await.is_ok();
11158 }
11159 if let Err(err) = target.sink.try_send(frame) {
11160 warn!(
11161 module_id,
11162 target_connection_id = target.endpoint.connection_id.get(),
11163 error = %err,
11164 "module draining command was not delivered to peer"
11165 );
11166 return false;
11167 }
11168 true
11169}
11170
11171fn module_goodbye_frame(module_id: &str, negotiated_ver: u8) -> Option<Frame> {
11173 match Frame::build_with_version(
11174 negotiated_ver,
11175 FrameType::Goodbye,
11176 control_flags(),
11177 0,
11178 0,
11179 0,
11180 Vec::new(),
11181 ) {
11182 Ok(frame) => Some(frame),
11183 Err(err) => {
11184 warn!(
11185 module_id,
11186 error = %err,
11187 "failed to build module GOODBYE frame"
11188 );
11189 None
11190 }
11191 }
11192}
11193
11194#[cfg(unix)]
11208async fn send_module_goodbyes_for_daemon_shutdown(
11209 forwarding: &Arc<ForwardingTable>,
11210 reason: &CloseReason,
11211 wait_for_flush: bool,
11212) {
11213 const GOODBYE_BUDGET: Duration = Duration::from_millis(500);
11214 let targets = match forwarding.module_connections() {
11215 Ok(targets) => targets,
11216 Err(err) => {
11217 warn!(error = %err, "could not list module connections for shutdown GOODBYE");
11218 return;
11219 }
11220 };
11221 let deadline = Instant::now() + GOODBYE_BUDGET;
11222 let mut sends = tokio::task::JoinSet::new();
11223 for target in targets {
11224 let Some(frame) = module_goodbye_frame(&target.module_id, target.negotiated_ver) else {
11225 continue;
11226 };
11227 if !wait_for_flush {
11228 if let Err(err) = target.sink.try_send(frame) {
11229 debug!(
11230 module_id = %target.module_id,
11231 error = %err,
11232 "shutdown module GOODBYE was not queued"
11233 );
11234 }
11235 continue;
11236 }
11237 let forwarding = Arc::clone(forwarding);
11238 let reason = reason.clone();
11239 sends.spawn(async move {
11240 match timeout_at(deadline, target.sink.send_flushed(frame)).await {
11241 Ok(Ok(())) => {}
11242 Ok(Err(err)) => debug!(
11243 module_id = %target.module_id,
11244 error = %err,
11245 "module connection closed before its shutdown GOODBYE was written"
11246 ),
11247 Err(_) => warn!(
11248 module_id = %target.module_id,
11249 budget = ?GOODBYE_BUDGET,
11250 "shutdown module GOODBYE was not written within its budget; closing anyway"
11251 ),
11252 }
11253 forwarding.request_connection_close(target.endpoint.connection_id, reason);
11254 });
11255 }
11256 while sends.join_next().await.is_some() {}
11258}
11259
11260fn send_module_goodbye(module_id: &str, forwarding: &ForwardingTable, target: &ModuleDrainTarget) {
11261 let Some(frame) = module_goodbye_frame(module_id, target.negotiated_ver) else {
11262 return;
11263 };
11264 if let Err(err) = target.sink.try_send(frame) {
11265 warn!(
11266 module_id,
11267 target_connection_id = target.endpoint.connection_id.get(),
11268 error = %err,
11269 "supervisor drain module GOODBYE was not delivered to peer; closing module connection"
11270 );
11271 forwarding.request_connection_close(
11272 target.endpoint.connection_id,
11273 CloseReason::new(
11274 "module_goodbye_delivery_failed",
11275 format!("failed to enqueue supervisor drain module GOODBYE for module '{module_id}': {err}"),
11276 ),
11277 );
11278 }
11279}
11280
11281#[derive(Clone, Copy)]
11282struct ForwardingDrainContext<'a> {
11283 spec: &'a ModuleSpec,
11284 runtime: &'a SupervisorRuntimeConfig,
11285 registry: &'a Registry,
11286 scope: DrainScope,
11287}
11288
11289#[derive(Debug, Clone, Copy, PartialEq, Eq)]
11291enum DrainScope {
11292 Active,
11295 Endpoint(crate::ModuleEndpointId),
11300}
11301
11302#[derive(Debug, Clone, Copy, PartialEq, Eq)]
11310enum StopNotice {
11311 SentOverConnection,
11314 NoConnection,
11318 NotSent,
11322}
11323
11324async fn begin_forwarding_drain(
11325 spec: &ModuleSpec,
11326 runtime: &SupervisorRuntimeConfig,
11327 registry: &Registry,
11328 snapshot: &SharedSnapshot,
11329 enabled: Option<bool>,
11330 reason: RouteCloseReason,
11331) -> Result<StopNotice, SuperviseError> {
11332 let Some(forwarding) = runtime.forwarding.as_ref() else {
11333 return Err(SuperviseError::ReloadUnavailable {
11334 module_id: spec.module_id.clone(),
11335 reason: "supervisor was not configured with a forwarding table".to_string(),
11336 });
11337 };
11338
11339 begin_forwarding_drain_with(
11340 forwarding,
11341 ForwardingDrainContext {
11342 spec,
11343 runtime,
11344 registry,
11345 scope: DrainScope::Active,
11346 },
11347 snapshot,
11348 enabled,
11349 reason,
11350 runtime.drain_timeout,
11351 )
11352 .await
11353}
11354
11355async fn begin_forwarding_drain_if_configured(
11356 spec: &ModuleSpec,
11357 runtime: &SupervisorRuntimeConfig,
11358 registry: &Registry,
11359 snapshot: &SharedSnapshot,
11360 enabled: Option<bool>,
11361 reason: RouteCloseReason,
11362) -> Result<StopNotice, SuperviseError> {
11363 begin_forwarding_drain_with_timeout(
11364 spec,
11365 runtime,
11366 registry,
11367 snapshot,
11368 enabled,
11369 reason,
11370 runtime.drain_timeout,
11371 )
11372 .await
11373}
11374
11375async fn begin_forwarding_drain_with_timeout(
11379 spec: &ModuleSpec,
11380 runtime: &SupervisorRuntimeConfig,
11381 registry: &Registry,
11382 snapshot: &SharedSnapshot,
11383 enabled: Option<bool>,
11384 reason: RouteCloseReason,
11385 drain_timeout: Duration,
11386) -> Result<StopNotice, SuperviseError> {
11387 let Some(forwarding) = runtime.forwarding.as_ref() else {
11388 return Ok(StopNotice::NotSent);
11389 };
11390
11391 begin_forwarding_drain_with(
11392 forwarding,
11393 ForwardingDrainContext {
11394 spec,
11395 runtime,
11396 registry,
11397 scope: DrainScope::Active,
11398 },
11399 snapshot,
11400 enabled,
11401 reason,
11402 drain_timeout,
11403 )
11404 .await
11405}
11406
11407async fn begin_forwarding_drain_with(
11408 forwarding: &ForwardingTable,
11409 context: ForwardingDrainContext<'_>,
11410 snapshot: &SharedSnapshot,
11411 enabled: Option<bool>,
11412 reason: RouteCloseReason,
11413 drain_timeout: Duration,
11414) -> Result<StopNotice, SuperviseError> {
11415 let ForwardingDrainContext {
11416 spec,
11417 runtime,
11418 registry,
11419 scope,
11420 } = context;
11421 debug_assert_ne!(reason, RouteCloseReason::Crash);
11422 let terminal = matches!(reason, RouteCloseReason::Disable);
11423 let drain_started_at = Instant::now();
11424 let drain_deadline = drain_started_at + drain_timeout;
11425 let deadline_ms =
11426 unix_ms_now().saturating_add(u64::try_from(drain_timeout.as_millis()).unwrap_or(u64::MAX));
11427 let busy_gauges = match scope {
11428 DrainScope::Active => declared_busy_gauges(registry, &spec.module_id)?,
11429 DrainScope::Endpoint(endpoint) => {
11430 declared_busy_gauges_for_connection(registry, endpoint.connection_id)?
11431 }
11432 };
11433
11434 let gate_started = Instant::now();
11437 let drain_target = match scope {
11438 DrainScope::Active => forwarding.begin_module_drain(&spec.module_id, reason),
11439 DrainScope::Endpoint(endpoint) => forwarding.begin_endpoint_drain(endpoint, reason),
11440 }
11441 .map_err(SuperviseError::Forwarding)?;
11442 info!(
11447 module_id = %spec.module_id,
11448 ?reason,
11449 gate_ms = u64::try_from(gate_started.elapsed().as_millis()).unwrap_or(u64::MAX),
11450 connected = drain_target.is_some(),
11451 "module drain began; route admission closed"
11452 );
11453 if scope == DrainScope::Active {
11454 update_snapshot(snapshot, Some(&spec.module_id), |state| {
11455 state.state = ModuleState::Draining;
11456 state.draining_to_replace =
11457 matches!(reason, RouteCloseReason::Restart | RouteCloseReason::Reload);
11458 if let Some(enabled) = enabled {
11459 state.enabled = enabled;
11460 }
11461 })?;
11462 }
11463
11464 let Some(target) = drain_target.as_ref() else {
11465 return Ok(StopNotice::NoConnection);
11469 };
11470 {
11471 send_module_draining(&spec.module_id, reason, deadline_ms, target, false).await;
11472 #[cfg(feature = "test-support")]
11473 forwarding.wait_before_route_closing_for_test().await;
11474 let routes = forwarding
11475 .endpoint_routes(target.endpoint)
11476 .map_err(SuperviseError::Forwarding)?;
11477 let routes_notified = routes.len();
11478 crate::control::send_route_control_pushes(
11479 forwarding,
11480 routes.clone(),
11481 ClientControlPush::RouteClosing {
11482 module_id: spec.module_id.clone(),
11483 channels: Vec::new(),
11484 reason,
11485 },
11486 );
11487 send_route_goodbyes(forwarding, target.abandoned_bindings.clone());
11488
11489 let mut gauge_trace = DrainGaugeTrace::new(&busy_gauges);
11495 let wait_result = wait_for_forwarding_quiescence(
11496 forwarding,
11497 &spec.module_id,
11498 runtime,
11499 target.endpoint,
11500 drain_deadline,
11501 &busy_gauges,
11502 scope,
11503 &mut gauge_trace,
11504 )
11505 .await;
11506 let drained = drained_after_quiescence_wait(&wait_result);
11507 if let Err(err) = &wait_result {
11508 error!(
11509 module_id = %spec.module_id,
11510 ?reason,
11511 error = %err,
11512 "forwarding quiescence wait failed after route.closing; forcing route.closed(drained: false) so the client is not left waiting on an unfulfilled promise"
11513 );
11514 } else if !drained {
11515 let holdouts = forwarding
11517 .endpoint_drain_holdouts(target.endpoint)
11518 .unwrap_or_default();
11519 warn_drain_timed_out(
11520 &spec.module_id,
11521 drain_timeout,
11522 reason,
11523 &holdouts,
11524 &gauge_trace,
11525 );
11526 }
11527 crate::control::send_route_control_pushes(
11528 forwarding,
11529 routes,
11530 ClientControlPush::RouteClosed {
11531 module_id: spec.module_id.clone(),
11532 channels: Vec::new(),
11533 reason,
11534 drained,
11535 abandoned: target.abandoned_bindings.len() as u32,
11536 excluded_subscriptions: target.excluded_subscriptions,
11537 terminal: Some(terminal),
11538 },
11539 );
11540 wait_result?;
11541
11542 let released_routes = match forwarding.release_module_endpoint_routes(target.endpoint) {
11548 Ok(routes) => routes,
11549 Err(err) => {
11550 warn!(
11551 module_id = %spec.module_id,
11552 ?reason,
11553 error = %err,
11554 "failed to release module endpoint routes after route.closed; module GOODBYE will still be sent"
11555 );
11556 send_module_goodbye(&spec.module_id, forwarding, target);
11557 return Err(SuperviseError::Forwarding(err));
11558 }
11559 };
11560 let route_goodbye_count = released_routes.len();
11561 send_route_goodbyes(forwarding, released_routes);
11562 send_module_goodbye(&spec.module_id, forwarding, target);
11563
11564 info!(
11570 module_id = %spec.module_id,
11571 ?reason,
11572 routes_notified,
11573 route_goodbyes = route_goodbye_count,
11574 abandoned_reservations = target.abandoned_bindings.len(),
11575 excluded_subscriptions = target.excluded_subscriptions,
11576 drained,
11577 "module drain complete; consumers notified via route.closing/route.closed pushes and per-route GOODBYE frames"
11578 );
11579 }
11580
11581 Ok(StopNotice::SentOverConnection)
11582}
11583
11584async fn wait_for_registration_after_reload(
11587 registry: &Registry,
11588 module_id: &str,
11589 snapshot: &SharedSnapshot,
11590 child: &mut SupervisedChild,
11591 wait: Duration,
11592) -> Result<RegistrationWaitOutcome, SuperviseError> {
11593 wait_for_slot_registration(
11594 registry,
11595 crate::registry::RegistrationSlot::Active(module_id),
11596 module_id,
11597 snapshot,
11598 child,
11599 wait,
11600 )
11601 .await
11602}
11603
11604async fn wait_for_slot_registration(
11612 registry: &Registry,
11613 slot: crate::registry::RegistrationSlot<'_>,
11614 module_id: &str,
11615 snapshot: &SharedSnapshot,
11616 child: &mut SupervisedChild,
11617 wait: Duration,
11618) -> Result<RegistrationWaitOutcome, SuperviseError> {
11619 let deadline = Instant::now() + wait;
11620 loop {
11621 if registry
11622 .registration(slot)
11623 .map_err(SuperviseError::Registry)?
11624 .is_some()
11625 {
11626 return Ok(RegistrationWaitOutcome::Registered);
11627 }
11628
11629 let now = Instant::now();
11630 if now >= deadline {
11631 return Ok(RegistrationWaitOutcome::TimedOut);
11632 }
11633 let remaining = deadline.saturating_duration_since(now);
11634 let poll = remaining.min(REGISTRY_RELEASE_POLL);
11635
11636 tokio::select! {
11637 wait_result = child.wait() => {
11638 let status = wait_result.map_err(|source| SuperviseError::Wait {
11639 module_id: module_id.to_string(),
11640 source,
11641 })?;
11642 return Ok(RegistrationWaitOutcome::Exited(classify_reaped_child_exit(
11643 snapshot,
11644 child,
11645 &status,
11646 )));
11647 }
11648 _ = sleep(poll) => {}
11649 }
11650 }
11651}
11652
11653fn registration_failure_exit_report(mut exit_report: ExitReport) -> ExitReport {
11654 if exit_report.kind != ExitKind::DeliberateSeverance {
11658 exit_report.kind = ExitKind::Crash;
11659 }
11660 exit_report
11661}
11662
11663async fn handle_reload_child_registration_failure(
11664 spec: &ModuleSpec,
11665 runtime: &SupervisorRuntimeConfig,
11666 registry: &Registry,
11667 process_liveness: &SupervisorProcessLiveness,
11668 snapshot: &SharedSnapshot,
11669 _child: &mut Option<SupervisedChild>,
11670 failure: ReloadRegistrationFailure,
11671) -> Result<(), SuperviseError> {
11672 let ReloadRegistrationFailure {
11673 exit_report,
11674 reason,
11675 } = failure;
11676 match on_child_exit(
11677 spec,
11678 runtime.restart_policy,
11679 registry,
11680 snapshot,
11681 &runtime.terminal_ring,
11682 &runtime.spawn_events,
11683 &runtime.child_roster,
11684 exit_report,
11685 runtime.recovery_is_daemon,
11686 )
11687 .await
11688 {
11689 NextAction::Stop {
11690 registration_released,
11691 } => {
11692 if registration_released {
11693 process_liveness.untrack_if_current(&spec.module_id, snapshot);
11694 }
11695 }
11696 NextAction::Restart { schedule } => {
11697 let delay = schedule.map_or(runtime.restart_policy.delay_for_restart(0), |schedule| {
11698 schedule.delay
11699 });
11700 if let Some(schedule) = schedule {
11701 log_crash_respawn(&spec.module_id, schedule);
11702 }
11703 schedule_respawn(
11704 runtime,
11705 snapshot,
11706 &spec.module_id,
11707 delay,
11708 RespawnKind::Spawn,
11709 true,
11710 )?;
11711 }
11712 }
11713 Err(SuperviseError::ReloadFailed {
11714 module_id: spec.module_id.clone(),
11715 reason,
11716 })
11717}
11718
11719async fn handle_reload_spawn_failure(
11720 spec: &ModuleSpec,
11721 runtime: &SupervisorRuntimeConfig,
11722 process_liveness: &SupervisorProcessLiveness,
11723 snapshot: &SharedSnapshot,
11724 _child: &mut Option<SupervisedChild>,
11725 reason: String,
11726) -> Result<(), SuperviseError> {
11727 let now = Instant::now();
11728 let mut schedule = None;
11729 update_snapshot(snapshot, Some(&spec.module_id), |state| {
11730 clear_current_process_facts(state);
11731 if state.enabled {
11732 schedule = state.next_crash_restart(&runtime.restart_policy, now);
11733 state.state = if schedule.is_some() {
11734 ModuleState::Restarting
11735 } else {
11736 if runtime
11737 .failure_context(runtime.recovery_is_daemon)
11738 .may_retry()
11739 {
11740 state.failed_retry.schedule(&spec.module_id);
11741 }
11742 ModuleState::Failed
11743 };
11744 } else {
11745 state.state = ModuleState::Disabled;
11746 }
11747 })?;
11748 if let Some(schedule) = schedule {
11749 schedule_respawn(
11750 runtime,
11751 snapshot,
11752 &spec.module_id,
11753 schedule.delay,
11754 RespawnKind::Spawn,
11755 true,
11756 )?;
11757 } else {
11758 process_liveness.untrack_if_current(&spec.module_id, snapshot);
11759 }
11760 Err(SuperviseError::ReloadFailed {
11761 module_id: spec.module_id.clone(),
11762 reason,
11763 })
11764}
11765
11766fn control_flags() -> Flags {
11767 Flags::new(false, Priority::Passive, false)
11768}
11769
11770#[allow(clippy::too_many_arguments)]
11771async fn drain_optional_child(
11772 module_id: &str,
11773 protocol: ModuleProtocol,
11774 stop_notice: StopNotice,
11775 registry: &Registry,
11776 forwarding: Option<&ForwardingTable>,
11777 snapshot: &SharedSnapshot,
11778 terminal_ring: &Arc<Mutex<TerminalRing>>,
11779 spawn_events: &SpawnEventFeed,
11780 child: &mut Option<SupervisedChild>,
11781 drain_timeout: Duration,
11782 final_state: ModuleState,
11783 enabled: Option<bool>,
11784 failure: &FailureContext,
11785) -> Result<(), SuperviseError> {
11786 if let Some(child) = child.take() {
11787 drain_child_to_state(
11788 module_id,
11789 protocol,
11790 stop_notice,
11791 registry,
11792 forwarding,
11793 snapshot,
11794 terminal_ring,
11795 spawn_events,
11796 child,
11797 drain_timeout,
11798 final_state,
11799 enabled,
11800 failure,
11801 )
11802 .await
11803 } else {
11804 update_snapshot(snapshot, Some(module_id), |state| {
11805 state.state = final_state;
11806 if final_state == ModuleState::Failed {
11807 state.failed_retry.failed_at = Some(Instant::now());
11808 }
11809 if let Some(enabled) = enabled {
11810 state.enabled = enabled;
11811 }
11812 clear_current_process_facts(state);
11813 })?;
11814 release_dead_registration(registry, forwarding, snapshot, module_id, failure).await
11815 }
11816}
11817
11818#[allow(clippy::too_many_arguments)]
11819async fn drain_child_to_state(
11820 module_id: &str,
11821 _protocol: ModuleProtocol,
11822 stop_notice: StopNotice,
11823 registry: &Registry,
11824 forwarding: Option<&ForwardingTable>,
11825 snapshot: &SharedSnapshot,
11826 terminal_ring: &Arc<Mutex<TerminalRing>>,
11827 spawn_events: &SpawnEventFeed,
11828 mut child: SupervisedChild,
11829 drain_timeout: Duration,
11830 final_state: ModuleState,
11831 enabled: Option<bool>,
11832 failure: &FailureContext,
11833) -> Result<(), SuperviseError> {
11834 let protocol = child.protocol;
11835 update_snapshot(snapshot, Some(module_id), |state| {
11836 state.state = ModuleState::Draining;
11837 state.draining_to_replace = final_state == ModuleState::Restarting;
11838 if let Some(enabled) = enabled {
11839 state.enabled = enabled;
11840 }
11841 })?;
11842
11843 if stop_notice != StopNotice::SentOverConnection {
11854 if protocol == ModuleProtocol::Subc && stop_notice == StopNotice::NoConnection {
11855 info!(
11856 module_id,
11857 pid = child.pid,
11858 budget_ms = u64::try_from(drain_timeout.as_millis()).unwrap_or(u64::MAX),
11859 "module has no connection yet; requesting stop by signal"
11860 );
11861 }
11862 request_graceful_stop(module_id, &child);
11863 }
11864
11865 let exit_report = match timeout(drain_timeout, child.wait()).await {
11866 Ok(Ok(status)) => classify_reaped_child_exit(snapshot, &child, &status),
11867 Ok(Err(source)) => {
11868 fail_after_wait_error(
11869 module_id,
11870 snapshot,
11871 terminal_ring,
11872 spawn_events,
11873 format!("failed to wait during drain: {source}"),
11874 failure,
11875 );
11876 return Err(SuperviseError::Wait {
11877 module_id: module_id.to_string(),
11878 source,
11879 });
11880 }
11881 Err(_) => {
11882 warn!(
11895 module_id,
11896 pid = child.pid,
11897 budget_ms = u64::try_from(drain_timeout.as_millis()).unwrap_or(u64::MAX),
11898 reason = ?final_state,
11899 ?stop_notice,
11900 "drain budget expired before the module exited; killing it"
11901 );
11902 let kill =
11903 if final_state == ModuleState::Restarting && child.child.container().is_some() {
11904 child.kill_leader()
11905 } else {
11906 child.start_kill()
11907 };
11908 kill.map_err(|source| {
11909 fail_after_wait_error(
11910 module_id,
11911 snapshot,
11912 terminal_ring,
11913 spawn_events,
11914 format!("failed to kill during drain: {source}"),
11915 failure,
11916 );
11917 SuperviseError::Kill {
11918 module_id: module_id.to_string(),
11919 source,
11920 }
11921 })?;
11922 let status = child.wait().await.map_err(|source| {
11923 fail_after_wait_error(
11924 module_id,
11925 snapshot,
11926 terminal_ring,
11927 spawn_events,
11928 format!("failed to wait after drain kill: {source}"),
11929 failure,
11930 );
11931 SuperviseError::Wait {
11932 module_id: module_id.to_string(),
11933 source,
11934 }
11935 })?;
11936 classify_reaped_child_exit(snapshot, &child, &status)
11937 }
11938 };
11939
11940 let detail = lock_snapshot(snapshot)?.drain_disposition_detail.take();
11941 publish_after_terminal_record(
11942 module_id,
11943 || {
11944 record_terminal_with_detail(
11945 module_id,
11946 terminal_ring,
11947 spawn_events,
11948 &exit_report,
11949 terminal_disposition(final_state),
11950 detail,
11951 )
11952 },
11953 || {
11954 update_snapshot(snapshot, Some(module_id), |state| {
11955 state.state = final_state;
11956 if final_state == ModuleState::Failed {
11957 state.failed_retry.failed_at = Some(Instant::now());
11958 }
11959 if let Some(enabled) = enabled {
11960 state.enabled = enabled;
11961 }
11962 clear_current_process_facts(state);
11963 state.last_exit = Some(exit_report.clone());
11964 if exit_report.kind == ExitKind::DeliberateSeverance {
11965 state.lifetime_restarts += 1;
11966 }
11967 })
11968 },
11969 )?;
11970 child.drain_stderr(module_id).await;
11971
11972 let released =
11973 release_dead_registration(registry, forwarding, snapshot, module_id, failure).await;
11974 child
11975 .finish_exit(
11976 released.is_ok()
11977 && final_state == ModuleState::Restarting
11978 && child.spawn_failure.is_none(),
11979 )
11980 .await
11981 .map_err(|source| SuperviseError::Wait {
11982 module_id: module_id.to_owned(),
11983 source,
11984 })?;
11985 released
11986}
11987
11988#[cfg(unix)]
12008fn request_graceful_stop(module_id: &str, child: &SupervisedChild) {
12009 let Some(pid) = child
12010 .id()
12011 .and_then(|pid| i32::try_from(pid).ok())
12012 .and_then(rustix::process::Pid::from_raw)
12013 else {
12014 debug!(
12015 module_id,
12016 "no pid to signal for teardown; falling through to the drain wait"
12017 );
12018 return;
12019 };
12020 match rustix::process::kill_process(pid, rustix::process::Signal::TERM) {
12021 Ok(()) => debug!(
12022 module_id,
12023 "sent SIGTERM to a module nothing else asked to stop"
12024 ),
12025 Err(err) => debug!(
12026 module_id,
12027 error = %err,
12028 "SIGTERM to module failed; the drain wait and kill still apply"
12029 ),
12030 }
12031}
12032
12033#[cfg(not(unix))]
12041fn request_graceful_stop(module_id: &str, _child: &SupervisedChild) {
12042 debug!(
12043 module_id,
12044 "no graceful stop signal exists on this platform; teardown of a module nothing asked to stop waits, then kills"
12045 );
12046}
12047
12048fn terminal_disposition(final_state: ModuleState) -> TerminalDisposition {
12049 match final_state {
12050 ModuleState::Stopped => TerminalDisposition::Stopped,
12051 ModuleState::Disabled => TerminalDisposition::Disabled,
12052 ModuleState::Restarting => TerminalDisposition::Restarting,
12053 ModuleState::Failed => TerminalDisposition::Failed,
12054 ModuleState::Starting
12055 | ModuleState::Running
12056 | ModuleState::Unresponsive
12057 | ModuleState::Draining => {
12058 unreachable!("terminal exits only finish in terminal or restarting states")
12059 }
12060 }
12061}
12062
12063async fn release_dead_registration(
12071 registry: &Registry,
12072 forwarding: Option<&ForwardingTable>,
12073 snapshot: &SharedSnapshot,
12074 module_id: &str,
12075 failure: &FailureContext,
12076) -> Result<(), SuperviseError> {
12077 let result = async {
12078 let registration = registry
12079 .get_module(module_id)
12080 .map_err(SuperviseError::Registry)?;
12081 match wait_for_registration_release(registry, module_id, REGISTRY_RELEASE_TIMEOUT).await {
12082 Ok(()) => return Ok(()),
12083 Err(SuperviseError::RegistrationStillActive { .. }) => {}
12084 Err(err) => return Err(err),
12085 }
12086 let pid = lock_snapshot(snapshot)?.reaped_pid;
12087 if let (Some(registration), Some(forwarding), Some(pid)) = (registration, forwarding, pid) {
12088 warn!(
12089 module_id,
12090 pid,
12091 connection_id = registration.connection_id.get(),
12092 "reaped module registration outlived release grace; closing dead connection"
12093 );
12094 forwarding.request_connection_close(
12095 registration.connection_id,
12096 CloseReason::new(
12097 "supervised_process_reaped",
12098 format!("module '{module_id}' pid {pid} exited"),
12099 ),
12100 );
12101 wait_for_slot_registration_release(
12102 registry,
12103 crate::registry::RegistrationSlot::Connection(registration.connection_id),
12104 REGISTRY_RELEASE_TIMEOUT,
12105 )
12106 .await?;
12107 }
12108 wait_for_registration_release(registry, module_id, Duration::ZERO).await
12109 }
12110 .await;
12111 if let Err(err) = &result {
12112 fail_snapshot(snapshot, Some(module_id), None, failure);
12113 error!(module_id, error = %err, "registration release failed after child exit; module is failed and start can retry");
12114 }
12115 result
12116}
12117
12118async fn wait_for_registration_release(
12121 registry: &Registry,
12122 module_id: &str,
12123 wait: Duration,
12124) -> Result<(), SuperviseError> {
12125 wait_for_slot_registration_release(
12126 registry,
12127 crate::registry::RegistrationSlot::Active(module_id),
12128 wait,
12129 )
12130 .await
12131}
12132
12133async fn wait_for_slot_registration_release(
12141 registry: &Registry,
12142 slot: crate::registry::RegistrationSlot<'_>,
12143 wait: Duration,
12144) -> Result<(), SuperviseError> {
12145 let deadline = Instant::now() + wait;
12146 let mut release_events = registration_release_events().subscribe();
12147 let still_active = |registration: &crate::registry::ModuleRegistration| {
12148 SuperviseError::RegistrationStillActive {
12149 module_id: registration.manifest.module_id.clone(),
12150 waited: wait,
12151 }
12152 };
12153 loop {
12154 let _observed_generation = *release_events.borrow_and_update();
12155 let Some(registration) = registry
12156 .registration(slot)
12157 .map_err(SuperviseError::Registry)?
12158 else {
12159 return Ok(());
12160 };
12161
12162 let now = Instant::now();
12163 if now >= deadline {
12164 return Err(still_active(®istration));
12165 }
12166
12167 let remaining = deadline.saturating_duration_since(now);
12168 match timeout(remaining, release_events.changed()).await {
12169 Ok(Ok(())) | Ok(Err(_)) => {}
12170 Err(_) => return Err(still_active(®istration)),
12171 }
12172 }
12173}
12174
12175#[cfg(test)]
12176mod slot_registration_wait_tests {
12177 use super::*;
12178 use crate::registry::{ConnectionId, RegistrationSlot};
12179 use subc_protocol::manifest::ModuleManifest;
12180
12181 #[tokio::test]
12182 async fn enable_release_failure_is_failed_and_a_second_enable_retries() {
12183 let registry = Arc::new(Registry::default());
12184 let supervisor = Supervisor::new_for_test(Arc::clone(®istry), RestartPolicy::default());
12185 let runtime = supervisor.runtime_config();
12186 let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::disabled()));
12187 let spec = ModuleSpec {
12188 module_id: "enable-stale-registration".to_string(),
12189 program: PathBuf::from("/missing/enable-retry-test"),
12190 args: Vec::new(),
12191 env: Vec::new(),
12192 reserved: false,
12193 reserved_prefixes: Vec::new(),
12194 protocol: ModuleProtocol::Subc,
12195 overlap: Default::default(),
12196 carry_containment: false,
12197 };
12198 let connection = ConnectionId::new(90);
12199 registry
12200 .register_with_control_ops(
12201 ModuleManifest::builder(&spec.module_id, "0.1.0").build(),
12202 1,
12203 connection,
12204 Vec::new(),
12205 )
12206 .unwrap();
12207 let mut child = None;
12208 let err = set_child_enabled(
12209 &spec,
12210 &runtime,
12211 ®istry,
12212 &supervisor.process_liveness,
12213 &snapshot,
12214 &mut child,
12215 true,
12216 &runtime.failure_context(false),
12217 )
12218 .await
12219 .unwrap_err();
12220 assert!(matches!(
12221 err,
12222 SuperviseError::RegistrationStillActive { .. }
12223 ));
12224 assert_eq!(lock_snapshot(&snapshot).unwrap().state, ModuleState::Failed);
12225 assert!(child.is_none());
12226 registry.deregister_connection(connection).unwrap();
12227 let err = set_child_enabled(
12228 &spec,
12229 &runtime,
12230 ®istry,
12231 &supervisor.process_liveness,
12232 &snapshot,
12233 &mut child,
12234 true,
12235 &runtime.failure_context(false),
12236 )
12237 .await
12238 .unwrap_err();
12239 assert!(
12240 matches!(err, SuperviseError::Spawn { .. }),
12241 "second enable must attempt a spawn: {err}"
12242 );
12243 assert_eq!(lock_snapshot(&snapshot).unwrap().state, ModuleState::Failed);
12244 }
12245
12246 const INCUMBENT: u64 = 1;
12247 const CANDIDATE: u64 = 2;
12248
12249 fn swapped_registry() -> Arc<Registry> {
12250 let registry = Arc::new(Registry::default());
12251 let manifest = ModuleManifest::builder("m", "0.1.0").build();
12252 registry
12253 .register_with_control_ops(
12254 manifest.clone(),
12255 1,
12256 ConnectionId::new(INCUMBENT),
12257 Vec::new(),
12258 )
12259 .unwrap();
12260 registry
12261 .register_candidate_with_control_ops(
12262 manifest,
12263 1,
12264 ConnectionId::new(CANDIDATE),
12265 Vec::new(),
12266 )
12267 .unwrap();
12268 registry
12269 }
12270
12271 #[tokio::test]
12275 async fn incumbent_release_is_awaited_by_connection_not_by_module_id() {
12276 let registry = swapped_registry();
12277 registry.promote_candidate("m").unwrap().unwrap();
12278
12279 assert!(matches!(
12280 wait_for_registration_release(®istry, "m", Duration::from_millis(50)).await,
12281 Err(SuperviseError::RegistrationStillActive { .. })
12282 ));
12283
12284 assert!(matches!(
12286 wait_for_slot_registration_release(
12287 ®istry,
12288 RegistrationSlot::Connection(ConnectionId::new(INCUMBENT)),
12289 Duration::from_millis(50),
12290 )
12291 .await,
12292 Err(SuperviseError::RegistrationStillActive { .. })
12293 ));
12294
12295 let releaser = Arc::clone(®istry);
12296 let release = tokio::spawn(async move {
12297 sleep(Duration::from_millis(20)).await;
12298 releaser
12299 .deregister_connection(ConnectionId::new(INCUMBENT))
12300 .unwrap();
12301 notify_registration_release();
12302 });
12303 wait_for_slot_registration_release(
12304 ®istry,
12305 RegistrationSlot::Connection(ConnectionId::new(INCUMBENT)),
12306 Duration::from_secs(5),
12307 )
12308 .await
12309 .expect("the incumbent's own registration is released");
12310 release.await.unwrap();
12311 assert!(registry.get_module("m").unwrap().is_some());
12312 }
12313
12314 #[tokio::test]
12317 async fn candidate_slot_wait_ignores_the_incumbents_registration() {
12318 let registry = swapped_registry();
12319 assert!(matches!(
12320 wait_for_slot_registration_release(
12321 ®istry,
12322 RegistrationSlot::Candidate("m"),
12323 Duration::from_millis(50),
12324 )
12325 .await,
12326 Err(SuperviseError::RegistrationStillActive { .. })
12327 ));
12328 registry
12329 .deregister_connection(ConnectionId::new(CANDIDATE))
12330 .unwrap();
12331 wait_for_slot_registration_release(
12332 ®istry,
12333 RegistrationSlot::Candidate("m"),
12334 Duration::from_millis(50),
12335 )
12336 .await
12337 .expect("a candidate slot with no candidate is released");
12338 assert!(registry
12339 .registration(RegistrationSlot::Active("m"))
12340 .unwrap()
12341 .is_some());
12342 }
12343}
12344
12345fn classify_exit(status: &ExitStatus) -> ExitReport {
12346 ExitReport {
12347 kind: if status.success() {
12348 ExitKind::Clean
12349 } else {
12350 ExitKind::Crash
12351 },
12352 code: status.code(),
12353 signal: exit_signal(status),
12354 at_ms: unix_ms_now(),
12355 }
12356}
12357
12358fn wait_error_exit_report() -> ExitReport {
12364 ExitReport {
12365 kind: ExitKind::Crash,
12366 code: None,
12367 signal: None,
12368 at_ms: unix_ms_now(),
12369 }
12370}
12371
12372#[cfg(unix)]
12373fn exit_signal(status: &ExitStatus) -> Option<i32> {
12374 use std::os::unix::process::ExitStatusExt;
12375
12376 status.signal()
12377}
12378
12379#[cfg(not(unix))]
12380fn exit_signal(_status: &ExitStatus) -> Option<i32> {
12381 None
12382}
12383
12384fn reset_restart_count(snapshot: &SharedSnapshot, module_id: &str) -> Result<(), SuperviseError> {
12390 update_snapshot(snapshot, Some(module_id), |state| {
12391 state.clear_crash_restarts();
12392 })
12393}
12394
12395fn set_running(
12396 snapshot: &SharedSnapshot,
12397 child: &SupervisedChild,
12398 module_id: &str,
12399 spawn_events: &SpawnEventFeed,
12400) -> Result<(), SuperviseError> {
12401 let mut state = snapshot.lock().map_err(|_| SuperviseError::StatePoisoned {
12402 module_id: Some(module_id.to_string()),
12403 })?;
12404 state.spawn_generation = spawn_events.emit_spawned(module_id, child.pid, child.spawned_at_ms);
12405 if std::mem::take(&mut state.coalesced_restart_pending) {
12406 let generation = state.spawn_generation;
12407 info!(module_id, "restart already satisfied by generation {generation}; coalesced pending request completed");
12408 }
12409 state.drain_disposition_detail = None;
12410 state.spawn_failure = None;
12411 state.waiting_for_previous_pid = None;
12412 state.waiting_for_previous_detail = None;
12413 state.in_alternate_slot = false;
12416 state.configuration_updated_since_spawn = false;
12417 state.spawned_protocol = Some(child.protocol);
12418 state.state = ModuleState::Running;
12419 state.enabled = true;
12420 state.process_alive = true;
12421 state.pid = child.id();
12422 state.subtree_target = child.subtree_target.clone();
12423 state.health.status = SupervisorHealthStatus::Unknown;
12426 state.health.consecutive_failures = 0;
12427 state.health.detail = None;
12428 state.health.metrics = None;
12429 state.health.last_probe_ms = None;
12430 state.failed_retry.healthy_since = None;
12431 state.failed_retry.failed_at = None;
12432 state.failed_retry.running_since = Some(child.spawned_at);
12433 #[cfg(windows)]
12434 {
12435 state.pipe_consumed = child
12436 .nonce_delivery
12437 .as_ref()
12438 .map(|delivery| delivery.consumed());
12439 }
12440 #[cfg(target_os = "macos")]
12441 {
12442 state.report_ready = Some(Arc::clone(&child.report_ready));
12443 }
12444 state.spawned_at_ms = Some(child.spawned_at_ms);
12445 state.exec_outcome = Some(Arc::clone(&child.exec.0));
12446 state.spawned_from = Some(child.spawned_from.clone());
12447 state.launch_spec_digest = Some(child.launch_spec_digest.clone());
12448 state.spawned_file_identity = child.spawned_file_identity.clone();
12449 state.process_start_time = child.process_start_time;
12450 Ok(())
12451}
12452
12453fn clear_current_process_facts(state: &mut SupervisorSnapshot) {
12454 state.subtree_target = crate::subtree_resources::Target::Unavailable("not_running");
12455 state.failed_retry.healthy_since = None;
12456 state.failed_retry.running_since = None;
12457 state.waiting_for_previous_pid = None;
12458 state.waiting_for_previous_detail = None;
12459 state.process_alive = false;
12460 state.spawned_protocol = None;
12461 state.pid = None;
12462 #[cfg(windows)]
12463 {
12464 state.pipe_consumed = None;
12465 }
12466 #[cfg(target_os = "macos")]
12467 {
12468 state.report_ready = None;
12469 }
12470 state.spawned_at_ms = None;
12471 state.exec_outcome = None;
12472 state.spawned_from = None;
12473 state.launch_spec_digest = None;
12474 state.spawned_file_identity = None;
12475 state.process_start_time = None;
12476 state.deliberate_severance = None;
12477}
12478
12479#[cfg(test)]
12480fn record_deliberate_severance(
12481 snapshot: &SharedSnapshot,
12482 identity: ProcessIdentity,
12483) -> Result<(), SuperviseError> {
12484 update_snapshot(snapshot, None, |state| {
12485 state.deliberate_severance = Some(identity);
12486 })
12487}
12488
12489fn apply_deliberate_severance_marker(
12490 snapshot: &SharedSnapshot,
12491 exited_identity: Option<ProcessIdentity>,
12492 mut exit_report: ExitReport,
12493) -> ExitReport {
12494 let marker = lock_snapshot(snapshot)
12495 .ok()
12496 .and_then(|mut state| state.deliberate_severance.take());
12497 if marker.is_some() && marker == exited_identity {
12498 exit_report.kind = ExitKind::DeliberateSeverance;
12499 }
12500 exit_report
12501}
12502
12503fn classify_reaped_child_exit(
12504 snapshot: &SharedSnapshot,
12505 child: &SupervisedChild,
12506 status: &ExitStatus,
12507) -> ExitReport {
12508 let _ = update_snapshot(snapshot, None, |state| {
12509 state.reaped_pid = Some(child.pid);
12510 state.spawn_failure = child.spawn_failure.clone();
12511 });
12512 if child.spawn_failure.as_deref() == Some("registration_timeout") {
12513 registration_failure_exit_report(classify_exit(status))
12514 } else {
12515 apply_deliberate_severance_marker(snapshot, child.process_identity(), classify_exit(status))
12516 }
12517}
12518
12519fn fail_snapshot(
12520 snapshot: &SharedSnapshot,
12521 module_id: Option<&str>,
12522 last_exit: Option<ExitReport>,
12523 failure: &FailureContext,
12524) {
12525 if let Err(err) = update_snapshot(snapshot, module_id, |state| {
12526 state.state = ModuleState::Failed;
12527 clear_current_process_facts(state);
12528 if failure.may_retry() && state.enabled {
12529 state.failed_retry.schedule(module_id.unwrap_or("unknown"));
12530 } else if failure.shutdown.is_set() {
12531 state.failed_retry.cancel();
12532 }
12533 if let Some(last_exit) = last_exit {
12534 state.last_exit = Some(last_exit);
12535 }
12536 }) {
12537 error!(error = %err, "failed to mark supervisor state failed");
12538 }
12539}
12540
12541fn update_snapshot(
12542 snapshot: &SharedSnapshot,
12543 module_id: Option<&str>,
12544 update: impl FnOnce(&mut SupervisorSnapshot),
12545) -> Result<(), SuperviseError> {
12546 let mut state = snapshot.lock().map_err(|_| SuperviseError::StatePoisoned {
12547 module_id: module_id.map(ToOwned::to_owned),
12548 })?;
12549 update(&mut state);
12550 Ok(())
12551}
12552
12553const SLOW_SNAPSHOT_LOCK_THRESHOLD: Duration = Duration::from_millis(250);
12554
12555fn lock_snapshot_for_control<'a>(
12556 snapshot: &'a SharedSnapshot,
12557 module_id: &str,
12558 caller: &'static str,
12559) -> Result<std::sync::MutexGuard<'a, SupervisorSnapshot>, SuperviseError> {
12560 let started_at = Instant::now();
12561 let guard = lock_snapshot(snapshot)?;
12562 let waited = started_at.elapsed();
12563 if waited >= SLOW_SNAPSHOT_LOCK_THRESHOLD {
12564 warn!(
12565 module_id = %module_id,
12566 waited_ms = waited.as_millis() as u64,
12567 caller = %caller,
12568 "slow snapshot lock"
12569 );
12570 }
12571 Ok(guard)
12572}
12573
12574fn lock_snapshot(
12575 snapshot: &SharedSnapshot,
12576) -> Result<std::sync::MutexGuard<'_, SupervisorSnapshot>, SuperviseError> {
12577 snapshot
12578 .lock()
12579 .map_err(|_| SuperviseError::StatePoisoned { module_id: None })
12580}
12581
12582#[cfg(test)]
12583mod terminal_history_tests {
12584 use std::{
12585 path::PathBuf,
12586 sync::Arc,
12587 time::{Duration, Instant},
12588 };
12589
12590 use tokio::{sync::mpsc, time::sleep};
12591
12592 use super::{
12593 apply_deliberate_severance_marker, daemon_will_restart, drain_child_to_state,
12594 drained_after_quiescence_wait, handle_reload_spawn_failure, health_restart_child,
12595 lock_snapshot, on_child_exit, record_deliberate_severance, record_wait_error_terminal,
12596 reset_restart_count, spawn_and_mark_running, update_snapshot, wait_error_exit_report,
12597 ExitKind, ExitReport, ModuleProtocol, ModuleSpec, ModuleState, NextAction, ProcessIdentity,
12598 RestartPolicy, SpawnEventKind, StopNotice, SuperviseError, SupervisedConfiguration,
12599 SupervisedModule, SupervisedModuleInner, Supervisor, SupervisorHandle,
12600 SupervisorHealthStatus, SupervisorSnapshot,
12601 };
12602 use super::Instant as ClockInstant;
12607 use crate::{
12608 registry::Registry,
12609 terminal_ring::{TerminalRing, TerminalRingConfig},
12610 };
12611 use std::sync::Mutex;
12612 use subc_control::TerminalDisposition;
12613
12614 pub(super) fn fake_aft_stub_path() -> PathBuf {
12619 let mut path = std::env::current_exe().expect("current_exe available in tests");
12620 path.pop();
12621 path.pop();
12622 path.push(if cfg!(windows) {
12623 "fake-aft-stub.exe"
12624 } else {
12625 "fake-aft-stub"
12626 });
12627 assert!(
12628 path.exists(),
12629 "fake-aft-stub not built at {}: run `cargo test -p subc-core` (which builds \
12630 [[bin]] targets) rather than `cargo test -p subc-core --lib` (which does not)",
12631 path.display()
12632 );
12633 path
12634 }
12635
12636 #[test]
12637 fn reserved_never_spawned_refuses_every_hello() {
12638 let supervisor = SupervisorHandle::default();
12643 supervisor.apply_identity_configuration(&ModuleSpec {
12644 module_id: "never-spawned".to_string(),
12645 program: PathBuf::from("/usr/bin/false"),
12646 args: Vec::new(),
12647 env: Vec::new(),
12648 reserved: true,
12649 reserved_prefixes: Vec::new(),
12650 protocol: ModuleProtocol::Subc,
12651 overlap: Default::default(),
12652 carry_containment: false,
12653 });
12654 assert!(
12655 supervisor
12656 .reserved_hello_verification("never-spawned", Some("any-forged-nonce"))
12657 .is_err(),
12658 "forged nonce must refuse on a reserved never-spawned id"
12659 );
12660 assert!(
12661 supervisor
12662 .reserved_hello_verification("never-spawned", None)
12663 .is_err(),
12664 "absent nonce must refuse on a reserved never-spawned id"
12665 );
12666 supervisor.set_spawn_nonce("never-spawned", "minted".to_string());
12668 supervisor.apply_identity_configuration(&ModuleSpec {
12669 module_id: "never-spawned".to_string(),
12670 program: PathBuf::from("/usr/bin/false"),
12671 args: Vec::new(),
12672 env: Vec::new(),
12673 reserved: true,
12674 reserved_prefixes: Vec::new(),
12675 protocol: ModuleProtocol::Subc,
12676 overlap: Default::default(),
12677 carry_containment: false,
12678 });
12679 assert!(supervisor
12680 .reserved_hello_verification("never-spawned", Some("minted"))
12681 .is_ok());
12682 assert!(supervisor
12683 .reserved_hello_verification("never-spawned", Some("forged"))
12684 .is_err());
12685 }
12686
12687 fn seed_crash_restarts(state: &mut SupervisorSnapshot, count: u32) {
12690 let now = ClockInstant::now();
12691 for _ in 0..count {
12692 state.crash_restarts.push_back(now);
12693 }
12694 }
12695
12696 fn age_oldest_crash_restart_out_of_window(state: &mut SupervisorSnapshot, window: Duration) {
12700 let aged = state
12701 .crash_restarts
12702 .front()
12703 .expect("a crash restart must be recorded before it can be aged")
12704 .checked_sub(window + Duration::from_secs(1))
12705 .expect("the test clock is far enough from its origin to age an instant");
12706 state.crash_restarts[0] = aged;
12707 }
12708
12709 fn snapshot_with_restarts(enabled: bool, count: u32) -> SupervisorSnapshot {
12710 let mut state = SupervisorSnapshot::new(ModuleState::Running, enabled);
12711 seed_crash_restarts(&mut state, count);
12712 state
12713 }
12714
12715 #[test]
12716 fn daemon_owned_recovery_predicate_uses_the_pre_increment_budget() {
12717 let policy = RestartPolicy::new(3, Duration::ZERO);
12718 let now = ClockInstant::now();
12719 assert!(daemon_will_restart(
12720 &mut snapshot_with_restarts(true, 2),
12721 &policy,
12722 now
12723 ));
12724 assert!(!daemon_will_restart(
12725 &mut snapshot_with_restarts(true, 3),
12726 &policy,
12727 now
12728 ));
12729 assert!(!daemon_will_restart(
12730 &mut snapshot_with_restarts(false, 0),
12731 &policy,
12732 now
12733 ));
12734 }
12735
12736 #[test]
12737 fn crash_restart_backoff_escalates_with_in_window_count() {
12738 let policy = RestartPolicy::new(4, Duration::from_millis(100))
12739 .with_max_backoff(Duration::from_secs(30));
12740 let now = ClockInstant::now();
12741 let mut state = SupervisorSnapshot::new(ModuleState::Running, true);
12742 let schedules = (0..4)
12743 .map(|_| {
12744 state
12745 .next_crash_restart(&policy, now)
12746 .expect("the test policy allows four crash restarts")
12747 })
12748 .collect::<Vec<_>>();
12749
12750 assert_eq!(
12751 schedules
12752 .iter()
12753 .map(|schedule| schedule.restart_in_window)
12754 .collect::<Vec<_>>(),
12755 vec![0, 1, 2, 3]
12756 );
12757 assert_eq!(
12758 schedules
12759 .iter()
12760 .map(|schedule| schedule.delay)
12761 .collect::<Vec<_>>(),
12762 vec![
12763 Duration::from_millis(100),
12764 Duration::from_secs(1),
12765 Duration::from_secs(10),
12766 Duration::from_secs(30),
12767 ]
12768 );
12769 }
12770
12771 #[test]
12772 fn crash_restart_backoff_resets_after_ring_clear() {
12773 let policy = RestartPolicy::new(3, Duration::from_millis(100));
12774 let now = ClockInstant::now();
12775 let mut state = SupervisorSnapshot::new(ModuleState::Running, true);
12776 assert_eq!(
12777 state.next_crash_restart(&policy, now).unwrap().delay,
12778 Duration::from_millis(100)
12779 );
12780 assert_eq!(
12781 state.next_crash_restart(&policy, now).unwrap().delay,
12782 Duration::from_secs(1)
12783 );
12784
12785 state.clear_crash_restarts();
12786 let schedule = state
12787 .next_crash_restart(&policy, now)
12788 .expect("a cleared ring must allow another restart");
12789 assert_eq!(schedule.restart_in_window, 0);
12790 assert_eq!(schedule.delay, Duration::from_millis(100));
12791 }
12792
12793 #[test]
12794 fn crash_restart_backoff_ignores_aged_restarts() {
12795 let policy = RestartPolicy::new(3, Duration::from_millis(100));
12796 let now = ClockInstant::now();
12797 let mut state = SupervisorSnapshot::new(ModuleState::Running, true);
12798 state
12799 .next_crash_restart(&policy, now)
12800 .expect("the first restart is allowed");
12801 state
12802 .next_crash_restart(&policy, now)
12803 .expect("the second restart is allowed");
12804 state.crash_restarts[0] = now
12805 .checked_sub(policy.window + Duration::from_secs(1))
12806 .expect("the fake clock can age a restart past the window");
12807
12808 let schedule = state
12809 .next_crash_restart(&policy, now)
12810 .expect("an aged restart must release its slot");
12811 assert_eq!(schedule.restart_in_window, 1);
12812 assert_eq!(schedule.delay, Duration::from_secs(1));
12813 assert_eq!(state.crash_restarts.len(), 2);
12814 }
12815
12816 #[test]
12820 fn a_budget_spent_before_the_window_no_longer_refuses() {
12821 let policy = RestartPolicy::new(3, Duration::ZERO);
12822 let mut state = snapshot_with_restarts(true, 3);
12823 let now = ClockInstant::now();
12824 assert!(!daemon_will_restart(&mut state, &policy, now));
12825
12826 assert!(daemon_will_restart(
12827 &mut state,
12828 &policy,
12829 now + policy.window + Duration::from_secs(1)
12830 ));
12831 assert!(
12832 state.crash_restarts.is_empty(),
12833 "reading the budget must drop the instants that left the window"
12834 );
12835 }
12836
12837 fn module_with_recovery_snapshot(
12838 state: ModuleState,
12839 enabled: bool,
12840 restart_count: u32,
12841 ) -> SupervisedModule {
12842 let registry = Arc::new(Registry::default());
12843 let supervisor =
12844 Supervisor::new_for_test(Arc::clone(®istry), RestartPolicy::new(3, Duration::ZERO));
12845 let runtime = supervisor.runtime_config();
12846 let spec = ModuleSpec {
12847 module_id: "recovery-snapshot".to_string(),
12848 program: fake_aft_stub_path(),
12849 args: Vec::new(),
12850 env: Vec::new(),
12851 reserved: false,
12852 reserved_prefixes: Vec::new(),
12853 protocol: ModuleProtocol::Subc,
12854 overlap: Default::default(),
12855 carry_containment: false,
12856 };
12857 let mut snapshot = SupervisorSnapshot::new(state, enabled);
12858 seed_crash_restarts(&mut snapshot, restart_count);
12859 let (commands, _rx) = mpsc::channel(4);
12862 SupervisedModule {
12863 inner: Arc::new(SupervisedModuleInner {
12864 module_id: spec.module_id.clone(),
12865 registry,
12866 snapshot: Arc::new(Mutex::new(snapshot)),
12867 configuration: Arc::new(Mutex::new(SupervisedConfiguration {
12868 spec,
12869 health: runtime.health,
12870 })),
12871 stderr_ring: runtime.stderr_ring,
12872 terminal_ring: runtime.terminal_ring,
12873 commands,
12874 monitor: Mutex::new(None),
12875 restart_policy: runtime.restart_policy,
12876 effective_drain_timeout: runtime.effective_drain_timeout,
12877 provenance_probe: supervisor.provenance_probe.clone(),
12878 retained: runtime.child_roster.retained_view(),
12879 }),
12880 }
12881 }
12882
12883 #[cfg(target_os = "linux")]
12884 #[tokio::test]
12885 async fn no_cgroup_placement_does_not_block_fake_aft_stub_spawn() {
12886 let supervisor =
12887 Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default())
12888 .with_cgroup_placement(None);
12889 let result = supervisor.spawn(ModuleSpec {
12890 module_id: "no-cgroup-placement".to_string(),
12891 program: fake_aft_stub_path(),
12892 args: Vec::new(),
12893 env: Vec::new(),
12894 reserved: false,
12895 reserved_prefixes: Vec::new(),
12896 protocol: ModuleProtocol::Subc,
12897 overlap: Default::default(),
12898 carry_containment: false,
12899 });
12900
12901 assert!(
12902 result.is_ok(),
12903 "no delegation must not turn an otherwise valid spawn into a failure: {result:?}"
12904 );
12905 }
12906
12907 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
12908 async fn undecided_snapshot_uses_shared_restart_predicate() {
12909 assert!(module_with_recovery_snapshot(ModuleState::Running, true, 2)
12910 .will_recover_after_connection_loss()
12911 .unwrap());
12912 assert!(
12913 !module_with_recovery_snapshot(ModuleState::Running, true, 3)
12914 .will_recover_after_connection_loss()
12915 .unwrap()
12916 );
12917 }
12918
12919 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
12920 async fn restarting_snapshot_at_exhausted_budget_is_non_terminal() {
12921 assert!(
12922 module_with_recovery_snapshot(ModuleState::Restarting, true, 3)
12923 .will_recover_after_connection_loss()
12924 .unwrap()
12925 );
12926 }
12927
12928 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
12929 async fn terminal_phase_snapshots_are_terminal_before_budget_exhaustion() {
12930 assert!(!module_with_recovery_snapshot(ModuleState::Failed, true, 0)
12931 .will_recover_after_connection_loss()
12932 .unwrap());
12933 assert!(
12934 !module_with_recovery_snapshot(ModuleState::Disabled, true, 0)
12935 .will_recover_after_connection_loss()
12936 .unwrap()
12937 );
12938 }
12939
12940 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
12941 async fn warming_snapshot_is_limited_to_startup_phases() {
12942 for state in [
12943 ModuleState::Starting,
12944 ModuleState::Running,
12945 ModuleState::Restarting,
12946 ] {
12947 assert!(
12948 module_with_recovery_snapshot(state, true, 0)
12949 .is_warming()
12950 .unwrap(),
12951 "{state:?} should be warming"
12952 );
12953 }
12954 for state in [
12955 ModuleState::Unresponsive,
12956 ModuleState::Draining,
12957 ModuleState::Stopped,
12958 ModuleState::Failed,
12959 ModuleState::Disabled,
12960 ] {
12961 assert!(
12962 !module_with_recovery_snapshot(state, true, 0)
12963 .is_warming()
12964 .unwrap(),
12965 "{state:?} should not be warming"
12966 );
12967 }
12968 }
12969
12970 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
12971 async fn terminal_history_survives_respawn_and_keeps_both_crashes_in_order() {
12972 let registry = Arc::new(Registry::default());
12973 let supervisor =
12974 Supervisor::new_for_test(Arc::clone(®istry), RestartPolicy::new(1, Duration::ZERO));
12975 let module = supervisor
12976 .spawn(ModuleSpec {
12977 module_id: "terminal-history".to_string(),
12978 program: fake_aft_stub_path(),
12979 args: Vec::new(),
12980 env: vec![("FAKE_AFT_EXIT_CODE".to_string(), "23".to_string())],
12981 reserved: false,
12982 reserved_prefixes: Vec::new(),
12983 protocol: ModuleProtocol::Subc,
12984 overlap: Default::default(),
12985 carry_containment: false,
12986 })
12987 .unwrap();
12988
12989 let deadline = Instant::now() + Duration::from_secs(5);
12990 loop {
12991 let history = module.terminal_history();
12992 if history.entries.len() == 2 {
12993 assert_eq!(module.status().unwrap().state, ModuleState::Failed);
12994 assert_eq!(history.dropped, 0);
12995 assert_eq!(
12996 history
12997 .entries
12998 .iter()
12999 .map(|entry| entry.exit_code)
13000 .collect::<Vec<_>>(),
13001 vec![Some(23), Some(23)]
13002 );
13003 assert!(history.entries[0].at_ms <= history.entries[1].at_ms);
13004 return;
13005 }
13006 assert!(
13007 Instant::now() < deadline,
13008 "module did not retain two terminal exits: {history:?}"
13009 );
13010 sleep(Duration::from_millis(10)).await;
13011 }
13012 }
13013
13014 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
13018 async fn disable_during_crash_backoff_cancels_pending_respawn() {
13019 let backoff = Duration::from_secs(2);
13020 let supervisor = Supervisor::new_for_test(
13021 Arc::new(Registry::default()),
13022 RestartPolicy::new(10, backoff),
13023 );
13024 let module = supervisor
13025 .spawn(ModuleSpec {
13026 module_id: "disable-during-backoff".to_string(),
13027 program: fake_aft_stub_path(),
13028 args: Vec::new(),
13029 env: vec![("FAKE_AFT_EXIT_CODE".to_string(), "23".to_string())],
13030 reserved: false,
13031 reserved_prefixes: Vec::new(),
13032 protocol: ModuleProtocol::Subc,
13033 overlap: Default::default(),
13034 carry_containment: false,
13035 })
13036 .unwrap();
13037
13038 let deadline = Instant::now() + Duration::from_secs(5);
13040 loop {
13041 if module.status().unwrap().state == ModuleState::Restarting {
13042 break;
13043 }
13044 assert!(
13045 Instant::now() < deadline,
13046 "module never entered the crash backoff"
13047 );
13048 sleep(Duration::from_millis(10)).await;
13049 }
13050
13051 let started = Instant::now();
13052 module.set_enabled(false).await.unwrap();
13053 let waited = started.elapsed();
13054
13055 assert!(
13056 waited < backoff / 2,
13057 "disable waited {waited:?} behind the {backoff:?} crash backoff; the operator command must preempt the pending respawn"
13058 );
13059 assert_eq!(module.status().unwrap().state, ModuleState::Disabled);
13060
13061 sleep(backoff + Duration::from_millis(500)).await;
13063 let status = module.status().unwrap();
13064 assert_eq!(status.state, ModuleState::Disabled);
13065 assert_eq!(
13066 status.spawn_generation, 1,
13067 "module respawned after the operator disabled it"
13068 );
13069 }
13070
13071 #[cfg(unix)]
13074 fn protocol_none_sigterm_exits_clean_spec(
13075 module_id: &str,
13076 dir: &std::path::Path,
13077 ) -> (ModuleSpec, PathBuf, PathBuf) {
13078 let ready = dir.join("ready");
13079 let marker = dir.join("sigterm");
13080 let spec = ModuleSpec {
13081 module_id: module_id.to_string(),
13082 program: fake_aft_stub_path(),
13083 args: Vec::new(),
13084 env: vec![
13085 ("FAKE_AFT_NEVER_CONNECT".to_string(), "1".to_string()),
13086 (
13087 "FAKE_AFT_SIGTERM_MARKER_PATH".to_string(),
13088 marker.display().to_string(),
13089 ),
13090 (
13091 "FAKE_AFT_NEVER_CONNECT_READY_PATH".to_string(),
13092 ready.display().to_string(),
13093 ),
13094 ],
13095 reserved: false,
13096 reserved_prefixes: Vec::new(),
13097 protocol: ModuleProtocol::None,
13098 overlap: Default::default(),
13099 carry_containment: false,
13100 };
13101 (spec, ready, marker)
13102 }
13103
13104 #[cfg(unix)]
13108 async fn wait_for_file(path: &std::path::Path) {
13109 let deadline = Instant::now() + Duration::from_secs(10);
13110 while !path.exists() {
13111 assert!(
13112 Instant::now() < deadline,
13113 "{} never appeared",
13114 path.display()
13115 );
13116 sleep(Duration::from_millis(10)).await;
13117 }
13118 }
13119
13120 #[cfg(unix)]
13124 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
13125 async fn protocol_none_unrequested_clean_exit_restarts_as_a_crash() {
13126 let dir = cortexkit_test_support::ScratchDir::new("none-unrequested-clean-exit");
13127 let (spec, ready, marker) =
13128 protocol_none_sigterm_exits_clean_spec("none-unrequested-clean-exit", dir.path());
13129 let supervisor = Supervisor::new_for_test(
13130 Arc::new(Registry::default()),
13131 RestartPolicy::new(3, Duration::ZERO),
13132 );
13133 let module = supervisor.spawn(spec).unwrap();
13134 wait_for_file(&ready).await;
13135 let deadline = Instant::now() + Duration::from_secs(10);
13139 let first_pid = loop {
13140 let status = module.status().unwrap();
13141 if status.state == ModuleState::Running {
13142 if let Some(pid) = status.pid {
13143 break pid;
13144 }
13145 }
13146 assert!(
13147 Instant::now() < deadline,
13148 "a running module must report its pid after exec confirmation: {status:?}"
13149 );
13150 sleep(Duration::from_millis(10)).await;
13151 };
13152
13153 rustix::process::kill_process(
13154 rustix::process::Pid::from_raw(i32::try_from(first_pid).unwrap()).unwrap(),
13155 rustix::process::Signal::TERM,
13156 )
13157 .unwrap();
13158
13159 let deadline = Instant::now() + Duration::from_secs(10);
13160 let respawned = loop {
13161 let status = module.status().unwrap();
13162 if status.state == ModuleState::Running
13163 && status.pid.is_some_and(|pid| pid != first_pid)
13164 {
13165 break status;
13166 }
13167 assert!(
13168 Instant::now() < deadline,
13169 "protocol-none module was not respawned after an unrequested clean exit: {status:?}"
13170 );
13171 sleep(Duration::from_millis(10)).await;
13172 };
13173 assert_eq!(respawned.spawn_generation, 2);
13174 assert!(
13175 marker.exists(),
13176 "the child must have exited through its SIGTERM handler (exit 0), or this proves nothing about clean exits"
13177 );
13178
13179 let history = module.terminal_history();
13180 assert_eq!(history.entries.len(), 1, "{history:?}");
13181 let entry = &history.entries[0];
13182 assert_eq!(entry.exit_code, Some(0));
13183 assert_eq!(entry.exit_kind, subc_control::TerminalExitKind::Clean);
13184 assert_eq!(entry.disposition, TerminalDisposition::Restarting);
13185
13186 module.stop().await.unwrap();
13187 }
13188
13189 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
13193 async fn protocol_none_repeated_clean_exits_exhaust_the_restart_budget() {
13194 let supervisor = Supervisor::new_for_test(
13195 Arc::new(Registry::default()),
13196 RestartPolicy::new(1, Duration::ZERO),
13197 );
13198 let module = supervisor
13199 .spawn(ModuleSpec {
13200 module_id: "none-clean-exit-budget".to_string(),
13201 program: fake_aft_stub_path(),
13202 args: Vec::new(),
13203 env: vec![("FAKE_AFT_EXIT_CODE".to_string(), "0".to_string())],
13204 reserved: false,
13205 reserved_prefixes: Vec::new(),
13206 protocol: ModuleProtocol::None,
13207 overlap: Default::default(),
13208 carry_containment: false,
13209 })
13210 .unwrap();
13211
13212 let deadline = Instant::now() + Duration::from_secs(10);
13215 loop {
13216 let status = module.status().unwrap();
13217 if status.state == ModuleState::Failed {
13218 break;
13219 }
13220 assert!(
13221 Instant::now() < deadline,
13222 "module never exhausted its budget: {status:?} {:?}",
13223 module.terminal_history()
13224 );
13225 sleep(Duration::from_millis(10)).await;
13226 }
13227 let history = module.terminal_history();
13228 assert_eq!(
13229 history
13230 .entries
13231 .iter()
13232 .map(|entry| (entry.exit_code, entry.disposition.clone()))
13233 .collect::<Vec<_>>(),
13234 vec![
13235 (Some(0), TerminalDisposition::Restarting),
13236 (Some(0), TerminalDisposition::Failed),
13237 ]
13238 );
13239 let detail = history.entries[1]
13240 .disposition_detail
13241 .as_deref()
13242 .expect("a budget failure names the budget");
13243 assert!(detail.contains("max_restarts=1"), "{detail}");
13244 assert_eq!(module.status().unwrap().spawn_generation, 2);
13245 }
13246
13247 #[test]
13248 fn restart_budget_failure_is_published_after_its_terminal_record() {
13249 use std::io::{Read, Write};
13250 let supervisor = Supervisor::new_for_test(
13251 Arc::new(Registry::default()),
13252 RestartPolicy::new(0, Duration::ZERO),
13253 );
13254 let runtime = supervisor.runtime_config();
13255 let spec = ModuleSpec {
13256 protocol: ModuleProtocol::None,
13257 ..windowed_crash_spec("budget-publication-order")
13258 };
13259 let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::new(
13260 ModuleState::Running,
13261 true,
13262 )));
13263 lock_snapshot(&snapshot).unwrap().process_alive = true;
13264 let reaped_snapshot = snapshot.clone();
13265 let mut barrier = terminal_record_barrier(&runtime.spawn_events);
13266 let history = runtime.terminal_ring.clone();
13267 let reap = std::thread::spawn(move || {
13268 tokio::runtime::Builder::new_current_thread()
13269 .enable_all()
13270 .build()
13271 .unwrap()
13272 .block_on(on_child_exit(
13273 &spec,
13274 runtime.restart_policy,
13275 &supervisor.registry,
13276 &reaped_snapshot,
13277 &runtime.terminal_ring,
13278 &runtime.spawn_events,
13279 &runtime.child_roster,
13280 ExitReport {
13281 kind: ExitKind::Clean,
13282 code: Some(0),
13283 signal: None,
13284 at_ms: 1,
13285 },
13286 true,
13287 ))
13288 });
13289 let mut reached = [0];
13290 barrier.read_exact(&mut reached).unwrap();
13291 assert_eq!(&reached, b"R");
13292 let (before_state, before_alive) = {
13293 let state = lock_snapshot(&snapshot).unwrap();
13294 (state.state, state.process_alive)
13295 };
13296 let before_history = history.lock().unwrap().snapshot();
13297 barrier.write_all(b"X").unwrap();
13298 assert!(matches!(reap.join().unwrap(), NextAction::Stop { .. }));
13299 assert!(before_history.entries.is_empty());
13300 assert_ne!(
13301 before_state,
13302 ModuleState::Failed,
13303 "Failed was visible before its terminal record could be written"
13304 );
13305 assert!(before_alive);
13306 assert_eq!(lock_snapshot(&snapshot).unwrap().state, ModuleState::Failed);
13307 let history = history.lock().unwrap().snapshot();
13308 assert_eq!(history.entries.len(), 1);
13309 assert_eq!(history.entries[0].exit_code, Some(0));
13310 assert_eq!(history.entries[0].disposition, TerminalDisposition::Failed);
13311 }
13312
13313 #[test]
13314 fn crash_restart_at_budget_limit_remains_recoverable_while_recording() {
13315 use std::io::{Read, Write};
13316
13317 let supervisor = Supervisor::new_for_test(
13318 Arc::new(Registry::default()),
13319 RestartPolicy::new(3, Duration::ZERO),
13320 );
13321 let mut runtime = supervisor.runtime_config();
13322 let module = module_with_recovery_snapshot(ModuleState::Running, true, 2);
13323 let snapshot = module.inner.snapshot.clone();
13324 lock_snapshot(&snapshot).unwrap().process_alive = true;
13325 runtime.terminal_ring = module.inner.terminal_ring.clone();
13326 let spec = windowed_crash_spec(module.module_id());
13327 let mut barrier = terminal_record_barrier(&runtime.spawn_events);
13328 let reaped_snapshot = snapshot.clone();
13329 let reap = std::thread::spawn(move || {
13330 tokio::runtime::Builder::new_current_thread()
13331 .enable_all()
13332 .build()
13333 .unwrap()
13334 .block_on(on_child_exit(
13335 &spec,
13336 runtime.restart_policy,
13337 &supervisor.registry,
13338 &reaped_snapshot,
13339 &runtime.terminal_ring,
13340 &runtime.spawn_events,
13341 &runtime.child_roster,
13342 ExitReport {
13343 kind: ExitKind::Crash,
13344 code: Some(2),
13345 signal: None,
13346 at_ms: 1,
13347 },
13348 true,
13349 ))
13350 });
13351
13352 let mut reached = [0];
13356 barrier.read_exact(&mut reached).unwrap();
13357 assert_eq!(&reached, b"R");
13358 let before_status = module.status().unwrap();
13359 let before_history = module.terminal_history();
13360 let before_recovers = module.will_recover_after_connection_loss().unwrap();
13361 barrier.write_all(b"X").unwrap();
13362 assert!(matches!(reap.join().unwrap(), NextAction::Restart { .. }));
13363
13364 assert_eq!(before_status.state, ModuleState::Running);
13365 assert!(before_status.process_alive);
13366 assert_eq!(before_status.restart_count, 3);
13367 assert!(before_history.entries.is_empty());
13368 assert!(
13369 before_recovers,
13370 "connection cleanup would report a terminal close for a scheduled crash restart"
13371 );
13372 let restarting = module.status().unwrap();
13373 assert_eq!(restarting.state, ModuleState::Restarting);
13374 assert_eq!(restarting.restart_count, 3);
13375 assert!(module.will_recover_after_connection_loss().unwrap());
13376 let history = module.terminal_history();
13377 assert_eq!(history.entries.len(), 1);
13378 assert_eq!(
13379 history.entries[0].disposition,
13380 TerminalDisposition::Restarting
13381 );
13382
13383 update_snapshot(&snapshot, Some(module.module_id()), |state| {
13386 state.state = ModuleState::Running;
13387 state.respawn_pending = false;
13388 })
13389 .unwrap();
13390 assert!(!module.will_recover_after_connection_loss().unwrap());
13391 update_snapshot(&snapshot, Some(module.module_id()), |state| {
13392 state.enabled = false;
13393 state.respawn_pending = true;
13394 })
13395 .unwrap();
13396 assert!(!module.will_recover_after_connection_loss().unwrap());
13397 }
13398
13399 fn terminal_record_barrier(events: &super::SpawnEventFeed) -> std::net::TcpStream {
13400 use std::net::{TcpListener, TcpStream};
13401 let listener = TcpListener::bind("127.0.0.1:0").unwrap();
13402 let writer = TcpStream::connect(listener.local_addr().unwrap()).unwrap();
13403 let (barrier, _) = listener.accept().unwrap();
13404 for stream in [&writer, &barrier] {
13405 stream
13406 .set_read_timeout(Some(Duration::from_secs(10)))
13407 .unwrap();
13408 stream
13409 .set_write_timeout(Some(Duration::from_secs(10)))
13410 .unwrap();
13411 }
13412 *events.1.lock().unwrap() = Some(writer);
13413 barrier
13414 }
13415
13416 pub(super) fn before_terminal_record(events: &super::SpawnEventFeed) {
13417 use std::io::{Read, Write};
13418 let barrier = events.1.lock().unwrap().take();
13419 if let Some(mut barrier) = barrier {
13420 barrier.write_all(b"R").unwrap();
13421 let mut release = [0];
13422 barrier.read_exact(&mut release).unwrap();
13423 assert_eq!(&release, b"X");
13424 }
13425 }
13426
13427 #[test]
13428 fn health_restart_budget_failure_is_published_after_its_terminal_record() {
13429 use std::io::{Read, Write};
13430
13431 let supervisor = Supervisor::new_for_test(
13432 Arc::new(Registry::default()),
13433 RestartPolicy::new(0, Duration::ZERO),
13434 );
13435 let mut runtime = supervisor.runtime_config();
13436 let module = module_with_recovery_snapshot(ModuleState::Running, true, 0);
13437 let snapshot = module.inner.snapshot.clone();
13438 runtime.terminal_ring = module.inner.terminal_ring.clone();
13439 let spec = ModuleSpec {
13440 module_id: module.module_id().into(),
13441 program: fake_aft_stub_path(),
13442 protocol: ModuleProtocol::None,
13443 env: vec![("FAKE_AFT_EXIT_CODE".into(), "17".into())],
13444 ..windowed_crash_spec(module.module_id())
13445 };
13446 let mut barrier = terminal_record_barrier(&runtime.spawn_events);
13447 let reap = std::thread::spawn(move || {
13448 tokio::runtime::Builder::new_current_thread()
13449 .enable_all()
13450 .build()
13451 .unwrap()
13452 .block_on(async {
13453 let mut child =
13454 Some(spawn_and_mark_running(&spec, &runtime, &snapshot).unwrap());
13455 health_restart_child(
13456 &spec,
13457 &runtime,
13458 &supervisor.registry,
13459 &supervisor.process_liveness,
13460 &snapshot,
13461 &mut child,
13462 SupervisorHealthStatus::Failing,
13463 Some("health probe did not answer"),
13464 1,
13465 )
13466 .await
13467 .unwrap();
13468 assert!(child.is_none());
13469 });
13470 });
13471 let mut reached = [0];
13476 barrier.read_exact(&mut reached).unwrap();
13477 assert_eq!(&reached, b"R");
13478 let before_status = module.status().unwrap();
13479 let before_history = module.terminal_history();
13480 barrier.write_all(b"X").unwrap();
13481 reap.join().unwrap();
13482 assert!(before_history.entries.is_empty());
13483 assert_ne!(
13484 before_status.state,
13485 ModuleState::Failed,
13486 "Failed was visible before its terminal record could be written"
13487 );
13488 assert!(
13489 before_status.process_alive,
13490 "process death was visible before its terminal record could be written"
13491 );
13492 let failed = module.status().unwrap();
13493 assert_eq!(failed.state, ModuleState::Failed);
13494 assert!(!failed.process_alive);
13495 let history = module.terminal_history();
13496 assert_eq!(history.entries.len(), 1);
13497 assert_eq!(history.entries[0].disposition, TerminalDisposition::Failed);
13498 assert_eq!(
13499 history.entries[0].disposition_detail.as_deref(),
13500 Some("crash budget exhausted: max_restarts=0 within window_secs=600")
13501 );
13502 }
13503
13504 #[test]
13505 fn observed_exits_publish_process_death_after_terminal_history() {
13506 use std::io::{Read, Write};
13507 for (kind, enabled, shutdown, wait_error, expected_state) in [
13508 (ExitKind::Clean, true, false, false, ModuleState::Stopped),
13509 (ExitKind::Crash, true, false, false, ModuleState::Restarting),
13510 (ExitKind::Crash, false, false, false, ModuleState::Disabled),
13511 (
13512 ExitKind::DeliberateSeverance,
13513 true,
13514 false,
13515 false,
13516 ModuleState::Restarting,
13517 ),
13518 #[cfg(unix)]
13519 (ExitKind::Crash, true, true, false, ModuleState::Stopped),
13520 (ExitKind::Crash, true, false, true, ModuleState::Failed),
13521 ] {
13522 let supervisor = Supervisor::new_for_test(
13523 Arc::new(Registry::default()),
13524 RestartPolicy::new(1, Duration::ZERO),
13525 );
13526 let runtime = supervisor.runtime_config();
13527 #[cfg(unix)]
13528 if shutdown {
13529 runtime.child_roster.close();
13530 }
13531 #[cfg(not(unix))]
13532 let _ = shutdown;
13533 let spec = windowed_crash_spec("exit-publication-order");
13534 let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::new(
13535 ModuleState::Running,
13536 enabled,
13537 )));
13538 lock_snapshot(&snapshot).unwrap().process_alive = true;
13539 let reaped_snapshot = snapshot.clone();
13540 let history = runtime.terminal_ring.clone();
13541 let mut barrier = terminal_record_barrier(&runtime.spawn_events);
13542 let reap = std::thread::spawn(move || {
13543 if wait_error {
13544 super::fail_after_wait_error(
13545 &spec.module_id,
13546 &reaped_snapshot,
13547 &runtime.terminal_ring,
13548 &runtime.spawn_events,
13549 "wait failed".into(),
13550 &runtime.failure_context(true),
13551 );
13552 } else {
13553 tokio::runtime::Builder::new_current_thread()
13554 .enable_all()
13555 .build()
13556 .unwrap()
13557 .block_on(on_child_exit(
13558 &spec,
13559 runtime.restart_policy,
13560 &supervisor.registry,
13561 &reaped_snapshot,
13562 &runtime.terminal_ring,
13563 &runtime.spawn_events,
13564 &runtime.child_roster,
13565 ExitReport {
13566 kind,
13567 code: Some(0),
13568 signal: None,
13569 at_ms: 1,
13570 },
13571 true,
13572 ));
13573 }
13574 });
13575 let mut reached = [0];
13576 barrier.read_exact(&mut reached).unwrap();
13577 assert_eq!(&reached, b"R");
13578 let before_alive = lock_snapshot(&snapshot).unwrap().process_alive;
13579 let before_history = history.lock().unwrap().snapshot();
13580 barrier.write_all(b"X").unwrap();
13581 reap.join().unwrap();
13582 assert!(
13583 before_alive,
13584 "{kind:?}: death was published before recording"
13585 );
13586 assert!(before_history.entries.is_empty());
13587 assert_eq!(lock_snapshot(&snapshot).unwrap().state, expected_state);
13588 assert!(!lock_snapshot(&snapshot).unwrap().process_alive);
13589 assert_eq!(history.lock().unwrap().snapshot().entries.len(), 1);
13590 }
13591 }
13592
13593 #[tokio::test]
13594 async fn health_restart_budget_failure_remains_failed_when_journal_append_fails() {
13595 let dir = cortexkit_test_support::ScratchDir::new("health-budget-journal-failure");
13596 let path = dir.join("terminals.jsonl");
13597 std::fs::create_dir(&path).unwrap();
13598 let supervisor = Supervisor::new_for_test(
13599 Arc::new(Registry::default()),
13600 RestartPolicy::new(0, Duration::ZERO),
13601 )
13602 .with_terminal_journal(path, "health-budget-journal-failure".into());
13603 let runtime = supervisor.runtime_config();
13604 let spec = ModuleSpec {
13605 program: fake_aft_stub_path(),
13606 protocol: ModuleProtocol::None,
13607 env: vec![("FAKE_AFT_EXIT_CODE".into(), "17".into())],
13608 ..windowed_crash_spec("health-budget-journal-failure")
13609 };
13610 let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::starting()));
13611 let mut child = Some(spawn_and_mark_running(&spec, &runtime, &snapshot).unwrap());
13612 health_restart_child(
13613 &spec,
13614 &runtime,
13615 &supervisor.registry,
13616 &supervisor.process_liveness,
13617 &snapshot,
13618 &mut child,
13619 SupervisorHealthStatus::Failing,
13620 None,
13621 1,
13622 )
13623 .await
13624 .unwrap();
13625 assert_eq!(lock_snapshot(&snapshot).unwrap().state, ModuleState::Failed);
13626 assert!(!lock_snapshot(&snapshot).unwrap().process_alive);
13627 let history = runtime
13628 .terminal_ring
13629 .lock()
13630 .unwrap()
13631 .durable_history(&spec.module_id);
13632 assert!(history.journal_write_failures > 0);
13633 assert_eq!(history.entries.len(), 1);
13634 assert_eq!(history.entries[0].disposition, TerminalDisposition::Failed);
13635 }
13636
13637 #[test]
13638 fn health_restart_budget_failure_remains_failed_when_exit_recording_panics() {
13639 let supervisor = Supervisor::new_for_test(
13640 Arc::new(Registry::default()),
13641 RestartPolicy::new(0, Duration::ZERO),
13642 );
13643 let runtime = supervisor.runtime_config();
13644 let spec = ModuleSpec {
13645 program: fake_aft_stub_path(),
13646 protocol: ModuleProtocol::None,
13647 env: vec![("FAKE_AFT_EXIT_CODE".into(), "17".into())],
13648 ..windowed_crash_spec("health-budget-recording-panic")
13649 };
13650 let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::starting()));
13651 let result = std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| {
13652 tokio::runtime::Builder::new_current_thread()
13653 .enable_all()
13654 .build()
13655 .unwrap()
13656 .block_on(async {
13657 let mut child =
13658 Some(spawn_and_mark_running(&spec, &runtime, &snapshot).unwrap());
13659 runtime.spawn_events.0.lock().unwrap().seq = u64::MAX;
13660 health_restart_child(
13661 &spec,
13662 &runtime,
13663 &supervisor.registry,
13664 &supervisor.process_liveness,
13665 &snapshot,
13666 &mut child,
13667 SupervisorHealthStatus::Failing,
13668 None,
13669 1,
13670 )
13671 .await
13672 .unwrap();
13673 });
13674 }));
13675 let panic = result.expect_err("recording must still unwind");
13676 assert_eq!(
13677 panic.downcast_ref::<String>().map(String::as_str),
13678 Some("spawn event sequence exhausted")
13679 );
13680 let state = lock_snapshot(&snapshot).unwrap();
13681 assert_eq!(state.state, ModuleState::Failed);
13682 assert!(!state.process_alive);
13683 assert!(state.last_exit.is_some());
13684 }
13685
13686 #[tokio::test]
13687 async fn restart_budget_failure_remains_failed_when_journal_append_fails() {
13688 let dir = cortexkit_test_support::ScratchDir::new("budget-journal-failure");
13689 let path = dir.join("terminals.jsonl");
13690 std::fs::create_dir(&path).unwrap();
13691 let supervisor = Supervisor::new_for_test(
13692 Arc::new(Registry::default()),
13693 RestartPolicy::new(0, Duration::ZERO),
13694 )
13695 .with_terminal_journal(path, "budget-journal-failure".into());
13696 let runtime = supervisor.runtime_config();
13697 let spec = windowed_crash_spec("budget-journal-failure");
13698 let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::starting()));
13699 assert!(matches!(
13700 on_child_exit(
13701 &spec,
13702 runtime.restart_policy,
13703 &supervisor.registry,
13704 &snapshot,
13705 &runtime.terminal_ring,
13706 &runtime.spawn_events,
13707 &runtime.child_roster,
13708 crash_exit_report(1),
13709 true,
13710 )
13711 .await,
13712 NextAction::Stop { .. }
13713 ));
13714 assert_eq!(lock_snapshot(&snapshot).unwrap().state, ModuleState::Failed);
13715 let history = runtime
13716 .terminal_ring
13717 .lock()
13718 .unwrap()
13719 .durable_history(&spec.module_id);
13720 assert!(history.journal_write_failures > 0);
13721 assert_eq!(history.entries.len(), 1);
13722 assert_eq!(history.entries[0].disposition, TerminalDisposition::Failed);
13723 }
13724
13725 #[test]
13726 fn restart_budget_failure_remains_failed_when_exit_recording_panics() {
13727 let supervisor = Supervisor::new_for_test(
13728 Arc::new(Registry::default()),
13729 RestartPolicy::new(0, Duration::ZERO),
13730 );
13731 let runtime = supervisor.runtime_config();
13732 let spec = windowed_crash_spec("budget-recording-panic");
13733 let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::starting()));
13734 runtime.spawn_events.emit_spawned(&spec.module_id, 1, 1);
13735 runtime.spawn_events.0.lock().unwrap().seq = u64::MAX;
13738 let result = std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| {
13739 tokio::runtime::Builder::new_current_thread()
13740 .enable_all()
13741 .build()
13742 .unwrap()
13743 .block_on(on_child_exit(
13744 &spec,
13745 runtime.restart_policy,
13746 &supervisor.registry,
13747 &snapshot,
13748 &runtime.terminal_ring,
13749 &runtime.spawn_events,
13750 &runtime.child_roster,
13751 crash_exit_report(1),
13752 true,
13753 ))
13754 }));
13755 let panic = result.err().expect("recording must still unwind");
13756 assert_eq!(
13757 panic.downcast_ref::<String>().map(String::as_str),
13758 Some("spawn event sequence exhausted")
13759 );
13760 assert_eq!(lock_snapshot(&snapshot).unwrap().state, ModuleState::Failed);
13761 }
13762
13763 #[cfg(unix)]
13766 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
13767 async fn protocol_none_requested_stop_and_disable_do_not_respawn() {
13768 for disable in [false, true] {
13769 let label = if disable {
13770 "none-requested-disable"
13771 } else {
13772 "none-requested-stop"
13773 };
13774 let dir = cortexkit_test_support::ScratchDir::new(label);
13775 let (spec, ready, marker) = protocol_none_sigterm_exits_clean_spec(label, dir.path());
13776 let supervisor = Supervisor::new_for_test(
13777 Arc::new(Registry::default()),
13778 RestartPolicy::new(3, Duration::ZERO),
13779 );
13780 let module = supervisor.spawn(spec).unwrap();
13781 wait_for_file(&ready).await;
13782
13783 if disable {
13784 module.set_enabled(false).await.unwrap();
13785 } else {
13786 module.stop().await.unwrap();
13787 }
13788 assert!(
13789 marker.exists(),
13790 "{label}: the child must have left through its SIGTERM handler with exit 0"
13791 );
13792
13793 sleep(Duration::from_millis(500)).await;
13796 let status = module.status().unwrap();
13797 let expected = if disable {
13798 ModuleState::Disabled
13799 } else {
13800 ModuleState::Stopped
13801 };
13802 assert_eq!(status.state, expected, "{label}");
13803 assert_eq!(
13804 status.spawn_generation, 1,
13805 "{label}: respawned after a requested stop"
13806 );
13807 let history = module.terminal_history();
13808 assert_eq!(history.entries.len(), 1, "{label}: {history:?}");
13809 assert_eq!(history.entries[0].exit_code, Some(0), "{label}");
13810 assert_ne!(
13811 history.entries[0].disposition,
13812 TerminalDisposition::Restarting,
13813 "{label}"
13814 );
13815 }
13816 }
13817
13818 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
13821 async fn subc_wire_clean_exit_is_still_a_stop() {
13822 let supervisor = Supervisor::new_for_test(
13823 Arc::new(Registry::default()),
13824 RestartPolicy::new(3, Duration::ZERO),
13825 );
13826 let module = supervisor
13827 .spawn(ModuleSpec {
13828 module_id: "wire-clean-exit".to_string(),
13829 program: fake_aft_stub_path(),
13830 args: Vec::new(),
13831 env: vec![("FAKE_AFT_EXIT_CODE".to_string(), "0".to_string())],
13832 reserved: false,
13833 reserved_prefixes: Vec::new(),
13834 protocol: ModuleProtocol::Subc,
13835 overlap: Default::default(),
13836 carry_containment: false,
13837 })
13838 .unwrap();
13839
13840 let deadline = Instant::now() + Duration::from_secs(10);
13841 while module.terminal_history().entries.is_empty() {
13842 assert!(Instant::now() < deadline, "module never exited");
13843 sleep(Duration::from_millis(10)).await;
13844 }
13845 sleep(Duration::from_millis(500)).await;
13847 let status = module.status().unwrap();
13848 assert_eq!(status.state, ModuleState::Stopped);
13849 assert_eq!(status.spawn_generation, 1);
13850 let history = module.terminal_history();
13851 assert_eq!(history.entries.len(), 1, "{history:?}");
13852 assert_eq!(history.entries[0].exit_code, Some(0));
13853 assert_eq!(history.entries[0].disposition, TerminalDisposition::Stopped);
13854 }
13855
13856 #[cfg(unix)]
13857 #[tokio::test]
13858 async fn reload_exit_keeps_roster_until_terminal_is_recorded_during_shutdown() {
13859 let dir = cortexkit_test_support::ScratchDir::new("reload-roster-terminal-order");
13860 let record = dir.join("live-children.json");
13861 let supervisor = Supervisor::new_for_test(
13862 Arc::new(Registry::default()),
13863 RestartPolicy::new(0, Duration::ZERO),
13864 );
13865 let mut runtime = supervisor.runtime_config();
13866 runtime.child_roster.record_to(record.clone());
13867 let gate = Arc::new(super::ReloadExitRecordGate::default());
13868 runtime.test_reload_exit_record_gate = Some(Arc::clone(&gate));
13869 let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::starting()));
13870 let spec = ModuleSpec {
13871 module_id: "reload-exit-roster".into(),
13872 program: fake_aft_stub_path(),
13873 args: Vec::new(),
13874 env: vec![("FAKE_AFT_EXIT_CODE".into(), "23".into())],
13875 reserved: false,
13876 reserved_prefixes: Vec::new(),
13877 protocol: ModuleProtocol::Subc,
13878 overlap: Default::default(),
13879 carry_containment: false,
13880 };
13881 let mut child = None;
13882 let reload = super::finish_reload_child(
13883 &spec,
13884 &runtime,
13885 &supervisor.registry,
13886 &supervisor.process_liveness,
13887 &snapshot,
13888 &mut child,
13889 );
13890 tokio::pin!(reload);
13891 tokio::select! {
13892 result = &mut reload => panic!("reload missed the before-record gate: {result:?}"),
13893 _ = gate.reached.notified() => {}
13894 }
13895 assert!(runtime
13896 .terminal_ring
13897 .lock()
13898 .unwrap()
13899 .snapshot()
13900 .entries
13901 .is_empty());
13902 assert_eq!(
13903 crate::live_children::read_record(&record).unwrap().len(),
13904 1,
13905 "shutdown must still wait for the reaped child until its terminal record exists"
13906 );
13907 runtime.child_roster.close();
13908 gate.resume.notify_one();
13909 assert!(reload.await.is_err());
13910 assert!(crate::live_children::read_record(&record)
13911 .unwrap()
13912 .is_empty());
13913 let history = runtime.terminal_ring.lock().unwrap().snapshot();
13914 assert_eq!(history.entries.len(), 1);
13915 assert_eq!(
13916 history.entries[0].disposition,
13917 TerminalDisposition::DaemonShutdown
13918 );
13919 }
13920
13921 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
13925 async fn every_restart_increment_path_advances_lifetime_count() {
13926 let supervisor = Supervisor::new_for_test(
13927 Arc::new(Registry::default()),
13928 RestartPolicy::new(1, Duration::ZERO),
13929 );
13930 let runtime = supervisor.runtime_config();
13931 let spec = ModuleSpec {
13932 module_id: "lifetime-increment-path".to_string(),
13933 program: PathBuf::from("/unused/lifetime-increment-path"),
13934 args: Vec::new(),
13935 env: Vec::new(),
13936 reserved: false,
13937 reserved_prefixes: Vec::new(),
13938 protocol: ModuleProtocol::Subc,
13939 overlap: Default::default(),
13940 carry_containment: false,
13941 };
13942
13943 let crash_snapshot = Arc::new(Mutex::new(SupervisorSnapshot::starting()));
13944 assert!(matches!(
13945 on_child_exit(
13946 &spec,
13947 runtime.restart_policy,
13948 &supervisor.registry,
13949 &crash_snapshot,
13950 &runtime.terminal_ring,
13951 &runtime.spawn_events,
13952 &runtime.child_roster,
13953 ExitReport {
13954 kind: ExitKind::Crash,
13955 code: Some(1),
13956 signal: None,
13957 at_ms: 1,
13958 },
13959 true,
13960 )
13961 .await,
13962 NextAction::Restart { schedule: _ }
13963 ));
13964 let (crash_restarts, crash_lifetime) = {
13965 let state = lock_snapshot(&crash_snapshot).unwrap();
13966 (state.crash_restarts.len(), state.lifetime_restarts)
13967 };
13968 assert_eq!(crash_restarts, 1);
13969 assert_eq!(crash_lifetime, 1);
13970
13971 let health_snapshot = Arc::new(Mutex::new(SupervisorSnapshot::starting()));
13972 let mut health_child = None;
13973 assert!(matches!(
13974 health_restart_child(
13975 &spec,
13976 &runtime,
13977 &supervisor.registry,
13978 &supervisor.process_liveness,
13979 &health_snapshot,
13980 &mut health_child,
13981 SupervisorHealthStatus::Failing,
13982 None,
13983 2,
13984 )
13985 .await,
13986 Ok(())
13987 ));
13988 assert!(health_child.is_none());
13989 assert!(lock_snapshot(&health_snapshot).unwrap().respawn_pending);
13990 assert!(runtime.scheduled_respawn.lock().unwrap().take().is_some());
13991 let (health_restarts, health_lifetime) = {
13992 let state = lock_snapshot(&health_snapshot).unwrap();
13993 (state.crash_restarts.len(), state.lifetime_restarts)
13994 };
13995 assert_eq!(health_restarts, 1);
13996 assert_eq!(health_lifetime, 1);
13997
13998 let reload_snapshot = Arc::new(Mutex::new(SupervisorSnapshot::starting()));
13999 let mut reload_child = None;
14000 assert!(matches!(
14001 handle_reload_spawn_failure(
14002 &spec,
14003 &runtime,
14004 &supervisor.process_liveness,
14005 &reload_snapshot,
14006 &mut reload_child,
14007 "forced reload spawn failure".to_string(),
14008 )
14009 .await,
14010 Err(SuperviseError::ReloadFailed { .. })
14011 ));
14012 let (reload_restarts, reload_lifetime) = {
14013 let state = lock_snapshot(&reload_snapshot).unwrap();
14014 (state.crash_restarts.len(), state.lifetime_restarts)
14015 };
14016 assert_eq!(reload_restarts, 1);
14017 assert_eq!(reload_lifetime, 1);
14018 }
14019
14020 #[tokio::test]
14021 async fn deliberately_severed_live_child_records_lifetime_without_spending_restart_budget() {
14022 let supervisor = Supervisor::new_for_test(
14023 Arc::new(Registry::default()),
14024 RestartPolicy::new(3, Duration::ZERO),
14025 );
14026 let runtime = supervisor.runtime_config();
14027 let spec = ModuleSpec {
14028 module_id: "deliberately-severed".to_string(),
14029 program: PathBuf::from("/unused/deliberately-severed"),
14030 args: Vec::new(),
14031 env: Vec::new(),
14032 reserved: false,
14033 reserved_prefixes: Vec::new(),
14034 protocol: ModuleProtocol::Subc,
14035 overlap: Default::default(),
14036 carry_containment: false,
14037 };
14038 let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::starting()));
14039 let process = ProcessIdentity {
14040 pid: 41,
14041 start_time: 101,
14042 };
14043 record_deliberate_severance(&snapshot, process).unwrap();
14044 let exit_report = apply_deliberate_severance_marker(
14045 &snapshot,
14046 Some(process),
14047 ExitReport {
14048 kind: ExitKind::Crash,
14049 code: Some(1),
14050 signal: None,
14051 at_ms: 1,
14052 },
14053 );
14054 assert_eq!(exit_report.kind, ExitKind::DeliberateSeverance);
14055
14056 assert!(matches!(
14057 on_child_exit(
14058 &spec,
14059 runtime.restart_policy,
14060 &supervisor.registry,
14061 &snapshot,
14062 &runtime.terminal_ring,
14063 &runtime.spawn_events,
14064 &runtime.child_roster,
14065 exit_report,
14066 true,
14067 )
14068 .await,
14069 NextAction::Restart { schedule: _ }
14070 ));
14071 let state = lock_snapshot(&snapshot).unwrap();
14072 assert_eq!(state.lifetime_restarts, 1);
14073 assert_eq!(state.crash_restarts.len(), 0);
14074 }
14075
14076 #[tokio::test]
14077 async fn genuine_crash_spends_restart_budget_and_records_lifetime() {
14078 let supervisor = Supervisor::new_for_test(
14079 Arc::new(Registry::default()),
14080 RestartPolicy::new(3, Duration::ZERO),
14081 );
14082 let runtime = supervisor.runtime_config();
14083 let spec = ModuleSpec {
14084 module_id: "genuine-crash".to_string(),
14085 program: PathBuf::from("/unused/genuine-crash"),
14086 args: Vec::new(),
14087 env: Vec::new(),
14088 reserved: false,
14089 reserved_prefixes: Vec::new(),
14090 protocol: ModuleProtocol::Subc,
14091 overlap: Default::default(),
14092 carry_containment: false,
14093 };
14094 let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::starting()));
14095
14096 assert!(matches!(
14097 on_child_exit(
14098 &spec,
14099 runtime.restart_policy,
14100 &supervisor.registry,
14101 &snapshot,
14102 &runtime.terminal_ring,
14103 &runtime.spawn_events,
14104 &runtime.child_roster,
14105 ExitReport {
14106 kind: ExitKind::Crash,
14107 code: Some(1),
14108 signal: None,
14109 at_ms: 1,
14110 },
14111 true,
14112 )
14113 .await,
14114 NextAction::Restart { schedule: _ }
14115 ));
14116 let state = lock_snapshot(&snapshot).unwrap();
14117 assert_eq!(state.lifetime_restarts, 1);
14118 assert_eq!(state.crash_restarts.len(), 1);
14119 }
14120
14121 fn crash_exit_report(at_ms: u64) -> ExitReport {
14122 ExitReport {
14123 kind: ExitKind::Crash,
14124 code: Some(1),
14125 signal: None,
14126 at_ms,
14127 }
14128 }
14129
14130 fn windowed_crash_spec(module_id: &str) -> ModuleSpec {
14131 ModuleSpec {
14132 module_id: module_id.to_string(),
14133 program: PathBuf::from("/unused").join(module_id),
14134 args: Vec::new(),
14135 env: Vec::new(),
14136 reserved: false,
14137 reserved_prefixes: Vec::new(),
14138 protocol: ModuleProtocol::Subc,
14139 overlap: Default::default(),
14140 carry_containment: false,
14141 }
14142 }
14143
14144 #[tokio::test]
14150 async fn three_crashes_inside_the_window_stop_the_module_and_name_the_window() {
14151 let (logs, _guard) = crate::router::test_log::log_capture(tracing::Level::ERROR);
14152 let supervisor = Supervisor::new_for_test(
14153 Arc::new(Registry::default()),
14154 RestartPolicy::new(2, Duration::ZERO),
14155 );
14156 let runtime = supervisor.runtime_config();
14157 let spec = windowed_crash_spec("crash-loop-in-window");
14158 let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::starting()));
14159
14160 for attempt in 1..=2 {
14161 assert!(
14162 matches!(
14163 on_child_exit(
14164 &spec,
14165 runtime.restart_policy,
14166 &supervisor.registry,
14167 &snapshot,
14168 &runtime.terminal_ring,
14169 &runtime.spawn_events,
14170 &runtime.child_roster,
14171 crash_exit_report(attempt),
14172 true,
14173 )
14174 .await,
14175 NextAction::Restart { schedule: _ }
14176 ),
14177 "crash {attempt} is inside the budget and must respawn"
14178 );
14179 }
14180
14181 assert!(matches!(
14182 on_child_exit(
14183 &spec,
14184 runtime.restart_policy,
14185 &supervisor.registry,
14186 &snapshot,
14187 &runtime.terminal_ring,
14188 &runtime.spawn_events,
14189 &runtime.child_roster,
14190 crash_exit_report(3),
14191 true,
14192 )
14193 .await,
14194 NextAction::Stop { .. }
14195 ));
14196
14197 {
14198 let state = lock_snapshot(&snapshot).unwrap();
14199 assert_eq!(state.state, ModuleState::Failed);
14200 assert_eq!(state.crash_restarts.len(), 2);
14201 assert_eq!(state.lifetime_restarts, 2);
14202 }
14203
14204 let history = runtime
14205 .terminal_ring
14206 .lock()
14207 .expect("terminal ring is not poisoned")
14208 .snapshot();
14209 let last = history
14210 .entries
14211 .last()
14212 .expect("the refused crash is retained");
14213 assert_eq!(last.disposition, TerminalDisposition::Failed);
14214 assert_eq!(
14215 last.disposition_detail.as_deref(),
14216 Some("crash budget exhausted: max_restarts=2 within window_secs=600")
14217 );
14218
14219 let captured = crate::router::test_log::captured_logs(&logs);
14220 assert!(
14221 captured.contains("crash budget exhausted: max_restarts=2 within window_secs=600"),
14222 "the stop must be logged with its window: {captured}"
14223 );
14224 }
14225
14226 #[tokio::test]
14234 async fn a_crash_older_than_the_window_frees_its_slot_for_a_later_crash() {
14235 let supervisor = Supervisor::new_for_test(
14236 Arc::new(Registry::default()),
14237 RestartPolicy::new(2, Duration::ZERO),
14238 );
14239 let runtime = supervisor.runtime_config();
14240 let spec = windowed_crash_spec("crash-across-windows");
14241 let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::starting()));
14242
14243 for attempt in 1..=2 {
14244 assert!(matches!(
14245 on_child_exit(
14246 &spec,
14247 runtime.restart_policy,
14248 &supervisor.registry,
14249 &snapshot,
14250 &runtime.terminal_ring,
14251 &runtime.spawn_events,
14252 &runtime.child_roster,
14253 crash_exit_report(attempt),
14254 true,
14255 )
14256 .await,
14257 NextAction::Restart { schedule: _ }
14258 ));
14259 }
14260
14261 update_snapshot(&snapshot, Some(&spec.module_id), |state| {
14264 age_oldest_crash_restart_out_of_window(state, runtime.restart_policy.window);
14265 })
14266 .unwrap();
14267
14268 assert!(
14269 matches!(
14270 on_child_exit(
14271 &spec,
14272 runtime.restart_policy,
14273 &supervisor.registry,
14274 &snapshot,
14275 &runtime.terminal_ring,
14276 &runtime.spawn_events,
14277 &runtime.child_roster,
14278 crash_exit_report(3),
14279 true,
14280 )
14281 .await,
14282 NextAction::Restart { schedule: _ }
14283 ),
14284 "a crash older than the window must not hold a budget slot"
14285 );
14286
14287 let state = lock_snapshot(&snapshot).unwrap();
14288 assert_eq!(state.state, ModuleState::Restarting);
14289 assert_eq!(
14290 state.crash_restarts.len(),
14291 2,
14292 "the aged instant is dropped and the new one takes its place"
14293 );
14294 assert_eq!(
14295 state.lifetime_restarts, 3,
14296 "the ledger counts every restart, including the ones the window forgot"
14297 );
14298 }
14299
14300 #[tokio::test]
14305 async fn an_operator_restart_clears_the_ring_and_leaves_the_ledger_alone() {
14306 let supervisor = Supervisor::new_for_test(
14307 Arc::new(Registry::default()),
14308 RestartPolicy::new(2, Duration::ZERO),
14309 );
14310 let runtime = supervisor.runtime_config();
14311 let spec = windowed_crash_spec("operator-cleared-budget");
14312 let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::starting()));
14313
14314 for attempt in 1..=2 {
14315 assert!(matches!(
14316 on_child_exit(
14317 &spec,
14318 runtime.restart_policy,
14319 &supervisor.registry,
14320 &snapshot,
14321 &runtime.terminal_ring,
14322 &runtime.spawn_events,
14323 &runtime.child_roster,
14324 crash_exit_report(attempt),
14325 true,
14326 )
14327 .await,
14328 NextAction::Restart { schedule: _ }
14329 ));
14330 }
14331
14332 reset_restart_count(&snapshot, &spec.module_id).unwrap();
14333 {
14334 let state = lock_snapshot(&snapshot).unwrap();
14335 assert!(
14336 state.crash_restarts.is_empty(),
14337 "an operator restart returns the full budget"
14338 );
14339 assert_eq!(
14340 state.lifetime_restarts, 2,
14341 "clearing the budget must not unmake the crashes"
14342 );
14343 }
14344
14345 assert!(
14346 matches!(
14347 on_child_exit(
14348 &spec,
14349 runtime.restart_policy,
14350 &supervisor.registry,
14351 &snapshot,
14352 &runtime.terminal_ring,
14353 &runtime.spawn_events,
14354 &runtime.child_roster,
14355 crash_exit_report(3),
14356 true,
14357 )
14358 .await,
14359 NextAction::Restart { schedule: _ }
14360 ),
14361 "the cleared budget must be spendable again"
14362 );
14363 let state = lock_snapshot(&snapshot).unwrap();
14364 assert_eq!(state.crash_restarts.len(), 1);
14365 assert_eq!(state.lifetime_restarts, 3);
14366 }
14367
14368 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
14369 async fn severance_marker_for_a_dead_child_does_not_label_its_successor() {
14370 let severed = ProcessIdentity {
14371 pid: 41,
14372 start_time: 101,
14373 };
14374 let successor = ProcessIdentity {
14375 pid: 41,
14376 start_time: 202,
14377 };
14378 let module = module_with_recovery_snapshot(ModuleState::Running, true, 0);
14379 update_snapshot(&module.inner.snapshot, Some("recovery-snapshot"), |state| {
14380 state.pid = Some(successor.pid);
14381 state.process_start_time = Some(successor.start_time);
14382 })
14383 .unwrap();
14384 assert!(!module.record_deliberate_severance(severed).unwrap());
14385
14386 let exit_report = apply_deliberate_severance_marker(
14387 &module.inner.snapshot,
14388 Some(successor),
14389 ExitReport {
14390 kind: ExitKind::Crash,
14391 code: Some(1),
14392 signal: None,
14393 at_ms: 1,
14394 },
14395 );
14396
14397 assert_eq!(exit_report.kind, ExitKind::Crash);
14398 }
14399
14400 #[tokio::test]
14401 async fn drain_reap_marks_deliberate_severance_and_records_lifetime_without_budget() {
14402 let registry = Registry::default();
14403 let supervisor = Supervisor::new_for_test(
14404 Arc::new(Registry::default()),
14405 RestartPolicy::new(3, Duration::ZERO),
14406 );
14407 let runtime = supervisor.runtime_config();
14408 let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::starting()));
14409 let spec = ModuleSpec {
14410 module_id: "drain-deliberate-severance".to_string(),
14411 program: fake_aft_stub_path(),
14412 args: Vec::new(),
14413 env: vec![("FAKE_AFT_EXIT_CODE".to_string(), "23".to_string())],
14414 reserved: false,
14415 reserved_prefixes: Vec::new(),
14416 protocol: ModuleProtocol::Subc,
14417 overlap: Default::default(),
14418 carry_containment: false,
14419 };
14420 let mut child = spawn_and_mark_running(&spec, &runtime, &snapshot).unwrap();
14421 let process = ProcessIdentity {
14422 pid: 41,
14423 start_time: 101,
14424 };
14425 child.process_identity = Some(process);
14426 update_snapshot(&snapshot, Some(&spec.module_id), |state| {
14427 state.pid = Some(process.pid);
14428 state.process_start_time = Some(process.start_time);
14429 })
14430 .unwrap();
14431 record_deliberate_severance(&snapshot, process).unwrap();
14432
14433 drain_child_to_state(
14434 &spec.module_id,
14435 spec.protocol,
14436 StopNotice::SentOverConnection,
14439 ®istry,
14440 None,
14441 &snapshot,
14442 &runtime.terminal_ring,
14443 &runtime.spawn_events,
14444 child,
14445 Duration::from_secs(1),
14446 ModuleState::Stopped,
14447 Some(false),
14448 &super::FailureContext::operator(),
14449 )
14450 .await
14451 .unwrap();
14452
14453 let state = lock_snapshot(&snapshot).unwrap();
14454 assert_eq!(
14455 state.last_exit.as_ref().map(|exit| exit.kind),
14456 Some(ExitKind::DeliberateSeverance)
14457 );
14458 assert_eq!(state.lifetime_restarts, 1);
14459 assert_eq!(state.crash_restarts.len(), 0);
14460 drop(state);
14461 let history = runtime.terminal_ring.lock().unwrap().snapshot();
14462 assert_eq!(
14463 history.entries[0].exit_kind,
14464 subc_control::TerminalExitKind::DeliberateSeverance
14465 );
14466 }
14467
14468 #[tokio::test]
14469 async fn ordinary_drain_reap_does_not_record_a_lifetime_restart() {
14470 let registry = Registry::default();
14471 let supervisor = Supervisor::new_for_test(
14472 Arc::new(Registry::default()),
14473 RestartPolicy::new(3, Duration::ZERO),
14474 );
14475 let runtime = supervisor.runtime_config();
14476 let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::starting()));
14477 let spec = ModuleSpec {
14478 module_id: "ordinary-drain".to_string(),
14479 program: fake_aft_stub_path(),
14480 args: Vec::new(),
14481 env: vec![("FAKE_AFT_EXIT_CODE".to_string(), "23".to_string())],
14482 reserved: false,
14483 reserved_prefixes: Vec::new(),
14484 protocol: ModuleProtocol::Subc,
14485 overlap: Default::default(),
14486 carry_containment: false,
14487 };
14488 let child = spawn_and_mark_running(&spec, &runtime, &snapshot).unwrap();
14489
14490 drain_child_to_state(
14491 &spec.module_id,
14492 spec.protocol,
14493 StopNotice::SentOverConnection,
14496 ®istry,
14497 None,
14498 &snapshot,
14499 &runtime.terminal_ring,
14500 &runtime.spawn_events,
14501 child,
14502 Duration::from_secs(1),
14503 ModuleState::Stopped,
14504 Some(false),
14505 &super::FailureContext::operator(),
14506 )
14507 .await
14508 .unwrap();
14509
14510 let state = lock_snapshot(&snapshot).unwrap();
14511 assert_eq!(
14512 state.last_exit.as_ref().map(|exit| exit.kind),
14513 Some(ExitKind::Crash)
14514 );
14515 assert_eq!(state.lifetime_restarts, 0);
14516 assert_eq!(state.crash_restarts.len(), 0);
14517 }
14518
14519 #[test]
14520 fn fatal_connection_teardown_cannot_arm_a_marker_for_a_surviving_process() {
14521 assert!(!include_str!("server.rs")
14527 .contains("router.record_deliberate_connection_severance(ctx.connection_id)"));
14528 }
14529
14530 #[test]
14537 fn drained_after_quiescence_wait_passes_ok_through_and_forces_false_on_err() {
14538 assert!(drained_after_quiescence_wait(&Ok(true)));
14539 assert!(!drained_after_quiescence_wait(&Ok(false)));
14540 assert!(!drained_after_quiescence_wait(&Err(
14541 SuperviseError::StatePoisoned { module_id: None }
14542 )));
14543 }
14544
14545 #[test]
14554 fn wait_error_exit_report_records_a_failed_terminal_with_no_code_or_signal() {
14555 let ring = Arc::new(Mutex::new(TerminalRing::new(
14556 TerminalRingConfig::default(),
14557 0,
14558 )));
14559 record_wait_error_terminal("wait-error", &ring, &super::SpawnEventFeed::default());
14560
14561 let snapshot = ring.lock().unwrap().snapshot();
14562 assert_eq!(snapshot.entries.len(), 1);
14563 let entry = &snapshot.entries[0];
14564 assert_eq!(entry.exit_code, None);
14565 assert_eq!(entry.exit_signal, None);
14566 assert_eq!(entry.disposition, TerminalDisposition::Failed);
14567 }
14568
14569 #[test]
14570 fn wait_error_exit_path_preserves_spawn_event_density() {
14571 let feed = super::SpawnEventFeed::default();
14572 feed.configure_incarnation("wait-error-density".to_string());
14573 feed.emit_spawned("wait-error", 41, 1);
14574 let ring = Arc::new(Mutex::new(TerminalRing::new(
14575 TerminalRingConfig::default(),
14576 0,
14577 )));
14578
14579 record_wait_error_terminal("wait-error", &ring, &feed);
14580 feed.emit_spawned("after-wait-error", 42, 2);
14581
14582 let state = feed.0.lock().unwrap();
14583 let sequences = state
14584 .events
14585 .iter()
14586 .map(|event| event.cursor.seq)
14587 .collect::<Vec<_>>();
14588 assert_eq!(sequences, vec![1, 2, 3]);
14589 assert_eq!(state.events[1].kind, SpawnEventKind::Exited);
14590 assert_eq!(state.events[1].exit_code, None);
14591 assert_eq!(state.events[1].exit_signal, None);
14592 }
14593
14594 #[test]
14598 fn wait_error_exit_report_is_classified_as_a_crash() {
14599 assert_eq!(wait_error_exit_report().kind, ExitKind::Crash);
14600 }
14601}
14602
14603#[cfg(all(test, unix))]
14604mod restart_ordering_tests {
14605 use super::*;
14606 use std::io::Write;
14607 use tokio::io::AsyncReadExt;
14608
14609 #[test]
14610 #[ignore = "child process fixture"]
14611 fn serving_child() {
14612 let Ok(address_file) = std::env::var("RESTART_TEST_ADDRESS_FILE") else {
14613 return;
14614 };
14615 let listener = std::net::TcpListener::bind("127.0.0.1:0").unwrap();
14616 std::fs::write(address_file, listener.local_addr().unwrap().to_string()).unwrap();
14617 for stream in listener.incoming() {
14618 writeln!(stream.unwrap(), "{}", std::process::id()).unwrap();
14619 }
14620 }
14621
14622 async fn restart_before_publication(requests: usize) {
14623 let home = cortexkit_test_support::ScratchDir::new("restart-before-publication");
14624 let address_file = home.path().join("address");
14625 let spec = ModuleSpec {
14626 module_id: "restart-serving-child".into(),
14627 program: std::env::current_exe().unwrap(),
14628 args: vec![
14629 "--exact".into(),
14630 "supervise::restart_ordering_tests::serving_child".into(),
14631 "--ignored".into(),
14632 "--nocapture".into(),
14633 ],
14634 env: vec![(
14635 "RESTART_TEST_ADDRESS_FILE".into(),
14636 address_file.display().to_string(),
14637 )],
14638 reserved: false,
14639 reserved_prefixes: vec![],
14640 protocol: ModuleProtocol::None,
14641 overlap: Default::default(),
14642 carry_containment: false,
14643 };
14644 let mut supervisor =
14645 Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default());
14646 let (spawned_tx, spawned_rx) = oneshot::channel();
14647 let spawned_tx = Mutex::new(Some(spawned_tx));
14648 let (release_tx, release_rx) = std::sync::mpsc::channel();
14649 let release_rx = Mutex::new(release_rx);
14650 supervisor.test_after_first_spawn = AfterFirstSpawnHook(Some(Arc::new(move |_| {
14653 spawned_tx.lock().unwrap().take().unwrap().send(()).unwrap();
14654 release_rx
14655 .lock()
14656 .unwrap()
14657 .recv_timeout(Duration::from_secs(20))
14658 .unwrap();
14659 })));
14660 let registered = supervisor
14661 .register_configured_with_health(
14662 spec.clone(),
14663 true,
14664 HealthConfig::default(),
14665 None,
14666 RestartPolicy::new(3, Duration::ZERO),
14667 DEFAULT_REGISTER_TIMEOUT,
14668 )
14669 .unwrap();
14670 let module = registered.module.clone();
14671 let startup = tokio::task::spawn_blocking(move || {
14672 supervisor.start_registered_configured(registered, None, true)
14673 });
14674 spawned_rx.await.unwrap();
14675 let old_pid = tokio::time::timeout(Duration::from_secs(10), async {
14676 let address = loop {
14677 if let Ok(address) = std::fs::read_to_string(&address_file) {
14678 if !address.is_empty() {
14679 break address;
14680 }
14681 }
14682 tokio::time::sleep(Duration::from_millis(1)).await;
14683 };
14684 let mut stream = tokio::net::TcpStream::connect(address).await.unwrap();
14685 let mut reply = String::new();
14686 stream.read_to_string(&mut reply).await.unwrap();
14687 reply.trim().parse::<u32>().unwrap()
14688 })
14689 .await
14690 .expect("child must serve before generation publication");
14691 assert_eq!(
14692 lock_snapshot(&module.inner.snapshot)
14693 .unwrap()
14694 .spawn_generation,
14695 0
14696 );
14697 let mut replies = Vec::new();
14698 for _ in 0..requests {
14699 let mut restart = Box::pin(module.restart(Some(0)));
14700 std::future::poll_fn(|cx| {
14703 assert!(std::future::Future::poll(restart.as_mut(), cx).is_pending());
14704 std::task::Poll::Ready(())
14705 })
14706 .await;
14707 replies.push(restart);
14708 }
14709 release_tx.send(()).unwrap();
14710 startup.await.unwrap().unwrap();
14711 for reply in replies {
14712 reply.await.unwrap();
14713 }
14714 let replacement = tokio::time::timeout(Duration::from_secs(10), async {
14717 loop {
14718 if lock_snapshot(&module.inner.snapshot)
14719 .unwrap()
14720 .spawn_generation
14721 > 1
14722 {
14723 break;
14724 }
14725 tokio::time::sleep(Duration::from_millis(1)).await;
14726 }
14727 })
14728 .await;
14729 module.update_spec_for_test(spec).await.unwrap();
14733 let (generation, pid) = {
14734 let state = lock_snapshot(&module.inner.snapshot).unwrap();
14735 (state.spawn_generation, state.pid)
14736 };
14737 module.stop().await.unwrap();
14738 replacement.expect("restart must publish a replacement for the already-serving child");
14739 assert_ne!(
14740 pid,
14741 Some(old_pid),
14742 "a serving child predating the restart must be replaced"
14743 );
14744 assert_eq!(generation, 2, "queued restarts must share one replacement");
14745 }
14746
14747 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
14748 async fn restart_received_after_serving_before_generation_publication_replaces_child() {
14749 restart_before_publication(1).await;
14750 }
14751
14752 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
14753 async fn restart_queued_behind_restart_does_not_kill_its_replacement() {
14754 restart_before_publication(2).await;
14755 }
14756
14757 #[tokio::test]
14758 async fn restart_queued_before_replacement_spawn_keeps_the_new_child() {
14759 let supervisor =
14760 Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default());
14761 let registered = supervisor
14762 .register_configured_with_health(
14763 ModuleSpec {
14764 module_id: "queued-before-replacement".into(),
14765 program: "/bin/sleep".into(),
14770 args: vec!["600".into()],
14771 env: vec![],
14772 reserved: false,
14773 reserved_prefixes: vec![],
14774 protocol: ModuleProtocol::None,
14775 overlap: Default::default(),
14776 carry_containment: false,
14777 },
14778 true,
14779 HealthConfig::default(),
14780 None,
14781 RestartPolicy::default(),
14782 DEFAULT_REGISTER_TIMEOUT,
14783 )
14784 .unwrap();
14785 let RegisteredConfiguredModule {
14786 mut spec,
14787 mut runtime,
14788 module,
14789 mut commands,
14790 } = registered;
14791 let snapshot = &module.inner.snapshot;
14792 let mut restart = Box::pin(module.restart(Some(0)));
14793 std::future::poll_fn(|cx| {
14794 assert!(std::future::Future::poll(restart.as_mut(), cx).is_pending());
14795 std::task::Poll::Ready(())
14796 })
14797 .await;
14798 let mut child = Some(spawn_and_mark_running(&spec, &runtime, snapshot).unwrap());
14801 let replacement_pid = child.as_ref().unwrap().id();
14802 let command = commands.recv().await.unwrap();
14803 assert!(
14804 handle_supervisor_command(
14805 command,
14806 &mut spec,
14807 &mut runtime,
14808 &supervisor.registry,
14809 &supervisor.process_liveness,
14810 snapshot,
14811 &mut child,
14812 &mut commands,
14813 &mut VecDeque::new(),
14814 )
14815 .await
14816 );
14817 restart.await.unwrap();
14818 let kept_pid = child.as_ref().and_then(SupervisedChild::id);
14819 let pending = lock_snapshot(snapshot).unwrap().respawn_pending;
14820 if let Some(mut child) = child {
14821 child.start_kill().unwrap();
14822 child.wait().await.unwrap();
14823 }
14824 assert_eq!(
14825 kept_pid, replacement_pid,
14826 "a newer replacement must survive"
14827 );
14828 assert!(
14829 !pending,
14830 "a satisfied restart must not schedule another spawn"
14831 );
14832 }
14833}
14834
14835#[cfg(all(test, unix))]
14836mod health_event_tests {
14837 use super::*;
14838 use cortexkit_test_support::ScratchDir;
14839 use subc_protocol::{manifest::Concurrency, session::ModuleControlResponse};
14840
14841 struct Actor {
14842 module: SupervisedModule,
14843 registry: Arc<Registry>,
14844 forwarding: Arc<ForwardingTable>,
14845 spec: ModuleSpec,
14846 health: HealthConfig,
14847 rx: mpsc::Receiver<crate::router::OutboundFrame>,
14848 roster: ChildRoster,
14849 _home: ScratchDir,
14850 }
14851
14852 async fn wait_for<T>(reason: &str, mut observe: impl FnMut() -> Option<T>) -> T {
14853 let deadline = std::time::Instant::now() + Duration::from_secs(10);
14857 let tick = Instant::now();
14858 loop {
14859 if let Some(value) = observe() {
14860 assert_eq!(Instant::now(), tick, "{reason} must not wait for a timer");
14861 return value;
14862 }
14863 assert!(
14864 std::time::Instant::now() < deadline,
14865 "timed out waiting for {reason}"
14866 );
14867 tokio::task::yield_now().await;
14868 }
14869 }
14870
14871 impl Actor {
14872 async fn start(protocol: ModuleProtocol, cadence: Duration) -> Self {
14873 Self::start_with_policy(
14874 protocol,
14875 cadence,
14876 DEFAULT_REGISTER_TIMEOUT,
14877 RestartPolicy::new(3, Duration::ZERO),
14878 )
14879 .await
14880 }
14881
14882 async fn start_with_policy(
14883 protocol: ModuleProtocol,
14884 cadence: Duration,
14885 register_timeout: Duration,
14886 policy: RestartPolicy,
14887 ) -> Self {
14888 let home = ScratchDir::new("health-events");
14889 let registry = Arc::new(Registry::default());
14890 let forwarding = Arc::new(ForwardingTable::default());
14891 let health = HealthConfig {
14892 cadence,
14893 deadline: Duration::from_secs(3600),
14894 ..HealthConfig::default()
14895 };
14896 let supervisor = Supervisor::new_for_test(registry.clone(), policy)
14897 .with_forwarding(forwarding.clone())
14898 .with_health_config(health.clone());
14899 let spec = ModuleSpec {
14900 module_id: "event-health-child".into(),
14901 program: PathBuf::from("/bin/sleep"),
14909 args: vec!["600".into()],
14912 env: ["XDG_DATA_HOME", "XDG_RUNTIME_DIR", "XDG_CONFIG_HOME"]
14913 .into_iter()
14914 .map(|key| (key.into(), home.path().display().to_string()))
14915 .collect(),
14916 reserved: false,
14917 reserved_prefixes: vec![],
14918 protocol,
14919 overlap: Default::default(),
14920 carry_containment: false,
14921 };
14922 let module = supervisor
14923 .supervise_configured_requested(
14924 spec.clone(),
14925 true,
14926 health.clone(),
14927 Some(0),
14928 policy,
14929 register_timeout,
14930 None,
14931 )
14932 .unwrap();
14933 let (_, rx) = mpsc::channel(8);
14934 let actor = Self {
14935 module,
14936 registry,
14937 forwarding,
14938 spec,
14939 health,
14940 rx,
14941 roster: supervisor.child_roster.clone(),
14942 _home: home,
14943 };
14944 actor
14945 .wait_for_select(|checkpoint| checkpoint.generation > 0)
14946 .await;
14947 actor
14948 }
14949
14950 async fn expire_registration_wait(&self, remaining: Duration) {
14951 tokio::time::advance(remaining).await;
14952 wait_for("registration timeout TERM grace", || {
14953 lock_snapshot(&self.module.inner.snapshot)
14954 .unwrap()
14955 .actor_select
14956 .is_none()
14957 .then_some(())
14958 })
14959 .await;
14960 tokio::time::advance(crate::child_roster::TERM_TO_KILL).await;
14961 }
14962
14963 fn turns(&self) -> u64 {
14964 lock_snapshot(&self.module.inner.snapshot)
14965 .unwrap()
14966 .actor_turns
14967 }
14968
14969 async fn wait_for_select(
14970 &self,
14971 expected: impl Fn(&ActorSelectCheckpoint) -> bool,
14972 ) -> ActorSelectCheckpoint {
14973 wait_for("actor select checkpoint", || {
14974 let state = lock_snapshot(&self.module.inner.snapshot).unwrap();
14975 state.actor_select.filter(|checkpoint| {
14976 checkpoint.generation == state.spawn_generation
14977 && state.state == ModuleState::Running
14978 && state.process_alive
14979 && state.reported_pid().is_some()
14980 && expected(checkpoint)
14981 })
14982 })
14983 .await
14984 }
14985
14986 fn hello(&mut self, connection: u64, health: bool) {
14987 let manifest =
14988 subc_protocol::manifest::ModuleManifest::builder(&self.spec.module_id, "1.0.0")
14989 .protocol_ver(subc_protocol::PROTOCOL_VERSION)
14990 .build();
14991 let connection = ConnectionId::new(connection);
14992 let (tx, rx) = mpsc::channel(8);
14993 self.rx = rx;
14994 self.forwarding
14995 .register_module_connection(
14996 connection,
14997 self.spec.module_id.clone(),
14998 subc_protocol::PROTOCOL_VERSION,
14999 Concurrency::ModuleManaged,
15000 FrameSink::new(tx),
15001 )
15002 .unwrap();
15003 self.registry
15004 .register_with_control_ops(
15005 manifest,
15006 subc_protocol::PROTOCOL_VERSION,
15007 connection,
15008 if health {
15009 vec![MODULE_CONTROL_OP_HEALTH_CHECK.into()]
15010 } else {
15011 vec![]
15012 },
15013 )
15014 .unwrap();
15015 }
15016
15017 async fn probe(&mut self) -> crate::router::OutboundFrame {
15018 let frame = wait_for("outbound health probe", || match self.rx.try_recv() {
15019 Ok(frame) => Some(frame),
15020 Err(mpsc::error::TryRecvError::Empty) => None,
15021 Err(mpsc::error::TryRecvError::Disconnected) => panic!("health peer disconnected"),
15022 })
15023 .await;
15024 let body: Value = serde_json::from_slice(&frame.body).unwrap();
15025 assert_eq!(body["op"], MODULE_CONTROL_OP_HEALTH_CHECK);
15026 frame
15027 }
15028
15029 fn no_probe(&mut self) {
15030 assert!(matches!(
15031 self.rx.try_recv(),
15032 Err(mpsc::error::TryRecvError::Empty)
15033 ));
15034 }
15035
15036 fn answer(&self, connection: u64, frame: crate::router::OutboundFrame) {
15037 self.forwarding
15038 .complete_module_control_rpc(
15039 ConnectionId::new(connection),
15040 frame.header.corr,
15041 Some(MODULE_CONTROL_OP_HEALTH_CHECK),
15042 ModuleControlRpcOutcome::Response(ModuleControlResponse::HealthCheck {
15043 status: HealthStatus::Ok,
15044 detail: None,
15045 metrics: None,
15046 }),
15047 )
15048 .unwrap();
15049 }
15050 }
15051
15052 #[tokio::test(start_paused = true)]
15053 async fn registration_timeout_kills_at_120_seconds_and_respawns() {
15054 let actor = Actor::start(ModuleProtocol::Subc, Duration::from_secs(30)).await;
15055 let initial = actor.module.status().unwrap();
15056 tokio::time::advance(Duration::from_secs(119)).await;
15057 tokio::task::yield_now().await;
15058 assert_eq!(actor.module.status().unwrap().pid, initial.pid);
15059 assert!(actor.module.terminal_history().entries.is_empty());
15060 actor.expire_registration_wait(Duration::from_secs(1)).await;
15061 actor
15062 .wait_for_select(|at| at.generation > initial.spawn_generation)
15063 .await;
15064 let status = actor.module.status().unwrap();
15065 assert_ne!(status.pid, initial.pid);
15066 assert!(!cortexkit_test_support::process_alive(
15067 initial.pid.unwrap() as i32
15068 ));
15069 assert_eq!(status.restart_count, 1);
15070 assert_eq!(status.last_exit.unwrap().kind, ExitKind::Crash);
15071 let entries = actor.module.terminal_history().entries;
15072 assert_eq!(entries.len(), 1);
15073 assert_eq!(
15074 entries[0].disposition_detail.as_deref(),
15075 Some("registration_timeout")
15076 );
15077 actor.module.stop().await.unwrap();
15078 }
15079
15080 #[tokio::test(start_paused = true)]
15081 async fn registration_timeout_is_cancelled_by_hello_at_119_seconds() {
15082 let mut actor = Actor::start(ModuleProtocol::Subc, Duration::from_secs(30)).await;
15083 let pid = actor.module.status().unwrap().pid;
15084 tokio::time::advance(Duration::from_secs(119)).await;
15085 actor.hello(1, false);
15086 let registered = actor
15087 .wait_for_select(|at| at.registered_connection.is_some())
15088 .await;
15089 assert_eq!(
15090 registered.wake_after, None,
15091 "HELLO must cancel the deadline"
15092 );
15093 tokio::time::advance(Duration::from_secs(300)).await;
15094 tokio::task::yield_now().await;
15095 assert_eq!(actor.module.status().unwrap().pid, pid);
15096 assert!(actor.module.terminal_history().entries.is_empty());
15097 actor
15099 .registry
15100 .deregister_connection(ConnectionId::new(1))
15101 .unwrap();
15102 actor
15103 .wait_for_select(|at| at.registered_connection.is_none())
15104 .await;
15105 tokio::time::advance(Duration::from_secs(300)).await;
15106 tokio::task::yield_now().await;
15107 assert_eq!(actor.module.status().unwrap().pid, pid);
15108 assert!(actor.module.terminal_history().entries.is_empty());
15109 actor.module.stop().await.unwrap();
15110
15111 let mut transient = Actor::start(ModuleProtocol::Subc, Duration::from_secs(30)).await;
15112 let pid = transient.module.status().unwrap().pid;
15113 tokio::time::advance(Duration::from_secs(119)).await;
15114 let turns = transient.turns();
15115 transient.hello(1, false);
15118 transient
15119 .registry
15120 .deregister_connection(ConnectionId::new(1))
15121 .unwrap();
15122 let parked = transient.wait_for_select(|at| at.turn > turns).await;
15123 assert_eq!(
15124 parked.wake_after, None,
15125 "transient HELLO must cancel the deadline"
15126 );
15127 tokio::time::advance(Duration::from_secs(300)).await;
15128 tokio::task::yield_now().await;
15129 assert_eq!(transient.module.status().unwrap().pid, pid);
15130 assert!(transient.module.terminal_history().entries.is_empty());
15131 transient.module.stop().await.unwrap();
15132 }
15133
15134 #[tokio::test(start_paused = true)]
15135 async fn registration_timeouts_exhaust_budget_and_schedule_slow_retry() {
15136 let actor = Actor::start_with_policy(
15137 ModuleProtocol::Subc,
15138 Duration::from_secs(30),
15139 Duration::from_secs(10),
15140 RestartPolicy::new(1, Duration::ZERO),
15141 )
15142 .await;
15143 let generation = actor.module.status().unwrap().spawn_generation;
15144 actor
15145 .expire_registration_wait(Duration::from_secs(10))
15146 .await;
15147 actor.wait_for_select(|at| at.generation > generation).await;
15148 actor
15149 .expire_registration_wait(Duration::from_secs(10))
15150 .await;
15151 let retry = wait_for("failed retry schedule", || {
15152 let state = lock_snapshot(&actor.module.inner.snapshot).unwrap();
15153 (state.state == ModuleState::Failed)
15154 .then_some(state.failed_retry.pending)
15155 .flatten()
15156 })
15157 .await;
15158 assert_eq!(retry.delay, Duration::from_secs(600));
15159 assert_eq!(retry.attempt, 1);
15160 let entries = actor.module.terminal_history().entries;
15161 assert_eq!(entries.len(), 2);
15162 assert_eq!(entries[1].disposition, TerminalDisposition::Failed);
15163 assert!(entries[1]
15164 .disposition_detail
15165 .as_deref()
15166 .unwrap()
15167 .contains("registration_timeout"));
15168 tokio::time::advance(Duration::from_secs(599)).await;
15169 tokio::task::yield_now().await;
15170 assert_eq!(actor.module.state().unwrap(), ModuleState::Failed);
15171 tokio::time::advance(Duration::from_secs(1)).await;
15172 actor
15173 .wait_for_select(|at| at.generation > generation + 1)
15174 .await;
15175 actor
15177 .expire_registration_wait(Duration::from_secs(10))
15178 .await;
15179 wait_for("slow retry registration timeout", || {
15180 (actor.module.terminal_history().entries.len() == 3).then_some(())
15181 })
15182 .await;
15183 actor.module.stop().await.unwrap();
15184 }
15185
15186 #[tokio::test(start_paused = true)]
15187 async fn registration_timeout_honours_ten_second_override() {
15188 let actor = Actor::start_with_policy(
15189 ModuleProtocol::Subc,
15190 Duration::from_secs(30),
15191 Duration::from_secs(10),
15192 RestartPolicy::new(3, Duration::ZERO),
15193 )
15194 .await;
15195 let generation = actor.module.status().unwrap().spawn_generation;
15196 tokio::time::advance(Duration::from_secs(9)).await;
15197 tokio::task::yield_now().await;
15198 assert!(actor.module.terminal_history().entries.is_empty());
15199 actor.expire_registration_wait(Duration::from_secs(1)).await;
15200 actor.wait_for_select(|at| at.generation > generation).await;
15201 assert_eq!(actor.module.terminal_history().entries.len(), 1);
15202 actor.module.stop().await.unwrap();
15203 }
15204
15205 #[tokio::test]
15206 async fn registration_timeout_kills_term_ignoring_helper_before_reaping_leader() {
15207 use std::os::unix::process::{CommandExt, ExitStatusExt};
15208
15209 struct Helper(std::process::Child);
15210 impl Drop for Helper {
15211 fn drop(&mut self) {
15212 if matches!(self.0.try_wait(), Ok(None)) {
15215 let _ = self.0.kill();
15216 let _ = self.0.wait();
15217 }
15218 }
15219 }
15220
15221 let home = ScratchDir::new("registration-timeout-helper");
15222 let supervisor = Supervisor::new_for_test(
15223 Arc::new(Registry::default()),
15224 RestartPolicy::new(0, Duration::ZERO),
15225 );
15226 let spec = ModuleSpec {
15227 module_id: "registration-timeout-group".into(),
15228 program: "/bin/sleep".into(),
15229 args: vec!["120".into()],
15230 env: ["XDG_DATA_HOME", "XDG_RUNTIME_DIR", "XDG_CONFIG_HOME"]
15231 .into_iter()
15232 .map(|key| (key.into(), home.path().display().to_string()))
15233 .collect(),
15234 reserved: false,
15235 reserved_prefixes: vec![],
15236 protocol: ModuleProtocol::Subc,
15237 overlap: Default::default(),
15238 carry_containment: false,
15239 };
15240 let module = supervisor
15241 .supervise_configured_requested(
15242 spec,
15243 true,
15244 HealthConfig::default(),
15245 None,
15246 RestartPolicy::new(0, Duration::ZERO),
15247 Duration::from_secs(10),
15248 None,
15249 )
15250 .unwrap();
15251 let leader = timeout(Duration::from_secs(5), async {
15252 loop {
15253 if let Some(pid) = module.status().unwrap().pid {
15254 break pid;
15255 }
15256 sleep(Duration::from_millis(5)).await;
15257 }
15258 })
15259 .await
15260 .expect("supervised group leader must be reported before the deadline");
15261 assert!(leader > 1);
15262 let ready = home.join("helper.ready");
15263 let mut helper = Helper(
15266 std::process::Command::new("/bin/sh")
15267 .args([
15268 "-c",
15269 "trap '' TERM; : > \"$1\"; exec /bin/sleep 120",
15270 "helper",
15271 ])
15272 .arg(&ready)
15273 .process_group(i32::try_from(leader).unwrap())
15274 .stdout(std::process::Stdio::null())
15275 .stderr(std::process::Stdio::null())
15276 .spawn()
15277 .unwrap(),
15278 );
15279 timeout(Duration::from_secs(5), async {
15280 while !ready.exists() {
15281 assert!(helper.0.try_wait().unwrap().is_none());
15282 sleep(Duration::from_millis(5)).await;
15283 }
15284 })
15285 .await
15286 .expect("helper must install TERM ignore before the registration deadline");
15287 let helper_pid =
15288 rustix::process::Pid::from_raw(i32::try_from(helper.0.id()).unwrap()).unwrap();
15289 rustix::process::kill_process(helper_pid, rustix::process::Signal::TERM).unwrap();
15290 sleep(Duration::from_millis(20)).await;
15291 assert!(
15292 helper.0.try_wait().unwrap().is_none(),
15293 "helper must ignore TERM"
15294 );
15295
15296 timeout(Duration::from_secs(20), async {
15297 while module.state().unwrap() != ModuleState::Failed {
15298 sleep(Duration::from_millis(5)).await;
15299 }
15300 })
15301 .await
15302 .expect("registration timeout must stop and reap the group leader");
15303 let helper_exit = timeout(Duration::from_secs(3), async {
15304 loop {
15305 if let Some(status) = helper.0.try_wait().unwrap() {
15306 break status;
15307 }
15308 sleep(Duration::from_millis(5)).await;
15309 }
15310 })
15311 .await
15312 .expect("TERM-ignoring helper survived the registration timeout's group KILL");
15313 assert_eq!(helper_exit.signal(), Some(9));
15314 assert!(!cortexkit_test_support::process_alive(
15315 i32::try_from(leader).unwrap()
15316 ));
15317 assert_eq!(
15318 module.status().unwrap().last_exit.unwrap().kind,
15319 ExitKind::Crash
15320 );
15321 assert!(module.terminal_history().entries[0]
15322 .disposition_detail
15323 .as_deref()
15324 .unwrap()
15325 .contains("registration_timeout"));
15326 module.stop().await.unwrap();
15327 }
15328
15329 #[tokio::test(start_paused = true)]
15330 async fn registration_timeout_does_not_count_an_earlier_exit() {
15331 let actor = Actor::start_with_policy(
15332 ModuleProtocol::Subc,
15333 Duration::from_secs(30),
15334 DEFAULT_REGISTER_TIMEOUT,
15335 RestartPolicy::new(0, Duration::ZERO),
15336 )
15337 .await;
15338 let pid = actor.module.status().unwrap().pid.unwrap();
15339 rustix::process::kill_process(
15340 rustix::process::Pid::from_raw(pid as i32).unwrap(),
15341 rustix::process::Signal::TERM,
15342 )
15343 .unwrap();
15344 wait_for("early exit", || {
15345 (actor.module.state().unwrap() == ModuleState::Failed).then_some(())
15346 })
15347 .await;
15348 tokio::time::advance(Duration::from_secs(121)).await;
15349 tokio::task::yield_now().await;
15350 let entries = actor.module.terminal_history().entries;
15351 assert_eq!(entries.len(), 1);
15352 assert!(!entries[0]
15353 .disposition_detail
15354 .as_deref()
15355 .unwrap()
15356 .contains("registration_timeout"));
15357 assert_eq!(actor.module.status().unwrap().lifetime_restarts, 0);
15358 actor.module.stop().await.unwrap();
15359 }
15360
15361 #[tokio::test(start_paused = true)]
15362 async fn registration_timeout_is_cancelled_by_daemon_shutdown() {
15363 let actor = Actor::start(ModuleProtocol::Subc, Duration::from_secs(30)).await;
15364 let pid = actor.module.status().unwrap().pid;
15365 tokio::time::advance(Duration::from_secs(119)).await;
15366 actor.roster.close();
15367 let turns = actor.turns();
15368 tokio::time::advance(Duration::from_secs(300)).await;
15369 let parked = actor.wait_for_select(|at| at.turn > turns).await;
15370 assert_eq!(parked.wake_after, None);
15371 assert_eq!(actor.module.status().unwrap().pid, pid);
15372 assert!(actor.module.terminal_history().entries.is_empty());
15373 actor.module.stop().await.unwrap();
15374 let status = actor.module.status().unwrap();
15375 assert_eq!(status.restart_count, 0);
15376 assert_eq!(
15377 actor.module.terminal_history().entries[0].disposition,
15378 TerminalDisposition::DaemonShutdown
15379 );
15380 }
15381
15382 #[tokio::test(start_paused = true)]
15383 async fn no_health_children_park_for_sixty_seconds() {
15384 let none = Actor::start(ModuleProtocol::None, Duration::from_secs(30)).await;
15385 let unregistered = Actor::start(ModuleProtocol::Subc, Duration::from_secs(30)).await;
15386 let mut unadvertised = Actor::start(ModuleProtocol::Subc, Duration::from_secs(30)).await;
15387 unadvertised.hello(1, false);
15388 for _ in 0..6000 {
15393 tokio::time::advance(Duration::from_millis(10)).await;
15394 tokio::task::yield_now().await;
15395 }
15396 for actor in [&none, &unregistered, &unadvertised] {
15397 assert!(
15398 actor.turns() <= 5,
15399 "no-health actor woke {} times",
15400 actor.turns()
15401 );
15402 assert_eq!(actor.module.state().unwrap(), ModuleState::Running);
15403 }
15404 }
15405
15406 #[tokio::test(start_paused = true)]
15407 async fn late_hello_starts_a_due_probe_in_the_notification_tick() {
15408 let mut actor = Actor::start(ModuleProtocol::Subc, Duration::ZERO).await;
15412 let tick = Instant::now();
15413 actor.hello(1, true);
15414 actor.probe().await;
15415 assert_eq!(Instant::now(), tick, "HELLO must not wait for a poll");
15416 }
15417
15418 #[tokio::test(start_paused = true)]
15419 async fn hello_keeps_the_jittered_cadence_and_catalog_updates_do_not_reset_it() {
15420 let mut actor = Actor::start(ModuleProtocol::Subc, Duration::from_secs(30)).await;
15421 let hello_tick = Instant::now();
15422 actor.hello(1, true);
15423 let armed = actor
15424 .wait_for_select(|checkpoint| {
15425 checkpoint.registered_connection == Some(ConnectionId::new(1))
15426 && checkpoint.next_probe_at.is_some()
15427 })
15428 .await;
15429 let delay = jittered_health_delay(&actor.spec.module_id, 0, actor.health.cadence);
15430 assert!((Duration::from_secs(30)..Duration::from_secs(33)).contains(&delay));
15431 let first_deadline = hello_tick + delay;
15432 assert_eq!(armed.next_probe_at, Some(first_deadline));
15433 tokio::time::advance(Duration::from_secs(29)).await;
15434 let turns = actor.turns();
15435 actor
15436 .registry
15437 .replace_catalog_for_connection(ConnectionId::new(1), vec![], None, Some(true))
15438 .unwrap();
15439 let updated = actor
15440 .wait_for_select(|checkpoint| checkpoint.turn > turns)
15441 .await;
15442 assert_eq!(
15443 updated.next_probe_at,
15444 Some(first_deadline),
15445 "catalog update must not reset cadence"
15446 );
15447 actor.no_probe();
15448 tokio::time::advance(first_deadline - Instant::now()).await;
15449 let frame = actor.probe().await;
15450 actor.answer(1, frame);
15451 let next = jittered_health_delay(&actor.spec.module_id, 1, actor.health.cadence);
15452 let next_deadline = Instant::now() + next;
15453 let rearmed = actor
15454 .wait_for_select(|checkpoint| {
15455 checkpoint
15456 .next_probe_at
15457 .is_some_and(|deadline| deadline > first_deadline)
15458 })
15459 .await;
15460 assert_eq!(rearmed.next_probe_at, Some(next_deadline));
15461 tokio::time::advance(next - Duration::from_nanos(1)).await;
15462 actor.no_probe();
15463 tokio::time::advance(Duration::from_nanos(1)).await;
15464 actor.probe().await;
15465 }
15466
15467 #[tokio::test(start_paused = true)]
15468 async fn disconnect_disarms_and_reconnect_rearms_health() {
15469 let mut actor = Actor::start(ModuleProtocol::Subc, Duration::from_secs(30)).await;
15470 actor.hello(1, true);
15471 actor
15472 .wait_for_select(|checkpoint| {
15473 checkpoint.registered_connection == Some(ConnectionId::new(1))
15474 && checkpoint.next_probe_at.is_some()
15475 })
15476 .await;
15477 let turns = actor.turns();
15478 let tick = Instant::now();
15479 actor
15480 .registry
15481 .deregister_connection(ConnectionId::new(1))
15482 .unwrap();
15483 let disarmed = actor
15484 .wait_for_select(|checkpoint| {
15485 checkpoint.turn > turns && checkpoint.registered_connection.is_none()
15486 })
15487 .await;
15488 assert_eq!(disarmed.next_probe_at, None);
15489 assert_eq!(disarmed.wake_after, None);
15490 assert_eq!(
15491 Instant::now(),
15492 tick,
15493 "disconnect must disarm in the notification tick"
15494 );
15495 tokio::time::advance(Duration::from_secs(60)).await;
15496 actor.no_probe();
15497 let reconnect_tick = Instant::now();
15498 actor.hello(2, true);
15499 let rearmed = actor
15500 .wait_for_select(|checkpoint| {
15501 checkpoint.registered_connection == Some(ConnectionId::new(2))
15502 && checkpoint.next_probe_at.is_some()
15503 })
15504 .await;
15505 let delay = jittered_health_delay(&actor.spec.module_id, 0, actor.health.cadence);
15506 assert_eq!(rearmed.next_probe_at, Some(reconnect_tick + delay));
15507 tokio::time::advance(delay).await;
15508 actor.probe().await;
15509 }
15510
15511 #[tokio::test(start_paused = true)]
15512 async fn reregistration_without_health_disarms_the_old_schedule() {
15513 let mut actor = Actor::start(ModuleProtocol::Subc, Duration::from_secs(30)).await;
15514 actor.hello(1, true);
15515 actor
15516 .wait_for_select(|checkpoint| {
15517 checkpoint.registered_connection == Some(ConnectionId::new(1))
15518 && checkpoint.next_probe_at.is_some()
15519 })
15520 .await;
15521 actor
15522 .registry
15523 .deregister_connection(ConnectionId::new(1))
15524 .unwrap();
15525 actor.hello(2, false);
15526 let disarmed = actor
15527 .wait_for_select(|checkpoint| {
15528 checkpoint.registered_connection == Some(ConnectionId::new(2))
15529 })
15530 .await;
15531 assert_eq!(disarmed.next_probe_at, None);
15532 assert_eq!(disarmed.wake_after, None);
15533 tokio::time::advance(Duration::from_secs(60)).await;
15534 actor.no_probe();
15535 }
15536
15537 #[tokio::test(start_paused = true)]
15538 async fn restart_waits_for_its_own_hello_before_rearming_health() {
15539 let mut actor = Actor::start(ModuleProtocol::Subc, Duration::from_secs(30)).await;
15540 actor.hello(1, true);
15541 let armed = actor
15542 .wait_for_select(|checkpoint| {
15543 checkpoint.registered_connection == Some(ConnectionId::new(1))
15544 && checkpoint.next_probe_at.is_some()
15545 })
15546 .await;
15547 actor
15551 .registry
15552 .deregister_connection(ConnectionId::new(1))
15553 .unwrap();
15554 actor.module.restart(Some(0)).await.unwrap();
15555 let parked = actor
15556 .wait_for_select(|checkpoint| {
15557 checkpoint.generation > armed.generation
15558 && checkpoint.registered_connection.is_none()
15559 && checkpoint.next_probe_at.is_none()
15560 })
15561 .await;
15562 assert_eq!(parked.wake_after, Some(DEFAULT_REGISTER_TIMEOUT));
15563 actor.rx.close();
15564 while actor.rx.try_recv().is_ok() {}
15567 let turns = actor.turns();
15568 tokio::time::advance(Duration::from_secs(60)).await;
15569 let after = actor.turns();
15570 eprintln!(
15571 "restart park turns: before={turns} after={after} delta={}",
15572 after - turns
15573 );
15574 assert_eq!(after, turns, "replacement must park before HELLO");
15575 let hello_tick = Instant::now();
15576 actor.hello(2, true);
15577 let rearmed = actor
15578 .wait_for_select(|checkpoint| {
15579 checkpoint.registered_connection == Some(ConnectionId::new(2))
15580 && checkpoint.next_probe_at.is_some()
15581 })
15582 .await;
15583 let delay = jittered_health_delay(&actor.spec.module_id, 0, actor.health.cadence);
15584 assert_eq!(rearmed.next_probe_at, Some(hello_tick + delay));
15585 tokio::time::advance(delay).await;
15586 actor.probe().await;
15587 }
15588
15589 #[tokio::test(start_paused = true)]
15590 async fn http_probe_waits_a_full_cadence_for_each_spawn() {
15591 use tokio::{
15592 io::{AsyncReadExt, AsyncWriteExt},
15593 net::TcpListener,
15594 };
15595 let mut actor = Actor::start(ModuleProtocol::None, Duration::from_secs(30)).await;
15596 let listener = TcpListener::bind("127.0.0.1:0").await.unwrap();
15597 actor.health.http = Some(format!("http://{}/healthz", listener.local_addr().unwrap()));
15598 let (started_tx, mut started_rx) = mpsc::channel(8);
15599 let peer = tokio::spawn(async move {
15600 loop {
15601 let (mut socket, _) = listener.accept().await.unwrap();
15602 let mut request = [0; 1024];
15603 assert!(socket.read(&mut request).await.unwrap() > 0);
15604 started_tx.send(Instant::now()).await.unwrap();
15605 socket
15606 .write_all(b"HTTP/1.1 200 OK\r\nContent-Length: 0\r\n\r\n")
15607 .await
15608 .unwrap();
15609 }
15610 });
15611 actor
15612 .module
15613 .update_configuration(actor.spec.clone(), actor.health.clone(), None)
15614 .await
15615 .unwrap();
15616 let first_tick = Instant::now();
15617 let first = actor
15618 .wait_for_select(|checkpoint| checkpoint.next_probe_at.is_some())
15619 .await;
15620 assert!(first.next_probe_at.unwrap() >= first_tick + actor.health.cadence);
15621 assert!(started_rx.try_recv().is_err());
15622
15623 tokio::time::advance(
15626 first.next_probe_at.unwrap() - Instant::now() - Duration::from_secs(1),
15627 )
15628 .await;
15629 actor.module.restart(Some(0)).await.unwrap();
15630 let spawn_tick = Instant::now();
15631 let replacement = actor
15632 .wait_for_select(|checkpoint| checkpoint.generation > first.generation)
15633 .await;
15634 let deadline = replacement.next_probe_at.unwrap();
15635 assert!(
15636 deadline >= spawn_tick + actor.health.cadence,
15637 "a replacement must not inherit its predecessor's probe deadline"
15638 );
15639 tokio::time::advance(actor.health.cadence - Duration::from_nanos(1)).await;
15640 assert!(
15641 started_rx.try_recv().is_err(),
15642 "no HTTP probe during startup grace"
15643 );
15644 tokio::time::advance(deadline - Instant::now()).await;
15645 let landed = wait_for("first replacement HTTP probe", || {
15646 started_rx.try_recv().ok()
15647 })
15648 .await;
15649 assert!(landed >= spawn_tick + actor.health.cadence);
15650 wait_for("healthy replacement", || {
15651 (actor.module.status().unwrap().health.status == SupervisorHealthStatus::Ok)
15652 .then_some(())
15653 })
15654 .await;
15655 peer.abort();
15656 }
15657
15658 #[tokio::test(start_paused = true)]
15659 async fn rescan_adds_and_removes_http_health_without_polling() {
15660 use tokio::{
15661 io::{AsyncReadExt, AsyncWriteExt},
15662 net::TcpListener,
15663 };
15664 let mut actor = Actor::start(ModuleProtocol::None, Duration::from_secs(30)).await;
15665 let listener = TcpListener::bind("127.0.0.1:0").await.unwrap();
15666 actor.health.http = Some(format!("http://{}/health", listener.local_addr().unwrap()));
15667 actor.health.cadence = Duration::from_millis(1);
15669 let (started_tx, mut started_rx) = mpsc::channel(8);
15670 let peer = tokio::spawn(async move {
15671 loop {
15672 let (mut socket, _) = listener.accept().await.unwrap();
15673 let mut request = [0; 1024];
15674 assert!(socket.read(&mut request).await.unwrap() > 0);
15675 started_tx.send(()).await.unwrap();
15676 socket
15677 .write_all(b"HTTP/1.1 200 OK\r\nContent-Length: 0\r\n\r\n")
15678 .await
15679 .unwrap();
15680 }
15681 });
15682 let tick = Instant::now();
15683 let turns = actor.turns();
15684 actor
15685 .module
15686 .update_configuration(actor.spec.clone(), actor.health.clone(), None)
15687 .await
15688 .unwrap();
15689 let armed = actor
15690 .wait_for_select(|checkpoint| {
15691 checkpoint.turn > turns && checkpoint.next_probe_at.is_some()
15692 })
15693 .await;
15694 assert_eq!(armed.next_probe_at, Some(tick + Duration::from_millis(1)));
15695 assert_eq!(Instant::now(), tick);
15696 tokio::time::advance(Duration::from_millis(1)).await;
15697 let mut started = false;
15698 wait_for("successful HTTP probe", || {
15699 started |= started_rx.try_recv().is_ok();
15700 (started && actor.module.status().unwrap().health.status == SupervisorHealthStatus::Ok)
15701 .then_some(())
15702 })
15703 .await;
15704 assert!(started, "added HTTP check must start at its first deadline");
15705 assert_eq!(
15706 actor.module.status().unwrap().health.status,
15707 SupervisorHealthStatus::Ok
15708 );
15709 assert_eq!(Instant::now(), tick + Duration::from_millis(1));
15710 actor.health.http = None;
15711 let turns = actor.turns();
15712 actor
15713 .module
15714 .update_configuration(actor.spec.clone(), actor.health.clone(), None)
15715 .await
15716 .unwrap();
15717 let parked = actor
15718 .wait_for_select(|checkpoint| {
15719 checkpoint.turn > turns && checkpoint.next_probe_at.is_none()
15720 })
15721 .await;
15722 assert_eq!(parked.wake_after, None);
15723 let turns = actor.turns();
15724 tokio::time::advance(Duration::from_secs(60)).await;
15725 assert_eq!(
15726 actor.turns(),
15727 turns,
15728 "removed HTTP check must park the actor"
15729 );
15730 assert!(
15731 started_rx.try_recv().is_err(),
15732 "removed HTTP check must stay disarmed"
15733 );
15734 assert_eq!(
15735 actor.module.status().unwrap().health,
15736 ModuleHealthStatus::default()
15737 );
15738 peer.abort();
15739 }
15740}
15741
15742#[cfg(test)]
15743mod drain_busy_reprobe_tests {
15744 use super::*;
15748 use std::collections::BTreeMap;
15749 use std::sync::Mutex as StdMutex;
15750 use subc_protocol::{manifest::Concurrency, session::ModuleControlResponse};
15751
15752 const MODULE_ID: &str = "drain-reprobe-module";
15753 const CONNECTION: u64 = 7;
15754 const GAUGE: &str = "runs_in_flight";
15755 const CADENCE: Duration = Duration::from_secs(30);
15758 const CEILING: Duration = Duration::from_secs(30);
15759
15760 #[derive(Clone, Copy)]
15762 enum Answer {
15763 BusyUntil { until: Instant, latency: Duration },
15766 OmitsGauge,
15768 Silent,
15770 }
15771
15772 struct Drain {
15773 forwarding: Arc<ForwardingTable>,
15774 runtime: SupervisorRuntimeConfig,
15775 endpoint: crate::ModuleEndpointId,
15776 probes: Arc<StdMutex<Vec<Instant>>>,
15778 responder: tokio::task::JoinHandle<()>,
15779 }
15780
15781 impl Drain {
15782 fn start(answer: Answer) -> Self {
15783 let registry = Arc::new(Registry::default());
15784 let forwarding = Arc::new(ForwardingTable::default());
15785 let health = HealthConfig {
15786 cadence: CADENCE,
15787 ..HealthConfig::default()
15788 };
15789 let runtime = Supervisor::new_for_test(registry, RestartPolicy::new(3, Duration::ZERO))
15790 .with_forwarding(forwarding.clone())
15791 .with_health_config(health)
15792 .runtime_config();
15793 let (tx, mut rx) = mpsc::channel::<crate::router::OutboundFrame>(8);
15794 let endpoint = forwarding
15795 .register_module_connection(
15796 ConnectionId::new(CONNECTION),
15797 MODULE_ID.to_string(),
15798 subc_protocol::PROTOCOL_VERSION,
15799 Concurrency::ModuleManaged,
15800 FrameSink::new(tx),
15801 )
15802 .unwrap();
15803 let probes = Arc::new(StdMutex::new(Vec::new()));
15804 let responder = tokio::spawn({
15805 let forwarding = forwarding.clone();
15806 let probes = probes.clone();
15807 async move {
15808 while let Some(frame) = rx.recv().await {
15809 let body: Value = serde_json::from_slice(&frame.body).unwrap();
15810 assert_eq!(body["op"], MODULE_CONTROL_OP_HEALTH_CHECK);
15811 probes.lock().unwrap().push(Instant::now());
15812 let metrics = match answer {
15813 Answer::BusyUntil { until, latency } => {
15814 sleep(latency).await;
15815 let busy = u64::from(Instant::now() < until);
15816 serde_json::json!({ GAUGE: busy })
15817 }
15818 Answer::OmitsGauge => serde_json::json!({ "queue_depth": 0 }),
15819 Answer::Silent => continue,
15820 };
15821 let _ = forwarding.complete_module_control_rpc(
15824 ConnectionId::new(CONNECTION),
15825 frame.header.corr,
15826 Some(MODULE_CONTROL_OP_HEALTH_CHECK),
15827 ModuleControlRpcOutcome::Response(ModuleControlResponse::HealthCheck {
15828 status: HealthStatus::Ok,
15829 detail: None,
15830 metrics: Some(metrics),
15831 }),
15832 );
15833 }
15834 }
15835 });
15836 Self {
15837 forwarding,
15838 runtime,
15839 endpoint,
15840 probes,
15841 responder,
15842 }
15843 }
15844
15845 async fn run(&self, ceiling: Duration) -> (bool, Duration) {
15848 let (drained, elapsed, _) = self.run_traced(ceiling).await;
15849 (drained, elapsed)
15850 }
15851
15852 async fn run_traced(&self, ceiling: Duration) -> (bool, Duration, DrainGaugeTrace) {
15854 let started = Instant::now();
15855 let mut trace = DrainGaugeTrace::new(&[GAUGE.to_string()]);
15856 let drained = wait_for_forwarding_quiescence(
15857 &self.forwarding,
15858 MODULE_ID,
15859 &self.runtime,
15860 self.endpoint,
15861 started + ceiling,
15862 &[GAUGE.to_string()],
15863 DrainScope::Active,
15864 &mut trace,
15865 )
15866 .await
15867 .unwrap();
15868 (drained, started.elapsed(), trace)
15869 }
15870
15871 fn probes(&self) -> Vec<Instant> {
15872 self.probes.lock().unwrap().clone()
15873 }
15874 }
15875
15876 impl Drop for Drain {
15877 fn drop(&mut self) {
15878 self.responder.abort();
15879 }
15880 }
15881
15882 #[tokio::test(start_paused = true)]
15883 async fn busy_at_first_probe_then_quiet_drains_long_before_the_ceiling() {
15884 let drain = Drain::start(Answer::BusyUntil {
15885 until: Instant::now() + Duration::from_secs(1),
15886 latency: Duration::ZERO,
15887 });
15888 let (drained, elapsed) = drain.run(CEILING).await;
15889 assert!(
15890 drained,
15891 "gauges that read zero during the drain must let it finish drained, \
15892 not wait out the {CADENCE:?} health cadence (elapsed {elapsed:?})"
15893 );
15894 assert!(
15895 elapsed < Duration::from_secs(3),
15896 "drain took {elapsed:?} after the gauges went quiet at 1s"
15897 );
15898 let probes = drain.probes();
15899 assert!(
15900 probes.len() >= 2,
15901 "the module was busy at the first probe, so a quiet verdict needs a second: {probes:?}"
15902 );
15903 }
15904
15905 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
15910 async fn an_instant_busy_first_answer_does_not_stop_later_probes_in_real_time() {
15911 let drain = Drain::start(Answer::BusyUntil {
15912 until: Instant::now() + Duration::from_millis(600),
15913 latency: Duration::ZERO,
15914 });
15915 let (drained, elapsed, trace) = drain.run_traced(CEILING).await;
15916 assert!(drained, "the gauge fell at 600ms: {trace}");
15917 assert!(
15918 elapsed < Duration::from_secs(3),
15919 "drain took {elapsed:?} after the gauge fell at 600ms: {trace}"
15920 );
15921 let probes = drain.probes();
15922 assert!(
15923 probes.len() >= 2 && trace.probes as usize == probes.len(),
15924 "the drain must re-probe after the first busy answer: {probes:?}, {trace}"
15925 );
15926 }
15927
15928 #[tokio::test(start_paused = true)]
15929 async fn busy_throughout_ends_undrained_at_the_ceiling_with_bounded_probes() {
15930 let started = Instant::now();
15931 let drain = Drain::start(Answer::BusyUntil {
15932 until: started + Duration::from_secs(3600),
15933 latency: Duration::ZERO,
15934 });
15935 let (drained, elapsed) = drain.run(CEILING).await;
15936 assert!(!drained, "a module busy throughout must not drain");
15937 assert!(
15938 elapsed >= CEILING && elapsed < CEILING + drain_busy_reprobe_interval(),
15939 "a busy module is held until the ceiling and no longer: {elapsed:?}"
15940 );
15941 let probes = drain.probes();
15944 let max_probes = (CEILING.as_millis() / drain_busy_reprobe_interval().as_millis()) + 1;
15945 assert!(
15946 probes.len() as u128 <= max_probes,
15947 "{} probes in a {CEILING:?} drain exceeds the {max_probes} the interval allows",
15948 probes.len()
15949 );
15950 for pair in probes.windows(2) {
15951 assert!(
15952 pair[1] - pair[0] >= drain_busy_reprobe_interval(),
15953 "consecutive probes {:?} apart",
15954 pair[1] - pair[0]
15955 );
15956 }
15957 }
15958
15959 #[tokio::test(start_paused = true)]
15960 async fn a_slow_probe_is_never_overlapped_by_the_next() {
15961 let latency = Duration::from_secs(1);
15964 let drain = Drain::start(Answer::BusyUntil {
15965 until: Instant::now() + Duration::from_secs(3600),
15966 latency,
15967 });
15968 let (drained, _) = drain.run(Duration::from_secs(5)).await;
15969 assert!(!drained);
15970 let probes = drain.probes();
15971 assert!(probes.len() >= 2, "the drain must re-probe: {probes:?}");
15972 for pair in probes.windows(2) {
15973 assert!(
15974 pair[1] - pair[0] >= latency + drain_busy_reprobe_interval(),
15975 "a probe started {:?} after the previous one, before it could finish",
15976 pair[1] - pair[0]
15977 );
15978 }
15979 }
15980
15981 #[tokio::test(start_paused = true)]
15982 async fn an_unanswered_probe_cannot_hold_the_drain_past_its_ceiling() {
15983 let drain = Drain::start(Answer::Silent);
15984 let ceiling = Duration::from_secs(2);
15985 let (drained, elapsed) = drain.run(ceiling).await;
15986 assert!(!drained, "an unanswered probe counts as busy");
15987 assert!(
15988 elapsed >= ceiling && elapsed < ceiling + drain_busy_reprobe_interval(),
15989 "the probe deadline is capped at the drain ceiling: {elapsed:?}"
15990 );
15991 assert_eq!(
15992 drain.probes().len(),
15993 1,
15994 "the unanswered probe holds the only slot until the ceiling"
15995 );
15996 }
15997
15998 #[derive(Clone, Default)]
16000 struct LogCapture {
16001 events: Arc<StdMutex<Vec<BTreeMap<String, String>>>>,
16002 }
16003
16004 impl<S: tracing::Subscriber> tracing_subscriber::Layer<S> for LogCapture {
16005 fn on_event(
16006 &self,
16007 event: &tracing::Event<'_>,
16008 _context: tracing_subscriber::layer::Context<'_, S>,
16009 ) {
16010 struct Fields(BTreeMap<String, String>);
16011 impl tracing::field::Visit for Fields {
16012 fn record_debug(&mut self, field: &tracing::field::Field, value: &dyn fmt::Debug) {
16013 self.0
16014 .insert(field.name().to_string(), format!("{value:?}"));
16015 }
16016 }
16017 let mut fields = Fields(BTreeMap::new());
16018 event.record(&mut fields);
16019 self.events.lock().unwrap().push(fields.0);
16020 }
16021 }
16022
16023 impl LogCapture {
16024 fn install(&self) -> tracing::subscriber::DefaultGuard {
16025 use tracing_subscriber::layer::SubscriberExt;
16026 tracing::subscriber::set_default(tracing_subscriber::registry().with(self.clone()))
16027 }
16028
16029 fn with_message(&self, needle: &str) -> Vec<BTreeMap<String, String>> {
16030 self.events
16031 .lock()
16032 .unwrap()
16033 .iter()
16034 .filter(|fields| fields.get("message").is_some_and(|m| m.contains(needle)))
16035 .cloned()
16036 .collect()
16037 }
16038 }
16039
16040 #[tokio::test(start_paused = true)]
16045 async fn a_missing_declared_gauge_is_named_on_the_drain_timeout_line() {
16046 let capture = LogCapture::default();
16047 let _guard = capture.install();
16048 let drain = Drain::start(Answer::OmitsGauge);
16049 let (drained, _, trace) = drain.run_traced(Duration::from_secs(2)).await;
16050 assert!(!drained, "a gauge never reported cannot read quiet");
16051 assert!(trace.probes >= 2, "the drain must re-probe: {trace:?}");
16052
16053 let missing = capture.with_message("lacks a declared busy gauge");
16054 assert_eq!(missing.len(), 1, "one warning per drain: {missing:?}");
16055 assert_eq!(
16056 missing[0].get("missing_gauges").map(String::as_str),
16057 Some(GAUGE)
16058 );
16059
16060 warn_drain_timed_out(
16061 MODULE_ID,
16062 Duration::from_secs(2),
16063 RouteCloseReason::Restart,
16064 &Default::default(),
16065 &trace,
16066 );
16067 let timeouts = capture.with_message("route drain timed out");
16068 assert_eq!(timeouts.len(), 1, "{timeouts:?}");
16069 let gauges = &timeouts[0]["busy_gauges"];
16070 assert!(
16071 gauges.starts_with(&format!("{GAUGE}: missing_from_health_body @")),
16072 "the timeout line must name the gauge and why it read busy: {gauges}"
16073 );
16074 assert!(
16075 gauges.ends_with("; no value ever read"),
16076 "no value was ever read for the gauge: {gauges}"
16077 );
16078 assert_eq!(timeouts[0]["gauge_probes"], trace.probes.to_string());
16079 }
16080
16081 #[tokio::test(start_paused = true)]
16085 async fn the_drain_trace_names_a_busy_value_and_an_unanswered_probe() {
16086 let busy = Drain::start(Answer::BusyUntil {
16087 until: Instant::now() + Duration::from_secs(3600),
16088 latency: Duration::ZERO,
16089 });
16090 let (_, _, trace) = busy.run_traced(Duration::from_secs(1)).await;
16091 let rendered = trace.to_string();
16092 assert!(
16093 rendered.starts_with(&format!("{GAUGE}: non_zero(1) @"))
16094 && rendered.contains("; last value non_zero(1) @"),
16095 "{rendered}"
16096 );
16097
16098 let silent = Drain::start(Answer::Silent);
16099 let (_, _, trace) = silent.run_traced(Duration::from_secs(1)).await;
16100 assert_eq!(
16103 trace.to_string(),
16104 format!("{GAUGE}: probe_failed(no-answer) @1000ms; no value ever read")
16105 );
16106 assert_eq!(trace.probes, 1);
16107 }
16108
16109 #[test]
16111 fn a_gauge_that_is_not_a_count_is_busy_and_shown_as_sent() {
16112 let gauges = [GAUGE.to_string()];
16113 let verdicts = gauge_verdicts(Some(&serde_json::json!({ GAUGE: 0.0 })), &gauges);
16114 assert_eq!(verdicts, vec![GaugeVerdict::NotACount("0.0".to_string())]);
16115 assert_eq!(
16116 busy_gauge_observation(&verdicts),
16117 BusyGaugeObservation::Busy
16118 );
16119 let zero = gauge_verdicts(Some(&serde_json::json!({ GAUGE: 0 })), &gauges);
16120 assert_eq!(
16121 busy_gauge_observation(&zero),
16122 BusyGaugeObservation::Quiescent
16123 );
16124 let none = gauge_verdicts(None, &gauges);
16125 assert_eq!(busy_gauge_observation(&none), BusyGaugeObservation::Omitted);
16126 }
16127}
16128
16129#[cfg(test)]
16130mod health_evidence_tests {
16131 use super::{HealthProbeError, HealthProbeEvidence};
16132 use std::collections::HashSet;
16133
16134 #[test]
16142 fn only_a_dead_lane_is_proof_of_death() {
16143 assert!(HealthProbeError::lane_dead("gone").is_proof_of_death());
16144 assert!(!HealthProbeError::no_answer("timed out").is_proof_of_death());
16148 assert!(!HealthProbeError::bad_answer("garbage").is_proof_of_death());
16149 assert!(!HealthProbeError::misconfigured("no table").is_proof_of_death());
16150 }
16151
16152 #[test]
16158 fn every_evidence_class_has_a_distinct_label() {
16159 let labels = [
16160 HealthProbeError::lane_dead("").label(),
16161 HealthProbeError::no_answer("").label(),
16162 HealthProbeError::bad_answer("").label(),
16163 HealthProbeError::misconfigured("").label(),
16164 ];
16165 let unique: HashSet<_> = labels.iter().collect();
16166 assert_eq!(unique.len(), labels.len(), "labels collided: {labels:?}");
16167 }
16168
16169 #[test]
16175 fn classification_preserves_the_original_message() {
16176 let err = HealthProbeError::no_answer("module did not answer within 5s");
16177 assert_eq!(err.to_string(), "module did not answer within 5s");
16178 assert!(matches!(err.evidence, HealthProbeEvidence::NoAnswer));
16179 }
16180}
16181
16182#[cfg(test)]
16183mod health_tombstone_tests {
16184 use std::{path::PathBuf, sync::Arc, time::Duration};
16185
16186 use subc_protocol::{
16187 manifest::Concurrency,
16188 session::{HealthStatus, ModuleControlResponse},
16189 };
16190 use tokio::sync::mpsc;
16191
16192 use super::{
16193 probe_module_health, HealthAction, HealthConfig, HealthProbeEvidence, ModuleProtocol,
16194 ModuleSpec, RestartPolicy, Supervisor, SupervisorRuntimeConfig,
16195 };
16196 use crate::{
16197 control::ControlHandler,
16198 forwarding::{ForwardingTable, ModuleControlRpcCompletion, ModuleControlRpcOutcome},
16199 registry::{ConnectionId, Registry},
16200 router::FrameSink,
16201 };
16202
16203 struct ProbeHarness {
16204 spec: ModuleSpec,
16205 runtime: SupervisorRuntimeConfig,
16206 forwarding: Arc<ForwardingTable>,
16207 module_connection: ConnectionId,
16208 module_rx: mpsc::Receiver<crate::router::OutboundFrame>,
16209 handler: ControlHandler,
16210 module: super::SupervisedModule,
16211 }
16212
16213 fn probe_harness() -> ProbeHarness {
16214 let registry = Arc::new(Registry::default());
16215 let forwarding = Arc::new(ForwardingTable::default());
16216 let supervisor_handle = super::SupervisorHandle::new();
16217 let health = HealthConfig {
16218 http: None,
16219 cadence: Duration::from_secs(30),
16220 deadline: Duration::from_secs(5),
16221 failure_threshold: 3,
16222 on_degraded: HealthAction::Report,
16223 on_failing: HealthAction::Report,
16224 critical: false,
16225 };
16226 let supervisor = Supervisor::new_for_test(Arc::clone(®istry), RestartPolicy::default())
16227 .with_forwarding(Arc::clone(&forwarding))
16228 .with_handle(supervisor_handle.clone())
16229 .with_health_config(health);
16230 let spec = ModuleSpec {
16231 module_id: "late-health-module".to_string(),
16232 program: PathBuf::from("disabled-module"),
16233 args: Vec::new(),
16234 env: Vec::new(),
16235 reserved: false,
16236 reserved_prefixes: Vec::new(),
16237 protocol: ModuleProtocol::Subc,
16238 overlap: Default::default(),
16239 carry_containment: false,
16240 };
16241 let module = supervisor
16242 .supervise_configured(spec.clone(), false)
16243 .unwrap();
16244 let runtime = supervisor.runtime_config();
16245 let handler = ControlHandler::with_forwarding(registry, Arc::clone(&forwarding))
16246 .with_supervisor(supervisor_handle);
16247 let module_connection = ConnectionId::new(700);
16248 let (module_tx, module_rx) = mpsc::channel(8);
16249 forwarding
16250 .register_module_connection(
16251 module_connection,
16252 spec.module_id.clone(),
16253 subc_protocol::PROTOCOL_VERSION,
16254 Concurrency::ModuleManaged,
16255 FrameSink::new(module_tx),
16256 )
16257 .unwrap();
16258
16259 ProbeHarness {
16260 spec,
16261 runtime,
16262 forwarding,
16263 module_connection,
16264 module_rx,
16265 handler,
16266 module,
16267 }
16268 }
16269
16270 async fn finish_after(
16271 harness: &mut ProbeHarness,
16272 stall: Duration,
16273 ) -> ModuleControlRpcCompletion {
16274 assert!(stall > harness.runtime.health.deadline);
16275 let deadline = harness.runtime.health.deadline;
16276 let probe = probe_module_health(&harness.spec.module_id, &harness.runtime, None);
16277 let answer = async {
16278 let frame = harness.module_rx.recv().await.expect("health.check frame");
16279 tokio::time::advance(deadline).await;
16280 tokio::task::yield_now().await;
16281 tokio::time::advance(stall - deadline).await;
16282 harness
16283 .forwarding
16284 .complete_module_control_rpc(
16285 harness.module_connection,
16286 frame.header.corr,
16287 Some("health.check"),
16288 ModuleControlRpcOutcome::Response(ModuleControlResponse::HealthCheck {
16289 status: HealthStatus::Ok,
16290 detail: None,
16291 metrics: None,
16292 }),
16293 )
16294 .unwrap()
16295 };
16296 let (probe_result, completion) = tokio::join!(probe, answer);
16297 let err = probe_result.expect_err("probe must miss its deadline");
16298 assert!(matches!(err.evidence, HealthProbeEvidence::NoAnswer));
16299 completion
16300 }
16301
16302 async fn time_out_without_answer(harness: &mut ProbeHarness) {
16303 let deadline = harness.runtime.health.deadline;
16304 let probe = probe_module_health(&harness.spec.module_id, &harness.runtime, None);
16305 let exhaust_deadline = async {
16306 let _frame = harness.module_rx.recv().await.expect("health.check frame");
16307 tokio::time::advance(deadline).await;
16308 tokio::task::yield_now().await;
16309 };
16310 let (probe_result, ()) = tokio::join!(probe, exhaust_deadline);
16311 let err = probe_result.expect_err("probe must miss its deadline");
16312 assert!(matches!(err.evidence, HealthProbeEvidence::NoAnswer));
16313 }
16314
16315 async fn run_probe_cycle(harness: &mut ProbeHarness, answer: bool) {
16316 let registry = Arc::clone(&harness.module.inner.registry);
16317 let snapshot = Arc::clone(&harness.module.inner.snapshot);
16318 let process_liveness = super::SupervisorProcessLiveness::default();
16319 let mut child = None;
16320 let cycle = super::run_health_probe_cycle(
16321 &harness.spec,
16322 &harness.runtime,
16323 ®istry,
16324 &process_liveness,
16325 &snapshot,
16326 &mut child,
16327 );
16328 let peer = async {
16329 let frame = harness.module_rx.recv().await.expect("health.check frame");
16330 if answer {
16331 harness
16332 .forwarding
16333 .complete_module_control_rpc(
16334 harness.module_connection,
16335 frame.header.corr,
16336 Some("health.check"),
16337 ModuleControlRpcOutcome::Response(ModuleControlResponse::HealthCheck {
16338 status: HealthStatus::Ok,
16339 detail: None,
16340 metrics: Some(serde_json::json!({"ready": true})),
16341 }),
16342 )
16343 .unwrap();
16344 } else {
16345 tokio::time::advance(harness.runtime.health.deadline).await;
16346 tokio::task::yield_now().await;
16347 }
16348 };
16349 tokio::join!(cycle, peer);
16350 }
16351
16352 #[tokio::test(start_paused = true)]
16353 async fn unanswered_probe_is_unknown_until_threshold_and_ok_report_recovers() {
16354 let mut harness = probe_harness();
16355 let monitor = harness.module.inner.monitor.lock().unwrap().take().unwrap();
16359 monitor.abort();
16360 let _ = monitor.await;
16361 super::update_snapshot(&harness.module.inner.snapshot, None, |state| {
16362 state.enabled = true;
16363 state.state = super::ModuleState::Running;
16364 state.process_alive = true;
16365 })
16366 .unwrap();
16367
16368 run_probe_cycle(&mut harness, true).await;
16369 assert_eq!(
16370 harness.module.status().unwrap().health.status,
16371 super::SupervisorHealthStatus::Ok
16372 );
16373
16374 for failures in 1..harness.runtime.health.failure_threshold {
16375 run_probe_cycle(&mut harness, false).await;
16376 let status = harness.module.status().unwrap();
16377 assert_eq!(status.health.status, super::SupervisorHealthStatus::Unknown);
16378 assert_eq!(status.health.consecutive_failures, failures);
16379 assert!(status.health.last_probe_ms.is_some());
16380 assert!(status.health.detail.unwrap().starts_with("[no-answer]"));
16381 assert!(status.health.metrics.is_none());
16382 assert_eq!(status.state, super::ModuleState::Running);
16383 assert!(status.process_alive);
16384 assert_eq!(status.restart_count, 0);
16385 assert_eq!(status.lifetime_restarts, 0);
16386 assert!(status.health.last_action.is_none());
16387 assert!(harness.runtime.scheduled_respawn.lock().unwrap().is_none());
16388 }
16389
16390 run_probe_cycle(&mut harness, true).await;
16391 let recovered = harness.module.status().unwrap();
16392 assert_eq!(recovered.health.status, super::SupervisorHealthStatus::Ok);
16393 assert_eq!(recovered.health.consecutive_failures, 0);
16394 assert!(recovered.health.detail.is_none());
16395 assert_eq!(
16396 recovered.health.metrics,
16397 Some(serde_json::json!({"ready": true}))
16398 );
16399 assert_eq!(recovered.lifetime_restarts, 0);
16400
16401 for failures in 1..=harness.runtime.health.failure_threshold {
16402 run_probe_cycle(&mut harness, false).await;
16403 let status = harness.module.status().unwrap();
16404 assert_eq!(status.health.consecutive_failures, failures);
16405 if failures < harness.runtime.health.failure_threshold {
16406 assert_eq!(status.health.status, super::SupervisorHealthStatus::Unknown);
16407 assert_eq!(status.state, super::ModuleState::Running);
16408 assert_eq!(status.lifetime_restarts, 0);
16409 assert!(harness.runtime.scheduled_respawn.lock().unwrap().is_none());
16410 } else {
16411 assert_eq!(
16412 status.health.status,
16413 super::SupervisorHealthStatus::Unresponsive
16414 );
16415 assert_eq!(status.state, super::ModuleState::Restarting);
16416 assert_eq!(status.restart_count, 1);
16417 assert_eq!(status.lifetime_restarts, 1);
16418 assert_eq!(status.health.last_action.as_deref(), Some("restart"));
16419 assert!(harness.runtime.scheduled_respawn.lock().unwrap().is_some());
16420 }
16421 }
16422 }
16423
16424 #[tokio::test(start_paused = true)]
16425 async fn late_health_answers_record_start_anchored_latency_for_two_stalls() {
16426 let mut harness = probe_harness();
16427
16428 let first = finish_after(&mut harness, Duration::from_secs(8)).await;
16429 let first_latency = match &first {
16430 ModuleControlRpcCompletion::LateHealthAnswer { latency, .. } => *latency,
16431 other => panic!("late answer was not retained: {other:?}"),
16432 };
16433 assert!(harness.handler.observe_module_control_completion(first));
16434
16435 let second = finish_after(&mut harness, Duration::from_secs(11)).await;
16436 let second_latency = match &second {
16437 ModuleControlRpcCompletion::LateHealthAnswer { latency, .. } => *latency,
16438 other => panic!("late answer was not retained: {other:?}"),
16439 };
16440 assert!(harness.handler.observe_module_control_completion(second));
16441
16442 assert_eq!(first_latency, Duration::from_secs(8));
16443 assert_eq!(
16444 second_latency - first_latency,
16445 Duration::from_secs(3),
16446 "latency must grow linearly with the additional stall"
16447 );
16448 let health = harness.module.status().unwrap().health;
16449 assert_eq!(health.late_answer_count, 2);
16450 assert_eq!(health.last_late_answer_latency_ms, Some(11_000));
16451 }
16452
16453 #[tokio::test(start_paused = true)]
16461 async fn late_answer_clears_the_consecutive_failure_streak() {
16462 let mut harness = probe_harness();
16463
16464 time_out_without_answer(&mut harness).await;
16466 harness
16467 .module
16468 .record_health_probe_failure_for_test("[no-answer] test miss")
16469 .unwrap();
16470 assert_eq!(
16471 harness.module.status().unwrap().health.consecutive_failures,
16472 1,
16473 "precondition: the miss must be on the streak before the late answer"
16474 );
16475
16476 let late = finish_after(&mut harness, Duration::from_secs(9)).await;
16478 assert!(matches!(
16479 late,
16480 ModuleControlRpcCompletion::LateHealthAnswer { .. }
16481 ));
16482 assert!(harness.handler.observe_module_control_completion(late));
16483
16484 let health = harness.module.status().unwrap().health;
16485 assert_eq!(
16486 health.consecutive_failures, 0,
16487 "a late answer is an answer: the streak must reset"
16488 );
16489 assert_eq!(health.late_answer_count, 1);
16490 }
16491
16492 #[tokio::test(start_paused = true)]
16493 async fn repeated_serial_probe_cycles_keep_one_tombstone_per_endpoint() {
16494 let mut harness = probe_harness();
16495
16496 for _ in 0..20 {
16497 time_out_without_answer(&mut harness).await;
16498 assert_eq!(
16499 harness.forwarding.health_probe_tombstone_count().unwrap(),
16500 1
16501 );
16502 }
16503 }
16504}
16505
16506#[cfg(test)]
16507mod child_env_tests {
16508 #[cfg(windows)]
16509 use super::{
16510 clear_current_process_facts, observed_launch_nonce_source, ModuleState, SupervisorSnapshot,
16511 };
16512 #[cfg(windows)]
16513 use std::sync::{Arc, OnceLock};
16514 #[cfg(windows)]
16515 #[test]
16516 fn windows_wire_spawn_prepares_pipe_and_keeps_phase_one_env() {
16517 let spec = spec(Vec::new());
16518 let mut command = Command::new("unused");
16519 let handoff = apply_wire_spawn_args(&mut command, &spec, None, None)
16520 .unwrap()
16521 .unwrap();
16522 let name = command
16523 .as_std()
16524 .get_envs()
16525 .find(|(key, _)| *key == OsStr::new(subc_os::launch_nonce::LAUNCH_NONCE_PIPE_ENV))
16526 .and_then(|(_, value)| value)
16527 .unwrap();
16528 assert_eq!(name, OsStr::new(handoff.name()));
16529 assert!(handoff.name().starts_with(r"\\.\pipe\subc-launch-"));
16530 assert!(command.as_std().get_envs().any(|(key, value)| key
16531 == OsStr::new(subc_os::LAUNCH_NONCE_PIPE_FALLBACK_ENV)
16532 && value == Some(OsStr::new("env"))));
16533 assert!(command
16534 .as_std()
16535 .get_envs()
16536 .any(|(key, value)| key == OsStr::new(SUBC_LAUNCH_NONCE_ENV) && value.is_some()));
16537 }
16538
16539 #[cfg(windows)]
16540 #[test]
16541 fn windows_source_reporting_tracks_consumption_and_resets_per_spawn() {
16542 let mut snapshot = SupervisorSnapshot::new(ModuleState::Running, true);
16543 snapshot.process_alive = true;
16544 snapshot.spawned_protocol = Some(ModuleProtocol::Subc);
16545 assert_eq!(
16546 observed_launch_nonce_source(&snapshot).as_deref(),
16547 Some("env")
16548 );
16549 let consumed = Arc::new(OnceLock::new());
16550 snapshot.pipe_consumed = Some(consumed.clone());
16551 assert_eq!(
16552 observed_launch_nonce_source(&snapshot).as_deref(),
16553 Some("env")
16554 );
16555 consumed.set(()).unwrap();
16556 assert_eq!(
16557 observed_launch_nonce_source(&snapshot).as_deref(),
16558 Some("pipe")
16559 );
16560 snapshot.pipe_consumed = Some(Arc::new(OnceLock::new()));
16561 assert_eq!(
16562 observed_launch_nonce_source(&snapshot).as_deref(),
16563 Some("env")
16564 );
16565 clear_current_process_facts(&mut snapshot);
16566 assert_eq!(observed_launch_nonce_source(&snapshot), None);
16567 }
16568 use super::{
16569 apply_child_env, apply_spawn_role, apply_wire_spawn_args, LaunchSpecDigest, ModuleProtocol,
16570 ModuleSpec, SpawnRole, SupervisorHandle, SPAWN_ROLE_SWAP_CANDIDATE, SUBC_ARG,
16571 SUBC_LAUNCH_NONCE_ENV, SUBC_MODULE_ID_ENV, SUBC_SPAWN_ROLE_ENV,
16572 };
16573 use std::{ffi::OsStr, path::PathBuf};
16574 use tokio::process::Command;
16575
16576 fn spec(env: Vec<(String, String)>) -> ModuleSpec {
16577 ModuleSpec {
16578 module_id: "env-plan".to_string(),
16579 program: PathBuf::from("/nonexistent"),
16580 args: Vec::new(),
16581 env,
16582 reserved: false,
16583 reserved_prefixes: Vec::new(),
16584 protocol: ModuleProtocol::Subc,
16585 overlap: Default::default(),
16586 carry_containment: false,
16587 }
16588 }
16589
16590 #[test]
16591 fn configured_launch_spec_digest_is_canonical_and_excludes_injected_inputs() {
16592 let base = spec(vec![("A".into(), "B=C".into()), ("Z".into(), "v".into())]);
16593 let digest = LaunchSpecDigest::from_spec(&base);
16594 let mut reordered = base.clone();
16595 reordered.env.reverse();
16596 reordered.module_id = "different-id".into();
16597 reordered.reserved = true;
16598 reordered.overlap = Default::default();
16599 assert_eq!(digest, LaunchSpecDigest::from_spec(&reordered));
16600 let mut delimiter_collision = base.clone();
16601 delimiter_collision.env[0] = ("A=B".into(), "C".into());
16602 assert_ne!(digest, LaunchSpecDigest::from_spec(&delimiter_collision));
16603 let mut changed = base.clone();
16604 changed.protocol = ModuleProtocol::None;
16605 assert_ne!(digest, LaunchSpecDigest::from_spec(&changed));
16606 changed = base.clone();
16607 changed.args.push("--flag".into());
16608 assert_ne!(digest, LaunchSpecDigest::from_spec(&changed));
16609 changed = base.clone();
16610 changed.program = PathBuf::from("/other");
16611 assert_ne!(digest, LaunchSpecDigest::from_spec(&changed));
16612 changed = base.clone();
16613 changed.env[1].1 = "other".into();
16614 assert_ne!(digest, LaunchSpecDigest::from_spec(&changed));
16615 assert!(!format!("{digest:?}").contains("B=C"));
16616 assert!(!format!("{digest:?}").contains("[u8"));
16617 }
16618
16619 #[cfg(unix)]
16620 #[test]
16621 fn native_program_bytes_do_not_collapse_through_lossy_utf8() {
16622 use std::os::unix::ffi::OsStringExt;
16623 let mut left = spec(Vec::new());
16624 left.program = std::ffi::OsString::from_vec(vec![b'/', 0xff]).into();
16625 let mut right = left.clone();
16626 right.program = std::ffi::OsString::from_vec(vec![b'/', 0xfe]).into();
16627 assert_eq!(
16628 left.program.to_string_lossy(),
16629 right.program.to_string_lossy()
16630 );
16631 assert_ne!(
16632 LaunchSpecDigest::from_spec(&left),
16633 LaunchSpecDigest::from_spec(&right)
16634 );
16635 }
16636
16637 #[test]
16651 fn ambient_ck_log_is_removed_and_a_configured_one_survives() {
16652 let mut command = Command::new("/nonexistent");
16653 apply_child_env(&mut command, &spec(Vec::new()));
16654 let removed = command
16655 .as_std()
16656 .get_envs()
16657 .any(|(key, value)| key == OsStr::new("CK_LOG") && value.is_none());
16658 assert!(
16659 removed,
16660 "ambient CK_LOG must be explicitly removed for an unconfigured module"
16661 );
16662
16663 let mut configured = Command::new("/nonexistent");
16664 apply_child_env(
16665 &mut configured,
16666 &spec(vec![("CK_LOG".to_string(), "debug".to_string())]),
16667 );
16668 let effective = configured
16669 .as_std()
16670 .get_envs()
16671 .filter(|(key, _)| *key == OsStr::new("CK_LOG"))
16672 .last()
16673 .map(|(_, value)| value.map(|v| v.to_string_lossy().into_owned()));
16674 assert_eq!(
16675 effective,
16676 Some(Some("debug".to_string())),
16677 "a module's configured CK_LOG must survive the ambient removal"
16678 );
16679 }
16680
16681 #[test]
16690 fn protocol_none_spawn_carries_no_subc_argument_and_no_nonce() {
16691 let connection_file = std::path::Path::new("/run/subc-connection.json");
16692 let handle = SupervisorHandle::new();
16693
16694 let mut none_spec = spec(Vec::new());
16695 none_spec.protocol = ModuleProtocol::None;
16696 let mut none = Command::new("/nonexistent");
16697 let none_handoff =
16698 apply_wire_spawn_args(&mut none, &none_spec, Some(connection_file), Some(&handle))
16699 .expect("protocol-none spawn args apply");
16700 assert!(
16701 none_handoff.is_none(),
16702 "protocol:none spawn must not receive a nonce descriptor"
16703 );
16704 #[cfg(windows)]
16705 assert!(!none.as_std().get_envs().any(|(key, value)| key
16706 == OsStr::new(subc_os::LAUNCH_NONCE_PIPE_FALLBACK_ENV)
16707 && value.is_some()));
16708 assert!(
16709 !none.as_std().get_envs().any(|(key, value)| key
16710 == OsStr::new(subc_os::LAUNCH_NONCE_FD_ENV)
16711 && value.is_some()),
16712 "protocol:none spawn must not name a nonce descriptor"
16713 );
16714 let none_args: Vec<String> = none
16715 .as_std()
16716 .get_args()
16717 .map(|a| a.to_string_lossy().into_owned())
16718 .collect();
16719 assert!(
16720 !none_args.iter().any(|a| a == SUBC_ARG),
16721 "protocol:none argv must not carry --subc; got {none_args:?}"
16722 );
16723 let none_has_nonce = none
16724 .as_std()
16725 .get_envs()
16726 .any(|(key, value)| key == OsStr::new(SUBC_LAUNCH_NONCE_ENV) && value.is_some());
16727 assert!(
16728 !none_has_nonce,
16729 "protocol:none spawn must not receive a launch nonce"
16730 );
16731 let none_has_module_id = none
16732 .as_std()
16733 .get_envs()
16734 .any(|(key, value)| key == OsStr::new(SUBC_MODULE_ID_ENV) && value.is_some());
16735 assert!(
16736 none_has_module_id,
16737 "SUBC_MODULE_ID is inert and stays on every path"
16738 );
16739 assert!(
16740 handle.spawn_nonce(&none_spec.module_id).is_none(),
16741 "no nonce record for a process that will never present one"
16742 );
16743
16744 let wire_spec = spec(Vec::new());
16746 let mut wire = Command::new("/nonexistent");
16747 let wire_handoff =
16748 apply_wire_spawn_args(&mut wire, &wire_spec, Some(connection_file), Some(&handle))
16749 .expect("subc-wire spawn args apply");
16750 let wire_fd_env = wire
16751 .as_std()
16752 .get_envs()
16753 .find(|(key, _)| *key == OsStr::new(subc_os::LAUNCH_NONCE_FD_ENV))
16754 .and_then(|(_, value)| value.map(|v| v.to_string_lossy().into_owned()));
16755 #[cfg(unix)]
16756 assert_eq!(
16757 wire_fd_env,
16758 Some(wire_handoff.expect("a descriptor handoff").fd_env_value()),
16759 "a subc-wire spawn names the pipe it will receive at descriptor 3"
16760 );
16761 #[cfg(windows)]
16762 assert!(wire_handoff.is_some() && wire_fd_env.is_none());
16763 #[cfg(not(any(unix, windows)))]
16764 assert!(wire_handoff.is_none() && wire_fd_env.is_none());
16765 let wire_args: Vec<String> = wire
16766 .as_std()
16767 .get_args()
16768 .map(|a| a.to_string_lossy().into_owned())
16769 .collect();
16770 assert_eq!(
16771 wire_args,
16772 vec![
16773 SUBC_ARG.to_string(),
16774 connection_file.to_string_lossy().into_owned()
16775 ],
16776 "a subc-wire spawn still carries --subc <path>"
16777 );
16778 assert_eq!(
16779 wire.as_std()
16780 .get_envs()
16781 .any(|(key, value)| key == OsStr::new(SUBC_LAUNCH_NONCE_ENV) && value.is_some()),
16782 !cfg!(unix),
16783 "only Windows supplies the environment nonce"
16784 );
16785 assert!(handle.spawn_nonce(&wire_spec.module_id).is_some());
16786 }
16787
16788 #[test]
16798 fn plain_spawn_removes_the_spawn_role_even_when_the_spec_sets_it() {
16799 let role = |command: &Command| {
16800 command
16801 .as_std()
16802 .get_envs()
16803 .filter(|(key, _)| *key == OsStr::new(SUBC_SPAWN_ROLE_ENV))
16804 .last()
16805 .map(|(_, value)| value.map(|v| v.to_string_lossy().into_owned()))
16806 };
16807 let forged = spec(vec![(
16808 SUBC_SPAWN_ROLE_ENV.to_string(),
16809 SPAWN_ROLE_SWAP_CANDIDATE.to_string(),
16810 )]);
16811
16812 let mut plain = Command::new("/nonexistent");
16813 apply_child_env(&mut plain, &forged);
16814 apply_spawn_role(&mut plain, SpawnRole::Plain);
16815 assert_eq!(
16816 role(&plain),
16817 Some(None),
16818 "a plain spawn must remove SUBC_SPAWN_ROLE, whatever the spec says"
16819 );
16820
16821 let mut candidate = Command::new("/nonexistent");
16822 apply_child_env(&mut candidate, &spec(Vec::new()));
16823 apply_spawn_role(&mut candidate, SpawnRole::SwapCandidate);
16824 assert_eq!(
16825 role(&candidate),
16826 Some(Some(SPAWN_ROLE_SWAP_CANDIDATE.to_string()))
16827 );
16828 }
16829
16830 #[test]
16836 fn daemon_private_capture_keys_are_not_passed_to_the_child() {
16837 let mut command = Command::new("/nonexistent");
16838 apply_child_env(
16839 &mut command,
16840 &spec(vec![
16841 (super::CAPTURE_KEEP_ENV.to_string(), "5".to_string()),
16842 ("KEPT".to_string(), "yes".to_string()),
16843 ]),
16844 );
16845 let keys: Vec<String> = command
16846 .as_std()
16847 .get_envs()
16848 .filter(|(_, value)| value.is_some())
16849 .map(|(key, _)| key.to_string_lossy().into_owned())
16850 .collect();
16851 assert!(keys.contains(&"KEPT".to_string()), "got {keys:?}");
16852 assert!(
16853 !keys.contains(&super::CAPTURE_KEEP_ENV.to_string()),
16854 "daemon-private capture key leaked to the child: {keys:?}"
16855 );
16856 }
16857}
16858
16859#[cfg(test)]
16860mod jitter_tests {
16861 use super::jittered_health_delay;
16862 use std::{collections::HashSet, time::Duration};
16863
16864 const FLEET: [&str; 14] = [
16873 "aft",
16874 "alfonso-core",
16875 "magic-context",
16876 "broca",
16877 "thalamus",
16878 "quota",
16879 "engram",
16880 "plexus",
16881 "cerebellum",
16882 "astrocyte",
16883 "synapse",
16884 "subc-mcp",
16885 "cortexkit-credentials",
16886 "subc-federation",
16887 ];
16888
16889 #[test]
16897 fn probe_delays_disperse_across_the_fleet() {
16898 let cadence = Duration::from_secs(30);
16899 let delays: HashSet<Duration> = FLEET
16900 .iter()
16901 .map(|id| jittered_health_delay(id, 0, cadence))
16902 .collect();
16903 assert_eq!(
16904 delays.len(),
16905 FLEET.len(),
16906 "every supervised module must land on its own probe offset"
16907 );
16908 }
16909
16910 #[test]
16916 fn jitter_only_delays_and_stays_within_one_tenth_of_cadence() {
16917 let cadence = Duration::from_secs(30);
16918 let span = cadence / 10;
16919 for id in FLEET {
16920 for probe_index in 0..8 {
16921 let delay = jittered_health_delay(id, probe_index, cadence);
16922 assert!(
16923 delay >= cadence,
16924 "{id}#{probe_index}: jitter must not shorten the cadence"
16925 );
16926 assert!(
16927 delay < cadence + span,
16928 "{id}#{probe_index}: jitter must stay inside one tenth of the cadence"
16929 );
16930 }
16931 }
16932 }
16933
16934 #[test]
16940 fn a_module_offset_is_stable_across_restarts() {
16941 let cadence = Duration::from_secs(30);
16942 for id in FLEET {
16943 assert_eq!(
16944 jittered_health_delay(id, 0, cadence),
16945 jittered_health_delay(id, 0, cadence),
16946 "{id}: the same module and probe index must produce the same offset"
16947 );
16948 }
16949 }
16950
16951 #[test]
16953 fn zero_cadence_yields_zero_delay() {
16954 assert_eq!(
16955 jittered_health_delay("aft", 0, Duration::ZERO),
16956 Duration::ZERO
16957 );
16958 }
16959}
16960
16961#[cfg(all(test, target_os = "linux"))]
16962mod cgroup_placement_tests {
16963 use super::{
16964 apply_cgroup_placement, remove_module_cgroup, LaunchSpecDigest, ModuleProtocol, ModuleSpec,
16965 SuperviseError, SupervisedChild,
16966 };
16967 use crate::stderr_tail::{StderrRing, StderrTailConfig};
16968 use cortexkit_test_support::ScratchDir;
16969 use std::{
16970 fs, io,
16971 path::{Path, PathBuf},
16972 sync::{Arc, Mutex},
16973 };
16974 use tokio::process::Command;
16975
16976 #[tokio::test]
16977 async fn unique_spawn_cgroups_clean_up_across_restarts_and_shutdown() {
16978 use super::*;
16979 let dir = ScratchDir::new("unique-spawn-cgroups");
16980 let root = PathBuf::from(format!(
16981 "/sys/fs/cgroup/subc-unique-test-{}-{}",
16982 std::process::id(),
16983 unix_ms_now()
16984 ));
16985 if let Err(error) = fs::create_dir(&root) {
16986 assert!(
16987 std::env::var_os("SUBC_REQUIRE_CGROUP_TEST").is_none(),
16988 "required cgroup test cannot execute: {error}"
16989 );
16990 eprintln!("SKIP unique_spawn_cgroups_clean_up_across_restarts_and_shutdown: {error}");
16991 return;
16992 }
16993 let placement = subc_cgroup::prepare_at(&root).unwrap().unwrap();
16994 let group_count = || {
16995 fs::read_dir(root.join("subc-modules"))
16996 .unwrap()
16997 .map(|entry| entry.unwrap().file_type().unwrap())
16998 .filter(|kind| kind.is_dir())
16999 .count()
17000 };
17001 let supervisor =
17002 Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default())
17003 .with_cgroup_placement(Some(placement.clone()));
17004 let runtime = supervisor.runtime_config();
17005 let mut spec = ModuleSpec {
17006 module_id: "unique-spawn".into(),
17007 program: PathBuf::from("/bin/sleep"),
17008 args: vec!["60".into()],
17009 env: ["XDG_DATA_HOME", "XDG_RUNTIME_DIR", "XDG_CONFIG_HOME"]
17010 .into_iter()
17011 .map(|key| (key.into(), dir.to_string_lossy().into_owned()))
17012 .collect(),
17013 reserved: false,
17014 reserved_prefixes: vec![],
17015 protocol: ModuleProtocol::None,
17016 overlap: Default::default(),
17017 carry_containment: false,
17018 };
17019 let spawn = |spec: &ModuleSpec| {
17020 spawn_child(
17021 spec,
17022 None,
17023 None,
17024 &runtime.stderr_ring,
17025 None,
17026 &runtime.child_roster,
17027 Some(&placement),
17028 )
17029 .unwrap()
17030 };
17031 let mut live = spawn(&spec);
17032 for _ in 0..3 {
17033 let next = spawn(&spec);
17035 assert_ne!(live.module_id, next.module_id);
17036 live.start_kill().unwrap();
17037 live.wait().await.unwrap();
17038 live = next;
17039 assert!(
17040 live.child.try_wait().unwrap().is_none(),
17041 "retiring the old slot must not kill the replacement"
17042 );
17043 assert_eq!(
17044 group_count(),
17045 1,
17046 "only the live spawn's cgroup should remain"
17047 );
17048 }
17049 supervisor.begin_daemon_shutdown();
17050 let reap = tokio::spawn(async move {
17051 live.wait().await.unwrap();
17052 });
17053 supervisor
17054 .end_children_for_daemon_shutdown(false, std::future::pending())
17055 .await;
17056 reap.await.unwrap();
17057 assert_eq!(group_count(), 0);
17058 spec.program = PathBuf::from("/bin/true");
17060 spec.args.clear();
17061 let fresh_roster = ChildRoster::default();
17062 let mut short = spawn_child(
17063 &spec,
17064 None,
17065 None,
17066 &runtime.stderr_ring,
17067 None,
17068 &fresh_roster,
17069 Some(&placement),
17070 )
17071 .unwrap();
17072 short.wait().await.unwrap();
17073 assert_eq!(group_count(), 0);
17074 spec.module_id = "_".repeat(255);
17075 let mut long_id = spawn_child(
17076 &spec,
17077 None,
17078 None,
17079 &runtime.stderr_ring,
17080 None,
17081 &fresh_roster,
17082 Some(&placement),
17083 )
17084 .unwrap();
17085 long_id.wait().await.unwrap();
17086 assert_eq!(
17087 group_count(),
17088 0,
17089 "valid long module IDs must not exceed cgroup NAME_MAX"
17090 );
17091 fs::remove_dir(root.join("subc-modules")).unwrap();
17092 fs::remove_dir(root).unwrap();
17093 eprintln!("EXECUTED unique_spawn_cgroups_clean_up_across_restarts_and_shutdown");
17094 }
17095
17096 #[test]
17097 fn failed_parent_cgroup_open_is_a_cgroup_supervision_error() {
17098 let path = Path::new("/definitely-missing-subc-cgroup");
17099 let mut command = Command::new("true");
17100 let error = apply_cgroup_placement(
17101 &mut command,
17102 &ModuleSpec {
17103 module_id: "broken-cgroup".to_string(),
17104 program: PathBuf::from("true"),
17105 args: Vec::new(),
17106 env: Vec::new(),
17107 reserved: false,
17108 reserved_prefixes: Vec::new(),
17109 protocol: ModuleProtocol::Subc,
17110 overlap: Default::default(),
17111 carry_containment: false,
17112 },
17113 path,
17114 )
17115 .expect_err("a parent cgroup open failure must reject the supervised spawn");
17116 let reason = error.to_string();
17117
17118 assert!(
17119 matches!(error, SuperviseError::Cgroup { .. }),
17120 "parent cgroup open must be reported as a cgroup supervision error: {reason}"
17121 );
17122 assert!(
17123 reason.contains("/definitely-missing-subc-cgroup/cgroup.procs"),
17124 "parent cgroup open failure must name cgroup.procs: {reason}"
17125 );
17126 }
17127
17128 #[tokio::test]
17129 async fn reaping_a_child_removes_its_empty_module_cgroup() {
17130 let root = ScratchDir::new("supervisor-reap-cgroup");
17131 fs::write(root.join("cgroup.procs"), b"").expect("write scratch cgroup marker");
17132 let placement = subc_cgroup::prepare_at(&root)
17133 .expect("prepare scratch cgroup root")
17134 .expect("scratch root has a cgroup.procs marker");
17135 let module_id = "reaped-module";
17136 let module = placement
17137 .module_path(module_id)
17138 .expect("create scratch module cgroup");
17139 let child = Command::new("true")
17140 .env("XDG_DATA_HOME", root.path())
17141 .env("XDG_RUNTIME_DIR", root.path())
17142 .env("XDG_CONFIG_HOME", root.path())
17143 .spawn()
17144 .expect("spawn short-lived child");
17145 let pid = child.id().expect("spawned child has pid");
17146 let mut child = SupervisedChild {
17147 subtree_target: crate::subtree_resources::Target::Unavailable("not_running"),
17148 child: crate::supervise::containment::ProcessChild::plain(child, pid),
17149 exec: crate::activity::ExecOwner(Arc::new(crate::activity::ExecOutcome::new())),
17150 protocol: ModuleProtocol::Subc,
17151 module_id: module_id.to_string(),
17152 cgroup_placement: Some(placement),
17153 stdout_pump: None,
17154 stderr_pump: None,
17155 stderr_ring: Arc::new(Mutex::new(StderrRing::new(StderrTailConfig::default()))),
17156 spawned_at: tokio::time::Instant::now(),
17157 registration_deadline: None,
17158 registration_watch: None,
17159 spawned_at_ms: 0,
17160 spawned_from: PathBuf::from("true"),
17161 launch_spec_digest: LaunchSpecDigest([0; 32]),
17162 spawned_file_identity: None,
17163 process_start_time: None,
17164 process_identity: None,
17165 pid,
17166 roster_guard: None,
17167 #[cfg(target_os = "macos")]
17168 privacy_exec: None,
17169 spawn_failure: None,
17170 };
17171
17172 child.wait().await.expect("reap short-lived child");
17173
17174 assert!(
17175 !module.exists(),
17176 "waiting for a non-carried child must remove its empty cgroup before returning"
17177 );
17178 child.finish_exit(false).await.unwrap();
17179 }
17180
17181 #[test]
17182 fn non_empty_cgroup_removal_is_reported_without_blocking_teardown() {
17183 let root = ScratchDir::new("supervisor-non-empty-cgroup");
17184 fs::write(root.join("cgroup.procs"), b"").expect("write scratch cgroup marker");
17185 let placement = subc_cgroup::prepare_at(&root)
17186 .expect("prepare scratch cgroup root")
17187 .expect("scratch root has a cgroup.procs marker");
17188 let module = placement
17189 .module_path("surviving-module")
17190 .expect("create scratch module cgroup");
17191 fs::write(module.join("surviving-process"), b"still present")
17192 .expect("make scratch cgroup non-empty");
17193 let (logs, _guard) = crate::router::test_log::log_capture(tracing::Level::WARN);
17194
17195 remove_module_cgroup(&placement, "surviving-module");
17196
17197 let logs = crate::router::test_log::captured_logs(&logs);
17198 assert!(
17199 module.exists(),
17200 "failed removal must leave the cgroup intact"
17201 );
17202 assert!(
17203 logs.contains("could not remove module cgroup after process exit; continuing teardown")
17204 && logs.contains("surviving-module"),
17205 "best-effort removal must report the failure without returning it: {logs}"
17206 );
17207 }
17208
17209 #[test]
17210 fn cgroup_pre_exec_spawn_failure_names_the_cgroup_path() {
17211 let cgroup_path = PathBuf::from("/sys/fs/cgroup/subc-modules/broken-module");
17212 let reason = SuperviseError::Spawn {
17213 program: PathBuf::from("/bin/true"),
17214 source: io::Error::from_raw_os_error(13),
17215 cgroup_path: Some(cgroup_path.clone()),
17216 }
17217 .to_string();
17218
17219 assert!(
17220 reason.contains(&cgroup_path.display().to_string()),
17221 "a pre_exec spawn failure must name the cgroup path: {reason}"
17222 );
17223 }
17224}
17225
17226#[cfg(test)]
17227mod spawn_subscriber_lag_tests {
17228 use super::*;
17229
17230 #[tokio::test]
17235 async fn lagged_spawn_subscriber_receives_a_terminal_lagged_error_after_its_queued_frames() {
17236 let feed = SpawnEventFeed::default();
17237 feed.configure_incarnation("lag-incarnation".to_string());
17238 let (tx, mut rx) = mpsc::channel(1);
17241 feed.subscribe(ConnectionId::new(1), 7, 1, None, FrameSink::new(tx))
17242 .expect("subscribe");
17243 let emitted = SPAWN_SUBSCRIBER_BUFFER + 16;
17244 for index in 0..emitted {
17245 feed.emit_spawned(&format!("lag-module-{index}"), 1000, 0);
17246 tokio::task::yield_now().await;
17249 }
17250 assert_eq!(
17251 feed.subscriber_count(),
17252 0,
17253 "the lagged subscriber must be removed"
17254 );
17255
17256 let mut data = Vec::new();
17257 let mut last = None;
17258 loop {
17259 let next = tokio::time::timeout(Duration::from_secs(5), rx.recv())
17260 .await
17261 .expect("the forwarder must finish once the subscriber is dropped");
17262 let Some(outbound) = next else { break };
17263 let frame = outbound.frame;
17264 if frame.header.ty == FrameType::StreamData {
17265 assert!(last.is_none(), "no data may follow the terminal frame");
17266 let event: SpawnEvent = serde_json::from_slice(&frame.body).unwrap();
17267 data.push(event.cursor.seq);
17268 } else {
17269 assert!(last.is_none(), "exactly one terminal frame");
17270 last = Some(frame);
17271 }
17272 }
17273 assert!(!data.is_empty(), "queued frames drain before the terminal");
17274 for pair in data.windows(2) {
17275 assert_eq!(
17276 pair[1],
17277 pair[0] + 1,
17278 "queued frames arrive dense and in order"
17279 );
17280 }
17281 let terminal = last.expect("a lagged subscriber must receive a terminal frame");
17282 assert_eq!(terminal.header.ty, FrameType::Error);
17283 assert_eq!(terminal.header.corr, 7);
17284 let body: subc_protocol::ErrorBody = serde_json::from_slice(&terminal.body).unwrap();
17285 assert_eq!(body.code, SPAWN_SUBSCRIBER_LAGGED_CODE);
17286 let detail = body.detail.expect("lagged error carries detail");
17287 assert_eq!(
17288 detail["first_undelivered_cursor"]["seq"],
17289 data.last().unwrap() + 1,
17290 "the named cursor is the first event the subscriber did not receive"
17291 );
17292 assert_eq!(
17293 detail["first_undelivered_cursor"]["daemon_incarnation"],
17294 "lag-incarnation"
17295 );
17296 }
17297}
17298
17299#[cfg(test)]
17300mod terminal_history_read_concurrency_tests {
17301 use super::*;
17302 use crate::terminal_journal::read_pause;
17303 use cortexkit_test_support::ScratchDir;
17304 use std::sync::mpsc as std_mpsc;
17305
17306 fn journaled_ring(
17307 journal: &Arc<crate::terminal_journal::TerminalJournal>,
17308 ) -> Arc<Mutex<TerminalRing>> {
17309 Arc::new(Mutex::new(
17310 TerminalRing::new(TerminalRingConfig::default(), 1)
17311 .with_journal(Some(Arc::clone(journal))),
17312 ))
17313 }
17314
17315 fn crash(at_ms: u64) -> ExitReport {
17316 ExitReport {
17317 kind: ExitKind::Crash,
17318 code: Some(1),
17319 signal: None,
17320 at_ms,
17321 }
17322 }
17323
17324 fn record_within(
17327 module_id: &'static str,
17328 ring: &Arc<Mutex<TerminalRing>>,
17329 at_ms: u64,
17330 bound: Duration,
17331 ) -> bool {
17332 let ring = Arc::clone(ring);
17333 let (done, done_rx) = std_mpsc::channel();
17334 std::thread::spawn(move || {
17335 record_terminal(
17336 module_id,
17337 &ring,
17338 &SpawnEventFeed::default(),
17339 &crash(at_ms),
17340 TerminalDisposition::Restarting,
17341 );
17342 let _ = done.send(());
17343 });
17344 done_rx.recv_timeout(bound).is_ok()
17345 }
17346
17347 #[test]
17352 fn exits_recorded_during_a_paused_history_read_are_not_blocked_or_half_merged() {
17353 let dir = ScratchDir::new("terminal-history-concurrent-read");
17354 let path = dir.join("terminals.jsonl");
17355 let journal = Arc::new(crate::terminal_journal::TerminalJournal::open(
17356 path.clone(),
17357 "daemon".into(),
17358 ));
17359 let reader_ring = journaled_ring(&journal);
17360 let other_ring = journaled_ring(&journal);
17361 assert!(record_within(
17362 "reader-module",
17363 &reader_ring,
17364 10,
17365 Duration::from_secs(5)
17366 ));
17367
17368 let (started, release) = read_pause::install(&path);
17369 let reading = {
17370 let ring = Arc::clone(&reader_ring);
17371 std::thread::spawn(move || durable_terminal_history_of(&ring, "reader-module"))
17372 };
17373 started
17374 .recv_timeout(Duration::from_secs(5))
17375 .expect("the history read reached its pause");
17376
17377 let bound = Duration::from_secs(1);
17378 assert!(
17379 record_within("other-module", &other_ring, 20, bound),
17380 "another module's exit waited on a history read (journal writer held)"
17381 );
17382 assert!(
17383 record_within("reader-module", &reader_ring, 30, bound),
17384 "the read module's own exit waited on its history read (ring held)"
17385 );
17386
17387 drop(release);
17388 let paused = reading.join().unwrap();
17389 assert_eq!(
17390 paused.entries.iter().map(|e| e.at_ms).collect::<Vec<_>>(),
17391 vec![10],
17392 "an exit recorded after the read began lands in neither half of it"
17393 );
17394 assert_eq!(paused.journal_skipped_lines, 0);
17395 assert_eq!(paused.journal_read_errors, 0);
17396
17397 let after = durable_terminal_history_of(&reader_ring, "reader-module");
17398 assert_eq!(
17399 after.entries.iter().map(|e| e.at_ms).collect::<Vec<_>>(),
17400 vec![10, 30],
17401 "the next read merges ring and journal with no duplicate"
17402 );
17403 assert_eq!(after.journal_skipped_lines, 0);
17404 }
17405}
17406
17407#[cfg(test)]
17412mod stderr_settle_tests {
17413 use std::{
17414 future::Future,
17415 io,
17416 pin::Pin,
17417 sync::{Arc, Mutex},
17418 task::{Context, Poll},
17419 time::Duration,
17420 };
17421
17422 use tokio::{
17423 io::{AsyncRead, ReadBuf},
17424 sync::oneshot,
17425 time::Instant,
17426 };
17427
17428 use super::{settle_stderr_pump, settle_stdout_pump, StderrPump};
17429 use crate::stderr_tail::{
17430 pump_stderr_to, pump_stdout_to, split_capture_stamp, untimed, CaptureState,
17431 ChildOutputSink, OutputSink, StderrRing, StderrTailConfig, TailEntry,
17432 };
17433
17434 const BOUND: Duration = Duration::from_millis(250);
17435
17436 struct HeldReader {
17440 before: Option<Vec<u8>>,
17441 gate: Option<oneshot::Receiver<()>>,
17442 after: io::Cursor<Vec<u8>>,
17443 }
17444
17445 impl AsyncRead for HeldReader {
17446 fn poll_read(
17447 mut self: Pin<&mut Self>,
17448 cx: &mut Context<'_>,
17449 buf: &mut ReadBuf<'_>,
17450 ) -> Poll<io::Result<()>> {
17451 if let Some(bytes) = self.before.take() {
17452 buf.put_slice(&bytes);
17453 return Poll::Ready(Ok(()));
17454 }
17455 if let Some(gate) = self.gate.as_mut() {
17456 match Pin::new(gate).poll(cx) {
17457 Poll::Pending => return Poll::Pending,
17458 Poll::Ready(_) => self.gate = None,
17459 }
17460 }
17461 Pin::new(&mut self.after).poll_read(cx, buf)
17462 }
17463 }
17464
17465 struct DiscardSink;
17466
17467 impl OutputSink for DiscardSink {
17468 fn write_line(&mut self, _line: &[u8]) {}
17469 }
17470
17471 fn line(text: &str) -> TailEntry {
17472 TailEntry::Line {
17473 text: text.to_string(),
17474 truncated: false,
17475 at_ms: None,
17476 }
17477 }
17478
17479 fn lock(ring: &Arc<Mutex<StderrRing>>) -> std::sync::MutexGuard<'_, StderrRing> {
17480 ring.lock().unwrap()
17481 }
17482
17483 fn held_pump(
17487 ring: &Arc<Mutex<StderrRing>>,
17488 before: &str,
17489 after: &str,
17490 ) -> (StderrPump, oneshot::Sender<()>) {
17491 let generation = lock(ring).begin_process();
17492 let (release, gate) = oneshot::channel();
17493 let reader = HeldReader {
17494 before: Some(before.as_bytes().to_vec()),
17495 gate: Some(gate),
17496 after: io::Cursor::new(after.as_bytes().to_vec()),
17497 };
17498 let task = tokio::spawn(pump_stderr_to(
17499 reader,
17500 Arc::clone(ring),
17501 generation,
17502 DiscardSink,
17503 ));
17504 (StderrPump { task, generation }, release)
17505 }
17506
17507 async fn wait_until(ring: &Arc<Mutex<StderrRing>>, done: impl Fn(&StderrRing) -> bool) {
17508 for _ in 0..1000 {
17509 if done(&lock(ring)) {
17510 return;
17511 }
17512 tokio::time::sleep(Duration::from_millis(1)).await;
17513 }
17514 panic!(
17515 "ring never reached the expected state: {:?}",
17516 lock(ring).snapshot(None, None)
17517 );
17518 }
17519
17520 #[tokio::test(start_paused = true)]
17521 async fn a_crash_line_the_reader_had_not_reached_by_the_bound_is_kept_before_the_restart() {
17522 let ring = Arc::new(Mutex::new(StderrRing::new(StderrTailConfig::default())));
17523 let (pump, release) = held_pump(&ring, "booting\n", "config error: missing storage\n");
17524
17525 settle_stderr_pump("crasher", &ring, pump, BOUND).await;
17526 let before_release = lock(&ring).snapshot(None, None);
17527 assert!(
17528 matches!(before_release.capture, CaptureState::Incomplete { .. }),
17529 "a reader that has not reached EOF cannot claim a whole tail: {before_release:?}"
17530 );
17531
17532 let next = lock(&ring).begin_process();
17535 lock(&ring).push_line_from(next, "next process booting");
17536 release.send(()).unwrap();
17537 wait_until(&ring, |ring| {
17538 ring.snapshot(None, None).capture == CaptureState::Captured
17539 })
17540 .await;
17541
17542 assert_eq!(
17543 untimed(lock(&ring).snapshot(None, None).entries),
17544 vec![
17545 line("booting"),
17546 line("config error: missing storage"),
17547 TailEntry::ProcessStart,
17548 line("next process booting"),
17549 ],
17550 "the crash's last line must survive a slow reader and stay in the crashed process's section"
17551 );
17552 }
17553
17554 #[tokio::test(start_paused = true)]
17555 async fn a_pipe_held_open_by_a_descendant_reads_incomplete_without_delaying_the_restart_past_the_bound(
17556 ) {
17557 let ring = Arc::new(Mutex::new(StderrRing::new(StderrTailConfig::default())));
17558 let (pump, _held) = held_pump(&ring, "parent exiting\n", "");
17561
17562 let started = Instant::now();
17563 settle_stderr_pump("orphaning", &ring, pump, BOUND).await;
17564 assert_eq!(
17565 started.elapsed(),
17566 BOUND,
17567 "the restart must wait exactly the bound for a pipe that stays open, no longer"
17568 );
17569
17570 let next = lock(&ring).begin_process();
17571 lock(&ring).push_line_from(next, "next process booting");
17572 tokio::time::sleep(Duration::from_secs(60)).await;
17573
17574 let snapshot = lock(&ring).snapshot(None, None);
17575 match &snapshot.capture {
17576 CaptureState::Incomplete { reason } => assert!(
17577 reason.contains("had not reached EOF") && reason.contains("250ms"),
17578 "the reason must say what is missing and after how long: {reason}"
17579 ),
17580 other => panic!("expected Incomplete while the pipe is held open, got {other:?}"),
17581 }
17582 assert_eq!(
17583 untimed(snapshot.entries),
17584 vec![
17585 line("parent exiting"),
17586 TailEntry::ProcessStart,
17587 line("next process booting"),
17588 ]
17589 );
17590 }
17591
17592 #[tokio::test(start_paused = true)]
17593 async fn a_reader_that_reaches_eof_within_the_bound_leaves_the_tail_captured() {
17594 let ring = Arc::new(Mutex::new(StderrRing::new(StderrTailConfig::default())));
17595 let (pump, release) = held_pump(&ring, "one\n", "two\n");
17596 release.send(()).unwrap();
17597
17598 settle_stderr_pump("clean", &ring, pump, BOUND).await;
17599
17600 let snapshot = lock(&ring).snapshot(None, None);
17601 assert_eq!(snapshot.capture, CaptureState::Captured);
17602 assert_eq!(untimed(snapshot.entries), vec![line("one"), line("two")]);
17603 }
17604
17605 #[tokio::test(start_paused = true)]
17606 async fn stdout_buffered_past_the_exit_drain_bound_reaches_the_capture_file() {
17607 const BURST: usize = 150;
17608 let capture = cortexkit_test_support::ScratchDir::new("stdout-exit-drain");
17609 let path = capture.join("two-pipe.stderr.log");
17610 let sink = ChildOutputSink::open(&path, cortexkit_log::Retention::default()).unwrap();
17611 let ring = Arc::new(Mutex::new(StderrRing::new(StderrTailConfig::default())));
17612 let generation = lock(&ring).begin_process();
17613 let stdout = (0..BURST)
17614 .map(|i| format!("out-{i:04}-{}\n", "o".repeat(64)))
17615 .collect::<String>();
17616 let stderr = (0..BURST)
17617 .map(|i| format!("err-{i:04}-{}\n", "e".repeat(64)))
17618 .collect::<String>();
17619 let first_chunk = stdout.find('\n').unwrap() + 1 + "out-0001-".len();
17624 let (release, gate) = oneshot::channel();
17625 let reader = HeldReader {
17626 before: Some(stdout.as_bytes()[..first_chunk].to_vec()),
17627 gate: Some(gate),
17628 after: io::Cursor::new(stdout.as_bytes()[first_chunk..stdout.len() - 1].to_vec()),
17629 };
17630 let (finished, completion) = oneshot::channel();
17631 let stdout_sink = sink.clone();
17632 let pump = tokio::spawn(async move {
17633 pump_stdout_to(reader, stdout_sink).await;
17634 finished.send(()).unwrap();
17635 });
17636 pump_stderr_to(io::Cursor::new(stderr.into_bytes()), ring, generation, sink).await;
17637
17638 let started = Instant::now();
17639 settle_stdout_pump("two-pipe", pump, BOUND).await;
17640 assert_eq!(
17641 started.elapsed(),
17642 BOUND,
17643 "a late reader must not hold up restart"
17644 );
17645 assert_eq!(
17646 std::fs::read_to_string(&path).unwrap().lines().count(),
17647 BURST + 1
17648 );
17649
17650 release
17651 .send(())
17652 .expect("stdout reader must survive the exit drain bound");
17653 tokio::time::timeout(Duration::from_secs(1), completion)
17654 .await
17655 .expect("released stdout reader must reach EOF")
17656 .expect("stdout capture must finish at EOF, not be cancelled");
17657 let contents = std::fs::read_to_string(&path).unwrap();
17658 let mut lines = contents
17659 .lines()
17660 .map(|line| split_capture_stamp(line).expect("capture stamp").1)
17661 .collect::<Vec<_>>();
17662 let mut expected = (0..BURST)
17663 .flat_map(|i| {
17664 [
17665 format!("out-{i:04}-{}", "o".repeat(64)),
17666 format!("err-{i:04}-{}", "e".repeat(64)),
17667 ]
17668 })
17669 .collect::<Vec<_>>();
17670 lines.sort_unstable();
17671 expected.sort_unstable();
17672 assert_eq!(
17673 lines, expected,
17674 "both pipes must retain every complete line exactly once"
17675 );
17676 }
17677}
17678
17679#[cfg(all(test, windows))]
17693mod job_containment_tests {
17694 use super::*;
17695 use cortexkit_test_support::ScratchDir;
17696 use std::{
17697 path::{Path, PathBuf},
17698 sync::{Arc, Mutex},
17699 time::{Duration, Instant},
17700 };
17701
17702 fn stub_path() -> PathBuf {
17708 let mut path = std::env::current_exe().expect("current_exe available in tests");
17709 path.pop();
17710 path.pop();
17711 path.push("fake-aft-stub.exe");
17712 assert!(
17713 path.exists(),
17714 "fake-aft-stub not built at {}: run `cargo test -p subc-core` (which builds \
17715 [[bin]] targets) rather than `cargo test -p subc-core --lib` (which does not)",
17716 path.display()
17717 );
17718 path
17719 }
17720
17721 fn read_grandchild_pid(path: &Path) -> u32 {
17723 let deadline = Instant::now() + Duration::from_secs(10);
17724 loop {
17725 if let Ok(contents) = std::fs::read_to_string(path) {
17726 if let Ok(pid) = contents.trim().parse() {
17727 return pid;
17728 }
17729 }
17730 assert!(
17731 Instant::now() < deadline,
17732 "the stub never recorded a grandchild pid at {}",
17733 path.display()
17734 );
17735 std::thread::sleep(Duration::from_millis(10));
17736 }
17737 }
17738
17739 struct Fixture {
17742 _dir: ScratchDir,
17743 module_id: String,
17744 grandchild: u32,
17745 child: Option<SupervisedChild>,
17746 registry: Arc<Registry>,
17747 snapshot: Arc<Mutex<SupervisorSnapshot>>,
17748 terminal_ring: Arc<Mutex<TerminalRing>>,
17749 spawn_events: SpawnEventFeed,
17750 }
17751
17752 fn fixture(label: &str, module_id: &str) -> Fixture {
17753 let dir = ScratchDir::new(label);
17754 let pid_file = dir.join("grandchild.pid");
17755 let supervisor = Supervisor::new_for_test(
17756 Arc::new(Registry::default()),
17757 RestartPolicy::new(3, Duration::ZERO),
17758 );
17759 let runtime = supervisor.runtime_config();
17760 let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::starting()));
17761 let spec = ModuleSpec {
17762 module_id: module_id.to_string(),
17763 program: stub_path(),
17764 args: Vec::new(),
17768 env: vec![
17769 ("FAKE_AFT_NEVER_CONNECT".to_string(), "1".to_string()),
17770 (
17771 "FAKE_AFT_GRANDCHILD_PID_FILE".to_string(),
17772 pid_file.display().to_string(),
17773 ),
17774 ],
17775 reserved: false,
17776 reserved_prefixes: Vec::new(),
17777 protocol: ModuleProtocol::Subc,
17778 overlap: Default::default(),
17779 carry_containment: false,
17780 };
17781 let child = spawn_and_mark_running(&spec, &runtime, &snapshot)
17782 .expect("spawn the supervised fixture");
17783 let grandchild = read_grandchild_pid(&pid_file);
17784 Fixture {
17785 _dir: dir,
17786 module_id: module_id.to_string(),
17787 grandchild,
17788 child: Some(child),
17789 registry: Arc::new(Registry::default()),
17790 snapshot,
17791 terminal_ring: Arc::clone(&runtime.terminal_ring),
17792 spawn_events: SpawnEventFeed::default(),
17793 }
17794 }
17795
17796 impl Fixture {
17797 async fn drain(&mut self) {
17799 let child = self
17800 .child
17801 .take()
17802 .expect("the fixture child is still present");
17803 drain_child_to_state(
17804 &self.module_id,
17805 ModuleProtocol::Subc,
17806 StopNotice::NotSent,
17809 &self.registry,
17810 None,
17811 &self.snapshot,
17812 &self.terminal_ring,
17813 &self.spawn_events,
17814 child,
17815 Duration::from_millis(500),
17816 ModuleState::Stopped,
17817 Some(false),
17818 &FailureContext::operator(),
17819 )
17820 .await
17821 .expect("drain the supervised fixture");
17822 }
17823 }
17824
17825 #[tokio::test]
17831 async fn teardown_reaps_the_grandchild() {
17832 let mut fixture = fixture("teardown-grandchild", "tree-teardown");
17833 let grandchild = fixture.grandchild;
17834
17835 assert!(
17836 subc_jobobject::process_exists(grandchild),
17837 "grandchild {grandchild} must be alive before teardown, or this proves nothing"
17838 );
17839
17840 fixture.drain().await;
17841
17842 assert!(
17843 subc_jobobject::wait_for_process_exit(grandchild, Duration::from_secs(10)),
17844 "grandchild {grandchild} outlived module teardown: the tree was not contained"
17845 );
17846 }
17847
17848 #[test]
17861 fn an_uncontained_grandchild_survives_a_direct_child_kill() {
17862 let dir = ScratchDir::new("teardown-uncontained");
17863 let pid_file = dir.join("grandchild.pid");
17864 let mut child = std::process::Command::new(stub_path())
17865 .env("FAKE_AFT_NEVER_CONNECT", "1")
17866 .env(
17867 "FAKE_AFT_GRANDCHILD_PID_FILE",
17868 pid_file.display().to_string(),
17869 )
17870 .stdin(std::process::Stdio::null())
17871 .stdout(std::process::Stdio::null())
17872 .stderr(std::process::Stdio::null())
17873 .spawn()
17874 .expect("spawn the uncontained fixture");
17875 let grandchild = read_grandchild_pid(&pid_file);
17876
17877 child.kill().expect("kill the direct child");
17879 let _ = child.wait();
17880
17881 assert!(
17882 subc_jobobject::process_exists(grandchild),
17883 "grandchild {grandchild} died with the direct child, so this control no longer \
17884 distinguishes contained from uncontained teardown and the regression test is \
17885 passing vacuously"
17886 );
17887
17888 kill_tree(grandchild);
17891 }
17892
17893 #[tokio::test]
17902 async fn dropping_containment_reaps_the_grandchild() {
17903 let mut fixture = fixture("drop-containment", "tree-drop");
17904 let grandchild = fixture.grandchild;
17905
17906 assert!(subc_jobobject::process_exists(grandchild));
17907
17908 fixture.child.as_mut().expect("child present").job = None;
17910
17911 assert!(
17912 subc_jobobject::wait_for_process_exit(grandchild, Duration::from_secs(10)),
17913 "grandchild {grandchild} survived the containment handle closing, so a daemon \
17914 crash would leave the tree behind"
17915 );
17916 }
17917
17918 fn kill_tree(pid: u32) {
17920 let _ = std::process::Command::new("taskkill.exe")
17921 .args(["/PID", &pid.to_string(), "/T", "/F"])
17922 .stdin(std::process::Stdio::null())
17923 .stdout(std::process::Stdio::null())
17924 .stderr(std::process::Stdio::null())
17925 .status();
17926 assert!(
17927 subc_jobobject::wait_for_process_exit(pid, Duration::from_secs(10)),
17928 "could not clean up grandchild {pid}"
17929 );
17930 }
17931}
17932
17933#[cfg(test)]
17934mod privacy_trampoline_configuration_tests {
17935 #[tokio::test]
17936 #[cfg_attr(not(target_os = "macos"), ignore = "requires macOS privacy trampoline")]
17937 async fn macos_spawn_without_a_configured_trampoline_refuses_by_name() {
17938 #[cfg(target_os = "macos")]
17939 {
17940 let supervisor = super::Supervisor::new(
17941 std::sync::Arc::new(crate::Registry::default()),
17942 super::RestartPolicy::default(),
17943 );
17944 let error = supervisor.spawn(spec()).unwrap_err();
17945 assert!(
17946 error
17947 .to_string()
17948 .contains("no privacy trampoline configured"),
17949 "{error}"
17950 );
17951 }
17952 }
17953
17954 #[tokio::test]
17955 #[cfg_attr(not(target_os = "macos"), ignore = "requires macOS privacy trampoline")]
17956 async fn macos_wrong_trampoline_is_rejected_by_the_startup_probe() {
17957 #[cfg(target_os = "macos")]
17958 {
17959 let supervisor = super::Supervisor::new(
17960 std::sync::Arc::new(crate::Registry::default()),
17961 super::RestartPolicy::default(),
17962 )
17963 .with_privacy_trampoline(std::env::current_exe().unwrap());
17964 let error = supervisor.spawn(spec()).unwrap_err();
17965 assert!(
17966 error
17967 .to_string()
17968 .contains("binary does not implement the privacy trampoline protocol"),
17969 "{error}"
17970 );
17971 }
17972 }
17973
17974 #[cfg(target_os = "macos")]
17975 fn spec() -> super::ModuleSpec {
17976 super::ModuleSpec {
17977 module_id: "privacy-configuration".into(),
17978 program: "/bin/sleep".into(),
17979 args: vec!["30".into()],
17980 env: vec![],
17981 reserved: false,
17982 reserved_prefixes: vec![],
17983 protocol: subc_control::ModuleProtocol::None,
17984 overlap: super::ModuleOverlap::Exclusive,
17985 carry_containment: false,
17986 }
17987 }
17988}
17989
17990#[cfg(test)]
17991mod privacy_exec_boundary_tests {
17992 #[cfg(target_os = "macos")]
17993 use super::*;
17994 #[cfg(target_os = "macos")]
17995 use std::{
17996 io::{Read, Write},
17997 net::{TcpListener, TcpStream},
17998 };
17999
18000 #[cfg(target_os = "macos")]
18003 pub(super) fn before_image_sample(spec: &ModuleSpec, pid: u32) {
18004 if let Some((_, path)) = spec
18005 .env
18006 .iter()
18007 .find(|(key, _)| key == "SUBC_TEST_PRIVACY_SAMPLE_BARRIER")
18008 {
18009 let mut barrier = TcpStream::connect(path).unwrap();
18010 barrier
18011 .set_read_timeout(Some(Duration::from_secs(30)))
18012 .unwrap();
18013 barrier.write_all(&pid.to_ne_bytes()).unwrap();
18014 let mut release = [0];
18015 barrier.read_exact(&mut release).unwrap();
18016 assert_eq!(&release, b"X");
18017 }
18018 }
18019
18020 #[cfg(target_os = "macos")]
18021 fn spec(program: &str, args: &[&str]) -> ModuleSpec {
18022 ModuleSpec {
18023 module_id: "privacy-boundary".into(),
18024 program: program.into(),
18025 args: args.iter().map(|arg| (*arg).into()).collect(),
18026 env: vec![],
18027 reserved: false,
18028 reserved_prefixes: vec![],
18029 protocol: ModuleProtocol::None,
18030 overlap: ModuleOverlap::Exclusive,
18031 carry_containment: false,
18032 }
18033 }
18034
18035 #[cfg(target_os = "macos")]
18036 async fn accept(listener: TcpListener) -> TcpStream {
18037 let listener = tokio::net::TcpListener::from_std({
18039 listener.set_nonblocking(true).unwrap();
18040 listener
18041 })
18042 .unwrap();
18043 let (stream, _) = tokio::time::timeout(Duration::from_secs(30), listener.accept())
18044 .await
18045 .unwrap()
18046 .unwrap();
18047 let stream = stream.into_std().unwrap();
18048 stream.set_nonblocking(false).unwrap();
18049 stream
18050 .set_read_timeout(Some(Duration::from_secs(30)))
18051 .unwrap();
18052 stream
18053 }
18054
18055 #[tokio::test]
18056 #[cfg_attr(not(target_os = "macos"), ignore = "requires macOS privacy trampoline")]
18057 async fn macos_roster_withholds_a_nonnull_trampoline_image_until_exec_confirmation() {
18058 #[cfg(target_os = "macos")]
18059 {
18060 let root = cortexkit_test_support::ScratchDir::new("privacy-roster-barrier");
18061 let exec_listener = TcpListener::bind("127.0.0.1:0").unwrap();
18064 let sample_listener = TcpListener::bind("127.0.0.1:0").unwrap();
18065 let record = root.join("live-children.json");
18066 let supervisor =
18067 Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default())
18068 .with_live_children_record(&record);
18069 let runtime = supervisor.runtime_config();
18070 let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::starting()));
18071 let mut spec = spec("/bin/sleep", &["30"]);
18072 spec.env = vec![
18073 (
18074 "SUBC_TEST_PRIVACY_EXEC_BARRIER".into(),
18075 exec_listener.local_addr().unwrap().to_string(),
18076 ),
18077 (
18078 "SUBC_TEST_PRIVACY_SAMPLE_BARRIER".into(),
18079 sample_listener.local_addr().unwrap().to_string(),
18080 ),
18081 ];
18082 let spawn = tokio::task::spawn_blocking(move || {
18083 spawn_and_mark_running(&spec, &runtime, &snapshot).unwrap()
18084 });
18085 let mut sample = accept(sample_listener).await;
18086 let mut pid = [0; 4];
18087 sample.read_exact(&mut pid).unwrap();
18088 let pid = u32::from_ne_bytes(pid);
18089 let mut exec = accept(exec_listener).await;
18090 let mut ready = [0];
18091 exec.read_exact(&mut ready).unwrap();
18092 assert_eq!(&ready, b"R");
18093 let trampoline = subc_os::file_identity(&test_privacy_trampoline()).unwrap();
18096 assert_eq!(
18097 observe_spawned_image(pid).unwrap().executable,
18098 Some(trampoline)
18099 );
18100 sample.write_all(b"X").unwrap();
18101 let mut child = spawn.await.unwrap();
18102 let early = crate::live_children::read_record(&record).unwrap();
18103 assert_eq!(early.len(), 1);
18104 assert_eq!(early[0].pid, pid);
18105 assert_eq!(
18106 early[0].executable, None,
18107 "unconfirmed trampoline image entered the roster"
18108 );
18109 assert!(child.report_ready.get().is_none());
18110 child.privacy_exec.as_mut().unwrap().deadline =
18113 tokio::time::Instant::now() + Duration::from_secs(30);
18114 exec.write_all(b"X").unwrap();
18115 child.confirm_privacy_exec().await;
18116 assert_eq!(child.spawn_failure, None);
18117 assert!(child.report_ready.get().is_some());
18118 let confirmed = crate::live_children::read_record(&record).unwrap();
18119 let module = subc_os::file_identity(std::path::Path::new("/bin/sleep")).unwrap();
18120 assert_ne!(module, trampoline);
18121 assert_eq!(confirmed[0].executable, Some(module.into()));
18122 child.start_kill().unwrap();
18123 child.wait().await.unwrap();
18124 child.release_roster();
18125 }
18126 }
18127
18128 #[tokio::test]
18129 #[cfg_attr(not(target_os = "macos"), ignore = "requires macOS privacy trampoline")]
18130 async fn macos_already_exited_121_in_try_wait_is_a_module_exit_not_a_trampoline_refusal() {
18131 #[cfg(target_os = "macos")]
18132 {
18133 let registry = Arc::new(Registry::default());
18134 let policy = RestartPolicy::new(0, Duration::ZERO);
18135 let supervisor = Supervisor::new_for_test(Arc::clone(®istry), policy);
18136 let runtime = supervisor.runtime_config();
18137 let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::starting()));
18138 let spec = spec("/bin/sh", &["-c", "exit 121"]);
18139 let mut child = spawn_and_mark_running(&spec, &runtime, &snapshot).unwrap();
18143 let pid = child.pid;
18144 tokio::task::spawn_blocking(move || {
18145 subc_os::fork_exec_test::wait_for_child_exit_without_reaping(pid)
18146 })
18147 .await
18148 .unwrap()
18149 .unwrap();
18150 child.privacy_exec.as_mut().unwrap().deadline =
18151 tokio::time::Instant::now() + Duration::from_secs(30);
18152 let status = child.wait().await.unwrap();
18153 assert_eq!(status.code(), Some(121));
18154 assert!(child.privacy_exec.is_none());
18155 assert!(
18156 child.report_ready.get().is_none(),
18157 "an exited module must not publish a live pid"
18158 );
18159 let report = classify_reaped_child_exit(&snapshot, &child, &status);
18160 on_child_exit(
18161 &spec,
18162 policy,
18163 ®istry,
18164 &snapshot,
18165 &runtime.terminal_ring,
18166 &runtime.spawn_events,
18167 &runtime.child_roster,
18168 report,
18169 true,
18170 )
18171 .await;
18172 let state = lock_snapshot(&snapshot).unwrap();
18173 assert_eq!(state.state, ModuleState::Failed);
18174 assert_eq!(state.last_exit.as_ref().unwrap().code, Some(121));
18175 assert_eq!(state.reported_pid(), None);
18176 drop(state);
18177 let history = runtime.terminal_ring.lock().unwrap().snapshot();
18178 assert_eq!(history.entries.len(), 1);
18179 let terminal = &history.entries[0];
18180 assert_eq!(terminal.exit_code, Some(121));
18181 assert_eq!(terminal.exit_kind, subc_control::TerminalExitKind::Crash);
18182 assert_eq!(terminal.disposition, TerminalDisposition::Failed);
18183 assert_eq!(
18184 terminal.disposition_detail.as_deref(),
18185 Some(policy.budget_exhausted_detail().as_str()),
18186 "module exit 121 was classified as a trampoline refusal: {terminal:?}"
18187 );
18188 assert_eq!(child.spawn_failure, None);
18189 child.release_roster();
18190 }
18191 }
18192}
18193
18194#[cfg(all(test, unix))]
18198mod launch_nonce_descriptor_tests {
18199 use super::{ChildRoster, ModuleProtocol, ModuleSpec, SupervisorHandle};
18200 use crate::stderr_tail::{StderrRing, StderrTailConfig};
18201 use cortexkit_test_support::ScratchDir;
18202 use std::{
18203 path::PathBuf,
18204 sync::{Arc, Mutex},
18205 time::{Duration, Instant},
18206 };
18207
18208 async fn probe(role: super::SpawnRole) {
18209 let scratch = ScratchDir::new("launch-nonce-descriptor");
18210 let fd_copy = scratch.join("from-descriptor");
18211 let env_copy = scratch.join("environment");
18212 let script = format!(
18213 "env > '{env}.tmp' && mv '{env}.tmp' '{env}'; cat <&3 > '{fd}.tmp' && mv '{fd}.tmp' '{fd}'; sleep 30",
18214 fd = fd_copy.display(), env = env_copy.display(),
18215 );
18216 let xdg = |name: &str| (name.to_string(), scratch.join(name).display().to_string());
18217 let spec = ModuleSpec {
18218 module_id: "nonce-descriptor-probe".to_string(),
18219 program: PathBuf::from("/bin/sh"),
18220 args: vec!["-c".to_string(), script],
18221 env: vec![
18222 xdg("XDG_DATA_HOME"),
18223 xdg("XDG_RUNTIME_DIR"),
18224 xdg("XDG_CONFIG_HOME"),
18225 ("SUBC_LAUNCH_NONCE".to_string(), "must-not-leak".to_string()),
18226 ],
18227 reserved: true,
18228 reserved_prefixes: Vec::new(),
18229 protocol: ModuleProtocol::Subc,
18230 overlap: Default::default(),
18231 carry_containment: false,
18232 };
18233 let handle = SupervisorHandle::new();
18234 let ring = Arc::new(Mutex::new(StderrRing::new(StderrTailConfig::default())));
18235 let roster = ChildRoster::default();
18236 #[cfg(target_os = "macos")]
18237 {
18238 let path = super::test_privacy_trampoline();
18239 roster.set_privacy_trampoline(super::probe_privacy_trampoline(&path).map(|()| path));
18240 }
18241 let child = super::spawn_child_in_slot(
18242 &spec,
18243 None,
18244 Some(&handle),
18245 &ring,
18246 None,
18247 &roster,
18248 #[cfg(target_os = "linux")]
18249 None,
18250 role,
18251 matches!(role, super::SpawnRole::SwapCandidate),
18252 )
18253 .expect("spawn probe");
18254 let deadline = Instant::now() + Duration::from_secs(10);
18255 while !(fd_copy.exists() && env_copy.exists()) {
18256 assert!(Instant::now() < deadline, "probe never wrote its copies");
18257 tokio::time::sleep(Duration::from_millis(20)).await;
18258 }
18259 let nonce = std::fs::read_to_string(fd_copy).unwrap();
18260 assert!(!nonce.is_empty());
18261 let environment = std::fs::read_to_string(env_copy).unwrap();
18262 assert!(environment
18263 .lines()
18264 .any(|line| line.starts_with("SUBC_LAUNCH_NONCE_FD=3:")));
18265 let copy = environment
18266 .lines()
18267 .find_map(|line| line.strip_prefix("SUBC_LAUNCH_NONCE="));
18268 assert_eq!(
18269 copy, None,
18270 "Unix children must never receive the environment nonce"
18271 );
18272 if matches!(role, super::SpawnRole::Plain) {
18273 assert_eq!(
18274 handle.spawn_nonce(&spec.module_id).as_deref(),
18275 Some(nonce.as_str())
18276 );
18277 }
18278 drop(child);
18279 }
18280
18281 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
18282 async fn a_spawned_module_receives_its_nonce_only_on_descriptor_3() {
18283 probe(super::SpawnRole::Plain).await;
18284 }
18285
18286 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
18287 async fn swap_candidate_receives_its_nonce_only_on_descriptor_3() {
18288 probe(super::SpawnRole::SwapCandidate).await;
18289 }
18290}
18291
18292#[cfg(all(test, target_os = "linux"))]
18293#[path = "supervise/cgroup_reap_tests.rs"]
18294mod cgroup_reap_tests;
18295
18296#[cfg(all(test, target_os = "linux"))]
18297mod cgroup_containment_tests {
18298 use super::*;
18299 use cortexkit_test_support::ScratchDir;
18300
18301 #[tokio::test]
18302 async fn linux_subtree_memory_covers_module_children() {
18303 let dir = ScratchDir::new("subtree-memory");
18304 let root = PathBuf::from(format!(
18305 "/sys/fs/cgroup/subc-memory-test-{}-{}",
18306 std::process::id(),
18307 unix_ms_now()
18308 ));
18309 if let Err(error) = std::fs::create_dir(&root) {
18310 assert!(
18311 std::env::var_os("SUBC_REQUIRE_CGROUP_TEST").is_none(),
18312 "required cgroup test cannot execute: {error}"
18313 );
18314 eprintln!("SKIP linux_subtree_memory_covers_module_children: {error}");
18315 return;
18316 }
18317 let placement = subc_cgroup::prepare_at(&root)
18318 .unwrap()
18319 .expect("delegated cgroup");
18320 let supervisor = Supervisor::new_for_test(
18321 Arc::new(Registry::default()),
18322 RestartPolicy::new(3, Duration::ZERO),
18323 )
18324 .with_cgroup_placement(Some(placement));
18325 let ready = dir.join("ready");
18326 let module = supervisor
18327 .spawn(ModuleSpec {
18328 module_id: "memory-tree".into(),
18329 program: PathBuf::from("/bin/sh"),
18330 args: vec![
18331 "-c".into(),
18332 "sleep 600 & sleep 600 & sleep 600 & echo ready > \"$1\"; wait".into(),
18333 "fixture".into(),
18334 ready.display().to_string(),
18335 ],
18336 env: ["XDG_DATA_HOME", "XDG_RUNTIME_DIR", "XDG_CONFIG_HOME"]
18337 .into_iter()
18338 .map(|key| (key.into(), dir.display().to_string()))
18339 .collect(),
18340 reserved: false,
18341 reserved_prefixes: Vec::new(),
18342 protocol: ModuleProtocol::None,
18343 overlap: Default::default(),
18344 carry_containment: false,
18345 })
18346 .unwrap();
18347 let deadline = tokio::time::Instant::now() + Duration::from_secs(10);
18348 while !ready.exists() {
18349 assert!(
18350 tokio::time::Instant::now() < deadline,
18351 "children never started"
18352 );
18353 tokio::time::sleep(Duration::from_millis(10)).await;
18354 }
18355 let readings = crate::subtree_resources::read(vec![module.subtree_target()]).await;
18356 let reading = &readings[0];
18357 assert!(
18358 reading.process_count.is_some_and(|count| count >= 4),
18359 "{reading:?}"
18360 );
18361 assert!(
18362 reading.memory_bytes.is_some_and(|bytes| bytes > 0),
18363 "{reading:?}"
18364 );
18365 module.stop().await.unwrap();
18366 drop(module);
18367 std::fs::remove_dir(root.join("subc-modules")).unwrap();
18368 std::fs::remove_dir(&root).unwrap();
18369 eprintln!("EXECUTED linux_subtree_memory_covers_module_children: {reading:?}");
18370 }
18371
18372 fn running(pid: u32) -> bool {
18373 std::fs::read_to_string(format!("/proc/{pid}/stat"))
18375 .ok()
18376 .and_then(|stat| {
18377 stat.rsplit_once(") ")
18378 .map(|(_, rest)| rest.starts_with('Z'))
18379 })
18380 .is_some_and(|zombie| !zombie)
18381 }
18382
18383 #[tokio::test]
18384 async fn linux_teardown_reaps_the_grandchild() {
18385 teardown_tree("linux_teardown_reaps_the_grandchild", false).await;
18386 }
18387
18388 #[tokio::test]
18389 async fn linux_shutdown_straggler_reaps_the_grandchild() {
18390 teardown_tree("linux_shutdown_straggler_reaps_the_grandchild", true).await;
18391 }
18392
18393 async fn teardown_tree(test_name: &str, shutdown: bool) {
18394 let dir = ScratchDir::new(test_name);
18395 let root = PathBuf::from(format!(
18396 "/sys/fs/cgroup/subc-tree-test-{test_name}-{}-{}",
18397 std::process::id(),
18398 unix_ms_now()
18399 ));
18400 if let Err(error) = std::fs::create_dir(&root) {
18401 assert!(
18402 std::env::var_os("SUBC_REQUIRE_CGROUP_TEST").is_none(),
18403 "required cgroup test cannot execute: {error}"
18404 );
18405 eprintln!(
18406 "SKIP {test_name}: no writable delegated cgroup at {}: {error}",
18407 root.display()
18408 );
18409 return;
18410 }
18411 let placement = subc_cgroup::prepare_at(&root)
18412 .expect("prepare isolated kernel cgroup")
18413 .expect("isolated cgroup is delegated");
18414 let module_id = "tree-teardown";
18415 let module = placement
18416 .module_path(module_id)
18417 .expect("create isolated module cgroup");
18418 if !module.join("cgroup.kill").exists() {
18419 std::fs::remove_dir(&module).unwrap();
18420 std::fs::remove_dir(root.join("subc-modules")).unwrap();
18421 std::fs::remove_dir(&root).unwrap();
18422 assert!(
18423 std::env::var_os("SUBC_REQUIRE_CGROUP_TEST").is_none(),
18424 "required cgroup.kill interface unavailable"
18425 );
18426 eprintln!("SKIP {test_name}: cgroup.kill unavailable (kernel < 5.14)");
18427 return;
18428 }
18429 let supervisor = Supervisor::new_for_test(
18430 Arc::new(Registry::default()),
18431 RestartPolicy::new(3, Duration::ZERO),
18432 )
18433 .with_cgroup_placement(Some(placement));
18434 let mut runtime = supervisor.runtime_config();
18435 runtime.child_roster = runtime
18436 .child_roster
18437 .for_module(Arc::new(Mutex::new(Duration::from_millis(100))));
18438 let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::starting()));
18439 let pid_file = dir.join("grandchild.pid");
18440 let spec = ModuleSpec {
18441 module_id: module_id.to_string(),
18442 program: PathBuf::from("/bin/sh"),
18443 args: vec![
18444 "-c".into(),
18445 "trap '' TERM; sleep 600 & echo $! > \"$1\"; wait".into(),
18446 "fixture".into(),
18447 pid_file.display().to_string(),
18448 ],
18449 env: ["XDG_DATA_HOME", "XDG_RUNTIME_DIR", "XDG_CONFIG_HOME"]
18450 .into_iter()
18451 .map(|key| (key.to_string(), dir.display().to_string()))
18452 .collect(),
18453 reserved: false,
18454 reserved_prefixes: Vec::new(),
18455 protocol: ModuleProtocol::None,
18456 overlap: Default::default(),
18457 carry_containment: false,
18458 };
18459 let child =
18460 spawn_and_mark_running(&spec, &runtime, &snapshot).expect("spawn supervised tree");
18461 let deadline = tokio::time::Instant::now() + Duration::from_secs(10);
18462 let grandchild: u32 = loop {
18463 if let Ok(contents) = std::fs::read_to_string(&pid_file) {
18464 if let Ok(pid) = contents.trim().parse() {
18465 break pid;
18466 }
18467 }
18468 assert!(
18469 tokio::time::Instant::now() < deadline,
18470 "grandchild pid was not recorded"
18471 );
18472 tokio::time::sleep(Duration::from_millis(10)).await;
18473 };
18474 assert!(
18475 running(grandchild),
18476 "grandchild must be alive before teardown"
18477 );
18478 if shutdown {
18479 let mut child = child;
18480 crate::child_roster::end_children_for_daemon_shutdown(
18481 &runtime.child_roster,
18482 false,
18483 std::future::pending(),
18484 )
18485 .await;
18486 child.wait().await.expect("reap shutdown straggler");
18487 } else {
18488 drain_child_to_state(
18489 module_id,
18490 ModuleProtocol::None,
18491 StopNotice::NotSent,
18492 &Registry::default(),
18493 None,
18494 &snapshot,
18495 &runtime.terminal_ring,
18496 &SpawnEventFeed::default(),
18497 child,
18498 Duration::from_millis(100),
18499 ModuleState::Stopped,
18500 Some(false),
18501 &FailureContext::operator(),
18502 )
18503 .await
18504 .expect("real supervisor teardown");
18505 }
18506 let deadline = tokio::time::Instant::now() + Duration::from_secs(5);
18507 while running(grandchild) && tokio::time::Instant::now() < deadline {
18508 tokio::time::sleep(Duration::from_millis(10)).await;
18509 }
18510 let survived = running(grandchild);
18511 if survived {
18513 let pid = rustix::process::Pid::from_raw(grandchild as i32).unwrap();
18514 let _ = rustix::process::kill_process(pid, rustix::process::Signal::KILL);
18515 tokio::time::sleep(Duration::from_millis(100)).await;
18516 }
18517 if module.exists() {
18518 std::fs::remove_dir(&module).expect("remove empty module cgroup");
18519 }
18520 std::fs::remove_dir(root.join("subc-modules")).unwrap();
18521 std::fs::remove_dir(&root).unwrap();
18522 assert!(
18523 !survived,
18524 "grandchild {grandchild} outlived module teardown"
18525 );
18526 eprintln!("EXECUTED {test_name}: grandchild {grandchild} killed in isolated cgroup");
18527 }
18528}
18529
18530#[cfg(all(test, unix))]
18533mod record_write_stall_tests {
18534 use super::*;
18535 use std::sync::atomic::{AtomicBool, Ordering};
18536 use subc_transport::{authenticate_client, ConnectionInfo, Endpoint, SCHEMA_VERSION};
18537
18538 const MODULE_AUTH_DEADLINE: Duration = Duration::from_secs(2);
18541 const HELD_RECORD_WRITE: Duration = Duration::from_secs(3);
18545
18546 fn sleeping_spec() -> ModuleSpec {
18558 ModuleSpec {
18559 module_id: "record-write-stall".into(),
18560 program: "/bin/sleep".into(),
18561 args: vec!["600".into()],
18562 env: vec![],
18563 reserved: false,
18564 reserved_prefixes: vec![],
18565 protocol: ModuleProtocol::None,
18566 overlap: Default::default(),
18567 carry_containment: false,
18568 }
18569 }
18570
18571 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
18572 async fn module_auth_completes_while_the_restart_waits_on_the_record_write() {
18573 let home = cortexkit_test_support::ScratchDir::new("record-write-stall");
18574 let supervisor =
18575 Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default())
18576 .with_live_children_record(home.join("live-children.json"));
18577 let module = supervisor.spawn(sleeping_spec()).unwrap();
18578 let old_pid = tokio::time::timeout(Duration::from_secs(10), async {
18583 loop {
18584 if let Some(pid) = lock_snapshot(&module.inner.snapshot)
18585 .unwrap()
18586 .reported_pid()
18587 {
18588 return pid;
18589 }
18590 tokio::time::sleep(Duration::from_millis(10)).await;
18591 }
18592 })
18593 .await
18594 .expect("the first process was admitted as the module");
18595
18596 let key = vec![0x42; 32];
18599 let daemon_id = [0x24; 16];
18600 let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
18601 let port = listener.local_addr().unwrap().port();
18602 let server = tokio::spawn(crate::server::serve_listener(
18603 listener,
18604 Arc::new(crate::router::Router::with_default_self_handler()),
18605 crate::server::ServerAuth::with_limits(
18606 key.clone(),
18607 daemon_id,
18608 "test",
18609 crate::server::DEFAULT_AUTH_DEADLINE,
18610 4,
18611 ),
18612 ));
18613
18614 let (held_tx, held_rx) = std::sync::mpsc::channel();
18617 let armed = AtomicBool::new(true);
18618 supervisor
18619 .child_roster
18620 .set_record_write_hook_for_test(move |live| {
18621 let admitting_replacement = live.values().any(|entry| entry.pid != old_pid);
18622 if admitting_replacement && armed.swap(false, Ordering::SeqCst) {
18623 let _ = held_tx.send(());
18624 std::thread::sleep(HELD_RECORD_WRITE);
18625 }
18626 });
18627
18628 let module_process = std::thread::spawn(move || {
18631 held_rx
18632 .recv_timeout(Duration::from_secs(30))
18633 .expect("the restart reached the replacement's record write");
18634 let conn = ConnectionInfo {
18635 schema: SCHEMA_VERSION,
18636 wire_version: None,
18637 endpoints: vec![Endpoint {
18638 host: "127.0.0.1".to_owned(),
18639 port,
18640 }],
18641 key,
18642 daemon_id,
18643 pid: std::process::id(),
18644 daemon_ver: "test".to_owned(),
18645 };
18646 tokio::runtime::Builder::new_current_thread()
18647 .enable_all()
18648 .build()
18649 .unwrap()
18650 .block_on(async {
18651 let started = Instant::now();
18652 let mut stream = tokio::net::TcpStream::connect(("127.0.0.1", port))
18653 .await
18654 .unwrap();
18655 let result =
18656 authenticate_client(&mut stream, &conn, MODULE_AUTH_DEADLINE).await;
18657 (result, started.elapsed())
18658 })
18659 });
18660
18661 module.restart(Some(0)).await.unwrap();
18662 let (auth, elapsed) = tokio::task::spawn_blocking(move || module_process.join().unwrap())
18663 .await
18664 .unwrap();
18665 assert!(
18666 auth.is_ok(),
18667 "the daemon must answer a module's handshake while a restart waits on the \
18668 live-children record write; got {auth:?} after {elapsed:?}"
18669 );
18670
18671 tokio::time::timeout(Duration::from_secs(20), async {
18672 while lock_snapshot(&module.inner.snapshot)
18673 .unwrap()
18674 .reported_pid()
18675 .is_none_or(|pid| pid == old_pid)
18676 {
18677 tokio::time::sleep(Duration::from_millis(10)).await;
18678 }
18679 })
18680 .await
18681 .expect("the restart published its replacement");
18682 module.stop().await.unwrap();
18683 server.abort();
18684 }
18685}
18686
18687#[cfg(all(test, unix))]
18688mod retained_status_tests {
18689 use super::*;
18690
18691 #[tokio::test]
18692 async fn cached_container_block_is_visible_in_status_and_health_without_rechecking() {
18693 use crate::live_children::{ContainerIdentity, RetainedReason};
18694 for (reason, carried, expected) in [
18695 (
18696 RetainedReason::ContainerSurvivors(vec![4242]),
18697 true,
18698 Some("waiting: container survivors (pids [4242])".to_owned()),
18699 ),
18700 (
18701 RetainedReason::ContainerUnverifiable("member inspection denied".into()),
18702 true,
18703 Some(
18704 "waiting: container survivors could not be verified: member inspection denied"
18705 .to_owned(),
18706 ),
18707 ),
18708 (RetainedReason::Survived, false, None),
18709 ] {
18710 let supervisor =
18711 Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default());
18712 let spec = ModuleSpec {
18713 module_id: "retained-status-fixture".into(),
18714 program: PathBuf::from("/not-spawned"),
18715 args: Vec::new(),
18716 env: Vec::new(),
18717 reserved: false,
18718 reserved_prefixes: Vec::new(),
18719 protocol: ModuleProtocol::None,
18720 overlap: ModuleOverlap::Exclusive,
18721 carry_containment: carried,
18722 };
18723 let module = supervisor
18724 .supervise_configured(spec.clone(), false)
18725 .unwrap();
18726 let guard = supervisor
18730 .child_roster
18731 .admit(
18732 spec.module_id,
18733 0,
18734 ModuleProtocol::None,
18735 None,
18736 crate::child_roster::RecordedIdentity {
18737 container: carried.then_some(ContainerIdentity {
18738 pgid: Some(0),
18739 created_start_time: Some(0),
18740 cgroup_path: None,
18741 ..ContainerIdentity::test_metadata()
18742 }),
18743 ..crate::child_roster::RecordedIdentity::default()
18744 },
18745 )
18746 .unwrap();
18747 guard.retain(reason);
18748 let status = module.status().unwrap();
18749 assert_eq!(status.waiting_for_previous_detail, expected.clone());
18750 assert_eq!(status.health.detail, expected);
18751 assert!(!status.process_alive);
18752 }
18753 }
18754}