1mod resource_launch;
2pub(crate) use resource_launch::{ResourceCommitError, ResourceLaunch};
3
4use std::{
5 collections::{HashMap, HashSet, VecDeque},
6 error::Error,
7 fmt, io,
8 path::PathBuf,
9 process::{ExitStatus, Stdio},
10 sync::{Arc, Mutex, OnceLock},
11 time::{Duration, SystemTime, UNIX_EPOCH},
12};
13
14use cortexkit_log::Retention;
15use serde_json::Value;
16use sha2::{Digest, Sha256};
17use subc_control::{
18 ClientControlPush, LiveSpawn, ModuleProtocol, RouteCloseReason, SpawnCursor, SpawnEvent,
19 SpawnEventKind, SpawnSnapshot, SupervisorHealthStatus, TerminalDisposition, TerminalExitKind,
20};
21use subc_protocol::{
22 manifest::{SelfSignalKind, SignalAnchor},
23 session::{
24 HealthReport, HealthStatus, ModuleControlCommand, ModuleControlRequest,
25 MODULE_CONTROL_OP_HEALTH_CHECK,
26 },
27 Flags, FrameType, Principal, Priority, SUBC_LAUNCH_NONCE_ENV, SUBC_MODULE_ID_ENV,
28};
29use tokio::{
30 process::{Child, Command},
31 sync::{mpsc, oneshot, watch, Mutex as AsyncMutex},
32 task::JoinHandle,
33 time::{sleep, sleep_until, timeout, timeout_at, Instant},
34};
35use tracing::{debug, error, info, warn};
36
37use crate::{
38 child_roster::ChildRoster,
39 daemon_config::{
40 CAPTURE_KEEP_ENV, CAPTURE_MAX_AGE_DAYS_ENV, CAPTURE_MAX_FILE_MB_ENV, CK_LOG_ENV,
41 },
42 forwarding::{
43 CloseReason, ForwardingError, ForwardingTable, GoodbyeTarget, ModuleControlRpcOutcome,
44 ModuleDrainTarget, PendingModuleControlRpc,
45 },
46 provenance::{ExecutableIdentityProbe, SpawnedFileIdentity},
47 registry::{ConnectionId, RegistrationEndReason, RegistryError},
48 stderr_tail::{
49 pump_stderr_to, pump_stdout_to, ChildOutputSink, StderrRing, StderrTailConfig,
50 StderrTailSnapshot,
51 },
52 terminal_ring::{TerminalHistorySnapshot, TerminalRecord, TerminalRing, TerminalRingConfig},
53 Frame, FrameSink, Registry,
54};
55
56#[cfg(not(windows))]
57use crate::provenance::spawned_file_identity;
58
59#[path = "supervise_swap.rs"]
60mod swap;
61
62#[path = "supervise_activity.rs"]
63mod activity;
64
65#[cfg(unix)]
66#[path = "supervise/admission.rs"]
67mod admission;
68#[cfg(all(test, windows))]
69mod carried_job_tests;
70pub(crate) mod containment;
71#[cfg(all(test, unix))]
72mod containment_tests;
73mod slow_retry;
74use slow_retry::{FailedRetry, FailureContext};
75
76#[cfg(all(test, unix))]
77mod slow_retry_tests;
78
79pub const SUBC_ARG: &str = "--subc";
85
86const DEFAULT_MAX_RESTARTS: u32 = 3;
87const DEFAULT_BACKOFF: Duration = Duration::from_millis(100);
88const DEFAULT_MAX_BACKOFF: Duration = Duration::from_secs(30);
89const DEFAULT_RESTART_WINDOW: Duration = Duration::from_secs(600);
93pub const DEFAULT_DRAIN_TIMEOUT: Duration = Duration::from_secs(30);
104pub(crate) const DEFAULT_REGISTER_TIMEOUT: Duration = Duration::from_secs(120);
105const REGISTRY_RELEASE_TIMEOUT: Duration = Duration::from_secs(1);
106const REGISTRY_RELEASE_POLL: Duration = Duration::from_millis(10);
107const DRAIN_BUSY_REPROBE_INTERVAL: Duration = Duration::from_millis(250);
119const STDERR_PUMP_DRAIN_TIMEOUT: Duration = Duration::from_millis(250);
142pub const SPAWN_EVENT_RING_CAPACITY: usize = 4096;
144const SPAWN_SUBSCRIBER_BUFFER: usize = SPAWN_EVENT_RING_CAPACITY + 1;
145pub(crate) const SPAWN_SUBSCRIBER_LAGGED_CODE: &str = "spawn_subscriber_lagged";
150
151#[cfg(target_os = "macos")]
152struct PrivacyExec {
153 reader: tokio::io::unix::AsyncFd<std::io::PipeReader>,
154 deadline: tokio::time::Instant,
155 expected: Option<subc_os::FileIdentity>,
156 trampoline: Option<subc_os::FileIdentity>,
157 script: bool,
158 module_id: String,
159 #[cfg(test)]
160 confirmation_pause: Option<(String, Option<tokio::net::TcpStream>)>,
161}
162
163#[cfg(all(test, target_os = "macos"))]
164pub(crate) fn test_privacy_trampoline() -> PathBuf {
165 let path = std::env::current_exe()
168 .unwrap()
169 .parent()
170 .unwrap()
171 .parent()
172 .unwrap()
173 .join("privacy-trampoline-fixture");
174 assert!(
178 path.exists(),
179 "privacy-trampoline-fixture not built at {}: run `cargo build -p subc-daemon \
180 --bins --features test-support` or `cargo test -p subc-daemon` first",
181 path.display()
182 );
183 path
184}
185
186#[cfg(target_os = "macos")]
187fn probe_privacy_trampoline(path: &std::path::Path) -> Result<(), String> {
188 use std::io::Read;
189 let mut probe = std::process::Command::new(path)
190 .args(["__disclaim-exec", "--probe"])
191 .stdin(Stdio::null())
192 .stdout(Stdio::piped())
193 .stderr(Stdio::piped())
194 .spawn()
195 .map_err(|error| {
196 format!(
197 "privacy trampoline probe failed for {}: {error}",
198 path.display()
199 )
200 })?;
201 let deadline = std::time::Instant::now() + Duration::from_secs(5);
202 let status = loop {
203 match probe.try_wait() {
204 Ok(Some(status)) => break status,
205 Ok(None) if std::time::Instant::now() < deadline => {
206 std::thread::sleep(Duration::from_millis(5))
207 }
208 result => {
209 let _ = probe.kill();
210 let _ = probe.wait();
211 return Err(format!(
212 "privacy trampoline probe failed or timed out for {}: {result:?}",
213 path.display()
214 ));
215 }
216 }
217 };
218 let mut answer = String::new();
219 if let Some(stdout) = probe.stdout.take() {
220 let _ = stdout.take(256).read_to_string(&mut answer);
221 }
222 if status.success() && answer.trim() == subc_os::privacy_identity::TRAMPOLINE_PROBE {
223 return Ok(());
224 }
225 let mut diagnostic = String::new();
226 if let Some(stderr) = probe.stderr.take() {
227 let _ = stderr.take(1024).read_to_string(&mut diagnostic);
228 }
229 let cause = diagnostic
230 .trim()
231 .strip_prefix("ck-subc: own privacy identity refused: ")
232 .unwrap_or("binary does not implement the privacy trampoline protocol");
233 Err(format!(
234 "{cause}: probe of {} exited {status}",
235 path.display()
236 ))
237}
238
239#[cfg(target_os = "macos")]
240fn privacy_command(
241 spec: &ModuleSpec,
242 roster: &ChildRoster,
243) -> Result<
244 (
245 Command,
246 Option<PrivacyExec>,
247 subc_os::privacy_identity::ExecAcknowledgement,
248 ),
249 SuperviseError,
250> {
251 let failure = |cause: String| {
252 warn!(module_id = %spec.module_id, %cause, "privacy identity trampoline refused module spawn");
253 SuperviseError::Spawn {
254 program: spec.program.clone(),
255 source: io::Error::other(cause),
256 cgroup_path: None,
257 }
258 };
259 let trampoline = roster.privacy_trampoline().map_err(failure)?;
260 let program = if spec.program.components().count() == 1 && !spec.program.is_absolute() {
265 let path = spec
266 .env
267 .iter()
268 .find(|(key, _)| key == "PATH")
269 .map(|(_, value)| std::ffi::OsString::from(value))
270 .or_else(|| std::env::var_os("PATH"))
271 .unwrap_or_else(|| "/usr/bin:/bin".into());
272 std::env::split_paths(&path)
273 .map(|dir| dir.join(&spec.program))
274 .find(|path| path.is_file())
275 .unwrap_or_else(|| spec.program.clone())
276 } else {
277 spec.program.clone()
278 };
279 let expected = subc_os::file_identity(&program);
280 let trampoline_image = subc_os::file_identity(&trampoline);
281 let script = {
282 use std::io::Read;
283 let mut prefix = [0u8; 2];
284 std::fs::File::open(&program)
285 .is_ok_and(|mut file| file.read_exact(&mut prefix).is_ok() && &prefix == b"#!")
286 };
287 if expected.is_none() || expected == subc_os::file_identity(&trampoline) {
288 return Err(failure(
289 "privacy identity module executable is missing or is the trampoline itself".to_string(),
290 ));
291 }
292 let (reader, ack) = subc_os::privacy_identity::ExecAcknowledgement::pipe()
293 .map_err(|error| failure(error.to_string()))?;
294 let reader =
295 tokio::io::unix::AsyncFd::new(reader).map_err(|error| failure(error.to_string()))?;
296 let mut command = Command::new(&trampoline);
297 command
298 .arg("__disclaim-exec")
299 .arg(ack.fd().to_string())
300 .arg(&program);
301 Ok((
302 command,
303 Some(PrivacyExec {
304 reader,
305 deadline: tokio::time::Instant::now() + Duration::from_secs(5),
306 expected,
307 trampoline: trampoline_image,
308 script,
309 module_id: spec.module_id.clone(),
310 #[cfg(test)]
311 confirmation_pause: spec
312 .env
313 .iter()
314 .find(|(key, _)| key == "SUBC_TEST_BEFORE_EXEC_CONFIRM")
315 .map(|(_, address)| (address.clone(), None)),
316 }),
317 ack,
318 ))
319}
320
321struct SupervisedChild {
322 subtree_target: crate::subtree_resources::Target,
323 child: containment::ProcessChild,
324 exec: crate::activity::ExecOwner,
325 #[cfg(windows)]
326 nonce_delivery: Option<subc_os::LaunchNoncePipeDelivery>,
327 #[cfg(target_os = "macos")]
328 privacy_exec: Option<PrivacyExec>,
329 #[cfg(target_os = "macos")]
336 report_ready: Arc<OnceLock<()>>,
337 spawn_failure: Option<String>,
340 protocol: ModuleProtocol,
345 #[cfg(target_os = "linux")]
352 module_id: String,
353 #[cfg(target_os = "linux")]
354 cgroup_placement: Option<subc_cgroup::Placement>,
355 #[cfg(windows)]
378 job: Option<Arc<Mutex<subc_jobobject::JobObject>>>,
379 stdout_pump: Option<JoinHandle<()>>,
380 stderr_pump: Option<StderrPump>,
381 stderr_ring: Arc<Mutex<StderrRing>>,
382 spawned_at: Instant,
385 registration_deadline: Option<Instant>,
388 registration_watch: Option<RegistrationWatch>,
389 spawned_at_ms: u64,
390 spawned_from: PathBuf,
391 launch_spec_digest: LaunchSpecDigest,
392 spawned_file_identity: Option<SpawnedFileIdentity>,
393 process_start_time: Option<u64>,
394 process_identity: Option<ProcessIdentity>,
395 pid: u32,
396 roster_guard: Option<crate::child_roster::RosterGuard>,
399}
400
401impl SupervisedChild {
402 fn registered_since_spawn(&self) -> bool {
403 self.registration_watch
404 .as_ref()
405 .is_some_and(|watch| *watch.changes.borrow() != watch.baseline)
406 }
407
408 fn id(&self) -> Option<u32> {
409 Some(self.pid)
410 }
411
412 fn process_identity(&self) -> Option<ProcessIdentity> {
413 self.process_identity
414 }
415
416 async fn wait(&mut self) -> io::Result<ExitStatus> {
417 #[cfg(target_os = "macos")]
418 self.confirm_privacy_exec().await;
419 let result = self.child.wait().await;
427 #[cfg(target_os = "linux")]
428 if result.is_ok() && self.child.container().is_none() {
429 if let Some(placement) = self.cgroup_placement.as_ref() {
430 cleanup_reaped_cgroup(placement, &self.module_id).await;
434 self.cgroup_placement = None;
435 }
436 }
437 result
438 }
439
440 async fn finish_exit(&mut self, carry: bool) -> io::Result<()> {
441 if self.child.container().is_some() {
442 if !carry {
443 self.child.terminate_container().await;
444 }
445 } else {
446 #[cfg(target_os = "linux")]
447 if let Some(placement) = self.cgroup_placement.as_ref() {
448 cleanup_reaped_cgroup(placement, &self.module_id).await;
449 self.cgroup_placement = None;
450 }
451 }
452 self.child.reap().await?;
453 self.release_roster();
454 Ok(())
455 }
456
457 #[cfg(target_os = "macos")]
458 async fn confirm_privacy_exec(&mut self) {
459 let Some(pending) = &mut self.privacy_exec else {
460 return;
461 };
462 let result = tokio::time::timeout_at(pending.deadline, async {
463 let mut record = Vec::new();
464 loop {
465 let mut ready = pending.reader.readable().await?;
466 let read = ready.try_io(|reader| {
467 use std::io::Read;
468 let mut reader = reader.get_ref();
469 let mut buffer = [0u8; 256];
470 reader.read(&mut buffer).map(|count| (count, buffer))
471 });
472 match read {
473 Ok(Ok((0, _))) => return Ok::<_, io::Error>(record),
474 Ok(Ok((count, buffer))) => {
475 if record.len() + count > 1024 {
476 return Err(io::Error::other(
477 "privacy exec refusal record is too long",
478 ));
479 }
480 record.extend_from_slice(&buffer[..count]);
481 }
482 Ok(Err(error)) => return Err(error),
483 Err(_) => continue,
484 }
485 }
486 })
487 .await;
488 #[cfg(test)]
489 let result = if let Some((address, stream)) = &mut pending.confirmation_pause {
490 use tokio::io::{AsyncReadExt, AsyncWriteExt};
491 if stream.is_none() {
492 *stream = Some(
493 tokio::net::TcpStream::connect(address.as_str())
494 .await
495 .unwrap(),
496 );
497 stream.as_mut().unwrap().write_all(b"R").await.unwrap();
498 stream
499 .as_mut()
500 .unwrap()
501 .write_all(&self.pid.to_ne_bytes())
502 .await
503 .unwrap();
504 }
505 let mut reply = [0];
506 stream
507 .as_mut()
508 .unwrap()
509 .read_exact(&mut reply)
510 .await
511 .unwrap();
512 pending.confirmation_pause = None;
513 if &reply == b"F" {
514 Ok(Err(io::Error::other("injected late exec refusal")))
515 } else {
516 assert_eq!(&reply, b"X");
517 result
518 }
519 } else {
520 result
521 };
522 let pending = self.privacy_exec.as_ref().expect("pending exec");
525 let cause = match result {
526 Err(_) => Some("privacy identity trampoline did not exec within 5s".to_string()),
527 Ok(Err(error)) => Some(format!(
528 "privacy identity exec acknowledgement failed: {error}"
529 )),
530 Ok(Ok(record)) if !record.is_empty() => Some(
531 std::str::from_utf8(&record)
532 .ok()
533 .and_then(|record| {
534 record
535 .trim()
536 .strip_prefix(subc_os::privacy_identity::EXEC_REFUSAL_TAG)
537 })
538 .filter(|cause| !cause.is_empty())
539 .unwrap_or("invalid privacy exec refusal record")
540 .to_string(),
541 ),
542 Ok(Ok(_)) => match self.child.try_wait() {
543 Ok(Some(_status)) => None,
547 Err(error) => Some(format!("privacy identity trampoline wait failed: {error}")),
548 Ok(None) => {
549 let image = observe_spawned_image(self.pid);
550 if let Some(image) = image.filter(|image| {
551 image.executable.is_some()
552 && image.executable != pending.trampoline
553 && (image.executable == pending.expected || pending.script)
554 }) {
555 if let Err(error) = self
556 .roster_guard
557 .as_ref()
558 .map(|guard| guard.confirm_executable(image))
559 .transpose()
560 {
561 Some(format!(
562 "could not durably record confirmed module image: {error}"
563 ))
564 } else {
565 let _ = self.report_ready.set(());
566 info!(module_id = %pending.module_id, pid = self.pid,
567 "module spawned with own privacy identity (responsibility disclaimed)");
568 None
569 }
570 } else if image.is_none()
571 || image.is_some_and(|image| image.executable.is_none())
572 {
573 match tokio::time::timeout_at(pending.deadline, self.child.wait()).await {
580 Ok(Ok(_status)) => None,
581 Ok(Err(error)) => Some(format!("privacy identity trampoline wait failed: {error}")),
582 Err(_) => Some("privacy identity module executable remained unreadable until the 5s exec deadline".to_string()),
583 }
584 } else {
585 Some("privacy identity trampoline executable mismatch: expected the module program, not ck-subc".to_string())
586 }
587 }
588 },
589 };
590 let pending = self.privacy_exec.take().expect("pending exec");
591 self.exec.0.finish(if cause.is_some() {
594 crate::activity::ExecState::Refused
595 } else {
596 crate::activity::ExecState::CommittedExec
597 });
598 if let Some(cause) = cause {
599 warn!(module_id = %pending.module_id, pid = self.pid, %cause, "privacy identity trampoline refused module spawn");
600 self.spawn_failure = Some(cause);
601 if let Some(container) = self.child.container() {
604 container.hard_kill();
605 } else if let Some(pid) = rustix::process::Pid::from_raw(self.pid as i32) {
606 let _ = rustix::process::kill_process_group(pid, rustix::process::Signal::KILL);
607 }
608 let _ = self.child.start_kill();
609 }
610 }
611
612 fn release_roster(&mut self) {
616 self.roster_guard = None;
617 }
618
619 fn start_kill(&mut self) -> io::Result<()> {
632 if let Some(container) = self.child.container() {
633 container.hard_kill();
634 return self.kill_leader();
635 }
636 #[cfg(windows)]
637 if let Some(job) = &self.job {
638 if let Err(error) = job.lock().unwrap_or_else(|p| p.into_inner()).terminate() {
639 debug!(
640 error = %error,
641 "job termination failed; the direct-child kill still owns the outcome"
642 );
643 }
644 }
645 #[cfg(target_os = "linux")]
646 kill_module_cgroup(self.cgroup_placement.as_ref(), &self.module_id);
647 self.child.start_kill()
648 }
649
650 fn kill_leader(&mut self) -> io::Result<()> {
651 self.child.start_kill()
652 }
653
654 async fn drain_stderr(&mut self, module_id: &str) {
655 if let Some(pump) = self.stdout_pump.take() {
656 settle_stdout_pump(module_id, pump, STDERR_PUMP_DRAIN_TIMEOUT).await;
657 }
658
659 let Some(pump) = self.stderr_pump.take() else {
660 return;
661 };
662 settle_stderr_pump(
663 module_id,
664 &self.stderr_ring,
665 pump,
666 STDERR_PUMP_DRAIN_TIMEOUT,
667 )
668 .await;
669 }
670}
671
672async fn settle_stdout_pump(module_id: &str, mut pump: JoinHandle<()>, bound: Duration) {
676 match timeout(bound, &mut pump).await {
677 Ok(Ok(())) => {}
678 Ok(Err(error)) => {
679 warn!(module_id, error = %error, "stdout pump ended unexpectedly");
680 }
681 Err(_) => {
682 drop(pump);
683 warn!(
684 module_id,
685 waited = ?bound,
686 "stdout pipe of the exited process has not drained; its reader keeps running without delaying the restart"
687 );
688 }
689 }
690}
691
692struct StderrPump {
695 task: JoinHandle<()>,
696 generation: u64,
697}
698
699async fn settle_stderr_pump(
705 module_id: &str,
706 ring: &Arc<Mutex<StderrRing>>,
707 pump: StderrPump,
708 bound: Duration,
709) {
710 let lock = || ring.lock().unwrap_or_else(|poisoned| poisoned.into_inner());
711 let StderrPump {
712 mut task,
713 generation,
714 } = pump;
715 lock().retire_pump(generation);
716 match timeout(bound, &mut task).await {
717 Ok(Ok(())) => {}
718 Ok(Err(err)) => {
719 let mut ring = lock();
720 ring.mark_incomplete_from(generation, format!("stderr pump ended unexpectedly: {err}"));
721 ring.finish_pump(generation);
722 warn!(module_id, error = %err, "stderr pump ended before clean EOF");
723 }
724 Err(_) => {
725 drop(task);
727 lock().mark_pump_late(
728 generation,
729 format!(
730 "stderr of the exited process had not reached EOF {bound:?} after it was \
731 retired (a descendant may still hold the pipe open); lines it still \
732 writes are kept in that process's section"
733 ),
734 );
735 warn!(
736 module_id,
737 waited = ?bound,
738 "stderr pipe of the exited process is still open; its reader keeps running without delaying the restart"
739 );
740 }
741 }
742}
743
744fn registration_release_events() -> &'static watch::Sender<u64> {
745 static EVENTS: OnceLock<watch::Sender<u64>> = OnceLock::new();
746 EVENTS.get_or_init(|| {
747 let (sender, _receiver) = watch::channel(0);
748 sender
749 })
750}
751
752pub(crate) fn notify_registration_release() {
753 let events = registration_release_events();
754 let next_generation = (*events.borrow()).wrapping_add(1);
755 events.send_replace(next_generation);
756}
757
758#[derive(Debug, Clone, PartialEq, Eq)]
760pub struct ModuleSpec {
761 pub module_id: String,
762 pub program: PathBuf,
763 pub args: Vec<String>,
764 pub env: Vec<(String, String)>,
765 pub reserved: bool,
770 pub reserved_prefixes: Vec<String>,
775 pub protocol: ModuleProtocol,
794 pub overlap: ModuleOverlap,
799 pub carry_containment: bool,
803}
804
805#[derive(Clone, PartialEq, Eq)]
808pub(crate) struct LaunchSpecDigest([u8; 32]);
809
810impl fmt::Debug for LaunchSpecDigest {
811 fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
812 f.write_str("LaunchSpecDigest([redacted])")
813 }
814}
815
816impl LaunchSpecDigest {
817 pub(crate) fn from_spec(spec: &ModuleSpec) -> Self {
818 fn field(hash: &mut Sha256, bytes: &[u8]) {
819 hash.update((bytes.len() as u64).to_le_bytes());
820 hash.update(bytes);
821 }
822 let mut hash = Sha256::new();
823 if let Some(program) = spec.program.to_str() {
824 field(&mut hash, b"utf8-program");
825 field(&mut hash, program.as_bytes());
826 } else {
827 #[cfg(unix)]
830 {
831 use std::os::unix::ffi::OsStrExt;
832 field(&mut hash, b"unix-program");
833 field(&mut hash, spec.program.as_os_str().as_bytes());
834 }
835 #[cfg(windows)]
836 {
837 use std::os::windows::ffi::OsStrExt;
838 field(&mut hash, b"windows-program");
839 let bytes = spec
840 .program
841 .as_os_str()
842 .encode_wide()
843 .flat_map(u16::to_le_bytes)
844 .collect::<Vec<_>>();
845 field(&mut hash, &bytes);
846 }
847 #[cfg(not(any(unix, windows)))]
848 field(&mut hash, spec.program.to_string_lossy().as_bytes());
849 }
850 hash.update((spec.args.len() as u64).to_le_bytes());
851 for arg in &spec.args {
852 field(&mut hash, arg.as_bytes());
853 }
854 let mut env = spec.env.iter().collect::<Vec<_>>();
855 env.sort_unstable();
856 hash.update((env.len() as u64).to_le_bytes());
857 for (key, value) in env {
858 field(&mut hash, key.as_bytes());
859 field(&mut hash, value.as_bytes());
860 }
861 field(
862 &mut hash,
863 match spec.protocol {
864 ModuleProtocol::Subc => b"subc",
865 ModuleProtocol::None => b"none",
866 },
867 );
868 Self(hash.finalize().into())
869 }
870}
871
872#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)]
879pub enum ModuleOverlap {
880 #[default]
882 Exclusive,
883 Safe,
895}
896
897impl ModuleOverlap {
898 pub fn as_str(self) -> &'static str {
899 match self {
900 Self::Exclusive => "exclusive",
901 Self::Safe => "safe",
902 }
903 }
904}
905
906pub const SUBC_SPAWN_ROLE_ENV: &str = "SUBC_SPAWN_ROLE";
916pub const SPAWN_ROLE_SWAP_CANDIDATE: &str = "swap_candidate";
918pub const DEFAULT_SWAP_READY_TIMEOUT: Duration = Duration::from_secs(100);
923
924#[derive(Debug, Clone, Copy, PartialEq, Eq)]
942pub struct RestartPolicy {
943 pub max_restarts: u32,
944 pub backoff: Duration,
947 pub max_backoff: Duration,
949 pub window: Duration,
953}
954
955impl RestartPolicy {
956 pub fn new(max_restarts: u32, backoff: Duration) -> Self {
960 Self {
961 max_restarts,
962 backoff,
963 max_backoff: DEFAULT_MAX_BACKOFF,
964 window: DEFAULT_RESTART_WINDOW,
965 }
966 }
967
968 pub fn with_max_backoff(mut self, max_backoff: Duration) -> Self {
969 self.max_backoff = max_backoff;
970 self
971 }
972
973 pub fn with_window(mut self, window: Duration) -> Self {
974 self.window = window;
975 self
976 }
977
978 fn delay_for_restart(&self, restart_in_window: u32) -> Duration {
983 if self.backoff.is_zero() || self.max_backoff.is_zero() {
984 return Duration::ZERO;
985 }
986
987 let mut delay = self.backoff;
988 for _ in 0..restart_in_window {
989 if delay >= self.max_backoff {
990 return self.max_backoff;
991 }
992 delay = delay
993 .checked_mul(10)
994 .unwrap_or(self.max_backoff)
995 .min(self.max_backoff);
996 }
997 delay.min(self.max_backoff)
998 }
999
1000 fn budget_exhausted_detail(&self) -> String {
1005 format!(
1006 "crash budget exhausted: max_restarts={} within window_secs={}",
1007 self.max_restarts,
1008 self.window.as_secs()
1009 )
1010 }
1011}
1012
1013impl Default for RestartPolicy {
1014 fn default() -> Self {
1015 Self {
1016 max_restarts: DEFAULT_MAX_RESTARTS,
1017 backoff: DEFAULT_BACKOFF,
1018 max_backoff: DEFAULT_MAX_BACKOFF,
1019 window: DEFAULT_RESTART_WINDOW,
1020 }
1021 }
1022}
1023
1024#[derive(Debug, Clone, Copy, PartialEq, Eq)]
1025struct CrashRestartSchedule {
1026 restart_in_window: u32,
1027 delay: Duration,
1028}
1029
1030fn daemon_will_restart(
1037 state: &mut SupervisorSnapshot,
1038 policy: &RestartPolicy,
1039 now: Instant,
1040) -> bool {
1041 state.enabled && state.crash_restarts_in_window(policy.window, now) < policy.max_restarts
1042}
1043
1044const DEFAULT_HEALTH_CADENCE: Duration = Duration::from_secs(30);
1045const DEFAULT_HEALTH_DEADLINE: Duration = Duration::from_secs(5);
1046const DEFAULT_HEALTH_FAILURE_THRESHOLD: u32 = 3;
1047const MAX_HEALTH_METRICS_BYTES: usize = 16 * 1024;
1048
1049#[derive(Debug, Clone, Copy, PartialEq, Eq)]
1050pub enum HealthAction {
1051 Report,
1052 Restart,
1053 Alert,
1054}
1055
1056impl fmt::Display for HealthAction {
1057 fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
1058 f.write_str(match self {
1059 Self::Report => "report",
1060 Self::Restart => "restart",
1061 Self::Alert => "alert",
1062 })
1063 }
1064}
1065
1066#[derive(Debug, Clone, PartialEq, Eq)]
1067pub struct HealthConfig {
1068 pub http: Option<String>,
1072 pub cadence: Duration,
1073 pub deadline: Duration,
1074 pub failure_threshold: u32,
1075 pub on_degraded: HealthAction,
1076 pub on_failing: HealthAction,
1077 pub critical: bool,
1078}
1079
1080impl Default for HealthConfig {
1081 fn default() -> Self {
1082 Self {
1083 http: None,
1084 cadence: DEFAULT_HEALTH_CADENCE,
1085 deadline: DEFAULT_HEALTH_DEADLINE,
1086 failure_threshold: DEFAULT_HEALTH_FAILURE_THRESHOLD,
1087 on_degraded: HealthAction::Report,
1088 on_failing: HealthAction::Report,
1089 critical: false,
1090 }
1091 }
1092}
1093
1094#[derive(Debug, Clone, PartialEq)]
1112pub struct ModuleHealthStatus {
1113 pub status: SupervisorHealthStatus,
1114 pub last_probe_ms: Option<u64>,
1115 pub detail: Option<String>,
1116 pub metrics: Option<Value>,
1117 pub consecutive_failures: u32,
1118 pub late_answer_count: u64,
1121 pub last_late_answer_latency_ms: Option<u64>,
1123 pub last_action: Option<String>,
1124 pub last_action_ms: Option<u64>,
1128}
1129
1130impl Default for ModuleHealthStatus {
1131 fn default() -> Self {
1132 Self {
1133 status: SupervisorHealthStatus::Unknown,
1134 last_probe_ms: None,
1135 detail: None,
1136 metrics: None,
1137 consecutive_failures: 0,
1138 late_answer_count: 0,
1139 last_late_answer_latency_ms: None,
1140 last_action: None,
1141 last_action_ms: None,
1142 }
1143 }
1144}
1145
1146#[derive(Debug, Clone, Copy, PartialEq, Eq)]
1148pub enum ModuleState {
1149 Starting,
1150 Running,
1151 Unresponsive,
1152 Restarting,
1153 Draining,
1154 Stopped,
1155 Failed,
1156 Disabled,
1157}
1158
1159impl fmt::Display for ModuleState {
1160 fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
1161 f.write_str(match self {
1162 Self::Starting => "starting",
1163 Self::Running => "running",
1164 Self::Unresponsive => "unresponsive",
1165 Self::Restarting => "restarting",
1166 Self::Draining => "draining",
1167 Self::Stopped => "stopped",
1168 Self::Failed => "failed",
1169 Self::Disabled => "disabled",
1170 })
1171 }
1172}
1173
1174#[derive(Debug, Clone, Copy, PartialEq, Eq)]
1176pub enum ExitKind {
1177 Clean,
1178 Crash,
1179 DeliberateSeverance,
1180}
1181
1182impl From<ExitKind> for TerminalExitKind {
1183 fn from(kind: ExitKind) -> Self {
1184 match kind {
1185 ExitKind::Clean => Self::Clean,
1186 ExitKind::Crash => Self::Crash,
1187 ExitKind::DeliberateSeverance => Self::DeliberateSeverance,
1188 }
1189 }
1190}
1191
1192#[derive(Debug, Clone, Copy, PartialEq, Eq)]
1195pub(crate) struct ProcessIdentity {
1196 pub(crate) pid: u32,
1197 pub(crate) start_time: u64,
1198}
1199
1200#[derive(Debug, Clone, PartialEq, Eq)]
1202pub struct ExitReport {
1203 pub kind: ExitKind,
1204 pub code: Option<i32>,
1205 pub signal: Option<i32>,
1206 pub at_ms: u64,
1207}
1208
1209#[derive(Debug, Clone, PartialEq)]
1212pub struct ModuleStatus {
1213 pub module_id: String,
1214 pub waiting_for_previous_pid: Option<u32>,
1216 pub waiting_for_previous_detail: Option<String>,
1218 pub launch_nonce_source: Option<String>,
1223 pub state: ModuleState,
1224 pub enabled: bool,
1225 pub process_alive: bool,
1226 pub registration_active: bool,
1227 pub protocol: ModuleProtocol,
1232 pub live: bool,
1243 pub restart_count: u32,
1247 pub lifetime_restarts: u32,
1251 pub spawn_generation: u64,
1252 pub max_restarts: u32,
1257 pub restart_window: Duration,
1261 pub drain_timeout: Duration,
1265 pub restart_backoff: Duration,
1266 pub restart_max_backoff: Duration,
1267 pub pid: Option<u32>,
1272 pub spawned_at_ms: Option<u64>,
1273 pub spawned_from: Option<PathBuf>,
1274 pub(crate) launch_spec_digest: Option<LaunchSpecDigest>,
1275 pub process_start_time: Option<u64>,
1276 pub last_exit: Option<ExitReport>,
1277 pub health: ModuleHealthStatus,
1278}
1279
1280#[cfg(test)]
1281#[derive(Debug, Clone, Copy, PartialEq)]
1282struct ActorSelectCheckpoint {
1283 generation: u64,
1284 turn: u64,
1285 registered_connection: Option<ConnectionId>,
1286 next_probe_at: Option<Instant>,
1287 wake_after: Option<Duration>,
1288}
1289
1290#[derive(Debug, Clone, PartialEq)]
1291struct SupervisorSnapshot {
1292 subtree_target: crate::subtree_resources::Target,
1293 failed_retry: FailedRetry,
1294 exec_outcome: Option<Arc<crate::activity::ExecOutcome>>,
1295 #[cfg(windows)]
1296 pipe_consumed: Option<Arc<OnceLock<()>>>,
1297 #[cfg(test)]
1299 actor_turns: u64,
1300 #[cfg(test)]
1303 actor_select: Option<ActorSelectCheckpoint>,
1304 state: ModuleState,
1305 enabled: bool,
1306 process_alive: bool,
1307 spawned_protocol: Option<ModuleProtocol>,
1308 spawn_failure: Option<String>,
1309 waiting_for_previous_pid: Option<u32>,
1310 waiting_for_previous_detail: Option<String>,
1311 crash_restarts: VecDeque<Instant>,
1317 lifetime_restarts: u32,
1318 spawn_generation: u64,
1327 pid: Option<u32>,
1328 #[cfg(target_os = "macos")]
1329 report_ready: Option<Arc<OnceLock<()>>>,
1330 reaped_pid: Option<u32>,
1332 respawn_pending: bool,
1334 coalesced_restart_pending: bool,
1336 spawned_at_ms: Option<u64>,
1337 spawned_from: Option<PathBuf>,
1338 launch_spec_digest: Option<LaunchSpecDigest>,
1339 spawned_file_identity: Option<SpawnedFileIdentity>,
1340 process_start_time: Option<u64>,
1341 deliberate_severance: Option<ProcessIdentity>,
1342 last_exit: Option<ExitReport>,
1343 drain_disposition_detail: Option<String>,
1345 health: ModuleHealthStatus,
1346 in_alternate_slot: bool,
1351 draining_to_replace: bool,
1358 configuration_updated_since_spawn: bool,
1364}
1365
1366impl SupervisorSnapshot {
1367 fn reported_pid(&self) -> Option<u32> {
1372 #[cfg(target_os = "macos")]
1373 if self
1374 .report_ready
1375 .as_ref()
1376 .is_some_and(|ready| ready.get().is_none())
1377 {
1378 return None;
1379 }
1380 self.pid
1381 }
1382
1383 fn starting() -> Self {
1384 Self::new(ModuleState::Starting, true)
1385 }
1386
1387 fn disabled() -> Self {
1388 Self::new(ModuleState::Disabled, false)
1389 }
1390
1391 fn crash_restarts_in_window(&mut self, window: Duration, now: Instant) -> u32 {
1395 while let Some(oldest) = self.crash_restarts.front() {
1396 if now.duration_since(*oldest) > window {
1397 self.crash_restarts.pop_front();
1398 } else {
1399 break;
1400 }
1401 }
1402 u32::try_from(self.crash_restarts.len()).unwrap_or(u32::MAX)
1403 }
1404
1405 fn record_crash_restart(&mut self, policy: &RestartPolicy, now: Instant) {
1411 self.crash_restarts.push_back(now);
1412 while self.crash_restarts.len() > policy.max_restarts as usize {
1413 self.crash_restarts.pop_front();
1414 }
1415 self.lifetime_restarts += 1;
1416 }
1417
1418 fn next_crash_restart(
1422 &mut self,
1423 policy: &RestartPolicy,
1424 now: Instant,
1425 ) -> Option<CrashRestartSchedule> {
1426 let restart_in_window = self.crash_restarts_in_window(policy.window, now);
1427 if restart_in_window >= policy.max_restarts {
1428 return None;
1429 }
1430 self.record_crash_restart(policy, now);
1431 Some(CrashRestartSchedule {
1432 restart_in_window,
1433 delay: policy.delay_for_restart(restart_in_window),
1434 })
1435 }
1436
1437 fn clear_crash_restarts(&mut self) {
1442 self.crash_restarts.clear();
1443 }
1444
1445 fn new(state: ModuleState, enabled: bool) -> Self {
1446 Self {
1447 subtree_target: crate::subtree_resources::Target::Unavailable("not_running"),
1448 failed_retry: FailedRetry::default(),
1449 exec_outcome: None,
1450 #[cfg(test)]
1451 actor_turns: 0,
1452 #[cfg(test)]
1453 actor_select: None,
1454 state,
1455 enabled,
1456 process_alive: false,
1457 #[cfg(windows)]
1458 pipe_consumed: None,
1459 spawned_protocol: None,
1460 spawn_failure: None,
1461 waiting_for_previous_pid: None,
1462 waiting_for_previous_detail: None,
1463 crash_restarts: VecDeque::new(),
1464 lifetime_restarts: 0,
1465 spawn_generation: 0,
1466 pid: None,
1467 #[cfg(target_os = "macos")]
1468 report_ready: None,
1469 reaped_pid: None,
1470 respawn_pending: false,
1471 coalesced_restart_pending: false,
1472 spawned_at_ms: None,
1473 spawned_from: None,
1474 launch_spec_digest: None,
1475 spawned_file_identity: None,
1476 process_start_time: None,
1477 deliberate_severance: None,
1478 last_exit: None,
1479 drain_disposition_detail: None,
1480 health: ModuleHealthStatus::default(),
1481 in_alternate_slot: false,
1482 draining_to_replace: false,
1483 configuration_updated_since_spawn: false,
1484 }
1485 }
1486}
1487
1488type SharedSnapshot = Arc<Mutex<SupervisorSnapshot>>;
1489
1490type SpawnSubscriberKey = (ConnectionId, u64);
1491
1492#[derive(Debug)]
1493struct SpawnSubscriber {
1494 version: u8,
1495 frames: mpsc::Sender<Frame>,
1496 lagged: Option<oneshot::Sender<SpawnCursor>>,
1500}
1501
1502#[derive(Debug)]
1503struct SpawnEventState {
1504 daemon_incarnation: String,
1505 seq: u64,
1506 capacity: usize,
1507 live: HashMap<String, LiveSpawn>,
1508 generations: HashMap<String, u64>,
1509 events: VecDeque<SpawnEvent>,
1510 subscribers: HashMap<SpawnSubscriberKey, SpawnSubscriber>,
1511}
1512
1513impl Default for SpawnEventState {
1514 fn default() -> Self {
1515 Self {
1516 daemon_incarnation: "unconfigured".to_string(),
1517 seq: 0,
1518 capacity: SPAWN_EVENT_RING_CAPACITY,
1519 live: HashMap::new(),
1520 generations: HashMap::new(),
1521 events: VecDeque::new(),
1522 subscribers: HashMap::new(),
1523 }
1524 }
1525}
1526
1527#[derive(Debug, Clone, Default)]
1528struct SpawnEventFeed(
1529 Arc<Mutex<SpawnEventState>>,
1530 #[cfg(test)] Arc<Mutex<Option<std::net::TcpStream>>>,
1531);
1532
1533#[derive(Debug, Clone, PartialEq, Eq)]
1534pub(crate) enum SpawnSubscribeRefusal {
1535 ForeignIncarnation { current: String },
1536 TooOld { oldest: SpawnCursor },
1537 Frame(String),
1538}
1539
1540impl SpawnEventFeed {
1541 fn configure_incarnation(&self, daemon_incarnation: String) {
1542 let mut state = self.0.lock().unwrap_or_else(|p| p.into_inner());
1543 state.daemon_incarnation = daemon_incarnation;
1544 state.seq = 0;
1545 state.live.clear();
1546 state.generations.clear();
1547 state.events.clear();
1548 state.subscribers.clear();
1549 }
1550
1551 fn cursor(state: &SpawnEventState) -> SpawnCursor {
1552 SpawnCursor {
1553 daemon_incarnation: state.daemon_incarnation.clone(),
1554 seq: state.seq,
1555 }
1556 }
1557
1558 fn snapshot(&self) -> SpawnSnapshot {
1559 let state = self.0.lock().unwrap_or_else(|p| p.into_inner());
1560 let mut live = state.live.values().cloned().collect::<Vec<_>>();
1561 live.sort_by(|left, right| left.module_id.cmp(&right.module_id));
1562 SpawnSnapshot {
1563 cursor: Self::cursor(&state),
1564 ring_bound: state.capacity as u64,
1565 live,
1566 }
1567 }
1568
1569 fn emit_spawned(&self, module_id: &str, pid: u32, spawned_at_ms: u64) -> u64 {
1570 let mut state = self.0.lock().unwrap_or_else(|p| p.into_inner());
1571 let generation = state
1572 .generations
1573 .get(module_id)
1574 .copied()
1575 .unwrap_or(0)
1576 .checked_add(1)
1577 .expect("spawn generation exhausted");
1578 state.generations.insert(module_id.to_string(), generation);
1579 let live = LiveSpawn {
1580 module_id: module_id.to_string(),
1581 spawn_generation: generation,
1582 pid,
1583 spawned_at_ms,
1584 };
1585 state.live.insert(module_id.to_string(), live);
1586 Self::emit_locked(
1587 &mut state,
1588 SpawnEventKind::Spawned,
1589 module_id.to_string(),
1590 generation,
1591 pid,
1592 None,
1593 None,
1594 );
1595 generation
1596 }
1597
1598 fn emit_exited(&self, module_id: &str, exit_code: Option<i32>, exit_signal: Option<i32>) {
1599 let mut state = self.0.lock().unwrap_or_else(|p| p.into_inner());
1600 let Some(live) = state.live.remove(module_id) else {
1601 warn!(
1602 module_id,
1603 "terminal record had no live spawn event identity"
1604 );
1605 return;
1606 };
1607 Self::emit_locked(
1608 &mut state,
1609 SpawnEventKind::Exited,
1610 module_id.to_string(),
1611 live.spawn_generation,
1612 live.pid,
1613 exit_code,
1614 exit_signal,
1615 );
1616 }
1617
1618 fn emit_superseded_exited(
1625 &self,
1626 module_id: &str,
1627 spawn_generation: u64,
1628 pid: u32,
1629 exit_code: Option<i32>,
1630 exit_signal: Option<i32>,
1631 ) {
1632 let mut state = self.0.lock().unwrap_or_else(|p| p.into_inner());
1633 if state
1634 .live
1635 .get(module_id)
1636 .is_some_and(|live| live.spawn_generation == spawn_generation)
1637 {
1638 state.live.remove(module_id);
1639 }
1640 Self::emit_locked(
1641 &mut state,
1642 SpawnEventKind::Exited,
1643 module_id.to_string(),
1644 spawn_generation,
1645 pid,
1646 exit_code,
1647 exit_signal,
1648 );
1649 }
1650
1651 #[allow(clippy::too_many_arguments)]
1652 fn emit_locked(
1653 state: &mut SpawnEventState,
1654 kind: SpawnEventKind,
1655 module_id: String,
1656 spawn_generation: u64,
1657 pid: u32,
1658 exit_code: Option<i32>,
1659 exit_signal: Option<i32>,
1660 ) {
1661 state.seq = state
1662 .seq
1663 .checked_add(1)
1664 .expect("spawn event sequence exhausted");
1665 let event = SpawnEvent {
1666 cursor: Self::cursor(state),
1667 kind,
1668 module_id,
1669 spawn_generation,
1670 pid,
1671 exit_code,
1672 exit_signal,
1673 };
1674 state.events.push_back(event.clone());
1675 while state.events.len() > state.capacity {
1676 state.events.pop_front();
1677 }
1678 let body = match serde_json::to_vec(&event) {
1679 Ok(body) => body,
1680 Err(error) => {
1681 error!(%error, "failed to serialize supervisor spawn event");
1682 return;
1683 }
1684 };
1685 state.subscribers.retain(|(connection_id, corr), subscriber| {
1686 let frame = Frame::build_with_version(
1687 subscriber.version,
1688 FrameType::StreamData,
1689 control_flags(),
1690 0,
1691 0,
1692 *corr,
1693 body.clone(),
1694 );
1695 match frame {
1696 Ok(frame) => {
1697 if subscriber.frames.try_send(frame).is_ok() {
1698 true
1699 } else {
1700 warn!(connection_id = connection_id.get(), corr, "dropping lagged supervisor spawn subscriber");
1701 if let Some(lagged) = subscriber.lagged.take() {
1702 let _ = lagged.send(event.cursor.clone());
1703 }
1704 false
1705 }
1706 }
1707 Err(error) => {
1708 warn!(connection_id = connection_id.get(), corr, %error, "dropping supervisor spawn subscriber after frame build failure");
1709 false
1710 }
1711 }
1712 });
1713 }
1714
1715 fn subscribe(
1716 &self,
1717 connection_id: ConnectionId,
1718 corr: u64,
1719 version: u8,
1720 since: Option<SpawnCursor>,
1721 sink: FrameSink,
1722 ) -> Result<(), SpawnSubscribeRefusal> {
1723 let (frames, mut receiver) = mpsc::channel(SPAWN_SUBSCRIBER_BUFFER);
1724 let (lagged, mut lagged_rx) = oneshot::channel::<SpawnCursor>();
1725 {
1726 let mut state = self.0.lock().unwrap_or_else(|p| p.into_inner());
1727 let replay = if let Some(since) = since {
1728 if since.daemon_incarnation != state.daemon_incarnation {
1729 return Err(SpawnSubscribeRefusal::ForeignIncarnation {
1730 current: state.daemon_incarnation.clone(),
1731 });
1732 }
1733 if let Some(oldest) = state.events.front().map(|event| event.cursor.clone()) {
1734 if since.seq < oldest.seq.saturating_sub(1) {
1735 return Err(SpawnSubscribeRefusal::TooOld { oldest });
1736 }
1737 }
1738 state
1739 .events
1740 .iter()
1741 .filter(|event| event.cursor.seq > since.seq)
1742 .cloned()
1743 .collect::<Vec<_>>()
1744 } else {
1745 Vec::new()
1746 };
1747 for event in replay {
1748 let body = serde_json::to_vec(&event)
1749 .map_err(|error| SpawnSubscribeRefusal::Frame(error.to_string()))?;
1750 let frame = Frame::build_with_version(
1751 version,
1752 FrameType::StreamData,
1753 control_flags(),
1754 0,
1755 0,
1756 corr,
1757 body,
1758 )
1759 .map_err(|error| SpawnSubscribeRefusal::Frame(error.to_string()))?;
1760 frames
1761 .try_send(frame)
1762 .map_err(|error| SpawnSubscribeRefusal::Frame(error.to_string()))?;
1763 }
1764 state.subscribers.insert(
1765 (connection_id, corr),
1766 SpawnSubscriber {
1767 version,
1768 frames,
1769 lagged: Some(lagged),
1770 },
1771 );
1772 }
1773 tokio::spawn(async move {
1784 while let Some(frame) = receiver.recv().await {
1785 if sink.send(frame).await.is_err() {
1786 return;
1787 }
1788 }
1789 let Ok(first_undelivered) = lagged_rx.try_recv() else {
1790 return;
1791 };
1792 match spawn_subscriber_lagged_frame(version, corr, first_undelivered) {
1793 Ok(frame) => {
1794 let _ = sink.send(frame).await;
1795 }
1796 Err(error) => {
1797 error!(%error, corr, "failed to build lagged spawn subscriber terminal frame");
1798 }
1799 }
1800 });
1801 Ok(())
1802 }
1803
1804 fn cancel(&self, connection_id: ConnectionId, corr: u64) -> bool {
1805 let Some(subscriber) = self
1806 .0
1807 .lock()
1808 .unwrap_or_else(|p| p.into_inner())
1809 .subscribers
1810 .remove(&(connection_id, corr))
1811 else {
1812 return false;
1813 };
1814 if let Ok(frame) = Frame::build_with_version(
1815 subscriber.version,
1816 FrameType::StreamEnd,
1817 control_flags(),
1818 0,
1819 0,
1820 corr,
1821 Vec::new(),
1822 ) {
1823 tokio::spawn(async move {
1824 let _ = subscriber.frames.send(frame).await;
1825 });
1826 }
1827 true
1828 }
1829
1830 fn remove_connection(&self, connection_id: ConnectionId) {
1831 self.0
1832 .lock()
1833 .unwrap_or_else(|p| p.into_inner())
1834 .subscribers
1835 .retain(|(subscriber_connection, _), _| *subscriber_connection != connection_id);
1836 }
1837
1838 #[cfg(any(test, feature = "test-support"))]
1839 fn set_capacity(&self, capacity: usize) {
1840 self.0.lock().unwrap_or_else(|p| p.into_inner()).capacity = capacity;
1841 }
1842
1843 #[cfg(any(test, feature = "test-support"))]
1844 fn subscriber_count(&self) -> usize {
1845 self.0
1846 .lock()
1847 .unwrap_or_else(|p| p.into_inner())
1848 .subscribers
1849 .len()
1850 }
1851}
1852
1853fn spawn_subscriber_lagged_frame(
1856 version: u8,
1857 corr: u64,
1858 first_undelivered: SpawnCursor,
1859) -> Result<Frame, String> {
1860 let body = serde_json::to_vec(&subc_protocol::ErrorBody {
1862 code: SPAWN_SUBSCRIBER_LAGGED_CODE.to_string(),
1863 message: "spawn subscriber fell behind and was dropped; resubscribe from the last cursor received"
1864 .to_string(),
1865 detail: Some(serde_json::json!({
1866 "first_undelivered_cursor": first_undelivered
1867 })),
1868 })
1869 .map_err(|error| error.to_string())?;
1870 Frame::build_with_version(
1871 version,
1872 FrameType::Error,
1873 crate::router::daemon_error_flags(),
1874 0,
1875 0,
1876 corr,
1877 body,
1878 )
1879 .map_err(|error| error.to_string())
1880}
1881
1882pub trait ModuleProcessLiveness: Send + Sync {
1883 fn process_live(&self, module_id: &str) -> Option<bool>;
1884
1885 fn process_replacing(&self, _module_id: &str) -> bool {
1891 false
1892 }
1893}
1894
1895#[derive(Debug, Clone, Default)]
1897pub struct SupervisorProcessLiveness {
1898 snapshots: Arc<Mutex<HashMap<String, SharedSnapshot>>>,
1899}
1900
1901impl SupervisorProcessLiveness {
1902 pub fn new() -> Self {
1903 Self::default()
1904 }
1905
1906 fn track(&self, module_id: String, snapshot: SharedSnapshot) {
1907 let mut snapshots = self
1908 .snapshots
1909 .lock()
1910 .unwrap_or_else(|poisoned| poisoned.into_inner());
1911 snapshots.insert(module_id, snapshot);
1912 }
1913
1914 fn untrack_if_current(&self, module_id: &str, snapshot: &SharedSnapshot) {
1915 let mut snapshots = self
1916 .snapshots
1917 .lock()
1918 .unwrap_or_else(|poisoned| poisoned.into_inner());
1919 let is_current = snapshots
1920 .get(module_id)
1921 .map(|tracked| Arc::ptr_eq(tracked, snapshot))
1922 .unwrap_or(false);
1923 if is_current {
1924 snapshots.remove(module_id);
1925 }
1926 }
1927}
1928
1929impl ModuleProcessLiveness for SupervisorProcessLiveness {
1930 fn process_live(&self, module_id: &str) -> Option<bool> {
1931 let snapshot = {
1932 let snapshots = self
1933 .snapshots
1934 .lock()
1935 .unwrap_or_else(|poisoned| poisoned.into_inner());
1936 snapshots.get(module_id).cloned()
1937 }?;
1938 let snapshot = snapshot
1939 .lock()
1940 .unwrap_or_else(|poisoned| poisoned.into_inner());
1941 Some(snapshot.state == ModuleState::Running && snapshot.process_alive)
1942 }
1943
1944 fn process_replacing(&self, module_id: &str) -> bool {
1945 let Some(snapshot) = self
1946 .snapshots
1947 .lock()
1948 .unwrap_or_else(|poisoned| poisoned.into_inner())
1949 .get(module_id)
1950 .cloned()
1951 else {
1952 return false;
1953 };
1954 let snapshot = snapshot
1955 .lock()
1956 .unwrap_or_else(|poisoned| poisoned.into_inner());
1957 snapshot.enabled
1958 && match snapshot.state {
1959 ModuleState::Restarting => true,
1960 ModuleState::Draining => snapshot.draining_to_replace,
1961 ModuleState::Starting
1962 | ModuleState::Running
1963 | ModuleState::Unresponsive
1964 | ModuleState::Stopped
1965 | ModuleState::Failed
1966 | ModuleState::Disabled => false,
1967 }
1968 }
1969}
1970
1971#[cfg(test)]
1972#[derive(Debug, Default)]
1973struct ReloadExitRecordGate {
1974 reached: tokio::sync::Notify,
1975 resume: tokio::sync::Notify,
1976}
1977
1978#[derive(Debug, Clone, Copy)]
1979enum RespawnKind {
1980 Spawn,
1981 Reload,
1982}
1983
1984#[derive(Debug, Clone, Copy)]
1985struct PendingRespawn {
1986 deadline: Instant,
1987 kind: RespawnKind,
1988 daemon_initiated: bool,
1989}
1990
1991type ReloadReply = oneshot::Sender<Result<(), SuperviseError>>;
1992
1993#[derive(Debug, Clone)]
1994struct SupervisorRuntimeConfig {
1995 registry: Arc<Registry>,
1996 recovery_is_daemon: bool,
1998 start_requester: Option<Principal>,
1999 pending_activity_restart: bool,
2000 provenance_probe: ExecutableIdentityProbe,
2001 scheduled_respawn: Arc<Mutex<Option<PendingRespawn>>>,
2003 deferred_reload_reply: Arc<Mutex<Option<ReloadReply>>>,
2005 restart_policy: RestartPolicy,
2006 register_timeout: Duration,
2007 drain_timeout: Duration,
2010 effective_drain_timeout: Arc<Mutex<Duration>>,
2013 default_drain_timeout: Duration,
2016 health: HealthConfig,
2017 connection_file_path: Option<PathBuf>,
2018 capture_logs_dir: Option<PathBuf>,
2019 forwarding: Option<Arc<ForwardingTable>>,
2020 supervisor_handle: Option<SupervisorHandle>,
2023 stderr_ring: Arc<Mutex<StderrRing>>,
2030 terminal_ring: Arc<Mutex<TerminalRing>>,
2031 spawn_events: SpawnEventFeed,
2032 child_roster: ChildRoster,
2033 #[cfg(target_os = "linux")]
2034 cgroup_placement: Option<subc_cgroup::Placement>,
2035 #[cfg(test)]
2036 test_seed_stale_facts_before_enable_spawn: bool,
2037 #[cfg(test)]
2038 test_reload_exit_record_gate: Option<Arc<ReloadExitRecordGate>>,
2039}
2040
2041impl SupervisorRuntimeConfig {
2042 fn failure_context(&self, daemon_initiated: bool) -> FailureContext {
2043 FailureContext {
2044 daemon_initiated,
2045 shutdown: self.child_roster.shutdown_flag(),
2046 }
2047 }
2048}
2049
2050#[derive(Debug, Clone, PartialEq, Eq)]
2051struct SupervisedConfiguration {
2052 spec: ModuleSpec,
2053 health: HealthConfig,
2054}
2055
2056#[derive(Debug, Clone, Default)]
2062pub struct SupervisorHandle {
2063 restart_intent: Arc<Mutex<DaemonRestartIntent>>,
2064 activity: Arc<Mutex<Option<crate::activity::ActivityLedger>>>,
2065 activity_images: Arc<Mutex<HashMap<String, activity::ActivityImage>>>,
2066 modules: Arc<Mutex<HashMap<String, SupervisedModule>>>,
2067 configured_ids: Arc<Mutex<HashSet<String>>>,
2079 spawn_events: SpawnEventFeed,
2080 reserved_nonces: Arc<Mutex<HashMap<String, Option<String>>>>,
2091 removal_tombstones: Arc<Mutex<HashMap<String, u64>>>,
2097 spawn_nonces: Arc<Mutex<HashMap<String, String>>>,
2101 reserved_prefix_owners: Arc<Mutex<HashMap<String, String>>>,
2109 swaps: Arc<Mutex<HashMap<String, OpenSwap>>>,
2115 promotion_observer: PromotionObserverSlot,
2117 operation_lock: Arc<AsyncMutex<()>>,
2121}
2122
2123pub(crate) trait SwapPromotionObserver: Send + Sync {
2132 fn swap_promoted(&self, registration: &crate::registry::ModuleRegistration);
2133}
2134
2135#[derive(Clone, Default)]
2139struct PromotionObserverSlot(Arc<Mutex<Option<std::sync::Weak<dyn SwapPromotionObserver>>>>);
2140
2141impl fmt::Debug for PromotionObserverSlot {
2142 fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
2143 f.write_str("PromotionObserverSlot")
2144 }
2145}
2146
2147#[derive(Debug, Clone)]
2149struct OpenSwap {
2150 candidate_nonce: String,
2153 incumbent_nonce: Option<String>,
2158 candidate_admitted: bool,
2162}
2163
2164#[derive(Debug, Clone, Copy, PartialEq, Eq)]
2167pub(crate) enum SwapHelloAdmission {
2168 NotSwapping,
2171 Candidate,
2173 Refused,
2176}
2177
2178#[derive(Debug, Clone, PartialEq, Eq)]
2179pub(crate) enum ReservedHelloRejection {
2180 Exact {
2181 module_id: String,
2182 },
2183 Prefix {
2184 prefix: String,
2185 owner_module_id: String,
2186 },
2187}
2188
2189#[derive(Debug, Default)]
2191pub(crate) struct ReservedHelloVerification {
2192 pub(crate) nonce_verified: bool,
2193 pub(crate) prefix_owner: Option<String>,
2194}
2195
2196#[derive(Debug)]
2197struct DaemonRestartIntent {
2198 expires: Option<Duration>,
2199 window: Duration,
2200}
2201
2202impl Default for DaemonRestartIntent {
2203 fn default() -> Self {
2204 Self {
2205 expires: None,
2206 window: Duration::from_secs(120),
2207 }
2208 }
2209}
2210
2211impl SupervisorHandle {
2212 pub(crate) fn announce_daemon_restart(&self) -> u64 {
2213 let mut intent = self
2214 .restart_intent
2215 .lock()
2216 .unwrap_or_else(|p| p.into_inner());
2217 intent.expires = Some(subc_uptime::suspend_inclusive_now() + intent.window);
2218 unix_ms_now().saturating_add(intent.window.as_millis() as u64)
2219 }
2220
2221 #[cfg(feature = "test-support")]
2222 pub(crate) fn set_shutdown_intent_window(&self, window: Duration) {
2223 self.restart_intent
2224 .lock()
2225 .unwrap_or_else(|p| p.into_inner())
2226 .window = window;
2227 }
2228
2229 #[cfg(unix)]
2230 fn take_daemon_shutdown_reason(&self) -> RouteCloseReason {
2231 let expiry = self
2232 .restart_intent
2233 .lock()
2234 .unwrap_or_else(|p| p.into_inner())
2235 .expires
2236 .take();
2237 if expiry.is_some_and(|expiry| subc_uptime::suspend_inclusive_now() < expiry) {
2238 RouteCloseReason::Restart
2239 } else {
2240 RouteCloseReason::Disable
2241 }
2242 }
2243
2244 pub fn new() -> Self {
2245 Self::default()
2246 }
2247
2248 pub(crate) fn spawn_snapshot(&self) -> SpawnSnapshot {
2249 self.spawn_events.snapshot()
2250 }
2251
2252 pub(crate) fn subscribe_spawns(
2253 &self,
2254 connection_id: ConnectionId,
2255 corr: u64,
2256 version: u8,
2257 since: Option<SpawnCursor>,
2258 sink: FrameSink,
2259 ) -> Result<(), SpawnSubscribeRefusal> {
2260 self.spawn_events
2261 .subscribe(connection_id, corr, version, since, sink)
2262 }
2263
2264 pub(crate) fn cancel_spawn_subscription(&self, connection_id: ConnectionId, corr: u64) -> bool {
2265 self.spawn_events.cancel(connection_id, corr)
2266 }
2267
2268 pub(crate) fn remove_spawn_subscribers(&self, connection_id: ConnectionId) {
2269 self.spawn_events.remove_connection(connection_id);
2270 }
2271
2272 #[cfg(any(test, feature = "test-support"))]
2273 pub fn set_spawn_event_capacity_for_test(&self, capacity: usize) {
2274 assert!(capacity > 0, "spawn event capacity must be non-zero");
2275 self.spawn_events.set_capacity(capacity);
2276 }
2277
2278 #[cfg(any(test, feature = "test-support"))]
2279 pub fn spawn_subscriber_count_for_test(&self) -> usize {
2280 self.spawn_events.subscriber_count()
2281 }
2282
2283 pub fn set_spawn_nonce(&self, module_id: &str, nonce: String) {
2286 self.spawn_nonces
2287 .lock()
2288 .unwrap_or_else(|poisoned| poisoned.into_inner())
2289 .insert(module_id.to_string(), nonce);
2290 }
2291
2292 pub fn set_reserved_nonce(&self, module_id: &str, nonce: String) {
2295 self.reserved_nonces
2296 .lock()
2297 .unwrap_or_else(|poisoned| poisoned.into_inner())
2298 .insert(module_id.to_string(), Some(nonce));
2299 }
2300
2301 pub fn set_reserved_prefixes(&self, owner_module_id: &str, prefixes: &[String]) {
2303 let mut owners = self
2304 .reserved_prefix_owners
2305 .lock()
2306 .unwrap_or_else(|poisoned| poisoned.into_inner());
2307 owners.retain(|_, owner| owner != owner_module_id);
2308 for prefix in prefixes {
2309 owners.insert(prefix.clone(), owner_module_id.to_string());
2310 }
2311 }
2312
2313 #[cfg(test)]
2315 pub(crate) fn spawn_nonce(&self, module_id: &str) -> Option<String> {
2316 self.spawn_nonces
2317 .lock()
2318 .unwrap_or_else(|poisoned| poisoned.into_inner())
2319 .get(module_id)
2320 .cloned()
2321 }
2322
2323 fn apply_identity_configuration(&self, spec: &ModuleSpec) {
2324 self.set_reserved_prefixes(&spec.module_id, &spec.reserved_prefixes);
2325 let spawn_nonce = self
2326 .spawn_nonces
2327 .lock()
2328 .unwrap_or_else(|poisoned| poisoned.into_inner())
2329 .get(&spec.module_id)
2330 .cloned();
2331 let mut reserved_nonces = self
2332 .reserved_nonces
2333 .lock()
2334 .unwrap_or_else(|poisoned| poisoned.into_inner());
2335 if spec.reserved {
2336 reserved_nonces.insert(spec.module_id.clone(), spawn_nonce);
2341 }
2342 drop(reserved_nonces);
2343 self.removal_tombstones
2347 .lock()
2348 .unwrap_or_else(|poisoned| poisoned.into_inner())
2349 .remove(&spec.module_id);
2350 }
2351
2352 pub fn reserved_hello_authorized(&self, module_id: &str, presented: Option<&str>) -> bool {
2357 self.reserved_hello_verification(module_id, presented)
2358 .is_ok()
2359 }
2360
2361 pub(crate) fn reserved_hello_verification(
2364 &self,
2365 module_id: &str,
2366 presented: Option<&str>,
2367 ) -> Result<ReservedHelloVerification, ReservedHelloRejection> {
2368 let nonces = self
2369 .reserved_nonces
2370 .lock()
2371 .unwrap_or_else(|poisoned| poisoned.into_inner());
2372 if let Some(expected) = nonces.get(module_id) {
2373 let authorized = match expected {
2377 Some(expected) => {
2378 presented.is_some_and(|p| constant_time_eq(expected.as_bytes(), p.as_bytes()))
2379 }
2380 None => false,
2381 };
2382 if authorized {
2383 return Ok(ReservedHelloVerification {
2384 nonce_verified: true,
2385 prefix_owner: None,
2386 });
2387 }
2388 return Err(ReservedHelloRejection::Exact {
2389 module_id: module_id.to_string(),
2390 });
2391 }
2392 drop(nonces);
2393
2394 let matched_prefix = self
2395 .reserved_prefix_owners
2396 .lock()
2397 .unwrap_or_else(|poisoned| poisoned.into_inner())
2398 .iter()
2399 .filter(|(prefix, _)| module_id.starts_with(prefix.as_str()))
2400 .max_by_key(|(prefix, _)| prefix.len())
2401 .map(|(prefix, owner)| (prefix.clone(), owner.clone()));
2402 let Some((prefix, owner_module_id)) = matched_prefix else {
2403 return Ok(ReservedHelloVerification::default());
2404 };
2405
2406 let authorized = presented.is_some_and(|presented| {
2407 self.spawn_nonces
2408 .lock()
2409 .unwrap_or_else(|poisoned| poisoned.into_inner())
2410 .get(&owner_module_id)
2411 .is_some_and(|expected| constant_time_eq(expected.as_bytes(), presented.as_bytes()))
2412 || self.swap_nonce_matches(&owner_module_id, presented)
2415 });
2416 if authorized {
2417 Ok(ReservedHelloVerification {
2418 nonce_verified: true,
2419 prefix_owner: Some(owner_module_id),
2420 })
2421 } else {
2422 Err(ReservedHelloRejection::Prefix {
2423 prefix,
2424 owner_module_id,
2425 })
2426 }
2427 }
2428
2429 pub fn spawned_consumer_authorized(&self, module_id: &str, presented: &str) -> bool {
2434 if presented.is_empty() {
2435 return false;
2436 }
2437 let nonces = self
2438 .spawn_nonces
2439 .lock()
2440 .unwrap_or_else(|poisoned| poisoned.into_inner());
2441 let current = nonces
2442 .get(module_id)
2443 .is_some_and(|expected| constant_time_eq(expected.as_bytes(), presented.as_bytes()));
2444 drop(nonces);
2445 current || self.swap_nonce_matches(module_id, presented)
2450 }
2451
2452 fn swap_nonce_matches(&self, module_id: &str, presented: &str) -> bool {
2454 let swaps = self
2455 .swaps
2456 .lock()
2457 .unwrap_or_else(|poisoned| poisoned.into_inner());
2458 swaps.get(module_id).is_some_and(|swap| {
2459 constant_time_eq(swap.candidate_nonce.as_bytes(), presented.as_bytes())
2460 || swap.incumbent_nonce.as_deref().is_some_and(|incumbent| {
2461 constant_time_eq(incumbent.as_bytes(), presented.as_bytes())
2462 })
2463 })
2464 }
2465
2466 pub(crate) fn open_swap(&self, module_id: &str, candidate_nonce: String) {
2469 let incumbent_nonce = self
2470 .spawn_nonces
2471 .lock()
2472 .unwrap_or_else(|poisoned| poisoned.into_inner())
2473 .get(module_id)
2474 .cloned();
2475 self.swaps
2476 .lock()
2477 .unwrap_or_else(|poisoned| poisoned.into_inner())
2478 .insert(
2479 module_id.to_string(),
2480 OpenSwap {
2481 candidate_nonce,
2482 incumbent_nonce,
2483 candidate_admitted: false,
2484 },
2485 );
2486 }
2487
2488 pub(crate) fn close_swap(&self, module_id: &str) {
2491 self.swaps
2492 .lock()
2493 .unwrap_or_else(|poisoned| poisoned.into_inner())
2494 .remove(module_id);
2495 }
2496
2497 pub(crate) fn set_swap_promotion_observer(
2500 &self,
2501 observer: std::sync::Weak<dyn SwapPromotionObserver>,
2502 ) {
2503 *self
2504 .promotion_observer
2505 .0
2506 .lock()
2507 .unwrap_or_else(|poisoned| poisoned.into_inner()) = Some(observer);
2508 }
2509
2510 fn notify_swap_promoted(&self, registration: &crate::registry::ModuleRegistration) {
2513 let observer = self
2514 .promotion_observer
2515 .0
2516 .lock()
2517 .unwrap_or_else(|poisoned| poisoned.into_inner())
2518 .as_ref()
2519 .and_then(std::sync::Weak::upgrade);
2520 if let Some(observer) = observer {
2521 observer.swap_promoted(registration);
2522 }
2523 }
2524
2525 pub(crate) fn swap_open(&self, module_id: &str) -> bool {
2527 self.swaps
2528 .lock()
2529 .unwrap_or_else(|poisoned| poisoned.into_inner())
2530 .contains_key(module_id)
2531 }
2532
2533 fn promote_swap_nonce(&self, module_id: &str, reserved: bool) {
2538 let candidate_nonce = self
2539 .swaps
2540 .lock()
2541 .unwrap_or_else(|poisoned| poisoned.into_inner())
2542 .get(module_id)
2543 .map(|swap| swap.candidate_nonce.clone());
2544 let Some(nonce) = candidate_nonce else {
2545 return;
2546 };
2547 self.set_spawn_nonce(module_id, nonce.clone());
2548 if reserved {
2549 self.set_reserved_nonce(module_id, nonce);
2550 }
2551 }
2552
2553 pub(crate) fn swap_hello_admission(
2568 &self,
2569 module_id: &str,
2570 presented: Option<&str>,
2571 ) -> SwapHelloAdmission {
2572 let swaps = self
2573 .swaps
2574 .lock()
2575 .unwrap_or_else(|poisoned| poisoned.into_inner());
2576 let Some(swap) = swaps.get(module_id) else {
2577 return SwapHelloAdmission::NotSwapping;
2578 };
2579 let Some(presented) = presented else {
2580 return SwapHelloAdmission::Refused;
2581 };
2582 if constant_time_eq(swap.candidate_nonce.as_bytes(), presented.as_bytes()) {
2583 return if swap.candidate_admitted {
2584 SwapHelloAdmission::Refused
2585 } else {
2586 SwapHelloAdmission::Candidate
2587 };
2588 }
2589 if swap
2590 .incumbent_nonce
2591 .as_deref()
2592 .is_some_and(|incumbent| constant_time_eq(incumbent.as_bytes(), presented.as_bytes()))
2593 {
2594 return SwapHelloAdmission::NotSwapping;
2595 }
2596 SwapHelloAdmission::Refused
2597 }
2598
2599 pub(crate) fn mark_swap_candidate_admitted(&self, module_id: &str) {
2602 if let Some(swap) = self
2603 .swaps
2604 .lock()
2605 .unwrap_or_else(|poisoned| poisoned.into_inner())
2606 .get_mut(module_id)
2607 {
2608 swap.candidate_admitted = true;
2609 }
2610 }
2611
2612 pub fn spawn_launch_nonce_for(&self, module_id: &str) -> Option<String> {
2614 self.spawn_nonces
2615 .lock()
2616 .unwrap_or_else(|poisoned| poisoned.into_inner())
2617 .get(module_id)
2618 .cloned()
2619 }
2620
2621 pub fn reserved_launch_nonce_for(&self, module_id: &str) -> Option<String> {
2623 self.reserved_nonces
2624 .lock()
2625 .unwrap_or_else(|poisoned| poisoned.into_inner())
2626 .get(module_id)
2627 .cloned()
2628 .flatten()
2629 }
2630
2631 pub fn insert(&self, module: SupervisedModule) -> Option<SupervisedModule> {
2632 self.mark_configured(module.module_id());
2636 let mut modules = self
2637 .modules
2638 .lock()
2639 .unwrap_or_else(|poisoned| poisoned.into_inner());
2640 modules.insert(module.module_id().to_string(), module)
2641 }
2642
2643 fn mark_configured(&self, module_id: &str) {
2647 self.configured_ids
2648 .lock()
2649 .unwrap_or_else(|poisoned| poisoned.into_inner())
2650 .insert(module_id.to_string());
2651 }
2652
2653 fn unmark_configured_unless_rostered(&self, module_id: &str) {
2657 let modules = self
2658 .modules
2659 .lock()
2660 .unwrap_or_else(|poisoned| poisoned.into_inner());
2661 if !modules.contains_key(module_id) {
2662 self.configured_ids
2663 .lock()
2664 .unwrap_or_else(|poisoned| poisoned.into_inner())
2665 .remove(module_id);
2666 }
2667 }
2668
2669 pub(crate) fn is_configured(&self, module_id: &str) -> bool {
2677 self.configured_ids
2678 .lock()
2679 .unwrap_or_else(|poisoned| poisoned.into_inner())
2680 .contains(module_id)
2681 }
2682
2683 pub fn get(&self, module_id: &str) -> Option<SupervisedModule> {
2684 let modules = self
2685 .modules
2686 .lock()
2687 .unwrap_or_else(|poisoned| poisoned.into_inner());
2688 modules.get(module_id).cloned()
2689 }
2690
2691 pub(crate) fn record_late_health_answer(
2692 &self,
2693 module_id: &str,
2694 latency_ms: u64,
2695 ) -> Result<bool, SuperviseError> {
2696 let Some(module) = self.get(module_id) else {
2697 return Ok(false);
2698 };
2699 update_snapshot(&module.inner.snapshot, Some(module_id), |state| {
2700 state.health.late_answer_count = state.health.late_answer_count.saturating_add(1);
2701 state.health.last_late_answer_latency_ms = Some(latency_ms);
2702 state.health.consecutive_failures = 0;
2710 })?;
2711 Ok(true)
2712 }
2713
2714 pub fn record_deliberate_severance(&self, module_id: &str) -> Result<bool, SuperviseError> {
2720 let Some(module) = self.get(module_id) else {
2721 return Ok(false);
2722 };
2723 let snapshot = lock_snapshot(&module.inner.snapshot)?;
2724 let Some((pid, start_time)) = snapshot.pid.zip(snapshot.process_start_time) else {
2725 return Ok(false);
2726 };
2727 drop(snapshot);
2728 module.record_deliberate_severance(ProcessIdentity { pid, start_time })
2729 }
2730
2731 pub fn list(&self) -> Vec<SupervisedModule> {
2732 let modules = self
2733 .modules
2734 .lock()
2735 .unwrap_or_else(|poisoned| poisoned.into_inner());
2736 let mut modules = modules.values().cloned().collect::<Vec<_>>();
2737 modules.sort_by(|left, right| left.module_id().cmp(right.module_id()));
2738 modules
2739 }
2740
2741 pub(crate) fn retire(&self, module_id: &str) -> Option<SupervisedModule> {
2742 self.activity_images
2743 .lock()
2744 .unwrap_or_else(|p| p.into_inner())
2745 .remove(module_id);
2746 self.spawn_nonces
2747 .lock()
2748 .unwrap_or_else(|poisoned| poisoned.into_inner())
2749 .remove(module_id);
2750 self.close_swap(module_id);
2751 let mut reserved_nonces = self
2752 .reserved_nonces
2753 .lock()
2754 .unwrap_or_else(|poisoned| poisoned.into_inner());
2755 if reserved_nonces.contains_key(module_id) {
2756 reserved_nonces.insert(module_id.to_string(), None);
2759 }
2760 drop(reserved_nonces);
2761 self.reserved_prefix_owners
2762 .lock()
2763 .unwrap_or_else(|poisoned| poisoned.into_inner())
2764 .retain(|_, owner| owner != module_id);
2765 let removed = self
2766 .modules
2767 .lock()
2768 .unwrap_or_else(|poisoned| poisoned.into_inner())
2769 .remove(module_id);
2770 self.configured_ids
2771 .lock()
2772 .unwrap_or_else(|poisoned| poisoned.into_inner())
2773 .remove(module_id);
2774 removed
2775 }
2776
2777 pub(crate) fn record_rescan_removal(&self, module_id: &str) {
2780 self.removal_tombstones
2781 .lock()
2782 .unwrap_or_else(|poisoned| poisoned.into_inner())
2783 .insert(module_id.to_string(), unix_ms_now());
2784 }
2785
2786 pub(crate) fn removal_tombstone_age_ms(&self, module_id: &str) -> Option<u64> {
2788 self.removal_tombstones
2789 .lock()
2790 .unwrap_or_else(|poisoned| poisoned.into_inner())
2791 .get(module_id)
2792 .copied()
2793 .map(|removed_at_ms| unix_ms_now().saturating_sub(removed_at_ms))
2794 }
2795
2796 pub(crate) fn release_retained_reserved_gate(&self, module_id: &str) -> bool {
2801 if self.get(module_id).is_some() {
2802 return false;
2803 }
2804 let mut reserved_nonces = self
2805 .reserved_nonces
2806 .lock()
2807 .unwrap_or_else(|poisoned| poisoned.into_inner());
2808 if !matches!(reserved_nonces.get(module_id), Some(None)) {
2809 return false;
2810 }
2811 reserved_nonces.remove(module_id);
2812 true
2813 }
2814
2815 pub(crate) fn operation_lock(&self) -> Arc<AsyncMutex<()>> {
2816 Arc::clone(&self.operation_lock)
2817 }
2818}
2819
2820#[derive(Debug, Clone)]
2822pub struct Supervisor {
2823 registry: Arc<Registry>,
2824 restart_policy: RestartPolicy,
2825 drain_timeout: Duration,
2826 connection_file_path: Option<PathBuf>,
2827 capture_logs_dir: Option<PathBuf>,
2828 forwarding: Option<Arc<ForwardingTable>>,
2829 process_liveness: Arc<SupervisorProcessLiveness>,
2830 supervisor_handle: Option<SupervisorHandle>,
2831 health: HealthConfig,
2832 daemon_start_clock: crate::clock::StartClock,
2833 terminal_journal: Option<Arc<crate::terminal_journal::TerminalJournal>>,
2834 spawn_events: SpawnEventFeed,
2835 provenance_probe: ExecutableIdentityProbe,
2836 child_roster: ChildRoster,
2839 #[cfg(target_os = "linux")]
2840 cgroup_placement: Option<subc_cgroup::Placement>,
2841 #[cfg(test)]
2842 test_after_first_spawn: AfterFirstSpawnHook,
2843}
2844
2845pub(crate) struct RegisteredConfiguredModule {
2849 spec: ModuleSpec,
2850 runtime: SupervisorRuntimeConfig,
2851 module: SupervisedModule,
2852 commands: mpsc::Receiver<SupervisorCommand>,
2853}
2854
2855#[cfg(test)]
2861#[derive(Clone, Default)]
2862struct AfterFirstSpawnHook(Option<AfterFirstSpawnFn>);
2863
2864#[cfg(test)]
2865type AfterFirstSpawnFn = Arc<dyn Fn(&str) + Send + Sync>;
2866
2867#[cfg(test)]
2868impl fmt::Debug for AfterFirstSpawnHook {
2869 fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
2870 f.write_str("AfterFirstSpawnHook")
2871 }
2872}
2873
2874#[cfg(test)]
2875impl AfterFirstSpawnHook {
2876 fn run(&self, module_id: &str) {
2877 if let Some(hook) = &self.0 {
2878 hook(module_id);
2879 }
2880 }
2881}
2882
2883impl Supervisor {
2884 #[cfg(test)]
2885 pub(crate) fn new_for_test(registry: Arc<Registry>, policy: RestartPolicy) -> Self {
2886 let supervisor = Self::new(registry, policy);
2887 #[cfg(target_os = "macos")]
2888 let supervisor = supervisor.with_privacy_trampoline(test_privacy_trampoline());
2889 supervisor
2890 }
2891 pub fn with_privacy_trampoline(self, path: impl Into<PathBuf>) -> Self {
2896 let path = path.into();
2897 #[cfg(target_os = "macos")]
2898 {
2899 let result = probe_privacy_trampoline(&path).map(|()| path);
2900 if let Err(cause) = &result {
2901 error!(%cause, "macOS privacy identity unavailable; supervised launches will refuse");
2902 }
2903 self.child_roster.set_privacy_trampoline(result);
2904 }
2905 #[cfg(not(target_os = "macos"))]
2906 let _ = path;
2907 self
2908 }
2909 pub(crate) fn daemon_shutdown_flag(&self) -> crate::child_roster::DaemonShutdownFlag {
2912 self.child_roster.shutdown_flag()
2913 }
2914
2915 #[cfg(unix)]
2926 pub(crate) fn begin_daemon_shutdown(&self) {
2927 self.child_roster.close();
2928 if let Some(journal) = &self.terminal_journal {
2929 journal.stamp_shutdown();
2930 }
2931 }
2932
2933 #[cfg(unix)]
2938 pub(crate) async fn drain_for_daemon_shutdown(&self) -> Result<(), SuperviseError> {
2939 const NOTICE_BUDGET: Duration = Duration::from_millis(500);
2940 const DRAIN_BUDGET: Duration = Duration::from_secs(2);
2941 let Some(forwarding) = &self.forwarding else {
2942 return Ok(());
2943 };
2944 let notice_started = Instant::now();
2945 let modules = forwarding
2946 .begin_daemon_drain()
2947 .map_err(SuperviseError::Forwarding)?;
2948 let deadline_ms =
2949 unix_ms_now().saturating_add((NOTICE_BUDGET + DRAIN_BUDGET).as_millis() as u64);
2950 let shutdown_reason = self
2951 .supervisor_handle
2952 .as_ref()
2953 .map(SupervisorHandle::take_daemon_shutdown_reason)
2954 .unwrap_or(RouteCloseReason::Disable);
2955 let mut modules_told = 0;
2956 let mut notices = tokio::task::JoinSet::new();
2957 let mut drains = Vec::new();
2958 for connection in forwarding
2959 .module_connections()
2960 .map_err(SuperviseError::Forwarding)?
2961 {
2962 let target = ModuleDrainTarget {
2963 endpoint: connection.endpoint,
2964 sink: connection.sink,
2965 negotiated_ver: connection.negotiated_ver,
2966 abandoned_bindings: Vec::new(),
2967 excluded_subscriptions: 0,
2968 };
2969 notices.spawn(async move {
2970 (
2971 true,
2972 send_module_draining(
2973 &connection.module_id,
2974 shutdown_reason,
2975 deadline_ms,
2976 &target,
2977 true,
2978 )
2979 .await,
2980 )
2981 });
2982 }
2983 for (module_id, endpoint) in modules {
2984 let Some(target) = forwarding
2985 .begin_endpoint_drain(endpoint, RouteCloseReason::Restart)
2986 .map_err(SuperviseError::Forwarding)?
2987 else {
2988 continue;
2989 };
2990 let routes = forwarding
2991 .endpoint_routes(target.endpoint)
2992 .map_err(SuperviseError::Forwarding)?;
2993 let mut recipients = Vec::new();
2996 let mut clients: Vec<(crate::forwarding::GoodbyeTarget, Vec<u16>)> = Vec::new();
2997 for route in routes {
2998 let client = route.goodbye_target;
2999 if let Some((_, channels)) = clients
3000 .iter_mut()
3001 .find(|(existing, _)| existing.connection_id == client.connection_id)
3002 {
3003 channels.push(client.channel);
3004 } else {
3005 let channel = client.channel;
3006 clients.push((client, vec![channel]));
3007 }
3008 }
3009 for (client, mut channels) in clients {
3010 channels.sort_unstable();
3011 channels.dedup();
3012 let closing = serde_json::to_vec(&ClientControlPush::RouteClosing {
3013 module_id: module_id.clone(),
3014 channels,
3015 reason: RouteCloseReason::Restart,
3016 })
3017 .expect("route closing serializes");
3018 recipients.push((client.sink, client.negotiated_ver, closing));
3019 }
3020 for (sink, version, body) in recipients {
3021 notices.spawn(async move {
3022 let frame = Frame::build_with_version(
3023 version,
3024 FrameType::Push,
3025 control_flags(),
3026 0,
3027 0,
3028 0,
3029 body,
3030 )
3031 .expect("bounded lifecycle notice frame builds");
3032 (false, sink.send_flushed(frame).await.is_ok())
3033 });
3034 }
3035 let gauges = declared_busy_gauges(&self.registry, &module_id)?;
3036 drains.push((module_id, target.endpoint, gauges));
3037 }
3038 let notice_deadline = Instant::now() + NOTICE_BUDGET;
3041 while let Ok(Some(result)) = timeout_at(notice_deadline, notices.join_next()).await {
3042 match result {
3043 Ok((true, true)) => modules_told += 1,
3044 Ok((false, true)) => {}
3045 result => warn!(?result, "daemon shutdown notice delivery failed"),
3046 }
3047 }
3048 notices.abort_all();
3049 info!(
3050 ?shutdown_reason,
3051 modules_told, "daemon shutdown module notices delivered"
3052 );
3053 let notice_elapsed = notice_started.elapsed();
3054 let drain_started = Instant::now();
3055 let deadline = drain_started + DRAIN_BUDGET;
3056 let mut waits = tokio::task::JoinSet::new();
3057 for (module_id, endpoint, gauges) in drains {
3058 let forwarding = Arc::clone(forwarding);
3059 let mut runtime = self.runtime_config();
3060 runtime.health.cadence = Duration::from_millis(100);
3061 waits.spawn(async move {
3062 let wait_result = wait_for_forwarding_quiescence(
3063 &forwarding,
3064 &module_id,
3065 &runtime,
3066 endpoint,
3067 deadline,
3068 &gauges,
3069 DrainScope::Endpoint(endpoint),
3070 &mut DrainGaugeTrace::new(&gauges),
3071 )
3072 .await;
3073 forwarding
3077 .record_daemon_drain_result(
3078 endpoint,
3079 drained_after_quiescence_wait(&wait_result),
3080 )
3081 .map_err(SuperviseError::Forwarding)?;
3082 wait_result
3083 });
3084 }
3085 while let Ok(Some(result)) = timeout_at(deadline, waits.join_next()).await {
3086 if !matches!(result, Ok(Ok(true))) {
3087 warn!(?result, "daemon shutdown drain did not reach quiescence");
3088 }
3089 }
3090 info!(
3093 notice_elapsed_ms = notice_elapsed.as_millis() as u64,
3094 drain_elapsed_ms = drain_started.elapsed().as_millis() as u64,
3095 "daemon shutdown forwarding drain complete"
3096 );
3097 Ok(())
3098 }
3099
3100 #[cfg(unix)]
3112 pub(crate) async fn end_children_for_daemon_shutdown(
3113 &self,
3114 already_escalated: bool,
3115 escalate: impl std::future::Future<Output = ()>,
3116 ) {
3117 tokio::pin!(escalate);
3118 let mut escalated = already_escalated;
3119 if let Some(forwarding) = &self.forwarding {
3120 let reason = CloseReason::new(
3121 "daemon_shutdown",
3122 "the daemon is exiting after its shutdown notice and drain",
3123 );
3124 if escalated {
3125 send_module_goodbyes_for_daemon_shutdown(forwarding, &reason, false).await;
3128 } else {
3129 tokio::select! {
3130 biased;
3131 _ = escalate.as_mut() => {
3132 info!("second SIGTERM: abandoning module GOODBYE delivery");
3133 escalated = true;
3134 }
3135 _ = send_module_goodbyes_for_daemon_shutdown(forwarding, &reason, true) => {}
3136 }
3137 }
3138 let closed = forwarding.close_all_connections(&reason);
3139 debug!(closed, "closed established connections for daemon shutdown");
3140 }
3141 let escalated_here = escalated && !already_escalated;
3145 let remaining_escalate = async move {
3146 if escalated_here {
3147 std::future::pending::<()>().await;
3148 } else {
3149 escalate.await;
3150 }
3151 };
3152 crate::child_roster::end_children_for_daemon_shutdown(
3153 &self.child_roster,
3154 escalated,
3155 remaining_escalate,
3156 )
3157 .await;
3158 }
3159
3160 pub fn new(registry: Arc<Registry>, restart_policy: RestartPolicy) -> Self {
3161 Self {
3162 registry,
3163 restart_policy,
3164 drain_timeout: DEFAULT_DRAIN_TIMEOUT,
3165 connection_file_path: None,
3166 capture_logs_dir: None,
3167 forwarding: None,
3168 process_liveness: Arc::new(SupervisorProcessLiveness::default()),
3169 supervisor_handle: None,
3170 health: HealthConfig::default(),
3171 daemon_start_clock: crate::clock::StartClock::capture(),
3172 terminal_journal: None,
3173 spawn_events: SpawnEventFeed::default(),
3174 provenance_probe: ExecutableIdentityProbe::default(),
3175 child_roster: ChildRoster::default(),
3176 #[cfg(target_os = "linux")]
3177 cgroup_placement: None,
3178 #[cfg(test)]
3179 test_after_first_spawn: AfterFirstSpawnHook::default(),
3180 }
3181 }
3182
3183 pub fn with_drain_timeout(mut self, drain_timeout: Duration) -> Self {
3184 self.drain_timeout = drain_timeout;
3185 self
3186 }
3187
3188 pub(crate) fn default_drain_timeout(&self) -> Duration {
3190 self.drain_timeout
3191 }
3192
3193 pub fn with_process_liveness(
3194 mut self,
3195 process_liveness: Arc<SupervisorProcessLiveness>,
3196 ) -> Self {
3197 self.process_liveness = process_liveness;
3198 self
3199 }
3200
3201 pub fn with_connection_file_path(mut self, connection_file_path: impl Into<PathBuf>) -> Self {
3202 self.connection_file_path = Some(connection_file_path.into());
3203 self
3204 }
3205
3206 pub fn with_capture_logs_dir(mut self, logs_dir: impl Into<PathBuf>) -> Self {
3208 self.capture_logs_dir = Some(logs_dir.into());
3209 self
3210 }
3211
3212 pub fn with_daemon_incarnation(self, daemon_incarnation: String) -> Self {
3215 self.spawn_events.configure_incarnation(daemon_incarnation);
3219 self
3220 }
3221
3222 pub fn with_terminal_journal(self, path: PathBuf, daemon_incarnation: String) -> Self {
3225 let mut this = self.with_daemon_incarnation(daemon_incarnation.clone());
3226 this.terminal_journal = Some(Arc::new(crate::terminal_journal::TerminalJournal::open(
3227 path,
3228 daemon_incarnation,
3229 )));
3230 this
3231 }
3232
3233 pub fn with_forwarding(mut self, forwarding: Arc<ForwardingTable>) -> Self {
3234 self.child_roster.counters = forwarding.counters();
3235 self.forwarding = Some(forwarding);
3236 self
3237 }
3238
3239 pub fn with_handle(mut self, supervisor_handle: SupervisorHandle) -> Self {
3240 self.spawn_events = supervisor_handle.spawn_events.clone();
3241 self.supervisor_handle = Some(supervisor_handle);
3242 self
3243 }
3244
3245 pub fn with_health_config(mut self, health: HealthConfig) -> Self {
3246 self.health = health;
3247 self
3248 }
3249
3250 pub fn with_live_children_record(self, path: impl Into<PathBuf>) -> Self {
3254 self.child_roster.record_to(path.into());
3255 self
3256 }
3257
3258 #[cfg(target_os = "linux")]
3259 pub fn with_cgroup_placement(
3260 mut self,
3261 cgroup_placement: Option<subc_cgroup::Placement>,
3262 ) -> Self {
3263 self.child_roster
3264 .set_cgroup_placement(cgroup_placement.clone());
3265 self.cgroup_placement = cgroup_placement;
3266 self
3267 }
3268
3269 pub fn spawn(&self, spec: ModuleSpec) -> Result<SupervisedModule, SuperviseError> {
3275 validate_spec(&spec)?;
3276 self.establish_identity(&spec);
3277
3278 let runtime = self.runtime_config();
3279 let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::starting()));
3280 let activity_spawn = runtime.begin_activity_spawn(&spec.module_id);
3281 let registration_watch = watch_registration(&self.registry, &spec);
3282 let spawned = spawn_child(
3283 &spec,
3284 runtime.connection_file_path.as_deref(),
3285 self.supervisor_handle.as_ref(),
3286 &runtime.stderr_ring,
3287 runtime.capture_logs_dir.as_deref(),
3288 &runtime.child_roster,
3289 #[cfg(target_os = "linux")]
3290 runtime.cgroup_placement.as_ref(),
3291 );
3292 #[cfg(test)]
3293 self.test_after_first_spawn.run(&spec.module_id);
3294 let mut child = match spawned {
3295 Ok(child) => child,
3296 Err(err) => {
3297 self.abandon_unrostered(&spec.module_id);
3300 return Err(err);
3301 }
3302 };
3303 arm_registration_deadline(&mut child, runtime.register_timeout, registration_watch);
3304 self.mark_first_process_running(&spec, &runtime, &snapshot, &mut child)?;
3305 if let Some(spawn) = activity_spawn {
3306 spawn.complete(&snapshot);
3307 }
3308
3309 Ok(self.supervised_module(spec, runtime, snapshot, Some(child)))
3310 }
3311
3312 fn establish_identity(&self, spec: &ModuleSpec) {
3325 if let Some(supervisor_handle) = &self.supervisor_handle {
3326 supervisor_handle.apply_identity_configuration(spec);
3327 supervisor_handle.mark_configured(&spec.module_id);
3328 }
3329 }
3330
3331 fn abandon_unrostered(&self, module_id: &str) {
3334 if let Some(supervisor_handle) = &self.supervisor_handle {
3335 supervisor_handle.unmark_configured_unless_rostered(module_id);
3336 }
3337 }
3338
3339 fn mark_first_process_running(
3342 &self,
3343 spec: &ModuleSpec,
3344 runtime: &SupervisorRuntimeConfig,
3345 snapshot: &SharedSnapshot,
3346 child: &mut SupervisedChild,
3347 ) -> Result<(), SuperviseError> {
3348 if let Err(err) = set_running(snapshot, child, &spec.module_id, &runtime.spawn_events) {
3349 containment::terminate_blocking(&runtime.child_roster.containment);
3350 self.abandon_unrostered(&spec.module_id);
3351 return Err(err);
3352 }
3353 self.process_liveness
3354 .track(spec.module_id.clone(), Arc::clone(snapshot));
3355 Ok(())
3356 }
3357
3358 pub fn supervise_configured(
3364 &self,
3365 spec: ModuleSpec,
3366 enabled: bool,
3367 ) -> Result<SupervisedModule, SuperviseError> {
3368 self.supervise_configured_requested(
3369 spec,
3370 enabled,
3371 self.health.clone(),
3372 None,
3373 self.restart_policy,
3374 DEFAULT_REGISTER_TIMEOUT,
3375 None,
3376 )
3377 }
3378
3379 pub fn supervise_configured_with_health(
3385 &self,
3386 spec: ModuleSpec,
3387 enabled: bool,
3388 health: HealthConfig,
3389 drain_timeout_ms: Option<u64>,
3390 restart_policy: RestartPolicy,
3391 ) -> Result<SupervisedModule, SuperviseError> {
3392 self.supervise_configured_requested(
3393 spec,
3394 enabled,
3395 health,
3396 drain_timeout_ms,
3397 restart_policy,
3398 DEFAULT_REGISTER_TIMEOUT,
3399 None,
3400 )
3401 }
3402
3403 #[allow(clippy::too_many_arguments)]
3404 pub(crate) fn supervise_configured_requested(
3405 &self,
3406 spec: ModuleSpec,
3407 enabled: bool,
3408 health: HealthConfig,
3409 drain_timeout_ms: Option<u64>,
3410 restart_policy: RestartPolicy,
3411 register_timeout: Duration,
3412 requester: Option<Principal>,
3413 ) -> Result<SupervisedModule, SuperviseError> {
3414 let registered = self.register_configured_with_health(
3415 spec,
3416 enabled,
3417 health,
3418 drain_timeout_ms,
3419 restart_policy,
3420 register_timeout,
3421 )?;
3422 let daemon_initiated = requester.is_none();
3423 self.start_registered_configured(registered, requester, daemon_initiated)
3424 }
3425
3426 pub(crate) fn register_configured_with_health(
3431 &self,
3432 spec: ModuleSpec,
3433 enabled: bool,
3434 health: HealthConfig,
3435 drain_timeout_ms: Option<u64>,
3436 restart_policy: RestartPolicy,
3437 register_timeout: Duration,
3438 ) -> Result<RegisteredConfiguredModule, SuperviseError> {
3439 validate_spec(&spec)?;
3440 self.establish_identity(&spec);
3441
3442 let mut runtime = self.runtime_config();
3443 runtime.health = health;
3444 runtime.restart_policy = restart_policy;
3445 runtime.register_timeout = register_timeout;
3446 if let Some(ms) = drain_timeout_ms {
3447 runtime.drain_timeout = Duration::from_millis(ms);
3448 *runtime
3449 .effective_drain_timeout
3450 .lock()
3451 .unwrap_or_else(|poisoned| poisoned.into_inner()) = runtime.drain_timeout;
3452 }
3453 let snapshot = if enabled {
3454 SupervisorSnapshot::starting()
3455 } else {
3456 SupervisorSnapshot::disabled()
3457 };
3458 let snapshot = {
3459 let mut snapshot = snapshot;
3460 if enabled {
3461 if let Some((pid, reason)) = runtime.child_roster.previous_instance(&spec.module_id)
3462 {
3463 snapshot.waiting_for_previous_pid = Some(pid);
3464 snapshot.waiting_for_previous_detail = Some(reason.waiting_message(pid));
3465 }
3466 }
3467 snapshot
3468 };
3469 let (module, commands) =
3470 self.register_module(&spec, &runtime, Arc::new(Mutex::new(snapshot)));
3471 Ok(RegisteredConfiguredModule {
3472 spec,
3473 runtime,
3474 module,
3475 commands,
3476 })
3477 }
3478
3479 pub(crate) fn start_registered_configured(
3482 &self,
3483 registered: RegisteredConfiguredModule,
3484 requester: Option<Principal>,
3485 daemon_initiated: bool,
3486 ) -> Result<SupervisedModule, SuperviseError> {
3487 let RegisteredConfiguredModule {
3488 spec,
3489 mut runtime,
3490 module,
3491 commands,
3492 } = registered;
3493 runtime.start_requester = requester;
3494 runtime.recovery_is_daemon = daemon_initiated;
3495 let snapshot = &module.inner.snapshot;
3496 let should_spawn = {
3497 let state = lock_snapshot(snapshot)?;
3498 state.enabled && state.waiting_for_previous_pid.is_none()
3499 };
3500 let child = if should_spawn {
3501 let activity_spawn = runtime.begin_activity_spawn(&spec.module_id);
3502 let registration_watch = watch_registration(&self.registry, &spec);
3503 let spawned = spawn_child(
3504 &spec,
3505 runtime.connection_file_path.as_deref(),
3506 self.supervisor_handle.as_ref(),
3507 &runtime.stderr_ring,
3508 runtime.capture_logs_dir.as_deref(),
3509 &runtime.child_roster,
3510 #[cfg(target_os = "linux")]
3511 runtime.cgroup_placement.as_ref(),
3512 );
3513 #[cfg(test)]
3514 self.test_after_first_spawn.run(&spec.module_id);
3515 match spawned {
3516 Ok(mut child) => {
3517 arm_registration_deadline(
3518 &mut child,
3519 runtime.register_timeout,
3520 registration_watch,
3521 );
3522 self.mark_first_process_running(&spec, &runtime, snapshot, &mut child)?;
3523 if let Some(spawn) = activity_spawn {
3524 spawn.complete(snapshot);
3525 }
3526 Some(child)
3527 }
3528 Err(err) => {
3529 if runtime.health.critical {
3530 error!(
3531 module_id = %spec.module_id,
3532 program = %spec.program.display(),
3533 error = %err,
3534 "critical configured module failed to spawn; marking failed and alerting"
3535 );
3536 } else {
3537 error!(
3538 module_id = %spec.module_id,
3539 program = %spec.program.display(),
3540 error = %err,
3541 "configured module failed to spawn; marking failed and continuing"
3542 );
3543 }
3544 fail_snapshot(
3545 snapshot,
3546 Some(&spec.module_id),
3547 None,
3548 &runtime.failure_context(daemon_initiated),
3549 );
3550 None
3551 }
3552 }
3553 } else {
3554 None
3555 };
3556 self.start_monitor(spec, runtime, &module, child, commands);
3557 Ok(module)
3558 }
3559
3560 fn runtime_config(&self) -> SupervisorRuntimeConfig {
3561 let effective_drain_timeout = Arc::new(Mutex::new(self.drain_timeout));
3562 SupervisorRuntimeConfig {
3563 registry: self.registry.clone(),
3564 recovery_is_daemon: true,
3565 start_requester: None,
3566 pending_activity_restart: false,
3567 provenance_probe: self.provenance_probe.clone(),
3568 scheduled_respawn: Arc::default(),
3569 deferred_reload_reply: Arc::default(),
3570 restart_policy: self.restart_policy,
3571 register_timeout: DEFAULT_REGISTER_TIMEOUT,
3572 drain_timeout: self.drain_timeout,
3573 child_roster: self
3576 .child_roster
3577 .for_module(Arc::clone(&effective_drain_timeout)),
3578 effective_drain_timeout,
3579 default_drain_timeout: self.drain_timeout,
3580 health: self.health.clone(),
3581 connection_file_path: self.connection_file_path.clone(),
3582 capture_logs_dir: self.capture_logs_dir.clone(),
3583 forwarding: self.forwarding.clone(),
3584 supervisor_handle: self.supervisor_handle.clone(),
3585 stderr_ring: Arc::new(Mutex::new(StderrRing::new(StderrTailConfig::default()))),
3586 terminal_ring: Arc::new(Mutex::new(
3587 TerminalRing::new(
3588 TerminalRingConfig::default(),
3589 self.daemon_start_clock.started_at_ms(),
3590 )
3591 .with_start_clock(self.daemon_start_clock)
3592 .with_journal(self.terminal_journal.clone())
3593 .with_daemon_shutdown(self.child_roster.shutdown_flag()),
3594 )),
3595 spawn_events: self.spawn_events.clone(),
3596 #[cfg(target_os = "linux")]
3597 cgroup_placement: self.cgroup_placement.clone(),
3598 #[cfg(test)]
3599 test_seed_stale_facts_before_enable_spawn: false,
3600 #[cfg(test)]
3601 test_reload_exit_record_gate: None,
3602 }
3603 }
3604
3605 fn supervised_module(
3606 &self,
3607 spec: ModuleSpec,
3608 runtime: SupervisorRuntimeConfig,
3609 snapshot: SharedSnapshot,
3610 child: Option<SupervisedChild>,
3611 ) -> SupervisedModule {
3612 let (module, commands) = self.register_module(&spec, &runtime, snapshot);
3613 self.start_monitor(spec, runtime, &module, child, commands);
3614 module
3615 }
3616
3617 fn register_module(
3618 &self,
3619 spec: &ModuleSpec,
3620 runtime: &SupervisorRuntimeConfig,
3621 snapshot: SharedSnapshot,
3622 ) -> (SupervisedModule, mpsc::Receiver<SupervisorCommand>) {
3623 let configuration = Arc::new(Mutex::new(SupervisedConfiguration {
3624 spec: spec.clone(),
3625 health: runtime.health.clone(),
3626 }));
3627 let stderr_ring = Arc::clone(&runtime.stderr_ring);
3628 let terminal_ring = Arc::clone(&runtime.terminal_ring);
3629 let restart_policy = runtime.restart_policy;
3633 let effective_drain_timeout = Arc::clone(&runtime.effective_drain_timeout);
3634 let (tx, rx) = mpsc::channel(4);
3635 let module_id = spec.module_id.clone();
3636 let module = SupervisedModule {
3637 inner: Arc::new(SupervisedModuleInner {
3638 module_id: module_id.clone(),
3639 registry: Arc::clone(&self.registry),
3640 snapshot,
3641 configuration,
3642 stderr_ring,
3643 terminal_ring,
3644 commands: tx,
3645 monitor: Mutex::new(None),
3646 restart_policy,
3647 effective_drain_timeout,
3648 provenance_probe: self.provenance_probe.clone(),
3649 retained: runtime.child_roster.retained_view(),
3650 }),
3651 };
3652 if let Some(supervisor_handle) = &self.supervisor_handle {
3655 supervisor_handle.insert(module.clone());
3656 }
3657 (module, rx)
3658 }
3659
3660 fn start_monitor(
3661 &self,
3662 spec: ModuleSpec,
3663 runtime: SupervisorRuntimeConfig,
3664 module: &SupervisedModule,
3665 child: Option<SupervisedChild>,
3666 commands: mpsc::Receiver<SupervisorCommand>,
3667 ) {
3668 let monitor = tokio::spawn(supervise_loop(
3669 spec,
3670 runtime,
3671 Arc::clone(&self.registry),
3672 Arc::clone(&self.process_liveness),
3673 Arc::clone(&module.inner.snapshot),
3674 child,
3675 commands,
3676 ));
3677 *module
3678 .inner
3679 .monitor
3680 .lock()
3681 .unwrap_or_else(|p| p.into_inner()) = Some(monitor);
3682 }
3683}
3684
3685impl Default for Supervisor {
3686 fn default() -> Self {
3687 Self::new(Arc::new(Registry::default()), RestartPolicy::default())
3688 }
3689}
3690
3691#[derive(Clone)]
3693pub struct SupervisedModule {
3694 inner: Arc<SupervisedModuleInner>,
3695}
3696
3697struct SupervisedModuleInner {
3698 retained: crate::child_roster::RetainedView,
3699 module_id: String,
3700 registry: Arc<Registry>,
3701 snapshot: SharedSnapshot,
3702 configuration: Arc<Mutex<SupervisedConfiguration>>,
3703 stderr_ring: Arc<Mutex<StderrRing>>,
3704 terminal_ring: Arc<Mutex<TerminalRing>>,
3705 commands: mpsc::Sender<SupervisorCommand>,
3706 monitor: Mutex<Option<JoinHandle<()>>>,
3707 restart_policy: RestartPolicy,
3711 effective_drain_timeout: Arc<Mutex<Duration>>,
3712 provenance_probe: ExecutableIdentityProbe,
3713}
3714
3715impl fmt::Debug for SupervisedModule {
3716 fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
3717 f.debug_struct("SupervisedModule")
3718 .field("module_id", &self.inner.module_id)
3719 .field("status", &self.status())
3720 .finish_non_exhaustive()
3721 }
3722}
3723
3724impl SupervisedModule {
3725 pub fn module_id(&self) -> &str {
3726 &self.inner.module_id
3727 }
3728
3729 #[cfg(test)]
3733 pub(crate) fn record_health_probe_failure_for_test(
3734 &self,
3735 detail: &str,
3736 ) -> Result<(), SuperviseError> {
3737 update_snapshot(&self.inner.snapshot, Some(&self.inner.module_id), |state| {
3738 state.health.consecutive_failures = state.health.consecutive_failures.saturating_add(1);
3739 state.health.detail = Some(detail.to_string());
3740 })
3741 }
3742
3743 pub fn state(&self) -> Result<ModuleState, SuperviseError> {
3744 Ok(lock_snapshot(&self.inner.snapshot)?.state)
3745 }
3746
3747 pub(crate) fn registration_end_reason(
3748 &self,
3749 ) -> Result<Option<RegistrationEndReason>, SuperviseError> {
3750 let snapshot = lock_snapshot(&self.inner.snapshot)?;
3751 Ok(match snapshot.state {
3752 ModuleState::Draining if snapshot.draining_to_replace => {
3753 Some(RegistrationEndReason::SupervisorRestart)
3754 }
3755 ModuleState::Draining => Some(RegistrationEndReason::SupervisorStop),
3756 ModuleState::Restarting if snapshot.draining_to_replace => {
3757 Some(RegistrationEndReason::SupervisorRestart)
3758 }
3759 _ => None,
3760 })
3761 }
3762
3763 pub fn stderr_tail(
3770 &self,
3771 max_lines: Option<usize>,
3772 max_bytes: Option<usize>,
3773 ) -> StderrTailSnapshot {
3774 self.inner
3775 .stderr_ring
3776 .lock()
3777 .unwrap_or_else(|poisoned| poisoned.into_inner())
3778 .snapshot(max_lines, max_bytes)
3779 }
3780
3781 pub fn terminal_history(&self) -> TerminalHistorySnapshot {
3786 self.inner
3787 .terminal_ring
3788 .lock()
3789 .unwrap_or_else(|poisoned| poisoned.into_inner())
3790 .snapshot()
3791 }
3792
3793 pub fn durable_terminal_history(&self) -> subc_control::TerminalHistory {
3798 durable_terminal_history_of(&self.inner.terminal_ring, &self.inner.module_id)
3799 }
3800
3801 pub(crate) async fn read_durable_terminal_history(
3806 &self,
3807 ) -> Result<subc_control::TerminalHistory, tokio::task::JoinError> {
3808 let terminal_ring = Arc::clone(&self.inner.terminal_ring);
3809 let module_id = self.inner.module_id.clone();
3810 tokio::task::spawn_blocking(move || durable_terminal_history_of(&terminal_ring, &module_id))
3811 .await
3812 }
3813
3814 pub fn status(&self) -> Result<ModuleStatus, SuperviseError> {
3815 self.status_with_snapshot_lock(&self.inner.snapshot, None)
3816 .map(|(status, _)| status)
3817 }
3818
3819 pub(crate) fn next_retry_at_ms(&self) -> Result<Option<u64>, SuperviseError> {
3820 let state = lock_snapshot(&self.inner.snapshot)?;
3821 Ok(state
3822 .failed_retry
3823 .pending
3824 .filter(|_| state.state == ModuleState::Failed && state.enabled)
3825 .map(|retry| {
3826 unix_ms_now().saturating_add(
3827 u64::try_from(
3828 retry
3829 .deadline
3830 .saturating_duration_since(Instant::now())
3831 .as_millis(),
3832 )
3833 .unwrap_or(u64::MAX),
3834 )
3835 }))
3836 }
3837
3838 pub(crate) fn record_deliberate_severance(
3839 &self,
3840 identity: ProcessIdentity,
3841 ) -> Result<bool, SuperviseError> {
3842 let mut snapshot = lock_snapshot(&self.inner.snapshot)?;
3843 if snapshot.pid != Some(identity.pid)
3844 || snapshot.process_start_time != Some(identity.start_time)
3845 {
3846 return Ok(false);
3847 }
3848 snapshot.deliberate_severance = Some(identity);
3849 Ok(true)
3850 }
3851
3852 pub(crate) fn status_for_control(
3857 &self,
3858 caller: &'static str,
3859 ) -> Result<ModuleStatus, SuperviseError> {
3860 self.status_with_snapshot_lock(&self.inner.snapshot, Some(caller))
3861 .map(|(status, _)| status)
3862 }
3863
3864 fn status_with_snapshot_lock(
3865 &self,
3866 snapshot: &SharedSnapshot,
3867 caller: Option<&'static str>,
3868 ) -> Result<(ModuleStatus, Option<SpawnedFileIdentity>), SuperviseError> {
3869 let mut guard = match caller {
3870 Some(caller) => lock_snapshot_for_control(snapshot, &self.inner.module_id, caller)?,
3871 None => lock_snapshot(snapshot)?,
3872 };
3873 let restart_count =
3876 guard.crash_restarts_in_window(self.inner.restart_policy.window, Instant::now());
3877 let snapshot = guard.clone();
3878 drop(guard);
3879 let drain_timeout = *self.inner.effective_drain_timeout.lock().map_err(|_| {
3880 SuperviseError::StatePoisoned {
3881 module_id: Some(self.inner.module_id.clone()),
3882 }
3883 })?;
3884 let registration_active = self
3885 .inner
3886 .registry
3887 .get_module(&self.inner.module_id)
3888 .map_err(SuperviseError::Registry)?
3889 .is_some();
3890 let protocol = snapshot
3891 .spawned_protocol
3892 .unwrap_or(self.declared_protocol()?);
3893 let running_process =
3894 snapshot.enabled && snapshot.state == ModuleState::Running && snapshot.process_alive;
3895 let live = match protocol {
3901 ModuleProtocol::Subc => running_process && registration_active,
3902 ModuleProtocol::None => running_process,
3903 };
3904 let retained = self.inner.retained.waiting_detail(&self.inner.module_id);
3907 let carry_configured = self
3908 .inner
3909 .configuration
3910 .lock()
3911 .map_err(|_| SuperviseError::StatePoisoned {
3912 module_id: Some(self.inner.module_id.clone()),
3913 })?
3914 .spec
3915 .carry_containment;
3916 let recovery_block = carries_containment(
3917 carry_configured,
3918 self.inner
3919 .retained
3920 .has_carried_container(&self.inner.module_id),
3921 )
3922 .then(|| self.inner.retained.recovery_block())
3923 .flatten();
3924 let launch_nonce_source = observed_launch_nonce_source(&snapshot);
3925 let pid = snapshot.reported_pid();
3926 let mut health = snapshot.health;
3927 if let Some(detail) = recovery_block
3928 .as_ref()
3929 .or_else(|| retained.as_ref().map(|(_, detail)| detail))
3930 {
3931 health.detail = Some(match health.detail {
3932 Some(previous) => format!("{previous}; {detail}"),
3933 None => detail.clone(),
3934 });
3935 }
3936
3937 Ok((
3938 ModuleStatus {
3939 waiting_for_previous_pid: retained
3940 .as_ref()
3941 .map(|(pid, _)| *pid)
3942 .filter(|pid| *pid > 0)
3943 .or(snapshot.waiting_for_previous_pid),
3944 waiting_for_previous_detail: recovery_block
3945 .or_else(|| retained.map(|(_, detail)| detail))
3946 .or(snapshot.waiting_for_previous_detail),
3947 module_id: self.inner.module_id.clone(),
3948 launch_nonce_source,
3949 state: snapshot.state,
3950 enabled: snapshot.enabled,
3951 process_alive: snapshot.process_alive,
3952 registration_active,
3953 protocol,
3954 live,
3955 restart_count,
3956 lifetime_restarts: snapshot.lifetime_restarts,
3957 spawn_generation: snapshot.spawn_generation,
3958 max_restarts: self.inner.restart_policy.max_restarts,
3959 restart_window: self.inner.restart_policy.window,
3960 drain_timeout,
3961 restart_backoff: self.inner.restart_policy.backoff,
3962 restart_max_backoff: self.inner.restart_policy.max_backoff,
3963 pid,
3964 spawned_at_ms: snapshot.spawned_at_ms,
3965 spawned_from: snapshot.spawned_from,
3966 launch_spec_digest: snapshot.launch_spec_digest,
3967 process_start_time: snapshot.process_start_time,
3968 last_exit: snapshot.last_exit,
3969 health,
3970 },
3971 snapshot.spawned_file_identity.clone(),
3972 ))
3973 }
3974
3975 #[cfg(test)]
3976 pub(crate) fn hold_snapshot_for_test(
3977 &self,
3978 acquired: std::sync::mpsc::Sender<()>,
3979 hold: Duration,
3980 ) -> std::thread::JoinHandle<()> {
3981 let snapshot = Arc::clone(&self.inner.snapshot);
3982 std::thread::spawn(move || {
3983 let _guard = snapshot.lock().expect("test snapshot lock is not poisoned");
3984 acquired
3985 .send(())
3986 .expect("test receiver waits for snapshot lock");
3987 std::thread::sleep(hold);
3988 })
3989 }
3990
3991 pub(crate) async fn status_and_running_image_agreement(
3997 &self,
3998 ) -> Result<(ModuleStatus, subc_control::RunningImageAgreement), SuperviseError> {
3999 let (status, identity) = self.status_with_snapshot_lock(&self.inner.snapshot, None)?;
4000 let image = self
4001 .inner
4002 .provenance_probe
4003 .observe(
4004 status.pid,
4005 status.spawned_from.as_deref(),
4006 identity,
4007 status.process_start_time,
4008 )
4009 .await;
4010 Ok((status, image))
4011 }
4012
4013 pub(crate) async fn running_image_agreement(&self) -> subc_control::RunningImageAgreement {
4014 let snapshot = match lock_snapshot(&self.inner.snapshot) {
4015 Ok(snapshot) => snapshot.clone(),
4016 Err(_) => {
4017 return subc_control::RunningImageAgreement::Unavailable {
4018 reason: subc_control::RunningImageUnavailableReason::NotRunning,
4019 };
4020 }
4021 };
4022 self.inner
4023 .provenance_probe
4024 .observe(
4025 snapshot.reported_pid(),
4026 snapshot.spawned_from.as_deref(),
4027 snapshot.spawned_file_identity.clone(),
4028 snapshot.process_start_time,
4029 )
4030 .await
4031 }
4032
4033 pub(crate) fn child_resource_usage(&self) -> subc_control::ChildResourceUsage {
4036 #[cfg(windows)]
4037 {
4038 let image = lock_snapshot(&self.inner.snapshot)
4039 .ok()
4040 .and_then(|snapshot| {
4041 snapshot
4042 .spawned_file_identity
4043 .as_ref()
4044 .and_then(|identity| identity.windows_image.clone())
4045 });
4046 match image {
4047 Some(image) => {
4048 crate::child_resources::read_windows_handle(image.process(), image.start_time())
4049 }
4050 None => subc_control::ChildResourceUsage::Unavailable {
4051 reason: subc_control::ChildResourceUnavailableReason::NotRunning,
4052 },
4053 }
4054 }
4055 #[cfg(not(windows))]
4056 {
4057 let (pid, start_time) = match lock_snapshot(&self.inner.snapshot) {
4058 Ok(snapshot) => (snapshot.reported_pid(), snapshot.process_start_time),
4059 Err(_) => {
4060 return subc_control::ChildResourceUsage::Unavailable {
4061 reason: subc_control::ChildResourceUnavailableReason::Unreadable,
4062 }
4063 }
4064 };
4065 crate::child_resources::read(pid, start_time)
4066 }
4067 }
4068
4069 #[cfg(all(test, any(target_os = "linux", windows)))]
4070 pub(crate) fn subtree_target(&self) -> crate::subtree_resources::Target {
4071 self.inner
4072 .snapshot
4073 .try_lock()
4074 .map(|snapshot| snapshot.subtree_target.clone())
4075 .unwrap_or(crate::subtree_resources::Target::Unavailable("read_failed"))
4076 }
4077
4078 #[cfg(test)]
4079 pub(crate) fn set_subtree_target_for_test(&self, target: crate::subtree_resources::Target) {
4080 self.inner.snapshot.lock().unwrap().subtree_target = target;
4081 }
4082
4083 pub(crate) fn will_recover_after_connection_loss(&self) -> Result<bool, SuperviseError> {
4084 let mut snapshot = lock_snapshot(&self.inner.snapshot)?;
4085 Ok(match snapshot.state {
4086 ModuleState::Restarting => true,
4087 ModuleState::Failed | ModuleState::Disabled => false,
4088 _ if snapshot.enabled && snapshot.respawn_pending => true,
4089 _ => daemon_will_restart(&mut snapshot, &self.inner.restart_policy, Instant::now()),
4090 })
4091 }
4092
4093 #[cfg(test)]
4094 pub(crate) fn is_warming(&self) -> Result<bool, SuperviseError> {
4095 self.is_warming_with_snapshot_lock(None)
4096 }
4097
4098 pub(crate) fn is_warming_for_control(
4099 &self,
4100 caller: &'static str,
4101 ) -> Result<bool, SuperviseError> {
4102 self.is_warming_with_snapshot_lock(Some(caller))
4103 }
4104
4105 fn is_warming_with_snapshot_lock(
4106 &self,
4107 caller: Option<&'static str>,
4108 ) -> Result<bool, SuperviseError> {
4109 let snapshot = match caller {
4110 Some(caller) => {
4111 lock_snapshot_for_control(&self.inner.snapshot, &self.inner.module_id, caller)?
4112 }
4113 None => lock_snapshot(&self.inner.snapshot)?,
4114 }
4115 .clone();
4116 Ok(matches!(
4117 snapshot.state,
4118 ModuleState::Starting | ModuleState::Running | ModuleState::Restarting
4119 ))
4120 }
4121
4122 pub async fn drain(&self) -> Result<(), SuperviseError> {
4124 self.stop().await
4125 }
4126
4127 pub(crate) async fn retire_requested(
4128 &self,
4129 requester: Option<Principal>,
4130 ) -> Result<(), SuperviseError> {
4131 match self.state()? {
4132 ModuleState::Stopped => return Ok(()),
4133 ModuleState::Failed
4134 if lock_snapshot(&self.inner.snapshot)?
4135 .failed_retry
4136 .pending
4137 .is_none() =>
4138 {
4139 return Ok(())
4140 }
4141 ModuleState::Failed
4142 | ModuleState::Starting
4143 | ModuleState::Running
4144 | ModuleState::Unresponsive
4145 | ModuleState::Restarting
4146 | ModuleState::Draining
4147 | ModuleState::Disabled => {}
4148 }
4149
4150 let (reply_tx, reply_rx) = oneshot::channel();
4151 self.inner
4152 .commands
4153 .send(SupervisorCommand::Retire {
4154 requester,
4155 reply: reply_tx,
4156 })
4157 .await
4158 .map_err(|_| SuperviseError::CommandClosed {
4159 module_id: self.inner.module_id.clone(),
4160 })?;
4161 reply_rx.await.map_err(|_| SuperviseError::CommandClosed {
4162 module_id: self.inner.module_id.clone(),
4163 })?
4164 }
4165
4166 pub async fn stop(&self) -> Result<(), SuperviseError> {
4167 self.stop_requested(None).await
4168 }
4169
4170 pub(crate) async fn stop_requested(
4171 &self,
4172 requester: Option<Principal>,
4173 ) -> Result<(), SuperviseError> {
4174 match self.state()? {
4175 ModuleState::Stopped => return Ok(()),
4176 ModuleState::Failed
4177 if lock_snapshot(&self.inner.snapshot)?
4178 .failed_retry
4179 .pending
4180 .is_none() =>
4181 {
4182 return Ok(())
4183 }
4184 ModuleState::Failed
4185 | ModuleState::Starting
4186 | ModuleState::Running
4187 | ModuleState::Unresponsive
4188 | ModuleState::Restarting
4189 | ModuleState::Draining
4190 | ModuleState::Disabled => {}
4191 }
4192
4193 let (reply_tx, reply_rx) = oneshot::channel();
4194 self.inner
4195 .commands
4196 .send(SupervisorCommand::Drain {
4197 requester,
4198 reply: reply_tx,
4199 })
4200 .await
4201 .map_err(|_| SuperviseError::CommandClosed {
4202 module_id: self.inner.module_id.clone(),
4203 })?;
4204 reply_rx.await.map_err(|_| SuperviseError::CommandClosed {
4205 module_id: self.inner.module_id.clone(),
4206 })?
4207 }
4208
4209 pub async fn restart(&self, drain_timeout_ms: Option<u64>) -> Result<(), SuperviseError> {
4210 self.restart_requested(drain_timeout_ms, None).await
4211 }
4212
4213 pub(crate) async fn restart_requested(
4214 &self,
4215 drain_timeout_ms: Option<u64>,
4216 requester: Option<Principal>,
4217 ) -> Result<(), SuperviseError> {
4218 let queued_at = Instant::now();
4219 let received_at_generation = lock_snapshot(&self.inner.snapshot)?.spawn_generation;
4220 let (reply_tx, reply_rx) = oneshot::channel();
4221 self.inner
4222 .commands
4223 .send(SupervisorCommand::Restart {
4224 requester,
4225 drain_timeout_ms,
4226 received_at_generation,
4227 queued_at,
4228 reply: reply_tx,
4229 })
4230 .await
4231 .map_err(|_| SuperviseError::CommandClosed {
4232 module_id: self.inner.module_id.clone(),
4233 })?;
4234 reply_rx.await.map_err(|_| SuperviseError::CommandClosed {
4235 module_id: self.inner.module_id.clone(),
4236 })?
4237 }
4238
4239 pub async fn swap(&self, ready_timeout: Option<Duration>) -> Result<(), SuperviseError> {
4244 self.swap_requested(ready_timeout, None).await
4245 }
4246
4247 pub(crate) async fn swap_requested(
4248 &self,
4249 ready_timeout: Option<Duration>,
4250 requester: Option<Principal>,
4251 ) -> Result<(), SuperviseError> {
4252 let (reply_tx, reply_rx) = oneshot::channel();
4253 self.inner
4254 .commands
4255 .send(SupervisorCommand::Swap {
4256 requester,
4257 ready_timeout,
4258 reply: reply_tx,
4259 })
4260 .await
4261 .map_err(|_| SuperviseError::CommandClosed {
4262 module_id: self.inner.module_id.clone(),
4263 })?;
4264 reply_rx.await.map_err(|_| SuperviseError::CommandClosed {
4265 module_id: self.inner.module_id.clone(),
4266 })?
4267 }
4268
4269 pub async fn reload(&self) -> Result<(), SuperviseError> {
4270 self.reload_requested(None).await
4271 }
4272
4273 pub(crate) async fn reload_requested(
4274 &self,
4275 requester: Option<Principal>,
4276 ) -> Result<(), SuperviseError> {
4277 let (reply_tx, reply_rx) = oneshot::channel();
4278 self.inner
4279 .commands
4280 .send(SupervisorCommand::Reload {
4281 requester,
4282 reply: reply_tx,
4283 })
4284 .await
4285 .map_err(|_| SuperviseError::CommandClosed {
4286 module_id: self.inner.module_id.clone(),
4287 })?;
4288 reply_rx.await.map_err(|_| SuperviseError::CommandClosed {
4289 module_id: self.inner.module_id.clone(),
4290 })?
4291 }
4292
4293 pub async fn set_enabled(&self, enabled: bool) -> Result<bool, SuperviseError> {
4294 self.set_enabled_requested(enabled, None).await
4295 }
4296
4297 pub(crate) async fn set_enabled_requested(
4298 &self,
4299 enabled: bool,
4300 requester: Option<Principal>,
4301 ) -> Result<bool, SuperviseError> {
4302 let (reply_tx, reply_rx) = oneshot::channel();
4303 self.inner
4304 .commands
4305 .send(SupervisorCommand::SetEnabled {
4306 requester,
4307 enabled,
4308 reply: reply_tx,
4309 })
4310 .await
4311 .map_err(|_| SuperviseError::CommandClosed {
4312 module_id: self.inner.module_id.clone(),
4313 })?;
4314 reply_rx.await.map_err(|_| SuperviseError::CommandClosed {
4315 module_id: self.inner.module_id.clone(),
4316 })?
4317 }
4318
4319 pub(crate) fn declared_protocol(&self) -> Result<ModuleProtocol, SuperviseError> {
4323 let configured = self
4324 .inner
4325 .configuration
4326 .lock()
4327 .map_err(|_| SuperviseError::StatePoisoned {
4328 module_id: Some(self.inner.module_id.clone()),
4329 })?
4330 .spec
4331 .protocol;
4332 let state = lock_snapshot(&self.inner.snapshot)?;
4333 Ok(state.spawned_protocol.unwrap_or(configured))
4334 }
4335
4336 pub(crate) fn configuration(&self) -> Result<(ModuleSpec, HealthConfig), SuperviseError> {
4337 let configuration =
4338 self.inner
4339 .configuration
4340 .lock()
4341 .map_err(|_| SuperviseError::StatePoisoned {
4342 module_id: Some(self.inner.module_id.clone()),
4343 })?;
4344 Ok((configuration.spec.clone(), configuration.health.clone()))
4345 }
4346
4347 #[cfg(any(test, feature = "test-support"))]
4351 pub async fn update_spec_for_test(&self, spec: ModuleSpec) -> Result<(), SuperviseError> {
4352 let (_, health) = self.configuration()?;
4353 let drain_timeout_ms = u64::try_from(
4354 self.inner
4355 .effective_drain_timeout
4356 .lock()
4357 .unwrap_or_else(|poisoned| poisoned.into_inner())
4358 .as_millis(),
4359 )
4360 .ok();
4361 self.update_configuration(spec, health, drain_timeout_ms)
4362 .await
4363 }
4364
4365 pub(crate) async fn update_configuration(
4366 &self,
4367 spec: ModuleSpec,
4368 health: HealthConfig,
4369 drain_timeout_ms: Option<u64>,
4370 ) -> Result<(), SuperviseError> {
4371 if spec.module_id != self.inner.module_id {
4372 return Err(SuperviseError::InvalidSpec {
4373 reason: "a supervised module's module_id cannot be changed".to_string(),
4374 });
4375 }
4376 validate_spec(&spec)?;
4377 let (reply_tx, reply_rx) = oneshot::channel();
4378 self.inner
4379 .commands
4380 .send(SupervisorCommand::UpdateConfiguration {
4381 spec: spec.clone(),
4382 health: health.clone(),
4383 drain_timeout_ms,
4384 reply: reply_tx,
4385 })
4386 .await
4387 .map_err(|_| SuperviseError::CommandClosed {
4388 module_id: self.inner.module_id.clone(),
4389 })?;
4390 reply_rx.await.map_err(|_| SuperviseError::CommandClosed {
4391 module_id: self.inner.module_id.clone(),
4392 })?;
4393 let mut configuration =
4394 self.inner
4395 .configuration
4396 .lock()
4397 .map_err(|_| SuperviseError::StatePoisoned {
4398 module_id: Some(self.inner.module_id.clone()),
4399 })?;
4400 configuration.spec = spec;
4401 configuration.health = health;
4402 Ok(())
4403 }
4404}
4405
4406impl Drop for SupervisedModuleInner {
4407 fn drop(&mut self) {
4408 let Ok(mut monitor) = self.monitor.lock() else {
4409 return;
4410 };
4411 if let Some(monitor) = monitor.as_ref().filter(|monitor| !monitor.is_finished()) {
4412 let _ = update_snapshot(&self.snapshot, Some(&self.module_id), |state| {
4413 state.state = ModuleState::Stopped;
4414 clear_current_process_facts(state);
4415 });
4416 monitor.abort();
4417 }
4418 let _ = monitor.take();
4419 }
4420}
4421
4422#[derive(Debug)]
4423enum SupervisorCommand {
4424 Drain {
4425 requester: Option<Principal>,
4426 reply: oneshot::Sender<Result<(), SuperviseError>>,
4427 },
4428 Retire {
4429 requester: Option<Principal>,
4430 reply: oneshot::Sender<Result<(), SuperviseError>>,
4431 },
4432 Restart {
4433 requester: Option<Principal>,
4434 drain_timeout_ms: Option<u64>,
4439 received_at_generation: u64,
4443 queued_at: Instant,
4446 reply: oneshot::Sender<Result<(), SuperviseError>>,
4447 },
4448 Reload {
4449 requester: Option<Principal>,
4450 reply: oneshot::Sender<Result<(), SuperviseError>>,
4451 },
4452 SetEnabled {
4453 requester: Option<Principal>,
4454 enabled: bool,
4455 reply: oneshot::Sender<Result<bool, SuperviseError>>,
4456 },
4457 UpdateConfiguration {
4458 spec: ModuleSpec,
4459 health: HealthConfig,
4460 drain_timeout_ms: Option<u64>,
4463 reply: oneshot::Sender<()>,
4464 },
4465 Swap {
4466 requester: Option<Principal>,
4467 ready_timeout: Option<Duration>,
4470 reply: oneshot::Sender<Result<(), SuperviseError>>,
4472 },
4473}
4474
4475#[derive(Debug)]
4476pub enum SuperviseError {
4477 InvalidSpec {
4478 reason: String,
4479 },
4480 Spawn {
4481 program: PathBuf,
4482 source: io::Error,
4483 cgroup_path: Option<PathBuf>,
4484 },
4485 Cgroup {
4486 module_id: String,
4487 source: io::Error,
4488 },
4489 LaunchNonce {
4492 reason: String,
4493 },
4494 Wait {
4495 module_id: String,
4496 source: io::Error,
4497 },
4498 Kill {
4499 module_id: String,
4500 source: io::Error,
4501 },
4502 Forwarding(ForwardingError),
4503 Registry(RegistryError),
4504 ReloadUnavailable {
4505 module_id: String,
4506 reason: String,
4507 },
4508 Disabled {
4513 module_id: String,
4514 },
4515 ReloadFailed {
4516 module_id: String,
4517 reason: String,
4518 },
4519 RegistrationStillActive {
4520 module_id: String,
4521 waited: Duration,
4522 },
4523 StatePoisoned {
4524 module_id: Option<String>,
4525 },
4526 CommandClosed {
4527 module_id: String,
4528 },
4529 SwapInProgress {
4533 module_id: String,
4534 },
4535 SwapRefused {
4537 module_id: String,
4538 reason: SwapRefusal,
4539 },
4540 SwapFailed {
4544 module_id: String,
4545 arm: SwapFailureArm,
4546 detail: String,
4547 candidate_exit: Option<ExitReport>,
4550 },
4551}
4552
4553#[derive(Debug, Clone, Copy, PartialEq, Eq)]
4555pub enum SwapRefusal {
4556 OverlapExclusive,
4558 NotRegistered,
4561 ProtocolNone,
4564 NotConfigured,
4567 AlreadySwapping,
4569}
4570
4571impl SwapRefusal {
4572 pub fn as_str(self) -> &'static str {
4573 match self {
4574 Self::OverlapExclusive => "overlap_exclusive",
4575 Self::NotRegistered => "not_registered",
4576 Self::ProtocolNone => "protocol_none",
4577 Self::NotConfigured => "not_configured",
4578 Self::AlreadySwapping => "already_swapping",
4579 }
4580 }
4581}
4582
4583#[derive(Debug, Clone, Copy, PartialEq, Eq)]
4586pub enum SwapFailureArm {
4587 SpawnFailed,
4589 NeverRegistered,
4591 NeverReady,
4593 CandidateExited,
4595 CandidateUnhealthy,
4597 Interrupted,
4601 CutoverLost,
4606}
4607
4608impl SwapFailureArm {
4609 pub fn as_str(self) -> &'static str {
4610 match self {
4611 Self::SpawnFailed => "spawn_failed",
4612 Self::NeverRegistered => "never_registered",
4613 Self::NeverReady => "never_ready",
4614 Self::CandidateExited => "candidate_exited",
4615 Self::CandidateUnhealthy => "candidate_unhealthy",
4616 Self::Interrupted => "interrupted",
4617 Self::CutoverLost => "cutover_lost",
4618 }
4619 }
4620}
4621
4622impl fmt::Display for SuperviseError {
4623 fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
4624 match self {
4625 Self::InvalidSpec { reason } => write!(f, "invalid module spec: {reason}"),
4626 Self::Spawn {
4627 program,
4628 source,
4629 cgroup_path: Some(cgroup_path),
4630 } => write!(
4631 f,
4632 "failed to place module in cgroup '{}' while spawning '{}': {source}",
4633 cgroup_path.display(),
4634 program.display()
4635 ),
4636 Self::Spawn {
4637 program,
4638 source,
4639 cgroup_path: None,
4640 } => write!(
4641 f,
4642 "failed to spawn module '{}': {source}",
4643 program.display()
4644 ),
4645 Self::Cgroup { module_id, source } => {
4646 write!(
4647 f,
4648 "failed to prepare cgroup for module '{module_id}': {source}"
4649 )
4650 }
4651 Self::LaunchNonce { reason } => {
4652 write!(
4653 f,
4654 "failed to generate reserved-module launch nonce: {reason}"
4655 )
4656 }
4657 Self::Wait { module_id, source } => {
4658 write!(f, "failed to wait for module '{module_id}': {source}")
4659 }
4660 Self::Kill { module_id, source } => {
4661 write!(f, "failed to kill module '{module_id}': {source}")
4662 }
4663 Self::Forwarding(err) => write!(f, "forwarding error: {err}"),
4664 Self::Registry(err) => write!(f, "registry error: {err}"),
4665 Self::ReloadUnavailable { module_id, reason } => {
4666 write!(f, "reload unavailable for module '{module_id}': {reason}")
4667 }
4668 Self::Disabled { module_id } => {
4669 write!(
4670 f,
4671 "module '{module_id}' is disabled; enable it before restart or reload"
4672 )
4673 }
4674 Self::ReloadFailed { module_id, reason } => {
4675 write!(f, "reload failed for module '{module_id}': {reason}")
4676 }
4677 Self::RegistrationStillActive { module_id, waited } => write!(
4678 f,
4679 "module '{module_id}' registration remained active after waiting {waited:?}"
4680 ),
4681 Self::StatePoisoned { module_id } => match module_id {
4682 Some(module_id) => {
4683 write!(f, "supervisor state for module '{module_id}' was poisoned")
4684 }
4685 None => write!(f, "supervisor state was poisoned"),
4686 },
4687 Self::CommandClosed { module_id } => {
4688 write!(
4689 f,
4690 "supervisor command channel for module '{module_id}' is closed"
4691 )
4692 }
4693 Self::SwapInProgress { module_id } => write!(
4694 f,
4695 "module '{module_id}' is being swapped; retry once the swap has cut over or failed, or stop the module to abort the swap"
4696 ),
4697 Self::SwapRefused { module_id, reason } => match reason {
4698 SwapRefusal::OverlapExclusive => write!(
4699 f,
4700 "module '{module_id}' is declared overlap: \"exclusive\" (the default): two processes of it must not run at once, so it cannot be swapped; use a plain restart, or declare overlap: \"safe\" in its config if it really tolerates a second process"
4701 ),
4702 SwapRefusal::NotRegistered => write!(
4703 f,
4704 "module '{module_id}' is not registered, so there is no serving process to keep while a replacement warms; use a plain restart"
4705 ),
4706 SwapRefusal::ProtocolNone => write!(
4707 f,
4708 "module '{module_id}' is protocol: \"none\" and never registers, so a swap could never see its replacement become ready; use a plain restart"
4709 ),
4710 SwapRefusal::NotConfigured => write!(
4711 f,
4712 "module '{module_id}' cannot be swapped: the supervisor was built without the forwarding table or shared handle a swap needs"
4713 ),
4714 SwapRefusal::AlreadySwapping => {
4715 write!(f, "module '{module_id}' is already being swapped")
4716 }
4717 },
4718 Self::SwapFailed {
4719 module_id,
4720 arm,
4721 detail,
4722 ..
4723 } => write!(
4724 f,
4725 "swap of module '{module_id}' failed ({}): {detail}; the running process was left serving",
4726 arm.as_str()
4727 ),
4728 }
4729 }
4730}
4731
4732impl Error for SuperviseError {
4733 fn source(&self) -> Option<&(dyn Error + 'static)> {
4734 match self {
4735 Self::Spawn { source, .. }
4736 | Self::Cgroup { source, .. }
4737 | Self::Wait { source, .. }
4738 | Self::Kill { source, .. } => Some(source),
4739 Self::Forwarding(err) => Some(err),
4740 Self::Registry(err) => Some(err),
4741 Self::LaunchNonce { .. }
4742 | Self::InvalidSpec { .. }
4743 | Self::ReloadUnavailable { .. }
4744 | Self::Disabled { .. }
4745 | Self::ReloadFailed { .. }
4746 | Self::RegistrationStillActive { .. }
4747 | Self::StatePoisoned { .. }
4748 | Self::CommandClosed { .. }
4749 | Self::SwapInProgress { .. }
4750 | Self::SwapRefused { .. }
4751 | Self::SwapFailed { .. } => None,
4752 }
4753 }
4754}
4755
4756pub(crate) fn validate_spec(spec: &ModuleSpec) -> Result<(), SuperviseError> {
4757 if spec.carry_containment && spec.overlap == ModuleOverlap::Safe {
4758 return Err(SuperviseError::InvalidSpec {
4759 reason: "carry_containment cannot be combined with overlap: safe".to_owned(),
4760 });
4761 }
4762 if spec.module_id.trim().is_empty() {
4763 return Err(SuperviseError::InvalidSpec {
4764 reason: "module_id must not be empty".to_string(),
4765 });
4766 }
4767
4768 Ok(())
4769}
4770
4771#[derive(Debug, Default)]
4772struct HealthProbeRuntime {
4773 spawn_generation: Option<u64>,
4774 configured_health: Option<HealthConfig>,
4775 registered_connection: Option<crate::ConnectionId>,
4776 advertised: bool,
4777 next_probe_at: Option<Instant>,
4778 probe_index: u64,
4779}
4780
4781fn running_protocol(spec: &ModuleSpec, snapshot: &SharedSnapshot) -> ModuleProtocol {
4782 lock_snapshot(snapshot)
4783 .ok()
4784 .and_then(|state| state.spawned_protocol)
4785 .unwrap_or(spec.protocol)
4786}
4787
4788impl HealthProbeRuntime {
4789 fn refresh_registration(
4790 &mut self,
4791 spec: &ModuleSpec,
4792 runtime: &SupervisorRuntimeConfig,
4793 registry: &Registry,
4794 snapshot: &SharedSnapshot,
4795 ) {
4796 let generation = lock_snapshot(snapshot)
4797 .ok()
4798 .map(|state| state.spawn_generation);
4799 if self.spawn_generation != generation {
4800 *self = Self {
4804 spawn_generation: generation,
4805 ..Self::default()
4806 };
4807 }
4808 if self.configured_health.as_ref() != Some(&runtime.health) {
4809 self.configured_health = Some(runtime.health.clone());
4810 self.next_probe_at = None;
4811 self.registered_connection = None;
4812 self.probe_index = 0;
4813 }
4814 if running_protocol(spec, snapshot) == ModuleProtocol::None {
4818 self.registered_connection = None;
4819 self.advertised = runtime.health.http.is_some();
4820 if !self.advertised {
4821 self.next_probe_at = None;
4822 let _ = update_snapshot(snapshot, Some(&spec.module_id), |state| {
4823 let unknown = ModuleHealthStatus::default();
4824 if state.health != unknown {
4825 state.health = unknown;
4826 }
4827 });
4828 } else if self.next_probe_at.is_none() {
4829 self.next_probe_at = Some(
4830 Instant::now()
4831 + jittered_health_delay(&spec.module_id, 0, runtime.health.cadence),
4832 );
4833 }
4834 return;
4835 }
4836
4837 let registration = match registry.get_module(&spec.module_id) {
4838 Ok(registration) => registration,
4839 Err(err) => {
4840 warn!(module_id = %spec.module_id, error = %err, "health prober could not read registry");
4841 self.advertised = false;
4842 self.next_probe_at = None;
4843 return;
4844 }
4845 };
4846
4847 let Some(registration) = registration else {
4848 self.registered_connection = None;
4849 self.advertised = false;
4850 self.next_probe_at = None;
4851 return;
4852 };
4853
4854 let advertised = registration
4855 .control_ops
4856 .iter()
4857 .any(|op| op == MODULE_CONTROL_OP_HEALTH_CHECK);
4858 if !advertised {
4859 self.registered_connection = Some(registration.connection_id);
4860 self.advertised = false;
4861 self.next_probe_at = None;
4862 let _ = update_snapshot(snapshot, Some(&spec.module_id), |state| {
4863 if state.health.status != SupervisorHealthStatus::Unknown
4864 || state.health.consecutive_failures != 0
4865 || state.health.last_probe_ms.is_some()
4866 || state.health.detail.is_some()
4867 || state.health.metrics.is_some()
4868 {
4869 state.health.status = SupervisorHealthStatus::Unknown;
4870 state.health.consecutive_failures = 0;
4871 state.health.last_probe_ms = None;
4872 state.health.detail = None;
4873 state.health.metrics = None;
4874 }
4875 });
4876 return;
4877 }
4878
4879 let reregistered = self.registered_connection != Some(registration.connection_id);
4880 self.registered_connection = Some(registration.connection_id);
4881 self.advertised = true;
4882 if reregistered || self.next_probe_at.is_none() {
4883 self.probe_index = 0;
4884 self.next_probe_at = Some(
4885 Instant::now() + jittered_health_delay(&spec.module_id, 0, runtime.health.cadence),
4886 );
4887 let _ = update_snapshot(snapshot, Some(&spec.module_id), |state| {
4888 state.health.status = SupervisorHealthStatus::Unknown;
4889 state.health.consecutive_failures = 0;
4890 state.health.detail = None;
4891 state.health.metrics = None;
4892 });
4893 }
4894 }
4895
4896 fn wake_at(&self) -> Option<Instant> {
4897 if !self.advertised {
4898 return None;
4899 }
4900 self.next_probe_at
4901 }
4902
4903 fn due(&self) -> bool {
4904 self.advertised
4905 && self
4906 .next_probe_at
4907 .is_some_and(|next| Instant::now() >= next)
4908 }
4909
4910 fn schedule_next(&mut self, spec: &ModuleSpec, cadence: Duration) {
4911 self.probe_index = self.probe_index.wrapping_add(1);
4912 self.next_probe_at = Some(
4913 Instant::now() + jittered_health_delay(&spec.module_id, self.probe_index, cadence),
4914 );
4915 }
4916}
4917
4918#[derive(Debug)]
4953enum HealthProbeEvidence {
4954 LaneDead,
4956 NoAnswer,
4958 BadAnswer,
4960 Misconfigured,
4962}
4963
4964#[derive(Debug)]
4965struct HealthProbeError {
4966 evidence: HealthProbeEvidence,
4967 message: String,
4968}
4969
4970impl HealthProbeError {
4971 fn lane_dead(message: impl Into<String>) -> Self {
4972 Self::with(HealthProbeEvidence::LaneDead, message)
4973 }
4974
4975 fn no_answer(message: impl Into<String>) -> Self {
4976 Self::with(HealthProbeEvidence::NoAnswer, message)
4977 }
4978
4979 fn bad_answer(message: impl Into<String>) -> Self {
4980 Self::with(HealthProbeEvidence::BadAnswer, message)
4981 }
4982
4983 fn misconfigured(message: impl Into<String>) -> Self {
4984 Self::with(HealthProbeEvidence::Misconfigured, message)
4985 }
4986
4987 fn with(evidence: HealthProbeEvidence, message: impl Into<String>) -> Self {
4988 Self {
4989 evidence,
4990 message: message.into(),
4991 }
4992 }
4993
4994 #[allow(dead_code)]
5008 fn is_proof_of_death(&self) -> bool {
5009 matches!(self.evidence, HealthProbeEvidence::LaneDead)
5010 }
5011
5012 fn label(&self) -> &'static str {
5020 match self.evidence {
5021 HealthProbeEvidence::LaneDead => "lane-dead",
5022 HealthProbeEvidence::NoAnswer => "no-answer",
5023 HealthProbeEvidence::BadAnswer => "bad-answer",
5024 HealthProbeEvidence::Misconfigured => "daemon-misconfigured",
5025 }
5026 }
5027}
5028
5029impl fmt::Display for HealthProbeError {
5030 fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
5031 f.write_str(&self.message)
5032 }
5033}
5034
5035async fn run_health_probe_cycle(
5036 spec: &ModuleSpec,
5037 runtime: &SupervisorRuntimeConfig,
5038 registry: &Registry,
5039 process_liveness: &SupervisorProcessLiveness,
5040 snapshot: &SharedSnapshot,
5041 child: &mut Option<SupervisedChild>,
5042) {
5043 let now_ms = unix_ms_now();
5044 let http = (running_protocol(spec, snapshot) == ModuleProtocol::None)
5045 .then_some(runtime.health.http.as_deref())
5046 .flatten();
5047 let result = match http {
5048 Some(url) => probe_http_health(url, runtime.health.deadline).await,
5049 None => probe_module_health(&spec.module_id, runtime, None).await,
5050 };
5051 match result {
5052 Ok(report) => {
5053 handle_health_report(
5054 spec,
5055 runtime,
5056 registry,
5057 process_liveness,
5058 snapshot,
5059 child,
5060 report,
5061 now_ms,
5062 )
5063 .await;
5064 }
5065 Err(err) => {
5066 if http.is_some() {
5067 let _ = update_snapshot(snapshot, Some(&spec.module_id), |state| {
5068 state.health.status = SupervisorHealthStatus::Failing;
5069 });
5070 }
5071 handle_health_probe_failure(
5072 spec,
5073 runtime,
5074 registry,
5075 process_liveness,
5076 snapshot,
5077 child,
5078 err,
5079 now_ms,
5080 )
5081 .await;
5082 }
5083 }
5084}
5085
5086pub(crate) struct HttpProbeTarget<'a> {
5087 address: std::net::SocketAddr,
5088 localhost: bool,
5089 authority: &'a str,
5090 path: String,
5091}
5092
5093pub(crate) fn parse_http_probe_url(url: &str) -> Result<HttpProbeTarget<'_>, String> {
5096 if url.bytes().any(|b| b <= b' ' || b == 127) || url.contains('#') {
5097 return Err("must not contain whitespace, controls, or a fragment".into());
5098 }
5099 let rest = url
5100 .strip_prefix("http://")
5101 .ok_or("must use plain http://")?;
5102 let split = rest.find(['/', '?']).unwrap_or(rest.len());
5103 let (authority, suffix) = rest.split_at(split);
5104 let (host, port) = if let Some(rest) = authority.strip_prefix("[::1]") {
5105 ("::1", rest)
5106 } else {
5107 let split = authority.find(':').unwrap_or(authority.len());
5108 authority.split_at(split)
5109 };
5110 let ip: std::net::IpAddr = match host {
5111 "127.0.0.1" | "localhost" => std::net::Ipv4Addr::LOCALHOST.into(),
5112 "::1" => std::net::Ipv6Addr::LOCALHOST.into(),
5113 _ => return Err("host must be 127.0.0.1, [::1], or localhost".into()),
5114 };
5115 let port = if port.is_empty() {
5116 80
5117 } else {
5118 port.strip_prefix(':')
5119 .and_then(|p| p.parse::<u16>().ok())
5120 .filter(|p| *p > 0)
5121 .ok_or("must have a valid nonzero TCP port")?
5122 };
5123 let path = if suffix.is_empty() {
5124 "/".into()
5125 } else if suffix.starts_with('?') {
5126 format!("/{suffix}")
5127 } else {
5128 suffix.into()
5129 };
5130 Ok(HttpProbeTarget {
5131 address: std::net::SocketAddr::new(ip, port),
5132 localhost: host == "localhost",
5133 authority,
5134 path,
5135 })
5136}
5137
5138async fn probe_http_health(
5139 url: &str,
5140 deadline: Duration,
5141) -> Result<HealthReport, HealthProbeError> {
5142 use tokio::io::{AsyncReadExt, AsyncWriteExt, BufReader};
5143 let target = parse_http_probe_url(url).map_err(HealthProbeError::misconfigured)?;
5144 let mut response_status = String::new();
5147 let mut body = Vec::new();
5148 let probe = async {
5149 let connection = match tokio::net::TcpStream::connect(target.address).await {
5152 Err(_) if target.localhost => {
5153 tokio::net::TcpStream::connect((
5154 std::net::Ipv6Addr::LOCALHOST,
5155 target.address.port(),
5156 ))
5157 .await
5158 }
5159 result => result,
5160 };
5161 let mut stream = connection.map_err(|error| {
5162 HealthProbeError::no_answer(format!("HTTP connect failed: {error}"))
5163 })?;
5164 stream.write_all(format!("GET {} HTTP/1.1\r\nHost: {}\r\nAccept-Encoding: identity\r\nConnection: close\r\n\r\n", target.path, target.authority).as_bytes()).await
5165 .map_err(|error| HealthProbeError::no_answer(format!("HTTP write failed: {error}")))?;
5166 let mut reader = BufReader::new(stream);
5167 let mut budget = 16 * 1024;
5168 let status = http_line(&mut reader, &mut budget).await?;
5169 let mut words = status.split_ascii_whitespace();
5170 let version = words.next();
5171 let code = words
5172 .next()
5173 .filter(|word| word.len() == 3)
5174 .and_then(|word| word.parse::<u16>().ok());
5175 if !matches!(version, Some("HTTP/1.1" | "HTTP/1.0"))
5176 || !code.is_some_and(|code| (100..600).contains(&code))
5177 {
5178 return Err(HealthProbeError::bad_answer(format!(
5179 "invalid HTTP status: {status}"
5180 )));
5181 }
5182 let code = code.expect("validated status code");
5183 response_status = status.clone();
5184 let mut length = None;
5185 let mut chunked = false;
5186 loop {
5187 let line = http_line(&mut reader, &mut budget).await?;
5188 if line.is_empty() {
5189 break;
5190 }
5191 if let Some((name, value)) = line.split_once(':') {
5192 if name.eq_ignore_ascii_case("content-length") {
5193 length = Some(value.trim().parse::<u64>().map_err(|_| {
5194 HealthProbeError::bad_answer("invalid HTTP Content-Length")
5195 })?);
5196 } else if name.eq_ignore_ascii_case("transfer-encoding") {
5197 chunked = value.trim().eq_ignore_ascii_case("chunked");
5198 }
5199 }
5200 }
5201 if chunked {
5202 while body.len() < 200 {
5203 let line = http_line(&mut reader, &mut budget).await?;
5204 let size = u64::from_str_radix(line.split(';').next().unwrap_or("").trim(), 16)
5205 .map_err(|_| HealthProbeError::bad_answer("invalid HTTP chunk size"))?;
5206 if size == 0 {
5207 break;
5208 }
5209 let count = size.min((200 - body.len()) as u64) as usize;
5210 let start = body.len();
5211 (&mut reader)
5212 .take(count as u64)
5213 .read_to_end(&mut body)
5214 .await
5215 .map_err(|error| {
5216 HealthProbeError::no_answer(format!("HTTP body read failed: {error}"))
5217 })?;
5218 if body.len() - start != count {
5219 return Err(HealthProbeError::bad_answer("truncated HTTP chunk"));
5220 }
5221 if size > count as u64 || body.len() == 200 {
5222 break;
5223 }
5224 if !http_line(&mut reader, &mut budget).await?.is_empty() {
5225 return Err(HealthProbeError::bad_answer("invalid HTTP chunk delimiter"));
5226 }
5227 }
5228 } else {
5229 reader
5230 .take(length.unwrap_or(200).min(200))
5231 .read_to_end(&mut body)
5232 .await
5233 .map_err(|error| {
5234 HealthProbeError::no_answer(format!("HTTP body read failed: {error}"))
5235 })?;
5236 }
5237 if (200..300).contains(&code) {
5238 Ok(HealthReport::ok())
5239 } else {
5240 Err(HealthProbeError::bad_answer(
5241 "HTTP health endpoint returned non-2xx",
5242 ))
5243 }
5244 };
5245 let mut result = timeout(deadline, probe).await.unwrap_or_else(|_| {
5246 Err(HealthProbeError::no_answer(format!(
5247 "HTTP probe timed out after {deadline:?}"
5248 )))
5249 });
5250 if let Err(error) = &mut result {
5251 if !response_status.is_empty() {
5252 error.message = format!(
5253 "{}; {response_status}: {}",
5254 error.message,
5255 String::from_utf8_lossy(&body)
5256 );
5257 }
5258 }
5259 result
5260}
5261
5262async fn http_line(
5263 reader: &mut tokio::io::BufReader<tokio::net::TcpStream>,
5264 remaining: &mut usize,
5265) -> Result<String, HealthProbeError> {
5266 use tokio::io::{AsyncBufReadExt, AsyncReadExt};
5267 let mut line = Vec::new();
5268 (&mut *reader)
5269 .take(*remaining as u64)
5270 .read_until(b'\n', &mut line)
5271 .await
5272 .map_err(|error| {
5273 HealthProbeError::no_answer(format!("HTTP header read failed: {error}"))
5274 })?;
5275 *remaining -= line.len();
5276 if !line.ends_with(b"\r\n") {
5277 return Err(HealthProbeError::bad_answer(
5278 "HTTP headers are incomplete or exceed 16 KiB",
5279 ));
5280 }
5281 line.truncate(line.len() - 2);
5282 String::from_utf8(line).map_err(|_| HealthProbeError::bad_answer("HTTP header is not UTF-8"))
5283}
5284
5285async fn probe_module_health(
5286 module_id: &str,
5287 runtime: &SupervisorRuntimeConfig,
5288 drain_deadline: Option<Instant>,
5289) -> Result<HealthReport, HealthProbeError> {
5290 let Some(forwarding) = runtime.forwarding.as_ref() else {
5291 return Err(HealthProbeError::misconfigured(
5292 "supervisor was not configured with a forwarding table",
5293 ));
5294 };
5295 let probe_started_at = Instant::now();
5296 let mut deadline = probe_started_at + runtime.health.deadline;
5297 if let Some(drain_deadline) = drain_deadline {
5298 deadline = deadline.min(drain_deadline);
5299 }
5300 let pending = if drain_deadline.is_some() {
5301 forwarding.begin_drain_health_probe_rpc_for(
5302 module_id,
5303 MODULE_CONTROL_OP_HEALTH_CHECK,
5304 probe_started_at,
5305 deadline,
5306 )
5307 } else {
5308 forwarding.begin_health_probe_rpc_for(
5309 module_id,
5310 MODULE_CONTROL_OP_HEALTH_CHECK,
5311 probe_started_at,
5312 deadline,
5313 )
5314 }
5315 .map_err(|err| {
5316 HealthProbeError::lane_dead(format!("failed to begin health.check RPC: {err}"))
5319 })?;
5320 await_health_probe(forwarding, pending, deadline, runtime.health.deadline).await
5321}
5322
5323async fn probe_endpoint_health(
5330 endpoint: crate::ModuleEndpointId,
5331 runtime: &SupervisorRuntimeConfig,
5332 deadline_cap: Option<Instant>,
5333) -> Result<HealthReport, HealthProbeError> {
5334 let Some(forwarding) = runtime.forwarding.as_ref() else {
5335 return Err(HealthProbeError::misconfigured(
5336 "supervisor was not configured with a forwarding table",
5337 ));
5338 };
5339 let probe_started_at = Instant::now();
5340 let mut deadline = probe_started_at + runtime.health.deadline;
5341 if let Some(cap) = deadline_cap {
5342 deadline = deadline.min(cap);
5343 }
5344 let pending = forwarding
5345 .begin_endpoint_health_probe_rpc_for(
5346 endpoint,
5347 MODULE_CONTROL_OP_HEALTH_CHECK,
5348 probe_started_at,
5349 deadline,
5350 )
5351 .map_err(|err| {
5352 HealthProbeError::lane_dead(format!("failed to begin health.check RPC: {err}"))
5353 })?;
5354 await_health_probe(forwarding, pending, deadline, runtime.health.deadline).await
5355}
5356
5357async fn await_health_probe(
5359 forwarding: &ForwardingTable,
5360 pending: PendingModuleControlRpc,
5361 deadline: Instant,
5362 probe_budget: Duration,
5363) -> Result<HealthReport, HealthProbeError> {
5364 let PendingModuleControlRpc {
5365 endpoint,
5366 module_sink,
5367 negotiated_ver,
5368 corr,
5369 receiver,
5370 } = pending;
5371 let body = serde_json::to_vec(&ModuleControlRequest::HealthCheck {}).map_err(|err| {
5372 HealthProbeError::misconfigured(format!("failed to encode health.check: {err}"))
5373 })?;
5374 let frame = Frame::build_with_version(
5375 negotiated_ver,
5376 FrameType::Request,
5377 control_flags(),
5378 0,
5379 0,
5380 corr,
5381 body,
5382 )
5383 .map_err(|err| {
5384 HealthProbeError::misconfigured(format!("failed to build health.check frame: {err}"))
5385 })?;
5386
5387 match timeout_at(deadline, module_sink.send(frame)).await {
5393 Ok(Ok(())) => {}
5394 Ok(Err(err)) => {
5395 let _ = forwarding.cancel_module_control_rpc(endpoint, corr);
5396 return Err(HealthProbeError::lane_dead(format!(
5399 "failed to send health.check: {err}"
5400 )));
5401 }
5402 Err(_elapsed) => {
5403 let _ = forwarding.cancel_module_control_rpc(endpoint, corr);
5404 return Err(HealthProbeError::no_answer(
5408 "health.check send timed out before enqueue (module egress full)",
5409 ));
5410 }
5411 }
5412
5413 match timeout_at(deadline, receiver).await {
5414 Ok(Ok(ModuleControlRpcOutcome::Response(response))) => {
5418 response.health_report().ok_or_else(|| {
5419 HealthProbeError::bad_answer("health.check RPC returned a non-health response")
5420 })
5421 }
5422 Ok(Ok(ModuleControlRpcOutcome::Rejected(body))) => Err(HealthProbeError::bad_answer(
5423 format!("health.check rejected: {}", body.message),
5424 )),
5425 Ok(Ok(ModuleControlRpcOutcome::ModuleGone(message))) => {
5426 Err(HealthProbeError::lane_dead(message))
5427 }
5428 Ok(Ok(ModuleControlRpcOutcome::MalformedResponse(message))) => {
5429 Err(HealthProbeError::bad_answer(message))
5430 }
5431 Ok(Ok(ModuleControlRpcOutcome::UnexpectedOp { expected, actual })) => {
5432 Err(HealthProbeError::bad_answer(format!(
5433 "expected module-control op '{expected}', got '{actual}'"
5434 )))
5435 }
5436 Ok(Ok(ModuleControlRpcOutcome::DeadlineElapsed)) => Err(HealthProbeError::bad_answer(
5440 "module answered health.check after its daemon deadline",
5441 )),
5442 Ok(Err(_)) => Err(HealthProbeError::misconfigured(
5443 "health.check waiter was canceled before the module responded",
5444 )),
5445 Err(_) => {
5446 let _ = forwarding.tombstone_health_probe_rpc(endpoint, corr);
5447 Err(HealthProbeError::no_answer(format!(
5448 "module did not answer health.check within {probe_budget:?}"
5449 )))
5450 }
5451 }
5452}
5453
5454#[allow(clippy::too_many_arguments)]
5455async fn handle_health_report(
5456 spec: &ModuleSpec,
5457 runtime: &SupervisorRuntimeConfig,
5458 registry: &Registry,
5459 process_liveness: &SupervisorProcessLiveness,
5460 snapshot: &SharedSnapshot,
5461 child: &mut Option<SupervisedChild>,
5462 report: HealthReport,
5463 now_ms: u64,
5464) {
5465 let status = supervisor_health_status(report.status);
5466 let detail = report.detail.clone();
5467 let metrics = truncate_health_metrics(report.metrics);
5468 let _ = update_snapshot(snapshot, Some(&spec.module_id), |state| {
5469 state.health.status = status;
5470 state.health.last_probe_ms = Some(now_ms);
5471 state.health.detail = detail.clone();
5472 state.health.metrics = metrics.clone();
5473 state.health.consecutive_failures = 0;
5474 state.failed_retry.observe_health(
5475 report.status == HealthStatus::Ok,
5476 runtime.restart_policy.window,
5477 );
5478 });
5479
5480 let action = match report.status {
5481 HealthStatus::Ok => return,
5482 HealthStatus::Degraded => runtime.health.on_degraded,
5483 HealthStatus::Failing => runtime.health.on_failing,
5484 };
5485 apply_l3_health_action(
5486 spec,
5487 runtime,
5488 registry,
5489 process_liveness,
5490 snapshot,
5491 child,
5492 status,
5493 detail.as_deref(),
5494 action,
5495 now_ms,
5496 )
5497 .await;
5498}
5499
5500#[allow(clippy::too_many_arguments)]
5501async fn handle_health_probe_failure(
5502 spec: &ModuleSpec,
5503 runtime: &SupervisorRuntimeConfig,
5504 registry: &Registry,
5505 process_liveness: &SupervisorProcessLiveness,
5506 snapshot: &SharedSnapshot,
5507 child: &mut Option<SupervisedChild>,
5508 err: HealthProbeError,
5509 now_ms: u64,
5510) {
5511 let threshold = runtime.health.failure_threshold.max(1);
5512 let mut failures = 0;
5513 let detail = format!("[{}] {err}", err.label());
5518 let _ = update_snapshot(snapshot, Some(&spec.module_id), |state| {
5519 state
5520 .failed_retry
5521 .observe_health(false, runtime.restart_policy.window);
5522 if state.spawned_protocol.unwrap_or(spec.protocol) == ModuleProtocol::Subc {
5525 state.health.status = SupervisorHealthStatus::Unknown;
5526 }
5527 state.health.last_probe_ms = Some(now_ms);
5528 state.health.consecutive_failures = state.health.consecutive_failures.saturating_add(1);
5529 state.health.detail = Some(detail.clone());
5530 state.health.metrics = None;
5531 failures = state.health.consecutive_failures;
5532 });
5533
5534 if failures < threshold {
5535 warn!(
5536 module_id = %spec.module_id,
5537 consecutive_failures = failures,
5538 threshold,
5539 evidence = err.label(),
5540 detail = %detail,
5541 "health.check probe failed"
5542 );
5543 return;
5544 }
5545
5546 let _ = update_snapshot(snapshot, Some(&spec.module_id), |state| {
5547 state.state = ModuleState::Unresponsive;
5548 state.health.status = SupervisorHealthStatus::Unresponsive;
5549 });
5550 if runtime.health.critical {
5554 error!(
5555 module_id = %spec.module_id,
5556 status = "unresponsive",
5557 evidence = err.label(),
5558 detail = %detail,
5559 "critical module health alert"
5560 );
5561 } else {
5562 warn!(
5563 module_id = %spec.module_id,
5564 status = "unresponsive",
5565 evidence = err.label(),
5566 detail = %detail,
5567 "module health threshold breached"
5568 );
5569 }
5570 if let Err(err) = health_restart_child(
5571 spec,
5572 runtime,
5573 registry,
5574 process_liveness,
5575 snapshot,
5576 child,
5577 SupervisorHealthStatus::Unresponsive,
5578 Some(&detail),
5579 now_ms,
5580 )
5581 .await
5582 {
5583 error!(module_id = %spec.module_id, error = %err, "health-triggered restart failed");
5584 }
5585}
5586
5587#[allow(clippy::too_many_arguments)]
5588async fn apply_l3_health_action(
5589 spec: &ModuleSpec,
5590 runtime: &SupervisorRuntimeConfig,
5591 registry: &Registry,
5592 process_liveness: &SupervisorProcessLiveness,
5593 snapshot: &SharedSnapshot,
5594 child: &mut Option<SupervisedChild>,
5595 status: SupervisorHealthStatus,
5596 detail: Option<&str>,
5597 action: HealthAction,
5598 now_ms: u64,
5599) {
5600 record_health_action(snapshot, &spec.module_id, action.to_string(), now_ms);
5601 match action {
5602 HealthAction::Report => {
5603 info!(
5604 module_id = %spec.module_id,
5605 status = ?status,
5606 detail,
5607 "module reported non-ok health"
5608 );
5609 }
5610 HealthAction::Alert => {
5611 error!(
5612 module_id = %spec.module_id,
5613 status = ?status,
5614 detail,
5615 "module health alert"
5616 );
5617 }
5618 HealthAction::Restart => {
5619 if let Err(err) = health_restart_child(
5620 spec,
5621 runtime,
5622 registry,
5623 process_liveness,
5624 snapshot,
5625 child,
5626 status,
5627 detail,
5628 now_ms,
5629 )
5630 .await
5631 {
5632 error!(module_id = %spec.module_id, error = %err, "health-triggered restart failed");
5633 }
5634 }
5635 }
5636}
5637
5638#[allow(clippy::too_many_arguments)]
5639async fn health_restart_child(
5640 spec: &ModuleSpec,
5641 runtime: &SupervisorRuntimeConfig,
5642 registry: &Registry,
5643 process_liveness: &SupervisorProcessLiveness,
5644 snapshot: &SharedSnapshot,
5645 child: &mut Option<SupervisedChild>,
5646 status: SupervisorHealthStatus,
5647 detail: Option<&str>,
5648 now_ms: u64,
5649) -> Result<(), SuperviseError> {
5650 let (enabled, schedule) = {
5651 let mut state = lock_snapshot(snapshot)?;
5652 let enabled = state.enabled;
5653 let schedule = if enabled {
5654 state.next_crash_restart(&runtime.restart_policy, Instant::now())
5655 } else {
5656 None
5657 };
5658 (enabled, schedule)
5659 };
5660
5661 if !enabled {
5662 return Err(SuperviseError::Disabled {
5663 module_id: spec.module_id.clone(),
5664 });
5665 }
5666
5667 if schedule.is_none() {
5668 record_health_action(snapshot, &spec.module_id, "failed".to_string(), now_ms);
5669 error!(
5670 module_id = %spec.module_id,
5671 status = ?status,
5672 detail,
5673 max_restarts = runtime.restart_policy.max_restarts,
5674 window_secs = runtime.restart_policy.window.as_secs(),
5675 reason = %runtime.restart_policy.budget_exhausted_detail(),
5676 "health restart budget exhausted; marking module failed"
5677 );
5678 let stop_notice = begin_forwarding_drain_if_configured(
5679 spec,
5680 runtime,
5681 registry,
5682 snapshot,
5683 Some(true),
5684 RouteCloseReason::Disable,
5685 )
5686 .await?;
5687 update_snapshot(snapshot, Some(&spec.module_id), |state| {
5688 state.drain_disposition_detail = Some(runtime.restart_policy.budget_exhausted_detail());
5689 })?;
5690 let result = drain_optional_child(
5691 &spec.module_id,
5692 spec.protocol,
5693 stop_notice,
5694 registry,
5695 runtime.forwarding.as_deref(),
5696 snapshot,
5697 &runtime.terminal_ring,
5698 &runtime.spawn_events,
5699 child,
5700 runtime.drain_timeout,
5701 ModuleState::Failed,
5702 Some(true),
5703 &runtime.failure_context(true),
5704 )
5705 .await;
5706 update_snapshot(snapshot, Some(&spec.module_id), |state| {
5707 if state.state == ModuleState::Failed && !runtime.child_roster.is_closed() {
5708 state.failed_retry.schedule(&spec.module_id);
5709 }
5710 })?;
5711 result?;
5712 process_liveness.untrack_if_current(&spec.module_id, snapshot);
5713 return Ok(());
5714 }
5715
5716 let schedule = schedule.expect("a health restart must have a crash-restart schedule");
5717 let mut restart_count = 0;
5718 update_snapshot(snapshot, Some(&spec.module_id), |state| {
5719 restart_count = state.crash_restarts.len();
5720 state.state = ModuleState::Unresponsive;
5721 state.health.status = status;
5722 state.health.last_action = Some(HealthAction::Restart.to_string());
5723 state.health.last_action_ms = Some(now_ms);
5724 })?;
5725 warn!(
5726 module_id = %spec.module_id,
5727 status = ?status,
5728 detail,
5729 restart_count,
5730 restart_in_window = schedule.restart_in_window,
5731 delay_ms = schedule.delay.as_millis() as u64,
5732 "health-triggered module restart"
5733 );
5734
5735 let stop_notice = begin_forwarding_drain_if_configured(
5736 spec,
5737 runtime,
5738 registry,
5739 snapshot,
5740 Some(true),
5741 RouteCloseReason::Restart,
5742 )
5743 .await?;
5744 drain_optional_child(
5745 &spec.module_id,
5746 spec.protocol,
5747 stop_notice,
5748 registry,
5749 runtime.forwarding.as_deref(),
5750 snapshot,
5751 &runtime.terminal_ring,
5752 &runtime.spawn_events,
5753 child,
5754 runtime.drain_timeout,
5755 ModuleState::Restarting,
5756 Some(true),
5757 &runtime.failure_context(true),
5758 )
5759 .await?;
5760 schedule_respawn(
5761 runtime,
5762 snapshot,
5763 &spec.module_id,
5764 schedule.delay,
5765 RespawnKind::Spawn,
5766 true,
5767 )
5768}
5769
5770fn cancel_deferred_reload(runtime: &SupervisorRuntimeConfig, module_id: &str, reason: &str) {
5771 if let Some(reply) = runtime
5772 .deferred_reload_reply
5773 .lock()
5774 .unwrap_or_else(|p| p.into_inner())
5775 .take()
5776 {
5777 let _ = reply.send(Err(SuperviseError::ReloadFailed {
5778 module_id: module_id.to_string(),
5779 reason: reason.to_string(),
5780 }));
5781 }
5782}
5783
5784fn schedule_respawn(
5785 runtime: &SupervisorRuntimeConfig,
5786 snapshot: &SharedSnapshot,
5787 module_id: &str,
5788 delay: Duration,
5789 kind: RespawnKind,
5790 daemon_initiated: bool,
5791) -> Result<(), SuperviseError> {
5792 cancel_deferred_reload(runtime, module_id, "respawn superseded by another restart");
5793 update_snapshot(snapshot, Some(module_id), |state| {
5794 state.respawn_pending = true
5795 })?;
5796 *runtime
5797 .scheduled_respawn
5798 .lock()
5799 .unwrap_or_else(|p| p.into_inner()) = Some(PendingRespawn {
5800 deadline: Instant::now() + delay,
5801 kind,
5802 daemon_initiated,
5803 });
5804 Ok(())
5805}
5806
5807fn record_health_action(snapshot: &SharedSnapshot, module_id: &str, action: String, now_ms: u64) {
5808 let _ = update_snapshot(snapshot, Some(module_id), |state| {
5809 state.health.last_action = Some(action);
5810 state.health.last_action_ms = Some(now_ms);
5811 });
5812}
5813
5814fn supervisor_health_status(status: HealthStatus) -> SupervisorHealthStatus {
5815 match status {
5816 HealthStatus::Ok => SupervisorHealthStatus::Ok,
5817 HealthStatus::Degraded => SupervisorHealthStatus::Degraded,
5818 HealthStatus::Failing => SupervisorHealthStatus::Failing,
5819 }
5820}
5821
5822fn truncate_health_metrics(metrics: Option<Value>) -> Option<Value> {
5834 let metrics = metrics?;
5835 match serde_json::to_vec(&metrics) {
5836 Ok(encoded) if encoded.len() > MAX_HEALTH_METRICS_BYTES => Some(serde_json::json!({
5837 "truncated": true,
5838 "original_bytes": encoded.len(),
5839 })),
5840 Ok(_) | Err(_) => Some(metrics),
5841 }
5842}
5843
5844fn jittered_health_delay(module_id: &str, probe_index: u64, cadence: Duration) -> Duration {
5850 if cadence.is_zero() {
5851 return Duration::ZERO;
5852 }
5853 let cadence_ms = cadence.as_millis() as u64;
5854 if cadence_ms == 0 {
5870 return cadence;
5871 }
5872 let jitter_span = (cadence_ms / 10).max(1);
5887 let hash = module_id.as_bytes().iter().fold(
5888 probe_index.wrapping_mul(0x9E37_79B9_7F4A_7C15),
5889 |acc, byte| {
5890 acc.wrapping_mul(1099511628211)
5891 .wrapping_add(u64::from(*byte))
5892 },
5893 );
5894 cadence + Duration::from_millis(hash % jitter_span)
5895}
5896
5897#[cfg(test)]
5898mod tests {
5899 use super::*;
5900
5901 #[test]
5902 fn readding_a_module_clears_its_rescan_removal_tombstone() {
5903 let handle = SupervisorHandle::new();
5904 let module_id = "readded-tombstone";
5905 handle.record_rescan_removal(module_id);
5906 assert!(handle.removal_tombstone_age_ms(module_id).is_some());
5907
5908 handle.apply_identity_configuration(&ModuleSpec {
5909 module_id: module_id.to_string(),
5910 program: PathBuf::from("/test/module"),
5911 args: Vec::new(),
5912 env: Vec::new(),
5913 reserved: false,
5914 reserved_prefixes: Vec::new(),
5915 protocol: ModuleProtocol::Subc,
5916 overlap: Default::default(),
5917 carry_containment: false,
5918 });
5919
5920 assert!(
5921 handle.removal_tombstone_age_ms(module_id).is_none(),
5922 "a re-added module must not retain a stale removal tombstone"
5923 );
5924 }
5925
5926 #[derive(Debug, PartialEq, Eq)]
5929 struct OwnerInSpawnWindow {
5930 module_id: String,
5931 configured: bool,
5932 on_roster: bool,
5933 admission_refusal: Option<&'static str>,
5934 }
5935
5936 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
5947 async fn a_new_module_is_configured_before_its_first_process_can_register() {
5948 use crate::scopes::ScopeTable;
5949 use subc_protocol::{error_codes, scope::ScopeSelector, Principal};
5950
5951 let dir = cortexkit_test_support::ScratchDir::new("configured-before-spawn");
5952 let stub = |module_id: &str, program: PathBuf| ModuleSpec {
5953 module_id: module_id.to_string(),
5954 program,
5955 args: Vec::new(),
5956 env: ["XDG_DATA_HOME", "XDG_RUNTIME_DIR", "XDG_CONFIG_HOME"]
5957 .into_iter()
5958 .map(|key| (key.to_string(), dir.path().display().to_string()))
5959 .chain([("FAKE_AFT_NEVER_CONNECT".to_string(), "1".to_string())])
5960 .collect(),
5961 reserved: false,
5962 reserved_prefixes: Vec::new(),
5963 protocol: ModuleProtocol::Subc,
5964 overlap: Default::default(),
5965 carry_containment: false,
5966 };
5967 let live = super::terminal_history_tests::fake_aft_stub_path();
5968 let missing = dir.path().join("definitely-missing-module");
5969
5970 let handle = SupervisorHandle::new();
5971 let mut supervisor =
5972 Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default())
5973 .with_handle(handle.clone());
5974 let observed = Arc::new(Mutex::new(Vec::<OwnerInSpawnWindow>::new()));
5975 let hook_handle = handle.clone();
5976 let hook_observed = Arc::clone(&observed);
5977 supervisor.test_after_first_spawn = AfterFirstSpawnHook(Some(Arc::new(move |module_id| {
5978 let configured = hook_handle.is_configured(module_id);
5981 let selector = ScopeSelector::new(
5982 Principal::Reserved {
5983 module_id: module_id.to_string(),
5984 },
5985 "s".to_string(),
5986 Some(1),
5987 );
5988 let carrier = Principal::Reserved {
5989 module_id: "carrier".to_string(),
5990 };
5991 let admission_refusal = match ScopeTable::new(Vec::<String>::new())
5992 .admit(&carrier, module_id, &selector, configured)
5993 {
5994 Ok(_) => None,
5995 Err(refusal) => Some(refusal.code),
5996 };
5997 hook_observed.lock().unwrap().push(OwnerInSpawnWindow {
5998 module_id: module_id.to_string(),
5999 configured,
6000 on_roster: hook_handle.get(module_id).is_some(),
6001 admission_refusal,
6002 });
6003 })));
6004
6005 let plain = supervisor.spawn(stub("plain", live.clone())).unwrap();
6006 let configured = supervisor
6007 .supervise_configured(stub("configured", live.clone()), true)
6008 .unwrap();
6009 let with_health = supervisor
6010 .supervise_configured_with_health(
6011 stub("with-health", live.clone()),
6012 true,
6013 HealthConfig::default(),
6014 None,
6015 RestartPolicy::default(),
6016 )
6017 .unwrap();
6018 let failed = supervisor
6021 .supervise_configured_with_health(
6022 stub("failed-spawn", missing.clone()),
6023 true,
6024 HealthConfig::default(),
6025 None,
6026 RestartPolicy::default(),
6027 )
6028 .unwrap();
6029 assert!(supervisor.spawn(stub("spawn-error", missing)).is_err());
6032
6033 let expected = [
6034 "plain",
6035 "configured",
6036 "with-health",
6037 "failed-spawn",
6038 "spawn-error",
6039 ]
6040 .into_iter()
6041 .map(|module_id| OwnerInSpawnWindow {
6042 module_id: module_id.to_string(),
6043 configured: true,
6044 on_roster: !matches!(module_id, "plain" | "spawn-error"),
6045 admission_refusal: Some(error_codes::SCOPE_NOT_SYNCED),
6046 })
6047 .collect::<Vec<_>>();
6048 assert_eq!(*observed.lock().unwrap(), expected);
6049
6050 for module_id in ["plain", "configured", "with-health", "failed-spawn"] {
6051 assert!(
6052 handle.get(module_id).is_some(),
6053 "{module_id} is on the roster"
6054 );
6055 assert!(
6056 handle.is_configured(module_id),
6057 "{module_id} stays configured"
6058 );
6059 }
6060 assert!(handle.get("spawn-error").is_none());
6061 assert!(
6062 !handle.is_configured("spawn-error"),
6063 "a plain spawn that failed must not leave its module marked configured"
6064 );
6065
6066 handle.retire("failed-spawn");
6068 assert!(!handle.is_configured("failed-spawn"));
6069
6070 for module in [plain, configured, with_health] {
6071 module.stop().await.unwrap();
6072 }
6073 drop(failed);
6074 }
6075
6076 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
6083 async fn a_configured_module_reads_starting_while_its_first_process_already_runs() {
6084 let dir = cortexkit_test_support::ScratchDir::new("starting-while-spawned");
6085 let spec = ModuleSpec {
6086 module_id: "spawned-not-admitted".to_string(),
6087 program: super::terminal_history_tests::fake_aft_stub_path(),
6088 args: Vec::new(),
6089 env: ["XDG_DATA_HOME", "XDG_RUNTIME_DIR", "XDG_CONFIG_HOME"]
6090 .into_iter()
6091 .map(|key| (key.to_string(), dir.path().display().to_string()))
6092 .chain([("FAKE_AFT_NEVER_CONNECT".to_string(), "1".to_string())])
6093 .collect(),
6094 reserved: false,
6095 reserved_prefixes: Vec::new(),
6096 protocol: ModuleProtocol::Subc,
6097 overlap: Default::default(),
6098 carry_containment: false,
6099 };
6100 let handle = SupervisorHandle::new();
6101 let mut supervisor =
6102 Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default())
6103 .with_handle(handle.clone());
6104 let observed = Arc::new(Mutex::new(None));
6105 let hook_handle = handle.clone();
6106 let hook_observed = Arc::clone(&observed);
6107 supervisor.test_after_first_spawn = AfterFirstSpawnHook(Some(Arc::new(move |module_id| {
6108 let state = hook_handle
6109 .get(module_id)
6110 .expect("a configured module is on the roster before it is spawned")
6111 .state()
6112 .unwrap();
6113 *hook_observed.lock().unwrap() = Some(state);
6114 })));
6115
6116 let module = supervisor.supervise_configured(spec, true).unwrap();
6117 assert_eq!(
6118 *observed.lock().unwrap(),
6119 Some(ModuleState::Starting),
6120 "between the spawn and its admission the supervisor still reports starting"
6121 );
6122 assert_eq!(module.state().unwrap(), ModuleState::Running);
6123 module.stop().await.unwrap();
6124 }
6125
6126 fn stale_process_snapshot(state: ModuleState, enabled: bool) -> SharedSnapshot {
6127 let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::new(state, enabled)));
6128 update_snapshot(&snapshot, Some("stale-process-facts"), |snapshot| {
6129 snapshot.process_alive = true;
6130 snapshot.pid = Some(41);
6131 snapshot.spawned_at_ms = Some(42);
6132 snapshot.spawned_from = Some(PathBuf::from("/spawned/module"));
6133 snapshot.launch_spec_digest = Some(LaunchSpecDigest([0xA5; 32]));
6134 snapshot.spawned_file_identity = Some(SpawnedFileIdentity {
6135 device: 43,
6136 inode: 44,
6137 #[cfg(windows)]
6138 windows_image: None,
6139 });
6140 })
6141 .unwrap();
6142 snapshot
6143 }
6144
6145 fn assert_snapshot_process_facts_cleared(snapshot: &SharedSnapshot) {
6146 let snapshot = lock_snapshot(snapshot).unwrap();
6147 assert!(!snapshot.process_alive);
6148 assert_eq!(snapshot.pid, None);
6149 assert_eq!(snapshot.spawned_at_ms, None);
6150 assert_eq!(snapshot.spawned_from, None);
6151 assert_eq!(snapshot.launch_spec_digest, None);
6152 assert_eq!(snapshot.spawned_file_identity, None);
6153 }
6154
6155 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
6156 async fn failed_enable_spawn_clears_preexisting_current_process_facts() {
6157 let supervisor =
6158 Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default());
6159 let mut runtime = supervisor.runtime_config();
6160 runtime.test_seed_stale_facts_before_enable_spawn = true;
6161 let snapshot = stale_process_snapshot(ModuleState::Disabled, false);
6162 let mut child = None;
6163 let spec = ModuleSpec {
6164 module_id: "failed-enable-clears-facts".to_string(),
6165 program: PathBuf::from("/definitely/missing/failed-enable-module"),
6166 args: Vec::new(),
6167 env: Vec::new(),
6168 reserved: false,
6169 reserved_prefixes: Vec::new(),
6170 protocol: ModuleProtocol::Subc,
6171 overlap: Default::default(),
6172 carry_containment: false,
6173 };
6174
6175 let result = set_child_enabled(
6176 &spec,
6177 &runtime,
6178 &supervisor.registry,
6179 &supervisor.process_liveness,
6180 &snapshot,
6181 &mut child,
6182 true,
6183 &runtime.failure_context(false),
6184 )
6185 .await;
6186
6187 assert!(matches!(result, Err(SuperviseError::Spawn { .. })));
6188 assert_eq!(lock_snapshot(&snapshot).unwrap().state, ModuleState::Failed);
6189 assert_snapshot_process_facts_cleared(&snapshot);
6190 }
6191
6192 #[tokio::test]
6193 async fn start_revives_stranded_restarting_but_not_pending_backoff() {
6194 let supervisor =
6195 Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default());
6196 let runtime = supervisor.runtime_config();
6197 let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::new(
6198 ModuleState::Restarting,
6199 true,
6200 )));
6201 let spec = ModuleSpec {
6202 module_id: "start-stranded-restarting".to_string(),
6203 program: super::terminal_history_tests::fake_aft_stub_path(),
6204 args: Vec::new(),
6205 env: vec![("FAKE_AFT_NEVER_CONNECT".to_string(), "1".to_string())],
6206 reserved: false,
6207 reserved_prefixes: Vec::new(),
6208 protocol: ModuleProtocol::None,
6209 overlap: Default::default(),
6210 carry_containment: false,
6211 };
6212 let mut child = None;
6213 lock_snapshot(&snapshot).unwrap().respawn_pending = true;
6214 assert!(!super::set_child_enabled(
6215 &spec,
6216 &runtime,
6217 &Registry::default(),
6218 &supervisor.process_liveness,
6219 &snapshot,
6220 &mut child,
6221 true,
6222 &runtime.failure_context(false),
6223 )
6224 .await
6225 .unwrap());
6226 assert!(child.is_none());
6227 lock_snapshot(&snapshot).unwrap().respawn_pending = false;
6228 assert!(super::set_child_enabled(
6229 &spec,
6230 &runtime,
6231 &Registry::default(),
6232 &supervisor.process_liveness,
6233 &snapshot,
6234 &mut child,
6235 true,
6236 &runtime.failure_context(false),
6237 )
6238 .await
6239 .unwrap());
6240 assert_eq!(
6241 lock_snapshot(&snapshot).unwrap().state,
6242 ModuleState::Running
6243 );
6244 let mut child = child.unwrap();
6245 child.start_kill().unwrap();
6246 child.wait().await.unwrap();
6247 }
6248
6249 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
6250 async fn failed_reload_spawn_clears_current_process_facts() {
6251 let supervisor =
6252 Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default());
6253 let mut runtime = supervisor.runtime_config();
6254 runtime.restart_policy = RestartPolicy::new(0, Duration::ZERO);
6255 let snapshot = stale_process_snapshot(ModuleState::Running, true);
6256 let mut child = None;
6257 let spec = ModuleSpec {
6258 module_id: "failed-reload-clears-facts".to_string(),
6259 program: PathBuf::from("/unused/failed-reload-module"),
6260 args: Vec::new(),
6261 env: Vec::new(),
6262 reserved: false,
6263 reserved_prefixes: Vec::new(),
6264 protocol: ModuleProtocol::Subc,
6265 overlap: Default::default(),
6266 carry_containment: false,
6267 };
6268
6269 let result = handle_reload_spawn_failure(
6270 &spec,
6271 &runtime,
6272 &supervisor.process_liveness,
6273 &snapshot,
6274 &mut child,
6275 "forced reload spawn failure".to_string(),
6276 )
6277 .await;
6278
6279 assert!(matches!(result, Err(SuperviseError::ReloadFailed { .. })));
6280 assert_eq!(lock_snapshot(&snapshot).unwrap().state, ModuleState::Failed);
6281 assert_snapshot_process_facts_cleared(&snapshot);
6282 }
6283
6284 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
6285 async fn dropping_a_module_with_an_active_monitor_clears_current_process_facts() {
6286 let supervisor =
6287 Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default());
6288 let snapshot = stale_process_snapshot(ModuleState::Running, true);
6289 let module = supervisor.supervised_module(
6290 ModuleSpec {
6291 module_id: "drop-clears-facts".to_string(),
6292 program: PathBuf::from("/unused/drop-module"),
6293 args: Vec::new(),
6294 env: Vec::new(),
6295 reserved: false,
6296 reserved_prefixes: Vec::new(),
6297 protocol: ModuleProtocol::Subc,
6298 overlap: Default::default(),
6299 carry_containment: false,
6300 },
6301 supervisor.runtime_config(),
6302 Arc::clone(&snapshot),
6303 None,
6304 );
6305 assert!(!module
6306 .inner
6307 .monitor
6308 .lock()
6309 .unwrap()
6310 .as_ref()
6311 .unwrap()
6312 .is_finished());
6313
6314 drop(module);
6315
6316 assert_eq!(
6317 lock_snapshot(&snapshot).unwrap().state,
6318 ModuleState::Stopped
6319 );
6320 assert_snapshot_process_facts_cleared(&snapshot);
6321 }
6322
6323 #[cfg(unix)]
6324 #[tokio::test]
6325 async fn rescan_preserves_running_protocol_until_respawn() {
6326 let dir = cortexkit_test_support::ScratchDir::new("rescan-protocol");
6327 let initial = ModuleSpec {
6328 module_id: "rescan-protocol".into(),
6329 program: PathBuf::from("/bin/sleep"),
6330 args: vec!["60".into()],
6331 env: ["XDG_DATA_HOME", "XDG_RUNTIME_DIR", "XDG_CONFIG_HOME"]
6332 .into_iter()
6333 .map(|key| (key.into(), dir.path().to_string_lossy().into_owned()))
6334 .collect(),
6335 reserved: false,
6336 reserved_prefixes: vec![],
6337 protocol: ModuleProtocol::None,
6338 overlap: Default::default(),
6339 carry_containment: false,
6340 };
6341 let supervisor =
6342 Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default());
6343 let module = supervisor.spawn(initial.clone()).unwrap();
6344 assert!(module.status().unwrap().live);
6345 let mut next = initial;
6346 next.protocol = ModuleProtocol::Subc;
6347 module
6348 .update_configuration(next.clone(), HealthConfig::default(), None)
6349 .await
6350 .unwrap();
6351 assert!(
6352 module.status().unwrap().live,
6353 "rescan must not require HELLO from the old non-wire process"
6354 );
6355 let runtime = supervisor.runtime_config();
6356 let action = on_child_exit(
6357 &next,
6358 RestartPolicy::default(),
6359 &supervisor.registry,
6360 &module.inner.snapshot,
6361 &runtime.terminal_ring,
6362 &runtime.spawn_events,
6363 &runtime.child_roster,
6364 ExitReport {
6365 kind: ExitKind::Clean,
6366 code: Some(0),
6367 signal: None,
6368 at_ms: unix_ms_now(),
6369 },
6370 true,
6371 )
6372 .await;
6373 assert!(
6374 matches!(action, NextAction::Restart { .. }),
6375 "the old non-wire process's clean exit must restart"
6376 );
6377 module.drain().await.unwrap();
6378 }
6379
6380 #[cfg(unix)]
6381 #[tokio::test]
6382 async fn orphan_identity_matches_path_names_and_shebang_interpreters() {
6383 use std::os::unix::fs::PermissionsExt;
6384 let dir = cortexkit_test_support::ScratchDir::new("spawn-image-identity");
6385 let script = dir.join("module.sh");
6386 std::fs::write(&script, "#!/bin/sh\nwhile :; do sleep 0.1; done\n").unwrap();
6387 std::fs::set_permissions(&script, std::fs::Permissions::from_mode(0o700)).unwrap();
6388 let record_path = dir.join("live-children.json");
6389 let supervisor =
6390 Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default())
6391 .with_live_children_record(&record_path);
6392 for (program, args) in [
6393 (PathBuf::from("sleep"), vec!["60".into()]),
6394 (script, vec![]),
6395 ] {
6396 let spec = ModuleSpec {
6397 module_id: "image-identity".into(),
6398 program: program.clone(),
6399 args,
6400 env: ["XDG_DATA_HOME", "XDG_RUNTIME_DIR", "XDG_CONFIG_HOME"]
6401 .into_iter()
6402 .map(|key| (key.into(), dir.to_string_lossy().into_owned()))
6403 .collect(),
6404 reserved: false,
6405 reserved_prefixes: vec![],
6406 protocol: ModuleProtocol::None,
6407 overlap: Default::default(),
6408 carry_containment: false,
6409 };
6410 let module = supervisor.spawn(spec).unwrap();
6411 #[cfg(target_os = "macos")]
6412 {
6413 let deadline = Instant::now() + Duration::from_secs(5);
6416 while crate::live_children::read_record(&record_path)
6417 .unwrap()
6418 .iter()
6419 .all(|entry| entry.executable.is_none())
6420 {
6421 assert!(Instant::now() < deadline, "module image was not confirmed");
6422 tokio::time::sleep(Duration::from_millis(5)).await;
6423 }
6424 }
6425 let entry = crate::live_children::read_record(&record_path)
6426 .unwrap()
6427 .pop()
6428 .unwrap();
6429 let observed = subc_os::Process::open(entry.pid)
6430 .unwrap()
6431 .unwrap()
6432 .observe()
6433 .unwrap();
6434 let verdict = crate::live_children::identity_verdict(&entry, entry.pid, &observed);
6435 module.drain().await.unwrap();
6436 assert_eq!(
6437 verdict,
6438 crate::live_children::IdentityVerdict::Matches,
6439 "program {program:?}: recorded {entry:?}, observed {observed:?}"
6440 );
6441 }
6442 }
6443
6444 #[cfg(unix)]
6445 fn http_fixture(
6446 dir: &std::path::Path,
6447 url: &str,
6448 threshold: u32,
6449 ) -> crate::daemon_config::ConfiguredModule {
6450 let path = dir.join("subc.jsonc");
6451 std::fs::write(&path, serde_json::json!({"version": 1, "modules": {"http-process": {
6452 "program": "/bin/sleep", "args": ["60"], "protocol": "none",
6453 "env": {"XDG_DATA_HOME":dir,"XDG_RUNTIME_DIR":dir,"XDG_CONFIG_HOME":dir},
6454 "health": {"http":url,"cadence_ms":20,"deadline_ms":200,"failure_threshold":threshold},
6455 "restart": {"backoff_ms":1,"max_backoff_ms":1}, "drain_timeout_ms":10
6456 }}}).to_string()).unwrap();
6457 crate::daemon_config::load(&path)
6458 .unwrap()
6459 .unwrap()
6460 .modules
6461 .pop()
6462 .unwrap()
6463 }
6464
6465 #[cfg(unix)]
6466 async fn wait_http_health(module: &SupervisedModule, status: SupervisorHealthStatus) {
6467 timeout(Duration::from_secs(5), async {
6468 loop {
6469 if module.status().unwrap().health.status == status {
6470 break;
6471 }
6472 sleep(Duration::from_millis(5)).await;
6473 }
6474 })
6475 .await
6476 .unwrap_or_else(|_| {
6477 panic!(
6478 "expected {status:?}, got {:?}",
6479 module.status().unwrap().health
6480 )
6481 });
6482 }
6483
6484 #[cfg(unix)]
6485 #[tokio::test]
6486 async fn http_health_status_flips_ok_failing_ok() {
6487 use tokio::io::{AsyncReadExt, AsyncWriteExt};
6488 let dir = cortexkit_test_support::ScratchDir::new("http-status-flips");
6489 let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
6490 let status = Arc::new(std::sync::atomic::AtomicU16::new(200));
6491 let serving_status = status.clone();
6492 let url = format!("http://{}/healthz", listener.local_addr().unwrap());
6493 let server = tokio::spawn(async move {
6494 loop {
6495 let (mut stream, _) = listener.accept().await.unwrap();
6496 let mut request = [0u8; 2048];
6497 let count = stream.read(&mut request).await.unwrap();
6498 assert!(count > 0, "a probe must send an HTTP request");
6499 let code = serving_status.load(std::sync::atomic::Ordering::SeqCst);
6500 let body = if code == 200 {
6501 "ready"
6502 } else {
6503 "{\"status\":\"unavailable\",\"error\":\"scratch failure\"}"
6504 };
6505 let response = format!("HTTP/1.1 {code} scratch\r\nContent-Length: {}\r\nConnection: close\r\n\r\n{body}", body.len());
6506 let _ = stream.write_all(response.as_bytes()).await;
6507 }
6508 });
6509 let configured = http_fixture(&dir, &url, 1000);
6510 let module =
6511 Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default())
6512 .supervise_configured_with_health(
6513 configured.module_spec(),
6514 true,
6515 configured.health,
6516 configured.drain_timeout_ms,
6517 configured.restart,
6518 )
6519 .unwrap();
6520 wait_http_health(&module, SupervisorHealthStatus::Ok).await;
6521 status.store(503, std::sync::atomic::Ordering::SeqCst);
6522 wait_http_health(&module, SupervisorHealthStatus::Failing).await;
6523 assert!(module
6524 .status()
6525 .unwrap()
6526 .health
6527 .detail
6528 .unwrap()
6529 .contains("scratch failure"));
6530 status.store(200, std::sync::atomic::Ordering::SeqCst);
6531 wait_http_health(&module, SupervisorHealthStatus::Ok).await;
6532 assert_eq!(module.status().unwrap().health.consecutive_failures, 0);
6533 let before = module.status().unwrap();
6534 let (spec, mut health) = module.configuration().unwrap();
6535 health.http = None;
6536 module
6537 .update_configuration(spec.clone(), health.clone(), Some(10))
6538 .await
6539 .unwrap();
6540 wait_http_health(&module, SupervisorHealthStatus::Unknown).await;
6541 health.http = Some(url);
6542 module
6543 .update_configuration(spec, health, Some(10))
6544 .await
6545 .unwrap();
6546 wait_http_health(&module, SupervisorHealthStatus::Ok).await;
6547 assert_eq!(
6548 module.status().unwrap().pid,
6549 before.pid,
6550 "changing a probe must apply live, not restart its process"
6551 );
6552 let (spec, mut health) = module.configuration().unwrap();
6553 health.failure_threshold = 2;
6554 module
6555 .update_configuration(spec, health, Some(10))
6556 .await
6557 .unwrap();
6558 status.store(503, std::sync::atomic::Ordering::SeqCst);
6559 timeout(Duration::from_secs(5), async {
6560 while module.status().unwrap().spawn_generation == before.spawn_generation {
6561 sleep(Duration::from_millis(5)).await;
6562 }
6563 })
6564 .await
6565 .expect("sustained HTTP 503 responses must trigger the health restart policy");
6566 module.drain().await.unwrap();
6567 server.abort();
6568 }
6569
6570 #[cfg(unix)]
6571 #[tokio::test]
6572 async fn http_health_replacement_recovers_after_one_restart() {
6573 use tokio::io::{AsyncReadExt, AsyncWriteExt};
6574 let dir = cortexkit_test_support::ScratchDir::new("http-replacement-recovers");
6575 let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
6576 let url = format!("http://{}/healthz", listener.local_addr().unwrap());
6577 let mut configured = http_fixture(&dir, &url, 2);
6578 configured.restart.max_restarts = 2;
6579 let module =
6580 Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default())
6581 .supervise_configured_with_health(
6582 configured.module_spec(),
6583 true,
6584 configured.health,
6585 configured.drain_timeout_ms,
6586 configured.restart,
6587 )
6588 .unwrap();
6589 let first_generation = module.status().unwrap().spawn_generation;
6590 let observed = module.clone();
6591 let (healthy_tx, mut healthy_rx) = mpsc::channel(8);
6592 let server = tokio::spawn(async move {
6593 let mut generation = first_generation;
6594 let mut probes = 0;
6595 loop {
6596 let (mut stream, _) = listener.accept().await.unwrap();
6597 let mut request = [0; 2048];
6598 assert!(stream.read(&mut request).await.unwrap() > 0);
6599 let current = observed.status().unwrap().spawn_generation;
6600 if current != generation {
6601 generation = current;
6602 probes = 0;
6603 }
6604 probes += 1;
6605 let healthy = generation != first_generation && probes > 1;
6608 let code = if healthy { 200 } else { 503 };
6609 let response = format!(
6610 "HTTP/1.1 {code} scratch\r\nContent-Length: 0\r\nConnection: close\r\n\r\n"
6611 );
6612 let _ = stream.write_all(response.as_bytes()).await;
6613 if healthy {
6614 healthy_tx.send(()).await.unwrap();
6615 }
6616 }
6617 });
6618 timeout(Duration::from_secs(5), async {
6619 let mut healthy_probes = 0;
6620 loop {
6621 if module.state().unwrap() == ModuleState::Failed {
6622 break;
6623 }
6624 while healthy_rx.try_recv().is_ok() {
6625 healthy_probes += 1;
6626 }
6627 if healthy_probes >= 3
6628 && module.status().unwrap().health.status == SupervisorHealthStatus::Ok
6629 {
6630 break;
6631 }
6632 sleep(Duration::from_millis(5)).await;
6633 }
6634 })
6635 .await
6636 .expect("the replacement must settle healthy without exhausting its restart budget");
6637 let status = module.status().unwrap();
6638 module.drain().await.unwrap();
6639 server.abort();
6640 assert_eq!(status.state, ModuleState::Running);
6641 assert_eq!(status.health.status, SupervisorHealthStatus::Ok);
6642 assert_eq!(status.health.consecutive_failures, 0);
6643 assert_eq!(status.lifetime_restarts, 1);
6644 assert_eq!(status.restart_count, 1);
6645 }
6646
6647 #[cfg(unix)]
6648 #[tokio::test]
6649 async fn http_health_sustained_failure_exhausts_restart_budget() {
6650 use std::collections::BTreeMap;
6651 use tokio::io::{AsyncReadExt, AsyncWriteExt};
6652 let dir = cortexkit_test_support::ScratchDir::new("http-health-budget");
6653 let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
6654 let url = format!("http://{}/healthz", listener.local_addr().unwrap());
6655 let mut configured = http_fixture(&dir, &url, 2);
6656 configured.restart.max_restarts = 2;
6657 let module =
6658 Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default())
6659 .supervise_configured_with_health(
6660 configured.module_spec(),
6661 true,
6662 configured.health,
6663 configured.drain_timeout_ms,
6664 configured.restart,
6665 )
6666 .unwrap();
6667 let observed = module.clone();
6668 let probes = Arc::new(Mutex::new(BTreeMap::<u64, u32>::new()));
6669 let served = probes.clone();
6670 let server = tokio::spawn(async move {
6671 loop {
6672 let (mut stream, _) = listener.accept().await.unwrap();
6673 let mut request = [0; 2048];
6674 assert!(stream.read(&mut request).await.unwrap() > 0);
6675 *served
6676 .lock()
6677 .unwrap()
6678 .entry(observed.status().unwrap().spawn_generation)
6679 .or_default() += 1;
6680 let _ = stream
6681 .write_all(b"HTTP/1.1 503 Unavailable\r\nContent-Length: 0\r\nConnection: close\r\n\r\n")
6682 .await;
6683 }
6684 });
6685 timeout(Duration::from_secs(5), async {
6686 while module.state().unwrap() != ModuleState::Failed {
6687 sleep(Duration::from_millis(5)).await;
6688 }
6689 })
6690 .await
6691 .expect("sustained failures must still exhaust the health restart budget");
6692 let status = module.status().unwrap();
6693 server.abort();
6694 assert_eq!(status.restart_count, 2);
6695 assert_eq!(status.lifetime_restarts, 2);
6696 assert!(!status.process_alive);
6697 assert_eq!(
6698 probes.lock().unwrap().values().copied().collect::<Vec<_>>(),
6699 vec![2, 2, 2]
6700 );
6701 }
6702
6703 #[cfg(unix)]
6704 #[tokio::test]
6705 async fn spawn_clears_the_previous_process_health() {
6706 let dir = cortexkit_test_support::ScratchDir::new("spawn-fresh-health");
6707 let mut configured = http_fixture(&dir, "http://127.0.0.1:1/healthz", 2);
6708 configured.health.cadence = Duration::from_secs(3600);
6709 let module =
6710 Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default())
6711 .supervise_configured_with_health(
6712 configured.module_spec(),
6713 false,
6714 configured.health,
6715 configured.drain_timeout_ms,
6716 configured.restart,
6717 )
6718 .unwrap();
6719 update_snapshot(&module.inner.snapshot, Some(module.module_id()), |state| {
6720 state.health.status = SupervisorHealthStatus::Unresponsive;
6721 state.health.consecutive_failures = 3;
6722 state.health.detail = Some("old process refused connections".into());
6723 state.health.metrics = Some(serde_json::json!({"old": true}));
6724 state.health.last_probe_ms = Some(123);
6725 state.health.last_action = Some("restart".into());
6726 state.health.last_action_ms = Some(124);
6727 })
6728 .unwrap();
6729 module.set_enabled(true).await.unwrap();
6730 let health = module.status().unwrap().health;
6731 module.drain().await.unwrap();
6732 assert_eq!(health.consecutive_failures, 0);
6733 assert_eq!(health.status, SupervisorHealthStatus::Unknown);
6734 assert_eq!(health.detail, None);
6735 assert_eq!(health.metrics, None);
6736 assert_eq!(health.last_probe_ms, None);
6737 assert_eq!(health.last_action.as_deref(), Some("restart"));
6738 assert_eq!(health.last_action_ms, Some(124));
6739 }
6740
6741 #[cfg(unix)]
6742 #[tokio::test]
6743 async fn http_health_no_listener_restarts_after_consecutive_failures() {
6744 let dir = cortexkit_test_support::ScratchDir::new("http-refused");
6745 let unused = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
6746 let url = format!("http://{}/healthz", unused.local_addr().unwrap());
6747 drop(unused);
6748 let configured = http_fixture(&dir, &url, 2);
6749 let module =
6750 Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default())
6751 .supervise_configured_with_health(
6752 configured.module_spec(),
6753 true,
6754 configured.health,
6755 configured.drain_timeout_ms,
6756 configured.restart,
6757 )
6758 .unwrap();
6759 let before = module.status().unwrap().spawn_generation;
6760 timeout(Duration::from_secs(5), async {
6761 loop {
6762 let status = module.status().unwrap();
6763 if status.spawn_generation > before {
6764 assert!(status.lifetime_restarts > 0);
6765 break;
6766 }
6767 sleep(Duration::from_millis(5)).await;
6768 }
6769 })
6770 .await
6771 .expect("sustained HTTP refusal must trigger the health restart policy");
6772 module.drain().await.unwrap();
6773 }
6774
6775 #[cfg(unix)]
6776 #[tokio::test]
6777 async fn http_health_timeout_honours_deadline() {
6778 let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
6779 let url = format!("http://{}/healthz", listener.local_addr().unwrap());
6780 let server = tokio::spawn(async move {
6781 let _held = listener.accept().await.unwrap();
6782 std::future::pending::<()>().await;
6783 });
6784 let error = timeout(
6785 Duration::from_secs(1),
6786 probe_http_health(&url, Duration::from_millis(10)),
6787 )
6788 .await
6789 .expect("the probe must enforce its own deadline")
6790 .unwrap_err();
6791 server.abort();
6792 assert!(error.to_string().contains("timed out"));
6793 }
6794
6795 #[cfg(unix)]
6796 #[tokio::test]
6797 async fn http_health_localhost_reaches_an_ipv6_only_listener() {
6798 use tokio::io::{AsyncReadExt, AsyncWriteExt};
6799 let listener = tokio::net::TcpListener::bind("[::1]:0").await.unwrap();
6800 let url = format!(
6801 "http://localhost:{}/healthz",
6802 listener.local_addr().unwrap().port()
6803 );
6804 let server = tokio::spawn(async move {
6805 let (mut stream, _) = listener.accept().await.unwrap();
6806 let mut request = [0u8; 2048];
6807 assert!(stream.read(&mut request).await.unwrap() > 0);
6808 stream
6809 .write_all(b"HTTP/1.1 200 OK\r\nContent-Length: 2\r\n\r\nok")
6810 .await
6811 .unwrap();
6812 });
6813 assert_eq!(
6818 probe_http_health(&url, Duration::from_secs(10))
6819 .await
6820 .unwrap()
6821 .status,
6822 HealthStatus::Ok
6823 );
6824 server.await.unwrap();
6825 }
6826
6827 #[cfg(unix)]
6828 #[tokio::test]
6829 async fn http_health_timeout_keeps_partial_status_and_body() {
6830 use tokio::io::{AsyncReadExt, AsyncWriteExt};
6831 let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
6832 let url = format!("http://{}/healthz", listener.local_addr().unwrap());
6833 let server = tokio::spawn(async move {
6834 let (mut stream, _) = listener.accept().await.unwrap();
6835 let mut request = [0u8; 2048];
6836 assert!(stream.read(&mut request).await.unwrap() > 0);
6837 stream
6838 .write_all(
6839 b"HTTP/1.1 503 Unavailable\r\nContent-Length: 100\r\n\r\npartial diagnostic",
6840 )
6841 .await
6842 .unwrap();
6843 std::future::pending::<()>().await;
6844 });
6845 let error = probe_http_health(&url, Duration::from_secs(1))
6846 .await
6847 .unwrap_err()
6848 .to_string();
6849 server.abort();
6850 assert!(
6851 error.contains("timed out")
6852 && error.contains("503 Unavailable")
6853 && error.contains("partial diagnostic"),
6854 "{error}"
6855 );
6856 }
6857
6858 #[cfg(unix)]
6859 #[tokio::test]
6860 async fn http_health_diagnostic_body_is_bounded_to_200_bytes() {
6861 use tokio::io::{AsyncReadExt, AsyncWriteExt};
6862 let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
6863 let url = format!("http://{}/healthz", listener.local_addr().unwrap());
6864 let server = tokio::spawn(async move {
6865 let (mut stream, _) = listener.accept().await.unwrap();
6866 let mut request = [0u8; 2048];
6867 assert!(stream.read(&mut request).await.unwrap() > 0);
6868 let body = format!("{}not-in-diagnostic", "x".repeat(400));
6869 let response = format!(
6870 "HTTP/1.1 503 Unavailable\r\nContent-Length: {}\r\n\r\n{body}",
6871 body.len()
6872 );
6873 stream.write_all(response.as_bytes()).await.unwrap();
6874 });
6875 let error = probe_http_health(&url, Duration::from_secs(1))
6876 .await
6877 .unwrap_err()
6878 .to_string();
6879 server.await.unwrap();
6880 assert!(error.ends_with(&"x".repeat(200)), "{error}");
6881 assert_eq!(error.split(": ").last().unwrap().len(), 200);
6882 assert!(!error.contains("not-in-diagnostic"));
6883 }
6884
6885 #[cfg(unix)]
6886 #[tokio::test]
6887 async fn http_health_real_nats_server_monitoring() {
6888 if std::process::Command::new("nats-server")
6889 .arg("--version")
6890 .env("XDG_DATA_HOME", std::env::temp_dir())
6891 .env("XDG_RUNTIME_DIR", std::env::temp_dir())
6892 .env("XDG_CONFIG_HOME", std::env::temp_dir())
6893 .output()
6894 .is_err()
6895 {
6896 eprintln!(
6897 "skipped: http_health_real_nats_server_monitoring (nats-server is not on PATH)"
6898 );
6899 return;
6900 }
6901 let dir = cortexkit_test_support::ScratchDir::new("nats-http-monitoring");
6902 let monitor = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
6903 let port = monitor.local_addr().unwrap().port();
6904 let client = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
6905 let client_port = client.local_addr().unwrap().port();
6906 let config = dir.join("server.conf");
6907 std::fs::write(&config, format!("listen: 127.0.0.1:{client_port}\nhttp: 127.0.0.1:{port}\njetstream {{ store_dir: \"{}\" }}\n", dir.join("jetstream").display())).unwrap();
6908 let mut configured = http_fixture(&dir, &format!("http://127.0.0.1:{port}/healthz"), 1000);
6909 configured.program = PathBuf::from("nats-server");
6910 configured.args = vec!["-c".into(), config.to_string_lossy().into_owned()];
6911 drop(monitor);
6912 drop(client);
6913 let module =
6914 Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default())
6915 .supervise_configured_with_health(
6916 configured.module_spec(),
6917 true,
6918 configured.health,
6919 configured.drain_timeout_ms,
6920 configured.restart,
6921 )
6922 .unwrap();
6923 wait_http_health(&module, SupervisorHealthStatus::Ok).await;
6924 module.drain().await.unwrap();
6925 }
6926
6927 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
6928 async fn configuration_update_does_not_replace_captured_running_process_facts() {
6929 let supervisor =
6930 Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default());
6931 let snapshot = stale_process_snapshot(ModuleState::Running, true);
6932 let initial = ModuleSpec {
6933 module_id: "rescan-preserves-spawn-facts".to_string(),
6934 program: PathBuf::from("/spawned/module"),
6935 args: Vec::new(),
6936 env: Vec::new(),
6937 reserved: false,
6938 reserved_prefixes: Vec::new(),
6939 protocol: ModuleProtocol::Subc,
6940 overlap: Default::default(),
6941 carry_containment: false,
6942 };
6943 let module = supervisor.supervised_module(
6944 initial.clone(),
6945 supervisor.runtime_config(),
6946 snapshot,
6947 None,
6948 );
6949 let before = module.status().unwrap();
6950 let mut replacement = initial;
6951 replacement.program = PathBuf::from("/rescanned/replacement-module");
6952
6953 module
6954 .update_configuration(replacement, HealthConfig::default(), None)
6955 .await
6956 .unwrap();
6957
6958 let after = module.status().unwrap();
6959 assert_eq!(after.pid, before.pid);
6960 assert_eq!(after.spawned_at_ms, before.spawned_at_ms);
6961 assert_eq!(after.spawned_from, before.spawned_from);
6962 drop(module);
6963 }
6964}
6965
6966fn unix_ms_now() -> u64 {
6967 SystemTime::now()
6968 .duration_since(UNIX_EPOCH)
6969 .map(|duration| duration.as_millis().min(u128::from(u64::MAX)) as u64)
6970 .unwrap_or(0)
6971}
6972
6973async fn supervise_loop(
6974 mut spec: ModuleSpec,
6975 mut runtime: SupervisorRuntimeConfig,
6976 registry: Arc<Registry>,
6977 process_liveness: Arc<SupervisorProcessLiveness>,
6978 snapshot: SharedSnapshot,
6979 mut child: Option<SupervisedChild>,
6980 mut commands: mpsc::Receiver<SupervisorCommand>,
6981) {
6982 let mut health_probe = HealthProbeRuntime::default();
6983 let mut registration_changes = match registry.subscribe_module_changes(&spec.module_id) {
6987 Ok(changes) => Some(changes),
6988 Err(err) => {
6989 warn!(module_id = %spec.module_id, error = %err, "health prober could not subscribe to registry");
6992 None
6993 }
6994 };
6995 let mut pending_respawn: Option<PendingRespawn> = None;
6999 let mut requeued: VecDeque<SupervisorCommand> = VecDeque::new();
7002 let mut previous_instance_recheck = tokio::time::interval_at(
7003 Instant::now() + Duration::from_secs(5),
7004 Duration::from_secs(5),
7005 );
7006 previous_instance_recheck.set_missed_tick_behavior(tokio::time::MissedTickBehavior::Skip);
7007 loop {
7008 #[cfg(test)]
7009 {
7010 lock_snapshot(&snapshot).unwrap().actor_select = None;
7011 }
7012 #[cfg(target_os = "macos")]
7013 if let Some(active) = child.as_mut() {
7014 active.confirm_privacy_exec().await;
7015 }
7016 if let Some(scheduled) = runtime
7017 .scheduled_respawn
7018 .lock()
7019 .unwrap_or_else(|p| p.into_inner())
7020 .take()
7021 {
7022 runtime.recovery_is_daemon = scheduled.daemon_initiated;
7023 pending_respawn = Some(scheduled);
7024 }
7025 if pending_respawn.is_some() && (child.is_some() || !respawn_still_pending(&snapshot)) {
7026 pending_respawn = None;
7027 cancel_deferred_reload(
7028 &runtime,
7029 &spec.module_id,
7030 "respawn cancelled by a supervisor command",
7031 );
7032 }
7033 if child.is_none()
7034 && pending_respawn.is_none()
7035 && lock_snapshot(&snapshot).is_ok_and(|state| state.waiting_for_previous_pid.is_none())
7036 {
7037 cancel_deferred_reload(
7038 &runtime,
7039 &spec.module_id,
7040 "respawn cancelled before a replacement was spawned",
7041 );
7042 let mut stranded = false;
7043 let _ = update_snapshot(&snapshot, Some(&spec.module_id), |state| {
7044 state.respawn_pending = false;
7045 state.coalesced_restart_pending = false;
7046 if matches!(
7047 state.state,
7048 ModuleState::Restarting
7049 | ModuleState::Starting
7050 | ModuleState::Draining
7051 | ModuleState::Unresponsive
7052 ) {
7053 error!(module_id = %spec.module_id, state = ?state.state,
7054 "supervision operation ended without a child or pending respawn; marking failed so start can retry");
7055 stranded = true;
7056 }
7057 });
7058 if stranded {
7059 fail_snapshot(
7060 &snapshot,
7061 Some(&spec.module_id),
7062 None,
7063 &runtime.failure_context(runtime.recovery_is_daemon),
7064 );
7065 }
7066 }
7067 if child.is_none()
7068 && (runtime.child_roster.is_closed()
7069 || pending_respawn.is_none()
7070 || !lock_snapshot(&snapshot)
7071 .is_ok_and(|state| state.enabled && state.state == ModuleState::Restarting))
7072 {
7073 containment::terminate(&runtime.child_roster.containment).await;
7074 }
7075 if let Some(command) = requeued.pop_front() {
7076 if !handle_supervisor_command(
7077 command,
7078 &mut spec,
7079 &mut runtime,
7080 ®istry,
7081 &process_liveness,
7082 &snapshot,
7083 &mut child,
7084 &mut commands,
7085 &mut requeued,
7086 )
7087 .await
7088 {
7089 containment::terminate(&runtime.child_roster.containment).await;
7090 return;
7091 }
7092 if child.is_some() || !respawn_still_pending(&snapshot) {
7093 pending_respawn = None;
7094 let _ = update_snapshot(&snapshot, Some(&spec.module_id), |state| {
7095 state.respawn_pending = false
7096 });
7097 }
7098 continue;
7099 }
7100 if child.is_some() {
7101 runtime.recovery_is_daemon = true;
7102 #[cfg(test)]
7103 {
7104 lock_snapshot(&snapshot).unwrap().actor_turns += 1;
7105 }
7106 health_probe.refresh_registration(&spec, &runtime, ®istry, &snapshot);
7107 let active = child.as_mut().expect("child checked above");
7108 if runtime.child_roster.is_closed() || active.registered_since_spawn() {
7109 active.registration_deadline = None;
7110 }
7111 let registration_deadline = active.registration_deadline;
7112 let no_checks = runtime.health.http.is_none()
7113 && (running_protocol(&spec, &snapshot) == ModuleProtocol::None
7114 || (health_probe.registered_connection.is_some() && !health_probe.advertised));
7115 let uptime_reset = lock_snapshot(&snapshot).ok().and_then(|mut state| {
7116 state
7117 .failed_retry
7118 .uptime_reset_at(no_checks, runtime.restart_policy.window)
7119 });
7120 let wake_at = match (health_probe.wake_at(), uptime_reset) {
7121 (probe, Some(reset)) => Some(probe.map_or(reset, |probe| probe.min(reset))),
7122 (probe, None) => probe,
7123 };
7124 let wake_at = match (wake_at, registration_deadline) {
7125 (Some(wake), Some(register)) => Some(wake.min(register)),
7126 (wake, register) => wake.or(register),
7127 };
7128 #[cfg(test)]
7129 let wake_after = wake_at.map(|at| at.saturating_duration_since(Instant::now()));
7130 let probe_sleep = supervisor_sleep(wake_at);
7131 tokio::pin!(probe_sleep);
7132 let wire_child = running_protocol(&spec, &snapshot) == ModuleProtocol::Subc;
7137 #[cfg(test)]
7138 {
7139 let registration_pending = wire_child
7140 && registration_changes
7141 .as_ref()
7142 .is_some_and(|changes| changes.has_changed().unwrap());
7143 let mut state = lock_snapshot(&snapshot).unwrap();
7144 state.actor_select = (!registration_pending).then_some(ActorSelectCheckpoint {
7145 generation: state.spawn_generation,
7146 turn: state.actor_turns,
7147 registered_connection: health_probe.registered_connection,
7148 next_probe_at: health_probe.next_probe_at,
7149 wake_after,
7150 });
7151 }
7152 let active_child = child.as_mut().expect("child checked above");
7153 tokio::select! {
7154 wait_result = active_child.wait() => {
7155 if let Ok(mut state) = lock_snapshot(&snapshot) {
7156 state.failed_retry.uptime_reset_at(no_checks, runtime.restart_policy.window);
7157 }
7158 #[cfg(test)]
7159 { lock_snapshot(&snapshot).unwrap().actor_select = None; }
7160 let exit_report = match wait_result {
7169 Ok(status) => classify_reaped_child_exit(&snapshot, active_child, &status),
7170 Err(err) => {
7171 active_child.drain_stderr(&spec.module_id).await;
7172 fail_after_wait_error(
7173 &spec.module_id,
7174 &snapshot,
7175 &runtime.terminal_ring,
7176 &runtime.spawn_events,
7177 format!("failed to wait for supervised module: {err}"),
7178 &runtime.failure_context(true),
7179 );
7180 untrack_if_registration_released(
7181 &process_liveness,
7182 ®istry,
7183 &spec.module_id,
7184 &snapshot,
7185 );
7186 error!(module_id = %spec.module_id, error = %err, "failed to wait for supervised module");
7187 child = None;
7188 continue;
7189 }
7190 };
7191 active_child.drain_stderr(&spec.module_id).await;
7192
7193 let next = on_child_exit(
7194 &spec,
7195 runtime.restart_policy,
7196 ®istry,
7197 &snapshot,
7198 &runtime.terminal_ring,
7199 &runtime.spawn_events,
7200 &runtime.child_roster,
7201 exit_report,
7202 true,
7203 ).await;
7204 runtime.start_requester = None;
7205 runtime.pending_activity_restart = false;
7206 let carry = matches!(next, NextAction::Restart { .. })
7209 && active_child.spawn_failure.is_none()
7210 && !runtime.child_roster.is_closed();
7211 if let Err(error) = active_child.finish_exit(carry).await {
7212 containment::terminate(&runtime.child_roster.containment).await;
7213 fail_after_wait_error(&spec.module_id, &snapshot, &runtime.terminal_ring,
7214 &runtime.spawn_events, format!("failed to reap classified exit: {error}"),
7215 &runtime.failure_context(true));
7216 child = None;
7217 continue;
7218 }
7219 match next {
7220 NextAction::Stop { registration_released } => {
7221 if registration_released {
7222 process_liveness.untrack_if_current(&spec.module_id, &snapshot);
7223 }
7224 child = None;
7225 }
7226 NextAction::Restart { schedule } => {
7227 let delay = schedule.map_or(
7228 runtime.restart_policy.delay_for_restart(0),
7229 |schedule| schedule.delay,
7230 );
7231 if let Some(schedule) = schedule {
7232 log_crash_respawn(&spec.module_id, schedule);
7233 }
7234 child = None;
7242 pending_respawn = Some(PendingRespawn { deadline: Instant::now() + delay, kind: RespawnKind::Spawn, daemon_initiated: true });
7243 let _ = update_snapshot(&snapshot, Some(&spec.module_id), |state| state.respawn_pending = true);
7244 }
7245 }
7246 }
7247 command = commands.recv() => {
7248 #[cfg(test)]
7249 { lock_snapshot(&snapshot).unwrap().actor_select = None; }
7250 let Some(command) = command else {
7251 containment::terminate(&runtime.child_roster.containment).await;
7252 return;
7253 };
7254 if !handle_supervisor_command(
7255 command,
7256 &mut spec,
7257 &mut runtime,
7258 ®istry,
7259 &process_liveness,
7260 &snapshot,
7261 &mut child,
7262 &mut commands,
7263 &mut requeued,
7264 ).await {
7265 containment::terminate(&runtime.child_roster.containment).await;
7266 return;
7267 }
7268 }
7269 _ = async {
7270 match registration_changes.as_mut() {
7271 Some(changes) => { let _ = changes.changed().await; }
7272 None => std::future::pending().await,
7273 }
7274 }, if wire_child => {
7275 #[cfg(test)]
7276 { lock_snapshot(&snapshot).unwrap().actor_select = None; }
7277 }
7278 _ = &mut probe_sleep => {
7279 #[cfg(test)]
7280 { lock_snapshot(&snapshot).unwrap().actor_select = None; }
7281 if registration_deadline.is_some_and(|deadline| Instant::now() >= deadline) {
7282 active_child.registration_deadline = None;
7285 if !runtime.child_roster.is_closed()
7286 && !active_child.registered_since_spawn()
7287 && active_child.child.try_wait().is_ok_and(|status| status.is_none())
7288 {
7289 warn!(module_id = %spec.module_id, pid = active_child.pid,
7290 wait_secs = runtime.register_timeout.as_secs(),
7291 "supervised module registration_timeout: first HELLO deadline expired; stopping child");
7292 active_child.spawn_failure = Some("registration_timeout".into());
7293 if let Err(error) = stop_unregistered_child(active_child).await {
7294 error!(module_id = %spec.module_id, %error, "failed to stop unregistered child");
7295 fail_after_wait_error(&spec.module_id, &snapshot, &runtime.terminal_ring,
7296 &runtime.spawn_events, format!("registration_timeout: {error}"),
7297 &runtime.failure_context(true));
7298 child = None;
7299 }
7300 continue;
7303 }
7304 }
7305 if health_probe.due() {
7306 run_health_probe_cycle(
7307 &spec,
7308 &runtime,
7309 ®istry,
7310 &process_liveness,
7311 &snapshot,
7312 &mut child,
7313 ).await;
7314 if child.is_none() {
7315 runtime.start_requester = None;
7316 runtime.pending_activity_restart = false;
7317 }
7318 if child.is_some() {
7319 health_probe.schedule_next(&spec, runtime.health.cadence);
7320 }
7321 }
7322 }
7323 }
7324 } else if let Some(pending) = pending_respawn {
7325 tokio::select! {
7326 _ = runtime.child_roster.closed() => {
7327 pending_respawn = None;
7328 let _ = update_snapshot(&snapshot, Some(&spec.module_id), |state| {
7329 state.state = ModuleState::Stopped;
7330 state.respawn_pending = false;
7331 });
7332 containment::terminate(&runtime.child_roster.containment).await;
7333 }
7334 _ = sleep_until(pending.deadline) => {
7335 pending_respawn = None;
7336 let _ = update_snapshot(&snapshot, Some(&spec.module_id), |state| state.respawn_pending = false);
7337 if !respawn_still_pending(&snapshot) {
7341 continue;
7342 }
7343 if runtime.child_roster.is_closed() {
7348 let _ = update_snapshot(&snapshot, Some(&spec.module_id), |state| {
7349 state.state = ModuleState::Stopped;
7350 });
7351 debug!(module_id = %spec.module_id, "crash respawn cancelled by daemon shutdown");
7352 continue;
7353 }
7354 if let Err(err) = release_dead_registration(
7355 ®istry,
7356 runtime.forwarding.as_deref(),
7357 &snapshot,
7358 &spec.module_id,
7359 &runtime.failure_context(runtime.recovery_is_daemon),
7360 ).await {
7361 error!(module_id = %spec.module_id, error = %err, "registration did not release before restart");
7362 continue;
7363 }
7364
7365 if matches!(pending.kind, RespawnKind::Reload) {
7366 let reply = runtime.deferred_reload_reply.lock().unwrap_or_else(|p| p.into_inner()).take();
7367 let result = finish_reload_child(&spec, &runtime, ®istry, &process_liveness, &snapshot, &mut child).await;
7368 if result.is_ok() && runtime.pending_activity_restart {
7369 runtime.record_activity_transition("subc.module_restarted", &spec.module_id, runtime.start_requester.clone());
7370 }
7371 runtime.pending_activity_restart = false;
7372 if let Some(reply) = reply { let _ = reply.send(result); }
7373 continue;
7374 }
7375 process_liveness.track(spec.module_id.clone(), Arc::clone(&snapshot));
7376 match spawn_and_mark_running(&spec, &runtime, &snapshot) {
7377 Ok(next_child) => {
7378 child = Some(next_child);
7379 if runtime.pending_activity_restart {
7380 runtime.record_activity_transition("subc.module_restarted", &spec.module_id, runtime.start_requester.clone());
7381 runtime.pending_activity_restart = false;
7382 }
7383 debug!(module_id = %spec.module_id, "supervised module restarted after crash");
7384 }
7385 Err(err) => {
7386 fail_snapshot(&snapshot, Some(&spec.module_id), None, &runtime.failure_context(runtime.recovery_is_daemon));
7387 process_liveness.untrack_if_current(&spec.module_id, &snapshot);
7388 error!(module_id = %spec.module_id, error = %err, "failed to restart supervised module");
7389 }
7390 }
7391 }
7392 command = commands.recv() => {
7393 let Some(command) = command else {
7394 containment::terminate(&runtime.child_roster.containment).await;
7395 return;
7396 };
7397 if !handle_supervisor_command(
7398 command,
7399 &mut spec,
7400 &mut runtime,
7401 ®istry,
7402 &process_liveness,
7403 &snapshot,
7404 &mut child,
7405 &mut commands,
7406 &mut requeued,
7407 ).await {
7408 containment::terminate(&runtime.child_roster.containment).await;
7409 return;
7410 }
7411 if child.is_some() || !respawn_still_pending(&snapshot) {
7416 pending_respawn = None;
7417 let _ = update_snapshot(&snapshot, Some(&spec.module_id), |state| state.respawn_pending = false);
7418 }
7419 }
7420 }
7421 } else {
7422 let slow_retry = lock_snapshot(&snapshot).ok().and_then(|state| {
7423 (state.enabled && state.state == ModuleState::Failed)
7424 .then_some(state.failed_retry.pending)
7425 .flatten()
7426 });
7427 let command = if lock_snapshot(&snapshot)
7428 .is_ok_and(|state| state.waiting_for_previous_pid.is_some())
7429 {
7430 tokio::select! {
7431 command = commands.recv() => command,
7432 _ = previous_instance_recheck.tick() => {
7433 if runtime.child_roster.previous_instance_pid(&spec.module_id).is_none() {
7434 let enabled = lock_snapshot(&snapshot).is_ok_and(|state| state.enabled);
7435 if enabled && !runtime.child_roster.is_closed() {
7436 process_liveness.track(spec.module_id.clone(), Arc::clone(&snapshot));
7437 match spawn_and_mark_running(&spec, &runtime, &snapshot) {
7438 Ok(next_child) => child = Some(next_child),
7439 Err(error) => {
7440 fail_snapshot(&snapshot, Some(&spec.module_id), None, &runtime.failure_context(true));
7441 error!(module_id = %spec.module_id, %error, "failed to start module after its previous instance exited");
7442 }
7443 }
7444 } else {
7445 let _ = update_snapshot(&snapshot, Some(&spec.module_id), |state| {
7446 state.waiting_for_previous_pid = None;
7447 state.waiting_for_previous_detail = None;
7448 });
7449 }
7450 }
7451 continue;
7452 }
7453 }
7454 } else {
7455 tokio::select! {
7456 biased;
7457 command = commands.recv() => command,
7458 _ = async {
7459 match slow_retry {
7460 Some(retry) => sleep_until(retry.deadline).await,
7461 None => std::future::pending().await,
7462 }
7463 } => {
7464 let retry = slow_retry.expect("only a scheduled slow retry can wake");
7465 let _ = update_snapshot(&snapshot, Some(&spec.module_id), |state| {
7466 state.failed_retry.pending = None;
7467 });
7468 if runtime.child_roster.is_closed() {
7469 continue;
7470 }
7471 info!(module_id = %spec.module_id, attempt = retry.attempt,
7472 delay_secs = retry.delay.as_secs(), "starting failed module retry");
7473 if let Err(error) = set_child_enabled(
7474 &spec, &runtime, ®istry, &process_liveness, &snapshot, &mut child, true,
7475 &runtime.failure_context(true),
7476 ).await {
7477 error!(module_id = %spec.module_id, %error, "failed module retry could not start");
7478 }
7479 continue;
7480 }
7481 }
7482 };
7483 let Some(command) = command else {
7484 containment::terminate(&runtime.child_roster.containment).await;
7485 return;
7486 };
7487 if !handle_supervisor_command(
7488 command,
7489 &mut spec,
7490 &mut runtime,
7491 ®istry,
7492 &process_liveness,
7493 &snapshot,
7494 &mut child,
7495 &mut commands,
7496 &mut requeued,
7497 )
7498 .await
7499 {
7500 containment::terminate(&runtime.child_roster.containment).await;
7501 return;
7502 }
7503 }
7504 }
7505}
7506
7507async fn supervisor_sleep(wake_at: Option<Instant>) {
7510 match wake_at {
7511 Some(deadline) => sleep_until(deadline).await,
7512 None => std::future::pending().await,
7513 }
7514}
7515
7516fn log_crash_respawn(module_id: &str, schedule: CrashRestartSchedule) {
7517 info!(
7518 module_id,
7519 restart_in_window = schedule.restart_in_window,
7520 delay_ms = schedule.delay.as_millis() as u64,
7521 "respawning after crash"
7522 );
7523}
7524
7525fn respawn_still_pending(snapshot: &SharedSnapshot) -> bool {
7531 matches!(
7532 lock_snapshot(snapshot),
7533 Ok(state) if state.enabled && state.state == ModuleState::Restarting
7534 )
7535}
7536
7537enum NextAction {
7538 Stop {
7539 registration_released: bool,
7540 },
7541 Restart {
7542 schedule: Option<CrashRestartSchedule>,
7543 },
7544}
7545
7546#[allow(clippy::too_many_arguments)]
7547async fn handle_supervisor_command(
7548 command: SupervisorCommand,
7549 spec: &mut ModuleSpec,
7550 runtime: &mut SupervisorRuntimeConfig,
7551 registry: &Arc<Registry>,
7552 process_liveness: &SupervisorProcessLiveness,
7553 snapshot: &SharedSnapshot,
7554 child: &mut Option<SupervisedChild>,
7555 commands: &mut mpsc::Receiver<SupervisorCommand>,
7556 requeued: &mut VecDeque<SupervisorCommand>,
7557) -> bool {
7558 if !matches!(command, SupervisorCommand::UpdateConfiguration { .. }) {
7559 runtime.recovery_is_daemon = false;
7560 }
7561 if matches!(
7562 command,
7563 SupervisorCommand::Drain { .. }
7564 | SupervisorCommand::Retire { .. }
7565 | SupervisorCommand::SetEnabled { .. }
7566 | SupervisorCommand::Restart { .. }
7567 ) {
7568 let _ = update_snapshot(snapshot, Some(&spec.module_id), |state| {
7569 state.failed_retry.cancel()
7570 });
7571 }
7572 let blocked = recovery_block_error(spec, &runtime.child_roster).or_else(|| {
7573 runtime
7574 .child_roster
7575 .previous_instance(&spec.module_id)
7576 .map(|(pid, reason)| previous_instance_error(spec, pid, reason))
7577 });
7578 let command = if let Some(error) = blocked {
7579 match command {
7580 SupervisorCommand::Restart { reply, .. }
7581 | SupervisorCommand::Reload { reply, .. }
7582 | SupervisorCommand::Swap { reply, .. } => {
7583 let _ = reply.send(Err(error));
7584 return true;
7585 }
7586 SupervisorCommand::SetEnabled {
7587 enabled: true,
7588 reply,
7589 ..
7590 } => {
7591 let _ = reply.send(Err(error));
7592 return true;
7593 }
7594 other => other,
7595 }
7596 } else {
7597 command
7598 };
7599 match command {
7600 SupervisorCommand::Drain { requester, reply } => {
7601 let result = drain_optional_child(
7604 &spec.module_id,
7605 spec.protocol,
7606 StopNotice::NotSent,
7607 registry,
7608 runtime.forwarding.as_deref(),
7609 snapshot,
7610 &runtime.terminal_ring,
7611 &runtime.spawn_events,
7612 child,
7613 runtime.drain_timeout,
7614 ModuleState::Stopped,
7615 None,
7616 &FailureContext::operator(),
7617 )
7618 .await;
7619 containment::terminate(&runtime.child_roster.containment).await;
7620 let registration_released = result.is_ok();
7621 if registration_released {
7622 runtime.record_activity_transition(
7623 "subc.module_stopped",
7624 &spec.module_id,
7625 requester,
7626 );
7627 }
7628 let _ = reply.send(result);
7629 if registration_released {
7630 process_liveness.untrack_if_current(&spec.module_id, snapshot);
7631 }
7632 false
7633 }
7634 SupervisorCommand::Retire { requester, reply } => {
7635 let result = async {
7636 let stop_notice = begin_forwarding_drain_if_configured(
7637 spec,
7638 runtime,
7639 registry,
7640 snapshot,
7641 None,
7642 RouteCloseReason::Disable,
7643 )
7644 .await?;
7645 drain_optional_child(
7646 &spec.module_id,
7647 spec.protocol,
7648 stop_notice,
7649 registry,
7650 runtime.forwarding.as_deref(),
7651 snapshot,
7652 &runtime.terminal_ring,
7653 &runtime.spawn_events,
7654 child,
7655 runtime.drain_timeout,
7656 ModuleState::Stopped,
7657 None,
7658 &FailureContext::operator(),
7659 )
7660 .await
7661 }
7662 .await;
7663 containment::terminate(&runtime.child_roster.containment).await;
7664 let registration_released = result.is_ok();
7665 if registration_released {
7666 runtime.record_activity_transition(
7667 "subc.module_stopped",
7668 &spec.module_id,
7669 requester,
7670 );
7671 }
7672 let _ = reply.send(result);
7673 if registration_released {
7674 process_liveness.untrack_if_current(&spec.module_id, snapshot);
7675 }
7676 false
7677 }
7678 SupervisorCommand::Restart {
7679 requester,
7680 drain_timeout_ms,
7681 received_at_generation,
7682 queued_at,
7683 reply,
7684 } => {
7685 info!(
7689 module_id = %spec.module_id,
7690 queued_ms = u64::try_from(queued_at.elapsed().as_millis()).unwrap_or(u64::MAX),
7691 "restart command dequeued"
7692 );
7693 let validation = match lock_snapshot(snapshot) {
7705 Ok(state) if !state.enabled => Err(SuperviseError::Disabled {
7706 module_id: spec.module_id.clone(),
7707 }),
7708 Ok(_) => Ok(()),
7709 Err(err) => Err(err),
7710 };
7711 let initiated = validation.is_ok();
7712 let _ = reply.send(validation);
7713 let satisfied_by_generation = if initiated
7727 && child
7728 .as_ref()
7729 .is_some_and(|child| child.spawned_at > queued_at)
7730 {
7731 lock_snapshot(snapshot).ok().and_then(|state| {
7732 (!state.configuration_updated_since_spawn).then_some(state.spawn_generation)
7733 })
7734 } else {
7735 None
7736 };
7737 let satisfied_by_pending = initiated
7738 && child.is_none()
7739 && lock_snapshot(snapshot).ok().is_some_and(|mut state| {
7740 let pending = state.respawn_pending && !state.configuration_updated_since_spawn;
7744 if pending {
7745 state.coalesced_restart_pending = true;
7746 }
7747 pending
7748 });
7749 if satisfied_by_pending {
7750 debug!(module_id = %spec.module_id, "restart coalesced into the pending replacement");
7751 } else if let Some(generation) = satisfied_by_generation {
7752 info!(
7753 module_id = %spec.module_id,
7754 received_at_generation,
7755 "restart already satisfied by generation {generation}; not restarting again"
7756 );
7757 } else if initiated {
7758 let drain_timeout = drain_timeout_ms
7761 .map(Duration::from_millis)
7762 .unwrap_or(runtime.drain_timeout);
7763 if let Err(err) = restart_child(
7764 spec,
7765 runtime,
7766 registry,
7767 process_liveness,
7768 snapshot,
7769 child,
7770 drain_timeout,
7771 )
7772 .await
7773 {
7774 warn!(
7775 module_id = %spec.module_id,
7776 error = %err,
7777 "operator restart failed after initiation ack; module state carries the outcome"
7778 );
7779 let _ = update_snapshot(snapshot, Some(&spec.module_id), |state| {
7780 state.state = ModuleState::Failed;
7781 clear_current_process_facts(state);
7782 });
7783 } else {
7784 runtime.start_requester = requester;
7785 runtime.pending_activity_restart = true;
7786 }
7787 }
7788 true
7789 }
7790 SupervisorCommand::Reload { requester, reply } => {
7791 let result =
7792 reload_child(spec, runtime, registry, process_liveness, snapshot, child).await;
7793 if result.is_ok() {
7794 runtime.start_requester = requester;
7795 runtime.pending_activity_restart = true;
7796 }
7797 if result.is_ok()
7798 && runtime
7799 .scheduled_respawn
7800 .lock()
7801 .unwrap_or_else(|p| p.into_inner())
7802 .as_ref()
7803 .is_some_and(|pending| matches!(pending.kind, RespawnKind::Reload))
7804 {
7805 *runtime
7806 .deferred_reload_reply
7807 .lock()
7808 .unwrap_or_else(|p| p.into_inner()) = Some(reply);
7809 } else {
7810 let _ = reply.send(result);
7811 }
7812 true
7813 }
7814 SupervisorCommand::SetEnabled {
7815 enabled,
7816 requester,
7817 reply,
7818 } => {
7819 let previous_requester = runtime.start_requester.clone();
7820 if enabled {
7821 runtime.start_requester = requester.clone();
7822 }
7823 let result = set_child_enabled(
7824 spec,
7825 runtime,
7826 registry,
7827 process_liveness,
7828 snapshot,
7829 child,
7830 enabled,
7831 &runtime.failure_context(false),
7832 )
7833 .await;
7834 if !enabled {
7835 containment::terminate(&runtime.child_roster.containment).await;
7836 }
7837 if matches!(result, Ok(true)) {
7838 runtime.pending_activity_restart = false;
7839 if enabled {
7840 runtime.start_requester = requester;
7841 } else {
7842 runtime.start_requester = None;
7843 runtime.record_activity_transition(
7844 "subc.module_disabled",
7845 &spec.module_id,
7846 requester,
7847 );
7848 }
7849 } else {
7850 runtime.start_requester = previous_requester;
7851 }
7852 let _ = reply.send(result);
7853 true
7854 }
7855 SupervisorCommand::UpdateConfiguration {
7856 spec: next_spec,
7857 health,
7858 drain_timeout_ms,
7859 reply,
7860 } => {
7861 if let Some(handle) = &runtime.supervisor_handle {
7862 handle.apply_identity_configuration(&next_spec);
7863 }
7864 *spec = next_spec;
7865 let _ = update_snapshot(snapshot, Some(&spec.module_id), |state| {
7866 state.configuration_updated_since_spawn = true;
7867 });
7868 let health_changed = runtime.health != health;
7869 runtime.health = health;
7870 if health_changed {
7873 let _ = update_snapshot(snapshot, Some(&spec.module_id), |state| {
7874 state.health = ModuleHealthStatus::default();
7875 });
7876 }
7877 runtime.drain_timeout = drain_timeout_ms
7878 .map(Duration::from_millis)
7879 .unwrap_or(runtime.default_drain_timeout);
7880 *runtime
7881 .effective_drain_timeout
7882 .lock()
7883 .unwrap_or_else(|poisoned| poisoned.into_inner()) = runtime.drain_timeout;
7884 let _ = reply.send(());
7885 true
7886 }
7887 SupervisorCommand::Swap {
7888 requester,
7889 ready_timeout,
7890 reply,
7891 } => {
7892 if spec.carry_containment
7893 || containment::current(&runtime.child_roster.containment).is_some()
7894 {
7895 let _ = reply.send(Err(SuperviseError::InvalidSpec {
7896 reason: "a carried container cannot overlap a swap candidate".to_owned(),
7897 }));
7898 return true;
7899 }
7900 let mut swap_runtime = runtime.clone();
7901 swap_runtime.start_requester = requester;
7902 let before = lock_snapshot(snapshot)
7903 .ok()
7904 .map(|state| state.spawn_generation);
7905 let end = swap::run_swap(
7906 spec,
7907 &swap_runtime,
7908 registry,
7909 process_liveness,
7910 snapshot,
7911 child,
7912 commands,
7913 ready_timeout.unwrap_or(DEFAULT_SWAP_READY_TIMEOUT),
7914 reply,
7915 )
7916 .await;
7917 if before
7918 != lock_snapshot(snapshot)
7919 .ok()
7920 .map(|state| state.spawn_generation)
7921 {
7922 runtime.start_requester = swap_runtime.start_requester;
7923 }
7924 requeued.extend(end.requeue);
7925 true
7926 }
7927 }
7928}
7929
7930async fn restart_child(
7931 spec: &ModuleSpec,
7932 runtime: &SupervisorRuntimeConfig,
7933 registry: &Registry,
7934 process_liveness: &SupervisorProcessLiveness,
7935 snapshot: &SharedSnapshot,
7936 child: &mut Option<SupervisedChild>,
7937 drain_timeout: Duration,
7938) -> Result<(), SuperviseError> {
7939 if !lock_snapshot(snapshot)?.enabled {
7941 return Err(SuperviseError::Disabled {
7942 module_id: spec.module_id.clone(),
7943 });
7944 }
7945 let stop_notice = begin_forwarding_drain_with_timeout(
7946 spec,
7947 runtime,
7948 registry,
7949 snapshot,
7950 None,
7951 RouteCloseReason::Restart,
7952 drain_timeout,
7953 )
7954 .await?;
7955
7956 if child.is_some() {
7957 drain_optional_child(
7958 &spec.module_id,
7959 spec.protocol,
7960 stop_notice,
7961 registry,
7962 runtime.forwarding.as_deref(),
7963 snapshot,
7964 &runtime.terminal_ring,
7965 &runtime.spawn_events,
7966 child,
7967 drain_timeout,
7968 ModuleState::Restarting,
7969 Some(true),
7970 &FailureContext::operator(),
7971 )
7972 .await?;
7973 } else {
7974 update_snapshot(snapshot, Some(&spec.module_id), |state| {
7975 state.enabled = true;
7976 state.state = ModuleState::Restarting;
7977 clear_current_process_facts(state);
7978 })?;
7979 release_dead_registration(
7980 registry,
7981 runtime.forwarding.as_deref(),
7982 snapshot,
7983 &spec.module_id,
7984 &runtime.failure_context(false),
7985 )
7986 .await?;
7987 }
7988
7989 reset_restart_count(snapshot, &spec.module_id)?;
7990 process_liveness.track(spec.module_id.clone(), Arc::clone(snapshot));
7991 schedule_respawn(
7992 runtime,
7993 snapshot,
7994 &spec.module_id,
7995 runtime.restart_policy.backoff,
7996 RespawnKind::Spawn,
7997 false,
7998 )
7999}
8000
8001async fn reload_child(
8002 spec: &ModuleSpec,
8003 runtime: &SupervisorRuntimeConfig,
8004 registry: &Registry,
8005 process_liveness: &SupervisorProcessLiveness,
8006 snapshot: &SharedSnapshot,
8007 child: &mut Option<SupervisedChild>,
8008) -> Result<(), SuperviseError> {
8009 if !lock_snapshot(snapshot)?.enabled {
8011 return Err(SuperviseError::Disabled {
8012 module_id: spec.module_id.clone(),
8013 });
8014 }
8015 let stop_notice = begin_forwarding_drain(
8016 spec,
8017 runtime,
8018 registry,
8019 snapshot,
8020 Some(true),
8021 RouteCloseReason::Reload,
8022 )
8023 .await?;
8024
8025 if child.is_some() {
8026 drain_optional_child(
8027 &spec.module_id,
8028 spec.protocol,
8029 stop_notice,
8030 registry,
8031 runtime.forwarding.as_deref(),
8032 snapshot,
8033 &runtime.terminal_ring,
8034 &runtime.spawn_events,
8035 child,
8036 runtime.drain_timeout,
8037 ModuleState::Restarting,
8038 Some(true),
8039 &FailureContext::operator(),
8040 )
8041 .await?;
8042 } else {
8043 update_snapshot(snapshot, Some(&spec.module_id), |state| {
8044 state.enabled = true;
8045 state.state = ModuleState::Restarting;
8046 clear_current_process_facts(state);
8047 })?;
8048 release_dead_registration(
8049 registry,
8050 runtime.forwarding.as_deref(),
8051 snapshot,
8052 &spec.module_id,
8053 &runtime.failure_context(false),
8054 )
8055 .await?;
8056 }
8057
8058 reset_restart_count(snapshot, &spec.module_id)?;
8059 process_liveness.track(spec.module_id.clone(), Arc::clone(snapshot));
8060 schedule_respawn(
8061 runtime,
8062 snapshot,
8063 &spec.module_id,
8064 runtime.restart_policy.backoff,
8065 RespawnKind::Reload,
8066 false,
8067 )
8068}
8069
8070async fn finish_reload_child(
8071 spec: &ModuleSpec,
8072 runtime: &SupervisorRuntimeConfig,
8073 registry: &Registry,
8074 process_liveness: &SupervisorProcessLiveness,
8075 snapshot: &SharedSnapshot,
8076 child: &mut Option<SupervisedChild>,
8077) -> Result<(), SuperviseError> {
8078 process_liveness.track(spec.module_id.clone(), Arc::clone(snapshot));
8079 let next_child = match spawn_and_mark_running(spec, runtime, snapshot) {
8080 Ok(next_child) => next_child,
8081 Err(err) => {
8082 return handle_reload_spawn_failure(
8083 spec,
8084 runtime,
8085 process_liveness,
8086 snapshot,
8087 child,
8088 format!("new child failed to spawn: {err}"),
8089 )
8090 .await;
8091 }
8092 };
8093 *child = Some(next_child);
8094
8095 let wait_outcome = {
8096 let active_child = child.as_mut().expect("new reload child was just stored");
8097 wait_for_registration_after_reload(
8098 registry,
8099 &spec.module_id,
8100 snapshot,
8101 active_child,
8102 REGISTRY_RELEASE_TIMEOUT,
8103 )
8104 .await?
8105 };
8106
8107 match wait_outcome {
8108 RegistrationWaitOutcome::Registered => {
8109 child
8110 .as_mut()
8111 .expect("registered reload child")
8112 .registration_deadline = None;
8113 debug!(module_id = %spec.module_id, "supervised module reloaded and registered");
8114 Ok(())
8115 }
8116 RegistrationWaitOutcome::Exited(exit_report) => {
8117 if let Some(active_child) = child.as_mut() {
8118 active_child.drain_stderr(&spec.module_id).await;
8119 }
8120 let mut exited_child = child.take().expect("exited reload child is still stored");
8123 #[cfg(test)]
8124 if let Some(gate) = &runtime.test_reload_exit_record_gate {
8125 gate.reached.notify_one();
8126 gate.resume.notified().await;
8127 }
8128 let result = handle_reload_child_registration_failure(
8129 spec,
8130 runtime,
8131 registry,
8132 process_liveness,
8133 snapshot,
8134 child,
8135 ReloadRegistrationFailure {
8136 exit_report: registration_failure_exit_report(exit_report),
8137 reason: exited_child
8138 .spawn_failure
8139 .clone()
8140 .unwrap_or_else(|| "new child exited before registering".to_string()),
8141 },
8142 )
8143 .await;
8144 exited_child
8145 .finish_exit(false)
8146 .await
8147 .map_err(|source| SuperviseError::Wait {
8148 module_id: spec.module_id.clone(),
8149 source,
8150 })?;
8151 result
8152 }
8153 RegistrationWaitOutcome::TimedOut => {
8154 let mut timed_out_child = child
8155 .take()
8156 .expect("timed-out reload child is still running");
8157 timed_out_child
8158 .start_kill()
8159 .map_err(|source| SuperviseError::Kill {
8160 module_id: spec.module_id.clone(),
8161 source,
8162 })?;
8163 let status = timed_out_child
8164 .wait()
8165 .await
8166 .map_err(|source| SuperviseError::Wait {
8167 module_id: spec.module_id.clone(),
8168 source,
8169 })?;
8170 timed_out_child.drain_stderr(&spec.module_id).await;
8171 let result = handle_reload_child_registration_failure(
8172 spec,
8173 runtime,
8174 registry,
8175 process_liveness,
8176 snapshot,
8177 child,
8178 ReloadRegistrationFailure {
8179 exit_report: registration_failure_exit_report(classify_reaped_child_exit(
8180 snapshot,
8181 &timed_out_child,
8182 &status,
8183 )),
8184 reason: format!(
8185 "new child did not register within {:?}",
8186 REGISTRY_RELEASE_TIMEOUT
8187 ),
8188 },
8189 )
8190 .await;
8191 timed_out_child
8192 .finish_exit(false)
8193 .await
8194 .map_err(|source| SuperviseError::Wait {
8195 module_id: spec.module_id.clone(),
8196 source,
8197 })?;
8198 result
8199 }
8200 }
8201}
8202
8203#[allow(clippy::too_many_arguments)]
8204async fn set_child_enabled(
8205 spec: &ModuleSpec,
8206 runtime: &SupervisorRuntimeConfig,
8207 registry: &Registry,
8208 process_liveness: &SupervisorProcessLiveness,
8209 snapshot: &SharedSnapshot,
8210 child: &mut Option<SupervisedChild>,
8211 enabled: bool,
8212 failure: &FailureContext,
8213) -> Result<bool, SuperviseError> {
8214 let (current_enabled, current_state, respawn_pending) = {
8215 let state = lock_snapshot(snapshot)?;
8216 (state.enabled, state.state, state.respawn_pending)
8217 };
8218 let revive_terminal = enabled
8226 && current_enabled
8227 && child.is_none()
8228 && (matches!(current_state, ModuleState::Failed | ModuleState::Stopped)
8229 || (current_state == ModuleState::Restarting && !respawn_pending));
8230 if current_enabled == enabled && !revive_terminal {
8231 return Ok(false);
8232 }
8233
8234 if enabled {
8235 update_snapshot(snapshot, Some(&spec.module_id), |state| {
8236 state.enabled = true;
8237 state.state = ModuleState::Starting;
8238 clear_current_process_facts(state);
8239 })?;
8240 #[cfg(test)]
8241 if runtime.test_seed_stale_facts_before_enable_spawn {
8242 update_snapshot(snapshot, Some(&spec.module_id), |state| {
8243 state.process_alive = true;
8244 state.pid = Some(41);
8245 state.spawned_at_ms = Some(42);
8246 state.spawned_from = Some(PathBuf::from("/spawned/module"));
8247 state.spawned_file_identity = Some(SpawnedFileIdentity {
8248 device: 43,
8249 inode: 44,
8250 #[cfg(windows)]
8251 windows_image: None,
8252 });
8253 })?;
8254 }
8255 release_dead_registration(
8256 registry,
8257 runtime.forwarding.as_deref(),
8258 snapshot,
8259 &spec.module_id,
8260 failure,
8261 )
8262 .await?;
8263 reset_restart_count(snapshot, &spec.module_id)?;
8264 process_liveness.track(spec.module_id.clone(), Arc::clone(snapshot));
8265 let next_child = match spawn_and_mark_running(spec, runtime, snapshot) {
8266 Ok(next_child) => next_child,
8267 Err(err) => {
8268 fail_snapshot(snapshot, Some(&spec.module_id), None, failure);
8269 process_liveness.untrack_if_current(&spec.module_id, snapshot);
8270 return Err(err);
8271 }
8272 };
8273 *child = Some(next_child);
8274 debug!(module_id = %spec.module_id, "supervised module enabled");
8275 Ok(true)
8276 } else {
8277 let stop_notice = begin_forwarding_drain_if_configured(
8278 spec,
8279 runtime,
8280 registry,
8281 snapshot,
8282 Some(false),
8283 RouteCloseReason::Disable,
8284 )
8285 .await?;
8286 drain_optional_child(
8287 &spec.module_id,
8288 spec.protocol,
8289 stop_notice,
8290 registry,
8291 runtime.forwarding.as_deref(),
8292 snapshot,
8293 &runtime.terminal_ring,
8294 &runtime.spawn_events,
8295 child,
8296 runtime.drain_timeout,
8297 ModuleState::Disabled,
8298 Some(false),
8299 failure,
8300 )
8301 .await?;
8302 debug!(module_id = %spec.module_id, "supervised module disabled");
8303 Ok(true)
8304 }
8305}
8306
8307#[allow(clippy::too_many_arguments)]
8308async fn on_child_exit(
8309 spec: &ModuleSpec,
8310 policy: RestartPolicy,
8311 registry: &Registry,
8312 snapshot: &SharedSnapshot,
8313 terminal_ring: &Arc<Mutex<TerminalRing>>,
8314 spawn_events: &SpawnEventFeed,
8315 roster: &ChildRoster,
8316 exit_report: ExitReport,
8317 daemon_initiated: bool,
8318) -> NextAction {
8319 if roster.is_closed() {
8325 return on_child_exit_during_daemon_shutdown(
8326 spec,
8327 registry,
8328 snapshot,
8329 terminal_ring,
8330 spawn_events,
8331 exit_report,
8332 )
8333 .await;
8334 }
8335 let unrequested_clean_exit_of_protocol_none = exit_report.kind == ExitKind::Clean
8351 && running_protocol(spec, snapshot) == ModuleProtocol::None;
8352 match exit_report.kind {
8353 ExitKind::Clean if !unrequested_clean_exit_of_protocol_none => {
8354 info!(
8355 module_id = %spec.module_id,
8356 exit_code = ?exit_report.code,
8357 exit_signal = ?exit_report.signal,
8358 "supervised module exited cleanly"
8359 );
8360 if let Err(err) = publish_after_terminal_record(
8361 &spec.module_id,
8362 || {
8363 record_terminal(
8364 &spec.module_id,
8365 terminal_ring,
8366 spawn_events,
8367 &exit_report,
8368 TerminalDisposition::Stopped,
8369 )
8370 },
8371 || {
8372 update_snapshot(snapshot, Some(&spec.module_id), |state| {
8373 state.state = ModuleState::Stopped;
8374 clear_current_process_facts(state);
8375 state.last_exit = Some(exit_report.clone());
8376 })
8377 },
8378 ) {
8379 error!(module_id = %spec.module_id, error = %err, "failed to publish clean module exit");
8380 }
8381 let registration_released = match wait_for_registration_release(
8382 registry,
8383 &spec.module_id,
8384 REGISTRY_RELEASE_TIMEOUT,
8385 )
8386 .await
8387 {
8388 Ok(()) => true,
8389 Err(err) => {
8390 warn!(module_id = %spec.module_id, error = %err, "registration still active after clean exit");
8391 false
8392 }
8393 };
8394 NextAction::Stop {
8395 registration_released,
8396 }
8397 }
8398 ExitKind::Clean | ExitKind::Crash => {
8399 if unrequested_clean_exit_of_protocol_none {
8400 warn!(
8401 module_id = %spec.module_id,
8402 exit_code = ?exit_report.code,
8403 exit_signal = ?exit_report.signal,
8404 "protocol-none module exited cleanly without a stop request; handling it as a crash"
8405 );
8406 } else {
8407 warn!(
8408 module_id = %spec.module_id,
8409 exit_code = ?exit_report.code,
8410 exit_signal = ?exit_report.signal,
8411 "supervised module exited abnormally (crash)"
8412 );
8413 }
8414 let mut restart_schedule = None;
8415 let mut disposition = TerminalDisposition::Disabled;
8416 let mut final_state = ModuleState::Disabled;
8417 let mut disposition_detail = lock_snapshot(snapshot)
8421 .ok()
8422 .and_then(|mut state| state.spawn_failure.take());
8423 let now = Instant::now();
8424 if let Err(err) = update_snapshot(snapshot, Some(&spec.module_id), |state| {
8425 if state.enabled {
8426 if let Some(schedule) = state.next_crash_restart(&policy, now) {
8427 state.respawn_pending = true;
8433 final_state = ModuleState::Restarting;
8434 restart_schedule = Some(schedule);
8435 disposition = TerminalDisposition::Restarting;
8436 } else {
8437 final_state = ModuleState::Failed;
8438 disposition = TerminalDisposition::Failed;
8439 let budget = policy.budget_exhausted_detail();
8440 disposition_detail =
8441 Some(disposition_detail.take().map_or_else(
8442 || budget.clone(),
8443 |cause| format!("{cause}; {budget}"),
8444 ));
8445 }
8446 } else {
8447 disposition = TerminalDisposition::Disabled;
8448 }
8449 }) {
8450 error!(module_id = %spec.module_id, error = %err, "failed to record crashed module exit");
8451 return NextAction::Stop {
8452 registration_released: false,
8453 };
8454 }
8455 if disposition == TerminalDisposition::Failed {
8456 error!(
8461 module_id = %spec.module_id,
8462 max_restarts = policy.max_restarts,
8463 window_secs = policy.window.as_secs(),
8464 "module stopped: {}",
8465 policy.budget_exhausted_detail()
8466 );
8467 }
8468 if let Err(err) = publish_after_terminal_record(
8469 &spec.module_id,
8470 || {
8471 record_terminal_with_detail(
8472 &spec.module_id,
8473 terminal_ring,
8474 spawn_events,
8475 &exit_report,
8476 disposition,
8477 disposition_detail,
8478 )
8479 },
8480 || {
8481 update_snapshot(snapshot, Some(&spec.module_id), |state| {
8482 state.state = final_state;
8483 clear_current_process_facts(state);
8484 if final_state == ModuleState::Failed
8485 && daemon_initiated
8486 && !roster.is_closed()
8487 {
8488 state.failed_retry.schedule(&spec.module_id);
8489 }
8490 state.last_exit = Some(exit_report.clone());
8491 })
8492 },
8493 ) {
8494 error!(module_id = %spec.module_id, error = %err, "failed to publish crashed module exit");
8495 }
8496
8497 if let Some(schedule) = restart_schedule {
8498 NextAction::Restart {
8499 schedule: Some(schedule),
8500 }
8501 } else {
8502 let registration_released = match wait_for_registration_release(
8503 registry,
8504 &spec.module_id,
8505 REGISTRY_RELEASE_TIMEOUT,
8506 )
8507 .await
8508 {
8509 Ok(()) => true,
8510 Err(err) => {
8511 warn!(module_id = %spec.module_id, error = %err, "registration still active after failed module");
8512 false
8513 }
8514 };
8515 NextAction::Stop {
8516 registration_released,
8517 }
8518 }
8519 }
8520 ExitKind::DeliberateSeverance => {
8521 warn!(
8522 module_id = %spec.module_id,
8523 exit_code = ?exit_report.code,
8524 exit_signal = ?exit_report.signal,
8525 "supervised module exited after deliberate connection severance"
8526 );
8527 let should_restart = match lock_snapshot(snapshot) {
8528 Ok(state) => state.enabled,
8529 Err(err) => {
8530 error!(module_id = %spec.module_id, error = %err, "failed to read deliberately severed module state");
8531 return NextAction::Stop {
8532 registration_released: false,
8533 };
8534 }
8535 };
8536 let final_state = if should_restart {
8537 ModuleState::Restarting
8538 } else {
8539 ModuleState::Disabled
8540 };
8541 if let Err(err) = publish_after_terminal_record(
8542 &spec.module_id,
8543 || {
8544 record_terminal(
8545 &spec.module_id,
8546 terminal_ring,
8547 spawn_events,
8548 &exit_report,
8549 terminal_disposition(final_state),
8550 )
8551 },
8552 || {
8553 update_snapshot(snapshot, Some(&spec.module_id), |state| {
8554 state.state = final_state;
8555 clear_current_process_facts(state);
8556 state.last_exit = Some(exit_report.clone());
8557 state.lifetime_restarts += 1;
8558 })
8559 },
8560 ) {
8561 error!(module_id = %spec.module_id, error = %err, "failed to publish deliberately severed module exit");
8562 return NextAction::Stop {
8563 registration_released: false,
8564 };
8565 }
8566
8567 if should_restart {
8568 NextAction::Restart { schedule: None }
8569 } else {
8570 let registration_released = match wait_for_registration_release(
8571 registry,
8572 &spec.module_id,
8573 REGISTRY_RELEASE_TIMEOUT,
8574 )
8575 .await
8576 {
8577 Ok(()) => true,
8578 Err(err) => {
8579 warn!(module_id = %spec.module_id, error = %err, "registration still active after deliberately severed module exit");
8580 false
8581 }
8582 };
8583 NextAction::Stop {
8584 registration_released,
8585 }
8586 }
8587 }
8588 }
8589}
8590
8591async fn on_child_exit_during_daemon_shutdown(
8592 spec: &ModuleSpec,
8593 registry: &Registry,
8594 snapshot: &SharedSnapshot,
8595 terminal_ring: &Arc<Mutex<TerminalRing>>,
8596 spawn_events: &SpawnEventFeed,
8597 exit_report: ExitReport,
8598) -> NextAction {
8599 info!(
8600 module_id = %spec.module_id,
8601 exit_code = ?exit_report.code,
8602 exit_signal = ?exit_report.signal,
8603 exit_kind = ?exit_report.kind,
8604 "supervised module exited during daemon shutdown; not restarting it"
8605 );
8606 if let Err(err) = publish_after_terminal_record(
8607 &spec.module_id,
8608 || {
8609 record_terminal(
8610 &spec.module_id,
8611 terminal_ring,
8612 spawn_events,
8613 &exit_report,
8614 TerminalDisposition::DaemonShutdown,
8615 )
8616 },
8617 || {
8618 update_snapshot(snapshot, Some(&spec.module_id), |state| {
8619 state.state = ModuleState::Stopped;
8620 clear_current_process_facts(state);
8621 state.last_exit = Some(exit_report.clone());
8622 })
8623 },
8624 ) {
8625 error!(module_id = %spec.module_id, error = %err, "failed to publish module exit during daemon shutdown");
8626 }
8627 let registration_released =
8628 wait_for_registration_release(registry, &spec.module_id, REGISTRY_RELEASE_TIMEOUT)
8629 .await
8630 .is_ok();
8631 NextAction::Stop {
8632 registration_released,
8633 }
8634}
8635
8636fn publish_after_terminal_record<T>(
8641 module_id: &str,
8642 record: impl FnOnce(),
8643 publish: impl FnOnce() -> T,
8644) -> T {
8645 let recorded = std::panic::catch_unwind(std::panic::AssertUnwindSafe(record));
8646 let published = publish();
8647 if let Err(panic) = recorded {
8648 error!(
8649 module_id,
8650 "terminal recording panicked; published final state before resuming unwind"
8651 );
8652 std::panic::resume_unwind(panic);
8653 }
8654 published
8655}
8656
8657fn fail_after_wait_error(
8658 module_id: &str,
8659 snapshot: &SharedSnapshot,
8660 terminal_ring: &Arc<Mutex<TerminalRing>>,
8661 spawn_events: &SpawnEventFeed,
8662 detail: String,
8663 failure: &FailureContext,
8664) {
8665 let report = wait_error_exit_report();
8666 publish_after_terminal_record(
8667 module_id,
8668 || {
8669 record_terminal_with_detail(
8670 module_id,
8671 terminal_ring,
8672 spawn_events,
8673 &report,
8674 TerminalDisposition::Failed,
8675 Some(detail),
8676 )
8677 },
8678 || fail_snapshot(snapshot, Some(module_id), Some(report.clone()), failure),
8679 );
8680}
8681
8682#[cfg(test)]
8683fn record_wait_error_terminal(
8684 module_id: &str,
8685 terminal_ring: &Arc<Mutex<TerminalRing>>,
8686 spawn_events: &SpawnEventFeed,
8687) {
8688 record_terminal(
8689 module_id,
8690 terminal_ring,
8691 spawn_events,
8692 &wait_error_exit_report(),
8693 TerminalDisposition::Failed,
8694 );
8695}
8696
8697fn record_terminal(
8698 module_id: &str,
8699 terminal_ring: &Arc<Mutex<TerminalRing>>,
8700 spawn_events: &SpawnEventFeed,
8701 exit_report: &ExitReport,
8702 disposition: TerminalDisposition,
8703) {
8704 record_terminal_with_detail(
8705 module_id,
8706 terminal_ring,
8707 spawn_events,
8708 exit_report,
8709 disposition,
8710 None,
8711 );
8712}
8713
8714fn durable_terminal_history_of(
8718 terminal_ring: &Mutex<TerminalRing>,
8719 module_id: &str,
8720) -> subc_control::TerminalHistory {
8721 let read = terminal_ring
8722 .lock()
8723 .unwrap_or_else(|p| p.into_inner())
8724 .capture_durable_history();
8725 read.read(module_id)
8726}
8727
8728fn record_terminal_with_detail(
8729 module_id: &str,
8730 terminal_ring: &Arc<Mutex<TerminalRing>>,
8731 spawn_events: &SpawnEventFeed,
8732 exit_report: &ExitReport,
8733 disposition: TerminalDisposition,
8734 disposition_detail: Option<String>,
8735) {
8736 #[cfg(test)]
8737 terminal_history_tests::before_terminal_record(spawn_events);
8738 spawn_events.emit_exited(module_id, exit_report.code, exit_report.signal);
8739 let record = TerminalRecord {
8740 exit_code: exit_report.code,
8741 exit_signal: exit_report.signal,
8742 at_ms: exit_report.at_ms,
8743 disposition,
8744 exit_kind: exit_report.kind.into(),
8745 disposition_detail,
8746 };
8747 terminal_ring
8748 .lock()
8749 .unwrap_or_else(|poisoned| poisoned.into_inner())
8750 .record_exit(module_id, record);
8751}
8752
8753fn untrack_if_registration_released(
8754 process_liveness: &SupervisorProcessLiveness,
8755 registry: &Registry,
8756 module_id: &str,
8757 snapshot: &SharedSnapshot,
8758) {
8759 match registry.get_module(module_id) {
8760 Ok(None) => process_liveness.untrack_if_current(module_id, snapshot),
8761 Ok(Some(_)) => {}
8762 Err(err) => {
8763 warn!(module_id, error = %err, "could not determine whether supervisor liveness can be untracked");
8764 }
8765 }
8766}
8767
8768#[cfg(test)]
8782fn apply_wire_spawn_args(
8783 command: &mut Command,
8784 spec: &ModuleSpec,
8785 connection_file_path: Option<&std::path::Path>,
8786 handle: Option<&SupervisorHandle>,
8787) -> Result<Option<NonceHandoff>, SuperviseError> {
8788 apply_wire_spawn_args_for_role(
8789 command,
8790 spec,
8791 connection_file_path,
8792 handle,
8793 SpawnRole::Plain,
8794 )
8795}
8796
8797#[cfg(unix)]
8802type NonceHandoff = subc_os::LaunchNonceHandoff;
8803#[cfg(windows)]
8804type NonceHandoff = subc_os::LaunchNoncePipeHandoff;
8805#[cfg(not(any(unix, windows)))]
8806type NonceHandoff = std::convert::Infallible;
8807
8808fn apply_wire_spawn_args_for_role(
8829 command: &mut Command,
8830 spec: &ModuleSpec,
8831 connection_file_path: Option<&std::path::Path>,
8832 handle: Option<&SupervisorHandle>,
8833 role: SpawnRole,
8834) -> Result<Option<NonceHandoff>, SuperviseError> {
8835 command.env(SUBC_MODULE_ID_ENV, &spec.module_id);
8836 command.env_remove(subc_os::LAUNCH_NONCE_FD_ENV);
8841 #[cfg(windows)]
8842 {
8843 command.env_remove(subc_os::LAUNCH_NONCE_PIPE_ENV);
8844 command.env_remove(subc_os::LAUNCH_NONCE_PIPE_FALLBACK_ENV);
8845 }
8846 command.env_remove(SUBC_LAUNCH_NONCE_ENV);
8848 if spec.protocol == ModuleProtocol::None {
8849 return Ok(None);
8850 }
8851 if let Some(connection_file_path) = connection_file_path {
8852 command.arg(SUBC_ARG).arg(connection_file_path);
8853 }
8854
8855 let nonce = generate_launch_nonce()?;
8859 if let Some(handle) = handle {
8860 match role {
8861 SpawnRole::Plain => {
8862 handle.set_spawn_nonce(&spec.module_id, nonce.clone());
8863 if spec.reserved {
8864 handle.set_reserved_nonce(&spec.module_id, nonce.clone());
8865 }
8866 }
8867 SpawnRole::SwapCandidate => handle.open_swap(&spec.module_id, nonce.clone()),
8868 }
8869 }
8870 #[cfg(unix)]
8871 let handoff = {
8872 let handoff =
8873 subc_os::LaunchNonceHandoff::new(&nonce).map_err(|source| SuperviseError::Spawn {
8874 program: spec.program.clone(),
8875 source,
8876 cgroup_path: None,
8877 })?;
8878 command.env(subc_os::LAUNCH_NONCE_FD_ENV, handoff.fd_env_value());
8879 Some(handoff)
8880 };
8881 #[cfg(windows)]
8882 let handoff = prepare_windows_nonce_pipe(command, spec, &nonce);
8883 #[cfg(not(any(unix, windows)))]
8884 let handoff = None;
8885 #[cfg(not(unix))]
8889 command.env(SUBC_LAUNCH_NONCE_ENV, nonce);
8890 Ok(handoff)
8891}
8892
8893fn apply_child_env(command: &mut Command, spec: &ModuleSpec) {
8894 command.env_remove(CK_LOG_ENV);
8895 command.env_remove(SUBC_SPAWN_ROLE_ENV);
8902 for (key, value) in &spec.env {
8903 if matches!(
8907 key.as_str(),
8908 CAPTURE_MAX_FILE_MB_ENV | CAPTURE_KEEP_ENV | CAPTURE_MAX_AGE_DAYS_ENV
8909 ) || key == SUBC_SPAWN_ROLE_ENV
8910 {
8911 continue;
8912 }
8913 command.env(key, value);
8914 }
8915}
8916
8917#[cfg(windows)]
8918fn prepare_windows_nonce_pipe(
8919 command: &mut Command,
8920 spec: &ModuleSpec,
8921 nonce: &str,
8922) -> Option<NonceHandoff> {
8923 match subc_os::LaunchNoncePipeHandoff::new(nonce) {
8924 Ok(handoff) => {
8925 command.env(subc_os::LAUNCH_NONCE_PIPE_ENV, handoff.name());
8926 command.env(subc_os::LAUNCH_NONCE_PIPE_FALLBACK_ENV, "env");
8931 Some(handoff)
8932 }
8933 Err(error) => {
8934 warn!(module_id = %spec.module_id, %error, "could not prepare launch nonce pipe; retaining the environment handoff");
8935 None
8936 }
8937 }
8938}
8939
8940#[cfg(windows)]
8941fn start_windows_nonce_pipe(
8942 handoff: Option<NonceHandoff>,
8943 pid: u32,
8944 spec: &ModuleSpec,
8945) -> Option<subc_os::LaunchNoncePipeDelivery> {
8946 handoff.and_then(|handoff| match handoff.serve(pid, REGISTRY_RELEASE_TIMEOUT) {
8950 Ok(delivery) => Some(delivery),
8951 Err(error) => {
8952 warn!(module_id = %spec.module_id, %error, "could not serve launch nonce pipe; retaining the environment handoff");
8953 None
8954 }
8955 })
8956}
8957
8958fn observed_launch_nonce_source(snapshot: &SupervisorSnapshot) -> Option<String> {
8959 #[cfg(windows)]
8960 if snapshot.process_alive && snapshot.spawned_protocol == Some(ModuleProtocol::Subc) {
8961 return Some(
8962 if snapshot
8963 .pipe_consumed
8964 .as_ref()
8965 .is_some_and(|consumed| consumed.get().is_some())
8966 {
8967 "pipe"
8968 } else {
8969 "env"
8970 }
8971 .to_owned(),
8972 );
8973 }
8974 let _ = snapshot;
8975 None
8976}
8977
8978#[derive(Debug, Clone, Copy, PartialEq, Eq)]
8981enum SpawnRole {
8982 Plain,
8983 SwapCandidate,
8984}
8985
8986fn apply_spawn_role(command: &mut Command, role: SpawnRole) {
8989 if role == SpawnRole::SwapCandidate {
8990 command.env(SUBC_SPAWN_ROLE_ENV, SPAWN_ROLE_SWAP_CANDIDATE);
8991 }
8992}
8993
8994fn previous_instance_error(
8995 spec: &ModuleSpec,
8996 pid: u32,
8997 reason: crate::live_children::RetainedReason,
8998) -> SuperviseError {
8999 SuperviseError::Spawn {
9000 program: spec.program.clone(),
9001 source: io::Error::new(io::ErrorKind::WouldBlock, reason.waiting_message(pid)),
9002 cgroup_path: None,
9003 }
9004}
9005
9006fn carries_containment(configured: bool, has_container: bool) -> bool {
9007 configured || has_container
9008}
9009
9010fn recovery_block_error(spec: &ModuleSpec, roster: &ChildRoster) -> Option<SuperviseError> {
9011 if !carries_containment(
9012 spec.carry_containment,
9013 containment::current(&roster.containment).is_some(),
9014 ) {
9015 return None;
9016 }
9017 Some(SuperviseError::Spawn {
9018 program: spec.program.clone(),
9019 source: io::Error::new(io::ErrorKind::WouldBlock, roster.recovery_block()?),
9020 cgroup_path: None,
9021 })
9022}
9023
9024fn spawn_child(
9025 spec: &ModuleSpec,
9026 connection_file_path: Option<&std::path::Path>,
9027 handle: Option<&SupervisorHandle>,
9028 ring: &Arc<Mutex<StderrRing>>,
9029 capture_logs_dir: Option<&std::path::Path>,
9030 roster: &ChildRoster,
9031 #[cfg(target_os = "linux")] cgroup_placement: Option<&subc_cgroup::Placement>,
9032) -> Result<SupervisedChild, SuperviseError> {
9033 #[cfg(all(test, unix))]
9034 if let Some(source) = containment_tests::join_failure(spec, roster) {
9035 containment::terminate_blocking(&roster.containment);
9036 return Err(SuperviseError::Spawn {
9037 program: spec.program.clone(),
9038 source,
9039 cgroup_path: None,
9040 });
9041 }
9042 let result = spawn_child_in_slot(
9043 spec,
9044 connection_file_path,
9045 handle,
9046 ring,
9047 capture_logs_dir,
9048 roster,
9049 #[cfg(target_os = "linux")]
9050 cgroup_placement,
9051 SpawnRole::Plain,
9052 false,
9053 );
9054 if result.is_err() {
9055 if roster.admission_pending(&spec.module_id) {
9056 containment::terminate_deferred(&roster.containment);
9057 } else {
9058 containment::terminate_blocking(&roster.containment);
9059 }
9060 }
9061 result
9062}
9063
9064#[allow(clippy::too_many_arguments)]
9077fn spawn_child_in_slot(
9078 spec: &ModuleSpec,
9079 connection_file_path: Option<&std::path::Path>,
9080 handle: Option<&SupervisorHandle>,
9081 ring: &Arc<Mutex<StderrRing>>,
9082 capture_logs_dir: Option<&std::path::Path>,
9083 roster: &ChildRoster,
9084 #[cfg(target_os = "linux")] cgroup_placement: Option<&subc_cgroup::Placement>,
9085 role: SpawnRole,
9086 alternate_slot: bool,
9087) -> Result<SupervisedChild, SuperviseError> {
9088 if let Some(error) = recovery_block_error(spec, roster) {
9089 return Err(error);
9090 }
9091 if let Some((pid, reason)) = roster.previous_instance(&spec.module_id) {
9092 return Err(previous_instance_error(spec, pid, reason));
9093 }
9094 if roster.is_closed() {
9095 return Err(SuperviseError::Spawn {
9096 program: spec.program.clone(),
9097 source: io::Error::other("the daemon is shutting down; not starting a new process"),
9098 cgroup_path: None,
9099 });
9100 }
9101 let carried = containment::current(&roster.containment);
9102 let carrying = carries_containment(spec.carry_containment, carried.is_some());
9103 if role != SpawnRole::Plain && carrying {
9104 return Err(SuperviseError::InvalidSpec {
9105 reason: "a carried container cannot overlap a swap candidate".to_owned(),
9106 });
9107 }
9108 #[cfg(unix)]
9109 let launch_identity = if carrying && carried.is_none() {
9110 Some(
9111 containment::LaunchIdentity::capture().map_err(|source| SuperviseError::Spawn {
9112 program: spec.program.clone(),
9113 source,
9114 cgroup_path: None,
9115 })?,
9116 )
9117 } else {
9118 None
9119 };
9120 #[cfg(target_os = "linux")]
9121 let cgroup_placement = match &carried {
9122 Some(container) => container.cgroup().map(|(placement, _)| placement),
9123 None => cgroup_placement,
9124 };
9125 #[cfg(unix)]
9126 let group_target = if let Some(container) = carried
9127 .as_ref()
9128 .filter(|container| container.group().is_some())
9129 {
9130 containment::GroupTarget::Pinned(Arc::clone(container))
9131 } else {
9132 #[cfg(target_os = "linux")]
9133 let cgroup_container = carrying && cgroup_placement.is_some();
9134 #[cfg(not(target_os = "linux"))]
9135 let cgroup_container = false;
9136 if cgroup_container {
9137 containment::GroupTarget::NoGroup
9138 } else {
9139 containment::GroupTarget::OwnLeader
9140 }
9141 };
9142 #[cfg(target_os = "linux")]
9143 let cgroup_name = {
9144 if let Some((_, name)) = carried.as_ref().and_then(|container| container.cgroup()) {
9149 name.to_owned()
9150 } else if cgroup_placement.is_none() {
9151 swap::cgroup_name(&spec.module_id, alternate_slot)
9152 } else {
9153 let nonce = generate_launch_nonce()?;
9154 let suffix = u128::from_str_radix(&nonce[..32], 16).expect("hex launch nonce");
9155 let mut end = spec.module_id.len().min(64);
9158 while !spec.module_id.is_char_boundary(end) {
9159 end -= 1;
9160 }
9161 format!(
9162 "{}_{suffix}",
9163 swap::cgroup_name(&spec.module_id[..end], alternate_slot)
9164 )
9165 }
9166 };
9167 #[cfg(not(target_os = "linux"))]
9168 let _ = alternate_slot;
9169 #[cfg(target_os = "macos")]
9170 let (mut command, privacy_exec, exec_ack) = privacy_command(spec, roster)?;
9171 #[cfg(windows)]
9172 let image_capture = windows_spawn_capture(spec).map_err(|source| SuperviseError::Spawn {
9173 program: spec.program.clone(),
9174 source,
9175 cgroup_path: None,
9176 })?;
9177 #[cfg(windows)]
9178 let mut command = Command::new(image_capture.path());
9179 #[cfg(not(any(target_os = "macos", windows)))]
9180 let mut command = Command::new(&spec.program);
9181 command.args(&spec.args);
9182 apply_child_env(&mut command, spec);
9212 apply_spawn_role(&mut command, role);
9213 let nonce_handoff =
9214 apply_wire_spawn_args_for_role(&mut command, spec, connection_file_path, handle, role)?;
9215
9216 #[cfg(target_os = "linux")]
9217 let cgroup_path = cgroup_placement
9218 .map(|placement| {
9219 if let Some(container) = &carried {
9220 let path = container
9221 .cgroup_path()
9222 .ok_or_else(|| io::Error::other("carried container has no cgroup path"))?;
9223 placement.existing_module_path(path)?.ok_or_else(|| {
9224 io::Error::new(
9225 io::ErrorKind::NotFound,
9226 "carried cgroup disappeared before replacement",
9227 )
9228 })
9229 } else {
9230 if carrying {
9231 containment::prepare_cgroup_path(placement, &cgroup_name)
9232 } else {
9233 placement.module_path(&cgroup_name)
9234 }
9235 }
9236 })
9237 .transpose()
9238 .map_err(|source| SuperviseError::Cgroup {
9239 module_id: spec.module_id.clone(),
9240 source,
9241 })?;
9242 #[cfg(not(target_os = "linux"))]
9243 let cgroup_path: Option<PathBuf> = None;
9244 #[cfg(unix)]
9245 let mut admission_gate = if carrying {
9246 Some(
9247 admission::Gate::install(&mut command, spec, roster).map_err(|source| {
9248 SuperviseError::Spawn {
9249 program: spec.program.clone(),
9250 source,
9251 cgroup_path: cgroup_path.clone(),
9252 }
9253 })?,
9254 )
9255 } else {
9256 None
9257 };
9258 #[cfg(target_os = "macos")]
9259 exec_ack.install(command.as_std_mut());
9260 #[cfg(target_os = "linux")]
9261 if let Some(path) = &cgroup_path {
9262 if let Err(error) = apply_cgroup_placement(&mut command, spec, path) {
9263 if let Some(placement) = cgroup_placement.filter(|_| carried.is_none()) {
9264 remove_module_cgroup(placement, &cgroup_name);
9265 }
9266 return Err(error);
9267 }
9268 }
9269
9270 let output_sink = if let Some(logs_dir) = capture_logs_dir {
9271 let path = logs_dir.join(format!("{}.stderr.log", spec.module_id));
9272 match ChildOutputSink::open(&path, capture_retention(spec)) {
9273 Ok(sink) => sink,
9274 Err(error) => {
9275 warn!(
9276 module_id = %spec.module_id,
9277 path = %path.display(),
9278 error = %error,
9279 "could not open child output capture file; forwarding to stderr"
9280 );
9281 ChildOutputSink::Stderr
9282 }
9283 }
9284 } else {
9285 ChildOutputSink::Stderr
9286 };
9287
9288 command.stdout(Stdio::piped());
9289 command.stderr(Stdio::piped());
9290 command.kill_on_drop(!carrying);
9291 #[cfg(unix)]
9308 command.process_group(
9309 carried
9310 .as_ref()
9311 .map(|container| container.verified_join_group())
9312 .transpose()
9313 .map_err(|source| SuperviseError::Spawn {
9314 program: spec.program.clone(),
9315 source,
9316 cgroup_path: cgroup_path.clone(),
9317 })?
9318 .flatten()
9319 .unwrap_or(0) as i32,
9320 );
9321 command.stdin(Stdio::null());
9322 #[cfg(unix)]
9323 if let Some(gate) = &mut admission_gate {
9324 gate.install_placement(&mut command);
9325 }
9326 #[cfg(unix)]
9330 if let Some(handoff) = nonce_handoff {
9331 handoff.install_last(command.as_std_mut());
9332 }
9333 #[cfg(not(any(unix, windows)))]
9334 let _ = nonce_handoff;
9335
9336 #[cfg(windows)]
9341 {
9342 subc_jobobject::ensure_root_job().map_err(|source| SuperviseError::Spawn {
9343 program: spec.program.clone(),
9344 source,
9345 cgroup_path: None,
9346 })?;
9347 subc_jobobject::suspend_on_create_async(&mut command);
9348 }
9349 let spawned_at = Instant::now();
9353 #[cfg(all(test, unix))]
9354 containment_tests::record_spawn_attempt(spec, &mut command);
9355 #[cfg(unix)]
9356 let spawned = match admission_gate {
9357 Some(gate) => gate
9358 .spawn(
9359 command,
9360 spec,
9361 roster,
9362 launch_identity,
9363 #[cfg(target_os = "linux")]
9364 cgroup_placement
9365 .zip(cgroup_path.as_ref())
9366 .map(|(placement, path)| {
9367 (placement.clone(), cgroup_name.clone(), path.clone())
9368 }),
9369 )
9370 .map(|(child, admitted)| (child, Some(admitted))),
9371 None => command.spawn().map(|child| (child, None)),
9372 };
9373 #[cfg(not(unix))]
9374 let spawned = command.spawn();
9375 let spawned = match spawned {
9376 Ok(child) => child,
9377 Err(source) => {
9378 if carried
9379 .as_ref()
9380 .and_then(|container| container.group())
9381 .is_some()
9382 {
9383 containment::terminate_blocking(&roster.containment);
9384 }
9385 #[cfg(target_os = "linux")]
9386 if let Some(placement) = cgroup_placement.filter(|_| carried.is_none()) {
9387 remove_module_cgroup(placement, &cgroup_name);
9388 }
9389 return Err(SuperviseError::Spawn {
9390 program: spec.program.clone(),
9391 source,
9392 cgroup_path,
9393 });
9394 }
9395 };
9396 #[cfg(unix)]
9397 let (mut child, mut admitted) = spawned;
9398 #[cfg(unix)]
9399 let group_target = admitted
9400 .as_ref()
9401 .map(|attempt| attempt.group_target())
9402 .unwrap_or(group_target);
9403 #[cfg(not(unix))]
9404 let mut child = spawned;
9405 #[cfg(test)]
9406 spawn_record_tests::after_create(&child, spec);
9407 #[cfg(target_os = "macos")]
9412 drop(exec_ack);
9413
9414 #[cfg(windows)]
9419 let windows_image = match image_capture.bind_tokio(&child) {
9420 Ok(image) => Arc::new(image),
9421 Err(source) => {
9422 kill_and_reap_rejected_child(&mut child);
9423 return Err(SuperviseError::Spawn {
9424 program: spec.program.clone(),
9425 source,
9426 cgroup_path: None,
9427 });
9428 }
9429 };
9430 #[cfg(windows)]
9433 let job = contain_spawned_child(&mut child, spec, carried.as_ref())?;
9434 let spawned_at_ms = unix_ms_now();
9435 #[cfg(not(windows))]
9436 let spawned_from = spec.program.clone();
9437 #[cfg(windows)]
9438 let spawned_from = windows_image.path().to_path_buf();
9439 #[cfg(not(windows))]
9440 let spawned_file_identity = spawned_file_identity(&spawned_from);
9441 #[cfg(windows)]
9442 let spawned_file_identity = Some(SpawnedFileIdentity {
9443 device: 0,
9444 inode: 0,
9445 windows_image: Some(Arc::clone(&windows_image)),
9446 });
9447 let pid = child.id().ok_or_else(|| SuperviseError::Spawn {
9448 program: spec.program.clone(),
9449 source: io::Error::other("spawned child exposed no live pid"),
9450 cgroup_path: cgroup_path.clone(),
9451 })?;
9452 #[cfg(windows)]
9453 let nonce_delivery = start_windows_nonce_pipe(nonce_handoff, pid, spec);
9454 #[cfg(not(windows))]
9455 let process_start_time = admitted
9456 .as_ref()
9457 .map(|admitted| admitted.birth)
9458 .or_else(|| crate::provenance::process_start_time(pid));
9459 #[cfg(windows)]
9460 let process_start_time = Some(windows_image.start_time());
9461 let process_identity = process_start_time.map(|start_time| ProcessIdentity { pid, start_time });
9462 #[cfg(all(test, target_os = "macos"))]
9463 privacy_exec_boundary_tests::before_image_sample(spec, pid);
9464 #[cfg(not(windows))]
9469 let recorded_image = observe_spawned_image(pid);
9470 #[cfg(target_os = "macos")]
9474 let recorded_image = if privacy_exec.is_some() {
9475 None
9476 } else {
9477 recorded_image
9478 };
9479 #[cfg(target_os = "linux")]
9480 let recorded_cgroup_name = cgroup_path.as_ref().map(|_| cgroup_name.clone());
9481 #[cfg(not(target_os = "linux"))]
9482 let recorded_cgroup_name = None;
9483 #[cfg(unix)]
9484 let preadmitted = admitted
9485 .as_mut()
9486 .and_then(|admitted| admitted.leader.take());
9487 #[cfg(not(unix))]
9488 let preadmitted: Option<crate::child_roster::RosterGuard> = None;
9489 let roster_guard = if let Some(guard) = preadmitted {
9490 Ok(guard)
9491 } else {
9492 roster.admit(
9493 spec.module_id.clone(),
9494 pid,
9495 spec.protocol,
9496 process_start_time,
9497 crate::child_roster::RecordedIdentity {
9498 container: None,
9499 #[cfg(unix)]
9500 carried: None,
9501 #[cfg(not(windows))]
9502 start_time: recorded_image.map(|image| image.start_time),
9503 #[cfg(windows)]
9504 start_time: process_start_time,
9505 #[cfg(not(windows))]
9506 executable: recorded_image
9507 .and_then(|image| image.executable)
9508 .map(crate::live_children::ExecutableIdentity::from),
9509 #[cfg(windows)]
9510 executable: Some(crate::live_children::ExecutableIdentity::from(
9511 windows_image.identity(),
9512 )),
9513 cgroup_name: recorded_cgroup_name,
9514 #[cfg(target_os = "linux")]
9515 cgroup_placement: cgroup_placement.cloned(),
9516 },
9517 )
9518 };
9519 let roster_guard = match roster_guard {
9520 Ok(guard) => guard,
9521 Err(failure) => {
9522 return failure.with_cleanup(|source| {
9523 #[cfg(windows)]
9524 if let Some(job) = &job {
9525 let _ = job.lock().unwrap_or_else(|p| p.into_inner()).terminate();
9526 }
9527 #[cfg(target_os = "linux")]
9528 kill_module_cgroup(cgroup_placement, &cgroup_name);
9529 #[cfg(unix)]
9530 let rejected_group = group_target.clone();
9531 kill_and_reap_rejected_child(
9532 &mut child,
9533 #[cfg(unix)]
9534 rejected_group,
9535 );
9536 #[cfg(target_os = "linux")]
9537 if let Some(placement) = cgroup_placement.filter(|_| carried.is_none()) {
9538 remove_module_cgroup(placement, &cgroup_name);
9539 }
9540 Err(SuperviseError::Spawn {
9541 program: spec.program.clone(),
9542 source: io::Error::other(format!(
9543 "could not durably record module child: {source}"
9544 )),
9545 cgroup_path,
9546 })
9547 });
9548 }
9549 };
9550 #[cfg(unix)]
9551 if admitted.is_some() {
9552 if let Some(image) = recorded_image {
9553 if let Err(source) = roster_guard.confirm_executable(image) {
9554 kill_and_reap_rejected_child(&mut child, group_target.clone());
9555 return Err(SuperviseError::Spawn {
9556 program: spec.program.clone(),
9557 source,
9558 cgroup_path,
9559 });
9560 }
9561 }
9562 }
9563 if roster.is_closed() {
9568 if carrying {
9569 #[cfg(unix)]
9570 let rejected_group = admitted
9571 .as_ref()
9572 .map(|admitted| admitted.rejection_target())
9573 .unwrap_or_else(|| group_target.clone());
9574 #[cfg(windows)]
9575 if let Some(job) = &job {
9576 let _ = job.lock().unwrap_or_else(|p| p.into_inner()).terminate();
9577 }
9578 kill_and_reap_rejected_child(
9579 &mut child,
9580 #[cfg(unix)]
9581 rejected_group,
9582 );
9583 #[cfg(all(test, unix))]
9584 let _ =
9585 roster.admission_event_for_test(&spec.module_id, "after_rejected_child", Some(pid));
9586 drop(roster_guard);
9587 return Err(SuperviseError::Spawn {
9588 program: spec.program.clone(),
9589 source: io::Error::other(
9590 "the daemon began shutting down during carried admission; ended the container",
9591 ),
9592 cgroup_path,
9593 });
9594 }
9595 #[cfg(target_os = "linux")]
9597 kill_module_cgroup(cgroup_placement, &cgroup_name);
9598 if let Err(error) = child.start_kill() {
9599 debug!(module_id = %spec.module_id, pid, %error, "kill of a process spawned during daemon shutdown failed; it may already have exited");
9600 }
9601 #[cfg(target_os = "linux")]
9602 if let Some(placement) = cgroup_placement.filter(|_| carried.is_none()) {
9603 while matches!(child.try_wait(), Ok(None)) {
9607 std::thread::yield_now();
9608 }
9609 if matches!(
9610 subc_cgroup::kill_module(Some(placement), &cgroup_name),
9611 subc_cgroup::KillOutcome::Killed
9612 ) {
9613 if let Ok(path) = placement.module_path(&cgroup_name) {
9614 while std::fs::read_to_string(path.join("cgroup.events"))
9615 .ok()
9616 .is_some_and(|events| events.lines().any(|line| line == "populated 1"))
9617 {
9618 std::thread::yield_now();
9619 }
9620 }
9621 }
9622 remove_module_cgroup(placement, &cgroup_name);
9623 }
9624 drop(roster_guard);
9625 return Err(SuperviseError::Spawn {
9626 program: spec.program.clone(),
9627 source: io::Error::other(
9628 "the daemon began shutting down while this process was starting; ended it",
9629 ),
9630 cgroup_path,
9631 });
9632 }
9633
9634 let stdout_pump = match child.stdout.take() {
9635 Some(stdout) => Some(tokio::spawn(pump_stdout_to(stdout, output_sink.clone()))),
9636 None => {
9637 warn!(
9638 module_id = %spec.module_id,
9639 "spawned child exposed no stdout pipe; file capture will be incomplete"
9640 );
9641 None
9642 }
9643 };
9644 let stderr_pump = match child.stderr.take() {
9645 Some(stderr) => {
9646 let generation = ring
9647 .lock()
9648 .unwrap_or_else(|poisoned| poisoned.into_inner())
9649 .begin_process();
9650 Some(StderrPump {
9651 task: tokio::spawn(pump_stderr_to(
9652 stderr,
9653 Arc::clone(ring),
9654 generation,
9655 output_sink,
9656 )),
9657 generation,
9658 })
9659 }
9660 None => {
9661 ring.lock()
9665 .unwrap_or_else(|poisoned| poisoned.into_inner())
9666 .mark_not_captured("stderr pipe was not available on spawn");
9667 warn!(
9668 module_id = %spec.module_id,
9669 "spawned child exposed no stderr pipe; tail will be unavailable"
9670 );
9671 None
9672 }
9673 };
9674
9675 let exec = Arc::new(crate::activity::ExecOutcome::new());
9676 #[cfg(target_os = "macos")]
9677 let pending_exec = privacy_exec.is_some();
9678 #[cfg(not(target_os = "macos"))]
9679 let pending_exec = false;
9680 if !pending_exec {
9681 exec.finish(crate::activity::ExecState::CommittedExec);
9682 }
9683 #[cfg(windows)]
9684 let subtree_target = job
9685 .as_ref()
9686 .map(|job| crate::subtree_resources::Target::Job(Arc::downgrade(job)))
9687 .unwrap_or(crate::subtree_resources::Target::Unavailable("no_job"));
9688 #[cfg(target_os = "linux")]
9689 let subtree_target = cgroup_path
9690 .map(crate::subtree_resources::Target::Cgroup)
9691 .unwrap_or(crate::subtree_resources::Target::Unavailable("no_cgroup"));
9692 #[cfg(not(any(target_os = "linux", windows)))]
9693 let subtree_target = crate::subtree_resources::Target::Unavailable("unsupported_platform");
9694 let mut supervised = SupervisedChild {
9695 subtree_target,
9696 child: containment::ProcessChild::plain(child, pid),
9697 exec: crate::activity::ExecOwner(exec),
9698 #[cfg(windows)]
9699 nonce_delivery,
9700 protocol: spec.protocol,
9701 #[cfg(target_os = "linux")]
9702 module_id: cgroup_name,
9703 #[cfg(target_os = "linux")]
9704 cgroup_placement: cgroup_placement.cloned(),
9705 #[cfg(windows)]
9706 job,
9707 stdout_pump,
9708 stderr_pump,
9709 stderr_ring: Arc::clone(ring),
9710 spawned_at,
9711 registration_deadline: None,
9712 registration_watch: None,
9713 spawned_at_ms,
9714 spawned_from,
9715 launch_spec_digest: LaunchSpecDigest::from_spec(spec),
9716 spawned_file_identity,
9717 process_start_time,
9718 process_identity,
9719 pid,
9720 roster_guard: Some(roster_guard),
9721 #[cfg(target_os = "macos")]
9722 privacy_exec,
9723 #[cfg(target_os = "macos")]
9724 report_ready: Arc::new(OnceLock::new()),
9725 spawn_failure: None,
9726 };
9727 if let Err(error) = containment::adopt(&mut supervised, spec, roster) {
9728 supervised.child.reject(
9729 #[cfg(unix)]
9730 group_target,
9731 );
9732 #[cfg(all(test, unix))]
9733 let _ = roster.admission_event_for_test(&spec.module_id, "after_rejected_child", Some(pid));
9734 return Err(error);
9735 }
9736 #[cfg(unix)]
9737 if let Some(admitted) = admitted.take() {
9738 admitted.disarm();
9739 }
9740 #[cfg(windows)]
9741 if let Err(error) = resume_suspended_child(pid, spec) {
9742 containment::terminate_blocking(&roster.containment);
9743 let _ = supervised.child.start_kill();
9744 return Err(error);
9745 }
9746 Ok(supervised)
9747}
9748
9749#[cfg(test)]
9750mod spawn_record_tests {
9751 use super::*;
9752
9753 thread_local! {
9754 static LAST_PID: std::cell::Cell<Option<u32>> = const { std::cell::Cell::new(None) };
9755 }
9756
9757 pub(super) fn after_create(child: &Child, _spec: &ModuleSpec) {
9758 LAST_PID.set(child.id());
9759 #[cfg(windows)]
9760 if let Some((_, address)) = _spec
9761 .env
9762 .iter()
9763 .find(|(key, _)| key == "SUBC_TEST_AFTER_CREATE")
9764 {
9765 use std::io::{Read, Write};
9766 let mut stream = std::net::TcpStream::connect(address).unwrap();
9767 stream
9768 .write_all(&child.id().unwrap().to_ne_bytes())
9769 .unwrap();
9770 let mut release = [0];
9771 stream.read_exact(&mut release).unwrap();
9772 }
9773 }
9774
9775 #[cfg(windows)]
9776 pub(super) fn assignment_failpoint(spec: &ModuleSpec) -> io::Result<()> {
9777 if spec
9778 .env
9779 .iter()
9780 .any(|(key, _)| key == "SUBC_TEST_ASSIGNMENT_FAILURE")
9781 {
9782 Err(io::Error::other("injected module job assignment failure"))
9783 } else {
9784 Ok(())
9785 }
9786 }
9787
9788 fn spec() -> ModuleSpec {
9789 ModuleSpec {
9790 module_id: "spawn-record-worker".into(),
9791 #[cfg(unix)]
9792 program: "/bin/sleep".into(),
9793 #[cfg(windows)]
9794 program: std::env::current_exe().unwrap(),
9795 #[cfg(unix)]
9796 args: vec!["60".into()],
9797 #[cfg(windows)]
9798 args: vec![
9799 "--exact".into(),
9800 "supervise::spawn_record_tests::windows_store_touch_fixture".into(),
9801 "--nocapture".into(),
9802 ],
9803 env: vec![],
9804 reserved: false,
9805 reserved_prefixes: vec![],
9806 protocol: ModuleProtocol::None,
9807 overlap: Default::default(),
9808 carry_containment: false,
9809 }
9810 }
9811
9812 #[tokio::test]
9813 async fn record_sync_failure_fails_spawn_and_reaps_child() {
9814 let home = cortexkit_test_support::ScratchDir::new("record-sync-failure");
9815 let supervisor =
9816 Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default())
9817 .with_live_children_record(home.join("live-children.json"));
9818 crate::live_children::fail_next_file_sync();
9819 let result = supervisor.spawn(spec());
9820 let failed = result.is_err();
9821 if let Ok(module) = result {
9822 module.stop().await.unwrap();
9823 }
9824 assert!(failed, "record failure must fail spawn");
9825 let pid = LAST_PID.get().expect("the real OS child was created");
9826 assert!(
9827 subc_os::Process::open(pid).unwrap().is_none(),
9828 "child {pid} survived rejected admission or was not reaped"
9829 );
9830 }
9831
9832 #[cfg(target_os = "macos")]
9833 #[test]
9834 #[ignore = "manual on-device latency measurement"]
9835 fn durable_record_write_latency_50() {
9836 let home = cortexkit_test_support::ScratchDir::new("record-latency");
9837 let path = home.join("live-children.json");
9838 let children = vec![crate::live_children::LiveChild {
9839 module_id: "latency-worker".into(),
9840 pid: std::process::id(),
9841 protocol: ModuleProtocol::None,
9842 start_time: Some(1),
9843 executable: None,
9844 cgroup_name: None,
9845 container: None,
9846 }];
9847 let mut samples = Vec::new();
9848 for _ in 0..50 {
9849 let start = std::time::Instant::now();
9850 crate::live_children::write_record(&path, &children).unwrap();
9851 samples.push(start.elapsed());
9852 }
9853 samples.sort();
9854 eprintln!(
9855 "durable_record_write_latency_50: n=50 median={:?} p95={:?}",
9856 (samples[24] + samples[25]) / 2,
9857 samples[47]
9858 );
9859 }
9860
9861 #[cfg(windows)]
9862 #[test]
9863 fn windows_store_touch_fixture() {
9864 if let Some(path) = std::env::var_os("SUBC_TEST_STORE_TOUCH") {
9865 std::fs::write(path, b"module ran").unwrap();
9866 }
9867 }
9868
9869 #[cfg(windows)]
9870 #[tokio::test]
9871 async fn windows_daemon_fixture() {
9872 let Ok(address) = std::env::var("SUBC_TEST_DAEMON_BARRIER") else {
9873 return;
9874 };
9875 let mut spec = spec();
9876 spec.env.push(("SUBC_TEST_AFTER_CREATE".into(), address));
9877 let supervisor =
9878 Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default());
9879 let _module = supervisor.spawn(spec).unwrap();
9880 }
9881
9882 #[cfg(windows)]
9883 #[test]
9884 fn windows_daemon_death_before_module_job_assignment_leaves_no_child() {
9885 use std::io::Read;
9886 let listener = std::net::TcpListener::bind("127.0.0.1:0").unwrap();
9887 let mut command = std::process::Command::new(std::env::current_exe().unwrap());
9890 command
9891 .args([
9892 "--exact",
9893 "supervise::spawn_record_tests::windows_daemon_fixture",
9894 "--nocapture",
9895 ])
9896 .env(
9897 "SUBC_TEST_DAEMON_BARRIER",
9898 listener.local_addr().unwrap().to_string(),
9899 );
9900 let mut daemon = subc_jobobject::spawn_contained(&mut command).unwrap();
9901 listener.set_nonblocking(true).unwrap();
9902 let deadline = std::time::Instant::now() + Duration::from_secs(30);
9903 let mut stream = loop {
9904 if let Ok((stream, _)) = listener.accept() {
9905 break stream;
9906 }
9907 assert!(
9908 daemon.child.try_wait().unwrap().is_none(),
9909 "daemon failed before rendezvous"
9910 );
9911 assert!(
9912 std::time::Instant::now() < deadline,
9913 "daemon did not reach CreateProcess failpoint"
9914 );
9915 std::thread::yield_now();
9916 };
9917 stream
9918 .set_read_timeout(Some(Duration::from_secs(30)))
9919 .unwrap();
9920 let mut pid = [0; 4];
9921 stream.read_exact(&mut pid).unwrap();
9922 let pid = u32::from_ne_bytes(pid);
9923 assert!(subc_jobobject::process_exists(pid));
9924 daemon.child.kill().unwrap();
9927 daemon.child.wait().unwrap();
9928 let exited = subc_jobobject::wait_for_process_exit(pid, Duration::from_secs(10));
9929 if !exited {
9930 daemon.job.terminate().unwrap();
9931 }
9932 assert!(exited, "suspended child {pid} outlived hard-killed daemon");
9933 }
9934
9935 #[cfg(windows)]
9936 #[tokio::test]
9937 async fn windows_containment_failure_never_resumes() {
9938 let home = cortexkit_test_support::ScratchDir::new("assignment-refusal");
9939 let sentinel = home.join("store-touched");
9940 let mut spec = spec();
9941 spec.env
9942 .push(("SUBC_TEST_ASSIGNMENT_FAILURE".into(), "1".into()));
9943 spec.env.push((
9944 "SUBC_TEST_STORE_TOUCH".into(),
9945 sentinel.display().to_string(),
9946 ));
9947 let supervisor =
9948 Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default());
9949 let result = supervisor.spawn(spec);
9950 if let Ok(module) = result {
9953 let _ = tokio::time::timeout(Duration::from_secs(10), async {
9954 while !sentinel.exists() && module.state().unwrap() == ModuleState::Running {
9955 tokio::task::yield_now().await;
9956 }
9957 })
9958 .await;
9959 let _ = module.stop().await;
9960 panic!(
9961 "assignment failure resumed a module; store touched={}",
9962 sentinel.exists()
9963 );
9964 }
9965 assert!(
9966 !sentinel.exists(),
9967 "refused module executed store-touch sentinel"
9968 );
9969 assert!(!subc_jobobject::process_exists(LAST_PID.get().unwrap()));
9970 }
9971
9972 #[cfg(windows)]
9973 #[tokio::test]
9974 async fn windows_nested_daemon_fixture() {
9975 let Some(sentinel) = std::env::var_os("SUBC_TEST_NESTED_STORE") else {
9976 return;
9977 };
9978 let mut spec = spec();
9979 spec.env.push((
9980 "SUBC_TEST_STORE_TOUCH".into(),
9981 sentinel.to_str().unwrap().into(),
9982 ));
9983 let supervisor =
9984 Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default());
9985 let module = supervisor
9986 .spawn(spec)
9987 .expect("per-module job must nest below daemon and host jobs");
9988 tokio::time::timeout(Duration::from_secs(10), async {
9989 while !PathBuf::from(&sentinel).exists() {
9990 tokio::task::yield_now().await;
9991 }
9992 })
9993 .await
9994 .unwrap();
9995 module.stop().await.unwrap();
9996 }
9997
9998 #[cfg(windows)]
9999 #[test]
10000 fn windows_module_job_nests_under_host_job() {
10001 let home = cortexkit_test_support::ScratchDir::new("nested-jobs");
10002 let sentinel = home.join("nested-store-touched");
10003 let mut command = std::process::Command::new(std::env::current_exe().unwrap());
10004 command
10005 .args([
10006 "--exact",
10007 "supervise::spawn_record_tests::windows_nested_daemon_fixture",
10008 "--nocapture",
10009 ])
10010 .env("SUBC_TEST_NESTED_STORE", &sentinel);
10011 let mut daemon = subc_jobobject::spawn_contained(&mut command).unwrap();
10012 assert!(daemon.child.wait().unwrap().success());
10013 assert!(sentinel.exists(), "nested protocol:none module never ran");
10014 }
10015}
10016
10017#[cfg(target_os = "linux")]
10018pub(crate) fn kill_module_cgroup(placement: Option<&subc_cgroup::Placement>, module_id: &str) {
10019 use subc_cgroup::KillOutcome;
10020 match subc_cgroup::kill_module(placement, module_id) {
10021 KillOutcome::Killed => {}
10022 KillOutcome::NotPlaced | KillOutcome::Unsupported => {
10023 debug!(
10024 module_id,
10025 "cgroup tree kill unavailable; using direct-child kill"
10026 );
10027 }
10028 KillOutcome::IoError { path, error } => {
10029 warn!(module_id, path = %path.display(), %error, "cgroup tree kill failed; using direct-child kill");
10030 }
10031 }
10032}
10033
10034#[cfg(windows)]
10042fn contain_spawned_child(
10043 child: &mut Child,
10044 spec: &ModuleSpec,
10045 carried: Option<&Arc<containment::CarriedContainment>>,
10046) -> Result<Option<Arc<Mutex<subc_jobobject::JobObject>>>, SuperviseError> {
10047 let assignment = || {
10048 let job = match carried {
10049 Some(container) => container.job(),
10050 None => Arc::new(Mutex::new(subc_jobobject::JobObject::new()?)),
10051 };
10052 #[cfg(test)]
10053 spawn_record_tests::assignment_failpoint(spec)?;
10054 job.lock()
10055 .unwrap_or_else(|p| p.into_inner())
10056 .assign(child)?;
10057 Ok::<_, io::Error>(job)
10058 };
10059 match assignment() {
10060 Ok(job) => Ok(Some(job)),
10061 Err(source) => {
10062 kill_and_reap_rejected_child(child);
10063 Err(SuperviseError::Spawn {
10064 program: spec.program.clone(),
10065 source: io::Error::other(format!(
10066 "Windows module job containment failed: {source}"
10067 )),
10068 cgroup_path: None,
10069 })
10070 }
10071 }
10072}
10073
10074fn kill_and_reap_rejected_child(child: &mut Child, #[cfg(unix)] group: containment::GroupTarget) {
10077 #[cfg(unix)]
10078 group.signal_rejected(child);
10079 let _ = child.start_kill();
10080 while matches!(child.try_wait(), Ok(None)) {
10081 std::thread::yield_now();
10082 }
10083}
10084
10085#[cfg(windows)]
10090fn resume_suspended_child(pid: u32, spec: &ModuleSpec) -> Result<(), SuperviseError> {
10091 if let Err(source) = subc_jobobject::resume_main_thread(pid) {
10092 let _ = std::process::Command::new("taskkill.exe")
10096 .args(["/PID", &pid.to_string(), "/T", "/F"])
10097 .stdin(Stdio::null())
10098 .stdout(Stdio::null())
10099 .stderr(Stdio::null())
10100 .status();
10101 return Err(SuperviseError::Spawn {
10102 program: spec.program.clone(),
10103 source,
10104 cgroup_path: None,
10105 });
10106 }
10107 Ok(())
10108}
10109
10110#[cfg(target_os = "linux")]
10111fn remove_module_cgroup(placement: &subc_cgroup::Placement, module_id: &str) {
10112 match placement.remove_module(module_id) {
10113 Ok(()) => debug!(module_id, "removed module cgroup after process exit"),
10114 Err(error) => warn!(
10115 module_id,
10116 error = %error,
10117 "could not remove module cgroup after process exit; continuing teardown"
10118 ),
10119 }
10120}
10121
10122#[cfg(target_os = "linux")]
10123const REAPED_CGROUP_TERM_GRACE: Duration = Duration::from_secs(2);
10124#[cfg(target_os = "linux")]
10125const REAPED_CGROUP_FREEZE_TIMEOUT: Duration = Duration::from_millis(100);
10126
10127#[cfg(target_os = "linux")]
10128async fn cleanup_reaped_cgroup(placement: &subc_cgroup::Placement, module_id: &str) {
10129 let path = placement.module_path(module_id).ok();
10130 if path.as_ref().is_some_and(|path| {
10131 std::fs::read_to_string(path.join("cgroup.events"))
10132 .ok()
10133 .is_some_and(|events| events.lines().any(|line| line == "populated 0"))
10134 }) {
10135 remove_module_cgroup(placement, module_id);
10136 return;
10137 }
10138
10139 if terminate_reaped_cgroup(placement, module_id).await {
10142 if let Some(path) = &path {
10143 let deadline = Instant::now() + REAPED_CGROUP_TERM_GRACE;
10144 while cgroup_is_populated(path) && Instant::now() < deadline {
10145 sleep_until((Instant::now() + Duration::from_millis(1)).min(deadline)).await;
10146 }
10147 }
10148 }
10149
10150 if matches!(
10154 subc_cgroup::kill_module(Some(placement), module_id),
10155 subc_cgroup::KillOutcome::Killed
10156 ) {
10157 if let Some(path) = &path {
10158 while cgroup_is_populated(path) {
10159 sleep(Duration::from_millis(1)).await;
10160 }
10161 }
10162 }
10163 remove_module_cgroup(placement, module_id);
10164}
10165
10166#[cfg(target_os = "linux")]
10167fn cgroup_is_populated(path: &std::path::Path) -> bool {
10168 std::fs::read_to_string(path.join("cgroup.events"))
10169 .ok()
10170 .is_some_and(|events| events.lines().any(|line| line == "populated 1"))
10171}
10172
10173#[cfg(target_os = "linux")]
10174async fn terminate_reaped_cgroup(placement: &subc_cgroup::Placement, module_id: &str) -> bool {
10175 let frozen =
10176 match subc_cgroup::freeze_module(Some(placement), module_id, REAPED_CGROUP_FREEZE_TIMEOUT)
10177 .await
10178 {
10179 subc_cgroup::FreezeOutcome::Frozen(frozen) => frozen,
10180 outcome => {
10181 tracing::debug!(
10182 module_id,
10183 ?outcome,
10184 "cgroup freeze failed; skipping SIGTERM grace"
10185 );
10186 return false;
10187 }
10188 };
10189 let listed = match frozen.processes() {
10190 subc_cgroup::ProcsOutcome::Listed(pids) => {
10191 for pid in pids.into_iter().filter(|pid| *pid > 1) {
10192 if let Some(pid) = rustix::process::Pid::from_raw(pid) {
10193 if let Err(error) =
10194 rustix::process::kill_process(pid, rustix::process::Signal::TERM)
10195 {
10196 tracing::debug!(module_id, %pid, %error, "leftover cgroup SIGTERM failed");
10197 }
10198 }
10199 }
10200 true
10201 }
10202 outcome => {
10203 tracing::debug!(
10204 module_id,
10205 ?outcome,
10206 "cgroup process listing failed; skipping SIGTERM grace"
10207 );
10208 false
10209 }
10210 };
10211 match frozen.thaw() {
10212 subc_cgroup::ThawOutcome::Thawed => listed,
10213 outcome => {
10214 tracing::debug!(
10215 module_id,
10216 ?outcome,
10217 "cgroup thaw failed; proceeding to cgroup.kill"
10218 );
10219 false
10220 }
10221 }
10222}
10223
10224#[cfg(target_os = "linux")]
10225fn apply_cgroup_placement(
10226 command: &mut Command,
10227 spec: &ModuleSpec,
10228 path: &std::path::Path,
10229) -> Result<(), SuperviseError> {
10230 subc_cgroup::apply(command, path).map_err(|source| SuperviseError::Cgroup {
10231 module_id: spec.module_id.clone(),
10232 source,
10233 })
10234}
10235
10236fn capture_retention(spec: &ModuleSpec) -> Retention {
10237 let defaults = Retention::default();
10238 let value = |name: &str| {
10239 spec.env
10240 .iter()
10241 .rev()
10242 .find_map(|(key, value)| (key == name).then_some(value.as_str()))
10243 };
10244 Retention {
10245 max_file_mb: value(CAPTURE_MAX_FILE_MB_ENV)
10246 .and_then(|value| value.parse().ok())
10247 .unwrap_or(defaults.max_file_mb),
10248 keep: value(CAPTURE_KEEP_ENV)
10249 .and_then(|value| value.parse().ok())
10250 .unwrap_or(defaults.keep),
10251 max_age_days: value(CAPTURE_MAX_AGE_DAYS_ENV)
10252 .and_then(|value| value.parse().ok())
10253 .unwrap_or(defaults.max_age_days),
10254 }
10255}
10256
10257fn generate_launch_nonce() -> Result<String, SuperviseError> {
10260 let mut bytes = [0u8; 32];
10261 getrandom::getrandom(&mut bytes).map_err(|source| SuperviseError::LaunchNonce {
10262 reason: source.to_string(),
10263 })?;
10264 let mut hex = String::with_capacity(64);
10265 for b in bytes {
10266 use std::fmt::Write;
10267 let _ = write!(hex, "{b:02x}");
10268 }
10269 Ok(hex)
10270}
10271
10272fn constant_time_eq(a: &[u8], b: &[u8]) -> bool {
10275 if a.len() != b.len() {
10276 return false;
10277 }
10278 let mut diff = 0u8;
10279 for (x, y) in a.iter().zip(b.iter()) {
10280 diff |= x ^ y;
10281 }
10282 diff == 0
10283}
10284
10285#[cfg(not(windows))]
10289fn observe_spawned_image(pid: u32) -> Option<subc_os::Observation> {
10290 let process = subc_os::Process::open(pid).ok().flatten()?;
10291 #[cfg(target_os = "linux")]
10292 return sample_after_linux_exec(
10293 || {
10294 let stat = std::fs::read_to_string(format!("/proc/{pid}/stat")).ok()?;
10295 let committed = linux_exec_committed(&stat);
10296 #[cfg(test)]
10297 spawn_image_tests::report_exec_state(committed);
10298 committed
10299 },
10300 || process.observe(),
10301 );
10302 #[cfg(not(target_os = "linux"))]
10303 process.observe()
10304}
10305
10306#[cfg(target_os = "linux")]
10310fn linux_exec_committed(stat: &str) -> Option<bool> {
10311 const PF_FORKNOEXEC: u64 = 0x40;
10312 let flags = stat.rsplit_once(')')?.1.split_whitespace().nth(6)?;
10313 Some(flags.parse::<u64>().ok()? & PF_FORKNOEXEC == 0)
10314}
10315
10316#[cfg(target_os = "linux")]
10317fn sample_after_linux_exec(
10318 mut exec_committed: impl FnMut() -> Option<bool>,
10319 observe: impl FnOnce() -> Option<subc_os::Observation>,
10320) -> Option<subc_os::Observation> {
10321 let deadline = std::time::Instant::now() + Duration::from_secs(1);
10322 while !exec_committed()? {
10323 if std::time::Instant::now() >= deadline {
10327 return None;
10328 }
10329 std::thread::sleep(Duration::from_micros(50));
10330 }
10331 observe()
10332}
10333
10334#[cfg(all(test, target_os = "linux"))]
10335mod spawn_image_tests {
10336 use super::*;
10337 use std::cell::{Cell, RefCell};
10338
10339 enum ExecEvent {
10340 Uncommitted,
10341 Image(Option<subc_os::Observation>),
10342 }
10343
10344 thread_local! {
10345 static EXEC_EVENTS: RefCell<Option<std::sync::mpsc::Sender<ExecEvent>>> = const { RefCell::new(None) };
10346 }
10347
10348 pub(super) fn report_exec_state(committed: Option<bool>) {
10349 if committed == Some(false) {
10350 EXEC_EVENTS.with(|events| {
10351 if let Some(events) = events.borrow_mut().take() {
10352 let _ = events.send(ExecEvent::Uncommitted);
10353 }
10354 });
10355 }
10356 }
10357
10358 #[test]
10359 fn linux_spawn_observation_waits_for_a_paused_fork_to_exec() {
10360 use std::io::{Read, Write};
10361 use std::os::fd::AsRawFd;
10362
10363 let descriptor = std::fs::File::open("/dev/null").unwrap();
10364 let mut command = std::process::Command::new("/bin/sleep");
10365 command.arg("30");
10366 let (mut ready, mut release) =
10367 subc_os::fork_exec_test::pause_before_exec(&mut command, descriptor.as_raw_fd())
10368 .unwrap();
10369 let spawn = std::thread::spawn(move || command.spawn());
10370 let mut message = [0u8; 8];
10371 ready.read_exact(&mut message).unwrap();
10372 let pid = i32::from_ne_bytes(message[..4].try_into().unwrap()) as u32;
10373 let before = std::fs::read_to_string(format!("/proc/{pid}/stat"));
10374 let inherited = subc_os::Process::open(pid).unwrap().unwrap().observe();
10375 let (events, received) = std::sync::mpsc::channel();
10376 let observer = std::thread::spawn(move || {
10377 EXEC_EVENTS.with(|slot| *slot.borrow_mut() = Some(events.clone()));
10378 let image = observe_spawned_image(pid);
10379 let _ = events.send(ExecEvent::Image(image));
10380 });
10381 let first = received.recv_timeout(Duration::from_secs(5));
10385 release.write_all(&[1]).unwrap();
10386 let mut child = spawn.join().unwrap().unwrap();
10387 let after = match first {
10388 Ok(ExecEvent::Uncommitted) => match received.recv_timeout(Duration::from_secs(5)) {
10389 Ok(ExecEvent::Image(image)) => image,
10390 _ => None,
10391 },
10392 Ok(ExecEvent::Image(image)) => image,
10393 Err(_) => None,
10394 };
10395 observer.join().unwrap();
10396 child.kill().unwrap();
10397 child.wait().unwrap();
10398
10399 assert_eq!(linux_exec_committed(&before.unwrap()), Some(false));
10400 assert_eq!(
10401 inherited.unwrap().executable,
10402 subc_os::file_identity(&std::env::current_exe().unwrap())
10403 );
10404 assert_eq!(
10405 after.unwrap().executable,
10406 subc_os::file_identity(std::path::Path::new("/bin/sleep"))
10407 );
10408 }
10409
10410 #[test]
10411 fn spawned_image_is_not_sampled_until_linux_exec_commits() {
10412 let committed = Cell::new(false);
10413 let inherited = subc_os::Observation {
10414 start_time: 23,
10415 executable: Some(subc_os::FileIdentity {
10416 device: 29,
10417 inode: 31,
10418 }),
10419 };
10420 let module = subc_os::Observation {
10421 start_time: 23,
10422 executable: Some(subc_os::FileIdentity {
10423 device: 29,
10424 inode: 37,
10425 }),
10426 };
10427 let polls = Cell::new(0);
10428 let image = sample_after_linux_exec(
10429 || {
10430 polls.set(polls.get() + 1);
10431 committed.set(polls.get() == 2);
10434 Some(committed.get())
10435 },
10436 || Some(if committed.get() { module } else { inherited }),
10437 );
10438 assert_eq!(image, Some(module));
10439 assert_eq!(polls.get(), 2);
10440 }
10441
10442 #[test]
10443 fn linux_exec_flag_is_read_from_flags_after_the_last_parenthesis() {
10444 assert_eq!(
10445 linux_exec_committed("17 (a ) b) R 1 2 3 4 5 4194368"),
10446 Some(false)
10447 );
10448 assert_eq!(
10449 linux_exec_committed("17 (a ) b) R 1 2 3 4 5 4194304"),
10450 Some(true)
10451 );
10452 assert_eq!(linux_exec_committed("17 (a) R 1 2 3 4 5 invalid"), None);
10453 assert_eq!(linux_exec_committed("17 (a) R"), None);
10454 }
10455
10456 #[test]
10457 fn unreadable_exec_state_never_samples_an_inherited_image() {
10458 assert_eq!(
10459 sample_after_linux_exec(
10460 || None,
10461 || panic!("must not sample before exec confirmation")
10462 ),
10463 None
10464 );
10465 }
10466
10467 #[test]
10468 fn uncommitted_exec_times_out_without_sampling_an_inherited_image() {
10469 assert_eq!(
10470 sample_after_linux_exec(
10471 || Some(false),
10472 || panic!("must not sample after an exec timeout")
10473 ),
10474 None
10475 );
10476 }
10477}
10478
10479#[cfg(windows)]
10482fn windows_spawn_capture(spec: &ModuleSpec) -> io::Result<subc_os::ExecutableCapture> {
10483 let mut program = spec.program.clone();
10484 if program.is_absolute() || program.components().count() > 1 {
10485 match subc_os::ExecutableCapture::open(&program) {
10488 Ok(capture) => return Ok(capture),
10489 Err(error)
10490 if error.kind() == io::ErrorKind::NotFound && program.extension().is_none() => {}
10491 Err(error) => return Err(error),
10492 }
10493 program.set_extension("exe");
10494 return subc_os::ExecutableCapture::open(&program);
10495 }
10496 if program.extension().is_none() {
10497 program.set_extension("exe");
10498 }
10499 let custom_path = spec
10500 .env
10501 .iter()
10502 .rev()
10503 .find(|(key, _)| key.eq_ignore_ascii_case("PATH"));
10504 let mut directories = Vec::new();
10505 if let Some(parent) = std::env::current_exe()?.parent() {
10508 directories.push(parent.to_path_buf());
10509 }
10510 directories.push(std::env::current_dir()?);
10511 if let Some(windows) = std::env::var_os("SystemRoot") {
10512 let windows = PathBuf::from(windows);
10513 directories.push(windows.join("System32"));
10514 directories.push(windows.clone());
10515 }
10516 let path = custom_path
10517 .map(|(_, value)| std::ffi::OsString::from(value))
10518 .or_else(|| std::env::var_os("PATH"));
10519 if let Some(path) = path {
10520 directories.extend(std::env::split_paths(&path));
10521 }
10522 for directory in directories {
10523 match subc_os::ExecutableCapture::open(&directory.join(&program)) {
10524 Ok(capture) => return Ok(capture),
10525 Err(error) if error.kind() == io::ErrorKind::NotFound => continue,
10526 Err(error) => return Err(error),
10527 }
10528 }
10529 Err(io::Error::new(
10530 io::ErrorKind::NotFound,
10531 "resolved executable not found",
10532 ))
10533}
10534
10535fn spawn_and_mark_running(
10536 spec: &ModuleSpec,
10537 runtime: &SupervisorRuntimeConfig,
10538 snapshot: &SharedSnapshot,
10539) -> Result<SupervisedChild, SuperviseError> {
10540 let activity_spawn = runtime.begin_activity_spawn(&spec.module_id);
10541 let registration_watch = watch_registration(&runtime.registry, spec);
10542 #[cfg(unix)]
10543 let joining_group = containment::current(&runtime.child_roster.containment)
10544 .is_some_and(|container| container.group().is_some());
10545 let spawn = || {
10546 spawn_child(
10547 spec,
10548 runtime.connection_file_path.as_deref(),
10549 runtime.supervisor_handle.as_ref(),
10550 &runtime.stderr_ring,
10551 runtime.capture_logs_dir.as_deref(),
10552 &runtime.child_roster,
10553 #[cfg(target_os = "linux")]
10554 runtime.cgroup_placement.as_ref(),
10555 )
10556 };
10557 let result = spawn();
10558 #[cfg(unix)]
10559 let result = if result.is_err()
10560 && joining_group
10561 && !runtime.child_roster.is_closed()
10562 && lock_snapshot(snapshot).is_ok_and(|state| state.enabled)
10563 && runtime
10564 .child_roster
10565 .previous_instance(&spec.module_id)
10566 .is_none()
10567 {
10568 warn!(module_id = %spec.module_id, "carried process-group spawn failed; trying one fresh container");
10572 spawn()
10573 } else {
10574 result
10575 };
10576 let child = result?;
10577 let mut child = child;
10578 arm_registration_deadline(&mut child, runtime.register_timeout, registration_watch);
10579 if let Err(error) = set_running(snapshot, &child, &spec.module_id, &runtime.spawn_events) {
10580 containment::terminate_blocking(&runtime.child_roster.containment);
10581 return Err(error);
10582 }
10583 if let Some(spawn) = activity_spawn {
10584 spawn.complete(snapshot);
10585 }
10586 Ok(child)
10587}
10588
10589struct RegistrationWatch {
10590 changes: watch::Receiver<u64>,
10591 baseline: u64,
10592}
10593
10594fn watch_registration(registry: &Registry, spec: &ModuleSpec) -> Option<RegistrationWatch> {
10598 if spec.protocol != ModuleProtocol::Subc {
10599 return None;
10600 }
10601 let changes = registry.subscribe_module_changes(&spec.module_id).ok()?;
10602 let baseline = *changes.borrow();
10603 Some(RegistrationWatch { changes, baseline })
10604}
10605
10606fn arm_registration_deadline(
10607 child: &mut SupervisedChild,
10608 budget: Duration,
10609 registration_watch: Option<RegistrationWatch>,
10610) {
10611 child.registration_deadline =
10612 (child.protocol == ModuleProtocol::Subc).then_some(child.spawned_at + budget);
10613 child.registration_watch = registration_watch;
10614}
10615
10616async fn stop_unregistered_child(child: &mut SupervisedChild) -> io::Result<()> {
10617 if let Some(container) = child.child.container() {
10618 container.stop_members().await;
10619 child.kill_leader()?;
10620 child.wait().await?;
10621 return Ok(());
10622 }
10623 #[cfg(unix)]
10624 {
10625 let group = unreaped_registration_group(child)?;
10628 let _ = rustix::process::kill_process_group(group, rustix::process::Signal::TERM);
10629 sleep(crate::child_roster::TERM_TO_KILL).await;
10634 let group = unreaped_registration_group(child)?;
10635 let _ = rustix::process::kill_process_group(group, rustix::process::Signal::KILL);
10636 }
10637 child.start_kill()?;
10640 child.wait().await?;
10641 Ok(())
10642}
10643
10644#[cfg(unix)]
10645fn unreaped_registration_group(child: &SupervisedChild) -> io::Result<rustix::process::Pid> {
10646 if let Some(container) = child.child.container() {
10647 return container
10648 .group()
10649 .and_then(|pid| rustix::process::Pid::from_raw(pid as i32))
10650 .ok_or_else(|| io::Error::other("a carried cgroup has no signalable process group"));
10651 }
10652 let group = i32::try_from(child.pid)
10655 .ok()
10656 .filter(|pid| *pid > 1)
10657 .and_then(rustix::process::Pid::from_raw)
10658 .ok_or_else(|| {
10659 io::Error::new(
10660 io::ErrorKind::InvalidInput,
10661 "invalid supervised process group",
10662 )
10663 })?;
10664 if child.child.id() != Some(child.pid) {
10665 return Err(io::Error::new(
10666 io::ErrorKind::InvalidInput,
10667 "refusing process-group signal after supervised child was reaped",
10668 ));
10669 }
10670 Ok(group)
10671}
10672
10673enum RegistrationWaitOutcome {
10674 Registered,
10675 Exited(ExitReport),
10676 TimedOut,
10677}
10678
10679struct ReloadRegistrationFailure {
10680 exit_report: ExitReport,
10681 reason: String,
10682}
10683
10684#[derive(Debug, Clone, Copy, PartialEq, Eq)]
10685enum BusyGaugeObservation {
10686 Quiescent,
10687 Busy,
10688 Omitted,
10689}
10690
10691#[derive(Debug, Clone, PartialEq, Eq)]
10693enum GaugeVerdict {
10694 Zero,
10696 NonZero(u64),
10698 NotACount(String),
10702 Missing,
10704 ProbeFailed(&'static str),
10707}
10708
10709impl GaugeVerdict {
10710 fn reads_quiet(&self) -> bool {
10711 matches!(self, Self::Zero)
10712 }
10713}
10714
10715impl fmt::Display for GaugeVerdict {
10716 fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
10717 match self {
10718 Self::Zero => f.write_str("zero"),
10719 Self::NonZero(value) => write!(f, "non_zero({value})"),
10720 Self::NotACount(value) => write!(f, "not_a_count({value})"),
10721 Self::Missing => f.write_str("missing_from_health_body"),
10722 Self::ProbeFailed(kind) => write!(f, "probe_failed({kind})"),
10723 }
10724 }
10725}
10726
10727const GAUGE_VALUE_TEXT_LIMIT: usize = 32;
10730
10731fn gauge_verdicts(metrics: Option<&Value>, gauges: &[String]) -> Vec<GaugeVerdict> {
10733 let metrics = metrics.and_then(Value::as_object);
10734 gauges
10735 .iter()
10736 .map(
10737 |gauge| match metrics.and_then(|metrics| metrics.get(gauge)) {
10738 None => GaugeVerdict::Missing,
10739 Some(value) => match value.as_u64() {
10740 Some(0) => GaugeVerdict::Zero,
10741 Some(count) => GaugeVerdict::NonZero(count),
10742 None => {
10743 let mut text = value.to_string();
10744 if text.len() > GAUGE_VALUE_TEXT_LIMIT {
10745 let mut end = GAUGE_VALUE_TEXT_LIMIT;
10746 while !text.is_char_boundary(end) {
10747 end -= 1;
10748 }
10749 text.truncate(end);
10750 text.push_str("...");
10751 }
10752 GaugeVerdict::NotACount(text)
10753 }
10754 },
10755 },
10756 )
10757 .collect()
10758}
10759
10760fn busy_gauge_observation(verdicts: &[GaugeVerdict]) -> BusyGaugeObservation {
10763 if verdicts
10764 .iter()
10765 .any(|verdict| matches!(verdict, GaugeVerdict::Missing))
10766 {
10767 BusyGaugeObservation::Omitted
10768 } else if verdicts.iter().all(GaugeVerdict::reads_quiet) {
10769 BusyGaugeObservation::Quiescent
10770 } else {
10771 BusyGaugeObservation::Busy
10772 }
10773}
10774
10775#[derive(Debug, Clone, Default, PartialEq, Eq)]
10780struct GaugeTrace {
10781 latest: Option<(GaugeVerdict, Duration)>,
10782 last_value: Option<(GaugeVerdict, Duration)>,
10783}
10784
10785#[derive(Debug, Clone, Default, PartialEq, Eq)]
10791struct DrainGaugeTrace {
10792 probes: u32,
10794 gauges: Vec<(String, GaugeTrace)>,
10796}
10797
10798impl DrainGaugeTrace {
10799 fn new(gauges: &[String]) -> Self {
10800 Self {
10801 probes: 0,
10802 gauges: gauges
10803 .iter()
10804 .map(|gauge| (gauge.clone(), GaugeTrace::default()))
10805 .collect(),
10806 }
10807 }
10808
10809 fn record(&mut self, verdicts: &[GaugeVerdict], at: Duration) {
10811 self.probes = self.probes.saturating_add(1);
10812 for ((_, trace), verdict) in self.gauges.iter_mut().zip(verdicts) {
10813 trace.latest = Some((verdict.clone(), at));
10814 if matches!(
10815 verdict,
10816 GaugeVerdict::Zero | GaugeVerdict::NonZero(_) | GaugeVerdict::NotACount(_)
10817 ) {
10818 trace.last_value = Some((verdict.clone(), at));
10819 }
10820 }
10821 }
10822}
10823
10824impl fmt::Display for DrainGaugeTrace {
10829 fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
10830 if self.gauges.is_empty() {
10831 return f.write_str("none declared");
10832 }
10833 for (index, (name, trace)) in self.gauges.iter().enumerate() {
10834 if index > 0 {
10835 f.write_str(" | ")?;
10836 }
10837 write!(f, "{name}: ")?;
10838 match &trace.latest {
10839 Some((verdict, at)) => write!(f, "{verdict} @{}ms", at.as_millis())?,
10840 None => f.write_str("never probed")?,
10841 }
10842 match &trace.last_value {
10843 Some((value, at)) => write!(f, "; last value {value} @{}ms", at.as_millis())?,
10844 None => f.write_str("; no value ever read")?,
10845 }
10846 }
10847 Ok(())
10848 }
10849}
10850
10851pub(crate) fn declared_busy_gauges(
10852 registry: &Registry,
10853 module_id: &str,
10854) -> Result<Vec<String>, SuperviseError> {
10855 busy_gauges_of(
10856 registry
10857 .get_module(module_id)
10858 .map_err(SuperviseError::Registry)?,
10859 )
10860}
10861
10862fn declared_busy_gauges_for_connection(
10866 registry: &Registry,
10867 connection_id: ConnectionId,
10868) -> Result<Vec<String>, SuperviseError> {
10869 busy_gauges_of(
10870 registry
10871 .get_module_by_connection(connection_id)
10872 .map_err(SuperviseError::Registry)?,
10873 )
10874}
10875
10876fn busy_gauges_of(
10877 registration: Option<crate::registry::ModuleRegistration>,
10878) -> Result<Vec<String>, SuperviseError> {
10879 let Some(registration) = registration else {
10880 return Ok(Vec::new());
10881 };
10882 let Some(self_signals) = registration.manifest.self_signals else {
10883 return Ok(Vec::new());
10884 };
10885
10886 let mut gauges = Vec::new();
10887 for declaration in self_signals {
10888 if declaration.kind != SelfSignalKind::Busy {
10889 continue;
10890 }
10891 match declaration.anchored_to {
10892 SignalAnchor::HealthGauges { gauges: declared } if !declared.is_empty() => {
10893 gauges.extend(declared)
10894 }
10895 _ => {
10896 gauges.push(String::new());
10899 }
10900 }
10901 }
10902 Ok(gauges)
10903}
10904
10905#[allow(clippy::too_many_arguments)]
10914async fn wait_for_forwarding_quiescence(
10915 forwarding: &ForwardingTable,
10916 module_id: &str,
10917 runtime: &SupervisorRuntimeConfig,
10918 endpoint: crate::ModuleEndpointId,
10919 deadline: Instant,
10920 busy_gauges: &[String],
10921 scope: DrainScope,
10922 trace: &mut DrainGaugeTrace,
10923) -> Result<bool, SuperviseError> {
10924 let wait_started = Instant::now();
10925 let mut gauges_quiescent = busy_gauges.is_empty();
10926 let mut next_probe_at = wait_started;
10927 let mut omission_reported = false;
10928
10929 loop {
10930 let now = Instant::now();
10931 if !busy_gauges.is_empty() && now >= next_probe_at && now < deadline {
10932 let report = match scope {
10933 DrainScope::Active => probe_module_health(module_id, runtime, Some(deadline)).await,
10934 DrainScope::Endpoint(endpoint) => {
10935 probe_endpoint_health(endpoint, runtime, Some(deadline)).await
10936 }
10937 };
10938 let verdicts = match &report {
10939 Ok(report) => gauge_verdicts(report.metrics.as_ref(), busy_gauges),
10940 Err(err) => vec![GaugeVerdict::ProbeFailed(err.label()); busy_gauges.len()],
10941 };
10942 trace.record(&verdicts, wait_started.elapsed());
10943 gauges_quiescent = match report {
10944 Ok(_) => match busy_gauge_observation(&verdicts) {
10945 BusyGaugeObservation::Quiescent => true,
10946 BusyGaugeObservation::Busy => false,
10947 BusyGaugeObservation::Omitted => {
10948 if !omission_reported {
10951 forwarding
10952 .counters()
10953 .increment_drains_with_undeclared_gauge();
10954 let missing = busy_gauges
10955 .iter()
10956 .zip(&verdicts)
10957 .filter(|(_, verdict)| matches!(verdict, GaugeVerdict::Missing))
10958 .map(|(gauge, _)| gauge.as_str())
10959 .collect::<Vec<_>>()
10960 .join(",");
10961 warn!(
10962 module_id,
10963 missing_gauges = %missing,
10964 "drain health.check answer lacks a declared busy gauge; the drain treats the module as busy until it reports it"
10965 );
10966 omission_reported = true;
10967 }
10968 false
10969 }
10970 },
10971 Err(err) => {
10972 warn!(
10973 module_id,
10974 error = %err,
10975 "drain health.check did not produce declared busy gauges; treating module as busy"
10976 );
10977 false
10978 }
10979 };
10980 next_probe_at = Instant::now() + drain_busy_reprobe_interval();
10983 }
10984
10985 let in_flight = forwarding
10986 .endpoint_in_flight_count(endpoint)
10987 .map_err(SuperviseError::Forwarding)?;
10988 if in_flight == 0 && gauges_quiescent {
10989 return Ok(true);
10990 }
10991
10992 let now = Instant::now();
10993 if now >= deadline {
10994 return Ok(false);
10995 }
10996 let mut wait = deadline
10997 .saturating_duration_since(now)
10998 .min(REGISTRY_RELEASE_POLL);
10999 if !busy_gauges.is_empty() {
11000 wait = wait.min(next_probe_at.saturating_duration_since(now));
11001 }
11002 sleep(wait).await;
11003 }
11004}
11005
11006fn drain_busy_reprobe_interval() -> Duration {
11010 DRAIN_BUSY_REPROBE_INTERVAL.max(REGISTRY_RELEASE_POLL)
11011}
11012
11013fn warn_drain_timed_out(
11028 module_id: &str,
11029 drain_timeout: Duration,
11030 reason: RouteCloseReason,
11031 holdouts: &crate::forwarding::DrainHoldouts,
11032 gauges: &DrainGaugeTrace,
11033) {
11034 warn!(
11035 module_id = %module_id,
11036 waited = ?drain_timeout,
11037 ?reason,
11038 held_requests = holdouts.requests,
11039 held_routes = holdouts.routes,
11040 total_routes = holdouts.total_routes,
11041 top_connections = ?holdouts.top_connections,
11042 held = %holdouts
11045 .held
11046 .iter()
11047 .map(|(channel, corr)| format!("{channel}:{corr}"))
11048 .collect::<Vec<_>>()
11049 .join(","),
11050 gauge_probes = gauges.probes,
11051 busy_gauges = %gauges,
11052 "route drain timed out before request quiescence; forcing teardown"
11053 );
11054}
11055
11056fn drained_after_quiescence_wait(wait_result: &Result<bool, SuperviseError>) -> bool {
11064 match wait_result {
11065 Ok(drained) => *drained,
11066 Err(_) => false,
11067 }
11068}
11069
11070fn send_route_goodbyes(forwarding: &ForwardingTable, released_routes: Vec<GoodbyeTarget>) {
11071 for released in released_routes {
11072 let frame = match Frame::build_with_version(
11073 released.negotiated_ver,
11074 FrameType::Goodbye,
11075 control_flags(),
11076 released.channel,
11077 released.epoch,
11078 0,
11079 Vec::new(),
11080 ) {
11081 Ok(frame) => frame,
11082 Err(err) => {
11083 warn!(
11084 route_channel = released.channel,
11085 error = %err,
11086 "failed to build supervisor drain route GOODBYE frame"
11087 );
11088 continue;
11089 }
11090 };
11091 if !released.close_on_delivery_failure() {
11092 crate::forwarding::send_module_route_goodbye(
11093 &forwarding.counters(),
11094 &released.sink,
11095 frame,
11096 released.module_id.as_deref(),
11097 "supervisor drain",
11098 );
11099 continue;
11100 }
11101 if let Err(err) = released.sink.try_send(frame) {
11102 warn!(
11103 target_connection_id = released.connection_id.get(),
11104 route_channel = released.channel,
11105 error = %err,
11106 "supervisor drain route GOODBYE was not delivered to client; closing target connection"
11107 );
11108 let _ = forwarding.escalate_client_delivery_failure(
11109 released.connection_id,
11110 released.channel,
11111 released.epoch,
11112 CloseReason::new(
11113 "route_goodbye_delivery_failed",
11114 format!(
11115 "failed to enqueue supervisor drain route GOODBYE for channel {}: {err}",
11116 released.channel
11117 ),
11118 ),
11119 crate::forwarding::UndeliveredFrame {
11120 module_id: released.module_id.as_deref(),
11121 sink: &released.sink,
11122 },
11123 );
11124 }
11125 }
11126}
11127
11128async fn send_module_draining(
11129 module_id: &str,
11130 reason: RouteCloseReason,
11131 deadline_ms: u64,
11132 target: &ModuleDrainTarget,
11133 _wait_for_flush: bool,
11134) -> bool {
11135 let body = match serde_json::to_vec(&ModuleControlCommand::Draining {
11136 reason,
11137 deadline_ms,
11138 }) {
11139 Ok(body) => body,
11140 Err(err) => {
11141 warn!(
11142 module_id,
11143 error = %err,
11144 "failed to encode module draining command"
11145 );
11146 return false;
11147 }
11148 };
11149 let frame = match Frame::build_with_version(
11150 target.negotiated_ver,
11151 FrameType::Push,
11152 control_flags(),
11153 0,
11154 0,
11155 0,
11156 body,
11157 ) {
11158 Ok(frame) => frame,
11159 Err(err) => {
11160 warn!(
11161 module_id,
11162 error = %err,
11163 "failed to build module draining command frame"
11164 );
11165 return false;
11166 }
11167 };
11168 #[cfg(unix)]
11169 if _wait_for_flush {
11170 return target.sink.send_flushed(frame).await.is_ok();
11171 }
11172 if let Err(err) = target.sink.try_send(frame) {
11173 warn!(
11174 module_id,
11175 target_connection_id = target.endpoint.connection_id.get(),
11176 error = %err,
11177 "module draining command was not delivered to peer"
11178 );
11179 return false;
11180 }
11181 true
11182}
11183
11184fn module_goodbye_frame(module_id: &str, negotiated_ver: u8) -> Option<Frame> {
11186 match Frame::build_with_version(
11187 negotiated_ver,
11188 FrameType::Goodbye,
11189 control_flags(),
11190 0,
11191 0,
11192 0,
11193 Vec::new(),
11194 ) {
11195 Ok(frame) => Some(frame),
11196 Err(err) => {
11197 warn!(
11198 module_id,
11199 error = %err,
11200 "failed to build module GOODBYE frame"
11201 );
11202 None
11203 }
11204 }
11205}
11206
11207#[cfg(unix)]
11221async fn send_module_goodbyes_for_daemon_shutdown(
11222 forwarding: &Arc<ForwardingTable>,
11223 reason: &CloseReason,
11224 wait_for_flush: bool,
11225) {
11226 const GOODBYE_BUDGET: Duration = Duration::from_millis(500);
11227 let targets = match forwarding.module_connections() {
11228 Ok(targets) => targets,
11229 Err(err) => {
11230 warn!(error = %err, "could not list module connections for shutdown GOODBYE");
11231 return;
11232 }
11233 };
11234 let deadline = Instant::now() + GOODBYE_BUDGET;
11235 let mut sends = tokio::task::JoinSet::new();
11236 for target in targets {
11237 let Some(frame) = module_goodbye_frame(&target.module_id, target.negotiated_ver) else {
11238 continue;
11239 };
11240 if !wait_for_flush {
11241 if let Err(err) = target.sink.try_send(frame) {
11242 debug!(
11243 module_id = %target.module_id,
11244 error = %err,
11245 "shutdown module GOODBYE was not queued"
11246 );
11247 }
11248 continue;
11249 }
11250 let forwarding = Arc::clone(forwarding);
11251 let reason = reason.clone();
11252 sends.spawn(async move {
11253 match timeout_at(deadline, target.sink.send_flushed(frame)).await {
11254 Ok(Ok(())) => {}
11255 Ok(Err(err)) => debug!(
11256 module_id = %target.module_id,
11257 error = %err,
11258 "module connection closed before its shutdown GOODBYE was written"
11259 ),
11260 Err(_) => warn!(
11261 module_id = %target.module_id,
11262 budget = ?GOODBYE_BUDGET,
11263 "shutdown module GOODBYE was not written within its budget; closing anyway"
11264 ),
11265 }
11266 forwarding.request_connection_close(target.endpoint.connection_id, reason);
11267 });
11268 }
11269 while sends.join_next().await.is_some() {}
11271}
11272
11273fn send_module_goodbye(module_id: &str, forwarding: &ForwardingTable, target: &ModuleDrainTarget) {
11274 let Some(frame) = module_goodbye_frame(module_id, target.negotiated_ver) else {
11275 return;
11276 };
11277 if let Err(err) = target.sink.try_send(frame) {
11278 warn!(
11279 module_id,
11280 target_connection_id = target.endpoint.connection_id.get(),
11281 error = %err,
11282 "supervisor drain module GOODBYE was not delivered to peer; closing module connection"
11283 );
11284 forwarding.request_connection_close(
11285 target.endpoint.connection_id,
11286 CloseReason::new(
11287 "module_goodbye_delivery_failed",
11288 format!("failed to enqueue supervisor drain module GOODBYE for module '{module_id}': {err}"),
11289 ),
11290 );
11291 }
11292}
11293
11294#[derive(Clone, Copy)]
11295struct ForwardingDrainContext<'a> {
11296 spec: &'a ModuleSpec,
11297 runtime: &'a SupervisorRuntimeConfig,
11298 registry: &'a Registry,
11299 scope: DrainScope,
11300}
11301
11302#[derive(Debug, Clone, Copy, PartialEq, Eq)]
11304enum DrainScope {
11305 Active,
11308 Endpoint(crate::ModuleEndpointId),
11313}
11314
11315#[derive(Debug, Clone, Copy, PartialEq, Eq)]
11323enum StopNotice {
11324 SentOverConnection,
11327 NoConnection,
11331 NotSent,
11335}
11336
11337async fn begin_forwarding_drain(
11338 spec: &ModuleSpec,
11339 runtime: &SupervisorRuntimeConfig,
11340 registry: &Registry,
11341 snapshot: &SharedSnapshot,
11342 enabled: Option<bool>,
11343 reason: RouteCloseReason,
11344) -> Result<StopNotice, SuperviseError> {
11345 let Some(forwarding) = runtime.forwarding.as_ref() else {
11346 return Err(SuperviseError::ReloadUnavailable {
11347 module_id: spec.module_id.clone(),
11348 reason: "supervisor was not configured with a forwarding table".to_string(),
11349 });
11350 };
11351
11352 begin_forwarding_drain_with(
11353 forwarding,
11354 ForwardingDrainContext {
11355 spec,
11356 runtime,
11357 registry,
11358 scope: DrainScope::Active,
11359 },
11360 snapshot,
11361 enabled,
11362 reason,
11363 runtime.drain_timeout,
11364 )
11365 .await
11366}
11367
11368async fn begin_forwarding_drain_if_configured(
11369 spec: &ModuleSpec,
11370 runtime: &SupervisorRuntimeConfig,
11371 registry: &Registry,
11372 snapshot: &SharedSnapshot,
11373 enabled: Option<bool>,
11374 reason: RouteCloseReason,
11375) -> Result<StopNotice, SuperviseError> {
11376 begin_forwarding_drain_with_timeout(
11377 spec,
11378 runtime,
11379 registry,
11380 snapshot,
11381 enabled,
11382 reason,
11383 runtime.drain_timeout,
11384 )
11385 .await
11386}
11387
11388async fn begin_forwarding_drain_with_timeout(
11392 spec: &ModuleSpec,
11393 runtime: &SupervisorRuntimeConfig,
11394 registry: &Registry,
11395 snapshot: &SharedSnapshot,
11396 enabled: Option<bool>,
11397 reason: RouteCloseReason,
11398 drain_timeout: Duration,
11399) -> Result<StopNotice, SuperviseError> {
11400 let Some(forwarding) = runtime.forwarding.as_ref() else {
11401 return Ok(StopNotice::NotSent);
11402 };
11403
11404 begin_forwarding_drain_with(
11405 forwarding,
11406 ForwardingDrainContext {
11407 spec,
11408 runtime,
11409 registry,
11410 scope: DrainScope::Active,
11411 },
11412 snapshot,
11413 enabled,
11414 reason,
11415 drain_timeout,
11416 )
11417 .await
11418}
11419
11420async fn begin_forwarding_drain_with(
11421 forwarding: &ForwardingTable,
11422 context: ForwardingDrainContext<'_>,
11423 snapshot: &SharedSnapshot,
11424 enabled: Option<bool>,
11425 reason: RouteCloseReason,
11426 drain_timeout: Duration,
11427) -> Result<StopNotice, SuperviseError> {
11428 let ForwardingDrainContext {
11429 spec,
11430 runtime,
11431 registry,
11432 scope,
11433 } = context;
11434 debug_assert_ne!(reason, RouteCloseReason::Crash);
11435 let terminal = matches!(reason, RouteCloseReason::Disable);
11436 let drain_started_at = Instant::now();
11437 let drain_deadline = drain_started_at + drain_timeout;
11438 let deadline_ms =
11439 unix_ms_now().saturating_add(u64::try_from(drain_timeout.as_millis()).unwrap_or(u64::MAX));
11440 let busy_gauges = match scope {
11441 DrainScope::Active => declared_busy_gauges(registry, &spec.module_id)?,
11442 DrainScope::Endpoint(endpoint) => {
11443 declared_busy_gauges_for_connection(registry, endpoint.connection_id)?
11444 }
11445 };
11446
11447 let gate_started = Instant::now();
11450 let drain_target = match scope {
11451 DrainScope::Active => forwarding.begin_module_drain(&spec.module_id, reason),
11452 DrainScope::Endpoint(endpoint) => forwarding.begin_endpoint_drain(endpoint, reason),
11453 }
11454 .map_err(SuperviseError::Forwarding)?;
11455 info!(
11460 module_id = %spec.module_id,
11461 ?reason,
11462 gate_ms = u64::try_from(gate_started.elapsed().as_millis()).unwrap_or(u64::MAX),
11463 connected = drain_target.is_some(),
11464 "module drain began; route admission closed"
11465 );
11466 if scope == DrainScope::Active {
11467 update_snapshot(snapshot, Some(&spec.module_id), |state| {
11468 state.state = ModuleState::Draining;
11469 state.draining_to_replace =
11470 matches!(reason, RouteCloseReason::Restart | RouteCloseReason::Reload);
11471 if let Some(enabled) = enabled {
11472 state.enabled = enabled;
11473 }
11474 })?;
11475 }
11476
11477 let Some(target) = drain_target.as_ref() else {
11478 return Ok(StopNotice::NoConnection);
11482 };
11483 {
11484 send_module_draining(&spec.module_id, reason, deadline_ms, target, false).await;
11485 #[cfg(feature = "test-support")]
11486 forwarding.wait_before_route_closing_for_test().await;
11487 let routes = forwarding
11488 .endpoint_routes(target.endpoint)
11489 .map_err(SuperviseError::Forwarding)?;
11490 let routes_notified = routes.len();
11491 crate::control::send_route_control_pushes(
11492 forwarding,
11493 routes.clone(),
11494 ClientControlPush::RouteClosing {
11495 module_id: spec.module_id.clone(),
11496 channels: Vec::new(),
11497 reason,
11498 },
11499 );
11500 send_route_goodbyes(forwarding, target.abandoned_bindings.clone());
11501
11502 let mut gauge_trace = DrainGaugeTrace::new(&busy_gauges);
11508 let wait_result = wait_for_forwarding_quiescence(
11509 forwarding,
11510 &spec.module_id,
11511 runtime,
11512 target.endpoint,
11513 drain_deadline,
11514 &busy_gauges,
11515 scope,
11516 &mut gauge_trace,
11517 )
11518 .await;
11519 let drained = drained_after_quiescence_wait(&wait_result);
11520 if let Err(err) = &wait_result {
11521 error!(
11522 module_id = %spec.module_id,
11523 ?reason,
11524 error = %err,
11525 "forwarding quiescence wait failed after route.closing; forcing route.closed(drained: false) so the client is not left waiting on an unfulfilled promise"
11526 );
11527 } else if !drained {
11528 let holdouts = forwarding
11530 .endpoint_drain_holdouts(target.endpoint)
11531 .unwrap_or_default();
11532 warn_drain_timed_out(
11533 &spec.module_id,
11534 drain_timeout,
11535 reason,
11536 &holdouts,
11537 &gauge_trace,
11538 );
11539 }
11540 crate::control::send_route_control_pushes(
11541 forwarding,
11542 routes,
11543 ClientControlPush::RouteClosed {
11544 module_id: spec.module_id.clone(),
11545 channels: Vec::new(),
11546 reason,
11547 drained,
11548 abandoned: target.abandoned_bindings.len() as u32,
11549 excluded_subscriptions: target.excluded_subscriptions,
11550 terminal: Some(terminal),
11551 },
11552 );
11553 wait_result?;
11554
11555 let released_routes = match forwarding.release_module_endpoint_routes(target.endpoint) {
11561 Ok(routes) => routes,
11562 Err(err) => {
11563 warn!(
11564 module_id = %spec.module_id,
11565 ?reason,
11566 error = %err,
11567 "failed to release module endpoint routes after route.closed; module GOODBYE will still be sent"
11568 );
11569 send_module_goodbye(&spec.module_id, forwarding, target);
11570 return Err(SuperviseError::Forwarding(err));
11571 }
11572 };
11573 let route_goodbye_count = released_routes.len();
11574 send_route_goodbyes(forwarding, released_routes);
11575 send_module_goodbye(&spec.module_id, forwarding, target);
11576
11577 info!(
11583 module_id = %spec.module_id,
11584 ?reason,
11585 routes_notified,
11586 route_goodbyes = route_goodbye_count,
11587 abandoned_reservations = target.abandoned_bindings.len(),
11588 excluded_subscriptions = target.excluded_subscriptions,
11589 drained,
11590 "module drain complete; consumers notified via route.closing/route.closed pushes and per-route GOODBYE frames"
11591 );
11592 }
11593
11594 Ok(StopNotice::SentOverConnection)
11595}
11596
11597async fn wait_for_registration_after_reload(
11600 registry: &Registry,
11601 module_id: &str,
11602 snapshot: &SharedSnapshot,
11603 child: &mut SupervisedChild,
11604 wait: Duration,
11605) -> Result<RegistrationWaitOutcome, SuperviseError> {
11606 wait_for_slot_registration(
11607 registry,
11608 crate::registry::RegistrationSlot::Active(module_id),
11609 module_id,
11610 snapshot,
11611 child,
11612 wait,
11613 )
11614 .await
11615}
11616
11617async fn wait_for_slot_registration(
11625 registry: &Registry,
11626 slot: crate::registry::RegistrationSlot<'_>,
11627 module_id: &str,
11628 snapshot: &SharedSnapshot,
11629 child: &mut SupervisedChild,
11630 wait: Duration,
11631) -> Result<RegistrationWaitOutcome, SuperviseError> {
11632 let deadline = Instant::now() + wait;
11633 loop {
11634 if registry
11635 .registration(slot)
11636 .map_err(SuperviseError::Registry)?
11637 .is_some()
11638 {
11639 return Ok(RegistrationWaitOutcome::Registered);
11640 }
11641
11642 let now = Instant::now();
11643 if now >= deadline {
11644 return Ok(RegistrationWaitOutcome::TimedOut);
11645 }
11646 let remaining = deadline.saturating_duration_since(now);
11647 let poll = remaining.min(REGISTRY_RELEASE_POLL);
11648
11649 tokio::select! {
11650 wait_result = child.wait() => {
11651 let status = wait_result.map_err(|source| SuperviseError::Wait {
11652 module_id: module_id.to_string(),
11653 source,
11654 })?;
11655 return Ok(RegistrationWaitOutcome::Exited(classify_reaped_child_exit(
11656 snapshot,
11657 child,
11658 &status,
11659 )));
11660 }
11661 _ = sleep(poll) => {}
11662 }
11663 }
11664}
11665
11666fn registration_failure_exit_report(mut exit_report: ExitReport) -> ExitReport {
11667 if exit_report.kind != ExitKind::DeliberateSeverance {
11671 exit_report.kind = ExitKind::Crash;
11672 }
11673 exit_report
11674}
11675
11676async fn handle_reload_child_registration_failure(
11677 spec: &ModuleSpec,
11678 runtime: &SupervisorRuntimeConfig,
11679 registry: &Registry,
11680 process_liveness: &SupervisorProcessLiveness,
11681 snapshot: &SharedSnapshot,
11682 _child: &mut Option<SupervisedChild>,
11683 failure: ReloadRegistrationFailure,
11684) -> Result<(), SuperviseError> {
11685 let ReloadRegistrationFailure {
11686 exit_report,
11687 reason,
11688 } = failure;
11689 match on_child_exit(
11690 spec,
11691 runtime.restart_policy,
11692 registry,
11693 snapshot,
11694 &runtime.terminal_ring,
11695 &runtime.spawn_events,
11696 &runtime.child_roster,
11697 exit_report,
11698 runtime.recovery_is_daemon,
11699 )
11700 .await
11701 {
11702 NextAction::Stop {
11703 registration_released,
11704 } => {
11705 if registration_released {
11706 process_liveness.untrack_if_current(&spec.module_id, snapshot);
11707 }
11708 }
11709 NextAction::Restart { schedule } => {
11710 let delay = schedule.map_or(runtime.restart_policy.delay_for_restart(0), |schedule| {
11711 schedule.delay
11712 });
11713 if let Some(schedule) = schedule {
11714 log_crash_respawn(&spec.module_id, schedule);
11715 }
11716 schedule_respawn(
11717 runtime,
11718 snapshot,
11719 &spec.module_id,
11720 delay,
11721 RespawnKind::Spawn,
11722 true,
11723 )?;
11724 }
11725 }
11726 Err(SuperviseError::ReloadFailed {
11727 module_id: spec.module_id.clone(),
11728 reason,
11729 })
11730}
11731
11732async fn handle_reload_spawn_failure(
11733 spec: &ModuleSpec,
11734 runtime: &SupervisorRuntimeConfig,
11735 process_liveness: &SupervisorProcessLiveness,
11736 snapshot: &SharedSnapshot,
11737 _child: &mut Option<SupervisedChild>,
11738 reason: String,
11739) -> Result<(), SuperviseError> {
11740 let now = Instant::now();
11741 let mut schedule = None;
11742 update_snapshot(snapshot, Some(&spec.module_id), |state| {
11743 clear_current_process_facts(state);
11744 if state.enabled {
11745 schedule = state.next_crash_restart(&runtime.restart_policy, now);
11746 state.state = if schedule.is_some() {
11747 ModuleState::Restarting
11748 } else {
11749 if runtime
11750 .failure_context(runtime.recovery_is_daemon)
11751 .may_retry()
11752 {
11753 state.failed_retry.schedule(&spec.module_id);
11754 }
11755 ModuleState::Failed
11756 };
11757 } else {
11758 state.state = ModuleState::Disabled;
11759 }
11760 })?;
11761 if let Some(schedule) = schedule {
11762 schedule_respawn(
11763 runtime,
11764 snapshot,
11765 &spec.module_id,
11766 schedule.delay,
11767 RespawnKind::Spawn,
11768 true,
11769 )?;
11770 } else {
11771 process_liveness.untrack_if_current(&spec.module_id, snapshot);
11772 }
11773 Err(SuperviseError::ReloadFailed {
11774 module_id: spec.module_id.clone(),
11775 reason,
11776 })
11777}
11778
11779fn control_flags() -> Flags {
11780 Flags::new(false, Priority::Passive, false)
11781}
11782
11783#[allow(clippy::too_many_arguments)]
11784async fn drain_optional_child(
11785 module_id: &str,
11786 protocol: ModuleProtocol,
11787 stop_notice: StopNotice,
11788 registry: &Registry,
11789 forwarding: Option<&ForwardingTable>,
11790 snapshot: &SharedSnapshot,
11791 terminal_ring: &Arc<Mutex<TerminalRing>>,
11792 spawn_events: &SpawnEventFeed,
11793 child: &mut Option<SupervisedChild>,
11794 drain_timeout: Duration,
11795 final_state: ModuleState,
11796 enabled: Option<bool>,
11797 failure: &FailureContext,
11798) -> Result<(), SuperviseError> {
11799 if let Some(child) = child.take() {
11800 drain_child_to_state(
11801 module_id,
11802 protocol,
11803 stop_notice,
11804 registry,
11805 forwarding,
11806 snapshot,
11807 terminal_ring,
11808 spawn_events,
11809 child,
11810 drain_timeout,
11811 final_state,
11812 enabled,
11813 failure,
11814 )
11815 .await
11816 } else {
11817 update_snapshot(snapshot, Some(module_id), |state| {
11818 state.state = final_state;
11819 if final_state == ModuleState::Failed {
11820 state.failed_retry.failed_at = Some(Instant::now());
11821 }
11822 if let Some(enabled) = enabled {
11823 state.enabled = enabled;
11824 }
11825 clear_current_process_facts(state);
11826 })?;
11827 release_dead_registration(registry, forwarding, snapshot, module_id, failure).await
11828 }
11829}
11830
11831#[allow(clippy::too_many_arguments)]
11832async fn drain_child_to_state(
11833 module_id: &str,
11834 _protocol: ModuleProtocol,
11835 stop_notice: StopNotice,
11836 registry: &Registry,
11837 forwarding: Option<&ForwardingTable>,
11838 snapshot: &SharedSnapshot,
11839 terminal_ring: &Arc<Mutex<TerminalRing>>,
11840 spawn_events: &SpawnEventFeed,
11841 mut child: SupervisedChild,
11842 drain_timeout: Duration,
11843 final_state: ModuleState,
11844 enabled: Option<bool>,
11845 failure: &FailureContext,
11846) -> Result<(), SuperviseError> {
11847 let protocol = child.protocol;
11848 update_snapshot(snapshot, Some(module_id), |state| {
11849 state.state = ModuleState::Draining;
11850 state.draining_to_replace = final_state == ModuleState::Restarting;
11851 if let Some(enabled) = enabled {
11852 state.enabled = enabled;
11853 }
11854 })?;
11855
11856 if stop_notice != StopNotice::SentOverConnection {
11867 if protocol == ModuleProtocol::Subc && stop_notice == StopNotice::NoConnection {
11868 info!(
11869 module_id,
11870 pid = child.pid,
11871 budget_ms = u64::try_from(drain_timeout.as_millis()).unwrap_or(u64::MAX),
11872 "module has no connection yet; requesting stop by signal"
11873 );
11874 }
11875 request_graceful_stop(module_id, &child);
11876 }
11877
11878 let exit_report = match timeout(drain_timeout, child.wait()).await {
11879 Ok(Ok(status)) => classify_reaped_child_exit(snapshot, &child, &status),
11880 Ok(Err(source)) => {
11881 fail_after_wait_error(
11882 module_id,
11883 snapshot,
11884 terminal_ring,
11885 spawn_events,
11886 format!("failed to wait during drain: {source}"),
11887 failure,
11888 );
11889 return Err(SuperviseError::Wait {
11890 module_id: module_id.to_string(),
11891 source,
11892 });
11893 }
11894 Err(_) => {
11895 warn!(
11908 module_id,
11909 pid = child.pid,
11910 budget_ms = u64::try_from(drain_timeout.as_millis()).unwrap_or(u64::MAX),
11911 reason = ?final_state,
11912 ?stop_notice,
11913 "drain budget expired before the module exited; killing it"
11914 );
11915 let kill =
11916 if final_state == ModuleState::Restarting && child.child.container().is_some() {
11917 child.kill_leader()
11918 } else {
11919 child.start_kill()
11920 };
11921 kill.map_err(|source| {
11922 fail_after_wait_error(
11923 module_id,
11924 snapshot,
11925 terminal_ring,
11926 spawn_events,
11927 format!("failed to kill during drain: {source}"),
11928 failure,
11929 );
11930 SuperviseError::Kill {
11931 module_id: module_id.to_string(),
11932 source,
11933 }
11934 })?;
11935 let status = child.wait().await.map_err(|source| {
11936 fail_after_wait_error(
11937 module_id,
11938 snapshot,
11939 terminal_ring,
11940 spawn_events,
11941 format!("failed to wait after drain kill: {source}"),
11942 failure,
11943 );
11944 SuperviseError::Wait {
11945 module_id: module_id.to_string(),
11946 source,
11947 }
11948 })?;
11949 classify_reaped_child_exit(snapshot, &child, &status)
11950 }
11951 };
11952
11953 let detail = lock_snapshot(snapshot)?.drain_disposition_detail.take();
11954 publish_after_terminal_record(
11955 module_id,
11956 || {
11957 record_terminal_with_detail(
11958 module_id,
11959 terminal_ring,
11960 spawn_events,
11961 &exit_report,
11962 terminal_disposition(final_state),
11963 detail,
11964 )
11965 },
11966 || {
11967 update_snapshot(snapshot, Some(module_id), |state| {
11968 state.state = final_state;
11969 if final_state == ModuleState::Failed {
11970 state.failed_retry.failed_at = Some(Instant::now());
11971 }
11972 if let Some(enabled) = enabled {
11973 state.enabled = enabled;
11974 }
11975 clear_current_process_facts(state);
11976 state.last_exit = Some(exit_report.clone());
11977 if exit_report.kind == ExitKind::DeliberateSeverance {
11978 state.lifetime_restarts += 1;
11979 }
11980 })
11981 },
11982 )?;
11983 child.drain_stderr(module_id).await;
11984
11985 let released =
11986 release_dead_registration(registry, forwarding, snapshot, module_id, failure).await;
11987 child
11988 .finish_exit(
11989 released.is_ok()
11990 && final_state == ModuleState::Restarting
11991 && child.spawn_failure.is_none(),
11992 )
11993 .await
11994 .map_err(|source| SuperviseError::Wait {
11995 module_id: module_id.to_owned(),
11996 source,
11997 })?;
11998 released
11999}
12000
12001#[cfg(unix)]
12021fn request_graceful_stop(module_id: &str, child: &SupervisedChild) {
12022 let Some(pid) = child
12023 .id()
12024 .and_then(|pid| i32::try_from(pid).ok())
12025 .and_then(rustix::process::Pid::from_raw)
12026 else {
12027 debug!(
12028 module_id,
12029 "no pid to signal for teardown; falling through to the drain wait"
12030 );
12031 return;
12032 };
12033 match rustix::process::kill_process(pid, rustix::process::Signal::TERM) {
12034 Ok(()) => debug!(
12035 module_id,
12036 "sent SIGTERM to a module nothing else asked to stop"
12037 ),
12038 Err(err) => debug!(
12039 module_id,
12040 error = %err,
12041 "SIGTERM to module failed; the drain wait and kill still apply"
12042 ),
12043 }
12044}
12045
12046#[cfg(not(unix))]
12054fn request_graceful_stop(module_id: &str, _child: &SupervisedChild) {
12055 debug!(
12056 module_id,
12057 "no graceful stop signal exists on this platform; teardown of a module nothing asked to stop waits, then kills"
12058 );
12059}
12060
12061fn terminal_disposition(final_state: ModuleState) -> TerminalDisposition {
12062 match final_state {
12063 ModuleState::Stopped => TerminalDisposition::Stopped,
12064 ModuleState::Disabled => TerminalDisposition::Disabled,
12065 ModuleState::Restarting => TerminalDisposition::Restarting,
12066 ModuleState::Failed => TerminalDisposition::Failed,
12067 ModuleState::Starting
12068 | ModuleState::Running
12069 | ModuleState::Unresponsive
12070 | ModuleState::Draining => {
12071 unreachable!("terminal exits only finish in terminal or restarting states")
12072 }
12073 }
12074}
12075
12076async fn release_dead_registration(
12084 registry: &Registry,
12085 forwarding: Option<&ForwardingTable>,
12086 snapshot: &SharedSnapshot,
12087 module_id: &str,
12088 failure: &FailureContext,
12089) -> Result<(), SuperviseError> {
12090 let result = async {
12091 let registration = registry
12092 .get_module(module_id)
12093 .map_err(SuperviseError::Registry)?;
12094 match wait_for_registration_release(registry, module_id, REGISTRY_RELEASE_TIMEOUT).await {
12095 Ok(()) => return Ok(()),
12096 Err(SuperviseError::RegistrationStillActive { .. }) => {}
12097 Err(err) => return Err(err),
12098 }
12099 let pid = lock_snapshot(snapshot)?.reaped_pid;
12100 if let (Some(registration), Some(forwarding), Some(pid)) = (registration, forwarding, pid) {
12101 warn!(
12102 module_id,
12103 pid,
12104 connection_id = registration.connection_id.get(),
12105 "reaped module registration outlived release grace; closing dead connection"
12106 );
12107 forwarding.request_connection_close(
12108 registration.connection_id,
12109 CloseReason::new(
12110 "supervised_process_reaped",
12111 format!("module '{module_id}' pid {pid} exited"),
12112 ),
12113 );
12114 wait_for_slot_registration_release(
12115 registry,
12116 crate::registry::RegistrationSlot::Connection(registration.connection_id),
12117 REGISTRY_RELEASE_TIMEOUT,
12118 )
12119 .await?;
12120 }
12121 wait_for_registration_release(registry, module_id, Duration::ZERO).await
12122 }
12123 .await;
12124 if let Err(err) = &result {
12125 fail_snapshot(snapshot, Some(module_id), None, failure);
12126 error!(module_id, error = %err, "registration release failed after child exit; module is failed and start can retry");
12127 }
12128 result
12129}
12130
12131async fn wait_for_registration_release(
12134 registry: &Registry,
12135 module_id: &str,
12136 wait: Duration,
12137) -> Result<(), SuperviseError> {
12138 wait_for_slot_registration_release(
12139 registry,
12140 crate::registry::RegistrationSlot::Active(module_id),
12141 wait,
12142 )
12143 .await
12144}
12145
12146async fn wait_for_slot_registration_release(
12154 registry: &Registry,
12155 slot: crate::registry::RegistrationSlot<'_>,
12156 wait: Duration,
12157) -> Result<(), SuperviseError> {
12158 let deadline = Instant::now() + wait;
12159 let mut release_events = registration_release_events().subscribe();
12160 let still_active = |registration: &crate::registry::ModuleRegistration| {
12161 SuperviseError::RegistrationStillActive {
12162 module_id: registration.manifest.module_id.clone(),
12163 waited: wait,
12164 }
12165 };
12166 loop {
12167 let _observed_generation = *release_events.borrow_and_update();
12168 let Some(registration) = registry
12169 .registration(slot)
12170 .map_err(SuperviseError::Registry)?
12171 else {
12172 return Ok(());
12173 };
12174
12175 let now = Instant::now();
12176 if now >= deadline {
12177 return Err(still_active(®istration));
12178 }
12179
12180 let remaining = deadline.saturating_duration_since(now);
12181 match timeout(remaining, release_events.changed()).await {
12182 Ok(Ok(())) | Ok(Err(_)) => {}
12183 Err(_) => return Err(still_active(®istration)),
12184 }
12185 }
12186}
12187
12188#[cfg(test)]
12189mod slot_registration_wait_tests {
12190 use super::*;
12191 use crate::registry::{ConnectionId, RegistrationSlot};
12192 use subc_protocol::manifest::ModuleManifest;
12193
12194 #[tokio::test]
12195 async fn enable_release_failure_is_failed_and_a_second_enable_retries() {
12196 let registry = Arc::new(Registry::default());
12197 let supervisor = Supervisor::new_for_test(Arc::clone(®istry), RestartPolicy::default());
12198 let runtime = supervisor.runtime_config();
12199 let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::disabled()));
12200 let spec = ModuleSpec {
12201 module_id: "enable-stale-registration".to_string(),
12202 program: PathBuf::from("/missing/enable-retry-test"),
12203 args: Vec::new(),
12204 env: Vec::new(),
12205 reserved: false,
12206 reserved_prefixes: Vec::new(),
12207 protocol: ModuleProtocol::Subc,
12208 overlap: Default::default(),
12209 carry_containment: false,
12210 };
12211 let connection = ConnectionId::new(90);
12212 registry
12213 .register_with_control_ops(
12214 ModuleManifest::builder(&spec.module_id, "0.1.0").build(),
12215 1,
12216 connection,
12217 Vec::new(),
12218 )
12219 .unwrap();
12220 let mut child = None;
12221 let err = set_child_enabled(
12222 &spec,
12223 &runtime,
12224 ®istry,
12225 &supervisor.process_liveness,
12226 &snapshot,
12227 &mut child,
12228 true,
12229 &runtime.failure_context(false),
12230 )
12231 .await
12232 .unwrap_err();
12233 assert!(matches!(
12234 err,
12235 SuperviseError::RegistrationStillActive { .. }
12236 ));
12237 assert_eq!(lock_snapshot(&snapshot).unwrap().state, ModuleState::Failed);
12238 assert!(child.is_none());
12239 registry.deregister_connection(connection).unwrap();
12240 let err = set_child_enabled(
12241 &spec,
12242 &runtime,
12243 ®istry,
12244 &supervisor.process_liveness,
12245 &snapshot,
12246 &mut child,
12247 true,
12248 &runtime.failure_context(false),
12249 )
12250 .await
12251 .unwrap_err();
12252 assert!(
12253 matches!(err, SuperviseError::Spawn { .. }),
12254 "second enable must attempt a spawn: {err}"
12255 );
12256 assert_eq!(lock_snapshot(&snapshot).unwrap().state, ModuleState::Failed);
12257 }
12258
12259 const INCUMBENT: u64 = 1;
12260 const CANDIDATE: u64 = 2;
12261
12262 fn swapped_registry() -> Arc<Registry> {
12263 let registry = Arc::new(Registry::default());
12264 let manifest = ModuleManifest::builder("m", "0.1.0").build();
12265 registry
12266 .register_with_control_ops(
12267 manifest.clone(),
12268 1,
12269 ConnectionId::new(INCUMBENT),
12270 Vec::new(),
12271 )
12272 .unwrap();
12273 registry
12274 .register_candidate_with_control_ops(
12275 manifest,
12276 1,
12277 ConnectionId::new(CANDIDATE),
12278 Vec::new(),
12279 )
12280 .unwrap();
12281 registry
12282 }
12283
12284 #[tokio::test]
12288 async fn incumbent_release_is_awaited_by_connection_not_by_module_id() {
12289 let registry = swapped_registry();
12290 registry.promote_candidate("m").unwrap().unwrap();
12291
12292 assert!(matches!(
12293 wait_for_registration_release(®istry, "m", Duration::from_millis(50)).await,
12294 Err(SuperviseError::RegistrationStillActive { .. })
12295 ));
12296
12297 assert!(matches!(
12299 wait_for_slot_registration_release(
12300 ®istry,
12301 RegistrationSlot::Connection(ConnectionId::new(INCUMBENT)),
12302 Duration::from_millis(50),
12303 )
12304 .await,
12305 Err(SuperviseError::RegistrationStillActive { .. })
12306 ));
12307
12308 let releaser = Arc::clone(®istry);
12309 let release = tokio::spawn(async move {
12310 sleep(Duration::from_millis(20)).await;
12311 releaser
12312 .deregister_connection(ConnectionId::new(INCUMBENT))
12313 .unwrap();
12314 notify_registration_release();
12315 });
12316 wait_for_slot_registration_release(
12317 ®istry,
12318 RegistrationSlot::Connection(ConnectionId::new(INCUMBENT)),
12319 Duration::from_secs(5),
12320 )
12321 .await
12322 .expect("the incumbent's own registration is released");
12323 release.await.unwrap();
12324 assert!(registry.get_module("m").unwrap().is_some());
12325 }
12326
12327 #[tokio::test]
12330 async fn candidate_slot_wait_ignores_the_incumbents_registration() {
12331 let registry = swapped_registry();
12332 assert!(matches!(
12333 wait_for_slot_registration_release(
12334 ®istry,
12335 RegistrationSlot::Candidate("m"),
12336 Duration::from_millis(50),
12337 )
12338 .await,
12339 Err(SuperviseError::RegistrationStillActive { .. })
12340 ));
12341 registry
12342 .deregister_connection(ConnectionId::new(CANDIDATE))
12343 .unwrap();
12344 wait_for_slot_registration_release(
12345 ®istry,
12346 RegistrationSlot::Candidate("m"),
12347 Duration::from_millis(50),
12348 )
12349 .await
12350 .expect("a candidate slot with no candidate is released");
12351 assert!(registry
12352 .registration(RegistrationSlot::Active("m"))
12353 .unwrap()
12354 .is_some());
12355 }
12356}
12357
12358fn classify_exit(status: &ExitStatus) -> ExitReport {
12359 ExitReport {
12360 kind: if status.success() {
12361 ExitKind::Clean
12362 } else {
12363 ExitKind::Crash
12364 },
12365 code: status.code(),
12366 signal: exit_signal(status),
12367 at_ms: unix_ms_now(),
12368 }
12369}
12370
12371fn wait_error_exit_report() -> ExitReport {
12377 ExitReport {
12378 kind: ExitKind::Crash,
12379 code: None,
12380 signal: None,
12381 at_ms: unix_ms_now(),
12382 }
12383}
12384
12385#[cfg(unix)]
12386fn exit_signal(status: &ExitStatus) -> Option<i32> {
12387 use std::os::unix::process::ExitStatusExt;
12388
12389 status.signal()
12390}
12391
12392#[cfg(not(unix))]
12393fn exit_signal(_status: &ExitStatus) -> Option<i32> {
12394 None
12395}
12396
12397fn reset_restart_count(snapshot: &SharedSnapshot, module_id: &str) -> Result<(), SuperviseError> {
12403 update_snapshot(snapshot, Some(module_id), |state| {
12404 state.clear_crash_restarts();
12405 })
12406}
12407
12408fn set_running(
12409 snapshot: &SharedSnapshot,
12410 child: &SupervisedChild,
12411 module_id: &str,
12412 spawn_events: &SpawnEventFeed,
12413) -> Result<(), SuperviseError> {
12414 let mut state = snapshot.lock().map_err(|_| SuperviseError::StatePoisoned {
12415 module_id: Some(module_id.to_string()),
12416 })?;
12417 state.spawn_generation = spawn_events.emit_spawned(module_id, child.pid, child.spawned_at_ms);
12418 if std::mem::take(&mut state.coalesced_restart_pending) {
12419 let generation = state.spawn_generation;
12420 info!(module_id, "restart already satisfied by generation {generation}; coalesced pending request completed");
12421 }
12422 state.drain_disposition_detail = None;
12423 state.spawn_failure = None;
12424 state.waiting_for_previous_pid = None;
12425 state.waiting_for_previous_detail = None;
12426 state.in_alternate_slot = false;
12429 state.configuration_updated_since_spawn = false;
12430 state.spawned_protocol = Some(child.protocol);
12431 state.state = ModuleState::Running;
12432 state.enabled = true;
12433 state.process_alive = true;
12434 state.pid = child.id();
12435 state.subtree_target = child.subtree_target.clone();
12436 state.health.status = SupervisorHealthStatus::Unknown;
12439 state.health.consecutive_failures = 0;
12440 state.health.detail = None;
12441 state.health.metrics = None;
12442 state.health.last_probe_ms = None;
12443 state.failed_retry.healthy_since = None;
12444 state.failed_retry.failed_at = None;
12445 state.failed_retry.running_since = Some(child.spawned_at);
12446 #[cfg(windows)]
12447 {
12448 state.pipe_consumed = child
12449 .nonce_delivery
12450 .as_ref()
12451 .map(|delivery| delivery.consumed());
12452 }
12453 #[cfg(target_os = "macos")]
12454 {
12455 state.report_ready = Some(Arc::clone(&child.report_ready));
12456 }
12457 state.spawned_at_ms = Some(child.spawned_at_ms);
12458 state.exec_outcome = Some(Arc::clone(&child.exec.0));
12459 state.spawned_from = Some(child.spawned_from.clone());
12460 state.launch_spec_digest = Some(child.launch_spec_digest.clone());
12461 state.spawned_file_identity = child.spawned_file_identity.clone();
12462 state.process_start_time = child.process_start_time;
12463 Ok(())
12464}
12465
12466fn clear_current_process_facts(state: &mut SupervisorSnapshot) {
12467 state.subtree_target = crate::subtree_resources::Target::Unavailable("not_running");
12468 state.failed_retry.healthy_since = None;
12469 state.failed_retry.running_since = None;
12470 state.waiting_for_previous_pid = None;
12471 state.waiting_for_previous_detail = None;
12472 state.process_alive = false;
12473 state.spawned_protocol = None;
12474 state.pid = None;
12475 #[cfg(windows)]
12476 {
12477 state.pipe_consumed = None;
12478 }
12479 #[cfg(target_os = "macos")]
12480 {
12481 state.report_ready = None;
12482 }
12483 state.spawned_at_ms = None;
12484 state.exec_outcome = None;
12485 state.spawned_from = None;
12486 state.launch_spec_digest = None;
12487 state.spawned_file_identity = None;
12488 state.process_start_time = None;
12489 state.deliberate_severance = None;
12490}
12491
12492#[cfg(test)]
12493fn record_deliberate_severance(
12494 snapshot: &SharedSnapshot,
12495 identity: ProcessIdentity,
12496) -> Result<(), SuperviseError> {
12497 update_snapshot(snapshot, None, |state| {
12498 state.deliberate_severance = Some(identity);
12499 })
12500}
12501
12502fn apply_deliberate_severance_marker(
12503 snapshot: &SharedSnapshot,
12504 exited_identity: Option<ProcessIdentity>,
12505 mut exit_report: ExitReport,
12506) -> ExitReport {
12507 let marker = lock_snapshot(snapshot)
12508 .ok()
12509 .and_then(|mut state| state.deliberate_severance.take());
12510 if marker.is_some() && marker == exited_identity {
12511 exit_report.kind = ExitKind::DeliberateSeverance;
12512 }
12513 exit_report
12514}
12515
12516fn classify_reaped_child_exit(
12517 snapshot: &SharedSnapshot,
12518 child: &SupervisedChild,
12519 status: &ExitStatus,
12520) -> ExitReport {
12521 let _ = update_snapshot(snapshot, None, |state| {
12522 state.reaped_pid = Some(child.pid);
12523 state.spawn_failure = child.spawn_failure.clone();
12524 });
12525 if child.spawn_failure.as_deref() == Some("registration_timeout") {
12526 registration_failure_exit_report(classify_exit(status))
12527 } else {
12528 apply_deliberate_severance_marker(snapshot, child.process_identity(), classify_exit(status))
12529 }
12530}
12531
12532fn fail_snapshot(
12533 snapshot: &SharedSnapshot,
12534 module_id: Option<&str>,
12535 last_exit: Option<ExitReport>,
12536 failure: &FailureContext,
12537) {
12538 if let Err(err) = update_snapshot(snapshot, module_id, |state| {
12539 state.state = ModuleState::Failed;
12540 clear_current_process_facts(state);
12541 if failure.may_retry() && state.enabled {
12542 state.failed_retry.schedule(module_id.unwrap_or("unknown"));
12543 } else if failure.shutdown.is_set() {
12544 state.failed_retry.cancel();
12545 }
12546 if let Some(last_exit) = last_exit {
12547 state.last_exit = Some(last_exit);
12548 }
12549 }) {
12550 error!(error = %err, "failed to mark supervisor state failed");
12551 }
12552}
12553
12554fn update_snapshot(
12555 snapshot: &SharedSnapshot,
12556 module_id: Option<&str>,
12557 update: impl FnOnce(&mut SupervisorSnapshot),
12558) -> Result<(), SuperviseError> {
12559 let mut state = snapshot.lock().map_err(|_| SuperviseError::StatePoisoned {
12560 module_id: module_id.map(ToOwned::to_owned),
12561 })?;
12562 update(&mut state);
12563 Ok(())
12564}
12565
12566const SLOW_SNAPSHOT_LOCK_THRESHOLD: Duration = Duration::from_millis(250);
12567
12568fn lock_snapshot_for_control<'a>(
12569 snapshot: &'a SharedSnapshot,
12570 module_id: &str,
12571 caller: &'static str,
12572) -> Result<std::sync::MutexGuard<'a, SupervisorSnapshot>, SuperviseError> {
12573 let started_at = Instant::now();
12574 let guard = lock_snapshot(snapshot)?;
12575 let waited = started_at.elapsed();
12576 if waited >= SLOW_SNAPSHOT_LOCK_THRESHOLD {
12577 warn!(
12578 module_id = %module_id,
12579 waited_ms = waited.as_millis() as u64,
12580 caller = %caller,
12581 "slow snapshot lock"
12582 );
12583 }
12584 Ok(guard)
12585}
12586
12587fn lock_snapshot(
12588 snapshot: &SharedSnapshot,
12589) -> Result<std::sync::MutexGuard<'_, SupervisorSnapshot>, SuperviseError> {
12590 snapshot
12591 .lock()
12592 .map_err(|_| SuperviseError::StatePoisoned { module_id: None })
12593}
12594
12595#[cfg(test)]
12596mod terminal_history_tests {
12597 use std::{
12598 path::PathBuf,
12599 sync::Arc,
12600 time::{Duration, Instant},
12601 };
12602
12603 use tokio::{sync::mpsc, time::sleep};
12604
12605 use super::{
12606 apply_deliberate_severance_marker, daemon_will_restart, drain_child_to_state,
12607 drained_after_quiescence_wait, handle_reload_spawn_failure, health_restart_child,
12608 lock_snapshot, on_child_exit, record_deliberate_severance, record_wait_error_terminal,
12609 reset_restart_count, spawn_and_mark_running, update_snapshot, wait_error_exit_report,
12610 ExitKind, ExitReport, ModuleProtocol, ModuleSpec, ModuleState, NextAction, ProcessIdentity,
12611 RestartPolicy, SpawnEventKind, StopNotice, SuperviseError, SupervisedConfiguration,
12612 SupervisedModule, SupervisedModuleInner, Supervisor, SupervisorHandle,
12613 SupervisorHealthStatus, SupervisorSnapshot,
12614 };
12615 use super::Instant as ClockInstant;
12620 use crate::{
12621 registry::Registry,
12622 terminal_ring::{TerminalRing, TerminalRingConfig},
12623 };
12624 use std::sync::Mutex;
12625 use subc_control::TerminalDisposition;
12626
12627 pub(super) fn fake_aft_stub_path() -> PathBuf {
12632 let mut path = std::env::current_exe().expect("current_exe available in tests");
12633 path.pop();
12634 path.pop();
12635 path.push(if cfg!(windows) {
12636 "fake-aft-stub.exe"
12637 } else {
12638 "fake-aft-stub"
12639 });
12640 assert!(
12641 path.exists(),
12642 "fake-aft-stub not built at {}: run `cargo test -p subc-core` (which builds \
12643 [[bin]] targets) rather than `cargo test -p subc-core --lib` (which does not)",
12644 path.display()
12645 );
12646 path
12647 }
12648
12649 #[test]
12650 fn reserved_never_spawned_refuses_every_hello() {
12651 let supervisor = SupervisorHandle::default();
12656 supervisor.apply_identity_configuration(&ModuleSpec {
12657 module_id: "never-spawned".to_string(),
12658 program: PathBuf::from("/usr/bin/false"),
12659 args: Vec::new(),
12660 env: Vec::new(),
12661 reserved: true,
12662 reserved_prefixes: Vec::new(),
12663 protocol: ModuleProtocol::Subc,
12664 overlap: Default::default(),
12665 carry_containment: false,
12666 });
12667 assert!(
12668 supervisor
12669 .reserved_hello_verification("never-spawned", Some("any-forged-nonce"))
12670 .is_err(),
12671 "forged nonce must refuse on a reserved never-spawned id"
12672 );
12673 assert!(
12674 supervisor
12675 .reserved_hello_verification("never-spawned", None)
12676 .is_err(),
12677 "absent nonce must refuse on a reserved never-spawned id"
12678 );
12679 supervisor.set_spawn_nonce("never-spawned", "minted".to_string());
12681 supervisor.apply_identity_configuration(&ModuleSpec {
12682 module_id: "never-spawned".to_string(),
12683 program: PathBuf::from("/usr/bin/false"),
12684 args: Vec::new(),
12685 env: Vec::new(),
12686 reserved: true,
12687 reserved_prefixes: Vec::new(),
12688 protocol: ModuleProtocol::Subc,
12689 overlap: Default::default(),
12690 carry_containment: false,
12691 });
12692 assert!(supervisor
12693 .reserved_hello_verification("never-spawned", Some("minted"))
12694 .is_ok());
12695 assert!(supervisor
12696 .reserved_hello_verification("never-spawned", Some("forged"))
12697 .is_err());
12698 }
12699
12700 fn seed_crash_restarts(state: &mut SupervisorSnapshot, count: u32) {
12703 let now = ClockInstant::now();
12704 for _ in 0..count {
12705 state.crash_restarts.push_back(now);
12706 }
12707 }
12708
12709 fn age_oldest_crash_restart_out_of_window(state: &mut SupervisorSnapshot, window: Duration) {
12713 let aged = state
12714 .crash_restarts
12715 .front()
12716 .expect("a crash restart must be recorded before it can be aged")
12717 .checked_sub(window + Duration::from_secs(1))
12718 .expect("the test clock is far enough from its origin to age an instant");
12719 state.crash_restarts[0] = aged;
12720 }
12721
12722 fn snapshot_with_restarts(enabled: bool, count: u32) -> SupervisorSnapshot {
12723 let mut state = SupervisorSnapshot::new(ModuleState::Running, enabled);
12724 seed_crash_restarts(&mut state, count);
12725 state
12726 }
12727
12728 #[test]
12729 fn daemon_owned_recovery_predicate_uses_the_pre_increment_budget() {
12730 let policy = RestartPolicy::new(3, Duration::ZERO);
12731 let now = ClockInstant::now();
12732 assert!(daemon_will_restart(
12733 &mut snapshot_with_restarts(true, 2),
12734 &policy,
12735 now
12736 ));
12737 assert!(!daemon_will_restart(
12738 &mut snapshot_with_restarts(true, 3),
12739 &policy,
12740 now
12741 ));
12742 assert!(!daemon_will_restart(
12743 &mut snapshot_with_restarts(false, 0),
12744 &policy,
12745 now
12746 ));
12747 }
12748
12749 #[test]
12750 fn crash_restart_backoff_escalates_with_in_window_count() {
12751 let policy = RestartPolicy::new(4, Duration::from_millis(100))
12752 .with_max_backoff(Duration::from_secs(30));
12753 let now = ClockInstant::now();
12754 let mut state = SupervisorSnapshot::new(ModuleState::Running, true);
12755 let schedules = (0..4)
12756 .map(|_| {
12757 state
12758 .next_crash_restart(&policy, now)
12759 .expect("the test policy allows four crash restarts")
12760 })
12761 .collect::<Vec<_>>();
12762
12763 assert_eq!(
12764 schedules
12765 .iter()
12766 .map(|schedule| schedule.restart_in_window)
12767 .collect::<Vec<_>>(),
12768 vec![0, 1, 2, 3]
12769 );
12770 assert_eq!(
12771 schedules
12772 .iter()
12773 .map(|schedule| schedule.delay)
12774 .collect::<Vec<_>>(),
12775 vec![
12776 Duration::from_millis(100),
12777 Duration::from_secs(1),
12778 Duration::from_secs(10),
12779 Duration::from_secs(30),
12780 ]
12781 );
12782 }
12783
12784 #[test]
12785 fn crash_restart_backoff_resets_after_ring_clear() {
12786 let policy = RestartPolicy::new(3, Duration::from_millis(100));
12787 let now = ClockInstant::now();
12788 let mut state = SupervisorSnapshot::new(ModuleState::Running, true);
12789 assert_eq!(
12790 state.next_crash_restart(&policy, now).unwrap().delay,
12791 Duration::from_millis(100)
12792 );
12793 assert_eq!(
12794 state.next_crash_restart(&policy, now).unwrap().delay,
12795 Duration::from_secs(1)
12796 );
12797
12798 state.clear_crash_restarts();
12799 let schedule = state
12800 .next_crash_restart(&policy, now)
12801 .expect("a cleared ring must allow another restart");
12802 assert_eq!(schedule.restart_in_window, 0);
12803 assert_eq!(schedule.delay, Duration::from_millis(100));
12804 }
12805
12806 #[test]
12807 fn crash_restart_backoff_ignores_aged_restarts() {
12808 let policy = RestartPolicy::new(3, Duration::from_millis(100));
12809 let now = ClockInstant::now();
12810 let mut state = SupervisorSnapshot::new(ModuleState::Running, true);
12811 state
12812 .next_crash_restart(&policy, now)
12813 .expect("the first restart is allowed");
12814 state
12815 .next_crash_restart(&policy, now)
12816 .expect("the second restart is allowed");
12817 state.crash_restarts[0] = now
12818 .checked_sub(policy.window + Duration::from_secs(1))
12819 .expect("the fake clock can age a restart past the window");
12820
12821 let schedule = state
12822 .next_crash_restart(&policy, now)
12823 .expect("an aged restart must release its slot");
12824 assert_eq!(schedule.restart_in_window, 1);
12825 assert_eq!(schedule.delay, Duration::from_secs(1));
12826 assert_eq!(state.crash_restarts.len(), 2);
12827 }
12828
12829 #[test]
12833 fn a_budget_spent_before_the_window_no_longer_refuses() {
12834 let policy = RestartPolicy::new(3, Duration::ZERO);
12835 let mut state = snapshot_with_restarts(true, 3);
12836 let now = ClockInstant::now();
12837 assert!(!daemon_will_restart(&mut state, &policy, now));
12838
12839 assert!(daemon_will_restart(
12840 &mut state,
12841 &policy,
12842 now + policy.window + Duration::from_secs(1)
12843 ));
12844 assert!(
12845 state.crash_restarts.is_empty(),
12846 "reading the budget must drop the instants that left the window"
12847 );
12848 }
12849
12850 fn module_with_recovery_snapshot(
12851 state: ModuleState,
12852 enabled: bool,
12853 restart_count: u32,
12854 ) -> SupervisedModule {
12855 let registry = Arc::new(Registry::default());
12856 let supervisor =
12857 Supervisor::new_for_test(Arc::clone(®istry), RestartPolicy::new(3, Duration::ZERO));
12858 let runtime = supervisor.runtime_config();
12859 let spec = ModuleSpec {
12860 module_id: "recovery-snapshot".to_string(),
12861 program: fake_aft_stub_path(),
12862 args: Vec::new(),
12863 env: Vec::new(),
12864 reserved: false,
12865 reserved_prefixes: Vec::new(),
12866 protocol: ModuleProtocol::Subc,
12867 overlap: Default::default(),
12868 carry_containment: false,
12869 };
12870 let mut snapshot = SupervisorSnapshot::new(state, enabled);
12871 seed_crash_restarts(&mut snapshot, restart_count);
12872 let (commands, _rx) = mpsc::channel(4);
12875 SupervisedModule {
12876 inner: Arc::new(SupervisedModuleInner {
12877 module_id: spec.module_id.clone(),
12878 registry,
12879 snapshot: Arc::new(Mutex::new(snapshot)),
12880 configuration: Arc::new(Mutex::new(SupervisedConfiguration {
12881 spec,
12882 health: runtime.health,
12883 })),
12884 stderr_ring: runtime.stderr_ring,
12885 terminal_ring: runtime.terminal_ring,
12886 commands,
12887 monitor: Mutex::new(None),
12888 restart_policy: runtime.restart_policy,
12889 effective_drain_timeout: runtime.effective_drain_timeout,
12890 provenance_probe: supervisor.provenance_probe.clone(),
12891 retained: runtime.child_roster.retained_view(),
12892 }),
12893 }
12894 }
12895
12896 #[cfg(target_os = "linux")]
12897 #[tokio::test]
12898 async fn no_cgroup_placement_does_not_block_fake_aft_stub_spawn() {
12899 let supervisor =
12900 Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default())
12901 .with_cgroup_placement(None);
12902 let result = supervisor.spawn(ModuleSpec {
12903 module_id: "no-cgroup-placement".to_string(),
12904 program: fake_aft_stub_path(),
12905 args: Vec::new(),
12906 env: Vec::new(),
12907 reserved: false,
12908 reserved_prefixes: Vec::new(),
12909 protocol: ModuleProtocol::Subc,
12910 overlap: Default::default(),
12911 carry_containment: false,
12912 });
12913
12914 assert!(
12915 result.is_ok(),
12916 "no delegation must not turn an otherwise valid spawn into a failure: {result:?}"
12917 );
12918 }
12919
12920 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
12921 async fn undecided_snapshot_uses_shared_restart_predicate() {
12922 assert!(module_with_recovery_snapshot(ModuleState::Running, true, 2)
12923 .will_recover_after_connection_loss()
12924 .unwrap());
12925 assert!(
12926 !module_with_recovery_snapshot(ModuleState::Running, true, 3)
12927 .will_recover_after_connection_loss()
12928 .unwrap()
12929 );
12930 }
12931
12932 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
12933 async fn restarting_snapshot_at_exhausted_budget_is_non_terminal() {
12934 assert!(
12935 module_with_recovery_snapshot(ModuleState::Restarting, true, 3)
12936 .will_recover_after_connection_loss()
12937 .unwrap()
12938 );
12939 }
12940
12941 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
12942 async fn terminal_phase_snapshots_are_terminal_before_budget_exhaustion() {
12943 assert!(!module_with_recovery_snapshot(ModuleState::Failed, true, 0)
12944 .will_recover_after_connection_loss()
12945 .unwrap());
12946 assert!(
12947 !module_with_recovery_snapshot(ModuleState::Disabled, true, 0)
12948 .will_recover_after_connection_loss()
12949 .unwrap()
12950 );
12951 }
12952
12953 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
12954 async fn warming_snapshot_is_limited_to_startup_phases() {
12955 for state in [
12956 ModuleState::Starting,
12957 ModuleState::Running,
12958 ModuleState::Restarting,
12959 ] {
12960 assert!(
12961 module_with_recovery_snapshot(state, true, 0)
12962 .is_warming()
12963 .unwrap(),
12964 "{state:?} should be warming"
12965 );
12966 }
12967 for state in [
12968 ModuleState::Unresponsive,
12969 ModuleState::Draining,
12970 ModuleState::Stopped,
12971 ModuleState::Failed,
12972 ModuleState::Disabled,
12973 ] {
12974 assert!(
12975 !module_with_recovery_snapshot(state, true, 0)
12976 .is_warming()
12977 .unwrap(),
12978 "{state:?} should not be warming"
12979 );
12980 }
12981 }
12982
12983 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
12984 async fn terminal_history_survives_respawn_and_keeps_both_crashes_in_order() {
12985 let registry = Arc::new(Registry::default());
12986 let supervisor =
12987 Supervisor::new_for_test(Arc::clone(®istry), RestartPolicy::new(1, Duration::ZERO));
12988 let module = supervisor
12989 .spawn(ModuleSpec {
12990 module_id: "terminal-history".to_string(),
12991 program: fake_aft_stub_path(),
12992 args: Vec::new(),
12993 env: vec![("FAKE_AFT_EXIT_CODE".to_string(), "23".to_string())],
12994 reserved: false,
12995 reserved_prefixes: Vec::new(),
12996 protocol: ModuleProtocol::Subc,
12997 overlap: Default::default(),
12998 carry_containment: false,
12999 })
13000 .unwrap();
13001
13002 let deadline = Instant::now() + Duration::from_secs(5);
13003 loop {
13004 let history = module.terminal_history();
13005 if history.entries.len() == 2 {
13006 assert_eq!(module.status().unwrap().state, ModuleState::Failed);
13007 assert_eq!(history.dropped, 0);
13008 assert_eq!(
13009 history
13010 .entries
13011 .iter()
13012 .map(|entry| entry.exit_code)
13013 .collect::<Vec<_>>(),
13014 vec![Some(23), Some(23)]
13015 );
13016 assert!(history.entries[0].at_ms <= history.entries[1].at_ms);
13017 return;
13018 }
13019 assert!(
13020 Instant::now() < deadline,
13021 "module did not retain two terminal exits: {history:?}"
13022 );
13023 sleep(Duration::from_millis(10)).await;
13024 }
13025 }
13026
13027 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
13031 async fn disable_during_crash_backoff_cancels_pending_respawn() {
13032 let backoff = Duration::from_secs(2);
13033 let supervisor = Supervisor::new_for_test(
13034 Arc::new(Registry::default()),
13035 RestartPolicy::new(10, backoff),
13036 );
13037 let module = supervisor
13038 .spawn(ModuleSpec {
13039 module_id: "disable-during-backoff".to_string(),
13040 program: fake_aft_stub_path(),
13041 args: Vec::new(),
13042 env: vec![("FAKE_AFT_EXIT_CODE".to_string(), "23".to_string())],
13043 reserved: false,
13044 reserved_prefixes: Vec::new(),
13045 protocol: ModuleProtocol::Subc,
13046 overlap: Default::default(),
13047 carry_containment: false,
13048 })
13049 .unwrap();
13050
13051 let deadline = Instant::now() + Duration::from_secs(5);
13053 loop {
13054 if module.status().unwrap().state == ModuleState::Restarting {
13055 break;
13056 }
13057 assert!(
13058 Instant::now() < deadline,
13059 "module never entered the crash backoff"
13060 );
13061 sleep(Duration::from_millis(10)).await;
13062 }
13063
13064 let started = Instant::now();
13065 module.set_enabled(false).await.unwrap();
13066 let waited = started.elapsed();
13067
13068 assert!(
13069 waited < backoff / 2,
13070 "disable waited {waited:?} behind the {backoff:?} crash backoff; the operator command must preempt the pending respawn"
13071 );
13072 assert_eq!(module.status().unwrap().state, ModuleState::Disabled);
13073
13074 sleep(backoff + Duration::from_millis(500)).await;
13076 let status = module.status().unwrap();
13077 assert_eq!(status.state, ModuleState::Disabled);
13078 assert_eq!(
13079 status.spawn_generation, 1,
13080 "module respawned after the operator disabled it"
13081 );
13082 }
13083
13084 #[cfg(unix)]
13087 fn protocol_none_sigterm_exits_clean_spec(
13088 module_id: &str,
13089 dir: &std::path::Path,
13090 ) -> (ModuleSpec, PathBuf, PathBuf) {
13091 let ready = dir.join("ready");
13092 let marker = dir.join("sigterm");
13093 let spec = ModuleSpec {
13094 module_id: module_id.to_string(),
13095 program: fake_aft_stub_path(),
13096 args: Vec::new(),
13097 env: vec![
13098 ("FAKE_AFT_NEVER_CONNECT".to_string(), "1".to_string()),
13099 (
13100 "FAKE_AFT_SIGTERM_MARKER_PATH".to_string(),
13101 marker.display().to_string(),
13102 ),
13103 (
13104 "FAKE_AFT_NEVER_CONNECT_READY_PATH".to_string(),
13105 ready.display().to_string(),
13106 ),
13107 ],
13108 reserved: false,
13109 reserved_prefixes: Vec::new(),
13110 protocol: ModuleProtocol::None,
13111 overlap: Default::default(),
13112 carry_containment: false,
13113 };
13114 (spec, ready, marker)
13115 }
13116
13117 #[cfg(unix)]
13121 async fn wait_for_file(path: &std::path::Path) {
13122 let deadline = Instant::now() + Duration::from_secs(10);
13123 while !path.exists() {
13124 assert!(
13125 Instant::now() < deadline,
13126 "{} never appeared",
13127 path.display()
13128 );
13129 sleep(Duration::from_millis(10)).await;
13130 }
13131 }
13132
13133 #[cfg(unix)]
13137 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
13138 async fn protocol_none_unrequested_clean_exit_restarts_as_a_crash() {
13139 let dir = cortexkit_test_support::ScratchDir::new("none-unrequested-clean-exit");
13140 let (spec, ready, marker) =
13141 protocol_none_sigterm_exits_clean_spec("none-unrequested-clean-exit", dir.path());
13142 let supervisor = Supervisor::new_for_test(
13143 Arc::new(Registry::default()),
13144 RestartPolicy::new(3, Duration::ZERO),
13145 );
13146 let module = supervisor.spawn(spec).unwrap();
13147 wait_for_file(&ready).await;
13148 let deadline = Instant::now() + Duration::from_secs(10);
13152 let first_pid = loop {
13153 let status = module.status().unwrap();
13154 if status.state == ModuleState::Running {
13155 if let Some(pid) = status.pid {
13156 break pid;
13157 }
13158 }
13159 assert!(
13160 Instant::now() < deadline,
13161 "a running module must report its pid after exec confirmation: {status:?}"
13162 );
13163 sleep(Duration::from_millis(10)).await;
13164 };
13165
13166 rustix::process::kill_process(
13167 rustix::process::Pid::from_raw(i32::try_from(first_pid).unwrap()).unwrap(),
13168 rustix::process::Signal::TERM,
13169 )
13170 .unwrap();
13171
13172 let deadline = Instant::now() + Duration::from_secs(10);
13173 let respawned = loop {
13174 let status = module.status().unwrap();
13175 if status.state == ModuleState::Running
13176 && status.pid.is_some_and(|pid| pid != first_pid)
13177 {
13178 break status;
13179 }
13180 assert!(
13181 Instant::now() < deadline,
13182 "protocol-none module was not respawned after an unrequested clean exit: {status:?}"
13183 );
13184 sleep(Duration::from_millis(10)).await;
13185 };
13186 assert_eq!(respawned.spawn_generation, 2);
13187 assert!(
13188 marker.exists(),
13189 "the child must have exited through its SIGTERM handler (exit 0), or this proves nothing about clean exits"
13190 );
13191
13192 let history = module.terminal_history();
13193 assert_eq!(history.entries.len(), 1, "{history:?}");
13194 let entry = &history.entries[0];
13195 assert_eq!(entry.exit_code, Some(0));
13196 assert_eq!(entry.exit_kind, subc_control::TerminalExitKind::Clean);
13197 assert_eq!(entry.disposition, TerminalDisposition::Restarting);
13198
13199 module.stop().await.unwrap();
13200 }
13201
13202 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
13206 async fn protocol_none_repeated_clean_exits_exhaust_the_restart_budget() {
13207 let supervisor = Supervisor::new_for_test(
13208 Arc::new(Registry::default()),
13209 RestartPolicy::new(1, Duration::ZERO),
13210 );
13211 let module = supervisor
13212 .spawn(ModuleSpec {
13213 module_id: "none-clean-exit-budget".to_string(),
13214 program: fake_aft_stub_path(),
13215 args: Vec::new(),
13216 env: vec![("FAKE_AFT_EXIT_CODE".to_string(), "0".to_string())],
13217 reserved: false,
13218 reserved_prefixes: Vec::new(),
13219 protocol: ModuleProtocol::None,
13220 overlap: Default::default(),
13221 carry_containment: false,
13222 })
13223 .unwrap();
13224
13225 let deadline = Instant::now() + Duration::from_secs(10);
13228 loop {
13229 let status = module.status().unwrap();
13230 if status.state == ModuleState::Failed {
13231 break;
13232 }
13233 assert!(
13234 Instant::now() < deadline,
13235 "module never exhausted its budget: {status:?} {:?}",
13236 module.terminal_history()
13237 );
13238 sleep(Duration::from_millis(10)).await;
13239 }
13240 let history = module.terminal_history();
13241 assert_eq!(
13242 history
13243 .entries
13244 .iter()
13245 .map(|entry| (entry.exit_code, entry.disposition.clone()))
13246 .collect::<Vec<_>>(),
13247 vec![
13248 (Some(0), TerminalDisposition::Restarting),
13249 (Some(0), TerminalDisposition::Failed),
13250 ]
13251 );
13252 let detail = history.entries[1]
13253 .disposition_detail
13254 .as_deref()
13255 .expect("a budget failure names the budget");
13256 assert!(detail.contains("max_restarts=1"), "{detail}");
13257 assert_eq!(module.status().unwrap().spawn_generation, 2);
13258 }
13259
13260 #[test]
13261 fn restart_budget_failure_is_published_after_its_terminal_record() {
13262 use std::io::{Read, Write};
13263 let supervisor = Supervisor::new_for_test(
13264 Arc::new(Registry::default()),
13265 RestartPolicy::new(0, Duration::ZERO),
13266 );
13267 let runtime = supervisor.runtime_config();
13268 let spec = ModuleSpec {
13269 protocol: ModuleProtocol::None,
13270 ..windowed_crash_spec("budget-publication-order")
13271 };
13272 let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::new(
13273 ModuleState::Running,
13274 true,
13275 )));
13276 lock_snapshot(&snapshot).unwrap().process_alive = true;
13277 let reaped_snapshot = snapshot.clone();
13278 let mut barrier = terminal_record_barrier(&runtime.spawn_events);
13279 let history = runtime.terminal_ring.clone();
13280 let reap = std::thread::spawn(move || {
13281 tokio::runtime::Builder::new_current_thread()
13282 .enable_all()
13283 .build()
13284 .unwrap()
13285 .block_on(on_child_exit(
13286 &spec,
13287 runtime.restart_policy,
13288 &supervisor.registry,
13289 &reaped_snapshot,
13290 &runtime.terminal_ring,
13291 &runtime.spawn_events,
13292 &runtime.child_roster,
13293 ExitReport {
13294 kind: ExitKind::Clean,
13295 code: Some(0),
13296 signal: None,
13297 at_ms: 1,
13298 },
13299 true,
13300 ))
13301 });
13302 let mut reached = [0];
13303 barrier.read_exact(&mut reached).unwrap();
13304 assert_eq!(&reached, b"R");
13305 let (before_state, before_alive) = {
13306 let state = lock_snapshot(&snapshot).unwrap();
13307 (state.state, state.process_alive)
13308 };
13309 let before_history = history.lock().unwrap().snapshot();
13310 barrier.write_all(b"X").unwrap();
13311 assert!(matches!(reap.join().unwrap(), NextAction::Stop { .. }));
13312 assert!(before_history.entries.is_empty());
13313 assert_ne!(
13314 before_state,
13315 ModuleState::Failed,
13316 "Failed was visible before its terminal record could be written"
13317 );
13318 assert!(before_alive);
13319 assert_eq!(lock_snapshot(&snapshot).unwrap().state, ModuleState::Failed);
13320 let history = history.lock().unwrap().snapshot();
13321 assert_eq!(history.entries.len(), 1);
13322 assert_eq!(history.entries[0].exit_code, Some(0));
13323 assert_eq!(history.entries[0].disposition, TerminalDisposition::Failed);
13324 }
13325
13326 #[test]
13327 fn crash_restart_at_budget_limit_remains_recoverable_while_recording() {
13328 use std::io::{Read, Write};
13329
13330 let supervisor = Supervisor::new_for_test(
13331 Arc::new(Registry::default()),
13332 RestartPolicy::new(3, Duration::ZERO),
13333 );
13334 let mut runtime = supervisor.runtime_config();
13335 let module = module_with_recovery_snapshot(ModuleState::Running, true, 2);
13336 let snapshot = module.inner.snapshot.clone();
13337 lock_snapshot(&snapshot).unwrap().process_alive = true;
13338 runtime.terminal_ring = module.inner.terminal_ring.clone();
13339 let spec = windowed_crash_spec(module.module_id());
13340 let mut barrier = terminal_record_barrier(&runtime.spawn_events);
13341 let reaped_snapshot = snapshot.clone();
13342 let reap = std::thread::spawn(move || {
13343 tokio::runtime::Builder::new_current_thread()
13344 .enable_all()
13345 .build()
13346 .unwrap()
13347 .block_on(on_child_exit(
13348 &spec,
13349 runtime.restart_policy,
13350 &supervisor.registry,
13351 &reaped_snapshot,
13352 &runtime.terminal_ring,
13353 &runtime.spawn_events,
13354 &runtime.child_roster,
13355 ExitReport {
13356 kind: ExitKind::Crash,
13357 code: Some(2),
13358 signal: None,
13359 at_ms: 1,
13360 },
13361 true,
13362 ))
13363 });
13364
13365 let mut reached = [0];
13369 barrier.read_exact(&mut reached).unwrap();
13370 assert_eq!(&reached, b"R");
13371 let before_status = module.status().unwrap();
13372 let before_history = module.terminal_history();
13373 let before_recovers = module.will_recover_after_connection_loss().unwrap();
13374 barrier.write_all(b"X").unwrap();
13375 assert!(matches!(reap.join().unwrap(), NextAction::Restart { .. }));
13376
13377 assert_eq!(before_status.state, ModuleState::Running);
13378 assert!(before_status.process_alive);
13379 assert_eq!(before_status.restart_count, 3);
13380 assert!(before_history.entries.is_empty());
13381 assert!(
13382 before_recovers,
13383 "connection cleanup would report a terminal close for a scheduled crash restart"
13384 );
13385 let restarting = module.status().unwrap();
13386 assert_eq!(restarting.state, ModuleState::Restarting);
13387 assert_eq!(restarting.restart_count, 3);
13388 assert!(module.will_recover_after_connection_loss().unwrap());
13389 let history = module.terminal_history();
13390 assert_eq!(history.entries.len(), 1);
13391 assert_eq!(
13392 history.entries[0].disposition,
13393 TerminalDisposition::Restarting
13394 );
13395
13396 update_snapshot(&snapshot, Some(module.module_id()), |state| {
13399 state.state = ModuleState::Running;
13400 state.respawn_pending = false;
13401 })
13402 .unwrap();
13403 assert!(!module.will_recover_after_connection_loss().unwrap());
13404 update_snapshot(&snapshot, Some(module.module_id()), |state| {
13405 state.enabled = false;
13406 state.respawn_pending = true;
13407 })
13408 .unwrap();
13409 assert!(!module.will_recover_after_connection_loss().unwrap());
13410 }
13411
13412 fn terminal_record_barrier(events: &super::SpawnEventFeed) -> std::net::TcpStream {
13413 use std::net::{TcpListener, TcpStream};
13414 let listener = TcpListener::bind("127.0.0.1:0").unwrap();
13415 let writer = TcpStream::connect(listener.local_addr().unwrap()).unwrap();
13416 let (barrier, _) = listener.accept().unwrap();
13417 for stream in [&writer, &barrier] {
13418 stream
13419 .set_read_timeout(Some(Duration::from_secs(10)))
13420 .unwrap();
13421 stream
13422 .set_write_timeout(Some(Duration::from_secs(10)))
13423 .unwrap();
13424 }
13425 *events.1.lock().unwrap() = Some(writer);
13426 barrier
13427 }
13428
13429 pub(super) fn before_terminal_record(events: &super::SpawnEventFeed) {
13430 use std::io::{Read, Write};
13431 let barrier = events.1.lock().unwrap().take();
13432 if let Some(mut barrier) = barrier {
13433 barrier.write_all(b"R").unwrap();
13434 let mut release = [0];
13435 barrier.read_exact(&mut release).unwrap();
13436 assert_eq!(&release, b"X");
13437 }
13438 }
13439
13440 #[test]
13441 fn health_restart_budget_failure_is_published_after_its_terminal_record() {
13442 use std::io::{Read, Write};
13443
13444 let supervisor = Supervisor::new_for_test(
13445 Arc::new(Registry::default()),
13446 RestartPolicy::new(0, Duration::ZERO),
13447 );
13448 let mut runtime = supervisor.runtime_config();
13449 let module = module_with_recovery_snapshot(ModuleState::Running, true, 0);
13450 let snapshot = module.inner.snapshot.clone();
13451 runtime.terminal_ring = module.inner.terminal_ring.clone();
13452 let spec = ModuleSpec {
13453 module_id: module.module_id().into(),
13454 program: fake_aft_stub_path(),
13455 protocol: ModuleProtocol::None,
13456 env: vec![("FAKE_AFT_EXIT_CODE".into(), "17".into())],
13457 ..windowed_crash_spec(module.module_id())
13458 };
13459 let mut barrier = terminal_record_barrier(&runtime.spawn_events);
13460 let reap = std::thread::spawn(move || {
13461 tokio::runtime::Builder::new_current_thread()
13462 .enable_all()
13463 .build()
13464 .unwrap()
13465 .block_on(async {
13466 let mut child =
13467 Some(spawn_and_mark_running(&spec, &runtime, &snapshot).unwrap());
13468 health_restart_child(
13469 &spec,
13470 &runtime,
13471 &supervisor.registry,
13472 &supervisor.process_liveness,
13473 &snapshot,
13474 &mut child,
13475 SupervisorHealthStatus::Failing,
13476 Some("health probe did not answer"),
13477 1,
13478 )
13479 .await
13480 .unwrap();
13481 assert!(child.is_none());
13482 });
13483 });
13484 let mut reached = [0];
13489 barrier.read_exact(&mut reached).unwrap();
13490 assert_eq!(&reached, b"R");
13491 let before_status = module.status().unwrap();
13492 let before_history = module.terminal_history();
13493 barrier.write_all(b"X").unwrap();
13494 reap.join().unwrap();
13495 assert!(before_history.entries.is_empty());
13496 assert_ne!(
13497 before_status.state,
13498 ModuleState::Failed,
13499 "Failed was visible before its terminal record could be written"
13500 );
13501 assert!(
13502 before_status.process_alive,
13503 "process death was visible before its terminal record could be written"
13504 );
13505 let failed = module.status().unwrap();
13506 assert_eq!(failed.state, ModuleState::Failed);
13507 assert!(!failed.process_alive);
13508 let history = module.terminal_history();
13509 assert_eq!(history.entries.len(), 1);
13510 assert_eq!(history.entries[0].disposition, TerminalDisposition::Failed);
13511 assert_eq!(
13512 history.entries[0].disposition_detail.as_deref(),
13513 Some("crash budget exhausted: max_restarts=0 within window_secs=600")
13514 );
13515 }
13516
13517 #[test]
13518 fn observed_exits_publish_process_death_after_terminal_history() {
13519 use std::io::{Read, Write};
13520 for (kind, enabled, shutdown, wait_error, expected_state) in [
13521 (ExitKind::Clean, true, false, false, ModuleState::Stopped),
13522 (ExitKind::Crash, true, false, false, ModuleState::Restarting),
13523 (ExitKind::Crash, false, false, false, ModuleState::Disabled),
13524 (
13525 ExitKind::DeliberateSeverance,
13526 true,
13527 false,
13528 false,
13529 ModuleState::Restarting,
13530 ),
13531 #[cfg(unix)]
13532 (ExitKind::Crash, true, true, false, ModuleState::Stopped),
13533 (ExitKind::Crash, true, false, true, ModuleState::Failed),
13534 ] {
13535 let supervisor = Supervisor::new_for_test(
13536 Arc::new(Registry::default()),
13537 RestartPolicy::new(1, Duration::ZERO),
13538 );
13539 let runtime = supervisor.runtime_config();
13540 #[cfg(unix)]
13541 if shutdown {
13542 runtime.child_roster.close();
13543 }
13544 #[cfg(not(unix))]
13545 let _ = shutdown;
13546 let spec = windowed_crash_spec("exit-publication-order");
13547 let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::new(
13548 ModuleState::Running,
13549 enabled,
13550 )));
13551 lock_snapshot(&snapshot).unwrap().process_alive = true;
13552 let reaped_snapshot = snapshot.clone();
13553 let history = runtime.terminal_ring.clone();
13554 let mut barrier = terminal_record_barrier(&runtime.spawn_events);
13555 let reap = std::thread::spawn(move || {
13556 if wait_error {
13557 super::fail_after_wait_error(
13558 &spec.module_id,
13559 &reaped_snapshot,
13560 &runtime.terminal_ring,
13561 &runtime.spawn_events,
13562 "wait failed".into(),
13563 &runtime.failure_context(true),
13564 );
13565 } else {
13566 tokio::runtime::Builder::new_current_thread()
13567 .enable_all()
13568 .build()
13569 .unwrap()
13570 .block_on(on_child_exit(
13571 &spec,
13572 runtime.restart_policy,
13573 &supervisor.registry,
13574 &reaped_snapshot,
13575 &runtime.terminal_ring,
13576 &runtime.spawn_events,
13577 &runtime.child_roster,
13578 ExitReport {
13579 kind,
13580 code: Some(0),
13581 signal: None,
13582 at_ms: 1,
13583 },
13584 true,
13585 ));
13586 }
13587 });
13588 let mut reached = [0];
13589 barrier.read_exact(&mut reached).unwrap();
13590 assert_eq!(&reached, b"R");
13591 let before_alive = lock_snapshot(&snapshot).unwrap().process_alive;
13592 let before_history = history.lock().unwrap().snapshot();
13593 barrier.write_all(b"X").unwrap();
13594 reap.join().unwrap();
13595 assert!(
13596 before_alive,
13597 "{kind:?}: death was published before recording"
13598 );
13599 assert!(before_history.entries.is_empty());
13600 assert_eq!(lock_snapshot(&snapshot).unwrap().state, expected_state);
13601 assert!(!lock_snapshot(&snapshot).unwrap().process_alive);
13602 assert_eq!(history.lock().unwrap().snapshot().entries.len(), 1);
13603 }
13604 }
13605
13606 #[tokio::test]
13607 async fn health_restart_budget_failure_remains_failed_when_journal_append_fails() {
13608 let dir = cortexkit_test_support::ScratchDir::new("health-budget-journal-failure");
13609 let path = dir.join("terminals.jsonl");
13610 std::fs::create_dir(&path).unwrap();
13611 let supervisor = Supervisor::new_for_test(
13612 Arc::new(Registry::default()),
13613 RestartPolicy::new(0, Duration::ZERO),
13614 )
13615 .with_terminal_journal(path, "health-budget-journal-failure".into());
13616 let runtime = supervisor.runtime_config();
13617 let spec = ModuleSpec {
13618 program: fake_aft_stub_path(),
13619 protocol: ModuleProtocol::None,
13620 env: vec![("FAKE_AFT_EXIT_CODE".into(), "17".into())],
13621 ..windowed_crash_spec("health-budget-journal-failure")
13622 };
13623 let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::starting()));
13624 let mut child = Some(spawn_and_mark_running(&spec, &runtime, &snapshot).unwrap());
13625 health_restart_child(
13626 &spec,
13627 &runtime,
13628 &supervisor.registry,
13629 &supervisor.process_liveness,
13630 &snapshot,
13631 &mut child,
13632 SupervisorHealthStatus::Failing,
13633 None,
13634 1,
13635 )
13636 .await
13637 .unwrap();
13638 assert_eq!(lock_snapshot(&snapshot).unwrap().state, ModuleState::Failed);
13639 assert!(!lock_snapshot(&snapshot).unwrap().process_alive);
13640 let history = runtime
13641 .terminal_ring
13642 .lock()
13643 .unwrap()
13644 .durable_history(&spec.module_id);
13645 assert!(history.journal_write_failures > 0);
13646 assert_eq!(history.entries.len(), 1);
13647 assert_eq!(history.entries[0].disposition, TerminalDisposition::Failed);
13648 }
13649
13650 #[test]
13651 fn health_restart_budget_failure_remains_failed_when_exit_recording_panics() {
13652 let supervisor = Supervisor::new_for_test(
13653 Arc::new(Registry::default()),
13654 RestartPolicy::new(0, Duration::ZERO),
13655 );
13656 let runtime = supervisor.runtime_config();
13657 let spec = ModuleSpec {
13658 program: fake_aft_stub_path(),
13659 protocol: ModuleProtocol::None,
13660 env: vec![("FAKE_AFT_EXIT_CODE".into(), "17".into())],
13661 ..windowed_crash_spec("health-budget-recording-panic")
13662 };
13663 let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::starting()));
13664 let result = std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| {
13665 tokio::runtime::Builder::new_current_thread()
13666 .enable_all()
13667 .build()
13668 .unwrap()
13669 .block_on(async {
13670 let mut child =
13671 Some(spawn_and_mark_running(&spec, &runtime, &snapshot).unwrap());
13672 runtime.spawn_events.0.lock().unwrap().seq = u64::MAX;
13673 health_restart_child(
13674 &spec,
13675 &runtime,
13676 &supervisor.registry,
13677 &supervisor.process_liveness,
13678 &snapshot,
13679 &mut child,
13680 SupervisorHealthStatus::Failing,
13681 None,
13682 1,
13683 )
13684 .await
13685 .unwrap();
13686 });
13687 }));
13688 let panic = result.expect_err("recording must still unwind");
13689 assert_eq!(
13690 panic.downcast_ref::<String>().map(String::as_str),
13691 Some("spawn event sequence exhausted")
13692 );
13693 let state = lock_snapshot(&snapshot).unwrap();
13694 assert_eq!(state.state, ModuleState::Failed);
13695 assert!(!state.process_alive);
13696 assert!(state.last_exit.is_some());
13697 }
13698
13699 #[tokio::test]
13700 async fn restart_budget_failure_remains_failed_when_journal_append_fails() {
13701 let dir = cortexkit_test_support::ScratchDir::new("budget-journal-failure");
13702 let path = dir.join("terminals.jsonl");
13703 std::fs::create_dir(&path).unwrap();
13704 let supervisor = Supervisor::new_for_test(
13705 Arc::new(Registry::default()),
13706 RestartPolicy::new(0, Duration::ZERO),
13707 )
13708 .with_terminal_journal(path, "budget-journal-failure".into());
13709 let runtime = supervisor.runtime_config();
13710 let spec = windowed_crash_spec("budget-journal-failure");
13711 let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::starting()));
13712 assert!(matches!(
13713 on_child_exit(
13714 &spec,
13715 runtime.restart_policy,
13716 &supervisor.registry,
13717 &snapshot,
13718 &runtime.terminal_ring,
13719 &runtime.spawn_events,
13720 &runtime.child_roster,
13721 crash_exit_report(1),
13722 true,
13723 )
13724 .await,
13725 NextAction::Stop { .. }
13726 ));
13727 assert_eq!(lock_snapshot(&snapshot).unwrap().state, ModuleState::Failed);
13728 let history = runtime
13729 .terminal_ring
13730 .lock()
13731 .unwrap()
13732 .durable_history(&spec.module_id);
13733 assert!(history.journal_write_failures > 0);
13734 assert_eq!(history.entries.len(), 1);
13735 assert_eq!(history.entries[0].disposition, TerminalDisposition::Failed);
13736 }
13737
13738 #[test]
13739 fn restart_budget_failure_remains_failed_when_exit_recording_panics() {
13740 let supervisor = Supervisor::new_for_test(
13741 Arc::new(Registry::default()),
13742 RestartPolicy::new(0, Duration::ZERO),
13743 );
13744 let runtime = supervisor.runtime_config();
13745 let spec = windowed_crash_spec("budget-recording-panic");
13746 let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::starting()));
13747 runtime.spawn_events.emit_spawned(&spec.module_id, 1, 1);
13748 runtime.spawn_events.0.lock().unwrap().seq = u64::MAX;
13751 let result = std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| {
13752 tokio::runtime::Builder::new_current_thread()
13753 .enable_all()
13754 .build()
13755 .unwrap()
13756 .block_on(on_child_exit(
13757 &spec,
13758 runtime.restart_policy,
13759 &supervisor.registry,
13760 &snapshot,
13761 &runtime.terminal_ring,
13762 &runtime.spawn_events,
13763 &runtime.child_roster,
13764 crash_exit_report(1),
13765 true,
13766 ))
13767 }));
13768 let panic = result.err().expect("recording must still unwind");
13769 assert_eq!(
13770 panic.downcast_ref::<String>().map(String::as_str),
13771 Some("spawn event sequence exhausted")
13772 );
13773 assert_eq!(lock_snapshot(&snapshot).unwrap().state, ModuleState::Failed);
13774 }
13775
13776 #[cfg(unix)]
13779 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
13780 async fn protocol_none_requested_stop_and_disable_do_not_respawn() {
13781 for disable in [false, true] {
13782 let label = if disable {
13783 "none-requested-disable"
13784 } else {
13785 "none-requested-stop"
13786 };
13787 let dir = cortexkit_test_support::ScratchDir::new(label);
13788 let (spec, ready, marker) = protocol_none_sigterm_exits_clean_spec(label, dir.path());
13789 let supervisor = Supervisor::new_for_test(
13790 Arc::new(Registry::default()),
13791 RestartPolicy::new(3, Duration::ZERO),
13792 );
13793 let module = supervisor.spawn(spec).unwrap();
13794 wait_for_file(&ready).await;
13795
13796 if disable {
13797 module.set_enabled(false).await.unwrap();
13798 } else {
13799 module.stop().await.unwrap();
13800 }
13801 assert!(
13802 marker.exists(),
13803 "{label}: the child must have left through its SIGTERM handler with exit 0"
13804 );
13805
13806 sleep(Duration::from_millis(500)).await;
13809 let status = module.status().unwrap();
13810 let expected = if disable {
13811 ModuleState::Disabled
13812 } else {
13813 ModuleState::Stopped
13814 };
13815 assert_eq!(status.state, expected, "{label}");
13816 assert_eq!(
13817 status.spawn_generation, 1,
13818 "{label}: respawned after a requested stop"
13819 );
13820 let history = module.terminal_history();
13821 assert_eq!(history.entries.len(), 1, "{label}: {history:?}");
13822 assert_eq!(history.entries[0].exit_code, Some(0), "{label}");
13823 assert_ne!(
13824 history.entries[0].disposition,
13825 TerminalDisposition::Restarting,
13826 "{label}"
13827 );
13828 }
13829 }
13830
13831 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
13834 async fn subc_wire_clean_exit_is_still_a_stop() {
13835 let supervisor = Supervisor::new_for_test(
13836 Arc::new(Registry::default()),
13837 RestartPolicy::new(3, Duration::ZERO),
13838 );
13839 let module = supervisor
13840 .spawn(ModuleSpec {
13841 module_id: "wire-clean-exit".to_string(),
13842 program: fake_aft_stub_path(),
13843 args: Vec::new(),
13844 env: vec![("FAKE_AFT_EXIT_CODE".to_string(), "0".to_string())],
13845 reserved: false,
13846 reserved_prefixes: Vec::new(),
13847 protocol: ModuleProtocol::Subc,
13848 overlap: Default::default(),
13849 carry_containment: false,
13850 })
13851 .unwrap();
13852
13853 let deadline = Instant::now() + Duration::from_secs(10);
13854 while module.terminal_history().entries.is_empty() {
13855 assert!(Instant::now() < deadline, "module never exited");
13856 sleep(Duration::from_millis(10)).await;
13857 }
13858 sleep(Duration::from_millis(500)).await;
13860 let status = module.status().unwrap();
13861 assert_eq!(status.state, ModuleState::Stopped);
13862 assert_eq!(status.spawn_generation, 1);
13863 let history = module.terminal_history();
13864 assert_eq!(history.entries.len(), 1, "{history:?}");
13865 assert_eq!(history.entries[0].exit_code, Some(0));
13866 assert_eq!(history.entries[0].disposition, TerminalDisposition::Stopped);
13867 }
13868
13869 #[cfg(unix)]
13870 #[tokio::test]
13871 async fn reload_exit_keeps_roster_until_terminal_is_recorded_during_shutdown() {
13872 let dir = cortexkit_test_support::ScratchDir::new("reload-roster-terminal-order");
13873 let record = dir.join("live-children.json");
13874 let supervisor = Supervisor::new_for_test(
13875 Arc::new(Registry::default()),
13876 RestartPolicy::new(0, Duration::ZERO),
13877 );
13878 let mut runtime = supervisor.runtime_config();
13879 runtime.child_roster.record_to(record.clone());
13880 let gate = Arc::new(super::ReloadExitRecordGate::default());
13881 runtime.test_reload_exit_record_gate = Some(Arc::clone(&gate));
13882 let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::starting()));
13883 let spec = ModuleSpec {
13884 module_id: "reload-exit-roster".into(),
13885 program: fake_aft_stub_path(),
13886 args: Vec::new(),
13887 env: vec![("FAKE_AFT_EXIT_CODE".into(), "23".into())],
13888 reserved: false,
13889 reserved_prefixes: Vec::new(),
13890 protocol: ModuleProtocol::Subc,
13891 overlap: Default::default(),
13892 carry_containment: false,
13893 };
13894 let mut child = None;
13895 let reload = super::finish_reload_child(
13896 &spec,
13897 &runtime,
13898 &supervisor.registry,
13899 &supervisor.process_liveness,
13900 &snapshot,
13901 &mut child,
13902 );
13903 tokio::pin!(reload);
13904 tokio::select! {
13905 result = &mut reload => panic!("reload missed the before-record gate: {result:?}"),
13906 _ = gate.reached.notified() => {}
13907 }
13908 assert!(runtime
13909 .terminal_ring
13910 .lock()
13911 .unwrap()
13912 .snapshot()
13913 .entries
13914 .is_empty());
13915 assert_eq!(
13916 crate::live_children::read_record(&record).unwrap().len(),
13917 1,
13918 "shutdown must still wait for the reaped child until its terminal record exists"
13919 );
13920 runtime.child_roster.close();
13921 gate.resume.notify_one();
13922 assert!(reload.await.is_err());
13923 assert!(crate::live_children::read_record(&record)
13924 .unwrap()
13925 .is_empty());
13926 let history = runtime.terminal_ring.lock().unwrap().snapshot();
13927 assert_eq!(history.entries.len(), 1);
13928 assert_eq!(
13929 history.entries[0].disposition,
13930 TerminalDisposition::DaemonShutdown
13931 );
13932 }
13933
13934 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
13938 async fn every_restart_increment_path_advances_lifetime_count() {
13939 let supervisor = Supervisor::new_for_test(
13940 Arc::new(Registry::default()),
13941 RestartPolicy::new(1, Duration::ZERO),
13942 );
13943 let runtime = supervisor.runtime_config();
13944 let spec = ModuleSpec {
13945 module_id: "lifetime-increment-path".to_string(),
13946 program: PathBuf::from("/unused/lifetime-increment-path"),
13947 args: Vec::new(),
13948 env: Vec::new(),
13949 reserved: false,
13950 reserved_prefixes: Vec::new(),
13951 protocol: ModuleProtocol::Subc,
13952 overlap: Default::default(),
13953 carry_containment: false,
13954 };
13955
13956 let crash_snapshot = Arc::new(Mutex::new(SupervisorSnapshot::starting()));
13957 assert!(matches!(
13958 on_child_exit(
13959 &spec,
13960 runtime.restart_policy,
13961 &supervisor.registry,
13962 &crash_snapshot,
13963 &runtime.terminal_ring,
13964 &runtime.spawn_events,
13965 &runtime.child_roster,
13966 ExitReport {
13967 kind: ExitKind::Crash,
13968 code: Some(1),
13969 signal: None,
13970 at_ms: 1,
13971 },
13972 true,
13973 )
13974 .await,
13975 NextAction::Restart { schedule: _ }
13976 ));
13977 let (crash_restarts, crash_lifetime) = {
13978 let state = lock_snapshot(&crash_snapshot).unwrap();
13979 (state.crash_restarts.len(), state.lifetime_restarts)
13980 };
13981 assert_eq!(crash_restarts, 1);
13982 assert_eq!(crash_lifetime, 1);
13983
13984 let health_snapshot = Arc::new(Mutex::new(SupervisorSnapshot::starting()));
13985 let mut health_child = None;
13986 assert!(matches!(
13987 health_restart_child(
13988 &spec,
13989 &runtime,
13990 &supervisor.registry,
13991 &supervisor.process_liveness,
13992 &health_snapshot,
13993 &mut health_child,
13994 SupervisorHealthStatus::Failing,
13995 None,
13996 2,
13997 )
13998 .await,
13999 Ok(())
14000 ));
14001 assert!(health_child.is_none());
14002 assert!(lock_snapshot(&health_snapshot).unwrap().respawn_pending);
14003 assert!(runtime.scheduled_respawn.lock().unwrap().take().is_some());
14004 let (health_restarts, health_lifetime) = {
14005 let state = lock_snapshot(&health_snapshot).unwrap();
14006 (state.crash_restarts.len(), state.lifetime_restarts)
14007 };
14008 assert_eq!(health_restarts, 1);
14009 assert_eq!(health_lifetime, 1);
14010
14011 let reload_snapshot = Arc::new(Mutex::new(SupervisorSnapshot::starting()));
14012 let mut reload_child = None;
14013 assert!(matches!(
14014 handle_reload_spawn_failure(
14015 &spec,
14016 &runtime,
14017 &supervisor.process_liveness,
14018 &reload_snapshot,
14019 &mut reload_child,
14020 "forced reload spawn failure".to_string(),
14021 )
14022 .await,
14023 Err(SuperviseError::ReloadFailed { .. })
14024 ));
14025 let (reload_restarts, reload_lifetime) = {
14026 let state = lock_snapshot(&reload_snapshot).unwrap();
14027 (state.crash_restarts.len(), state.lifetime_restarts)
14028 };
14029 assert_eq!(reload_restarts, 1);
14030 assert_eq!(reload_lifetime, 1);
14031 }
14032
14033 #[tokio::test]
14034 async fn deliberately_severed_live_child_records_lifetime_without_spending_restart_budget() {
14035 let supervisor = Supervisor::new_for_test(
14036 Arc::new(Registry::default()),
14037 RestartPolicy::new(3, Duration::ZERO),
14038 );
14039 let runtime = supervisor.runtime_config();
14040 let spec = ModuleSpec {
14041 module_id: "deliberately-severed".to_string(),
14042 program: PathBuf::from("/unused/deliberately-severed"),
14043 args: Vec::new(),
14044 env: Vec::new(),
14045 reserved: false,
14046 reserved_prefixes: Vec::new(),
14047 protocol: ModuleProtocol::Subc,
14048 overlap: Default::default(),
14049 carry_containment: false,
14050 };
14051 let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::starting()));
14052 let process = ProcessIdentity {
14053 pid: 41,
14054 start_time: 101,
14055 };
14056 record_deliberate_severance(&snapshot, process).unwrap();
14057 let exit_report = apply_deliberate_severance_marker(
14058 &snapshot,
14059 Some(process),
14060 ExitReport {
14061 kind: ExitKind::Crash,
14062 code: Some(1),
14063 signal: None,
14064 at_ms: 1,
14065 },
14066 );
14067 assert_eq!(exit_report.kind, ExitKind::DeliberateSeverance);
14068
14069 assert!(matches!(
14070 on_child_exit(
14071 &spec,
14072 runtime.restart_policy,
14073 &supervisor.registry,
14074 &snapshot,
14075 &runtime.terminal_ring,
14076 &runtime.spawn_events,
14077 &runtime.child_roster,
14078 exit_report,
14079 true,
14080 )
14081 .await,
14082 NextAction::Restart { schedule: _ }
14083 ));
14084 let state = lock_snapshot(&snapshot).unwrap();
14085 assert_eq!(state.lifetime_restarts, 1);
14086 assert_eq!(state.crash_restarts.len(), 0);
14087 }
14088
14089 #[tokio::test]
14090 async fn genuine_crash_spends_restart_budget_and_records_lifetime() {
14091 let supervisor = Supervisor::new_for_test(
14092 Arc::new(Registry::default()),
14093 RestartPolicy::new(3, Duration::ZERO),
14094 );
14095 let runtime = supervisor.runtime_config();
14096 let spec = ModuleSpec {
14097 module_id: "genuine-crash".to_string(),
14098 program: PathBuf::from("/unused/genuine-crash"),
14099 args: Vec::new(),
14100 env: Vec::new(),
14101 reserved: false,
14102 reserved_prefixes: Vec::new(),
14103 protocol: ModuleProtocol::Subc,
14104 overlap: Default::default(),
14105 carry_containment: false,
14106 };
14107 let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::starting()));
14108
14109 assert!(matches!(
14110 on_child_exit(
14111 &spec,
14112 runtime.restart_policy,
14113 &supervisor.registry,
14114 &snapshot,
14115 &runtime.terminal_ring,
14116 &runtime.spawn_events,
14117 &runtime.child_roster,
14118 ExitReport {
14119 kind: ExitKind::Crash,
14120 code: Some(1),
14121 signal: None,
14122 at_ms: 1,
14123 },
14124 true,
14125 )
14126 .await,
14127 NextAction::Restart { schedule: _ }
14128 ));
14129 let state = lock_snapshot(&snapshot).unwrap();
14130 assert_eq!(state.lifetime_restarts, 1);
14131 assert_eq!(state.crash_restarts.len(), 1);
14132 }
14133
14134 fn crash_exit_report(at_ms: u64) -> ExitReport {
14135 ExitReport {
14136 kind: ExitKind::Crash,
14137 code: Some(1),
14138 signal: None,
14139 at_ms,
14140 }
14141 }
14142
14143 fn windowed_crash_spec(module_id: &str) -> ModuleSpec {
14144 ModuleSpec {
14145 module_id: module_id.to_string(),
14146 program: PathBuf::from("/unused").join(module_id),
14147 args: Vec::new(),
14148 env: Vec::new(),
14149 reserved: false,
14150 reserved_prefixes: Vec::new(),
14151 protocol: ModuleProtocol::Subc,
14152 overlap: Default::default(),
14153 carry_containment: false,
14154 }
14155 }
14156
14157 #[tokio::test]
14163 async fn three_crashes_inside_the_window_stop_the_module_and_name_the_window() {
14164 let (logs, _guard) = crate::router::test_log::log_capture(tracing::Level::ERROR);
14165 let supervisor = Supervisor::new_for_test(
14166 Arc::new(Registry::default()),
14167 RestartPolicy::new(2, Duration::ZERO),
14168 );
14169 let runtime = supervisor.runtime_config();
14170 let spec = windowed_crash_spec("crash-loop-in-window");
14171 let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::starting()));
14172
14173 for attempt in 1..=2 {
14174 assert!(
14175 matches!(
14176 on_child_exit(
14177 &spec,
14178 runtime.restart_policy,
14179 &supervisor.registry,
14180 &snapshot,
14181 &runtime.terminal_ring,
14182 &runtime.spawn_events,
14183 &runtime.child_roster,
14184 crash_exit_report(attempt),
14185 true,
14186 )
14187 .await,
14188 NextAction::Restart { schedule: _ }
14189 ),
14190 "crash {attempt} is inside the budget and must respawn"
14191 );
14192 }
14193
14194 assert!(matches!(
14195 on_child_exit(
14196 &spec,
14197 runtime.restart_policy,
14198 &supervisor.registry,
14199 &snapshot,
14200 &runtime.terminal_ring,
14201 &runtime.spawn_events,
14202 &runtime.child_roster,
14203 crash_exit_report(3),
14204 true,
14205 )
14206 .await,
14207 NextAction::Stop { .. }
14208 ));
14209
14210 {
14211 let state = lock_snapshot(&snapshot).unwrap();
14212 assert_eq!(state.state, ModuleState::Failed);
14213 assert_eq!(state.crash_restarts.len(), 2);
14214 assert_eq!(state.lifetime_restarts, 2);
14215 }
14216
14217 let history = runtime
14218 .terminal_ring
14219 .lock()
14220 .expect("terminal ring is not poisoned")
14221 .snapshot();
14222 let last = history
14223 .entries
14224 .last()
14225 .expect("the refused crash is retained");
14226 assert_eq!(last.disposition, TerminalDisposition::Failed);
14227 assert_eq!(
14228 last.disposition_detail.as_deref(),
14229 Some("crash budget exhausted: max_restarts=2 within window_secs=600")
14230 );
14231
14232 let captured = crate::router::test_log::captured_logs(&logs);
14233 assert!(
14234 captured.contains("crash budget exhausted: max_restarts=2 within window_secs=600"),
14235 "the stop must be logged with its window: {captured}"
14236 );
14237 }
14238
14239 #[tokio::test]
14247 async fn a_crash_older_than_the_window_frees_its_slot_for_a_later_crash() {
14248 let supervisor = Supervisor::new_for_test(
14249 Arc::new(Registry::default()),
14250 RestartPolicy::new(2, Duration::ZERO),
14251 );
14252 let runtime = supervisor.runtime_config();
14253 let spec = windowed_crash_spec("crash-across-windows");
14254 let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::starting()));
14255
14256 for attempt in 1..=2 {
14257 assert!(matches!(
14258 on_child_exit(
14259 &spec,
14260 runtime.restart_policy,
14261 &supervisor.registry,
14262 &snapshot,
14263 &runtime.terminal_ring,
14264 &runtime.spawn_events,
14265 &runtime.child_roster,
14266 crash_exit_report(attempt),
14267 true,
14268 )
14269 .await,
14270 NextAction::Restart { schedule: _ }
14271 ));
14272 }
14273
14274 update_snapshot(&snapshot, Some(&spec.module_id), |state| {
14277 age_oldest_crash_restart_out_of_window(state, runtime.restart_policy.window);
14278 })
14279 .unwrap();
14280
14281 assert!(
14282 matches!(
14283 on_child_exit(
14284 &spec,
14285 runtime.restart_policy,
14286 &supervisor.registry,
14287 &snapshot,
14288 &runtime.terminal_ring,
14289 &runtime.spawn_events,
14290 &runtime.child_roster,
14291 crash_exit_report(3),
14292 true,
14293 )
14294 .await,
14295 NextAction::Restart { schedule: _ }
14296 ),
14297 "a crash older than the window must not hold a budget slot"
14298 );
14299
14300 let state = lock_snapshot(&snapshot).unwrap();
14301 assert_eq!(state.state, ModuleState::Restarting);
14302 assert_eq!(
14303 state.crash_restarts.len(),
14304 2,
14305 "the aged instant is dropped and the new one takes its place"
14306 );
14307 assert_eq!(
14308 state.lifetime_restarts, 3,
14309 "the ledger counts every restart, including the ones the window forgot"
14310 );
14311 }
14312
14313 #[tokio::test]
14318 async fn an_operator_restart_clears_the_ring_and_leaves_the_ledger_alone() {
14319 let supervisor = Supervisor::new_for_test(
14320 Arc::new(Registry::default()),
14321 RestartPolicy::new(2, Duration::ZERO),
14322 );
14323 let runtime = supervisor.runtime_config();
14324 let spec = windowed_crash_spec("operator-cleared-budget");
14325 let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::starting()));
14326
14327 for attempt in 1..=2 {
14328 assert!(matches!(
14329 on_child_exit(
14330 &spec,
14331 runtime.restart_policy,
14332 &supervisor.registry,
14333 &snapshot,
14334 &runtime.terminal_ring,
14335 &runtime.spawn_events,
14336 &runtime.child_roster,
14337 crash_exit_report(attempt),
14338 true,
14339 )
14340 .await,
14341 NextAction::Restart { schedule: _ }
14342 ));
14343 }
14344
14345 reset_restart_count(&snapshot, &spec.module_id).unwrap();
14346 {
14347 let state = lock_snapshot(&snapshot).unwrap();
14348 assert!(
14349 state.crash_restarts.is_empty(),
14350 "an operator restart returns the full budget"
14351 );
14352 assert_eq!(
14353 state.lifetime_restarts, 2,
14354 "clearing the budget must not unmake the crashes"
14355 );
14356 }
14357
14358 assert!(
14359 matches!(
14360 on_child_exit(
14361 &spec,
14362 runtime.restart_policy,
14363 &supervisor.registry,
14364 &snapshot,
14365 &runtime.terminal_ring,
14366 &runtime.spawn_events,
14367 &runtime.child_roster,
14368 crash_exit_report(3),
14369 true,
14370 )
14371 .await,
14372 NextAction::Restart { schedule: _ }
14373 ),
14374 "the cleared budget must be spendable again"
14375 );
14376 let state = lock_snapshot(&snapshot).unwrap();
14377 assert_eq!(state.crash_restarts.len(), 1);
14378 assert_eq!(state.lifetime_restarts, 3);
14379 }
14380
14381 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
14382 async fn severance_marker_for_a_dead_child_does_not_label_its_successor() {
14383 let severed = ProcessIdentity {
14384 pid: 41,
14385 start_time: 101,
14386 };
14387 let successor = ProcessIdentity {
14388 pid: 41,
14389 start_time: 202,
14390 };
14391 let module = module_with_recovery_snapshot(ModuleState::Running, true, 0);
14392 update_snapshot(&module.inner.snapshot, Some("recovery-snapshot"), |state| {
14393 state.pid = Some(successor.pid);
14394 state.process_start_time = Some(successor.start_time);
14395 })
14396 .unwrap();
14397 assert!(!module.record_deliberate_severance(severed).unwrap());
14398
14399 let exit_report = apply_deliberate_severance_marker(
14400 &module.inner.snapshot,
14401 Some(successor),
14402 ExitReport {
14403 kind: ExitKind::Crash,
14404 code: Some(1),
14405 signal: None,
14406 at_ms: 1,
14407 },
14408 );
14409
14410 assert_eq!(exit_report.kind, ExitKind::Crash);
14411 }
14412
14413 #[tokio::test]
14414 async fn drain_reap_marks_deliberate_severance_and_records_lifetime_without_budget() {
14415 let registry = Registry::default();
14416 let supervisor = Supervisor::new_for_test(
14417 Arc::new(Registry::default()),
14418 RestartPolicy::new(3, Duration::ZERO),
14419 );
14420 let runtime = supervisor.runtime_config();
14421 let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::starting()));
14422 let spec = ModuleSpec {
14423 module_id: "drain-deliberate-severance".to_string(),
14424 program: fake_aft_stub_path(),
14425 args: Vec::new(),
14426 env: vec![("FAKE_AFT_EXIT_CODE".to_string(), "23".to_string())],
14427 reserved: false,
14428 reserved_prefixes: Vec::new(),
14429 protocol: ModuleProtocol::Subc,
14430 overlap: Default::default(),
14431 carry_containment: false,
14432 };
14433 let mut child = spawn_and_mark_running(&spec, &runtime, &snapshot).unwrap();
14434 let process = ProcessIdentity {
14435 pid: 41,
14436 start_time: 101,
14437 };
14438 child.process_identity = Some(process);
14439 update_snapshot(&snapshot, Some(&spec.module_id), |state| {
14440 state.pid = Some(process.pid);
14441 state.process_start_time = Some(process.start_time);
14442 })
14443 .unwrap();
14444 record_deliberate_severance(&snapshot, process).unwrap();
14445
14446 drain_child_to_state(
14447 &spec.module_id,
14448 spec.protocol,
14449 StopNotice::SentOverConnection,
14452 ®istry,
14453 None,
14454 &snapshot,
14455 &runtime.terminal_ring,
14456 &runtime.spawn_events,
14457 child,
14458 Duration::from_secs(1),
14459 ModuleState::Stopped,
14460 Some(false),
14461 &super::FailureContext::operator(),
14462 )
14463 .await
14464 .unwrap();
14465
14466 let state = lock_snapshot(&snapshot).unwrap();
14467 assert_eq!(
14468 state.last_exit.as_ref().map(|exit| exit.kind),
14469 Some(ExitKind::DeliberateSeverance)
14470 );
14471 assert_eq!(state.lifetime_restarts, 1);
14472 assert_eq!(state.crash_restarts.len(), 0);
14473 drop(state);
14474 let history = runtime.terminal_ring.lock().unwrap().snapshot();
14475 assert_eq!(
14476 history.entries[0].exit_kind,
14477 subc_control::TerminalExitKind::DeliberateSeverance
14478 );
14479 }
14480
14481 #[tokio::test]
14482 async fn ordinary_drain_reap_does_not_record_a_lifetime_restart() {
14483 let registry = Registry::default();
14484 let supervisor = Supervisor::new_for_test(
14485 Arc::new(Registry::default()),
14486 RestartPolicy::new(3, Duration::ZERO),
14487 );
14488 let runtime = supervisor.runtime_config();
14489 let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::starting()));
14490 let spec = ModuleSpec {
14491 module_id: "ordinary-drain".to_string(),
14492 program: fake_aft_stub_path(),
14493 args: Vec::new(),
14494 env: vec![("FAKE_AFT_EXIT_CODE".to_string(), "23".to_string())],
14495 reserved: false,
14496 reserved_prefixes: Vec::new(),
14497 protocol: ModuleProtocol::Subc,
14498 overlap: Default::default(),
14499 carry_containment: false,
14500 };
14501 let child = spawn_and_mark_running(&spec, &runtime, &snapshot).unwrap();
14502
14503 drain_child_to_state(
14504 &spec.module_id,
14505 spec.protocol,
14506 StopNotice::SentOverConnection,
14509 ®istry,
14510 None,
14511 &snapshot,
14512 &runtime.terminal_ring,
14513 &runtime.spawn_events,
14514 child,
14515 Duration::from_secs(1),
14516 ModuleState::Stopped,
14517 Some(false),
14518 &super::FailureContext::operator(),
14519 )
14520 .await
14521 .unwrap();
14522
14523 let state = lock_snapshot(&snapshot).unwrap();
14524 assert_eq!(
14525 state.last_exit.as_ref().map(|exit| exit.kind),
14526 Some(ExitKind::Crash)
14527 );
14528 assert_eq!(state.lifetime_restarts, 0);
14529 assert_eq!(state.crash_restarts.len(), 0);
14530 }
14531
14532 #[test]
14533 fn fatal_connection_teardown_cannot_arm_a_marker_for_a_surviving_process() {
14534 assert!(!include_str!("server.rs")
14540 .contains("router.record_deliberate_connection_severance(ctx.connection_id)"));
14541 }
14542
14543 #[test]
14550 fn drained_after_quiescence_wait_passes_ok_through_and_forces_false_on_err() {
14551 assert!(drained_after_quiescence_wait(&Ok(true)));
14552 assert!(!drained_after_quiescence_wait(&Ok(false)));
14553 assert!(!drained_after_quiescence_wait(&Err(
14554 SuperviseError::StatePoisoned { module_id: None }
14555 )));
14556 }
14557
14558 #[test]
14567 fn wait_error_exit_report_records_a_failed_terminal_with_no_code_or_signal() {
14568 let ring = Arc::new(Mutex::new(TerminalRing::new(
14569 TerminalRingConfig::default(),
14570 0,
14571 )));
14572 record_wait_error_terminal("wait-error", &ring, &super::SpawnEventFeed::default());
14573
14574 let snapshot = ring.lock().unwrap().snapshot();
14575 assert_eq!(snapshot.entries.len(), 1);
14576 let entry = &snapshot.entries[0];
14577 assert_eq!(entry.exit_code, None);
14578 assert_eq!(entry.exit_signal, None);
14579 assert_eq!(entry.disposition, TerminalDisposition::Failed);
14580 }
14581
14582 #[test]
14583 fn wait_error_exit_path_preserves_spawn_event_density() {
14584 let feed = super::SpawnEventFeed::default();
14585 feed.configure_incarnation("wait-error-density".to_string());
14586 feed.emit_spawned("wait-error", 41, 1);
14587 let ring = Arc::new(Mutex::new(TerminalRing::new(
14588 TerminalRingConfig::default(),
14589 0,
14590 )));
14591
14592 record_wait_error_terminal("wait-error", &ring, &feed);
14593 feed.emit_spawned("after-wait-error", 42, 2);
14594
14595 let state = feed.0.lock().unwrap();
14596 let sequences = state
14597 .events
14598 .iter()
14599 .map(|event| event.cursor.seq)
14600 .collect::<Vec<_>>();
14601 assert_eq!(sequences, vec![1, 2, 3]);
14602 assert_eq!(state.events[1].kind, SpawnEventKind::Exited);
14603 assert_eq!(state.events[1].exit_code, None);
14604 assert_eq!(state.events[1].exit_signal, None);
14605 }
14606
14607 #[test]
14611 fn wait_error_exit_report_is_classified_as_a_crash() {
14612 assert_eq!(wait_error_exit_report().kind, ExitKind::Crash);
14613 }
14614}
14615
14616#[cfg(all(test, unix))]
14617mod restart_ordering_tests {
14618 use super::*;
14619 use std::io::Write;
14620 use tokio::io::AsyncReadExt;
14621
14622 #[test]
14623 #[ignore = "child process fixture"]
14624 fn serving_child() {
14625 let Ok(address_file) = std::env::var("RESTART_TEST_ADDRESS_FILE") else {
14626 return;
14627 };
14628 let listener = std::net::TcpListener::bind("127.0.0.1:0").unwrap();
14629 std::fs::write(address_file, listener.local_addr().unwrap().to_string()).unwrap();
14630 for stream in listener.incoming() {
14631 writeln!(stream.unwrap(), "{}", std::process::id()).unwrap();
14632 }
14633 }
14634
14635 async fn restart_before_publication(requests: usize) {
14636 let home = cortexkit_test_support::ScratchDir::new("restart-before-publication");
14637 let address_file = home.path().join("address");
14638 let spec = ModuleSpec {
14639 module_id: "restart-serving-child".into(),
14640 program: std::env::current_exe().unwrap(),
14641 args: vec![
14642 "--exact".into(),
14643 "supervise::restart_ordering_tests::serving_child".into(),
14644 "--ignored".into(),
14645 "--nocapture".into(),
14646 ],
14647 env: vec![(
14648 "RESTART_TEST_ADDRESS_FILE".into(),
14649 address_file.display().to_string(),
14650 )],
14651 reserved: false,
14652 reserved_prefixes: vec![],
14653 protocol: ModuleProtocol::None,
14654 overlap: Default::default(),
14655 carry_containment: false,
14656 };
14657 let mut supervisor =
14658 Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default());
14659 let (spawned_tx, spawned_rx) = oneshot::channel();
14660 let spawned_tx = Mutex::new(Some(spawned_tx));
14661 let (release_tx, release_rx) = std::sync::mpsc::channel();
14662 let release_rx = Mutex::new(release_rx);
14663 supervisor.test_after_first_spawn = AfterFirstSpawnHook(Some(Arc::new(move |_| {
14666 spawned_tx.lock().unwrap().take().unwrap().send(()).unwrap();
14667 release_rx
14668 .lock()
14669 .unwrap()
14670 .recv_timeout(Duration::from_secs(20))
14671 .unwrap();
14672 })));
14673 let registered = supervisor
14674 .register_configured_with_health(
14675 spec.clone(),
14676 true,
14677 HealthConfig::default(),
14678 None,
14679 RestartPolicy::new(3, Duration::ZERO),
14680 DEFAULT_REGISTER_TIMEOUT,
14681 )
14682 .unwrap();
14683 let module = registered.module.clone();
14684 let startup = tokio::task::spawn_blocking(move || {
14685 supervisor.start_registered_configured(registered, None, true)
14686 });
14687 spawned_rx.await.unwrap();
14688 let old_pid = tokio::time::timeout(Duration::from_secs(10), async {
14689 let address = loop {
14690 if let Ok(address) = std::fs::read_to_string(&address_file) {
14691 if !address.is_empty() {
14692 break address;
14693 }
14694 }
14695 tokio::time::sleep(Duration::from_millis(1)).await;
14696 };
14697 let mut stream = tokio::net::TcpStream::connect(address).await.unwrap();
14698 let mut reply = String::new();
14699 stream.read_to_string(&mut reply).await.unwrap();
14700 reply.trim().parse::<u32>().unwrap()
14701 })
14702 .await
14703 .expect("child must serve before generation publication");
14704 assert_eq!(
14705 lock_snapshot(&module.inner.snapshot)
14706 .unwrap()
14707 .spawn_generation,
14708 0
14709 );
14710 let mut replies = Vec::new();
14711 for _ in 0..requests {
14712 let mut restart = Box::pin(module.restart(Some(0)));
14713 std::future::poll_fn(|cx| {
14716 assert!(std::future::Future::poll(restart.as_mut(), cx).is_pending());
14717 std::task::Poll::Ready(())
14718 })
14719 .await;
14720 replies.push(restart);
14721 }
14722 release_tx.send(()).unwrap();
14723 startup.await.unwrap().unwrap();
14724 for reply in replies {
14725 reply.await.unwrap();
14726 }
14727 let replacement = tokio::time::timeout(Duration::from_secs(10), async {
14730 loop {
14731 if lock_snapshot(&module.inner.snapshot)
14732 .unwrap()
14733 .spawn_generation
14734 > 1
14735 {
14736 break;
14737 }
14738 tokio::time::sleep(Duration::from_millis(1)).await;
14739 }
14740 })
14741 .await;
14742 module.update_spec_for_test(spec).await.unwrap();
14746 let (generation, pid) = {
14747 let state = lock_snapshot(&module.inner.snapshot).unwrap();
14748 (state.spawn_generation, state.pid)
14749 };
14750 module.stop().await.unwrap();
14751 replacement.expect("restart must publish a replacement for the already-serving child");
14752 assert_ne!(
14753 pid,
14754 Some(old_pid),
14755 "a serving child predating the restart must be replaced"
14756 );
14757 assert_eq!(generation, 2, "queued restarts must share one replacement");
14758 }
14759
14760 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
14761 async fn restart_received_after_serving_before_generation_publication_replaces_child() {
14762 restart_before_publication(1).await;
14763 }
14764
14765 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
14766 async fn restart_queued_behind_restart_does_not_kill_its_replacement() {
14767 restart_before_publication(2).await;
14768 }
14769
14770 #[tokio::test]
14771 async fn restart_queued_before_replacement_spawn_keeps_the_new_child() {
14772 let supervisor =
14773 Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default());
14774 let registered = supervisor
14775 .register_configured_with_health(
14776 ModuleSpec {
14777 module_id: "queued-before-replacement".into(),
14778 program: "/bin/sleep".into(),
14783 args: vec!["600".into()],
14784 env: vec![],
14785 reserved: false,
14786 reserved_prefixes: vec![],
14787 protocol: ModuleProtocol::None,
14788 overlap: Default::default(),
14789 carry_containment: false,
14790 },
14791 true,
14792 HealthConfig::default(),
14793 None,
14794 RestartPolicy::default(),
14795 DEFAULT_REGISTER_TIMEOUT,
14796 )
14797 .unwrap();
14798 let RegisteredConfiguredModule {
14799 mut spec,
14800 mut runtime,
14801 module,
14802 mut commands,
14803 } = registered;
14804 let snapshot = &module.inner.snapshot;
14805 let mut restart = Box::pin(module.restart(Some(0)));
14806 std::future::poll_fn(|cx| {
14807 assert!(std::future::Future::poll(restart.as_mut(), cx).is_pending());
14808 std::task::Poll::Ready(())
14809 })
14810 .await;
14811 let mut child = Some(spawn_and_mark_running(&spec, &runtime, snapshot).unwrap());
14814 let replacement_pid = child.as_ref().unwrap().id();
14815 let command = commands.recv().await.unwrap();
14816 assert!(
14817 handle_supervisor_command(
14818 command,
14819 &mut spec,
14820 &mut runtime,
14821 &supervisor.registry,
14822 &supervisor.process_liveness,
14823 snapshot,
14824 &mut child,
14825 &mut commands,
14826 &mut VecDeque::new(),
14827 )
14828 .await
14829 );
14830 restart.await.unwrap();
14831 let kept_pid = child.as_ref().and_then(SupervisedChild::id);
14832 let pending = lock_snapshot(snapshot).unwrap().respawn_pending;
14833 if let Some(mut child) = child {
14834 child.start_kill().unwrap();
14835 child.wait().await.unwrap();
14836 }
14837 assert_eq!(
14838 kept_pid, replacement_pid,
14839 "a newer replacement must survive"
14840 );
14841 assert!(
14842 !pending,
14843 "a satisfied restart must not schedule another spawn"
14844 );
14845 }
14846}
14847
14848#[cfg(all(test, unix))]
14849mod health_event_tests {
14850 use super::*;
14851 use cortexkit_test_support::ScratchDir;
14852 use subc_protocol::{manifest::Concurrency, session::ModuleControlResponse};
14853
14854 struct Actor {
14855 module: SupervisedModule,
14856 registry: Arc<Registry>,
14857 forwarding: Arc<ForwardingTable>,
14858 spec: ModuleSpec,
14859 health: HealthConfig,
14860 rx: mpsc::Receiver<crate::router::OutboundFrame>,
14861 roster: ChildRoster,
14862 _home: ScratchDir,
14863 }
14864
14865 async fn wait_for<T>(reason: &str, mut observe: impl FnMut() -> Option<T>) -> T {
14866 let deadline = std::time::Instant::now() + Duration::from_secs(10);
14870 let tick = Instant::now();
14871 loop {
14872 if let Some(value) = observe() {
14873 assert_eq!(Instant::now(), tick, "{reason} must not wait for a timer");
14874 return value;
14875 }
14876 assert!(
14877 std::time::Instant::now() < deadline,
14878 "timed out waiting for {reason}"
14879 );
14880 tokio::task::yield_now().await;
14881 }
14882 }
14883
14884 impl Actor {
14885 async fn start(protocol: ModuleProtocol, cadence: Duration) -> Self {
14886 Self::start_with_policy(
14887 protocol,
14888 cadence,
14889 DEFAULT_REGISTER_TIMEOUT,
14890 RestartPolicy::new(3, Duration::ZERO),
14891 )
14892 .await
14893 }
14894
14895 async fn start_with_policy(
14896 protocol: ModuleProtocol,
14897 cadence: Duration,
14898 register_timeout: Duration,
14899 policy: RestartPolicy,
14900 ) -> Self {
14901 let home = ScratchDir::new("health-events");
14902 let registry = Arc::new(Registry::default());
14903 let forwarding = Arc::new(ForwardingTable::default());
14904 let health = HealthConfig {
14905 cadence,
14906 deadline: Duration::from_secs(3600),
14907 ..HealthConfig::default()
14908 };
14909 let supervisor = Supervisor::new_for_test(registry.clone(), policy)
14910 .with_forwarding(forwarding.clone())
14911 .with_health_config(health.clone());
14912 let spec = ModuleSpec {
14913 module_id: "event-health-child".into(),
14914 program: PathBuf::from("/bin/sleep"),
14922 args: vec!["600".into()],
14925 env: ["XDG_DATA_HOME", "XDG_RUNTIME_DIR", "XDG_CONFIG_HOME"]
14926 .into_iter()
14927 .map(|key| (key.into(), home.path().display().to_string()))
14928 .collect(),
14929 reserved: false,
14930 reserved_prefixes: vec![],
14931 protocol,
14932 overlap: Default::default(),
14933 carry_containment: false,
14934 };
14935 let module = supervisor
14936 .supervise_configured_requested(
14937 spec.clone(),
14938 true,
14939 health.clone(),
14940 Some(0),
14941 policy,
14942 register_timeout,
14943 None,
14944 )
14945 .unwrap();
14946 let (_, rx) = mpsc::channel(8);
14947 let actor = Self {
14948 module,
14949 registry,
14950 forwarding,
14951 spec,
14952 health,
14953 rx,
14954 roster: supervisor.child_roster.clone(),
14955 _home: home,
14956 };
14957 actor
14958 .wait_for_select(|checkpoint| checkpoint.generation > 0)
14959 .await;
14960 actor
14961 }
14962
14963 async fn expire_registration_wait(&self, remaining: Duration) {
14964 tokio::time::advance(remaining).await;
14965 wait_for("registration timeout TERM grace", || {
14966 lock_snapshot(&self.module.inner.snapshot)
14967 .unwrap()
14968 .actor_select
14969 .is_none()
14970 .then_some(())
14971 })
14972 .await;
14973 tokio::time::advance(crate::child_roster::TERM_TO_KILL).await;
14974 }
14975
14976 fn turns(&self) -> u64 {
14977 lock_snapshot(&self.module.inner.snapshot)
14978 .unwrap()
14979 .actor_turns
14980 }
14981
14982 async fn wait_for_select(
14983 &self,
14984 expected: impl Fn(&ActorSelectCheckpoint) -> bool,
14985 ) -> ActorSelectCheckpoint {
14986 wait_for("actor select checkpoint", || {
14987 let state = lock_snapshot(&self.module.inner.snapshot).unwrap();
14988 state.actor_select.filter(|checkpoint| {
14989 checkpoint.generation == state.spawn_generation
14990 && state.state == ModuleState::Running
14991 && state.process_alive
14992 && state.reported_pid().is_some()
14993 && expected(checkpoint)
14994 })
14995 })
14996 .await
14997 }
14998
14999 fn hello(&mut self, connection: u64, health: bool) {
15000 let manifest =
15001 subc_protocol::manifest::ModuleManifest::builder(&self.spec.module_id, "1.0.0")
15002 .protocol_ver(subc_protocol::PROTOCOL_VERSION)
15003 .build();
15004 let connection = ConnectionId::new(connection);
15005 let (tx, rx) = mpsc::channel(8);
15006 self.rx = rx;
15007 self.forwarding
15008 .register_module_connection(
15009 connection,
15010 self.spec.module_id.clone(),
15011 subc_protocol::PROTOCOL_VERSION,
15012 Concurrency::ModuleManaged,
15013 FrameSink::new(tx),
15014 )
15015 .unwrap();
15016 self.registry
15017 .register_with_control_ops(
15018 manifest,
15019 subc_protocol::PROTOCOL_VERSION,
15020 connection,
15021 if health {
15022 vec![MODULE_CONTROL_OP_HEALTH_CHECK.into()]
15023 } else {
15024 vec![]
15025 },
15026 )
15027 .unwrap();
15028 }
15029
15030 async fn probe(&mut self) -> crate::router::OutboundFrame {
15031 let frame = wait_for("outbound health probe", || match self.rx.try_recv() {
15032 Ok(frame) => Some(frame),
15033 Err(mpsc::error::TryRecvError::Empty) => None,
15034 Err(mpsc::error::TryRecvError::Disconnected) => panic!("health peer disconnected"),
15035 })
15036 .await;
15037 let body: Value = serde_json::from_slice(&frame.body).unwrap();
15038 assert_eq!(body["op"], MODULE_CONTROL_OP_HEALTH_CHECK);
15039 frame
15040 }
15041
15042 fn no_probe(&mut self) {
15043 assert!(matches!(
15044 self.rx.try_recv(),
15045 Err(mpsc::error::TryRecvError::Empty)
15046 ));
15047 }
15048
15049 fn answer(&self, connection: u64, frame: crate::router::OutboundFrame) {
15050 self.forwarding
15051 .complete_module_control_rpc(
15052 ConnectionId::new(connection),
15053 frame.header.corr,
15054 Some(MODULE_CONTROL_OP_HEALTH_CHECK),
15055 ModuleControlRpcOutcome::Response(ModuleControlResponse::HealthCheck {
15056 status: HealthStatus::Ok,
15057 detail: None,
15058 metrics: None,
15059 }),
15060 )
15061 .unwrap();
15062 }
15063 }
15064
15065 #[tokio::test(start_paused = true)]
15066 async fn registration_timeout_kills_at_120_seconds_and_respawns() {
15067 let actor = Actor::start(ModuleProtocol::Subc, Duration::from_secs(30)).await;
15068 let initial = actor.module.status().unwrap();
15069 tokio::time::advance(Duration::from_secs(119)).await;
15070 tokio::task::yield_now().await;
15071 assert_eq!(actor.module.status().unwrap().pid, initial.pid);
15072 assert!(actor.module.terminal_history().entries.is_empty());
15073 actor.expire_registration_wait(Duration::from_secs(1)).await;
15074 actor
15075 .wait_for_select(|at| at.generation > initial.spawn_generation)
15076 .await;
15077 let status = actor.module.status().unwrap();
15078 assert_ne!(status.pid, initial.pid);
15079 assert!(!cortexkit_test_support::process_alive(
15080 initial.pid.unwrap() as i32
15081 ));
15082 assert_eq!(status.restart_count, 1);
15083 assert_eq!(status.last_exit.unwrap().kind, ExitKind::Crash);
15084 let entries = actor.module.terminal_history().entries;
15085 assert_eq!(entries.len(), 1);
15086 assert_eq!(
15087 entries[0].disposition_detail.as_deref(),
15088 Some("registration_timeout")
15089 );
15090 actor.module.stop().await.unwrap();
15091 }
15092
15093 #[tokio::test(start_paused = true)]
15094 async fn registration_timeout_is_cancelled_by_hello_at_119_seconds() {
15095 let mut actor = Actor::start(ModuleProtocol::Subc, Duration::from_secs(30)).await;
15096 let pid = actor.module.status().unwrap().pid;
15097 tokio::time::advance(Duration::from_secs(119)).await;
15098 actor.hello(1, false);
15099 let registered = actor
15100 .wait_for_select(|at| at.registered_connection.is_some())
15101 .await;
15102 assert_eq!(
15103 registered.wake_after, None,
15104 "HELLO must cancel the deadline"
15105 );
15106 tokio::time::advance(Duration::from_secs(300)).await;
15107 tokio::task::yield_now().await;
15108 assert_eq!(actor.module.status().unwrap().pid, pid);
15109 assert!(actor.module.terminal_history().entries.is_empty());
15110 actor
15112 .registry
15113 .deregister_connection(ConnectionId::new(1))
15114 .unwrap();
15115 actor
15116 .wait_for_select(|at| at.registered_connection.is_none())
15117 .await;
15118 tokio::time::advance(Duration::from_secs(300)).await;
15119 tokio::task::yield_now().await;
15120 assert_eq!(actor.module.status().unwrap().pid, pid);
15121 assert!(actor.module.terminal_history().entries.is_empty());
15122 actor.module.stop().await.unwrap();
15123
15124 let mut transient = Actor::start(ModuleProtocol::Subc, Duration::from_secs(30)).await;
15125 let pid = transient.module.status().unwrap().pid;
15126 tokio::time::advance(Duration::from_secs(119)).await;
15127 let turns = transient.turns();
15128 transient.hello(1, false);
15131 transient
15132 .registry
15133 .deregister_connection(ConnectionId::new(1))
15134 .unwrap();
15135 let parked = transient.wait_for_select(|at| at.turn > turns).await;
15136 assert_eq!(
15137 parked.wake_after, None,
15138 "transient HELLO must cancel the deadline"
15139 );
15140 tokio::time::advance(Duration::from_secs(300)).await;
15141 tokio::task::yield_now().await;
15142 assert_eq!(transient.module.status().unwrap().pid, pid);
15143 assert!(transient.module.terminal_history().entries.is_empty());
15144 transient.module.stop().await.unwrap();
15145 }
15146
15147 #[tokio::test(start_paused = true)]
15148 async fn registration_timeouts_exhaust_budget_and_schedule_slow_retry() {
15149 let actor = Actor::start_with_policy(
15150 ModuleProtocol::Subc,
15151 Duration::from_secs(30),
15152 Duration::from_secs(10),
15153 RestartPolicy::new(1, Duration::ZERO),
15154 )
15155 .await;
15156 let generation = actor.module.status().unwrap().spawn_generation;
15157 actor
15158 .expire_registration_wait(Duration::from_secs(10))
15159 .await;
15160 actor.wait_for_select(|at| at.generation > generation).await;
15161 actor
15162 .expire_registration_wait(Duration::from_secs(10))
15163 .await;
15164 let retry = wait_for("failed retry schedule", || {
15165 let state = lock_snapshot(&actor.module.inner.snapshot).unwrap();
15166 (state.state == ModuleState::Failed)
15167 .then_some(state.failed_retry.pending)
15168 .flatten()
15169 })
15170 .await;
15171 assert_eq!(retry.delay, Duration::from_secs(600));
15172 assert_eq!(retry.attempt, 1);
15173 let entries = actor.module.terminal_history().entries;
15174 assert_eq!(entries.len(), 2);
15175 assert_eq!(entries[1].disposition, TerminalDisposition::Failed);
15176 assert!(entries[1]
15177 .disposition_detail
15178 .as_deref()
15179 .unwrap()
15180 .contains("registration_timeout"));
15181 tokio::time::advance(Duration::from_secs(599)).await;
15182 tokio::task::yield_now().await;
15183 assert_eq!(actor.module.state().unwrap(), ModuleState::Failed);
15184 tokio::time::advance(Duration::from_secs(1)).await;
15185 actor
15186 .wait_for_select(|at| at.generation > generation + 1)
15187 .await;
15188 actor
15190 .expire_registration_wait(Duration::from_secs(10))
15191 .await;
15192 wait_for("slow retry registration timeout", || {
15193 (actor.module.terminal_history().entries.len() == 3).then_some(())
15194 })
15195 .await;
15196 actor.module.stop().await.unwrap();
15197 }
15198
15199 #[tokio::test(start_paused = true)]
15200 async fn registration_timeout_honours_ten_second_override() {
15201 let actor = Actor::start_with_policy(
15202 ModuleProtocol::Subc,
15203 Duration::from_secs(30),
15204 Duration::from_secs(10),
15205 RestartPolicy::new(3, Duration::ZERO),
15206 )
15207 .await;
15208 let generation = actor.module.status().unwrap().spawn_generation;
15209 tokio::time::advance(Duration::from_secs(9)).await;
15210 tokio::task::yield_now().await;
15211 assert!(actor.module.terminal_history().entries.is_empty());
15212 actor.expire_registration_wait(Duration::from_secs(1)).await;
15213 actor.wait_for_select(|at| at.generation > generation).await;
15214 assert_eq!(actor.module.terminal_history().entries.len(), 1);
15215 actor.module.stop().await.unwrap();
15216 }
15217
15218 #[tokio::test]
15219 async fn registration_timeout_kills_term_ignoring_helper_before_reaping_leader() {
15220 use std::os::unix::process::{CommandExt, ExitStatusExt};
15221
15222 struct Helper(std::process::Child);
15223 impl Drop for Helper {
15224 fn drop(&mut self) {
15225 if matches!(self.0.try_wait(), Ok(None)) {
15228 let _ = self.0.kill();
15229 let _ = self.0.wait();
15230 }
15231 }
15232 }
15233
15234 let home = ScratchDir::new("registration-timeout-helper");
15235 let supervisor = Supervisor::new_for_test(
15236 Arc::new(Registry::default()),
15237 RestartPolicy::new(0, Duration::ZERO),
15238 );
15239 let spec = ModuleSpec {
15240 module_id: "registration-timeout-group".into(),
15241 program: "/bin/sleep".into(),
15242 args: vec!["120".into()],
15243 env: ["XDG_DATA_HOME", "XDG_RUNTIME_DIR", "XDG_CONFIG_HOME"]
15244 .into_iter()
15245 .map(|key| (key.into(), home.path().display().to_string()))
15246 .collect(),
15247 reserved: false,
15248 reserved_prefixes: vec![],
15249 protocol: ModuleProtocol::Subc,
15250 overlap: Default::default(),
15251 carry_containment: false,
15252 };
15253 let module = supervisor
15254 .supervise_configured_requested(
15255 spec,
15256 true,
15257 HealthConfig::default(),
15258 None,
15259 RestartPolicy::new(0, Duration::ZERO),
15260 Duration::from_secs(10),
15261 None,
15262 )
15263 .unwrap();
15264 let leader = timeout(Duration::from_secs(5), async {
15265 loop {
15266 if let Some(pid) = module.status().unwrap().pid {
15267 break pid;
15268 }
15269 sleep(Duration::from_millis(5)).await;
15270 }
15271 })
15272 .await
15273 .expect("supervised group leader must be reported before the deadline");
15274 assert!(leader > 1);
15275 let ready = home.join("helper.ready");
15276 let mut helper = Helper(
15279 std::process::Command::new("/bin/sh")
15280 .args([
15281 "-c",
15282 "trap '' TERM; : > \"$1\"; exec /bin/sleep 120",
15283 "helper",
15284 ])
15285 .arg(&ready)
15286 .process_group(i32::try_from(leader).unwrap())
15287 .stdout(std::process::Stdio::null())
15288 .stderr(std::process::Stdio::null())
15289 .spawn()
15290 .unwrap(),
15291 );
15292 timeout(Duration::from_secs(5), async {
15293 while !ready.exists() {
15294 assert!(helper.0.try_wait().unwrap().is_none());
15295 sleep(Duration::from_millis(5)).await;
15296 }
15297 })
15298 .await
15299 .expect("helper must install TERM ignore before the registration deadline");
15300 let helper_pid =
15301 rustix::process::Pid::from_raw(i32::try_from(helper.0.id()).unwrap()).unwrap();
15302 rustix::process::kill_process(helper_pid, rustix::process::Signal::TERM).unwrap();
15303 sleep(Duration::from_millis(20)).await;
15304 assert!(
15305 helper.0.try_wait().unwrap().is_none(),
15306 "helper must ignore TERM"
15307 );
15308
15309 timeout(Duration::from_secs(20), async {
15310 while module.state().unwrap() != ModuleState::Failed {
15311 sleep(Duration::from_millis(5)).await;
15312 }
15313 })
15314 .await
15315 .expect("registration timeout must stop and reap the group leader");
15316 let helper_exit = timeout(Duration::from_secs(3), async {
15317 loop {
15318 if let Some(status) = helper.0.try_wait().unwrap() {
15319 break status;
15320 }
15321 sleep(Duration::from_millis(5)).await;
15322 }
15323 })
15324 .await
15325 .expect("TERM-ignoring helper survived the registration timeout's group KILL");
15326 assert_eq!(helper_exit.signal(), Some(9));
15327 assert!(!cortexkit_test_support::process_alive(
15328 i32::try_from(leader).unwrap()
15329 ));
15330 assert_eq!(
15331 module.status().unwrap().last_exit.unwrap().kind,
15332 ExitKind::Crash
15333 );
15334 assert!(module.terminal_history().entries[0]
15335 .disposition_detail
15336 .as_deref()
15337 .unwrap()
15338 .contains("registration_timeout"));
15339 module.stop().await.unwrap();
15340 }
15341
15342 #[tokio::test(start_paused = true)]
15343 async fn registration_timeout_does_not_count_an_earlier_exit() {
15344 let actor = Actor::start_with_policy(
15345 ModuleProtocol::Subc,
15346 Duration::from_secs(30),
15347 DEFAULT_REGISTER_TIMEOUT,
15348 RestartPolicy::new(0, Duration::ZERO),
15349 )
15350 .await;
15351 let pid = actor.module.status().unwrap().pid.unwrap();
15352 rustix::process::kill_process(
15353 rustix::process::Pid::from_raw(pid as i32).unwrap(),
15354 rustix::process::Signal::TERM,
15355 )
15356 .unwrap();
15357 wait_for("early exit", || {
15358 (actor.module.state().unwrap() == ModuleState::Failed).then_some(())
15359 })
15360 .await;
15361 tokio::time::advance(Duration::from_secs(121)).await;
15362 tokio::task::yield_now().await;
15363 let entries = actor.module.terminal_history().entries;
15364 assert_eq!(entries.len(), 1);
15365 assert!(!entries[0]
15366 .disposition_detail
15367 .as_deref()
15368 .unwrap()
15369 .contains("registration_timeout"));
15370 assert_eq!(actor.module.status().unwrap().lifetime_restarts, 0);
15371 actor.module.stop().await.unwrap();
15372 }
15373
15374 #[tokio::test(start_paused = true)]
15375 async fn registration_timeout_is_cancelled_by_daemon_shutdown() {
15376 let actor = Actor::start(ModuleProtocol::Subc, Duration::from_secs(30)).await;
15377 let pid = actor.module.status().unwrap().pid;
15378 tokio::time::advance(Duration::from_secs(119)).await;
15379 actor.roster.close();
15380 let turns = actor.turns();
15381 tokio::time::advance(Duration::from_secs(300)).await;
15382 let parked = actor.wait_for_select(|at| at.turn > turns).await;
15383 assert_eq!(parked.wake_after, None);
15384 assert_eq!(actor.module.status().unwrap().pid, pid);
15385 assert!(actor.module.terminal_history().entries.is_empty());
15386 actor.module.stop().await.unwrap();
15387 let status = actor.module.status().unwrap();
15388 assert_eq!(status.restart_count, 0);
15389 assert_eq!(
15390 actor.module.terminal_history().entries[0].disposition,
15391 TerminalDisposition::DaemonShutdown
15392 );
15393 }
15394
15395 #[tokio::test(start_paused = true)]
15396 async fn no_health_children_park_for_sixty_seconds() {
15397 let none = Actor::start(ModuleProtocol::None, Duration::from_secs(30)).await;
15398 let unregistered = Actor::start(ModuleProtocol::Subc, Duration::from_secs(30)).await;
15399 let mut unadvertised = Actor::start(ModuleProtocol::Subc, Duration::from_secs(30)).await;
15400 unadvertised.hello(1, false);
15401 for _ in 0..6000 {
15406 tokio::time::advance(Duration::from_millis(10)).await;
15407 tokio::task::yield_now().await;
15408 }
15409 for actor in [&none, &unregistered, &unadvertised] {
15410 assert!(
15411 actor.turns() <= 5,
15412 "no-health actor woke {} times",
15413 actor.turns()
15414 );
15415 assert_eq!(actor.module.state().unwrap(), ModuleState::Running);
15416 }
15417 }
15418
15419 #[tokio::test(start_paused = true)]
15420 async fn late_hello_starts_a_due_probe_in_the_notification_tick() {
15421 let mut actor = Actor::start(ModuleProtocol::Subc, Duration::ZERO).await;
15425 let tick = Instant::now();
15426 actor.hello(1, true);
15427 actor.probe().await;
15428 assert_eq!(Instant::now(), tick, "HELLO must not wait for a poll");
15429 }
15430
15431 #[tokio::test(start_paused = true)]
15432 async fn hello_keeps_the_jittered_cadence_and_catalog_updates_do_not_reset_it() {
15433 let mut actor = Actor::start(ModuleProtocol::Subc, Duration::from_secs(30)).await;
15434 let hello_tick = Instant::now();
15435 actor.hello(1, true);
15436 let armed = actor
15437 .wait_for_select(|checkpoint| {
15438 checkpoint.registered_connection == Some(ConnectionId::new(1))
15439 && checkpoint.next_probe_at.is_some()
15440 })
15441 .await;
15442 let delay = jittered_health_delay(&actor.spec.module_id, 0, actor.health.cadence);
15443 assert!((Duration::from_secs(30)..Duration::from_secs(33)).contains(&delay));
15444 let first_deadline = hello_tick + delay;
15445 assert_eq!(armed.next_probe_at, Some(first_deadline));
15446 tokio::time::advance(Duration::from_secs(29)).await;
15447 let turns = actor.turns();
15448 actor
15449 .registry
15450 .replace_catalog_for_connection(ConnectionId::new(1), vec![], None, Some(true))
15451 .unwrap();
15452 let updated = actor
15453 .wait_for_select(|checkpoint| checkpoint.turn > turns)
15454 .await;
15455 assert_eq!(
15456 updated.next_probe_at,
15457 Some(first_deadline),
15458 "catalog update must not reset cadence"
15459 );
15460 actor.no_probe();
15461 tokio::time::advance(first_deadline - Instant::now()).await;
15462 let frame = actor.probe().await;
15463 actor.answer(1, frame);
15464 let next = jittered_health_delay(&actor.spec.module_id, 1, actor.health.cadence);
15465 let next_deadline = Instant::now() + next;
15466 let rearmed = actor
15467 .wait_for_select(|checkpoint| {
15468 checkpoint
15469 .next_probe_at
15470 .is_some_and(|deadline| deadline > first_deadline)
15471 })
15472 .await;
15473 assert_eq!(rearmed.next_probe_at, Some(next_deadline));
15474 tokio::time::advance(next - Duration::from_nanos(1)).await;
15475 actor.no_probe();
15476 tokio::time::advance(Duration::from_nanos(1)).await;
15477 actor.probe().await;
15478 }
15479
15480 #[tokio::test(start_paused = true)]
15481 async fn disconnect_disarms_and_reconnect_rearms_health() {
15482 let mut actor = Actor::start(ModuleProtocol::Subc, Duration::from_secs(30)).await;
15483 actor.hello(1, true);
15484 actor
15485 .wait_for_select(|checkpoint| {
15486 checkpoint.registered_connection == Some(ConnectionId::new(1))
15487 && checkpoint.next_probe_at.is_some()
15488 })
15489 .await;
15490 let turns = actor.turns();
15491 let tick = Instant::now();
15492 actor
15493 .registry
15494 .deregister_connection(ConnectionId::new(1))
15495 .unwrap();
15496 let disarmed = actor
15497 .wait_for_select(|checkpoint| {
15498 checkpoint.turn > turns && checkpoint.registered_connection.is_none()
15499 })
15500 .await;
15501 assert_eq!(disarmed.next_probe_at, None);
15502 assert_eq!(disarmed.wake_after, None);
15503 assert_eq!(
15504 Instant::now(),
15505 tick,
15506 "disconnect must disarm in the notification tick"
15507 );
15508 tokio::time::advance(Duration::from_secs(60)).await;
15509 actor.no_probe();
15510 let reconnect_tick = Instant::now();
15511 actor.hello(2, true);
15512 let rearmed = actor
15513 .wait_for_select(|checkpoint| {
15514 checkpoint.registered_connection == Some(ConnectionId::new(2))
15515 && checkpoint.next_probe_at.is_some()
15516 })
15517 .await;
15518 let delay = jittered_health_delay(&actor.spec.module_id, 0, actor.health.cadence);
15519 assert_eq!(rearmed.next_probe_at, Some(reconnect_tick + delay));
15520 tokio::time::advance(delay).await;
15521 actor.probe().await;
15522 }
15523
15524 #[tokio::test(start_paused = true)]
15525 async fn reregistration_without_health_disarms_the_old_schedule() {
15526 let mut actor = Actor::start(ModuleProtocol::Subc, Duration::from_secs(30)).await;
15527 actor.hello(1, true);
15528 actor
15529 .wait_for_select(|checkpoint| {
15530 checkpoint.registered_connection == Some(ConnectionId::new(1))
15531 && checkpoint.next_probe_at.is_some()
15532 })
15533 .await;
15534 actor
15535 .registry
15536 .deregister_connection(ConnectionId::new(1))
15537 .unwrap();
15538 actor.hello(2, false);
15539 let disarmed = actor
15540 .wait_for_select(|checkpoint| {
15541 checkpoint.registered_connection == Some(ConnectionId::new(2))
15542 })
15543 .await;
15544 assert_eq!(disarmed.next_probe_at, None);
15545 assert_eq!(disarmed.wake_after, None);
15546 tokio::time::advance(Duration::from_secs(60)).await;
15547 actor.no_probe();
15548 }
15549
15550 #[tokio::test(start_paused = true)]
15551 async fn restart_waits_for_its_own_hello_before_rearming_health() {
15552 let mut actor = Actor::start(ModuleProtocol::Subc, Duration::from_secs(30)).await;
15553 actor.hello(1, true);
15554 let armed = actor
15555 .wait_for_select(|checkpoint| {
15556 checkpoint.registered_connection == Some(ConnectionId::new(1))
15557 && checkpoint.next_probe_at.is_some()
15558 })
15559 .await;
15560 actor
15564 .registry
15565 .deregister_connection(ConnectionId::new(1))
15566 .unwrap();
15567 actor.module.restart(Some(0)).await.unwrap();
15568 let parked = actor
15569 .wait_for_select(|checkpoint| {
15570 checkpoint.generation > armed.generation
15571 && checkpoint.registered_connection.is_none()
15572 && checkpoint.next_probe_at.is_none()
15573 })
15574 .await;
15575 assert_eq!(parked.wake_after, Some(DEFAULT_REGISTER_TIMEOUT));
15576 actor.rx.close();
15577 while actor.rx.try_recv().is_ok() {}
15580 let turns = actor.turns();
15581 tokio::time::advance(Duration::from_secs(60)).await;
15582 let after = actor.turns();
15583 eprintln!(
15584 "restart park turns: before={turns} after={after} delta={}",
15585 after - turns
15586 );
15587 assert_eq!(after, turns, "replacement must park before HELLO");
15588 let hello_tick = Instant::now();
15589 actor.hello(2, true);
15590 let rearmed = actor
15591 .wait_for_select(|checkpoint| {
15592 checkpoint.registered_connection == Some(ConnectionId::new(2))
15593 && checkpoint.next_probe_at.is_some()
15594 })
15595 .await;
15596 let delay = jittered_health_delay(&actor.spec.module_id, 0, actor.health.cadence);
15597 assert_eq!(rearmed.next_probe_at, Some(hello_tick + delay));
15598 tokio::time::advance(delay).await;
15599 actor.probe().await;
15600 }
15601
15602 #[tokio::test(start_paused = true)]
15603 async fn http_probe_waits_a_full_cadence_for_each_spawn() {
15604 use tokio::{
15605 io::{AsyncReadExt, AsyncWriteExt},
15606 net::TcpListener,
15607 };
15608 let mut actor = Actor::start(ModuleProtocol::None, Duration::from_secs(30)).await;
15609 let listener = TcpListener::bind("127.0.0.1:0").await.unwrap();
15610 actor.health.http = Some(format!("http://{}/healthz", listener.local_addr().unwrap()));
15611 let (started_tx, mut started_rx) = mpsc::channel(8);
15612 let peer = tokio::spawn(async move {
15613 loop {
15614 let (mut socket, _) = listener.accept().await.unwrap();
15615 let mut request = [0; 1024];
15616 assert!(socket.read(&mut request).await.unwrap() > 0);
15617 started_tx.send(Instant::now()).await.unwrap();
15618 socket
15619 .write_all(b"HTTP/1.1 200 OK\r\nContent-Length: 0\r\n\r\n")
15620 .await
15621 .unwrap();
15622 }
15623 });
15624 actor
15625 .module
15626 .update_configuration(actor.spec.clone(), actor.health.clone(), None)
15627 .await
15628 .unwrap();
15629 let first_tick = Instant::now();
15630 let first = actor
15631 .wait_for_select(|checkpoint| checkpoint.next_probe_at.is_some())
15632 .await;
15633 assert!(first.next_probe_at.unwrap() >= first_tick + actor.health.cadence);
15634 assert!(started_rx.try_recv().is_err());
15635
15636 tokio::time::advance(
15639 first.next_probe_at.unwrap() - Instant::now() - Duration::from_secs(1),
15640 )
15641 .await;
15642 actor.module.restart(Some(0)).await.unwrap();
15643 let spawn_tick = Instant::now();
15644 let replacement = actor
15645 .wait_for_select(|checkpoint| checkpoint.generation > first.generation)
15646 .await;
15647 let deadline = replacement.next_probe_at.unwrap();
15648 assert!(
15649 deadline >= spawn_tick + actor.health.cadence,
15650 "a replacement must not inherit its predecessor's probe deadline"
15651 );
15652 tokio::time::advance(actor.health.cadence - Duration::from_nanos(1)).await;
15653 assert!(
15654 started_rx.try_recv().is_err(),
15655 "no HTTP probe during startup grace"
15656 );
15657 tokio::time::advance(deadline - Instant::now()).await;
15658 let landed = wait_for("first replacement HTTP probe", || {
15659 started_rx.try_recv().ok()
15660 })
15661 .await;
15662 assert!(landed >= spawn_tick + actor.health.cadence);
15663 wait_for("healthy replacement", || {
15664 (actor.module.status().unwrap().health.status == SupervisorHealthStatus::Ok)
15665 .then_some(())
15666 })
15667 .await;
15668 peer.abort();
15669 }
15670
15671 #[tokio::test(start_paused = true)]
15672 async fn rescan_adds_and_removes_http_health_without_polling() {
15673 use tokio::{
15674 io::{AsyncReadExt, AsyncWriteExt},
15675 net::TcpListener,
15676 };
15677 let mut actor = Actor::start(ModuleProtocol::None, Duration::from_secs(30)).await;
15678 let listener = TcpListener::bind("127.0.0.1:0").await.unwrap();
15679 actor.health.http = Some(format!("http://{}/health", listener.local_addr().unwrap()));
15680 actor.health.cadence = Duration::from_millis(1);
15682 let (started_tx, mut started_rx) = mpsc::channel(8);
15683 let peer = tokio::spawn(async move {
15684 loop {
15685 let (mut socket, _) = listener.accept().await.unwrap();
15686 let mut request = [0; 1024];
15687 assert!(socket.read(&mut request).await.unwrap() > 0);
15688 started_tx.send(()).await.unwrap();
15689 socket
15690 .write_all(b"HTTP/1.1 200 OK\r\nContent-Length: 0\r\n\r\n")
15691 .await
15692 .unwrap();
15693 }
15694 });
15695 let tick = Instant::now();
15696 let turns = actor.turns();
15697 actor
15698 .module
15699 .update_configuration(actor.spec.clone(), actor.health.clone(), None)
15700 .await
15701 .unwrap();
15702 let armed = actor
15703 .wait_for_select(|checkpoint| {
15704 checkpoint.turn > turns && checkpoint.next_probe_at.is_some()
15705 })
15706 .await;
15707 assert_eq!(armed.next_probe_at, Some(tick + Duration::from_millis(1)));
15708 assert_eq!(Instant::now(), tick);
15709 tokio::time::advance(Duration::from_millis(1)).await;
15710 let mut started = false;
15711 wait_for("successful HTTP probe", || {
15712 started |= started_rx.try_recv().is_ok();
15713 (started && actor.module.status().unwrap().health.status == SupervisorHealthStatus::Ok)
15714 .then_some(())
15715 })
15716 .await;
15717 assert!(started, "added HTTP check must start at its first deadline");
15718 assert_eq!(
15719 actor.module.status().unwrap().health.status,
15720 SupervisorHealthStatus::Ok
15721 );
15722 assert_eq!(Instant::now(), tick + Duration::from_millis(1));
15723 actor.health.http = None;
15724 let turns = actor.turns();
15725 actor
15726 .module
15727 .update_configuration(actor.spec.clone(), actor.health.clone(), None)
15728 .await
15729 .unwrap();
15730 let parked = actor
15731 .wait_for_select(|checkpoint| {
15732 checkpoint.turn > turns && checkpoint.next_probe_at.is_none()
15733 })
15734 .await;
15735 assert_eq!(parked.wake_after, None);
15736 let turns = actor.turns();
15737 tokio::time::advance(Duration::from_secs(60)).await;
15738 assert_eq!(
15739 actor.turns(),
15740 turns,
15741 "removed HTTP check must park the actor"
15742 );
15743 assert!(
15744 started_rx.try_recv().is_err(),
15745 "removed HTTP check must stay disarmed"
15746 );
15747 assert_eq!(
15748 actor.module.status().unwrap().health,
15749 ModuleHealthStatus::default()
15750 );
15751 peer.abort();
15752 }
15753}
15754
15755#[cfg(test)]
15756mod drain_busy_reprobe_tests {
15757 use super::*;
15761 use std::collections::BTreeMap;
15762 use std::sync::Mutex as StdMutex;
15763 use subc_protocol::{manifest::Concurrency, session::ModuleControlResponse};
15764
15765 const MODULE_ID: &str = "drain-reprobe-module";
15766 const CONNECTION: u64 = 7;
15767 const GAUGE: &str = "runs_in_flight";
15768 const CADENCE: Duration = Duration::from_secs(30);
15771 const CEILING: Duration = Duration::from_secs(30);
15772
15773 #[derive(Clone, Copy)]
15775 enum Answer {
15776 BusyUntil { until: Instant, latency: Duration },
15779 OmitsGauge,
15781 Silent,
15783 }
15784
15785 struct Drain {
15786 forwarding: Arc<ForwardingTable>,
15787 runtime: SupervisorRuntimeConfig,
15788 endpoint: crate::ModuleEndpointId,
15789 probes: Arc<StdMutex<Vec<Instant>>>,
15791 responder: tokio::task::JoinHandle<()>,
15792 }
15793
15794 impl Drain {
15795 fn start(answer: Answer) -> Self {
15796 let registry = Arc::new(Registry::default());
15797 let forwarding = Arc::new(ForwardingTable::default());
15798 let health = HealthConfig {
15799 cadence: CADENCE,
15800 ..HealthConfig::default()
15801 };
15802 let runtime = Supervisor::new_for_test(registry, RestartPolicy::new(3, Duration::ZERO))
15803 .with_forwarding(forwarding.clone())
15804 .with_health_config(health)
15805 .runtime_config();
15806 let (tx, mut rx) = mpsc::channel::<crate::router::OutboundFrame>(8);
15807 let endpoint = forwarding
15808 .register_module_connection(
15809 ConnectionId::new(CONNECTION),
15810 MODULE_ID.to_string(),
15811 subc_protocol::PROTOCOL_VERSION,
15812 Concurrency::ModuleManaged,
15813 FrameSink::new(tx),
15814 )
15815 .unwrap();
15816 let probes = Arc::new(StdMutex::new(Vec::new()));
15817 let responder = tokio::spawn({
15818 let forwarding = forwarding.clone();
15819 let probes = probes.clone();
15820 async move {
15821 while let Some(frame) = rx.recv().await {
15822 let body: Value = serde_json::from_slice(&frame.body).unwrap();
15823 assert_eq!(body["op"], MODULE_CONTROL_OP_HEALTH_CHECK);
15824 probes.lock().unwrap().push(Instant::now());
15825 let metrics = match answer {
15826 Answer::BusyUntil { until, latency } => {
15827 sleep(latency).await;
15828 let busy = u64::from(Instant::now() < until);
15829 serde_json::json!({ GAUGE: busy })
15830 }
15831 Answer::OmitsGauge => serde_json::json!({ "queue_depth": 0 }),
15832 Answer::Silent => continue,
15833 };
15834 let _ = forwarding.complete_module_control_rpc(
15837 ConnectionId::new(CONNECTION),
15838 frame.header.corr,
15839 Some(MODULE_CONTROL_OP_HEALTH_CHECK),
15840 ModuleControlRpcOutcome::Response(ModuleControlResponse::HealthCheck {
15841 status: HealthStatus::Ok,
15842 detail: None,
15843 metrics: Some(metrics),
15844 }),
15845 );
15846 }
15847 }
15848 });
15849 Self {
15850 forwarding,
15851 runtime,
15852 endpoint,
15853 probes,
15854 responder,
15855 }
15856 }
15857
15858 async fn run(&self, ceiling: Duration) -> (bool, Duration) {
15861 let (drained, elapsed, _) = self.run_traced(ceiling).await;
15862 (drained, elapsed)
15863 }
15864
15865 async fn run_traced(&self, ceiling: Duration) -> (bool, Duration, DrainGaugeTrace) {
15867 let started = Instant::now();
15868 let mut trace = DrainGaugeTrace::new(&[GAUGE.to_string()]);
15869 let drained = wait_for_forwarding_quiescence(
15870 &self.forwarding,
15871 MODULE_ID,
15872 &self.runtime,
15873 self.endpoint,
15874 started + ceiling,
15875 &[GAUGE.to_string()],
15876 DrainScope::Active,
15877 &mut trace,
15878 )
15879 .await
15880 .unwrap();
15881 (drained, started.elapsed(), trace)
15882 }
15883
15884 fn probes(&self) -> Vec<Instant> {
15885 self.probes.lock().unwrap().clone()
15886 }
15887 }
15888
15889 impl Drop for Drain {
15890 fn drop(&mut self) {
15891 self.responder.abort();
15892 }
15893 }
15894
15895 #[tokio::test(start_paused = true)]
15896 async fn busy_at_first_probe_then_quiet_drains_long_before_the_ceiling() {
15897 let drain = Drain::start(Answer::BusyUntil {
15898 until: Instant::now() + Duration::from_secs(1),
15899 latency: Duration::ZERO,
15900 });
15901 let (drained, elapsed) = drain.run(CEILING).await;
15902 assert!(
15903 drained,
15904 "gauges that read zero during the drain must let it finish drained, \
15905 not wait out the {CADENCE:?} health cadence (elapsed {elapsed:?})"
15906 );
15907 assert!(
15908 elapsed < Duration::from_secs(3),
15909 "drain took {elapsed:?} after the gauges went quiet at 1s"
15910 );
15911 let probes = drain.probes();
15912 assert!(
15913 probes.len() >= 2,
15914 "the module was busy at the first probe, so a quiet verdict needs a second: {probes:?}"
15915 );
15916 }
15917
15918 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
15923 async fn an_instant_busy_first_answer_does_not_stop_later_probes_in_real_time() {
15924 let drain = Drain::start(Answer::BusyUntil {
15925 until: Instant::now() + Duration::from_millis(600),
15926 latency: Duration::ZERO,
15927 });
15928 let (drained, elapsed, trace) = drain.run_traced(CEILING).await;
15929 assert!(drained, "the gauge fell at 600ms: {trace}");
15930 assert!(
15931 elapsed < Duration::from_secs(3),
15932 "drain took {elapsed:?} after the gauge fell at 600ms: {trace}"
15933 );
15934 let probes = drain.probes();
15935 assert!(
15936 probes.len() >= 2 && trace.probes as usize == probes.len(),
15937 "the drain must re-probe after the first busy answer: {probes:?}, {trace}"
15938 );
15939 }
15940
15941 #[tokio::test(start_paused = true)]
15942 async fn busy_throughout_ends_undrained_at_the_ceiling_with_bounded_probes() {
15943 let started = Instant::now();
15944 let drain = Drain::start(Answer::BusyUntil {
15945 until: started + Duration::from_secs(3600),
15946 latency: Duration::ZERO,
15947 });
15948 let (drained, elapsed) = drain.run(CEILING).await;
15949 assert!(!drained, "a module busy throughout must not drain");
15950 assert!(
15951 elapsed >= CEILING && elapsed < CEILING + drain_busy_reprobe_interval(),
15952 "a busy module is held until the ceiling and no longer: {elapsed:?}"
15953 );
15954 let probes = drain.probes();
15957 let max_probes = (CEILING.as_millis() / drain_busy_reprobe_interval().as_millis()) + 1;
15958 assert!(
15959 probes.len() as u128 <= max_probes,
15960 "{} probes in a {CEILING:?} drain exceeds the {max_probes} the interval allows",
15961 probes.len()
15962 );
15963 for pair in probes.windows(2) {
15964 assert!(
15965 pair[1] - pair[0] >= drain_busy_reprobe_interval(),
15966 "consecutive probes {:?} apart",
15967 pair[1] - pair[0]
15968 );
15969 }
15970 }
15971
15972 #[tokio::test(start_paused = true)]
15973 async fn a_slow_probe_is_never_overlapped_by_the_next() {
15974 let latency = Duration::from_secs(1);
15977 let drain = Drain::start(Answer::BusyUntil {
15978 until: Instant::now() + Duration::from_secs(3600),
15979 latency,
15980 });
15981 let (drained, _) = drain.run(Duration::from_secs(5)).await;
15982 assert!(!drained);
15983 let probes = drain.probes();
15984 assert!(probes.len() >= 2, "the drain must re-probe: {probes:?}");
15985 for pair in probes.windows(2) {
15986 assert!(
15987 pair[1] - pair[0] >= latency + drain_busy_reprobe_interval(),
15988 "a probe started {:?} after the previous one, before it could finish",
15989 pair[1] - pair[0]
15990 );
15991 }
15992 }
15993
15994 #[tokio::test(start_paused = true)]
15995 async fn an_unanswered_probe_cannot_hold_the_drain_past_its_ceiling() {
15996 let drain = Drain::start(Answer::Silent);
15997 let ceiling = Duration::from_secs(2);
15998 let (drained, elapsed) = drain.run(ceiling).await;
15999 assert!(!drained, "an unanswered probe counts as busy");
16000 assert!(
16001 elapsed >= ceiling && elapsed < ceiling + drain_busy_reprobe_interval(),
16002 "the probe deadline is capped at the drain ceiling: {elapsed:?}"
16003 );
16004 assert_eq!(
16005 drain.probes().len(),
16006 1,
16007 "the unanswered probe holds the only slot until the ceiling"
16008 );
16009 }
16010
16011 #[derive(Clone, Default)]
16013 struct LogCapture {
16014 events: Arc<StdMutex<Vec<BTreeMap<String, String>>>>,
16015 }
16016
16017 impl<S: tracing::Subscriber> tracing_subscriber::Layer<S> for LogCapture {
16018 fn on_event(
16019 &self,
16020 event: &tracing::Event<'_>,
16021 _context: tracing_subscriber::layer::Context<'_, S>,
16022 ) {
16023 struct Fields(BTreeMap<String, String>);
16024 impl tracing::field::Visit for Fields {
16025 fn record_debug(&mut self, field: &tracing::field::Field, value: &dyn fmt::Debug) {
16026 self.0
16027 .insert(field.name().to_string(), format!("{value:?}"));
16028 }
16029 }
16030 let mut fields = Fields(BTreeMap::new());
16031 event.record(&mut fields);
16032 self.events.lock().unwrap().push(fields.0);
16033 }
16034 }
16035
16036 impl LogCapture {
16037 fn install(&self) -> tracing::subscriber::DefaultGuard {
16038 use tracing_subscriber::layer::SubscriberExt;
16039 tracing::subscriber::set_default(tracing_subscriber::registry().with(self.clone()))
16040 }
16041
16042 fn with_message(&self, needle: &str) -> Vec<BTreeMap<String, String>> {
16043 self.events
16044 .lock()
16045 .unwrap()
16046 .iter()
16047 .filter(|fields| fields.get("message").is_some_and(|m| m.contains(needle)))
16048 .cloned()
16049 .collect()
16050 }
16051 }
16052
16053 #[tokio::test(start_paused = true)]
16058 async fn a_missing_declared_gauge_is_named_on_the_drain_timeout_line() {
16059 let capture = LogCapture::default();
16060 let _guard = capture.install();
16061 let drain = Drain::start(Answer::OmitsGauge);
16062 let (drained, _, trace) = drain.run_traced(Duration::from_secs(2)).await;
16063 assert!(!drained, "a gauge never reported cannot read quiet");
16064 assert!(trace.probes >= 2, "the drain must re-probe: {trace:?}");
16065
16066 let missing = capture.with_message("lacks a declared busy gauge");
16067 assert_eq!(missing.len(), 1, "one warning per drain: {missing:?}");
16068 assert_eq!(
16069 missing[0].get("missing_gauges").map(String::as_str),
16070 Some(GAUGE)
16071 );
16072
16073 warn_drain_timed_out(
16074 MODULE_ID,
16075 Duration::from_secs(2),
16076 RouteCloseReason::Restart,
16077 &Default::default(),
16078 &trace,
16079 );
16080 let timeouts = capture.with_message("route drain timed out");
16081 assert_eq!(timeouts.len(), 1, "{timeouts:?}");
16082 let gauges = &timeouts[0]["busy_gauges"];
16083 assert!(
16084 gauges.starts_with(&format!("{GAUGE}: missing_from_health_body @")),
16085 "the timeout line must name the gauge and why it read busy: {gauges}"
16086 );
16087 assert!(
16088 gauges.ends_with("; no value ever read"),
16089 "no value was ever read for the gauge: {gauges}"
16090 );
16091 assert_eq!(timeouts[0]["gauge_probes"], trace.probes.to_string());
16092 }
16093
16094 #[tokio::test(start_paused = true)]
16098 async fn the_drain_trace_names_a_busy_value_and_an_unanswered_probe() {
16099 let busy = Drain::start(Answer::BusyUntil {
16100 until: Instant::now() + Duration::from_secs(3600),
16101 latency: Duration::ZERO,
16102 });
16103 let (_, _, trace) = busy.run_traced(Duration::from_secs(1)).await;
16104 let rendered = trace.to_string();
16105 assert!(
16106 rendered.starts_with(&format!("{GAUGE}: non_zero(1) @"))
16107 && rendered.contains("; last value non_zero(1) @"),
16108 "{rendered}"
16109 );
16110
16111 let silent = Drain::start(Answer::Silent);
16112 let (_, _, trace) = silent.run_traced(Duration::from_secs(1)).await;
16113 assert_eq!(
16116 trace.to_string(),
16117 format!("{GAUGE}: probe_failed(no-answer) @1000ms; no value ever read")
16118 );
16119 assert_eq!(trace.probes, 1);
16120 }
16121
16122 #[test]
16124 fn a_gauge_that_is_not_a_count_is_busy_and_shown_as_sent() {
16125 let gauges = [GAUGE.to_string()];
16126 let verdicts = gauge_verdicts(Some(&serde_json::json!({ GAUGE: 0.0 })), &gauges);
16127 assert_eq!(verdicts, vec![GaugeVerdict::NotACount("0.0".to_string())]);
16128 assert_eq!(
16129 busy_gauge_observation(&verdicts),
16130 BusyGaugeObservation::Busy
16131 );
16132 let zero = gauge_verdicts(Some(&serde_json::json!({ GAUGE: 0 })), &gauges);
16133 assert_eq!(
16134 busy_gauge_observation(&zero),
16135 BusyGaugeObservation::Quiescent
16136 );
16137 let none = gauge_verdicts(None, &gauges);
16138 assert_eq!(busy_gauge_observation(&none), BusyGaugeObservation::Omitted);
16139 }
16140}
16141
16142#[cfg(test)]
16143mod health_evidence_tests {
16144 use super::{HealthProbeError, HealthProbeEvidence};
16145 use std::collections::HashSet;
16146
16147 #[test]
16155 fn only_a_dead_lane_is_proof_of_death() {
16156 assert!(HealthProbeError::lane_dead("gone").is_proof_of_death());
16157 assert!(!HealthProbeError::no_answer("timed out").is_proof_of_death());
16161 assert!(!HealthProbeError::bad_answer("garbage").is_proof_of_death());
16162 assert!(!HealthProbeError::misconfigured("no table").is_proof_of_death());
16163 }
16164
16165 #[test]
16171 fn every_evidence_class_has_a_distinct_label() {
16172 let labels = [
16173 HealthProbeError::lane_dead("").label(),
16174 HealthProbeError::no_answer("").label(),
16175 HealthProbeError::bad_answer("").label(),
16176 HealthProbeError::misconfigured("").label(),
16177 ];
16178 let unique: HashSet<_> = labels.iter().collect();
16179 assert_eq!(unique.len(), labels.len(), "labels collided: {labels:?}");
16180 }
16181
16182 #[test]
16188 fn classification_preserves_the_original_message() {
16189 let err = HealthProbeError::no_answer("module did not answer within 5s");
16190 assert_eq!(err.to_string(), "module did not answer within 5s");
16191 assert!(matches!(err.evidence, HealthProbeEvidence::NoAnswer));
16192 }
16193}
16194
16195#[cfg(test)]
16196mod health_tombstone_tests {
16197 use std::{path::PathBuf, sync::Arc, time::Duration};
16198
16199 use subc_protocol::{
16200 manifest::Concurrency,
16201 session::{HealthStatus, ModuleControlResponse},
16202 };
16203 use tokio::sync::mpsc;
16204
16205 use super::{
16206 probe_module_health, HealthAction, HealthConfig, HealthProbeEvidence, ModuleProtocol,
16207 ModuleSpec, RestartPolicy, Supervisor, SupervisorRuntimeConfig,
16208 };
16209 use crate::{
16210 control::ControlHandler,
16211 forwarding::{ForwardingTable, ModuleControlRpcCompletion, ModuleControlRpcOutcome},
16212 registry::{ConnectionId, Registry},
16213 router::FrameSink,
16214 };
16215
16216 struct ProbeHarness {
16217 spec: ModuleSpec,
16218 runtime: SupervisorRuntimeConfig,
16219 forwarding: Arc<ForwardingTable>,
16220 module_connection: ConnectionId,
16221 module_rx: mpsc::Receiver<crate::router::OutboundFrame>,
16222 handler: ControlHandler,
16223 module: super::SupervisedModule,
16224 }
16225
16226 fn probe_harness() -> ProbeHarness {
16227 let registry = Arc::new(Registry::default());
16228 let forwarding = Arc::new(ForwardingTable::default());
16229 let supervisor_handle = super::SupervisorHandle::new();
16230 let health = HealthConfig {
16231 http: None,
16232 cadence: Duration::from_secs(30),
16233 deadline: Duration::from_secs(5),
16234 failure_threshold: 3,
16235 on_degraded: HealthAction::Report,
16236 on_failing: HealthAction::Report,
16237 critical: false,
16238 };
16239 let supervisor = Supervisor::new_for_test(Arc::clone(®istry), RestartPolicy::default())
16240 .with_forwarding(Arc::clone(&forwarding))
16241 .with_handle(supervisor_handle.clone())
16242 .with_health_config(health);
16243 let spec = ModuleSpec {
16244 module_id: "late-health-module".to_string(),
16245 program: PathBuf::from("disabled-module"),
16246 args: Vec::new(),
16247 env: Vec::new(),
16248 reserved: false,
16249 reserved_prefixes: Vec::new(),
16250 protocol: ModuleProtocol::Subc,
16251 overlap: Default::default(),
16252 carry_containment: false,
16253 };
16254 let module = supervisor
16255 .supervise_configured(spec.clone(), false)
16256 .unwrap();
16257 let runtime = supervisor.runtime_config();
16258 let handler = ControlHandler::with_forwarding(registry, Arc::clone(&forwarding))
16259 .with_supervisor(supervisor_handle);
16260 let module_connection = ConnectionId::new(700);
16261 let (module_tx, module_rx) = mpsc::channel(8);
16262 forwarding
16263 .register_module_connection(
16264 module_connection,
16265 spec.module_id.clone(),
16266 subc_protocol::PROTOCOL_VERSION,
16267 Concurrency::ModuleManaged,
16268 FrameSink::new(module_tx),
16269 )
16270 .unwrap();
16271
16272 ProbeHarness {
16273 spec,
16274 runtime,
16275 forwarding,
16276 module_connection,
16277 module_rx,
16278 handler,
16279 module,
16280 }
16281 }
16282
16283 async fn finish_after(
16284 harness: &mut ProbeHarness,
16285 stall: Duration,
16286 ) -> ModuleControlRpcCompletion {
16287 assert!(stall > harness.runtime.health.deadline);
16288 let deadline = harness.runtime.health.deadline;
16289 let probe = probe_module_health(&harness.spec.module_id, &harness.runtime, None);
16290 let answer = async {
16291 let frame = harness.module_rx.recv().await.expect("health.check frame");
16292 tokio::time::advance(deadline).await;
16293 tokio::task::yield_now().await;
16294 tokio::time::advance(stall - deadline).await;
16295 harness
16296 .forwarding
16297 .complete_module_control_rpc(
16298 harness.module_connection,
16299 frame.header.corr,
16300 Some("health.check"),
16301 ModuleControlRpcOutcome::Response(ModuleControlResponse::HealthCheck {
16302 status: HealthStatus::Ok,
16303 detail: None,
16304 metrics: None,
16305 }),
16306 )
16307 .unwrap()
16308 };
16309 let (probe_result, completion) = tokio::join!(probe, answer);
16310 let err = probe_result.expect_err("probe must miss its deadline");
16311 assert!(matches!(err.evidence, HealthProbeEvidence::NoAnswer));
16312 completion
16313 }
16314
16315 async fn time_out_without_answer(harness: &mut ProbeHarness) {
16316 let deadline = harness.runtime.health.deadline;
16317 let probe = probe_module_health(&harness.spec.module_id, &harness.runtime, None);
16318 let exhaust_deadline = async {
16319 let _frame = harness.module_rx.recv().await.expect("health.check frame");
16320 tokio::time::advance(deadline).await;
16321 tokio::task::yield_now().await;
16322 };
16323 let (probe_result, ()) = tokio::join!(probe, exhaust_deadline);
16324 let err = probe_result.expect_err("probe must miss its deadline");
16325 assert!(matches!(err.evidence, HealthProbeEvidence::NoAnswer));
16326 }
16327
16328 async fn run_probe_cycle(harness: &mut ProbeHarness, answer: bool) {
16329 let registry = Arc::clone(&harness.module.inner.registry);
16330 let snapshot = Arc::clone(&harness.module.inner.snapshot);
16331 let process_liveness = super::SupervisorProcessLiveness::default();
16332 let mut child = None;
16333 let cycle = super::run_health_probe_cycle(
16334 &harness.spec,
16335 &harness.runtime,
16336 ®istry,
16337 &process_liveness,
16338 &snapshot,
16339 &mut child,
16340 );
16341 let peer = async {
16342 let frame = harness.module_rx.recv().await.expect("health.check frame");
16343 if answer {
16344 harness
16345 .forwarding
16346 .complete_module_control_rpc(
16347 harness.module_connection,
16348 frame.header.corr,
16349 Some("health.check"),
16350 ModuleControlRpcOutcome::Response(ModuleControlResponse::HealthCheck {
16351 status: HealthStatus::Ok,
16352 detail: None,
16353 metrics: Some(serde_json::json!({"ready": true})),
16354 }),
16355 )
16356 .unwrap();
16357 } else {
16358 tokio::time::advance(harness.runtime.health.deadline).await;
16359 tokio::task::yield_now().await;
16360 }
16361 };
16362 tokio::join!(cycle, peer);
16363 }
16364
16365 #[tokio::test(start_paused = true)]
16366 async fn unanswered_probe_is_unknown_until_threshold_and_ok_report_recovers() {
16367 let mut harness = probe_harness();
16368 let monitor = harness.module.inner.monitor.lock().unwrap().take().unwrap();
16372 monitor.abort();
16373 let _ = monitor.await;
16374 super::update_snapshot(&harness.module.inner.snapshot, None, |state| {
16375 state.enabled = true;
16376 state.state = super::ModuleState::Running;
16377 state.process_alive = true;
16378 })
16379 .unwrap();
16380
16381 run_probe_cycle(&mut harness, true).await;
16382 assert_eq!(
16383 harness.module.status().unwrap().health.status,
16384 super::SupervisorHealthStatus::Ok
16385 );
16386
16387 for failures in 1..harness.runtime.health.failure_threshold {
16388 run_probe_cycle(&mut harness, false).await;
16389 let status = harness.module.status().unwrap();
16390 assert_eq!(status.health.status, super::SupervisorHealthStatus::Unknown);
16391 assert_eq!(status.health.consecutive_failures, failures);
16392 assert!(status.health.last_probe_ms.is_some());
16393 assert!(status.health.detail.unwrap().starts_with("[no-answer]"));
16394 assert!(status.health.metrics.is_none());
16395 assert_eq!(status.state, super::ModuleState::Running);
16396 assert!(status.process_alive);
16397 assert_eq!(status.restart_count, 0);
16398 assert_eq!(status.lifetime_restarts, 0);
16399 assert!(status.health.last_action.is_none());
16400 assert!(harness.runtime.scheduled_respawn.lock().unwrap().is_none());
16401 }
16402
16403 run_probe_cycle(&mut harness, true).await;
16404 let recovered = harness.module.status().unwrap();
16405 assert_eq!(recovered.health.status, super::SupervisorHealthStatus::Ok);
16406 assert_eq!(recovered.health.consecutive_failures, 0);
16407 assert!(recovered.health.detail.is_none());
16408 assert_eq!(
16409 recovered.health.metrics,
16410 Some(serde_json::json!({"ready": true}))
16411 );
16412 assert_eq!(recovered.lifetime_restarts, 0);
16413
16414 for failures in 1..=harness.runtime.health.failure_threshold {
16415 run_probe_cycle(&mut harness, false).await;
16416 let status = harness.module.status().unwrap();
16417 assert_eq!(status.health.consecutive_failures, failures);
16418 if failures < harness.runtime.health.failure_threshold {
16419 assert_eq!(status.health.status, super::SupervisorHealthStatus::Unknown);
16420 assert_eq!(status.state, super::ModuleState::Running);
16421 assert_eq!(status.lifetime_restarts, 0);
16422 assert!(harness.runtime.scheduled_respawn.lock().unwrap().is_none());
16423 } else {
16424 assert_eq!(
16425 status.health.status,
16426 super::SupervisorHealthStatus::Unresponsive
16427 );
16428 assert_eq!(status.state, super::ModuleState::Restarting);
16429 assert_eq!(status.restart_count, 1);
16430 assert_eq!(status.lifetime_restarts, 1);
16431 assert_eq!(status.health.last_action.as_deref(), Some("restart"));
16432 assert!(harness.runtime.scheduled_respawn.lock().unwrap().is_some());
16433 }
16434 }
16435 }
16436
16437 #[tokio::test(start_paused = true)]
16438 async fn late_health_answers_record_start_anchored_latency_for_two_stalls() {
16439 let mut harness = probe_harness();
16440
16441 let first = finish_after(&mut harness, Duration::from_secs(8)).await;
16442 let first_latency = match &first {
16443 ModuleControlRpcCompletion::LateHealthAnswer { latency, .. } => *latency,
16444 other => panic!("late answer was not retained: {other:?}"),
16445 };
16446 assert!(harness.handler.observe_module_control_completion(first));
16447
16448 let second = finish_after(&mut harness, Duration::from_secs(11)).await;
16449 let second_latency = match &second {
16450 ModuleControlRpcCompletion::LateHealthAnswer { latency, .. } => *latency,
16451 other => panic!("late answer was not retained: {other:?}"),
16452 };
16453 assert!(harness.handler.observe_module_control_completion(second));
16454
16455 assert_eq!(first_latency, Duration::from_secs(8));
16456 assert_eq!(
16457 second_latency - first_latency,
16458 Duration::from_secs(3),
16459 "latency must grow linearly with the additional stall"
16460 );
16461 let health = harness.module.status().unwrap().health;
16462 assert_eq!(health.late_answer_count, 2);
16463 assert_eq!(health.last_late_answer_latency_ms, Some(11_000));
16464 }
16465
16466 #[tokio::test(start_paused = true)]
16474 async fn late_answer_clears_the_consecutive_failure_streak() {
16475 let mut harness = probe_harness();
16476
16477 time_out_without_answer(&mut harness).await;
16479 harness
16480 .module
16481 .record_health_probe_failure_for_test("[no-answer] test miss")
16482 .unwrap();
16483 assert_eq!(
16484 harness.module.status().unwrap().health.consecutive_failures,
16485 1,
16486 "precondition: the miss must be on the streak before the late answer"
16487 );
16488
16489 let late = finish_after(&mut harness, Duration::from_secs(9)).await;
16491 assert!(matches!(
16492 late,
16493 ModuleControlRpcCompletion::LateHealthAnswer { .. }
16494 ));
16495 assert!(harness.handler.observe_module_control_completion(late));
16496
16497 let health = harness.module.status().unwrap().health;
16498 assert_eq!(
16499 health.consecutive_failures, 0,
16500 "a late answer is an answer: the streak must reset"
16501 );
16502 assert_eq!(health.late_answer_count, 1);
16503 }
16504
16505 #[tokio::test(start_paused = true)]
16506 async fn repeated_serial_probe_cycles_keep_one_tombstone_per_endpoint() {
16507 let mut harness = probe_harness();
16508
16509 for _ in 0..20 {
16510 time_out_without_answer(&mut harness).await;
16511 assert_eq!(
16512 harness.forwarding.health_probe_tombstone_count().unwrap(),
16513 1
16514 );
16515 }
16516 }
16517}
16518
16519#[cfg(test)]
16520mod child_env_tests {
16521 #[cfg(windows)]
16522 use super::{
16523 clear_current_process_facts, observed_launch_nonce_source, ModuleState, SupervisorSnapshot,
16524 };
16525 #[cfg(windows)]
16526 use std::sync::{Arc, OnceLock};
16527 #[cfg(windows)]
16528 #[test]
16529 fn windows_wire_spawn_prepares_pipe_and_keeps_phase_one_env() {
16530 let spec = spec(Vec::new());
16531 let mut command = Command::new("unused");
16532 let handoff = apply_wire_spawn_args(&mut command, &spec, None, None)
16533 .unwrap()
16534 .unwrap();
16535 let name = command
16536 .as_std()
16537 .get_envs()
16538 .find(|(key, _)| *key == OsStr::new(subc_os::launch_nonce::LAUNCH_NONCE_PIPE_ENV))
16539 .and_then(|(_, value)| value)
16540 .unwrap();
16541 assert_eq!(name, OsStr::new(handoff.name()));
16542 assert!(handoff.name().starts_with(r"\\.\pipe\subc-launch-"));
16543 assert!(command.as_std().get_envs().any(|(key, value)| key
16544 == OsStr::new(subc_os::LAUNCH_NONCE_PIPE_FALLBACK_ENV)
16545 && value == Some(OsStr::new("env"))));
16546 assert!(command
16547 .as_std()
16548 .get_envs()
16549 .any(|(key, value)| key == OsStr::new(SUBC_LAUNCH_NONCE_ENV) && value.is_some()));
16550 }
16551
16552 #[cfg(windows)]
16553 #[test]
16554 fn windows_source_reporting_tracks_consumption_and_resets_per_spawn() {
16555 let mut snapshot = SupervisorSnapshot::new(ModuleState::Running, true);
16556 snapshot.process_alive = true;
16557 snapshot.spawned_protocol = Some(ModuleProtocol::Subc);
16558 assert_eq!(
16559 observed_launch_nonce_source(&snapshot).as_deref(),
16560 Some("env")
16561 );
16562 let consumed = Arc::new(OnceLock::new());
16563 snapshot.pipe_consumed = Some(consumed.clone());
16564 assert_eq!(
16565 observed_launch_nonce_source(&snapshot).as_deref(),
16566 Some("env")
16567 );
16568 consumed.set(()).unwrap();
16569 assert_eq!(
16570 observed_launch_nonce_source(&snapshot).as_deref(),
16571 Some("pipe")
16572 );
16573 snapshot.pipe_consumed = Some(Arc::new(OnceLock::new()));
16574 assert_eq!(
16575 observed_launch_nonce_source(&snapshot).as_deref(),
16576 Some("env")
16577 );
16578 clear_current_process_facts(&mut snapshot);
16579 assert_eq!(observed_launch_nonce_source(&snapshot), None);
16580 }
16581 use super::{
16582 apply_child_env, apply_spawn_role, apply_wire_spawn_args, LaunchSpecDigest, ModuleProtocol,
16583 ModuleSpec, SpawnRole, SupervisorHandle, SPAWN_ROLE_SWAP_CANDIDATE, SUBC_ARG,
16584 SUBC_LAUNCH_NONCE_ENV, SUBC_MODULE_ID_ENV, SUBC_SPAWN_ROLE_ENV,
16585 };
16586 use std::{ffi::OsStr, path::PathBuf};
16587 use tokio::process::Command;
16588
16589 fn spec(env: Vec<(String, String)>) -> ModuleSpec {
16590 ModuleSpec {
16591 module_id: "env-plan".to_string(),
16592 program: PathBuf::from("/nonexistent"),
16593 args: Vec::new(),
16594 env,
16595 reserved: false,
16596 reserved_prefixes: Vec::new(),
16597 protocol: ModuleProtocol::Subc,
16598 overlap: Default::default(),
16599 carry_containment: false,
16600 }
16601 }
16602
16603 #[test]
16604 fn configured_launch_spec_digest_is_canonical_and_excludes_injected_inputs() {
16605 let base = spec(vec![("A".into(), "B=C".into()), ("Z".into(), "v".into())]);
16606 let digest = LaunchSpecDigest::from_spec(&base);
16607 let mut reordered = base.clone();
16608 reordered.env.reverse();
16609 reordered.module_id = "different-id".into();
16610 reordered.reserved = true;
16611 reordered.overlap = Default::default();
16612 assert_eq!(digest, LaunchSpecDigest::from_spec(&reordered));
16613 let mut delimiter_collision = base.clone();
16614 delimiter_collision.env[0] = ("A=B".into(), "C".into());
16615 assert_ne!(digest, LaunchSpecDigest::from_spec(&delimiter_collision));
16616 let mut changed = base.clone();
16617 changed.protocol = ModuleProtocol::None;
16618 assert_ne!(digest, LaunchSpecDigest::from_spec(&changed));
16619 changed = base.clone();
16620 changed.args.push("--flag".into());
16621 assert_ne!(digest, LaunchSpecDigest::from_spec(&changed));
16622 changed = base.clone();
16623 changed.program = PathBuf::from("/other");
16624 assert_ne!(digest, LaunchSpecDigest::from_spec(&changed));
16625 changed = base.clone();
16626 changed.env[1].1 = "other".into();
16627 assert_ne!(digest, LaunchSpecDigest::from_spec(&changed));
16628 assert!(!format!("{digest:?}").contains("B=C"));
16629 assert!(!format!("{digest:?}").contains("[u8"));
16630 }
16631
16632 #[cfg(unix)]
16633 #[test]
16634 fn native_program_bytes_do_not_collapse_through_lossy_utf8() {
16635 use std::os::unix::ffi::OsStringExt;
16636 let mut left = spec(Vec::new());
16637 left.program = std::ffi::OsString::from_vec(vec![b'/', 0xff]).into();
16638 let mut right = left.clone();
16639 right.program = std::ffi::OsString::from_vec(vec![b'/', 0xfe]).into();
16640 assert_eq!(
16641 left.program.to_string_lossy(),
16642 right.program.to_string_lossy()
16643 );
16644 assert_ne!(
16645 LaunchSpecDigest::from_spec(&left),
16646 LaunchSpecDigest::from_spec(&right)
16647 );
16648 }
16649
16650 #[test]
16664 fn ambient_ck_log_is_removed_and_a_configured_one_survives() {
16665 let mut command = Command::new("/nonexistent");
16666 apply_child_env(&mut command, &spec(Vec::new()));
16667 let removed = command
16668 .as_std()
16669 .get_envs()
16670 .any(|(key, value)| key == OsStr::new("CK_LOG") && value.is_none());
16671 assert!(
16672 removed,
16673 "ambient CK_LOG must be explicitly removed for an unconfigured module"
16674 );
16675
16676 let mut configured = Command::new("/nonexistent");
16677 apply_child_env(
16678 &mut configured,
16679 &spec(vec![("CK_LOG".to_string(), "debug".to_string())]),
16680 );
16681 let effective = configured
16682 .as_std()
16683 .get_envs()
16684 .filter(|(key, _)| *key == OsStr::new("CK_LOG"))
16685 .last()
16686 .map(|(_, value)| value.map(|v| v.to_string_lossy().into_owned()));
16687 assert_eq!(
16688 effective,
16689 Some(Some("debug".to_string())),
16690 "a module's configured CK_LOG must survive the ambient removal"
16691 );
16692 }
16693
16694 #[test]
16703 fn protocol_none_spawn_carries_no_subc_argument_and_no_nonce() {
16704 let connection_file = std::path::Path::new("/run/subc-connection.json");
16705 let handle = SupervisorHandle::new();
16706
16707 let mut none_spec = spec(Vec::new());
16708 none_spec.protocol = ModuleProtocol::None;
16709 let mut none = Command::new("/nonexistent");
16710 let none_handoff =
16711 apply_wire_spawn_args(&mut none, &none_spec, Some(connection_file), Some(&handle))
16712 .expect("protocol-none spawn args apply");
16713 assert!(
16714 none_handoff.is_none(),
16715 "protocol:none spawn must not receive a nonce descriptor"
16716 );
16717 #[cfg(windows)]
16718 assert!(!none.as_std().get_envs().any(|(key, value)| key
16719 == OsStr::new(subc_os::LAUNCH_NONCE_PIPE_FALLBACK_ENV)
16720 && value.is_some()));
16721 assert!(
16722 !none.as_std().get_envs().any(|(key, value)| key
16723 == OsStr::new(subc_os::LAUNCH_NONCE_FD_ENV)
16724 && value.is_some()),
16725 "protocol:none spawn must not name a nonce descriptor"
16726 );
16727 let none_args: Vec<String> = none
16728 .as_std()
16729 .get_args()
16730 .map(|a| a.to_string_lossy().into_owned())
16731 .collect();
16732 assert!(
16733 !none_args.iter().any(|a| a == SUBC_ARG),
16734 "protocol:none argv must not carry --subc; got {none_args:?}"
16735 );
16736 let none_has_nonce = none
16737 .as_std()
16738 .get_envs()
16739 .any(|(key, value)| key == OsStr::new(SUBC_LAUNCH_NONCE_ENV) && value.is_some());
16740 assert!(
16741 !none_has_nonce,
16742 "protocol:none spawn must not receive a launch nonce"
16743 );
16744 let none_has_module_id = none
16745 .as_std()
16746 .get_envs()
16747 .any(|(key, value)| key == OsStr::new(SUBC_MODULE_ID_ENV) && value.is_some());
16748 assert!(
16749 none_has_module_id,
16750 "SUBC_MODULE_ID is inert and stays on every path"
16751 );
16752 assert!(
16753 handle.spawn_nonce(&none_spec.module_id).is_none(),
16754 "no nonce record for a process that will never present one"
16755 );
16756
16757 let wire_spec = spec(Vec::new());
16759 let mut wire = Command::new("/nonexistent");
16760 let wire_handoff =
16761 apply_wire_spawn_args(&mut wire, &wire_spec, Some(connection_file), Some(&handle))
16762 .expect("subc-wire spawn args apply");
16763 let wire_fd_env = wire
16764 .as_std()
16765 .get_envs()
16766 .find(|(key, _)| *key == OsStr::new(subc_os::LAUNCH_NONCE_FD_ENV))
16767 .and_then(|(_, value)| value.map(|v| v.to_string_lossy().into_owned()));
16768 #[cfg(unix)]
16769 assert_eq!(
16770 wire_fd_env,
16771 Some(wire_handoff.expect("a descriptor handoff").fd_env_value()),
16772 "a subc-wire spawn names the pipe it will receive at descriptor 3"
16773 );
16774 #[cfg(windows)]
16775 assert!(wire_handoff.is_some() && wire_fd_env.is_none());
16776 #[cfg(not(any(unix, windows)))]
16777 assert!(wire_handoff.is_none() && wire_fd_env.is_none());
16778 let wire_args: Vec<String> = wire
16779 .as_std()
16780 .get_args()
16781 .map(|a| a.to_string_lossy().into_owned())
16782 .collect();
16783 assert_eq!(
16784 wire_args,
16785 vec![
16786 SUBC_ARG.to_string(),
16787 connection_file.to_string_lossy().into_owned()
16788 ],
16789 "a subc-wire spawn still carries --subc <path>"
16790 );
16791 assert_eq!(
16792 wire.as_std()
16793 .get_envs()
16794 .any(|(key, value)| key == OsStr::new(SUBC_LAUNCH_NONCE_ENV) && value.is_some()),
16795 !cfg!(unix),
16796 "only Windows supplies the environment nonce"
16797 );
16798 assert!(handle.spawn_nonce(&wire_spec.module_id).is_some());
16799 }
16800
16801 #[test]
16811 fn plain_spawn_removes_the_spawn_role_even_when_the_spec_sets_it() {
16812 let role = |command: &Command| {
16813 command
16814 .as_std()
16815 .get_envs()
16816 .filter(|(key, _)| *key == OsStr::new(SUBC_SPAWN_ROLE_ENV))
16817 .last()
16818 .map(|(_, value)| value.map(|v| v.to_string_lossy().into_owned()))
16819 };
16820 let forged = spec(vec![(
16821 SUBC_SPAWN_ROLE_ENV.to_string(),
16822 SPAWN_ROLE_SWAP_CANDIDATE.to_string(),
16823 )]);
16824
16825 let mut plain = Command::new("/nonexistent");
16826 apply_child_env(&mut plain, &forged);
16827 apply_spawn_role(&mut plain, SpawnRole::Plain);
16828 assert_eq!(
16829 role(&plain),
16830 Some(None),
16831 "a plain spawn must remove SUBC_SPAWN_ROLE, whatever the spec says"
16832 );
16833
16834 let mut candidate = Command::new("/nonexistent");
16835 apply_child_env(&mut candidate, &spec(Vec::new()));
16836 apply_spawn_role(&mut candidate, SpawnRole::SwapCandidate);
16837 assert_eq!(
16838 role(&candidate),
16839 Some(Some(SPAWN_ROLE_SWAP_CANDIDATE.to_string()))
16840 );
16841 }
16842
16843 #[test]
16849 fn daemon_private_capture_keys_are_not_passed_to_the_child() {
16850 let mut command = Command::new("/nonexistent");
16851 apply_child_env(
16852 &mut command,
16853 &spec(vec![
16854 (super::CAPTURE_KEEP_ENV.to_string(), "5".to_string()),
16855 ("KEPT".to_string(), "yes".to_string()),
16856 ]),
16857 );
16858 let keys: Vec<String> = command
16859 .as_std()
16860 .get_envs()
16861 .filter(|(_, value)| value.is_some())
16862 .map(|(key, _)| key.to_string_lossy().into_owned())
16863 .collect();
16864 assert!(keys.contains(&"KEPT".to_string()), "got {keys:?}");
16865 assert!(
16866 !keys.contains(&super::CAPTURE_KEEP_ENV.to_string()),
16867 "daemon-private capture key leaked to the child: {keys:?}"
16868 );
16869 }
16870}
16871
16872#[cfg(test)]
16873mod jitter_tests {
16874 use super::jittered_health_delay;
16875 use std::{collections::HashSet, time::Duration};
16876
16877 const FLEET: [&str; 14] = [
16886 "aft",
16887 "alfonso-core",
16888 "magic-context",
16889 "broca",
16890 "thalamus",
16891 "quota",
16892 "engram",
16893 "plexus",
16894 "cerebellum",
16895 "astrocyte",
16896 "synapse",
16897 "subc-mcp",
16898 "cortexkit-credentials",
16899 "subc-federation",
16900 ];
16901
16902 #[test]
16910 fn probe_delays_disperse_across_the_fleet() {
16911 let cadence = Duration::from_secs(30);
16912 let delays: HashSet<Duration> = FLEET
16913 .iter()
16914 .map(|id| jittered_health_delay(id, 0, cadence))
16915 .collect();
16916 assert_eq!(
16917 delays.len(),
16918 FLEET.len(),
16919 "every supervised module must land on its own probe offset"
16920 );
16921 }
16922
16923 #[test]
16929 fn jitter_only_delays_and_stays_within_one_tenth_of_cadence() {
16930 let cadence = Duration::from_secs(30);
16931 let span = cadence / 10;
16932 for id in FLEET {
16933 for probe_index in 0..8 {
16934 let delay = jittered_health_delay(id, probe_index, cadence);
16935 assert!(
16936 delay >= cadence,
16937 "{id}#{probe_index}: jitter must not shorten the cadence"
16938 );
16939 assert!(
16940 delay < cadence + span,
16941 "{id}#{probe_index}: jitter must stay inside one tenth of the cadence"
16942 );
16943 }
16944 }
16945 }
16946
16947 #[test]
16953 fn a_module_offset_is_stable_across_restarts() {
16954 let cadence = Duration::from_secs(30);
16955 for id in FLEET {
16956 assert_eq!(
16957 jittered_health_delay(id, 0, cadence),
16958 jittered_health_delay(id, 0, cadence),
16959 "{id}: the same module and probe index must produce the same offset"
16960 );
16961 }
16962 }
16963
16964 #[test]
16966 fn zero_cadence_yields_zero_delay() {
16967 assert_eq!(
16968 jittered_health_delay("aft", 0, Duration::ZERO),
16969 Duration::ZERO
16970 );
16971 }
16972}
16973
16974#[cfg(all(test, target_os = "linux"))]
16975mod cgroup_placement_tests {
16976 use super::{
16977 apply_cgroup_placement, remove_module_cgroup, LaunchSpecDigest, ModuleProtocol, ModuleSpec,
16978 SuperviseError, SupervisedChild,
16979 };
16980 use crate::stderr_tail::{StderrRing, StderrTailConfig};
16981 use cortexkit_test_support::ScratchDir;
16982 use std::{
16983 fs, io,
16984 path::{Path, PathBuf},
16985 sync::{Arc, Mutex},
16986 };
16987 use tokio::process::Command;
16988
16989 #[tokio::test]
16990 async fn unique_spawn_cgroups_clean_up_across_restarts_and_shutdown() {
16991 use super::*;
16992 let dir = ScratchDir::new("unique-spawn-cgroups");
16993 let root = PathBuf::from(format!(
16994 "/sys/fs/cgroup/subc-unique-test-{}-{}",
16995 std::process::id(),
16996 unix_ms_now()
16997 ));
16998 if let Err(error) = fs::create_dir(&root) {
16999 assert!(
17000 std::env::var_os("SUBC_REQUIRE_CGROUP_TEST").is_none(),
17001 "required cgroup test cannot execute: {error}"
17002 );
17003 eprintln!("SKIP unique_spawn_cgroups_clean_up_across_restarts_and_shutdown: {error}");
17004 return;
17005 }
17006 let placement = subc_cgroup::prepare_at(&root).unwrap().unwrap();
17007 let group_count = || {
17008 fs::read_dir(root.join("subc-modules"))
17009 .unwrap()
17010 .map(|entry| entry.unwrap().file_type().unwrap())
17011 .filter(|kind| kind.is_dir())
17012 .count()
17013 };
17014 let supervisor =
17015 Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default())
17016 .with_cgroup_placement(Some(placement.clone()));
17017 let runtime = supervisor.runtime_config();
17018 let mut spec = ModuleSpec {
17019 module_id: "unique-spawn".into(),
17020 program: PathBuf::from("/bin/sleep"),
17021 args: vec!["60".into()],
17022 env: ["XDG_DATA_HOME", "XDG_RUNTIME_DIR", "XDG_CONFIG_HOME"]
17023 .into_iter()
17024 .map(|key| (key.into(), dir.to_string_lossy().into_owned()))
17025 .collect(),
17026 reserved: false,
17027 reserved_prefixes: vec![],
17028 protocol: ModuleProtocol::None,
17029 overlap: Default::default(),
17030 carry_containment: false,
17031 };
17032 let spawn = |spec: &ModuleSpec| {
17033 spawn_child(
17034 spec,
17035 None,
17036 None,
17037 &runtime.stderr_ring,
17038 None,
17039 &runtime.child_roster,
17040 Some(&placement),
17041 )
17042 .unwrap()
17043 };
17044 let mut live = spawn(&spec);
17045 for _ in 0..3 {
17046 let next = spawn(&spec);
17048 assert_ne!(live.module_id, next.module_id);
17049 live.start_kill().unwrap();
17050 live.wait().await.unwrap();
17051 live = next;
17052 assert!(
17053 live.child.try_wait().unwrap().is_none(),
17054 "retiring the old slot must not kill the replacement"
17055 );
17056 assert_eq!(
17057 group_count(),
17058 1,
17059 "only the live spawn's cgroup should remain"
17060 );
17061 }
17062 supervisor.begin_daemon_shutdown();
17063 let reap = tokio::spawn(async move {
17064 live.wait().await.unwrap();
17065 });
17066 supervisor
17067 .end_children_for_daemon_shutdown(false, std::future::pending())
17068 .await;
17069 reap.await.unwrap();
17070 assert_eq!(group_count(), 0);
17071 spec.program = PathBuf::from("/bin/true");
17073 spec.args.clear();
17074 let fresh_roster = ChildRoster::default();
17075 let mut short = spawn_child(
17076 &spec,
17077 None,
17078 None,
17079 &runtime.stderr_ring,
17080 None,
17081 &fresh_roster,
17082 Some(&placement),
17083 )
17084 .unwrap();
17085 short.wait().await.unwrap();
17086 assert_eq!(group_count(), 0);
17087 spec.module_id = "_".repeat(255);
17088 let mut long_id = spawn_child(
17089 &spec,
17090 None,
17091 None,
17092 &runtime.stderr_ring,
17093 None,
17094 &fresh_roster,
17095 Some(&placement),
17096 )
17097 .unwrap();
17098 long_id.wait().await.unwrap();
17099 assert_eq!(
17100 group_count(),
17101 0,
17102 "valid long module IDs must not exceed cgroup NAME_MAX"
17103 );
17104 fs::remove_dir(root.join("subc-modules")).unwrap();
17105 fs::remove_dir(root).unwrap();
17106 eprintln!("EXECUTED unique_spawn_cgroups_clean_up_across_restarts_and_shutdown");
17107 }
17108
17109 #[test]
17110 fn failed_parent_cgroup_open_is_a_cgroup_supervision_error() {
17111 let path = Path::new("/definitely-missing-subc-cgroup");
17112 let mut command = Command::new("true");
17113 let error = apply_cgroup_placement(
17114 &mut command,
17115 &ModuleSpec {
17116 module_id: "broken-cgroup".to_string(),
17117 program: PathBuf::from("true"),
17118 args: Vec::new(),
17119 env: Vec::new(),
17120 reserved: false,
17121 reserved_prefixes: Vec::new(),
17122 protocol: ModuleProtocol::Subc,
17123 overlap: Default::default(),
17124 carry_containment: false,
17125 },
17126 path,
17127 )
17128 .expect_err("a parent cgroup open failure must reject the supervised spawn");
17129 let reason = error.to_string();
17130
17131 assert!(
17132 matches!(error, SuperviseError::Cgroup { .. }),
17133 "parent cgroup open must be reported as a cgroup supervision error: {reason}"
17134 );
17135 assert!(
17136 reason.contains("/definitely-missing-subc-cgroup/cgroup.procs"),
17137 "parent cgroup open failure must name cgroup.procs: {reason}"
17138 );
17139 }
17140
17141 #[tokio::test]
17142 async fn reaping_a_child_removes_its_empty_module_cgroup() {
17143 let root = ScratchDir::new("supervisor-reap-cgroup");
17144 fs::write(root.join("cgroup.procs"), b"").expect("write scratch cgroup marker");
17145 let placement = subc_cgroup::prepare_at(&root)
17146 .expect("prepare scratch cgroup root")
17147 .expect("scratch root has a cgroup.procs marker");
17148 let module_id = "reaped-module";
17149 let module = placement
17150 .module_path(module_id)
17151 .expect("create scratch module cgroup");
17152 let child = Command::new("true")
17153 .env("XDG_DATA_HOME", root.path())
17154 .env("XDG_RUNTIME_DIR", root.path())
17155 .env("XDG_CONFIG_HOME", root.path())
17156 .spawn()
17157 .expect("spawn short-lived child");
17158 let pid = child.id().expect("spawned child has pid");
17159 let mut child = SupervisedChild {
17160 subtree_target: crate::subtree_resources::Target::Unavailable("not_running"),
17161 child: crate::supervise::containment::ProcessChild::plain(child, pid),
17162 exec: crate::activity::ExecOwner(Arc::new(crate::activity::ExecOutcome::new())),
17163 protocol: ModuleProtocol::Subc,
17164 module_id: module_id.to_string(),
17165 cgroup_placement: Some(placement),
17166 stdout_pump: None,
17167 stderr_pump: None,
17168 stderr_ring: Arc::new(Mutex::new(StderrRing::new(StderrTailConfig::default()))),
17169 spawned_at: tokio::time::Instant::now(),
17170 registration_deadline: None,
17171 registration_watch: None,
17172 spawned_at_ms: 0,
17173 spawned_from: PathBuf::from("true"),
17174 launch_spec_digest: LaunchSpecDigest([0; 32]),
17175 spawned_file_identity: None,
17176 process_start_time: None,
17177 process_identity: None,
17178 pid,
17179 roster_guard: None,
17180 #[cfg(target_os = "macos")]
17181 privacy_exec: None,
17182 spawn_failure: None,
17183 };
17184
17185 child.wait().await.expect("reap short-lived child");
17186
17187 assert!(
17188 !module.exists(),
17189 "waiting for a non-carried child must remove its empty cgroup before returning"
17190 );
17191 child.finish_exit(false).await.unwrap();
17192 }
17193
17194 #[test]
17195 fn non_empty_cgroup_removal_is_reported_without_blocking_teardown() {
17196 let root = ScratchDir::new("supervisor-non-empty-cgroup");
17197 fs::write(root.join("cgroup.procs"), b"").expect("write scratch cgroup marker");
17198 let placement = subc_cgroup::prepare_at(&root)
17199 .expect("prepare scratch cgroup root")
17200 .expect("scratch root has a cgroup.procs marker");
17201 let module = placement
17202 .module_path("surviving-module")
17203 .expect("create scratch module cgroup");
17204 fs::write(module.join("surviving-process"), b"still present")
17205 .expect("make scratch cgroup non-empty");
17206 let (logs, _guard) = crate::router::test_log::log_capture(tracing::Level::WARN);
17207
17208 remove_module_cgroup(&placement, "surviving-module");
17209
17210 let logs = crate::router::test_log::captured_logs(&logs);
17211 assert!(
17212 module.exists(),
17213 "failed removal must leave the cgroup intact"
17214 );
17215 assert!(
17216 logs.contains("could not remove module cgroup after process exit; continuing teardown")
17217 && logs.contains("surviving-module"),
17218 "best-effort removal must report the failure without returning it: {logs}"
17219 );
17220 }
17221
17222 #[test]
17223 fn cgroup_pre_exec_spawn_failure_names_the_cgroup_path() {
17224 let cgroup_path = PathBuf::from("/sys/fs/cgroup/subc-modules/broken-module");
17225 let reason = SuperviseError::Spawn {
17226 program: PathBuf::from("/bin/true"),
17227 source: io::Error::from_raw_os_error(13),
17228 cgroup_path: Some(cgroup_path.clone()),
17229 }
17230 .to_string();
17231
17232 assert!(
17233 reason.contains(&cgroup_path.display().to_string()),
17234 "a pre_exec spawn failure must name the cgroup path: {reason}"
17235 );
17236 }
17237}
17238
17239#[cfg(test)]
17240mod spawn_subscriber_lag_tests {
17241 use super::*;
17242
17243 #[tokio::test]
17248 async fn lagged_spawn_subscriber_receives_a_terminal_lagged_error_after_its_queued_frames() {
17249 let feed = SpawnEventFeed::default();
17250 feed.configure_incarnation("lag-incarnation".to_string());
17251 let (tx, mut rx) = mpsc::channel(1);
17254 feed.subscribe(ConnectionId::new(1), 7, 1, None, FrameSink::new(tx))
17255 .expect("subscribe");
17256 let emitted = SPAWN_SUBSCRIBER_BUFFER + 16;
17257 for index in 0..emitted {
17258 feed.emit_spawned(&format!("lag-module-{index}"), 1000, 0);
17259 tokio::task::yield_now().await;
17262 }
17263 assert_eq!(
17264 feed.subscriber_count(),
17265 0,
17266 "the lagged subscriber must be removed"
17267 );
17268
17269 let mut data = Vec::new();
17270 let mut last = None;
17271 loop {
17272 let next = tokio::time::timeout(Duration::from_secs(5), rx.recv())
17273 .await
17274 .expect("the forwarder must finish once the subscriber is dropped");
17275 let Some(outbound) = next else { break };
17276 let frame = outbound.frame;
17277 if frame.header.ty == FrameType::StreamData {
17278 assert!(last.is_none(), "no data may follow the terminal frame");
17279 let event: SpawnEvent = serde_json::from_slice(&frame.body).unwrap();
17280 data.push(event.cursor.seq);
17281 } else {
17282 assert!(last.is_none(), "exactly one terminal frame");
17283 last = Some(frame);
17284 }
17285 }
17286 assert!(!data.is_empty(), "queued frames drain before the terminal");
17287 for pair in data.windows(2) {
17288 assert_eq!(
17289 pair[1],
17290 pair[0] + 1,
17291 "queued frames arrive dense and in order"
17292 );
17293 }
17294 let terminal = last.expect("a lagged subscriber must receive a terminal frame");
17295 assert_eq!(terminal.header.ty, FrameType::Error);
17296 assert_eq!(terminal.header.corr, 7);
17297 let body: subc_protocol::ErrorBody = serde_json::from_slice(&terminal.body).unwrap();
17298 assert_eq!(body.code, SPAWN_SUBSCRIBER_LAGGED_CODE);
17299 let detail = body.detail.expect("lagged error carries detail");
17300 assert_eq!(
17301 detail["first_undelivered_cursor"]["seq"],
17302 data.last().unwrap() + 1,
17303 "the named cursor is the first event the subscriber did not receive"
17304 );
17305 assert_eq!(
17306 detail["first_undelivered_cursor"]["daemon_incarnation"],
17307 "lag-incarnation"
17308 );
17309 }
17310}
17311
17312#[cfg(test)]
17313mod terminal_history_read_concurrency_tests {
17314 use super::*;
17315 use crate::terminal_journal::read_pause;
17316 use cortexkit_test_support::ScratchDir;
17317 use std::sync::mpsc as std_mpsc;
17318
17319 fn journaled_ring(
17320 journal: &Arc<crate::terminal_journal::TerminalJournal>,
17321 ) -> Arc<Mutex<TerminalRing>> {
17322 Arc::new(Mutex::new(
17323 TerminalRing::new(TerminalRingConfig::default(), 1)
17324 .with_journal(Some(Arc::clone(journal))),
17325 ))
17326 }
17327
17328 fn crash(at_ms: u64) -> ExitReport {
17329 ExitReport {
17330 kind: ExitKind::Crash,
17331 code: Some(1),
17332 signal: None,
17333 at_ms,
17334 }
17335 }
17336
17337 fn record_within(
17340 module_id: &'static str,
17341 ring: &Arc<Mutex<TerminalRing>>,
17342 at_ms: u64,
17343 bound: Duration,
17344 ) -> bool {
17345 let ring = Arc::clone(ring);
17346 let (done, done_rx) = std_mpsc::channel();
17347 std::thread::spawn(move || {
17348 record_terminal(
17349 module_id,
17350 &ring,
17351 &SpawnEventFeed::default(),
17352 &crash(at_ms),
17353 TerminalDisposition::Restarting,
17354 );
17355 let _ = done.send(());
17356 });
17357 done_rx.recv_timeout(bound).is_ok()
17358 }
17359
17360 #[test]
17365 fn exits_recorded_during_a_paused_history_read_are_not_blocked_or_half_merged() {
17366 let dir = ScratchDir::new("terminal-history-concurrent-read");
17367 let path = dir.join("terminals.jsonl");
17368 let journal = Arc::new(crate::terminal_journal::TerminalJournal::open(
17369 path.clone(),
17370 "daemon".into(),
17371 ));
17372 let reader_ring = journaled_ring(&journal);
17373 let other_ring = journaled_ring(&journal);
17374 assert!(record_within(
17375 "reader-module",
17376 &reader_ring,
17377 10,
17378 Duration::from_secs(5)
17379 ));
17380
17381 let (started, release) = read_pause::install(&path);
17382 let reading = {
17383 let ring = Arc::clone(&reader_ring);
17384 std::thread::spawn(move || durable_terminal_history_of(&ring, "reader-module"))
17385 };
17386 started
17387 .recv_timeout(Duration::from_secs(5))
17388 .expect("the history read reached its pause");
17389
17390 let bound = Duration::from_secs(1);
17391 assert!(
17392 record_within("other-module", &other_ring, 20, bound),
17393 "another module's exit waited on a history read (journal writer held)"
17394 );
17395 assert!(
17396 record_within("reader-module", &reader_ring, 30, bound),
17397 "the read module's own exit waited on its history read (ring held)"
17398 );
17399
17400 drop(release);
17401 let paused = reading.join().unwrap();
17402 assert_eq!(
17403 paused.entries.iter().map(|e| e.at_ms).collect::<Vec<_>>(),
17404 vec![10],
17405 "an exit recorded after the read began lands in neither half of it"
17406 );
17407 assert_eq!(paused.journal_skipped_lines, 0);
17408 assert_eq!(paused.journal_read_errors, 0);
17409
17410 let after = durable_terminal_history_of(&reader_ring, "reader-module");
17411 assert_eq!(
17412 after.entries.iter().map(|e| e.at_ms).collect::<Vec<_>>(),
17413 vec![10, 30],
17414 "the next read merges ring and journal with no duplicate"
17415 );
17416 assert_eq!(after.journal_skipped_lines, 0);
17417 }
17418}
17419
17420#[cfg(test)]
17425mod stderr_settle_tests {
17426 use std::{
17427 future::Future,
17428 io,
17429 pin::Pin,
17430 sync::{Arc, Mutex},
17431 task::{Context, Poll},
17432 time::Duration,
17433 };
17434
17435 use tokio::{
17436 io::{AsyncRead, ReadBuf},
17437 sync::oneshot,
17438 time::Instant,
17439 };
17440
17441 use super::{settle_stderr_pump, settle_stdout_pump, StderrPump};
17442 use crate::stderr_tail::{
17443 pump_stderr_to, pump_stdout_to, split_capture_stamp, untimed, CaptureState,
17444 ChildOutputSink, OutputSink, StderrRing, StderrTailConfig, TailEntry,
17445 };
17446
17447 const BOUND: Duration = Duration::from_millis(250);
17448
17449 struct HeldReader {
17453 before: Option<Vec<u8>>,
17454 gate: Option<oneshot::Receiver<()>>,
17455 after: io::Cursor<Vec<u8>>,
17456 }
17457
17458 impl AsyncRead for HeldReader {
17459 fn poll_read(
17460 mut self: Pin<&mut Self>,
17461 cx: &mut Context<'_>,
17462 buf: &mut ReadBuf<'_>,
17463 ) -> Poll<io::Result<()>> {
17464 if let Some(bytes) = self.before.take() {
17465 buf.put_slice(&bytes);
17466 return Poll::Ready(Ok(()));
17467 }
17468 if let Some(gate) = self.gate.as_mut() {
17469 match Pin::new(gate).poll(cx) {
17470 Poll::Pending => return Poll::Pending,
17471 Poll::Ready(_) => self.gate = None,
17472 }
17473 }
17474 Pin::new(&mut self.after).poll_read(cx, buf)
17475 }
17476 }
17477
17478 struct DiscardSink;
17479
17480 impl OutputSink for DiscardSink {
17481 fn write_line(&mut self, _line: &[u8]) {}
17482 }
17483
17484 fn line(text: &str) -> TailEntry {
17485 TailEntry::Line {
17486 text: text.to_string(),
17487 truncated: false,
17488 at_ms: None,
17489 }
17490 }
17491
17492 fn lock(ring: &Arc<Mutex<StderrRing>>) -> std::sync::MutexGuard<'_, StderrRing> {
17493 ring.lock().unwrap()
17494 }
17495
17496 fn held_pump(
17500 ring: &Arc<Mutex<StderrRing>>,
17501 before: &str,
17502 after: &str,
17503 ) -> (StderrPump, oneshot::Sender<()>) {
17504 let generation = lock(ring).begin_process();
17505 let (release, gate) = oneshot::channel();
17506 let reader = HeldReader {
17507 before: Some(before.as_bytes().to_vec()),
17508 gate: Some(gate),
17509 after: io::Cursor::new(after.as_bytes().to_vec()),
17510 };
17511 let task = tokio::spawn(pump_stderr_to(
17512 reader,
17513 Arc::clone(ring),
17514 generation,
17515 DiscardSink,
17516 ));
17517 (StderrPump { task, generation }, release)
17518 }
17519
17520 async fn wait_until(ring: &Arc<Mutex<StderrRing>>, done: impl Fn(&StderrRing) -> bool) {
17521 for _ in 0..1000 {
17522 if done(&lock(ring)) {
17523 return;
17524 }
17525 tokio::time::sleep(Duration::from_millis(1)).await;
17526 }
17527 panic!(
17528 "ring never reached the expected state: {:?}",
17529 lock(ring).snapshot(None, None)
17530 );
17531 }
17532
17533 #[tokio::test(start_paused = true)]
17534 async fn a_crash_line_the_reader_had_not_reached_by_the_bound_is_kept_before_the_restart() {
17535 let ring = Arc::new(Mutex::new(StderrRing::new(StderrTailConfig::default())));
17536 let (pump, release) = held_pump(&ring, "booting\n", "config error: missing storage\n");
17537
17538 settle_stderr_pump("crasher", &ring, pump, BOUND).await;
17539 let before_release = lock(&ring).snapshot(None, None);
17540 assert!(
17541 matches!(before_release.capture, CaptureState::Incomplete { .. }),
17542 "a reader that has not reached EOF cannot claim a whole tail: {before_release:?}"
17543 );
17544
17545 let next = lock(&ring).begin_process();
17548 lock(&ring).push_line_from(next, "next process booting");
17549 release.send(()).unwrap();
17550 wait_until(&ring, |ring| {
17551 ring.snapshot(None, None).capture == CaptureState::Captured
17552 })
17553 .await;
17554
17555 assert_eq!(
17556 untimed(lock(&ring).snapshot(None, None).entries),
17557 vec![
17558 line("booting"),
17559 line("config error: missing storage"),
17560 TailEntry::ProcessStart,
17561 line("next process booting"),
17562 ],
17563 "the crash's last line must survive a slow reader and stay in the crashed process's section"
17564 );
17565 }
17566
17567 #[tokio::test(start_paused = true)]
17568 async fn a_pipe_held_open_by_a_descendant_reads_incomplete_without_delaying_the_restart_past_the_bound(
17569 ) {
17570 let ring = Arc::new(Mutex::new(StderrRing::new(StderrTailConfig::default())));
17571 let (pump, _held) = held_pump(&ring, "parent exiting\n", "");
17574
17575 let started = Instant::now();
17576 settle_stderr_pump("orphaning", &ring, pump, BOUND).await;
17577 assert_eq!(
17578 started.elapsed(),
17579 BOUND,
17580 "the restart must wait exactly the bound for a pipe that stays open, no longer"
17581 );
17582
17583 let next = lock(&ring).begin_process();
17584 lock(&ring).push_line_from(next, "next process booting");
17585 tokio::time::sleep(Duration::from_secs(60)).await;
17586
17587 let snapshot = lock(&ring).snapshot(None, None);
17588 match &snapshot.capture {
17589 CaptureState::Incomplete { reason } => assert!(
17590 reason.contains("had not reached EOF") && reason.contains("250ms"),
17591 "the reason must say what is missing and after how long: {reason}"
17592 ),
17593 other => panic!("expected Incomplete while the pipe is held open, got {other:?}"),
17594 }
17595 assert_eq!(
17596 untimed(snapshot.entries),
17597 vec![
17598 line("parent exiting"),
17599 TailEntry::ProcessStart,
17600 line("next process booting"),
17601 ]
17602 );
17603 }
17604
17605 #[tokio::test(start_paused = true)]
17606 async fn a_reader_that_reaches_eof_within_the_bound_leaves_the_tail_captured() {
17607 let ring = Arc::new(Mutex::new(StderrRing::new(StderrTailConfig::default())));
17608 let (pump, release) = held_pump(&ring, "one\n", "two\n");
17609 release.send(()).unwrap();
17610
17611 settle_stderr_pump("clean", &ring, pump, BOUND).await;
17612
17613 let snapshot = lock(&ring).snapshot(None, None);
17614 assert_eq!(snapshot.capture, CaptureState::Captured);
17615 assert_eq!(untimed(snapshot.entries), vec![line("one"), line("two")]);
17616 }
17617
17618 #[tokio::test(start_paused = true)]
17619 async fn stdout_buffered_past_the_exit_drain_bound_reaches_the_capture_file() {
17620 const BURST: usize = 150;
17621 let capture = cortexkit_test_support::ScratchDir::new("stdout-exit-drain");
17622 let path = capture.join("two-pipe.stderr.log");
17623 let sink = ChildOutputSink::open(&path, cortexkit_log::Retention::default()).unwrap();
17624 let ring = Arc::new(Mutex::new(StderrRing::new(StderrTailConfig::default())));
17625 let generation = lock(&ring).begin_process();
17626 let stdout = (0..BURST)
17627 .map(|i| format!("out-{i:04}-{}\n", "o".repeat(64)))
17628 .collect::<String>();
17629 let stderr = (0..BURST)
17630 .map(|i| format!("err-{i:04}-{}\n", "e".repeat(64)))
17631 .collect::<String>();
17632 let first_chunk = stdout.find('\n').unwrap() + 1 + "out-0001-".len();
17637 let (release, gate) = oneshot::channel();
17638 let reader = HeldReader {
17639 before: Some(stdout.as_bytes()[..first_chunk].to_vec()),
17640 gate: Some(gate),
17641 after: io::Cursor::new(stdout.as_bytes()[first_chunk..stdout.len() - 1].to_vec()),
17642 };
17643 let (finished, completion) = oneshot::channel();
17644 let stdout_sink = sink.clone();
17645 let pump = tokio::spawn(async move {
17646 pump_stdout_to(reader, stdout_sink).await;
17647 finished.send(()).unwrap();
17648 });
17649 pump_stderr_to(io::Cursor::new(stderr.into_bytes()), ring, generation, sink).await;
17650
17651 let started = Instant::now();
17652 settle_stdout_pump("two-pipe", pump, BOUND).await;
17653 assert_eq!(
17654 started.elapsed(),
17655 BOUND,
17656 "a late reader must not hold up restart"
17657 );
17658 assert_eq!(
17659 std::fs::read_to_string(&path).unwrap().lines().count(),
17660 BURST + 1
17661 );
17662
17663 release
17664 .send(())
17665 .expect("stdout reader must survive the exit drain bound");
17666 tokio::time::timeout(Duration::from_secs(1), completion)
17667 .await
17668 .expect("released stdout reader must reach EOF")
17669 .expect("stdout capture must finish at EOF, not be cancelled");
17670 let contents = std::fs::read_to_string(&path).unwrap();
17671 let mut lines = contents
17672 .lines()
17673 .map(|line| split_capture_stamp(line).expect("capture stamp").1)
17674 .collect::<Vec<_>>();
17675 let mut expected = (0..BURST)
17676 .flat_map(|i| {
17677 [
17678 format!("out-{i:04}-{}", "o".repeat(64)),
17679 format!("err-{i:04}-{}", "e".repeat(64)),
17680 ]
17681 })
17682 .collect::<Vec<_>>();
17683 lines.sort_unstable();
17684 expected.sort_unstable();
17685 assert_eq!(
17686 lines, expected,
17687 "both pipes must retain every complete line exactly once"
17688 );
17689 }
17690}
17691
17692#[cfg(all(test, windows))]
17706mod job_containment_tests {
17707 use super::*;
17708 use cortexkit_test_support::ScratchDir;
17709 use std::{
17710 path::{Path, PathBuf},
17711 sync::{Arc, Mutex},
17712 time::{Duration, Instant},
17713 };
17714
17715 fn stub_path() -> PathBuf {
17721 let mut path = std::env::current_exe().expect("current_exe available in tests");
17722 path.pop();
17723 path.pop();
17724 path.push("fake-aft-stub.exe");
17725 assert!(
17726 path.exists(),
17727 "fake-aft-stub not built at {}: run `cargo test -p subc-core` (which builds \
17728 [[bin]] targets) rather than `cargo test -p subc-core --lib` (which does not)",
17729 path.display()
17730 );
17731 path
17732 }
17733
17734 fn read_grandchild_pid(path: &Path) -> u32 {
17736 let deadline = Instant::now() + Duration::from_secs(10);
17737 loop {
17738 if let Ok(contents) = std::fs::read_to_string(path) {
17739 if let Ok(pid) = contents.trim().parse() {
17740 return pid;
17741 }
17742 }
17743 assert!(
17744 Instant::now() < deadline,
17745 "the stub never recorded a grandchild pid at {}",
17746 path.display()
17747 );
17748 std::thread::sleep(Duration::from_millis(10));
17749 }
17750 }
17751
17752 struct Fixture {
17755 _dir: ScratchDir,
17756 module_id: String,
17757 grandchild: u32,
17758 child: Option<SupervisedChild>,
17759 registry: Arc<Registry>,
17760 snapshot: Arc<Mutex<SupervisorSnapshot>>,
17761 terminal_ring: Arc<Mutex<TerminalRing>>,
17762 spawn_events: SpawnEventFeed,
17763 }
17764
17765 fn fixture(label: &str, module_id: &str) -> Fixture {
17766 let dir = ScratchDir::new(label);
17767 let pid_file = dir.join("grandchild.pid");
17768 let supervisor = Supervisor::new_for_test(
17769 Arc::new(Registry::default()),
17770 RestartPolicy::new(3, Duration::ZERO),
17771 );
17772 let runtime = supervisor.runtime_config();
17773 let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::starting()));
17774 let spec = ModuleSpec {
17775 module_id: module_id.to_string(),
17776 program: stub_path(),
17777 args: Vec::new(),
17781 env: vec![
17782 ("FAKE_AFT_NEVER_CONNECT".to_string(), "1".to_string()),
17783 (
17784 "FAKE_AFT_GRANDCHILD_PID_FILE".to_string(),
17785 pid_file.display().to_string(),
17786 ),
17787 ],
17788 reserved: false,
17789 reserved_prefixes: Vec::new(),
17790 protocol: ModuleProtocol::Subc,
17791 overlap: Default::default(),
17792 carry_containment: false,
17793 };
17794 let child = spawn_and_mark_running(&spec, &runtime, &snapshot)
17795 .expect("spawn the supervised fixture");
17796 let grandchild = read_grandchild_pid(&pid_file);
17797 Fixture {
17798 _dir: dir,
17799 module_id: module_id.to_string(),
17800 grandchild,
17801 child: Some(child),
17802 registry: Arc::new(Registry::default()),
17803 snapshot,
17804 terminal_ring: Arc::clone(&runtime.terminal_ring),
17805 spawn_events: SpawnEventFeed::default(),
17806 }
17807 }
17808
17809 impl Fixture {
17810 async fn drain(&mut self) {
17812 let child = self
17813 .child
17814 .take()
17815 .expect("the fixture child is still present");
17816 drain_child_to_state(
17817 &self.module_id,
17818 ModuleProtocol::Subc,
17819 StopNotice::NotSent,
17822 &self.registry,
17823 None,
17824 &self.snapshot,
17825 &self.terminal_ring,
17826 &self.spawn_events,
17827 child,
17828 Duration::from_millis(500),
17829 ModuleState::Stopped,
17830 Some(false),
17831 &FailureContext::operator(),
17832 )
17833 .await
17834 .expect("drain the supervised fixture");
17835 }
17836 }
17837
17838 #[tokio::test]
17844 async fn teardown_reaps_the_grandchild() {
17845 let mut fixture = fixture("teardown-grandchild", "tree-teardown");
17846 let grandchild = fixture.grandchild;
17847
17848 assert!(
17849 subc_jobobject::process_exists(grandchild),
17850 "grandchild {grandchild} must be alive before teardown, or this proves nothing"
17851 );
17852
17853 fixture.drain().await;
17854
17855 assert!(
17856 subc_jobobject::wait_for_process_exit(grandchild, Duration::from_secs(10)),
17857 "grandchild {grandchild} outlived module teardown: the tree was not contained"
17858 );
17859 }
17860
17861 #[test]
17874 fn an_uncontained_grandchild_survives_a_direct_child_kill() {
17875 let dir = ScratchDir::new("teardown-uncontained");
17876 let pid_file = dir.join("grandchild.pid");
17877 let mut child = std::process::Command::new(stub_path())
17878 .env("FAKE_AFT_NEVER_CONNECT", "1")
17879 .env(
17880 "FAKE_AFT_GRANDCHILD_PID_FILE",
17881 pid_file.display().to_string(),
17882 )
17883 .stdin(std::process::Stdio::null())
17884 .stdout(std::process::Stdio::null())
17885 .stderr(std::process::Stdio::null())
17886 .spawn()
17887 .expect("spawn the uncontained fixture");
17888 let grandchild = read_grandchild_pid(&pid_file);
17889
17890 child.kill().expect("kill the direct child");
17892 let _ = child.wait();
17893
17894 assert!(
17895 subc_jobobject::process_exists(grandchild),
17896 "grandchild {grandchild} died with the direct child, so this control no longer \
17897 distinguishes contained from uncontained teardown and the regression test is \
17898 passing vacuously"
17899 );
17900
17901 kill_tree(grandchild);
17904 }
17905
17906 #[tokio::test]
17915 async fn dropping_containment_reaps_the_grandchild() {
17916 let mut fixture = fixture("drop-containment", "tree-drop");
17917 let grandchild = fixture.grandchild;
17918
17919 assert!(subc_jobobject::process_exists(grandchild));
17920
17921 fixture.child.as_mut().expect("child present").job = None;
17923
17924 assert!(
17925 subc_jobobject::wait_for_process_exit(grandchild, Duration::from_secs(10)),
17926 "grandchild {grandchild} survived the containment handle closing, so a daemon \
17927 crash would leave the tree behind"
17928 );
17929 }
17930
17931 fn kill_tree(pid: u32) {
17933 let _ = std::process::Command::new("taskkill.exe")
17934 .args(["/PID", &pid.to_string(), "/T", "/F"])
17935 .stdin(std::process::Stdio::null())
17936 .stdout(std::process::Stdio::null())
17937 .stderr(std::process::Stdio::null())
17938 .status();
17939 assert!(
17940 subc_jobobject::wait_for_process_exit(pid, Duration::from_secs(10)),
17941 "could not clean up grandchild {pid}"
17942 );
17943 }
17944}
17945
17946#[cfg(test)]
17947mod privacy_trampoline_configuration_tests {
17948 #[tokio::test]
17949 #[cfg_attr(not(target_os = "macos"), ignore = "requires macOS privacy trampoline")]
17950 async fn macos_spawn_without_a_configured_trampoline_refuses_by_name() {
17951 #[cfg(target_os = "macos")]
17952 {
17953 let supervisor = super::Supervisor::new(
17954 std::sync::Arc::new(crate::Registry::default()),
17955 super::RestartPolicy::default(),
17956 );
17957 let error = supervisor.spawn(spec()).unwrap_err();
17958 assert!(
17959 error
17960 .to_string()
17961 .contains("no privacy trampoline configured"),
17962 "{error}"
17963 );
17964 }
17965 }
17966
17967 #[tokio::test]
17968 #[cfg_attr(not(target_os = "macos"), ignore = "requires macOS privacy trampoline")]
17969 async fn macos_wrong_trampoline_is_rejected_by_the_startup_probe() {
17970 #[cfg(target_os = "macos")]
17971 {
17972 let supervisor = super::Supervisor::new(
17973 std::sync::Arc::new(crate::Registry::default()),
17974 super::RestartPolicy::default(),
17975 )
17976 .with_privacy_trampoline(std::env::current_exe().unwrap());
17977 let error = supervisor.spawn(spec()).unwrap_err();
17978 assert!(
17979 error
17980 .to_string()
17981 .contains("binary does not implement the privacy trampoline protocol"),
17982 "{error}"
17983 );
17984 }
17985 }
17986
17987 #[cfg(target_os = "macos")]
17988 fn spec() -> super::ModuleSpec {
17989 super::ModuleSpec {
17990 module_id: "privacy-configuration".into(),
17991 program: "/bin/sleep".into(),
17992 args: vec!["30".into()],
17993 env: vec![],
17994 reserved: false,
17995 reserved_prefixes: vec![],
17996 protocol: subc_control::ModuleProtocol::None,
17997 overlap: super::ModuleOverlap::Exclusive,
17998 carry_containment: false,
17999 }
18000 }
18001}
18002
18003#[cfg(test)]
18004mod privacy_exec_boundary_tests {
18005 #[cfg(target_os = "macos")]
18006 use super::*;
18007 #[cfg(target_os = "macos")]
18008 use std::{
18009 io::{Read, Write},
18010 net::{TcpListener, TcpStream},
18011 };
18012
18013 #[cfg(target_os = "macos")]
18016 pub(super) fn before_image_sample(spec: &ModuleSpec, pid: u32) {
18017 if let Some((_, path)) = spec
18018 .env
18019 .iter()
18020 .find(|(key, _)| key == "SUBC_TEST_PRIVACY_SAMPLE_BARRIER")
18021 {
18022 let mut barrier = TcpStream::connect(path).unwrap();
18023 barrier
18024 .set_read_timeout(Some(Duration::from_secs(30)))
18025 .unwrap();
18026 barrier.write_all(&pid.to_ne_bytes()).unwrap();
18027 let mut release = [0];
18028 barrier.read_exact(&mut release).unwrap();
18029 assert_eq!(&release, b"X");
18030 }
18031 }
18032
18033 #[cfg(target_os = "macos")]
18034 fn spec(program: &str, args: &[&str]) -> ModuleSpec {
18035 ModuleSpec {
18036 module_id: "privacy-boundary".into(),
18037 program: program.into(),
18038 args: args.iter().map(|arg| (*arg).into()).collect(),
18039 env: vec![],
18040 reserved: false,
18041 reserved_prefixes: vec![],
18042 protocol: ModuleProtocol::None,
18043 overlap: ModuleOverlap::Exclusive,
18044 carry_containment: false,
18045 }
18046 }
18047
18048 #[cfg(target_os = "macos")]
18049 async fn accept(listener: TcpListener) -> TcpStream {
18050 let listener = tokio::net::TcpListener::from_std({
18052 listener.set_nonblocking(true).unwrap();
18053 listener
18054 })
18055 .unwrap();
18056 let (stream, _) = tokio::time::timeout(Duration::from_secs(30), listener.accept())
18057 .await
18058 .unwrap()
18059 .unwrap();
18060 let stream = stream.into_std().unwrap();
18061 stream.set_nonblocking(false).unwrap();
18062 stream
18063 .set_read_timeout(Some(Duration::from_secs(30)))
18064 .unwrap();
18065 stream
18066 }
18067
18068 #[tokio::test]
18069 #[cfg_attr(not(target_os = "macos"), ignore = "requires macOS privacy trampoline")]
18070 async fn macos_roster_withholds_a_nonnull_trampoline_image_until_exec_confirmation() {
18071 #[cfg(target_os = "macos")]
18072 {
18073 let root = cortexkit_test_support::ScratchDir::new("privacy-roster-barrier");
18074 let exec_listener = TcpListener::bind("127.0.0.1:0").unwrap();
18077 let sample_listener = TcpListener::bind("127.0.0.1:0").unwrap();
18078 let record = root.join("live-children.json");
18079 let supervisor =
18080 Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default())
18081 .with_live_children_record(&record);
18082 let runtime = supervisor.runtime_config();
18083 let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::starting()));
18084 let mut spec = spec("/bin/sleep", &["30"]);
18085 spec.env = vec![
18086 (
18087 "SUBC_TEST_PRIVACY_EXEC_BARRIER".into(),
18088 exec_listener.local_addr().unwrap().to_string(),
18089 ),
18090 (
18091 "SUBC_TEST_PRIVACY_SAMPLE_BARRIER".into(),
18092 sample_listener.local_addr().unwrap().to_string(),
18093 ),
18094 ];
18095 let spawn = tokio::task::spawn_blocking(move || {
18096 spawn_and_mark_running(&spec, &runtime, &snapshot).unwrap()
18097 });
18098 let mut sample = accept(sample_listener).await;
18099 let mut pid = [0; 4];
18100 sample.read_exact(&mut pid).unwrap();
18101 let pid = u32::from_ne_bytes(pid);
18102 let mut exec = accept(exec_listener).await;
18103 let mut ready = [0];
18104 exec.read_exact(&mut ready).unwrap();
18105 assert_eq!(&ready, b"R");
18106 let trampoline = subc_os::file_identity(&test_privacy_trampoline()).unwrap();
18109 assert_eq!(
18110 observe_spawned_image(pid).unwrap().executable,
18111 Some(trampoline)
18112 );
18113 sample.write_all(b"X").unwrap();
18114 let mut child = spawn.await.unwrap();
18115 let early = crate::live_children::read_record(&record).unwrap();
18116 assert_eq!(early.len(), 1);
18117 assert_eq!(early[0].pid, pid);
18118 assert_eq!(
18119 early[0].executable, None,
18120 "unconfirmed trampoline image entered the roster"
18121 );
18122 assert!(child.report_ready.get().is_none());
18123 child.privacy_exec.as_mut().unwrap().deadline =
18126 tokio::time::Instant::now() + Duration::from_secs(30);
18127 exec.write_all(b"X").unwrap();
18128 child.confirm_privacy_exec().await;
18129 assert_eq!(child.spawn_failure, None);
18130 assert!(child.report_ready.get().is_some());
18131 let confirmed = crate::live_children::read_record(&record).unwrap();
18132 let module = subc_os::file_identity(std::path::Path::new("/bin/sleep")).unwrap();
18133 assert_ne!(module, trampoline);
18134 assert_eq!(confirmed[0].executable, Some(module.into()));
18135 child.start_kill().unwrap();
18136 child.wait().await.unwrap();
18137 child.release_roster();
18138 }
18139 }
18140
18141 #[tokio::test]
18142 #[cfg_attr(not(target_os = "macos"), ignore = "requires macOS privacy trampoline")]
18143 async fn macos_already_exited_121_in_try_wait_is_a_module_exit_not_a_trampoline_refusal() {
18144 #[cfg(target_os = "macos")]
18145 {
18146 let registry = Arc::new(Registry::default());
18147 let policy = RestartPolicy::new(0, Duration::ZERO);
18148 let supervisor = Supervisor::new_for_test(Arc::clone(®istry), policy);
18149 let runtime = supervisor.runtime_config();
18150 let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::starting()));
18151 let spec = spec("/bin/sh", &["-c", "exit 121"]);
18152 let mut child = spawn_and_mark_running(&spec, &runtime, &snapshot).unwrap();
18156 let pid = child.pid;
18157 tokio::task::spawn_blocking(move || {
18158 subc_os::fork_exec_test::wait_for_child_exit_without_reaping(pid)
18159 })
18160 .await
18161 .unwrap()
18162 .unwrap();
18163 child.privacy_exec.as_mut().unwrap().deadline =
18164 tokio::time::Instant::now() + Duration::from_secs(30);
18165 let status = child.wait().await.unwrap();
18166 assert_eq!(status.code(), Some(121));
18167 assert!(child.privacy_exec.is_none());
18168 assert!(
18169 child.report_ready.get().is_none(),
18170 "an exited module must not publish a live pid"
18171 );
18172 let report = classify_reaped_child_exit(&snapshot, &child, &status);
18173 on_child_exit(
18174 &spec,
18175 policy,
18176 ®istry,
18177 &snapshot,
18178 &runtime.terminal_ring,
18179 &runtime.spawn_events,
18180 &runtime.child_roster,
18181 report,
18182 true,
18183 )
18184 .await;
18185 let state = lock_snapshot(&snapshot).unwrap();
18186 assert_eq!(state.state, ModuleState::Failed);
18187 assert_eq!(state.last_exit.as_ref().unwrap().code, Some(121));
18188 assert_eq!(state.reported_pid(), None);
18189 drop(state);
18190 let history = runtime.terminal_ring.lock().unwrap().snapshot();
18191 assert_eq!(history.entries.len(), 1);
18192 let terminal = &history.entries[0];
18193 assert_eq!(terminal.exit_code, Some(121));
18194 assert_eq!(terminal.exit_kind, subc_control::TerminalExitKind::Crash);
18195 assert_eq!(terminal.disposition, TerminalDisposition::Failed);
18196 assert_eq!(
18197 terminal.disposition_detail.as_deref(),
18198 Some(policy.budget_exhausted_detail().as_str()),
18199 "module exit 121 was classified as a trampoline refusal: {terminal:?}"
18200 );
18201 assert_eq!(child.spawn_failure, None);
18202 child.release_roster();
18203 }
18204 }
18205}
18206
18207#[cfg(all(test, unix))]
18211mod launch_nonce_descriptor_tests {
18212 use super::{ChildRoster, ModuleProtocol, ModuleSpec, SupervisorHandle};
18213 use crate::stderr_tail::{StderrRing, StderrTailConfig};
18214 use cortexkit_test_support::ScratchDir;
18215 use std::{
18216 path::PathBuf,
18217 sync::{Arc, Mutex},
18218 time::{Duration, Instant},
18219 };
18220
18221 async fn probe(role: super::SpawnRole) {
18222 let scratch = ScratchDir::new("launch-nonce-descriptor");
18223 let fd_copy = scratch.join("from-descriptor");
18224 let env_copy = scratch.join("environment");
18225 let script = format!(
18226 "env > '{env}.tmp' && mv '{env}.tmp' '{env}'; cat <&3 > '{fd}.tmp' && mv '{fd}.tmp' '{fd}'; sleep 30",
18227 fd = fd_copy.display(), env = env_copy.display(),
18228 );
18229 let xdg = |name: &str| (name.to_string(), scratch.join(name).display().to_string());
18230 let spec = ModuleSpec {
18231 module_id: "nonce-descriptor-probe".to_string(),
18232 program: PathBuf::from("/bin/sh"),
18233 args: vec!["-c".to_string(), script],
18234 env: vec![
18235 xdg("XDG_DATA_HOME"),
18236 xdg("XDG_RUNTIME_DIR"),
18237 xdg("XDG_CONFIG_HOME"),
18238 ("SUBC_LAUNCH_NONCE".to_string(), "must-not-leak".to_string()),
18239 ],
18240 reserved: true,
18241 reserved_prefixes: Vec::new(),
18242 protocol: ModuleProtocol::Subc,
18243 overlap: Default::default(),
18244 carry_containment: false,
18245 };
18246 let handle = SupervisorHandle::new();
18247 let ring = Arc::new(Mutex::new(StderrRing::new(StderrTailConfig::default())));
18248 let roster = ChildRoster::default();
18249 #[cfg(target_os = "macos")]
18250 {
18251 let path = super::test_privacy_trampoline();
18252 roster.set_privacy_trampoline(super::probe_privacy_trampoline(&path).map(|()| path));
18253 }
18254 let child = super::spawn_child_in_slot(
18255 &spec,
18256 None,
18257 Some(&handle),
18258 &ring,
18259 None,
18260 &roster,
18261 #[cfg(target_os = "linux")]
18262 None,
18263 role,
18264 matches!(role, super::SpawnRole::SwapCandidate),
18265 )
18266 .expect("spawn probe");
18267 let deadline = Instant::now() + Duration::from_secs(10);
18268 while !(fd_copy.exists() && env_copy.exists()) {
18269 assert!(Instant::now() < deadline, "probe never wrote its copies");
18270 tokio::time::sleep(Duration::from_millis(20)).await;
18271 }
18272 let nonce = std::fs::read_to_string(fd_copy).unwrap();
18273 assert!(!nonce.is_empty());
18274 let environment = std::fs::read_to_string(env_copy).unwrap();
18275 assert!(environment
18276 .lines()
18277 .any(|line| line.starts_with("SUBC_LAUNCH_NONCE_FD=3:")));
18278 let copy = environment
18279 .lines()
18280 .find_map(|line| line.strip_prefix("SUBC_LAUNCH_NONCE="));
18281 assert_eq!(
18282 copy, None,
18283 "Unix children must never receive the environment nonce"
18284 );
18285 if matches!(role, super::SpawnRole::Plain) {
18286 assert_eq!(
18287 handle.spawn_nonce(&spec.module_id).as_deref(),
18288 Some(nonce.as_str())
18289 );
18290 }
18291 drop(child);
18292 }
18293
18294 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
18295 async fn a_spawned_module_receives_its_nonce_only_on_descriptor_3() {
18296 probe(super::SpawnRole::Plain).await;
18297 }
18298
18299 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
18300 async fn swap_candidate_receives_its_nonce_only_on_descriptor_3() {
18301 probe(super::SpawnRole::SwapCandidate).await;
18302 }
18303}
18304
18305#[cfg(all(test, target_os = "linux"))]
18306#[path = "supervise/cgroup_reap_tests.rs"]
18307mod cgroup_reap_tests;
18308
18309#[cfg(all(test, target_os = "linux"))]
18310mod cgroup_containment_tests {
18311 use super::*;
18312 use cortexkit_test_support::ScratchDir;
18313
18314 #[tokio::test]
18315 async fn linux_subtree_memory_covers_module_children() {
18316 let dir = ScratchDir::new("subtree-memory");
18317 let root = PathBuf::from(format!(
18318 "/sys/fs/cgroup/subc-memory-test-{}-{}",
18319 std::process::id(),
18320 unix_ms_now()
18321 ));
18322 if let Err(error) = std::fs::create_dir(&root) {
18323 assert!(
18324 std::env::var_os("SUBC_REQUIRE_CGROUP_TEST").is_none(),
18325 "required cgroup test cannot execute: {error}"
18326 );
18327 eprintln!("SKIP linux_subtree_memory_covers_module_children: {error}");
18328 return;
18329 }
18330 let placement = subc_cgroup::prepare_at(&root)
18331 .unwrap()
18332 .expect("delegated cgroup");
18333 let supervisor = Supervisor::new_for_test(
18334 Arc::new(Registry::default()),
18335 RestartPolicy::new(3, Duration::ZERO),
18336 )
18337 .with_cgroup_placement(Some(placement));
18338 let ready = dir.join("ready");
18339 let module = supervisor
18340 .spawn(ModuleSpec {
18341 module_id: "memory-tree".into(),
18342 program: PathBuf::from("/bin/sh"),
18343 args: vec![
18344 "-c".into(),
18345 "sleep 600 & sleep 600 & sleep 600 & echo ready > \"$1\"; wait".into(),
18346 "fixture".into(),
18347 ready.display().to_string(),
18348 ],
18349 env: ["XDG_DATA_HOME", "XDG_RUNTIME_DIR", "XDG_CONFIG_HOME"]
18350 .into_iter()
18351 .map(|key| (key.into(), dir.display().to_string()))
18352 .collect(),
18353 reserved: false,
18354 reserved_prefixes: Vec::new(),
18355 protocol: ModuleProtocol::None,
18356 overlap: Default::default(),
18357 carry_containment: false,
18358 })
18359 .unwrap();
18360 let deadline = tokio::time::Instant::now() + Duration::from_secs(10);
18361 while !ready.exists() {
18362 assert!(
18363 tokio::time::Instant::now() < deadline,
18364 "children never started"
18365 );
18366 tokio::time::sleep(Duration::from_millis(10)).await;
18367 }
18368 let readings = crate::subtree_resources::read(vec![module.subtree_target()]).await;
18369 let reading = &readings[0];
18370 assert!(
18371 reading.process_count.is_some_and(|count| count >= 4),
18372 "{reading:?}"
18373 );
18374 assert!(
18375 reading.memory_bytes.is_some_and(|bytes| bytes > 0),
18376 "{reading:?}"
18377 );
18378 module.stop().await.unwrap();
18379 drop(module);
18380 std::fs::remove_dir(root.join("subc-modules")).unwrap();
18381 std::fs::remove_dir(&root).unwrap();
18382 eprintln!("EXECUTED linux_subtree_memory_covers_module_children: {reading:?}");
18383 }
18384
18385 fn running(pid: u32) -> bool {
18386 std::fs::read_to_string(format!("/proc/{pid}/stat"))
18388 .ok()
18389 .and_then(|stat| {
18390 stat.rsplit_once(") ")
18391 .map(|(_, rest)| rest.starts_with('Z'))
18392 })
18393 .is_some_and(|zombie| !zombie)
18394 }
18395
18396 #[tokio::test]
18397 async fn linux_teardown_reaps_the_grandchild() {
18398 teardown_tree("linux_teardown_reaps_the_grandchild", false).await;
18399 }
18400
18401 #[tokio::test]
18402 async fn linux_shutdown_straggler_reaps_the_grandchild() {
18403 teardown_tree("linux_shutdown_straggler_reaps_the_grandchild", true).await;
18404 }
18405
18406 async fn teardown_tree(test_name: &str, shutdown: bool) {
18407 let dir = ScratchDir::new(test_name);
18408 let root = PathBuf::from(format!(
18409 "/sys/fs/cgroup/subc-tree-test-{test_name}-{}-{}",
18410 std::process::id(),
18411 unix_ms_now()
18412 ));
18413 if let Err(error) = std::fs::create_dir(&root) {
18414 assert!(
18415 std::env::var_os("SUBC_REQUIRE_CGROUP_TEST").is_none(),
18416 "required cgroup test cannot execute: {error}"
18417 );
18418 eprintln!(
18419 "SKIP {test_name}: no writable delegated cgroup at {}: {error}",
18420 root.display()
18421 );
18422 return;
18423 }
18424 let placement = subc_cgroup::prepare_at(&root)
18425 .expect("prepare isolated kernel cgroup")
18426 .expect("isolated cgroup is delegated");
18427 let module_id = "tree-teardown";
18428 let module = placement
18429 .module_path(module_id)
18430 .expect("create isolated module cgroup");
18431 if !module.join("cgroup.kill").exists() {
18432 std::fs::remove_dir(&module).unwrap();
18433 std::fs::remove_dir(root.join("subc-modules")).unwrap();
18434 std::fs::remove_dir(&root).unwrap();
18435 assert!(
18436 std::env::var_os("SUBC_REQUIRE_CGROUP_TEST").is_none(),
18437 "required cgroup.kill interface unavailable"
18438 );
18439 eprintln!("SKIP {test_name}: cgroup.kill unavailable (kernel < 5.14)");
18440 return;
18441 }
18442 let supervisor = Supervisor::new_for_test(
18443 Arc::new(Registry::default()),
18444 RestartPolicy::new(3, Duration::ZERO),
18445 )
18446 .with_cgroup_placement(Some(placement));
18447 let mut runtime = supervisor.runtime_config();
18448 runtime.child_roster = runtime
18449 .child_roster
18450 .for_module(Arc::new(Mutex::new(Duration::from_millis(100))));
18451 let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::starting()));
18452 let pid_file = dir.join("grandchild.pid");
18453 let spec = ModuleSpec {
18454 module_id: module_id.to_string(),
18455 program: PathBuf::from("/bin/sh"),
18456 args: vec![
18457 "-c".into(),
18458 "trap '' TERM; sleep 600 & echo $! > \"$1\"; wait".into(),
18459 "fixture".into(),
18460 pid_file.display().to_string(),
18461 ],
18462 env: ["XDG_DATA_HOME", "XDG_RUNTIME_DIR", "XDG_CONFIG_HOME"]
18463 .into_iter()
18464 .map(|key| (key.to_string(), dir.display().to_string()))
18465 .collect(),
18466 reserved: false,
18467 reserved_prefixes: Vec::new(),
18468 protocol: ModuleProtocol::None,
18469 overlap: Default::default(),
18470 carry_containment: false,
18471 };
18472 let child =
18473 spawn_and_mark_running(&spec, &runtime, &snapshot).expect("spawn supervised tree");
18474 let deadline = tokio::time::Instant::now() + Duration::from_secs(10);
18475 let grandchild: u32 = loop {
18476 if let Ok(contents) = std::fs::read_to_string(&pid_file) {
18477 if let Ok(pid) = contents.trim().parse() {
18478 break pid;
18479 }
18480 }
18481 assert!(
18482 tokio::time::Instant::now() < deadline,
18483 "grandchild pid was not recorded"
18484 );
18485 tokio::time::sleep(Duration::from_millis(10)).await;
18486 };
18487 assert!(
18488 running(grandchild),
18489 "grandchild must be alive before teardown"
18490 );
18491 if shutdown {
18492 let mut child = child;
18493 crate::child_roster::end_children_for_daemon_shutdown(
18494 &runtime.child_roster,
18495 false,
18496 std::future::pending(),
18497 )
18498 .await;
18499 child.wait().await.expect("reap shutdown straggler");
18500 } else {
18501 drain_child_to_state(
18502 module_id,
18503 ModuleProtocol::None,
18504 StopNotice::NotSent,
18505 &Registry::default(),
18506 None,
18507 &snapshot,
18508 &runtime.terminal_ring,
18509 &SpawnEventFeed::default(),
18510 child,
18511 Duration::from_millis(100),
18512 ModuleState::Stopped,
18513 Some(false),
18514 &FailureContext::operator(),
18515 )
18516 .await
18517 .expect("real supervisor teardown");
18518 }
18519 let deadline = tokio::time::Instant::now() + Duration::from_secs(5);
18520 while running(grandchild) && tokio::time::Instant::now() < deadline {
18521 tokio::time::sleep(Duration::from_millis(10)).await;
18522 }
18523 let survived = running(grandchild);
18524 if survived {
18526 let pid = rustix::process::Pid::from_raw(grandchild as i32).unwrap();
18527 let _ = rustix::process::kill_process(pid, rustix::process::Signal::KILL);
18528 tokio::time::sleep(Duration::from_millis(100)).await;
18529 }
18530 if module.exists() {
18531 std::fs::remove_dir(&module).expect("remove empty module cgroup");
18532 }
18533 std::fs::remove_dir(root.join("subc-modules")).unwrap();
18534 std::fs::remove_dir(&root).unwrap();
18535 assert!(
18536 !survived,
18537 "grandchild {grandchild} outlived module teardown"
18538 );
18539 eprintln!("EXECUTED {test_name}: grandchild {grandchild} killed in isolated cgroup");
18540 }
18541}
18542
18543#[cfg(all(test, unix))]
18546mod record_write_stall_tests {
18547 use super::*;
18548 use std::sync::atomic::{AtomicBool, Ordering};
18549 use subc_transport::{authenticate_client, ConnectionInfo, Endpoint, SCHEMA_VERSION};
18550
18551 const MODULE_AUTH_DEADLINE: Duration = Duration::from_secs(2);
18554 const HELD_RECORD_WRITE: Duration = Duration::from_secs(3);
18558
18559 fn sleeping_spec() -> ModuleSpec {
18571 ModuleSpec {
18572 module_id: "record-write-stall".into(),
18573 program: "/bin/sleep".into(),
18574 args: vec!["600".into()],
18575 env: vec![],
18576 reserved: false,
18577 reserved_prefixes: vec![],
18578 protocol: ModuleProtocol::None,
18579 overlap: Default::default(),
18580 carry_containment: false,
18581 }
18582 }
18583
18584 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
18585 async fn module_auth_completes_while_the_restart_waits_on_the_record_write() {
18586 let home = cortexkit_test_support::ScratchDir::new("record-write-stall");
18587 let supervisor =
18588 Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default())
18589 .with_live_children_record(home.join("live-children.json"));
18590 let module = supervisor.spawn(sleeping_spec()).unwrap();
18591 let old_pid = tokio::time::timeout(Duration::from_secs(10), async {
18596 loop {
18597 if let Some(pid) = lock_snapshot(&module.inner.snapshot)
18598 .unwrap()
18599 .reported_pid()
18600 {
18601 return pid;
18602 }
18603 tokio::time::sleep(Duration::from_millis(10)).await;
18604 }
18605 })
18606 .await
18607 .expect("the first process was admitted as the module");
18608
18609 let key = vec![0x42; 32];
18612 let daemon_id = [0x24; 16];
18613 let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
18614 let port = listener.local_addr().unwrap().port();
18615 let server = tokio::spawn(crate::server::serve_listener(
18616 listener,
18617 Arc::new(crate::router::Router::with_default_self_handler()),
18618 crate::server::ServerAuth::with_limits(
18619 key.clone(),
18620 daemon_id,
18621 "test",
18622 crate::server::DEFAULT_AUTH_DEADLINE,
18623 4,
18624 ),
18625 ));
18626
18627 let (held_tx, held_rx) = std::sync::mpsc::channel();
18630 let armed = AtomicBool::new(true);
18631 supervisor
18632 .child_roster
18633 .set_record_write_hook_for_test(move |live| {
18634 let admitting_replacement = live.values().any(|entry| entry.pid != old_pid);
18635 if admitting_replacement && armed.swap(false, Ordering::SeqCst) {
18636 let _ = held_tx.send(());
18637 std::thread::sleep(HELD_RECORD_WRITE);
18638 }
18639 });
18640
18641 let module_process = std::thread::spawn(move || {
18644 held_rx
18645 .recv_timeout(Duration::from_secs(30))
18646 .expect("the restart reached the replacement's record write");
18647 let conn = ConnectionInfo {
18648 schema: SCHEMA_VERSION,
18649 wire_version: None,
18650 endpoints: vec![Endpoint {
18651 host: "127.0.0.1".to_owned(),
18652 port,
18653 }],
18654 key,
18655 daemon_id,
18656 pid: std::process::id(),
18657 daemon_ver: "test".to_owned(),
18658 };
18659 tokio::runtime::Builder::new_current_thread()
18660 .enable_all()
18661 .build()
18662 .unwrap()
18663 .block_on(async {
18664 let started = Instant::now();
18665 let mut stream = tokio::net::TcpStream::connect(("127.0.0.1", port))
18666 .await
18667 .unwrap();
18668 let result =
18669 authenticate_client(&mut stream, &conn, MODULE_AUTH_DEADLINE).await;
18670 (result, started.elapsed())
18671 })
18672 });
18673
18674 module.restart(Some(0)).await.unwrap();
18675 let (auth, elapsed) = tokio::task::spawn_blocking(move || module_process.join().unwrap())
18676 .await
18677 .unwrap();
18678 assert!(
18679 auth.is_ok(),
18680 "the daemon must answer a module's handshake while a restart waits on the \
18681 live-children record write; got {auth:?} after {elapsed:?}"
18682 );
18683
18684 tokio::time::timeout(Duration::from_secs(20), async {
18685 while lock_snapshot(&module.inner.snapshot)
18686 .unwrap()
18687 .reported_pid()
18688 .is_none_or(|pid| pid == old_pid)
18689 {
18690 tokio::time::sleep(Duration::from_millis(10)).await;
18691 }
18692 })
18693 .await
18694 .expect("the restart published its replacement");
18695 module.stop().await.unwrap();
18696 server.abort();
18697 }
18698}
18699
18700#[cfg(all(test, unix))]
18701mod retained_status_tests {
18702 use super::*;
18703
18704 #[tokio::test]
18705 async fn cached_container_block_is_visible_in_status_and_health_without_rechecking() {
18706 use crate::live_children::{ContainerIdentity, RetainedReason};
18707 for (reason, carried, expected) in [
18708 (
18709 RetainedReason::ContainerSurvivors(vec![4242]),
18710 true,
18711 Some("waiting: container survivors (pids [4242])".to_owned()),
18712 ),
18713 (
18714 RetainedReason::ContainerUnverifiable("member inspection denied".into()),
18715 true,
18716 Some(
18717 "waiting: container survivors could not be verified: member inspection denied"
18718 .to_owned(),
18719 ),
18720 ),
18721 (RetainedReason::Survived, false, None),
18722 ] {
18723 let supervisor =
18724 Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default());
18725 let spec = ModuleSpec {
18726 module_id: "retained-status-fixture".into(),
18727 program: PathBuf::from("/not-spawned"),
18728 args: Vec::new(),
18729 env: Vec::new(),
18730 reserved: false,
18731 reserved_prefixes: Vec::new(),
18732 protocol: ModuleProtocol::None,
18733 overlap: ModuleOverlap::Exclusive,
18734 carry_containment: carried,
18735 };
18736 let module = supervisor
18737 .supervise_configured(spec.clone(), false)
18738 .unwrap();
18739 let guard = supervisor
18743 .child_roster
18744 .admit(
18745 spec.module_id,
18746 0,
18747 ModuleProtocol::None,
18748 None,
18749 crate::child_roster::RecordedIdentity {
18750 container: carried.then_some(ContainerIdentity {
18751 pgid: Some(0),
18752 created_start_time: Some(0),
18753 cgroup_path: None,
18754 ..ContainerIdentity::test_metadata()
18755 }),
18756 ..crate::child_roster::RecordedIdentity::default()
18757 },
18758 )
18759 .unwrap();
18760 guard.retain(reason);
18761 let status = module.status().unwrap();
18762 assert_eq!(status.waiting_for_previous_detail, expected.clone());
18763 assert_eq!(status.health.detail, expected);
18764 assert!(!status.process_alive);
18765 }
18766 }
18767}