1mod resource_launch;
2pub(crate) use resource_launch::{ResourceCommitError, ResourceLaunch};
3
4use std::{
5 collections::{HashMap, HashSet, VecDeque},
6 error::Error,
7 fmt, io,
8 path::PathBuf,
9 process::{ExitStatus, Stdio},
10 sync::{Arc, Mutex, OnceLock},
11 time::{Duration, SystemTime, UNIX_EPOCH},
12};
13
14use cortexkit_log::Retention;
15use serde_json::Value;
16use sha2::{Digest, Sha256};
17use subc_control::{
18 ClientControlPush, LiveSpawn, ModuleProtocol, RouteCloseReason, SpawnCursor, SpawnEvent,
19 SpawnEventKind, SpawnSnapshot, SupervisorHealthStatus, TerminalDisposition, TerminalExitKind,
20};
21use subc_protocol::{
22 manifest::{SelfSignalKind, SignalAnchor},
23 session::{
24 HealthReport, HealthStatus, ModuleControlCommand, ModuleControlRequest,
25 MODULE_CONTROL_OP_HEALTH_CHECK,
26 },
27 Flags, FrameType, Principal, Priority, SUBC_LAUNCH_NONCE_ENV, SUBC_MODULE_ID_ENV,
28};
29use tokio::{
30 process::{Child, Command},
31 sync::{mpsc, oneshot, watch, Mutex as AsyncMutex},
32 task::JoinHandle,
33 time::{sleep, sleep_until, timeout, timeout_at, Instant},
34};
35use tracing::{debug, error, info, warn};
36
37use crate::{
38 child_roster::ChildRoster,
39 daemon_config::{
40 CAPTURE_KEEP_ENV, CAPTURE_MAX_AGE_DAYS_ENV, CAPTURE_MAX_FILE_MB_ENV, CK_LOG_ENV,
41 },
42 forwarding::{
43 CloseReason, ForwardingError, ForwardingTable, GoodbyeTarget, ModuleControlRpcOutcome,
44 ModuleDrainTarget, PendingModuleControlRpc,
45 },
46 provenance::{ExecutableIdentityProbe, SpawnedFileIdentity},
47 registry::{ConnectionId, RegistrationEndReason, RegistryError},
48 stderr_tail::{
49 pump_stderr_to, pump_stdout_to, ChildOutputSink, StderrRing, StderrTailConfig,
50 StderrTailSnapshot,
51 },
52 terminal_ring::{TerminalHistorySnapshot, TerminalRecord, TerminalRing, TerminalRingConfig},
53 Frame, FrameSink, Registry,
54};
55
56#[cfg(not(windows))]
57use crate::provenance::spawned_file_identity;
58
59#[path = "supervise_swap.rs"]
60mod swap;
61
62#[path = "supervise_activity.rs"]
63mod activity;
64
65#[cfg(unix)]
66#[path = "supervise/admission.rs"]
67mod admission;
68#[cfg(all(test, windows))]
69mod carried_job_tests;
70pub(crate) mod containment;
71#[cfg(all(test, unix))]
72mod containment_tests;
73mod slow_retry;
74use slow_retry::{FailedRetry, FailureContext};
75mod host_health;
76use host_health::HostHealth;
77
78#[cfg(all(test, unix))]
79mod slow_retry_tests;
80
81pub const SUBC_ARG: &str = "--subc";
87
88const DEFAULT_MAX_RESTARTS: u32 = 3;
89const DEFAULT_BACKOFF: Duration = Duration::from_millis(100);
90const DEFAULT_MAX_BACKOFF: Duration = Duration::from_secs(30);
91const DEFAULT_RESTART_WINDOW: Duration = Duration::from_secs(600);
95pub const DEFAULT_DRAIN_TIMEOUT: Duration = Duration::from_secs(30);
106pub(crate) const DEFAULT_REGISTER_TIMEOUT: Duration = Duration::from_secs(120);
107const REGISTRY_RELEASE_TIMEOUT: Duration = Duration::from_secs(1);
108const REGISTRY_RELEASE_POLL: Duration = Duration::from_millis(10);
109const DRAIN_BUSY_REPROBE_INTERVAL: Duration = Duration::from_millis(250);
121const STDERR_PUMP_DRAIN_TIMEOUT: Duration = Duration::from_millis(250);
144pub const SPAWN_EVENT_RING_CAPACITY: usize = 4096;
146const SPAWN_SUBSCRIBER_BUFFER: usize = SPAWN_EVENT_RING_CAPACITY + 1;
147pub(crate) const SPAWN_SUBSCRIBER_LAGGED_CODE: &str = "spawn_subscriber_lagged";
152
153#[cfg(target_os = "macos")]
154struct PrivacyExec {
155 reader: tokio::io::unix::AsyncFd<std::io::PipeReader>,
156 deadline: tokio::time::Instant,
157 expected: Option<subc_os::FileIdentity>,
158 trampoline: Option<subc_os::FileIdentity>,
159 script: bool,
160 module_id: String,
161 #[cfg(test)]
162 confirmation_pause: Option<(String, Option<tokio::net::TcpStream>)>,
163}
164
165#[cfg(all(test, target_os = "macos"))]
166pub(crate) fn test_privacy_trampoline() -> PathBuf {
167 let path = std::env::current_exe()
170 .unwrap()
171 .parent()
172 .unwrap()
173 .parent()
174 .unwrap()
175 .join("privacy-trampoline-fixture");
176 assert!(
180 path.exists(),
181 "privacy-trampoline-fixture not built at {}: run `cargo build -p subc-daemon \
182 --bins --features test-support` or `cargo test -p subc-daemon` first",
183 path.display()
184 );
185 path
186}
187
188#[cfg(target_os = "macos")]
189fn probe_privacy_trampoline(path: &std::path::Path) -> Result<(), String> {
190 use std::io::Read;
191 let mut probe = std::process::Command::new(path)
192 .args(["__disclaim-exec", "--probe"])
193 .stdin(Stdio::null())
194 .stdout(Stdio::piped())
195 .stderr(Stdio::piped())
196 .spawn()
197 .map_err(|error| {
198 format!(
199 "privacy trampoline probe failed for {}: {error}",
200 path.display()
201 )
202 })?;
203 let deadline = std::time::Instant::now() + Duration::from_secs(5);
204 let status = loop {
205 match probe.try_wait() {
206 Ok(Some(status)) => break status,
207 Ok(None) if std::time::Instant::now() < deadline => {
208 std::thread::sleep(Duration::from_millis(5))
209 }
210 result => {
211 let _ = probe.kill();
212 let _ = probe.wait();
213 return Err(format!(
214 "privacy trampoline probe failed or timed out for {}: {result:?}",
215 path.display()
216 ));
217 }
218 }
219 };
220 let mut answer = String::new();
221 if let Some(stdout) = probe.stdout.take() {
222 let _ = stdout.take(256).read_to_string(&mut answer);
223 }
224 if status.success() && answer.trim() == subc_os::privacy_identity::TRAMPOLINE_PROBE {
225 return Ok(());
226 }
227 let mut diagnostic = String::new();
228 if let Some(stderr) = probe.stderr.take() {
229 let _ = stderr.take(1024).read_to_string(&mut diagnostic);
230 }
231 let cause = diagnostic
232 .trim()
233 .strip_prefix("ck-subc: own privacy identity refused: ")
234 .unwrap_or("binary does not implement the privacy trampoline protocol");
235 Err(format!(
236 "{cause}: probe of {} exited {status}",
237 path.display()
238 ))
239}
240
241#[cfg(target_os = "macos")]
242fn privacy_command(
243 spec: &ModuleSpec,
244 roster: &ChildRoster,
245) -> Result<
246 (
247 Command,
248 Option<PrivacyExec>,
249 subc_os::privacy_identity::ExecAcknowledgement,
250 ),
251 SuperviseError,
252> {
253 let failure = |cause: String| {
254 warn!(module_id = %spec.module_id, %cause, "privacy identity trampoline refused module spawn");
255 SuperviseError::Spawn {
256 program: spec.program.clone(),
257 source: io::Error::other(cause),
258 cgroup_path: None,
259 }
260 };
261 let trampoline = roster.privacy_trampoline().map_err(failure)?;
262 let program = if spec.program.components().count() == 1 && !spec.program.is_absolute() {
267 let path = spec
268 .env
269 .iter()
270 .find(|(key, _)| key == "PATH")
271 .map(|(_, value)| std::ffi::OsString::from(value))
272 .or_else(|| std::env::var_os("PATH"))
273 .unwrap_or_else(|| "/usr/bin:/bin".into());
274 std::env::split_paths(&path)
275 .map(|dir| dir.join(&spec.program))
276 .find(|path| path.is_file())
277 .unwrap_or_else(|| spec.program.clone())
278 } else {
279 spec.program.clone()
280 };
281 let expected = subc_os::file_identity(&program);
282 let trampoline_image = subc_os::file_identity(&trampoline);
283 let script = {
284 use std::io::Read;
285 let mut prefix = [0u8; 2];
286 std::fs::File::open(&program)
287 .is_ok_and(|mut file| file.read_exact(&mut prefix).is_ok() && &prefix == b"#!")
288 };
289 if expected.is_none() || expected == subc_os::file_identity(&trampoline) {
290 return Err(failure(
291 "privacy identity module executable is missing or is the trampoline itself".to_string(),
292 ));
293 }
294 let (reader, ack) = subc_os::privacy_identity::ExecAcknowledgement::pipe()
295 .map_err(|error| failure(error.to_string()))?;
296 let reader =
297 tokio::io::unix::AsyncFd::new(reader).map_err(|error| failure(error.to_string()))?;
298 let mut command = Command::new(&trampoline);
299 command
300 .arg("__disclaim-exec")
301 .arg(ack.fd().to_string())
302 .arg(&program);
303 Ok((
304 command,
305 Some(PrivacyExec {
306 reader,
307 deadline: tokio::time::Instant::now() + Duration::from_secs(5),
308 expected,
309 trampoline: trampoline_image,
310 script,
311 module_id: spec.module_id.clone(),
312 #[cfg(test)]
313 confirmation_pause: spec
314 .env
315 .iter()
316 .find(|(key, _)| key == "SUBC_TEST_BEFORE_EXEC_CONFIRM")
317 .map(|(_, address)| (address.clone(), None)),
318 }),
319 ack,
320 ))
321}
322
323struct SupervisedChild {
324 subtree_target: crate::subtree_resources::Target,
325 child: containment::ProcessChild,
326 exec: crate::activity::ExecOwner,
327 #[cfg(windows)]
328 nonce_delivery: Option<subc_os::LaunchNoncePipeDelivery>,
329 #[cfg(target_os = "macos")]
330 privacy_exec: Option<PrivacyExec>,
331 #[cfg(target_os = "macos")]
338 report_ready: Arc<OnceLock<()>>,
339 spawn_failure: Option<String>,
342 protocol: ModuleProtocol,
347 #[cfg(target_os = "linux")]
354 module_id: String,
355 #[cfg(target_os = "linux")]
356 cgroup_placement: Option<subc_cgroup::Placement>,
357 #[cfg(windows)]
380 job: Option<Arc<Mutex<subc_jobobject::JobObject>>>,
381 stdout_pump: Option<JoinHandle<()>>,
382 stderr_pump: Option<StderrPump>,
383 stderr_ring: Arc<Mutex<StderrRing>>,
384 spawned_at: Instant,
387 registration_deadline: Option<Instant>,
390 registration_watch: Option<RegistrationWatch>,
391 spawned_at_ms: u64,
392 spawned_from: PathBuf,
393 launch_spec_digest: LaunchSpecDigest,
394 spawned_file_identity: Option<SpawnedFileIdentity>,
395 process_start_time: Option<u64>,
396 process_identity: Option<ProcessIdentity>,
397 pid: u32,
398 roster_guard: Option<crate::child_roster::RosterGuard>,
401}
402
403impl SupervisedChild {
404 fn registered_since_spawn(&self) -> bool {
405 self.registration_watch
406 .as_ref()
407 .is_some_and(|watch| *watch.changes.borrow() != watch.baseline)
408 }
409
410 fn id(&self) -> Option<u32> {
411 Some(self.pid)
412 }
413
414 fn process_identity(&self) -> Option<ProcessIdentity> {
415 self.process_identity
416 }
417
418 async fn wait(&mut self) -> io::Result<ExitStatus> {
419 #[cfg(target_os = "macos")]
420 self.confirm_privacy_exec().await;
421 let result = self.child.wait().await;
429 #[cfg(target_os = "linux")]
430 if result.is_ok() && self.child.container().is_none() {
431 if let Some(placement) = self.cgroup_placement.as_ref() {
432 cleanup_reaped_cgroup(placement, &self.module_id).await;
436 self.cgroup_placement = None;
437 }
438 }
439 result
440 }
441
442 async fn finish_exit(&mut self, carry: bool) -> io::Result<()> {
443 if self.child.container().is_some() {
444 if !carry {
445 self.child.terminate_container().await;
446 }
447 } else {
448 #[cfg(target_os = "linux")]
449 if let Some(placement) = self.cgroup_placement.as_ref() {
450 cleanup_reaped_cgroup(placement, &self.module_id).await;
451 self.cgroup_placement = None;
452 }
453 }
454 self.child.reap().await?;
455 self.release_roster();
456 Ok(())
457 }
458
459 #[cfg(target_os = "macos")]
460 async fn confirm_privacy_exec(&mut self) {
461 let Some(pending) = &mut self.privacy_exec else {
462 return;
463 };
464 let result = tokio::time::timeout_at(pending.deadline, async {
465 let mut record = Vec::new();
466 loop {
467 let mut ready = pending.reader.readable().await?;
468 let read = ready.try_io(|reader| {
469 use std::io::Read;
470 let mut reader = reader.get_ref();
471 let mut buffer = [0u8; 256];
472 reader.read(&mut buffer).map(|count| (count, buffer))
473 });
474 match read {
475 Ok(Ok((0, _))) => return Ok::<_, io::Error>(record),
476 Ok(Ok((count, buffer))) => {
477 if record.len() + count > 1024 {
478 return Err(io::Error::other(
479 "privacy exec refusal record is too long",
480 ));
481 }
482 record.extend_from_slice(&buffer[..count]);
483 }
484 Ok(Err(error)) => return Err(error),
485 Err(_) => continue,
486 }
487 }
488 })
489 .await;
490 #[cfg(test)]
491 let result = if let Some((address, stream)) = &mut pending.confirmation_pause {
492 use tokio::io::{AsyncReadExt, AsyncWriteExt};
493 if stream.is_none() {
494 *stream = Some(
495 tokio::net::TcpStream::connect(address.as_str())
496 .await
497 .unwrap(),
498 );
499 stream.as_mut().unwrap().write_all(b"R").await.unwrap();
500 stream
501 .as_mut()
502 .unwrap()
503 .write_all(&self.pid.to_ne_bytes())
504 .await
505 .unwrap();
506 }
507 let mut reply = [0];
508 stream
509 .as_mut()
510 .unwrap()
511 .read_exact(&mut reply)
512 .await
513 .unwrap();
514 pending.confirmation_pause = None;
515 if &reply == b"F" {
516 Ok(Err(io::Error::other("injected late exec refusal")))
517 } else {
518 assert_eq!(&reply, b"X");
519 result
520 }
521 } else {
522 result
523 };
524 let pending = self.privacy_exec.as_ref().expect("pending exec");
527 let cause = match result {
528 Err(_) => Some("privacy identity trampoline did not exec within 5s".to_string()),
529 Ok(Err(error)) => Some(format!(
530 "privacy identity exec acknowledgement failed: {error}"
531 )),
532 Ok(Ok(record)) if !record.is_empty() => Some(
533 std::str::from_utf8(&record)
534 .ok()
535 .and_then(|record| {
536 record
537 .trim()
538 .strip_prefix(subc_os::privacy_identity::EXEC_REFUSAL_TAG)
539 })
540 .filter(|cause| !cause.is_empty())
541 .unwrap_or("invalid privacy exec refusal record")
542 .to_string(),
543 ),
544 Ok(Ok(_)) => match self.child.try_wait() {
545 Ok(Some(_status)) => None,
549 Err(error) => Some(format!("privacy identity trampoline wait failed: {error}")),
550 Ok(None) => {
551 let image = observe_spawned_image(self.pid);
552 if let Some(image) = image.filter(|image| {
553 image.executable.is_some()
554 && image.executable != pending.trampoline
555 && (image.executable == pending.expected || pending.script)
556 }) {
557 if let Err(error) = self
558 .roster_guard
559 .as_ref()
560 .map(|guard| guard.confirm_executable(image))
561 .transpose()
562 {
563 Some(format!(
564 "could not durably record confirmed module image: {error}"
565 ))
566 } else {
567 let _ = self.report_ready.set(());
568 info!(module_id = %pending.module_id, pid = self.pid,
569 "module spawned with own privacy identity (responsibility disclaimed)");
570 None
571 }
572 } else if image.is_none()
573 || image.is_some_and(|image| image.executable.is_none())
574 {
575 match tokio::time::timeout_at(pending.deadline, self.child.wait()).await {
582 Ok(Ok(_status)) => None,
583 Ok(Err(error)) => Some(format!("privacy identity trampoline wait failed: {error}")),
584 Err(_) => Some("privacy identity module executable remained unreadable until the 5s exec deadline".to_string()),
585 }
586 } else {
587 Some("privacy identity trampoline executable mismatch: expected the module program, not ck-subc".to_string())
588 }
589 }
590 },
591 };
592 let pending = self.privacy_exec.take().expect("pending exec");
593 self.exec.0.finish(if cause.is_some() {
596 crate::activity::ExecState::Refused
597 } else {
598 crate::activity::ExecState::CommittedExec
599 });
600 if let Some(cause) = cause {
601 warn!(module_id = %pending.module_id, pid = self.pid, %cause, "privacy identity trampoline refused module spawn");
602 self.spawn_failure = Some(cause);
603 if let Some(container) = self.child.container() {
606 container.hard_kill();
607 } else if let Some(pid) = rustix::process::Pid::from_raw(self.pid as i32) {
608 let _ = rustix::process::kill_process_group(pid, rustix::process::Signal::KILL);
609 }
610 let _ = self.child.start_kill();
611 }
612 }
613
614 fn release_roster(&mut self) {
618 self.roster_guard = None;
619 }
620
621 fn start_kill(&mut self) -> io::Result<()> {
634 if let Some(container) = self.child.container() {
635 container.hard_kill();
636 return self.kill_leader();
637 }
638 #[cfg(windows)]
639 if let Some(job) = &self.job {
640 if let Err(error) = job.lock().unwrap_or_else(|p| p.into_inner()).terminate() {
641 debug!(
642 error = %error,
643 "job termination failed; the direct-child kill still owns the outcome"
644 );
645 }
646 }
647 #[cfg(target_os = "linux")]
648 kill_module_cgroup(self.cgroup_placement.as_ref(), &self.module_id);
649 self.child.start_kill()
650 }
651
652 fn kill_leader(&mut self) -> io::Result<()> {
653 self.child.start_kill()
654 }
655
656 async fn drain_stderr(&mut self, module_id: &str) {
657 if let Some(pump) = self.stdout_pump.take() {
658 settle_stdout_pump(module_id, pump, STDERR_PUMP_DRAIN_TIMEOUT).await;
659 }
660
661 let Some(pump) = self.stderr_pump.take() else {
662 return;
663 };
664 settle_stderr_pump(
665 module_id,
666 &self.stderr_ring,
667 pump,
668 STDERR_PUMP_DRAIN_TIMEOUT,
669 )
670 .await;
671 }
672}
673
674async fn settle_stdout_pump(module_id: &str, mut pump: JoinHandle<()>, bound: Duration) {
678 match timeout(bound, &mut pump).await {
679 Ok(Ok(())) => {}
680 Ok(Err(error)) => {
681 warn!(module_id, error = %error, "stdout pump ended unexpectedly");
682 }
683 Err(_) => {
684 drop(pump);
685 warn!(
686 module_id,
687 waited = ?bound,
688 "stdout pipe of the exited process has not drained; its reader keeps running without delaying the restart"
689 );
690 }
691 }
692}
693
694struct StderrPump {
697 task: JoinHandle<()>,
698 generation: u64,
699}
700
701async fn settle_stderr_pump(
707 module_id: &str,
708 ring: &Arc<Mutex<StderrRing>>,
709 pump: StderrPump,
710 bound: Duration,
711) {
712 let lock = || ring.lock().unwrap_or_else(|poisoned| poisoned.into_inner());
713 let StderrPump {
714 mut task,
715 generation,
716 } = pump;
717 lock().retire_pump(generation);
718 match timeout(bound, &mut task).await {
719 Ok(Ok(())) => {}
720 Ok(Err(err)) => {
721 let mut ring = lock();
722 ring.mark_incomplete_from(generation, format!("stderr pump ended unexpectedly: {err}"));
723 ring.finish_pump(generation);
724 warn!(module_id, error = %err, "stderr pump ended before clean EOF");
725 }
726 Err(_) => {
727 drop(task);
729 lock().mark_pump_late(
730 generation,
731 format!(
732 "stderr of the exited process had not reached EOF {bound:?} after it was \
733 retired (a descendant may still hold the pipe open); lines it still \
734 writes are kept in that process's section"
735 ),
736 );
737 warn!(
738 module_id,
739 waited = ?bound,
740 "stderr pipe of the exited process is still open; its reader keeps running without delaying the restart"
741 );
742 }
743 }
744}
745
746fn registration_release_events() -> &'static watch::Sender<u64> {
747 static EVENTS: OnceLock<watch::Sender<u64>> = OnceLock::new();
748 EVENTS.get_or_init(|| {
749 let (sender, _receiver) = watch::channel(0);
750 sender
751 })
752}
753
754pub(crate) fn notify_registration_release() {
755 let events = registration_release_events();
756 let next_generation = (*events.borrow()).wrapping_add(1);
757 events.send_replace(next_generation);
758}
759
760#[derive(Debug, Clone, PartialEq, Eq)]
762pub struct ModuleSpec {
763 pub module_id: String,
764 pub program: PathBuf,
765 pub args: Vec<String>,
766 pub env: Vec<(String, String)>,
767 pub reserved: bool,
772 pub reserved_prefixes: Vec<String>,
777 pub protocol: ModuleProtocol,
796 pub overlap: ModuleOverlap,
801 pub carry_containment: bool,
805}
806
807#[derive(Clone, PartialEq, Eq)]
810pub(crate) struct LaunchSpecDigest([u8; 32]);
811
812impl fmt::Debug for LaunchSpecDigest {
813 fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
814 f.write_str("LaunchSpecDigest([redacted])")
815 }
816}
817
818impl LaunchSpecDigest {
819 pub(crate) fn from_spec(spec: &ModuleSpec) -> Self {
820 fn field(hash: &mut Sha256, bytes: &[u8]) {
821 hash.update((bytes.len() as u64).to_le_bytes());
822 hash.update(bytes);
823 }
824 let mut hash = Sha256::new();
825 if let Some(program) = spec.program.to_str() {
826 field(&mut hash, b"utf8-program");
827 field(&mut hash, program.as_bytes());
828 } else {
829 #[cfg(unix)]
832 {
833 use std::os::unix::ffi::OsStrExt;
834 field(&mut hash, b"unix-program");
835 field(&mut hash, spec.program.as_os_str().as_bytes());
836 }
837 #[cfg(windows)]
838 {
839 use std::os::windows::ffi::OsStrExt;
840 field(&mut hash, b"windows-program");
841 let bytes = spec
842 .program
843 .as_os_str()
844 .encode_wide()
845 .flat_map(u16::to_le_bytes)
846 .collect::<Vec<_>>();
847 field(&mut hash, &bytes);
848 }
849 #[cfg(not(any(unix, windows)))]
850 field(&mut hash, spec.program.to_string_lossy().as_bytes());
851 }
852 hash.update((spec.args.len() as u64).to_le_bytes());
853 for arg in &spec.args {
854 field(&mut hash, arg.as_bytes());
855 }
856 let mut env = spec.env.iter().collect::<Vec<_>>();
857 env.sort_unstable();
858 hash.update((env.len() as u64).to_le_bytes());
859 for (key, value) in env {
860 field(&mut hash, key.as_bytes());
861 field(&mut hash, value.as_bytes());
862 }
863 field(
864 &mut hash,
865 match spec.protocol {
866 ModuleProtocol::Subc => b"subc",
867 ModuleProtocol::None => b"none",
868 },
869 );
870 Self(hash.finalize().into())
871 }
872}
873
874#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)]
881pub enum ModuleOverlap {
882 #[default]
884 Exclusive,
885 Safe,
897}
898
899impl ModuleOverlap {
900 pub fn as_str(self) -> &'static str {
901 match self {
902 Self::Exclusive => "exclusive",
903 Self::Safe => "safe",
904 }
905 }
906}
907
908pub const SUBC_SPAWN_ROLE_ENV: &str = "SUBC_SPAWN_ROLE";
918pub const SPAWN_ROLE_SWAP_CANDIDATE: &str = "swap_candidate";
920pub const DEFAULT_SWAP_READY_TIMEOUT: Duration = Duration::from_secs(100);
925
926#[derive(Debug, Clone, Copy, PartialEq, Eq)]
944pub struct RestartPolicy {
945 pub max_restarts: u32,
946 pub backoff: Duration,
949 pub max_backoff: Duration,
951 pub window: Duration,
955}
956
957impl RestartPolicy {
958 pub fn new(max_restarts: u32, backoff: Duration) -> Self {
962 Self {
963 max_restarts,
964 backoff,
965 max_backoff: DEFAULT_MAX_BACKOFF,
966 window: DEFAULT_RESTART_WINDOW,
967 }
968 }
969
970 pub fn with_max_backoff(mut self, max_backoff: Duration) -> Self {
971 self.max_backoff = max_backoff;
972 self
973 }
974
975 pub fn with_window(mut self, window: Duration) -> Self {
976 self.window = window;
977 self
978 }
979
980 fn delay_for_restart(&self, restart_in_window: u32) -> Duration {
985 if self.backoff.is_zero() || self.max_backoff.is_zero() {
986 return Duration::ZERO;
987 }
988
989 let mut delay = self.backoff;
990 for _ in 0..restart_in_window {
991 if delay >= self.max_backoff {
992 return self.max_backoff;
993 }
994 delay = delay
995 .checked_mul(10)
996 .unwrap_or(self.max_backoff)
997 .min(self.max_backoff);
998 }
999 delay.min(self.max_backoff)
1000 }
1001
1002 fn budget_exhausted_detail(&self) -> String {
1007 format!(
1008 "crash budget exhausted: max_restarts={} within window_secs={}",
1009 self.max_restarts,
1010 self.window.as_secs()
1011 )
1012 }
1013}
1014
1015impl Default for RestartPolicy {
1016 fn default() -> Self {
1017 Self {
1018 max_restarts: DEFAULT_MAX_RESTARTS,
1019 backoff: DEFAULT_BACKOFF,
1020 max_backoff: DEFAULT_MAX_BACKOFF,
1021 window: DEFAULT_RESTART_WINDOW,
1022 }
1023 }
1024}
1025
1026#[derive(Debug, Clone, Copy, PartialEq, Eq)]
1027struct CrashRestartSchedule {
1028 restart_in_window: u32,
1029 delay: Duration,
1030}
1031
1032fn daemon_will_restart(
1039 state: &mut SupervisorSnapshot,
1040 policy: &RestartPolicy,
1041 now: Instant,
1042) -> bool {
1043 state.enabled && state.crash_restarts_in_window(policy.window, now) < policy.max_restarts
1044}
1045
1046const DEFAULT_HEALTH_CADENCE: Duration = Duration::from_secs(30);
1047const DEFAULT_HEALTH_DEADLINE: Duration = Duration::from_secs(5);
1048const DEFAULT_HEALTH_FAILURE_THRESHOLD: u32 = 3;
1049const MAX_HEALTH_METRICS_BYTES: usize = 16 * 1024;
1050
1051#[derive(Debug, Clone, Copy, PartialEq, Eq)]
1052pub enum HealthAction {
1053 Report,
1054 Restart,
1055 Alert,
1056}
1057
1058impl fmt::Display for HealthAction {
1059 fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
1060 f.write_str(match self {
1061 Self::Report => "report",
1062 Self::Restart => "restart",
1063 Self::Alert => "alert",
1064 })
1065 }
1066}
1067
1068#[derive(Debug, Clone, PartialEq, Eq)]
1069pub struct HealthConfig {
1070 pub http: Option<String>,
1074 pub cadence: Duration,
1075 pub deadline: Duration,
1076 pub failure_threshold: u32,
1077 pub on_degraded: HealthAction,
1078 pub on_failing: HealthAction,
1079 pub critical: bool,
1080}
1081
1082impl Default for HealthConfig {
1083 fn default() -> Self {
1084 Self {
1085 http: None,
1086 cadence: DEFAULT_HEALTH_CADENCE,
1087 deadline: DEFAULT_HEALTH_DEADLINE,
1088 failure_threshold: DEFAULT_HEALTH_FAILURE_THRESHOLD,
1089 on_degraded: HealthAction::Report,
1090 on_failing: HealthAction::Report,
1091 critical: false,
1092 }
1093 }
1094}
1095
1096#[derive(Debug, Clone, PartialEq)]
1114pub struct ModuleHealthStatus {
1115 pub status: SupervisorHealthStatus,
1116 pub last_probe_ms: Option<u64>,
1117 pub detail: Option<String>,
1118 pub metrics: Option<Value>,
1119 pub consecutive_failures: u32,
1120 pub late_answer_count: u64,
1123 pub last_late_answer_latency_ms: Option<u64>,
1125 pub last_action: Option<String>,
1126 pub last_action_ms: Option<u64>,
1130}
1131
1132impl Default for ModuleHealthStatus {
1133 fn default() -> Self {
1134 Self {
1135 status: SupervisorHealthStatus::Unknown,
1136 last_probe_ms: None,
1137 detail: None,
1138 metrics: None,
1139 consecutive_failures: 0,
1140 late_answer_count: 0,
1141 last_late_answer_latency_ms: None,
1142 last_action: None,
1143 last_action_ms: None,
1144 }
1145 }
1146}
1147
1148#[derive(Debug, Clone, Copy, PartialEq, Eq)]
1150pub enum ModuleState {
1151 Starting,
1152 Running,
1153 Unresponsive,
1154 Restarting,
1155 Draining,
1156 Stopped,
1157 Failed,
1158 Disabled,
1159}
1160
1161impl fmt::Display for ModuleState {
1162 fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
1163 f.write_str(match self {
1164 Self::Starting => "starting",
1165 Self::Running => "running",
1166 Self::Unresponsive => "unresponsive",
1167 Self::Restarting => "restarting",
1168 Self::Draining => "draining",
1169 Self::Stopped => "stopped",
1170 Self::Failed => "failed",
1171 Self::Disabled => "disabled",
1172 })
1173 }
1174}
1175
1176#[derive(Debug, Clone, Copy, PartialEq, Eq)]
1178pub enum ExitKind {
1179 Clean,
1180 Crash,
1181 DeliberateSeverance,
1182}
1183
1184impl From<ExitKind> for TerminalExitKind {
1185 fn from(kind: ExitKind) -> Self {
1186 match kind {
1187 ExitKind::Clean => Self::Clean,
1188 ExitKind::Crash => Self::Crash,
1189 ExitKind::DeliberateSeverance => Self::DeliberateSeverance,
1190 }
1191 }
1192}
1193
1194#[derive(Debug, Clone, Copy, PartialEq, Eq)]
1197pub(crate) struct ProcessIdentity {
1198 pub(crate) pid: u32,
1199 pub(crate) start_time: u64,
1200}
1201
1202#[derive(Debug, Clone, PartialEq, Eq)]
1204pub struct ExitReport {
1205 pub kind: ExitKind,
1206 pub code: Option<i32>,
1207 pub signal: Option<i32>,
1208 pub at_ms: u64,
1209}
1210
1211#[derive(Debug, Clone, PartialEq)]
1214pub struct ModuleStatus {
1215 pub module_id: String,
1216 pub waiting_for_previous_pid: Option<u32>,
1218 pub waiting_for_previous_detail: Option<String>,
1220 pub launch_nonce_source: Option<String>,
1225 pub state: ModuleState,
1226 pub enabled: bool,
1227 pub process_alive: bool,
1228 pub registration_active: bool,
1229 pub protocol: ModuleProtocol,
1234 pub live: bool,
1245 pub restart_count: u32,
1249 pub lifetime_restarts: u32,
1253 pub spawn_generation: u64,
1254 pub max_restarts: u32,
1259 pub restart_window: Duration,
1263 pub drain_timeout: Duration,
1267 pub restart_backoff: Duration,
1268 pub restart_max_backoff: Duration,
1269 pub pid: Option<u32>,
1274 pub spawned_at_ms: Option<u64>,
1275 pub spawned_from: Option<PathBuf>,
1276 pub(crate) launch_spec_digest: Option<LaunchSpecDigest>,
1277 pub process_start_time: Option<u64>,
1278 pub last_exit: Option<ExitReport>,
1279 pub health: ModuleHealthStatus,
1280}
1281
1282#[cfg(test)]
1283#[derive(Debug, Clone, Copy, PartialEq)]
1284struct ActorSelectCheckpoint {
1285 generation: u64,
1286 turn: u64,
1287 registered_connection: Option<ConnectionId>,
1288 next_probe_at: Option<Instant>,
1289 wake_after: Option<Duration>,
1290}
1291
1292#[derive(Debug, Clone, PartialEq)]
1293struct SupervisorSnapshot {
1294 subtree_target: crate::subtree_resources::Target,
1295 failed_retry: FailedRetry,
1296 exec_outcome: Option<Arc<crate::activity::ExecOutcome>>,
1297 #[cfg(windows)]
1298 pipe_consumed: Option<Arc<OnceLock<()>>>,
1299 #[cfg(test)]
1301 actor_turns: u64,
1302 #[cfg(test)]
1305 actor_select: Option<ActorSelectCheckpoint>,
1306 state: ModuleState,
1307 enabled: bool,
1308 process_alive: bool,
1309 spawned_protocol: Option<ModuleProtocol>,
1310 spawn_failure: Option<String>,
1311 waiting_for_previous_pid: Option<u32>,
1312 waiting_for_previous_detail: Option<String>,
1313 crash_restarts: VecDeque<Instant>,
1319 lifetime_restarts: u32,
1320 spawn_generation: u64,
1329 pid: Option<u32>,
1330 #[cfg(target_os = "macos")]
1331 report_ready: Option<Arc<OnceLock<()>>>,
1332 reaped_pid: Option<u32>,
1334 respawn_pending: bool,
1336 coalesced_restart_pending: bool,
1338 spawned_at_ms: Option<u64>,
1339 spawned_from: Option<PathBuf>,
1340 launch_spec_digest: Option<LaunchSpecDigest>,
1341 spawned_file_identity: Option<SpawnedFileIdentity>,
1342 process_start_time: Option<u64>,
1343 deliberate_severance: Option<ProcessIdentity>,
1344 last_exit: Option<ExitReport>,
1345 drain_disposition_detail: Option<String>,
1347 health: ModuleHealthStatus,
1348 last_probe_evidence: Option<HealthProbeEvidence>,
1349 in_alternate_slot: bool,
1354 draining_to_replace: bool,
1361 configuration_updated_since_spawn: bool,
1367}
1368
1369impl SupervisorSnapshot {
1370 fn reported_pid(&self) -> Option<u32> {
1375 #[cfg(target_os = "macos")]
1376 if self
1377 .report_ready
1378 .as_ref()
1379 .is_some_and(|ready| ready.get().is_none())
1380 {
1381 return None;
1382 }
1383 self.pid
1384 }
1385
1386 fn starting() -> Self {
1387 Self::new(ModuleState::Starting, true)
1388 }
1389
1390 fn disabled() -> Self {
1391 Self::new(ModuleState::Disabled, false)
1392 }
1393
1394 fn crash_restarts_in_window(&mut self, window: Duration, now: Instant) -> u32 {
1398 while let Some(oldest) = self.crash_restarts.front() {
1399 if now.duration_since(*oldest) > window {
1400 self.crash_restarts.pop_front();
1401 } else {
1402 break;
1403 }
1404 }
1405 u32::try_from(self.crash_restarts.len()).unwrap_or(u32::MAX)
1406 }
1407
1408 fn record_crash_restart(&mut self, policy: &RestartPolicy, now: Instant) {
1414 self.crash_restarts.push_back(now);
1415 while self.crash_restarts.len() > policy.max_restarts as usize {
1416 self.crash_restarts.pop_front();
1417 }
1418 self.lifetime_restarts += 1;
1419 }
1420
1421 fn next_crash_restart(
1425 &mut self,
1426 policy: &RestartPolicy,
1427 now: Instant,
1428 ) -> Option<CrashRestartSchedule> {
1429 let restart_in_window = self.crash_restarts_in_window(policy.window, now);
1430 if restart_in_window >= policy.max_restarts {
1431 return None;
1432 }
1433 self.record_crash_restart(policy, now);
1434 Some(CrashRestartSchedule {
1435 restart_in_window,
1436 delay: policy.delay_for_restart(restart_in_window),
1437 })
1438 }
1439
1440 fn clear_crash_restarts(&mut self) {
1445 self.crash_restarts.clear();
1446 }
1447
1448 fn new(state: ModuleState, enabled: bool) -> Self {
1449 Self {
1450 subtree_target: crate::subtree_resources::Target::Unavailable("not_running"),
1451 failed_retry: FailedRetry::default(),
1452 exec_outcome: None,
1453 #[cfg(test)]
1454 actor_turns: 0,
1455 #[cfg(test)]
1456 actor_select: None,
1457 state,
1458 enabled,
1459 process_alive: false,
1460 #[cfg(windows)]
1461 pipe_consumed: None,
1462 spawned_protocol: None,
1463 spawn_failure: None,
1464 waiting_for_previous_pid: None,
1465 waiting_for_previous_detail: None,
1466 crash_restarts: VecDeque::new(),
1467 lifetime_restarts: 0,
1468 spawn_generation: 0,
1469 pid: None,
1470 #[cfg(target_os = "macos")]
1471 report_ready: None,
1472 reaped_pid: None,
1473 respawn_pending: false,
1474 coalesced_restart_pending: false,
1475 spawned_at_ms: None,
1476 spawned_from: None,
1477 launch_spec_digest: None,
1478 spawned_file_identity: None,
1479 process_start_time: None,
1480 deliberate_severance: None,
1481 last_exit: None,
1482 drain_disposition_detail: None,
1483 health: ModuleHealthStatus::default(),
1484 last_probe_evidence: None,
1485 in_alternate_slot: false,
1486 draining_to_replace: false,
1487 configuration_updated_since_spawn: false,
1488 }
1489 }
1490}
1491
1492type SharedSnapshot = Arc<Mutex<SupervisorSnapshot>>;
1493
1494type SpawnSubscriberKey = (ConnectionId, u64);
1495
1496#[derive(Debug)]
1497struct SpawnSubscriber {
1498 version: u8,
1499 frames: mpsc::Sender<Frame>,
1500 lagged: Option<oneshot::Sender<SpawnCursor>>,
1504}
1505
1506#[derive(Debug)]
1507struct SpawnEventState {
1508 daemon_incarnation: String,
1509 seq: u64,
1510 capacity: usize,
1511 live: HashMap<String, LiveSpawn>,
1512 generations: HashMap<String, u64>,
1513 events: VecDeque<SpawnEvent>,
1514 subscribers: HashMap<SpawnSubscriberKey, SpawnSubscriber>,
1515}
1516
1517impl Default for SpawnEventState {
1518 fn default() -> Self {
1519 Self {
1520 daemon_incarnation: "unconfigured".to_string(),
1521 seq: 0,
1522 capacity: SPAWN_EVENT_RING_CAPACITY,
1523 live: HashMap::new(),
1524 generations: HashMap::new(),
1525 events: VecDeque::new(),
1526 subscribers: HashMap::new(),
1527 }
1528 }
1529}
1530
1531#[derive(Debug, Clone, Default)]
1532struct SpawnEventFeed(
1533 Arc<Mutex<SpawnEventState>>,
1534 #[cfg(test)] Arc<Mutex<Option<std::net::TcpStream>>>,
1535);
1536
1537#[derive(Debug, Clone, PartialEq, Eq)]
1538pub(crate) enum SpawnSubscribeRefusal {
1539 ForeignIncarnation { current: String },
1540 TooOld { oldest: SpawnCursor },
1541 Frame(String),
1542}
1543
1544impl SpawnEventFeed {
1545 fn configure_incarnation(&self, daemon_incarnation: String) {
1546 let mut state = self.0.lock().unwrap_or_else(|p| p.into_inner());
1547 state.daemon_incarnation = daemon_incarnation;
1548 state.seq = 0;
1549 state.live.clear();
1550 state.generations.clear();
1551 state.events.clear();
1552 state.subscribers.clear();
1553 }
1554
1555 fn cursor(state: &SpawnEventState) -> SpawnCursor {
1556 SpawnCursor {
1557 daemon_incarnation: state.daemon_incarnation.clone(),
1558 seq: state.seq,
1559 }
1560 }
1561
1562 fn snapshot(&self) -> SpawnSnapshot {
1563 let state = self.0.lock().unwrap_or_else(|p| p.into_inner());
1564 let mut live = state.live.values().cloned().collect::<Vec<_>>();
1565 live.sort_by(|left, right| left.module_id.cmp(&right.module_id));
1566 SpawnSnapshot {
1567 cursor: Self::cursor(&state),
1568 ring_bound: state.capacity as u64,
1569 live,
1570 }
1571 }
1572
1573 fn emit_spawned(&self, module_id: &str, pid: u32, spawned_at_ms: u64) -> u64 {
1574 let mut state = self.0.lock().unwrap_or_else(|p| p.into_inner());
1575 let generation = state
1576 .generations
1577 .get(module_id)
1578 .copied()
1579 .unwrap_or(0)
1580 .checked_add(1)
1581 .expect("spawn generation exhausted");
1582 state.generations.insert(module_id.to_string(), generation);
1583 let live = LiveSpawn {
1584 module_id: module_id.to_string(),
1585 spawn_generation: generation,
1586 pid,
1587 spawned_at_ms,
1588 };
1589 state.live.insert(module_id.to_string(), live);
1590 Self::emit_locked(
1591 &mut state,
1592 SpawnEventKind::Spawned,
1593 module_id.to_string(),
1594 generation,
1595 pid,
1596 None,
1597 None,
1598 );
1599 generation
1600 }
1601
1602 fn emit_exited(&self, module_id: &str, exit_code: Option<i32>, exit_signal: Option<i32>) {
1603 let mut state = self.0.lock().unwrap_or_else(|p| p.into_inner());
1604 let Some(live) = state.live.remove(module_id) else {
1605 warn!(
1606 module_id,
1607 "terminal record had no live spawn event identity"
1608 );
1609 return;
1610 };
1611 Self::emit_locked(
1612 &mut state,
1613 SpawnEventKind::Exited,
1614 module_id.to_string(),
1615 live.spawn_generation,
1616 live.pid,
1617 exit_code,
1618 exit_signal,
1619 );
1620 }
1621
1622 fn emit_superseded_exited(
1629 &self,
1630 module_id: &str,
1631 spawn_generation: u64,
1632 pid: u32,
1633 exit_code: Option<i32>,
1634 exit_signal: Option<i32>,
1635 ) {
1636 let mut state = self.0.lock().unwrap_or_else(|p| p.into_inner());
1637 if state
1638 .live
1639 .get(module_id)
1640 .is_some_and(|live| live.spawn_generation == spawn_generation)
1641 {
1642 state.live.remove(module_id);
1643 }
1644 Self::emit_locked(
1645 &mut state,
1646 SpawnEventKind::Exited,
1647 module_id.to_string(),
1648 spawn_generation,
1649 pid,
1650 exit_code,
1651 exit_signal,
1652 );
1653 }
1654
1655 #[allow(clippy::too_many_arguments)]
1656 fn emit_locked(
1657 state: &mut SpawnEventState,
1658 kind: SpawnEventKind,
1659 module_id: String,
1660 spawn_generation: u64,
1661 pid: u32,
1662 exit_code: Option<i32>,
1663 exit_signal: Option<i32>,
1664 ) {
1665 state.seq = state
1666 .seq
1667 .checked_add(1)
1668 .expect("spawn event sequence exhausted");
1669 let event = SpawnEvent {
1670 cursor: Self::cursor(state),
1671 kind,
1672 module_id,
1673 spawn_generation,
1674 pid,
1675 exit_code,
1676 exit_signal,
1677 };
1678 state.events.push_back(event.clone());
1679 while state.events.len() > state.capacity {
1680 state.events.pop_front();
1681 }
1682 let body = match serde_json::to_vec(&event) {
1683 Ok(body) => body,
1684 Err(error) => {
1685 error!(%error, "failed to serialize supervisor spawn event");
1686 return;
1687 }
1688 };
1689 state.subscribers.retain(|(connection_id, corr), subscriber| {
1690 let frame = Frame::build_with_version(
1691 subscriber.version,
1692 FrameType::StreamData,
1693 control_flags(),
1694 0,
1695 0,
1696 *corr,
1697 body.clone(),
1698 );
1699 match frame {
1700 Ok(frame) => {
1701 if subscriber.frames.try_send(frame).is_ok() {
1702 true
1703 } else {
1704 warn!(connection_id = connection_id.get(), corr, "dropping lagged supervisor spawn subscriber");
1705 if let Some(lagged) = subscriber.lagged.take() {
1706 let _ = lagged.send(event.cursor.clone());
1707 }
1708 false
1709 }
1710 }
1711 Err(error) => {
1712 warn!(connection_id = connection_id.get(), corr, %error, "dropping supervisor spawn subscriber after frame build failure");
1713 false
1714 }
1715 }
1716 });
1717 }
1718
1719 fn subscribe(
1720 &self,
1721 connection_id: ConnectionId,
1722 corr: u64,
1723 version: u8,
1724 since: Option<SpawnCursor>,
1725 sink: FrameSink,
1726 ) -> Result<(), SpawnSubscribeRefusal> {
1727 let (frames, mut receiver) = mpsc::channel(SPAWN_SUBSCRIBER_BUFFER);
1728 let (lagged, mut lagged_rx) = oneshot::channel::<SpawnCursor>();
1729 {
1730 let mut state = self.0.lock().unwrap_or_else(|p| p.into_inner());
1731 let replay = if let Some(since) = since {
1732 if since.daemon_incarnation != state.daemon_incarnation {
1733 return Err(SpawnSubscribeRefusal::ForeignIncarnation {
1734 current: state.daemon_incarnation.clone(),
1735 });
1736 }
1737 if let Some(oldest) = state.events.front().map(|event| event.cursor.clone()) {
1738 if since.seq < oldest.seq.saturating_sub(1) {
1739 return Err(SpawnSubscribeRefusal::TooOld { oldest });
1740 }
1741 }
1742 state
1743 .events
1744 .iter()
1745 .filter(|event| event.cursor.seq > since.seq)
1746 .cloned()
1747 .collect::<Vec<_>>()
1748 } else {
1749 Vec::new()
1750 };
1751 for event in replay {
1752 let body = serde_json::to_vec(&event)
1753 .map_err(|error| SpawnSubscribeRefusal::Frame(error.to_string()))?;
1754 let frame = Frame::build_with_version(
1755 version,
1756 FrameType::StreamData,
1757 control_flags(),
1758 0,
1759 0,
1760 corr,
1761 body,
1762 )
1763 .map_err(|error| SpawnSubscribeRefusal::Frame(error.to_string()))?;
1764 frames
1765 .try_send(frame)
1766 .map_err(|error| SpawnSubscribeRefusal::Frame(error.to_string()))?;
1767 }
1768 state.subscribers.insert(
1769 (connection_id, corr),
1770 SpawnSubscriber {
1771 version,
1772 frames,
1773 lagged: Some(lagged),
1774 },
1775 );
1776 }
1777 tokio::spawn(async move {
1788 while let Some(frame) = receiver.recv().await {
1789 if sink.send(frame).await.is_err() {
1790 return;
1791 }
1792 }
1793 let Ok(first_undelivered) = lagged_rx.try_recv() else {
1794 return;
1795 };
1796 match spawn_subscriber_lagged_frame(version, corr, first_undelivered) {
1797 Ok(frame) => {
1798 let _ = sink.send(frame).await;
1799 }
1800 Err(error) => {
1801 error!(%error, corr, "failed to build lagged spawn subscriber terminal frame");
1802 }
1803 }
1804 });
1805 Ok(())
1806 }
1807
1808 fn cancel(&self, connection_id: ConnectionId, corr: u64) -> bool {
1809 let Some(subscriber) = self
1810 .0
1811 .lock()
1812 .unwrap_or_else(|p| p.into_inner())
1813 .subscribers
1814 .remove(&(connection_id, corr))
1815 else {
1816 return false;
1817 };
1818 if let Ok(frame) = Frame::build_with_version(
1819 subscriber.version,
1820 FrameType::StreamEnd,
1821 control_flags(),
1822 0,
1823 0,
1824 corr,
1825 Vec::new(),
1826 ) {
1827 tokio::spawn(async move {
1828 let _ = subscriber.frames.send(frame).await;
1829 });
1830 }
1831 true
1832 }
1833
1834 fn remove_connection(&self, connection_id: ConnectionId) {
1835 self.0
1836 .lock()
1837 .unwrap_or_else(|p| p.into_inner())
1838 .subscribers
1839 .retain(|(subscriber_connection, _), _| *subscriber_connection != connection_id);
1840 }
1841
1842 #[cfg(any(test, feature = "test-support"))]
1843 fn set_capacity(&self, capacity: usize) {
1844 self.0.lock().unwrap_or_else(|p| p.into_inner()).capacity = capacity;
1845 }
1846
1847 #[cfg(any(test, feature = "test-support"))]
1848 fn subscriber_count(&self) -> usize {
1849 self.0
1850 .lock()
1851 .unwrap_or_else(|p| p.into_inner())
1852 .subscribers
1853 .len()
1854 }
1855}
1856
1857fn spawn_subscriber_lagged_frame(
1860 version: u8,
1861 corr: u64,
1862 first_undelivered: SpawnCursor,
1863) -> Result<Frame, String> {
1864 let body = serde_json::to_vec(&subc_protocol::ErrorBody {
1866 code: SPAWN_SUBSCRIBER_LAGGED_CODE.to_string(),
1867 message: "spawn subscriber fell behind and was dropped; resubscribe from the last cursor received"
1868 .to_string(),
1869 detail: Some(serde_json::json!({
1870 "first_undelivered_cursor": first_undelivered
1871 })),
1872 })
1873 .map_err(|error| error.to_string())?;
1874 Frame::build_with_version(
1875 version,
1876 FrameType::Error,
1877 crate::router::daemon_error_flags(),
1878 0,
1879 0,
1880 corr,
1881 body,
1882 )
1883 .map_err(|error| error.to_string())
1884}
1885
1886pub trait ModuleProcessLiveness: Send + Sync {
1887 fn process_live(&self, module_id: &str) -> Option<bool>;
1888
1889 fn process_replacing(&self, _module_id: &str) -> bool {
1895 false
1896 }
1897}
1898
1899#[derive(Debug, Clone, Default)]
1901pub struct SupervisorProcessLiveness {
1902 snapshots: Arc<Mutex<HashMap<String, SharedSnapshot>>>,
1903}
1904
1905impl SupervisorProcessLiveness {
1906 pub fn new() -> Self {
1907 Self::default()
1908 }
1909
1910 fn track(&self, module_id: String, snapshot: SharedSnapshot) {
1911 let mut snapshots = self
1912 .snapshots
1913 .lock()
1914 .unwrap_or_else(|poisoned| poisoned.into_inner());
1915 snapshots.insert(module_id, snapshot);
1916 }
1917
1918 fn untrack_if_current(&self, module_id: &str, snapshot: &SharedSnapshot) {
1919 let mut snapshots = self
1920 .snapshots
1921 .lock()
1922 .unwrap_or_else(|poisoned| poisoned.into_inner());
1923 let is_current = snapshots
1924 .get(module_id)
1925 .map(|tracked| Arc::ptr_eq(tracked, snapshot))
1926 .unwrap_or(false);
1927 if is_current {
1928 snapshots.remove(module_id);
1929 }
1930 }
1931}
1932
1933impl ModuleProcessLiveness for SupervisorProcessLiveness {
1934 fn process_live(&self, module_id: &str) -> Option<bool> {
1935 let snapshot = {
1936 let snapshots = self
1937 .snapshots
1938 .lock()
1939 .unwrap_or_else(|poisoned| poisoned.into_inner());
1940 snapshots.get(module_id).cloned()
1941 }?;
1942 let snapshot = snapshot
1943 .lock()
1944 .unwrap_or_else(|poisoned| poisoned.into_inner());
1945 Some(snapshot.state == ModuleState::Running && snapshot.process_alive)
1946 }
1947
1948 fn process_replacing(&self, module_id: &str) -> bool {
1949 let Some(snapshot) = self
1950 .snapshots
1951 .lock()
1952 .unwrap_or_else(|poisoned| poisoned.into_inner())
1953 .get(module_id)
1954 .cloned()
1955 else {
1956 return false;
1957 };
1958 let snapshot = snapshot
1959 .lock()
1960 .unwrap_or_else(|poisoned| poisoned.into_inner());
1961 snapshot.enabled
1962 && match snapshot.state {
1963 ModuleState::Restarting => true,
1964 ModuleState::Draining => snapshot.draining_to_replace,
1965 ModuleState::Starting
1966 | ModuleState::Running
1967 | ModuleState::Unresponsive
1968 | ModuleState::Stopped
1969 | ModuleState::Failed
1970 | ModuleState::Disabled => false,
1971 }
1972 }
1973}
1974
1975#[cfg(test)]
1976#[derive(Debug, Default)]
1977struct ReloadExitRecordGate {
1978 reached: tokio::sync::Notify,
1979 resume: tokio::sync::Notify,
1980}
1981
1982#[derive(Debug, Clone, Copy)]
1983enum RespawnKind {
1984 Spawn,
1985 Reload,
1986}
1987
1988#[derive(Debug, Clone, Copy)]
1989struct PendingRespawn {
1990 deadline: Instant,
1991 kind: RespawnKind,
1992 daemon_initiated: bool,
1993}
1994
1995type ReloadReply = oneshot::Sender<Result<(), SuperviseError>>;
1996
1997#[derive(Debug, Clone)]
1998struct SupervisorRuntimeConfig {
1999 registry: Arc<Registry>,
2000 recovery_is_daemon: bool,
2002 start_requester: Option<Principal>,
2003 pending_activity_restart: bool,
2004 provenance_probe: ExecutableIdentityProbe,
2005 scheduled_respawn: Arc<Mutex<Option<PendingRespawn>>>,
2007 deferred_reload_reply: Arc<Mutex<Option<ReloadReply>>>,
2009 restart_policy: RestartPolicy,
2010 register_timeout: Duration,
2011 drain_timeout: Duration,
2014 effective_drain_timeout: Arc<Mutex<Duration>>,
2017 default_drain_timeout: Duration,
2020 health: HealthConfig,
2021 host_health: HostHealth,
2022 connection_file_path: Option<PathBuf>,
2023 capture_logs_dir: Option<PathBuf>,
2024 forwarding: Option<Arc<ForwardingTable>>,
2025 supervisor_handle: Option<SupervisorHandle>,
2028 stderr_ring: Arc<Mutex<StderrRing>>,
2035 terminal_ring: Arc<Mutex<TerminalRing>>,
2036 spawn_events: SpawnEventFeed,
2037 child_roster: ChildRoster,
2038 #[cfg(target_os = "linux")]
2039 cgroup_placement: Option<subc_cgroup::Placement>,
2040 #[cfg(test)]
2041 test_seed_stale_facts_before_enable_spawn: bool,
2042 #[cfg(test)]
2043 test_reload_exit_record_gate: Option<Arc<ReloadExitRecordGate>>,
2044}
2045
2046impl SupervisorRuntimeConfig {
2047 fn failure_context(&self, daemon_initiated: bool) -> FailureContext {
2048 FailureContext {
2049 daemon_initiated,
2050 shutdown: self.child_roster.shutdown_flag(),
2051 }
2052 }
2053}
2054
2055#[derive(Debug, Clone, PartialEq, Eq)]
2056struct SupervisedConfiguration {
2057 spec: ModuleSpec,
2058 health: HealthConfig,
2059}
2060
2061#[derive(Debug, Clone, Default)]
2067pub struct SupervisorHandle {
2068 host_health: HostHealth,
2069 restart_intent: Arc<Mutex<DaemonRestartIntent>>,
2070 activity: Arc<Mutex<Option<crate::activity::ActivityLedger>>>,
2071 activity_images: Arc<Mutex<HashMap<String, activity::ActivityImage>>>,
2072 modules: Arc<Mutex<HashMap<String, SupervisedModule>>>,
2073 configured_ids: Arc<Mutex<HashSet<String>>>,
2085 spawn_events: SpawnEventFeed,
2086 reserved_nonces: Arc<Mutex<HashMap<String, Option<String>>>>,
2097 removal_tombstones: Arc<Mutex<HashMap<String, u64>>>,
2103 spawn_nonces: Arc<Mutex<HashMap<String, String>>>,
2107 reserved_prefix_owners: Arc<Mutex<HashMap<String, String>>>,
2115 swaps: Arc<Mutex<HashMap<String, OpenSwap>>>,
2121 promotion_observer: PromotionObserverSlot,
2123 operation_lock: Arc<AsyncMutex<()>>,
2127}
2128
2129pub(crate) trait SwapPromotionObserver: Send + Sync {
2138 fn swap_promoted(&self, registration: &crate::registry::ModuleRegistration);
2139}
2140
2141#[derive(Clone, Default)]
2145struct PromotionObserverSlot(Arc<Mutex<Option<std::sync::Weak<dyn SwapPromotionObserver>>>>);
2146
2147impl fmt::Debug for PromotionObserverSlot {
2148 fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
2149 f.write_str("PromotionObserverSlot")
2150 }
2151}
2152
2153#[derive(Debug, Clone)]
2155struct OpenSwap {
2156 candidate_nonce: String,
2159 incumbent_nonce: Option<String>,
2164 candidate_admitted: bool,
2168}
2169
2170#[derive(Debug, Clone, Copy, PartialEq, Eq)]
2173pub(crate) enum SwapHelloAdmission {
2174 NotSwapping,
2177 Candidate,
2179 Refused,
2182}
2183
2184#[derive(Debug, Clone, PartialEq, Eq)]
2185pub(crate) enum ReservedHelloRejection {
2186 Exact {
2187 module_id: String,
2188 },
2189 Prefix {
2190 prefix: String,
2191 owner_module_id: String,
2192 },
2193}
2194
2195#[derive(Debug, Default)]
2197pub(crate) struct ReservedHelloVerification {
2198 pub(crate) nonce_verified: bool,
2199 pub(crate) prefix_owner: Option<String>,
2200}
2201
2202#[derive(Debug)]
2203struct DaemonRestartIntent {
2204 expires: Option<Duration>,
2205 window: Duration,
2206}
2207
2208impl Default for DaemonRestartIntent {
2209 fn default() -> Self {
2210 Self {
2211 expires: None,
2212 window: Duration::from_secs(120),
2213 }
2214 }
2215}
2216
2217impl SupervisorHandle {
2218 pub(crate) fn announce_daemon_restart(&self) -> u64 {
2219 let mut intent = self
2220 .restart_intent
2221 .lock()
2222 .unwrap_or_else(|p| p.into_inner());
2223 intent.expires = Some(subc_uptime::suspend_inclusive_now() + intent.window);
2224 unix_ms_now().saturating_add(intent.window.as_millis() as u64)
2225 }
2226
2227 #[cfg(feature = "test-support")]
2228 pub(crate) fn set_shutdown_intent_window(&self, window: Duration) {
2229 self.restart_intent
2230 .lock()
2231 .unwrap_or_else(|p| p.into_inner())
2232 .window = window;
2233 }
2234
2235 #[cfg(unix)]
2236 fn take_daemon_shutdown_reason(&self) -> RouteCloseReason {
2237 let expiry = self
2238 .restart_intent
2239 .lock()
2240 .unwrap_or_else(|p| p.into_inner())
2241 .expires
2242 .take();
2243 if expiry.is_some_and(|expiry| subc_uptime::suspend_inclusive_now() < expiry) {
2244 RouteCloseReason::Restart
2245 } else {
2246 RouteCloseReason::Disable
2247 }
2248 }
2249
2250 pub fn new() -> Self {
2251 Self::default()
2252 }
2253
2254 pub(crate) fn spawn_snapshot(&self) -> SpawnSnapshot {
2255 self.spawn_events.snapshot()
2256 }
2257
2258 pub(crate) fn subscribe_spawns(
2259 &self,
2260 connection_id: ConnectionId,
2261 corr: u64,
2262 version: u8,
2263 since: Option<SpawnCursor>,
2264 sink: FrameSink,
2265 ) -> Result<(), SpawnSubscribeRefusal> {
2266 self.spawn_events
2267 .subscribe(connection_id, corr, version, since, sink)
2268 }
2269
2270 pub(crate) fn cancel_spawn_subscription(&self, connection_id: ConnectionId, corr: u64) -> bool {
2271 self.spawn_events.cancel(connection_id, corr)
2272 }
2273
2274 pub(crate) fn remove_spawn_subscribers(&self, connection_id: ConnectionId) {
2275 self.spawn_events.remove_connection(connection_id);
2276 }
2277
2278 #[cfg(any(test, feature = "test-support"))]
2279 pub fn set_spawn_event_capacity_for_test(&self, capacity: usize) {
2280 assert!(capacity > 0, "spawn event capacity must be non-zero");
2281 self.spawn_events.set_capacity(capacity);
2282 }
2283
2284 #[cfg(any(test, feature = "test-support"))]
2285 pub fn spawn_subscriber_count_for_test(&self) -> usize {
2286 self.spawn_events.subscriber_count()
2287 }
2288
2289 pub fn set_spawn_nonce(&self, module_id: &str, nonce: String) {
2292 self.spawn_nonces
2293 .lock()
2294 .unwrap_or_else(|poisoned| poisoned.into_inner())
2295 .insert(module_id.to_string(), nonce);
2296 }
2297
2298 pub fn set_reserved_nonce(&self, module_id: &str, nonce: String) {
2301 self.reserved_nonces
2302 .lock()
2303 .unwrap_or_else(|poisoned| poisoned.into_inner())
2304 .insert(module_id.to_string(), Some(nonce));
2305 }
2306
2307 pub fn set_reserved_prefixes(&self, owner_module_id: &str, prefixes: &[String]) {
2309 let mut owners = self
2310 .reserved_prefix_owners
2311 .lock()
2312 .unwrap_or_else(|poisoned| poisoned.into_inner());
2313 owners.retain(|_, owner| owner != owner_module_id);
2314 for prefix in prefixes {
2315 owners.insert(prefix.clone(), owner_module_id.to_string());
2316 }
2317 }
2318
2319 #[cfg(test)]
2321 pub(crate) fn spawn_nonce(&self, module_id: &str) -> Option<String> {
2322 self.spawn_nonces
2323 .lock()
2324 .unwrap_or_else(|poisoned| poisoned.into_inner())
2325 .get(module_id)
2326 .cloned()
2327 }
2328
2329 fn apply_identity_configuration(&self, spec: &ModuleSpec) {
2330 self.set_reserved_prefixes(&spec.module_id, &spec.reserved_prefixes);
2331 let spawn_nonce = self
2332 .spawn_nonces
2333 .lock()
2334 .unwrap_or_else(|poisoned| poisoned.into_inner())
2335 .get(&spec.module_id)
2336 .cloned();
2337 let mut reserved_nonces = self
2338 .reserved_nonces
2339 .lock()
2340 .unwrap_or_else(|poisoned| poisoned.into_inner());
2341 if spec.reserved {
2342 reserved_nonces.insert(spec.module_id.clone(), spawn_nonce);
2347 }
2348 drop(reserved_nonces);
2349 self.removal_tombstones
2353 .lock()
2354 .unwrap_or_else(|poisoned| poisoned.into_inner())
2355 .remove(&spec.module_id);
2356 }
2357
2358 pub fn reserved_hello_authorized(&self, module_id: &str, presented: Option<&str>) -> bool {
2363 self.reserved_hello_verification(module_id, presented)
2364 .is_ok()
2365 }
2366
2367 pub(crate) fn reserved_hello_verification(
2370 &self,
2371 module_id: &str,
2372 presented: Option<&str>,
2373 ) -> Result<ReservedHelloVerification, ReservedHelloRejection> {
2374 let nonces = self
2375 .reserved_nonces
2376 .lock()
2377 .unwrap_or_else(|poisoned| poisoned.into_inner());
2378 if let Some(expected) = nonces.get(module_id) {
2379 let authorized = match expected {
2383 Some(expected) => {
2384 presented.is_some_and(|p| constant_time_eq(expected.as_bytes(), p.as_bytes()))
2385 }
2386 None => false,
2387 };
2388 if authorized {
2389 return Ok(ReservedHelloVerification {
2390 nonce_verified: true,
2391 prefix_owner: None,
2392 });
2393 }
2394 return Err(ReservedHelloRejection::Exact {
2395 module_id: module_id.to_string(),
2396 });
2397 }
2398 drop(nonces);
2399
2400 let matched_prefix = self
2401 .reserved_prefix_owners
2402 .lock()
2403 .unwrap_or_else(|poisoned| poisoned.into_inner())
2404 .iter()
2405 .filter(|(prefix, _)| module_id.starts_with(prefix.as_str()))
2406 .max_by_key(|(prefix, _)| prefix.len())
2407 .map(|(prefix, owner)| (prefix.clone(), owner.clone()));
2408 let Some((prefix, owner_module_id)) = matched_prefix else {
2409 return Ok(ReservedHelloVerification::default());
2410 };
2411
2412 let authorized = presented.is_some_and(|presented| {
2413 self.spawn_nonces
2414 .lock()
2415 .unwrap_or_else(|poisoned| poisoned.into_inner())
2416 .get(&owner_module_id)
2417 .is_some_and(|expected| constant_time_eq(expected.as_bytes(), presented.as_bytes()))
2418 || self.swap_nonce_matches(&owner_module_id, presented)
2421 });
2422 if authorized {
2423 Ok(ReservedHelloVerification {
2424 nonce_verified: true,
2425 prefix_owner: Some(owner_module_id),
2426 })
2427 } else {
2428 Err(ReservedHelloRejection::Prefix {
2429 prefix,
2430 owner_module_id,
2431 })
2432 }
2433 }
2434
2435 pub fn spawned_consumer_authorized(&self, module_id: &str, presented: &str) -> bool {
2440 if presented.is_empty() {
2441 return false;
2442 }
2443 let nonces = self
2444 .spawn_nonces
2445 .lock()
2446 .unwrap_or_else(|poisoned| poisoned.into_inner());
2447 let current = nonces
2448 .get(module_id)
2449 .is_some_and(|expected| constant_time_eq(expected.as_bytes(), presented.as_bytes()));
2450 drop(nonces);
2451 current || self.swap_nonce_matches(module_id, presented)
2456 }
2457
2458 fn swap_nonce_matches(&self, module_id: &str, presented: &str) -> bool {
2460 let swaps = self
2461 .swaps
2462 .lock()
2463 .unwrap_or_else(|poisoned| poisoned.into_inner());
2464 swaps.get(module_id).is_some_and(|swap| {
2465 constant_time_eq(swap.candidate_nonce.as_bytes(), presented.as_bytes())
2466 || swap.incumbent_nonce.as_deref().is_some_and(|incumbent| {
2467 constant_time_eq(incumbent.as_bytes(), presented.as_bytes())
2468 })
2469 })
2470 }
2471
2472 pub(crate) fn open_swap(&self, module_id: &str, candidate_nonce: String) {
2475 let incumbent_nonce = self
2476 .spawn_nonces
2477 .lock()
2478 .unwrap_or_else(|poisoned| poisoned.into_inner())
2479 .get(module_id)
2480 .cloned();
2481 self.swaps
2482 .lock()
2483 .unwrap_or_else(|poisoned| poisoned.into_inner())
2484 .insert(
2485 module_id.to_string(),
2486 OpenSwap {
2487 candidate_nonce,
2488 incumbent_nonce,
2489 candidate_admitted: false,
2490 },
2491 );
2492 }
2493
2494 pub(crate) fn close_swap(&self, module_id: &str) {
2497 self.swaps
2498 .lock()
2499 .unwrap_or_else(|poisoned| poisoned.into_inner())
2500 .remove(module_id);
2501 }
2502
2503 pub(crate) fn set_swap_promotion_observer(
2506 &self,
2507 observer: std::sync::Weak<dyn SwapPromotionObserver>,
2508 ) {
2509 *self
2510 .promotion_observer
2511 .0
2512 .lock()
2513 .unwrap_or_else(|poisoned| poisoned.into_inner()) = Some(observer);
2514 }
2515
2516 fn notify_swap_promoted(&self, registration: &crate::registry::ModuleRegistration) {
2519 let observer = self
2520 .promotion_observer
2521 .0
2522 .lock()
2523 .unwrap_or_else(|poisoned| poisoned.into_inner())
2524 .as_ref()
2525 .and_then(std::sync::Weak::upgrade);
2526 if let Some(observer) = observer {
2527 observer.swap_promoted(registration);
2528 }
2529 }
2530
2531 pub(crate) fn swap_open(&self, module_id: &str) -> bool {
2533 self.swaps
2534 .lock()
2535 .unwrap_or_else(|poisoned| poisoned.into_inner())
2536 .contains_key(module_id)
2537 }
2538
2539 fn promote_swap_nonce(&self, module_id: &str, reserved: bool) {
2544 let candidate_nonce = self
2545 .swaps
2546 .lock()
2547 .unwrap_or_else(|poisoned| poisoned.into_inner())
2548 .get(module_id)
2549 .map(|swap| swap.candidate_nonce.clone());
2550 let Some(nonce) = candidate_nonce else {
2551 return;
2552 };
2553 self.set_spawn_nonce(module_id, nonce.clone());
2554 if reserved {
2555 self.set_reserved_nonce(module_id, nonce);
2556 }
2557 }
2558
2559 pub(crate) fn swap_hello_admission(
2574 &self,
2575 module_id: &str,
2576 presented: Option<&str>,
2577 ) -> SwapHelloAdmission {
2578 let swaps = self
2579 .swaps
2580 .lock()
2581 .unwrap_or_else(|poisoned| poisoned.into_inner());
2582 let Some(swap) = swaps.get(module_id) else {
2583 return SwapHelloAdmission::NotSwapping;
2584 };
2585 let Some(presented) = presented else {
2586 return SwapHelloAdmission::Refused;
2587 };
2588 if constant_time_eq(swap.candidate_nonce.as_bytes(), presented.as_bytes()) {
2589 return if swap.candidate_admitted {
2590 SwapHelloAdmission::Refused
2591 } else {
2592 SwapHelloAdmission::Candidate
2593 };
2594 }
2595 if swap
2596 .incumbent_nonce
2597 .as_deref()
2598 .is_some_and(|incumbent| constant_time_eq(incumbent.as_bytes(), presented.as_bytes()))
2599 {
2600 return SwapHelloAdmission::NotSwapping;
2601 }
2602 SwapHelloAdmission::Refused
2603 }
2604
2605 pub(crate) fn mark_swap_candidate_admitted(&self, module_id: &str) {
2608 if let Some(swap) = self
2609 .swaps
2610 .lock()
2611 .unwrap_or_else(|poisoned| poisoned.into_inner())
2612 .get_mut(module_id)
2613 {
2614 swap.candidate_admitted = true;
2615 }
2616 }
2617
2618 pub fn spawn_launch_nonce_for(&self, module_id: &str) -> Option<String> {
2620 self.spawn_nonces
2621 .lock()
2622 .unwrap_or_else(|poisoned| poisoned.into_inner())
2623 .get(module_id)
2624 .cloned()
2625 }
2626
2627 pub fn reserved_launch_nonce_for(&self, module_id: &str) -> Option<String> {
2629 self.reserved_nonces
2630 .lock()
2631 .unwrap_or_else(|poisoned| poisoned.into_inner())
2632 .get(module_id)
2633 .cloned()
2634 .flatten()
2635 }
2636
2637 pub fn insert(&self, module: SupervisedModule) -> Option<SupervisedModule> {
2638 self.mark_configured(module.module_id());
2642 let mut modules = self
2643 .modules
2644 .lock()
2645 .unwrap_or_else(|poisoned| poisoned.into_inner());
2646 modules.insert(module.module_id().to_string(), module)
2647 }
2648
2649 fn mark_configured(&self, module_id: &str) {
2653 self.configured_ids
2654 .lock()
2655 .unwrap_or_else(|poisoned| poisoned.into_inner())
2656 .insert(module_id.to_string());
2657 }
2658
2659 fn unmark_configured_unless_rostered(&self, module_id: &str) {
2663 let modules = self
2664 .modules
2665 .lock()
2666 .unwrap_or_else(|poisoned| poisoned.into_inner());
2667 if !modules.contains_key(module_id) {
2668 self.configured_ids
2669 .lock()
2670 .unwrap_or_else(|poisoned| poisoned.into_inner())
2671 .remove(module_id);
2672 }
2673 }
2674
2675 pub(crate) fn is_configured(&self, module_id: &str) -> bool {
2683 self.configured_ids
2684 .lock()
2685 .unwrap_or_else(|poisoned| poisoned.into_inner())
2686 .contains(module_id)
2687 }
2688
2689 pub fn get(&self, module_id: &str) -> Option<SupervisedModule> {
2690 let modules = self
2691 .modules
2692 .lock()
2693 .unwrap_or_else(|poisoned| poisoned.into_inner());
2694 modules.get(module_id).cloned()
2695 }
2696
2697 pub(crate) fn record_late_health_answer(
2698 &self,
2699 module_id: &str,
2700 latency_ms: u64,
2701 ) -> Result<bool, SuperviseError> {
2702 let Some(module) = self.get(module_id) else {
2703 return Ok(false);
2704 };
2705 self.host_health.record_late_module(module_id);
2706 update_snapshot(&module.inner.snapshot, Some(module_id), |state| {
2707 state.health.late_answer_count = state.health.late_answer_count.saturating_add(1);
2708 state.health.last_late_answer_latency_ms = Some(latency_ms);
2709 state.health.consecutive_failures = 0;
2717 })?;
2718 Ok(true)
2719 }
2720
2721 pub(crate) fn spawn_host_health_monitor(&self) -> JoinHandle<()> {
2722 self.host_health.spawn_monitor()
2723 }
2724
2725 pub fn record_deliberate_severance(&self, module_id: &str) -> Result<bool, SuperviseError> {
2731 let Some(module) = self.get(module_id) else {
2732 return Ok(false);
2733 };
2734 let snapshot = lock_snapshot(&module.inner.snapshot)?;
2735 let Some((pid, start_time)) = snapshot.pid.zip(snapshot.process_start_time) else {
2736 return Ok(false);
2737 };
2738 drop(snapshot);
2739 module.record_deliberate_severance(ProcessIdentity { pid, start_time })
2740 }
2741
2742 pub fn list(&self) -> Vec<SupervisedModule> {
2743 let modules = self
2744 .modules
2745 .lock()
2746 .unwrap_or_else(|poisoned| poisoned.into_inner());
2747 let mut modules = modules.values().cloned().collect::<Vec<_>>();
2748 modules.sort_by(|left, right| left.module_id().cmp(right.module_id()));
2749 modules
2750 }
2751
2752 pub(crate) fn retire(&self, module_id: &str) -> Option<SupervisedModule> {
2753 self.activity_images
2754 .lock()
2755 .unwrap_or_else(|p| p.into_inner())
2756 .remove(module_id);
2757 self.spawn_nonces
2758 .lock()
2759 .unwrap_or_else(|poisoned| poisoned.into_inner())
2760 .remove(module_id);
2761 self.close_swap(module_id);
2762 let mut reserved_nonces = self
2763 .reserved_nonces
2764 .lock()
2765 .unwrap_or_else(|poisoned| poisoned.into_inner());
2766 if reserved_nonces.contains_key(module_id) {
2767 reserved_nonces.insert(module_id.to_string(), None);
2770 }
2771 drop(reserved_nonces);
2772 self.reserved_prefix_owners
2773 .lock()
2774 .unwrap_or_else(|poisoned| poisoned.into_inner())
2775 .retain(|_, owner| owner != module_id);
2776 let removed = self
2777 .modules
2778 .lock()
2779 .unwrap_or_else(|poisoned| poisoned.into_inner())
2780 .remove(module_id);
2781 self.configured_ids
2782 .lock()
2783 .unwrap_or_else(|poisoned| poisoned.into_inner())
2784 .remove(module_id);
2785 removed
2786 }
2787
2788 pub(crate) fn record_rescan_removal(&self, module_id: &str) {
2791 self.removal_tombstones
2792 .lock()
2793 .unwrap_or_else(|poisoned| poisoned.into_inner())
2794 .insert(module_id.to_string(), unix_ms_now());
2795 }
2796
2797 pub(crate) fn removal_tombstone_age_ms(&self, module_id: &str) -> Option<u64> {
2799 self.removal_tombstones
2800 .lock()
2801 .unwrap_or_else(|poisoned| poisoned.into_inner())
2802 .get(module_id)
2803 .copied()
2804 .map(|removed_at_ms| unix_ms_now().saturating_sub(removed_at_ms))
2805 }
2806
2807 pub(crate) fn release_retained_reserved_gate(&self, module_id: &str) -> bool {
2812 if self.get(module_id).is_some() {
2813 return false;
2814 }
2815 let mut reserved_nonces = self
2816 .reserved_nonces
2817 .lock()
2818 .unwrap_or_else(|poisoned| poisoned.into_inner());
2819 if !matches!(reserved_nonces.get(module_id), Some(None)) {
2820 return false;
2821 }
2822 reserved_nonces.remove(module_id);
2823 true
2824 }
2825
2826 pub(crate) fn operation_lock(&self) -> Arc<AsyncMutex<()>> {
2827 Arc::clone(&self.operation_lock)
2828 }
2829}
2830
2831#[derive(Debug, Clone)]
2833pub struct Supervisor {
2834 registry: Arc<Registry>,
2835 restart_policy: RestartPolicy,
2836 drain_timeout: Duration,
2837 connection_file_path: Option<PathBuf>,
2838 capture_logs_dir: Option<PathBuf>,
2839 forwarding: Option<Arc<ForwardingTable>>,
2840 process_liveness: Arc<SupervisorProcessLiveness>,
2841 supervisor_handle: Option<SupervisorHandle>,
2842 health: HealthConfig,
2843 host_health: HostHealth,
2844 daemon_start_clock: crate::clock::StartClock,
2845 terminal_journal: Option<Arc<crate::terminal_journal::TerminalJournal>>,
2846 spawn_events: SpawnEventFeed,
2847 provenance_probe: ExecutableIdentityProbe,
2848 child_roster: ChildRoster,
2851 #[cfg(target_os = "linux")]
2852 cgroup_placement: Option<subc_cgroup::Placement>,
2853 #[cfg(test)]
2854 test_after_first_spawn: AfterFirstSpawnHook,
2855}
2856
2857pub(crate) struct RegisteredConfiguredModule {
2861 spec: ModuleSpec,
2862 runtime: SupervisorRuntimeConfig,
2863 module: SupervisedModule,
2864 commands: mpsc::Receiver<SupervisorCommand>,
2865}
2866
2867#[cfg(test)]
2873#[derive(Clone, Default)]
2874struct AfterFirstSpawnHook(Option<AfterFirstSpawnFn>);
2875
2876#[cfg(test)]
2877type AfterFirstSpawnFn = Arc<dyn Fn(&str) + Send + Sync>;
2878
2879#[cfg(test)]
2880impl fmt::Debug for AfterFirstSpawnHook {
2881 fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
2882 f.write_str("AfterFirstSpawnHook")
2883 }
2884}
2885
2886#[cfg(test)]
2887impl AfterFirstSpawnHook {
2888 fn run(&self, module_id: &str) {
2889 if let Some(hook) = &self.0 {
2890 hook(module_id);
2891 }
2892 }
2893}
2894
2895impl Supervisor {
2896 #[cfg(test)]
2897 pub(crate) fn new_for_test(registry: Arc<Registry>, policy: RestartPolicy) -> Self {
2898 let supervisor = Self::new(registry, policy);
2899 #[cfg(target_os = "macos")]
2900 let supervisor = supervisor.with_privacy_trampoline(test_privacy_trampoline());
2901 supervisor
2902 }
2903 pub fn with_privacy_trampoline(self, path: impl Into<PathBuf>) -> Self {
2908 let path = path.into();
2909 #[cfg(target_os = "macos")]
2910 {
2911 let result = probe_privacy_trampoline(&path).map(|()| path);
2912 if let Err(cause) = &result {
2913 error!(%cause, "macOS privacy identity unavailable; supervised launches will refuse");
2914 }
2915 self.child_roster.set_privacy_trampoline(result);
2916 }
2917 #[cfg(not(target_os = "macos"))]
2918 let _ = path;
2919 self
2920 }
2921 pub(crate) fn daemon_shutdown_flag(&self) -> crate::child_roster::DaemonShutdownFlag {
2924 self.child_roster.shutdown_flag()
2925 }
2926
2927 #[cfg(unix)]
2938 pub(crate) fn begin_daemon_shutdown(&self) {
2939 self.child_roster.close();
2940 if let Some(journal) = &self.terminal_journal {
2941 journal.stamp_shutdown();
2942 }
2943 }
2944
2945 #[cfg(unix)]
2950 pub(crate) async fn drain_for_daemon_shutdown(&self) -> Result<(), SuperviseError> {
2951 const NOTICE_BUDGET: Duration = Duration::from_millis(500);
2952 const DRAIN_BUDGET: Duration = Duration::from_secs(2);
2953 let Some(forwarding) = &self.forwarding else {
2954 return Ok(());
2955 };
2956 let notice_started = Instant::now();
2957 let modules = forwarding
2958 .begin_daemon_drain()
2959 .map_err(SuperviseError::Forwarding)?;
2960 let deadline_ms =
2961 unix_ms_now().saturating_add((NOTICE_BUDGET + DRAIN_BUDGET).as_millis() as u64);
2962 let shutdown_reason = self
2963 .supervisor_handle
2964 .as_ref()
2965 .map(SupervisorHandle::take_daemon_shutdown_reason)
2966 .unwrap_or(RouteCloseReason::Disable);
2967 let mut modules_told = 0;
2968 let mut notices = tokio::task::JoinSet::new();
2969 let mut drains = Vec::new();
2970 for connection in forwarding
2971 .module_connections()
2972 .map_err(SuperviseError::Forwarding)?
2973 {
2974 let target = ModuleDrainTarget {
2975 endpoint: connection.endpoint,
2976 sink: connection.sink,
2977 negotiated_ver: connection.negotiated_ver,
2978 abandoned_bindings: Vec::new(),
2979 excluded_subscriptions: 0,
2980 };
2981 notices.spawn(async move {
2982 (
2983 true,
2984 send_module_draining(
2985 &connection.module_id,
2986 shutdown_reason,
2987 deadline_ms,
2988 &target,
2989 true,
2990 )
2991 .await,
2992 )
2993 });
2994 }
2995 for (module_id, endpoint) in modules {
2996 let Some(target) = forwarding
2997 .begin_endpoint_drain(endpoint, RouteCloseReason::Restart)
2998 .map_err(SuperviseError::Forwarding)?
2999 else {
3000 continue;
3001 };
3002 let routes = forwarding
3003 .endpoint_routes(target.endpoint)
3004 .map_err(SuperviseError::Forwarding)?;
3005 let mut recipients = Vec::new();
3008 let mut clients: Vec<(crate::forwarding::GoodbyeTarget, Vec<u16>)> = Vec::new();
3009 for route in routes {
3010 let client = route.goodbye_target;
3011 if let Some((_, channels)) = clients
3012 .iter_mut()
3013 .find(|(existing, _)| existing.connection_id == client.connection_id)
3014 {
3015 channels.push(client.channel);
3016 } else {
3017 let channel = client.channel;
3018 clients.push((client, vec![channel]));
3019 }
3020 }
3021 for (client, mut channels) in clients {
3022 channels.sort_unstable();
3023 channels.dedup();
3024 let closing = serde_json::to_vec(&ClientControlPush::RouteClosing {
3025 module_id: module_id.clone(),
3026 channels,
3027 reason: RouteCloseReason::Restart,
3028 })
3029 .expect("route closing serializes");
3030 recipients.push((client.sink, client.negotiated_ver, closing));
3031 }
3032 for (sink, version, body) in recipients {
3033 notices.spawn(async move {
3034 let frame = Frame::build_with_version(
3035 version,
3036 FrameType::Push,
3037 control_flags(),
3038 0,
3039 0,
3040 0,
3041 body,
3042 )
3043 .expect("bounded lifecycle notice frame builds");
3044 (false, sink.send_flushed(frame).await.is_ok())
3045 });
3046 }
3047 let gauges = declared_busy_gauges(&self.registry, &module_id)?;
3048 drains.push((module_id, target.endpoint, gauges));
3049 }
3050 let notice_deadline = Instant::now() + NOTICE_BUDGET;
3053 while let Ok(Some(result)) = timeout_at(notice_deadline, notices.join_next()).await {
3054 match result {
3055 Ok((true, true)) => modules_told += 1,
3056 Ok((false, true)) => {}
3057 result => warn!(?result, "daemon shutdown notice delivery failed"),
3058 }
3059 }
3060 notices.abort_all();
3061 info!(
3062 ?shutdown_reason,
3063 modules_told, "daemon shutdown module notices delivered"
3064 );
3065 let notice_elapsed = notice_started.elapsed();
3066 let drain_started = Instant::now();
3067 let deadline = drain_started + DRAIN_BUDGET;
3068 let mut waits = tokio::task::JoinSet::new();
3069 for (module_id, endpoint, gauges) in drains {
3070 let forwarding = Arc::clone(forwarding);
3071 let mut runtime = self.runtime_config();
3072 runtime.health.cadence = Duration::from_millis(100);
3073 waits.spawn(async move {
3074 let wait_result = wait_for_forwarding_quiescence(
3075 &forwarding,
3076 &module_id,
3077 &runtime,
3078 endpoint,
3079 deadline,
3080 &gauges,
3081 DrainScope::Endpoint(endpoint),
3082 &mut DrainGaugeTrace::new(&gauges),
3083 )
3084 .await;
3085 forwarding
3089 .record_daemon_drain_result(
3090 endpoint,
3091 drained_after_quiescence_wait(&wait_result),
3092 )
3093 .map_err(SuperviseError::Forwarding)?;
3094 wait_result
3095 });
3096 }
3097 while let Ok(Some(result)) = timeout_at(deadline, waits.join_next()).await {
3098 if !matches!(result, Ok(Ok(true))) {
3099 warn!(?result, "daemon shutdown drain did not reach quiescence");
3100 }
3101 }
3102 info!(
3105 notice_elapsed_ms = notice_elapsed.as_millis() as u64,
3106 drain_elapsed_ms = drain_started.elapsed().as_millis() as u64,
3107 "daemon shutdown forwarding drain complete"
3108 );
3109 Ok(())
3110 }
3111
3112 #[cfg(unix)]
3124 pub(crate) async fn end_children_for_daemon_shutdown(
3125 &self,
3126 already_escalated: bool,
3127 escalate: impl std::future::Future<Output = ()>,
3128 ) {
3129 tokio::pin!(escalate);
3130 let mut escalated = already_escalated;
3131 if let Some(forwarding) = &self.forwarding {
3132 let reason = CloseReason::new(
3133 "daemon_shutdown",
3134 "the daemon is exiting after its shutdown notice and drain",
3135 );
3136 if escalated {
3137 send_module_goodbyes_for_daemon_shutdown(forwarding, &reason, false).await;
3140 } else {
3141 tokio::select! {
3142 biased;
3143 _ = escalate.as_mut() => {
3144 info!("second SIGTERM: abandoning module GOODBYE delivery");
3145 escalated = true;
3146 }
3147 _ = send_module_goodbyes_for_daemon_shutdown(forwarding, &reason, true) => {}
3148 }
3149 }
3150 let closed = forwarding.close_all_connections(&reason);
3151 debug!(closed, "closed established connections for daemon shutdown");
3152 }
3153 let escalated_here = escalated && !already_escalated;
3157 let remaining_escalate = async move {
3158 if escalated_here {
3159 std::future::pending::<()>().await;
3160 } else {
3161 escalate.await;
3162 }
3163 };
3164 crate::child_roster::end_children_for_daemon_shutdown(
3165 &self.child_roster,
3166 escalated,
3167 remaining_escalate,
3168 )
3169 .await;
3170 }
3171
3172 pub fn new(registry: Arc<Registry>, restart_policy: RestartPolicy) -> Self {
3173 Self {
3174 registry,
3175 restart_policy,
3176 drain_timeout: DEFAULT_DRAIN_TIMEOUT,
3177 connection_file_path: None,
3178 capture_logs_dir: None,
3179 forwarding: None,
3180 process_liveness: Arc::new(SupervisorProcessLiveness::default()),
3181 supervisor_handle: None,
3182 health: HealthConfig::default(),
3183 host_health: HostHealth::default(),
3184 daemon_start_clock: crate::clock::StartClock::capture(),
3185 terminal_journal: None,
3186 spawn_events: SpawnEventFeed::default(),
3187 provenance_probe: ExecutableIdentityProbe::default(),
3188 child_roster: ChildRoster::default(),
3189 #[cfg(target_os = "linux")]
3190 cgroup_placement: None,
3191 #[cfg(test)]
3192 test_after_first_spawn: AfterFirstSpawnHook::default(),
3193 }
3194 }
3195
3196 pub fn with_drain_timeout(mut self, drain_timeout: Duration) -> Self {
3197 self.drain_timeout = drain_timeout;
3198 self
3199 }
3200
3201 pub(crate) fn default_drain_timeout(&self) -> Duration {
3203 self.drain_timeout
3204 }
3205
3206 pub fn with_process_liveness(
3207 mut self,
3208 process_liveness: Arc<SupervisorProcessLiveness>,
3209 ) -> Self {
3210 self.process_liveness = process_liveness;
3211 self
3212 }
3213
3214 pub fn with_connection_file_path(mut self, connection_file_path: impl Into<PathBuf>) -> Self {
3215 self.connection_file_path = Some(connection_file_path.into());
3216 self
3217 }
3218
3219 pub fn with_capture_logs_dir(mut self, logs_dir: impl Into<PathBuf>) -> Self {
3221 self.capture_logs_dir = Some(logs_dir.into());
3222 self
3223 }
3224
3225 pub fn with_daemon_incarnation(self, daemon_incarnation: String) -> Self {
3228 self.spawn_events.configure_incarnation(daemon_incarnation);
3232 self
3233 }
3234
3235 pub fn with_terminal_journal(self, path: PathBuf, daemon_incarnation: String) -> Self {
3238 let mut this = self.with_daemon_incarnation(daemon_incarnation.clone());
3239 this.terminal_journal = Some(Arc::new(crate::terminal_journal::TerminalJournal::open(
3240 path,
3241 daemon_incarnation,
3242 )));
3243 this
3244 }
3245
3246 pub fn with_forwarding(mut self, forwarding: Arc<ForwardingTable>) -> Self {
3247 self.child_roster.counters = forwarding.counters();
3248 self.forwarding = Some(forwarding);
3249 self
3250 }
3251
3252 pub fn with_handle(mut self, supervisor_handle: SupervisorHandle) -> Self {
3253 self.host_health = supervisor_handle.host_health.clone();
3254 self.spawn_events = supervisor_handle.spawn_events.clone();
3255 self.supervisor_handle = Some(supervisor_handle);
3256 self
3257 }
3258
3259 pub fn with_health_config(mut self, health: HealthConfig) -> Self {
3260 self.health = health;
3261 self
3262 }
3263
3264 pub fn with_live_children_record(self, path: impl Into<PathBuf>) -> Self {
3268 self.child_roster.record_to(path.into());
3269 self
3270 }
3271
3272 #[cfg(target_os = "linux")]
3273 pub fn with_cgroup_placement(
3274 mut self,
3275 cgroup_placement: Option<subc_cgroup::Placement>,
3276 ) -> Self {
3277 self.child_roster
3278 .set_cgroup_placement(cgroup_placement.clone());
3279 self.cgroup_placement = cgroup_placement;
3280 self
3281 }
3282
3283 pub fn spawn(&self, spec: ModuleSpec) -> Result<SupervisedModule, SuperviseError> {
3289 validate_spec(&spec)?;
3290 self.establish_identity(&spec);
3291
3292 let runtime = self.runtime_config();
3293 let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::starting()));
3294 let activity_spawn = runtime.begin_activity_spawn(&spec.module_id);
3295 let registration_watch = watch_registration(&self.registry, &spec);
3296 let spawned = spawn_child(
3297 &spec,
3298 runtime.connection_file_path.as_deref(),
3299 self.supervisor_handle.as_ref(),
3300 &runtime.stderr_ring,
3301 runtime.capture_logs_dir.as_deref(),
3302 &runtime.child_roster,
3303 #[cfg(target_os = "linux")]
3304 runtime.cgroup_placement.as_ref(),
3305 );
3306 #[cfg(test)]
3307 self.test_after_first_spawn.run(&spec.module_id);
3308 let mut child = match spawned {
3309 Ok(child) => child,
3310 Err(err) => {
3311 self.abandon_unrostered(&spec.module_id);
3314 return Err(err);
3315 }
3316 };
3317 arm_registration_deadline(&mut child, runtime.register_timeout, registration_watch);
3318 self.mark_first_process_running(&spec, &runtime, &snapshot, &mut child)?;
3319 if let Some(spawn) = activity_spawn {
3320 spawn.complete(&snapshot);
3321 }
3322
3323 Ok(self.supervised_module(spec, runtime, snapshot, Some(child)))
3324 }
3325
3326 fn establish_identity(&self, spec: &ModuleSpec) {
3339 if let Some(supervisor_handle) = &self.supervisor_handle {
3340 supervisor_handle.apply_identity_configuration(spec);
3341 supervisor_handle.mark_configured(&spec.module_id);
3342 }
3343 }
3344
3345 fn abandon_unrostered(&self, module_id: &str) {
3348 if let Some(supervisor_handle) = &self.supervisor_handle {
3349 supervisor_handle.unmark_configured_unless_rostered(module_id);
3350 }
3351 }
3352
3353 fn mark_first_process_running(
3356 &self,
3357 spec: &ModuleSpec,
3358 runtime: &SupervisorRuntimeConfig,
3359 snapshot: &SharedSnapshot,
3360 child: &mut SupervisedChild,
3361 ) -> Result<(), SuperviseError> {
3362 if let Err(err) = set_running(snapshot, child, &spec.module_id, &runtime.spawn_events) {
3363 containment::terminate_blocking(&runtime.child_roster.containment);
3364 self.abandon_unrostered(&spec.module_id);
3365 return Err(err);
3366 }
3367 self.process_liveness
3368 .track(spec.module_id.clone(), Arc::clone(snapshot));
3369 Ok(())
3370 }
3371
3372 pub fn supervise_configured(
3378 &self,
3379 spec: ModuleSpec,
3380 enabled: bool,
3381 ) -> Result<SupervisedModule, SuperviseError> {
3382 self.supervise_configured_requested(
3383 spec,
3384 enabled,
3385 self.health.clone(),
3386 None,
3387 self.restart_policy,
3388 DEFAULT_REGISTER_TIMEOUT,
3389 None,
3390 )
3391 }
3392
3393 pub fn supervise_configured_with_health(
3399 &self,
3400 spec: ModuleSpec,
3401 enabled: bool,
3402 health: HealthConfig,
3403 drain_timeout_ms: Option<u64>,
3404 restart_policy: RestartPolicy,
3405 ) -> Result<SupervisedModule, SuperviseError> {
3406 self.supervise_configured_requested(
3407 spec,
3408 enabled,
3409 health,
3410 drain_timeout_ms,
3411 restart_policy,
3412 DEFAULT_REGISTER_TIMEOUT,
3413 None,
3414 )
3415 }
3416
3417 #[allow(clippy::too_many_arguments)]
3418 pub(crate) fn supervise_configured_requested(
3419 &self,
3420 spec: ModuleSpec,
3421 enabled: bool,
3422 health: HealthConfig,
3423 drain_timeout_ms: Option<u64>,
3424 restart_policy: RestartPolicy,
3425 register_timeout: Duration,
3426 requester: Option<Principal>,
3427 ) -> Result<SupervisedModule, SuperviseError> {
3428 let registered = self.register_configured_with_health(
3429 spec,
3430 enabled,
3431 health,
3432 drain_timeout_ms,
3433 restart_policy,
3434 register_timeout,
3435 )?;
3436 let daemon_initiated = requester.is_none();
3437 self.start_registered_configured(registered, requester, daemon_initiated)
3438 }
3439
3440 pub(crate) fn register_configured_with_health(
3445 &self,
3446 spec: ModuleSpec,
3447 enabled: bool,
3448 health: HealthConfig,
3449 drain_timeout_ms: Option<u64>,
3450 restart_policy: RestartPolicy,
3451 register_timeout: Duration,
3452 ) -> Result<RegisteredConfiguredModule, SuperviseError> {
3453 validate_spec(&spec)?;
3454 self.establish_identity(&spec);
3455
3456 let mut runtime = self.runtime_config();
3457 runtime.health = health;
3458 runtime.restart_policy = restart_policy;
3459 runtime.register_timeout = register_timeout;
3460 if let Some(ms) = drain_timeout_ms {
3461 runtime.drain_timeout = Duration::from_millis(ms);
3462 *runtime
3463 .effective_drain_timeout
3464 .lock()
3465 .unwrap_or_else(|poisoned| poisoned.into_inner()) = runtime.drain_timeout;
3466 }
3467 let snapshot = if enabled {
3468 SupervisorSnapshot::starting()
3469 } else {
3470 SupervisorSnapshot::disabled()
3471 };
3472 let snapshot = {
3473 let mut snapshot = snapshot;
3474 if enabled {
3475 if let Some((pid, reason)) = runtime.child_roster.previous_instance(&spec.module_id)
3476 {
3477 snapshot.waiting_for_previous_pid = Some(pid);
3478 snapshot.waiting_for_previous_detail = Some(reason.waiting_message(pid));
3479 }
3480 }
3481 snapshot
3482 };
3483 let (module, commands) =
3484 self.register_module(&spec, &runtime, Arc::new(Mutex::new(snapshot)));
3485 Ok(RegisteredConfiguredModule {
3486 spec,
3487 runtime,
3488 module,
3489 commands,
3490 })
3491 }
3492
3493 pub(crate) fn start_registered_configured(
3496 &self,
3497 registered: RegisteredConfiguredModule,
3498 requester: Option<Principal>,
3499 daemon_initiated: bool,
3500 ) -> Result<SupervisedModule, SuperviseError> {
3501 let RegisteredConfiguredModule {
3502 spec,
3503 mut runtime,
3504 module,
3505 commands,
3506 } = registered;
3507 runtime.start_requester = requester;
3508 runtime.recovery_is_daemon = daemon_initiated;
3509 let snapshot = &module.inner.snapshot;
3510 let should_spawn = {
3511 let state = lock_snapshot(snapshot)?;
3512 state.enabled && state.waiting_for_previous_pid.is_none()
3513 };
3514 let child = if should_spawn {
3515 let activity_spawn = runtime.begin_activity_spawn(&spec.module_id);
3516 let registration_watch = watch_registration(&self.registry, &spec);
3517 let spawned = spawn_child(
3518 &spec,
3519 runtime.connection_file_path.as_deref(),
3520 self.supervisor_handle.as_ref(),
3521 &runtime.stderr_ring,
3522 runtime.capture_logs_dir.as_deref(),
3523 &runtime.child_roster,
3524 #[cfg(target_os = "linux")]
3525 runtime.cgroup_placement.as_ref(),
3526 );
3527 #[cfg(test)]
3528 self.test_after_first_spawn.run(&spec.module_id);
3529 match spawned {
3530 Ok(mut child) => {
3531 arm_registration_deadline(
3532 &mut child,
3533 runtime.register_timeout,
3534 registration_watch,
3535 );
3536 self.mark_first_process_running(&spec, &runtime, snapshot, &mut child)?;
3537 if let Some(spawn) = activity_spawn {
3538 spawn.complete(snapshot);
3539 }
3540 Some(child)
3541 }
3542 Err(err) => {
3543 if runtime.health.critical {
3544 error!(
3545 module_id = %spec.module_id,
3546 program = %spec.program.display(),
3547 error = %err,
3548 "critical configured module failed to spawn; marking failed and alerting"
3549 );
3550 } else {
3551 error!(
3552 module_id = %spec.module_id,
3553 program = %spec.program.display(),
3554 error = %err,
3555 "configured module failed to spawn; marking failed and continuing"
3556 );
3557 }
3558 fail_snapshot(
3559 snapshot,
3560 Some(&spec.module_id),
3561 None,
3562 &runtime.failure_context(daemon_initiated),
3563 );
3564 None
3565 }
3566 }
3567 } else {
3568 None
3569 };
3570 self.start_monitor(spec, runtime, &module, child, commands);
3571 Ok(module)
3572 }
3573
3574 fn runtime_config(&self) -> SupervisorRuntimeConfig {
3575 let effective_drain_timeout = Arc::new(Mutex::new(self.drain_timeout));
3576 SupervisorRuntimeConfig {
3577 registry: self.registry.clone(),
3578 recovery_is_daemon: true,
3579 start_requester: None,
3580 pending_activity_restart: false,
3581 provenance_probe: self.provenance_probe.clone(),
3582 scheduled_respawn: Arc::default(),
3583 deferred_reload_reply: Arc::default(),
3584 restart_policy: self.restart_policy,
3585 register_timeout: DEFAULT_REGISTER_TIMEOUT,
3586 drain_timeout: self.drain_timeout,
3587 child_roster: self
3590 .child_roster
3591 .for_module(Arc::clone(&effective_drain_timeout)),
3592 effective_drain_timeout,
3593 default_drain_timeout: self.drain_timeout,
3594 health: self.health.clone(),
3595 host_health: self.host_health.clone(),
3596 connection_file_path: self.connection_file_path.clone(),
3597 capture_logs_dir: self.capture_logs_dir.clone(),
3598 forwarding: self.forwarding.clone(),
3599 supervisor_handle: self.supervisor_handle.clone(),
3600 stderr_ring: Arc::new(Mutex::new(StderrRing::new(StderrTailConfig::default()))),
3601 terminal_ring: Arc::new(Mutex::new(
3602 TerminalRing::new(
3603 TerminalRingConfig::default(),
3604 self.daemon_start_clock.started_at_ms(),
3605 )
3606 .with_start_clock(self.daemon_start_clock)
3607 .with_journal(self.terminal_journal.clone())
3608 .with_daemon_shutdown(self.child_roster.shutdown_flag()),
3609 )),
3610 spawn_events: self.spawn_events.clone(),
3611 #[cfg(target_os = "linux")]
3612 cgroup_placement: self.cgroup_placement.clone(),
3613 #[cfg(test)]
3614 test_seed_stale_facts_before_enable_spawn: false,
3615 #[cfg(test)]
3616 test_reload_exit_record_gate: None,
3617 }
3618 }
3619
3620 fn supervised_module(
3621 &self,
3622 spec: ModuleSpec,
3623 runtime: SupervisorRuntimeConfig,
3624 snapshot: SharedSnapshot,
3625 child: Option<SupervisedChild>,
3626 ) -> SupervisedModule {
3627 let (module, commands) = self.register_module(&spec, &runtime, snapshot);
3628 self.start_monitor(spec, runtime, &module, child, commands);
3629 module
3630 }
3631
3632 fn register_module(
3633 &self,
3634 spec: &ModuleSpec,
3635 runtime: &SupervisorRuntimeConfig,
3636 snapshot: SharedSnapshot,
3637 ) -> (SupervisedModule, mpsc::Receiver<SupervisorCommand>) {
3638 runtime.host_health.track(spec, &snapshot);
3639 let configuration = Arc::new(Mutex::new(SupervisedConfiguration {
3640 spec: spec.clone(),
3641 health: runtime.health.clone(),
3642 }));
3643 let stderr_ring = Arc::clone(&runtime.stderr_ring);
3644 let terminal_ring = Arc::clone(&runtime.terminal_ring);
3645 let restart_policy = runtime.restart_policy;
3649 let effective_drain_timeout = Arc::clone(&runtime.effective_drain_timeout);
3650 let (tx, rx) = mpsc::channel(4);
3651 let module_id = spec.module_id.clone();
3652 let module = SupervisedModule {
3653 inner: Arc::new(SupervisedModuleInner {
3654 module_id: module_id.clone(),
3655 registry: Arc::clone(&self.registry),
3656 snapshot,
3657 configuration,
3658 stderr_ring,
3659 terminal_ring,
3660 commands: tx,
3661 monitor: Mutex::new(None),
3662 restart_policy,
3663 effective_drain_timeout,
3664 provenance_probe: self.provenance_probe.clone(),
3665 retained: runtime.child_roster.retained_view(),
3666 }),
3667 };
3668 if let Some(supervisor_handle) = &self.supervisor_handle {
3671 supervisor_handle.insert(module.clone());
3672 }
3673 (module, rx)
3674 }
3675
3676 fn start_monitor(
3677 &self,
3678 spec: ModuleSpec,
3679 runtime: SupervisorRuntimeConfig,
3680 module: &SupervisedModule,
3681 child: Option<SupervisedChild>,
3682 commands: mpsc::Receiver<SupervisorCommand>,
3683 ) {
3684 let monitor = tokio::spawn(supervise_loop(
3685 spec,
3686 runtime,
3687 Arc::clone(&self.registry),
3688 Arc::clone(&self.process_liveness),
3689 Arc::clone(&module.inner.snapshot),
3690 child,
3691 commands,
3692 ));
3693 *module
3694 .inner
3695 .monitor
3696 .lock()
3697 .unwrap_or_else(|p| p.into_inner()) = Some(monitor);
3698 }
3699}
3700
3701impl Default for Supervisor {
3702 fn default() -> Self {
3703 Self::new(Arc::new(Registry::default()), RestartPolicy::default())
3704 }
3705}
3706
3707#[derive(Clone)]
3709pub struct SupervisedModule {
3710 inner: Arc<SupervisedModuleInner>,
3711}
3712
3713struct SupervisedModuleInner {
3714 retained: crate::child_roster::RetainedView,
3715 module_id: String,
3716 registry: Arc<Registry>,
3717 snapshot: SharedSnapshot,
3718 configuration: Arc<Mutex<SupervisedConfiguration>>,
3719 stderr_ring: Arc<Mutex<StderrRing>>,
3720 terminal_ring: Arc<Mutex<TerminalRing>>,
3721 commands: mpsc::Sender<SupervisorCommand>,
3722 monitor: Mutex<Option<JoinHandle<()>>>,
3723 restart_policy: RestartPolicy,
3727 effective_drain_timeout: Arc<Mutex<Duration>>,
3728 provenance_probe: ExecutableIdentityProbe,
3729}
3730
3731impl fmt::Debug for SupervisedModule {
3732 fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
3733 f.debug_struct("SupervisedModule")
3734 .field("module_id", &self.inner.module_id)
3735 .field("status", &self.status())
3736 .finish_non_exhaustive()
3737 }
3738}
3739
3740impl SupervisedModule {
3741 pub fn module_id(&self) -> &str {
3742 &self.inner.module_id
3743 }
3744
3745 #[cfg(test)]
3749 pub(crate) fn record_health_probe_failure_for_test(
3750 &self,
3751 detail: &str,
3752 ) -> Result<(), SuperviseError> {
3753 update_snapshot(&self.inner.snapshot, Some(&self.inner.module_id), |state| {
3754 state.health.consecutive_failures = state.health.consecutive_failures.saturating_add(1);
3755 state.health.detail = Some(detail.to_string());
3756 })
3757 }
3758
3759 pub fn state(&self) -> Result<ModuleState, SuperviseError> {
3760 Ok(lock_snapshot(&self.inner.snapshot)?.state)
3761 }
3762
3763 pub(crate) fn registration_end_reason(
3764 &self,
3765 ) -> Result<Option<RegistrationEndReason>, SuperviseError> {
3766 let snapshot = lock_snapshot(&self.inner.snapshot)?;
3767 Ok(match snapshot.state {
3768 ModuleState::Draining if snapshot.draining_to_replace => {
3769 Some(RegistrationEndReason::SupervisorRestart)
3770 }
3771 ModuleState::Draining => Some(RegistrationEndReason::SupervisorStop),
3772 ModuleState::Restarting if snapshot.draining_to_replace => {
3773 Some(RegistrationEndReason::SupervisorRestart)
3774 }
3775 _ => None,
3776 })
3777 }
3778
3779 pub fn stderr_tail(
3786 &self,
3787 max_lines: Option<usize>,
3788 max_bytes: Option<usize>,
3789 ) -> StderrTailSnapshot {
3790 self.inner
3791 .stderr_ring
3792 .lock()
3793 .unwrap_or_else(|poisoned| poisoned.into_inner())
3794 .snapshot(max_lines, max_bytes)
3795 }
3796
3797 pub fn terminal_history(&self) -> TerminalHistorySnapshot {
3802 self.inner
3803 .terminal_ring
3804 .lock()
3805 .unwrap_or_else(|poisoned| poisoned.into_inner())
3806 .snapshot()
3807 }
3808
3809 pub fn durable_terminal_history(&self) -> subc_control::TerminalHistory {
3814 durable_terminal_history_of(&self.inner.terminal_ring, &self.inner.module_id)
3815 }
3816
3817 pub(crate) async fn read_durable_terminal_history(
3822 &self,
3823 ) -> Result<subc_control::TerminalHistory, tokio::task::JoinError> {
3824 let terminal_ring = Arc::clone(&self.inner.terminal_ring);
3825 let module_id = self.inner.module_id.clone();
3826 tokio::task::spawn_blocking(move || durable_terminal_history_of(&terminal_ring, &module_id))
3827 .await
3828 }
3829
3830 pub fn status(&self) -> Result<ModuleStatus, SuperviseError> {
3831 self.status_with_snapshot_lock(&self.inner.snapshot, None)
3832 .map(|(status, _)| status)
3833 }
3834
3835 pub(crate) fn next_retry_at_ms(&self) -> Result<Option<u64>, SuperviseError> {
3836 let state = lock_snapshot(&self.inner.snapshot)?;
3837 Ok(state
3838 .failed_retry
3839 .pending
3840 .filter(|_| state.state == ModuleState::Failed && state.enabled)
3841 .map(|retry| {
3842 unix_ms_now().saturating_add(
3843 u64::try_from(
3844 retry
3845 .deadline
3846 .saturating_duration_since(Instant::now())
3847 .as_millis(),
3848 )
3849 .unwrap_or(u64::MAX),
3850 )
3851 }))
3852 }
3853
3854 pub(crate) fn record_deliberate_severance(
3855 &self,
3856 identity: ProcessIdentity,
3857 ) -> Result<bool, SuperviseError> {
3858 let mut snapshot = lock_snapshot(&self.inner.snapshot)?;
3859 if snapshot.pid != Some(identity.pid)
3860 || snapshot.process_start_time != Some(identity.start_time)
3861 {
3862 return Ok(false);
3863 }
3864 snapshot.deliberate_severance = Some(identity);
3865 Ok(true)
3866 }
3867
3868 pub(crate) fn status_for_control(
3873 &self,
3874 caller: &'static str,
3875 ) -> Result<ModuleStatus, SuperviseError> {
3876 self.status_with_snapshot_lock(&self.inner.snapshot, Some(caller))
3877 .map(|(status, _)| status)
3878 }
3879
3880 fn status_with_snapshot_lock(
3881 &self,
3882 snapshot: &SharedSnapshot,
3883 caller: Option<&'static str>,
3884 ) -> Result<(ModuleStatus, Option<SpawnedFileIdentity>), SuperviseError> {
3885 let mut guard = match caller {
3886 Some(caller) => lock_snapshot_for_control(snapshot, &self.inner.module_id, caller)?,
3887 None => lock_snapshot(snapshot)?,
3888 };
3889 let restart_count =
3892 guard.crash_restarts_in_window(self.inner.restart_policy.window, Instant::now());
3893 let snapshot = guard.clone();
3894 drop(guard);
3895 let drain_timeout = *self.inner.effective_drain_timeout.lock().map_err(|_| {
3896 SuperviseError::StatePoisoned {
3897 module_id: Some(self.inner.module_id.clone()),
3898 }
3899 })?;
3900 let registration_active = self
3901 .inner
3902 .registry
3903 .get_module(&self.inner.module_id)
3904 .map_err(SuperviseError::Registry)?
3905 .is_some();
3906 let protocol = snapshot
3907 .spawned_protocol
3908 .unwrap_or(self.declared_protocol()?);
3909 let running_process =
3910 snapshot.enabled && snapshot.state == ModuleState::Running && snapshot.process_alive;
3911 let live = match protocol {
3917 ModuleProtocol::Subc => running_process && registration_active,
3918 ModuleProtocol::None => running_process,
3919 };
3920 let retained = self.inner.retained.waiting_detail(&self.inner.module_id);
3923 let carry_configured = self
3924 .inner
3925 .configuration
3926 .lock()
3927 .map_err(|_| SuperviseError::StatePoisoned {
3928 module_id: Some(self.inner.module_id.clone()),
3929 })?
3930 .spec
3931 .carry_containment;
3932 let recovery_block = carries_containment(
3933 carry_configured,
3934 self.inner
3935 .retained
3936 .has_carried_container(&self.inner.module_id),
3937 )
3938 .then(|| self.inner.retained.recovery_block())
3939 .flatten();
3940 let launch_nonce_source = observed_launch_nonce_source(&snapshot);
3941 let pid = snapshot.reported_pid();
3942 let mut health = snapshot.health;
3943 if let Some(detail) = recovery_block
3944 .as_ref()
3945 .or_else(|| retained.as_ref().map(|(_, detail)| detail))
3946 {
3947 health.detail = Some(match health.detail {
3948 Some(previous) => format!("{previous}; {detail}"),
3949 None => detail.clone(),
3950 });
3951 }
3952
3953 Ok((
3954 ModuleStatus {
3955 waiting_for_previous_pid: retained
3956 .as_ref()
3957 .map(|(pid, _)| *pid)
3958 .filter(|pid| *pid > 0)
3959 .or(snapshot.waiting_for_previous_pid),
3960 waiting_for_previous_detail: recovery_block
3961 .or_else(|| retained.map(|(_, detail)| detail))
3962 .or(snapshot.waiting_for_previous_detail),
3963 module_id: self.inner.module_id.clone(),
3964 launch_nonce_source,
3965 state: snapshot.state,
3966 enabled: snapshot.enabled,
3967 process_alive: snapshot.process_alive,
3968 registration_active,
3969 protocol,
3970 live,
3971 restart_count,
3972 lifetime_restarts: snapshot.lifetime_restarts,
3973 spawn_generation: snapshot.spawn_generation,
3974 max_restarts: self.inner.restart_policy.max_restarts,
3975 restart_window: self.inner.restart_policy.window,
3976 drain_timeout,
3977 restart_backoff: self.inner.restart_policy.backoff,
3978 restart_max_backoff: self.inner.restart_policy.max_backoff,
3979 pid,
3980 spawned_at_ms: snapshot.spawned_at_ms,
3981 spawned_from: snapshot.spawned_from,
3982 launch_spec_digest: snapshot.launch_spec_digest,
3983 process_start_time: snapshot.process_start_time,
3984 last_exit: snapshot.last_exit,
3985 health,
3986 },
3987 snapshot.spawned_file_identity.clone(),
3988 ))
3989 }
3990
3991 #[cfg(test)]
3992 pub(crate) fn hold_snapshot_for_test(
3993 &self,
3994 acquired: std::sync::mpsc::Sender<()>,
3995 hold: Duration,
3996 ) -> std::thread::JoinHandle<()> {
3997 let snapshot = Arc::clone(&self.inner.snapshot);
3998 std::thread::spawn(move || {
3999 let _guard = snapshot.lock().expect("test snapshot lock is not poisoned");
4000 acquired
4001 .send(())
4002 .expect("test receiver waits for snapshot lock");
4003 std::thread::sleep(hold);
4004 })
4005 }
4006
4007 pub(crate) async fn status_and_running_image_agreement(
4013 &self,
4014 ) -> Result<(ModuleStatus, subc_control::RunningImageAgreement), SuperviseError> {
4015 let (status, identity) = self.status_with_snapshot_lock(&self.inner.snapshot, None)?;
4016 let image = self
4017 .inner
4018 .provenance_probe
4019 .observe(
4020 status.pid,
4021 status.spawned_from.as_deref(),
4022 identity,
4023 status.process_start_time,
4024 )
4025 .await;
4026 Ok((status, image))
4027 }
4028
4029 pub(crate) async fn running_image_agreement(&self) -> subc_control::RunningImageAgreement {
4030 let snapshot = match lock_snapshot(&self.inner.snapshot) {
4031 Ok(snapshot) => snapshot.clone(),
4032 Err(_) => {
4033 return subc_control::RunningImageAgreement::Unavailable {
4034 reason: subc_control::RunningImageUnavailableReason::NotRunning,
4035 };
4036 }
4037 };
4038 self.inner
4039 .provenance_probe
4040 .observe(
4041 snapshot.reported_pid(),
4042 snapshot.spawned_from.as_deref(),
4043 snapshot.spawned_file_identity.clone(),
4044 snapshot.process_start_time,
4045 )
4046 .await
4047 }
4048
4049 pub(crate) fn child_resource_usage(&self) -> subc_control::ChildResourceUsage {
4052 #[cfg(windows)]
4053 {
4054 let image = lock_snapshot(&self.inner.snapshot)
4055 .ok()
4056 .and_then(|snapshot| {
4057 snapshot
4058 .spawned_file_identity
4059 .as_ref()
4060 .and_then(|identity| identity.windows_image.clone())
4061 });
4062 match image {
4063 Some(image) => {
4064 crate::child_resources::read_windows_handle(image.process(), image.start_time())
4065 }
4066 None => subc_control::ChildResourceUsage::Unavailable {
4067 reason: subc_control::ChildResourceUnavailableReason::NotRunning,
4068 },
4069 }
4070 }
4071 #[cfg(not(windows))]
4072 {
4073 let (pid, start_time) = match lock_snapshot(&self.inner.snapshot) {
4074 Ok(snapshot) => (snapshot.reported_pid(), snapshot.process_start_time),
4075 Err(_) => {
4076 return subc_control::ChildResourceUsage::Unavailable {
4077 reason: subc_control::ChildResourceUnavailableReason::Unreadable,
4078 }
4079 }
4080 };
4081 crate::child_resources::read(pid, start_time)
4082 }
4083 }
4084
4085 #[cfg(all(test, any(target_os = "linux", windows)))]
4086 pub(crate) fn subtree_target(&self) -> crate::subtree_resources::Target {
4087 self.inner
4088 .snapshot
4089 .try_lock()
4090 .map(|snapshot| snapshot.subtree_target.clone())
4091 .unwrap_or(crate::subtree_resources::Target::Unavailable("read_failed"))
4092 }
4093
4094 #[cfg(test)]
4095 pub(crate) fn set_subtree_target_for_test(&self, target: crate::subtree_resources::Target) {
4096 self.inner.snapshot.lock().unwrap().subtree_target = target;
4097 }
4098
4099 pub(crate) fn will_recover_after_connection_loss(&self) -> Result<bool, SuperviseError> {
4100 let mut snapshot = lock_snapshot(&self.inner.snapshot)?;
4101 Ok(match snapshot.state {
4102 ModuleState::Restarting => true,
4103 ModuleState::Failed | ModuleState::Disabled => false,
4104 _ if snapshot.enabled && snapshot.respawn_pending => true,
4105 _ => daemon_will_restart(&mut snapshot, &self.inner.restart_policy, Instant::now()),
4106 })
4107 }
4108
4109 #[cfg(test)]
4110 pub(crate) fn is_warming(&self) -> Result<bool, SuperviseError> {
4111 self.is_warming_with_snapshot_lock(None)
4112 }
4113
4114 pub(crate) fn is_warming_for_control(
4115 &self,
4116 caller: &'static str,
4117 ) -> Result<bool, SuperviseError> {
4118 self.is_warming_with_snapshot_lock(Some(caller))
4119 }
4120
4121 fn is_warming_with_snapshot_lock(
4122 &self,
4123 caller: Option<&'static str>,
4124 ) -> Result<bool, SuperviseError> {
4125 let snapshot = match caller {
4126 Some(caller) => {
4127 lock_snapshot_for_control(&self.inner.snapshot, &self.inner.module_id, caller)?
4128 }
4129 None => lock_snapshot(&self.inner.snapshot)?,
4130 }
4131 .clone();
4132 Ok(matches!(
4133 snapshot.state,
4134 ModuleState::Starting | ModuleState::Running | ModuleState::Restarting
4135 ))
4136 }
4137
4138 pub async fn drain(&self) -> Result<(), SuperviseError> {
4140 self.stop().await
4141 }
4142
4143 pub(crate) async fn retire_requested(
4144 &self,
4145 requester: Option<Principal>,
4146 ) -> Result<(), SuperviseError> {
4147 match self.state()? {
4148 ModuleState::Stopped => return Ok(()),
4149 ModuleState::Failed
4150 if lock_snapshot(&self.inner.snapshot)?
4151 .failed_retry
4152 .pending
4153 .is_none() =>
4154 {
4155 return Ok(())
4156 }
4157 ModuleState::Failed
4158 | ModuleState::Starting
4159 | ModuleState::Running
4160 | ModuleState::Unresponsive
4161 | ModuleState::Restarting
4162 | ModuleState::Draining
4163 | ModuleState::Disabled => {}
4164 }
4165
4166 let (reply_tx, reply_rx) = oneshot::channel();
4167 self.inner
4168 .commands
4169 .send(SupervisorCommand::Retire {
4170 requester,
4171 reply: reply_tx,
4172 })
4173 .await
4174 .map_err(|_| SuperviseError::CommandClosed {
4175 module_id: self.inner.module_id.clone(),
4176 })?;
4177 reply_rx.await.map_err(|_| SuperviseError::CommandClosed {
4178 module_id: self.inner.module_id.clone(),
4179 })?
4180 }
4181
4182 pub async fn stop(&self) -> Result<(), SuperviseError> {
4183 self.stop_requested(None).await
4184 }
4185
4186 pub(crate) async fn stop_requested(
4187 &self,
4188 requester: Option<Principal>,
4189 ) -> Result<(), SuperviseError> {
4190 match self.state()? {
4191 ModuleState::Stopped => return Ok(()),
4192 ModuleState::Failed
4193 if lock_snapshot(&self.inner.snapshot)?
4194 .failed_retry
4195 .pending
4196 .is_none() =>
4197 {
4198 return Ok(())
4199 }
4200 ModuleState::Failed
4201 | ModuleState::Starting
4202 | ModuleState::Running
4203 | ModuleState::Unresponsive
4204 | ModuleState::Restarting
4205 | ModuleState::Draining
4206 | ModuleState::Disabled => {}
4207 }
4208
4209 let (reply_tx, reply_rx) = oneshot::channel();
4210 self.inner
4211 .commands
4212 .send(SupervisorCommand::Drain {
4213 requester,
4214 reply: reply_tx,
4215 })
4216 .await
4217 .map_err(|_| SuperviseError::CommandClosed {
4218 module_id: self.inner.module_id.clone(),
4219 })?;
4220 reply_rx.await.map_err(|_| SuperviseError::CommandClosed {
4221 module_id: self.inner.module_id.clone(),
4222 })?
4223 }
4224
4225 pub async fn restart(&self, drain_timeout_ms: Option<u64>) -> Result<(), SuperviseError> {
4226 self.restart_requested(drain_timeout_ms, None).await
4227 }
4228
4229 pub(crate) async fn restart_requested(
4230 &self,
4231 drain_timeout_ms: Option<u64>,
4232 requester: Option<Principal>,
4233 ) -> Result<(), SuperviseError> {
4234 let queued_at = Instant::now();
4235 let received_at_generation = lock_snapshot(&self.inner.snapshot)?.spawn_generation;
4236 let (reply_tx, reply_rx) = oneshot::channel();
4237 self.inner
4238 .commands
4239 .send(SupervisorCommand::Restart {
4240 requester,
4241 drain_timeout_ms,
4242 received_at_generation,
4243 queued_at,
4244 reply: reply_tx,
4245 })
4246 .await
4247 .map_err(|_| SuperviseError::CommandClosed {
4248 module_id: self.inner.module_id.clone(),
4249 })?;
4250 reply_rx.await.map_err(|_| SuperviseError::CommandClosed {
4251 module_id: self.inner.module_id.clone(),
4252 })?
4253 }
4254
4255 pub async fn swap(&self, ready_timeout: Option<Duration>) -> Result<(), SuperviseError> {
4260 self.swap_requested(ready_timeout, None).await
4261 }
4262
4263 pub(crate) async fn swap_requested(
4264 &self,
4265 ready_timeout: Option<Duration>,
4266 requester: Option<Principal>,
4267 ) -> Result<(), SuperviseError> {
4268 let (reply_tx, reply_rx) = oneshot::channel();
4269 self.inner
4270 .commands
4271 .send(SupervisorCommand::Swap {
4272 requester,
4273 ready_timeout,
4274 reply: reply_tx,
4275 })
4276 .await
4277 .map_err(|_| SuperviseError::CommandClosed {
4278 module_id: self.inner.module_id.clone(),
4279 })?;
4280 reply_rx.await.map_err(|_| SuperviseError::CommandClosed {
4281 module_id: self.inner.module_id.clone(),
4282 })?
4283 }
4284
4285 pub async fn reload(&self) -> Result<(), SuperviseError> {
4286 self.reload_requested(None).await
4287 }
4288
4289 pub(crate) async fn reload_requested(
4290 &self,
4291 requester: Option<Principal>,
4292 ) -> Result<(), SuperviseError> {
4293 let (reply_tx, reply_rx) = oneshot::channel();
4294 self.inner
4295 .commands
4296 .send(SupervisorCommand::Reload {
4297 requester,
4298 reply: reply_tx,
4299 })
4300 .await
4301 .map_err(|_| SuperviseError::CommandClosed {
4302 module_id: self.inner.module_id.clone(),
4303 })?;
4304 reply_rx.await.map_err(|_| SuperviseError::CommandClosed {
4305 module_id: self.inner.module_id.clone(),
4306 })?
4307 }
4308
4309 pub async fn set_enabled(&self, enabled: bool) -> Result<bool, SuperviseError> {
4310 self.set_enabled_requested(enabled, None).await
4311 }
4312
4313 pub(crate) async fn set_enabled_requested(
4314 &self,
4315 enabled: bool,
4316 requester: Option<Principal>,
4317 ) -> Result<bool, SuperviseError> {
4318 let (reply_tx, reply_rx) = oneshot::channel();
4319 self.inner
4320 .commands
4321 .send(SupervisorCommand::SetEnabled {
4322 requester,
4323 enabled,
4324 reply: reply_tx,
4325 })
4326 .await
4327 .map_err(|_| SuperviseError::CommandClosed {
4328 module_id: self.inner.module_id.clone(),
4329 })?;
4330 reply_rx.await.map_err(|_| SuperviseError::CommandClosed {
4331 module_id: self.inner.module_id.clone(),
4332 })?
4333 }
4334
4335 pub(crate) fn declared_protocol(&self) -> Result<ModuleProtocol, SuperviseError> {
4339 let configured = self
4340 .inner
4341 .configuration
4342 .lock()
4343 .map_err(|_| SuperviseError::StatePoisoned {
4344 module_id: Some(self.inner.module_id.clone()),
4345 })?
4346 .spec
4347 .protocol;
4348 let state = lock_snapshot(&self.inner.snapshot)?;
4349 Ok(state.spawned_protocol.unwrap_or(configured))
4350 }
4351
4352 pub(crate) fn configuration(&self) -> Result<(ModuleSpec, HealthConfig), SuperviseError> {
4353 let configuration =
4354 self.inner
4355 .configuration
4356 .lock()
4357 .map_err(|_| SuperviseError::StatePoisoned {
4358 module_id: Some(self.inner.module_id.clone()),
4359 })?;
4360 Ok((configuration.spec.clone(), configuration.health.clone()))
4361 }
4362
4363 #[cfg(any(test, feature = "test-support"))]
4367 pub async fn update_spec_for_test(&self, spec: ModuleSpec) -> Result<(), SuperviseError> {
4368 let (_, health) = self.configuration()?;
4369 let drain_timeout_ms = u64::try_from(
4370 self.inner
4371 .effective_drain_timeout
4372 .lock()
4373 .unwrap_or_else(|poisoned| poisoned.into_inner())
4374 .as_millis(),
4375 )
4376 .ok();
4377 self.update_configuration(spec, health, drain_timeout_ms)
4378 .await
4379 }
4380
4381 pub(crate) async fn update_configuration(
4382 &self,
4383 spec: ModuleSpec,
4384 health: HealthConfig,
4385 drain_timeout_ms: Option<u64>,
4386 ) -> Result<(), SuperviseError> {
4387 if spec.module_id != self.inner.module_id {
4388 return Err(SuperviseError::InvalidSpec {
4389 reason: "a supervised module's module_id cannot be changed".to_string(),
4390 });
4391 }
4392 validate_spec(&spec)?;
4393 let (reply_tx, reply_rx) = oneshot::channel();
4394 self.inner
4395 .commands
4396 .send(SupervisorCommand::UpdateConfiguration {
4397 spec: spec.clone(),
4398 health: health.clone(),
4399 drain_timeout_ms,
4400 reply: reply_tx,
4401 })
4402 .await
4403 .map_err(|_| SuperviseError::CommandClosed {
4404 module_id: self.inner.module_id.clone(),
4405 })?;
4406 reply_rx.await.map_err(|_| SuperviseError::CommandClosed {
4407 module_id: self.inner.module_id.clone(),
4408 })?;
4409 let mut configuration =
4410 self.inner
4411 .configuration
4412 .lock()
4413 .map_err(|_| SuperviseError::StatePoisoned {
4414 module_id: Some(self.inner.module_id.clone()),
4415 })?;
4416 configuration.spec = spec;
4417 configuration.health = health;
4418 Ok(())
4419 }
4420}
4421
4422impl Drop for SupervisedModuleInner {
4423 fn drop(&mut self) {
4424 let Ok(mut monitor) = self.monitor.lock() else {
4425 return;
4426 };
4427 if let Some(monitor) = monitor.as_ref().filter(|monitor| !monitor.is_finished()) {
4428 let _ = update_snapshot(&self.snapshot, Some(&self.module_id), |state| {
4429 state.state = ModuleState::Stopped;
4430 clear_current_process_facts(state);
4431 });
4432 monitor.abort();
4433 }
4434 let _ = monitor.take();
4435 }
4436}
4437
4438#[derive(Debug)]
4439enum SupervisorCommand {
4440 Drain {
4441 requester: Option<Principal>,
4442 reply: oneshot::Sender<Result<(), SuperviseError>>,
4443 },
4444 Retire {
4445 requester: Option<Principal>,
4446 reply: oneshot::Sender<Result<(), SuperviseError>>,
4447 },
4448 Restart {
4449 requester: Option<Principal>,
4450 drain_timeout_ms: Option<u64>,
4455 received_at_generation: u64,
4459 queued_at: Instant,
4462 reply: oneshot::Sender<Result<(), SuperviseError>>,
4463 },
4464 Reload {
4465 requester: Option<Principal>,
4466 reply: oneshot::Sender<Result<(), SuperviseError>>,
4467 },
4468 SetEnabled {
4469 requester: Option<Principal>,
4470 enabled: bool,
4471 reply: oneshot::Sender<Result<bool, SuperviseError>>,
4472 },
4473 UpdateConfiguration {
4474 spec: ModuleSpec,
4475 health: HealthConfig,
4476 drain_timeout_ms: Option<u64>,
4479 reply: oneshot::Sender<()>,
4480 },
4481 Swap {
4482 requester: Option<Principal>,
4483 ready_timeout: Option<Duration>,
4486 reply: oneshot::Sender<Result<(), SuperviseError>>,
4488 },
4489}
4490
4491#[derive(Debug)]
4492pub enum SuperviseError {
4493 InvalidSpec {
4494 reason: String,
4495 },
4496 Spawn {
4497 program: PathBuf,
4498 source: io::Error,
4499 cgroup_path: Option<PathBuf>,
4500 },
4501 Cgroup {
4502 module_id: String,
4503 source: io::Error,
4504 },
4505 LaunchNonce {
4508 reason: String,
4509 },
4510 Wait {
4511 module_id: String,
4512 source: io::Error,
4513 },
4514 Kill {
4515 module_id: String,
4516 source: io::Error,
4517 },
4518 Forwarding(ForwardingError),
4519 Registry(RegistryError),
4520 ReloadUnavailable {
4521 module_id: String,
4522 reason: String,
4523 },
4524 Disabled {
4529 module_id: String,
4530 },
4531 ReloadFailed {
4532 module_id: String,
4533 reason: String,
4534 },
4535 RegistrationStillActive {
4536 module_id: String,
4537 waited: Duration,
4538 },
4539 StatePoisoned {
4540 module_id: Option<String>,
4541 },
4542 CommandClosed {
4543 module_id: String,
4544 },
4545 SwapInProgress {
4549 module_id: String,
4550 },
4551 SwapRefused {
4553 module_id: String,
4554 reason: SwapRefusal,
4555 },
4556 SwapFailed {
4560 module_id: String,
4561 arm: SwapFailureArm,
4562 detail: String,
4563 candidate_exit: Option<ExitReport>,
4566 },
4567}
4568
4569#[derive(Debug, Clone, Copy, PartialEq, Eq)]
4571pub enum SwapRefusal {
4572 OverlapExclusive,
4574 NotRegistered,
4577 ProtocolNone,
4580 NotConfigured,
4583 AlreadySwapping,
4585}
4586
4587impl SwapRefusal {
4588 pub fn as_str(self) -> &'static str {
4589 match self {
4590 Self::OverlapExclusive => "overlap_exclusive",
4591 Self::NotRegistered => "not_registered",
4592 Self::ProtocolNone => "protocol_none",
4593 Self::NotConfigured => "not_configured",
4594 Self::AlreadySwapping => "already_swapping",
4595 }
4596 }
4597}
4598
4599#[derive(Debug, Clone, Copy, PartialEq, Eq)]
4602pub enum SwapFailureArm {
4603 SpawnFailed,
4605 NeverRegistered,
4607 NeverReady,
4609 CandidateExited,
4611 CandidateUnhealthy,
4613 Interrupted,
4617 CutoverLost,
4622}
4623
4624impl SwapFailureArm {
4625 pub fn as_str(self) -> &'static str {
4626 match self {
4627 Self::SpawnFailed => "spawn_failed",
4628 Self::NeverRegistered => "never_registered",
4629 Self::NeverReady => "never_ready",
4630 Self::CandidateExited => "candidate_exited",
4631 Self::CandidateUnhealthy => "candidate_unhealthy",
4632 Self::Interrupted => "interrupted",
4633 Self::CutoverLost => "cutover_lost",
4634 }
4635 }
4636}
4637
4638impl fmt::Display for SuperviseError {
4639 fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
4640 match self {
4641 Self::InvalidSpec { reason } => write!(f, "invalid module spec: {reason}"),
4642 Self::Spawn {
4643 program,
4644 source,
4645 cgroup_path: Some(cgroup_path),
4646 } => write!(
4647 f,
4648 "failed to place module in cgroup '{}' while spawning '{}': {source}",
4649 cgroup_path.display(),
4650 program.display()
4651 ),
4652 Self::Spawn {
4653 program,
4654 source,
4655 cgroup_path: None,
4656 } => write!(
4657 f,
4658 "failed to spawn module '{}': {source}",
4659 program.display()
4660 ),
4661 Self::Cgroup { module_id, source } => {
4662 write!(
4663 f,
4664 "failed to prepare cgroup for module '{module_id}': {source}"
4665 )
4666 }
4667 Self::LaunchNonce { reason } => {
4668 write!(
4669 f,
4670 "failed to generate reserved-module launch nonce: {reason}"
4671 )
4672 }
4673 Self::Wait { module_id, source } => {
4674 write!(f, "failed to wait for module '{module_id}': {source}")
4675 }
4676 Self::Kill { module_id, source } => {
4677 write!(f, "failed to kill module '{module_id}': {source}")
4678 }
4679 Self::Forwarding(err) => write!(f, "forwarding error: {err}"),
4680 Self::Registry(err) => write!(f, "registry error: {err}"),
4681 Self::ReloadUnavailable { module_id, reason } => {
4682 write!(f, "reload unavailable for module '{module_id}': {reason}")
4683 }
4684 Self::Disabled { module_id } => {
4685 write!(
4686 f,
4687 "module '{module_id}' is disabled; enable it before restart or reload"
4688 )
4689 }
4690 Self::ReloadFailed { module_id, reason } => {
4691 write!(f, "reload failed for module '{module_id}': {reason}")
4692 }
4693 Self::RegistrationStillActive { module_id, waited } => write!(
4694 f,
4695 "module '{module_id}' registration remained active after waiting {waited:?}"
4696 ),
4697 Self::StatePoisoned { module_id } => match module_id {
4698 Some(module_id) => {
4699 write!(f, "supervisor state for module '{module_id}' was poisoned")
4700 }
4701 None => write!(f, "supervisor state was poisoned"),
4702 },
4703 Self::CommandClosed { module_id } => {
4704 write!(
4705 f,
4706 "supervisor command channel for module '{module_id}' is closed"
4707 )
4708 }
4709 Self::SwapInProgress { module_id } => write!(
4710 f,
4711 "module '{module_id}' is being swapped; retry once the swap has cut over or failed, or stop the module to abort the swap"
4712 ),
4713 Self::SwapRefused { module_id, reason } => match reason {
4714 SwapRefusal::OverlapExclusive => write!(
4715 f,
4716 "module '{module_id}' is declared overlap: \"exclusive\" (the default): two processes of it must not run at once, so it cannot be swapped; use a plain restart, or declare overlap: \"safe\" in its config if it really tolerates a second process"
4717 ),
4718 SwapRefusal::NotRegistered => write!(
4719 f,
4720 "module '{module_id}' is not registered, so there is no serving process to keep while a replacement warms; use a plain restart"
4721 ),
4722 SwapRefusal::ProtocolNone => write!(
4723 f,
4724 "module '{module_id}' is protocol: \"none\" and never registers, so a swap could never see its replacement become ready; use a plain restart"
4725 ),
4726 SwapRefusal::NotConfigured => write!(
4727 f,
4728 "module '{module_id}' cannot be swapped: the supervisor was built without the forwarding table or shared handle a swap needs"
4729 ),
4730 SwapRefusal::AlreadySwapping => {
4731 write!(f, "module '{module_id}' is already being swapped")
4732 }
4733 },
4734 Self::SwapFailed {
4735 module_id,
4736 arm,
4737 detail,
4738 ..
4739 } => write!(
4740 f,
4741 "swap of module '{module_id}' failed ({}): {detail}; the running process was left serving",
4742 arm.as_str()
4743 ),
4744 }
4745 }
4746}
4747
4748impl Error for SuperviseError {
4749 fn source(&self) -> Option<&(dyn Error + 'static)> {
4750 match self {
4751 Self::Spawn { source, .. }
4752 | Self::Cgroup { source, .. }
4753 | Self::Wait { source, .. }
4754 | Self::Kill { source, .. } => Some(source),
4755 Self::Forwarding(err) => Some(err),
4756 Self::Registry(err) => Some(err),
4757 Self::LaunchNonce { .. }
4758 | Self::InvalidSpec { .. }
4759 | Self::ReloadUnavailable { .. }
4760 | Self::Disabled { .. }
4761 | Self::ReloadFailed { .. }
4762 | Self::RegistrationStillActive { .. }
4763 | Self::StatePoisoned { .. }
4764 | Self::CommandClosed { .. }
4765 | Self::SwapInProgress { .. }
4766 | Self::SwapRefused { .. }
4767 | Self::SwapFailed { .. } => None,
4768 }
4769 }
4770}
4771
4772pub(crate) fn validate_spec(spec: &ModuleSpec) -> Result<(), SuperviseError> {
4773 if spec.carry_containment && spec.overlap == ModuleOverlap::Safe {
4774 return Err(SuperviseError::InvalidSpec {
4775 reason: "carry_containment cannot be combined with overlap: safe".to_owned(),
4776 });
4777 }
4778 if spec.module_id.trim().is_empty() {
4779 return Err(SuperviseError::InvalidSpec {
4780 reason: "module_id must not be empty".to_string(),
4781 });
4782 }
4783
4784 Ok(())
4785}
4786
4787#[derive(Debug, Default)]
4788struct HealthProbeRuntime {
4789 spawn_generation: Option<u64>,
4790 configured_health: Option<HealthConfig>,
4791 registered_connection: Option<crate::ConnectionId>,
4792 advertised: bool,
4793 next_probe_at: Option<Instant>,
4794 probe_index: u64,
4795}
4796
4797fn running_protocol(spec: &ModuleSpec, snapshot: &SharedSnapshot) -> ModuleProtocol {
4798 lock_snapshot(snapshot)
4799 .ok()
4800 .and_then(|state| state.spawned_protocol)
4801 .unwrap_or(spec.protocol)
4802}
4803
4804impl HealthProbeRuntime {
4805 fn refresh_registration(
4806 &mut self,
4807 spec: &ModuleSpec,
4808 runtime: &SupervisorRuntimeConfig,
4809 registry: &Registry,
4810 snapshot: &SharedSnapshot,
4811 ) {
4812 let generation = lock_snapshot(snapshot)
4813 .ok()
4814 .map(|state| state.spawn_generation);
4815 if self.spawn_generation != generation {
4816 *self = Self {
4820 spawn_generation: generation,
4821 ..Self::default()
4822 };
4823 }
4824 if self.configured_health.as_ref() != Some(&runtime.health) {
4825 self.configured_health = Some(runtime.health.clone());
4826 self.next_probe_at = None;
4827 self.registered_connection = None;
4828 self.probe_index = 0;
4829 }
4830 if running_protocol(spec, snapshot) == ModuleProtocol::None {
4834 self.registered_connection = None;
4835 self.advertised = runtime.health.http.is_some();
4836 if !self.advertised {
4837 self.next_probe_at = None;
4838 let _ = update_snapshot(snapshot, Some(&spec.module_id), |state| {
4839 let unknown = ModuleHealthStatus::default();
4840 if state.health != unknown {
4841 state.health = unknown;
4842 state.last_probe_evidence = None;
4843 }
4844 });
4845 } else if self.next_probe_at.is_none() {
4846 self.next_probe_at = Some(
4847 Instant::now()
4848 + jittered_health_delay(&spec.module_id, 0, runtime.health.cadence),
4849 );
4850 }
4851 return;
4852 }
4853
4854 let registration = match registry.get_module(&spec.module_id) {
4855 Ok(registration) => registration,
4856 Err(err) => {
4857 warn!(module_id = %spec.module_id, error = %err, "health prober could not read registry");
4858 self.advertised = false;
4859 self.next_probe_at = None;
4860 return;
4861 }
4862 };
4863
4864 let Some(registration) = registration else {
4865 self.registered_connection = None;
4866 self.advertised = false;
4867 self.next_probe_at = None;
4868 return;
4869 };
4870
4871 let advertised = registration
4872 .control_ops
4873 .iter()
4874 .any(|op| op == MODULE_CONTROL_OP_HEALTH_CHECK);
4875 if !advertised {
4876 self.registered_connection = Some(registration.connection_id);
4877 self.advertised = false;
4878 self.next_probe_at = None;
4879 let _ = update_snapshot(snapshot, Some(&spec.module_id), |state| {
4880 if state.health.status != SupervisorHealthStatus::Unknown
4881 || state.health.consecutive_failures != 0
4882 || state.health.last_probe_ms.is_some()
4883 || state.health.detail.is_some()
4884 || state.health.metrics.is_some()
4885 {
4886 state.health.status = SupervisorHealthStatus::Unknown;
4887 state.last_probe_evidence = None;
4888 state.health.consecutive_failures = 0;
4889 state.health.last_probe_ms = None;
4890 state.health.detail = None;
4891 state.health.metrics = None;
4892 }
4893 });
4894 return;
4895 }
4896
4897 let reregistered = self.registered_connection != Some(registration.connection_id);
4898 self.registered_connection = Some(registration.connection_id);
4899 self.advertised = true;
4900 if reregistered || self.next_probe_at.is_none() {
4901 self.probe_index = 0;
4902 self.next_probe_at = Some(
4903 Instant::now() + jittered_health_delay(&spec.module_id, 0, runtime.health.cadence),
4904 );
4905 let _ = update_snapshot(snapshot, Some(&spec.module_id), |state| {
4906 state.health.status = SupervisorHealthStatus::Unknown;
4907 state.last_probe_evidence = None;
4908 state.health.consecutive_failures = 0;
4909 state.health.detail = None;
4910 state.health.metrics = None;
4911 });
4912 }
4913 }
4914
4915 fn wake_at(&self) -> Option<Instant> {
4916 if !self.advertised {
4917 return None;
4918 }
4919 self.next_probe_at
4920 }
4921
4922 fn due(&self) -> bool {
4923 self.advertised
4924 && self
4925 .next_probe_at
4926 .is_some_and(|next| Instant::now() >= next)
4927 }
4928
4929 fn schedule_next(&mut self, spec: &ModuleSpec, cadence: Duration) {
4930 self.probe_index = self.probe_index.wrapping_add(1);
4931 self.next_probe_at = Some(
4932 Instant::now() + jittered_health_delay(&spec.module_id, self.probe_index, cadence),
4933 );
4934 }
4935}
4936
4937#[derive(Debug, Clone, Copy, PartialEq, Eq)]
4941enum HealthProbeEvidence {
4942 LaneDead,
4944 NoAnswer,
4946 BadAnswer,
4948 Misconfigured,
4950}
4951
4952#[derive(Debug)]
4953struct HealthProbeError {
4954 evidence: HealthProbeEvidence,
4955 message: String,
4956}
4957
4958impl HealthProbeError {
4959 fn lane_dead(message: impl Into<String>) -> Self {
4960 Self::with(HealthProbeEvidence::LaneDead, message)
4961 }
4962
4963 fn no_answer(message: impl Into<String>) -> Self {
4964 Self::with(HealthProbeEvidence::NoAnswer, message)
4965 }
4966
4967 fn bad_answer(message: impl Into<String>) -> Self {
4968 Self::with(HealthProbeEvidence::BadAnswer, message)
4969 }
4970
4971 fn misconfigured(message: impl Into<String>) -> Self {
4972 Self::with(HealthProbeEvidence::Misconfigured, message)
4973 }
4974
4975 fn with(evidence: HealthProbeEvidence, message: impl Into<String>) -> Self {
4976 Self {
4977 evidence,
4978 message: message.into(),
4979 }
4980 }
4981
4982 #[allow(dead_code)]
4991 fn is_proof_of_death(&self) -> bool {
4992 matches!(self.evidence, HealthProbeEvidence::LaneDead)
4993 }
4994
4995 fn label(&self) -> &'static str {
5003 match self.evidence {
5004 HealthProbeEvidence::LaneDead => "lane-dead",
5005 HealthProbeEvidence::NoAnswer => "no-answer",
5006 HealthProbeEvidence::BadAnswer => "bad-answer",
5007 HealthProbeEvidence::Misconfigured => "daemon-misconfigured",
5008 }
5009 }
5010}
5011
5012impl fmt::Display for HealthProbeError {
5013 fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
5014 f.write_str(&self.message)
5015 }
5016}
5017
5018async fn run_health_probe_cycle(
5019 spec: &ModuleSpec,
5020 runtime: &SupervisorRuntimeConfig,
5021 registry: &Registry,
5022 process_liveness: &SupervisorProcessLiveness,
5023 snapshot: &SharedSnapshot,
5024 child: &mut Option<SupervisedChild>,
5025) {
5026 let now_ms = unix_ms_now();
5027 let http = (running_protocol(spec, snapshot) == ModuleProtocol::None)
5028 .then_some(runtime.health.http.as_deref())
5029 .flatten();
5030 let result = match http {
5031 Some(url) => probe_http_health(url, runtime.health.deadline).await,
5032 None => probe_module_health(&spec.module_id, runtime, None).await,
5033 };
5034 match result {
5035 Ok(report) => {
5036 handle_health_report(
5037 spec,
5038 runtime,
5039 registry,
5040 process_liveness,
5041 snapshot,
5042 child,
5043 report,
5044 now_ms,
5045 )
5046 .await;
5047 }
5048 Err(err) => {
5049 if http.is_some() {
5050 let _ = update_snapshot(snapshot, Some(&spec.module_id), |state| {
5051 state.health.status = SupervisorHealthStatus::Failing;
5052 });
5053 }
5054 handle_health_probe_failure(
5055 spec,
5056 runtime,
5057 registry,
5058 process_liveness,
5059 snapshot,
5060 child,
5061 err,
5062 now_ms,
5063 )
5064 .await;
5065 }
5066 }
5067}
5068
5069pub(crate) struct HttpProbeTarget<'a> {
5070 address: std::net::SocketAddr,
5071 localhost: bool,
5072 authority: &'a str,
5073 path: String,
5074}
5075
5076pub(crate) fn parse_http_probe_url(url: &str) -> Result<HttpProbeTarget<'_>, String> {
5079 if url.bytes().any(|b| b <= b' ' || b == 127) || url.contains('#') {
5080 return Err("must not contain whitespace, controls, or a fragment".into());
5081 }
5082 let rest = url
5083 .strip_prefix("http://")
5084 .ok_or("must use plain http://")?;
5085 let split = rest.find(['/', '?']).unwrap_or(rest.len());
5086 let (authority, suffix) = rest.split_at(split);
5087 let (host, port) = if let Some(rest) = authority.strip_prefix("[::1]") {
5088 ("::1", rest)
5089 } else {
5090 let split = authority.find(':').unwrap_or(authority.len());
5091 authority.split_at(split)
5092 };
5093 let ip: std::net::IpAddr = match host {
5094 "127.0.0.1" | "localhost" => std::net::Ipv4Addr::LOCALHOST.into(),
5095 "::1" => std::net::Ipv6Addr::LOCALHOST.into(),
5096 _ => return Err("host must be 127.0.0.1, [::1], or localhost".into()),
5097 };
5098 let port = if port.is_empty() {
5099 80
5100 } else {
5101 port.strip_prefix(':')
5102 .and_then(|p| p.parse::<u16>().ok())
5103 .filter(|p| *p > 0)
5104 .ok_or("must have a valid nonzero TCP port")?
5105 };
5106 let path = if suffix.is_empty() {
5107 "/".into()
5108 } else if suffix.starts_with('?') {
5109 format!("/{suffix}")
5110 } else {
5111 suffix.into()
5112 };
5113 Ok(HttpProbeTarget {
5114 address: std::net::SocketAddr::new(ip, port),
5115 localhost: host == "localhost",
5116 authority,
5117 path,
5118 })
5119}
5120
5121async fn probe_http_health(
5122 url: &str,
5123 deadline: Duration,
5124) -> Result<HealthReport, HealthProbeError> {
5125 use tokio::io::{AsyncReadExt, AsyncWriteExt, BufReader};
5126 let target = parse_http_probe_url(url).map_err(HealthProbeError::misconfigured)?;
5127 let mut response_status = String::new();
5130 let mut body = Vec::new();
5131 let probe = async {
5132 let connection = match tokio::net::TcpStream::connect(target.address).await {
5135 Err(_) if target.localhost => {
5136 tokio::net::TcpStream::connect((
5137 std::net::Ipv6Addr::LOCALHOST,
5138 target.address.port(),
5139 ))
5140 .await
5141 }
5142 result => result,
5143 };
5144 let mut stream = connection.map_err(|error| {
5145 HealthProbeError::no_answer(format!("HTTP connect failed: {error}"))
5146 })?;
5147 stream.write_all(format!("GET {} HTTP/1.1\r\nHost: {}\r\nAccept-Encoding: identity\r\nConnection: close\r\n\r\n", target.path, target.authority).as_bytes()).await
5148 .map_err(|error| HealthProbeError::no_answer(format!("HTTP write failed: {error}")))?;
5149 let mut reader = BufReader::new(stream);
5150 let mut budget = 16 * 1024;
5151 let status = http_line(&mut reader, &mut budget).await?;
5152 let mut words = status.split_ascii_whitespace();
5153 let version = words.next();
5154 let code = words
5155 .next()
5156 .filter(|word| word.len() == 3)
5157 .and_then(|word| word.parse::<u16>().ok());
5158 if !matches!(version, Some("HTTP/1.1" | "HTTP/1.0"))
5159 || !code.is_some_and(|code| (100..600).contains(&code))
5160 {
5161 return Err(HealthProbeError::bad_answer(format!(
5162 "invalid HTTP status: {status}"
5163 )));
5164 }
5165 let code = code.expect("validated status code");
5166 response_status = status.clone();
5167 let mut length = None;
5168 let mut chunked = false;
5169 loop {
5170 let line = http_line(&mut reader, &mut budget).await?;
5171 if line.is_empty() {
5172 break;
5173 }
5174 if let Some((name, value)) = line.split_once(':') {
5175 if name.eq_ignore_ascii_case("content-length") {
5176 length = Some(value.trim().parse::<u64>().map_err(|_| {
5177 HealthProbeError::bad_answer("invalid HTTP Content-Length")
5178 })?);
5179 } else if name.eq_ignore_ascii_case("transfer-encoding") {
5180 chunked = value.trim().eq_ignore_ascii_case("chunked");
5181 }
5182 }
5183 }
5184 if chunked {
5185 while body.len() < 200 {
5186 let line = http_line(&mut reader, &mut budget).await?;
5187 let size = u64::from_str_radix(line.split(';').next().unwrap_or("").trim(), 16)
5188 .map_err(|_| HealthProbeError::bad_answer("invalid HTTP chunk size"))?;
5189 if size == 0 {
5190 break;
5191 }
5192 let count = size.min((200 - body.len()) as u64) as usize;
5193 let start = body.len();
5194 (&mut reader)
5195 .take(count as u64)
5196 .read_to_end(&mut body)
5197 .await
5198 .map_err(|error| {
5199 HealthProbeError::no_answer(format!("HTTP body read failed: {error}"))
5200 })?;
5201 if body.len() - start != count {
5202 return Err(HealthProbeError::bad_answer("truncated HTTP chunk"));
5203 }
5204 if size > count as u64 || body.len() == 200 {
5205 break;
5206 }
5207 if !http_line(&mut reader, &mut budget).await?.is_empty() {
5208 return Err(HealthProbeError::bad_answer("invalid HTTP chunk delimiter"));
5209 }
5210 }
5211 } else {
5212 reader
5213 .take(length.unwrap_or(200).min(200))
5214 .read_to_end(&mut body)
5215 .await
5216 .map_err(|error| {
5217 HealthProbeError::no_answer(format!("HTTP body read failed: {error}"))
5218 })?;
5219 }
5220 if (200..300).contains(&code) {
5221 Ok(HealthReport::ok())
5222 } else {
5223 Err(HealthProbeError::bad_answer(
5224 "HTTP health endpoint returned non-2xx",
5225 ))
5226 }
5227 };
5228 let mut result = timeout(deadline, probe).await.unwrap_or_else(|_| {
5229 Err(HealthProbeError::no_answer(format!(
5230 "HTTP probe timed out after {deadline:?}"
5231 )))
5232 });
5233 if let Err(error) = &mut result {
5234 if !response_status.is_empty() {
5235 error.message = format!(
5236 "{}; {response_status}: {}",
5237 error.message,
5238 String::from_utf8_lossy(&body)
5239 );
5240 }
5241 }
5242 result
5243}
5244
5245async fn http_line(
5246 reader: &mut tokio::io::BufReader<tokio::net::TcpStream>,
5247 remaining: &mut usize,
5248) -> Result<String, HealthProbeError> {
5249 use tokio::io::{AsyncBufReadExt, AsyncReadExt};
5250 let mut line = Vec::new();
5251 (&mut *reader)
5252 .take(*remaining as u64)
5253 .read_until(b'\n', &mut line)
5254 .await
5255 .map_err(|error| {
5256 HealthProbeError::no_answer(format!("HTTP header read failed: {error}"))
5257 })?;
5258 *remaining -= line.len();
5259 if !line.ends_with(b"\r\n") {
5260 return Err(HealthProbeError::bad_answer(
5261 "HTTP headers are incomplete or exceed 16 KiB",
5262 ));
5263 }
5264 line.truncate(line.len() - 2);
5265 String::from_utf8(line).map_err(|_| HealthProbeError::bad_answer("HTTP header is not UTF-8"))
5266}
5267
5268async fn probe_module_health(
5269 module_id: &str,
5270 runtime: &SupervisorRuntimeConfig,
5271 drain_deadline: Option<Instant>,
5272) -> Result<HealthReport, HealthProbeError> {
5273 let Some(forwarding) = runtime.forwarding.as_ref() else {
5274 return Err(HealthProbeError::misconfigured(
5275 "supervisor was not configured with a forwarding table",
5276 ));
5277 };
5278 let probe_started_at = Instant::now();
5279 let mut deadline = probe_started_at + runtime.health.deadline;
5280 if let Some(drain_deadline) = drain_deadline {
5281 deadline = deadline.min(drain_deadline);
5282 }
5283 let pending = if drain_deadline.is_some() {
5284 forwarding.begin_drain_health_probe_rpc_for(
5285 module_id,
5286 MODULE_CONTROL_OP_HEALTH_CHECK,
5287 probe_started_at,
5288 deadline,
5289 )
5290 } else {
5291 forwarding.begin_health_probe_rpc_for(
5292 module_id,
5293 MODULE_CONTROL_OP_HEALTH_CHECK,
5294 probe_started_at,
5295 deadline,
5296 )
5297 }
5298 .map_err(|err| {
5299 HealthProbeError::lane_dead(format!("failed to begin health.check RPC: {err}"))
5302 })?;
5303 await_health_probe(forwarding, pending, deadline, runtime.health.deadline).await
5304}
5305
5306async fn probe_endpoint_health(
5313 endpoint: crate::ModuleEndpointId,
5314 runtime: &SupervisorRuntimeConfig,
5315 deadline_cap: Option<Instant>,
5316) -> Result<HealthReport, HealthProbeError> {
5317 let Some(forwarding) = runtime.forwarding.as_ref() else {
5318 return Err(HealthProbeError::misconfigured(
5319 "supervisor was not configured with a forwarding table",
5320 ));
5321 };
5322 let probe_started_at = Instant::now();
5323 let mut deadline = probe_started_at + runtime.health.deadline;
5324 if let Some(cap) = deadline_cap {
5325 deadline = deadline.min(cap);
5326 }
5327 let pending = forwarding
5328 .begin_endpoint_health_probe_rpc_for(
5329 endpoint,
5330 MODULE_CONTROL_OP_HEALTH_CHECK,
5331 probe_started_at,
5332 deadline,
5333 )
5334 .map_err(|err| {
5335 HealthProbeError::lane_dead(format!("failed to begin health.check RPC: {err}"))
5336 })?;
5337 await_health_probe(forwarding, pending, deadline, runtime.health.deadline).await
5338}
5339
5340async fn await_health_probe(
5342 forwarding: &ForwardingTable,
5343 pending: PendingModuleControlRpc,
5344 deadline: Instant,
5345 probe_budget: Duration,
5346) -> Result<HealthReport, HealthProbeError> {
5347 let PendingModuleControlRpc {
5348 endpoint,
5349 module_sink,
5350 negotiated_ver,
5351 corr,
5352 receiver,
5353 } = pending;
5354 let body = serde_json::to_vec(&ModuleControlRequest::HealthCheck {}).map_err(|err| {
5355 HealthProbeError::misconfigured(format!("failed to encode health.check: {err}"))
5356 })?;
5357 let frame = Frame::build_with_version(
5358 negotiated_ver,
5359 FrameType::Request,
5360 control_flags(),
5361 0,
5362 0,
5363 corr,
5364 body,
5365 )
5366 .map_err(|err| {
5367 HealthProbeError::misconfigured(format!("failed to build health.check frame: {err}"))
5368 })?;
5369
5370 match timeout_at(deadline, module_sink.send(frame)).await {
5376 Ok(Ok(())) => {}
5377 Ok(Err(err)) => {
5378 let _ = forwarding.cancel_module_control_rpc(endpoint, corr);
5379 return Err(HealthProbeError::lane_dead(format!(
5382 "failed to send health.check: {err}"
5383 )));
5384 }
5385 Err(_elapsed) => {
5386 let _ = forwarding.cancel_module_control_rpc(endpoint, corr);
5387 return Err(HealthProbeError::no_answer(
5391 "health.check send timed out before enqueue (module egress full)",
5392 ));
5393 }
5394 }
5395
5396 match timeout_at(deadline, receiver).await {
5397 Ok(Ok(ModuleControlRpcOutcome::Response(response))) => {
5401 response.health_report().ok_or_else(|| {
5402 HealthProbeError::bad_answer("health.check RPC returned a non-health response")
5403 })
5404 }
5405 Ok(Ok(ModuleControlRpcOutcome::Rejected(body))) => Err(HealthProbeError::bad_answer(
5406 format!("health.check rejected: {}", body.message),
5407 )),
5408 Ok(Ok(ModuleControlRpcOutcome::ModuleGone(message))) => {
5409 Err(HealthProbeError::lane_dead(message))
5410 }
5411 Ok(Ok(ModuleControlRpcOutcome::MalformedResponse(message))) => {
5412 Err(HealthProbeError::bad_answer(message))
5413 }
5414 Ok(Ok(ModuleControlRpcOutcome::UnexpectedOp { expected, actual })) => {
5415 Err(HealthProbeError::bad_answer(format!(
5416 "expected module-control op '{expected}', got '{actual}'"
5417 )))
5418 }
5419 Ok(Ok(ModuleControlRpcOutcome::DeadlineElapsed)) => Err(HealthProbeError::bad_answer(
5423 "module answered health.check after its daemon deadline",
5424 )),
5425 Ok(Err(_)) => Err(HealthProbeError::misconfigured(
5426 "health.check waiter was canceled before the module responded",
5427 )),
5428 Err(_) => {
5429 let _ = forwarding.tombstone_health_probe_rpc(endpoint, corr);
5430 Err(HealthProbeError::no_answer(format!(
5431 "module did not answer health.check within {probe_budget:?}"
5432 )))
5433 }
5434 }
5435}
5436
5437#[allow(clippy::too_many_arguments)]
5438async fn handle_health_report(
5439 spec: &ModuleSpec,
5440 runtime: &SupervisorRuntimeConfig,
5441 registry: &Registry,
5442 process_liveness: &SupervisorProcessLiveness,
5443 snapshot: &SharedSnapshot,
5444 child: &mut Option<SupervisedChild>,
5445 report: HealthReport,
5446 now_ms: u64,
5447) {
5448 let status = supervisor_health_status(report.status);
5449 let detail = report.detail.clone();
5450 let metrics = truncate_health_metrics(report.metrics);
5451 let _ = update_snapshot(snapshot, Some(&spec.module_id), |state| {
5452 state.health.status = status;
5453 state.last_probe_evidence = None;
5454 state.health.last_probe_ms = Some(now_ms);
5455 state.health.detail = detail.clone();
5456 state.health.metrics = metrics.clone();
5457 state.health.consecutive_failures = 0;
5458 state.failed_retry.observe_health(
5459 report.status == HealthStatus::Ok,
5460 runtime.restart_policy.window,
5461 );
5462 });
5463
5464 let action = match report.status {
5465 HealthStatus::Ok => return,
5466 HealthStatus::Degraded => runtime.health.on_degraded,
5467 HealthStatus::Failing => runtime.health.on_failing,
5468 };
5469 apply_l3_health_action(
5470 spec,
5471 runtime,
5472 registry,
5473 process_liveness,
5474 snapshot,
5475 child,
5476 status,
5477 detail.as_deref(),
5478 action,
5479 now_ms,
5480 )
5481 .await;
5482}
5483
5484#[allow(clippy::too_many_arguments)]
5485async fn handle_health_probe_failure(
5486 spec: &ModuleSpec,
5487 runtime: &SupervisorRuntimeConfig,
5488 registry: &Registry,
5489 process_liveness: &SupervisorProcessLiveness,
5490 snapshot: &SharedSnapshot,
5491 child: &mut Option<SupervisedChild>,
5492 err: HealthProbeError,
5493 now_ms: u64,
5494) {
5495 let threshold = runtime.health.failure_threshold.max(1);
5496 let held = if err.evidence == HealthProbeEvidence::NoAnswer {
5497 runtime.host_health.record_late_module(&spec.module_id)
5498 } else {
5499 false
5500 };
5501 let mut failures = 0;
5502 let detail = if held {
5507 format!("[{}] {err}; host overloaded", err.label())
5508 } else {
5509 format!("[{}] {err}", err.label())
5510 };
5511 let _ = update_snapshot(snapshot, Some(&spec.module_id), |state| {
5512 state.last_probe_evidence = Some(err.evidence);
5513 state
5514 .failed_retry
5515 .observe_health(false, runtime.restart_policy.window);
5516 if held {
5519 state.health.status = SupervisorHealthStatus::Degraded;
5520 } else if state.spawned_protocol.unwrap_or(spec.protocol) == ModuleProtocol::Subc {
5521 state.health.status = SupervisorHealthStatus::Unknown;
5522 }
5523 state.health.last_probe_ms = Some(now_ms);
5524 state.health.consecutive_failures = if held {
5527 0
5528 } else {
5529 state.health.consecutive_failures.saturating_add(1)
5530 };
5531 state.health.detail = Some(detail.clone());
5532 state.health.metrics = None;
5533 failures = state.health.consecutive_failures;
5534 });
5535
5536 if held || failures < threshold {
5537 warn!(
5538 module_id = %spec.module_id,
5539 consecutive_failures = failures,
5540 threshold,
5541 evidence = err.label(),
5542 detail = %detail,
5543 "health.check probe failed"
5544 );
5545 return;
5546 }
5547
5548 let _ = update_snapshot(snapshot, Some(&spec.module_id), |state| {
5549 state.state = ModuleState::Unresponsive;
5550 state.health.status = SupervisorHealthStatus::Unresponsive;
5551 });
5552 if runtime.health.critical {
5556 error!(
5557 module_id = %spec.module_id,
5558 status = "unresponsive",
5559 evidence = err.label(),
5560 detail = %detail,
5561 "critical module health alert"
5562 );
5563 } else {
5564 warn!(
5565 module_id = %spec.module_id,
5566 status = "unresponsive",
5567 evidence = err.label(),
5568 detail = %detail,
5569 "module health threshold breached"
5570 );
5571 }
5572 if let Err(err) = health_restart_child(
5573 spec,
5574 runtime,
5575 registry,
5576 process_liveness,
5577 snapshot,
5578 child,
5579 SupervisorHealthStatus::Unresponsive,
5580 Some(&detail),
5581 now_ms,
5582 )
5583 .await
5584 {
5585 error!(module_id = %spec.module_id, error = %err, "health-triggered restart failed");
5586 }
5587}
5588
5589#[allow(clippy::too_many_arguments)]
5590async fn apply_l3_health_action(
5591 spec: &ModuleSpec,
5592 runtime: &SupervisorRuntimeConfig,
5593 registry: &Registry,
5594 process_liveness: &SupervisorProcessLiveness,
5595 snapshot: &SharedSnapshot,
5596 child: &mut Option<SupervisedChild>,
5597 status: SupervisorHealthStatus,
5598 detail: Option<&str>,
5599 action: HealthAction,
5600 now_ms: u64,
5601) {
5602 record_health_action(snapshot, &spec.module_id, action.to_string(), now_ms);
5603 match action {
5604 HealthAction::Report => {
5605 info!(
5606 module_id = %spec.module_id,
5607 status = ?status,
5608 detail,
5609 "module reported non-ok health"
5610 );
5611 }
5612 HealthAction::Alert => {
5613 error!(
5614 module_id = %spec.module_id,
5615 status = ?status,
5616 detail,
5617 "module health alert"
5618 );
5619 }
5620 HealthAction::Restart => {
5621 if let Err(err) = health_restart_child(
5622 spec,
5623 runtime,
5624 registry,
5625 process_liveness,
5626 snapshot,
5627 child,
5628 status,
5629 detail,
5630 now_ms,
5631 )
5632 .await
5633 {
5634 error!(module_id = %spec.module_id, error = %err, "health-triggered restart failed");
5635 }
5636 }
5637 }
5638}
5639
5640#[allow(clippy::too_many_arguments)]
5641async fn health_restart_child(
5642 spec: &ModuleSpec,
5643 runtime: &SupervisorRuntimeConfig,
5644 registry: &Registry,
5645 process_liveness: &SupervisorProcessLiveness,
5646 snapshot: &SharedSnapshot,
5647 child: &mut Option<SupervisedChild>,
5648 status: SupervisorHealthStatus,
5649 detail: Option<&str>,
5650 now_ms: u64,
5651) -> Result<(), SuperviseError> {
5652 let (enabled, schedule) = {
5653 let mut state = lock_snapshot(snapshot)?;
5654 let enabled = state.enabled;
5655 let schedule = if enabled {
5656 state.next_crash_restart(&runtime.restart_policy, Instant::now())
5657 } else {
5658 None
5659 };
5660 (enabled, schedule)
5661 };
5662
5663 if !enabled {
5664 return Err(SuperviseError::Disabled {
5665 module_id: spec.module_id.clone(),
5666 });
5667 }
5668
5669 if schedule.is_none() {
5670 record_health_action(snapshot, &spec.module_id, "failed".to_string(), now_ms);
5671 error!(
5672 module_id = %spec.module_id,
5673 status = ?status,
5674 detail,
5675 max_restarts = runtime.restart_policy.max_restarts,
5676 window_secs = runtime.restart_policy.window.as_secs(),
5677 reason = %runtime.restart_policy.budget_exhausted_detail(),
5678 "health restart budget exhausted; marking module failed"
5679 );
5680 let stop_notice = begin_forwarding_drain_if_configured(
5681 spec,
5682 runtime,
5683 registry,
5684 snapshot,
5685 Some(true),
5686 RouteCloseReason::Disable,
5687 )
5688 .await?;
5689 update_snapshot(snapshot, Some(&spec.module_id), |state| {
5690 state.drain_disposition_detail = Some(runtime.restart_policy.budget_exhausted_detail());
5691 })?;
5692 let result = drain_optional_child(
5693 &spec.module_id,
5694 spec.protocol,
5695 stop_notice,
5696 registry,
5697 runtime.forwarding.as_deref(),
5698 snapshot,
5699 &runtime.terminal_ring,
5700 &runtime.spawn_events,
5701 child,
5702 runtime.drain_timeout,
5703 ModuleState::Failed,
5704 Some(true),
5705 &runtime.failure_context(true),
5706 )
5707 .await;
5708 update_snapshot(snapshot, Some(&spec.module_id), |state| {
5709 if state.state == ModuleState::Failed && !runtime.child_roster.is_closed() {
5710 state.failed_retry.schedule(&spec.module_id);
5711 }
5712 })?;
5713 result?;
5714 process_liveness.untrack_if_current(&spec.module_id, snapshot);
5715 return Ok(());
5716 }
5717
5718 let schedule = schedule.expect("a health restart must have a crash-restart schedule");
5719 let mut restart_count = 0;
5720 update_snapshot(snapshot, Some(&spec.module_id), |state| {
5721 restart_count = state.crash_restarts.len();
5722 state.state = ModuleState::Unresponsive;
5723 state.health.status = status;
5724 state.health.last_action = Some(HealthAction::Restart.to_string());
5725 state.health.last_action_ms = Some(now_ms);
5726 })?;
5727 warn!(
5728 module_id = %spec.module_id,
5729 status = ?status,
5730 detail,
5731 restart_count,
5732 restart_in_window = schedule.restart_in_window,
5733 delay_ms = schedule.delay.as_millis() as u64,
5734 "health-triggered module restart"
5735 );
5736
5737 let stop_notice = begin_forwarding_drain_if_configured(
5738 spec,
5739 runtime,
5740 registry,
5741 snapshot,
5742 Some(true),
5743 RouteCloseReason::Restart,
5744 )
5745 .await?;
5746 drain_optional_child(
5747 &spec.module_id,
5748 spec.protocol,
5749 stop_notice,
5750 registry,
5751 runtime.forwarding.as_deref(),
5752 snapshot,
5753 &runtime.terminal_ring,
5754 &runtime.spawn_events,
5755 child,
5756 runtime.drain_timeout,
5757 ModuleState::Restarting,
5758 Some(true),
5759 &runtime.failure_context(true),
5760 )
5761 .await?;
5762 schedule_respawn(
5763 runtime,
5764 snapshot,
5765 &spec.module_id,
5766 schedule.delay,
5767 RespawnKind::Spawn,
5768 true,
5769 )
5770}
5771
5772fn cancel_deferred_reload(runtime: &SupervisorRuntimeConfig, module_id: &str, reason: &str) {
5773 if let Some(reply) = runtime
5774 .deferred_reload_reply
5775 .lock()
5776 .unwrap_or_else(|p| p.into_inner())
5777 .take()
5778 {
5779 let _ = reply.send(Err(SuperviseError::ReloadFailed {
5780 module_id: module_id.to_string(),
5781 reason: reason.to_string(),
5782 }));
5783 }
5784}
5785
5786fn schedule_respawn(
5787 runtime: &SupervisorRuntimeConfig,
5788 snapshot: &SharedSnapshot,
5789 module_id: &str,
5790 delay: Duration,
5791 kind: RespawnKind,
5792 daemon_initiated: bool,
5793) -> Result<(), SuperviseError> {
5794 cancel_deferred_reload(runtime, module_id, "respawn superseded by another restart");
5795 update_snapshot(snapshot, Some(module_id), |state| {
5796 state.respawn_pending = true
5797 })?;
5798 *runtime
5799 .scheduled_respawn
5800 .lock()
5801 .unwrap_or_else(|p| p.into_inner()) = Some(PendingRespawn {
5802 deadline: Instant::now() + delay,
5803 kind,
5804 daemon_initiated,
5805 });
5806 Ok(())
5807}
5808
5809fn record_health_action(snapshot: &SharedSnapshot, module_id: &str, action: String, now_ms: u64) {
5810 let _ = update_snapshot(snapshot, Some(module_id), |state| {
5811 state.health.last_action = Some(action);
5812 state.health.last_action_ms = Some(now_ms);
5813 });
5814}
5815
5816fn supervisor_health_status(status: HealthStatus) -> SupervisorHealthStatus {
5817 match status {
5818 HealthStatus::Ok => SupervisorHealthStatus::Ok,
5819 HealthStatus::Degraded => SupervisorHealthStatus::Degraded,
5820 HealthStatus::Failing => SupervisorHealthStatus::Failing,
5821 }
5822}
5823
5824fn truncate_health_metrics(metrics: Option<Value>) -> Option<Value> {
5836 let metrics = metrics?;
5837 match serde_json::to_vec(&metrics) {
5838 Ok(encoded) if encoded.len() > MAX_HEALTH_METRICS_BYTES => Some(serde_json::json!({
5839 "truncated": true,
5840 "original_bytes": encoded.len(),
5841 })),
5842 Ok(_) | Err(_) => Some(metrics),
5843 }
5844}
5845
5846fn jittered_health_delay(module_id: &str, probe_index: u64, cadence: Duration) -> Duration {
5852 if cadence.is_zero() {
5853 return Duration::ZERO;
5854 }
5855 let cadence_ms = cadence.as_millis() as u64;
5856 if cadence_ms == 0 {
5872 return cadence;
5873 }
5874 let jitter_span = (cadence_ms / 10).max(1);
5889 let hash = module_id.as_bytes().iter().fold(
5890 probe_index.wrapping_mul(0x9E37_79B9_7F4A_7C15),
5891 |acc, byte| {
5892 acc.wrapping_mul(1099511628211)
5893 .wrapping_add(u64::from(*byte))
5894 },
5895 );
5896 cadence + Duration::from_millis(hash % jitter_span)
5897}
5898
5899#[cfg(test)]
5900mod tests {
5901 use super::*;
5902
5903 #[test]
5904 fn readding_a_module_clears_its_rescan_removal_tombstone() {
5905 let handle = SupervisorHandle::new();
5906 let module_id = "readded-tombstone";
5907 handle.record_rescan_removal(module_id);
5908 assert!(handle.removal_tombstone_age_ms(module_id).is_some());
5909
5910 handle.apply_identity_configuration(&ModuleSpec {
5911 module_id: module_id.to_string(),
5912 program: PathBuf::from("/test/module"),
5913 args: Vec::new(),
5914 env: Vec::new(),
5915 reserved: false,
5916 reserved_prefixes: Vec::new(),
5917 protocol: ModuleProtocol::Subc,
5918 overlap: Default::default(),
5919 carry_containment: false,
5920 });
5921
5922 assert!(
5923 handle.removal_tombstone_age_ms(module_id).is_none(),
5924 "a re-added module must not retain a stale removal tombstone"
5925 );
5926 }
5927
5928 #[derive(Debug, PartialEq, Eq)]
5931 struct OwnerInSpawnWindow {
5932 module_id: String,
5933 configured: bool,
5934 on_roster: bool,
5935 admission_refusal: Option<&'static str>,
5936 }
5937
5938 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
5949 async fn a_new_module_is_configured_before_its_first_process_can_register() {
5950 use crate::scopes::ScopeTable;
5951 use subc_protocol::{error_codes, scope::ScopeSelector, Principal};
5952
5953 let dir = cortexkit_test_support::ScratchDir::new("configured-before-spawn");
5954 let stub = |module_id: &str, program: PathBuf| ModuleSpec {
5955 module_id: module_id.to_string(),
5956 program,
5957 args: Vec::new(),
5958 env: ["XDG_DATA_HOME", "XDG_RUNTIME_DIR", "XDG_CONFIG_HOME"]
5959 .into_iter()
5960 .map(|key| (key.to_string(), dir.path().display().to_string()))
5961 .chain([("FAKE_AFT_NEVER_CONNECT".to_string(), "1".to_string())])
5962 .collect(),
5963 reserved: false,
5964 reserved_prefixes: Vec::new(),
5965 protocol: ModuleProtocol::Subc,
5966 overlap: Default::default(),
5967 carry_containment: false,
5968 };
5969 let live = super::terminal_history_tests::fake_aft_stub_path();
5970 let missing = dir.path().join("definitely-missing-module");
5971
5972 let handle = SupervisorHandle::new();
5973 let mut supervisor =
5974 Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default())
5975 .with_handle(handle.clone());
5976 let observed = Arc::new(Mutex::new(Vec::<OwnerInSpawnWindow>::new()));
5977 let hook_handle = handle.clone();
5978 let hook_observed = Arc::clone(&observed);
5979 supervisor.test_after_first_spawn = AfterFirstSpawnHook(Some(Arc::new(move |module_id| {
5980 let configured = hook_handle.is_configured(module_id);
5983 let selector = ScopeSelector::new(
5984 Principal::Reserved {
5985 module_id: module_id.to_string(),
5986 },
5987 "s".to_string(),
5988 Some(1),
5989 );
5990 let carrier = Principal::Reserved {
5991 module_id: "carrier".to_string(),
5992 };
5993 let admission_refusal = match ScopeTable::new(Vec::<String>::new())
5994 .admit(&carrier, module_id, &selector, configured)
5995 {
5996 Ok(_) => None,
5997 Err(refusal) => Some(refusal.code),
5998 };
5999 hook_observed.lock().unwrap().push(OwnerInSpawnWindow {
6000 module_id: module_id.to_string(),
6001 configured,
6002 on_roster: hook_handle.get(module_id).is_some(),
6003 admission_refusal,
6004 });
6005 })));
6006
6007 let plain = supervisor.spawn(stub("plain", live.clone())).unwrap();
6008 let configured = supervisor
6009 .supervise_configured(stub("configured", live.clone()), true)
6010 .unwrap();
6011 let with_health = supervisor
6012 .supervise_configured_with_health(
6013 stub("with-health", live.clone()),
6014 true,
6015 HealthConfig::default(),
6016 None,
6017 RestartPolicy::default(),
6018 )
6019 .unwrap();
6020 let failed = supervisor
6023 .supervise_configured_with_health(
6024 stub("failed-spawn", missing.clone()),
6025 true,
6026 HealthConfig::default(),
6027 None,
6028 RestartPolicy::default(),
6029 )
6030 .unwrap();
6031 assert!(supervisor.spawn(stub("spawn-error", missing)).is_err());
6034
6035 let expected = [
6036 "plain",
6037 "configured",
6038 "with-health",
6039 "failed-spawn",
6040 "spawn-error",
6041 ]
6042 .into_iter()
6043 .map(|module_id| OwnerInSpawnWindow {
6044 module_id: module_id.to_string(),
6045 configured: true,
6046 on_roster: !matches!(module_id, "plain" | "spawn-error"),
6047 admission_refusal: Some(error_codes::SCOPE_NOT_SYNCED),
6048 })
6049 .collect::<Vec<_>>();
6050 assert_eq!(*observed.lock().unwrap(), expected);
6051
6052 for module_id in ["plain", "configured", "with-health", "failed-spawn"] {
6053 assert!(
6054 handle.get(module_id).is_some(),
6055 "{module_id} is on the roster"
6056 );
6057 assert!(
6058 handle.is_configured(module_id),
6059 "{module_id} stays configured"
6060 );
6061 }
6062 assert!(handle.get("spawn-error").is_none());
6063 assert!(
6064 !handle.is_configured("spawn-error"),
6065 "a plain spawn that failed must not leave its module marked configured"
6066 );
6067
6068 handle.retire("failed-spawn");
6070 assert!(!handle.is_configured("failed-spawn"));
6071
6072 for module in [plain, configured, with_health] {
6073 module.stop().await.unwrap();
6074 }
6075 drop(failed);
6076 }
6077
6078 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
6085 async fn a_configured_module_reads_starting_while_its_first_process_already_runs() {
6086 let dir = cortexkit_test_support::ScratchDir::new("starting-while-spawned");
6087 let spec = ModuleSpec {
6088 module_id: "spawned-not-admitted".to_string(),
6089 program: super::terminal_history_tests::fake_aft_stub_path(),
6090 args: Vec::new(),
6091 env: ["XDG_DATA_HOME", "XDG_RUNTIME_DIR", "XDG_CONFIG_HOME"]
6092 .into_iter()
6093 .map(|key| (key.to_string(), dir.path().display().to_string()))
6094 .chain([("FAKE_AFT_NEVER_CONNECT".to_string(), "1".to_string())])
6095 .collect(),
6096 reserved: false,
6097 reserved_prefixes: Vec::new(),
6098 protocol: ModuleProtocol::Subc,
6099 overlap: Default::default(),
6100 carry_containment: false,
6101 };
6102 let handle = SupervisorHandle::new();
6103 let mut supervisor =
6104 Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default())
6105 .with_handle(handle.clone());
6106 let observed = Arc::new(Mutex::new(None));
6107 let hook_handle = handle.clone();
6108 let hook_observed = Arc::clone(&observed);
6109 supervisor.test_after_first_spawn = AfterFirstSpawnHook(Some(Arc::new(move |module_id| {
6110 let state = hook_handle
6111 .get(module_id)
6112 .expect("a configured module is on the roster before it is spawned")
6113 .state()
6114 .unwrap();
6115 *hook_observed.lock().unwrap() = Some(state);
6116 })));
6117
6118 let module = supervisor.supervise_configured(spec, true).unwrap();
6119 assert_eq!(
6120 *observed.lock().unwrap(),
6121 Some(ModuleState::Starting),
6122 "between the spawn and its admission the supervisor still reports starting"
6123 );
6124 assert_eq!(module.state().unwrap(), ModuleState::Running);
6125 module.stop().await.unwrap();
6126 }
6127
6128 fn stale_process_snapshot(state: ModuleState, enabled: bool) -> SharedSnapshot {
6129 let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::new(state, enabled)));
6130 update_snapshot(&snapshot, Some("stale-process-facts"), |snapshot| {
6131 snapshot.process_alive = true;
6132 snapshot.pid = Some(41);
6133 snapshot.spawned_at_ms = Some(42);
6134 snapshot.spawned_from = Some(PathBuf::from("/spawned/module"));
6135 snapshot.launch_spec_digest = Some(LaunchSpecDigest([0xA5; 32]));
6136 snapshot.spawned_file_identity = Some(SpawnedFileIdentity {
6137 device: 43,
6138 inode: 44,
6139 #[cfg(windows)]
6140 windows_image: None,
6141 });
6142 })
6143 .unwrap();
6144 snapshot
6145 }
6146
6147 fn assert_snapshot_process_facts_cleared(snapshot: &SharedSnapshot) {
6148 let snapshot = lock_snapshot(snapshot).unwrap();
6149 assert!(!snapshot.process_alive);
6150 assert_eq!(snapshot.pid, None);
6151 assert_eq!(snapshot.spawned_at_ms, None);
6152 assert_eq!(snapshot.spawned_from, None);
6153 assert_eq!(snapshot.launch_spec_digest, None);
6154 assert_eq!(snapshot.spawned_file_identity, None);
6155 }
6156
6157 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
6158 async fn failed_enable_spawn_clears_preexisting_current_process_facts() {
6159 let supervisor =
6160 Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default());
6161 let mut runtime = supervisor.runtime_config();
6162 runtime.test_seed_stale_facts_before_enable_spawn = true;
6163 let snapshot = stale_process_snapshot(ModuleState::Disabled, false);
6164 let mut child = None;
6165 let spec = ModuleSpec {
6166 module_id: "failed-enable-clears-facts".to_string(),
6167 program: PathBuf::from("/definitely/missing/failed-enable-module"),
6168 args: Vec::new(),
6169 env: Vec::new(),
6170 reserved: false,
6171 reserved_prefixes: Vec::new(),
6172 protocol: ModuleProtocol::Subc,
6173 overlap: Default::default(),
6174 carry_containment: false,
6175 };
6176
6177 let result = set_child_enabled(
6178 &spec,
6179 &runtime,
6180 &supervisor.registry,
6181 &supervisor.process_liveness,
6182 &snapshot,
6183 &mut child,
6184 true,
6185 &runtime.failure_context(false),
6186 )
6187 .await;
6188
6189 assert!(matches!(result, Err(SuperviseError::Spawn { .. })));
6190 assert_eq!(lock_snapshot(&snapshot).unwrap().state, ModuleState::Failed);
6191 assert_snapshot_process_facts_cleared(&snapshot);
6192 }
6193
6194 #[tokio::test]
6195 async fn start_revives_stranded_restarting_but_not_pending_backoff() {
6196 let supervisor =
6197 Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default());
6198 let runtime = supervisor.runtime_config();
6199 let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::new(
6200 ModuleState::Restarting,
6201 true,
6202 )));
6203 let spec = ModuleSpec {
6204 module_id: "start-stranded-restarting".to_string(),
6205 program: super::terminal_history_tests::fake_aft_stub_path(),
6206 args: Vec::new(),
6207 env: vec![("FAKE_AFT_NEVER_CONNECT".to_string(), "1".to_string())],
6208 reserved: false,
6209 reserved_prefixes: Vec::new(),
6210 protocol: ModuleProtocol::None,
6211 overlap: Default::default(),
6212 carry_containment: false,
6213 };
6214 let mut child = None;
6215 lock_snapshot(&snapshot).unwrap().respawn_pending = true;
6216 assert!(!super::set_child_enabled(
6217 &spec,
6218 &runtime,
6219 &Registry::default(),
6220 &supervisor.process_liveness,
6221 &snapshot,
6222 &mut child,
6223 true,
6224 &runtime.failure_context(false),
6225 )
6226 .await
6227 .unwrap());
6228 assert!(child.is_none());
6229 lock_snapshot(&snapshot).unwrap().respawn_pending = false;
6230 assert!(super::set_child_enabled(
6231 &spec,
6232 &runtime,
6233 &Registry::default(),
6234 &supervisor.process_liveness,
6235 &snapshot,
6236 &mut child,
6237 true,
6238 &runtime.failure_context(false),
6239 )
6240 .await
6241 .unwrap());
6242 assert_eq!(
6243 lock_snapshot(&snapshot).unwrap().state,
6244 ModuleState::Running
6245 );
6246 let mut child = child.unwrap();
6247 child.start_kill().unwrap();
6248 child.wait().await.unwrap();
6249 }
6250
6251 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
6252 async fn failed_reload_spawn_clears_current_process_facts() {
6253 let supervisor =
6254 Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default());
6255 let mut runtime = supervisor.runtime_config();
6256 runtime.restart_policy = RestartPolicy::new(0, Duration::ZERO);
6257 let snapshot = stale_process_snapshot(ModuleState::Running, true);
6258 let mut child = None;
6259 let spec = ModuleSpec {
6260 module_id: "failed-reload-clears-facts".to_string(),
6261 program: PathBuf::from("/unused/failed-reload-module"),
6262 args: Vec::new(),
6263 env: Vec::new(),
6264 reserved: false,
6265 reserved_prefixes: Vec::new(),
6266 protocol: ModuleProtocol::Subc,
6267 overlap: Default::default(),
6268 carry_containment: false,
6269 };
6270
6271 let result = handle_reload_spawn_failure(
6272 &spec,
6273 &runtime,
6274 &supervisor.process_liveness,
6275 &snapshot,
6276 &mut child,
6277 "forced reload spawn failure".to_string(),
6278 )
6279 .await;
6280
6281 assert!(matches!(result, Err(SuperviseError::ReloadFailed { .. })));
6282 assert_eq!(lock_snapshot(&snapshot).unwrap().state, ModuleState::Failed);
6283 assert_snapshot_process_facts_cleared(&snapshot);
6284 }
6285
6286 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
6287 async fn dropping_a_module_with_an_active_monitor_clears_current_process_facts() {
6288 let supervisor =
6289 Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default());
6290 let snapshot = stale_process_snapshot(ModuleState::Running, true);
6291 let module = supervisor.supervised_module(
6292 ModuleSpec {
6293 module_id: "drop-clears-facts".to_string(),
6294 program: PathBuf::from("/unused/drop-module"),
6295 args: Vec::new(),
6296 env: Vec::new(),
6297 reserved: false,
6298 reserved_prefixes: Vec::new(),
6299 protocol: ModuleProtocol::Subc,
6300 overlap: Default::default(),
6301 carry_containment: false,
6302 },
6303 supervisor.runtime_config(),
6304 Arc::clone(&snapshot),
6305 None,
6306 );
6307 assert!(!module
6308 .inner
6309 .monitor
6310 .lock()
6311 .unwrap()
6312 .as_ref()
6313 .unwrap()
6314 .is_finished());
6315
6316 drop(module);
6317
6318 assert_eq!(
6319 lock_snapshot(&snapshot).unwrap().state,
6320 ModuleState::Stopped
6321 );
6322 assert_snapshot_process_facts_cleared(&snapshot);
6323 }
6324
6325 #[cfg(unix)]
6326 #[tokio::test]
6327 async fn rescan_preserves_running_protocol_until_respawn() {
6328 let dir = cortexkit_test_support::ScratchDir::new("rescan-protocol");
6329 let initial = ModuleSpec {
6330 module_id: "rescan-protocol".into(),
6331 program: PathBuf::from("/bin/sleep"),
6332 args: vec!["60".into()],
6333 env: ["XDG_DATA_HOME", "XDG_RUNTIME_DIR", "XDG_CONFIG_HOME"]
6334 .into_iter()
6335 .map(|key| (key.into(), dir.path().to_string_lossy().into_owned()))
6336 .collect(),
6337 reserved: false,
6338 reserved_prefixes: vec![],
6339 protocol: ModuleProtocol::None,
6340 overlap: Default::default(),
6341 carry_containment: false,
6342 };
6343 let supervisor =
6344 Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default());
6345 let module = supervisor.spawn(initial.clone()).unwrap();
6346 assert!(module.status().unwrap().live);
6347 let mut next = initial;
6348 next.protocol = ModuleProtocol::Subc;
6349 module
6350 .update_configuration(next.clone(), HealthConfig::default(), None)
6351 .await
6352 .unwrap();
6353 assert!(
6354 module.status().unwrap().live,
6355 "rescan must not require HELLO from the old non-wire process"
6356 );
6357 let runtime = supervisor.runtime_config();
6358 let action = on_child_exit(
6359 &next,
6360 RestartPolicy::default(),
6361 &supervisor.registry,
6362 &module.inner.snapshot,
6363 &runtime.terminal_ring,
6364 &runtime.spawn_events,
6365 &runtime.child_roster,
6366 ExitReport {
6367 kind: ExitKind::Clean,
6368 code: Some(0),
6369 signal: None,
6370 at_ms: unix_ms_now(),
6371 },
6372 true,
6373 )
6374 .await;
6375 assert!(
6376 matches!(action, NextAction::Restart { .. }),
6377 "the old non-wire process's clean exit must restart"
6378 );
6379 module.drain().await.unwrap();
6380 }
6381
6382 #[cfg(unix)]
6383 #[tokio::test]
6384 async fn orphan_identity_matches_path_names_and_shebang_interpreters() {
6385 use std::os::unix::fs::PermissionsExt;
6386 let dir = cortexkit_test_support::ScratchDir::new("spawn-image-identity");
6387 let script = dir.join("module.sh");
6388 std::fs::write(&script, "#!/bin/sh\nwhile :; do sleep 0.1; done\n").unwrap();
6389 std::fs::set_permissions(&script, std::fs::Permissions::from_mode(0o700)).unwrap();
6390 let record_path = dir.join("live-children.json");
6391 let supervisor =
6392 Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default())
6393 .with_live_children_record(&record_path);
6394 for (program, args) in [
6395 (PathBuf::from("sleep"), vec!["60".into()]),
6396 (script, vec![]),
6397 ] {
6398 let spec = ModuleSpec {
6399 module_id: "image-identity".into(),
6400 program: program.clone(),
6401 args,
6402 env: ["XDG_DATA_HOME", "XDG_RUNTIME_DIR", "XDG_CONFIG_HOME"]
6403 .into_iter()
6404 .map(|key| (key.into(), dir.to_string_lossy().into_owned()))
6405 .collect(),
6406 reserved: false,
6407 reserved_prefixes: vec![],
6408 protocol: ModuleProtocol::None,
6409 overlap: Default::default(),
6410 carry_containment: false,
6411 };
6412 let module = supervisor.spawn(spec).unwrap();
6413 #[cfg(target_os = "macos")]
6414 {
6415 let deadline = Instant::now() + Duration::from_secs(5);
6418 while crate::live_children::read_record(&record_path)
6419 .unwrap()
6420 .iter()
6421 .all(|entry| entry.executable.is_none())
6422 {
6423 assert!(Instant::now() < deadline, "module image was not confirmed");
6424 tokio::time::sleep(Duration::from_millis(5)).await;
6425 }
6426 }
6427 let entry = crate::live_children::read_record(&record_path)
6428 .unwrap()
6429 .pop()
6430 .unwrap();
6431 let observed = subc_os::Process::open(entry.pid)
6432 .unwrap()
6433 .unwrap()
6434 .observe()
6435 .unwrap();
6436 let verdict = crate::live_children::identity_verdict(&entry, entry.pid, &observed);
6437 module.drain().await.unwrap();
6438 assert_eq!(
6439 verdict,
6440 crate::live_children::IdentityVerdict::Matches,
6441 "program {program:?}: recorded {entry:?}, observed {observed:?}"
6442 );
6443 }
6444 }
6445
6446 #[cfg(unix)]
6447 fn http_fixture(
6448 dir: &std::path::Path,
6449 url: &str,
6450 threshold: u32,
6451 ) -> crate::daemon_config::ConfiguredModule {
6452 let path = dir.join("subc.jsonc");
6453 std::fs::write(&path, serde_json::json!({"version": 1, "modules": {"http-process": {
6454 "program": "/bin/sleep", "args": ["60"], "protocol": "none",
6455 "env": {"XDG_DATA_HOME":dir,"XDG_RUNTIME_DIR":dir,"XDG_CONFIG_HOME":dir},
6456 "health": {"http":url,"cadence_ms":20,"deadline_ms":200,"failure_threshold":threshold},
6457 "restart": {"backoff_ms":1,"max_backoff_ms":1}, "drain_timeout_ms":10
6458 }}}).to_string()).unwrap();
6459 crate::daemon_config::load(&path)
6460 .unwrap()
6461 .unwrap()
6462 .modules
6463 .pop()
6464 .unwrap()
6465 }
6466
6467 #[cfg(unix)]
6468 async fn wait_http_health(module: &SupervisedModule, status: SupervisorHealthStatus) {
6469 timeout(Duration::from_secs(5), async {
6470 loop {
6471 if module.status().unwrap().health.status == status {
6472 break;
6473 }
6474 sleep(Duration::from_millis(5)).await;
6475 }
6476 })
6477 .await
6478 .unwrap_or_else(|_| {
6479 panic!(
6480 "expected {status:?}, got {:?}",
6481 module.status().unwrap().health
6482 )
6483 });
6484 }
6485
6486 #[cfg(unix)]
6487 #[tokio::test]
6488 async fn http_health_status_flips_ok_failing_ok() {
6489 use tokio::io::{AsyncReadExt, AsyncWriteExt};
6490 let dir = cortexkit_test_support::ScratchDir::new("http-status-flips");
6491 let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
6492 let status = Arc::new(std::sync::atomic::AtomicU16::new(200));
6493 let serving_status = status.clone();
6494 let url = format!("http://{}/healthz", listener.local_addr().unwrap());
6495 let server = tokio::spawn(async move {
6496 loop {
6497 let (mut stream, _) = listener.accept().await.unwrap();
6498 let mut request = [0u8; 2048];
6499 let count = stream.read(&mut request).await.unwrap();
6500 assert!(count > 0, "a probe must send an HTTP request");
6501 let code = serving_status.load(std::sync::atomic::Ordering::SeqCst);
6502 let body = if code == 200 {
6503 "ready"
6504 } else {
6505 "{\"status\":\"unavailable\",\"error\":\"scratch failure\"}"
6506 };
6507 let response = format!("HTTP/1.1 {code} scratch\r\nContent-Length: {}\r\nConnection: close\r\n\r\n{body}", body.len());
6508 let _ = stream.write_all(response.as_bytes()).await;
6509 }
6510 });
6511 let configured = http_fixture(&dir, &url, 1000);
6512 let module =
6513 Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default())
6514 .supervise_configured_with_health(
6515 configured.module_spec(),
6516 true,
6517 configured.health,
6518 configured.drain_timeout_ms,
6519 configured.restart,
6520 )
6521 .unwrap();
6522 wait_http_health(&module, SupervisorHealthStatus::Ok).await;
6523 status.store(503, std::sync::atomic::Ordering::SeqCst);
6524 wait_http_health(&module, SupervisorHealthStatus::Failing).await;
6525 assert!(module
6526 .status()
6527 .unwrap()
6528 .health
6529 .detail
6530 .unwrap()
6531 .contains("scratch failure"));
6532 status.store(200, std::sync::atomic::Ordering::SeqCst);
6533 wait_http_health(&module, SupervisorHealthStatus::Ok).await;
6534 assert_eq!(module.status().unwrap().health.consecutive_failures, 0);
6535 let before = module.status().unwrap();
6536 let (spec, mut health) = module.configuration().unwrap();
6537 health.http = None;
6538 module
6539 .update_configuration(spec.clone(), health.clone(), Some(10))
6540 .await
6541 .unwrap();
6542 wait_http_health(&module, SupervisorHealthStatus::Unknown).await;
6543 health.http = Some(url);
6544 module
6545 .update_configuration(spec, health, Some(10))
6546 .await
6547 .unwrap();
6548 wait_http_health(&module, SupervisorHealthStatus::Ok).await;
6549 assert_eq!(
6550 module.status().unwrap().pid,
6551 before.pid,
6552 "changing a probe must apply live, not restart its process"
6553 );
6554 let (spec, mut health) = module.configuration().unwrap();
6555 health.failure_threshold = 2;
6556 module
6557 .update_configuration(spec, health, Some(10))
6558 .await
6559 .unwrap();
6560 status.store(503, std::sync::atomic::Ordering::SeqCst);
6561 timeout(Duration::from_secs(5), async {
6562 while module.status().unwrap().spawn_generation == before.spawn_generation {
6563 sleep(Duration::from_millis(5)).await;
6564 }
6565 })
6566 .await
6567 .expect("sustained HTTP 503 responses must trigger the health restart policy");
6568 module.drain().await.unwrap();
6569 server.abort();
6570 }
6571
6572 #[cfg(unix)]
6573 #[tokio::test]
6574 async fn http_health_replacement_recovers_after_one_restart() {
6575 use tokio::io::{AsyncReadExt, AsyncWriteExt};
6576 let dir = cortexkit_test_support::ScratchDir::new("http-replacement-recovers");
6577 let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
6578 let url = format!("http://{}/healthz", listener.local_addr().unwrap());
6579 let mut configured = http_fixture(&dir, &url, 2);
6580 configured.restart.max_restarts = 2;
6581 let module =
6582 Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default())
6583 .supervise_configured_with_health(
6584 configured.module_spec(),
6585 true,
6586 configured.health,
6587 configured.drain_timeout_ms,
6588 configured.restart,
6589 )
6590 .unwrap();
6591 let first_generation = module.status().unwrap().spawn_generation;
6592 let observed = module.clone();
6593 let (healthy_tx, mut healthy_rx) = mpsc::channel(8);
6594 let server = tokio::spawn(async move {
6595 let mut generation = first_generation;
6596 let mut probes = 0;
6597 loop {
6598 let (mut stream, _) = listener.accept().await.unwrap();
6599 let mut request = [0; 2048];
6600 assert!(stream.read(&mut request).await.unwrap() > 0);
6601 let current = observed.status().unwrap().spawn_generation;
6602 if current != generation {
6603 generation = current;
6604 probes = 0;
6605 }
6606 probes += 1;
6607 let healthy = generation != first_generation && probes > 1;
6610 let code = if healthy { 200 } else { 503 };
6611 let response = format!(
6612 "HTTP/1.1 {code} scratch\r\nContent-Length: 0\r\nConnection: close\r\n\r\n"
6613 );
6614 let _ = stream.write_all(response.as_bytes()).await;
6615 if healthy {
6616 healthy_tx.send(()).await.unwrap();
6617 }
6618 }
6619 });
6620 timeout(Duration::from_secs(5), async {
6621 let mut healthy_probes = 0;
6622 loop {
6623 if module.state().unwrap() == ModuleState::Failed {
6624 break;
6625 }
6626 while healthy_rx.try_recv().is_ok() {
6627 healthy_probes += 1;
6628 }
6629 if healthy_probes >= 3
6630 && module.status().unwrap().health.status == SupervisorHealthStatus::Ok
6631 {
6632 break;
6633 }
6634 sleep(Duration::from_millis(5)).await;
6635 }
6636 })
6637 .await
6638 .expect("the replacement must settle healthy without exhausting its restart budget");
6639 let status = module.status().unwrap();
6640 module.drain().await.unwrap();
6641 server.abort();
6642 assert_eq!(status.state, ModuleState::Running);
6643 assert_eq!(status.health.status, SupervisorHealthStatus::Ok);
6644 assert_eq!(status.health.consecutive_failures, 0);
6645 assert_eq!(status.lifetime_restarts, 1);
6646 assert_eq!(status.restart_count, 1);
6647 }
6648
6649 #[cfg(unix)]
6650 #[tokio::test]
6651 async fn http_health_sustained_failure_exhausts_restart_budget() {
6652 use std::collections::BTreeMap;
6653 use tokio::io::{AsyncReadExt, AsyncWriteExt};
6654 let dir = cortexkit_test_support::ScratchDir::new("http-health-budget");
6655 let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
6656 let url = format!("http://{}/healthz", listener.local_addr().unwrap());
6657 let mut configured = http_fixture(&dir, &url, 2);
6658 configured.restart.max_restarts = 2;
6659 let module =
6660 Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default())
6661 .supervise_configured_with_health(
6662 configured.module_spec(),
6663 true,
6664 configured.health,
6665 configured.drain_timeout_ms,
6666 configured.restart,
6667 )
6668 .unwrap();
6669 let observed = module.clone();
6670 let probes = Arc::new(Mutex::new(BTreeMap::<u64, u32>::new()));
6671 let served = probes.clone();
6672 let server = tokio::spawn(async move {
6673 loop {
6674 let (mut stream, _) = listener.accept().await.unwrap();
6675 let mut request = [0; 2048];
6676 assert!(stream.read(&mut request).await.unwrap() > 0);
6677 *served
6678 .lock()
6679 .unwrap()
6680 .entry(observed.status().unwrap().spawn_generation)
6681 .or_default() += 1;
6682 let _ = stream
6683 .write_all(b"HTTP/1.1 503 Unavailable\r\nContent-Length: 0\r\nConnection: close\r\n\r\n")
6684 .await;
6685 }
6686 });
6687 timeout(Duration::from_secs(5), async {
6688 while module.state().unwrap() != ModuleState::Failed {
6689 sleep(Duration::from_millis(5)).await;
6690 }
6691 })
6692 .await
6693 .expect("sustained failures must still exhaust the health restart budget");
6694 let status = module.status().unwrap();
6695 server.abort();
6696 assert_eq!(status.restart_count, 2);
6697 assert_eq!(status.lifetime_restarts, 2);
6698 assert!(!status.process_alive);
6699 assert_eq!(
6700 probes.lock().unwrap().values().copied().collect::<Vec<_>>(),
6701 vec![2, 2, 2]
6702 );
6703 }
6704
6705 #[cfg(unix)]
6706 #[tokio::test]
6707 async fn spawn_clears_the_previous_process_health() {
6708 let dir = cortexkit_test_support::ScratchDir::new("spawn-fresh-health");
6709 let mut configured = http_fixture(&dir, "http://127.0.0.1:1/healthz", 2);
6710 configured.health.cadence = Duration::from_secs(3600);
6711 let module =
6712 Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default())
6713 .supervise_configured_with_health(
6714 configured.module_spec(),
6715 false,
6716 configured.health,
6717 configured.drain_timeout_ms,
6718 configured.restart,
6719 )
6720 .unwrap();
6721 update_snapshot(&module.inner.snapshot, Some(module.module_id()), |state| {
6722 state.health.status = SupervisorHealthStatus::Unresponsive;
6723 state.health.consecutive_failures = 3;
6724 state.health.detail = Some("old process refused connections".into());
6725 state.health.metrics = Some(serde_json::json!({"old": true}));
6726 state.health.last_probe_ms = Some(123);
6727 state.health.last_action = Some("restart".into());
6728 state.health.last_action_ms = Some(124);
6729 })
6730 .unwrap();
6731 module.set_enabled(true).await.unwrap();
6732 let health = module.status().unwrap().health;
6733 module.drain().await.unwrap();
6734 assert_eq!(health.consecutive_failures, 0);
6735 assert_eq!(health.status, SupervisorHealthStatus::Unknown);
6736 assert_eq!(health.detail, None);
6737 assert_eq!(health.metrics, None);
6738 assert_eq!(health.last_probe_ms, None);
6739 assert_eq!(health.last_action.as_deref(), Some("restart"));
6740 assert_eq!(health.last_action_ms, Some(124));
6741 }
6742
6743 #[cfg(unix)]
6744 #[tokio::test]
6745 async fn http_health_no_listener_restarts_after_consecutive_failures() {
6746 let dir = cortexkit_test_support::ScratchDir::new("http-refused");
6747 let unused = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
6748 let url = format!("http://{}/healthz", unused.local_addr().unwrap());
6749 drop(unused);
6750 let configured = http_fixture(&dir, &url, 2);
6751 let module =
6752 Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default())
6753 .supervise_configured_with_health(
6754 configured.module_spec(),
6755 true,
6756 configured.health,
6757 configured.drain_timeout_ms,
6758 configured.restart,
6759 )
6760 .unwrap();
6761 let before = module.status().unwrap().spawn_generation;
6762 timeout(Duration::from_secs(5), async {
6763 loop {
6764 let status = module.status().unwrap();
6765 if status.spawn_generation > before {
6766 assert!(status.lifetime_restarts > 0);
6767 break;
6768 }
6769 sleep(Duration::from_millis(5)).await;
6770 }
6771 })
6772 .await
6773 .expect("sustained HTTP refusal must trigger the health restart policy");
6774 module.drain().await.unwrap();
6775 }
6776
6777 #[cfg(unix)]
6778 #[tokio::test]
6779 async fn http_health_timeout_honours_deadline() {
6780 let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
6781 let url = format!("http://{}/healthz", listener.local_addr().unwrap());
6782 let server = tokio::spawn(async move {
6783 let _held = listener.accept().await.unwrap();
6784 std::future::pending::<()>().await;
6785 });
6786 let error = timeout(
6787 Duration::from_secs(1),
6788 probe_http_health(&url, Duration::from_millis(10)),
6789 )
6790 .await
6791 .expect("the probe must enforce its own deadline")
6792 .unwrap_err();
6793 server.abort();
6794 assert!(error.to_string().contains("timed out"));
6795 }
6796
6797 #[cfg(unix)]
6798 #[tokio::test]
6799 async fn http_health_localhost_reaches_an_ipv6_only_listener() {
6800 use tokio::io::{AsyncReadExt, AsyncWriteExt};
6801 let listener = tokio::net::TcpListener::bind("[::1]:0").await.unwrap();
6802 let url = format!(
6803 "http://localhost:{}/healthz",
6804 listener.local_addr().unwrap().port()
6805 );
6806 let server = tokio::spawn(async move {
6807 let (mut stream, _) = listener.accept().await.unwrap();
6808 let mut request = [0u8; 2048];
6809 assert!(stream.read(&mut request).await.unwrap() > 0);
6810 stream
6811 .write_all(b"HTTP/1.1 200 OK\r\nContent-Length: 2\r\n\r\nok")
6812 .await
6813 .unwrap();
6814 });
6815 assert_eq!(
6820 probe_http_health(&url, Duration::from_secs(10))
6821 .await
6822 .unwrap()
6823 .status,
6824 HealthStatus::Ok
6825 );
6826 server.await.unwrap();
6827 }
6828
6829 #[cfg(unix)]
6830 #[tokio::test]
6831 async fn http_health_timeout_keeps_partial_status_and_body() {
6832 use tokio::io::{AsyncReadExt, AsyncWriteExt};
6833 let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
6834 let url = format!("http://{}/healthz", listener.local_addr().unwrap());
6835 let server = tokio::spawn(async move {
6836 let (mut stream, _) = listener.accept().await.unwrap();
6837 let mut request = [0u8; 2048];
6838 assert!(stream.read(&mut request).await.unwrap() > 0);
6839 stream
6840 .write_all(
6841 b"HTTP/1.1 503 Unavailable\r\nContent-Length: 100\r\n\r\npartial diagnostic",
6842 )
6843 .await
6844 .unwrap();
6845 std::future::pending::<()>().await;
6846 });
6847 let error = probe_http_health(&url, Duration::from_secs(1))
6848 .await
6849 .unwrap_err()
6850 .to_string();
6851 server.abort();
6852 assert!(
6853 error.contains("timed out")
6854 && error.contains("503 Unavailable")
6855 && error.contains("partial diagnostic"),
6856 "{error}"
6857 );
6858 }
6859
6860 #[cfg(unix)]
6861 #[tokio::test]
6862 async fn http_health_diagnostic_body_is_bounded_to_200_bytes() {
6863 use tokio::io::{AsyncReadExt, AsyncWriteExt};
6864 let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
6865 let url = format!("http://{}/healthz", listener.local_addr().unwrap());
6866 let server = tokio::spawn(async move {
6867 let (mut stream, _) = listener.accept().await.unwrap();
6868 let mut request = [0u8; 2048];
6869 assert!(stream.read(&mut request).await.unwrap() > 0);
6870 let body = format!("{}not-in-diagnostic", "x".repeat(400));
6871 let response = format!(
6872 "HTTP/1.1 503 Unavailable\r\nContent-Length: {}\r\n\r\n{body}",
6873 body.len()
6874 );
6875 stream.write_all(response.as_bytes()).await.unwrap();
6876 });
6877 let error = probe_http_health(&url, Duration::from_secs(1))
6878 .await
6879 .unwrap_err()
6880 .to_string();
6881 server.await.unwrap();
6882 assert!(error.ends_with(&"x".repeat(200)), "{error}");
6883 assert_eq!(error.split(": ").last().unwrap().len(), 200);
6884 assert!(!error.contains("not-in-diagnostic"));
6885 }
6886
6887 #[cfg(unix)]
6888 #[tokio::test]
6889 async fn http_health_real_nats_server_monitoring() {
6890 if std::process::Command::new("nats-server")
6891 .arg("--version")
6892 .env("XDG_DATA_HOME", std::env::temp_dir())
6893 .env("XDG_RUNTIME_DIR", std::env::temp_dir())
6894 .env("XDG_CONFIG_HOME", std::env::temp_dir())
6895 .output()
6896 .is_err()
6897 {
6898 eprintln!(
6899 "skipped: http_health_real_nats_server_monitoring (nats-server is not on PATH)"
6900 );
6901 return;
6902 }
6903 let dir = cortexkit_test_support::ScratchDir::new("nats-http-monitoring");
6904 let monitor = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
6905 let port = monitor.local_addr().unwrap().port();
6906 let client = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
6907 let client_port = client.local_addr().unwrap().port();
6908 let config = dir.join("server.conf");
6909 std::fs::write(&config, format!("listen: 127.0.0.1:{client_port}\nhttp: 127.0.0.1:{port}\njetstream {{ store_dir: \"{}\" }}\n", dir.join("jetstream").display())).unwrap();
6910 let mut configured = http_fixture(&dir, &format!("http://127.0.0.1:{port}/healthz"), 1000);
6911 configured.program = PathBuf::from("nats-server");
6912 configured.args = vec!["-c".into(), config.to_string_lossy().into_owned()];
6913 drop(monitor);
6914 drop(client);
6915 let module =
6916 Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default())
6917 .supervise_configured_with_health(
6918 configured.module_spec(),
6919 true,
6920 configured.health,
6921 configured.drain_timeout_ms,
6922 configured.restart,
6923 )
6924 .unwrap();
6925 wait_http_health(&module, SupervisorHealthStatus::Ok).await;
6926 module.drain().await.unwrap();
6927 }
6928
6929 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
6930 async fn configuration_update_does_not_replace_captured_running_process_facts() {
6931 let supervisor =
6932 Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default());
6933 let snapshot = stale_process_snapshot(ModuleState::Running, true);
6934 let initial = ModuleSpec {
6935 module_id: "rescan-preserves-spawn-facts".to_string(),
6936 program: PathBuf::from("/spawned/module"),
6937 args: Vec::new(),
6938 env: Vec::new(),
6939 reserved: false,
6940 reserved_prefixes: Vec::new(),
6941 protocol: ModuleProtocol::Subc,
6942 overlap: Default::default(),
6943 carry_containment: false,
6944 };
6945 let module = supervisor.supervised_module(
6946 initial.clone(),
6947 supervisor.runtime_config(),
6948 snapshot,
6949 None,
6950 );
6951 let before = module.status().unwrap();
6952 let mut replacement = initial;
6953 replacement.program = PathBuf::from("/rescanned/replacement-module");
6954
6955 module
6956 .update_configuration(replacement, HealthConfig::default(), None)
6957 .await
6958 .unwrap();
6959
6960 let after = module.status().unwrap();
6961 assert_eq!(after.pid, before.pid);
6962 assert_eq!(after.spawned_at_ms, before.spawned_at_ms);
6963 assert_eq!(after.spawned_from, before.spawned_from);
6964 drop(module);
6965 }
6966}
6967
6968fn unix_ms_now() -> u64 {
6969 SystemTime::now()
6970 .duration_since(UNIX_EPOCH)
6971 .map(|duration| duration.as_millis().min(u128::from(u64::MAX)) as u64)
6972 .unwrap_or(0)
6973}
6974
6975async fn supervise_loop(
6976 mut spec: ModuleSpec,
6977 mut runtime: SupervisorRuntimeConfig,
6978 registry: Arc<Registry>,
6979 process_liveness: Arc<SupervisorProcessLiveness>,
6980 snapshot: SharedSnapshot,
6981 mut child: Option<SupervisedChild>,
6982 mut commands: mpsc::Receiver<SupervisorCommand>,
6983) {
6984 let mut health_probe = HealthProbeRuntime::default();
6985 let mut registration_changes = match registry.subscribe_module_changes(&spec.module_id) {
6989 Ok(changes) => Some(changes),
6990 Err(err) => {
6991 warn!(module_id = %spec.module_id, error = %err, "health prober could not subscribe to registry");
6994 None
6995 }
6996 };
6997 let mut pending_respawn: Option<PendingRespawn> = None;
7001 let mut requeued: VecDeque<SupervisorCommand> = VecDeque::new();
7004 let mut previous_instance_recheck = tokio::time::interval_at(
7005 Instant::now() + Duration::from_secs(5),
7006 Duration::from_secs(5),
7007 );
7008 previous_instance_recheck.set_missed_tick_behavior(tokio::time::MissedTickBehavior::Skip);
7009 loop {
7010 #[cfg(test)]
7011 {
7012 lock_snapshot(&snapshot).unwrap().actor_select = None;
7013 }
7014 #[cfg(target_os = "macos")]
7015 if let Some(active) = child.as_mut() {
7016 active.confirm_privacy_exec().await;
7017 }
7018 if let Some(scheduled) = runtime
7019 .scheduled_respawn
7020 .lock()
7021 .unwrap_or_else(|p| p.into_inner())
7022 .take()
7023 {
7024 runtime.recovery_is_daemon = scheduled.daemon_initiated;
7025 pending_respawn = Some(scheduled);
7026 }
7027 if pending_respawn.is_some() && (child.is_some() || !respawn_still_pending(&snapshot)) {
7028 pending_respawn = None;
7029 cancel_deferred_reload(
7030 &runtime,
7031 &spec.module_id,
7032 "respawn cancelled by a supervisor command",
7033 );
7034 }
7035 if child.is_none()
7036 && pending_respawn.is_none()
7037 && lock_snapshot(&snapshot).is_ok_and(|state| state.waiting_for_previous_pid.is_none())
7038 {
7039 cancel_deferred_reload(
7040 &runtime,
7041 &spec.module_id,
7042 "respawn cancelled before a replacement was spawned",
7043 );
7044 let mut stranded = false;
7045 let _ = update_snapshot(&snapshot, Some(&spec.module_id), |state| {
7046 state.respawn_pending = false;
7047 state.coalesced_restart_pending = false;
7048 if matches!(
7049 state.state,
7050 ModuleState::Restarting
7051 | ModuleState::Starting
7052 | ModuleState::Draining
7053 | ModuleState::Unresponsive
7054 ) {
7055 error!(module_id = %spec.module_id, state = ?state.state,
7056 "supervision operation ended without a child or pending respawn; marking failed so start can retry");
7057 stranded = true;
7058 }
7059 });
7060 if stranded {
7061 fail_snapshot(
7062 &snapshot,
7063 Some(&spec.module_id),
7064 None,
7065 &runtime.failure_context(runtime.recovery_is_daemon),
7066 );
7067 }
7068 }
7069 if child.is_none()
7070 && (runtime.child_roster.is_closed()
7071 || pending_respawn.is_none()
7072 || !lock_snapshot(&snapshot)
7073 .is_ok_and(|state| state.enabled && state.state == ModuleState::Restarting))
7074 {
7075 containment::terminate(&runtime.child_roster.containment).await;
7076 }
7077 if let Some(command) = requeued.pop_front() {
7078 if !handle_supervisor_command(
7079 command,
7080 &mut spec,
7081 &mut runtime,
7082 ®istry,
7083 &process_liveness,
7084 &snapshot,
7085 &mut child,
7086 &mut commands,
7087 &mut requeued,
7088 )
7089 .await
7090 {
7091 containment::terminate(&runtime.child_roster.containment).await;
7092 return;
7093 }
7094 if child.is_some() || !respawn_still_pending(&snapshot) {
7095 pending_respawn = None;
7096 let _ = update_snapshot(&snapshot, Some(&spec.module_id), |state| {
7097 state.respawn_pending = false
7098 });
7099 }
7100 continue;
7101 }
7102 if child.is_some() {
7103 runtime.recovery_is_daemon = true;
7104 #[cfg(test)]
7105 {
7106 lock_snapshot(&snapshot).unwrap().actor_turns += 1;
7107 }
7108 health_probe.refresh_registration(&spec, &runtime, ®istry, &snapshot);
7109 let active = child.as_mut().expect("child checked above");
7110 if runtime.child_roster.is_closed() || active.registered_since_spawn() {
7111 active.registration_deadline = None;
7112 }
7113 let registration_deadline = active.registration_deadline;
7114 let no_checks = runtime.health.http.is_none()
7115 && (running_protocol(&spec, &snapshot) == ModuleProtocol::None
7116 || (health_probe.registered_connection.is_some() && !health_probe.advertised));
7117 let uptime_reset = lock_snapshot(&snapshot).ok().and_then(|mut state| {
7118 state
7119 .failed_retry
7120 .uptime_reset_at(no_checks, runtime.restart_policy.window)
7121 });
7122 let wake_at = match (health_probe.wake_at(), uptime_reset) {
7123 (probe, Some(reset)) => Some(probe.map_or(reset, |probe| probe.min(reset))),
7124 (probe, None) => probe,
7125 };
7126 let wake_at = match (wake_at, registration_deadline) {
7127 (Some(wake), Some(register)) => Some(wake.min(register)),
7128 (wake, register) => wake.or(register),
7129 };
7130 #[cfg(test)]
7131 let wake_after = wake_at.map(|at| at.saturating_duration_since(Instant::now()));
7132 let probe_sleep = supervisor_sleep(wake_at);
7133 tokio::pin!(probe_sleep);
7134 let wire_child = running_protocol(&spec, &snapshot) == ModuleProtocol::Subc;
7139 #[cfg(test)]
7140 {
7141 let registration_pending = wire_child
7142 && registration_changes
7143 .as_ref()
7144 .is_some_and(|changes| changes.has_changed().unwrap());
7145 let mut state = lock_snapshot(&snapshot).unwrap();
7146 state.actor_select = (!registration_pending).then_some(ActorSelectCheckpoint {
7147 generation: state.spawn_generation,
7148 turn: state.actor_turns,
7149 registered_connection: health_probe.registered_connection,
7150 next_probe_at: health_probe.next_probe_at,
7151 wake_after,
7152 });
7153 }
7154 let active_child = child.as_mut().expect("child checked above");
7155 tokio::select! {
7156 wait_result = active_child.wait() => {
7157 if let Ok(mut state) = lock_snapshot(&snapshot) {
7158 state.failed_retry.uptime_reset_at(no_checks, runtime.restart_policy.window);
7159 }
7160 #[cfg(test)]
7161 { lock_snapshot(&snapshot).unwrap().actor_select = None; }
7162 let exit_report = match wait_result {
7171 Ok(status) => classify_reaped_child_exit(&snapshot, active_child, &status),
7172 Err(err) => {
7173 active_child.drain_stderr(&spec.module_id).await;
7174 fail_after_wait_error(
7175 &spec.module_id,
7176 &snapshot,
7177 &runtime.terminal_ring,
7178 &runtime.spawn_events,
7179 format!("failed to wait for supervised module: {err}"),
7180 &runtime.failure_context(true),
7181 );
7182 untrack_if_registration_released(
7183 &process_liveness,
7184 ®istry,
7185 &spec.module_id,
7186 &snapshot,
7187 );
7188 error!(module_id = %spec.module_id, error = %err, "failed to wait for supervised module");
7189 child = None;
7190 continue;
7191 }
7192 };
7193 active_child.drain_stderr(&spec.module_id).await;
7194
7195 let next = on_child_exit(
7196 &spec,
7197 runtime.restart_policy,
7198 ®istry,
7199 &snapshot,
7200 &runtime.terminal_ring,
7201 &runtime.spawn_events,
7202 &runtime.child_roster,
7203 exit_report,
7204 true,
7205 ).await;
7206 runtime.start_requester = None;
7207 runtime.pending_activity_restart = false;
7208 let carry = matches!(next, NextAction::Restart { .. })
7211 && active_child.spawn_failure.is_none()
7212 && !runtime.child_roster.is_closed();
7213 if let Err(error) = active_child.finish_exit(carry).await {
7214 containment::terminate(&runtime.child_roster.containment).await;
7215 fail_after_wait_error(&spec.module_id, &snapshot, &runtime.terminal_ring,
7216 &runtime.spawn_events, format!("failed to reap classified exit: {error}"),
7217 &runtime.failure_context(true));
7218 child = None;
7219 continue;
7220 }
7221 match next {
7222 NextAction::Stop { registration_released } => {
7223 if registration_released {
7224 process_liveness.untrack_if_current(&spec.module_id, &snapshot);
7225 }
7226 child = None;
7227 }
7228 NextAction::Restart { schedule } => {
7229 let delay = schedule.map_or(
7230 runtime.restart_policy.delay_for_restart(0),
7231 |schedule| schedule.delay,
7232 );
7233 if let Some(schedule) = schedule {
7234 log_crash_respawn(&spec.module_id, schedule);
7235 }
7236 child = None;
7244 pending_respawn = Some(PendingRespawn { deadline: Instant::now() + delay, kind: RespawnKind::Spawn, daemon_initiated: true });
7245 let _ = update_snapshot(&snapshot, Some(&spec.module_id), |state| state.respawn_pending = true);
7246 }
7247 }
7248 }
7249 command = commands.recv() => {
7250 #[cfg(test)]
7251 { lock_snapshot(&snapshot).unwrap().actor_select = None; }
7252 let Some(command) = command else {
7253 containment::terminate(&runtime.child_roster.containment).await;
7254 return;
7255 };
7256 if !handle_supervisor_command(
7257 command,
7258 &mut spec,
7259 &mut runtime,
7260 ®istry,
7261 &process_liveness,
7262 &snapshot,
7263 &mut child,
7264 &mut commands,
7265 &mut requeued,
7266 ).await {
7267 containment::terminate(&runtime.child_roster.containment).await;
7268 return;
7269 }
7270 }
7271 _ = async {
7272 match registration_changes.as_mut() {
7273 Some(changes) => { let _ = changes.changed().await; }
7274 None => std::future::pending().await,
7275 }
7276 }, if wire_child => {
7277 #[cfg(test)]
7278 { lock_snapshot(&snapshot).unwrap().actor_select = None; }
7279 }
7280 _ = &mut probe_sleep => {
7281 #[cfg(test)]
7282 { lock_snapshot(&snapshot).unwrap().actor_select = None; }
7283 if registration_deadline.is_some_and(|deadline| Instant::now() >= deadline) {
7284 active_child.registration_deadline = None;
7287 if !runtime.child_roster.is_closed()
7288 && !active_child.registered_since_spawn()
7289 && active_child.child.try_wait().is_ok_and(|status| status.is_none())
7290 {
7291 warn!(module_id = %spec.module_id, pid = active_child.pid,
7292 wait_secs = runtime.register_timeout.as_secs(),
7293 "supervised module registration_timeout: first HELLO deadline expired; stopping child");
7294 active_child.spawn_failure = Some("registration_timeout".into());
7295 if let Err(error) = stop_unregistered_child(active_child).await {
7296 error!(module_id = %spec.module_id, %error, "failed to stop unregistered child");
7297 fail_after_wait_error(&spec.module_id, &snapshot, &runtime.terminal_ring,
7298 &runtime.spawn_events, format!("registration_timeout: {error}"),
7299 &runtime.failure_context(true));
7300 child = None;
7301 }
7302 continue;
7305 }
7306 }
7307 if health_probe.due() {
7308 run_health_probe_cycle(
7309 &spec,
7310 &runtime,
7311 ®istry,
7312 &process_liveness,
7313 &snapshot,
7314 &mut child,
7315 ).await;
7316 if child.is_none() {
7317 runtime.start_requester = None;
7318 runtime.pending_activity_restart = false;
7319 }
7320 if child.is_some() {
7321 health_probe.schedule_next(&spec, runtime.health.cadence);
7322 }
7323 }
7324 }
7325 }
7326 } else if let Some(pending) = pending_respawn {
7327 tokio::select! {
7328 _ = runtime.child_roster.closed() => {
7329 pending_respawn = None;
7330 let _ = update_snapshot(&snapshot, Some(&spec.module_id), |state| {
7331 state.state = ModuleState::Stopped;
7332 state.respawn_pending = false;
7333 });
7334 containment::terminate(&runtime.child_roster.containment).await;
7335 }
7336 _ = sleep_until(pending.deadline) => {
7337 pending_respawn = None;
7338 let _ = update_snapshot(&snapshot, Some(&spec.module_id), |state| state.respawn_pending = false);
7339 if !respawn_still_pending(&snapshot) {
7343 continue;
7344 }
7345 if runtime.child_roster.is_closed() {
7350 let _ = update_snapshot(&snapshot, Some(&spec.module_id), |state| {
7351 state.state = ModuleState::Stopped;
7352 });
7353 debug!(module_id = %spec.module_id, "crash respawn cancelled by daemon shutdown");
7354 continue;
7355 }
7356 if let Err(err) = release_dead_registration(
7357 ®istry,
7358 runtime.forwarding.as_deref(),
7359 &snapshot,
7360 &spec.module_id,
7361 &runtime.failure_context(runtime.recovery_is_daemon),
7362 ).await {
7363 error!(module_id = %spec.module_id, error = %err, "registration did not release before restart");
7364 continue;
7365 }
7366
7367 if matches!(pending.kind, RespawnKind::Reload) {
7368 let reply = runtime.deferred_reload_reply.lock().unwrap_or_else(|p| p.into_inner()).take();
7369 let result = finish_reload_child(&spec, &runtime, ®istry, &process_liveness, &snapshot, &mut child).await;
7370 if result.is_ok() && runtime.pending_activity_restart {
7371 runtime.record_activity_transition("subc.module_restarted", &spec.module_id, runtime.start_requester.clone());
7372 }
7373 runtime.pending_activity_restart = false;
7374 if let Some(reply) = reply { let _ = reply.send(result); }
7375 continue;
7376 }
7377 process_liveness.track(spec.module_id.clone(), Arc::clone(&snapshot));
7378 match spawn_and_mark_running(&spec, &runtime, &snapshot) {
7379 Ok(next_child) => {
7380 child = Some(next_child);
7381 if runtime.pending_activity_restart {
7382 runtime.record_activity_transition("subc.module_restarted", &spec.module_id, runtime.start_requester.clone());
7383 runtime.pending_activity_restart = false;
7384 }
7385 debug!(module_id = %spec.module_id, "supervised module restarted after crash");
7386 }
7387 Err(err) => {
7388 fail_snapshot(&snapshot, Some(&spec.module_id), None, &runtime.failure_context(runtime.recovery_is_daemon));
7389 process_liveness.untrack_if_current(&spec.module_id, &snapshot);
7390 error!(module_id = %spec.module_id, error = %err, "failed to restart supervised module");
7391 }
7392 }
7393 }
7394 command = commands.recv() => {
7395 let Some(command) = command else {
7396 containment::terminate(&runtime.child_roster.containment).await;
7397 return;
7398 };
7399 if !handle_supervisor_command(
7400 command,
7401 &mut spec,
7402 &mut runtime,
7403 ®istry,
7404 &process_liveness,
7405 &snapshot,
7406 &mut child,
7407 &mut commands,
7408 &mut requeued,
7409 ).await {
7410 containment::terminate(&runtime.child_roster.containment).await;
7411 return;
7412 }
7413 if child.is_some() || !respawn_still_pending(&snapshot) {
7418 pending_respawn = None;
7419 let _ = update_snapshot(&snapshot, Some(&spec.module_id), |state| state.respawn_pending = false);
7420 }
7421 }
7422 }
7423 } else {
7424 let slow_retry = lock_snapshot(&snapshot).ok().and_then(|state| {
7425 (state.enabled && state.state == ModuleState::Failed)
7426 .then_some(state.failed_retry.pending)
7427 .flatten()
7428 });
7429 let command = if lock_snapshot(&snapshot)
7430 .is_ok_and(|state| state.waiting_for_previous_pid.is_some())
7431 {
7432 tokio::select! {
7433 command = commands.recv() => command,
7434 _ = previous_instance_recheck.tick() => {
7435 if runtime.child_roster.previous_instance_pid(&spec.module_id).is_none() {
7436 let enabled = lock_snapshot(&snapshot).is_ok_and(|state| state.enabled);
7437 if enabled && !runtime.child_roster.is_closed() {
7438 process_liveness.track(spec.module_id.clone(), Arc::clone(&snapshot));
7439 match spawn_and_mark_running(&spec, &runtime, &snapshot) {
7440 Ok(next_child) => child = Some(next_child),
7441 Err(error) => {
7442 fail_snapshot(&snapshot, Some(&spec.module_id), None, &runtime.failure_context(true));
7443 error!(module_id = %spec.module_id, %error, "failed to start module after its previous instance exited");
7444 }
7445 }
7446 } else {
7447 let _ = update_snapshot(&snapshot, Some(&spec.module_id), |state| {
7448 state.waiting_for_previous_pid = None;
7449 state.waiting_for_previous_detail = None;
7450 });
7451 }
7452 }
7453 continue;
7454 }
7455 }
7456 } else {
7457 tokio::select! {
7458 biased;
7459 command = commands.recv() => command,
7460 _ = async {
7461 match slow_retry {
7462 Some(retry) => sleep_until(retry.deadline).await,
7463 None => std::future::pending().await,
7464 }
7465 } => {
7466 let retry = slow_retry.expect("only a scheduled slow retry can wake");
7467 let _ = update_snapshot(&snapshot, Some(&spec.module_id), |state| {
7468 state.failed_retry.pending = None;
7469 });
7470 if runtime.child_roster.is_closed() {
7471 continue;
7472 }
7473 info!(module_id = %spec.module_id, attempt = retry.attempt,
7474 delay_secs = retry.delay.as_secs(), "starting failed module retry");
7475 if let Err(error) = set_child_enabled(
7476 &spec, &runtime, ®istry, &process_liveness, &snapshot, &mut child, true,
7477 &runtime.failure_context(true),
7478 ).await {
7479 error!(module_id = %spec.module_id, %error, "failed module retry could not start");
7480 }
7481 continue;
7482 }
7483 }
7484 };
7485 let Some(command) = command else {
7486 containment::terminate(&runtime.child_roster.containment).await;
7487 return;
7488 };
7489 if !handle_supervisor_command(
7490 command,
7491 &mut spec,
7492 &mut runtime,
7493 ®istry,
7494 &process_liveness,
7495 &snapshot,
7496 &mut child,
7497 &mut commands,
7498 &mut requeued,
7499 )
7500 .await
7501 {
7502 containment::terminate(&runtime.child_roster.containment).await;
7503 return;
7504 }
7505 }
7506 }
7507}
7508
7509async fn supervisor_sleep(wake_at: Option<Instant>) {
7512 match wake_at {
7513 Some(deadline) => sleep_until(deadline).await,
7514 None => std::future::pending().await,
7515 }
7516}
7517
7518fn log_crash_respawn(module_id: &str, schedule: CrashRestartSchedule) {
7519 info!(
7520 module_id,
7521 restart_in_window = schedule.restart_in_window,
7522 delay_ms = schedule.delay.as_millis() as u64,
7523 "respawning after crash"
7524 );
7525}
7526
7527fn respawn_still_pending(snapshot: &SharedSnapshot) -> bool {
7533 matches!(
7534 lock_snapshot(snapshot),
7535 Ok(state) if state.enabled && state.state == ModuleState::Restarting
7536 )
7537}
7538
7539enum NextAction {
7540 Stop {
7541 registration_released: bool,
7542 },
7543 Restart {
7544 schedule: Option<CrashRestartSchedule>,
7545 },
7546}
7547
7548#[allow(clippy::too_many_arguments)]
7549async fn handle_supervisor_command(
7550 command: SupervisorCommand,
7551 spec: &mut ModuleSpec,
7552 runtime: &mut SupervisorRuntimeConfig,
7553 registry: &Arc<Registry>,
7554 process_liveness: &SupervisorProcessLiveness,
7555 snapshot: &SharedSnapshot,
7556 child: &mut Option<SupervisedChild>,
7557 commands: &mut mpsc::Receiver<SupervisorCommand>,
7558 requeued: &mut VecDeque<SupervisorCommand>,
7559) -> bool {
7560 if !matches!(command, SupervisorCommand::UpdateConfiguration { .. }) {
7561 runtime.recovery_is_daemon = false;
7562 }
7563 if matches!(
7564 command,
7565 SupervisorCommand::Drain { .. }
7566 | SupervisorCommand::Retire { .. }
7567 | SupervisorCommand::SetEnabled { .. }
7568 | SupervisorCommand::Restart { .. }
7569 ) {
7570 let _ = update_snapshot(snapshot, Some(&spec.module_id), |state| {
7571 state.failed_retry.cancel()
7572 });
7573 }
7574 let blocked = recovery_block_error(spec, &runtime.child_roster).or_else(|| {
7575 runtime
7576 .child_roster
7577 .previous_instance(&spec.module_id)
7578 .map(|(pid, reason)| previous_instance_error(spec, pid, reason))
7579 });
7580 let command = if let Some(error) = blocked {
7581 match command {
7582 SupervisorCommand::Restart { reply, .. }
7583 | SupervisorCommand::Reload { reply, .. }
7584 | SupervisorCommand::Swap { reply, .. } => {
7585 let _ = reply.send(Err(error));
7586 return true;
7587 }
7588 SupervisorCommand::SetEnabled {
7589 enabled: true,
7590 reply,
7591 ..
7592 } => {
7593 let _ = reply.send(Err(error));
7594 return true;
7595 }
7596 other => other,
7597 }
7598 } else {
7599 command
7600 };
7601 match command {
7602 SupervisorCommand::Drain { requester, reply } => {
7603 let result = drain_optional_child(
7606 &spec.module_id,
7607 spec.protocol,
7608 StopNotice::NotSent,
7609 registry,
7610 runtime.forwarding.as_deref(),
7611 snapshot,
7612 &runtime.terminal_ring,
7613 &runtime.spawn_events,
7614 child,
7615 runtime.drain_timeout,
7616 ModuleState::Stopped,
7617 None,
7618 &FailureContext::operator(),
7619 )
7620 .await;
7621 containment::terminate(&runtime.child_roster.containment).await;
7622 let registration_released = result.is_ok();
7623 if registration_released {
7624 runtime.record_activity_transition(
7625 "subc.module_stopped",
7626 &spec.module_id,
7627 requester,
7628 );
7629 }
7630 let _ = reply.send(result);
7631 if registration_released {
7632 process_liveness.untrack_if_current(&spec.module_id, snapshot);
7633 }
7634 false
7635 }
7636 SupervisorCommand::Retire { requester, reply } => {
7637 let result = async {
7638 let stop_notice = begin_forwarding_drain_if_configured(
7639 spec,
7640 runtime,
7641 registry,
7642 snapshot,
7643 None,
7644 RouteCloseReason::Disable,
7645 )
7646 .await?;
7647 drain_optional_child(
7648 &spec.module_id,
7649 spec.protocol,
7650 stop_notice,
7651 registry,
7652 runtime.forwarding.as_deref(),
7653 snapshot,
7654 &runtime.terminal_ring,
7655 &runtime.spawn_events,
7656 child,
7657 runtime.drain_timeout,
7658 ModuleState::Stopped,
7659 None,
7660 &FailureContext::operator(),
7661 )
7662 .await
7663 }
7664 .await;
7665 containment::terminate(&runtime.child_roster.containment).await;
7666 let registration_released = result.is_ok();
7667 if registration_released {
7668 runtime.record_activity_transition(
7669 "subc.module_stopped",
7670 &spec.module_id,
7671 requester,
7672 );
7673 }
7674 let _ = reply.send(result);
7675 if registration_released {
7676 process_liveness.untrack_if_current(&spec.module_id, snapshot);
7677 }
7678 false
7679 }
7680 SupervisorCommand::Restart {
7681 requester,
7682 drain_timeout_ms,
7683 received_at_generation,
7684 queued_at,
7685 reply,
7686 } => {
7687 info!(
7691 module_id = %spec.module_id,
7692 queued_ms = u64::try_from(queued_at.elapsed().as_millis()).unwrap_or(u64::MAX),
7693 "restart command dequeued"
7694 );
7695 let validation = match lock_snapshot(snapshot) {
7707 Ok(state) if !state.enabled => Err(SuperviseError::Disabled {
7708 module_id: spec.module_id.clone(),
7709 }),
7710 Ok(_) => Ok(()),
7711 Err(err) => Err(err),
7712 };
7713 let initiated = validation.is_ok();
7714 let _ = reply.send(validation);
7715 let satisfied_by_generation = if initiated
7729 && child
7730 .as_ref()
7731 .is_some_and(|child| child.spawned_at > queued_at)
7732 {
7733 lock_snapshot(snapshot).ok().and_then(|state| {
7734 (!state.configuration_updated_since_spawn).then_some(state.spawn_generation)
7735 })
7736 } else {
7737 None
7738 };
7739 let satisfied_by_pending = initiated
7740 && child.is_none()
7741 && lock_snapshot(snapshot).ok().is_some_and(|mut state| {
7742 let pending = state.respawn_pending && !state.configuration_updated_since_spawn;
7746 if pending {
7747 state.coalesced_restart_pending = true;
7748 }
7749 pending
7750 });
7751 if satisfied_by_pending {
7752 debug!(module_id = %spec.module_id, "restart coalesced into the pending replacement");
7753 } else if let Some(generation) = satisfied_by_generation {
7754 info!(
7755 module_id = %spec.module_id,
7756 received_at_generation,
7757 "restart already satisfied by generation {generation}; not restarting again"
7758 );
7759 } else if initiated {
7760 let drain_timeout = drain_timeout_ms
7763 .map(Duration::from_millis)
7764 .unwrap_or(runtime.drain_timeout);
7765 if let Err(err) = restart_child(
7766 spec,
7767 runtime,
7768 registry,
7769 process_liveness,
7770 snapshot,
7771 child,
7772 drain_timeout,
7773 )
7774 .await
7775 {
7776 warn!(
7777 module_id = %spec.module_id,
7778 error = %err,
7779 "operator restart failed after initiation ack; module state carries the outcome"
7780 );
7781 let _ = update_snapshot(snapshot, Some(&spec.module_id), |state| {
7782 state.state = ModuleState::Failed;
7783 clear_current_process_facts(state);
7784 });
7785 } else {
7786 runtime.start_requester = requester;
7787 runtime.pending_activity_restart = true;
7788 }
7789 }
7790 true
7791 }
7792 SupervisorCommand::Reload { requester, reply } => {
7793 let result =
7794 reload_child(spec, runtime, registry, process_liveness, snapshot, child).await;
7795 if result.is_ok() {
7796 runtime.start_requester = requester;
7797 runtime.pending_activity_restart = true;
7798 }
7799 if result.is_ok()
7800 && runtime
7801 .scheduled_respawn
7802 .lock()
7803 .unwrap_or_else(|p| p.into_inner())
7804 .as_ref()
7805 .is_some_and(|pending| matches!(pending.kind, RespawnKind::Reload))
7806 {
7807 *runtime
7808 .deferred_reload_reply
7809 .lock()
7810 .unwrap_or_else(|p| p.into_inner()) = Some(reply);
7811 } else {
7812 let _ = reply.send(result);
7813 }
7814 true
7815 }
7816 SupervisorCommand::SetEnabled {
7817 enabled,
7818 requester,
7819 reply,
7820 } => {
7821 let previous_requester = runtime.start_requester.clone();
7822 if enabled {
7823 runtime.start_requester = requester.clone();
7824 }
7825 let result = set_child_enabled(
7826 spec,
7827 runtime,
7828 registry,
7829 process_liveness,
7830 snapshot,
7831 child,
7832 enabled,
7833 &runtime.failure_context(false),
7834 )
7835 .await;
7836 if !enabled {
7837 containment::terminate(&runtime.child_roster.containment).await;
7838 }
7839 if matches!(result, Ok(true)) {
7840 runtime.pending_activity_restart = false;
7841 if enabled {
7842 runtime.start_requester = requester;
7843 } else {
7844 runtime.start_requester = None;
7845 runtime.record_activity_transition(
7846 "subc.module_disabled",
7847 &spec.module_id,
7848 requester,
7849 );
7850 }
7851 } else {
7852 runtime.start_requester = previous_requester;
7853 }
7854 let _ = reply.send(result);
7855 true
7856 }
7857 SupervisorCommand::UpdateConfiguration {
7858 spec: next_spec,
7859 health,
7860 drain_timeout_ms,
7861 reply,
7862 } => {
7863 if let Some(handle) = &runtime.supervisor_handle {
7864 handle.apply_identity_configuration(&next_spec);
7865 }
7866 *spec = next_spec;
7867 let _ = update_snapshot(snapshot, Some(&spec.module_id), |state| {
7868 state.configuration_updated_since_spawn = true;
7869 });
7870 let health_changed = runtime.health != health;
7871 runtime.health = health;
7872 if health_changed {
7875 let _ = update_snapshot(snapshot, Some(&spec.module_id), |state| {
7876 state.health = ModuleHealthStatus::default();
7877 state.last_probe_evidence = None;
7878 });
7879 }
7880 runtime.drain_timeout = drain_timeout_ms
7881 .map(Duration::from_millis)
7882 .unwrap_or(runtime.default_drain_timeout);
7883 *runtime
7884 .effective_drain_timeout
7885 .lock()
7886 .unwrap_or_else(|poisoned| poisoned.into_inner()) = runtime.drain_timeout;
7887 let _ = reply.send(());
7888 true
7889 }
7890 SupervisorCommand::Swap {
7891 requester,
7892 ready_timeout,
7893 reply,
7894 } => {
7895 if spec.carry_containment
7896 || containment::current(&runtime.child_roster.containment).is_some()
7897 {
7898 let _ = reply.send(Err(SuperviseError::InvalidSpec {
7899 reason: "a carried container cannot overlap a swap candidate".to_owned(),
7900 }));
7901 return true;
7902 }
7903 let mut swap_runtime = runtime.clone();
7904 swap_runtime.start_requester = requester;
7905 let before = lock_snapshot(snapshot)
7906 .ok()
7907 .map(|state| state.spawn_generation);
7908 let end = swap::run_swap(
7909 spec,
7910 &swap_runtime,
7911 registry,
7912 process_liveness,
7913 snapshot,
7914 child,
7915 commands,
7916 ready_timeout.unwrap_or(DEFAULT_SWAP_READY_TIMEOUT),
7917 reply,
7918 )
7919 .await;
7920 if before
7921 != lock_snapshot(snapshot)
7922 .ok()
7923 .map(|state| state.spawn_generation)
7924 {
7925 runtime.start_requester = swap_runtime.start_requester;
7926 }
7927 requeued.extend(end.requeue);
7928 true
7929 }
7930 }
7931}
7932
7933async fn restart_child(
7934 spec: &ModuleSpec,
7935 runtime: &SupervisorRuntimeConfig,
7936 registry: &Registry,
7937 process_liveness: &SupervisorProcessLiveness,
7938 snapshot: &SharedSnapshot,
7939 child: &mut Option<SupervisedChild>,
7940 drain_timeout: Duration,
7941) -> Result<(), SuperviseError> {
7942 if !lock_snapshot(snapshot)?.enabled {
7944 return Err(SuperviseError::Disabled {
7945 module_id: spec.module_id.clone(),
7946 });
7947 }
7948 let stop_notice = begin_forwarding_drain_with_timeout(
7949 spec,
7950 runtime,
7951 registry,
7952 snapshot,
7953 None,
7954 RouteCloseReason::Restart,
7955 drain_timeout,
7956 )
7957 .await?;
7958
7959 if child.is_some() {
7960 drain_optional_child(
7961 &spec.module_id,
7962 spec.protocol,
7963 stop_notice,
7964 registry,
7965 runtime.forwarding.as_deref(),
7966 snapshot,
7967 &runtime.terminal_ring,
7968 &runtime.spawn_events,
7969 child,
7970 drain_timeout,
7971 ModuleState::Restarting,
7972 Some(true),
7973 &FailureContext::operator(),
7974 )
7975 .await?;
7976 } else {
7977 update_snapshot(snapshot, Some(&spec.module_id), |state| {
7978 state.enabled = true;
7979 state.state = ModuleState::Restarting;
7980 clear_current_process_facts(state);
7981 })?;
7982 release_dead_registration(
7983 registry,
7984 runtime.forwarding.as_deref(),
7985 snapshot,
7986 &spec.module_id,
7987 &runtime.failure_context(false),
7988 )
7989 .await?;
7990 }
7991
7992 reset_restart_count(snapshot, &spec.module_id)?;
7993 process_liveness.track(spec.module_id.clone(), Arc::clone(snapshot));
7994 schedule_respawn(
7995 runtime,
7996 snapshot,
7997 &spec.module_id,
7998 runtime.restart_policy.backoff,
7999 RespawnKind::Spawn,
8000 false,
8001 )
8002}
8003
8004async fn reload_child(
8005 spec: &ModuleSpec,
8006 runtime: &SupervisorRuntimeConfig,
8007 registry: &Registry,
8008 process_liveness: &SupervisorProcessLiveness,
8009 snapshot: &SharedSnapshot,
8010 child: &mut Option<SupervisedChild>,
8011) -> Result<(), SuperviseError> {
8012 if !lock_snapshot(snapshot)?.enabled {
8014 return Err(SuperviseError::Disabled {
8015 module_id: spec.module_id.clone(),
8016 });
8017 }
8018 let stop_notice = begin_forwarding_drain(
8019 spec,
8020 runtime,
8021 registry,
8022 snapshot,
8023 Some(true),
8024 RouteCloseReason::Reload,
8025 )
8026 .await?;
8027
8028 if child.is_some() {
8029 drain_optional_child(
8030 &spec.module_id,
8031 spec.protocol,
8032 stop_notice,
8033 registry,
8034 runtime.forwarding.as_deref(),
8035 snapshot,
8036 &runtime.terminal_ring,
8037 &runtime.spawn_events,
8038 child,
8039 runtime.drain_timeout,
8040 ModuleState::Restarting,
8041 Some(true),
8042 &FailureContext::operator(),
8043 )
8044 .await?;
8045 } else {
8046 update_snapshot(snapshot, Some(&spec.module_id), |state| {
8047 state.enabled = true;
8048 state.state = ModuleState::Restarting;
8049 clear_current_process_facts(state);
8050 })?;
8051 release_dead_registration(
8052 registry,
8053 runtime.forwarding.as_deref(),
8054 snapshot,
8055 &spec.module_id,
8056 &runtime.failure_context(false),
8057 )
8058 .await?;
8059 }
8060
8061 reset_restart_count(snapshot, &spec.module_id)?;
8062 process_liveness.track(spec.module_id.clone(), Arc::clone(snapshot));
8063 schedule_respawn(
8064 runtime,
8065 snapshot,
8066 &spec.module_id,
8067 runtime.restart_policy.backoff,
8068 RespawnKind::Reload,
8069 false,
8070 )
8071}
8072
8073async fn finish_reload_child(
8074 spec: &ModuleSpec,
8075 runtime: &SupervisorRuntimeConfig,
8076 registry: &Registry,
8077 process_liveness: &SupervisorProcessLiveness,
8078 snapshot: &SharedSnapshot,
8079 child: &mut Option<SupervisedChild>,
8080) -> Result<(), SuperviseError> {
8081 process_liveness.track(spec.module_id.clone(), Arc::clone(snapshot));
8082 let next_child = match spawn_and_mark_running(spec, runtime, snapshot) {
8083 Ok(next_child) => next_child,
8084 Err(err) => {
8085 return handle_reload_spawn_failure(
8086 spec,
8087 runtime,
8088 process_liveness,
8089 snapshot,
8090 child,
8091 format!("new child failed to spawn: {err}"),
8092 )
8093 .await;
8094 }
8095 };
8096 *child = Some(next_child);
8097
8098 let wait_outcome = {
8099 let active_child = child.as_mut().expect("new reload child was just stored");
8100 wait_for_registration_after_reload(
8101 registry,
8102 &spec.module_id,
8103 snapshot,
8104 active_child,
8105 REGISTRY_RELEASE_TIMEOUT,
8106 )
8107 .await?
8108 };
8109
8110 match wait_outcome {
8111 RegistrationWaitOutcome::Registered => {
8112 child
8113 .as_mut()
8114 .expect("registered reload child")
8115 .registration_deadline = None;
8116 debug!(module_id = %spec.module_id, "supervised module reloaded and registered");
8117 Ok(())
8118 }
8119 RegistrationWaitOutcome::Exited(exit_report) => {
8120 if let Some(active_child) = child.as_mut() {
8121 active_child.drain_stderr(&spec.module_id).await;
8122 }
8123 let mut exited_child = child.take().expect("exited reload child is still stored");
8126 #[cfg(test)]
8127 if let Some(gate) = &runtime.test_reload_exit_record_gate {
8128 gate.reached.notify_one();
8129 gate.resume.notified().await;
8130 }
8131 let result = handle_reload_child_registration_failure(
8132 spec,
8133 runtime,
8134 registry,
8135 process_liveness,
8136 snapshot,
8137 child,
8138 ReloadRegistrationFailure {
8139 exit_report: registration_failure_exit_report(exit_report),
8140 reason: exited_child
8141 .spawn_failure
8142 .clone()
8143 .unwrap_or_else(|| "new child exited before registering".to_string()),
8144 },
8145 )
8146 .await;
8147 exited_child
8148 .finish_exit(false)
8149 .await
8150 .map_err(|source| SuperviseError::Wait {
8151 module_id: spec.module_id.clone(),
8152 source,
8153 })?;
8154 result
8155 }
8156 RegistrationWaitOutcome::TimedOut => {
8157 let mut timed_out_child = child
8158 .take()
8159 .expect("timed-out reload child is still running");
8160 timed_out_child
8161 .start_kill()
8162 .map_err(|source| SuperviseError::Kill {
8163 module_id: spec.module_id.clone(),
8164 source,
8165 })?;
8166 let status = timed_out_child
8167 .wait()
8168 .await
8169 .map_err(|source| SuperviseError::Wait {
8170 module_id: spec.module_id.clone(),
8171 source,
8172 })?;
8173 timed_out_child.drain_stderr(&spec.module_id).await;
8174 let result = handle_reload_child_registration_failure(
8175 spec,
8176 runtime,
8177 registry,
8178 process_liveness,
8179 snapshot,
8180 child,
8181 ReloadRegistrationFailure {
8182 exit_report: registration_failure_exit_report(classify_reaped_child_exit(
8183 snapshot,
8184 &timed_out_child,
8185 &status,
8186 )),
8187 reason: format!(
8188 "new child did not register within {:?}",
8189 REGISTRY_RELEASE_TIMEOUT
8190 ),
8191 },
8192 )
8193 .await;
8194 timed_out_child
8195 .finish_exit(false)
8196 .await
8197 .map_err(|source| SuperviseError::Wait {
8198 module_id: spec.module_id.clone(),
8199 source,
8200 })?;
8201 result
8202 }
8203 }
8204}
8205
8206#[allow(clippy::too_many_arguments)]
8207async fn set_child_enabled(
8208 spec: &ModuleSpec,
8209 runtime: &SupervisorRuntimeConfig,
8210 registry: &Registry,
8211 process_liveness: &SupervisorProcessLiveness,
8212 snapshot: &SharedSnapshot,
8213 child: &mut Option<SupervisedChild>,
8214 enabled: bool,
8215 failure: &FailureContext,
8216) -> Result<bool, SuperviseError> {
8217 let (current_enabled, current_state, respawn_pending) = {
8218 let state = lock_snapshot(snapshot)?;
8219 (state.enabled, state.state, state.respawn_pending)
8220 };
8221 let revive_terminal = enabled
8229 && current_enabled
8230 && child.is_none()
8231 && (matches!(current_state, ModuleState::Failed | ModuleState::Stopped)
8232 || (current_state == ModuleState::Restarting && !respawn_pending));
8233 if current_enabled == enabled && !revive_terminal {
8234 return Ok(false);
8235 }
8236
8237 if enabled {
8238 update_snapshot(snapshot, Some(&spec.module_id), |state| {
8239 state.enabled = true;
8240 state.state = ModuleState::Starting;
8241 clear_current_process_facts(state);
8242 })?;
8243 #[cfg(test)]
8244 if runtime.test_seed_stale_facts_before_enable_spawn {
8245 update_snapshot(snapshot, Some(&spec.module_id), |state| {
8246 state.process_alive = true;
8247 state.pid = Some(41);
8248 state.spawned_at_ms = Some(42);
8249 state.spawned_from = Some(PathBuf::from("/spawned/module"));
8250 state.spawned_file_identity = Some(SpawnedFileIdentity {
8251 device: 43,
8252 inode: 44,
8253 #[cfg(windows)]
8254 windows_image: None,
8255 });
8256 })?;
8257 }
8258 release_dead_registration(
8259 registry,
8260 runtime.forwarding.as_deref(),
8261 snapshot,
8262 &spec.module_id,
8263 failure,
8264 )
8265 .await?;
8266 reset_restart_count(snapshot, &spec.module_id)?;
8267 process_liveness.track(spec.module_id.clone(), Arc::clone(snapshot));
8268 let next_child = match spawn_and_mark_running(spec, runtime, snapshot) {
8269 Ok(next_child) => next_child,
8270 Err(err) => {
8271 fail_snapshot(snapshot, Some(&spec.module_id), None, failure);
8272 process_liveness.untrack_if_current(&spec.module_id, snapshot);
8273 return Err(err);
8274 }
8275 };
8276 *child = Some(next_child);
8277 debug!(module_id = %spec.module_id, "supervised module enabled");
8278 Ok(true)
8279 } else {
8280 let stop_notice = begin_forwarding_drain_if_configured(
8281 spec,
8282 runtime,
8283 registry,
8284 snapshot,
8285 Some(false),
8286 RouteCloseReason::Disable,
8287 )
8288 .await?;
8289 drain_optional_child(
8290 &spec.module_id,
8291 spec.protocol,
8292 stop_notice,
8293 registry,
8294 runtime.forwarding.as_deref(),
8295 snapshot,
8296 &runtime.terminal_ring,
8297 &runtime.spawn_events,
8298 child,
8299 runtime.drain_timeout,
8300 ModuleState::Disabled,
8301 Some(false),
8302 failure,
8303 )
8304 .await?;
8305 debug!(module_id = %spec.module_id, "supervised module disabled");
8306 Ok(true)
8307 }
8308}
8309
8310#[allow(clippy::too_many_arguments)]
8311async fn on_child_exit(
8312 spec: &ModuleSpec,
8313 policy: RestartPolicy,
8314 registry: &Registry,
8315 snapshot: &SharedSnapshot,
8316 terminal_ring: &Arc<Mutex<TerminalRing>>,
8317 spawn_events: &SpawnEventFeed,
8318 roster: &ChildRoster,
8319 exit_report: ExitReport,
8320 daemon_initiated: bool,
8321) -> NextAction {
8322 if roster.is_closed() {
8328 return on_child_exit_during_daemon_shutdown(
8329 spec,
8330 registry,
8331 snapshot,
8332 terminal_ring,
8333 spawn_events,
8334 exit_report,
8335 )
8336 .await;
8337 }
8338 let unrequested_clean_exit_of_protocol_none = exit_report.kind == ExitKind::Clean
8354 && running_protocol(spec, snapshot) == ModuleProtocol::None;
8355 match exit_report.kind {
8356 ExitKind::Clean if !unrequested_clean_exit_of_protocol_none => {
8357 info!(
8358 module_id = %spec.module_id,
8359 exit_code = ?exit_report.code,
8360 exit_signal = ?exit_report.signal,
8361 "supervised module exited cleanly"
8362 );
8363 if let Err(err) = publish_after_terminal_record(
8364 &spec.module_id,
8365 || {
8366 record_terminal(
8367 &spec.module_id,
8368 terminal_ring,
8369 spawn_events,
8370 &exit_report,
8371 TerminalDisposition::Stopped,
8372 )
8373 },
8374 || {
8375 update_snapshot(snapshot, Some(&spec.module_id), |state| {
8376 state.state = ModuleState::Stopped;
8377 clear_current_process_facts(state);
8378 state.last_exit = Some(exit_report.clone());
8379 })
8380 },
8381 ) {
8382 error!(module_id = %spec.module_id, error = %err, "failed to publish clean module exit");
8383 }
8384 let registration_released = match wait_for_registration_release(
8385 registry,
8386 &spec.module_id,
8387 REGISTRY_RELEASE_TIMEOUT,
8388 )
8389 .await
8390 {
8391 Ok(()) => true,
8392 Err(err) => {
8393 warn!(module_id = %spec.module_id, error = %err, "registration still active after clean exit");
8394 false
8395 }
8396 };
8397 NextAction::Stop {
8398 registration_released,
8399 }
8400 }
8401 ExitKind::Clean | ExitKind::Crash => {
8402 if unrequested_clean_exit_of_protocol_none {
8403 warn!(
8404 module_id = %spec.module_id,
8405 exit_code = ?exit_report.code,
8406 exit_signal = ?exit_report.signal,
8407 "protocol-none module exited cleanly without a stop request; handling it as a crash"
8408 );
8409 } else {
8410 warn!(
8411 module_id = %spec.module_id,
8412 exit_code = ?exit_report.code,
8413 exit_signal = ?exit_report.signal,
8414 "supervised module exited abnormally (crash)"
8415 );
8416 }
8417 let mut restart_schedule = None;
8418 let mut disposition = TerminalDisposition::Disabled;
8419 let mut final_state = ModuleState::Disabled;
8420 let mut disposition_detail = lock_snapshot(snapshot)
8424 .ok()
8425 .and_then(|mut state| state.spawn_failure.take());
8426 let now = Instant::now();
8427 if let Err(err) = update_snapshot(snapshot, Some(&spec.module_id), |state| {
8428 if state.enabled {
8429 if let Some(schedule) = state.next_crash_restart(&policy, now) {
8430 state.respawn_pending = true;
8436 final_state = ModuleState::Restarting;
8437 restart_schedule = Some(schedule);
8438 disposition = TerminalDisposition::Restarting;
8439 } else {
8440 final_state = ModuleState::Failed;
8441 disposition = TerminalDisposition::Failed;
8442 let budget = policy.budget_exhausted_detail();
8443 disposition_detail =
8444 Some(disposition_detail.take().map_or_else(
8445 || budget.clone(),
8446 |cause| format!("{cause}; {budget}"),
8447 ));
8448 }
8449 } else {
8450 disposition = TerminalDisposition::Disabled;
8451 }
8452 }) {
8453 error!(module_id = %spec.module_id, error = %err, "failed to record crashed module exit");
8454 return NextAction::Stop {
8455 registration_released: false,
8456 };
8457 }
8458 if disposition == TerminalDisposition::Failed {
8459 error!(
8464 module_id = %spec.module_id,
8465 max_restarts = policy.max_restarts,
8466 window_secs = policy.window.as_secs(),
8467 "module stopped: {}",
8468 policy.budget_exhausted_detail()
8469 );
8470 }
8471 if let Err(err) = publish_after_terminal_record(
8472 &spec.module_id,
8473 || {
8474 record_terminal_with_detail(
8475 &spec.module_id,
8476 terminal_ring,
8477 spawn_events,
8478 &exit_report,
8479 disposition,
8480 disposition_detail,
8481 )
8482 },
8483 || {
8484 update_snapshot(snapshot, Some(&spec.module_id), |state| {
8485 state.state = final_state;
8486 clear_current_process_facts(state);
8487 if final_state == ModuleState::Failed
8488 && daemon_initiated
8489 && !roster.is_closed()
8490 {
8491 state.failed_retry.schedule(&spec.module_id);
8492 }
8493 state.last_exit = Some(exit_report.clone());
8494 })
8495 },
8496 ) {
8497 error!(module_id = %spec.module_id, error = %err, "failed to publish crashed module exit");
8498 }
8499
8500 if let Some(schedule) = restart_schedule {
8501 NextAction::Restart {
8502 schedule: Some(schedule),
8503 }
8504 } else {
8505 let registration_released = match wait_for_registration_release(
8506 registry,
8507 &spec.module_id,
8508 REGISTRY_RELEASE_TIMEOUT,
8509 )
8510 .await
8511 {
8512 Ok(()) => true,
8513 Err(err) => {
8514 warn!(module_id = %spec.module_id, error = %err, "registration still active after failed module");
8515 false
8516 }
8517 };
8518 NextAction::Stop {
8519 registration_released,
8520 }
8521 }
8522 }
8523 ExitKind::DeliberateSeverance => {
8524 warn!(
8525 module_id = %spec.module_id,
8526 exit_code = ?exit_report.code,
8527 exit_signal = ?exit_report.signal,
8528 "supervised module exited after deliberate connection severance"
8529 );
8530 let should_restart = match lock_snapshot(snapshot) {
8531 Ok(state) => state.enabled,
8532 Err(err) => {
8533 error!(module_id = %spec.module_id, error = %err, "failed to read deliberately severed module state");
8534 return NextAction::Stop {
8535 registration_released: false,
8536 };
8537 }
8538 };
8539 let final_state = if should_restart {
8540 ModuleState::Restarting
8541 } else {
8542 ModuleState::Disabled
8543 };
8544 if let Err(err) = publish_after_terminal_record(
8545 &spec.module_id,
8546 || {
8547 record_terminal(
8548 &spec.module_id,
8549 terminal_ring,
8550 spawn_events,
8551 &exit_report,
8552 terminal_disposition(final_state),
8553 )
8554 },
8555 || {
8556 update_snapshot(snapshot, Some(&spec.module_id), |state| {
8557 state.state = final_state;
8558 clear_current_process_facts(state);
8559 state.last_exit = Some(exit_report.clone());
8560 state.lifetime_restarts += 1;
8561 })
8562 },
8563 ) {
8564 error!(module_id = %spec.module_id, error = %err, "failed to publish deliberately severed module exit");
8565 return NextAction::Stop {
8566 registration_released: false,
8567 };
8568 }
8569
8570 if should_restart {
8571 NextAction::Restart { schedule: None }
8572 } else {
8573 let registration_released = match wait_for_registration_release(
8574 registry,
8575 &spec.module_id,
8576 REGISTRY_RELEASE_TIMEOUT,
8577 )
8578 .await
8579 {
8580 Ok(()) => true,
8581 Err(err) => {
8582 warn!(module_id = %spec.module_id, error = %err, "registration still active after deliberately severed module exit");
8583 false
8584 }
8585 };
8586 NextAction::Stop {
8587 registration_released,
8588 }
8589 }
8590 }
8591 }
8592}
8593
8594async fn on_child_exit_during_daemon_shutdown(
8595 spec: &ModuleSpec,
8596 registry: &Registry,
8597 snapshot: &SharedSnapshot,
8598 terminal_ring: &Arc<Mutex<TerminalRing>>,
8599 spawn_events: &SpawnEventFeed,
8600 exit_report: ExitReport,
8601) -> NextAction {
8602 info!(
8603 module_id = %spec.module_id,
8604 exit_code = ?exit_report.code,
8605 exit_signal = ?exit_report.signal,
8606 exit_kind = ?exit_report.kind,
8607 "supervised module exited during daemon shutdown; not restarting it"
8608 );
8609 if let Err(err) = publish_after_terminal_record(
8610 &spec.module_id,
8611 || {
8612 record_terminal(
8613 &spec.module_id,
8614 terminal_ring,
8615 spawn_events,
8616 &exit_report,
8617 TerminalDisposition::DaemonShutdown,
8618 )
8619 },
8620 || {
8621 update_snapshot(snapshot, Some(&spec.module_id), |state| {
8622 state.state = ModuleState::Stopped;
8623 clear_current_process_facts(state);
8624 state.last_exit = Some(exit_report.clone());
8625 })
8626 },
8627 ) {
8628 error!(module_id = %spec.module_id, error = %err, "failed to publish module exit during daemon shutdown");
8629 }
8630 let registration_released =
8631 wait_for_registration_release(registry, &spec.module_id, REGISTRY_RELEASE_TIMEOUT)
8632 .await
8633 .is_ok();
8634 NextAction::Stop {
8635 registration_released,
8636 }
8637}
8638
8639fn publish_after_terminal_record<T>(
8644 module_id: &str,
8645 record: impl FnOnce(),
8646 publish: impl FnOnce() -> T,
8647) -> T {
8648 let recorded = std::panic::catch_unwind(std::panic::AssertUnwindSafe(record));
8649 let published = publish();
8650 if let Err(panic) = recorded {
8651 error!(
8652 module_id,
8653 "terminal recording panicked; published final state before resuming unwind"
8654 );
8655 std::panic::resume_unwind(panic);
8656 }
8657 published
8658}
8659
8660fn fail_after_wait_error(
8661 module_id: &str,
8662 snapshot: &SharedSnapshot,
8663 terminal_ring: &Arc<Mutex<TerminalRing>>,
8664 spawn_events: &SpawnEventFeed,
8665 detail: String,
8666 failure: &FailureContext,
8667) {
8668 let report = wait_error_exit_report();
8669 publish_after_terminal_record(
8670 module_id,
8671 || {
8672 record_terminal_with_detail(
8673 module_id,
8674 terminal_ring,
8675 spawn_events,
8676 &report,
8677 TerminalDisposition::Failed,
8678 Some(detail),
8679 )
8680 },
8681 || fail_snapshot(snapshot, Some(module_id), Some(report.clone()), failure),
8682 );
8683}
8684
8685#[cfg(test)]
8686fn record_wait_error_terminal(
8687 module_id: &str,
8688 terminal_ring: &Arc<Mutex<TerminalRing>>,
8689 spawn_events: &SpawnEventFeed,
8690) {
8691 record_terminal(
8692 module_id,
8693 terminal_ring,
8694 spawn_events,
8695 &wait_error_exit_report(),
8696 TerminalDisposition::Failed,
8697 );
8698}
8699
8700fn record_terminal(
8701 module_id: &str,
8702 terminal_ring: &Arc<Mutex<TerminalRing>>,
8703 spawn_events: &SpawnEventFeed,
8704 exit_report: &ExitReport,
8705 disposition: TerminalDisposition,
8706) {
8707 record_terminal_with_detail(
8708 module_id,
8709 terminal_ring,
8710 spawn_events,
8711 exit_report,
8712 disposition,
8713 None,
8714 );
8715}
8716
8717fn durable_terminal_history_of(
8721 terminal_ring: &Mutex<TerminalRing>,
8722 module_id: &str,
8723) -> subc_control::TerminalHistory {
8724 let read = terminal_ring
8725 .lock()
8726 .unwrap_or_else(|p| p.into_inner())
8727 .capture_durable_history();
8728 read.read(module_id)
8729}
8730
8731fn record_terminal_with_detail(
8732 module_id: &str,
8733 terminal_ring: &Arc<Mutex<TerminalRing>>,
8734 spawn_events: &SpawnEventFeed,
8735 exit_report: &ExitReport,
8736 disposition: TerminalDisposition,
8737 disposition_detail: Option<String>,
8738) {
8739 #[cfg(test)]
8740 terminal_history_tests::before_terminal_record(spawn_events);
8741 spawn_events.emit_exited(module_id, exit_report.code, exit_report.signal);
8742 let record = TerminalRecord {
8743 exit_code: exit_report.code,
8744 exit_signal: exit_report.signal,
8745 at_ms: exit_report.at_ms,
8746 disposition,
8747 exit_kind: exit_report.kind.into(),
8748 disposition_detail,
8749 };
8750 terminal_ring
8751 .lock()
8752 .unwrap_or_else(|poisoned| poisoned.into_inner())
8753 .record_exit(module_id, record);
8754}
8755
8756fn untrack_if_registration_released(
8757 process_liveness: &SupervisorProcessLiveness,
8758 registry: &Registry,
8759 module_id: &str,
8760 snapshot: &SharedSnapshot,
8761) {
8762 match registry.get_module(module_id) {
8763 Ok(None) => process_liveness.untrack_if_current(module_id, snapshot),
8764 Ok(Some(_)) => {}
8765 Err(err) => {
8766 warn!(module_id, error = %err, "could not determine whether supervisor liveness can be untracked");
8767 }
8768 }
8769}
8770
8771#[cfg(test)]
8785fn apply_wire_spawn_args(
8786 command: &mut Command,
8787 spec: &ModuleSpec,
8788 connection_file_path: Option<&std::path::Path>,
8789 handle: Option<&SupervisorHandle>,
8790) -> Result<Option<NonceHandoff>, SuperviseError> {
8791 apply_wire_spawn_args_for_role(
8792 command,
8793 spec,
8794 connection_file_path,
8795 handle,
8796 SpawnRole::Plain,
8797 )
8798}
8799
8800#[cfg(unix)]
8805type NonceHandoff = subc_os::LaunchNonceHandoff;
8806#[cfg(windows)]
8807type NonceHandoff = subc_os::LaunchNoncePipeHandoff;
8808#[cfg(not(any(unix, windows)))]
8809type NonceHandoff = std::convert::Infallible;
8810
8811fn apply_wire_spawn_args_for_role(
8832 command: &mut Command,
8833 spec: &ModuleSpec,
8834 connection_file_path: Option<&std::path::Path>,
8835 handle: Option<&SupervisorHandle>,
8836 role: SpawnRole,
8837) -> Result<Option<NonceHandoff>, SuperviseError> {
8838 command.env(SUBC_MODULE_ID_ENV, &spec.module_id);
8839 command.env_remove(subc_os::LAUNCH_NONCE_FD_ENV);
8844 #[cfg(windows)]
8845 {
8846 command.env_remove(subc_os::LAUNCH_NONCE_PIPE_ENV);
8847 command.env_remove(subc_os::LAUNCH_NONCE_PIPE_FALLBACK_ENV);
8848 }
8849 command.env_remove(SUBC_LAUNCH_NONCE_ENV);
8851 if spec.protocol == ModuleProtocol::None {
8852 return Ok(None);
8853 }
8854 if let Some(connection_file_path) = connection_file_path {
8855 command.arg(SUBC_ARG).arg(connection_file_path);
8856 }
8857
8858 let nonce = generate_launch_nonce()?;
8862 if let Some(handle) = handle {
8863 match role {
8864 SpawnRole::Plain => {
8865 handle.set_spawn_nonce(&spec.module_id, nonce.clone());
8866 if spec.reserved {
8867 handle.set_reserved_nonce(&spec.module_id, nonce.clone());
8868 }
8869 }
8870 SpawnRole::SwapCandidate => handle.open_swap(&spec.module_id, nonce.clone()),
8871 }
8872 }
8873 #[cfg(unix)]
8874 let handoff = {
8875 let handoff =
8876 subc_os::LaunchNonceHandoff::new(&nonce).map_err(|source| SuperviseError::Spawn {
8877 program: spec.program.clone(),
8878 source,
8879 cgroup_path: None,
8880 })?;
8881 command.env(subc_os::LAUNCH_NONCE_FD_ENV, handoff.fd_env_value());
8882 Some(handoff)
8883 };
8884 #[cfg(windows)]
8885 let handoff = prepare_windows_nonce_pipe(command, spec, &nonce);
8886 #[cfg(not(any(unix, windows)))]
8887 let handoff = None;
8888 #[cfg(not(unix))]
8892 command.env(SUBC_LAUNCH_NONCE_ENV, nonce);
8893 Ok(handoff)
8894}
8895
8896fn apply_child_env(command: &mut Command, spec: &ModuleSpec) {
8897 command.env_remove(CK_LOG_ENV);
8898 command.env_remove(SUBC_SPAWN_ROLE_ENV);
8905 for (key, value) in &spec.env {
8906 if matches!(
8910 key.as_str(),
8911 CAPTURE_MAX_FILE_MB_ENV | CAPTURE_KEEP_ENV | CAPTURE_MAX_AGE_DAYS_ENV
8912 ) || key == SUBC_SPAWN_ROLE_ENV
8913 {
8914 continue;
8915 }
8916 command.env(key, value);
8917 }
8918}
8919
8920#[cfg(windows)]
8921fn prepare_windows_nonce_pipe(
8922 command: &mut Command,
8923 spec: &ModuleSpec,
8924 nonce: &str,
8925) -> Option<NonceHandoff> {
8926 match subc_os::LaunchNoncePipeHandoff::new(nonce) {
8927 Ok(handoff) => {
8928 command.env(subc_os::LAUNCH_NONCE_PIPE_ENV, handoff.name());
8929 command.env(subc_os::LAUNCH_NONCE_PIPE_FALLBACK_ENV, "env");
8934 Some(handoff)
8935 }
8936 Err(error) => {
8937 warn!(module_id = %spec.module_id, %error, "could not prepare launch nonce pipe; retaining the environment handoff");
8938 None
8939 }
8940 }
8941}
8942
8943#[cfg(windows)]
8944fn start_windows_nonce_pipe(
8945 handoff: Option<NonceHandoff>,
8946 pid: u32,
8947 spec: &ModuleSpec,
8948) -> Option<subc_os::LaunchNoncePipeDelivery> {
8949 handoff.and_then(|handoff| match handoff.serve(pid, REGISTRY_RELEASE_TIMEOUT) {
8953 Ok(delivery) => Some(delivery),
8954 Err(error) => {
8955 warn!(module_id = %spec.module_id, %error, "could not serve launch nonce pipe; retaining the environment handoff");
8956 None
8957 }
8958 })
8959}
8960
8961fn observed_launch_nonce_source(snapshot: &SupervisorSnapshot) -> Option<String> {
8962 #[cfg(windows)]
8963 if snapshot.process_alive && snapshot.spawned_protocol == Some(ModuleProtocol::Subc) {
8964 return Some(
8965 if snapshot
8966 .pipe_consumed
8967 .as_ref()
8968 .is_some_and(|consumed| consumed.get().is_some())
8969 {
8970 "pipe"
8971 } else {
8972 "env"
8973 }
8974 .to_owned(),
8975 );
8976 }
8977 let _ = snapshot;
8978 None
8979}
8980
8981#[derive(Debug, Clone, Copy, PartialEq, Eq)]
8984enum SpawnRole {
8985 Plain,
8986 SwapCandidate,
8987}
8988
8989fn apply_spawn_role(command: &mut Command, role: SpawnRole) {
8992 if role == SpawnRole::SwapCandidate {
8993 command.env(SUBC_SPAWN_ROLE_ENV, SPAWN_ROLE_SWAP_CANDIDATE);
8994 }
8995}
8996
8997fn previous_instance_error(
8998 spec: &ModuleSpec,
8999 pid: u32,
9000 reason: crate::live_children::RetainedReason,
9001) -> SuperviseError {
9002 SuperviseError::Spawn {
9003 program: spec.program.clone(),
9004 source: io::Error::new(io::ErrorKind::WouldBlock, reason.waiting_message(pid)),
9005 cgroup_path: None,
9006 }
9007}
9008
9009fn carries_containment(configured: bool, has_container: bool) -> bool {
9010 configured || has_container
9011}
9012
9013fn recovery_block_error(spec: &ModuleSpec, roster: &ChildRoster) -> Option<SuperviseError> {
9014 if !carries_containment(
9015 spec.carry_containment,
9016 containment::current(&roster.containment).is_some(),
9017 ) {
9018 return None;
9019 }
9020 Some(SuperviseError::Spawn {
9021 program: spec.program.clone(),
9022 source: io::Error::new(io::ErrorKind::WouldBlock, roster.recovery_block()?),
9023 cgroup_path: None,
9024 })
9025}
9026
9027fn spawn_child(
9028 spec: &ModuleSpec,
9029 connection_file_path: Option<&std::path::Path>,
9030 handle: Option<&SupervisorHandle>,
9031 ring: &Arc<Mutex<StderrRing>>,
9032 capture_logs_dir: Option<&std::path::Path>,
9033 roster: &ChildRoster,
9034 #[cfg(target_os = "linux")] cgroup_placement: Option<&subc_cgroup::Placement>,
9035) -> Result<SupervisedChild, SuperviseError> {
9036 #[cfg(all(test, unix))]
9037 if let Some(source) = containment_tests::join_failure(spec, roster) {
9038 containment::terminate_blocking(&roster.containment);
9039 return Err(SuperviseError::Spawn {
9040 program: spec.program.clone(),
9041 source,
9042 cgroup_path: None,
9043 });
9044 }
9045 let result = spawn_child_in_slot(
9046 spec,
9047 connection_file_path,
9048 handle,
9049 ring,
9050 capture_logs_dir,
9051 roster,
9052 #[cfg(target_os = "linux")]
9053 cgroup_placement,
9054 SpawnRole::Plain,
9055 false,
9056 );
9057 if result.is_err() {
9058 if roster.admission_pending(&spec.module_id) {
9059 containment::terminate_deferred(&roster.containment);
9060 } else {
9061 containment::terminate_blocking(&roster.containment);
9062 }
9063 }
9064 result
9065}
9066
9067#[allow(clippy::too_many_arguments)]
9080fn spawn_child_in_slot(
9081 spec: &ModuleSpec,
9082 connection_file_path: Option<&std::path::Path>,
9083 handle: Option<&SupervisorHandle>,
9084 ring: &Arc<Mutex<StderrRing>>,
9085 capture_logs_dir: Option<&std::path::Path>,
9086 roster: &ChildRoster,
9087 #[cfg(target_os = "linux")] cgroup_placement: Option<&subc_cgroup::Placement>,
9088 role: SpawnRole,
9089 alternate_slot: bool,
9090) -> Result<SupervisedChild, SuperviseError> {
9091 if let Some(error) = recovery_block_error(spec, roster) {
9092 return Err(error);
9093 }
9094 if let Some((pid, reason)) = roster.previous_instance(&spec.module_id) {
9095 return Err(previous_instance_error(spec, pid, reason));
9096 }
9097 if roster.is_closed() {
9098 return Err(SuperviseError::Spawn {
9099 program: spec.program.clone(),
9100 source: io::Error::other("the daemon is shutting down; not starting a new process"),
9101 cgroup_path: None,
9102 });
9103 }
9104 let carried = containment::current(&roster.containment);
9105 let carrying = carries_containment(spec.carry_containment, carried.is_some());
9106 if role != SpawnRole::Plain && carrying {
9107 return Err(SuperviseError::InvalidSpec {
9108 reason: "a carried container cannot overlap a swap candidate".to_owned(),
9109 });
9110 }
9111 #[cfg(unix)]
9112 let launch_identity = if carrying && carried.is_none() {
9113 Some(
9114 containment::LaunchIdentity::capture().map_err(|source| SuperviseError::Spawn {
9115 program: spec.program.clone(),
9116 source,
9117 cgroup_path: None,
9118 })?,
9119 )
9120 } else {
9121 None
9122 };
9123 #[cfg(target_os = "linux")]
9124 let cgroup_placement = match &carried {
9125 Some(container) => container.cgroup().map(|(placement, _)| placement),
9126 None => cgroup_placement,
9127 };
9128 #[cfg(unix)]
9129 let group_target = if let Some(container) = carried
9130 .as_ref()
9131 .filter(|container| container.group().is_some())
9132 {
9133 containment::GroupTarget::Pinned(Arc::clone(container))
9134 } else {
9135 #[cfg(target_os = "linux")]
9136 let cgroup_container = carrying && cgroup_placement.is_some();
9137 #[cfg(not(target_os = "linux"))]
9138 let cgroup_container = false;
9139 if cgroup_container {
9140 containment::GroupTarget::NoGroup
9141 } else {
9142 containment::GroupTarget::OwnLeader
9143 }
9144 };
9145 #[cfg(target_os = "linux")]
9146 let cgroup_name = {
9147 if let Some((_, name)) = carried.as_ref().and_then(|container| container.cgroup()) {
9152 name.to_owned()
9153 } else if cgroup_placement.is_none() {
9154 swap::cgroup_name(&spec.module_id, alternate_slot)
9155 } else {
9156 let nonce = generate_launch_nonce()?;
9157 let suffix = u128::from_str_radix(&nonce[..32], 16).expect("hex launch nonce");
9158 let mut end = spec.module_id.len().min(64);
9161 while !spec.module_id.is_char_boundary(end) {
9162 end -= 1;
9163 }
9164 format!(
9165 "{}_{suffix}",
9166 swap::cgroup_name(&spec.module_id[..end], alternate_slot)
9167 )
9168 }
9169 };
9170 #[cfg(not(target_os = "linux"))]
9171 let _ = alternate_slot;
9172 #[cfg(target_os = "macos")]
9173 let (mut command, privacy_exec, exec_ack) = privacy_command(spec, roster)?;
9174 #[cfg(windows)]
9175 let image_capture = windows_spawn_capture(spec).map_err(|source| SuperviseError::Spawn {
9176 program: spec.program.clone(),
9177 source,
9178 cgroup_path: None,
9179 })?;
9180 #[cfg(windows)]
9181 let mut command = Command::new(image_capture.path());
9182 #[cfg(not(any(target_os = "macos", windows)))]
9183 let mut command = Command::new(&spec.program);
9184 command.args(&spec.args);
9185 apply_child_env(&mut command, spec);
9215 apply_spawn_role(&mut command, role);
9216 let nonce_handoff =
9217 apply_wire_spawn_args_for_role(&mut command, spec, connection_file_path, handle, role)?;
9218
9219 #[cfg(target_os = "linux")]
9220 let cgroup_path = cgroup_placement
9221 .map(|placement| {
9222 if let Some(container) = &carried {
9223 let path = container
9224 .cgroup_path()
9225 .ok_or_else(|| io::Error::other("carried container has no cgroup path"))?;
9226 placement.existing_module_path(path)?.ok_or_else(|| {
9227 io::Error::new(
9228 io::ErrorKind::NotFound,
9229 "carried cgroup disappeared before replacement",
9230 )
9231 })
9232 } else {
9233 if carrying {
9234 containment::prepare_cgroup_path(placement, &cgroup_name)
9235 } else {
9236 placement.module_path(&cgroup_name)
9237 }
9238 }
9239 })
9240 .transpose()
9241 .map_err(|source| SuperviseError::Cgroup {
9242 module_id: spec.module_id.clone(),
9243 source,
9244 })?;
9245 #[cfg(not(target_os = "linux"))]
9246 let cgroup_path: Option<PathBuf> = None;
9247 #[cfg(unix)]
9248 let mut admission_gate = if carrying {
9249 Some(
9250 admission::Gate::install(&mut command, spec, roster).map_err(|source| {
9251 SuperviseError::Spawn {
9252 program: spec.program.clone(),
9253 source,
9254 cgroup_path: cgroup_path.clone(),
9255 }
9256 })?,
9257 )
9258 } else {
9259 None
9260 };
9261 #[cfg(target_os = "macos")]
9262 exec_ack.install(command.as_std_mut());
9263 #[cfg(target_os = "linux")]
9264 if let Some(path) = &cgroup_path {
9265 if let Err(error) = apply_cgroup_placement(&mut command, spec, path) {
9266 if let Some(placement) = cgroup_placement.filter(|_| carried.is_none()) {
9267 remove_module_cgroup(placement, &cgroup_name);
9268 }
9269 return Err(error);
9270 }
9271 }
9272
9273 let output_sink = if let Some(logs_dir) = capture_logs_dir {
9274 let path = logs_dir.join(format!("{}.stderr.log", spec.module_id));
9275 match ChildOutputSink::open(&path, capture_retention(spec)) {
9276 Ok(sink) => sink,
9277 Err(error) => {
9278 warn!(
9279 module_id = %spec.module_id,
9280 path = %path.display(),
9281 error = %error,
9282 "could not open child output capture file; forwarding to stderr"
9283 );
9284 ChildOutputSink::Stderr
9285 }
9286 }
9287 } else {
9288 ChildOutputSink::Stderr
9289 };
9290
9291 command.stdout(Stdio::piped());
9292 command.stderr(Stdio::piped());
9293 command.kill_on_drop(!carrying);
9294 #[cfg(unix)]
9311 command.process_group(
9312 carried
9313 .as_ref()
9314 .map(|container| container.verified_join_group())
9315 .transpose()
9316 .map_err(|source| SuperviseError::Spawn {
9317 program: spec.program.clone(),
9318 source,
9319 cgroup_path: cgroup_path.clone(),
9320 })?
9321 .flatten()
9322 .unwrap_or(0) as i32,
9323 );
9324 command.stdin(Stdio::null());
9325 #[cfg(unix)]
9326 if let Some(gate) = &mut admission_gate {
9327 gate.install_placement(&mut command);
9328 }
9329 #[cfg(unix)]
9333 if let Some(handoff) = nonce_handoff {
9334 handoff.install_last(command.as_std_mut());
9335 }
9336 #[cfg(not(any(unix, windows)))]
9337 let _ = nonce_handoff;
9338
9339 #[cfg(windows)]
9344 {
9345 subc_jobobject::ensure_root_job().map_err(|source| SuperviseError::Spawn {
9346 program: spec.program.clone(),
9347 source,
9348 cgroup_path: None,
9349 })?;
9350 subc_jobobject::suspend_on_create_async(&mut command);
9351 }
9352 let spawned_at = Instant::now();
9356 #[cfg(all(test, unix))]
9357 containment_tests::record_spawn_attempt(spec, &mut command);
9358 #[cfg(unix)]
9359 let spawned = match admission_gate {
9360 Some(gate) => gate
9361 .spawn(
9362 command,
9363 spec,
9364 roster,
9365 launch_identity,
9366 #[cfg(target_os = "linux")]
9367 cgroup_placement
9368 .zip(cgroup_path.as_ref())
9369 .map(|(placement, path)| {
9370 (placement.clone(), cgroup_name.clone(), path.clone())
9371 }),
9372 )
9373 .map(|(child, admitted)| (child, Some(admitted))),
9374 None => command.spawn().map(|child| (child, None)),
9375 };
9376 #[cfg(not(unix))]
9377 let spawned = command.spawn();
9378 let spawned = match spawned {
9379 Ok(child) => child,
9380 Err(source) => {
9381 if carried
9382 .as_ref()
9383 .and_then(|container| container.group())
9384 .is_some()
9385 {
9386 containment::terminate_blocking(&roster.containment);
9387 }
9388 #[cfg(target_os = "linux")]
9389 if let Some(placement) = cgroup_placement.filter(|_| carried.is_none()) {
9390 remove_module_cgroup(placement, &cgroup_name);
9391 }
9392 return Err(SuperviseError::Spawn {
9393 program: spec.program.clone(),
9394 source,
9395 cgroup_path,
9396 });
9397 }
9398 };
9399 #[cfg(unix)]
9400 let (mut child, mut admitted) = spawned;
9401 #[cfg(unix)]
9402 let group_target = admitted
9403 .as_ref()
9404 .map(|attempt| attempt.group_target())
9405 .unwrap_or(group_target);
9406 #[cfg(not(unix))]
9407 let mut child = spawned;
9408 #[cfg(test)]
9409 spawn_record_tests::after_create(&child, spec);
9410 #[cfg(target_os = "macos")]
9415 drop(exec_ack);
9416
9417 #[cfg(windows)]
9422 let windows_image = match image_capture.bind_tokio(&child) {
9423 Ok(image) => Arc::new(image),
9424 Err(source) => {
9425 kill_and_reap_rejected_child(&mut child);
9426 return Err(SuperviseError::Spawn {
9427 program: spec.program.clone(),
9428 source,
9429 cgroup_path: None,
9430 });
9431 }
9432 };
9433 #[cfg(windows)]
9436 let job = contain_spawned_child(&mut child, spec, carried.as_ref())?;
9437 let spawned_at_ms = unix_ms_now();
9438 #[cfg(not(windows))]
9439 let spawned_from = spec.program.clone();
9440 #[cfg(windows)]
9441 let spawned_from = windows_image.path().to_path_buf();
9442 #[cfg(not(windows))]
9443 let spawned_file_identity = spawned_file_identity(&spawned_from);
9444 #[cfg(windows)]
9445 let spawned_file_identity = Some(SpawnedFileIdentity {
9446 device: 0,
9447 inode: 0,
9448 windows_image: Some(Arc::clone(&windows_image)),
9449 });
9450 let pid = child.id().ok_or_else(|| SuperviseError::Spawn {
9451 program: spec.program.clone(),
9452 source: io::Error::other("spawned child exposed no live pid"),
9453 cgroup_path: cgroup_path.clone(),
9454 })?;
9455 #[cfg(windows)]
9456 let nonce_delivery = start_windows_nonce_pipe(nonce_handoff, pid, spec);
9457 #[cfg(not(windows))]
9458 let process_start_time = admitted
9459 .as_ref()
9460 .map(|admitted| admitted.birth)
9461 .or_else(|| crate::provenance::process_start_time(pid));
9462 #[cfg(windows)]
9463 let process_start_time = Some(windows_image.start_time());
9464 let process_identity = process_start_time.map(|start_time| ProcessIdentity { pid, start_time });
9465 #[cfg(all(test, target_os = "macos"))]
9466 privacy_exec_boundary_tests::before_image_sample(spec, pid);
9467 #[cfg(not(windows))]
9472 let recorded_image = observe_spawned_image(pid);
9473 #[cfg(target_os = "macos")]
9477 let recorded_image = if privacy_exec.is_some() {
9478 None
9479 } else {
9480 recorded_image
9481 };
9482 #[cfg(target_os = "linux")]
9483 let recorded_cgroup_name = cgroup_path.as_ref().map(|_| cgroup_name.clone());
9484 #[cfg(not(target_os = "linux"))]
9485 let recorded_cgroup_name = None;
9486 #[cfg(unix)]
9487 let preadmitted = admitted
9488 .as_mut()
9489 .and_then(|admitted| admitted.leader.take());
9490 #[cfg(not(unix))]
9491 let preadmitted: Option<crate::child_roster::RosterGuard> = None;
9492 let roster_guard = if let Some(guard) = preadmitted {
9493 Ok(guard)
9494 } else {
9495 roster.admit(
9496 spec.module_id.clone(),
9497 pid,
9498 spec.protocol,
9499 process_start_time,
9500 crate::child_roster::RecordedIdentity {
9501 container: None,
9502 #[cfg(unix)]
9503 carried: None,
9504 #[cfg(not(windows))]
9505 start_time: recorded_image.map(|image| image.start_time),
9506 #[cfg(windows)]
9507 start_time: process_start_time,
9508 #[cfg(not(windows))]
9509 executable: recorded_image
9510 .and_then(|image| image.executable)
9511 .map(crate::live_children::ExecutableIdentity::from),
9512 #[cfg(windows)]
9513 executable: Some(crate::live_children::ExecutableIdentity::from(
9514 windows_image.identity(),
9515 )),
9516 cgroup_name: recorded_cgroup_name,
9517 #[cfg(target_os = "linux")]
9518 cgroup_placement: cgroup_placement.cloned(),
9519 },
9520 )
9521 };
9522 let roster_guard = match roster_guard {
9523 Ok(guard) => guard,
9524 Err(failure) => {
9525 return failure.with_cleanup(|source| {
9526 #[cfg(windows)]
9527 if let Some(job) = &job {
9528 let _ = job.lock().unwrap_or_else(|p| p.into_inner()).terminate();
9529 }
9530 #[cfg(target_os = "linux")]
9531 kill_module_cgroup(cgroup_placement, &cgroup_name);
9532 #[cfg(unix)]
9533 let rejected_group = group_target.clone();
9534 kill_and_reap_rejected_child(
9535 &mut child,
9536 #[cfg(unix)]
9537 rejected_group,
9538 );
9539 #[cfg(target_os = "linux")]
9540 if let Some(placement) = cgroup_placement.filter(|_| carried.is_none()) {
9541 remove_module_cgroup(placement, &cgroup_name);
9542 }
9543 Err(SuperviseError::Spawn {
9544 program: spec.program.clone(),
9545 source: io::Error::other(format!(
9546 "could not durably record module child: {source}"
9547 )),
9548 cgroup_path,
9549 })
9550 });
9551 }
9552 };
9553 #[cfg(unix)]
9554 if admitted.is_some() {
9555 if let Some(image) = recorded_image {
9556 if let Err(source) = roster_guard.confirm_executable(image) {
9557 kill_and_reap_rejected_child(&mut child, group_target.clone());
9558 return Err(SuperviseError::Spawn {
9559 program: spec.program.clone(),
9560 source,
9561 cgroup_path,
9562 });
9563 }
9564 }
9565 }
9566 if roster.is_closed() {
9571 if carrying {
9572 #[cfg(unix)]
9573 let rejected_group = admitted
9574 .as_ref()
9575 .map(|admitted| admitted.rejection_target())
9576 .unwrap_or_else(|| group_target.clone());
9577 #[cfg(windows)]
9578 if let Some(job) = &job {
9579 let _ = job.lock().unwrap_or_else(|p| p.into_inner()).terminate();
9580 }
9581 kill_and_reap_rejected_child(
9582 &mut child,
9583 #[cfg(unix)]
9584 rejected_group,
9585 );
9586 #[cfg(all(test, unix))]
9587 let _ =
9588 roster.admission_event_for_test(&spec.module_id, "after_rejected_child", Some(pid));
9589 drop(roster_guard);
9590 return Err(SuperviseError::Spawn {
9591 program: spec.program.clone(),
9592 source: io::Error::other(
9593 "the daemon began shutting down during carried admission; ended the container",
9594 ),
9595 cgroup_path,
9596 });
9597 }
9598 #[cfg(target_os = "linux")]
9600 kill_module_cgroup(cgroup_placement, &cgroup_name);
9601 if let Err(error) = child.start_kill() {
9602 debug!(module_id = %spec.module_id, pid, %error, "kill of a process spawned during daemon shutdown failed; it may already have exited");
9603 }
9604 #[cfg(target_os = "linux")]
9605 if let Some(placement) = cgroup_placement.filter(|_| carried.is_none()) {
9606 while matches!(child.try_wait(), Ok(None)) {
9610 std::thread::yield_now();
9611 }
9612 if matches!(
9613 subc_cgroup::kill_module(Some(placement), &cgroup_name),
9614 subc_cgroup::KillOutcome::Killed
9615 ) {
9616 if let Ok(path) = placement.module_path(&cgroup_name) {
9617 while std::fs::read_to_string(path.join("cgroup.events"))
9618 .ok()
9619 .is_some_and(|events| events.lines().any(|line| line == "populated 1"))
9620 {
9621 std::thread::yield_now();
9622 }
9623 }
9624 }
9625 remove_module_cgroup(placement, &cgroup_name);
9626 }
9627 drop(roster_guard);
9628 return Err(SuperviseError::Spawn {
9629 program: spec.program.clone(),
9630 source: io::Error::other(
9631 "the daemon began shutting down while this process was starting; ended it",
9632 ),
9633 cgroup_path,
9634 });
9635 }
9636
9637 let stdout_pump = match child.stdout.take() {
9638 Some(stdout) => Some(tokio::spawn(pump_stdout_to(stdout, output_sink.clone()))),
9639 None => {
9640 warn!(
9641 module_id = %spec.module_id,
9642 "spawned child exposed no stdout pipe; file capture will be incomplete"
9643 );
9644 None
9645 }
9646 };
9647 let stderr_pump = match child.stderr.take() {
9648 Some(stderr) => {
9649 let generation = ring
9650 .lock()
9651 .unwrap_or_else(|poisoned| poisoned.into_inner())
9652 .begin_process();
9653 Some(StderrPump {
9654 task: tokio::spawn(pump_stderr_to(
9655 stderr,
9656 Arc::clone(ring),
9657 generation,
9658 output_sink,
9659 )),
9660 generation,
9661 })
9662 }
9663 None => {
9664 ring.lock()
9668 .unwrap_or_else(|poisoned| poisoned.into_inner())
9669 .mark_not_captured("stderr pipe was not available on spawn");
9670 warn!(
9671 module_id = %spec.module_id,
9672 "spawned child exposed no stderr pipe; tail will be unavailable"
9673 );
9674 None
9675 }
9676 };
9677
9678 let exec = Arc::new(crate::activity::ExecOutcome::new());
9679 #[cfg(target_os = "macos")]
9680 let pending_exec = privacy_exec.is_some();
9681 #[cfg(not(target_os = "macos"))]
9682 let pending_exec = false;
9683 if !pending_exec {
9684 exec.finish(crate::activity::ExecState::CommittedExec);
9685 }
9686 #[cfg(windows)]
9687 let subtree_target = job
9688 .as_ref()
9689 .map(|job| crate::subtree_resources::Target::Job(Arc::downgrade(job)))
9690 .unwrap_or(crate::subtree_resources::Target::Unavailable("no_job"));
9691 #[cfg(target_os = "linux")]
9692 let subtree_target = cgroup_path
9693 .map(crate::subtree_resources::Target::Cgroup)
9694 .unwrap_or(crate::subtree_resources::Target::Unavailable("no_cgroup"));
9695 #[cfg(not(any(target_os = "linux", windows)))]
9696 let subtree_target = crate::subtree_resources::Target::Unavailable("unsupported_platform");
9697 let mut supervised = SupervisedChild {
9698 subtree_target,
9699 child: containment::ProcessChild::plain(child, pid),
9700 exec: crate::activity::ExecOwner(exec),
9701 #[cfg(windows)]
9702 nonce_delivery,
9703 protocol: spec.protocol,
9704 #[cfg(target_os = "linux")]
9705 module_id: cgroup_name,
9706 #[cfg(target_os = "linux")]
9707 cgroup_placement: cgroup_placement.cloned(),
9708 #[cfg(windows)]
9709 job,
9710 stdout_pump,
9711 stderr_pump,
9712 stderr_ring: Arc::clone(ring),
9713 spawned_at,
9714 registration_deadline: None,
9715 registration_watch: None,
9716 spawned_at_ms,
9717 spawned_from,
9718 launch_spec_digest: LaunchSpecDigest::from_spec(spec),
9719 spawned_file_identity,
9720 process_start_time,
9721 process_identity,
9722 pid,
9723 roster_guard: Some(roster_guard),
9724 #[cfg(target_os = "macos")]
9725 privacy_exec,
9726 #[cfg(target_os = "macos")]
9727 report_ready: Arc::new(OnceLock::new()),
9728 spawn_failure: None,
9729 };
9730 if let Err(error) = containment::adopt(&mut supervised, spec, roster) {
9731 supervised.child.reject(
9732 #[cfg(unix)]
9733 group_target,
9734 );
9735 #[cfg(all(test, unix))]
9736 let _ = roster.admission_event_for_test(&spec.module_id, "after_rejected_child", Some(pid));
9737 return Err(error);
9738 }
9739 #[cfg(unix)]
9740 if let Some(admitted) = admitted.take() {
9741 admitted.disarm();
9742 }
9743 #[cfg(windows)]
9744 if let Err(error) = resume_suspended_child(pid, spec) {
9745 containment::terminate_blocking(&roster.containment);
9746 let _ = supervised.child.start_kill();
9747 return Err(error);
9748 }
9749 Ok(supervised)
9750}
9751
9752#[cfg(test)]
9753mod spawn_record_tests {
9754 use super::*;
9755
9756 thread_local! {
9757 static LAST_PID: std::cell::Cell<Option<u32>> = const { std::cell::Cell::new(None) };
9758 }
9759
9760 pub(super) fn after_create(child: &Child, _spec: &ModuleSpec) {
9761 LAST_PID.set(child.id());
9762 #[cfg(windows)]
9763 if let Some((_, address)) = _spec
9764 .env
9765 .iter()
9766 .find(|(key, _)| key == "SUBC_TEST_AFTER_CREATE")
9767 {
9768 use std::io::{Read, Write};
9769 let mut stream = std::net::TcpStream::connect(address).unwrap();
9770 stream
9771 .write_all(&child.id().unwrap().to_ne_bytes())
9772 .unwrap();
9773 let mut release = [0];
9774 stream.read_exact(&mut release).unwrap();
9775 }
9776 }
9777
9778 #[cfg(windows)]
9779 pub(super) fn assignment_failpoint(spec: &ModuleSpec) -> io::Result<()> {
9780 if spec
9781 .env
9782 .iter()
9783 .any(|(key, _)| key == "SUBC_TEST_ASSIGNMENT_FAILURE")
9784 {
9785 Err(io::Error::other("injected module job assignment failure"))
9786 } else {
9787 Ok(())
9788 }
9789 }
9790
9791 fn spec() -> ModuleSpec {
9792 ModuleSpec {
9793 module_id: "spawn-record-worker".into(),
9794 #[cfg(unix)]
9795 program: "/bin/sleep".into(),
9796 #[cfg(windows)]
9797 program: std::env::current_exe().unwrap(),
9798 #[cfg(unix)]
9799 args: vec!["60".into()],
9800 #[cfg(windows)]
9801 args: vec![
9802 "--exact".into(),
9803 "supervise::spawn_record_tests::windows_store_touch_fixture".into(),
9804 "--nocapture".into(),
9805 ],
9806 env: vec![],
9807 reserved: false,
9808 reserved_prefixes: vec![],
9809 protocol: ModuleProtocol::None,
9810 overlap: Default::default(),
9811 carry_containment: false,
9812 }
9813 }
9814
9815 #[tokio::test]
9816 async fn record_sync_failure_fails_spawn_and_reaps_child() {
9817 let home = cortexkit_test_support::ScratchDir::new("record-sync-failure");
9818 let supervisor =
9819 Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default())
9820 .with_live_children_record(home.join("live-children.json"));
9821 crate::live_children::fail_next_file_sync();
9822 let result = supervisor.spawn(spec());
9823 let failed = result.is_err();
9824 if let Ok(module) = result {
9825 module.stop().await.unwrap();
9826 }
9827 assert!(failed, "record failure must fail spawn");
9828 let pid = LAST_PID.get().expect("the real OS child was created");
9829 assert!(
9830 subc_os::Process::open(pid).unwrap().is_none(),
9831 "child {pid} survived rejected admission or was not reaped"
9832 );
9833 }
9834
9835 #[cfg(target_os = "macos")]
9836 #[test]
9837 #[ignore = "manual on-device latency measurement"]
9838 fn durable_record_write_latency_50() {
9839 let home = cortexkit_test_support::ScratchDir::new("record-latency");
9840 let path = home.join("live-children.json");
9841 let children = vec![crate::live_children::LiveChild {
9842 module_id: "latency-worker".into(),
9843 pid: std::process::id(),
9844 protocol: ModuleProtocol::None,
9845 start_time: Some(1),
9846 executable: None,
9847 cgroup_name: None,
9848 container: None,
9849 }];
9850 let mut samples = Vec::new();
9851 for _ in 0..50 {
9852 let start = std::time::Instant::now();
9853 crate::live_children::write_record(&path, &children).unwrap();
9854 samples.push(start.elapsed());
9855 }
9856 samples.sort();
9857 eprintln!(
9858 "durable_record_write_latency_50: n=50 median={:?} p95={:?}",
9859 (samples[24] + samples[25]) / 2,
9860 samples[47]
9861 );
9862 }
9863
9864 #[cfg(windows)]
9865 #[test]
9866 fn windows_store_touch_fixture() {
9867 if let Some(path) = std::env::var_os("SUBC_TEST_STORE_TOUCH") {
9868 std::fs::write(path, b"module ran").unwrap();
9869 }
9870 }
9871
9872 #[cfg(windows)]
9873 #[tokio::test]
9874 async fn windows_daemon_fixture() {
9875 let Ok(address) = std::env::var("SUBC_TEST_DAEMON_BARRIER") else {
9876 return;
9877 };
9878 let mut spec = spec();
9879 spec.env.push(("SUBC_TEST_AFTER_CREATE".into(), address));
9880 let supervisor =
9881 Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default());
9882 let _module = supervisor.spawn(spec).unwrap();
9883 }
9884
9885 #[cfg(windows)]
9886 #[test]
9887 fn windows_daemon_death_before_module_job_assignment_leaves_no_child() {
9888 use std::io::Read;
9889 let listener = std::net::TcpListener::bind("127.0.0.1:0").unwrap();
9890 let mut command = std::process::Command::new(std::env::current_exe().unwrap());
9893 command
9894 .args([
9895 "--exact",
9896 "supervise::spawn_record_tests::windows_daemon_fixture",
9897 "--nocapture",
9898 ])
9899 .env(
9900 "SUBC_TEST_DAEMON_BARRIER",
9901 listener.local_addr().unwrap().to_string(),
9902 );
9903 let mut daemon = subc_jobobject::spawn_contained(&mut command).unwrap();
9904 listener.set_nonblocking(true).unwrap();
9905 let deadline = std::time::Instant::now() + Duration::from_secs(30);
9906 let mut stream = loop {
9907 if let Ok((stream, _)) = listener.accept() {
9908 break stream;
9909 }
9910 assert!(
9911 daemon.child.try_wait().unwrap().is_none(),
9912 "daemon failed before rendezvous"
9913 );
9914 assert!(
9915 std::time::Instant::now() < deadline,
9916 "daemon did not reach CreateProcess failpoint"
9917 );
9918 std::thread::yield_now();
9919 };
9920 stream
9921 .set_read_timeout(Some(Duration::from_secs(30)))
9922 .unwrap();
9923 let mut pid = [0; 4];
9924 stream.read_exact(&mut pid).unwrap();
9925 let pid = u32::from_ne_bytes(pid);
9926 assert!(subc_jobobject::process_exists(pid));
9927 daemon.child.kill().unwrap();
9930 daemon.child.wait().unwrap();
9931 let exited = subc_jobobject::wait_for_process_exit(pid, Duration::from_secs(10));
9932 if !exited {
9933 daemon.job.terminate().unwrap();
9934 }
9935 assert!(exited, "suspended child {pid} outlived hard-killed daemon");
9936 }
9937
9938 #[cfg(windows)]
9939 #[tokio::test]
9940 async fn windows_containment_failure_never_resumes() {
9941 let home = cortexkit_test_support::ScratchDir::new("assignment-refusal");
9942 let sentinel = home.join("store-touched");
9943 let mut spec = spec();
9944 spec.env
9945 .push(("SUBC_TEST_ASSIGNMENT_FAILURE".into(), "1".into()));
9946 spec.env.push((
9947 "SUBC_TEST_STORE_TOUCH".into(),
9948 sentinel.display().to_string(),
9949 ));
9950 let supervisor =
9951 Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default());
9952 let result = supervisor.spawn(spec);
9953 if let Ok(module) = result {
9956 let _ = tokio::time::timeout(Duration::from_secs(10), async {
9957 while !sentinel.exists() && module.state().unwrap() == ModuleState::Running {
9958 tokio::task::yield_now().await;
9959 }
9960 })
9961 .await;
9962 let _ = module.stop().await;
9963 panic!(
9964 "assignment failure resumed a module; store touched={}",
9965 sentinel.exists()
9966 );
9967 }
9968 assert!(
9969 !sentinel.exists(),
9970 "refused module executed store-touch sentinel"
9971 );
9972 assert!(!subc_jobobject::process_exists(LAST_PID.get().unwrap()));
9973 }
9974
9975 #[cfg(windows)]
9976 #[tokio::test]
9977 async fn windows_nested_daemon_fixture() {
9978 let Some(sentinel) = std::env::var_os("SUBC_TEST_NESTED_STORE") else {
9979 return;
9980 };
9981 let mut spec = spec();
9982 spec.env.push((
9983 "SUBC_TEST_STORE_TOUCH".into(),
9984 sentinel.to_str().unwrap().into(),
9985 ));
9986 let supervisor =
9987 Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default());
9988 let module = supervisor
9989 .spawn(spec)
9990 .expect("per-module job must nest below daemon and host jobs");
9991 tokio::time::timeout(Duration::from_secs(10), async {
9992 while !PathBuf::from(&sentinel).exists() {
9993 tokio::task::yield_now().await;
9994 }
9995 })
9996 .await
9997 .unwrap();
9998 module.stop().await.unwrap();
9999 }
10000
10001 #[cfg(windows)]
10002 #[test]
10003 fn windows_module_job_nests_under_host_job() {
10004 let home = cortexkit_test_support::ScratchDir::new("nested-jobs");
10005 let sentinel = home.join("nested-store-touched");
10006 let mut command = std::process::Command::new(std::env::current_exe().unwrap());
10007 command
10008 .args([
10009 "--exact",
10010 "supervise::spawn_record_tests::windows_nested_daemon_fixture",
10011 "--nocapture",
10012 ])
10013 .env("SUBC_TEST_NESTED_STORE", &sentinel);
10014 let mut daemon = subc_jobobject::spawn_contained(&mut command).unwrap();
10015 assert!(daemon.child.wait().unwrap().success());
10016 assert!(sentinel.exists(), "nested protocol:none module never ran");
10017 }
10018}
10019
10020#[cfg(target_os = "linux")]
10021pub(crate) fn kill_module_cgroup(placement: Option<&subc_cgroup::Placement>, module_id: &str) {
10022 use subc_cgroup::KillOutcome;
10023 match subc_cgroup::kill_module(placement, module_id) {
10024 KillOutcome::Killed => {}
10025 KillOutcome::NotPlaced | KillOutcome::Unsupported => {
10026 debug!(
10027 module_id,
10028 "cgroup tree kill unavailable; using direct-child kill"
10029 );
10030 }
10031 KillOutcome::IoError { path, error } => {
10032 warn!(module_id, path = %path.display(), %error, "cgroup tree kill failed; using direct-child kill");
10033 }
10034 }
10035}
10036
10037#[cfg(windows)]
10045fn contain_spawned_child(
10046 child: &mut Child,
10047 spec: &ModuleSpec,
10048 carried: Option<&Arc<containment::CarriedContainment>>,
10049) -> Result<Option<Arc<Mutex<subc_jobobject::JobObject>>>, SuperviseError> {
10050 let assignment = || {
10051 let job = match carried {
10052 Some(container) => container.job(),
10053 None => Arc::new(Mutex::new(subc_jobobject::JobObject::new()?)),
10054 };
10055 #[cfg(test)]
10056 spawn_record_tests::assignment_failpoint(spec)?;
10057 job.lock()
10058 .unwrap_or_else(|p| p.into_inner())
10059 .assign(child)?;
10060 Ok::<_, io::Error>(job)
10061 };
10062 match assignment() {
10063 Ok(job) => Ok(Some(job)),
10064 Err(source) => {
10065 kill_and_reap_rejected_child(child);
10066 Err(SuperviseError::Spawn {
10067 program: spec.program.clone(),
10068 source: io::Error::other(format!(
10069 "Windows module job containment failed: {source}"
10070 )),
10071 cgroup_path: None,
10072 })
10073 }
10074 }
10075}
10076
10077fn kill_and_reap_rejected_child(child: &mut Child, #[cfg(unix)] group: containment::GroupTarget) {
10080 #[cfg(unix)]
10081 group.signal_rejected(child);
10082 let _ = child.start_kill();
10083 while matches!(child.try_wait(), Ok(None)) {
10084 std::thread::yield_now();
10085 }
10086}
10087
10088#[cfg(windows)]
10093fn resume_suspended_child(pid: u32, spec: &ModuleSpec) -> Result<(), SuperviseError> {
10094 if let Err(source) = subc_jobobject::resume_main_thread(pid) {
10095 let _ = std::process::Command::new("taskkill.exe")
10099 .args(["/PID", &pid.to_string(), "/T", "/F"])
10100 .stdin(Stdio::null())
10101 .stdout(Stdio::null())
10102 .stderr(Stdio::null())
10103 .status();
10104 return Err(SuperviseError::Spawn {
10105 program: spec.program.clone(),
10106 source,
10107 cgroup_path: None,
10108 });
10109 }
10110 Ok(())
10111}
10112
10113#[cfg(target_os = "linux")]
10114fn remove_module_cgroup(placement: &subc_cgroup::Placement, module_id: &str) {
10115 match placement.remove_module(module_id) {
10116 Ok(()) => debug!(module_id, "removed module cgroup after process exit"),
10117 Err(error) => warn!(
10118 module_id,
10119 error = %error,
10120 "could not remove module cgroup after process exit; continuing teardown"
10121 ),
10122 }
10123}
10124
10125#[cfg(target_os = "linux")]
10126const REAPED_CGROUP_TERM_GRACE: Duration = Duration::from_secs(2);
10127#[cfg(target_os = "linux")]
10128const REAPED_CGROUP_FREEZE_TIMEOUT: Duration = Duration::from_millis(100);
10129
10130#[cfg(target_os = "linux")]
10131async fn cleanup_reaped_cgroup(placement: &subc_cgroup::Placement, module_id: &str) {
10132 let path = placement.module_path(module_id).ok();
10133 if path.as_ref().is_some_and(|path| {
10134 std::fs::read_to_string(path.join("cgroup.events"))
10135 .ok()
10136 .is_some_and(|events| events.lines().any(|line| line == "populated 0"))
10137 }) {
10138 remove_module_cgroup(placement, module_id);
10139 return;
10140 }
10141
10142 if terminate_reaped_cgroup(placement, module_id).await {
10145 if let Some(path) = &path {
10146 let deadline = Instant::now() + REAPED_CGROUP_TERM_GRACE;
10147 while cgroup_is_populated(path) && Instant::now() < deadline {
10148 sleep_until((Instant::now() + Duration::from_millis(1)).min(deadline)).await;
10149 }
10150 }
10151 }
10152
10153 if matches!(
10157 subc_cgroup::kill_module(Some(placement), module_id),
10158 subc_cgroup::KillOutcome::Killed
10159 ) {
10160 if let Some(path) = &path {
10161 while cgroup_is_populated(path) {
10162 sleep(Duration::from_millis(1)).await;
10163 }
10164 }
10165 }
10166 remove_module_cgroup(placement, module_id);
10167}
10168
10169#[cfg(target_os = "linux")]
10170fn cgroup_is_populated(path: &std::path::Path) -> bool {
10171 std::fs::read_to_string(path.join("cgroup.events"))
10172 .ok()
10173 .is_some_and(|events| events.lines().any(|line| line == "populated 1"))
10174}
10175
10176#[cfg(target_os = "linux")]
10177async fn terminate_reaped_cgroup(placement: &subc_cgroup::Placement, module_id: &str) -> bool {
10178 let frozen =
10179 match subc_cgroup::freeze_module(Some(placement), module_id, REAPED_CGROUP_FREEZE_TIMEOUT)
10180 .await
10181 {
10182 subc_cgroup::FreezeOutcome::Frozen(frozen) => frozen,
10183 outcome => {
10184 tracing::debug!(
10185 module_id,
10186 ?outcome,
10187 "cgroup freeze failed; skipping SIGTERM grace"
10188 );
10189 return false;
10190 }
10191 };
10192 let listed = match frozen.processes() {
10193 subc_cgroup::ProcsOutcome::Listed(pids) => {
10194 for pid in pids.into_iter().filter(|pid| *pid > 1) {
10195 if let Some(pid) = rustix::process::Pid::from_raw(pid) {
10196 if let Err(error) =
10197 rustix::process::kill_process(pid, rustix::process::Signal::TERM)
10198 {
10199 tracing::debug!(module_id, %pid, %error, "leftover cgroup SIGTERM failed");
10200 }
10201 }
10202 }
10203 true
10204 }
10205 outcome => {
10206 tracing::debug!(
10207 module_id,
10208 ?outcome,
10209 "cgroup process listing failed; skipping SIGTERM grace"
10210 );
10211 false
10212 }
10213 };
10214 match frozen.thaw() {
10215 subc_cgroup::ThawOutcome::Thawed => listed,
10216 outcome => {
10217 tracing::debug!(
10218 module_id,
10219 ?outcome,
10220 "cgroup thaw failed; proceeding to cgroup.kill"
10221 );
10222 false
10223 }
10224 }
10225}
10226
10227#[cfg(target_os = "linux")]
10228fn apply_cgroup_placement(
10229 command: &mut Command,
10230 spec: &ModuleSpec,
10231 path: &std::path::Path,
10232) -> Result<(), SuperviseError> {
10233 subc_cgroup::apply(command, path).map_err(|source| SuperviseError::Cgroup {
10234 module_id: spec.module_id.clone(),
10235 source,
10236 })
10237}
10238
10239fn capture_retention(spec: &ModuleSpec) -> Retention {
10240 let defaults = Retention::default();
10241 let value = |name: &str| {
10242 spec.env
10243 .iter()
10244 .rev()
10245 .find_map(|(key, value)| (key == name).then_some(value.as_str()))
10246 };
10247 Retention {
10248 max_file_mb: value(CAPTURE_MAX_FILE_MB_ENV)
10249 .and_then(|value| value.parse().ok())
10250 .unwrap_or(defaults.max_file_mb),
10251 keep: value(CAPTURE_KEEP_ENV)
10252 .and_then(|value| value.parse().ok())
10253 .unwrap_or(defaults.keep),
10254 max_age_days: value(CAPTURE_MAX_AGE_DAYS_ENV)
10255 .and_then(|value| value.parse().ok())
10256 .unwrap_or(defaults.max_age_days),
10257 }
10258}
10259
10260fn generate_launch_nonce() -> Result<String, SuperviseError> {
10263 let mut bytes = [0u8; 32];
10264 getrandom::getrandom(&mut bytes).map_err(|source| SuperviseError::LaunchNonce {
10265 reason: source.to_string(),
10266 })?;
10267 let mut hex = String::with_capacity(64);
10268 for b in bytes {
10269 use std::fmt::Write;
10270 let _ = write!(hex, "{b:02x}");
10271 }
10272 Ok(hex)
10273}
10274
10275fn constant_time_eq(a: &[u8], b: &[u8]) -> bool {
10278 if a.len() != b.len() {
10279 return false;
10280 }
10281 let mut diff = 0u8;
10282 for (x, y) in a.iter().zip(b.iter()) {
10283 diff |= x ^ y;
10284 }
10285 diff == 0
10286}
10287
10288#[cfg(not(windows))]
10292fn observe_spawned_image(pid: u32) -> Option<subc_os::Observation> {
10293 let process = subc_os::Process::open(pid).ok().flatten()?;
10294 #[cfg(target_os = "linux")]
10295 return sample_after_linux_exec(
10296 || {
10297 let stat = std::fs::read_to_string(format!("/proc/{pid}/stat")).ok()?;
10298 let committed = linux_exec_committed(&stat);
10299 #[cfg(test)]
10300 spawn_image_tests::report_exec_state(committed);
10301 committed
10302 },
10303 || process.observe(),
10304 );
10305 #[cfg(not(target_os = "linux"))]
10306 process.observe()
10307}
10308
10309#[cfg(target_os = "linux")]
10313fn linux_exec_committed(stat: &str) -> Option<bool> {
10314 const PF_FORKNOEXEC: u64 = 0x40;
10315 let flags = stat.rsplit_once(')')?.1.split_whitespace().nth(6)?;
10316 Some(flags.parse::<u64>().ok()? & PF_FORKNOEXEC == 0)
10317}
10318
10319#[cfg(target_os = "linux")]
10320fn sample_after_linux_exec(
10321 mut exec_committed: impl FnMut() -> Option<bool>,
10322 observe: impl FnOnce() -> Option<subc_os::Observation>,
10323) -> Option<subc_os::Observation> {
10324 let deadline = std::time::Instant::now() + Duration::from_secs(1);
10325 while !exec_committed()? {
10326 if std::time::Instant::now() >= deadline {
10330 return None;
10331 }
10332 std::thread::sleep(Duration::from_micros(50));
10333 }
10334 observe()
10335}
10336
10337#[cfg(all(test, target_os = "linux"))]
10338mod spawn_image_tests {
10339 use super::*;
10340 use std::cell::{Cell, RefCell};
10341
10342 enum ExecEvent {
10343 Uncommitted,
10344 Image(Option<subc_os::Observation>),
10345 }
10346
10347 thread_local! {
10348 static EXEC_EVENTS: RefCell<Option<std::sync::mpsc::Sender<ExecEvent>>> = const { RefCell::new(None) };
10349 }
10350
10351 pub(super) fn report_exec_state(committed: Option<bool>) {
10352 if committed == Some(false) {
10353 EXEC_EVENTS.with(|events| {
10354 if let Some(events) = events.borrow_mut().take() {
10355 let _ = events.send(ExecEvent::Uncommitted);
10356 }
10357 });
10358 }
10359 }
10360
10361 #[test]
10362 fn linux_spawn_observation_waits_for_a_paused_fork_to_exec() {
10363 use std::io::{Read, Write};
10364 use std::os::fd::AsRawFd;
10365
10366 let descriptor = std::fs::File::open("/dev/null").unwrap();
10367 let mut command = std::process::Command::new("/bin/sleep");
10368 command.arg("30");
10369 let (mut ready, mut release) =
10370 subc_os::fork_exec_test::pause_before_exec(&mut command, descriptor.as_raw_fd())
10371 .unwrap();
10372 let spawn = std::thread::spawn(move || command.spawn());
10373 let mut message = [0u8; 8];
10374 ready.read_exact(&mut message).unwrap();
10375 let pid = i32::from_ne_bytes(message[..4].try_into().unwrap()) as u32;
10376 let before = std::fs::read_to_string(format!("/proc/{pid}/stat"));
10377 let inherited = subc_os::Process::open(pid).unwrap().unwrap().observe();
10378 let (events, received) = std::sync::mpsc::channel();
10379 let observer = std::thread::spawn(move || {
10380 EXEC_EVENTS.with(|slot| *slot.borrow_mut() = Some(events.clone()));
10381 let image = observe_spawned_image(pid);
10382 let _ = events.send(ExecEvent::Image(image));
10383 });
10384 let first = received.recv_timeout(Duration::from_secs(5));
10388 release.write_all(&[1]).unwrap();
10389 let mut child = spawn.join().unwrap().unwrap();
10390 let after = match first {
10391 Ok(ExecEvent::Uncommitted) => match received.recv_timeout(Duration::from_secs(5)) {
10392 Ok(ExecEvent::Image(image)) => image,
10393 _ => None,
10394 },
10395 Ok(ExecEvent::Image(image)) => image,
10396 Err(_) => None,
10397 };
10398 observer.join().unwrap();
10399 child.kill().unwrap();
10400 child.wait().unwrap();
10401
10402 assert_eq!(linux_exec_committed(&before.unwrap()), Some(false));
10403 assert_eq!(
10404 inherited.unwrap().executable,
10405 subc_os::file_identity(&std::env::current_exe().unwrap())
10406 );
10407 assert_eq!(
10408 after.unwrap().executable,
10409 subc_os::file_identity(std::path::Path::new("/bin/sleep"))
10410 );
10411 }
10412
10413 #[test]
10414 fn spawned_image_is_not_sampled_until_linux_exec_commits() {
10415 let committed = Cell::new(false);
10416 let inherited = subc_os::Observation {
10417 start_time: 23,
10418 executable: Some(subc_os::FileIdentity {
10419 device: 29,
10420 inode: 31,
10421 }),
10422 };
10423 let module = subc_os::Observation {
10424 start_time: 23,
10425 executable: Some(subc_os::FileIdentity {
10426 device: 29,
10427 inode: 37,
10428 }),
10429 };
10430 let polls = Cell::new(0);
10431 let image = sample_after_linux_exec(
10432 || {
10433 polls.set(polls.get() + 1);
10434 committed.set(polls.get() == 2);
10437 Some(committed.get())
10438 },
10439 || Some(if committed.get() { module } else { inherited }),
10440 );
10441 assert_eq!(image, Some(module));
10442 assert_eq!(polls.get(), 2);
10443 }
10444
10445 #[test]
10446 fn linux_exec_flag_is_read_from_flags_after_the_last_parenthesis() {
10447 assert_eq!(
10448 linux_exec_committed("17 (a ) b) R 1 2 3 4 5 4194368"),
10449 Some(false)
10450 );
10451 assert_eq!(
10452 linux_exec_committed("17 (a ) b) R 1 2 3 4 5 4194304"),
10453 Some(true)
10454 );
10455 assert_eq!(linux_exec_committed("17 (a) R 1 2 3 4 5 invalid"), None);
10456 assert_eq!(linux_exec_committed("17 (a) R"), None);
10457 }
10458
10459 #[test]
10460 fn unreadable_exec_state_never_samples_an_inherited_image() {
10461 assert_eq!(
10462 sample_after_linux_exec(
10463 || None,
10464 || panic!("must not sample before exec confirmation")
10465 ),
10466 None
10467 );
10468 }
10469
10470 #[test]
10471 fn uncommitted_exec_times_out_without_sampling_an_inherited_image() {
10472 assert_eq!(
10473 sample_after_linux_exec(
10474 || Some(false),
10475 || panic!("must not sample after an exec timeout")
10476 ),
10477 None
10478 );
10479 }
10480}
10481
10482#[cfg(windows)]
10485fn windows_spawn_capture(spec: &ModuleSpec) -> io::Result<subc_os::ExecutableCapture> {
10486 let mut program = spec.program.clone();
10487 if program.is_absolute() || program.components().count() > 1 {
10488 match subc_os::ExecutableCapture::open(&program) {
10491 Ok(capture) => return Ok(capture),
10492 Err(error)
10493 if error.kind() == io::ErrorKind::NotFound && program.extension().is_none() => {}
10494 Err(error) => return Err(error),
10495 }
10496 program.set_extension("exe");
10497 return subc_os::ExecutableCapture::open(&program);
10498 }
10499 if program.extension().is_none() {
10500 program.set_extension("exe");
10501 }
10502 let custom_path = spec
10503 .env
10504 .iter()
10505 .rev()
10506 .find(|(key, _)| key.eq_ignore_ascii_case("PATH"));
10507 let mut directories = Vec::new();
10508 if let Some(parent) = std::env::current_exe()?.parent() {
10511 directories.push(parent.to_path_buf());
10512 }
10513 directories.push(std::env::current_dir()?);
10514 if let Some(windows) = std::env::var_os("SystemRoot") {
10515 let windows = PathBuf::from(windows);
10516 directories.push(windows.join("System32"));
10517 directories.push(windows.clone());
10518 }
10519 let path = custom_path
10520 .map(|(_, value)| std::ffi::OsString::from(value))
10521 .or_else(|| std::env::var_os("PATH"));
10522 if let Some(path) = path {
10523 directories.extend(std::env::split_paths(&path));
10524 }
10525 for directory in directories {
10526 match subc_os::ExecutableCapture::open(&directory.join(&program)) {
10527 Ok(capture) => return Ok(capture),
10528 Err(error) if error.kind() == io::ErrorKind::NotFound => continue,
10529 Err(error) => return Err(error),
10530 }
10531 }
10532 Err(io::Error::new(
10533 io::ErrorKind::NotFound,
10534 "resolved executable not found",
10535 ))
10536}
10537
10538fn spawn_and_mark_running(
10539 spec: &ModuleSpec,
10540 runtime: &SupervisorRuntimeConfig,
10541 snapshot: &SharedSnapshot,
10542) -> Result<SupervisedChild, SuperviseError> {
10543 let activity_spawn = runtime.begin_activity_spawn(&spec.module_id);
10544 let registration_watch = watch_registration(&runtime.registry, spec);
10545 #[cfg(unix)]
10546 let joining_group = containment::current(&runtime.child_roster.containment)
10547 .is_some_and(|container| container.group().is_some());
10548 let spawn = || {
10549 spawn_child(
10550 spec,
10551 runtime.connection_file_path.as_deref(),
10552 runtime.supervisor_handle.as_ref(),
10553 &runtime.stderr_ring,
10554 runtime.capture_logs_dir.as_deref(),
10555 &runtime.child_roster,
10556 #[cfg(target_os = "linux")]
10557 runtime.cgroup_placement.as_ref(),
10558 )
10559 };
10560 let result = spawn();
10561 #[cfg(unix)]
10562 let result = if result.is_err()
10563 && joining_group
10564 && !runtime.child_roster.is_closed()
10565 && lock_snapshot(snapshot).is_ok_and(|state| state.enabled)
10566 && runtime
10567 .child_roster
10568 .previous_instance(&spec.module_id)
10569 .is_none()
10570 {
10571 warn!(module_id = %spec.module_id, "carried process-group spawn failed; trying one fresh container");
10575 spawn()
10576 } else {
10577 result
10578 };
10579 let child = result?;
10580 let mut child = child;
10581 arm_registration_deadline(&mut child, runtime.register_timeout, registration_watch);
10582 if let Err(error) = set_running(snapshot, &child, &spec.module_id, &runtime.spawn_events) {
10583 containment::terminate_blocking(&runtime.child_roster.containment);
10584 return Err(error);
10585 }
10586 if let Some(spawn) = activity_spawn {
10587 spawn.complete(snapshot);
10588 }
10589 Ok(child)
10590}
10591
10592struct RegistrationWatch {
10593 changes: watch::Receiver<u64>,
10594 baseline: u64,
10595}
10596
10597fn watch_registration(registry: &Registry, spec: &ModuleSpec) -> Option<RegistrationWatch> {
10601 if spec.protocol != ModuleProtocol::Subc {
10602 return None;
10603 }
10604 let changes = registry.subscribe_module_changes(&spec.module_id).ok()?;
10605 let baseline = *changes.borrow();
10606 Some(RegistrationWatch { changes, baseline })
10607}
10608
10609fn arm_registration_deadline(
10610 child: &mut SupervisedChild,
10611 budget: Duration,
10612 registration_watch: Option<RegistrationWatch>,
10613) {
10614 child.registration_deadline =
10615 (child.protocol == ModuleProtocol::Subc).then_some(child.spawned_at + budget);
10616 child.registration_watch = registration_watch;
10617}
10618
10619async fn stop_unregistered_child(child: &mut SupervisedChild) -> io::Result<()> {
10620 if let Some(container) = child.child.container() {
10621 container.stop_members().await;
10622 child.kill_leader()?;
10623 child.wait().await?;
10624 return Ok(());
10625 }
10626 #[cfg(unix)]
10627 {
10628 let group = unreaped_registration_group(child)?;
10631 let _ = rustix::process::kill_process_group(group, rustix::process::Signal::TERM);
10632 sleep(crate::child_roster::TERM_TO_KILL).await;
10637 let group = unreaped_registration_group(child)?;
10638 let _ = rustix::process::kill_process_group(group, rustix::process::Signal::KILL);
10639 }
10640 child.start_kill()?;
10643 child.wait().await?;
10644 Ok(())
10645}
10646
10647#[cfg(unix)]
10648fn unreaped_registration_group(child: &SupervisedChild) -> io::Result<rustix::process::Pid> {
10649 if let Some(container) = child.child.container() {
10650 return container
10651 .group()
10652 .and_then(|pid| rustix::process::Pid::from_raw(pid as i32))
10653 .ok_or_else(|| io::Error::other("a carried cgroup has no signalable process group"));
10654 }
10655 let group = i32::try_from(child.pid)
10658 .ok()
10659 .filter(|pid| *pid > 1)
10660 .and_then(rustix::process::Pid::from_raw)
10661 .ok_or_else(|| {
10662 io::Error::new(
10663 io::ErrorKind::InvalidInput,
10664 "invalid supervised process group",
10665 )
10666 })?;
10667 if child.child.id() != Some(child.pid) {
10668 return Err(io::Error::new(
10669 io::ErrorKind::InvalidInput,
10670 "refusing process-group signal after supervised child was reaped",
10671 ));
10672 }
10673 Ok(group)
10674}
10675
10676enum RegistrationWaitOutcome {
10677 Registered,
10678 Exited(ExitReport),
10679 TimedOut,
10680}
10681
10682struct ReloadRegistrationFailure {
10683 exit_report: ExitReport,
10684 reason: String,
10685}
10686
10687#[derive(Debug, Clone, Copy, PartialEq, Eq)]
10688enum BusyGaugeObservation {
10689 Quiescent,
10690 Busy,
10691 Omitted,
10692}
10693
10694#[derive(Debug, Clone, PartialEq, Eq)]
10696enum GaugeVerdict {
10697 Zero,
10699 NonZero(u64),
10701 NotACount(String),
10705 Missing,
10707 ProbeFailed(&'static str),
10710}
10711
10712impl GaugeVerdict {
10713 fn reads_quiet(&self) -> bool {
10714 matches!(self, Self::Zero)
10715 }
10716}
10717
10718impl fmt::Display for GaugeVerdict {
10719 fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
10720 match self {
10721 Self::Zero => f.write_str("zero"),
10722 Self::NonZero(value) => write!(f, "non_zero({value})"),
10723 Self::NotACount(value) => write!(f, "not_a_count({value})"),
10724 Self::Missing => f.write_str("missing_from_health_body"),
10725 Self::ProbeFailed(kind) => write!(f, "probe_failed({kind})"),
10726 }
10727 }
10728}
10729
10730const GAUGE_VALUE_TEXT_LIMIT: usize = 32;
10733
10734fn gauge_verdicts(metrics: Option<&Value>, gauges: &[String]) -> Vec<GaugeVerdict> {
10736 let metrics = metrics.and_then(Value::as_object);
10737 gauges
10738 .iter()
10739 .map(
10740 |gauge| match metrics.and_then(|metrics| metrics.get(gauge)) {
10741 None => GaugeVerdict::Missing,
10742 Some(value) => match value.as_u64() {
10743 Some(0) => GaugeVerdict::Zero,
10744 Some(count) => GaugeVerdict::NonZero(count),
10745 None => {
10746 let mut text = value.to_string();
10747 if text.len() > GAUGE_VALUE_TEXT_LIMIT {
10748 let mut end = GAUGE_VALUE_TEXT_LIMIT;
10749 while !text.is_char_boundary(end) {
10750 end -= 1;
10751 }
10752 text.truncate(end);
10753 text.push_str("...");
10754 }
10755 GaugeVerdict::NotACount(text)
10756 }
10757 },
10758 },
10759 )
10760 .collect()
10761}
10762
10763fn busy_gauge_observation(verdicts: &[GaugeVerdict]) -> BusyGaugeObservation {
10766 if verdicts
10767 .iter()
10768 .any(|verdict| matches!(verdict, GaugeVerdict::Missing))
10769 {
10770 BusyGaugeObservation::Omitted
10771 } else if verdicts.iter().all(GaugeVerdict::reads_quiet) {
10772 BusyGaugeObservation::Quiescent
10773 } else {
10774 BusyGaugeObservation::Busy
10775 }
10776}
10777
10778#[derive(Debug, Clone, Default, PartialEq, Eq)]
10783struct GaugeTrace {
10784 latest: Option<(GaugeVerdict, Duration)>,
10785 last_value: Option<(GaugeVerdict, Duration)>,
10786}
10787
10788#[derive(Debug, Clone, Default, PartialEq, Eq)]
10794struct DrainGaugeTrace {
10795 probes: u32,
10797 gauges: Vec<(String, GaugeTrace)>,
10799}
10800
10801impl DrainGaugeTrace {
10802 fn new(gauges: &[String]) -> Self {
10803 Self {
10804 probes: 0,
10805 gauges: gauges
10806 .iter()
10807 .map(|gauge| (gauge.clone(), GaugeTrace::default()))
10808 .collect(),
10809 }
10810 }
10811
10812 fn record(&mut self, verdicts: &[GaugeVerdict], at: Duration) {
10814 self.probes = self.probes.saturating_add(1);
10815 for ((_, trace), verdict) in self.gauges.iter_mut().zip(verdicts) {
10816 trace.latest = Some((verdict.clone(), at));
10817 if matches!(
10818 verdict,
10819 GaugeVerdict::Zero | GaugeVerdict::NonZero(_) | GaugeVerdict::NotACount(_)
10820 ) {
10821 trace.last_value = Some((verdict.clone(), at));
10822 }
10823 }
10824 }
10825}
10826
10827impl fmt::Display for DrainGaugeTrace {
10832 fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
10833 if self.gauges.is_empty() {
10834 return f.write_str("none declared");
10835 }
10836 for (index, (name, trace)) in self.gauges.iter().enumerate() {
10837 if index > 0 {
10838 f.write_str(" | ")?;
10839 }
10840 write!(f, "{name}: ")?;
10841 match &trace.latest {
10842 Some((verdict, at)) => write!(f, "{verdict} @{}ms", at.as_millis())?,
10843 None => f.write_str("never probed")?,
10844 }
10845 match &trace.last_value {
10846 Some((value, at)) => write!(f, "; last value {value} @{}ms", at.as_millis())?,
10847 None => f.write_str("; no value ever read")?,
10848 }
10849 }
10850 Ok(())
10851 }
10852}
10853
10854pub(crate) fn declared_busy_gauges(
10855 registry: &Registry,
10856 module_id: &str,
10857) -> Result<Vec<String>, SuperviseError> {
10858 busy_gauges_of(
10859 registry
10860 .get_module(module_id)
10861 .map_err(SuperviseError::Registry)?,
10862 )
10863}
10864
10865fn declared_busy_gauges_for_connection(
10869 registry: &Registry,
10870 connection_id: ConnectionId,
10871) -> Result<Vec<String>, SuperviseError> {
10872 busy_gauges_of(
10873 registry
10874 .get_module_by_connection(connection_id)
10875 .map_err(SuperviseError::Registry)?,
10876 )
10877}
10878
10879fn busy_gauges_of(
10880 registration: Option<crate::registry::ModuleRegistration>,
10881) -> Result<Vec<String>, SuperviseError> {
10882 let Some(registration) = registration else {
10883 return Ok(Vec::new());
10884 };
10885 let Some(self_signals) = registration.manifest.self_signals else {
10886 return Ok(Vec::new());
10887 };
10888
10889 let mut gauges = Vec::new();
10890 for declaration in self_signals {
10891 if declaration.kind != SelfSignalKind::Busy {
10892 continue;
10893 }
10894 match declaration.anchored_to {
10895 SignalAnchor::HealthGauges { gauges: declared } if !declared.is_empty() => {
10896 gauges.extend(declared)
10897 }
10898 _ => {
10899 gauges.push(String::new());
10902 }
10903 }
10904 }
10905 Ok(gauges)
10906}
10907
10908#[allow(clippy::too_many_arguments)]
10917async fn wait_for_forwarding_quiescence(
10918 forwarding: &ForwardingTable,
10919 module_id: &str,
10920 runtime: &SupervisorRuntimeConfig,
10921 endpoint: crate::ModuleEndpointId,
10922 deadline: Instant,
10923 busy_gauges: &[String],
10924 scope: DrainScope,
10925 trace: &mut DrainGaugeTrace,
10926) -> Result<bool, SuperviseError> {
10927 let wait_started = Instant::now();
10928 let mut gauges_quiescent = busy_gauges.is_empty();
10929 let mut next_probe_at = wait_started;
10930 let mut omission_reported = false;
10931
10932 loop {
10933 let now = Instant::now();
10934 if !busy_gauges.is_empty() && now >= next_probe_at && now < deadline {
10935 let report = match scope {
10936 DrainScope::Active => probe_module_health(module_id, runtime, Some(deadline)).await,
10937 DrainScope::Endpoint(endpoint) => {
10938 probe_endpoint_health(endpoint, runtime, Some(deadline)).await
10939 }
10940 };
10941 let verdicts = match &report {
10942 Ok(report) => gauge_verdicts(report.metrics.as_ref(), busy_gauges),
10943 Err(err) => vec![GaugeVerdict::ProbeFailed(err.label()); busy_gauges.len()],
10944 };
10945 trace.record(&verdicts, wait_started.elapsed());
10946 gauges_quiescent = match report {
10947 Ok(_) => match busy_gauge_observation(&verdicts) {
10948 BusyGaugeObservation::Quiescent => true,
10949 BusyGaugeObservation::Busy => false,
10950 BusyGaugeObservation::Omitted => {
10951 if !omission_reported {
10954 forwarding
10955 .counters()
10956 .increment_drains_with_undeclared_gauge();
10957 let missing = busy_gauges
10958 .iter()
10959 .zip(&verdicts)
10960 .filter(|(_, verdict)| matches!(verdict, GaugeVerdict::Missing))
10961 .map(|(gauge, _)| gauge.as_str())
10962 .collect::<Vec<_>>()
10963 .join(",");
10964 warn!(
10965 module_id,
10966 missing_gauges = %missing,
10967 "drain health.check answer lacks a declared busy gauge; the drain treats the module as busy until it reports it"
10968 );
10969 omission_reported = true;
10970 }
10971 false
10972 }
10973 },
10974 Err(err) => {
10975 warn!(
10976 module_id,
10977 error = %err,
10978 "drain health.check did not produce declared busy gauges; treating module as busy"
10979 );
10980 false
10981 }
10982 };
10983 next_probe_at = Instant::now() + drain_busy_reprobe_interval();
10986 }
10987
10988 let in_flight = forwarding
10989 .endpoint_in_flight_count(endpoint)
10990 .map_err(SuperviseError::Forwarding)?;
10991 if in_flight == 0 && gauges_quiescent {
10992 return Ok(true);
10993 }
10994
10995 let now = Instant::now();
10996 if now >= deadline {
10997 return Ok(false);
10998 }
10999 let mut wait = deadline
11000 .saturating_duration_since(now)
11001 .min(REGISTRY_RELEASE_POLL);
11002 if !busy_gauges.is_empty() {
11003 wait = wait.min(next_probe_at.saturating_duration_since(now));
11004 }
11005 sleep(wait).await;
11006 }
11007}
11008
11009fn drain_busy_reprobe_interval() -> Duration {
11013 DRAIN_BUSY_REPROBE_INTERVAL.max(REGISTRY_RELEASE_POLL)
11014}
11015
11016fn warn_drain_timed_out(
11031 module_id: &str,
11032 drain_timeout: Duration,
11033 reason: RouteCloseReason,
11034 holdouts: &crate::forwarding::DrainHoldouts,
11035 gauges: &DrainGaugeTrace,
11036) {
11037 warn!(
11038 module_id = %module_id,
11039 waited = ?drain_timeout,
11040 ?reason,
11041 held_requests = holdouts.requests,
11042 held_routes = holdouts.routes,
11043 total_routes = holdouts.total_routes,
11044 top_connections = ?holdouts.top_connections,
11045 held = %holdouts
11048 .held
11049 .iter()
11050 .map(|(channel, corr)| format!("{channel}:{corr}"))
11051 .collect::<Vec<_>>()
11052 .join(","),
11053 gauge_probes = gauges.probes,
11054 busy_gauges = %gauges,
11055 "route drain timed out before request quiescence; forcing teardown"
11056 );
11057}
11058
11059fn drained_after_quiescence_wait(wait_result: &Result<bool, SuperviseError>) -> bool {
11067 match wait_result {
11068 Ok(drained) => *drained,
11069 Err(_) => false,
11070 }
11071}
11072
11073fn send_route_goodbyes(forwarding: &ForwardingTable, released_routes: Vec<GoodbyeTarget>) {
11074 for released in released_routes {
11075 let frame = match Frame::build_with_version(
11076 released.negotiated_ver,
11077 FrameType::Goodbye,
11078 control_flags(),
11079 released.channel,
11080 released.epoch,
11081 0,
11082 Vec::new(),
11083 ) {
11084 Ok(frame) => frame,
11085 Err(err) => {
11086 warn!(
11087 route_channel = released.channel,
11088 error = %err,
11089 "failed to build supervisor drain route GOODBYE frame"
11090 );
11091 continue;
11092 }
11093 };
11094 if !released.close_on_delivery_failure() {
11095 crate::forwarding::send_module_route_goodbye(
11096 &forwarding.counters(),
11097 &released.sink,
11098 frame,
11099 released.module_id.as_deref(),
11100 "supervisor drain",
11101 );
11102 continue;
11103 }
11104 if let Err(err) = released.sink.try_send(frame) {
11105 warn!(
11106 target_connection_id = released.connection_id.get(),
11107 route_channel = released.channel,
11108 error = %err,
11109 "supervisor drain route GOODBYE was not delivered to client; closing target connection"
11110 );
11111 let _ = forwarding.escalate_client_delivery_failure(
11112 released.connection_id,
11113 released.channel,
11114 released.epoch,
11115 CloseReason::new(
11116 "route_goodbye_delivery_failed",
11117 format!(
11118 "failed to enqueue supervisor drain route GOODBYE for channel {}: {err}",
11119 released.channel
11120 ),
11121 ),
11122 crate::forwarding::UndeliveredFrame {
11123 module_id: released.module_id.as_deref(),
11124 sink: &released.sink,
11125 },
11126 );
11127 }
11128 }
11129}
11130
11131async fn send_module_draining(
11132 module_id: &str,
11133 reason: RouteCloseReason,
11134 deadline_ms: u64,
11135 target: &ModuleDrainTarget,
11136 _wait_for_flush: bool,
11137) -> bool {
11138 let body = match serde_json::to_vec(&ModuleControlCommand::Draining {
11139 reason,
11140 deadline_ms,
11141 }) {
11142 Ok(body) => body,
11143 Err(err) => {
11144 warn!(
11145 module_id,
11146 error = %err,
11147 "failed to encode module draining command"
11148 );
11149 return false;
11150 }
11151 };
11152 let frame = match Frame::build_with_version(
11153 target.negotiated_ver,
11154 FrameType::Push,
11155 control_flags(),
11156 0,
11157 0,
11158 0,
11159 body,
11160 ) {
11161 Ok(frame) => frame,
11162 Err(err) => {
11163 warn!(
11164 module_id,
11165 error = %err,
11166 "failed to build module draining command frame"
11167 );
11168 return false;
11169 }
11170 };
11171 #[cfg(unix)]
11172 if _wait_for_flush {
11173 return target.sink.send_flushed(frame).await.is_ok();
11174 }
11175 if let Err(err) = target.sink.try_send(frame) {
11176 warn!(
11177 module_id,
11178 target_connection_id = target.endpoint.connection_id.get(),
11179 error = %err,
11180 "module draining command was not delivered to peer"
11181 );
11182 return false;
11183 }
11184 true
11185}
11186
11187fn module_goodbye_frame(module_id: &str, negotiated_ver: u8) -> Option<Frame> {
11189 match Frame::build_with_version(
11190 negotiated_ver,
11191 FrameType::Goodbye,
11192 control_flags(),
11193 0,
11194 0,
11195 0,
11196 Vec::new(),
11197 ) {
11198 Ok(frame) => Some(frame),
11199 Err(err) => {
11200 warn!(
11201 module_id,
11202 error = %err,
11203 "failed to build module GOODBYE frame"
11204 );
11205 None
11206 }
11207 }
11208}
11209
11210#[cfg(unix)]
11224async fn send_module_goodbyes_for_daemon_shutdown(
11225 forwarding: &Arc<ForwardingTable>,
11226 reason: &CloseReason,
11227 wait_for_flush: bool,
11228) {
11229 const GOODBYE_BUDGET: Duration = Duration::from_millis(500);
11230 let targets = match forwarding.module_connections() {
11231 Ok(targets) => targets,
11232 Err(err) => {
11233 warn!(error = %err, "could not list module connections for shutdown GOODBYE");
11234 return;
11235 }
11236 };
11237 let deadline = Instant::now() + GOODBYE_BUDGET;
11238 let mut sends = tokio::task::JoinSet::new();
11239 for target in targets {
11240 let Some(frame) = module_goodbye_frame(&target.module_id, target.negotiated_ver) else {
11241 continue;
11242 };
11243 if !wait_for_flush {
11244 if let Err(err) = target.sink.try_send(frame) {
11245 debug!(
11246 module_id = %target.module_id,
11247 error = %err,
11248 "shutdown module GOODBYE was not queued"
11249 );
11250 }
11251 continue;
11252 }
11253 let forwarding = Arc::clone(forwarding);
11254 let reason = reason.clone();
11255 sends.spawn(async move {
11256 match timeout_at(deadline, target.sink.send_flushed(frame)).await {
11257 Ok(Ok(())) => {}
11258 Ok(Err(err)) => debug!(
11259 module_id = %target.module_id,
11260 error = %err,
11261 "module connection closed before its shutdown GOODBYE was written"
11262 ),
11263 Err(_) => warn!(
11264 module_id = %target.module_id,
11265 budget = ?GOODBYE_BUDGET,
11266 "shutdown module GOODBYE was not written within its budget; closing anyway"
11267 ),
11268 }
11269 forwarding.request_connection_close(target.endpoint.connection_id, reason);
11270 });
11271 }
11272 while sends.join_next().await.is_some() {}
11274}
11275
11276fn send_module_goodbye(module_id: &str, forwarding: &ForwardingTable, target: &ModuleDrainTarget) {
11277 let Some(frame) = module_goodbye_frame(module_id, target.negotiated_ver) else {
11278 return;
11279 };
11280 if let Err(err) = target.sink.try_send(frame) {
11281 warn!(
11282 module_id,
11283 target_connection_id = target.endpoint.connection_id.get(),
11284 error = %err,
11285 "supervisor drain module GOODBYE was not delivered to peer; closing module connection"
11286 );
11287 forwarding.request_connection_close(
11288 target.endpoint.connection_id,
11289 CloseReason::new(
11290 "module_goodbye_delivery_failed",
11291 format!("failed to enqueue supervisor drain module GOODBYE for module '{module_id}': {err}"),
11292 ),
11293 );
11294 }
11295}
11296
11297#[derive(Clone, Copy)]
11298struct ForwardingDrainContext<'a> {
11299 spec: &'a ModuleSpec,
11300 runtime: &'a SupervisorRuntimeConfig,
11301 registry: &'a Registry,
11302 scope: DrainScope,
11303}
11304
11305#[derive(Debug, Clone, Copy, PartialEq, Eq)]
11307enum DrainScope {
11308 Active,
11311 Endpoint(crate::ModuleEndpointId),
11316}
11317
11318#[derive(Debug, Clone, Copy, PartialEq, Eq)]
11326enum StopNotice {
11327 SentOverConnection,
11330 NoConnection,
11334 NotSent,
11338}
11339
11340async fn begin_forwarding_drain(
11341 spec: &ModuleSpec,
11342 runtime: &SupervisorRuntimeConfig,
11343 registry: &Registry,
11344 snapshot: &SharedSnapshot,
11345 enabled: Option<bool>,
11346 reason: RouteCloseReason,
11347) -> Result<StopNotice, SuperviseError> {
11348 let Some(forwarding) = runtime.forwarding.as_ref() else {
11349 return Err(SuperviseError::ReloadUnavailable {
11350 module_id: spec.module_id.clone(),
11351 reason: "supervisor was not configured with a forwarding table".to_string(),
11352 });
11353 };
11354
11355 begin_forwarding_drain_with(
11356 forwarding,
11357 ForwardingDrainContext {
11358 spec,
11359 runtime,
11360 registry,
11361 scope: DrainScope::Active,
11362 },
11363 snapshot,
11364 enabled,
11365 reason,
11366 runtime.drain_timeout,
11367 )
11368 .await
11369}
11370
11371async fn begin_forwarding_drain_if_configured(
11372 spec: &ModuleSpec,
11373 runtime: &SupervisorRuntimeConfig,
11374 registry: &Registry,
11375 snapshot: &SharedSnapshot,
11376 enabled: Option<bool>,
11377 reason: RouteCloseReason,
11378) -> Result<StopNotice, SuperviseError> {
11379 begin_forwarding_drain_with_timeout(
11380 spec,
11381 runtime,
11382 registry,
11383 snapshot,
11384 enabled,
11385 reason,
11386 runtime.drain_timeout,
11387 )
11388 .await
11389}
11390
11391async fn begin_forwarding_drain_with_timeout(
11395 spec: &ModuleSpec,
11396 runtime: &SupervisorRuntimeConfig,
11397 registry: &Registry,
11398 snapshot: &SharedSnapshot,
11399 enabled: Option<bool>,
11400 reason: RouteCloseReason,
11401 drain_timeout: Duration,
11402) -> Result<StopNotice, SuperviseError> {
11403 let Some(forwarding) = runtime.forwarding.as_ref() else {
11404 return Ok(StopNotice::NotSent);
11405 };
11406
11407 begin_forwarding_drain_with(
11408 forwarding,
11409 ForwardingDrainContext {
11410 spec,
11411 runtime,
11412 registry,
11413 scope: DrainScope::Active,
11414 },
11415 snapshot,
11416 enabled,
11417 reason,
11418 drain_timeout,
11419 )
11420 .await
11421}
11422
11423async fn begin_forwarding_drain_with(
11424 forwarding: &ForwardingTable,
11425 context: ForwardingDrainContext<'_>,
11426 snapshot: &SharedSnapshot,
11427 enabled: Option<bool>,
11428 reason: RouteCloseReason,
11429 drain_timeout: Duration,
11430) -> Result<StopNotice, SuperviseError> {
11431 let ForwardingDrainContext {
11432 spec,
11433 runtime,
11434 registry,
11435 scope,
11436 } = context;
11437 debug_assert_ne!(reason, RouteCloseReason::Crash);
11438 let terminal = matches!(reason, RouteCloseReason::Disable);
11439 let drain_started_at = Instant::now();
11440 let drain_deadline = drain_started_at + drain_timeout;
11441 let deadline_ms =
11442 unix_ms_now().saturating_add(u64::try_from(drain_timeout.as_millis()).unwrap_or(u64::MAX));
11443 let busy_gauges = match scope {
11444 DrainScope::Active => declared_busy_gauges(registry, &spec.module_id)?,
11445 DrainScope::Endpoint(endpoint) => {
11446 declared_busy_gauges_for_connection(registry, endpoint.connection_id)?
11447 }
11448 };
11449
11450 let gate_started = Instant::now();
11453 let drain_target = match scope {
11454 DrainScope::Active => forwarding.begin_module_drain(&spec.module_id, reason),
11455 DrainScope::Endpoint(endpoint) => forwarding.begin_endpoint_drain(endpoint, reason),
11456 }
11457 .map_err(SuperviseError::Forwarding)?;
11458 info!(
11463 module_id = %spec.module_id,
11464 ?reason,
11465 gate_ms = u64::try_from(gate_started.elapsed().as_millis()).unwrap_or(u64::MAX),
11466 connected = drain_target.is_some(),
11467 "module drain began; route admission closed"
11468 );
11469 if scope == DrainScope::Active {
11470 update_snapshot(snapshot, Some(&spec.module_id), |state| {
11471 state.state = ModuleState::Draining;
11472 state.draining_to_replace =
11473 matches!(reason, RouteCloseReason::Restart | RouteCloseReason::Reload);
11474 if let Some(enabled) = enabled {
11475 state.enabled = enabled;
11476 }
11477 })?;
11478 }
11479
11480 let Some(target) = drain_target.as_ref() else {
11481 return Ok(StopNotice::NoConnection);
11485 };
11486 {
11487 send_module_draining(&spec.module_id, reason, deadline_ms, target, false).await;
11488 #[cfg(feature = "test-support")]
11489 forwarding.wait_before_route_closing_for_test().await;
11490 let routes = forwarding
11491 .endpoint_routes(target.endpoint)
11492 .map_err(SuperviseError::Forwarding)?;
11493 let routes_notified = routes.len();
11494 crate::control::send_route_control_pushes(
11495 forwarding,
11496 routes.clone(),
11497 ClientControlPush::RouteClosing {
11498 module_id: spec.module_id.clone(),
11499 channels: Vec::new(),
11500 reason,
11501 },
11502 );
11503 send_route_goodbyes(forwarding, target.abandoned_bindings.clone());
11504
11505 let mut gauge_trace = DrainGaugeTrace::new(&busy_gauges);
11511 let wait_result = wait_for_forwarding_quiescence(
11512 forwarding,
11513 &spec.module_id,
11514 runtime,
11515 target.endpoint,
11516 drain_deadline,
11517 &busy_gauges,
11518 scope,
11519 &mut gauge_trace,
11520 )
11521 .await;
11522 let drained = drained_after_quiescence_wait(&wait_result);
11523 if let Err(err) = &wait_result {
11524 error!(
11525 module_id = %spec.module_id,
11526 ?reason,
11527 error = %err,
11528 "forwarding quiescence wait failed after route.closing; forcing route.closed(drained: false) so the client is not left waiting on an unfulfilled promise"
11529 );
11530 } else if !drained {
11531 let holdouts = forwarding
11533 .endpoint_drain_holdouts(target.endpoint)
11534 .unwrap_or_default();
11535 warn_drain_timed_out(
11536 &spec.module_id,
11537 drain_timeout,
11538 reason,
11539 &holdouts,
11540 &gauge_trace,
11541 );
11542 }
11543 crate::control::send_route_control_pushes(
11544 forwarding,
11545 routes,
11546 ClientControlPush::RouteClosed {
11547 module_id: spec.module_id.clone(),
11548 channels: Vec::new(),
11549 reason,
11550 drained,
11551 abandoned: target.abandoned_bindings.len() as u32,
11552 excluded_subscriptions: target.excluded_subscriptions,
11553 terminal: Some(terminal),
11554 },
11555 );
11556 wait_result?;
11557
11558 let released_routes = match forwarding.release_module_endpoint_routes(target.endpoint) {
11564 Ok(routes) => routes,
11565 Err(err) => {
11566 warn!(
11567 module_id = %spec.module_id,
11568 ?reason,
11569 error = %err,
11570 "failed to release module endpoint routes after route.closed; module GOODBYE will still be sent"
11571 );
11572 send_module_goodbye(&spec.module_id, forwarding, target);
11573 return Err(SuperviseError::Forwarding(err));
11574 }
11575 };
11576 let route_goodbye_count = released_routes.len();
11577 send_route_goodbyes(forwarding, released_routes);
11578 send_module_goodbye(&spec.module_id, forwarding, target);
11579
11580 info!(
11586 module_id = %spec.module_id,
11587 ?reason,
11588 routes_notified,
11589 route_goodbyes = route_goodbye_count,
11590 abandoned_reservations = target.abandoned_bindings.len(),
11591 excluded_subscriptions = target.excluded_subscriptions,
11592 drained,
11593 "module drain complete; consumers notified via route.closing/route.closed pushes and per-route GOODBYE frames"
11594 );
11595 }
11596
11597 Ok(StopNotice::SentOverConnection)
11598}
11599
11600async fn wait_for_registration_after_reload(
11603 registry: &Registry,
11604 module_id: &str,
11605 snapshot: &SharedSnapshot,
11606 child: &mut SupervisedChild,
11607 wait: Duration,
11608) -> Result<RegistrationWaitOutcome, SuperviseError> {
11609 wait_for_slot_registration(
11610 registry,
11611 crate::registry::RegistrationSlot::Active(module_id),
11612 module_id,
11613 snapshot,
11614 child,
11615 wait,
11616 )
11617 .await
11618}
11619
11620async fn wait_for_slot_registration(
11628 registry: &Registry,
11629 slot: crate::registry::RegistrationSlot<'_>,
11630 module_id: &str,
11631 snapshot: &SharedSnapshot,
11632 child: &mut SupervisedChild,
11633 wait: Duration,
11634) -> Result<RegistrationWaitOutcome, SuperviseError> {
11635 let deadline = Instant::now() + wait;
11636 loop {
11637 if registry
11638 .registration(slot)
11639 .map_err(SuperviseError::Registry)?
11640 .is_some()
11641 {
11642 return Ok(RegistrationWaitOutcome::Registered);
11643 }
11644
11645 let now = Instant::now();
11646 if now >= deadline {
11647 return Ok(RegistrationWaitOutcome::TimedOut);
11648 }
11649 let remaining = deadline.saturating_duration_since(now);
11650 let poll = remaining.min(REGISTRY_RELEASE_POLL);
11651
11652 tokio::select! {
11653 wait_result = child.wait() => {
11654 let status = wait_result.map_err(|source| SuperviseError::Wait {
11655 module_id: module_id.to_string(),
11656 source,
11657 })?;
11658 return Ok(RegistrationWaitOutcome::Exited(classify_reaped_child_exit(
11659 snapshot,
11660 child,
11661 &status,
11662 )));
11663 }
11664 _ = sleep(poll) => {}
11665 }
11666 }
11667}
11668
11669fn registration_failure_exit_report(mut exit_report: ExitReport) -> ExitReport {
11670 if exit_report.kind != ExitKind::DeliberateSeverance {
11674 exit_report.kind = ExitKind::Crash;
11675 }
11676 exit_report
11677}
11678
11679async fn handle_reload_child_registration_failure(
11680 spec: &ModuleSpec,
11681 runtime: &SupervisorRuntimeConfig,
11682 registry: &Registry,
11683 process_liveness: &SupervisorProcessLiveness,
11684 snapshot: &SharedSnapshot,
11685 _child: &mut Option<SupervisedChild>,
11686 failure: ReloadRegistrationFailure,
11687) -> Result<(), SuperviseError> {
11688 let ReloadRegistrationFailure {
11689 exit_report,
11690 reason,
11691 } = failure;
11692 match on_child_exit(
11693 spec,
11694 runtime.restart_policy,
11695 registry,
11696 snapshot,
11697 &runtime.terminal_ring,
11698 &runtime.spawn_events,
11699 &runtime.child_roster,
11700 exit_report,
11701 runtime.recovery_is_daemon,
11702 )
11703 .await
11704 {
11705 NextAction::Stop {
11706 registration_released,
11707 } => {
11708 if registration_released {
11709 process_liveness.untrack_if_current(&spec.module_id, snapshot);
11710 }
11711 }
11712 NextAction::Restart { schedule } => {
11713 let delay = schedule.map_or(runtime.restart_policy.delay_for_restart(0), |schedule| {
11714 schedule.delay
11715 });
11716 if let Some(schedule) = schedule {
11717 log_crash_respawn(&spec.module_id, schedule);
11718 }
11719 schedule_respawn(
11720 runtime,
11721 snapshot,
11722 &spec.module_id,
11723 delay,
11724 RespawnKind::Spawn,
11725 true,
11726 )?;
11727 }
11728 }
11729 Err(SuperviseError::ReloadFailed {
11730 module_id: spec.module_id.clone(),
11731 reason,
11732 })
11733}
11734
11735async fn handle_reload_spawn_failure(
11736 spec: &ModuleSpec,
11737 runtime: &SupervisorRuntimeConfig,
11738 process_liveness: &SupervisorProcessLiveness,
11739 snapshot: &SharedSnapshot,
11740 _child: &mut Option<SupervisedChild>,
11741 reason: String,
11742) -> Result<(), SuperviseError> {
11743 let now = Instant::now();
11744 let mut schedule = None;
11745 update_snapshot(snapshot, Some(&spec.module_id), |state| {
11746 clear_current_process_facts(state);
11747 if state.enabled {
11748 schedule = state.next_crash_restart(&runtime.restart_policy, now);
11749 state.state = if schedule.is_some() {
11750 ModuleState::Restarting
11751 } else {
11752 if runtime
11753 .failure_context(runtime.recovery_is_daemon)
11754 .may_retry()
11755 {
11756 state.failed_retry.schedule(&spec.module_id);
11757 }
11758 ModuleState::Failed
11759 };
11760 } else {
11761 state.state = ModuleState::Disabled;
11762 }
11763 })?;
11764 if let Some(schedule) = schedule {
11765 schedule_respawn(
11766 runtime,
11767 snapshot,
11768 &spec.module_id,
11769 schedule.delay,
11770 RespawnKind::Spawn,
11771 true,
11772 )?;
11773 } else {
11774 process_liveness.untrack_if_current(&spec.module_id, snapshot);
11775 }
11776 Err(SuperviseError::ReloadFailed {
11777 module_id: spec.module_id.clone(),
11778 reason,
11779 })
11780}
11781
11782fn control_flags() -> Flags {
11783 Flags::new(false, Priority::Passive, false)
11784}
11785
11786#[allow(clippy::too_many_arguments)]
11787async fn drain_optional_child(
11788 module_id: &str,
11789 protocol: ModuleProtocol,
11790 stop_notice: StopNotice,
11791 registry: &Registry,
11792 forwarding: Option<&ForwardingTable>,
11793 snapshot: &SharedSnapshot,
11794 terminal_ring: &Arc<Mutex<TerminalRing>>,
11795 spawn_events: &SpawnEventFeed,
11796 child: &mut Option<SupervisedChild>,
11797 drain_timeout: Duration,
11798 final_state: ModuleState,
11799 enabled: Option<bool>,
11800 failure: &FailureContext,
11801) -> Result<(), SuperviseError> {
11802 if let Some(child) = child.take() {
11803 drain_child_to_state(
11804 module_id,
11805 protocol,
11806 stop_notice,
11807 registry,
11808 forwarding,
11809 snapshot,
11810 terminal_ring,
11811 spawn_events,
11812 child,
11813 drain_timeout,
11814 final_state,
11815 enabled,
11816 failure,
11817 )
11818 .await
11819 } else {
11820 update_snapshot(snapshot, Some(module_id), |state| {
11821 state.state = final_state;
11822 if final_state == ModuleState::Failed {
11823 state.failed_retry.failed_at = Some(Instant::now());
11824 }
11825 if let Some(enabled) = enabled {
11826 state.enabled = enabled;
11827 }
11828 clear_current_process_facts(state);
11829 })?;
11830 release_dead_registration(registry, forwarding, snapshot, module_id, failure).await
11831 }
11832}
11833
11834#[allow(clippy::too_many_arguments)]
11835async fn drain_child_to_state(
11836 module_id: &str,
11837 _protocol: ModuleProtocol,
11838 stop_notice: StopNotice,
11839 registry: &Registry,
11840 forwarding: Option<&ForwardingTable>,
11841 snapshot: &SharedSnapshot,
11842 terminal_ring: &Arc<Mutex<TerminalRing>>,
11843 spawn_events: &SpawnEventFeed,
11844 mut child: SupervisedChild,
11845 drain_timeout: Duration,
11846 final_state: ModuleState,
11847 enabled: Option<bool>,
11848 failure: &FailureContext,
11849) -> Result<(), SuperviseError> {
11850 let protocol = child.protocol;
11851 update_snapshot(snapshot, Some(module_id), |state| {
11852 state.state = ModuleState::Draining;
11853 state.draining_to_replace = final_state == ModuleState::Restarting;
11854 if let Some(enabled) = enabled {
11855 state.enabled = enabled;
11856 }
11857 })?;
11858
11859 if stop_notice != StopNotice::SentOverConnection {
11870 if protocol == ModuleProtocol::Subc && stop_notice == StopNotice::NoConnection {
11871 info!(
11872 module_id,
11873 pid = child.pid,
11874 budget_ms = u64::try_from(drain_timeout.as_millis()).unwrap_or(u64::MAX),
11875 "module has no connection yet; requesting stop by signal"
11876 );
11877 }
11878 request_graceful_stop(module_id, &child);
11879 }
11880
11881 let exit_report = match timeout(drain_timeout, child.wait()).await {
11882 Ok(Ok(status)) => classify_reaped_child_exit(snapshot, &child, &status),
11883 Ok(Err(source)) => {
11884 fail_after_wait_error(
11885 module_id,
11886 snapshot,
11887 terminal_ring,
11888 spawn_events,
11889 format!("failed to wait during drain: {source}"),
11890 failure,
11891 );
11892 return Err(SuperviseError::Wait {
11893 module_id: module_id.to_string(),
11894 source,
11895 });
11896 }
11897 Err(_) => {
11898 warn!(
11911 module_id,
11912 pid = child.pid,
11913 budget_ms = u64::try_from(drain_timeout.as_millis()).unwrap_or(u64::MAX),
11914 reason = ?final_state,
11915 ?stop_notice,
11916 "drain budget expired before the module exited; killing it"
11917 );
11918 let kill =
11919 if final_state == ModuleState::Restarting && child.child.container().is_some() {
11920 child.kill_leader()
11921 } else {
11922 child.start_kill()
11923 };
11924 kill.map_err(|source| {
11925 fail_after_wait_error(
11926 module_id,
11927 snapshot,
11928 terminal_ring,
11929 spawn_events,
11930 format!("failed to kill during drain: {source}"),
11931 failure,
11932 );
11933 SuperviseError::Kill {
11934 module_id: module_id.to_string(),
11935 source,
11936 }
11937 })?;
11938 let status = child.wait().await.map_err(|source| {
11939 fail_after_wait_error(
11940 module_id,
11941 snapshot,
11942 terminal_ring,
11943 spawn_events,
11944 format!("failed to wait after drain kill: {source}"),
11945 failure,
11946 );
11947 SuperviseError::Wait {
11948 module_id: module_id.to_string(),
11949 source,
11950 }
11951 })?;
11952 classify_reaped_child_exit(snapshot, &child, &status)
11953 }
11954 };
11955
11956 let detail = lock_snapshot(snapshot)?.drain_disposition_detail.take();
11957 publish_after_terminal_record(
11958 module_id,
11959 || {
11960 record_terminal_with_detail(
11961 module_id,
11962 terminal_ring,
11963 spawn_events,
11964 &exit_report,
11965 terminal_disposition(final_state),
11966 detail,
11967 )
11968 },
11969 || {
11970 update_snapshot(snapshot, Some(module_id), |state| {
11971 state.state = final_state;
11972 if final_state == ModuleState::Failed {
11973 state.failed_retry.failed_at = Some(Instant::now());
11974 }
11975 if let Some(enabled) = enabled {
11976 state.enabled = enabled;
11977 }
11978 clear_current_process_facts(state);
11979 state.last_exit = Some(exit_report.clone());
11980 if exit_report.kind == ExitKind::DeliberateSeverance {
11981 state.lifetime_restarts += 1;
11982 }
11983 })
11984 },
11985 )?;
11986 child.drain_stderr(module_id).await;
11987
11988 let released =
11989 release_dead_registration(registry, forwarding, snapshot, module_id, failure).await;
11990 child
11991 .finish_exit(
11992 released.is_ok()
11993 && final_state == ModuleState::Restarting
11994 && child.spawn_failure.is_none(),
11995 )
11996 .await
11997 .map_err(|source| SuperviseError::Wait {
11998 module_id: module_id.to_owned(),
11999 source,
12000 })?;
12001 released
12002}
12003
12004#[cfg(unix)]
12024fn request_graceful_stop(module_id: &str, child: &SupervisedChild) {
12025 let Some(pid) = child
12026 .id()
12027 .and_then(|pid| i32::try_from(pid).ok())
12028 .and_then(rustix::process::Pid::from_raw)
12029 else {
12030 debug!(
12031 module_id,
12032 "no pid to signal for teardown; falling through to the drain wait"
12033 );
12034 return;
12035 };
12036 match rustix::process::kill_process(pid, rustix::process::Signal::TERM) {
12037 Ok(()) => debug!(
12038 module_id,
12039 "sent SIGTERM to a module nothing else asked to stop"
12040 ),
12041 Err(err) => debug!(
12042 module_id,
12043 error = %err,
12044 "SIGTERM to module failed; the drain wait and kill still apply"
12045 ),
12046 }
12047}
12048
12049#[cfg(not(unix))]
12057fn request_graceful_stop(module_id: &str, _child: &SupervisedChild) {
12058 debug!(
12059 module_id,
12060 "no graceful stop signal exists on this platform; teardown of a module nothing asked to stop waits, then kills"
12061 );
12062}
12063
12064fn terminal_disposition(final_state: ModuleState) -> TerminalDisposition {
12065 match final_state {
12066 ModuleState::Stopped => TerminalDisposition::Stopped,
12067 ModuleState::Disabled => TerminalDisposition::Disabled,
12068 ModuleState::Restarting => TerminalDisposition::Restarting,
12069 ModuleState::Failed => TerminalDisposition::Failed,
12070 ModuleState::Starting
12071 | ModuleState::Running
12072 | ModuleState::Unresponsive
12073 | ModuleState::Draining => {
12074 unreachable!("terminal exits only finish in terminal or restarting states")
12075 }
12076 }
12077}
12078
12079async fn release_dead_registration(
12087 registry: &Registry,
12088 forwarding: Option<&ForwardingTable>,
12089 snapshot: &SharedSnapshot,
12090 module_id: &str,
12091 failure: &FailureContext,
12092) -> Result<(), SuperviseError> {
12093 let result = async {
12094 let registration = registry
12095 .get_module(module_id)
12096 .map_err(SuperviseError::Registry)?;
12097 match wait_for_registration_release(registry, module_id, REGISTRY_RELEASE_TIMEOUT).await {
12098 Ok(()) => return Ok(()),
12099 Err(SuperviseError::RegistrationStillActive { .. }) => {}
12100 Err(err) => return Err(err),
12101 }
12102 let pid = lock_snapshot(snapshot)?.reaped_pid;
12103 if let (Some(registration), Some(forwarding), Some(pid)) = (registration, forwarding, pid) {
12104 warn!(
12105 module_id,
12106 pid,
12107 connection_id = registration.connection_id.get(),
12108 "reaped module registration outlived release grace; closing dead connection"
12109 );
12110 forwarding.request_connection_close(
12111 registration.connection_id,
12112 CloseReason::new(
12113 "supervised_process_reaped",
12114 format!("module '{module_id}' pid {pid} exited"),
12115 ),
12116 );
12117 wait_for_slot_registration_release(
12118 registry,
12119 crate::registry::RegistrationSlot::Connection(registration.connection_id),
12120 REGISTRY_RELEASE_TIMEOUT,
12121 )
12122 .await?;
12123 }
12124 wait_for_registration_release(registry, module_id, Duration::ZERO).await
12125 }
12126 .await;
12127 if let Err(err) = &result {
12128 fail_snapshot(snapshot, Some(module_id), None, failure);
12129 error!(module_id, error = %err, "registration release failed after child exit; module is failed and start can retry");
12130 }
12131 result
12132}
12133
12134async fn wait_for_registration_release(
12137 registry: &Registry,
12138 module_id: &str,
12139 wait: Duration,
12140) -> Result<(), SuperviseError> {
12141 wait_for_slot_registration_release(
12142 registry,
12143 crate::registry::RegistrationSlot::Active(module_id),
12144 wait,
12145 )
12146 .await
12147}
12148
12149async fn wait_for_slot_registration_release(
12157 registry: &Registry,
12158 slot: crate::registry::RegistrationSlot<'_>,
12159 wait: Duration,
12160) -> Result<(), SuperviseError> {
12161 let deadline = Instant::now() + wait;
12162 let mut release_events = registration_release_events().subscribe();
12163 let still_active = |registration: &crate::registry::ModuleRegistration| {
12164 SuperviseError::RegistrationStillActive {
12165 module_id: registration.manifest.module_id.clone(),
12166 waited: wait,
12167 }
12168 };
12169 loop {
12170 let _observed_generation = *release_events.borrow_and_update();
12171 let Some(registration) = registry
12172 .registration(slot)
12173 .map_err(SuperviseError::Registry)?
12174 else {
12175 return Ok(());
12176 };
12177
12178 let now = Instant::now();
12179 if now >= deadline {
12180 return Err(still_active(®istration));
12181 }
12182
12183 let remaining = deadline.saturating_duration_since(now);
12184 match timeout(remaining, release_events.changed()).await {
12185 Ok(Ok(())) | Ok(Err(_)) => {}
12186 Err(_) => return Err(still_active(®istration)),
12187 }
12188 }
12189}
12190
12191#[cfg(test)]
12192mod slot_registration_wait_tests {
12193 use super::*;
12194 use crate::registry::{ConnectionId, RegistrationSlot};
12195 use subc_protocol::manifest::ModuleManifest;
12196
12197 #[tokio::test]
12198 async fn enable_release_failure_is_failed_and_a_second_enable_retries() {
12199 let registry = Arc::new(Registry::default());
12200 let supervisor = Supervisor::new_for_test(Arc::clone(®istry), RestartPolicy::default());
12201 let runtime = supervisor.runtime_config();
12202 let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::disabled()));
12203 let spec = ModuleSpec {
12204 module_id: "enable-stale-registration".to_string(),
12205 program: PathBuf::from("/missing/enable-retry-test"),
12206 args: Vec::new(),
12207 env: Vec::new(),
12208 reserved: false,
12209 reserved_prefixes: Vec::new(),
12210 protocol: ModuleProtocol::Subc,
12211 overlap: Default::default(),
12212 carry_containment: false,
12213 };
12214 let connection = ConnectionId::new(90);
12215 registry
12216 .register_with_control_ops(
12217 ModuleManifest::builder(&spec.module_id, "0.1.0").build(),
12218 1,
12219 connection,
12220 Vec::new(),
12221 )
12222 .unwrap();
12223 let mut child = None;
12224 let err = set_child_enabled(
12225 &spec,
12226 &runtime,
12227 ®istry,
12228 &supervisor.process_liveness,
12229 &snapshot,
12230 &mut child,
12231 true,
12232 &runtime.failure_context(false),
12233 )
12234 .await
12235 .unwrap_err();
12236 assert!(matches!(
12237 err,
12238 SuperviseError::RegistrationStillActive { .. }
12239 ));
12240 assert_eq!(lock_snapshot(&snapshot).unwrap().state, ModuleState::Failed);
12241 assert!(child.is_none());
12242 registry.deregister_connection(connection).unwrap();
12243 let err = set_child_enabled(
12244 &spec,
12245 &runtime,
12246 ®istry,
12247 &supervisor.process_liveness,
12248 &snapshot,
12249 &mut child,
12250 true,
12251 &runtime.failure_context(false),
12252 )
12253 .await
12254 .unwrap_err();
12255 assert!(
12256 matches!(err, SuperviseError::Spawn { .. }),
12257 "second enable must attempt a spawn: {err}"
12258 );
12259 assert_eq!(lock_snapshot(&snapshot).unwrap().state, ModuleState::Failed);
12260 }
12261
12262 const INCUMBENT: u64 = 1;
12263 const CANDIDATE: u64 = 2;
12264
12265 fn swapped_registry() -> Arc<Registry> {
12266 let registry = Arc::new(Registry::default());
12267 let manifest = ModuleManifest::builder("m", "0.1.0").build();
12268 registry
12269 .register_with_control_ops(
12270 manifest.clone(),
12271 1,
12272 ConnectionId::new(INCUMBENT),
12273 Vec::new(),
12274 )
12275 .unwrap();
12276 registry
12277 .register_candidate_with_control_ops(
12278 manifest,
12279 1,
12280 ConnectionId::new(CANDIDATE),
12281 Vec::new(),
12282 )
12283 .unwrap();
12284 registry
12285 }
12286
12287 #[tokio::test]
12291 async fn incumbent_release_is_awaited_by_connection_not_by_module_id() {
12292 let registry = swapped_registry();
12293 registry.promote_candidate("m").unwrap().unwrap();
12294
12295 assert!(matches!(
12296 wait_for_registration_release(®istry, "m", Duration::from_millis(50)).await,
12297 Err(SuperviseError::RegistrationStillActive { .. })
12298 ));
12299
12300 assert!(matches!(
12302 wait_for_slot_registration_release(
12303 ®istry,
12304 RegistrationSlot::Connection(ConnectionId::new(INCUMBENT)),
12305 Duration::from_millis(50),
12306 )
12307 .await,
12308 Err(SuperviseError::RegistrationStillActive { .. })
12309 ));
12310
12311 let releaser = Arc::clone(®istry);
12312 let release = tokio::spawn(async move {
12313 sleep(Duration::from_millis(20)).await;
12314 releaser
12315 .deregister_connection(ConnectionId::new(INCUMBENT))
12316 .unwrap();
12317 notify_registration_release();
12318 });
12319 wait_for_slot_registration_release(
12320 ®istry,
12321 RegistrationSlot::Connection(ConnectionId::new(INCUMBENT)),
12322 Duration::from_secs(5),
12323 )
12324 .await
12325 .expect("the incumbent's own registration is released");
12326 release.await.unwrap();
12327 assert!(registry.get_module("m").unwrap().is_some());
12328 }
12329
12330 #[tokio::test]
12333 async fn candidate_slot_wait_ignores_the_incumbents_registration() {
12334 let registry = swapped_registry();
12335 assert!(matches!(
12336 wait_for_slot_registration_release(
12337 ®istry,
12338 RegistrationSlot::Candidate("m"),
12339 Duration::from_millis(50),
12340 )
12341 .await,
12342 Err(SuperviseError::RegistrationStillActive { .. })
12343 ));
12344 registry
12345 .deregister_connection(ConnectionId::new(CANDIDATE))
12346 .unwrap();
12347 wait_for_slot_registration_release(
12348 ®istry,
12349 RegistrationSlot::Candidate("m"),
12350 Duration::from_millis(50),
12351 )
12352 .await
12353 .expect("a candidate slot with no candidate is released");
12354 assert!(registry
12355 .registration(RegistrationSlot::Active("m"))
12356 .unwrap()
12357 .is_some());
12358 }
12359}
12360
12361fn classify_exit(status: &ExitStatus) -> ExitReport {
12362 ExitReport {
12363 kind: if status.success() {
12364 ExitKind::Clean
12365 } else {
12366 ExitKind::Crash
12367 },
12368 code: status.code(),
12369 signal: exit_signal(status),
12370 at_ms: unix_ms_now(),
12371 }
12372}
12373
12374fn wait_error_exit_report() -> ExitReport {
12380 ExitReport {
12381 kind: ExitKind::Crash,
12382 code: None,
12383 signal: None,
12384 at_ms: unix_ms_now(),
12385 }
12386}
12387
12388#[cfg(unix)]
12389fn exit_signal(status: &ExitStatus) -> Option<i32> {
12390 use std::os::unix::process::ExitStatusExt;
12391
12392 status.signal()
12393}
12394
12395#[cfg(not(unix))]
12396fn exit_signal(_status: &ExitStatus) -> Option<i32> {
12397 None
12398}
12399
12400fn reset_restart_count(snapshot: &SharedSnapshot, module_id: &str) -> Result<(), SuperviseError> {
12406 update_snapshot(snapshot, Some(module_id), |state| {
12407 state.clear_crash_restarts();
12408 })
12409}
12410
12411fn set_running(
12412 snapshot: &SharedSnapshot,
12413 child: &SupervisedChild,
12414 module_id: &str,
12415 spawn_events: &SpawnEventFeed,
12416) -> Result<(), SuperviseError> {
12417 let mut state = snapshot.lock().map_err(|_| SuperviseError::StatePoisoned {
12418 module_id: Some(module_id.to_string()),
12419 })?;
12420 state.spawn_generation = spawn_events.emit_spawned(module_id, child.pid, child.spawned_at_ms);
12421 if std::mem::take(&mut state.coalesced_restart_pending) {
12422 let generation = state.spawn_generation;
12423 info!(module_id, "restart already satisfied by generation {generation}; coalesced pending request completed");
12424 }
12425 state.drain_disposition_detail = None;
12426 state.spawn_failure = None;
12427 state.waiting_for_previous_pid = None;
12428 state.waiting_for_previous_detail = None;
12429 state.in_alternate_slot = false;
12432 state.configuration_updated_since_spawn = false;
12433 state.spawned_protocol = Some(child.protocol);
12434 state.state = ModuleState::Running;
12435 state.enabled = true;
12436 state.process_alive = true;
12437 state.pid = child.id();
12438 state.subtree_target = child.subtree_target.clone();
12439 state.health.status = SupervisorHealthStatus::Unknown;
12442 state.last_probe_evidence = None;
12443 state.health.consecutive_failures = 0;
12444 state.health.detail = None;
12445 state.health.metrics = None;
12446 state.health.last_probe_ms = None;
12447 state.failed_retry.healthy_since = None;
12448 state.failed_retry.failed_at = None;
12449 state.failed_retry.running_since = Some(child.spawned_at);
12450 #[cfg(windows)]
12451 {
12452 state.pipe_consumed = child
12453 .nonce_delivery
12454 .as_ref()
12455 .map(|delivery| delivery.consumed());
12456 }
12457 #[cfg(target_os = "macos")]
12458 {
12459 state.report_ready = Some(Arc::clone(&child.report_ready));
12460 }
12461 state.spawned_at_ms = Some(child.spawned_at_ms);
12462 state.exec_outcome = Some(Arc::clone(&child.exec.0));
12463 state.spawned_from = Some(child.spawned_from.clone());
12464 state.launch_spec_digest = Some(child.launch_spec_digest.clone());
12465 state.spawned_file_identity = child.spawned_file_identity.clone();
12466 state.process_start_time = child.process_start_time;
12467 Ok(())
12468}
12469
12470fn clear_current_process_facts(state: &mut SupervisorSnapshot) {
12471 state.subtree_target = crate::subtree_resources::Target::Unavailable("not_running");
12472 state.failed_retry.healthy_since = None;
12473 state.failed_retry.running_since = None;
12474 state.waiting_for_previous_pid = None;
12475 state.waiting_for_previous_detail = None;
12476 state.process_alive = false;
12477 state.spawned_protocol = None;
12478 state.pid = None;
12479 #[cfg(windows)]
12480 {
12481 state.pipe_consumed = None;
12482 }
12483 #[cfg(target_os = "macos")]
12484 {
12485 state.report_ready = None;
12486 }
12487 state.spawned_at_ms = None;
12488 state.exec_outcome = None;
12489 state.spawned_from = None;
12490 state.launch_spec_digest = None;
12491 state.spawned_file_identity = None;
12492 state.process_start_time = None;
12493 state.deliberate_severance = None;
12494}
12495
12496#[cfg(test)]
12497fn record_deliberate_severance(
12498 snapshot: &SharedSnapshot,
12499 identity: ProcessIdentity,
12500) -> Result<(), SuperviseError> {
12501 update_snapshot(snapshot, None, |state| {
12502 state.deliberate_severance = Some(identity);
12503 })
12504}
12505
12506fn apply_deliberate_severance_marker(
12507 snapshot: &SharedSnapshot,
12508 exited_identity: Option<ProcessIdentity>,
12509 mut exit_report: ExitReport,
12510) -> ExitReport {
12511 let marker = lock_snapshot(snapshot)
12512 .ok()
12513 .and_then(|mut state| state.deliberate_severance.take());
12514 if marker.is_some() && marker == exited_identity {
12515 exit_report.kind = ExitKind::DeliberateSeverance;
12516 }
12517 exit_report
12518}
12519
12520fn classify_reaped_child_exit(
12521 snapshot: &SharedSnapshot,
12522 child: &SupervisedChild,
12523 status: &ExitStatus,
12524) -> ExitReport {
12525 let _ = update_snapshot(snapshot, None, |state| {
12526 state.reaped_pid = Some(child.pid);
12527 state.spawn_failure = child.spawn_failure.clone();
12528 });
12529 if child.spawn_failure.as_deref() == Some("registration_timeout") {
12530 registration_failure_exit_report(classify_exit(status))
12531 } else {
12532 apply_deliberate_severance_marker(snapshot, child.process_identity(), classify_exit(status))
12533 }
12534}
12535
12536fn fail_snapshot(
12537 snapshot: &SharedSnapshot,
12538 module_id: Option<&str>,
12539 last_exit: Option<ExitReport>,
12540 failure: &FailureContext,
12541) {
12542 if let Err(err) = update_snapshot(snapshot, module_id, |state| {
12543 state.state = ModuleState::Failed;
12544 clear_current_process_facts(state);
12545 if failure.may_retry() && state.enabled {
12546 state.failed_retry.schedule(module_id.unwrap_or("unknown"));
12547 } else if failure.shutdown.is_set() {
12548 state.failed_retry.cancel();
12549 }
12550 if let Some(last_exit) = last_exit {
12551 state.last_exit = Some(last_exit);
12552 }
12553 }) {
12554 error!(error = %err, "failed to mark supervisor state failed");
12555 }
12556}
12557
12558fn update_snapshot(
12559 snapshot: &SharedSnapshot,
12560 module_id: Option<&str>,
12561 update: impl FnOnce(&mut SupervisorSnapshot),
12562) -> Result<(), SuperviseError> {
12563 let mut state = snapshot.lock().map_err(|_| SuperviseError::StatePoisoned {
12564 module_id: module_id.map(ToOwned::to_owned),
12565 })?;
12566 update(&mut state);
12567 Ok(())
12568}
12569
12570const SLOW_SNAPSHOT_LOCK_THRESHOLD: Duration = Duration::from_millis(250);
12571
12572fn lock_snapshot_for_control<'a>(
12573 snapshot: &'a SharedSnapshot,
12574 module_id: &str,
12575 caller: &'static str,
12576) -> Result<std::sync::MutexGuard<'a, SupervisorSnapshot>, SuperviseError> {
12577 let started_at = Instant::now();
12578 let guard = lock_snapshot(snapshot)?;
12579 let waited = started_at.elapsed();
12580 if waited >= SLOW_SNAPSHOT_LOCK_THRESHOLD {
12581 warn!(
12582 module_id = %module_id,
12583 waited_ms = waited.as_millis() as u64,
12584 caller = %caller,
12585 "slow snapshot lock"
12586 );
12587 }
12588 Ok(guard)
12589}
12590
12591fn lock_snapshot(
12592 snapshot: &SharedSnapshot,
12593) -> Result<std::sync::MutexGuard<'_, SupervisorSnapshot>, SuperviseError> {
12594 snapshot
12595 .lock()
12596 .map_err(|_| SuperviseError::StatePoisoned { module_id: None })
12597}
12598
12599#[cfg(test)]
12600mod terminal_history_tests {
12601 use std::{
12602 path::PathBuf,
12603 sync::Arc,
12604 time::{Duration, Instant},
12605 };
12606
12607 use tokio::{sync::mpsc, time::sleep};
12608
12609 use super::{
12610 apply_deliberate_severance_marker, daemon_will_restart, drain_child_to_state,
12611 drained_after_quiescence_wait, handle_reload_spawn_failure, health_restart_child,
12612 lock_snapshot, on_child_exit, record_deliberate_severance, record_wait_error_terminal,
12613 reset_restart_count, spawn_and_mark_running, update_snapshot, wait_error_exit_report,
12614 ExitKind, ExitReport, ModuleProtocol, ModuleSpec, ModuleState, NextAction, ProcessIdentity,
12615 RestartPolicy, SpawnEventKind, StopNotice, SuperviseError, SupervisedConfiguration,
12616 SupervisedModule, SupervisedModuleInner, Supervisor, SupervisorHandle,
12617 SupervisorHealthStatus, SupervisorSnapshot,
12618 };
12619 use super::Instant as ClockInstant;
12624 use crate::{
12625 registry::Registry,
12626 terminal_ring::{TerminalRing, TerminalRingConfig},
12627 };
12628 use std::sync::Mutex;
12629 use subc_control::TerminalDisposition;
12630
12631 pub(super) fn fake_aft_stub_path() -> PathBuf {
12636 let mut path = std::env::current_exe().expect("current_exe available in tests");
12637 path.pop();
12638 path.pop();
12639 path.push(if cfg!(windows) {
12640 "fake-aft-stub.exe"
12641 } else {
12642 "fake-aft-stub"
12643 });
12644 assert!(
12645 path.exists(),
12646 "fake-aft-stub not built at {}: run `cargo test -p subc-core` (which builds \
12647 [[bin]] targets) rather than `cargo test -p subc-core --lib` (which does not)",
12648 path.display()
12649 );
12650 path
12651 }
12652
12653 #[test]
12654 fn reserved_never_spawned_refuses_every_hello() {
12655 let supervisor = SupervisorHandle::default();
12660 supervisor.apply_identity_configuration(&ModuleSpec {
12661 module_id: "never-spawned".to_string(),
12662 program: PathBuf::from("/usr/bin/false"),
12663 args: Vec::new(),
12664 env: Vec::new(),
12665 reserved: true,
12666 reserved_prefixes: Vec::new(),
12667 protocol: ModuleProtocol::Subc,
12668 overlap: Default::default(),
12669 carry_containment: false,
12670 });
12671 assert!(
12672 supervisor
12673 .reserved_hello_verification("never-spawned", Some("any-forged-nonce"))
12674 .is_err(),
12675 "forged nonce must refuse on a reserved never-spawned id"
12676 );
12677 assert!(
12678 supervisor
12679 .reserved_hello_verification("never-spawned", None)
12680 .is_err(),
12681 "absent nonce must refuse on a reserved never-spawned id"
12682 );
12683 supervisor.set_spawn_nonce("never-spawned", "minted".to_string());
12685 supervisor.apply_identity_configuration(&ModuleSpec {
12686 module_id: "never-spawned".to_string(),
12687 program: PathBuf::from("/usr/bin/false"),
12688 args: Vec::new(),
12689 env: Vec::new(),
12690 reserved: true,
12691 reserved_prefixes: Vec::new(),
12692 protocol: ModuleProtocol::Subc,
12693 overlap: Default::default(),
12694 carry_containment: false,
12695 });
12696 assert!(supervisor
12697 .reserved_hello_verification("never-spawned", Some("minted"))
12698 .is_ok());
12699 assert!(supervisor
12700 .reserved_hello_verification("never-spawned", Some("forged"))
12701 .is_err());
12702 }
12703
12704 fn seed_crash_restarts(state: &mut SupervisorSnapshot, count: u32) {
12707 let now = ClockInstant::now();
12708 for _ in 0..count {
12709 state.crash_restarts.push_back(now);
12710 }
12711 }
12712
12713 fn age_oldest_crash_restart_out_of_window(state: &mut SupervisorSnapshot, window: Duration) {
12717 let aged = state
12718 .crash_restarts
12719 .front()
12720 .expect("a crash restart must be recorded before it can be aged")
12721 .checked_sub(window + Duration::from_secs(1))
12722 .expect("the test clock is far enough from its origin to age an instant");
12723 state.crash_restarts[0] = aged;
12724 }
12725
12726 fn snapshot_with_restarts(enabled: bool, count: u32) -> SupervisorSnapshot {
12727 let mut state = SupervisorSnapshot::new(ModuleState::Running, enabled);
12728 seed_crash_restarts(&mut state, count);
12729 state
12730 }
12731
12732 #[test]
12733 fn daemon_owned_recovery_predicate_uses_the_pre_increment_budget() {
12734 let policy = RestartPolicy::new(3, Duration::ZERO);
12735 let now = ClockInstant::now();
12736 assert!(daemon_will_restart(
12737 &mut snapshot_with_restarts(true, 2),
12738 &policy,
12739 now
12740 ));
12741 assert!(!daemon_will_restart(
12742 &mut snapshot_with_restarts(true, 3),
12743 &policy,
12744 now
12745 ));
12746 assert!(!daemon_will_restart(
12747 &mut snapshot_with_restarts(false, 0),
12748 &policy,
12749 now
12750 ));
12751 }
12752
12753 #[test]
12754 fn crash_restart_backoff_escalates_with_in_window_count() {
12755 let policy = RestartPolicy::new(4, Duration::from_millis(100))
12756 .with_max_backoff(Duration::from_secs(30));
12757 let now = ClockInstant::now();
12758 let mut state = SupervisorSnapshot::new(ModuleState::Running, true);
12759 let schedules = (0..4)
12760 .map(|_| {
12761 state
12762 .next_crash_restart(&policy, now)
12763 .expect("the test policy allows four crash restarts")
12764 })
12765 .collect::<Vec<_>>();
12766
12767 assert_eq!(
12768 schedules
12769 .iter()
12770 .map(|schedule| schedule.restart_in_window)
12771 .collect::<Vec<_>>(),
12772 vec![0, 1, 2, 3]
12773 );
12774 assert_eq!(
12775 schedules
12776 .iter()
12777 .map(|schedule| schedule.delay)
12778 .collect::<Vec<_>>(),
12779 vec![
12780 Duration::from_millis(100),
12781 Duration::from_secs(1),
12782 Duration::from_secs(10),
12783 Duration::from_secs(30),
12784 ]
12785 );
12786 }
12787
12788 #[test]
12789 fn crash_restart_backoff_resets_after_ring_clear() {
12790 let policy = RestartPolicy::new(3, Duration::from_millis(100));
12791 let now = ClockInstant::now();
12792 let mut state = SupervisorSnapshot::new(ModuleState::Running, true);
12793 assert_eq!(
12794 state.next_crash_restart(&policy, now).unwrap().delay,
12795 Duration::from_millis(100)
12796 );
12797 assert_eq!(
12798 state.next_crash_restart(&policy, now).unwrap().delay,
12799 Duration::from_secs(1)
12800 );
12801
12802 state.clear_crash_restarts();
12803 let schedule = state
12804 .next_crash_restart(&policy, now)
12805 .expect("a cleared ring must allow another restart");
12806 assert_eq!(schedule.restart_in_window, 0);
12807 assert_eq!(schedule.delay, Duration::from_millis(100));
12808 }
12809
12810 #[test]
12811 fn crash_restart_backoff_ignores_aged_restarts() {
12812 let policy = RestartPolicy::new(3, Duration::from_millis(100));
12813 let now = ClockInstant::now();
12814 let mut state = SupervisorSnapshot::new(ModuleState::Running, true);
12815 state
12816 .next_crash_restart(&policy, now)
12817 .expect("the first restart is allowed");
12818 state
12819 .next_crash_restart(&policy, now)
12820 .expect("the second restart is allowed");
12821 state.crash_restarts[0] = now
12822 .checked_sub(policy.window + Duration::from_secs(1))
12823 .expect("the fake clock can age a restart past the window");
12824
12825 let schedule = state
12826 .next_crash_restart(&policy, now)
12827 .expect("an aged restart must release its slot");
12828 assert_eq!(schedule.restart_in_window, 1);
12829 assert_eq!(schedule.delay, Duration::from_secs(1));
12830 assert_eq!(state.crash_restarts.len(), 2);
12831 }
12832
12833 #[test]
12837 fn a_budget_spent_before_the_window_no_longer_refuses() {
12838 let policy = RestartPolicy::new(3, Duration::ZERO);
12839 let mut state = snapshot_with_restarts(true, 3);
12840 let now = ClockInstant::now();
12841 assert!(!daemon_will_restart(&mut state, &policy, now));
12842
12843 assert!(daemon_will_restart(
12844 &mut state,
12845 &policy,
12846 now + policy.window + Duration::from_secs(1)
12847 ));
12848 assert!(
12849 state.crash_restarts.is_empty(),
12850 "reading the budget must drop the instants that left the window"
12851 );
12852 }
12853
12854 fn module_with_recovery_snapshot(
12855 state: ModuleState,
12856 enabled: bool,
12857 restart_count: u32,
12858 ) -> SupervisedModule {
12859 let registry = Arc::new(Registry::default());
12860 let supervisor =
12861 Supervisor::new_for_test(Arc::clone(®istry), RestartPolicy::new(3, Duration::ZERO));
12862 let runtime = supervisor.runtime_config();
12863 let spec = ModuleSpec {
12864 module_id: "recovery-snapshot".to_string(),
12865 program: fake_aft_stub_path(),
12866 args: Vec::new(),
12867 env: Vec::new(),
12868 reserved: false,
12869 reserved_prefixes: Vec::new(),
12870 protocol: ModuleProtocol::Subc,
12871 overlap: Default::default(),
12872 carry_containment: false,
12873 };
12874 let mut snapshot = SupervisorSnapshot::new(state, enabled);
12875 seed_crash_restarts(&mut snapshot, restart_count);
12876 let (commands, _rx) = mpsc::channel(4);
12879 SupervisedModule {
12880 inner: Arc::new(SupervisedModuleInner {
12881 module_id: spec.module_id.clone(),
12882 registry,
12883 snapshot: Arc::new(Mutex::new(snapshot)),
12884 configuration: Arc::new(Mutex::new(SupervisedConfiguration {
12885 spec,
12886 health: runtime.health,
12887 })),
12888 stderr_ring: runtime.stderr_ring,
12889 terminal_ring: runtime.terminal_ring,
12890 commands,
12891 monitor: Mutex::new(None),
12892 restart_policy: runtime.restart_policy,
12893 effective_drain_timeout: runtime.effective_drain_timeout,
12894 provenance_probe: supervisor.provenance_probe.clone(),
12895 retained: runtime.child_roster.retained_view(),
12896 }),
12897 }
12898 }
12899
12900 #[cfg(target_os = "linux")]
12901 #[tokio::test]
12902 async fn no_cgroup_placement_does_not_block_fake_aft_stub_spawn() {
12903 let supervisor =
12904 Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default())
12905 .with_cgroup_placement(None);
12906 let result = supervisor.spawn(ModuleSpec {
12907 module_id: "no-cgroup-placement".to_string(),
12908 program: fake_aft_stub_path(),
12909 args: Vec::new(),
12910 env: Vec::new(),
12911 reserved: false,
12912 reserved_prefixes: Vec::new(),
12913 protocol: ModuleProtocol::Subc,
12914 overlap: Default::default(),
12915 carry_containment: false,
12916 });
12917
12918 assert!(
12919 result.is_ok(),
12920 "no delegation must not turn an otherwise valid spawn into a failure: {result:?}"
12921 );
12922 }
12923
12924 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
12925 async fn undecided_snapshot_uses_shared_restart_predicate() {
12926 assert!(module_with_recovery_snapshot(ModuleState::Running, true, 2)
12927 .will_recover_after_connection_loss()
12928 .unwrap());
12929 assert!(
12930 !module_with_recovery_snapshot(ModuleState::Running, true, 3)
12931 .will_recover_after_connection_loss()
12932 .unwrap()
12933 );
12934 }
12935
12936 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
12937 async fn restarting_snapshot_at_exhausted_budget_is_non_terminal() {
12938 assert!(
12939 module_with_recovery_snapshot(ModuleState::Restarting, true, 3)
12940 .will_recover_after_connection_loss()
12941 .unwrap()
12942 );
12943 }
12944
12945 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
12946 async fn terminal_phase_snapshots_are_terminal_before_budget_exhaustion() {
12947 assert!(!module_with_recovery_snapshot(ModuleState::Failed, true, 0)
12948 .will_recover_after_connection_loss()
12949 .unwrap());
12950 assert!(
12951 !module_with_recovery_snapshot(ModuleState::Disabled, true, 0)
12952 .will_recover_after_connection_loss()
12953 .unwrap()
12954 );
12955 }
12956
12957 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
12958 async fn warming_snapshot_is_limited_to_startup_phases() {
12959 for state in [
12960 ModuleState::Starting,
12961 ModuleState::Running,
12962 ModuleState::Restarting,
12963 ] {
12964 assert!(
12965 module_with_recovery_snapshot(state, true, 0)
12966 .is_warming()
12967 .unwrap(),
12968 "{state:?} should be warming"
12969 );
12970 }
12971 for state in [
12972 ModuleState::Unresponsive,
12973 ModuleState::Draining,
12974 ModuleState::Stopped,
12975 ModuleState::Failed,
12976 ModuleState::Disabled,
12977 ] {
12978 assert!(
12979 !module_with_recovery_snapshot(state, true, 0)
12980 .is_warming()
12981 .unwrap(),
12982 "{state:?} should not be warming"
12983 );
12984 }
12985 }
12986
12987 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
12988 async fn terminal_history_survives_respawn_and_keeps_both_crashes_in_order() {
12989 let registry = Arc::new(Registry::default());
12990 let supervisor =
12991 Supervisor::new_for_test(Arc::clone(®istry), RestartPolicy::new(1, Duration::ZERO));
12992 let module = supervisor
12993 .spawn(ModuleSpec {
12994 module_id: "terminal-history".to_string(),
12995 program: fake_aft_stub_path(),
12996 args: Vec::new(),
12997 env: vec![("FAKE_AFT_EXIT_CODE".to_string(), "23".to_string())],
12998 reserved: false,
12999 reserved_prefixes: Vec::new(),
13000 protocol: ModuleProtocol::Subc,
13001 overlap: Default::default(),
13002 carry_containment: false,
13003 })
13004 .unwrap();
13005
13006 let deadline = Instant::now() + Duration::from_secs(5);
13007 loop {
13008 let history = module.terminal_history();
13009 if history.entries.len() == 2 {
13010 assert_eq!(module.status().unwrap().state, ModuleState::Failed);
13011 assert_eq!(history.dropped, 0);
13012 assert_eq!(
13013 history
13014 .entries
13015 .iter()
13016 .map(|entry| entry.exit_code)
13017 .collect::<Vec<_>>(),
13018 vec![Some(23), Some(23)]
13019 );
13020 assert!(history.entries[0].at_ms <= history.entries[1].at_ms);
13021 return;
13022 }
13023 assert!(
13024 Instant::now() < deadline,
13025 "module did not retain two terminal exits: {history:?}"
13026 );
13027 sleep(Duration::from_millis(10)).await;
13028 }
13029 }
13030
13031 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
13035 async fn disable_during_crash_backoff_cancels_pending_respawn() {
13036 let backoff = Duration::from_secs(2);
13037 let supervisor = Supervisor::new_for_test(
13038 Arc::new(Registry::default()),
13039 RestartPolicy::new(10, backoff),
13040 );
13041 let module = supervisor
13042 .spawn(ModuleSpec {
13043 module_id: "disable-during-backoff".to_string(),
13044 program: fake_aft_stub_path(),
13045 args: Vec::new(),
13046 env: vec![("FAKE_AFT_EXIT_CODE".to_string(), "23".to_string())],
13047 reserved: false,
13048 reserved_prefixes: Vec::new(),
13049 protocol: ModuleProtocol::Subc,
13050 overlap: Default::default(),
13051 carry_containment: false,
13052 })
13053 .unwrap();
13054
13055 let deadline = Instant::now() + Duration::from_secs(5);
13057 loop {
13058 if module.status().unwrap().state == ModuleState::Restarting {
13059 break;
13060 }
13061 assert!(
13062 Instant::now() < deadline,
13063 "module never entered the crash backoff"
13064 );
13065 sleep(Duration::from_millis(10)).await;
13066 }
13067
13068 let started = Instant::now();
13069 module.set_enabled(false).await.unwrap();
13070 let waited = started.elapsed();
13071
13072 assert!(
13073 waited < backoff / 2,
13074 "disable waited {waited:?} behind the {backoff:?} crash backoff; the operator command must preempt the pending respawn"
13075 );
13076 assert_eq!(module.status().unwrap().state, ModuleState::Disabled);
13077
13078 sleep(backoff + Duration::from_millis(500)).await;
13080 let status = module.status().unwrap();
13081 assert_eq!(status.state, ModuleState::Disabled);
13082 assert_eq!(
13083 status.spawn_generation, 1,
13084 "module respawned after the operator disabled it"
13085 );
13086 }
13087
13088 #[cfg(unix)]
13091 fn protocol_none_sigterm_exits_clean_spec(
13092 module_id: &str,
13093 dir: &std::path::Path,
13094 ) -> (ModuleSpec, PathBuf, PathBuf) {
13095 let ready = dir.join("ready");
13096 let marker = dir.join("sigterm");
13097 let spec = ModuleSpec {
13098 module_id: module_id.to_string(),
13099 program: fake_aft_stub_path(),
13100 args: Vec::new(),
13101 env: vec![
13102 ("FAKE_AFT_NEVER_CONNECT".to_string(), "1".to_string()),
13103 (
13104 "FAKE_AFT_SIGTERM_MARKER_PATH".to_string(),
13105 marker.display().to_string(),
13106 ),
13107 (
13108 "FAKE_AFT_NEVER_CONNECT_READY_PATH".to_string(),
13109 ready.display().to_string(),
13110 ),
13111 ],
13112 reserved: false,
13113 reserved_prefixes: Vec::new(),
13114 protocol: ModuleProtocol::None,
13115 overlap: Default::default(),
13116 carry_containment: false,
13117 };
13118 (spec, ready, marker)
13119 }
13120
13121 #[cfg(unix)]
13125 async fn wait_for_file(path: &std::path::Path) {
13126 let deadline = Instant::now() + Duration::from_secs(10);
13127 while !path.exists() {
13128 assert!(
13129 Instant::now() < deadline,
13130 "{} never appeared",
13131 path.display()
13132 );
13133 sleep(Duration::from_millis(10)).await;
13134 }
13135 }
13136
13137 #[cfg(unix)]
13141 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
13142 async fn protocol_none_unrequested_clean_exit_restarts_as_a_crash() {
13143 let dir = cortexkit_test_support::ScratchDir::new("none-unrequested-clean-exit");
13144 let (spec, ready, marker) =
13145 protocol_none_sigterm_exits_clean_spec("none-unrequested-clean-exit", dir.path());
13146 let supervisor = Supervisor::new_for_test(
13147 Arc::new(Registry::default()),
13148 RestartPolicy::new(3, Duration::ZERO),
13149 );
13150 let module = supervisor.spawn(spec).unwrap();
13151 wait_for_file(&ready).await;
13152 let deadline = Instant::now() + Duration::from_secs(10);
13156 let first_pid = loop {
13157 let status = module.status().unwrap();
13158 if status.state == ModuleState::Running {
13159 if let Some(pid) = status.pid {
13160 break pid;
13161 }
13162 }
13163 assert!(
13164 Instant::now() < deadline,
13165 "a running module must report its pid after exec confirmation: {status:?}"
13166 );
13167 sleep(Duration::from_millis(10)).await;
13168 };
13169
13170 rustix::process::kill_process(
13171 rustix::process::Pid::from_raw(i32::try_from(first_pid).unwrap()).unwrap(),
13172 rustix::process::Signal::TERM,
13173 )
13174 .unwrap();
13175
13176 let deadline = Instant::now() + Duration::from_secs(10);
13177 let respawned = loop {
13178 let status = module.status().unwrap();
13179 if status.state == ModuleState::Running
13180 && status.pid.is_some_and(|pid| pid != first_pid)
13181 {
13182 break status;
13183 }
13184 assert!(
13185 Instant::now() < deadline,
13186 "protocol-none module was not respawned after an unrequested clean exit: {status:?}"
13187 );
13188 sleep(Duration::from_millis(10)).await;
13189 };
13190 assert_eq!(respawned.spawn_generation, 2);
13191 assert!(
13192 marker.exists(),
13193 "the child must have exited through its SIGTERM handler (exit 0), or this proves nothing about clean exits"
13194 );
13195
13196 let history = module.terminal_history();
13197 assert_eq!(history.entries.len(), 1, "{history:?}");
13198 let entry = &history.entries[0];
13199 assert_eq!(entry.exit_code, Some(0));
13200 assert_eq!(entry.exit_kind, subc_control::TerminalExitKind::Clean);
13201 assert_eq!(entry.disposition, TerminalDisposition::Restarting);
13202
13203 module.stop().await.unwrap();
13204 }
13205
13206 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
13210 async fn protocol_none_repeated_clean_exits_exhaust_the_restart_budget() {
13211 let supervisor = Supervisor::new_for_test(
13212 Arc::new(Registry::default()),
13213 RestartPolicy::new(1, Duration::ZERO),
13214 );
13215 let module = supervisor
13216 .spawn(ModuleSpec {
13217 module_id: "none-clean-exit-budget".to_string(),
13218 program: fake_aft_stub_path(),
13219 args: Vec::new(),
13220 env: vec![("FAKE_AFT_EXIT_CODE".to_string(), "0".to_string())],
13221 reserved: false,
13222 reserved_prefixes: Vec::new(),
13223 protocol: ModuleProtocol::None,
13224 overlap: Default::default(),
13225 carry_containment: false,
13226 })
13227 .unwrap();
13228
13229 let deadline = Instant::now() + Duration::from_secs(10);
13232 loop {
13233 let status = module.status().unwrap();
13234 if status.state == ModuleState::Failed {
13235 break;
13236 }
13237 assert!(
13238 Instant::now() < deadline,
13239 "module never exhausted its budget: {status:?} {:?}",
13240 module.terminal_history()
13241 );
13242 sleep(Duration::from_millis(10)).await;
13243 }
13244 let history = module.terminal_history();
13245 assert_eq!(
13246 history
13247 .entries
13248 .iter()
13249 .map(|entry| (entry.exit_code, entry.disposition.clone()))
13250 .collect::<Vec<_>>(),
13251 vec![
13252 (Some(0), TerminalDisposition::Restarting),
13253 (Some(0), TerminalDisposition::Failed),
13254 ]
13255 );
13256 let detail = history.entries[1]
13257 .disposition_detail
13258 .as_deref()
13259 .expect("a budget failure names the budget");
13260 assert!(detail.contains("max_restarts=1"), "{detail}");
13261 assert_eq!(module.status().unwrap().spawn_generation, 2);
13262 }
13263
13264 #[test]
13265 fn restart_budget_failure_is_published_after_its_terminal_record() {
13266 use std::io::{Read, Write};
13267 let supervisor = Supervisor::new_for_test(
13268 Arc::new(Registry::default()),
13269 RestartPolicy::new(0, Duration::ZERO),
13270 );
13271 let runtime = supervisor.runtime_config();
13272 let spec = ModuleSpec {
13273 protocol: ModuleProtocol::None,
13274 ..windowed_crash_spec("budget-publication-order")
13275 };
13276 let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::new(
13277 ModuleState::Running,
13278 true,
13279 )));
13280 lock_snapshot(&snapshot).unwrap().process_alive = true;
13281 let reaped_snapshot = snapshot.clone();
13282 let mut barrier = terminal_record_barrier(&runtime.spawn_events);
13283 let history = runtime.terminal_ring.clone();
13284 let reap = std::thread::spawn(move || {
13285 tokio::runtime::Builder::new_current_thread()
13286 .enable_all()
13287 .build()
13288 .unwrap()
13289 .block_on(on_child_exit(
13290 &spec,
13291 runtime.restart_policy,
13292 &supervisor.registry,
13293 &reaped_snapshot,
13294 &runtime.terminal_ring,
13295 &runtime.spawn_events,
13296 &runtime.child_roster,
13297 ExitReport {
13298 kind: ExitKind::Clean,
13299 code: Some(0),
13300 signal: None,
13301 at_ms: 1,
13302 },
13303 true,
13304 ))
13305 });
13306 let mut reached = [0];
13307 barrier.read_exact(&mut reached).unwrap();
13308 assert_eq!(&reached, b"R");
13309 let (before_state, before_alive) = {
13310 let state = lock_snapshot(&snapshot).unwrap();
13311 (state.state, state.process_alive)
13312 };
13313 let before_history = history.lock().unwrap().snapshot();
13314 barrier.write_all(b"X").unwrap();
13315 assert!(matches!(reap.join().unwrap(), NextAction::Stop { .. }));
13316 assert!(before_history.entries.is_empty());
13317 assert_ne!(
13318 before_state,
13319 ModuleState::Failed,
13320 "Failed was visible before its terminal record could be written"
13321 );
13322 assert!(before_alive);
13323 assert_eq!(lock_snapshot(&snapshot).unwrap().state, ModuleState::Failed);
13324 let history = history.lock().unwrap().snapshot();
13325 assert_eq!(history.entries.len(), 1);
13326 assert_eq!(history.entries[0].exit_code, Some(0));
13327 assert_eq!(history.entries[0].disposition, TerminalDisposition::Failed);
13328 }
13329
13330 #[test]
13331 fn crash_restart_at_budget_limit_remains_recoverable_while_recording() {
13332 use std::io::{Read, Write};
13333
13334 let supervisor = Supervisor::new_for_test(
13335 Arc::new(Registry::default()),
13336 RestartPolicy::new(3, Duration::ZERO),
13337 );
13338 let mut runtime = supervisor.runtime_config();
13339 let module = module_with_recovery_snapshot(ModuleState::Running, true, 2);
13340 let snapshot = module.inner.snapshot.clone();
13341 lock_snapshot(&snapshot).unwrap().process_alive = true;
13342 runtime.terminal_ring = module.inner.terminal_ring.clone();
13343 let spec = windowed_crash_spec(module.module_id());
13344 let mut barrier = terminal_record_barrier(&runtime.spawn_events);
13345 let reaped_snapshot = snapshot.clone();
13346 let reap = std::thread::spawn(move || {
13347 tokio::runtime::Builder::new_current_thread()
13348 .enable_all()
13349 .build()
13350 .unwrap()
13351 .block_on(on_child_exit(
13352 &spec,
13353 runtime.restart_policy,
13354 &supervisor.registry,
13355 &reaped_snapshot,
13356 &runtime.terminal_ring,
13357 &runtime.spawn_events,
13358 &runtime.child_roster,
13359 ExitReport {
13360 kind: ExitKind::Crash,
13361 code: Some(2),
13362 signal: None,
13363 at_ms: 1,
13364 },
13365 true,
13366 ))
13367 });
13368
13369 let mut reached = [0];
13373 barrier.read_exact(&mut reached).unwrap();
13374 assert_eq!(&reached, b"R");
13375 let before_status = module.status().unwrap();
13376 let before_history = module.terminal_history();
13377 let before_recovers = module.will_recover_after_connection_loss().unwrap();
13378 barrier.write_all(b"X").unwrap();
13379 assert!(matches!(reap.join().unwrap(), NextAction::Restart { .. }));
13380
13381 assert_eq!(before_status.state, ModuleState::Running);
13382 assert!(before_status.process_alive);
13383 assert_eq!(before_status.restart_count, 3);
13384 assert!(before_history.entries.is_empty());
13385 assert!(
13386 before_recovers,
13387 "connection cleanup would report a terminal close for a scheduled crash restart"
13388 );
13389 let restarting = module.status().unwrap();
13390 assert_eq!(restarting.state, ModuleState::Restarting);
13391 assert_eq!(restarting.restart_count, 3);
13392 assert!(module.will_recover_after_connection_loss().unwrap());
13393 let history = module.terminal_history();
13394 assert_eq!(history.entries.len(), 1);
13395 assert_eq!(
13396 history.entries[0].disposition,
13397 TerminalDisposition::Restarting
13398 );
13399
13400 update_snapshot(&snapshot, Some(module.module_id()), |state| {
13403 state.state = ModuleState::Running;
13404 state.respawn_pending = false;
13405 })
13406 .unwrap();
13407 assert!(!module.will_recover_after_connection_loss().unwrap());
13408 update_snapshot(&snapshot, Some(module.module_id()), |state| {
13409 state.enabled = false;
13410 state.respawn_pending = true;
13411 })
13412 .unwrap();
13413 assert!(!module.will_recover_after_connection_loss().unwrap());
13414 }
13415
13416 fn terminal_record_barrier(events: &super::SpawnEventFeed) -> std::net::TcpStream {
13417 use std::net::{TcpListener, TcpStream};
13418 let listener = TcpListener::bind("127.0.0.1:0").unwrap();
13419 let writer = TcpStream::connect(listener.local_addr().unwrap()).unwrap();
13420 let (barrier, _) = listener.accept().unwrap();
13421 for stream in [&writer, &barrier] {
13422 stream
13423 .set_read_timeout(Some(Duration::from_secs(10)))
13424 .unwrap();
13425 stream
13426 .set_write_timeout(Some(Duration::from_secs(10)))
13427 .unwrap();
13428 }
13429 *events.1.lock().unwrap() = Some(writer);
13430 barrier
13431 }
13432
13433 pub(super) fn before_terminal_record(events: &super::SpawnEventFeed) {
13434 use std::io::{Read, Write};
13435 let barrier = events.1.lock().unwrap().take();
13436 if let Some(mut barrier) = barrier {
13437 barrier.write_all(b"R").unwrap();
13438 let mut release = [0];
13439 barrier.read_exact(&mut release).unwrap();
13440 assert_eq!(&release, b"X");
13441 }
13442 }
13443
13444 #[test]
13445 fn health_restart_budget_failure_is_published_after_its_terminal_record() {
13446 use std::io::{Read, Write};
13447
13448 let supervisor = Supervisor::new_for_test(
13449 Arc::new(Registry::default()),
13450 RestartPolicy::new(0, Duration::ZERO),
13451 );
13452 let mut runtime = supervisor.runtime_config();
13453 let module = module_with_recovery_snapshot(ModuleState::Running, true, 0);
13454 let snapshot = module.inner.snapshot.clone();
13455 runtime.terminal_ring = module.inner.terminal_ring.clone();
13456 let spec = ModuleSpec {
13457 module_id: module.module_id().into(),
13458 program: fake_aft_stub_path(),
13459 protocol: ModuleProtocol::None,
13460 env: vec![("FAKE_AFT_EXIT_CODE".into(), "17".into())],
13461 ..windowed_crash_spec(module.module_id())
13462 };
13463 let mut barrier = terminal_record_barrier(&runtime.spawn_events);
13464 let reap = std::thread::spawn(move || {
13465 tokio::runtime::Builder::new_current_thread()
13466 .enable_all()
13467 .build()
13468 .unwrap()
13469 .block_on(async {
13470 let mut child =
13471 Some(spawn_and_mark_running(&spec, &runtime, &snapshot).unwrap());
13472 health_restart_child(
13473 &spec,
13474 &runtime,
13475 &supervisor.registry,
13476 &supervisor.process_liveness,
13477 &snapshot,
13478 &mut child,
13479 SupervisorHealthStatus::Failing,
13480 Some("health probe did not answer"),
13481 1,
13482 )
13483 .await
13484 .unwrap();
13485 assert!(child.is_none());
13486 });
13487 });
13488 let mut reached = [0];
13493 barrier.read_exact(&mut reached).unwrap();
13494 assert_eq!(&reached, b"R");
13495 let before_status = module.status().unwrap();
13496 let before_history = module.terminal_history();
13497 barrier.write_all(b"X").unwrap();
13498 reap.join().unwrap();
13499 assert!(before_history.entries.is_empty());
13500 assert_ne!(
13501 before_status.state,
13502 ModuleState::Failed,
13503 "Failed was visible before its terminal record could be written"
13504 );
13505 assert!(
13506 before_status.process_alive,
13507 "process death was visible before its terminal record could be written"
13508 );
13509 let failed = module.status().unwrap();
13510 assert_eq!(failed.state, ModuleState::Failed);
13511 assert!(!failed.process_alive);
13512 let history = module.terminal_history();
13513 assert_eq!(history.entries.len(), 1);
13514 assert_eq!(history.entries[0].disposition, TerminalDisposition::Failed);
13515 assert_eq!(
13516 history.entries[0].disposition_detail.as_deref(),
13517 Some("crash budget exhausted: max_restarts=0 within window_secs=600")
13518 );
13519 }
13520
13521 #[test]
13522 fn observed_exits_publish_process_death_after_terminal_history() {
13523 use std::io::{Read, Write};
13524 for (kind, enabled, shutdown, wait_error, expected_state) in [
13525 (ExitKind::Clean, true, false, false, ModuleState::Stopped),
13526 (ExitKind::Crash, true, false, false, ModuleState::Restarting),
13527 (ExitKind::Crash, false, false, false, ModuleState::Disabled),
13528 (
13529 ExitKind::DeliberateSeverance,
13530 true,
13531 false,
13532 false,
13533 ModuleState::Restarting,
13534 ),
13535 #[cfg(unix)]
13536 (ExitKind::Crash, true, true, false, ModuleState::Stopped),
13537 (ExitKind::Crash, true, false, true, ModuleState::Failed),
13538 ] {
13539 let supervisor = Supervisor::new_for_test(
13540 Arc::new(Registry::default()),
13541 RestartPolicy::new(1, Duration::ZERO),
13542 );
13543 let runtime = supervisor.runtime_config();
13544 #[cfg(unix)]
13545 if shutdown {
13546 runtime.child_roster.close();
13547 }
13548 #[cfg(not(unix))]
13549 let _ = shutdown;
13550 let spec = windowed_crash_spec("exit-publication-order");
13551 let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::new(
13552 ModuleState::Running,
13553 enabled,
13554 )));
13555 lock_snapshot(&snapshot).unwrap().process_alive = true;
13556 let reaped_snapshot = snapshot.clone();
13557 let history = runtime.terminal_ring.clone();
13558 let mut barrier = terminal_record_barrier(&runtime.spawn_events);
13559 let reap = std::thread::spawn(move || {
13560 if wait_error {
13561 super::fail_after_wait_error(
13562 &spec.module_id,
13563 &reaped_snapshot,
13564 &runtime.terminal_ring,
13565 &runtime.spawn_events,
13566 "wait failed".into(),
13567 &runtime.failure_context(true),
13568 );
13569 } else {
13570 tokio::runtime::Builder::new_current_thread()
13571 .enable_all()
13572 .build()
13573 .unwrap()
13574 .block_on(on_child_exit(
13575 &spec,
13576 runtime.restart_policy,
13577 &supervisor.registry,
13578 &reaped_snapshot,
13579 &runtime.terminal_ring,
13580 &runtime.spawn_events,
13581 &runtime.child_roster,
13582 ExitReport {
13583 kind,
13584 code: Some(0),
13585 signal: None,
13586 at_ms: 1,
13587 },
13588 true,
13589 ));
13590 }
13591 });
13592 let mut reached = [0];
13593 barrier.read_exact(&mut reached).unwrap();
13594 assert_eq!(&reached, b"R");
13595 let before_alive = lock_snapshot(&snapshot).unwrap().process_alive;
13596 let before_history = history.lock().unwrap().snapshot();
13597 barrier.write_all(b"X").unwrap();
13598 reap.join().unwrap();
13599 assert!(
13600 before_alive,
13601 "{kind:?}: death was published before recording"
13602 );
13603 assert!(before_history.entries.is_empty());
13604 assert_eq!(lock_snapshot(&snapshot).unwrap().state, expected_state);
13605 assert!(!lock_snapshot(&snapshot).unwrap().process_alive);
13606 assert_eq!(history.lock().unwrap().snapshot().entries.len(), 1);
13607 }
13608 }
13609
13610 #[tokio::test]
13611 async fn health_restart_budget_failure_remains_failed_when_journal_append_fails() {
13612 let dir = cortexkit_test_support::ScratchDir::new("health-budget-journal-failure");
13613 let path = dir.join("terminals.jsonl");
13614 std::fs::create_dir(&path).unwrap();
13615 let supervisor = Supervisor::new_for_test(
13616 Arc::new(Registry::default()),
13617 RestartPolicy::new(0, Duration::ZERO),
13618 )
13619 .with_terminal_journal(path, "health-budget-journal-failure".into());
13620 let runtime = supervisor.runtime_config();
13621 let spec = ModuleSpec {
13622 program: fake_aft_stub_path(),
13623 protocol: ModuleProtocol::None,
13624 env: vec![("FAKE_AFT_EXIT_CODE".into(), "17".into())],
13625 ..windowed_crash_spec("health-budget-journal-failure")
13626 };
13627 let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::starting()));
13628 let mut child = Some(spawn_and_mark_running(&spec, &runtime, &snapshot).unwrap());
13629 health_restart_child(
13630 &spec,
13631 &runtime,
13632 &supervisor.registry,
13633 &supervisor.process_liveness,
13634 &snapshot,
13635 &mut child,
13636 SupervisorHealthStatus::Failing,
13637 None,
13638 1,
13639 )
13640 .await
13641 .unwrap();
13642 assert_eq!(lock_snapshot(&snapshot).unwrap().state, ModuleState::Failed);
13643 assert!(!lock_snapshot(&snapshot).unwrap().process_alive);
13644 let history = runtime
13645 .terminal_ring
13646 .lock()
13647 .unwrap()
13648 .durable_history(&spec.module_id);
13649 assert!(history.journal_write_failures > 0);
13650 assert_eq!(history.entries.len(), 1);
13651 assert_eq!(history.entries[0].disposition, TerminalDisposition::Failed);
13652 }
13653
13654 #[test]
13655 fn health_restart_budget_failure_remains_failed_when_exit_recording_panics() {
13656 let supervisor = Supervisor::new_for_test(
13657 Arc::new(Registry::default()),
13658 RestartPolicy::new(0, Duration::ZERO),
13659 );
13660 let runtime = supervisor.runtime_config();
13661 let spec = ModuleSpec {
13662 program: fake_aft_stub_path(),
13663 protocol: ModuleProtocol::None,
13664 env: vec![("FAKE_AFT_EXIT_CODE".into(), "17".into())],
13665 ..windowed_crash_spec("health-budget-recording-panic")
13666 };
13667 let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::starting()));
13668 let result = std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| {
13669 tokio::runtime::Builder::new_current_thread()
13670 .enable_all()
13671 .build()
13672 .unwrap()
13673 .block_on(async {
13674 let mut child =
13675 Some(spawn_and_mark_running(&spec, &runtime, &snapshot).unwrap());
13676 runtime.spawn_events.0.lock().unwrap().seq = u64::MAX;
13677 health_restart_child(
13678 &spec,
13679 &runtime,
13680 &supervisor.registry,
13681 &supervisor.process_liveness,
13682 &snapshot,
13683 &mut child,
13684 SupervisorHealthStatus::Failing,
13685 None,
13686 1,
13687 )
13688 .await
13689 .unwrap();
13690 });
13691 }));
13692 let panic = result.expect_err("recording must still unwind");
13693 assert_eq!(
13694 panic.downcast_ref::<String>().map(String::as_str),
13695 Some("spawn event sequence exhausted")
13696 );
13697 let state = lock_snapshot(&snapshot).unwrap();
13698 assert_eq!(state.state, ModuleState::Failed);
13699 assert!(!state.process_alive);
13700 assert!(state.last_exit.is_some());
13701 }
13702
13703 #[tokio::test]
13704 async fn restart_budget_failure_remains_failed_when_journal_append_fails() {
13705 let dir = cortexkit_test_support::ScratchDir::new("budget-journal-failure");
13706 let path = dir.join("terminals.jsonl");
13707 std::fs::create_dir(&path).unwrap();
13708 let supervisor = Supervisor::new_for_test(
13709 Arc::new(Registry::default()),
13710 RestartPolicy::new(0, Duration::ZERO),
13711 )
13712 .with_terminal_journal(path, "budget-journal-failure".into());
13713 let runtime = supervisor.runtime_config();
13714 let spec = windowed_crash_spec("budget-journal-failure");
13715 let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::starting()));
13716 assert!(matches!(
13717 on_child_exit(
13718 &spec,
13719 runtime.restart_policy,
13720 &supervisor.registry,
13721 &snapshot,
13722 &runtime.terminal_ring,
13723 &runtime.spawn_events,
13724 &runtime.child_roster,
13725 crash_exit_report(1),
13726 true,
13727 )
13728 .await,
13729 NextAction::Stop { .. }
13730 ));
13731 assert_eq!(lock_snapshot(&snapshot).unwrap().state, ModuleState::Failed);
13732 let history = runtime
13733 .terminal_ring
13734 .lock()
13735 .unwrap()
13736 .durable_history(&spec.module_id);
13737 assert!(history.journal_write_failures > 0);
13738 assert_eq!(history.entries.len(), 1);
13739 assert_eq!(history.entries[0].disposition, TerminalDisposition::Failed);
13740 }
13741
13742 #[test]
13743 fn restart_budget_failure_remains_failed_when_exit_recording_panics() {
13744 let supervisor = Supervisor::new_for_test(
13745 Arc::new(Registry::default()),
13746 RestartPolicy::new(0, Duration::ZERO),
13747 );
13748 let runtime = supervisor.runtime_config();
13749 let spec = windowed_crash_spec("budget-recording-panic");
13750 let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::starting()));
13751 runtime.spawn_events.emit_spawned(&spec.module_id, 1, 1);
13752 runtime.spawn_events.0.lock().unwrap().seq = u64::MAX;
13755 let result = std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| {
13756 tokio::runtime::Builder::new_current_thread()
13757 .enable_all()
13758 .build()
13759 .unwrap()
13760 .block_on(on_child_exit(
13761 &spec,
13762 runtime.restart_policy,
13763 &supervisor.registry,
13764 &snapshot,
13765 &runtime.terminal_ring,
13766 &runtime.spawn_events,
13767 &runtime.child_roster,
13768 crash_exit_report(1),
13769 true,
13770 ))
13771 }));
13772 let panic = result.err().expect("recording must still unwind");
13773 assert_eq!(
13774 panic.downcast_ref::<String>().map(String::as_str),
13775 Some("spawn event sequence exhausted")
13776 );
13777 assert_eq!(lock_snapshot(&snapshot).unwrap().state, ModuleState::Failed);
13778 }
13779
13780 #[cfg(unix)]
13783 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
13784 async fn protocol_none_requested_stop_and_disable_do_not_respawn() {
13785 for disable in [false, true] {
13786 let label = if disable {
13787 "none-requested-disable"
13788 } else {
13789 "none-requested-stop"
13790 };
13791 let dir = cortexkit_test_support::ScratchDir::new(label);
13792 let (spec, ready, marker) = protocol_none_sigterm_exits_clean_spec(label, dir.path());
13793 let supervisor = Supervisor::new_for_test(
13794 Arc::new(Registry::default()),
13795 RestartPolicy::new(3, Duration::ZERO),
13796 );
13797 let module = supervisor.spawn(spec).unwrap();
13798 wait_for_file(&ready).await;
13799
13800 if disable {
13801 module.set_enabled(false).await.unwrap();
13802 } else {
13803 module.stop().await.unwrap();
13804 }
13805 assert!(
13806 marker.exists(),
13807 "{label}: the child must have left through its SIGTERM handler with exit 0"
13808 );
13809
13810 sleep(Duration::from_millis(500)).await;
13813 let status = module.status().unwrap();
13814 let expected = if disable {
13815 ModuleState::Disabled
13816 } else {
13817 ModuleState::Stopped
13818 };
13819 assert_eq!(status.state, expected, "{label}");
13820 assert_eq!(
13821 status.spawn_generation, 1,
13822 "{label}: respawned after a requested stop"
13823 );
13824 let history = module.terminal_history();
13825 assert_eq!(history.entries.len(), 1, "{label}: {history:?}");
13826 assert_eq!(history.entries[0].exit_code, Some(0), "{label}");
13827 assert_ne!(
13828 history.entries[0].disposition,
13829 TerminalDisposition::Restarting,
13830 "{label}"
13831 );
13832 }
13833 }
13834
13835 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
13838 async fn subc_wire_clean_exit_is_still_a_stop() {
13839 let supervisor = Supervisor::new_for_test(
13840 Arc::new(Registry::default()),
13841 RestartPolicy::new(3, Duration::ZERO),
13842 );
13843 let module = supervisor
13844 .spawn(ModuleSpec {
13845 module_id: "wire-clean-exit".to_string(),
13846 program: fake_aft_stub_path(),
13847 args: Vec::new(),
13848 env: vec![("FAKE_AFT_EXIT_CODE".to_string(), "0".to_string())],
13849 reserved: false,
13850 reserved_prefixes: Vec::new(),
13851 protocol: ModuleProtocol::Subc,
13852 overlap: Default::default(),
13853 carry_containment: false,
13854 })
13855 .unwrap();
13856
13857 let deadline = Instant::now() + Duration::from_secs(10);
13858 while module.terminal_history().entries.is_empty() {
13859 assert!(Instant::now() < deadline, "module never exited");
13860 sleep(Duration::from_millis(10)).await;
13861 }
13862 sleep(Duration::from_millis(500)).await;
13864 let status = module.status().unwrap();
13865 assert_eq!(status.state, ModuleState::Stopped);
13866 assert_eq!(status.spawn_generation, 1);
13867 let history = module.terminal_history();
13868 assert_eq!(history.entries.len(), 1, "{history:?}");
13869 assert_eq!(history.entries[0].exit_code, Some(0));
13870 assert_eq!(history.entries[0].disposition, TerminalDisposition::Stopped);
13871 }
13872
13873 #[cfg(unix)]
13874 #[tokio::test]
13875 async fn reload_exit_keeps_roster_until_terminal_is_recorded_during_shutdown() {
13876 let dir = cortexkit_test_support::ScratchDir::new("reload-roster-terminal-order");
13877 let record = dir.join("live-children.json");
13878 let supervisor = Supervisor::new_for_test(
13879 Arc::new(Registry::default()),
13880 RestartPolicy::new(0, Duration::ZERO),
13881 );
13882 let mut runtime = supervisor.runtime_config();
13883 runtime.child_roster.record_to(record.clone());
13884 let gate = Arc::new(super::ReloadExitRecordGate::default());
13885 runtime.test_reload_exit_record_gate = Some(Arc::clone(&gate));
13886 let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::starting()));
13887 let spec = ModuleSpec {
13888 module_id: "reload-exit-roster".into(),
13889 program: fake_aft_stub_path(),
13890 args: Vec::new(),
13891 env: vec![("FAKE_AFT_EXIT_CODE".into(), "23".into())],
13892 reserved: false,
13893 reserved_prefixes: Vec::new(),
13894 protocol: ModuleProtocol::Subc,
13895 overlap: Default::default(),
13896 carry_containment: false,
13897 };
13898 let mut child = None;
13899 let reload = super::finish_reload_child(
13900 &spec,
13901 &runtime,
13902 &supervisor.registry,
13903 &supervisor.process_liveness,
13904 &snapshot,
13905 &mut child,
13906 );
13907 tokio::pin!(reload);
13908 tokio::select! {
13909 result = &mut reload => panic!("reload missed the before-record gate: {result:?}"),
13910 _ = gate.reached.notified() => {}
13911 }
13912 assert!(runtime
13913 .terminal_ring
13914 .lock()
13915 .unwrap()
13916 .snapshot()
13917 .entries
13918 .is_empty());
13919 assert_eq!(
13920 crate::live_children::read_record(&record).unwrap().len(),
13921 1,
13922 "shutdown must still wait for the reaped child until its terminal record exists"
13923 );
13924 runtime.child_roster.close();
13925 gate.resume.notify_one();
13926 assert!(reload.await.is_err());
13927 assert!(crate::live_children::read_record(&record)
13928 .unwrap()
13929 .is_empty());
13930 let history = runtime.terminal_ring.lock().unwrap().snapshot();
13931 assert_eq!(history.entries.len(), 1);
13932 assert_eq!(
13933 history.entries[0].disposition,
13934 TerminalDisposition::DaemonShutdown
13935 );
13936 }
13937
13938 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
13942 async fn every_restart_increment_path_advances_lifetime_count() {
13943 let supervisor = Supervisor::new_for_test(
13944 Arc::new(Registry::default()),
13945 RestartPolicy::new(1, Duration::ZERO),
13946 );
13947 let runtime = supervisor.runtime_config();
13948 let spec = ModuleSpec {
13949 module_id: "lifetime-increment-path".to_string(),
13950 program: PathBuf::from("/unused/lifetime-increment-path"),
13951 args: Vec::new(),
13952 env: Vec::new(),
13953 reserved: false,
13954 reserved_prefixes: Vec::new(),
13955 protocol: ModuleProtocol::Subc,
13956 overlap: Default::default(),
13957 carry_containment: false,
13958 };
13959
13960 let crash_snapshot = Arc::new(Mutex::new(SupervisorSnapshot::starting()));
13961 assert!(matches!(
13962 on_child_exit(
13963 &spec,
13964 runtime.restart_policy,
13965 &supervisor.registry,
13966 &crash_snapshot,
13967 &runtime.terminal_ring,
13968 &runtime.spawn_events,
13969 &runtime.child_roster,
13970 ExitReport {
13971 kind: ExitKind::Crash,
13972 code: Some(1),
13973 signal: None,
13974 at_ms: 1,
13975 },
13976 true,
13977 )
13978 .await,
13979 NextAction::Restart { schedule: _ }
13980 ));
13981 let (crash_restarts, crash_lifetime) = {
13982 let state = lock_snapshot(&crash_snapshot).unwrap();
13983 (state.crash_restarts.len(), state.lifetime_restarts)
13984 };
13985 assert_eq!(crash_restarts, 1);
13986 assert_eq!(crash_lifetime, 1);
13987
13988 let health_snapshot = Arc::new(Mutex::new(SupervisorSnapshot::starting()));
13989 let mut health_child = None;
13990 assert!(matches!(
13991 health_restart_child(
13992 &spec,
13993 &runtime,
13994 &supervisor.registry,
13995 &supervisor.process_liveness,
13996 &health_snapshot,
13997 &mut health_child,
13998 SupervisorHealthStatus::Failing,
13999 None,
14000 2,
14001 )
14002 .await,
14003 Ok(())
14004 ));
14005 assert!(health_child.is_none());
14006 assert!(lock_snapshot(&health_snapshot).unwrap().respawn_pending);
14007 assert!(runtime.scheduled_respawn.lock().unwrap().take().is_some());
14008 let (health_restarts, health_lifetime) = {
14009 let state = lock_snapshot(&health_snapshot).unwrap();
14010 (state.crash_restarts.len(), state.lifetime_restarts)
14011 };
14012 assert_eq!(health_restarts, 1);
14013 assert_eq!(health_lifetime, 1);
14014
14015 let reload_snapshot = Arc::new(Mutex::new(SupervisorSnapshot::starting()));
14016 let mut reload_child = None;
14017 assert!(matches!(
14018 handle_reload_spawn_failure(
14019 &spec,
14020 &runtime,
14021 &supervisor.process_liveness,
14022 &reload_snapshot,
14023 &mut reload_child,
14024 "forced reload spawn failure".to_string(),
14025 )
14026 .await,
14027 Err(SuperviseError::ReloadFailed { .. })
14028 ));
14029 let (reload_restarts, reload_lifetime) = {
14030 let state = lock_snapshot(&reload_snapshot).unwrap();
14031 (state.crash_restarts.len(), state.lifetime_restarts)
14032 };
14033 assert_eq!(reload_restarts, 1);
14034 assert_eq!(reload_lifetime, 1);
14035 }
14036
14037 #[tokio::test]
14038 async fn deliberately_severed_live_child_records_lifetime_without_spending_restart_budget() {
14039 let supervisor = Supervisor::new_for_test(
14040 Arc::new(Registry::default()),
14041 RestartPolicy::new(3, Duration::ZERO),
14042 );
14043 let runtime = supervisor.runtime_config();
14044 let spec = ModuleSpec {
14045 module_id: "deliberately-severed".to_string(),
14046 program: PathBuf::from("/unused/deliberately-severed"),
14047 args: Vec::new(),
14048 env: Vec::new(),
14049 reserved: false,
14050 reserved_prefixes: Vec::new(),
14051 protocol: ModuleProtocol::Subc,
14052 overlap: Default::default(),
14053 carry_containment: false,
14054 };
14055 let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::starting()));
14056 let process = ProcessIdentity {
14057 pid: 41,
14058 start_time: 101,
14059 };
14060 record_deliberate_severance(&snapshot, process).unwrap();
14061 let exit_report = apply_deliberate_severance_marker(
14062 &snapshot,
14063 Some(process),
14064 ExitReport {
14065 kind: ExitKind::Crash,
14066 code: Some(1),
14067 signal: None,
14068 at_ms: 1,
14069 },
14070 );
14071 assert_eq!(exit_report.kind, ExitKind::DeliberateSeverance);
14072
14073 assert!(matches!(
14074 on_child_exit(
14075 &spec,
14076 runtime.restart_policy,
14077 &supervisor.registry,
14078 &snapshot,
14079 &runtime.terminal_ring,
14080 &runtime.spawn_events,
14081 &runtime.child_roster,
14082 exit_report,
14083 true,
14084 )
14085 .await,
14086 NextAction::Restart { schedule: _ }
14087 ));
14088 let state = lock_snapshot(&snapshot).unwrap();
14089 assert_eq!(state.lifetime_restarts, 1);
14090 assert_eq!(state.crash_restarts.len(), 0);
14091 }
14092
14093 #[tokio::test]
14094 async fn genuine_crash_spends_restart_budget_and_records_lifetime() {
14095 let supervisor = Supervisor::new_for_test(
14096 Arc::new(Registry::default()),
14097 RestartPolicy::new(3, Duration::ZERO),
14098 );
14099 let runtime = supervisor.runtime_config();
14100 let spec = ModuleSpec {
14101 module_id: "genuine-crash".to_string(),
14102 program: PathBuf::from("/unused/genuine-crash"),
14103 args: Vec::new(),
14104 env: Vec::new(),
14105 reserved: false,
14106 reserved_prefixes: Vec::new(),
14107 protocol: ModuleProtocol::Subc,
14108 overlap: Default::default(),
14109 carry_containment: false,
14110 };
14111 let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::starting()));
14112
14113 assert!(matches!(
14114 on_child_exit(
14115 &spec,
14116 runtime.restart_policy,
14117 &supervisor.registry,
14118 &snapshot,
14119 &runtime.terminal_ring,
14120 &runtime.spawn_events,
14121 &runtime.child_roster,
14122 ExitReport {
14123 kind: ExitKind::Crash,
14124 code: Some(1),
14125 signal: None,
14126 at_ms: 1,
14127 },
14128 true,
14129 )
14130 .await,
14131 NextAction::Restart { schedule: _ }
14132 ));
14133 let state = lock_snapshot(&snapshot).unwrap();
14134 assert_eq!(state.lifetime_restarts, 1);
14135 assert_eq!(state.crash_restarts.len(), 1);
14136 }
14137
14138 fn crash_exit_report(at_ms: u64) -> ExitReport {
14139 ExitReport {
14140 kind: ExitKind::Crash,
14141 code: Some(1),
14142 signal: None,
14143 at_ms,
14144 }
14145 }
14146
14147 fn windowed_crash_spec(module_id: &str) -> ModuleSpec {
14148 ModuleSpec {
14149 module_id: module_id.to_string(),
14150 program: PathBuf::from("/unused").join(module_id),
14151 args: Vec::new(),
14152 env: Vec::new(),
14153 reserved: false,
14154 reserved_prefixes: Vec::new(),
14155 protocol: ModuleProtocol::Subc,
14156 overlap: Default::default(),
14157 carry_containment: false,
14158 }
14159 }
14160
14161 #[tokio::test]
14167 async fn three_crashes_inside_the_window_stop_the_module_and_name_the_window() {
14168 let (logs, _guard) = crate::router::test_log::log_capture(tracing::Level::ERROR);
14169 let supervisor = Supervisor::new_for_test(
14170 Arc::new(Registry::default()),
14171 RestartPolicy::new(2, Duration::ZERO),
14172 );
14173 let runtime = supervisor.runtime_config();
14174 let spec = windowed_crash_spec("crash-loop-in-window");
14175 let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::starting()));
14176
14177 for attempt in 1..=2 {
14178 assert!(
14179 matches!(
14180 on_child_exit(
14181 &spec,
14182 runtime.restart_policy,
14183 &supervisor.registry,
14184 &snapshot,
14185 &runtime.terminal_ring,
14186 &runtime.spawn_events,
14187 &runtime.child_roster,
14188 crash_exit_report(attempt),
14189 true,
14190 )
14191 .await,
14192 NextAction::Restart { schedule: _ }
14193 ),
14194 "crash {attempt} is inside the budget and must respawn"
14195 );
14196 }
14197
14198 assert!(matches!(
14199 on_child_exit(
14200 &spec,
14201 runtime.restart_policy,
14202 &supervisor.registry,
14203 &snapshot,
14204 &runtime.terminal_ring,
14205 &runtime.spawn_events,
14206 &runtime.child_roster,
14207 crash_exit_report(3),
14208 true,
14209 )
14210 .await,
14211 NextAction::Stop { .. }
14212 ));
14213
14214 {
14215 let state = lock_snapshot(&snapshot).unwrap();
14216 assert_eq!(state.state, ModuleState::Failed);
14217 assert_eq!(state.crash_restarts.len(), 2);
14218 assert_eq!(state.lifetime_restarts, 2);
14219 }
14220
14221 let history = runtime
14222 .terminal_ring
14223 .lock()
14224 .expect("terminal ring is not poisoned")
14225 .snapshot();
14226 let last = history
14227 .entries
14228 .last()
14229 .expect("the refused crash is retained");
14230 assert_eq!(last.disposition, TerminalDisposition::Failed);
14231 assert_eq!(
14232 last.disposition_detail.as_deref(),
14233 Some("crash budget exhausted: max_restarts=2 within window_secs=600")
14234 );
14235
14236 let captured = crate::router::test_log::captured_logs(&logs);
14237 assert!(
14238 captured.contains("crash budget exhausted: max_restarts=2 within window_secs=600"),
14239 "the stop must be logged with its window: {captured}"
14240 );
14241 }
14242
14243 #[tokio::test]
14251 async fn a_crash_older_than_the_window_frees_its_slot_for_a_later_crash() {
14252 let supervisor = Supervisor::new_for_test(
14253 Arc::new(Registry::default()),
14254 RestartPolicy::new(2, Duration::ZERO),
14255 );
14256 let runtime = supervisor.runtime_config();
14257 let spec = windowed_crash_spec("crash-across-windows");
14258 let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::starting()));
14259
14260 for attempt in 1..=2 {
14261 assert!(matches!(
14262 on_child_exit(
14263 &spec,
14264 runtime.restart_policy,
14265 &supervisor.registry,
14266 &snapshot,
14267 &runtime.terminal_ring,
14268 &runtime.spawn_events,
14269 &runtime.child_roster,
14270 crash_exit_report(attempt),
14271 true,
14272 )
14273 .await,
14274 NextAction::Restart { schedule: _ }
14275 ));
14276 }
14277
14278 update_snapshot(&snapshot, Some(&spec.module_id), |state| {
14281 age_oldest_crash_restart_out_of_window(state, runtime.restart_policy.window);
14282 })
14283 .unwrap();
14284
14285 assert!(
14286 matches!(
14287 on_child_exit(
14288 &spec,
14289 runtime.restart_policy,
14290 &supervisor.registry,
14291 &snapshot,
14292 &runtime.terminal_ring,
14293 &runtime.spawn_events,
14294 &runtime.child_roster,
14295 crash_exit_report(3),
14296 true,
14297 )
14298 .await,
14299 NextAction::Restart { schedule: _ }
14300 ),
14301 "a crash older than the window must not hold a budget slot"
14302 );
14303
14304 let state = lock_snapshot(&snapshot).unwrap();
14305 assert_eq!(state.state, ModuleState::Restarting);
14306 assert_eq!(
14307 state.crash_restarts.len(),
14308 2,
14309 "the aged instant is dropped and the new one takes its place"
14310 );
14311 assert_eq!(
14312 state.lifetime_restarts, 3,
14313 "the ledger counts every restart, including the ones the window forgot"
14314 );
14315 }
14316
14317 #[tokio::test]
14322 async fn an_operator_restart_clears_the_ring_and_leaves_the_ledger_alone() {
14323 let supervisor = Supervisor::new_for_test(
14324 Arc::new(Registry::default()),
14325 RestartPolicy::new(2, Duration::ZERO),
14326 );
14327 let runtime = supervisor.runtime_config();
14328 let spec = windowed_crash_spec("operator-cleared-budget");
14329 let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::starting()));
14330
14331 for attempt in 1..=2 {
14332 assert!(matches!(
14333 on_child_exit(
14334 &spec,
14335 runtime.restart_policy,
14336 &supervisor.registry,
14337 &snapshot,
14338 &runtime.terminal_ring,
14339 &runtime.spawn_events,
14340 &runtime.child_roster,
14341 crash_exit_report(attempt),
14342 true,
14343 )
14344 .await,
14345 NextAction::Restart { schedule: _ }
14346 ));
14347 }
14348
14349 reset_restart_count(&snapshot, &spec.module_id).unwrap();
14350 {
14351 let state = lock_snapshot(&snapshot).unwrap();
14352 assert!(
14353 state.crash_restarts.is_empty(),
14354 "an operator restart returns the full budget"
14355 );
14356 assert_eq!(
14357 state.lifetime_restarts, 2,
14358 "clearing the budget must not unmake the crashes"
14359 );
14360 }
14361
14362 assert!(
14363 matches!(
14364 on_child_exit(
14365 &spec,
14366 runtime.restart_policy,
14367 &supervisor.registry,
14368 &snapshot,
14369 &runtime.terminal_ring,
14370 &runtime.spawn_events,
14371 &runtime.child_roster,
14372 crash_exit_report(3),
14373 true,
14374 )
14375 .await,
14376 NextAction::Restart { schedule: _ }
14377 ),
14378 "the cleared budget must be spendable again"
14379 );
14380 let state = lock_snapshot(&snapshot).unwrap();
14381 assert_eq!(state.crash_restarts.len(), 1);
14382 assert_eq!(state.lifetime_restarts, 3);
14383 }
14384
14385 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
14386 async fn severance_marker_for_a_dead_child_does_not_label_its_successor() {
14387 let severed = ProcessIdentity {
14388 pid: 41,
14389 start_time: 101,
14390 };
14391 let successor = ProcessIdentity {
14392 pid: 41,
14393 start_time: 202,
14394 };
14395 let module = module_with_recovery_snapshot(ModuleState::Running, true, 0);
14396 update_snapshot(&module.inner.snapshot, Some("recovery-snapshot"), |state| {
14397 state.pid = Some(successor.pid);
14398 state.process_start_time = Some(successor.start_time);
14399 })
14400 .unwrap();
14401 assert!(!module.record_deliberate_severance(severed).unwrap());
14402
14403 let exit_report = apply_deliberate_severance_marker(
14404 &module.inner.snapshot,
14405 Some(successor),
14406 ExitReport {
14407 kind: ExitKind::Crash,
14408 code: Some(1),
14409 signal: None,
14410 at_ms: 1,
14411 },
14412 );
14413
14414 assert_eq!(exit_report.kind, ExitKind::Crash);
14415 }
14416
14417 #[tokio::test]
14418 async fn drain_reap_marks_deliberate_severance_and_records_lifetime_without_budget() {
14419 let registry = Registry::default();
14420 let supervisor = Supervisor::new_for_test(
14421 Arc::new(Registry::default()),
14422 RestartPolicy::new(3, Duration::ZERO),
14423 );
14424 let runtime = supervisor.runtime_config();
14425 let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::starting()));
14426 let spec = ModuleSpec {
14427 module_id: "drain-deliberate-severance".to_string(),
14428 program: fake_aft_stub_path(),
14429 args: Vec::new(),
14430 env: vec![("FAKE_AFT_EXIT_CODE".to_string(), "23".to_string())],
14431 reserved: false,
14432 reserved_prefixes: Vec::new(),
14433 protocol: ModuleProtocol::Subc,
14434 overlap: Default::default(),
14435 carry_containment: false,
14436 };
14437 let mut child = spawn_and_mark_running(&spec, &runtime, &snapshot).unwrap();
14438 let process = ProcessIdentity {
14439 pid: 41,
14440 start_time: 101,
14441 };
14442 child.process_identity = Some(process);
14443 update_snapshot(&snapshot, Some(&spec.module_id), |state| {
14444 state.pid = Some(process.pid);
14445 state.process_start_time = Some(process.start_time);
14446 })
14447 .unwrap();
14448 record_deliberate_severance(&snapshot, process).unwrap();
14449
14450 drain_child_to_state(
14451 &spec.module_id,
14452 spec.protocol,
14453 StopNotice::SentOverConnection,
14456 ®istry,
14457 None,
14458 &snapshot,
14459 &runtime.terminal_ring,
14460 &runtime.spawn_events,
14461 child,
14462 Duration::from_secs(1),
14463 ModuleState::Stopped,
14464 Some(false),
14465 &super::FailureContext::operator(),
14466 )
14467 .await
14468 .unwrap();
14469
14470 let state = lock_snapshot(&snapshot).unwrap();
14471 assert_eq!(
14472 state.last_exit.as_ref().map(|exit| exit.kind),
14473 Some(ExitKind::DeliberateSeverance)
14474 );
14475 assert_eq!(state.lifetime_restarts, 1);
14476 assert_eq!(state.crash_restarts.len(), 0);
14477 drop(state);
14478 let history = runtime.terminal_ring.lock().unwrap().snapshot();
14479 assert_eq!(
14480 history.entries[0].exit_kind,
14481 subc_control::TerminalExitKind::DeliberateSeverance
14482 );
14483 }
14484
14485 #[tokio::test]
14486 async fn ordinary_drain_reap_does_not_record_a_lifetime_restart() {
14487 let registry = Registry::default();
14488 let supervisor = Supervisor::new_for_test(
14489 Arc::new(Registry::default()),
14490 RestartPolicy::new(3, Duration::ZERO),
14491 );
14492 let runtime = supervisor.runtime_config();
14493 let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::starting()));
14494 let spec = ModuleSpec {
14495 module_id: "ordinary-drain".to_string(),
14496 program: fake_aft_stub_path(),
14497 args: Vec::new(),
14498 env: vec![("FAKE_AFT_EXIT_CODE".to_string(), "23".to_string())],
14499 reserved: false,
14500 reserved_prefixes: Vec::new(),
14501 protocol: ModuleProtocol::Subc,
14502 overlap: Default::default(),
14503 carry_containment: false,
14504 };
14505 let child = spawn_and_mark_running(&spec, &runtime, &snapshot).unwrap();
14506
14507 drain_child_to_state(
14508 &spec.module_id,
14509 spec.protocol,
14510 StopNotice::SentOverConnection,
14513 ®istry,
14514 None,
14515 &snapshot,
14516 &runtime.terminal_ring,
14517 &runtime.spawn_events,
14518 child,
14519 Duration::from_secs(1),
14520 ModuleState::Stopped,
14521 Some(false),
14522 &super::FailureContext::operator(),
14523 )
14524 .await
14525 .unwrap();
14526
14527 let state = lock_snapshot(&snapshot).unwrap();
14528 assert_eq!(
14529 state.last_exit.as_ref().map(|exit| exit.kind),
14530 Some(ExitKind::Crash)
14531 );
14532 assert_eq!(state.lifetime_restarts, 0);
14533 assert_eq!(state.crash_restarts.len(), 0);
14534 }
14535
14536 #[test]
14537 fn fatal_connection_teardown_cannot_arm_a_marker_for_a_surviving_process() {
14538 assert!(!include_str!("server.rs")
14544 .contains("router.record_deliberate_connection_severance(ctx.connection_id)"));
14545 }
14546
14547 #[test]
14554 fn drained_after_quiescence_wait_passes_ok_through_and_forces_false_on_err() {
14555 assert!(drained_after_quiescence_wait(&Ok(true)));
14556 assert!(!drained_after_quiescence_wait(&Ok(false)));
14557 assert!(!drained_after_quiescence_wait(&Err(
14558 SuperviseError::StatePoisoned { module_id: None }
14559 )));
14560 }
14561
14562 #[test]
14571 fn wait_error_exit_report_records_a_failed_terminal_with_no_code_or_signal() {
14572 let ring = Arc::new(Mutex::new(TerminalRing::new(
14573 TerminalRingConfig::default(),
14574 0,
14575 )));
14576 record_wait_error_terminal("wait-error", &ring, &super::SpawnEventFeed::default());
14577
14578 let snapshot = ring.lock().unwrap().snapshot();
14579 assert_eq!(snapshot.entries.len(), 1);
14580 let entry = &snapshot.entries[0];
14581 assert_eq!(entry.exit_code, None);
14582 assert_eq!(entry.exit_signal, None);
14583 assert_eq!(entry.disposition, TerminalDisposition::Failed);
14584 }
14585
14586 #[test]
14587 fn wait_error_exit_path_preserves_spawn_event_density() {
14588 let feed = super::SpawnEventFeed::default();
14589 feed.configure_incarnation("wait-error-density".to_string());
14590 feed.emit_spawned("wait-error", 41, 1);
14591 let ring = Arc::new(Mutex::new(TerminalRing::new(
14592 TerminalRingConfig::default(),
14593 0,
14594 )));
14595
14596 record_wait_error_terminal("wait-error", &ring, &feed);
14597 feed.emit_spawned("after-wait-error", 42, 2);
14598
14599 let state = feed.0.lock().unwrap();
14600 let sequences = state
14601 .events
14602 .iter()
14603 .map(|event| event.cursor.seq)
14604 .collect::<Vec<_>>();
14605 assert_eq!(sequences, vec![1, 2, 3]);
14606 assert_eq!(state.events[1].kind, SpawnEventKind::Exited);
14607 assert_eq!(state.events[1].exit_code, None);
14608 assert_eq!(state.events[1].exit_signal, None);
14609 }
14610
14611 #[test]
14615 fn wait_error_exit_report_is_classified_as_a_crash() {
14616 assert_eq!(wait_error_exit_report().kind, ExitKind::Crash);
14617 }
14618}
14619
14620#[cfg(all(test, unix))]
14621mod restart_ordering_tests {
14622 use super::*;
14623 use std::io::Write;
14624 use tokio::io::AsyncReadExt;
14625
14626 #[test]
14627 #[ignore = "child process fixture"]
14628 fn serving_child() {
14629 let Ok(address_file) = std::env::var("RESTART_TEST_ADDRESS_FILE") else {
14630 return;
14631 };
14632 let listener = std::net::TcpListener::bind("127.0.0.1:0").unwrap();
14633 std::fs::write(address_file, listener.local_addr().unwrap().to_string()).unwrap();
14634 for stream in listener.incoming() {
14635 writeln!(stream.unwrap(), "{}", std::process::id()).unwrap();
14636 }
14637 }
14638
14639 async fn restart_before_publication(requests: usize) {
14640 let home = cortexkit_test_support::ScratchDir::new("restart-before-publication");
14641 let address_file = home.path().join("address");
14642 let spec = ModuleSpec {
14643 module_id: "restart-serving-child".into(),
14644 program: std::env::current_exe().unwrap(),
14645 args: vec![
14646 "--exact".into(),
14647 "supervise::restart_ordering_tests::serving_child".into(),
14648 "--ignored".into(),
14649 "--nocapture".into(),
14650 ],
14651 env: vec![(
14652 "RESTART_TEST_ADDRESS_FILE".into(),
14653 address_file.display().to_string(),
14654 )],
14655 reserved: false,
14656 reserved_prefixes: vec![],
14657 protocol: ModuleProtocol::None,
14658 overlap: Default::default(),
14659 carry_containment: false,
14660 };
14661 let mut supervisor =
14662 Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default());
14663 let (spawned_tx, spawned_rx) = oneshot::channel();
14664 let spawned_tx = Mutex::new(Some(spawned_tx));
14665 let (release_tx, release_rx) = std::sync::mpsc::channel();
14666 let release_rx = Mutex::new(release_rx);
14667 supervisor.test_after_first_spawn = AfterFirstSpawnHook(Some(Arc::new(move |_| {
14670 spawned_tx.lock().unwrap().take().unwrap().send(()).unwrap();
14671 release_rx
14672 .lock()
14673 .unwrap()
14674 .recv_timeout(Duration::from_secs(20))
14675 .unwrap();
14676 })));
14677 let registered = supervisor
14678 .register_configured_with_health(
14679 spec.clone(),
14680 true,
14681 HealthConfig::default(),
14682 None,
14683 RestartPolicy::new(3, Duration::ZERO),
14684 DEFAULT_REGISTER_TIMEOUT,
14685 )
14686 .unwrap();
14687 let module = registered.module.clone();
14688 let startup = tokio::task::spawn_blocking(move || {
14689 supervisor.start_registered_configured(registered, None, true)
14690 });
14691 spawned_rx.await.unwrap();
14692 let old_pid = tokio::time::timeout(Duration::from_secs(10), async {
14693 let address = loop {
14694 if let Ok(address) = std::fs::read_to_string(&address_file) {
14695 if !address.is_empty() {
14696 break address;
14697 }
14698 }
14699 tokio::time::sleep(Duration::from_millis(1)).await;
14700 };
14701 let mut stream = tokio::net::TcpStream::connect(address).await.unwrap();
14702 let mut reply = String::new();
14703 stream.read_to_string(&mut reply).await.unwrap();
14704 reply.trim().parse::<u32>().unwrap()
14705 })
14706 .await
14707 .expect("child must serve before generation publication");
14708 assert_eq!(
14709 lock_snapshot(&module.inner.snapshot)
14710 .unwrap()
14711 .spawn_generation,
14712 0
14713 );
14714 let mut replies = Vec::new();
14715 for _ in 0..requests {
14716 let mut restart = Box::pin(module.restart(Some(0)));
14717 std::future::poll_fn(|cx| {
14720 assert!(std::future::Future::poll(restart.as_mut(), cx).is_pending());
14721 std::task::Poll::Ready(())
14722 })
14723 .await;
14724 replies.push(restart);
14725 }
14726 release_tx.send(()).unwrap();
14727 startup.await.unwrap().unwrap();
14728 for reply in replies {
14729 reply.await.unwrap();
14730 }
14731 let replacement = tokio::time::timeout(Duration::from_secs(10), async {
14734 loop {
14735 if lock_snapshot(&module.inner.snapshot)
14736 .unwrap()
14737 .spawn_generation
14738 > 1
14739 {
14740 break;
14741 }
14742 tokio::time::sleep(Duration::from_millis(1)).await;
14743 }
14744 })
14745 .await;
14746 module.update_spec_for_test(spec).await.unwrap();
14750 let (generation, pid) = {
14751 let state = lock_snapshot(&module.inner.snapshot).unwrap();
14752 (state.spawn_generation, state.pid)
14753 };
14754 module.stop().await.unwrap();
14755 replacement.expect("restart must publish a replacement for the already-serving child");
14756 assert_ne!(
14757 pid,
14758 Some(old_pid),
14759 "a serving child predating the restart must be replaced"
14760 );
14761 assert_eq!(generation, 2, "queued restarts must share one replacement");
14762 }
14763
14764 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
14765 async fn restart_received_after_serving_before_generation_publication_replaces_child() {
14766 restart_before_publication(1).await;
14767 }
14768
14769 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
14770 async fn restart_queued_behind_restart_does_not_kill_its_replacement() {
14771 restart_before_publication(2).await;
14772 }
14773
14774 #[tokio::test]
14775 async fn restart_queued_before_replacement_spawn_keeps_the_new_child() {
14776 let supervisor =
14777 Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default());
14778 let registered = supervisor
14779 .register_configured_with_health(
14780 ModuleSpec {
14781 module_id: "queued-before-replacement".into(),
14782 program: "/bin/sleep".into(),
14787 args: vec!["600".into()],
14788 env: vec![],
14789 reserved: false,
14790 reserved_prefixes: vec![],
14791 protocol: ModuleProtocol::None,
14792 overlap: Default::default(),
14793 carry_containment: false,
14794 },
14795 true,
14796 HealthConfig::default(),
14797 None,
14798 RestartPolicy::default(),
14799 DEFAULT_REGISTER_TIMEOUT,
14800 )
14801 .unwrap();
14802 let RegisteredConfiguredModule {
14803 mut spec,
14804 mut runtime,
14805 module,
14806 mut commands,
14807 } = registered;
14808 let snapshot = &module.inner.snapshot;
14809 let mut restart = Box::pin(module.restart(Some(0)));
14810 std::future::poll_fn(|cx| {
14811 assert!(std::future::Future::poll(restart.as_mut(), cx).is_pending());
14812 std::task::Poll::Ready(())
14813 })
14814 .await;
14815 let mut child = Some(spawn_and_mark_running(&spec, &runtime, snapshot).unwrap());
14818 let replacement_pid = child.as_ref().unwrap().id();
14819 let command = commands.recv().await.unwrap();
14820 assert!(
14821 handle_supervisor_command(
14822 command,
14823 &mut spec,
14824 &mut runtime,
14825 &supervisor.registry,
14826 &supervisor.process_liveness,
14827 snapshot,
14828 &mut child,
14829 &mut commands,
14830 &mut VecDeque::new(),
14831 )
14832 .await
14833 );
14834 restart.await.unwrap();
14835 let kept_pid = child.as_ref().and_then(SupervisedChild::id);
14836 let pending = lock_snapshot(snapshot).unwrap().respawn_pending;
14837 if let Some(mut child) = child {
14838 child.start_kill().unwrap();
14839 child.wait().await.unwrap();
14840 }
14841 assert_eq!(
14842 kept_pid, replacement_pid,
14843 "a newer replacement must survive"
14844 );
14845 assert!(
14846 !pending,
14847 "a satisfied restart must not schedule another spawn"
14848 );
14849 }
14850}
14851
14852#[cfg(all(test, unix))]
14853mod health_event_tests {
14854 use super::*;
14855 use cortexkit_test_support::ScratchDir;
14856 use subc_protocol::{manifest::Concurrency, session::ModuleControlResponse};
14857
14858 struct Actor {
14859 module: SupervisedModule,
14860 registry: Arc<Registry>,
14861 forwarding: Arc<ForwardingTable>,
14862 spec: ModuleSpec,
14863 health: HealthConfig,
14864 rx: mpsc::Receiver<crate::router::OutboundFrame>,
14865 roster: ChildRoster,
14866 _home: ScratchDir,
14867 }
14868
14869 async fn wait_for<T>(reason: &str, mut observe: impl FnMut() -> Option<T>) -> T {
14870 let deadline = std::time::Instant::now() + Duration::from_secs(10);
14874 let tick = Instant::now();
14875 loop {
14876 if let Some(value) = observe() {
14877 assert_eq!(Instant::now(), tick, "{reason} must not wait for a timer");
14878 return value;
14879 }
14880 assert!(
14881 std::time::Instant::now() < deadline,
14882 "timed out waiting for {reason}"
14883 );
14884 tokio::task::yield_now().await;
14885 }
14886 }
14887
14888 impl Actor {
14889 async fn start(protocol: ModuleProtocol, cadence: Duration) -> Self {
14890 Self::start_with_policy(
14891 protocol,
14892 cadence,
14893 DEFAULT_REGISTER_TIMEOUT,
14894 RestartPolicy::new(3, Duration::ZERO),
14895 )
14896 .await
14897 }
14898
14899 async fn start_with_policy(
14900 protocol: ModuleProtocol,
14901 cadence: Duration,
14902 register_timeout: Duration,
14903 policy: RestartPolicy,
14904 ) -> Self {
14905 let home = ScratchDir::new("health-events");
14906 let registry = Arc::new(Registry::default());
14907 let forwarding = Arc::new(ForwardingTable::default());
14908 let health = HealthConfig {
14909 cadence,
14910 deadline: Duration::from_secs(3600),
14911 ..HealthConfig::default()
14912 };
14913 let supervisor = Supervisor::new_for_test(registry.clone(), policy)
14914 .with_forwarding(forwarding.clone())
14915 .with_health_config(health.clone());
14916 let spec = ModuleSpec {
14917 module_id: "event-health-child".into(),
14918 program: PathBuf::from("/bin/sleep"),
14926 args: vec!["600".into()],
14929 env: ["XDG_DATA_HOME", "XDG_RUNTIME_DIR", "XDG_CONFIG_HOME"]
14930 .into_iter()
14931 .map(|key| (key.into(), home.path().display().to_string()))
14932 .collect(),
14933 reserved: false,
14934 reserved_prefixes: vec![],
14935 protocol,
14936 overlap: Default::default(),
14937 carry_containment: false,
14938 };
14939 let module = supervisor
14940 .supervise_configured_requested(
14941 spec.clone(),
14942 true,
14943 health.clone(),
14944 Some(0),
14945 policy,
14946 register_timeout,
14947 None,
14948 )
14949 .unwrap();
14950 let (_, rx) = mpsc::channel(8);
14951 let actor = Self {
14952 module,
14953 registry,
14954 forwarding,
14955 spec,
14956 health,
14957 rx,
14958 roster: supervisor.child_roster.clone(),
14959 _home: home,
14960 };
14961 actor
14962 .wait_for_select(|checkpoint| checkpoint.generation > 0)
14963 .await;
14964 actor
14965 }
14966
14967 async fn expire_registration_wait(&self, remaining: Duration) {
14968 tokio::time::advance(remaining).await;
14969 wait_for("registration timeout TERM grace", || {
14970 lock_snapshot(&self.module.inner.snapshot)
14971 .unwrap()
14972 .actor_select
14973 .is_none()
14974 .then_some(())
14975 })
14976 .await;
14977 tokio::time::advance(crate::child_roster::TERM_TO_KILL).await;
14978 }
14979
14980 fn turns(&self) -> u64 {
14981 lock_snapshot(&self.module.inner.snapshot)
14982 .unwrap()
14983 .actor_turns
14984 }
14985
14986 async fn wait_for_select(
14987 &self,
14988 expected: impl Fn(&ActorSelectCheckpoint) -> bool,
14989 ) -> ActorSelectCheckpoint {
14990 wait_for("actor select checkpoint", || {
14991 let state = lock_snapshot(&self.module.inner.snapshot).unwrap();
14992 state.actor_select.filter(|checkpoint| {
14993 checkpoint.generation == state.spawn_generation
14994 && state.state == ModuleState::Running
14995 && state.process_alive
14996 && state.reported_pid().is_some()
14997 && expected(checkpoint)
14998 })
14999 })
15000 .await
15001 }
15002
15003 fn hello(&mut self, connection: u64, health: bool) {
15004 let manifest =
15005 subc_protocol::manifest::ModuleManifest::builder(&self.spec.module_id, "1.0.0")
15006 .protocol_ver(subc_protocol::PROTOCOL_VERSION)
15007 .build();
15008 let connection = ConnectionId::new(connection);
15009 let (tx, rx) = mpsc::channel(8);
15010 self.rx = rx;
15011 self.forwarding
15012 .register_module_connection(
15013 connection,
15014 self.spec.module_id.clone(),
15015 subc_protocol::PROTOCOL_VERSION,
15016 Concurrency::ModuleManaged,
15017 FrameSink::new(tx),
15018 )
15019 .unwrap();
15020 self.registry
15021 .register_with_control_ops(
15022 manifest,
15023 subc_protocol::PROTOCOL_VERSION,
15024 connection,
15025 if health {
15026 vec![MODULE_CONTROL_OP_HEALTH_CHECK.into()]
15027 } else {
15028 vec![]
15029 },
15030 )
15031 .unwrap();
15032 }
15033
15034 async fn probe(&mut self) -> crate::router::OutboundFrame {
15035 let frame = wait_for("outbound health probe", || match self.rx.try_recv() {
15036 Ok(frame) => Some(frame),
15037 Err(mpsc::error::TryRecvError::Empty) => None,
15038 Err(mpsc::error::TryRecvError::Disconnected) => panic!("health peer disconnected"),
15039 })
15040 .await;
15041 let body: Value = serde_json::from_slice(&frame.body).unwrap();
15042 assert_eq!(body["op"], MODULE_CONTROL_OP_HEALTH_CHECK);
15043 frame
15044 }
15045
15046 fn no_probe(&mut self) {
15047 assert!(matches!(
15048 self.rx.try_recv(),
15049 Err(mpsc::error::TryRecvError::Empty)
15050 ));
15051 }
15052
15053 fn answer(&self, connection: u64, frame: crate::router::OutboundFrame) {
15054 self.forwarding
15055 .complete_module_control_rpc(
15056 ConnectionId::new(connection),
15057 frame.header.corr,
15058 Some(MODULE_CONTROL_OP_HEALTH_CHECK),
15059 ModuleControlRpcOutcome::Response(ModuleControlResponse::HealthCheck {
15060 status: HealthStatus::Ok,
15061 detail: None,
15062 metrics: None,
15063 }),
15064 )
15065 .unwrap();
15066 }
15067 }
15068
15069 #[tokio::test(start_paused = true)]
15070 async fn registration_timeout_kills_at_120_seconds_and_respawns() {
15071 let actor = Actor::start(ModuleProtocol::Subc, Duration::from_secs(30)).await;
15072 let initial = actor.module.status().unwrap();
15073 tokio::time::advance(Duration::from_secs(119)).await;
15074 tokio::task::yield_now().await;
15075 assert_eq!(actor.module.status().unwrap().pid, initial.pid);
15076 assert!(actor.module.terminal_history().entries.is_empty());
15077 actor.expire_registration_wait(Duration::from_secs(1)).await;
15078 actor
15079 .wait_for_select(|at| at.generation > initial.spawn_generation)
15080 .await;
15081 let status = actor.module.status().unwrap();
15082 assert_ne!(status.pid, initial.pid);
15083 assert!(!cortexkit_test_support::process_alive(
15084 initial.pid.unwrap() as i32
15085 ));
15086 assert_eq!(status.restart_count, 1);
15087 assert_eq!(status.last_exit.unwrap().kind, ExitKind::Crash);
15088 let entries = actor.module.terminal_history().entries;
15089 assert_eq!(entries.len(), 1);
15090 assert_eq!(
15091 entries[0].disposition_detail.as_deref(),
15092 Some("registration_timeout")
15093 );
15094 actor.module.stop().await.unwrap();
15095 }
15096
15097 #[tokio::test(start_paused = true)]
15098 async fn registration_timeout_is_cancelled_by_hello_at_119_seconds() {
15099 let mut actor = Actor::start(ModuleProtocol::Subc, Duration::from_secs(30)).await;
15100 let pid = actor.module.status().unwrap().pid;
15101 tokio::time::advance(Duration::from_secs(119)).await;
15102 actor.hello(1, false);
15103 let registered = actor
15104 .wait_for_select(|at| at.registered_connection.is_some())
15105 .await;
15106 assert_eq!(
15107 registered.wake_after, None,
15108 "HELLO must cancel the deadline"
15109 );
15110 tokio::time::advance(Duration::from_secs(300)).await;
15111 tokio::task::yield_now().await;
15112 assert_eq!(actor.module.status().unwrap().pid, pid);
15113 assert!(actor.module.terminal_history().entries.is_empty());
15114 actor
15116 .registry
15117 .deregister_connection(ConnectionId::new(1))
15118 .unwrap();
15119 actor
15120 .wait_for_select(|at| at.registered_connection.is_none())
15121 .await;
15122 tokio::time::advance(Duration::from_secs(300)).await;
15123 tokio::task::yield_now().await;
15124 assert_eq!(actor.module.status().unwrap().pid, pid);
15125 assert!(actor.module.terminal_history().entries.is_empty());
15126 actor.module.stop().await.unwrap();
15127
15128 let mut transient = Actor::start(ModuleProtocol::Subc, Duration::from_secs(30)).await;
15129 let pid = transient.module.status().unwrap().pid;
15130 tokio::time::advance(Duration::from_secs(119)).await;
15131 let turns = transient.turns();
15132 transient.hello(1, false);
15135 transient
15136 .registry
15137 .deregister_connection(ConnectionId::new(1))
15138 .unwrap();
15139 let parked = transient.wait_for_select(|at| at.turn > turns).await;
15140 assert_eq!(
15141 parked.wake_after, None,
15142 "transient HELLO must cancel the deadline"
15143 );
15144 tokio::time::advance(Duration::from_secs(300)).await;
15145 tokio::task::yield_now().await;
15146 assert_eq!(transient.module.status().unwrap().pid, pid);
15147 assert!(transient.module.terminal_history().entries.is_empty());
15148 transient.module.stop().await.unwrap();
15149 }
15150
15151 #[tokio::test(start_paused = true)]
15152 async fn registration_timeouts_exhaust_budget_and_schedule_slow_retry() {
15153 let actor = Actor::start_with_policy(
15154 ModuleProtocol::Subc,
15155 Duration::from_secs(30),
15156 Duration::from_secs(10),
15157 RestartPolicy::new(1, Duration::ZERO),
15158 )
15159 .await;
15160 let generation = actor.module.status().unwrap().spawn_generation;
15161 actor
15162 .expire_registration_wait(Duration::from_secs(10))
15163 .await;
15164 actor.wait_for_select(|at| at.generation > generation).await;
15165 actor
15166 .expire_registration_wait(Duration::from_secs(10))
15167 .await;
15168 let retry = wait_for("failed retry schedule", || {
15169 let state = lock_snapshot(&actor.module.inner.snapshot).unwrap();
15170 (state.state == ModuleState::Failed)
15171 .then_some(state.failed_retry.pending)
15172 .flatten()
15173 })
15174 .await;
15175 assert_eq!(retry.delay, Duration::from_secs(600));
15176 assert_eq!(retry.attempt, 1);
15177 let entries = actor.module.terminal_history().entries;
15178 assert_eq!(entries.len(), 2);
15179 assert_eq!(entries[1].disposition, TerminalDisposition::Failed);
15180 assert!(entries[1]
15181 .disposition_detail
15182 .as_deref()
15183 .unwrap()
15184 .contains("registration_timeout"));
15185 tokio::time::advance(Duration::from_secs(599)).await;
15186 tokio::task::yield_now().await;
15187 assert_eq!(actor.module.state().unwrap(), ModuleState::Failed);
15188 tokio::time::advance(Duration::from_secs(1)).await;
15189 actor
15190 .wait_for_select(|at| at.generation > generation + 1)
15191 .await;
15192 actor
15194 .expire_registration_wait(Duration::from_secs(10))
15195 .await;
15196 wait_for("slow retry registration timeout", || {
15197 (actor.module.terminal_history().entries.len() == 3).then_some(())
15198 })
15199 .await;
15200 actor.module.stop().await.unwrap();
15201 }
15202
15203 #[tokio::test(start_paused = true)]
15204 async fn registration_timeout_honours_ten_second_override() {
15205 let actor = Actor::start_with_policy(
15206 ModuleProtocol::Subc,
15207 Duration::from_secs(30),
15208 Duration::from_secs(10),
15209 RestartPolicy::new(3, Duration::ZERO),
15210 )
15211 .await;
15212 let generation = actor.module.status().unwrap().spawn_generation;
15213 tokio::time::advance(Duration::from_secs(9)).await;
15214 tokio::task::yield_now().await;
15215 assert!(actor.module.terminal_history().entries.is_empty());
15216 actor.expire_registration_wait(Duration::from_secs(1)).await;
15217 actor.wait_for_select(|at| at.generation > generation).await;
15218 assert_eq!(actor.module.terminal_history().entries.len(), 1);
15219 actor.module.stop().await.unwrap();
15220 }
15221
15222 #[tokio::test]
15223 async fn registration_timeout_kills_term_ignoring_helper_before_reaping_leader() {
15224 use std::os::unix::process::{CommandExt, ExitStatusExt};
15225
15226 struct Helper(std::process::Child);
15227 impl Drop for Helper {
15228 fn drop(&mut self) {
15229 if matches!(self.0.try_wait(), Ok(None)) {
15232 let _ = self.0.kill();
15233 let _ = self.0.wait();
15234 }
15235 }
15236 }
15237
15238 let home = ScratchDir::new("registration-timeout-helper");
15239 let supervisor = Supervisor::new_for_test(
15240 Arc::new(Registry::default()),
15241 RestartPolicy::new(0, Duration::ZERO),
15242 );
15243 let spec = ModuleSpec {
15244 module_id: "registration-timeout-group".into(),
15245 program: "/bin/sleep".into(),
15246 args: vec!["120".into()],
15247 env: ["XDG_DATA_HOME", "XDG_RUNTIME_DIR", "XDG_CONFIG_HOME"]
15248 .into_iter()
15249 .map(|key| (key.into(), home.path().display().to_string()))
15250 .collect(),
15251 reserved: false,
15252 reserved_prefixes: vec![],
15253 protocol: ModuleProtocol::Subc,
15254 overlap: Default::default(),
15255 carry_containment: false,
15256 };
15257 let module = supervisor
15258 .supervise_configured_requested(
15259 spec,
15260 true,
15261 HealthConfig::default(),
15262 None,
15263 RestartPolicy::new(0, Duration::ZERO),
15264 Duration::from_secs(10),
15265 None,
15266 )
15267 .unwrap();
15268 let leader = timeout(Duration::from_secs(5), async {
15269 loop {
15270 if let Some(pid) = module.status().unwrap().pid {
15271 break pid;
15272 }
15273 sleep(Duration::from_millis(5)).await;
15274 }
15275 })
15276 .await
15277 .expect("supervised group leader must be reported before the deadline");
15278 assert!(leader > 1);
15279 let ready = home.join("helper.ready");
15280 let mut helper = Helper(
15283 std::process::Command::new("/bin/sh")
15284 .args([
15285 "-c",
15286 "trap '' TERM; : > \"$1\"; exec /bin/sleep 120",
15287 "helper",
15288 ])
15289 .arg(&ready)
15290 .process_group(i32::try_from(leader).unwrap())
15291 .stdout(std::process::Stdio::null())
15292 .stderr(std::process::Stdio::null())
15293 .spawn()
15294 .unwrap(),
15295 );
15296 timeout(Duration::from_secs(5), async {
15297 while !ready.exists() {
15298 assert!(helper.0.try_wait().unwrap().is_none());
15299 sleep(Duration::from_millis(5)).await;
15300 }
15301 })
15302 .await
15303 .expect("helper must install TERM ignore before the registration deadline");
15304 let helper_pid =
15305 rustix::process::Pid::from_raw(i32::try_from(helper.0.id()).unwrap()).unwrap();
15306 rustix::process::kill_process(helper_pid, rustix::process::Signal::TERM).unwrap();
15307 sleep(Duration::from_millis(20)).await;
15308 assert!(
15309 helper.0.try_wait().unwrap().is_none(),
15310 "helper must ignore TERM"
15311 );
15312
15313 timeout(Duration::from_secs(20), async {
15314 while module.state().unwrap() != ModuleState::Failed {
15315 sleep(Duration::from_millis(5)).await;
15316 }
15317 })
15318 .await
15319 .expect("registration timeout must stop and reap the group leader");
15320 let helper_exit = timeout(Duration::from_secs(3), async {
15321 loop {
15322 if let Some(status) = helper.0.try_wait().unwrap() {
15323 break status;
15324 }
15325 sleep(Duration::from_millis(5)).await;
15326 }
15327 })
15328 .await
15329 .expect("TERM-ignoring helper survived the registration timeout's group KILL");
15330 assert_eq!(helper_exit.signal(), Some(9));
15331 assert!(!cortexkit_test_support::process_alive(
15332 i32::try_from(leader).unwrap()
15333 ));
15334 assert_eq!(
15335 module.status().unwrap().last_exit.unwrap().kind,
15336 ExitKind::Crash
15337 );
15338 assert!(module.terminal_history().entries[0]
15339 .disposition_detail
15340 .as_deref()
15341 .unwrap()
15342 .contains("registration_timeout"));
15343 module.stop().await.unwrap();
15344 }
15345
15346 #[tokio::test(start_paused = true)]
15347 async fn registration_timeout_does_not_count_an_earlier_exit() {
15348 let actor = Actor::start_with_policy(
15349 ModuleProtocol::Subc,
15350 Duration::from_secs(30),
15351 DEFAULT_REGISTER_TIMEOUT,
15352 RestartPolicy::new(0, Duration::ZERO),
15353 )
15354 .await;
15355 let pid = actor.module.status().unwrap().pid.unwrap();
15356 rustix::process::kill_process(
15357 rustix::process::Pid::from_raw(pid as i32).unwrap(),
15358 rustix::process::Signal::TERM,
15359 )
15360 .unwrap();
15361 wait_for("early exit", || {
15362 (actor.module.state().unwrap() == ModuleState::Failed).then_some(())
15363 })
15364 .await;
15365 tokio::time::advance(Duration::from_secs(121)).await;
15366 tokio::task::yield_now().await;
15367 let entries = actor.module.terminal_history().entries;
15368 assert_eq!(entries.len(), 1);
15369 assert!(!entries[0]
15370 .disposition_detail
15371 .as_deref()
15372 .unwrap()
15373 .contains("registration_timeout"));
15374 assert_eq!(actor.module.status().unwrap().lifetime_restarts, 0);
15375 actor.module.stop().await.unwrap();
15376 }
15377
15378 #[tokio::test(start_paused = true)]
15379 async fn registration_timeout_is_cancelled_by_daemon_shutdown() {
15380 let actor = Actor::start(ModuleProtocol::Subc, Duration::from_secs(30)).await;
15381 let pid = actor.module.status().unwrap().pid;
15382 tokio::time::advance(Duration::from_secs(119)).await;
15383 actor.roster.close();
15384 let turns = actor.turns();
15385 tokio::time::advance(Duration::from_secs(300)).await;
15386 let parked = actor.wait_for_select(|at| at.turn > turns).await;
15387 assert_eq!(parked.wake_after, None);
15388 assert_eq!(actor.module.status().unwrap().pid, pid);
15389 assert!(actor.module.terminal_history().entries.is_empty());
15390 actor.module.stop().await.unwrap();
15391 let status = actor.module.status().unwrap();
15392 assert_eq!(status.restart_count, 0);
15393 assert_eq!(
15394 actor.module.terminal_history().entries[0].disposition,
15395 TerminalDisposition::DaemonShutdown
15396 );
15397 }
15398
15399 #[tokio::test(start_paused = true)]
15400 async fn no_health_children_park_for_sixty_seconds() {
15401 let none = Actor::start(ModuleProtocol::None, Duration::from_secs(30)).await;
15402 let unregistered = Actor::start(ModuleProtocol::Subc, Duration::from_secs(30)).await;
15403 let mut unadvertised = Actor::start(ModuleProtocol::Subc, Duration::from_secs(30)).await;
15404 unadvertised.hello(1, false);
15405 for _ in 0..6000 {
15410 tokio::time::advance(Duration::from_millis(10)).await;
15411 tokio::task::yield_now().await;
15412 }
15413 for actor in [&none, &unregistered, &unadvertised] {
15414 assert!(
15415 actor.turns() <= 5,
15416 "no-health actor woke {} times",
15417 actor.turns()
15418 );
15419 assert_eq!(actor.module.state().unwrap(), ModuleState::Running);
15420 }
15421 }
15422
15423 #[tokio::test(start_paused = true)]
15424 async fn late_hello_starts_a_due_probe_in_the_notification_tick() {
15425 let mut actor = Actor::start(ModuleProtocol::Subc, Duration::ZERO).await;
15429 let tick = Instant::now();
15430 actor.hello(1, true);
15431 actor.probe().await;
15432 assert_eq!(Instant::now(), tick, "HELLO must not wait for a poll");
15433 }
15434
15435 #[tokio::test(start_paused = true)]
15436 async fn hello_keeps_the_jittered_cadence_and_catalog_updates_do_not_reset_it() {
15437 let mut actor = Actor::start(ModuleProtocol::Subc, Duration::from_secs(30)).await;
15438 let hello_tick = Instant::now();
15439 actor.hello(1, true);
15440 let armed = actor
15441 .wait_for_select(|checkpoint| {
15442 checkpoint.registered_connection == Some(ConnectionId::new(1))
15443 && checkpoint.next_probe_at.is_some()
15444 })
15445 .await;
15446 let delay = jittered_health_delay(&actor.spec.module_id, 0, actor.health.cadence);
15447 assert!((Duration::from_secs(30)..Duration::from_secs(33)).contains(&delay));
15448 let first_deadline = hello_tick + delay;
15449 assert_eq!(armed.next_probe_at, Some(first_deadline));
15450 tokio::time::advance(Duration::from_secs(29)).await;
15451 let turns = actor.turns();
15452 actor
15453 .registry
15454 .replace_catalog_for_connection(ConnectionId::new(1), vec![], None, Some(true))
15455 .unwrap();
15456 let updated = actor
15457 .wait_for_select(|checkpoint| checkpoint.turn > turns)
15458 .await;
15459 assert_eq!(
15460 updated.next_probe_at,
15461 Some(first_deadline),
15462 "catalog update must not reset cadence"
15463 );
15464 actor.no_probe();
15465 tokio::time::advance(first_deadline - Instant::now()).await;
15466 let frame = actor.probe().await;
15467 actor.answer(1, frame);
15468 let next = jittered_health_delay(&actor.spec.module_id, 1, actor.health.cadence);
15469 let next_deadline = Instant::now() + next;
15470 let rearmed = actor
15471 .wait_for_select(|checkpoint| {
15472 checkpoint
15473 .next_probe_at
15474 .is_some_and(|deadline| deadline > first_deadline)
15475 })
15476 .await;
15477 assert_eq!(rearmed.next_probe_at, Some(next_deadline));
15478 tokio::time::advance(next - Duration::from_nanos(1)).await;
15479 actor.no_probe();
15480 tokio::time::advance(Duration::from_nanos(1)).await;
15481 actor.probe().await;
15482 }
15483
15484 #[tokio::test(start_paused = true)]
15485 async fn disconnect_disarms_and_reconnect_rearms_health() {
15486 let mut actor = Actor::start(ModuleProtocol::Subc, Duration::from_secs(30)).await;
15487 actor.hello(1, true);
15488 actor
15489 .wait_for_select(|checkpoint| {
15490 checkpoint.registered_connection == Some(ConnectionId::new(1))
15491 && checkpoint.next_probe_at.is_some()
15492 })
15493 .await;
15494 let turns = actor.turns();
15495 let tick = Instant::now();
15496 actor
15497 .registry
15498 .deregister_connection(ConnectionId::new(1))
15499 .unwrap();
15500 let disarmed = actor
15501 .wait_for_select(|checkpoint| {
15502 checkpoint.turn > turns && checkpoint.registered_connection.is_none()
15503 })
15504 .await;
15505 assert_eq!(disarmed.next_probe_at, None);
15506 assert_eq!(disarmed.wake_after, None);
15507 assert_eq!(
15508 Instant::now(),
15509 tick,
15510 "disconnect must disarm in the notification tick"
15511 );
15512 tokio::time::advance(Duration::from_secs(60)).await;
15513 actor.no_probe();
15514 let reconnect_tick = Instant::now();
15515 actor.hello(2, true);
15516 let rearmed = actor
15517 .wait_for_select(|checkpoint| {
15518 checkpoint.registered_connection == Some(ConnectionId::new(2))
15519 && checkpoint.next_probe_at.is_some()
15520 })
15521 .await;
15522 let delay = jittered_health_delay(&actor.spec.module_id, 0, actor.health.cadence);
15523 assert_eq!(rearmed.next_probe_at, Some(reconnect_tick + delay));
15524 tokio::time::advance(delay).await;
15525 actor.probe().await;
15526 }
15527
15528 #[tokio::test(start_paused = true)]
15529 async fn reregistration_without_health_disarms_the_old_schedule() {
15530 let mut actor = Actor::start(ModuleProtocol::Subc, Duration::from_secs(30)).await;
15531 actor.hello(1, true);
15532 actor
15533 .wait_for_select(|checkpoint| {
15534 checkpoint.registered_connection == Some(ConnectionId::new(1))
15535 && checkpoint.next_probe_at.is_some()
15536 })
15537 .await;
15538 actor
15539 .registry
15540 .deregister_connection(ConnectionId::new(1))
15541 .unwrap();
15542 actor.hello(2, false);
15543 let disarmed = actor
15544 .wait_for_select(|checkpoint| {
15545 checkpoint.registered_connection == Some(ConnectionId::new(2))
15546 })
15547 .await;
15548 assert_eq!(disarmed.next_probe_at, None);
15549 assert_eq!(disarmed.wake_after, None);
15550 tokio::time::advance(Duration::from_secs(60)).await;
15551 actor.no_probe();
15552 }
15553
15554 #[tokio::test(start_paused = true)]
15555 async fn restart_waits_for_its_own_hello_before_rearming_health() {
15556 let mut actor = Actor::start(ModuleProtocol::Subc, Duration::from_secs(30)).await;
15557 actor.hello(1, true);
15558 let armed = actor
15559 .wait_for_select(|checkpoint| {
15560 checkpoint.registered_connection == Some(ConnectionId::new(1))
15561 && checkpoint.next_probe_at.is_some()
15562 })
15563 .await;
15564 actor
15568 .registry
15569 .deregister_connection(ConnectionId::new(1))
15570 .unwrap();
15571 actor.module.restart(Some(0)).await.unwrap();
15572 let parked = actor
15573 .wait_for_select(|checkpoint| {
15574 checkpoint.generation > armed.generation
15575 && checkpoint.registered_connection.is_none()
15576 && checkpoint.next_probe_at.is_none()
15577 })
15578 .await;
15579 assert_eq!(parked.wake_after, Some(DEFAULT_REGISTER_TIMEOUT));
15580 actor.rx.close();
15581 while actor.rx.try_recv().is_ok() {}
15584 let turns = actor.turns();
15585 tokio::time::advance(Duration::from_secs(60)).await;
15586 let after = actor.turns();
15587 eprintln!(
15588 "restart park turns: before={turns} after={after} delta={}",
15589 after - turns
15590 );
15591 assert_eq!(after, turns, "replacement must park before HELLO");
15592 let hello_tick = Instant::now();
15593 actor.hello(2, true);
15594 let rearmed = actor
15595 .wait_for_select(|checkpoint| {
15596 checkpoint.registered_connection == Some(ConnectionId::new(2))
15597 && checkpoint.next_probe_at.is_some()
15598 })
15599 .await;
15600 let delay = jittered_health_delay(&actor.spec.module_id, 0, actor.health.cadence);
15601 assert_eq!(rearmed.next_probe_at, Some(hello_tick + delay));
15602 tokio::time::advance(delay).await;
15603 actor.probe().await;
15604 }
15605
15606 #[tokio::test(start_paused = true)]
15607 async fn http_probe_waits_a_full_cadence_for_each_spawn() {
15608 use tokio::{
15609 io::{AsyncReadExt, AsyncWriteExt},
15610 net::TcpListener,
15611 };
15612 let mut actor = Actor::start(ModuleProtocol::None, Duration::from_secs(30)).await;
15613 let listener = TcpListener::bind("127.0.0.1:0").await.unwrap();
15614 actor.health.http = Some(format!("http://{}/healthz", listener.local_addr().unwrap()));
15615 let (started_tx, mut started_rx) = mpsc::channel(8);
15616 let peer = tokio::spawn(async move {
15617 loop {
15618 let (mut socket, _) = listener.accept().await.unwrap();
15619 let mut request = [0; 1024];
15620 assert!(socket.read(&mut request).await.unwrap() > 0);
15621 started_tx.send(Instant::now()).await.unwrap();
15622 socket
15623 .write_all(b"HTTP/1.1 200 OK\r\nContent-Length: 0\r\n\r\n")
15624 .await
15625 .unwrap();
15626 }
15627 });
15628 actor
15629 .module
15630 .update_configuration(actor.spec.clone(), actor.health.clone(), None)
15631 .await
15632 .unwrap();
15633 let first_tick = Instant::now();
15634 let first = actor
15635 .wait_for_select(|checkpoint| checkpoint.next_probe_at.is_some())
15636 .await;
15637 assert!(first.next_probe_at.unwrap() >= first_tick + actor.health.cadence);
15638 assert!(started_rx.try_recv().is_err());
15639
15640 tokio::time::advance(
15643 first.next_probe_at.unwrap() - Instant::now() - Duration::from_secs(1),
15644 )
15645 .await;
15646 actor.module.restart(Some(0)).await.unwrap();
15647 let spawn_tick = Instant::now();
15648 let replacement = actor
15649 .wait_for_select(|checkpoint| checkpoint.generation > first.generation)
15650 .await;
15651 let deadline = replacement.next_probe_at.unwrap();
15652 assert!(
15653 deadline >= spawn_tick + actor.health.cadence,
15654 "a replacement must not inherit its predecessor's probe deadline"
15655 );
15656 tokio::time::advance(actor.health.cadence - Duration::from_nanos(1)).await;
15657 assert!(
15658 started_rx.try_recv().is_err(),
15659 "no HTTP probe during startup grace"
15660 );
15661 tokio::time::advance(deadline - Instant::now()).await;
15662 let landed = wait_for("first replacement HTTP probe", || {
15663 started_rx.try_recv().ok()
15664 })
15665 .await;
15666 assert!(landed >= spawn_tick + actor.health.cadence);
15667 wait_for("healthy replacement", || {
15668 (actor.module.status().unwrap().health.status == SupervisorHealthStatus::Ok)
15669 .then_some(())
15670 })
15671 .await;
15672 peer.abort();
15673 }
15674
15675 #[tokio::test(start_paused = true)]
15676 async fn rescan_adds_and_removes_http_health_without_polling() {
15677 use tokio::{
15678 io::{AsyncReadExt, AsyncWriteExt},
15679 net::TcpListener,
15680 };
15681 let mut actor = Actor::start(ModuleProtocol::None, Duration::from_secs(30)).await;
15682 let listener = TcpListener::bind("127.0.0.1:0").await.unwrap();
15683 actor.health.http = Some(format!("http://{}/health", listener.local_addr().unwrap()));
15684 actor.health.cadence = Duration::from_millis(1);
15686 let (started_tx, mut started_rx) = mpsc::channel(8);
15687 let peer = tokio::spawn(async move {
15688 loop {
15689 let (mut socket, _) = listener.accept().await.unwrap();
15690 let mut request = [0; 1024];
15691 assert!(socket.read(&mut request).await.unwrap() > 0);
15692 started_tx.send(()).await.unwrap();
15693 socket
15694 .write_all(b"HTTP/1.1 200 OK\r\nContent-Length: 0\r\n\r\n")
15695 .await
15696 .unwrap();
15697 }
15698 });
15699 let tick = Instant::now();
15700 let turns = actor.turns();
15701 actor
15702 .module
15703 .update_configuration(actor.spec.clone(), actor.health.clone(), None)
15704 .await
15705 .unwrap();
15706 let armed = actor
15707 .wait_for_select(|checkpoint| {
15708 checkpoint.turn > turns && checkpoint.next_probe_at.is_some()
15709 })
15710 .await;
15711 assert_eq!(armed.next_probe_at, Some(tick + Duration::from_millis(1)));
15712 assert_eq!(Instant::now(), tick);
15713 tokio::time::advance(Duration::from_millis(1)).await;
15714 let mut started = false;
15715 wait_for("successful HTTP probe", || {
15716 started |= started_rx.try_recv().is_ok();
15717 (started && actor.module.status().unwrap().health.status == SupervisorHealthStatus::Ok)
15718 .then_some(())
15719 })
15720 .await;
15721 assert!(started, "added HTTP check must start at its first deadline");
15722 assert_eq!(
15723 actor.module.status().unwrap().health.status,
15724 SupervisorHealthStatus::Ok
15725 );
15726 assert_eq!(Instant::now(), tick + Duration::from_millis(1));
15727 actor.health.http = None;
15728 let turns = actor.turns();
15729 actor
15730 .module
15731 .update_configuration(actor.spec.clone(), actor.health.clone(), None)
15732 .await
15733 .unwrap();
15734 let parked = actor
15735 .wait_for_select(|checkpoint| {
15736 checkpoint.turn > turns && checkpoint.next_probe_at.is_none()
15737 })
15738 .await;
15739 assert_eq!(parked.wake_after, None);
15740 let turns = actor.turns();
15741 tokio::time::advance(Duration::from_secs(60)).await;
15742 assert_eq!(
15743 actor.turns(),
15744 turns,
15745 "removed HTTP check must park the actor"
15746 );
15747 assert!(
15748 started_rx.try_recv().is_err(),
15749 "removed HTTP check must stay disarmed"
15750 );
15751 assert_eq!(
15752 actor.module.status().unwrap().health,
15753 ModuleHealthStatus::default()
15754 );
15755 peer.abort();
15756 }
15757}
15758
15759#[cfg(test)]
15760mod drain_busy_reprobe_tests {
15761 use super::*;
15765 use std::collections::BTreeMap;
15766 use std::sync::Mutex as StdMutex;
15767 use subc_protocol::{manifest::Concurrency, session::ModuleControlResponse};
15768
15769 const MODULE_ID: &str = "drain-reprobe-module";
15770 const CONNECTION: u64 = 7;
15771 const GAUGE: &str = "runs_in_flight";
15772 const CADENCE: Duration = Duration::from_secs(30);
15775 const CEILING: Duration = Duration::from_secs(30);
15776
15777 #[derive(Clone, Copy)]
15779 enum Answer {
15780 BusyUntil { until: Instant, latency: Duration },
15783 OmitsGauge,
15785 Silent,
15787 }
15788
15789 struct Drain {
15790 forwarding: Arc<ForwardingTable>,
15791 runtime: SupervisorRuntimeConfig,
15792 endpoint: crate::ModuleEndpointId,
15793 probes: Arc<StdMutex<Vec<Instant>>>,
15795 responder: tokio::task::JoinHandle<()>,
15796 }
15797
15798 impl Drain {
15799 fn start(answer: Answer) -> Self {
15800 let registry = Arc::new(Registry::default());
15801 let forwarding = Arc::new(ForwardingTable::default());
15802 let health = HealthConfig {
15803 cadence: CADENCE,
15804 ..HealthConfig::default()
15805 };
15806 let runtime = Supervisor::new_for_test(registry, RestartPolicy::new(3, Duration::ZERO))
15807 .with_forwarding(forwarding.clone())
15808 .with_health_config(health)
15809 .runtime_config();
15810 let (tx, mut rx) = mpsc::channel::<crate::router::OutboundFrame>(8);
15811 let endpoint = forwarding
15812 .register_module_connection(
15813 ConnectionId::new(CONNECTION),
15814 MODULE_ID.to_string(),
15815 subc_protocol::PROTOCOL_VERSION,
15816 Concurrency::ModuleManaged,
15817 FrameSink::new(tx),
15818 )
15819 .unwrap();
15820 let probes = Arc::new(StdMutex::new(Vec::new()));
15821 let responder = tokio::spawn({
15822 let forwarding = forwarding.clone();
15823 let probes = probes.clone();
15824 async move {
15825 while let Some(frame) = rx.recv().await {
15826 let body: Value = serde_json::from_slice(&frame.body).unwrap();
15827 assert_eq!(body["op"], MODULE_CONTROL_OP_HEALTH_CHECK);
15828 probes.lock().unwrap().push(Instant::now());
15829 let metrics = match answer {
15830 Answer::BusyUntil { until, latency } => {
15831 sleep(latency).await;
15832 let busy = u64::from(Instant::now() < until);
15833 serde_json::json!({ GAUGE: busy })
15834 }
15835 Answer::OmitsGauge => serde_json::json!({ "queue_depth": 0 }),
15836 Answer::Silent => continue,
15837 };
15838 let _ = forwarding.complete_module_control_rpc(
15841 ConnectionId::new(CONNECTION),
15842 frame.header.corr,
15843 Some(MODULE_CONTROL_OP_HEALTH_CHECK),
15844 ModuleControlRpcOutcome::Response(ModuleControlResponse::HealthCheck {
15845 status: HealthStatus::Ok,
15846 detail: None,
15847 metrics: Some(metrics),
15848 }),
15849 );
15850 }
15851 }
15852 });
15853 Self {
15854 forwarding,
15855 runtime,
15856 endpoint,
15857 probes,
15858 responder,
15859 }
15860 }
15861
15862 async fn run(&self, ceiling: Duration) -> (bool, Duration) {
15865 let (drained, elapsed, _) = self.run_traced(ceiling).await;
15866 (drained, elapsed)
15867 }
15868
15869 async fn run_traced(&self, ceiling: Duration) -> (bool, Duration, DrainGaugeTrace) {
15871 let started = Instant::now();
15872 let mut trace = DrainGaugeTrace::new(&[GAUGE.to_string()]);
15873 let drained = wait_for_forwarding_quiescence(
15874 &self.forwarding,
15875 MODULE_ID,
15876 &self.runtime,
15877 self.endpoint,
15878 started + ceiling,
15879 &[GAUGE.to_string()],
15880 DrainScope::Active,
15881 &mut trace,
15882 )
15883 .await
15884 .unwrap();
15885 (drained, started.elapsed(), trace)
15886 }
15887
15888 fn probes(&self) -> Vec<Instant> {
15889 self.probes.lock().unwrap().clone()
15890 }
15891 }
15892
15893 impl Drop for Drain {
15894 fn drop(&mut self) {
15895 self.responder.abort();
15896 }
15897 }
15898
15899 #[tokio::test(start_paused = true)]
15900 async fn busy_at_first_probe_then_quiet_drains_long_before_the_ceiling() {
15901 let drain = Drain::start(Answer::BusyUntil {
15902 until: Instant::now() + Duration::from_secs(1),
15903 latency: Duration::ZERO,
15904 });
15905 let (drained, elapsed) = drain.run(CEILING).await;
15906 assert!(
15907 drained,
15908 "gauges that read zero during the drain must let it finish drained, \
15909 not wait out the {CADENCE:?} health cadence (elapsed {elapsed:?})"
15910 );
15911 assert!(
15912 elapsed < Duration::from_secs(3),
15913 "drain took {elapsed:?} after the gauges went quiet at 1s"
15914 );
15915 let probes = drain.probes();
15916 assert!(
15917 probes.len() >= 2,
15918 "the module was busy at the first probe, so a quiet verdict needs a second: {probes:?}"
15919 );
15920 }
15921
15922 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
15927 async fn an_instant_busy_first_answer_does_not_stop_later_probes_in_real_time() {
15928 let drain = Drain::start(Answer::BusyUntil {
15929 until: Instant::now() + Duration::from_millis(600),
15930 latency: Duration::ZERO,
15931 });
15932 let (drained, elapsed, trace) = drain.run_traced(CEILING).await;
15933 assert!(drained, "the gauge fell at 600ms: {trace}");
15934 assert!(
15935 elapsed < Duration::from_secs(3),
15936 "drain took {elapsed:?} after the gauge fell at 600ms: {trace}"
15937 );
15938 let probes = drain.probes();
15939 assert!(
15940 probes.len() >= 2 && trace.probes as usize == probes.len(),
15941 "the drain must re-probe after the first busy answer: {probes:?}, {trace}"
15942 );
15943 }
15944
15945 #[tokio::test(start_paused = true)]
15946 async fn busy_throughout_ends_undrained_at_the_ceiling_with_bounded_probes() {
15947 let started = Instant::now();
15948 let drain = Drain::start(Answer::BusyUntil {
15949 until: started + Duration::from_secs(3600),
15950 latency: Duration::ZERO,
15951 });
15952 let (drained, elapsed) = drain.run(CEILING).await;
15953 assert!(!drained, "a module busy throughout must not drain");
15954 assert!(
15955 elapsed >= CEILING && elapsed < CEILING + drain_busy_reprobe_interval(),
15956 "a busy module is held until the ceiling and no longer: {elapsed:?}"
15957 );
15958 let probes = drain.probes();
15961 let max_probes = (CEILING.as_millis() / drain_busy_reprobe_interval().as_millis()) + 1;
15962 assert!(
15963 probes.len() as u128 <= max_probes,
15964 "{} probes in a {CEILING:?} drain exceeds the {max_probes} the interval allows",
15965 probes.len()
15966 );
15967 for pair in probes.windows(2) {
15968 assert!(
15969 pair[1] - pair[0] >= drain_busy_reprobe_interval(),
15970 "consecutive probes {:?} apart",
15971 pair[1] - pair[0]
15972 );
15973 }
15974 }
15975
15976 #[tokio::test(start_paused = true)]
15977 async fn a_slow_probe_is_never_overlapped_by_the_next() {
15978 let latency = Duration::from_secs(1);
15981 let drain = Drain::start(Answer::BusyUntil {
15982 until: Instant::now() + Duration::from_secs(3600),
15983 latency,
15984 });
15985 let (drained, _) = drain.run(Duration::from_secs(5)).await;
15986 assert!(!drained);
15987 let probes = drain.probes();
15988 assert!(probes.len() >= 2, "the drain must re-probe: {probes:?}");
15989 for pair in probes.windows(2) {
15990 assert!(
15991 pair[1] - pair[0] >= latency + drain_busy_reprobe_interval(),
15992 "a probe started {:?} after the previous one, before it could finish",
15993 pair[1] - pair[0]
15994 );
15995 }
15996 }
15997
15998 #[tokio::test(start_paused = true)]
15999 async fn an_unanswered_probe_cannot_hold_the_drain_past_its_ceiling() {
16000 let drain = Drain::start(Answer::Silent);
16001 let ceiling = Duration::from_secs(2);
16002 let (drained, elapsed) = drain.run(ceiling).await;
16003 assert!(!drained, "an unanswered probe counts as busy");
16004 assert!(
16005 elapsed >= ceiling && elapsed < ceiling + drain_busy_reprobe_interval(),
16006 "the probe deadline is capped at the drain ceiling: {elapsed:?}"
16007 );
16008 assert_eq!(
16009 drain.probes().len(),
16010 1,
16011 "the unanswered probe holds the only slot until the ceiling"
16012 );
16013 }
16014
16015 #[derive(Clone, Default)]
16017 struct LogCapture {
16018 events: Arc<StdMutex<Vec<BTreeMap<String, String>>>>,
16019 }
16020
16021 impl<S: tracing::Subscriber> tracing_subscriber::Layer<S> for LogCapture {
16022 fn on_event(
16023 &self,
16024 event: &tracing::Event<'_>,
16025 _context: tracing_subscriber::layer::Context<'_, S>,
16026 ) {
16027 struct Fields(BTreeMap<String, String>);
16028 impl tracing::field::Visit for Fields {
16029 fn record_debug(&mut self, field: &tracing::field::Field, value: &dyn fmt::Debug) {
16030 self.0
16031 .insert(field.name().to_string(), format!("{value:?}"));
16032 }
16033 }
16034 let mut fields = Fields(BTreeMap::new());
16035 event.record(&mut fields);
16036 self.events.lock().unwrap().push(fields.0);
16037 }
16038 }
16039
16040 impl LogCapture {
16041 fn install(&self) -> tracing::subscriber::DefaultGuard {
16042 use tracing_subscriber::layer::SubscriberExt;
16043 tracing::subscriber::set_default(tracing_subscriber::registry().with(self.clone()))
16044 }
16045
16046 fn with_message(&self, needle: &str) -> Vec<BTreeMap<String, String>> {
16047 self.events
16048 .lock()
16049 .unwrap()
16050 .iter()
16051 .filter(|fields| fields.get("message").is_some_and(|m| m.contains(needle)))
16052 .cloned()
16053 .collect()
16054 }
16055 }
16056
16057 #[tokio::test(start_paused = true)]
16062 async fn a_missing_declared_gauge_is_named_on_the_drain_timeout_line() {
16063 let capture = LogCapture::default();
16064 let _guard = capture.install();
16065 let drain = Drain::start(Answer::OmitsGauge);
16066 let (drained, _, trace) = drain.run_traced(Duration::from_secs(2)).await;
16067 assert!(!drained, "a gauge never reported cannot read quiet");
16068 assert!(trace.probes >= 2, "the drain must re-probe: {trace:?}");
16069
16070 let missing = capture.with_message("lacks a declared busy gauge");
16071 assert_eq!(missing.len(), 1, "one warning per drain: {missing:?}");
16072 assert_eq!(
16073 missing[0].get("missing_gauges").map(String::as_str),
16074 Some(GAUGE)
16075 );
16076
16077 warn_drain_timed_out(
16078 MODULE_ID,
16079 Duration::from_secs(2),
16080 RouteCloseReason::Restart,
16081 &Default::default(),
16082 &trace,
16083 );
16084 let timeouts = capture.with_message("route drain timed out");
16085 assert_eq!(timeouts.len(), 1, "{timeouts:?}");
16086 let gauges = &timeouts[0]["busy_gauges"];
16087 assert!(
16088 gauges.starts_with(&format!("{GAUGE}: missing_from_health_body @")),
16089 "the timeout line must name the gauge and why it read busy: {gauges}"
16090 );
16091 assert!(
16092 gauges.ends_with("; no value ever read"),
16093 "no value was ever read for the gauge: {gauges}"
16094 );
16095 assert_eq!(timeouts[0]["gauge_probes"], trace.probes.to_string());
16096 }
16097
16098 #[tokio::test(start_paused = true)]
16102 async fn the_drain_trace_names_a_busy_value_and_an_unanswered_probe() {
16103 let busy = Drain::start(Answer::BusyUntil {
16104 until: Instant::now() + Duration::from_secs(3600),
16105 latency: Duration::ZERO,
16106 });
16107 let (_, _, trace) = busy.run_traced(Duration::from_secs(1)).await;
16108 let rendered = trace.to_string();
16109 assert!(
16110 rendered.starts_with(&format!("{GAUGE}: non_zero(1) @"))
16111 && rendered.contains("; last value non_zero(1) @"),
16112 "{rendered}"
16113 );
16114
16115 let silent = Drain::start(Answer::Silent);
16116 let (_, _, trace) = silent.run_traced(Duration::from_secs(1)).await;
16117 assert_eq!(
16120 trace.to_string(),
16121 format!("{GAUGE}: probe_failed(no-answer) @1000ms; no value ever read")
16122 );
16123 assert_eq!(trace.probes, 1);
16124 }
16125
16126 #[test]
16128 fn a_gauge_that_is_not_a_count_is_busy_and_shown_as_sent() {
16129 let gauges = [GAUGE.to_string()];
16130 let verdicts = gauge_verdicts(Some(&serde_json::json!({ GAUGE: 0.0 })), &gauges);
16131 assert_eq!(verdicts, vec![GaugeVerdict::NotACount("0.0".to_string())]);
16132 assert_eq!(
16133 busy_gauge_observation(&verdicts),
16134 BusyGaugeObservation::Busy
16135 );
16136 let zero = gauge_verdicts(Some(&serde_json::json!({ GAUGE: 0 })), &gauges);
16137 assert_eq!(
16138 busy_gauge_observation(&zero),
16139 BusyGaugeObservation::Quiescent
16140 );
16141 let none = gauge_verdicts(None, &gauges);
16142 assert_eq!(busy_gauge_observation(&none), BusyGaugeObservation::Omitted);
16143 }
16144}
16145
16146#[cfg(test)]
16147mod health_evidence_tests {
16148 use super::{HealthProbeError, HealthProbeEvidence};
16149 use std::collections::HashSet;
16150
16151 #[test]
16159 fn only_a_dead_lane_is_proof_of_death() {
16160 assert!(HealthProbeError::lane_dead("gone").is_proof_of_death());
16161 assert!(!HealthProbeError::no_answer("timed out").is_proof_of_death());
16165 assert!(!HealthProbeError::bad_answer("garbage").is_proof_of_death());
16166 assert!(!HealthProbeError::misconfigured("no table").is_proof_of_death());
16167 }
16168
16169 #[test]
16175 fn every_evidence_class_has_a_distinct_label() {
16176 let labels = [
16177 HealthProbeError::lane_dead("").label(),
16178 HealthProbeError::no_answer("").label(),
16179 HealthProbeError::bad_answer("").label(),
16180 HealthProbeError::misconfigured("").label(),
16181 ];
16182 let unique: HashSet<_> = labels.iter().collect();
16183 assert_eq!(unique.len(), labels.len(), "labels collided: {labels:?}");
16184 }
16185
16186 #[test]
16192 fn classification_preserves_the_original_message() {
16193 let err = HealthProbeError::no_answer("module did not answer within 5s");
16194 assert_eq!(err.to_string(), "module did not answer within 5s");
16195 assert!(matches!(err.evidence, HealthProbeEvidence::NoAnswer));
16196 }
16197}
16198
16199#[cfg(test)]
16200mod health_tombstone_tests {
16201 use std::{path::PathBuf, sync::Arc, time::Duration};
16202
16203 use subc_protocol::{
16204 manifest::Concurrency,
16205 session::{HealthStatus, ModuleControlResponse},
16206 };
16207 use tokio::sync::mpsc;
16208
16209 use super::{
16210 probe_module_health, HealthAction, HealthConfig, HealthProbeEvidence, ModuleProtocol,
16211 ModuleSpec, RestartPolicy, Supervisor, SupervisorRuntimeConfig,
16212 };
16213 use crate::{
16214 control::ControlHandler,
16215 forwarding::{ForwardingTable, ModuleControlRpcCompletion, ModuleControlRpcOutcome},
16216 registry::{ConnectionId, Registry},
16217 router::FrameSink,
16218 };
16219
16220 struct ProbeHarness {
16221 spec: ModuleSpec,
16222 runtime: SupervisorRuntimeConfig,
16223 forwarding: Arc<ForwardingTable>,
16224 module_connection: ConnectionId,
16225 module_rx: mpsc::Receiver<crate::router::OutboundFrame>,
16226 handler: ControlHandler,
16227 module: super::SupervisedModule,
16228 }
16229
16230 fn probe_harness() -> ProbeHarness {
16231 let registry = Arc::new(Registry::default());
16232 let forwarding = Arc::new(ForwardingTable::default());
16233 let supervisor_handle = super::SupervisorHandle::new();
16234 let health = HealthConfig {
16235 http: None,
16236 cadence: Duration::from_secs(30),
16237 deadline: Duration::from_secs(5),
16238 failure_threshold: 3,
16239 on_degraded: HealthAction::Report,
16240 on_failing: HealthAction::Report,
16241 critical: false,
16242 };
16243 let supervisor = Supervisor::new_for_test(Arc::clone(®istry), RestartPolicy::default())
16244 .with_forwarding(Arc::clone(&forwarding))
16245 .with_handle(supervisor_handle.clone())
16246 .with_health_config(health);
16247 let spec = ModuleSpec {
16248 module_id: "late-health-module".to_string(),
16249 program: PathBuf::from("disabled-module"),
16250 args: Vec::new(),
16251 env: Vec::new(),
16252 reserved: false,
16253 reserved_prefixes: Vec::new(),
16254 protocol: ModuleProtocol::Subc,
16255 overlap: Default::default(),
16256 carry_containment: false,
16257 };
16258 let module = supervisor
16259 .supervise_configured(spec.clone(), false)
16260 .unwrap();
16261 let runtime = supervisor.runtime_config();
16262 let handler = ControlHandler::with_forwarding(registry, Arc::clone(&forwarding))
16263 .with_supervisor(supervisor_handle);
16264 let module_connection = ConnectionId::new(700);
16265 let (module_tx, module_rx) = mpsc::channel(8);
16266 forwarding
16267 .register_module_connection(
16268 module_connection,
16269 spec.module_id.clone(),
16270 subc_protocol::PROTOCOL_VERSION,
16271 Concurrency::ModuleManaged,
16272 FrameSink::new(module_tx),
16273 )
16274 .unwrap();
16275
16276 ProbeHarness {
16277 spec,
16278 runtime,
16279 forwarding,
16280 module_connection,
16281 module_rx,
16282 handler,
16283 module,
16284 }
16285 }
16286
16287 async fn finish_after(
16288 harness: &mut ProbeHarness,
16289 stall: Duration,
16290 ) -> ModuleControlRpcCompletion {
16291 assert!(stall > harness.runtime.health.deadline);
16292 let deadline = harness.runtime.health.deadline;
16293 let probe = probe_module_health(&harness.spec.module_id, &harness.runtime, None);
16294 let answer = async {
16295 let frame = harness.module_rx.recv().await.expect("health.check frame");
16296 tokio::time::advance(deadline).await;
16297 tokio::task::yield_now().await;
16298 tokio::time::advance(stall - deadline).await;
16299 harness
16300 .forwarding
16301 .complete_module_control_rpc(
16302 harness.module_connection,
16303 frame.header.corr,
16304 Some("health.check"),
16305 ModuleControlRpcOutcome::Response(ModuleControlResponse::HealthCheck {
16306 status: HealthStatus::Ok,
16307 detail: None,
16308 metrics: None,
16309 }),
16310 )
16311 .unwrap()
16312 };
16313 let (probe_result, completion) = tokio::join!(probe, answer);
16314 let err = probe_result.expect_err("probe must miss its deadline");
16315 assert!(matches!(err.evidence, HealthProbeEvidence::NoAnswer));
16316 completion
16317 }
16318
16319 async fn time_out_without_answer(harness: &mut ProbeHarness) {
16320 let deadline = harness.runtime.health.deadline;
16321 let probe = probe_module_health(&harness.spec.module_id, &harness.runtime, None);
16322 let exhaust_deadline = async {
16323 let _frame = harness.module_rx.recv().await.expect("health.check frame");
16324 tokio::time::advance(deadline).await;
16325 tokio::task::yield_now().await;
16326 };
16327 let (probe_result, ()) = tokio::join!(probe, exhaust_deadline);
16328 let err = probe_result.expect_err("probe must miss its deadline");
16329 assert!(matches!(err.evidence, HealthProbeEvidence::NoAnswer));
16330 }
16331
16332 async fn run_probe_cycle(harness: &mut ProbeHarness, answer: bool) {
16333 let registry = Arc::clone(&harness.module.inner.registry);
16334 let snapshot = Arc::clone(&harness.module.inner.snapshot);
16335 let process_liveness = super::SupervisorProcessLiveness::default();
16336 let mut child = None;
16337 let cycle = super::run_health_probe_cycle(
16338 &harness.spec,
16339 &harness.runtime,
16340 ®istry,
16341 &process_liveness,
16342 &snapshot,
16343 &mut child,
16344 );
16345 let peer = async {
16346 let frame = harness.module_rx.recv().await.expect("health.check frame");
16347 if answer {
16348 harness
16349 .forwarding
16350 .complete_module_control_rpc(
16351 harness.module_connection,
16352 frame.header.corr,
16353 Some("health.check"),
16354 ModuleControlRpcOutcome::Response(ModuleControlResponse::HealthCheck {
16355 status: HealthStatus::Ok,
16356 detail: None,
16357 metrics: Some(serde_json::json!({"ready": true})),
16358 }),
16359 )
16360 .unwrap();
16361 } else {
16362 tokio::time::advance(harness.runtime.health.deadline).await;
16363 tokio::task::yield_now().await;
16364 }
16365 };
16366 tokio::join!(cycle, peer);
16367 }
16368
16369 #[tokio::test(start_paused = true)]
16370 async fn unanswered_probe_is_unknown_until_threshold_and_ok_report_recovers() {
16371 let mut harness = probe_harness();
16372 let monitor = harness.module.inner.monitor.lock().unwrap().take().unwrap();
16376 monitor.abort();
16377 let _ = monitor.await;
16378 super::update_snapshot(&harness.module.inner.snapshot, None, |state| {
16379 state.enabled = true;
16380 state.state = super::ModuleState::Running;
16381 state.process_alive = true;
16382 })
16383 .unwrap();
16384
16385 run_probe_cycle(&mut harness, true).await;
16386 assert_eq!(
16387 harness.module.status().unwrap().health.status,
16388 super::SupervisorHealthStatus::Ok
16389 );
16390
16391 for failures in 1..harness.runtime.health.failure_threshold {
16392 run_probe_cycle(&mut harness, false).await;
16393 let status = harness.module.status().unwrap();
16394 assert_eq!(status.health.status, super::SupervisorHealthStatus::Unknown);
16395 assert_eq!(status.health.consecutive_failures, failures);
16396 assert!(status.health.last_probe_ms.is_some());
16397 assert!(status.health.detail.unwrap().starts_with("[no-answer]"));
16398 assert!(status.health.metrics.is_none());
16399 assert_eq!(status.state, super::ModuleState::Running);
16400 assert!(status.process_alive);
16401 assert_eq!(status.restart_count, 0);
16402 assert_eq!(status.lifetime_restarts, 0);
16403 assert!(status.health.last_action.is_none());
16404 assert!(harness.runtime.scheduled_respawn.lock().unwrap().is_none());
16405 }
16406
16407 run_probe_cycle(&mut harness, true).await;
16408 let recovered = harness.module.status().unwrap();
16409 assert_eq!(recovered.health.status, super::SupervisorHealthStatus::Ok);
16410 assert_eq!(recovered.health.consecutive_failures, 0);
16411 assert!(recovered.health.detail.is_none());
16412 assert_eq!(
16413 recovered.health.metrics,
16414 Some(serde_json::json!({"ready": true}))
16415 );
16416 assert_eq!(recovered.lifetime_restarts, 0);
16417
16418 for failures in 1..=harness.runtime.health.failure_threshold {
16419 run_probe_cycle(&mut harness, false).await;
16420 let status = harness.module.status().unwrap();
16421 assert_eq!(status.health.consecutive_failures, failures);
16422 if failures < harness.runtime.health.failure_threshold {
16423 assert_eq!(status.health.status, super::SupervisorHealthStatus::Unknown);
16424 assert_eq!(status.state, super::ModuleState::Running);
16425 assert_eq!(status.lifetime_restarts, 0);
16426 assert!(harness.runtime.scheduled_respawn.lock().unwrap().is_none());
16427 } else {
16428 assert_eq!(
16429 status.health.status,
16430 super::SupervisorHealthStatus::Unresponsive
16431 );
16432 assert_eq!(status.state, super::ModuleState::Restarting);
16433 assert_eq!(status.restart_count, 1);
16434 assert_eq!(status.lifetime_restarts, 1);
16435 assert_eq!(status.health.last_action.as_deref(), Some("restart"));
16436 assert!(harness.runtime.scheduled_respawn.lock().unwrap().is_some());
16437 }
16438 }
16439 }
16440
16441 #[tokio::test(start_paused = true)]
16442 async fn late_health_answers_record_start_anchored_latency_for_two_stalls() {
16443 let mut harness = probe_harness();
16444
16445 let first = finish_after(&mut harness, Duration::from_secs(8)).await;
16446 let first_latency = match &first {
16447 ModuleControlRpcCompletion::LateHealthAnswer { latency, .. } => *latency,
16448 other => panic!("late answer was not retained: {other:?}"),
16449 };
16450 assert!(harness.handler.observe_module_control_completion(first));
16451
16452 let second = finish_after(&mut harness, Duration::from_secs(11)).await;
16453 let second_latency = match &second {
16454 ModuleControlRpcCompletion::LateHealthAnswer { latency, .. } => *latency,
16455 other => panic!("late answer was not retained: {other:?}"),
16456 };
16457 assert!(harness.handler.observe_module_control_completion(second));
16458
16459 assert_eq!(first_latency, Duration::from_secs(8));
16460 assert_eq!(
16461 second_latency - first_latency,
16462 Duration::from_secs(3),
16463 "latency must grow linearly with the additional stall"
16464 );
16465 let health = harness.module.status().unwrap().health;
16466 assert_eq!(health.late_answer_count, 2);
16467 assert_eq!(health.last_late_answer_latency_ms, Some(11_000));
16468 }
16469
16470 #[tokio::test(start_paused = true)]
16478 async fn late_answer_clears_the_consecutive_failure_streak() {
16479 let mut harness = probe_harness();
16480
16481 time_out_without_answer(&mut harness).await;
16483 harness
16484 .module
16485 .record_health_probe_failure_for_test("[no-answer] test miss")
16486 .unwrap();
16487 assert_eq!(
16488 harness.module.status().unwrap().health.consecutive_failures,
16489 1,
16490 "precondition: the miss must be on the streak before the late answer"
16491 );
16492
16493 let late = finish_after(&mut harness, Duration::from_secs(9)).await;
16495 assert!(matches!(
16496 late,
16497 ModuleControlRpcCompletion::LateHealthAnswer { .. }
16498 ));
16499 assert!(harness.handler.observe_module_control_completion(late));
16500
16501 let health = harness.module.status().unwrap().health;
16502 assert_eq!(
16503 health.consecutive_failures, 0,
16504 "a late answer is an answer: the streak must reset"
16505 );
16506 assert_eq!(health.late_answer_count, 1);
16507 }
16508
16509 #[tokio::test(start_paused = true)]
16510 async fn repeated_serial_probe_cycles_keep_one_tombstone_per_endpoint() {
16511 let mut harness = probe_harness();
16512
16513 for _ in 0..20 {
16514 time_out_without_answer(&mut harness).await;
16515 assert_eq!(
16516 harness.forwarding.health_probe_tombstone_count().unwrap(),
16517 1
16518 );
16519 }
16520 }
16521}
16522
16523#[cfg(test)]
16524mod child_env_tests {
16525 #[cfg(windows)]
16526 use super::{
16527 clear_current_process_facts, observed_launch_nonce_source, ModuleState, SupervisorSnapshot,
16528 };
16529 #[cfg(windows)]
16530 use std::sync::{Arc, OnceLock};
16531 #[cfg(windows)]
16532 #[test]
16533 fn windows_wire_spawn_prepares_pipe_and_keeps_phase_one_env() {
16534 let spec = spec(Vec::new());
16535 let mut command = Command::new("unused");
16536 let handoff = apply_wire_spawn_args(&mut command, &spec, None, None)
16537 .unwrap()
16538 .unwrap();
16539 let name = command
16540 .as_std()
16541 .get_envs()
16542 .find(|(key, _)| *key == OsStr::new(subc_os::launch_nonce::LAUNCH_NONCE_PIPE_ENV))
16543 .and_then(|(_, value)| value)
16544 .unwrap();
16545 assert_eq!(name, OsStr::new(handoff.name()));
16546 assert!(handoff.name().starts_with(r"\\.\pipe\subc-launch-"));
16547 assert!(command.as_std().get_envs().any(|(key, value)| key
16548 == OsStr::new(subc_os::LAUNCH_NONCE_PIPE_FALLBACK_ENV)
16549 && value == Some(OsStr::new("env"))));
16550 assert!(command
16551 .as_std()
16552 .get_envs()
16553 .any(|(key, value)| key == OsStr::new(SUBC_LAUNCH_NONCE_ENV) && value.is_some()));
16554 }
16555
16556 #[cfg(windows)]
16557 #[test]
16558 fn windows_source_reporting_tracks_consumption_and_resets_per_spawn() {
16559 let mut snapshot = SupervisorSnapshot::new(ModuleState::Running, true);
16560 snapshot.process_alive = true;
16561 snapshot.spawned_protocol = Some(ModuleProtocol::Subc);
16562 assert_eq!(
16563 observed_launch_nonce_source(&snapshot).as_deref(),
16564 Some("env")
16565 );
16566 let consumed = Arc::new(OnceLock::new());
16567 snapshot.pipe_consumed = Some(consumed.clone());
16568 assert_eq!(
16569 observed_launch_nonce_source(&snapshot).as_deref(),
16570 Some("env")
16571 );
16572 consumed.set(()).unwrap();
16573 assert_eq!(
16574 observed_launch_nonce_source(&snapshot).as_deref(),
16575 Some("pipe")
16576 );
16577 snapshot.pipe_consumed = Some(Arc::new(OnceLock::new()));
16578 assert_eq!(
16579 observed_launch_nonce_source(&snapshot).as_deref(),
16580 Some("env")
16581 );
16582 clear_current_process_facts(&mut snapshot);
16583 assert_eq!(observed_launch_nonce_source(&snapshot), None);
16584 }
16585 use super::{
16586 apply_child_env, apply_spawn_role, apply_wire_spawn_args, LaunchSpecDigest, ModuleProtocol,
16587 ModuleSpec, SpawnRole, SupervisorHandle, SPAWN_ROLE_SWAP_CANDIDATE, SUBC_ARG,
16588 SUBC_LAUNCH_NONCE_ENV, SUBC_MODULE_ID_ENV, SUBC_SPAWN_ROLE_ENV,
16589 };
16590 use std::{ffi::OsStr, path::PathBuf};
16591 use tokio::process::Command;
16592
16593 fn spec(env: Vec<(String, String)>) -> ModuleSpec {
16594 ModuleSpec {
16595 module_id: "env-plan".to_string(),
16596 program: PathBuf::from("/nonexistent"),
16597 args: Vec::new(),
16598 env,
16599 reserved: false,
16600 reserved_prefixes: Vec::new(),
16601 protocol: ModuleProtocol::Subc,
16602 overlap: Default::default(),
16603 carry_containment: false,
16604 }
16605 }
16606
16607 #[test]
16608 fn configured_launch_spec_digest_is_canonical_and_excludes_injected_inputs() {
16609 let base = spec(vec![("A".into(), "B=C".into()), ("Z".into(), "v".into())]);
16610 let digest = LaunchSpecDigest::from_spec(&base);
16611 let mut reordered = base.clone();
16612 reordered.env.reverse();
16613 reordered.module_id = "different-id".into();
16614 reordered.reserved = true;
16615 reordered.overlap = Default::default();
16616 assert_eq!(digest, LaunchSpecDigest::from_spec(&reordered));
16617 let mut delimiter_collision = base.clone();
16618 delimiter_collision.env[0] = ("A=B".into(), "C".into());
16619 assert_ne!(digest, LaunchSpecDigest::from_spec(&delimiter_collision));
16620 let mut changed = base.clone();
16621 changed.protocol = ModuleProtocol::None;
16622 assert_ne!(digest, LaunchSpecDigest::from_spec(&changed));
16623 changed = base.clone();
16624 changed.args.push("--flag".into());
16625 assert_ne!(digest, LaunchSpecDigest::from_spec(&changed));
16626 changed = base.clone();
16627 changed.program = PathBuf::from("/other");
16628 assert_ne!(digest, LaunchSpecDigest::from_spec(&changed));
16629 changed = base.clone();
16630 changed.env[1].1 = "other".into();
16631 assert_ne!(digest, LaunchSpecDigest::from_spec(&changed));
16632 assert!(!format!("{digest:?}").contains("B=C"));
16633 assert!(!format!("{digest:?}").contains("[u8"));
16634 }
16635
16636 #[cfg(unix)]
16637 #[test]
16638 fn native_program_bytes_do_not_collapse_through_lossy_utf8() {
16639 use std::os::unix::ffi::OsStringExt;
16640 let mut left = spec(Vec::new());
16641 left.program = std::ffi::OsString::from_vec(vec![b'/', 0xff]).into();
16642 let mut right = left.clone();
16643 right.program = std::ffi::OsString::from_vec(vec![b'/', 0xfe]).into();
16644 assert_eq!(
16645 left.program.to_string_lossy(),
16646 right.program.to_string_lossy()
16647 );
16648 assert_ne!(
16649 LaunchSpecDigest::from_spec(&left),
16650 LaunchSpecDigest::from_spec(&right)
16651 );
16652 }
16653
16654 #[test]
16668 fn ambient_ck_log_is_removed_and_a_configured_one_survives() {
16669 let mut command = Command::new("/nonexistent");
16670 apply_child_env(&mut command, &spec(Vec::new()));
16671 let removed = command
16672 .as_std()
16673 .get_envs()
16674 .any(|(key, value)| key == OsStr::new("CK_LOG") && value.is_none());
16675 assert!(
16676 removed,
16677 "ambient CK_LOG must be explicitly removed for an unconfigured module"
16678 );
16679
16680 let mut configured = Command::new("/nonexistent");
16681 apply_child_env(
16682 &mut configured,
16683 &spec(vec![("CK_LOG".to_string(), "debug".to_string())]),
16684 );
16685 let effective = configured
16686 .as_std()
16687 .get_envs()
16688 .filter(|(key, _)| *key == OsStr::new("CK_LOG"))
16689 .last()
16690 .map(|(_, value)| value.map(|v| v.to_string_lossy().into_owned()));
16691 assert_eq!(
16692 effective,
16693 Some(Some("debug".to_string())),
16694 "a module's configured CK_LOG must survive the ambient removal"
16695 );
16696 }
16697
16698 #[test]
16707 fn protocol_none_spawn_carries_no_subc_argument_and_no_nonce() {
16708 let connection_file = std::path::Path::new("/run/subc-connection.json");
16709 let handle = SupervisorHandle::new();
16710
16711 let mut none_spec = spec(Vec::new());
16712 none_spec.protocol = ModuleProtocol::None;
16713 let mut none = Command::new("/nonexistent");
16714 let none_handoff =
16715 apply_wire_spawn_args(&mut none, &none_spec, Some(connection_file), Some(&handle))
16716 .expect("protocol-none spawn args apply");
16717 assert!(
16718 none_handoff.is_none(),
16719 "protocol:none spawn must not receive a nonce descriptor"
16720 );
16721 #[cfg(windows)]
16722 assert!(!none.as_std().get_envs().any(|(key, value)| key
16723 == OsStr::new(subc_os::LAUNCH_NONCE_PIPE_FALLBACK_ENV)
16724 && value.is_some()));
16725 assert!(
16726 !none.as_std().get_envs().any(|(key, value)| key
16727 == OsStr::new(subc_os::LAUNCH_NONCE_FD_ENV)
16728 && value.is_some()),
16729 "protocol:none spawn must not name a nonce descriptor"
16730 );
16731 let none_args: Vec<String> = none
16732 .as_std()
16733 .get_args()
16734 .map(|a| a.to_string_lossy().into_owned())
16735 .collect();
16736 assert!(
16737 !none_args.iter().any(|a| a == SUBC_ARG),
16738 "protocol:none argv must not carry --subc; got {none_args:?}"
16739 );
16740 let none_has_nonce = none
16741 .as_std()
16742 .get_envs()
16743 .any(|(key, value)| key == OsStr::new(SUBC_LAUNCH_NONCE_ENV) && value.is_some());
16744 assert!(
16745 !none_has_nonce,
16746 "protocol:none spawn must not receive a launch nonce"
16747 );
16748 let none_has_module_id = none
16749 .as_std()
16750 .get_envs()
16751 .any(|(key, value)| key == OsStr::new(SUBC_MODULE_ID_ENV) && value.is_some());
16752 assert!(
16753 none_has_module_id,
16754 "SUBC_MODULE_ID is inert and stays on every path"
16755 );
16756 assert!(
16757 handle.spawn_nonce(&none_spec.module_id).is_none(),
16758 "no nonce record for a process that will never present one"
16759 );
16760
16761 let wire_spec = spec(Vec::new());
16763 let mut wire = Command::new("/nonexistent");
16764 let wire_handoff =
16765 apply_wire_spawn_args(&mut wire, &wire_spec, Some(connection_file), Some(&handle))
16766 .expect("subc-wire spawn args apply");
16767 let wire_fd_env = wire
16768 .as_std()
16769 .get_envs()
16770 .find(|(key, _)| *key == OsStr::new(subc_os::LAUNCH_NONCE_FD_ENV))
16771 .and_then(|(_, value)| value.map(|v| v.to_string_lossy().into_owned()));
16772 #[cfg(unix)]
16773 assert_eq!(
16774 wire_fd_env,
16775 Some(wire_handoff.expect("a descriptor handoff").fd_env_value()),
16776 "a subc-wire spawn names the pipe it will receive at descriptor 3"
16777 );
16778 #[cfg(windows)]
16779 assert!(wire_handoff.is_some() && wire_fd_env.is_none());
16780 #[cfg(not(any(unix, windows)))]
16781 assert!(wire_handoff.is_none() && wire_fd_env.is_none());
16782 let wire_args: Vec<String> = wire
16783 .as_std()
16784 .get_args()
16785 .map(|a| a.to_string_lossy().into_owned())
16786 .collect();
16787 assert_eq!(
16788 wire_args,
16789 vec![
16790 SUBC_ARG.to_string(),
16791 connection_file.to_string_lossy().into_owned()
16792 ],
16793 "a subc-wire spawn still carries --subc <path>"
16794 );
16795 assert_eq!(
16796 wire.as_std()
16797 .get_envs()
16798 .any(|(key, value)| key == OsStr::new(SUBC_LAUNCH_NONCE_ENV) && value.is_some()),
16799 !cfg!(unix),
16800 "only Windows supplies the environment nonce"
16801 );
16802 assert!(handle.spawn_nonce(&wire_spec.module_id).is_some());
16803 }
16804
16805 #[test]
16815 fn plain_spawn_removes_the_spawn_role_even_when_the_spec_sets_it() {
16816 let role = |command: &Command| {
16817 command
16818 .as_std()
16819 .get_envs()
16820 .filter(|(key, _)| *key == OsStr::new(SUBC_SPAWN_ROLE_ENV))
16821 .last()
16822 .map(|(_, value)| value.map(|v| v.to_string_lossy().into_owned()))
16823 };
16824 let forged = spec(vec![(
16825 SUBC_SPAWN_ROLE_ENV.to_string(),
16826 SPAWN_ROLE_SWAP_CANDIDATE.to_string(),
16827 )]);
16828
16829 let mut plain = Command::new("/nonexistent");
16830 apply_child_env(&mut plain, &forged);
16831 apply_spawn_role(&mut plain, SpawnRole::Plain);
16832 assert_eq!(
16833 role(&plain),
16834 Some(None),
16835 "a plain spawn must remove SUBC_SPAWN_ROLE, whatever the spec says"
16836 );
16837
16838 let mut candidate = Command::new("/nonexistent");
16839 apply_child_env(&mut candidate, &spec(Vec::new()));
16840 apply_spawn_role(&mut candidate, SpawnRole::SwapCandidate);
16841 assert_eq!(
16842 role(&candidate),
16843 Some(Some(SPAWN_ROLE_SWAP_CANDIDATE.to_string()))
16844 );
16845 }
16846
16847 #[test]
16853 fn daemon_private_capture_keys_are_not_passed_to_the_child() {
16854 let mut command = Command::new("/nonexistent");
16855 apply_child_env(
16856 &mut command,
16857 &spec(vec![
16858 (super::CAPTURE_KEEP_ENV.to_string(), "5".to_string()),
16859 ("KEPT".to_string(), "yes".to_string()),
16860 ]),
16861 );
16862 let keys: Vec<String> = command
16863 .as_std()
16864 .get_envs()
16865 .filter(|(_, value)| value.is_some())
16866 .map(|(key, _)| key.to_string_lossy().into_owned())
16867 .collect();
16868 assert!(keys.contains(&"KEPT".to_string()), "got {keys:?}");
16869 assert!(
16870 !keys.contains(&super::CAPTURE_KEEP_ENV.to_string()),
16871 "daemon-private capture key leaked to the child: {keys:?}"
16872 );
16873 }
16874}
16875
16876#[cfg(test)]
16877mod jitter_tests {
16878 use super::jittered_health_delay;
16879 use std::{collections::HashSet, time::Duration};
16880
16881 const FLEET: [&str; 14] = [
16890 "aft",
16891 "alfonso-core",
16892 "magic-context",
16893 "broca",
16894 "thalamus",
16895 "quota",
16896 "engram",
16897 "plexus",
16898 "cerebellum",
16899 "astrocyte",
16900 "synapse",
16901 "subc-mcp",
16902 "cortexkit-credentials",
16903 "subc-federation",
16904 ];
16905
16906 #[test]
16914 fn probe_delays_disperse_across_the_fleet() {
16915 let cadence = Duration::from_secs(30);
16916 let delays: HashSet<Duration> = FLEET
16917 .iter()
16918 .map(|id| jittered_health_delay(id, 0, cadence))
16919 .collect();
16920 assert_eq!(
16921 delays.len(),
16922 FLEET.len(),
16923 "every supervised module must land on its own probe offset"
16924 );
16925 }
16926
16927 #[test]
16933 fn jitter_only_delays_and_stays_within_one_tenth_of_cadence() {
16934 let cadence = Duration::from_secs(30);
16935 let span = cadence / 10;
16936 for id in FLEET {
16937 for probe_index in 0..8 {
16938 let delay = jittered_health_delay(id, probe_index, cadence);
16939 assert!(
16940 delay >= cadence,
16941 "{id}#{probe_index}: jitter must not shorten the cadence"
16942 );
16943 assert!(
16944 delay < cadence + span,
16945 "{id}#{probe_index}: jitter must stay inside one tenth of the cadence"
16946 );
16947 }
16948 }
16949 }
16950
16951 #[test]
16957 fn a_module_offset_is_stable_across_restarts() {
16958 let cadence = Duration::from_secs(30);
16959 for id in FLEET {
16960 assert_eq!(
16961 jittered_health_delay(id, 0, cadence),
16962 jittered_health_delay(id, 0, cadence),
16963 "{id}: the same module and probe index must produce the same offset"
16964 );
16965 }
16966 }
16967
16968 #[test]
16970 fn zero_cadence_yields_zero_delay() {
16971 assert_eq!(
16972 jittered_health_delay("aft", 0, Duration::ZERO),
16973 Duration::ZERO
16974 );
16975 }
16976}
16977
16978#[cfg(all(test, target_os = "linux"))]
16979mod cgroup_placement_tests {
16980 use super::{
16981 apply_cgroup_placement, remove_module_cgroup, LaunchSpecDigest, ModuleProtocol, ModuleSpec,
16982 SuperviseError, SupervisedChild,
16983 };
16984 use crate::stderr_tail::{StderrRing, StderrTailConfig};
16985 use cortexkit_test_support::ScratchDir;
16986 use std::{
16987 fs, io,
16988 path::{Path, PathBuf},
16989 sync::{Arc, Mutex},
16990 };
16991 use tokio::process::Command;
16992
16993 #[tokio::test]
16994 async fn unique_spawn_cgroups_clean_up_across_restarts_and_shutdown() {
16995 use super::*;
16996 let dir = ScratchDir::new("unique-spawn-cgroups");
16997 let root = PathBuf::from(format!(
16998 "/sys/fs/cgroup/subc-unique-test-{}-{}",
16999 std::process::id(),
17000 unix_ms_now()
17001 ));
17002 if let Err(error) = fs::create_dir(&root) {
17003 assert!(
17004 std::env::var_os("SUBC_REQUIRE_CGROUP_TEST").is_none(),
17005 "required cgroup test cannot execute: {error}"
17006 );
17007 eprintln!("SKIP unique_spawn_cgroups_clean_up_across_restarts_and_shutdown: {error}");
17008 return;
17009 }
17010 let placement = subc_cgroup::prepare_at(&root).unwrap().unwrap();
17011 let group_count = || {
17012 fs::read_dir(root.join("subc-modules"))
17013 .unwrap()
17014 .map(|entry| entry.unwrap().file_type().unwrap())
17015 .filter(|kind| kind.is_dir())
17016 .count()
17017 };
17018 let supervisor =
17019 Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default())
17020 .with_cgroup_placement(Some(placement.clone()));
17021 let runtime = supervisor.runtime_config();
17022 let mut spec = ModuleSpec {
17023 module_id: "unique-spawn".into(),
17024 program: PathBuf::from("/bin/sleep"),
17025 args: vec!["60".into()],
17026 env: ["XDG_DATA_HOME", "XDG_RUNTIME_DIR", "XDG_CONFIG_HOME"]
17027 .into_iter()
17028 .map(|key| (key.into(), dir.to_string_lossy().into_owned()))
17029 .collect(),
17030 reserved: false,
17031 reserved_prefixes: vec![],
17032 protocol: ModuleProtocol::None,
17033 overlap: Default::default(),
17034 carry_containment: false,
17035 };
17036 let spawn = |spec: &ModuleSpec| {
17037 spawn_child(
17038 spec,
17039 None,
17040 None,
17041 &runtime.stderr_ring,
17042 None,
17043 &runtime.child_roster,
17044 Some(&placement),
17045 )
17046 .unwrap()
17047 };
17048 let mut live = spawn(&spec);
17049 for _ in 0..3 {
17050 let next = spawn(&spec);
17052 assert_ne!(live.module_id, next.module_id);
17053 live.start_kill().unwrap();
17054 live.wait().await.unwrap();
17055 live = next;
17056 assert!(
17057 live.child.try_wait().unwrap().is_none(),
17058 "retiring the old slot must not kill the replacement"
17059 );
17060 assert_eq!(
17061 group_count(),
17062 1,
17063 "only the live spawn's cgroup should remain"
17064 );
17065 }
17066 supervisor.begin_daemon_shutdown();
17067 let reap = tokio::spawn(async move {
17068 live.wait().await.unwrap();
17069 });
17070 supervisor
17071 .end_children_for_daemon_shutdown(false, std::future::pending())
17072 .await;
17073 reap.await.unwrap();
17074 assert_eq!(group_count(), 0);
17075 spec.program = PathBuf::from("/bin/true");
17077 spec.args.clear();
17078 let fresh_roster = ChildRoster::default();
17079 let mut short = spawn_child(
17080 &spec,
17081 None,
17082 None,
17083 &runtime.stderr_ring,
17084 None,
17085 &fresh_roster,
17086 Some(&placement),
17087 )
17088 .unwrap();
17089 short.wait().await.unwrap();
17090 assert_eq!(group_count(), 0);
17091 spec.module_id = "_".repeat(255);
17092 let mut long_id = spawn_child(
17093 &spec,
17094 None,
17095 None,
17096 &runtime.stderr_ring,
17097 None,
17098 &fresh_roster,
17099 Some(&placement),
17100 )
17101 .unwrap();
17102 long_id.wait().await.unwrap();
17103 assert_eq!(
17104 group_count(),
17105 0,
17106 "valid long module IDs must not exceed cgroup NAME_MAX"
17107 );
17108 fs::remove_dir(root.join("subc-modules")).unwrap();
17109 fs::remove_dir(root).unwrap();
17110 eprintln!("EXECUTED unique_spawn_cgroups_clean_up_across_restarts_and_shutdown");
17111 }
17112
17113 #[test]
17114 fn failed_parent_cgroup_open_is_a_cgroup_supervision_error() {
17115 let path = Path::new("/definitely-missing-subc-cgroup");
17116 let mut command = Command::new("true");
17117 let error = apply_cgroup_placement(
17118 &mut command,
17119 &ModuleSpec {
17120 module_id: "broken-cgroup".to_string(),
17121 program: PathBuf::from("true"),
17122 args: Vec::new(),
17123 env: Vec::new(),
17124 reserved: false,
17125 reserved_prefixes: Vec::new(),
17126 protocol: ModuleProtocol::Subc,
17127 overlap: Default::default(),
17128 carry_containment: false,
17129 },
17130 path,
17131 )
17132 .expect_err("a parent cgroup open failure must reject the supervised spawn");
17133 let reason = error.to_string();
17134
17135 assert!(
17136 matches!(error, SuperviseError::Cgroup { .. }),
17137 "parent cgroup open must be reported as a cgroup supervision error: {reason}"
17138 );
17139 assert!(
17140 reason.contains("/definitely-missing-subc-cgroup/cgroup.procs"),
17141 "parent cgroup open failure must name cgroup.procs: {reason}"
17142 );
17143 }
17144
17145 #[tokio::test]
17146 async fn reaping_a_child_removes_its_empty_module_cgroup() {
17147 let root = ScratchDir::new("supervisor-reap-cgroup");
17148 fs::write(root.join("cgroup.procs"), b"").expect("write scratch cgroup marker");
17149 let placement = subc_cgroup::prepare_at(&root)
17150 .expect("prepare scratch cgroup root")
17151 .expect("scratch root has a cgroup.procs marker");
17152 let module_id = "reaped-module";
17153 let module = placement
17154 .module_path(module_id)
17155 .expect("create scratch module cgroup");
17156 let child = Command::new("true")
17157 .env("XDG_DATA_HOME", root.path())
17158 .env("XDG_RUNTIME_DIR", root.path())
17159 .env("XDG_CONFIG_HOME", root.path())
17160 .spawn()
17161 .expect("spawn short-lived child");
17162 let pid = child.id().expect("spawned child has pid");
17163 let mut child = SupervisedChild {
17164 subtree_target: crate::subtree_resources::Target::Unavailable("not_running"),
17165 child: crate::supervise::containment::ProcessChild::plain(child, pid),
17166 exec: crate::activity::ExecOwner(Arc::new(crate::activity::ExecOutcome::new())),
17167 protocol: ModuleProtocol::Subc,
17168 module_id: module_id.to_string(),
17169 cgroup_placement: Some(placement),
17170 stdout_pump: None,
17171 stderr_pump: None,
17172 stderr_ring: Arc::new(Mutex::new(StderrRing::new(StderrTailConfig::default()))),
17173 spawned_at: tokio::time::Instant::now(),
17174 registration_deadline: None,
17175 registration_watch: None,
17176 spawned_at_ms: 0,
17177 spawned_from: PathBuf::from("true"),
17178 launch_spec_digest: LaunchSpecDigest([0; 32]),
17179 spawned_file_identity: None,
17180 process_start_time: None,
17181 process_identity: None,
17182 pid,
17183 roster_guard: None,
17184 #[cfg(target_os = "macos")]
17185 privacy_exec: None,
17186 spawn_failure: None,
17187 };
17188
17189 child.wait().await.expect("reap short-lived child");
17190
17191 assert!(
17192 !module.exists(),
17193 "waiting for a non-carried child must remove its empty cgroup before returning"
17194 );
17195 child.finish_exit(false).await.unwrap();
17196 }
17197
17198 #[test]
17199 fn non_empty_cgroup_removal_is_reported_without_blocking_teardown() {
17200 let root = ScratchDir::new("supervisor-non-empty-cgroup");
17201 fs::write(root.join("cgroup.procs"), b"").expect("write scratch cgroup marker");
17202 let placement = subc_cgroup::prepare_at(&root)
17203 .expect("prepare scratch cgroup root")
17204 .expect("scratch root has a cgroup.procs marker");
17205 let module = placement
17206 .module_path("surviving-module")
17207 .expect("create scratch module cgroup");
17208 fs::write(module.join("surviving-process"), b"still present")
17209 .expect("make scratch cgroup non-empty");
17210 let (logs, _guard) = crate::router::test_log::log_capture(tracing::Level::WARN);
17211
17212 remove_module_cgroup(&placement, "surviving-module");
17213
17214 let logs = crate::router::test_log::captured_logs(&logs);
17215 assert!(
17216 module.exists(),
17217 "failed removal must leave the cgroup intact"
17218 );
17219 assert!(
17220 logs.contains("could not remove module cgroup after process exit; continuing teardown")
17221 && logs.contains("surviving-module"),
17222 "best-effort removal must report the failure without returning it: {logs}"
17223 );
17224 }
17225
17226 #[test]
17227 fn cgroup_pre_exec_spawn_failure_names_the_cgroup_path() {
17228 let cgroup_path = PathBuf::from("/sys/fs/cgroup/subc-modules/broken-module");
17229 let reason = SuperviseError::Spawn {
17230 program: PathBuf::from("/bin/true"),
17231 source: io::Error::from_raw_os_error(13),
17232 cgroup_path: Some(cgroup_path.clone()),
17233 }
17234 .to_string();
17235
17236 assert!(
17237 reason.contains(&cgroup_path.display().to_string()),
17238 "a pre_exec spawn failure must name the cgroup path: {reason}"
17239 );
17240 }
17241}
17242
17243#[cfg(test)]
17244mod spawn_subscriber_lag_tests {
17245 use super::*;
17246
17247 #[tokio::test]
17252 async fn lagged_spawn_subscriber_receives_a_terminal_lagged_error_after_its_queued_frames() {
17253 let feed = SpawnEventFeed::default();
17254 feed.configure_incarnation("lag-incarnation".to_string());
17255 let (tx, mut rx) = mpsc::channel(1);
17258 feed.subscribe(ConnectionId::new(1), 7, 1, None, FrameSink::new(tx))
17259 .expect("subscribe");
17260 let emitted = SPAWN_SUBSCRIBER_BUFFER + 16;
17261 for index in 0..emitted {
17262 feed.emit_spawned(&format!("lag-module-{index}"), 1000, 0);
17263 tokio::task::yield_now().await;
17266 }
17267 assert_eq!(
17268 feed.subscriber_count(),
17269 0,
17270 "the lagged subscriber must be removed"
17271 );
17272
17273 let mut data = Vec::new();
17274 let mut last = None;
17275 loop {
17276 let next = tokio::time::timeout(Duration::from_secs(5), rx.recv())
17277 .await
17278 .expect("the forwarder must finish once the subscriber is dropped");
17279 let Some(outbound) = next else { break };
17280 let frame = outbound.frame;
17281 if frame.header.ty == FrameType::StreamData {
17282 assert!(last.is_none(), "no data may follow the terminal frame");
17283 let event: SpawnEvent = serde_json::from_slice(&frame.body).unwrap();
17284 data.push(event.cursor.seq);
17285 } else {
17286 assert!(last.is_none(), "exactly one terminal frame");
17287 last = Some(frame);
17288 }
17289 }
17290 assert!(!data.is_empty(), "queued frames drain before the terminal");
17291 for pair in data.windows(2) {
17292 assert_eq!(
17293 pair[1],
17294 pair[0] + 1,
17295 "queued frames arrive dense and in order"
17296 );
17297 }
17298 let terminal = last.expect("a lagged subscriber must receive a terminal frame");
17299 assert_eq!(terminal.header.ty, FrameType::Error);
17300 assert_eq!(terminal.header.corr, 7);
17301 let body: subc_protocol::ErrorBody = serde_json::from_slice(&terminal.body).unwrap();
17302 assert_eq!(body.code, SPAWN_SUBSCRIBER_LAGGED_CODE);
17303 let detail = body.detail.expect("lagged error carries detail");
17304 assert_eq!(
17305 detail["first_undelivered_cursor"]["seq"],
17306 data.last().unwrap() + 1,
17307 "the named cursor is the first event the subscriber did not receive"
17308 );
17309 assert_eq!(
17310 detail["first_undelivered_cursor"]["daemon_incarnation"],
17311 "lag-incarnation"
17312 );
17313 }
17314}
17315
17316#[cfg(test)]
17317mod terminal_history_read_concurrency_tests {
17318 use super::*;
17319 use crate::terminal_journal::read_pause;
17320 use cortexkit_test_support::ScratchDir;
17321 use std::sync::mpsc as std_mpsc;
17322
17323 fn journaled_ring(
17324 journal: &Arc<crate::terminal_journal::TerminalJournal>,
17325 ) -> Arc<Mutex<TerminalRing>> {
17326 Arc::new(Mutex::new(
17327 TerminalRing::new(TerminalRingConfig::default(), 1)
17328 .with_journal(Some(Arc::clone(journal))),
17329 ))
17330 }
17331
17332 fn crash(at_ms: u64) -> ExitReport {
17333 ExitReport {
17334 kind: ExitKind::Crash,
17335 code: Some(1),
17336 signal: None,
17337 at_ms,
17338 }
17339 }
17340
17341 fn record_within(
17344 module_id: &'static str,
17345 ring: &Arc<Mutex<TerminalRing>>,
17346 at_ms: u64,
17347 bound: Duration,
17348 ) -> bool {
17349 let ring = Arc::clone(ring);
17350 let (done, done_rx) = std_mpsc::channel();
17351 std::thread::spawn(move || {
17352 record_terminal(
17353 module_id,
17354 &ring,
17355 &SpawnEventFeed::default(),
17356 &crash(at_ms),
17357 TerminalDisposition::Restarting,
17358 );
17359 let _ = done.send(());
17360 });
17361 done_rx.recv_timeout(bound).is_ok()
17362 }
17363
17364 #[test]
17369 fn exits_recorded_during_a_paused_history_read_are_not_blocked_or_half_merged() {
17370 let dir = ScratchDir::new("terminal-history-concurrent-read");
17371 let path = dir.join("terminals.jsonl");
17372 let journal = Arc::new(crate::terminal_journal::TerminalJournal::open(
17373 path.clone(),
17374 "daemon".into(),
17375 ));
17376 let reader_ring = journaled_ring(&journal);
17377 let other_ring = journaled_ring(&journal);
17378 assert!(record_within(
17379 "reader-module",
17380 &reader_ring,
17381 10,
17382 Duration::from_secs(5)
17383 ));
17384
17385 let (started, release) = read_pause::install(&path);
17386 let reading = {
17387 let ring = Arc::clone(&reader_ring);
17388 std::thread::spawn(move || durable_terminal_history_of(&ring, "reader-module"))
17389 };
17390 started
17391 .recv_timeout(Duration::from_secs(5))
17392 .expect("the history read reached its pause");
17393
17394 let bound = Duration::from_secs(1);
17395 assert!(
17396 record_within("other-module", &other_ring, 20, bound),
17397 "another module's exit waited on a history read (journal writer held)"
17398 );
17399 assert!(
17400 record_within("reader-module", &reader_ring, 30, bound),
17401 "the read module's own exit waited on its history read (ring held)"
17402 );
17403
17404 drop(release);
17405 let paused = reading.join().unwrap();
17406 assert_eq!(
17407 paused.entries.iter().map(|e| e.at_ms).collect::<Vec<_>>(),
17408 vec![10],
17409 "an exit recorded after the read began lands in neither half of it"
17410 );
17411 assert_eq!(paused.journal_skipped_lines, 0);
17412 assert_eq!(paused.journal_read_errors, 0);
17413
17414 let after = durable_terminal_history_of(&reader_ring, "reader-module");
17415 assert_eq!(
17416 after.entries.iter().map(|e| e.at_ms).collect::<Vec<_>>(),
17417 vec![10, 30],
17418 "the next read merges ring and journal with no duplicate"
17419 );
17420 assert_eq!(after.journal_skipped_lines, 0);
17421 }
17422}
17423
17424#[cfg(test)]
17429mod stderr_settle_tests {
17430 use std::{
17431 future::Future,
17432 io,
17433 pin::Pin,
17434 sync::{Arc, Mutex},
17435 task::{Context, Poll},
17436 time::Duration,
17437 };
17438
17439 use tokio::{
17440 io::{AsyncRead, ReadBuf},
17441 sync::oneshot,
17442 time::Instant,
17443 };
17444
17445 use super::{settle_stderr_pump, settle_stdout_pump, StderrPump};
17446 use crate::stderr_tail::{
17447 pump_stderr_to, pump_stdout_to, split_capture_stamp, untimed, CaptureState,
17448 ChildOutputSink, OutputSink, StderrRing, StderrTailConfig, TailEntry,
17449 };
17450
17451 const BOUND: Duration = Duration::from_millis(250);
17452
17453 struct HeldReader {
17457 before: Option<Vec<u8>>,
17458 gate: Option<oneshot::Receiver<()>>,
17459 after: io::Cursor<Vec<u8>>,
17460 }
17461
17462 impl AsyncRead for HeldReader {
17463 fn poll_read(
17464 mut self: Pin<&mut Self>,
17465 cx: &mut Context<'_>,
17466 buf: &mut ReadBuf<'_>,
17467 ) -> Poll<io::Result<()>> {
17468 if let Some(bytes) = self.before.take() {
17469 buf.put_slice(&bytes);
17470 return Poll::Ready(Ok(()));
17471 }
17472 if let Some(gate) = self.gate.as_mut() {
17473 match Pin::new(gate).poll(cx) {
17474 Poll::Pending => return Poll::Pending,
17475 Poll::Ready(_) => self.gate = None,
17476 }
17477 }
17478 Pin::new(&mut self.after).poll_read(cx, buf)
17479 }
17480 }
17481
17482 struct DiscardSink;
17483
17484 impl OutputSink for DiscardSink {
17485 fn write_line(&mut self, _line: &[u8]) {}
17486 }
17487
17488 fn line(text: &str) -> TailEntry {
17489 TailEntry::Line {
17490 text: text.to_string(),
17491 truncated: false,
17492 at_ms: None,
17493 }
17494 }
17495
17496 fn lock(ring: &Arc<Mutex<StderrRing>>) -> std::sync::MutexGuard<'_, StderrRing> {
17497 ring.lock().unwrap()
17498 }
17499
17500 fn held_pump(
17504 ring: &Arc<Mutex<StderrRing>>,
17505 before: &str,
17506 after: &str,
17507 ) -> (StderrPump, oneshot::Sender<()>) {
17508 let generation = lock(ring).begin_process();
17509 let (release, gate) = oneshot::channel();
17510 let reader = HeldReader {
17511 before: Some(before.as_bytes().to_vec()),
17512 gate: Some(gate),
17513 after: io::Cursor::new(after.as_bytes().to_vec()),
17514 };
17515 let task = tokio::spawn(pump_stderr_to(
17516 reader,
17517 Arc::clone(ring),
17518 generation,
17519 DiscardSink,
17520 ));
17521 (StderrPump { task, generation }, release)
17522 }
17523
17524 async fn wait_until(ring: &Arc<Mutex<StderrRing>>, done: impl Fn(&StderrRing) -> bool) {
17525 for _ in 0..1000 {
17526 if done(&lock(ring)) {
17527 return;
17528 }
17529 tokio::time::sleep(Duration::from_millis(1)).await;
17530 }
17531 panic!(
17532 "ring never reached the expected state: {:?}",
17533 lock(ring).snapshot(None, None)
17534 );
17535 }
17536
17537 #[tokio::test(start_paused = true)]
17538 async fn a_crash_line_the_reader_had_not_reached_by_the_bound_is_kept_before_the_restart() {
17539 let ring = Arc::new(Mutex::new(StderrRing::new(StderrTailConfig::default())));
17540 let (pump, release) = held_pump(&ring, "booting\n", "config error: missing storage\n");
17541
17542 settle_stderr_pump("crasher", &ring, pump, BOUND).await;
17543 let before_release = lock(&ring).snapshot(None, None);
17544 assert!(
17545 matches!(before_release.capture, CaptureState::Incomplete { .. }),
17546 "a reader that has not reached EOF cannot claim a whole tail: {before_release:?}"
17547 );
17548
17549 let next = lock(&ring).begin_process();
17552 lock(&ring).push_line_from(next, "next process booting");
17553 release.send(()).unwrap();
17554 wait_until(&ring, |ring| {
17555 ring.snapshot(None, None).capture == CaptureState::Captured
17556 })
17557 .await;
17558
17559 assert_eq!(
17560 untimed(lock(&ring).snapshot(None, None).entries),
17561 vec![
17562 line("booting"),
17563 line("config error: missing storage"),
17564 TailEntry::ProcessStart,
17565 line("next process booting"),
17566 ],
17567 "the crash's last line must survive a slow reader and stay in the crashed process's section"
17568 );
17569 }
17570
17571 #[tokio::test(start_paused = true)]
17572 async fn a_pipe_held_open_by_a_descendant_reads_incomplete_without_delaying_the_restart_past_the_bound(
17573 ) {
17574 let ring = Arc::new(Mutex::new(StderrRing::new(StderrTailConfig::default())));
17575 let (pump, _held) = held_pump(&ring, "parent exiting\n", "");
17578
17579 let started = Instant::now();
17580 settle_stderr_pump("orphaning", &ring, pump, BOUND).await;
17581 assert_eq!(
17582 started.elapsed(),
17583 BOUND,
17584 "the restart must wait exactly the bound for a pipe that stays open, no longer"
17585 );
17586
17587 let next = lock(&ring).begin_process();
17588 lock(&ring).push_line_from(next, "next process booting");
17589 tokio::time::sleep(Duration::from_secs(60)).await;
17590
17591 let snapshot = lock(&ring).snapshot(None, None);
17592 match &snapshot.capture {
17593 CaptureState::Incomplete { reason } => assert!(
17594 reason.contains("had not reached EOF") && reason.contains("250ms"),
17595 "the reason must say what is missing and after how long: {reason}"
17596 ),
17597 other => panic!("expected Incomplete while the pipe is held open, got {other:?}"),
17598 }
17599 assert_eq!(
17600 untimed(snapshot.entries),
17601 vec![
17602 line("parent exiting"),
17603 TailEntry::ProcessStart,
17604 line("next process booting"),
17605 ]
17606 );
17607 }
17608
17609 #[tokio::test(start_paused = true)]
17610 async fn a_reader_that_reaches_eof_within_the_bound_leaves_the_tail_captured() {
17611 let ring = Arc::new(Mutex::new(StderrRing::new(StderrTailConfig::default())));
17612 let (pump, release) = held_pump(&ring, "one\n", "two\n");
17613 release.send(()).unwrap();
17614
17615 settle_stderr_pump("clean", &ring, pump, BOUND).await;
17616
17617 let snapshot = lock(&ring).snapshot(None, None);
17618 assert_eq!(snapshot.capture, CaptureState::Captured);
17619 assert_eq!(untimed(snapshot.entries), vec![line("one"), line("two")]);
17620 }
17621
17622 #[tokio::test(start_paused = true)]
17623 async fn stdout_buffered_past_the_exit_drain_bound_reaches_the_capture_file() {
17624 const BURST: usize = 150;
17625 let capture = cortexkit_test_support::ScratchDir::new("stdout-exit-drain");
17626 let path = capture.join("two-pipe.stderr.log");
17627 let sink = ChildOutputSink::open(&path, cortexkit_log::Retention::default()).unwrap();
17628 let ring = Arc::new(Mutex::new(StderrRing::new(StderrTailConfig::default())));
17629 let generation = lock(&ring).begin_process();
17630 let stdout = (0..BURST)
17631 .map(|i| format!("out-{i:04}-{}\n", "o".repeat(64)))
17632 .collect::<String>();
17633 let stderr = (0..BURST)
17634 .map(|i| format!("err-{i:04}-{}\n", "e".repeat(64)))
17635 .collect::<String>();
17636 let first_chunk = stdout.find('\n').unwrap() + 1 + "out-0001-".len();
17641 let (release, gate) = oneshot::channel();
17642 let reader = HeldReader {
17643 before: Some(stdout.as_bytes()[..first_chunk].to_vec()),
17644 gate: Some(gate),
17645 after: io::Cursor::new(stdout.as_bytes()[first_chunk..stdout.len() - 1].to_vec()),
17646 };
17647 let (finished, completion) = oneshot::channel();
17648 let stdout_sink = sink.clone();
17649 let pump = tokio::spawn(async move {
17650 pump_stdout_to(reader, stdout_sink).await;
17651 finished.send(()).unwrap();
17652 });
17653 pump_stderr_to(io::Cursor::new(stderr.into_bytes()), ring, generation, sink).await;
17654
17655 let started = Instant::now();
17656 settle_stdout_pump("two-pipe", pump, BOUND).await;
17657 assert_eq!(
17658 started.elapsed(),
17659 BOUND,
17660 "a late reader must not hold up restart"
17661 );
17662 assert_eq!(
17663 std::fs::read_to_string(&path).unwrap().lines().count(),
17664 BURST + 1
17665 );
17666
17667 release
17668 .send(())
17669 .expect("stdout reader must survive the exit drain bound");
17670 tokio::time::timeout(Duration::from_secs(1), completion)
17671 .await
17672 .expect("released stdout reader must reach EOF")
17673 .expect("stdout capture must finish at EOF, not be cancelled");
17674 let contents = std::fs::read_to_string(&path).unwrap();
17675 let mut lines = contents
17676 .lines()
17677 .map(|line| split_capture_stamp(line).expect("capture stamp").1)
17678 .collect::<Vec<_>>();
17679 let mut expected = (0..BURST)
17680 .flat_map(|i| {
17681 [
17682 format!("out-{i:04}-{}", "o".repeat(64)),
17683 format!("err-{i:04}-{}", "e".repeat(64)),
17684 ]
17685 })
17686 .collect::<Vec<_>>();
17687 lines.sort_unstable();
17688 expected.sort_unstable();
17689 assert_eq!(
17690 lines, expected,
17691 "both pipes must retain every complete line exactly once"
17692 );
17693 }
17694}
17695
17696#[cfg(all(test, windows))]
17710mod job_containment_tests {
17711 use super::*;
17712 use cortexkit_test_support::ScratchDir;
17713 use std::{
17714 path::{Path, PathBuf},
17715 sync::{Arc, Mutex},
17716 time::{Duration, Instant},
17717 };
17718
17719 fn stub_path() -> PathBuf {
17725 let mut path = std::env::current_exe().expect("current_exe available in tests");
17726 path.pop();
17727 path.pop();
17728 path.push("fake-aft-stub.exe");
17729 assert!(
17730 path.exists(),
17731 "fake-aft-stub not built at {}: run `cargo test -p subc-core` (which builds \
17732 [[bin]] targets) rather than `cargo test -p subc-core --lib` (which does not)",
17733 path.display()
17734 );
17735 path
17736 }
17737
17738 fn read_grandchild_pid(path: &Path) -> u32 {
17740 let deadline = Instant::now() + Duration::from_secs(10);
17741 loop {
17742 if let Ok(contents) = std::fs::read_to_string(path) {
17743 if let Ok(pid) = contents.trim().parse() {
17744 return pid;
17745 }
17746 }
17747 assert!(
17748 Instant::now() < deadline,
17749 "the stub never recorded a grandchild pid at {}",
17750 path.display()
17751 );
17752 std::thread::sleep(Duration::from_millis(10));
17753 }
17754 }
17755
17756 struct Fixture {
17759 _dir: ScratchDir,
17760 module_id: String,
17761 grandchild: u32,
17762 child: Option<SupervisedChild>,
17763 registry: Arc<Registry>,
17764 snapshot: Arc<Mutex<SupervisorSnapshot>>,
17765 terminal_ring: Arc<Mutex<TerminalRing>>,
17766 spawn_events: SpawnEventFeed,
17767 }
17768
17769 fn fixture(label: &str, module_id: &str) -> Fixture {
17770 let dir = ScratchDir::new(label);
17771 let pid_file = dir.join("grandchild.pid");
17772 let supervisor = Supervisor::new_for_test(
17773 Arc::new(Registry::default()),
17774 RestartPolicy::new(3, Duration::ZERO),
17775 );
17776 let runtime = supervisor.runtime_config();
17777 let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::starting()));
17778 let spec = ModuleSpec {
17779 module_id: module_id.to_string(),
17780 program: stub_path(),
17781 args: Vec::new(),
17785 env: vec![
17786 ("FAKE_AFT_NEVER_CONNECT".to_string(), "1".to_string()),
17787 (
17788 "FAKE_AFT_GRANDCHILD_PID_FILE".to_string(),
17789 pid_file.display().to_string(),
17790 ),
17791 ],
17792 reserved: false,
17793 reserved_prefixes: Vec::new(),
17794 protocol: ModuleProtocol::Subc,
17795 overlap: Default::default(),
17796 carry_containment: false,
17797 };
17798 let child = spawn_and_mark_running(&spec, &runtime, &snapshot)
17799 .expect("spawn the supervised fixture");
17800 let grandchild = read_grandchild_pid(&pid_file);
17801 Fixture {
17802 _dir: dir,
17803 module_id: module_id.to_string(),
17804 grandchild,
17805 child: Some(child),
17806 registry: Arc::new(Registry::default()),
17807 snapshot,
17808 terminal_ring: Arc::clone(&runtime.terminal_ring),
17809 spawn_events: SpawnEventFeed::default(),
17810 }
17811 }
17812
17813 impl Fixture {
17814 async fn drain(&mut self) {
17816 let child = self
17817 .child
17818 .take()
17819 .expect("the fixture child is still present");
17820 drain_child_to_state(
17821 &self.module_id,
17822 ModuleProtocol::Subc,
17823 StopNotice::NotSent,
17826 &self.registry,
17827 None,
17828 &self.snapshot,
17829 &self.terminal_ring,
17830 &self.spawn_events,
17831 child,
17832 Duration::from_millis(500),
17833 ModuleState::Stopped,
17834 Some(false),
17835 &FailureContext::operator(),
17836 )
17837 .await
17838 .expect("drain the supervised fixture");
17839 }
17840 }
17841
17842 #[tokio::test]
17848 async fn teardown_reaps_the_grandchild() {
17849 let mut fixture = fixture("teardown-grandchild", "tree-teardown");
17850 let grandchild = fixture.grandchild;
17851
17852 assert!(
17853 subc_jobobject::process_exists(grandchild),
17854 "grandchild {grandchild} must be alive before teardown, or this proves nothing"
17855 );
17856
17857 fixture.drain().await;
17858
17859 assert!(
17860 subc_jobobject::wait_for_process_exit(grandchild, Duration::from_secs(10)),
17861 "grandchild {grandchild} outlived module teardown: the tree was not contained"
17862 );
17863 }
17864
17865 #[test]
17878 fn an_uncontained_grandchild_survives_a_direct_child_kill() {
17879 let dir = ScratchDir::new("teardown-uncontained");
17880 let pid_file = dir.join("grandchild.pid");
17881 let mut child = std::process::Command::new(stub_path())
17882 .env("FAKE_AFT_NEVER_CONNECT", "1")
17883 .env(
17884 "FAKE_AFT_GRANDCHILD_PID_FILE",
17885 pid_file.display().to_string(),
17886 )
17887 .stdin(std::process::Stdio::null())
17888 .stdout(std::process::Stdio::null())
17889 .stderr(std::process::Stdio::null())
17890 .spawn()
17891 .expect("spawn the uncontained fixture");
17892 let grandchild = read_grandchild_pid(&pid_file);
17893
17894 child.kill().expect("kill the direct child");
17896 let _ = child.wait();
17897
17898 assert!(
17899 subc_jobobject::process_exists(grandchild),
17900 "grandchild {grandchild} died with the direct child, so this control no longer \
17901 distinguishes contained from uncontained teardown and the regression test is \
17902 passing vacuously"
17903 );
17904
17905 kill_tree(grandchild);
17908 }
17909
17910 #[tokio::test]
17919 async fn dropping_containment_reaps_the_grandchild() {
17920 let mut fixture = fixture("drop-containment", "tree-drop");
17921 let grandchild = fixture.grandchild;
17922
17923 assert!(subc_jobobject::process_exists(grandchild));
17924
17925 fixture.child.as_mut().expect("child present").job = None;
17927
17928 assert!(
17929 subc_jobobject::wait_for_process_exit(grandchild, Duration::from_secs(10)),
17930 "grandchild {grandchild} survived the containment handle closing, so a daemon \
17931 crash would leave the tree behind"
17932 );
17933 }
17934
17935 fn kill_tree(pid: u32) {
17937 let _ = std::process::Command::new("taskkill.exe")
17938 .args(["/PID", &pid.to_string(), "/T", "/F"])
17939 .stdin(std::process::Stdio::null())
17940 .stdout(std::process::Stdio::null())
17941 .stderr(std::process::Stdio::null())
17942 .status();
17943 assert!(
17944 subc_jobobject::wait_for_process_exit(pid, Duration::from_secs(10)),
17945 "could not clean up grandchild {pid}"
17946 );
17947 }
17948}
17949
17950#[cfg(test)]
17951mod privacy_trampoline_configuration_tests {
17952 #[tokio::test]
17953 #[cfg_attr(not(target_os = "macos"), ignore = "requires macOS privacy trampoline")]
17954 async fn macos_spawn_without_a_configured_trampoline_refuses_by_name() {
17955 #[cfg(target_os = "macos")]
17956 {
17957 let supervisor = super::Supervisor::new(
17958 std::sync::Arc::new(crate::Registry::default()),
17959 super::RestartPolicy::default(),
17960 );
17961 let error = supervisor.spawn(spec()).unwrap_err();
17962 assert!(
17963 error
17964 .to_string()
17965 .contains("no privacy trampoline configured"),
17966 "{error}"
17967 );
17968 }
17969 }
17970
17971 #[tokio::test]
17972 #[cfg_attr(not(target_os = "macos"), ignore = "requires macOS privacy trampoline")]
17973 async fn macos_wrong_trampoline_is_rejected_by_the_startup_probe() {
17974 #[cfg(target_os = "macos")]
17975 {
17976 let supervisor = super::Supervisor::new(
17977 std::sync::Arc::new(crate::Registry::default()),
17978 super::RestartPolicy::default(),
17979 )
17980 .with_privacy_trampoline(std::env::current_exe().unwrap());
17981 let error = supervisor.spawn(spec()).unwrap_err();
17982 assert!(
17983 error
17984 .to_string()
17985 .contains("binary does not implement the privacy trampoline protocol"),
17986 "{error}"
17987 );
17988 }
17989 }
17990
17991 #[cfg(target_os = "macos")]
17992 fn spec() -> super::ModuleSpec {
17993 super::ModuleSpec {
17994 module_id: "privacy-configuration".into(),
17995 program: "/bin/sleep".into(),
17996 args: vec!["30".into()],
17997 env: vec![],
17998 reserved: false,
17999 reserved_prefixes: vec![],
18000 protocol: subc_control::ModuleProtocol::None,
18001 overlap: super::ModuleOverlap::Exclusive,
18002 carry_containment: false,
18003 }
18004 }
18005}
18006
18007#[cfg(test)]
18008mod privacy_exec_boundary_tests {
18009 #[cfg(target_os = "macos")]
18010 use super::*;
18011 #[cfg(target_os = "macos")]
18012 use std::{
18013 io::{Read, Write},
18014 net::{TcpListener, TcpStream},
18015 };
18016
18017 #[cfg(target_os = "macos")]
18020 pub(super) fn before_image_sample(spec: &ModuleSpec, pid: u32) {
18021 if let Some((_, path)) = spec
18022 .env
18023 .iter()
18024 .find(|(key, _)| key == "SUBC_TEST_PRIVACY_SAMPLE_BARRIER")
18025 {
18026 let mut barrier = TcpStream::connect(path).unwrap();
18027 barrier
18028 .set_read_timeout(Some(Duration::from_secs(30)))
18029 .unwrap();
18030 barrier.write_all(&pid.to_ne_bytes()).unwrap();
18031 let mut release = [0];
18032 barrier.read_exact(&mut release).unwrap();
18033 assert_eq!(&release, b"X");
18034 }
18035 }
18036
18037 #[cfg(target_os = "macos")]
18038 fn spec(program: &str, args: &[&str]) -> ModuleSpec {
18039 ModuleSpec {
18040 module_id: "privacy-boundary".into(),
18041 program: program.into(),
18042 args: args.iter().map(|arg| (*arg).into()).collect(),
18043 env: vec![],
18044 reserved: false,
18045 reserved_prefixes: vec![],
18046 protocol: ModuleProtocol::None,
18047 overlap: ModuleOverlap::Exclusive,
18048 carry_containment: false,
18049 }
18050 }
18051
18052 #[cfg(target_os = "macos")]
18053 async fn accept(listener: TcpListener) -> TcpStream {
18054 let listener = tokio::net::TcpListener::from_std({
18056 listener.set_nonblocking(true).unwrap();
18057 listener
18058 })
18059 .unwrap();
18060 let (stream, _) = tokio::time::timeout(Duration::from_secs(30), listener.accept())
18061 .await
18062 .unwrap()
18063 .unwrap();
18064 let stream = stream.into_std().unwrap();
18065 stream.set_nonblocking(false).unwrap();
18066 stream
18067 .set_read_timeout(Some(Duration::from_secs(30)))
18068 .unwrap();
18069 stream
18070 }
18071
18072 #[tokio::test]
18073 #[cfg_attr(not(target_os = "macos"), ignore = "requires macOS privacy trampoline")]
18074 async fn macos_roster_withholds_a_nonnull_trampoline_image_until_exec_confirmation() {
18075 #[cfg(target_os = "macos")]
18076 {
18077 let root = cortexkit_test_support::ScratchDir::new("privacy-roster-barrier");
18078 let exec_listener = TcpListener::bind("127.0.0.1:0").unwrap();
18081 let sample_listener = TcpListener::bind("127.0.0.1:0").unwrap();
18082 let record = root.join("live-children.json");
18083 let supervisor =
18084 Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default())
18085 .with_live_children_record(&record);
18086 let runtime = supervisor.runtime_config();
18087 let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::starting()));
18088 let mut spec = spec("/bin/sleep", &["30"]);
18089 spec.env = vec![
18090 (
18091 "SUBC_TEST_PRIVACY_EXEC_BARRIER".into(),
18092 exec_listener.local_addr().unwrap().to_string(),
18093 ),
18094 (
18095 "SUBC_TEST_PRIVACY_SAMPLE_BARRIER".into(),
18096 sample_listener.local_addr().unwrap().to_string(),
18097 ),
18098 ];
18099 let spawn = tokio::task::spawn_blocking(move || {
18100 spawn_and_mark_running(&spec, &runtime, &snapshot).unwrap()
18101 });
18102 let mut sample = accept(sample_listener).await;
18103 let mut pid = [0; 4];
18104 sample.read_exact(&mut pid).unwrap();
18105 let pid = u32::from_ne_bytes(pid);
18106 let mut exec = accept(exec_listener).await;
18107 let mut ready = [0];
18108 exec.read_exact(&mut ready).unwrap();
18109 assert_eq!(&ready, b"R");
18110 let trampoline = subc_os::file_identity(&test_privacy_trampoline()).unwrap();
18113 assert_eq!(
18114 observe_spawned_image(pid).unwrap().executable,
18115 Some(trampoline)
18116 );
18117 sample.write_all(b"X").unwrap();
18118 let mut child = spawn.await.unwrap();
18119 let early = crate::live_children::read_record(&record).unwrap();
18120 assert_eq!(early.len(), 1);
18121 assert_eq!(early[0].pid, pid);
18122 assert_eq!(
18123 early[0].executable, None,
18124 "unconfirmed trampoline image entered the roster"
18125 );
18126 assert!(child.report_ready.get().is_none());
18127 child.privacy_exec.as_mut().unwrap().deadline =
18130 tokio::time::Instant::now() + Duration::from_secs(30);
18131 exec.write_all(b"X").unwrap();
18132 child.confirm_privacy_exec().await;
18133 assert_eq!(child.spawn_failure, None);
18134 assert!(child.report_ready.get().is_some());
18135 let confirmed = crate::live_children::read_record(&record).unwrap();
18136 let module = subc_os::file_identity(std::path::Path::new("/bin/sleep")).unwrap();
18137 assert_ne!(module, trampoline);
18138 assert_eq!(confirmed[0].executable, Some(module.into()));
18139 child.start_kill().unwrap();
18140 child.wait().await.unwrap();
18141 child.release_roster();
18142 }
18143 }
18144
18145 #[tokio::test]
18146 #[cfg_attr(not(target_os = "macos"), ignore = "requires macOS privacy trampoline")]
18147 async fn macos_already_exited_121_in_try_wait_is_a_module_exit_not_a_trampoline_refusal() {
18148 #[cfg(target_os = "macos")]
18149 {
18150 let registry = Arc::new(Registry::default());
18151 let policy = RestartPolicy::new(0, Duration::ZERO);
18152 let supervisor = Supervisor::new_for_test(Arc::clone(®istry), policy);
18153 let runtime = supervisor.runtime_config();
18154 let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::starting()));
18155 let spec = spec("/bin/sh", &["-c", "exit 121"]);
18156 let mut child = spawn_and_mark_running(&spec, &runtime, &snapshot).unwrap();
18160 let pid = child.pid;
18161 tokio::task::spawn_blocking(move || {
18162 subc_os::fork_exec_test::wait_for_child_exit_without_reaping(pid)
18163 })
18164 .await
18165 .unwrap()
18166 .unwrap();
18167 child.privacy_exec.as_mut().unwrap().deadline =
18168 tokio::time::Instant::now() + Duration::from_secs(30);
18169 let status = child.wait().await.unwrap();
18170 assert_eq!(status.code(), Some(121));
18171 assert!(child.privacy_exec.is_none());
18172 assert!(
18173 child.report_ready.get().is_none(),
18174 "an exited module must not publish a live pid"
18175 );
18176 let report = classify_reaped_child_exit(&snapshot, &child, &status);
18177 on_child_exit(
18178 &spec,
18179 policy,
18180 ®istry,
18181 &snapshot,
18182 &runtime.terminal_ring,
18183 &runtime.spawn_events,
18184 &runtime.child_roster,
18185 report,
18186 true,
18187 )
18188 .await;
18189 let state = lock_snapshot(&snapshot).unwrap();
18190 assert_eq!(state.state, ModuleState::Failed);
18191 assert_eq!(state.last_exit.as_ref().unwrap().code, Some(121));
18192 assert_eq!(state.reported_pid(), None);
18193 drop(state);
18194 let history = runtime.terminal_ring.lock().unwrap().snapshot();
18195 assert_eq!(history.entries.len(), 1);
18196 let terminal = &history.entries[0];
18197 assert_eq!(terminal.exit_code, Some(121));
18198 assert_eq!(terminal.exit_kind, subc_control::TerminalExitKind::Crash);
18199 assert_eq!(terminal.disposition, TerminalDisposition::Failed);
18200 assert_eq!(
18201 terminal.disposition_detail.as_deref(),
18202 Some(policy.budget_exhausted_detail().as_str()),
18203 "module exit 121 was classified as a trampoline refusal: {terminal:?}"
18204 );
18205 assert_eq!(child.spawn_failure, None);
18206 child.release_roster();
18207 }
18208 }
18209}
18210
18211#[cfg(all(test, unix))]
18215mod launch_nonce_descriptor_tests {
18216 use super::{ChildRoster, ModuleProtocol, ModuleSpec, SupervisorHandle};
18217 use crate::stderr_tail::{StderrRing, StderrTailConfig};
18218 use cortexkit_test_support::ScratchDir;
18219 use std::{
18220 path::PathBuf,
18221 sync::{Arc, Mutex},
18222 time::{Duration, Instant},
18223 };
18224
18225 async fn probe(role: super::SpawnRole) {
18226 let scratch = ScratchDir::new("launch-nonce-descriptor");
18227 let fd_copy = scratch.join("from-descriptor");
18228 let env_copy = scratch.join("environment");
18229 let script = format!(
18230 "env > '{env}.tmp' && mv '{env}.tmp' '{env}'; cat <&3 > '{fd}.tmp' && mv '{fd}.tmp' '{fd}'; sleep 30",
18231 fd = fd_copy.display(), env = env_copy.display(),
18232 );
18233 let xdg = |name: &str| (name.to_string(), scratch.join(name).display().to_string());
18234 let spec = ModuleSpec {
18235 module_id: "nonce-descriptor-probe".to_string(),
18236 program: PathBuf::from("/bin/sh"),
18237 args: vec!["-c".to_string(), script],
18238 env: vec![
18239 xdg("XDG_DATA_HOME"),
18240 xdg("XDG_RUNTIME_DIR"),
18241 xdg("XDG_CONFIG_HOME"),
18242 ("SUBC_LAUNCH_NONCE".to_string(), "must-not-leak".to_string()),
18243 ],
18244 reserved: true,
18245 reserved_prefixes: Vec::new(),
18246 protocol: ModuleProtocol::Subc,
18247 overlap: Default::default(),
18248 carry_containment: false,
18249 };
18250 let handle = SupervisorHandle::new();
18251 let ring = Arc::new(Mutex::new(StderrRing::new(StderrTailConfig::default())));
18252 let roster = ChildRoster::default();
18253 #[cfg(target_os = "macos")]
18254 {
18255 let path = super::test_privacy_trampoline();
18256 roster.set_privacy_trampoline(super::probe_privacy_trampoline(&path).map(|()| path));
18257 }
18258 let child = super::spawn_child_in_slot(
18259 &spec,
18260 None,
18261 Some(&handle),
18262 &ring,
18263 None,
18264 &roster,
18265 #[cfg(target_os = "linux")]
18266 None,
18267 role,
18268 matches!(role, super::SpawnRole::SwapCandidate),
18269 )
18270 .expect("spawn probe");
18271 let deadline = Instant::now() + Duration::from_secs(10);
18272 while !(fd_copy.exists() && env_copy.exists()) {
18273 assert!(Instant::now() < deadline, "probe never wrote its copies");
18274 tokio::time::sleep(Duration::from_millis(20)).await;
18275 }
18276 let nonce = std::fs::read_to_string(fd_copy).unwrap();
18277 assert!(!nonce.is_empty());
18278 let environment = std::fs::read_to_string(env_copy).unwrap();
18279 assert!(environment
18280 .lines()
18281 .any(|line| line.starts_with("SUBC_LAUNCH_NONCE_FD=3:")));
18282 let copy = environment
18283 .lines()
18284 .find_map(|line| line.strip_prefix("SUBC_LAUNCH_NONCE="));
18285 assert_eq!(
18286 copy, None,
18287 "Unix children must never receive the environment nonce"
18288 );
18289 if matches!(role, super::SpawnRole::Plain) {
18290 assert_eq!(
18291 handle.spawn_nonce(&spec.module_id).as_deref(),
18292 Some(nonce.as_str())
18293 );
18294 }
18295 drop(child);
18296 }
18297
18298 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
18299 async fn a_spawned_module_receives_its_nonce_only_on_descriptor_3() {
18300 probe(super::SpawnRole::Plain).await;
18301 }
18302
18303 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
18304 async fn swap_candidate_receives_its_nonce_only_on_descriptor_3() {
18305 probe(super::SpawnRole::SwapCandidate).await;
18306 }
18307}
18308
18309#[cfg(all(test, target_os = "linux"))]
18310#[path = "supervise/cgroup_reap_tests.rs"]
18311mod cgroup_reap_tests;
18312
18313#[cfg(all(test, target_os = "linux"))]
18314mod cgroup_containment_tests {
18315 use super::*;
18316 use cortexkit_test_support::ScratchDir;
18317
18318 #[tokio::test]
18319 async fn linux_subtree_memory_covers_module_children() {
18320 let dir = ScratchDir::new("subtree-memory");
18321 let root = PathBuf::from(format!(
18322 "/sys/fs/cgroup/subc-memory-test-{}-{}",
18323 std::process::id(),
18324 unix_ms_now()
18325 ));
18326 if let Err(error) = std::fs::create_dir(&root) {
18327 assert!(
18328 std::env::var_os("SUBC_REQUIRE_CGROUP_TEST").is_none(),
18329 "required cgroup test cannot execute: {error}"
18330 );
18331 eprintln!("SKIP linux_subtree_memory_covers_module_children: {error}");
18332 return;
18333 }
18334 let placement = subc_cgroup::prepare_at(&root)
18335 .unwrap()
18336 .expect("delegated cgroup");
18337 let supervisor = Supervisor::new_for_test(
18338 Arc::new(Registry::default()),
18339 RestartPolicy::new(3, Duration::ZERO),
18340 )
18341 .with_cgroup_placement(Some(placement));
18342 let ready = dir.join("ready");
18343 let module = supervisor
18344 .spawn(ModuleSpec {
18345 module_id: "memory-tree".into(),
18346 program: PathBuf::from("/bin/sh"),
18347 args: vec![
18348 "-c".into(),
18349 "sleep 600 & sleep 600 & sleep 600 & echo ready > \"$1\"; wait".into(),
18350 "fixture".into(),
18351 ready.display().to_string(),
18352 ],
18353 env: ["XDG_DATA_HOME", "XDG_RUNTIME_DIR", "XDG_CONFIG_HOME"]
18354 .into_iter()
18355 .map(|key| (key.into(), dir.display().to_string()))
18356 .collect(),
18357 reserved: false,
18358 reserved_prefixes: Vec::new(),
18359 protocol: ModuleProtocol::None,
18360 overlap: Default::default(),
18361 carry_containment: false,
18362 })
18363 .unwrap();
18364 let deadline = tokio::time::Instant::now() + Duration::from_secs(10);
18365 while !ready.exists() {
18366 assert!(
18367 tokio::time::Instant::now() < deadline,
18368 "children never started"
18369 );
18370 tokio::time::sleep(Duration::from_millis(10)).await;
18371 }
18372 let readings = crate::subtree_resources::read(vec![module.subtree_target()]).await;
18373 let reading = &readings[0];
18374 assert!(
18375 reading.process_count.is_some_and(|count| count >= 4),
18376 "{reading:?}"
18377 );
18378 assert!(
18379 reading.memory_bytes.is_some_and(|bytes| bytes > 0),
18380 "{reading:?}"
18381 );
18382 module.stop().await.unwrap();
18383 drop(module);
18384 std::fs::remove_dir(root.join("subc-modules")).unwrap();
18385 std::fs::remove_dir(&root).unwrap();
18386 eprintln!("EXECUTED linux_subtree_memory_covers_module_children: {reading:?}");
18387 }
18388
18389 fn running(pid: u32) -> bool {
18390 std::fs::read_to_string(format!("/proc/{pid}/stat"))
18392 .ok()
18393 .and_then(|stat| {
18394 stat.rsplit_once(") ")
18395 .map(|(_, rest)| rest.starts_with('Z'))
18396 })
18397 .is_some_and(|zombie| !zombie)
18398 }
18399
18400 #[tokio::test]
18401 async fn linux_teardown_reaps_the_grandchild() {
18402 teardown_tree("linux_teardown_reaps_the_grandchild", false).await;
18403 }
18404
18405 #[tokio::test]
18406 async fn linux_shutdown_straggler_reaps_the_grandchild() {
18407 teardown_tree("linux_shutdown_straggler_reaps_the_grandchild", true).await;
18408 }
18409
18410 async fn teardown_tree(test_name: &str, shutdown: bool) {
18411 let dir = ScratchDir::new(test_name);
18412 let root = PathBuf::from(format!(
18413 "/sys/fs/cgroup/subc-tree-test-{test_name}-{}-{}",
18414 std::process::id(),
18415 unix_ms_now()
18416 ));
18417 if let Err(error) = std::fs::create_dir(&root) {
18418 assert!(
18419 std::env::var_os("SUBC_REQUIRE_CGROUP_TEST").is_none(),
18420 "required cgroup test cannot execute: {error}"
18421 );
18422 eprintln!(
18423 "SKIP {test_name}: no writable delegated cgroup at {}: {error}",
18424 root.display()
18425 );
18426 return;
18427 }
18428 let placement = subc_cgroup::prepare_at(&root)
18429 .expect("prepare isolated kernel cgroup")
18430 .expect("isolated cgroup is delegated");
18431 let module_id = "tree-teardown";
18432 let module = placement
18433 .module_path(module_id)
18434 .expect("create isolated module cgroup");
18435 if !module.join("cgroup.kill").exists() {
18436 std::fs::remove_dir(&module).unwrap();
18437 std::fs::remove_dir(root.join("subc-modules")).unwrap();
18438 std::fs::remove_dir(&root).unwrap();
18439 assert!(
18440 std::env::var_os("SUBC_REQUIRE_CGROUP_TEST").is_none(),
18441 "required cgroup.kill interface unavailable"
18442 );
18443 eprintln!("SKIP {test_name}: cgroup.kill unavailable (kernel < 5.14)");
18444 return;
18445 }
18446 let supervisor = Supervisor::new_for_test(
18447 Arc::new(Registry::default()),
18448 RestartPolicy::new(3, Duration::ZERO),
18449 )
18450 .with_cgroup_placement(Some(placement));
18451 let mut runtime = supervisor.runtime_config();
18452 runtime.child_roster = runtime
18453 .child_roster
18454 .for_module(Arc::new(Mutex::new(Duration::from_millis(100))));
18455 let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::starting()));
18456 let pid_file = dir.join("grandchild.pid");
18457 let spec = ModuleSpec {
18458 module_id: module_id.to_string(),
18459 program: PathBuf::from("/bin/sh"),
18460 args: vec![
18461 "-c".into(),
18462 "trap '' TERM; sleep 600 & echo $! > \"$1\"; wait".into(),
18463 "fixture".into(),
18464 pid_file.display().to_string(),
18465 ],
18466 env: ["XDG_DATA_HOME", "XDG_RUNTIME_DIR", "XDG_CONFIG_HOME"]
18467 .into_iter()
18468 .map(|key| (key.to_string(), dir.display().to_string()))
18469 .collect(),
18470 reserved: false,
18471 reserved_prefixes: Vec::new(),
18472 protocol: ModuleProtocol::None,
18473 overlap: Default::default(),
18474 carry_containment: false,
18475 };
18476 let child =
18477 spawn_and_mark_running(&spec, &runtime, &snapshot).expect("spawn supervised tree");
18478 let deadline = tokio::time::Instant::now() + Duration::from_secs(10);
18479 let grandchild: u32 = loop {
18480 if let Ok(contents) = std::fs::read_to_string(&pid_file) {
18481 if let Ok(pid) = contents.trim().parse() {
18482 break pid;
18483 }
18484 }
18485 assert!(
18486 tokio::time::Instant::now() < deadline,
18487 "grandchild pid was not recorded"
18488 );
18489 tokio::time::sleep(Duration::from_millis(10)).await;
18490 };
18491 assert!(
18492 running(grandchild),
18493 "grandchild must be alive before teardown"
18494 );
18495 if shutdown {
18496 let mut child = child;
18497 crate::child_roster::end_children_for_daemon_shutdown(
18498 &runtime.child_roster,
18499 false,
18500 std::future::pending(),
18501 )
18502 .await;
18503 child.wait().await.expect("reap shutdown straggler");
18504 } else {
18505 drain_child_to_state(
18506 module_id,
18507 ModuleProtocol::None,
18508 StopNotice::NotSent,
18509 &Registry::default(),
18510 None,
18511 &snapshot,
18512 &runtime.terminal_ring,
18513 &SpawnEventFeed::default(),
18514 child,
18515 Duration::from_millis(100),
18516 ModuleState::Stopped,
18517 Some(false),
18518 &FailureContext::operator(),
18519 )
18520 .await
18521 .expect("real supervisor teardown");
18522 }
18523 let deadline = tokio::time::Instant::now() + Duration::from_secs(5);
18524 while running(grandchild) && tokio::time::Instant::now() < deadline {
18525 tokio::time::sleep(Duration::from_millis(10)).await;
18526 }
18527 let survived = running(grandchild);
18528 if survived {
18530 let pid = rustix::process::Pid::from_raw(grandchild as i32).unwrap();
18531 let _ = rustix::process::kill_process(pid, rustix::process::Signal::KILL);
18532 tokio::time::sleep(Duration::from_millis(100)).await;
18533 }
18534 if module.exists() {
18535 std::fs::remove_dir(&module).expect("remove empty module cgroup");
18536 }
18537 std::fs::remove_dir(root.join("subc-modules")).unwrap();
18538 std::fs::remove_dir(&root).unwrap();
18539 assert!(
18540 !survived,
18541 "grandchild {grandchild} outlived module teardown"
18542 );
18543 eprintln!("EXECUTED {test_name}: grandchild {grandchild} killed in isolated cgroup");
18544 }
18545}
18546
18547#[cfg(all(test, unix))]
18550mod record_write_stall_tests {
18551 use super::*;
18552 use std::sync::atomic::{AtomicBool, Ordering};
18553 use subc_transport::{authenticate_client, ConnectionInfo, Endpoint, SCHEMA_VERSION};
18554
18555 const MODULE_AUTH_DEADLINE: Duration = Duration::from_secs(2);
18558 const HELD_RECORD_WRITE: Duration = Duration::from_secs(3);
18562
18563 fn sleeping_spec() -> ModuleSpec {
18575 ModuleSpec {
18576 module_id: "record-write-stall".into(),
18577 program: "/bin/sleep".into(),
18578 args: vec!["600".into()],
18579 env: vec![],
18580 reserved: false,
18581 reserved_prefixes: vec![],
18582 protocol: ModuleProtocol::None,
18583 overlap: Default::default(),
18584 carry_containment: false,
18585 }
18586 }
18587
18588 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
18589 async fn module_auth_completes_while_the_restart_waits_on_the_record_write() {
18590 let home = cortexkit_test_support::ScratchDir::new("record-write-stall");
18591 let supervisor =
18592 Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default())
18593 .with_live_children_record(home.join("live-children.json"));
18594 let module = supervisor.spawn(sleeping_spec()).unwrap();
18595 let old_pid = tokio::time::timeout(Duration::from_secs(10), async {
18600 loop {
18601 if let Some(pid) = lock_snapshot(&module.inner.snapshot)
18602 .unwrap()
18603 .reported_pid()
18604 {
18605 return pid;
18606 }
18607 tokio::time::sleep(Duration::from_millis(10)).await;
18608 }
18609 })
18610 .await
18611 .expect("the first process was admitted as the module");
18612
18613 let key = vec![0x42; 32];
18616 let daemon_id = [0x24; 16];
18617 let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
18618 let port = listener.local_addr().unwrap().port();
18619 let server = tokio::spawn(crate::server::serve_listener(
18620 listener,
18621 Arc::new(crate::router::Router::with_default_self_handler()),
18622 crate::server::ServerAuth::with_limits(
18623 key.clone(),
18624 daemon_id,
18625 "test",
18626 crate::server::DEFAULT_AUTH_DEADLINE,
18627 4,
18628 ),
18629 ));
18630
18631 let (held_tx, held_rx) = std::sync::mpsc::channel();
18634 let armed = AtomicBool::new(true);
18635 supervisor
18636 .child_roster
18637 .set_record_write_hook_for_test(move |live| {
18638 let admitting_replacement = live.values().any(|entry| entry.pid != old_pid);
18639 if admitting_replacement && armed.swap(false, Ordering::SeqCst) {
18640 let _ = held_tx.send(());
18641 std::thread::sleep(HELD_RECORD_WRITE);
18642 }
18643 });
18644
18645 let module_process = std::thread::spawn(move || {
18648 held_rx
18649 .recv_timeout(Duration::from_secs(30))
18650 .expect("the restart reached the replacement's record write");
18651 let conn = ConnectionInfo {
18652 schema: SCHEMA_VERSION,
18653 wire_version: None,
18654 endpoints: vec![Endpoint {
18655 host: "127.0.0.1".to_owned(),
18656 port,
18657 }],
18658 key,
18659 daemon_id,
18660 pid: std::process::id(),
18661 daemon_ver: "test".to_owned(),
18662 };
18663 tokio::runtime::Builder::new_current_thread()
18664 .enable_all()
18665 .build()
18666 .unwrap()
18667 .block_on(async {
18668 let started = Instant::now();
18669 let mut stream = tokio::net::TcpStream::connect(("127.0.0.1", port))
18670 .await
18671 .unwrap();
18672 let result =
18673 authenticate_client(&mut stream, &conn, MODULE_AUTH_DEADLINE).await;
18674 (result, started.elapsed())
18675 })
18676 });
18677
18678 module.restart(Some(0)).await.unwrap();
18679 let (auth, elapsed) = tokio::task::spawn_blocking(move || module_process.join().unwrap())
18680 .await
18681 .unwrap();
18682 assert!(
18683 auth.is_ok(),
18684 "the daemon must answer a module's handshake while a restart waits on the \
18685 live-children record write; got {auth:?} after {elapsed:?}"
18686 );
18687
18688 tokio::time::timeout(Duration::from_secs(20), async {
18689 while lock_snapshot(&module.inner.snapshot)
18690 .unwrap()
18691 .reported_pid()
18692 .is_none_or(|pid| pid == old_pid)
18693 {
18694 tokio::time::sleep(Duration::from_millis(10)).await;
18695 }
18696 })
18697 .await
18698 .expect("the restart published its replacement");
18699 module.stop().await.unwrap();
18700 server.abort();
18701 }
18702}
18703
18704#[cfg(all(test, unix))]
18705mod retained_status_tests {
18706 use super::*;
18707
18708 #[tokio::test]
18709 async fn cached_container_block_is_visible_in_status_and_health_without_rechecking() {
18710 use crate::live_children::{ContainerIdentity, RetainedReason};
18711 for (reason, carried, expected) in [
18712 (
18713 RetainedReason::ContainerSurvivors(vec![4242]),
18714 true,
18715 Some("waiting: container survivors (pids [4242])".to_owned()),
18716 ),
18717 (
18718 RetainedReason::ContainerUnverifiable("member inspection denied".into()),
18719 true,
18720 Some(
18721 "waiting: container survivors could not be verified: member inspection denied"
18722 .to_owned(),
18723 ),
18724 ),
18725 (RetainedReason::Survived, false, None),
18726 ] {
18727 let supervisor =
18728 Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default());
18729 let spec = ModuleSpec {
18730 module_id: "retained-status-fixture".into(),
18731 program: PathBuf::from("/not-spawned"),
18732 args: Vec::new(),
18733 env: Vec::new(),
18734 reserved: false,
18735 reserved_prefixes: Vec::new(),
18736 protocol: ModuleProtocol::None,
18737 overlap: ModuleOverlap::Exclusive,
18738 carry_containment: carried,
18739 };
18740 let module = supervisor
18741 .supervise_configured(spec.clone(), false)
18742 .unwrap();
18743 let guard = supervisor
18747 .child_roster
18748 .admit(
18749 spec.module_id,
18750 0,
18751 ModuleProtocol::None,
18752 None,
18753 crate::child_roster::RecordedIdentity {
18754 container: carried.then_some(ContainerIdentity {
18755 pgid: Some(0),
18756 created_start_time: Some(0),
18757 cgroup_path: None,
18758 ..ContainerIdentity::test_metadata()
18759 }),
18760 ..crate::child_roster::RecordedIdentity::default()
18761 },
18762 )
18763 .unwrap();
18764 guard.retain(reason);
18765 let status = module.status().unwrap();
18766 assert_eq!(status.waiting_for_previous_detail, expected.clone());
18767 assert_eq!(status.health.detail, expected);
18768 assert!(!status.process_alive);
18769 }
18770 }
18771}