1use std::{
2 collections::{HashMap, HashSet, VecDeque},
3 error::Error,
4 fmt, io,
5 path::PathBuf,
6 process::{ExitStatus, Stdio},
7 sync::{Arc, Mutex, OnceLock},
8 time::{Duration, SystemTime, UNIX_EPOCH},
9};
10
11use cortexkit_log::Retention;
12use serde_json::Value;
13use subc_control::{
14 ClientControlPush, LiveSpawn, ModuleProtocol, RouteCloseReason, SpawnCursor, SpawnEvent,
15 SpawnEventKind, SpawnSnapshot, SupervisorHealthStatus, TerminalDisposition, TerminalExitKind,
16};
17use subc_protocol::{
18 manifest::{SelfSignalKind, SignalAnchor},
19 session::{
20 HealthReport, HealthStatus, ModuleControlCommand, ModuleControlRequest,
21 MODULE_CONTROL_OP_HEALTH_CHECK,
22 },
23 Flags, FrameType, Principal, Priority, SUBC_LAUNCH_NONCE_ENV, SUBC_MODULE_ID_ENV,
24};
25use tokio::{
26 process::{Child, Command},
27 sync::{mpsc, oneshot, watch, Mutex as AsyncMutex},
28 task::JoinHandle,
29 time::{sleep, sleep_until, timeout, timeout_at, Instant},
30};
31use tracing::{debug, error, info, warn};
32
33use crate::{
34 child_roster::ChildRoster,
35 daemon_config::{
36 CAPTURE_KEEP_ENV, CAPTURE_MAX_AGE_DAYS_ENV, CAPTURE_MAX_FILE_MB_ENV, CK_LOG_ENV,
37 },
38 forwarding::{
39 CloseReason, ForwardingError, ForwardingTable, GoodbyeTarget, ModuleControlRpcOutcome,
40 ModuleDrainTarget, PendingModuleControlRpc,
41 },
42 provenance::{ExecutableIdentityProbe, SpawnedFileIdentity},
43 registry::{ConnectionId, RegistrationEndReason, RegistryError},
44 stderr_tail::{
45 pump_stderr_to, pump_stdout_to, ChildOutputSink, StderrRing, StderrTailConfig,
46 StderrTailSnapshot,
47 },
48 terminal_ring::{TerminalHistorySnapshot, TerminalRecord, TerminalRing, TerminalRingConfig},
49 Frame, FrameSink, Registry,
50};
51
52#[cfg(not(windows))]
53use crate::provenance::spawned_file_identity;
54
55#[path = "supervise_swap.rs"]
56mod swap;
57
58#[path = "supervise_activity.rs"]
59mod activity;
60
61pub const SUBC_ARG: &str = "--subc";
67
68const DEFAULT_MAX_RESTARTS: u32 = 3;
69const DEFAULT_BACKOFF: Duration = Duration::from_millis(100);
70const DEFAULT_MAX_BACKOFF: Duration = Duration::from_secs(30);
71const DEFAULT_RESTART_WINDOW: Duration = Duration::from_secs(600);
75pub const DEFAULT_DRAIN_TIMEOUT: Duration = Duration::from_secs(30);
86const REGISTRY_RELEASE_TIMEOUT: Duration = Duration::from_secs(1);
87const REGISTRY_RELEASE_POLL: Duration = Duration::from_millis(10);
88const STDERR_PUMP_DRAIN_TIMEOUT: Duration = Duration::from_millis(250);
111pub const SPAWN_EVENT_RING_CAPACITY: usize = 4096;
113const SPAWN_SUBSCRIBER_BUFFER: usize = SPAWN_EVENT_RING_CAPACITY + 1;
114pub(crate) const SPAWN_SUBSCRIBER_LAGGED_CODE: &str = "spawn_subscriber_lagged";
119
120#[cfg(target_os = "macos")]
121struct PrivacyExec {
122 reader: tokio::io::unix::AsyncFd<std::io::PipeReader>,
123 deadline: tokio::time::Instant,
124 expected: Option<subc_os::FileIdentity>,
125 trampoline: Option<subc_os::FileIdentity>,
126 script: bool,
127 module_id: String,
128 #[cfg(test)]
129 confirmation_pause: Option<(String, Option<tokio::net::TcpStream>)>,
130}
131
132#[cfg(all(test, target_os = "macos"))]
133pub(crate) fn test_privacy_trampoline() -> PathBuf {
134 let path = std::env::current_exe()
137 .unwrap()
138 .parent()
139 .unwrap()
140 .parent()
141 .unwrap()
142 .join("privacy-trampoline-fixture");
143 assert!(
147 path.exists(),
148 "privacy-trampoline-fixture not built at {}: run `cargo build -p subc-daemon \
149 --bins --features test-support` or `cargo test -p subc-daemon` first",
150 path.display()
151 );
152 path
153}
154
155#[cfg(target_os = "macos")]
156fn probe_privacy_trampoline(path: &std::path::Path) -> Result<(), String> {
157 use std::io::Read;
158 let mut probe = std::process::Command::new(path)
159 .args(["__disclaim-exec", "--probe"])
160 .stdin(Stdio::null())
161 .stdout(Stdio::piped())
162 .stderr(Stdio::piped())
163 .spawn()
164 .map_err(|error| {
165 format!(
166 "privacy trampoline probe failed for {}: {error}",
167 path.display()
168 )
169 })?;
170 let deadline = std::time::Instant::now() + Duration::from_secs(5);
171 let status = loop {
172 match probe.try_wait() {
173 Ok(Some(status)) => break status,
174 Ok(None) if std::time::Instant::now() < deadline => {
175 std::thread::sleep(Duration::from_millis(5))
176 }
177 result => {
178 let _ = probe.kill();
179 let _ = probe.wait();
180 return Err(format!(
181 "privacy trampoline probe failed or timed out for {}: {result:?}",
182 path.display()
183 ));
184 }
185 }
186 };
187 let mut answer = String::new();
188 if let Some(stdout) = probe.stdout.take() {
189 let _ = stdout.take(256).read_to_string(&mut answer);
190 }
191 if status.success() && answer.trim() == subc_os::privacy_identity::TRAMPOLINE_PROBE {
192 return Ok(());
193 }
194 let mut diagnostic = String::new();
195 if let Some(stderr) = probe.stderr.take() {
196 let _ = stderr.take(1024).read_to_string(&mut diagnostic);
197 }
198 let cause = diagnostic
199 .trim()
200 .strip_prefix("ck-subc: own privacy identity refused: ")
201 .unwrap_or("binary does not implement the privacy trampoline protocol");
202 Err(format!(
203 "{cause}: probe of {} exited {status}",
204 path.display()
205 ))
206}
207
208#[cfg(target_os = "macos")]
209fn privacy_command(
210 spec: &ModuleSpec,
211 roster: &ChildRoster,
212) -> Result<
213 (
214 Command,
215 Option<PrivacyExec>,
216 subc_os::privacy_identity::ExecAcknowledgement,
217 ),
218 SuperviseError,
219> {
220 let failure = |cause: String| {
221 warn!(module_id = %spec.module_id, %cause, "privacy identity trampoline refused module spawn");
222 SuperviseError::Spawn {
223 program: spec.program.clone(),
224 source: io::Error::other(cause),
225 cgroup_path: None,
226 }
227 };
228 let trampoline = roster.privacy_trampoline().map_err(failure)?;
229 let program = if spec.program.components().count() == 1 && !spec.program.is_absolute() {
234 let path = spec
235 .env
236 .iter()
237 .find(|(key, _)| key == "PATH")
238 .map(|(_, value)| std::ffi::OsString::from(value))
239 .or_else(|| std::env::var_os("PATH"))
240 .unwrap_or_else(|| "/usr/bin:/bin".into());
241 std::env::split_paths(&path)
242 .map(|dir| dir.join(&spec.program))
243 .find(|path| path.is_file())
244 .unwrap_or_else(|| spec.program.clone())
245 } else {
246 spec.program.clone()
247 };
248 let expected = subc_os::file_identity(&program);
249 let trampoline_image = subc_os::file_identity(&trampoline);
250 let script = {
251 use std::io::Read;
252 let mut prefix = [0u8; 2];
253 std::fs::File::open(&program)
254 .is_ok_and(|mut file| file.read_exact(&mut prefix).is_ok() && &prefix == b"#!")
255 };
256 if expected.is_none() || expected == subc_os::file_identity(&trampoline) {
257 return Err(failure(
258 "privacy identity module executable is missing or is the trampoline itself".to_string(),
259 ));
260 }
261 let (reader, ack) = subc_os::privacy_identity::ExecAcknowledgement::pipe()
262 .map_err(|error| failure(error.to_string()))?;
263 let reader =
264 tokio::io::unix::AsyncFd::new(reader).map_err(|error| failure(error.to_string()))?;
265 let mut command = Command::new(&trampoline);
266 command
267 .arg("__disclaim-exec")
268 .arg(ack.fd().to_string())
269 .arg(&program);
270 ack.install(command.as_std_mut());
271 Ok((
272 command,
273 Some(PrivacyExec {
274 reader,
275 deadline: tokio::time::Instant::now() + Duration::from_secs(5),
276 expected,
277 trampoline: trampoline_image,
278 script,
279 module_id: spec.module_id.clone(),
280 #[cfg(test)]
281 confirmation_pause: spec
282 .env
283 .iter()
284 .find(|(key, _)| key == "SUBC_TEST_BEFORE_EXEC_CONFIRM")
285 .map(|(_, address)| (address.clone(), None)),
286 }),
287 ack,
288 ))
289}
290
291struct SupervisedChild {
292 child: Child,
293 exec: crate::activity::ExecOwner,
294 #[cfg(windows)]
295 nonce_delivery: Option<subc_os::LaunchNoncePipeDelivery>,
296 #[cfg(target_os = "macos")]
297 privacy_exec: Option<PrivacyExec>,
298 #[cfg(target_os = "macos")]
305 report_ready: Arc<OnceLock<()>>,
306 spawn_failure: Option<String>,
308 protocol: ModuleProtocol,
313 #[cfg(target_os = "linux")]
319 module_id: String,
320 #[cfg(target_os = "linux")]
321 cgroup_placement: Option<subc_cgroup::Placement>,
322 #[cfg(windows)]
344 job: Option<subc_jobobject::JobObject>,
345 stdout_pump: Option<JoinHandle<()>>,
346 stderr_pump: Option<StderrPump>,
347 stderr_ring: Arc<Mutex<StderrRing>>,
348 spawned_at_ms: u64,
349 spawned_from: PathBuf,
350 spawned_file_identity: Option<SpawnedFileIdentity>,
351 process_start_time: Option<u64>,
352 process_identity: Option<ProcessIdentity>,
353 pid: u32,
354 roster_guard: Option<crate::child_roster::RosterGuard>,
357}
358
359impl SupervisedChild {
360 fn id(&self) -> Option<u32> {
361 Some(self.pid)
362 }
363
364 fn process_identity(&self) -> Option<ProcessIdentity> {
365 self.process_identity
366 }
367
368 async fn wait(&mut self) -> io::Result<ExitStatus> {
369 #[cfg(target_os = "macos")]
370 self.confirm_privacy_exec().await;
371 let result = self.child.wait().await;
379 #[cfg(target_os = "linux")]
380 if result.is_ok() {
381 if let Some(placement) = self.cgroup_placement.as_ref() {
382 cleanup_reaped_cgroup(placement, &self.module_id).await;
383 self.cgroup_placement = None;
386 }
387 }
388 result
389 }
390
391 #[cfg(target_os = "macos")]
392 async fn confirm_privacy_exec(&mut self) {
393 let Some(pending) = &mut self.privacy_exec else {
394 return;
395 };
396 let result = tokio::time::timeout_at(pending.deadline, async {
397 let mut record = Vec::new();
398 loop {
399 let mut ready = pending.reader.readable().await?;
400 let read = ready.try_io(|reader| {
401 use std::io::Read;
402 let mut reader = reader.get_ref();
403 let mut buffer = [0u8; 256];
404 reader.read(&mut buffer).map(|count| (count, buffer))
405 });
406 match read {
407 Ok(Ok((0, _))) => return Ok::<_, io::Error>(record),
408 Ok(Ok((count, buffer))) => {
409 if record.len() + count > 1024 {
410 return Err(io::Error::other(
411 "privacy exec refusal record is too long",
412 ));
413 }
414 record.extend_from_slice(&buffer[..count]);
415 }
416 Ok(Err(error)) => return Err(error),
417 Err(_) => continue,
418 }
419 }
420 })
421 .await;
422 #[cfg(test)]
423 let result = if let Some((address, stream)) = &mut pending.confirmation_pause {
424 use tokio::io::{AsyncReadExt, AsyncWriteExt};
425 if stream.is_none() {
426 *stream = Some(
427 tokio::net::TcpStream::connect(address.as_str())
428 .await
429 .unwrap(),
430 );
431 stream.as_mut().unwrap().write_all(b"R").await.unwrap();
432 stream
433 .as_mut()
434 .unwrap()
435 .write_all(&self.pid.to_ne_bytes())
436 .await
437 .unwrap();
438 }
439 let mut reply = [0];
440 stream
441 .as_mut()
442 .unwrap()
443 .read_exact(&mut reply)
444 .await
445 .unwrap();
446 pending.confirmation_pause = None;
447 if &reply == b"F" {
448 Ok(Err(io::Error::other("injected late exec refusal")))
449 } else {
450 assert_eq!(&reply, b"X");
451 result
452 }
453 } else {
454 result
455 };
456 let pending = self.privacy_exec.as_ref().expect("pending exec");
459 let cause = match result {
460 Err(_) => Some("privacy identity trampoline did not exec within 5s".to_string()),
461 Ok(Err(error)) => Some(format!(
462 "privacy identity exec acknowledgement failed: {error}"
463 )),
464 Ok(Ok(record)) if !record.is_empty() => Some(
465 std::str::from_utf8(&record)
466 .ok()
467 .and_then(|record| {
468 record
469 .trim()
470 .strip_prefix(subc_os::privacy_identity::EXEC_REFUSAL_TAG)
471 })
472 .filter(|cause| !cause.is_empty())
473 .unwrap_or("invalid privacy exec refusal record")
474 .to_string(),
475 ),
476 Ok(Ok(_)) => match self.child.try_wait() {
477 Ok(Some(_status)) => None,
481 Err(error) => Some(format!("privacy identity trampoline wait failed: {error}")),
482 Ok(None) => {
483 let image = observe_spawned_image(self.pid);
484 if let Some(image) = image.filter(|image| {
485 image.executable.is_some()
486 && image.executable != pending.trampoline
487 && (image.executable == pending.expected || pending.script)
488 }) {
489 if let Err(error) = self
490 .roster_guard
491 .as_ref()
492 .map(|guard| guard.confirm_executable(image))
493 .transpose()
494 {
495 Some(format!(
496 "could not durably record confirmed module image: {error}"
497 ))
498 } else {
499 let _ = self.report_ready.set(());
500 info!(module_id = %pending.module_id, pid = self.pid,
501 "module spawned with own privacy identity (responsibility disclaimed)");
502 None
503 }
504 } else if image.is_none()
505 || image.is_some_and(|image| image.executable.is_none())
506 {
507 match tokio::time::timeout_at(pending.deadline, self.child.wait()).await {
514 Ok(Ok(_status)) => None,
515 Ok(Err(error)) => Some(format!("privacy identity trampoline wait failed: {error}")),
516 Err(_) => Some("privacy identity module executable remained unreadable until the 5s exec deadline".to_string()),
517 }
518 } else {
519 Some("privacy identity trampoline executable mismatch: expected the module program, not ck-subc".to_string())
520 }
521 }
522 },
523 };
524 let pending = self.privacy_exec.take().expect("pending exec");
525 self.exec.0.finish(if cause.is_some() {
528 crate::activity::ExecState::Refused
529 } else {
530 crate::activity::ExecState::CommittedExec
531 });
532 if let Some(cause) = cause {
533 warn!(module_id = %pending.module_id, pid = self.pid, %cause, "privacy identity trampoline refused module spawn");
534 self.spawn_failure = Some(cause);
535 if let Some(pid) = rustix::process::Pid::from_raw(self.pid as i32) {
538 let _ = rustix::process::kill_process_group(pid, rustix::process::Signal::KILL);
539 }
540 let _ = self.child.start_kill();
541 }
542 }
543
544 fn release_roster(&mut self) {
548 self.roster_guard = None;
549 }
550
551 fn start_kill(&mut self) -> io::Result<()> {
564 #[cfg(windows)]
565 if let Some(job) = &self.job {
566 if let Err(error) = job.terminate() {
567 debug!(
568 error = %error,
569 "job termination failed; the direct-child kill still owns the outcome"
570 );
571 }
572 }
573 #[cfg(target_os = "linux")]
574 kill_module_cgroup(self.cgroup_placement.as_ref(), &self.module_id);
575 self.child.start_kill()
576 }
577
578 async fn drain_stderr(&mut self, module_id: &str) {
579 if let Some(pump) = self.stdout_pump.take() {
580 settle_stdout_pump(module_id, pump, STDERR_PUMP_DRAIN_TIMEOUT).await;
581 }
582
583 let Some(pump) = self.stderr_pump.take() else {
584 return;
585 };
586 settle_stderr_pump(
587 module_id,
588 &self.stderr_ring,
589 pump,
590 STDERR_PUMP_DRAIN_TIMEOUT,
591 )
592 .await;
593 }
594}
595
596async fn settle_stdout_pump(module_id: &str, mut pump: JoinHandle<()>, bound: Duration) {
600 match timeout(bound, &mut pump).await {
601 Ok(Ok(())) => {}
602 Ok(Err(error)) => {
603 warn!(module_id, error = %error, "stdout pump ended unexpectedly");
604 }
605 Err(_) => {
606 drop(pump);
607 warn!(
608 module_id,
609 waited = ?bound,
610 "stdout pipe of the exited process has not drained; its reader keeps running without delaying the restart"
611 );
612 }
613 }
614}
615
616struct StderrPump {
619 task: JoinHandle<()>,
620 generation: u64,
621}
622
623async fn settle_stderr_pump(
629 module_id: &str,
630 ring: &Arc<Mutex<StderrRing>>,
631 pump: StderrPump,
632 bound: Duration,
633) {
634 let lock = || ring.lock().unwrap_or_else(|poisoned| poisoned.into_inner());
635 let StderrPump {
636 mut task,
637 generation,
638 } = pump;
639 lock().retire_pump(generation);
640 match timeout(bound, &mut task).await {
641 Ok(Ok(())) => {}
642 Ok(Err(err)) => {
643 let mut ring = lock();
644 ring.mark_incomplete_from(generation, format!("stderr pump ended unexpectedly: {err}"));
645 ring.finish_pump(generation);
646 warn!(module_id, error = %err, "stderr pump ended before clean EOF");
647 }
648 Err(_) => {
649 drop(task);
651 lock().mark_pump_late(
652 generation,
653 format!(
654 "stderr of the exited process had not reached EOF {bound:?} after it was \
655 retired (a descendant may still hold the pipe open); lines it still \
656 writes are kept in that process's section"
657 ),
658 );
659 warn!(
660 module_id,
661 waited = ?bound,
662 "stderr pipe of the exited process is still open; its reader keeps running without delaying the restart"
663 );
664 }
665 }
666}
667
668fn registration_release_events() -> &'static watch::Sender<u64> {
669 static EVENTS: OnceLock<watch::Sender<u64>> = OnceLock::new();
670 EVENTS.get_or_init(|| {
671 let (sender, _receiver) = watch::channel(0);
672 sender
673 })
674}
675
676pub(crate) fn notify_registration_release() {
677 let events = registration_release_events();
678 let next_generation = (*events.borrow()).wrapping_add(1);
679 events.send_replace(next_generation);
680}
681
682#[derive(Debug, Clone, PartialEq, Eq)]
684pub struct ModuleSpec {
685 pub module_id: String,
686 pub program: PathBuf,
687 pub args: Vec<String>,
688 pub env: Vec<(String, String)>,
689 pub reserved: bool,
694 pub reserved_prefixes: Vec<String>,
699 pub protocol: ModuleProtocol,
718 pub overlap: ModuleOverlap,
723}
724
725#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)]
732pub enum ModuleOverlap {
733 #[default]
735 Exclusive,
736 Safe,
748}
749
750impl ModuleOverlap {
751 pub fn as_str(self) -> &'static str {
752 match self {
753 Self::Exclusive => "exclusive",
754 Self::Safe => "safe",
755 }
756 }
757}
758
759pub const SUBC_SPAWN_ROLE_ENV: &str = "SUBC_SPAWN_ROLE";
769pub const SPAWN_ROLE_SWAP_CANDIDATE: &str = "swap_candidate";
771pub const DEFAULT_SWAP_READY_TIMEOUT: Duration = Duration::from_secs(100);
776
777#[derive(Debug, Clone, Copy, PartialEq, Eq)]
795pub struct RestartPolicy {
796 pub max_restarts: u32,
797 pub backoff: Duration,
800 pub max_backoff: Duration,
802 pub window: Duration,
806}
807
808impl RestartPolicy {
809 pub fn new(max_restarts: u32, backoff: Duration) -> Self {
813 Self {
814 max_restarts,
815 backoff,
816 max_backoff: DEFAULT_MAX_BACKOFF,
817 window: DEFAULT_RESTART_WINDOW,
818 }
819 }
820
821 pub fn with_max_backoff(mut self, max_backoff: Duration) -> Self {
822 self.max_backoff = max_backoff;
823 self
824 }
825
826 pub fn with_window(mut self, window: Duration) -> Self {
827 self.window = window;
828 self
829 }
830
831 fn delay_for_restart(&self, restart_in_window: u32) -> Duration {
836 if self.backoff.is_zero() || self.max_backoff.is_zero() {
837 return Duration::ZERO;
838 }
839
840 let mut delay = self.backoff;
841 for _ in 0..restart_in_window {
842 if delay >= self.max_backoff {
843 return self.max_backoff;
844 }
845 delay = delay
846 .checked_mul(10)
847 .unwrap_or(self.max_backoff)
848 .min(self.max_backoff);
849 }
850 delay.min(self.max_backoff)
851 }
852
853 fn budget_exhausted_detail(&self) -> String {
858 format!(
859 "crash budget exhausted: max_restarts={} within window_secs={}",
860 self.max_restarts,
861 self.window.as_secs()
862 )
863 }
864}
865
866impl Default for RestartPolicy {
867 fn default() -> Self {
868 Self {
869 max_restarts: DEFAULT_MAX_RESTARTS,
870 backoff: DEFAULT_BACKOFF,
871 max_backoff: DEFAULT_MAX_BACKOFF,
872 window: DEFAULT_RESTART_WINDOW,
873 }
874 }
875}
876
877#[derive(Debug, Clone, Copy, PartialEq, Eq)]
878struct CrashRestartSchedule {
879 restart_in_window: u32,
880 delay: Duration,
881}
882
883fn daemon_will_restart(
890 state: &mut SupervisorSnapshot,
891 policy: &RestartPolicy,
892 now: Instant,
893) -> bool {
894 state.enabled && state.crash_restarts_in_window(policy.window, now) < policy.max_restarts
895}
896
897const DEFAULT_HEALTH_CADENCE: Duration = Duration::from_secs(30);
898const DEFAULT_HEALTH_DEADLINE: Duration = Duration::from_secs(5);
899const DEFAULT_HEALTH_FAILURE_THRESHOLD: u32 = 3;
900const MAX_HEALTH_METRICS_BYTES: usize = 16 * 1024;
901
902#[derive(Debug, Clone, Copy, PartialEq, Eq)]
903pub enum HealthAction {
904 Report,
905 Restart,
906 Alert,
907}
908
909impl fmt::Display for HealthAction {
910 fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
911 f.write_str(match self {
912 Self::Report => "report",
913 Self::Restart => "restart",
914 Self::Alert => "alert",
915 })
916 }
917}
918
919#[derive(Debug, Clone, PartialEq, Eq)]
920pub struct HealthConfig {
921 pub http: Option<String>,
925 pub cadence: Duration,
926 pub deadline: Duration,
927 pub failure_threshold: u32,
928 pub on_degraded: HealthAction,
929 pub on_failing: HealthAction,
930 pub critical: bool,
931}
932
933impl Default for HealthConfig {
934 fn default() -> Self {
935 Self {
936 http: None,
937 cadence: DEFAULT_HEALTH_CADENCE,
938 deadline: DEFAULT_HEALTH_DEADLINE,
939 failure_threshold: DEFAULT_HEALTH_FAILURE_THRESHOLD,
940 on_degraded: HealthAction::Report,
941 on_failing: HealthAction::Report,
942 critical: false,
943 }
944 }
945}
946
947#[derive(Debug, Clone, PartialEq)]
965pub struct ModuleHealthStatus {
966 pub status: SupervisorHealthStatus,
967 pub last_probe_ms: Option<u64>,
968 pub detail: Option<String>,
969 pub metrics: Option<Value>,
970 pub consecutive_failures: u32,
971 pub late_answer_count: u64,
974 pub last_late_answer_latency_ms: Option<u64>,
976 pub last_action: Option<String>,
977 pub last_action_ms: Option<u64>,
981}
982
983impl Default for ModuleHealthStatus {
984 fn default() -> Self {
985 Self {
986 status: SupervisorHealthStatus::Unknown,
987 last_probe_ms: None,
988 detail: None,
989 metrics: None,
990 consecutive_failures: 0,
991 late_answer_count: 0,
992 last_late_answer_latency_ms: None,
993 last_action: None,
994 last_action_ms: None,
995 }
996 }
997}
998
999#[derive(Debug, Clone, Copy, PartialEq, Eq)]
1001pub enum ModuleState {
1002 Starting,
1003 Running,
1004 Unresponsive,
1005 Restarting,
1006 Draining,
1007 Stopped,
1008 Failed,
1009 Disabled,
1010}
1011
1012impl fmt::Display for ModuleState {
1013 fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
1014 f.write_str(match self {
1015 Self::Starting => "starting",
1016 Self::Running => "running",
1017 Self::Unresponsive => "unresponsive",
1018 Self::Restarting => "restarting",
1019 Self::Draining => "draining",
1020 Self::Stopped => "stopped",
1021 Self::Failed => "failed",
1022 Self::Disabled => "disabled",
1023 })
1024 }
1025}
1026
1027#[derive(Debug, Clone, Copy, PartialEq, Eq)]
1029pub enum ExitKind {
1030 Clean,
1031 Crash,
1032 DeliberateSeverance,
1033}
1034
1035impl From<ExitKind> for TerminalExitKind {
1036 fn from(kind: ExitKind) -> Self {
1037 match kind {
1038 ExitKind::Clean => Self::Clean,
1039 ExitKind::Crash => Self::Crash,
1040 ExitKind::DeliberateSeverance => Self::DeliberateSeverance,
1041 }
1042 }
1043}
1044
1045#[derive(Debug, Clone, Copy, PartialEq, Eq)]
1048pub(crate) struct ProcessIdentity {
1049 pub(crate) pid: u32,
1050 pub(crate) start_time: u64,
1051}
1052
1053#[derive(Debug, Clone, PartialEq, Eq)]
1055pub struct ExitReport {
1056 pub kind: ExitKind,
1057 pub code: Option<i32>,
1058 pub signal: Option<i32>,
1059 pub at_ms: u64,
1060}
1061
1062#[derive(Debug, Clone, PartialEq)]
1065pub struct ModuleStatus {
1066 pub module_id: String,
1067 pub waiting_for_previous_pid: Option<u32>,
1069 pub waiting_for_previous_detail: Option<String>,
1071 pub launch_nonce_source: Option<String>,
1076 pub state: ModuleState,
1077 pub enabled: bool,
1078 pub process_alive: bool,
1079 pub registration_active: bool,
1080 pub protocol: ModuleProtocol,
1085 pub live: bool,
1096 pub restart_count: u32,
1100 pub lifetime_restarts: u32,
1104 pub spawn_generation: u64,
1105 pub max_restarts: u32,
1110 pub restart_window: Duration,
1114 pub drain_timeout: Duration,
1118 pub restart_backoff: Duration,
1119 pub restart_max_backoff: Duration,
1120 pub pid: Option<u32>,
1125 pub spawned_at_ms: Option<u64>,
1126 pub spawned_from: Option<PathBuf>,
1127 pub process_start_time: Option<u64>,
1128 pub last_exit: Option<ExitReport>,
1129 pub health: ModuleHealthStatus,
1130}
1131
1132#[cfg(test)]
1133#[derive(Debug, Clone, Copy, PartialEq)]
1134struct ActorSelectCheckpoint {
1135 generation: u64,
1136 turn: u64,
1137 registered_connection: Option<ConnectionId>,
1138 next_probe_at: Option<Instant>,
1139 wake_after: Option<Duration>,
1140}
1141
1142#[derive(Debug, Clone, PartialEq)]
1143struct SupervisorSnapshot {
1144 exec_outcome: Option<Arc<crate::activity::ExecOutcome>>,
1145 #[cfg(windows)]
1146 pipe_consumed: Option<Arc<OnceLock<()>>>,
1147 #[cfg(test)]
1149 actor_turns: u64,
1150 #[cfg(test)]
1153 actor_select: Option<ActorSelectCheckpoint>,
1154 state: ModuleState,
1155 enabled: bool,
1156 process_alive: bool,
1157 spawned_protocol: Option<ModuleProtocol>,
1158 spawn_failure: Option<String>,
1159 waiting_for_previous_pid: Option<u32>,
1160 waiting_for_previous_detail: Option<String>,
1161 crash_restarts: VecDeque<Instant>,
1167 lifetime_restarts: u32,
1168 spawn_generation: u64,
1177 pid: Option<u32>,
1178 #[cfg(target_os = "macos")]
1179 report_ready: Option<Arc<OnceLock<()>>>,
1180 reaped_pid: Option<u32>,
1182 respawn_pending: bool,
1184 coalesced_restart_pending: bool,
1186 spawned_at_ms: Option<u64>,
1187 spawned_from: Option<PathBuf>,
1188 spawned_file_identity: Option<SpawnedFileIdentity>,
1189 process_start_time: Option<u64>,
1190 deliberate_severance: Option<ProcessIdentity>,
1191 last_exit: Option<ExitReport>,
1192 drain_disposition_detail: Option<String>,
1194 health: ModuleHealthStatus,
1195 in_alternate_slot: bool,
1200 draining_to_replace: bool,
1207 configuration_updated_since_spawn: bool,
1213}
1214
1215impl SupervisorSnapshot {
1216 fn reported_pid(&self) -> Option<u32> {
1221 #[cfg(target_os = "macos")]
1222 if self
1223 .report_ready
1224 .as_ref()
1225 .is_some_and(|ready| ready.get().is_none())
1226 {
1227 return None;
1228 }
1229 self.pid
1230 }
1231
1232 fn starting() -> Self {
1233 Self::new(ModuleState::Starting, true)
1234 }
1235
1236 fn disabled() -> Self {
1237 Self::new(ModuleState::Disabled, false)
1238 }
1239
1240 fn crash_restarts_in_window(&mut self, window: Duration, now: Instant) -> u32 {
1244 while let Some(oldest) = self.crash_restarts.front() {
1245 if now.duration_since(*oldest) > window {
1246 self.crash_restarts.pop_front();
1247 } else {
1248 break;
1249 }
1250 }
1251 u32::try_from(self.crash_restarts.len()).unwrap_or(u32::MAX)
1252 }
1253
1254 fn record_crash_restart(&mut self, policy: &RestartPolicy, now: Instant) {
1260 self.crash_restarts.push_back(now);
1261 while self.crash_restarts.len() > policy.max_restarts as usize {
1262 self.crash_restarts.pop_front();
1263 }
1264 self.lifetime_restarts += 1;
1265 }
1266
1267 fn next_crash_restart(
1271 &mut self,
1272 policy: &RestartPolicy,
1273 now: Instant,
1274 ) -> Option<CrashRestartSchedule> {
1275 let restart_in_window = self.crash_restarts_in_window(policy.window, now);
1276 if restart_in_window >= policy.max_restarts {
1277 return None;
1278 }
1279 self.record_crash_restart(policy, now);
1280 Some(CrashRestartSchedule {
1281 restart_in_window,
1282 delay: policy.delay_for_restart(restart_in_window),
1283 })
1284 }
1285
1286 fn clear_crash_restarts(&mut self) {
1291 self.crash_restarts.clear();
1292 }
1293
1294 fn new(state: ModuleState, enabled: bool) -> Self {
1295 Self {
1296 exec_outcome: None,
1297 #[cfg(test)]
1298 actor_turns: 0,
1299 #[cfg(test)]
1300 actor_select: None,
1301 state,
1302 enabled,
1303 process_alive: false,
1304 #[cfg(windows)]
1305 pipe_consumed: None,
1306 spawned_protocol: None,
1307 spawn_failure: None,
1308 waiting_for_previous_pid: None,
1309 waiting_for_previous_detail: None,
1310 crash_restarts: VecDeque::new(),
1311 lifetime_restarts: 0,
1312 spawn_generation: 0,
1313 pid: None,
1314 #[cfg(target_os = "macos")]
1315 report_ready: None,
1316 reaped_pid: None,
1317 respawn_pending: false,
1318 coalesced_restart_pending: false,
1319 spawned_at_ms: None,
1320 spawned_from: None,
1321 spawned_file_identity: None,
1322 process_start_time: None,
1323 deliberate_severance: None,
1324 last_exit: None,
1325 drain_disposition_detail: None,
1326 health: ModuleHealthStatus::default(),
1327 in_alternate_slot: false,
1328 draining_to_replace: false,
1329 configuration_updated_since_spawn: false,
1330 }
1331 }
1332}
1333
1334type SharedSnapshot = Arc<Mutex<SupervisorSnapshot>>;
1335
1336type SpawnSubscriberKey = (ConnectionId, u64);
1337
1338#[derive(Debug)]
1339struct SpawnSubscriber {
1340 version: u8,
1341 frames: mpsc::Sender<Frame>,
1342 lagged: Option<oneshot::Sender<SpawnCursor>>,
1346}
1347
1348#[derive(Debug)]
1349struct SpawnEventState {
1350 daemon_incarnation: String,
1351 seq: u64,
1352 capacity: usize,
1353 live: HashMap<String, LiveSpawn>,
1354 generations: HashMap<String, u64>,
1355 events: VecDeque<SpawnEvent>,
1356 subscribers: HashMap<SpawnSubscriberKey, SpawnSubscriber>,
1357}
1358
1359impl Default for SpawnEventState {
1360 fn default() -> Self {
1361 Self {
1362 daemon_incarnation: "unconfigured".to_string(),
1363 seq: 0,
1364 capacity: SPAWN_EVENT_RING_CAPACITY,
1365 live: HashMap::new(),
1366 generations: HashMap::new(),
1367 events: VecDeque::new(),
1368 subscribers: HashMap::new(),
1369 }
1370 }
1371}
1372
1373#[derive(Debug, Clone, Default)]
1374struct SpawnEventFeed(
1375 Arc<Mutex<SpawnEventState>>,
1376 #[cfg(test)] Arc<Mutex<Option<std::net::TcpStream>>>,
1377);
1378
1379#[derive(Debug, Clone, PartialEq, Eq)]
1380pub(crate) enum SpawnSubscribeRefusal {
1381 ForeignIncarnation { current: String },
1382 TooOld { oldest: SpawnCursor },
1383 Frame(String),
1384}
1385
1386impl SpawnEventFeed {
1387 fn configure_incarnation(&self, daemon_incarnation: String) {
1388 let mut state = self.0.lock().unwrap_or_else(|p| p.into_inner());
1389 state.daemon_incarnation = daemon_incarnation;
1390 state.seq = 0;
1391 state.live.clear();
1392 state.generations.clear();
1393 state.events.clear();
1394 state.subscribers.clear();
1395 }
1396
1397 fn cursor(state: &SpawnEventState) -> SpawnCursor {
1398 SpawnCursor {
1399 daemon_incarnation: state.daemon_incarnation.clone(),
1400 seq: state.seq,
1401 }
1402 }
1403
1404 fn snapshot(&self) -> SpawnSnapshot {
1405 let state = self.0.lock().unwrap_or_else(|p| p.into_inner());
1406 let mut live = state.live.values().cloned().collect::<Vec<_>>();
1407 live.sort_by(|left, right| left.module_id.cmp(&right.module_id));
1408 SpawnSnapshot {
1409 cursor: Self::cursor(&state),
1410 ring_bound: state.capacity as u64,
1411 live,
1412 }
1413 }
1414
1415 fn emit_spawned(&self, module_id: &str, pid: u32, spawned_at_ms: u64) -> u64 {
1416 let mut state = self.0.lock().unwrap_or_else(|p| p.into_inner());
1417 let generation = state
1418 .generations
1419 .get(module_id)
1420 .copied()
1421 .unwrap_or(0)
1422 .checked_add(1)
1423 .expect("spawn generation exhausted");
1424 state.generations.insert(module_id.to_string(), generation);
1425 let live = LiveSpawn {
1426 module_id: module_id.to_string(),
1427 spawn_generation: generation,
1428 pid,
1429 spawned_at_ms,
1430 };
1431 state.live.insert(module_id.to_string(), live);
1432 Self::emit_locked(
1433 &mut state,
1434 SpawnEventKind::Spawned,
1435 module_id.to_string(),
1436 generation,
1437 pid,
1438 None,
1439 None,
1440 );
1441 generation
1442 }
1443
1444 fn emit_exited(&self, module_id: &str, exit_code: Option<i32>, exit_signal: Option<i32>) {
1445 let mut state = self.0.lock().unwrap_or_else(|p| p.into_inner());
1446 let Some(live) = state.live.remove(module_id) else {
1447 warn!(
1448 module_id,
1449 "terminal record had no live spawn event identity"
1450 );
1451 return;
1452 };
1453 Self::emit_locked(
1454 &mut state,
1455 SpawnEventKind::Exited,
1456 module_id.to_string(),
1457 live.spawn_generation,
1458 live.pid,
1459 exit_code,
1460 exit_signal,
1461 );
1462 }
1463
1464 fn emit_superseded_exited(
1471 &self,
1472 module_id: &str,
1473 spawn_generation: u64,
1474 pid: u32,
1475 exit_code: Option<i32>,
1476 exit_signal: Option<i32>,
1477 ) {
1478 let mut state = self.0.lock().unwrap_or_else(|p| p.into_inner());
1479 if state
1480 .live
1481 .get(module_id)
1482 .is_some_and(|live| live.spawn_generation == spawn_generation)
1483 {
1484 state.live.remove(module_id);
1485 }
1486 Self::emit_locked(
1487 &mut state,
1488 SpawnEventKind::Exited,
1489 module_id.to_string(),
1490 spawn_generation,
1491 pid,
1492 exit_code,
1493 exit_signal,
1494 );
1495 }
1496
1497 #[allow(clippy::too_many_arguments)]
1498 fn emit_locked(
1499 state: &mut SpawnEventState,
1500 kind: SpawnEventKind,
1501 module_id: String,
1502 spawn_generation: u64,
1503 pid: u32,
1504 exit_code: Option<i32>,
1505 exit_signal: Option<i32>,
1506 ) {
1507 state.seq = state
1508 .seq
1509 .checked_add(1)
1510 .expect("spawn event sequence exhausted");
1511 let event = SpawnEvent {
1512 cursor: Self::cursor(state),
1513 kind,
1514 module_id,
1515 spawn_generation,
1516 pid,
1517 exit_code,
1518 exit_signal,
1519 };
1520 state.events.push_back(event.clone());
1521 while state.events.len() > state.capacity {
1522 state.events.pop_front();
1523 }
1524 let body = match serde_json::to_vec(&event) {
1525 Ok(body) => body,
1526 Err(error) => {
1527 error!(%error, "failed to serialize supervisor spawn event");
1528 return;
1529 }
1530 };
1531 state.subscribers.retain(|(connection_id, corr), subscriber| {
1532 let frame = Frame::build_with_version(
1533 subscriber.version,
1534 FrameType::StreamData,
1535 control_flags(),
1536 0,
1537 0,
1538 *corr,
1539 body.clone(),
1540 );
1541 match frame {
1542 Ok(frame) => {
1543 if subscriber.frames.try_send(frame).is_ok() {
1544 true
1545 } else {
1546 warn!(connection_id = connection_id.get(), corr, "dropping lagged supervisor spawn subscriber");
1547 if let Some(lagged) = subscriber.lagged.take() {
1548 let _ = lagged.send(event.cursor.clone());
1549 }
1550 false
1551 }
1552 }
1553 Err(error) => {
1554 warn!(connection_id = connection_id.get(), corr, %error, "dropping supervisor spawn subscriber after frame build failure");
1555 false
1556 }
1557 }
1558 });
1559 }
1560
1561 fn subscribe(
1562 &self,
1563 connection_id: ConnectionId,
1564 corr: u64,
1565 version: u8,
1566 since: Option<SpawnCursor>,
1567 sink: FrameSink,
1568 ) -> Result<(), SpawnSubscribeRefusal> {
1569 let (frames, mut receiver) = mpsc::channel(SPAWN_SUBSCRIBER_BUFFER);
1570 let (lagged, mut lagged_rx) = oneshot::channel::<SpawnCursor>();
1571 {
1572 let mut state = self.0.lock().unwrap_or_else(|p| p.into_inner());
1573 let replay = if let Some(since) = since {
1574 if since.daemon_incarnation != state.daemon_incarnation {
1575 return Err(SpawnSubscribeRefusal::ForeignIncarnation {
1576 current: state.daemon_incarnation.clone(),
1577 });
1578 }
1579 if let Some(oldest) = state.events.front().map(|event| event.cursor.clone()) {
1580 if since.seq < oldest.seq.saturating_sub(1) {
1581 return Err(SpawnSubscribeRefusal::TooOld { oldest });
1582 }
1583 }
1584 state
1585 .events
1586 .iter()
1587 .filter(|event| event.cursor.seq > since.seq)
1588 .cloned()
1589 .collect::<Vec<_>>()
1590 } else {
1591 Vec::new()
1592 };
1593 for event in replay {
1594 let body = serde_json::to_vec(&event)
1595 .map_err(|error| SpawnSubscribeRefusal::Frame(error.to_string()))?;
1596 let frame = Frame::build_with_version(
1597 version,
1598 FrameType::StreamData,
1599 control_flags(),
1600 0,
1601 0,
1602 corr,
1603 body,
1604 )
1605 .map_err(|error| SpawnSubscribeRefusal::Frame(error.to_string()))?;
1606 frames
1607 .try_send(frame)
1608 .map_err(|error| SpawnSubscribeRefusal::Frame(error.to_string()))?;
1609 }
1610 state.subscribers.insert(
1611 (connection_id, corr),
1612 SpawnSubscriber {
1613 version,
1614 frames,
1615 lagged: Some(lagged),
1616 },
1617 );
1618 }
1619 tokio::spawn(async move {
1630 while let Some(frame) = receiver.recv().await {
1631 if sink.send(frame).await.is_err() {
1632 return;
1633 }
1634 }
1635 let Ok(first_undelivered) = lagged_rx.try_recv() else {
1636 return;
1637 };
1638 match spawn_subscriber_lagged_frame(version, corr, first_undelivered) {
1639 Ok(frame) => {
1640 let _ = sink.send(frame).await;
1641 }
1642 Err(error) => {
1643 error!(%error, corr, "failed to build lagged spawn subscriber terminal frame");
1644 }
1645 }
1646 });
1647 Ok(())
1648 }
1649
1650 fn cancel(&self, connection_id: ConnectionId, corr: u64) -> bool {
1651 let Some(subscriber) = self
1652 .0
1653 .lock()
1654 .unwrap_or_else(|p| p.into_inner())
1655 .subscribers
1656 .remove(&(connection_id, corr))
1657 else {
1658 return false;
1659 };
1660 if let Ok(frame) = Frame::build_with_version(
1661 subscriber.version,
1662 FrameType::StreamEnd,
1663 control_flags(),
1664 0,
1665 0,
1666 corr,
1667 Vec::new(),
1668 ) {
1669 tokio::spawn(async move {
1670 let _ = subscriber.frames.send(frame).await;
1671 });
1672 }
1673 true
1674 }
1675
1676 fn remove_connection(&self, connection_id: ConnectionId) {
1677 self.0
1678 .lock()
1679 .unwrap_or_else(|p| p.into_inner())
1680 .subscribers
1681 .retain(|(subscriber_connection, _), _| *subscriber_connection != connection_id);
1682 }
1683
1684 #[cfg(any(test, feature = "test-support"))]
1685 fn set_capacity(&self, capacity: usize) {
1686 self.0.lock().unwrap_or_else(|p| p.into_inner()).capacity = capacity;
1687 }
1688
1689 #[cfg(any(test, feature = "test-support"))]
1690 fn subscriber_count(&self) -> usize {
1691 self.0
1692 .lock()
1693 .unwrap_or_else(|p| p.into_inner())
1694 .subscribers
1695 .len()
1696 }
1697}
1698
1699fn spawn_subscriber_lagged_frame(
1702 version: u8,
1703 corr: u64,
1704 first_undelivered: SpawnCursor,
1705) -> Result<Frame, String> {
1706 let body = serde_json::to_vec(&subc_protocol::ErrorBody {
1707 code: SPAWN_SUBSCRIBER_LAGGED_CODE.to_string(),
1708 message: "spawn subscriber fell behind and was dropped; resubscribe from the last cursor received"
1709 .to_string(),
1710 detail: Some(serde_json::json!({
1711 "first_undelivered_cursor": first_undelivered
1712 })),
1713 })
1714 .map_err(|error| error.to_string())?;
1715 Frame::build_with_version(version, FrameType::Error, control_flags(), 0, 0, corr, body)
1716 .map_err(|error| error.to_string())
1717}
1718
1719pub trait ModuleProcessLiveness: Send + Sync {
1720 fn process_live(&self, module_id: &str) -> Option<bool>;
1721
1722 fn process_replacing(&self, _module_id: &str) -> bool {
1728 false
1729 }
1730}
1731
1732#[derive(Debug, Clone, Default)]
1734pub struct SupervisorProcessLiveness {
1735 snapshots: Arc<Mutex<HashMap<String, SharedSnapshot>>>,
1736}
1737
1738impl SupervisorProcessLiveness {
1739 pub fn new() -> Self {
1740 Self::default()
1741 }
1742
1743 fn track(&self, module_id: String, snapshot: SharedSnapshot) {
1744 let mut snapshots = self
1745 .snapshots
1746 .lock()
1747 .unwrap_or_else(|poisoned| poisoned.into_inner());
1748 snapshots.insert(module_id, snapshot);
1749 }
1750
1751 fn untrack_if_current(&self, module_id: &str, snapshot: &SharedSnapshot) {
1752 let mut snapshots = self
1753 .snapshots
1754 .lock()
1755 .unwrap_or_else(|poisoned| poisoned.into_inner());
1756 let is_current = snapshots
1757 .get(module_id)
1758 .map(|tracked| Arc::ptr_eq(tracked, snapshot))
1759 .unwrap_or(false);
1760 if is_current {
1761 snapshots.remove(module_id);
1762 }
1763 }
1764}
1765
1766impl ModuleProcessLiveness for SupervisorProcessLiveness {
1767 fn process_live(&self, module_id: &str) -> Option<bool> {
1768 let snapshot = {
1769 let snapshots = self
1770 .snapshots
1771 .lock()
1772 .unwrap_or_else(|poisoned| poisoned.into_inner());
1773 snapshots.get(module_id).cloned()
1774 }?;
1775 let snapshot = snapshot
1776 .lock()
1777 .unwrap_or_else(|poisoned| poisoned.into_inner());
1778 Some(snapshot.state == ModuleState::Running && snapshot.process_alive)
1779 }
1780
1781 fn process_replacing(&self, module_id: &str) -> bool {
1782 let Some(snapshot) = self
1783 .snapshots
1784 .lock()
1785 .unwrap_or_else(|poisoned| poisoned.into_inner())
1786 .get(module_id)
1787 .cloned()
1788 else {
1789 return false;
1790 };
1791 let snapshot = snapshot
1792 .lock()
1793 .unwrap_or_else(|poisoned| poisoned.into_inner());
1794 snapshot.enabled
1795 && match snapshot.state {
1796 ModuleState::Restarting => true,
1797 ModuleState::Draining => snapshot.draining_to_replace,
1798 ModuleState::Starting
1799 | ModuleState::Running
1800 | ModuleState::Unresponsive
1801 | ModuleState::Stopped
1802 | ModuleState::Failed
1803 | ModuleState::Disabled => false,
1804 }
1805 }
1806}
1807
1808#[cfg(test)]
1809#[derive(Debug, Default)]
1810struct ReloadExitRecordGate {
1811 reached: tokio::sync::Notify,
1812 resume: tokio::sync::Notify,
1813}
1814
1815#[derive(Debug, Clone, Copy)]
1816enum RespawnKind {
1817 Spawn,
1818 Reload,
1819}
1820
1821#[derive(Debug, Clone, Copy)]
1822struct PendingRespawn {
1823 deadline: Instant,
1824 kind: RespawnKind,
1825}
1826
1827type ReloadReply = oneshot::Sender<Result<(), SuperviseError>>;
1828
1829#[derive(Debug, Clone)]
1830struct SupervisorRuntimeConfig {
1831 start_requester: Option<Principal>,
1832 pending_activity_restart: bool,
1833 provenance_probe: ExecutableIdentityProbe,
1834 scheduled_respawn: Arc<Mutex<Option<PendingRespawn>>>,
1836 deferred_reload_reply: Arc<Mutex<Option<ReloadReply>>>,
1838 restart_policy: RestartPolicy,
1839 drain_timeout: Duration,
1842 effective_drain_timeout: Arc<Mutex<Duration>>,
1845 default_drain_timeout: Duration,
1848 health: HealthConfig,
1849 connection_file_path: Option<PathBuf>,
1850 capture_logs_dir: Option<PathBuf>,
1851 forwarding: Option<Arc<ForwardingTable>>,
1852 supervisor_handle: Option<SupervisorHandle>,
1855 stderr_ring: Arc<Mutex<StderrRing>>,
1862 terminal_ring: Arc<Mutex<TerminalRing>>,
1863 spawn_events: SpawnEventFeed,
1864 child_roster: ChildRoster,
1865 #[cfg(target_os = "linux")]
1866 cgroup_placement: Option<subc_cgroup::Placement>,
1867 #[cfg(test)]
1868 test_seed_stale_facts_before_enable_spawn: bool,
1869 #[cfg(test)]
1870 test_reload_exit_record_gate: Option<Arc<ReloadExitRecordGate>>,
1871}
1872
1873#[derive(Debug, Clone, PartialEq, Eq)]
1874struct SupervisedConfiguration {
1875 spec: ModuleSpec,
1876 health: HealthConfig,
1877}
1878
1879#[derive(Debug, Clone, Default)]
1885pub struct SupervisorHandle {
1886 activity: Arc<Mutex<Option<crate::activity::ActivityLedger>>>,
1887 activity_images: Arc<Mutex<HashMap<String, activity::ActivityImage>>>,
1888 modules: Arc<Mutex<HashMap<String, SupervisedModule>>>,
1889 configured_ids: Arc<Mutex<HashSet<String>>>,
1901 spawn_events: SpawnEventFeed,
1902 reserved_nonces: Arc<Mutex<HashMap<String, Option<String>>>>,
1913 removal_tombstones: Arc<Mutex<HashMap<String, u64>>>,
1919 spawn_nonces: Arc<Mutex<HashMap<String, String>>>,
1923 reserved_prefix_owners: Arc<Mutex<HashMap<String, String>>>,
1931 swaps: Arc<Mutex<HashMap<String, OpenSwap>>>,
1937 promotion_observer: PromotionObserverSlot,
1939 operation_lock: Arc<AsyncMutex<()>>,
1943}
1944
1945pub(crate) trait SwapPromotionObserver: Send + Sync {
1954 fn swap_promoted(&self, registration: &crate::registry::ModuleRegistration);
1955}
1956
1957#[derive(Clone, Default)]
1961struct PromotionObserverSlot(Arc<Mutex<Option<std::sync::Weak<dyn SwapPromotionObserver>>>>);
1962
1963impl fmt::Debug for PromotionObserverSlot {
1964 fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
1965 f.write_str("PromotionObserverSlot")
1966 }
1967}
1968
1969#[derive(Debug, Clone)]
1971struct OpenSwap {
1972 candidate_nonce: String,
1975 incumbent_nonce: Option<String>,
1980 candidate_admitted: bool,
1984}
1985
1986#[derive(Debug, Clone, Copy, PartialEq, Eq)]
1989pub(crate) enum SwapHelloAdmission {
1990 NotSwapping,
1993 Candidate,
1995 Refused,
1998}
1999
2000#[derive(Debug, Clone, PartialEq, Eq)]
2001pub(crate) enum ReservedHelloRejection {
2002 Exact {
2003 module_id: String,
2004 },
2005 Prefix {
2006 prefix: String,
2007 owner_module_id: String,
2008 },
2009}
2010
2011impl SupervisorHandle {
2012 pub fn new() -> Self {
2013 Self::default()
2014 }
2015
2016 pub(crate) fn spawn_snapshot(&self) -> SpawnSnapshot {
2017 self.spawn_events.snapshot()
2018 }
2019
2020 pub(crate) fn subscribe_spawns(
2021 &self,
2022 connection_id: ConnectionId,
2023 corr: u64,
2024 version: u8,
2025 since: Option<SpawnCursor>,
2026 sink: FrameSink,
2027 ) -> Result<(), SpawnSubscribeRefusal> {
2028 self.spawn_events
2029 .subscribe(connection_id, corr, version, since, sink)
2030 }
2031
2032 pub(crate) fn cancel_spawn_subscription(&self, connection_id: ConnectionId, corr: u64) -> bool {
2033 self.spawn_events.cancel(connection_id, corr)
2034 }
2035
2036 pub(crate) fn remove_spawn_subscribers(&self, connection_id: ConnectionId) {
2037 self.spawn_events.remove_connection(connection_id);
2038 }
2039
2040 #[cfg(any(test, feature = "test-support"))]
2041 pub fn set_spawn_event_capacity_for_test(&self, capacity: usize) {
2042 assert!(capacity > 0, "spawn event capacity must be non-zero");
2043 self.spawn_events.set_capacity(capacity);
2044 }
2045
2046 #[cfg(any(test, feature = "test-support"))]
2047 pub fn spawn_subscriber_count_for_test(&self) -> usize {
2048 self.spawn_events.subscriber_count()
2049 }
2050
2051 pub fn set_spawn_nonce(&self, module_id: &str, nonce: String) {
2054 self.spawn_nonces
2055 .lock()
2056 .unwrap_or_else(|poisoned| poisoned.into_inner())
2057 .insert(module_id.to_string(), nonce);
2058 }
2059
2060 pub fn set_reserved_nonce(&self, module_id: &str, nonce: String) {
2063 self.reserved_nonces
2064 .lock()
2065 .unwrap_or_else(|poisoned| poisoned.into_inner())
2066 .insert(module_id.to_string(), Some(nonce));
2067 }
2068
2069 pub fn set_reserved_prefixes(&self, owner_module_id: &str, prefixes: &[String]) {
2071 let mut owners = self
2072 .reserved_prefix_owners
2073 .lock()
2074 .unwrap_or_else(|poisoned| poisoned.into_inner());
2075 owners.retain(|_, owner| owner != owner_module_id);
2076 for prefix in prefixes {
2077 owners.insert(prefix.clone(), owner_module_id.to_string());
2078 }
2079 }
2080
2081 #[cfg(test)]
2083 pub(crate) fn spawn_nonce(&self, module_id: &str) -> Option<String> {
2084 self.spawn_nonces
2085 .lock()
2086 .unwrap_or_else(|poisoned| poisoned.into_inner())
2087 .get(module_id)
2088 .cloned()
2089 }
2090
2091 fn apply_identity_configuration(&self, spec: &ModuleSpec) {
2092 self.set_reserved_prefixes(&spec.module_id, &spec.reserved_prefixes);
2093 let spawn_nonce = self
2094 .spawn_nonces
2095 .lock()
2096 .unwrap_or_else(|poisoned| poisoned.into_inner())
2097 .get(&spec.module_id)
2098 .cloned();
2099 let mut reserved_nonces = self
2100 .reserved_nonces
2101 .lock()
2102 .unwrap_or_else(|poisoned| poisoned.into_inner());
2103 if spec.reserved {
2104 reserved_nonces.insert(spec.module_id.clone(), spawn_nonce);
2109 }
2110 drop(reserved_nonces);
2111 self.removal_tombstones
2115 .lock()
2116 .unwrap_or_else(|poisoned| poisoned.into_inner())
2117 .remove(&spec.module_id);
2118 }
2119
2120 pub fn reserved_hello_authorized(&self, module_id: &str, presented: Option<&str>) -> bool {
2125 self.reserved_hello_rejection(module_id, presented)
2126 .is_none()
2127 }
2128
2129 pub(crate) fn reserved_hello_rejection(
2130 &self,
2131 module_id: &str,
2132 presented: Option<&str>,
2133 ) -> Option<ReservedHelloRejection> {
2134 let nonces = self
2135 .reserved_nonces
2136 .lock()
2137 .unwrap_or_else(|poisoned| poisoned.into_inner());
2138 if let Some(expected) = nonces.get(module_id) {
2139 let authorized = match expected {
2143 Some(expected) => {
2144 presented.is_some_and(|p| constant_time_eq(expected.as_bytes(), p.as_bytes()))
2145 }
2146 None => false,
2147 };
2148 if authorized {
2149 return None;
2150 }
2151 return Some(ReservedHelloRejection::Exact {
2152 module_id: module_id.to_string(),
2153 });
2154 }
2155 drop(nonces);
2156
2157 let matched_prefix = self
2158 .reserved_prefix_owners
2159 .lock()
2160 .unwrap_or_else(|poisoned| poisoned.into_inner())
2161 .iter()
2162 .filter(|(prefix, _)| module_id.starts_with(prefix.as_str()))
2163 .max_by_key(|(prefix, _)| prefix.len())
2164 .map(|(prefix, owner)| (prefix.clone(), owner.clone()));
2165 let (prefix, owner_module_id) = matched_prefix?;
2166
2167 let authorized = presented.is_some_and(|presented| {
2168 self.spawn_nonces
2169 .lock()
2170 .unwrap_or_else(|poisoned| poisoned.into_inner())
2171 .get(&owner_module_id)
2172 .is_some_and(|expected| constant_time_eq(expected.as_bytes(), presented.as_bytes()))
2173 || self.swap_nonce_matches(&owner_module_id, presented)
2176 });
2177 if authorized {
2178 None
2179 } else {
2180 Some(ReservedHelloRejection::Prefix {
2181 prefix,
2182 owner_module_id,
2183 })
2184 }
2185 }
2186
2187 pub fn spawned_consumer_authorized(&self, module_id: &str, presented: &str) -> bool {
2192 if presented.is_empty() {
2193 return false;
2194 }
2195 let nonces = self
2196 .spawn_nonces
2197 .lock()
2198 .unwrap_or_else(|poisoned| poisoned.into_inner());
2199 let current = nonces
2200 .get(module_id)
2201 .is_some_and(|expected| constant_time_eq(expected.as_bytes(), presented.as_bytes()));
2202 drop(nonces);
2203 current || self.swap_nonce_matches(module_id, presented)
2208 }
2209
2210 fn swap_nonce_matches(&self, module_id: &str, presented: &str) -> bool {
2212 let swaps = self
2213 .swaps
2214 .lock()
2215 .unwrap_or_else(|poisoned| poisoned.into_inner());
2216 swaps.get(module_id).is_some_and(|swap| {
2217 constant_time_eq(swap.candidate_nonce.as_bytes(), presented.as_bytes())
2218 || swap.incumbent_nonce.as_deref().is_some_and(|incumbent| {
2219 constant_time_eq(incumbent.as_bytes(), presented.as_bytes())
2220 })
2221 })
2222 }
2223
2224 pub(crate) fn open_swap(&self, module_id: &str, candidate_nonce: String) {
2227 let incumbent_nonce = self
2228 .spawn_nonces
2229 .lock()
2230 .unwrap_or_else(|poisoned| poisoned.into_inner())
2231 .get(module_id)
2232 .cloned();
2233 self.swaps
2234 .lock()
2235 .unwrap_or_else(|poisoned| poisoned.into_inner())
2236 .insert(
2237 module_id.to_string(),
2238 OpenSwap {
2239 candidate_nonce,
2240 incumbent_nonce,
2241 candidate_admitted: false,
2242 },
2243 );
2244 }
2245
2246 pub(crate) fn close_swap(&self, module_id: &str) {
2249 self.swaps
2250 .lock()
2251 .unwrap_or_else(|poisoned| poisoned.into_inner())
2252 .remove(module_id);
2253 }
2254
2255 pub(crate) fn set_swap_promotion_observer(
2258 &self,
2259 observer: std::sync::Weak<dyn SwapPromotionObserver>,
2260 ) {
2261 *self
2262 .promotion_observer
2263 .0
2264 .lock()
2265 .unwrap_or_else(|poisoned| poisoned.into_inner()) = Some(observer);
2266 }
2267
2268 fn notify_swap_promoted(&self, registration: &crate::registry::ModuleRegistration) {
2271 let observer = self
2272 .promotion_observer
2273 .0
2274 .lock()
2275 .unwrap_or_else(|poisoned| poisoned.into_inner())
2276 .as_ref()
2277 .and_then(std::sync::Weak::upgrade);
2278 if let Some(observer) = observer {
2279 observer.swap_promoted(registration);
2280 }
2281 }
2282
2283 pub(crate) fn swap_open(&self, module_id: &str) -> bool {
2285 self.swaps
2286 .lock()
2287 .unwrap_or_else(|poisoned| poisoned.into_inner())
2288 .contains_key(module_id)
2289 }
2290
2291 fn promote_swap_nonce(&self, module_id: &str, reserved: bool) {
2296 let candidate_nonce = self
2297 .swaps
2298 .lock()
2299 .unwrap_or_else(|poisoned| poisoned.into_inner())
2300 .get(module_id)
2301 .map(|swap| swap.candidate_nonce.clone());
2302 let Some(nonce) = candidate_nonce else {
2303 return;
2304 };
2305 self.set_spawn_nonce(module_id, nonce.clone());
2306 if reserved {
2307 self.set_reserved_nonce(module_id, nonce);
2308 }
2309 }
2310
2311 pub(crate) fn swap_hello_admission(
2326 &self,
2327 module_id: &str,
2328 presented: Option<&str>,
2329 ) -> SwapHelloAdmission {
2330 let swaps = self
2331 .swaps
2332 .lock()
2333 .unwrap_or_else(|poisoned| poisoned.into_inner());
2334 let Some(swap) = swaps.get(module_id) else {
2335 return SwapHelloAdmission::NotSwapping;
2336 };
2337 let Some(presented) = presented else {
2338 return SwapHelloAdmission::Refused;
2339 };
2340 if constant_time_eq(swap.candidate_nonce.as_bytes(), presented.as_bytes()) {
2341 return if swap.candidate_admitted {
2342 SwapHelloAdmission::Refused
2343 } else {
2344 SwapHelloAdmission::Candidate
2345 };
2346 }
2347 if swap
2348 .incumbent_nonce
2349 .as_deref()
2350 .is_some_and(|incumbent| constant_time_eq(incumbent.as_bytes(), presented.as_bytes()))
2351 {
2352 return SwapHelloAdmission::NotSwapping;
2353 }
2354 SwapHelloAdmission::Refused
2355 }
2356
2357 pub(crate) fn mark_swap_candidate_admitted(&self, module_id: &str) {
2360 if let Some(swap) = self
2361 .swaps
2362 .lock()
2363 .unwrap_or_else(|poisoned| poisoned.into_inner())
2364 .get_mut(module_id)
2365 {
2366 swap.candidate_admitted = true;
2367 }
2368 }
2369
2370 pub fn spawn_launch_nonce_for(&self, module_id: &str) -> Option<String> {
2372 self.spawn_nonces
2373 .lock()
2374 .unwrap_or_else(|poisoned| poisoned.into_inner())
2375 .get(module_id)
2376 .cloned()
2377 }
2378
2379 pub fn reserved_launch_nonce_for(&self, module_id: &str) -> Option<String> {
2381 self.reserved_nonces
2382 .lock()
2383 .unwrap_or_else(|poisoned| poisoned.into_inner())
2384 .get(module_id)
2385 .cloned()
2386 .flatten()
2387 }
2388
2389 pub fn insert(&self, module: SupervisedModule) -> Option<SupervisedModule> {
2390 self.mark_configured(module.module_id());
2394 let mut modules = self
2395 .modules
2396 .lock()
2397 .unwrap_or_else(|poisoned| poisoned.into_inner());
2398 modules.insert(module.module_id().to_string(), module)
2399 }
2400
2401 fn mark_configured(&self, module_id: &str) {
2405 self.configured_ids
2406 .lock()
2407 .unwrap_or_else(|poisoned| poisoned.into_inner())
2408 .insert(module_id.to_string());
2409 }
2410
2411 fn unmark_configured_unless_rostered(&self, module_id: &str) {
2415 let modules = self
2416 .modules
2417 .lock()
2418 .unwrap_or_else(|poisoned| poisoned.into_inner());
2419 if !modules.contains_key(module_id) {
2420 self.configured_ids
2421 .lock()
2422 .unwrap_or_else(|poisoned| poisoned.into_inner())
2423 .remove(module_id);
2424 }
2425 }
2426
2427 pub(crate) fn is_configured(&self, module_id: &str) -> bool {
2435 self.configured_ids
2436 .lock()
2437 .unwrap_or_else(|poisoned| poisoned.into_inner())
2438 .contains(module_id)
2439 }
2440
2441 pub fn get(&self, module_id: &str) -> Option<SupervisedModule> {
2442 let modules = self
2443 .modules
2444 .lock()
2445 .unwrap_or_else(|poisoned| poisoned.into_inner());
2446 modules.get(module_id).cloned()
2447 }
2448
2449 pub(crate) fn record_late_health_answer(
2450 &self,
2451 module_id: &str,
2452 latency_ms: u64,
2453 ) -> Result<bool, SuperviseError> {
2454 let Some(module) = self.get(module_id) else {
2455 return Ok(false);
2456 };
2457 update_snapshot(&module.inner.snapshot, Some(module_id), |state| {
2458 state.health.late_answer_count = state.health.late_answer_count.saturating_add(1);
2459 state.health.last_late_answer_latency_ms = Some(latency_ms);
2460 state.health.consecutive_failures = 0;
2468 })?;
2469 Ok(true)
2470 }
2471
2472 pub fn record_deliberate_severance(&self, module_id: &str) -> Result<bool, SuperviseError> {
2478 let Some(module) = self.get(module_id) else {
2479 return Ok(false);
2480 };
2481 let snapshot = lock_snapshot(&module.inner.snapshot)?;
2482 let Some((pid, start_time)) = snapshot.pid.zip(snapshot.process_start_time) else {
2483 return Ok(false);
2484 };
2485 drop(snapshot);
2486 module.record_deliberate_severance(ProcessIdentity { pid, start_time })
2487 }
2488
2489 pub fn list(&self) -> Vec<SupervisedModule> {
2490 let modules = self
2491 .modules
2492 .lock()
2493 .unwrap_or_else(|poisoned| poisoned.into_inner());
2494 let mut modules = modules.values().cloned().collect::<Vec<_>>();
2495 modules.sort_by(|left, right| left.module_id().cmp(right.module_id()));
2496 modules
2497 }
2498
2499 pub(crate) fn retire(&self, module_id: &str) -> Option<SupervisedModule> {
2500 self.activity_images
2501 .lock()
2502 .unwrap_or_else(|p| p.into_inner())
2503 .remove(module_id);
2504 self.spawn_nonces
2505 .lock()
2506 .unwrap_or_else(|poisoned| poisoned.into_inner())
2507 .remove(module_id);
2508 self.close_swap(module_id);
2509 let mut reserved_nonces = self
2510 .reserved_nonces
2511 .lock()
2512 .unwrap_or_else(|poisoned| poisoned.into_inner());
2513 if reserved_nonces.contains_key(module_id) {
2514 reserved_nonces.insert(module_id.to_string(), None);
2517 }
2518 drop(reserved_nonces);
2519 self.reserved_prefix_owners
2520 .lock()
2521 .unwrap_or_else(|poisoned| poisoned.into_inner())
2522 .retain(|_, owner| owner != module_id);
2523 let removed = self
2524 .modules
2525 .lock()
2526 .unwrap_or_else(|poisoned| poisoned.into_inner())
2527 .remove(module_id);
2528 self.configured_ids
2529 .lock()
2530 .unwrap_or_else(|poisoned| poisoned.into_inner())
2531 .remove(module_id);
2532 removed
2533 }
2534
2535 pub(crate) fn record_rescan_removal(&self, module_id: &str) {
2538 self.removal_tombstones
2539 .lock()
2540 .unwrap_or_else(|poisoned| poisoned.into_inner())
2541 .insert(module_id.to_string(), unix_ms_now());
2542 }
2543
2544 pub(crate) fn removal_tombstone_age_ms(&self, module_id: &str) -> Option<u64> {
2546 self.removal_tombstones
2547 .lock()
2548 .unwrap_or_else(|poisoned| poisoned.into_inner())
2549 .get(module_id)
2550 .copied()
2551 .map(|removed_at_ms| unix_ms_now().saturating_sub(removed_at_ms))
2552 }
2553
2554 pub(crate) fn release_retained_reserved_gate(&self, module_id: &str) -> bool {
2559 if self.get(module_id).is_some() {
2560 return false;
2561 }
2562 let mut reserved_nonces = self
2563 .reserved_nonces
2564 .lock()
2565 .unwrap_or_else(|poisoned| poisoned.into_inner());
2566 if !matches!(reserved_nonces.get(module_id), Some(None)) {
2567 return false;
2568 }
2569 reserved_nonces.remove(module_id);
2570 true
2571 }
2572
2573 pub(crate) fn operation_lock(&self) -> Arc<AsyncMutex<()>> {
2574 Arc::clone(&self.operation_lock)
2575 }
2576}
2577
2578#[derive(Debug, Clone)]
2580pub struct Supervisor {
2581 registry: Arc<Registry>,
2582 restart_policy: RestartPolicy,
2583 drain_timeout: Duration,
2584 connection_file_path: Option<PathBuf>,
2585 capture_logs_dir: Option<PathBuf>,
2586 forwarding: Option<Arc<ForwardingTable>>,
2587 process_liveness: Arc<SupervisorProcessLiveness>,
2588 supervisor_handle: Option<SupervisorHandle>,
2589 health: HealthConfig,
2590 daemon_start_clock: crate::clock::StartClock,
2591 terminal_journal: Option<Arc<crate::terminal_journal::TerminalJournal>>,
2592 spawn_events: SpawnEventFeed,
2593 provenance_probe: ExecutableIdentityProbe,
2594 child_roster: ChildRoster,
2597 #[cfg(target_os = "linux")]
2598 cgroup_placement: Option<subc_cgroup::Placement>,
2599 #[cfg(test)]
2600 test_after_first_spawn: AfterFirstSpawnHook,
2601}
2602
2603pub(crate) struct RegisteredConfiguredModule {
2607 spec: ModuleSpec,
2608 runtime: SupervisorRuntimeConfig,
2609 module: SupervisedModule,
2610 commands: mpsc::Receiver<SupervisorCommand>,
2611}
2612
2613#[cfg(test)]
2619#[derive(Clone, Default)]
2620struct AfterFirstSpawnHook(Option<AfterFirstSpawnFn>);
2621
2622#[cfg(test)]
2623type AfterFirstSpawnFn = Arc<dyn Fn(&str) + Send + Sync>;
2624
2625#[cfg(test)]
2626impl fmt::Debug for AfterFirstSpawnHook {
2627 fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
2628 f.write_str("AfterFirstSpawnHook")
2629 }
2630}
2631
2632#[cfg(test)]
2633impl AfterFirstSpawnHook {
2634 fn run(&self, module_id: &str) {
2635 if let Some(hook) = &self.0 {
2636 hook(module_id);
2637 }
2638 }
2639}
2640
2641impl Supervisor {
2642 #[cfg(test)]
2643 pub(crate) fn new_for_test(registry: Arc<Registry>, policy: RestartPolicy) -> Self {
2644 let supervisor = Self::new(registry, policy);
2645 #[cfg(target_os = "macos")]
2646 let supervisor = supervisor.with_privacy_trampoline(test_privacy_trampoline());
2647 supervisor
2648 }
2649 pub fn with_privacy_trampoline(self, path: impl Into<PathBuf>) -> Self {
2654 let path = path.into();
2655 #[cfg(target_os = "macos")]
2656 {
2657 let result = probe_privacy_trampoline(&path).map(|()| path);
2658 if let Err(cause) = &result {
2659 error!(%cause, "macOS privacy identity unavailable; supervised launches will refuse");
2660 }
2661 self.child_roster.set_privacy_trampoline(result);
2662 }
2663 #[cfg(not(target_os = "macos"))]
2664 let _ = path;
2665 self
2666 }
2667 #[cfg(unix)]
2678 pub(crate) fn begin_daemon_shutdown(&self) {
2679 self.child_roster.close();
2680 if let Some(journal) = &self.terminal_journal {
2681 journal.stamp_shutdown();
2682 }
2683 }
2684
2685 #[cfg(unix)]
2689 pub(crate) async fn drain_for_daemon_shutdown(&self) -> Result<(), SuperviseError> {
2690 const NOTICE_BUDGET: Duration = Duration::from_millis(500);
2691 const DRAIN_BUDGET: Duration = Duration::from_secs(2);
2692 let Some(forwarding) = &self.forwarding else {
2693 return Ok(());
2694 };
2695 let modules = forwarding
2696 .begin_daemon_drain()
2697 .map_err(SuperviseError::Forwarding)?;
2698 let deadline_ms =
2699 unix_ms_now().saturating_add((NOTICE_BUDGET + DRAIN_BUDGET).as_millis() as u64);
2700 let mut notices = tokio::task::JoinSet::new();
2701 let mut drains = Vec::new();
2702 for (module_id, endpoint) in modules {
2703 let Some(target) = forwarding
2704 .begin_endpoint_drain(endpoint, RouteCloseReason::Restart)
2705 .map_err(SuperviseError::Forwarding)?
2706 else {
2707 continue;
2708 };
2709 let routes = forwarding
2710 .endpoint_routes(target.endpoint)
2711 .map_err(SuperviseError::Forwarding)?;
2712 let command = serde_json::to_vec(&ModuleControlCommand::Draining {
2718 reason: RouteCloseReason::Restart,
2719 deadline_ms,
2720 })
2721 .expect("module draining serializes");
2722 let mut recipients = vec![(target.sink.clone(), target.negotiated_ver, command)];
2723 let mut clients: Vec<(crate::forwarding::GoodbyeTarget, Vec<u16>)> = Vec::new();
2724 for route in routes {
2725 let client = route.goodbye_target;
2726 if let Some((_, channels)) = clients
2727 .iter_mut()
2728 .find(|(existing, _)| existing.connection_id == client.connection_id)
2729 {
2730 channels.push(client.channel);
2731 } else {
2732 let channel = client.channel;
2733 clients.push((client, vec![channel]));
2734 }
2735 }
2736 for (client, mut channels) in clients {
2737 channels.sort_unstable();
2738 channels.dedup();
2739 let closing = serde_json::to_vec(&ClientControlPush::RouteClosing {
2740 module_id: module_id.clone(),
2741 channels,
2742 reason: RouteCloseReason::Restart,
2743 })
2744 .expect("route closing serializes");
2745 recipients.push((client.sink, client.negotiated_ver, closing));
2746 }
2747 for (sink, version, body) in recipients {
2748 notices.spawn(async move {
2749 let frame = Frame::build_with_version(
2750 version,
2751 FrameType::Push,
2752 control_flags(),
2753 0,
2754 0,
2755 0,
2756 body,
2757 )
2758 .expect("bounded lifecycle notice frame builds");
2759 sink.send_flushed(frame).await
2760 });
2761 }
2762 let gauges = declared_busy_gauges(&self.registry, &module_id)?;
2763 drains.push((module_id, target.endpoint, gauges));
2764 }
2765 let notice_deadline = Instant::now() + NOTICE_BUDGET;
2768 while let Ok(Some(result)) = timeout_at(notice_deadline, notices.join_next()).await {
2769 if !matches!(result, Ok(Ok(()))) {
2770 warn!(?result, "daemon shutdown notice delivery failed");
2771 }
2772 }
2773 notices.abort_all();
2774 let deadline = Instant::now() + DRAIN_BUDGET;
2775 let mut waits = tokio::task::JoinSet::new();
2776 for (module_id, endpoint, gauges) in drains {
2777 let forwarding = Arc::clone(forwarding);
2778 let mut runtime = self.runtime_config();
2779 runtime.health.cadence = Duration::from_millis(100);
2780 waits.spawn(async move {
2781 let wait_result = wait_for_forwarding_quiescence(
2782 &forwarding,
2783 &module_id,
2784 &runtime,
2785 endpoint,
2786 deadline,
2787 &gauges,
2788 DrainScope::Endpoint(endpoint),
2789 )
2790 .await;
2791 forwarding
2795 .record_daemon_drain_result(
2796 endpoint,
2797 drained_after_quiescence_wait(&wait_result),
2798 )
2799 .map_err(SuperviseError::Forwarding)?;
2800 wait_result
2801 });
2802 }
2803 while let Ok(Some(result)) = timeout_at(deadline, waits.join_next()).await {
2804 if !matches!(result, Ok(Ok(true))) {
2805 warn!(?result, "daemon shutdown drain did not reach quiescence");
2806 }
2807 }
2808 Ok(())
2809 }
2810
2811 #[cfg(unix)]
2823 pub(crate) async fn end_children_for_daemon_shutdown(
2824 &self,
2825 already_escalated: bool,
2826 escalate: impl std::future::Future<Output = ()>,
2827 ) {
2828 tokio::pin!(escalate);
2829 let mut escalated = already_escalated;
2830 if let Some(forwarding) = &self.forwarding {
2831 let reason = CloseReason::new(
2832 "daemon_shutdown",
2833 "the daemon is exiting after its shutdown notice and drain",
2834 );
2835 if escalated {
2836 send_module_goodbyes_for_daemon_shutdown(forwarding, &reason, false).await;
2839 } else {
2840 tokio::select! {
2841 biased;
2842 _ = escalate.as_mut() => {
2843 info!("second SIGTERM: abandoning module GOODBYE delivery");
2844 escalated = true;
2845 }
2846 _ = send_module_goodbyes_for_daemon_shutdown(forwarding, &reason, true) => {}
2847 }
2848 }
2849 let closed = forwarding.close_all_connections(&reason);
2850 debug!(closed, "closed established connections for daemon shutdown");
2851 }
2852 let escalated_here = escalated && !already_escalated;
2856 let remaining_escalate = async move {
2857 if escalated_here {
2858 std::future::pending::<()>().await;
2859 } else {
2860 escalate.await;
2861 }
2862 };
2863 crate::child_roster::end_children_for_daemon_shutdown(
2864 &self.child_roster,
2865 escalated,
2866 remaining_escalate,
2867 )
2868 .await;
2869 }
2870
2871 pub fn new(registry: Arc<Registry>, restart_policy: RestartPolicy) -> Self {
2872 Self {
2873 registry,
2874 restart_policy,
2875 drain_timeout: DEFAULT_DRAIN_TIMEOUT,
2876 connection_file_path: None,
2877 capture_logs_dir: None,
2878 forwarding: None,
2879 process_liveness: Arc::new(SupervisorProcessLiveness::default()),
2880 supervisor_handle: None,
2881 health: HealthConfig::default(),
2882 daemon_start_clock: crate::clock::StartClock::capture(),
2883 terminal_journal: None,
2884 spawn_events: SpawnEventFeed::default(),
2885 provenance_probe: ExecutableIdentityProbe::default(),
2886 child_roster: ChildRoster::default(),
2887 #[cfg(target_os = "linux")]
2888 cgroup_placement: None,
2889 #[cfg(test)]
2890 test_after_first_spawn: AfterFirstSpawnHook::default(),
2891 }
2892 }
2893
2894 pub fn with_drain_timeout(mut self, drain_timeout: Duration) -> Self {
2895 self.drain_timeout = drain_timeout;
2896 self
2897 }
2898
2899 pub fn with_process_liveness(
2900 mut self,
2901 process_liveness: Arc<SupervisorProcessLiveness>,
2902 ) -> Self {
2903 self.process_liveness = process_liveness;
2904 self
2905 }
2906
2907 pub fn with_connection_file_path(mut self, connection_file_path: impl Into<PathBuf>) -> Self {
2908 self.connection_file_path = Some(connection_file_path.into());
2909 self
2910 }
2911
2912 pub fn with_capture_logs_dir(mut self, logs_dir: impl Into<PathBuf>) -> Self {
2914 self.capture_logs_dir = Some(logs_dir.into());
2915 self
2916 }
2917
2918 pub fn with_daemon_incarnation(self, daemon_incarnation: String) -> Self {
2921 self.spawn_events.configure_incarnation(daemon_incarnation);
2925 self
2926 }
2927
2928 pub fn with_terminal_journal(self, path: PathBuf, daemon_incarnation: String) -> Self {
2931 let mut this = self.with_daemon_incarnation(daemon_incarnation.clone());
2932 this.terminal_journal = Some(Arc::new(crate::terminal_journal::TerminalJournal::open(
2933 path,
2934 daemon_incarnation,
2935 )));
2936 this
2937 }
2938
2939 pub fn with_forwarding(mut self, forwarding: Arc<ForwardingTable>) -> Self {
2940 self.forwarding = Some(forwarding);
2941 self
2942 }
2943
2944 pub fn with_handle(mut self, supervisor_handle: SupervisorHandle) -> Self {
2945 self.spawn_events = supervisor_handle.spawn_events.clone();
2946 self.supervisor_handle = Some(supervisor_handle);
2947 self
2948 }
2949
2950 pub fn with_health_config(mut self, health: HealthConfig) -> Self {
2951 self.health = health;
2952 self
2953 }
2954
2955 pub fn with_live_children_record(self, path: impl Into<PathBuf>) -> Self {
2959 self.child_roster.record_to(path.into());
2960 self
2961 }
2962
2963 #[cfg(target_os = "linux")]
2964 pub fn with_cgroup_placement(
2965 mut self,
2966 cgroup_placement: Option<subc_cgroup::Placement>,
2967 ) -> Self {
2968 self.cgroup_placement = cgroup_placement;
2969 self
2970 }
2971
2972 pub fn spawn(&self, spec: ModuleSpec) -> Result<SupervisedModule, SuperviseError> {
2978 validate_spec(&spec)?;
2979 self.establish_identity(&spec);
2980
2981 let runtime = self.runtime_config();
2982 let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::starting()));
2983 let activity_spawn = runtime.begin_activity_spawn(&spec.module_id);
2984 let spawned = spawn_child(
2985 &spec,
2986 runtime.connection_file_path.as_deref(),
2987 self.supervisor_handle.as_ref(),
2988 &runtime.stderr_ring,
2989 runtime.capture_logs_dir.as_deref(),
2990 &runtime.child_roster,
2991 #[cfg(target_os = "linux")]
2992 runtime.cgroup_placement.as_ref(),
2993 );
2994 #[cfg(test)]
2995 self.test_after_first_spawn.run(&spec.module_id);
2996 let child = match spawned {
2997 Ok(child) => child,
2998 Err(err) => {
2999 self.abandon_unrostered(&spec.module_id);
3002 return Err(err);
3003 }
3004 };
3005 self.mark_first_process_running(&spec, &runtime, &snapshot, &child)?;
3006 if let Some(spawn) = activity_spawn {
3007 spawn.complete(&snapshot);
3008 }
3009
3010 Ok(self.supervised_module(spec, runtime, snapshot, Some(child)))
3011 }
3012
3013 fn establish_identity(&self, spec: &ModuleSpec) {
3026 if let Some(supervisor_handle) = &self.supervisor_handle {
3027 supervisor_handle.apply_identity_configuration(spec);
3028 supervisor_handle.mark_configured(&spec.module_id);
3029 }
3030 }
3031
3032 fn abandon_unrostered(&self, module_id: &str) {
3035 if let Some(supervisor_handle) = &self.supervisor_handle {
3036 supervisor_handle.unmark_configured_unless_rostered(module_id);
3037 }
3038 }
3039
3040 fn mark_first_process_running(
3043 &self,
3044 spec: &ModuleSpec,
3045 runtime: &SupervisorRuntimeConfig,
3046 snapshot: &SharedSnapshot,
3047 child: &SupervisedChild,
3048 ) -> Result<(), SuperviseError> {
3049 if let Err(err) = set_running(snapshot, child, &spec.module_id, &runtime.spawn_events) {
3050 self.abandon_unrostered(&spec.module_id);
3051 return Err(err);
3052 }
3053 self.process_liveness
3054 .track(spec.module_id.clone(), Arc::clone(snapshot));
3055 Ok(())
3056 }
3057
3058 pub fn supervise_configured(
3064 &self,
3065 spec: ModuleSpec,
3066 enabled: bool,
3067 ) -> Result<SupervisedModule, SuperviseError> {
3068 self.supervise_configured_requested(
3069 spec,
3070 enabled,
3071 self.health.clone(),
3072 None,
3073 self.restart_policy,
3074 None,
3075 )
3076 }
3077
3078 pub fn supervise_configured_with_health(
3084 &self,
3085 spec: ModuleSpec,
3086 enabled: bool,
3087 health: HealthConfig,
3088 drain_timeout_ms: Option<u64>,
3089 restart_policy: RestartPolicy,
3090 ) -> Result<SupervisedModule, SuperviseError> {
3091 self.supervise_configured_requested(
3092 spec,
3093 enabled,
3094 health,
3095 drain_timeout_ms,
3096 restart_policy,
3097 None,
3098 )
3099 }
3100
3101 pub(crate) fn supervise_configured_requested(
3102 &self,
3103 spec: ModuleSpec,
3104 enabled: bool,
3105 health: HealthConfig,
3106 drain_timeout_ms: Option<u64>,
3107 restart_policy: RestartPolicy,
3108 requester: Option<Principal>,
3109 ) -> Result<SupervisedModule, SuperviseError> {
3110 let registered = self.register_configured_with_health(
3111 spec,
3112 enabled,
3113 health,
3114 drain_timeout_ms,
3115 restart_policy,
3116 )?;
3117 self.start_registered_configured(registered, requester)
3118 }
3119
3120 pub(crate) fn register_configured_with_health(
3125 &self,
3126 spec: ModuleSpec,
3127 enabled: bool,
3128 health: HealthConfig,
3129 drain_timeout_ms: Option<u64>,
3130 restart_policy: RestartPolicy,
3131 ) -> Result<RegisteredConfiguredModule, SuperviseError> {
3132 validate_spec(&spec)?;
3133 self.establish_identity(&spec);
3134
3135 let mut runtime = self.runtime_config();
3136 runtime.health = health;
3137 runtime.restart_policy = restart_policy;
3138 if let Some(ms) = drain_timeout_ms {
3139 runtime.drain_timeout = Duration::from_millis(ms);
3140 *runtime
3141 .effective_drain_timeout
3142 .lock()
3143 .unwrap_or_else(|poisoned| poisoned.into_inner()) = runtime.drain_timeout;
3144 }
3145 let snapshot = if enabled {
3146 SupervisorSnapshot::starting()
3147 } else {
3148 SupervisorSnapshot::disabled()
3149 };
3150 let snapshot = {
3151 let mut snapshot = snapshot;
3152 if enabled {
3153 if let Some((pid, reason)) = runtime.child_roster.previous_instance(&spec.module_id)
3154 {
3155 snapshot.waiting_for_previous_pid = Some(pid);
3156 snapshot.waiting_for_previous_detail = Some(reason.waiting_message(pid));
3157 }
3158 }
3159 snapshot
3160 };
3161 let (module, commands) =
3162 self.register_module(&spec, &runtime, Arc::new(Mutex::new(snapshot)));
3163 Ok(RegisteredConfiguredModule {
3164 spec,
3165 runtime,
3166 module,
3167 commands,
3168 })
3169 }
3170
3171 pub(crate) fn start_registered_configured(
3174 &self,
3175 registered: RegisteredConfiguredModule,
3176 requester: Option<Principal>,
3177 ) -> Result<SupervisedModule, SuperviseError> {
3178 let RegisteredConfiguredModule {
3179 spec,
3180 mut runtime,
3181 module,
3182 commands,
3183 } = registered;
3184 runtime.start_requester = requester;
3185 let snapshot = &module.inner.snapshot;
3186 let should_spawn = {
3187 let state = lock_snapshot(snapshot)?;
3188 state.enabled && state.waiting_for_previous_pid.is_none()
3189 };
3190 let child = if should_spawn {
3191 let activity_spawn = runtime.begin_activity_spawn(&spec.module_id);
3192 let spawned = spawn_child(
3193 &spec,
3194 runtime.connection_file_path.as_deref(),
3195 self.supervisor_handle.as_ref(),
3196 &runtime.stderr_ring,
3197 runtime.capture_logs_dir.as_deref(),
3198 &runtime.child_roster,
3199 #[cfg(target_os = "linux")]
3200 runtime.cgroup_placement.as_ref(),
3201 );
3202 #[cfg(test)]
3203 self.test_after_first_spawn.run(&spec.module_id);
3204 match spawned {
3205 Ok(child) => {
3206 self.mark_first_process_running(&spec, &runtime, snapshot, &child)?;
3207 if let Some(spawn) = activity_spawn {
3208 spawn.complete(snapshot);
3209 }
3210 Some(child)
3211 }
3212 Err(err) => {
3213 if runtime.health.critical {
3214 error!(
3215 module_id = %spec.module_id,
3216 program = %spec.program.display(),
3217 error = %err,
3218 "critical configured module failed to spawn; marking failed and alerting"
3219 );
3220 } else {
3221 error!(
3222 module_id = %spec.module_id,
3223 program = %spec.program.display(),
3224 error = %err,
3225 "configured module failed to spawn; marking failed and continuing"
3226 );
3227 }
3228 update_snapshot(snapshot, Some(&spec.module_id), |state| {
3229 state.state = ModuleState::Failed;
3230 clear_current_process_facts(state);
3231 })?;
3232 None
3233 }
3234 }
3235 } else {
3236 None
3237 };
3238 self.start_monitor(spec, runtime, &module, child, commands);
3239 Ok(module)
3240 }
3241
3242 fn runtime_config(&self) -> SupervisorRuntimeConfig {
3243 let effective_drain_timeout = Arc::new(Mutex::new(self.drain_timeout));
3244 SupervisorRuntimeConfig {
3245 start_requester: None,
3246 pending_activity_restart: false,
3247 provenance_probe: self.provenance_probe.clone(),
3248 scheduled_respawn: Arc::default(),
3249 deferred_reload_reply: Arc::default(),
3250 restart_policy: self.restart_policy,
3251 drain_timeout: self.drain_timeout,
3252 child_roster: self
3255 .child_roster
3256 .for_module(Arc::clone(&effective_drain_timeout)),
3257 effective_drain_timeout,
3258 default_drain_timeout: self.drain_timeout,
3259 health: self.health.clone(),
3260 connection_file_path: self.connection_file_path.clone(),
3261 capture_logs_dir: self.capture_logs_dir.clone(),
3262 forwarding: self.forwarding.clone(),
3263 supervisor_handle: self.supervisor_handle.clone(),
3264 stderr_ring: Arc::new(Mutex::new(StderrRing::new(StderrTailConfig::default()))),
3265 terminal_ring: Arc::new(Mutex::new(
3266 TerminalRing::new(
3267 TerminalRingConfig::default(),
3268 self.daemon_start_clock.started_at_ms(),
3269 )
3270 .with_start_clock(self.daemon_start_clock)
3271 .with_journal(self.terminal_journal.clone())
3272 .with_daemon_shutdown(self.child_roster.shutdown_flag()),
3273 )),
3274 spawn_events: self.spawn_events.clone(),
3275 #[cfg(target_os = "linux")]
3276 cgroup_placement: self.cgroup_placement.clone(),
3277 #[cfg(test)]
3278 test_seed_stale_facts_before_enable_spawn: false,
3279 #[cfg(test)]
3280 test_reload_exit_record_gate: None,
3281 }
3282 }
3283
3284 fn supervised_module(
3285 &self,
3286 spec: ModuleSpec,
3287 runtime: SupervisorRuntimeConfig,
3288 snapshot: SharedSnapshot,
3289 child: Option<SupervisedChild>,
3290 ) -> SupervisedModule {
3291 let (module, commands) = self.register_module(&spec, &runtime, snapshot);
3292 self.start_monitor(spec, runtime, &module, child, commands);
3293 module
3294 }
3295
3296 fn register_module(
3297 &self,
3298 spec: &ModuleSpec,
3299 runtime: &SupervisorRuntimeConfig,
3300 snapshot: SharedSnapshot,
3301 ) -> (SupervisedModule, mpsc::Receiver<SupervisorCommand>) {
3302 let configuration = Arc::new(Mutex::new(SupervisedConfiguration {
3303 spec: spec.clone(),
3304 health: runtime.health.clone(),
3305 }));
3306 let stderr_ring = Arc::clone(&runtime.stderr_ring);
3307 let terminal_ring = Arc::clone(&runtime.terminal_ring);
3308 let restart_policy = runtime.restart_policy;
3312 let effective_drain_timeout = Arc::clone(&runtime.effective_drain_timeout);
3313 let (tx, rx) = mpsc::channel(4);
3314 let module_id = spec.module_id.clone();
3315 let module = SupervisedModule {
3316 inner: Arc::new(SupervisedModuleInner {
3317 module_id: module_id.clone(),
3318 registry: Arc::clone(&self.registry),
3319 snapshot,
3320 configuration,
3321 stderr_ring,
3322 terminal_ring,
3323 commands: tx,
3324 monitor: Mutex::new(None),
3325 restart_policy,
3326 effective_drain_timeout,
3327 provenance_probe: self.provenance_probe.clone(),
3328 }),
3329 };
3330 if let Some(supervisor_handle) = &self.supervisor_handle {
3333 supervisor_handle.insert(module.clone());
3334 }
3335 (module, rx)
3336 }
3337
3338 fn start_monitor(
3339 &self,
3340 spec: ModuleSpec,
3341 runtime: SupervisorRuntimeConfig,
3342 module: &SupervisedModule,
3343 child: Option<SupervisedChild>,
3344 commands: mpsc::Receiver<SupervisorCommand>,
3345 ) {
3346 let monitor = tokio::spawn(supervise_loop(
3347 spec,
3348 runtime,
3349 Arc::clone(&self.registry),
3350 Arc::clone(&self.process_liveness),
3351 Arc::clone(&module.inner.snapshot),
3352 child,
3353 commands,
3354 ));
3355 *module
3356 .inner
3357 .monitor
3358 .lock()
3359 .unwrap_or_else(|p| p.into_inner()) = Some(monitor);
3360 }
3361}
3362
3363impl Default for Supervisor {
3364 fn default() -> Self {
3365 Self::new(Arc::new(Registry::default()), RestartPolicy::default())
3366 }
3367}
3368
3369#[derive(Clone)]
3371pub struct SupervisedModule {
3372 inner: Arc<SupervisedModuleInner>,
3373}
3374
3375struct SupervisedModuleInner {
3376 module_id: String,
3377 registry: Arc<Registry>,
3378 snapshot: SharedSnapshot,
3379 configuration: Arc<Mutex<SupervisedConfiguration>>,
3380 stderr_ring: Arc<Mutex<StderrRing>>,
3381 terminal_ring: Arc<Mutex<TerminalRing>>,
3382 commands: mpsc::Sender<SupervisorCommand>,
3383 monitor: Mutex<Option<JoinHandle<()>>>,
3384 restart_policy: RestartPolicy,
3388 effective_drain_timeout: Arc<Mutex<Duration>>,
3389 provenance_probe: ExecutableIdentityProbe,
3390}
3391
3392impl fmt::Debug for SupervisedModule {
3393 fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
3394 f.debug_struct("SupervisedModule")
3395 .field("module_id", &self.inner.module_id)
3396 .field("status", &self.status())
3397 .finish_non_exhaustive()
3398 }
3399}
3400
3401impl SupervisedModule {
3402 pub fn module_id(&self) -> &str {
3403 &self.inner.module_id
3404 }
3405
3406 #[cfg(test)]
3410 pub(crate) fn record_health_probe_failure_for_test(
3411 &self,
3412 detail: &str,
3413 ) -> Result<(), SuperviseError> {
3414 update_snapshot(&self.inner.snapshot, Some(&self.inner.module_id), |state| {
3415 state.health.consecutive_failures = state.health.consecutive_failures.saturating_add(1);
3416 state.health.detail = Some(detail.to_string());
3417 })
3418 }
3419
3420 pub fn state(&self) -> Result<ModuleState, SuperviseError> {
3421 Ok(lock_snapshot(&self.inner.snapshot)?.state)
3422 }
3423
3424 pub(crate) fn registration_end_reason(
3425 &self,
3426 ) -> Result<Option<RegistrationEndReason>, SuperviseError> {
3427 let snapshot = lock_snapshot(&self.inner.snapshot)?;
3428 Ok(match snapshot.state {
3429 ModuleState::Draining if snapshot.draining_to_replace => {
3430 Some(RegistrationEndReason::SupervisorRestart)
3431 }
3432 ModuleState::Draining => Some(RegistrationEndReason::SupervisorStop),
3433 ModuleState::Restarting if snapshot.draining_to_replace => {
3434 Some(RegistrationEndReason::SupervisorRestart)
3435 }
3436 _ => None,
3437 })
3438 }
3439
3440 pub fn stderr_tail(
3447 &self,
3448 max_lines: Option<usize>,
3449 max_bytes: Option<usize>,
3450 ) -> StderrTailSnapshot {
3451 self.inner
3452 .stderr_ring
3453 .lock()
3454 .unwrap_or_else(|poisoned| poisoned.into_inner())
3455 .snapshot(max_lines, max_bytes)
3456 }
3457
3458 pub fn terminal_history(&self) -> TerminalHistorySnapshot {
3463 self.inner
3464 .terminal_ring
3465 .lock()
3466 .unwrap_or_else(|poisoned| poisoned.into_inner())
3467 .snapshot()
3468 }
3469
3470 pub fn durable_terminal_history(&self) -> subc_control::TerminalHistory {
3475 durable_terminal_history_of(&self.inner.terminal_ring, &self.inner.module_id)
3476 }
3477
3478 pub(crate) async fn read_durable_terminal_history(
3483 &self,
3484 ) -> Result<subc_control::TerminalHistory, tokio::task::JoinError> {
3485 let terminal_ring = Arc::clone(&self.inner.terminal_ring);
3486 let module_id = self.inner.module_id.clone();
3487 tokio::task::spawn_blocking(move || durable_terminal_history_of(&terminal_ring, &module_id))
3488 .await
3489 }
3490
3491 pub fn status(&self) -> Result<ModuleStatus, SuperviseError> {
3492 self.status_with_snapshot_lock(&self.inner.snapshot, None)
3493 .map(|(status, _)| status)
3494 }
3495
3496 pub(crate) fn record_deliberate_severance(
3497 &self,
3498 identity: ProcessIdentity,
3499 ) -> Result<bool, SuperviseError> {
3500 let mut snapshot = lock_snapshot(&self.inner.snapshot)?;
3501 if snapshot.pid != Some(identity.pid)
3502 || snapshot.process_start_time != Some(identity.start_time)
3503 {
3504 return Ok(false);
3505 }
3506 snapshot.deliberate_severance = Some(identity);
3507 Ok(true)
3508 }
3509
3510 pub(crate) fn status_for_control(
3515 &self,
3516 caller: &'static str,
3517 ) -> Result<ModuleStatus, SuperviseError> {
3518 self.status_with_snapshot_lock(&self.inner.snapshot, Some(caller))
3519 .map(|(status, _)| status)
3520 }
3521
3522 fn status_with_snapshot_lock(
3523 &self,
3524 snapshot: &SharedSnapshot,
3525 caller: Option<&'static str>,
3526 ) -> Result<(ModuleStatus, Option<SpawnedFileIdentity>), SuperviseError> {
3527 let mut guard = match caller {
3528 Some(caller) => lock_snapshot_for_control(snapshot, &self.inner.module_id, caller)?,
3529 None => lock_snapshot(snapshot)?,
3530 };
3531 let restart_count =
3534 guard.crash_restarts_in_window(self.inner.restart_policy.window, Instant::now());
3535 let snapshot = guard.clone();
3536 drop(guard);
3537 let drain_timeout = *self.inner.effective_drain_timeout.lock().map_err(|_| {
3538 SuperviseError::StatePoisoned {
3539 module_id: Some(self.inner.module_id.clone()),
3540 }
3541 })?;
3542 let registration_active = self
3543 .inner
3544 .registry
3545 .get_module(&self.inner.module_id)
3546 .map_err(SuperviseError::Registry)?
3547 .is_some();
3548 let protocol = snapshot
3549 .spawned_protocol
3550 .unwrap_or(self.declared_protocol()?);
3551 let running_process =
3552 snapshot.enabled && snapshot.state == ModuleState::Running && snapshot.process_alive;
3553 let live = match protocol {
3559 ModuleProtocol::Subc => running_process && registration_active,
3560 ModuleProtocol::None => running_process,
3561 };
3562
3563 Ok((
3564 ModuleStatus {
3565 waiting_for_previous_pid: snapshot.waiting_for_previous_pid,
3566 waiting_for_previous_detail: snapshot.waiting_for_previous_detail.clone(),
3567 module_id: self.inner.module_id.clone(),
3568 launch_nonce_source: observed_launch_nonce_source(&snapshot),
3569 state: snapshot.state,
3570 enabled: snapshot.enabled,
3571 process_alive: snapshot.process_alive,
3572 registration_active,
3573 protocol,
3574 live,
3575 restart_count,
3576 lifetime_restarts: snapshot.lifetime_restarts,
3577 spawn_generation: snapshot.spawn_generation,
3578 max_restarts: self.inner.restart_policy.max_restarts,
3579 restart_window: self.inner.restart_policy.window,
3580 drain_timeout,
3581 restart_backoff: self.inner.restart_policy.backoff,
3582 restart_max_backoff: self.inner.restart_policy.max_backoff,
3583 pid: snapshot.reported_pid(),
3584 spawned_at_ms: snapshot.spawned_at_ms,
3585 spawned_from: snapshot.spawned_from,
3586 process_start_time: snapshot.process_start_time,
3587 last_exit: snapshot.last_exit,
3588 health: snapshot.health,
3589 },
3590 snapshot.spawned_file_identity.clone(),
3591 ))
3592 }
3593
3594 #[cfg(test)]
3595 pub(crate) fn hold_snapshot_for_test(
3596 &self,
3597 acquired: std::sync::mpsc::Sender<()>,
3598 hold: Duration,
3599 ) -> std::thread::JoinHandle<()> {
3600 let snapshot = Arc::clone(&self.inner.snapshot);
3601 std::thread::spawn(move || {
3602 let _guard = snapshot.lock().expect("test snapshot lock is not poisoned");
3603 acquired
3604 .send(())
3605 .expect("test receiver waits for snapshot lock");
3606 std::thread::sleep(hold);
3607 })
3608 }
3609
3610 pub(crate) async fn status_and_running_image_agreement(
3616 &self,
3617 ) -> Result<(ModuleStatus, subc_control::RunningImageAgreement), SuperviseError> {
3618 let (status, identity) = self.status_with_snapshot_lock(&self.inner.snapshot, None)?;
3619 let image = self
3620 .inner
3621 .provenance_probe
3622 .observe(
3623 status.pid,
3624 status.spawned_from.as_deref(),
3625 identity,
3626 status.process_start_time,
3627 )
3628 .await;
3629 Ok((status, image))
3630 }
3631
3632 pub(crate) async fn running_image_agreement(&self) -> subc_control::RunningImageAgreement {
3633 let snapshot = match lock_snapshot(&self.inner.snapshot) {
3634 Ok(snapshot) => snapshot.clone(),
3635 Err(_) => {
3636 return subc_control::RunningImageAgreement::Unavailable {
3637 reason: subc_control::RunningImageUnavailableReason::NotRunning,
3638 };
3639 }
3640 };
3641 self.inner
3642 .provenance_probe
3643 .observe(
3644 snapshot.reported_pid(),
3645 snapshot.spawned_from.as_deref(),
3646 snapshot.spawned_file_identity.clone(),
3647 snapshot.process_start_time,
3648 )
3649 .await
3650 }
3651
3652 pub(crate) fn child_resource_usage(&self) -> subc_control::ChildResourceUsage {
3655 #[cfg(windows)]
3656 {
3657 let image = lock_snapshot(&self.inner.snapshot)
3658 .ok()
3659 .and_then(|snapshot| {
3660 snapshot
3661 .spawned_file_identity
3662 .as_ref()
3663 .and_then(|identity| identity.windows_image.clone())
3664 });
3665 match image {
3666 Some(image) => {
3667 crate::child_resources::read_windows_handle(image.process(), image.start_time())
3668 }
3669 None => subc_control::ChildResourceUsage::Unavailable {
3670 reason: subc_control::ChildResourceUnavailableReason::NotRunning,
3671 },
3672 }
3673 }
3674 #[cfg(not(windows))]
3675 {
3676 let (pid, start_time) = match lock_snapshot(&self.inner.snapshot) {
3677 Ok(snapshot) => (snapshot.reported_pid(), snapshot.process_start_time),
3678 Err(_) => {
3679 return subc_control::ChildResourceUsage::Unavailable {
3680 reason: subc_control::ChildResourceUnavailableReason::Unreadable,
3681 }
3682 }
3683 };
3684 crate::child_resources::read(pid, start_time)
3685 }
3686 }
3687
3688 pub(crate) fn will_recover_after_connection_loss(&self) -> Result<bool, SuperviseError> {
3689 let mut snapshot = lock_snapshot(&self.inner.snapshot)?;
3690 Ok(match snapshot.state {
3691 ModuleState::Restarting => true,
3692 ModuleState::Failed | ModuleState::Disabled => false,
3693 _ if snapshot.enabled && snapshot.respawn_pending => true,
3694 _ => daemon_will_restart(&mut snapshot, &self.inner.restart_policy, Instant::now()),
3695 })
3696 }
3697
3698 #[cfg(test)]
3699 pub(crate) fn is_warming(&self) -> Result<bool, SuperviseError> {
3700 self.is_warming_with_snapshot_lock(None)
3701 }
3702
3703 pub(crate) fn is_warming_for_control(
3704 &self,
3705 caller: &'static str,
3706 ) -> Result<bool, SuperviseError> {
3707 self.is_warming_with_snapshot_lock(Some(caller))
3708 }
3709
3710 fn is_warming_with_snapshot_lock(
3711 &self,
3712 caller: Option<&'static str>,
3713 ) -> Result<bool, SuperviseError> {
3714 let snapshot = match caller {
3715 Some(caller) => {
3716 lock_snapshot_for_control(&self.inner.snapshot, &self.inner.module_id, caller)?
3717 }
3718 None => lock_snapshot(&self.inner.snapshot)?,
3719 }
3720 .clone();
3721 Ok(matches!(
3722 snapshot.state,
3723 ModuleState::Starting | ModuleState::Running | ModuleState::Restarting
3724 ))
3725 }
3726
3727 pub async fn drain(&self) -> Result<(), SuperviseError> {
3729 self.stop().await
3730 }
3731
3732 pub(crate) async fn retire_requested(
3733 &self,
3734 requester: Option<Principal>,
3735 ) -> Result<(), SuperviseError> {
3736 match self.state()? {
3737 ModuleState::Stopped | ModuleState::Failed => return Ok(()),
3738 ModuleState::Starting
3739 | ModuleState::Running
3740 | ModuleState::Unresponsive
3741 | ModuleState::Restarting
3742 | ModuleState::Draining
3743 | ModuleState::Disabled => {}
3744 }
3745
3746 let (reply_tx, reply_rx) = oneshot::channel();
3747 self.inner
3748 .commands
3749 .send(SupervisorCommand::Retire {
3750 requester,
3751 reply: reply_tx,
3752 })
3753 .await
3754 .map_err(|_| SuperviseError::CommandClosed {
3755 module_id: self.inner.module_id.clone(),
3756 })?;
3757 reply_rx.await.map_err(|_| SuperviseError::CommandClosed {
3758 module_id: self.inner.module_id.clone(),
3759 })?
3760 }
3761
3762 pub async fn stop(&self) -> Result<(), SuperviseError> {
3763 self.stop_requested(None).await
3764 }
3765
3766 pub(crate) async fn stop_requested(
3767 &self,
3768 requester: Option<Principal>,
3769 ) -> Result<(), SuperviseError> {
3770 match self.state()? {
3771 ModuleState::Stopped | ModuleState::Failed => return Ok(()),
3772 ModuleState::Starting
3773 | ModuleState::Running
3774 | ModuleState::Unresponsive
3775 | ModuleState::Restarting
3776 | ModuleState::Draining
3777 | ModuleState::Disabled => {}
3778 }
3779
3780 let (reply_tx, reply_rx) = oneshot::channel();
3781 self.inner
3782 .commands
3783 .send(SupervisorCommand::Drain {
3784 requester,
3785 reply: reply_tx,
3786 })
3787 .await
3788 .map_err(|_| SuperviseError::CommandClosed {
3789 module_id: self.inner.module_id.clone(),
3790 })?;
3791 reply_rx.await.map_err(|_| SuperviseError::CommandClosed {
3792 module_id: self.inner.module_id.clone(),
3793 })?
3794 }
3795
3796 pub async fn restart(&self, drain_timeout_ms: Option<u64>) -> Result<(), SuperviseError> {
3797 self.restart_requested(drain_timeout_ms, None).await
3798 }
3799
3800 pub(crate) async fn restart_requested(
3801 &self,
3802 drain_timeout_ms: Option<u64>,
3803 requester: Option<Principal>,
3804 ) -> Result<(), SuperviseError> {
3805 let received_at_generation = lock_snapshot(&self.inner.snapshot)?.spawn_generation;
3806 let (reply_tx, reply_rx) = oneshot::channel();
3807 self.inner
3808 .commands
3809 .send(SupervisorCommand::Restart {
3810 requester,
3811 drain_timeout_ms,
3812 received_at_generation,
3813 queued_at: Instant::now(),
3814 reply: reply_tx,
3815 })
3816 .await
3817 .map_err(|_| SuperviseError::CommandClosed {
3818 module_id: self.inner.module_id.clone(),
3819 })?;
3820 reply_rx.await.map_err(|_| SuperviseError::CommandClosed {
3821 module_id: self.inner.module_id.clone(),
3822 })?
3823 }
3824
3825 pub async fn swap(&self, ready_timeout: Option<Duration>) -> Result<(), SuperviseError> {
3830 self.swap_requested(ready_timeout, None).await
3831 }
3832
3833 pub(crate) async fn swap_requested(
3834 &self,
3835 ready_timeout: Option<Duration>,
3836 requester: Option<Principal>,
3837 ) -> Result<(), SuperviseError> {
3838 let (reply_tx, reply_rx) = oneshot::channel();
3839 self.inner
3840 .commands
3841 .send(SupervisorCommand::Swap {
3842 requester,
3843 ready_timeout,
3844 reply: reply_tx,
3845 })
3846 .await
3847 .map_err(|_| SuperviseError::CommandClosed {
3848 module_id: self.inner.module_id.clone(),
3849 })?;
3850 reply_rx.await.map_err(|_| SuperviseError::CommandClosed {
3851 module_id: self.inner.module_id.clone(),
3852 })?
3853 }
3854
3855 pub async fn reload(&self) -> Result<(), SuperviseError> {
3856 self.reload_requested(None).await
3857 }
3858
3859 pub(crate) async fn reload_requested(
3860 &self,
3861 requester: Option<Principal>,
3862 ) -> Result<(), SuperviseError> {
3863 let (reply_tx, reply_rx) = oneshot::channel();
3864 self.inner
3865 .commands
3866 .send(SupervisorCommand::Reload {
3867 requester,
3868 reply: reply_tx,
3869 })
3870 .await
3871 .map_err(|_| SuperviseError::CommandClosed {
3872 module_id: self.inner.module_id.clone(),
3873 })?;
3874 reply_rx.await.map_err(|_| SuperviseError::CommandClosed {
3875 module_id: self.inner.module_id.clone(),
3876 })?
3877 }
3878
3879 pub async fn set_enabled(&self, enabled: bool) -> Result<bool, SuperviseError> {
3880 self.set_enabled_requested(enabled, None).await
3881 }
3882
3883 pub(crate) async fn set_enabled_requested(
3884 &self,
3885 enabled: bool,
3886 requester: Option<Principal>,
3887 ) -> Result<bool, SuperviseError> {
3888 let (reply_tx, reply_rx) = oneshot::channel();
3889 self.inner
3890 .commands
3891 .send(SupervisorCommand::SetEnabled {
3892 requester,
3893 enabled,
3894 reply: reply_tx,
3895 })
3896 .await
3897 .map_err(|_| SuperviseError::CommandClosed {
3898 module_id: self.inner.module_id.clone(),
3899 })?;
3900 reply_rx.await.map_err(|_| SuperviseError::CommandClosed {
3901 module_id: self.inner.module_id.clone(),
3902 })?
3903 }
3904
3905 pub(crate) fn declared_protocol(&self) -> Result<ModuleProtocol, SuperviseError> {
3909 let configured = self
3910 .inner
3911 .configuration
3912 .lock()
3913 .map_err(|_| SuperviseError::StatePoisoned {
3914 module_id: Some(self.inner.module_id.clone()),
3915 })?
3916 .spec
3917 .protocol;
3918 let state = lock_snapshot(&self.inner.snapshot)?;
3919 Ok(state.spawned_protocol.unwrap_or(configured))
3920 }
3921
3922 pub(crate) fn configuration(&self) -> Result<(ModuleSpec, HealthConfig), SuperviseError> {
3923 let configuration =
3924 self.inner
3925 .configuration
3926 .lock()
3927 .map_err(|_| SuperviseError::StatePoisoned {
3928 module_id: Some(self.inner.module_id.clone()),
3929 })?;
3930 Ok((configuration.spec.clone(), configuration.health.clone()))
3931 }
3932
3933 #[cfg(any(test, feature = "test-support"))]
3937 pub async fn update_spec_for_test(&self, spec: ModuleSpec) -> Result<(), SuperviseError> {
3938 let (_, health) = self.configuration()?;
3939 let drain_timeout_ms = u64::try_from(
3940 self.inner
3941 .effective_drain_timeout
3942 .lock()
3943 .unwrap_or_else(|poisoned| poisoned.into_inner())
3944 .as_millis(),
3945 )
3946 .ok();
3947 self.update_configuration(spec, health, drain_timeout_ms)
3948 .await
3949 }
3950
3951 pub(crate) async fn update_configuration(
3952 &self,
3953 spec: ModuleSpec,
3954 health: HealthConfig,
3955 drain_timeout_ms: Option<u64>,
3956 ) -> Result<(), SuperviseError> {
3957 if spec.module_id != self.inner.module_id {
3958 return Err(SuperviseError::InvalidSpec {
3959 reason: "a supervised module's module_id cannot be changed".to_string(),
3960 });
3961 }
3962 validate_spec(&spec)?;
3963 let (reply_tx, reply_rx) = oneshot::channel();
3964 self.inner
3965 .commands
3966 .send(SupervisorCommand::UpdateConfiguration {
3967 spec: spec.clone(),
3968 health: health.clone(),
3969 drain_timeout_ms,
3970 reply: reply_tx,
3971 })
3972 .await
3973 .map_err(|_| SuperviseError::CommandClosed {
3974 module_id: self.inner.module_id.clone(),
3975 })?;
3976 reply_rx.await.map_err(|_| SuperviseError::CommandClosed {
3977 module_id: self.inner.module_id.clone(),
3978 })?;
3979 let mut configuration =
3980 self.inner
3981 .configuration
3982 .lock()
3983 .map_err(|_| SuperviseError::StatePoisoned {
3984 module_id: Some(self.inner.module_id.clone()),
3985 })?;
3986 configuration.spec = spec;
3987 configuration.health = health;
3988 Ok(())
3989 }
3990}
3991
3992impl Drop for SupervisedModuleInner {
3993 fn drop(&mut self) {
3994 let Ok(mut monitor) = self.monitor.lock() else {
3995 return;
3996 };
3997 if let Some(monitor) = monitor.as_ref().filter(|monitor| !monitor.is_finished()) {
3998 let _ = update_snapshot(&self.snapshot, Some(&self.module_id), |state| {
3999 state.state = ModuleState::Stopped;
4000 clear_current_process_facts(state);
4001 });
4002 monitor.abort();
4003 }
4004 let _ = monitor.take();
4005 }
4006}
4007
4008#[derive(Debug)]
4009enum SupervisorCommand {
4010 Drain {
4011 requester: Option<Principal>,
4012 reply: oneshot::Sender<Result<(), SuperviseError>>,
4013 },
4014 Retire {
4015 requester: Option<Principal>,
4016 reply: oneshot::Sender<Result<(), SuperviseError>>,
4017 },
4018 Restart {
4019 requester: Option<Principal>,
4020 drain_timeout_ms: Option<u64>,
4025 received_at_generation: u64,
4029 queued_at: Instant,
4032 reply: oneshot::Sender<Result<(), SuperviseError>>,
4033 },
4034 Reload {
4035 requester: Option<Principal>,
4036 reply: oneshot::Sender<Result<(), SuperviseError>>,
4037 },
4038 SetEnabled {
4039 requester: Option<Principal>,
4040 enabled: bool,
4041 reply: oneshot::Sender<Result<bool, SuperviseError>>,
4042 },
4043 UpdateConfiguration {
4044 spec: ModuleSpec,
4045 health: HealthConfig,
4046 drain_timeout_ms: Option<u64>,
4049 reply: oneshot::Sender<()>,
4050 },
4051 Swap {
4052 requester: Option<Principal>,
4053 ready_timeout: Option<Duration>,
4056 reply: oneshot::Sender<Result<(), SuperviseError>>,
4058 },
4059}
4060
4061#[derive(Debug)]
4062pub enum SuperviseError {
4063 InvalidSpec {
4064 reason: String,
4065 },
4066 Spawn {
4067 program: PathBuf,
4068 source: io::Error,
4069 cgroup_path: Option<PathBuf>,
4070 },
4071 Cgroup {
4072 module_id: String,
4073 source: io::Error,
4074 },
4075 LaunchNonce {
4078 reason: String,
4079 },
4080 Wait {
4081 module_id: String,
4082 source: io::Error,
4083 },
4084 Kill {
4085 module_id: String,
4086 source: io::Error,
4087 },
4088 Forwarding(ForwardingError),
4089 Registry(RegistryError),
4090 ReloadUnavailable {
4091 module_id: String,
4092 reason: String,
4093 },
4094 Disabled {
4099 module_id: String,
4100 },
4101 ReloadFailed {
4102 module_id: String,
4103 reason: String,
4104 },
4105 RegistrationStillActive {
4106 module_id: String,
4107 waited: Duration,
4108 },
4109 StatePoisoned {
4110 module_id: Option<String>,
4111 },
4112 CommandClosed {
4113 module_id: String,
4114 },
4115 SwapInProgress {
4119 module_id: String,
4120 },
4121 SwapRefused {
4123 module_id: String,
4124 reason: SwapRefusal,
4125 },
4126 SwapFailed {
4130 module_id: String,
4131 arm: SwapFailureArm,
4132 detail: String,
4133 candidate_exit: Option<ExitReport>,
4136 },
4137}
4138
4139#[derive(Debug, Clone, Copy, PartialEq, Eq)]
4141pub enum SwapRefusal {
4142 OverlapExclusive,
4144 NotRegistered,
4147 ProtocolNone,
4150 NotConfigured,
4153 AlreadySwapping,
4155}
4156
4157impl SwapRefusal {
4158 pub fn as_str(self) -> &'static str {
4159 match self {
4160 Self::OverlapExclusive => "overlap_exclusive",
4161 Self::NotRegistered => "not_registered",
4162 Self::ProtocolNone => "protocol_none",
4163 Self::NotConfigured => "not_configured",
4164 Self::AlreadySwapping => "already_swapping",
4165 }
4166 }
4167}
4168
4169#[derive(Debug, Clone, Copy, PartialEq, Eq)]
4172pub enum SwapFailureArm {
4173 SpawnFailed,
4175 NeverRegistered,
4177 NeverReady,
4179 CandidateExited,
4181 CandidateUnhealthy,
4183 Interrupted,
4187 CutoverLost,
4192}
4193
4194impl SwapFailureArm {
4195 pub fn as_str(self) -> &'static str {
4196 match self {
4197 Self::SpawnFailed => "spawn_failed",
4198 Self::NeverRegistered => "never_registered",
4199 Self::NeverReady => "never_ready",
4200 Self::CandidateExited => "candidate_exited",
4201 Self::CandidateUnhealthy => "candidate_unhealthy",
4202 Self::Interrupted => "interrupted",
4203 Self::CutoverLost => "cutover_lost",
4204 }
4205 }
4206}
4207
4208impl fmt::Display for SuperviseError {
4209 fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
4210 match self {
4211 Self::InvalidSpec { reason } => write!(f, "invalid module spec: {reason}"),
4212 Self::Spawn {
4213 program,
4214 source,
4215 cgroup_path: Some(cgroup_path),
4216 } => write!(
4217 f,
4218 "failed to place module in cgroup '{}' while spawning '{}': {source}",
4219 cgroup_path.display(),
4220 program.display()
4221 ),
4222 Self::Spawn {
4223 program,
4224 source,
4225 cgroup_path: None,
4226 } => write!(
4227 f,
4228 "failed to spawn module '{}': {source}",
4229 program.display()
4230 ),
4231 Self::Cgroup { module_id, source } => {
4232 write!(
4233 f,
4234 "failed to prepare cgroup for module '{module_id}': {source}"
4235 )
4236 }
4237 Self::LaunchNonce { reason } => {
4238 write!(
4239 f,
4240 "failed to generate reserved-module launch nonce: {reason}"
4241 )
4242 }
4243 Self::Wait { module_id, source } => {
4244 write!(f, "failed to wait for module '{module_id}': {source}")
4245 }
4246 Self::Kill { module_id, source } => {
4247 write!(f, "failed to kill module '{module_id}': {source}")
4248 }
4249 Self::Forwarding(err) => write!(f, "forwarding error: {err}"),
4250 Self::Registry(err) => write!(f, "registry error: {err}"),
4251 Self::ReloadUnavailable { module_id, reason } => {
4252 write!(f, "reload unavailable for module '{module_id}': {reason}")
4253 }
4254 Self::Disabled { module_id } => {
4255 write!(
4256 f,
4257 "module '{module_id}' is disabled; enable it before restart or reload"
4258 )
4259 }
4260 Self::ReloadFailed { module_id, reason } => {
4261 write!(f, "reload failed for module '{module_id}': {reason}")
4262 }
4263 Self::RegistrationStillActive { module_id, waited } => write!(
4264 f,
4265 "module '{module_id}' registration remained active after waiting {waited:?}"
4266 ),
4267 Self::StatePoisoned { module_id } => match module_id {
4268 Some(module_id) => {
4269 write!(f, "supervisor state for module '{module_id}' was poisoned")
4270 }
4271 None => write!(f, "supervisor state was poisoned"),
4272 },
4273 Self::CommandClosed { module_id } => {
4274 write!(
4275 f,
4276 "supervisor command channel for module '{module_id}' is closed"
4277 )
4278 }
4279 Self::SwapInProgress { module_id } => write!(
4280 f,
4281 "module '{module_id}' is being swapped; retry once the swap has cut over or failed, or stop the module to abort the swap"
4282 ),
4283 Self::SwapRefused { module_id, reason } => match reason {
4284 SwapRefusal::OverlapExclusive => write!(
4285 f,
4286 "module '{module_id}' is declared overlap: \"exclusive\" (the default): two processes of it must not run at once, so it cannot be swapped; use a plain restart, or declare overlap: \"safe\" in its config if it really tolerates a second process"
4287 ),
4288 SwapRefusal::NotRegistered => write!(
4289 f,
4290 "module '{module_id}' is not registered, so there is no serving process to keep while a replacement warms; use a plain restart"
4291 ),
4292 SwapRefusal::ProtocolNone => write!(
4293 f,
4294 "module '{module_id}' is protocol: \"none\" and never registers, so a swap could never see its replacement become ready; use a plain restart"
4295 ),
4296 SwapRefusal::NotConfigured => write!(
4297 f,
4298 "module '{module_id}' cannot be swapped: the supervisor was built without the forwarding table or shared handle a swap needs"
4299 ),
4300 SwapRefusal::AlreadySwapping => {
4301 write!(f, "module '{module_id}' is already being swapped")
4302 }
4303 },
4304 Self::SwapFailed {
4305 module_id,
4306 arm,
4307 detail,
4308 ..
4309 } => write!(
4310 f,
4311 "swap of module '{module_id}' failed ({}): {detail}; the running process was left serving",
4312 arm.as_str()
4313 ),
4314 }
4315 }
4316}
4317
4318impl Error for SuperviseError {
4319 fn source(&self) -> Option<&(dyn Error + 'static)> {
4320 match self {
4321 Self::Spawn { source, .. }
4322 | Self::Cgroup { source, .. }
4323 | Self::Wait { source, .. }
4324 | Self::Kill { source, .. } => Some(source),
4325 Self::Forwarding(err) => Some(err),
4326 Self::Registry(err) => Some(err),
4327 Self::LaunchNonce { .. }
4328 | Self::InvalidSpec { .. }
4329 | Self::ReloadUnavailable { .. }
4330 | Self::Disabled { .. }
4331 | Self::ReloadFailed { .. }
4332 | Self::RegistrationStillActive { .. }
4333 | Self::StatePoisoned { .. }
4334 | Self::CommandClosed { .. }
4335 | Self::SwapInProgress { .. }
4336 | Self::SwapRefused { .. }
4337 | Self::SwapFailed { .. } => None,
4338 }
4339 }
4340}
4341
4342pub(crate) fn validate_spec(spec: &ModuleSpec) -> Result<(), SuperviseError> {
4343 if spec.module_id.trim().is_empty() {
4344 return Err(SuperviseError::InvalidSpec {
4345 reason: "module_id must not be empty".to_string(),
4346 });
4347 }
4348
4349 Ok(())
4350}
4351
4352#[derive(Debug, Default)]
4353struct HealthProbeRuntime {
4354 configured_health: Option<HealthConfig>,
4355 registered_connection: Option<crate::ConnectionId>,
4356 advertised: bool,
4357 next_probe_at: Option<Instant>,
4358 probe_index: u64,
4359}
4360
4361fn running_protocol(spec: &ModuleSpec, snapshot: &SharedSnapshot) -> ModuleProtocol {
4362 lock_snapshot(snapshot)
4363 .ok()
4364 .and_then(|state| state.spawned_protocol)
4365 .unwrap_or(spec.protocol)
4366}
4367
4368impl HealthProbeRuntime {
4369 fn refresh_registration(
4370 &mut self,
4371 spec: &ModuleSpec,
4372 runtime: &SupervisorRuntimeConfig,
4373 registry: &Registry,
4374 snapshot: &SharedSnapshot,
4375 ) {
4376 if self.configured_health.as_ref() != Some(&runtime.health) {
4377 self.configured_health = Some(runtime.health.clone());
4378 self.next_probe_at = None;
4379 self.registered_connection = None;
4380 self.probe_index = 0;
4381 }
4382 if running_protocol(spec, snapshot) == ModuleProtocol::None {
4386 self.registered_connection = None;
4387 self.advertised = runtime.health.http.is_some();
4388 if !self.advertised {
4389 self.next_probe_at = None;
4390 let _ = update_snapshot(snapshot, Some(&spec.module_id), |state| {
4391 let unknown = ModuleHealthStatus::default();
4392 if state.health != unknown {
4393 state.health = unknown;
4394 }
4395 });
4396 } else if self.next_probe_at.is_none() {
4397 self.next_probe_at = Some(
4398 Instant::now()
4399 + jittered_health_delay(&spec.module_id, 0, runtime.health.cadence),
4400 );
4401 }
4402 return;
4403 }
4404
4405 let registration = match registry.get_module(&spec.module_id) {
4406 Ok(registration) => registration,
4407 Err(err) => {
4408 warn!(module_id = %spec.module_id, error = %err, "health prober could not read registry");
4409 self.advertised = false;
4410 self.next_probe_at = None;
4411 return;
4412 }
4413 };
4414
4415 let Some(registration) = registration else {
4416 self.registered_connection = None;
4417 self.advertised = false;
4418 self.next_probe_at = None;
4419 return;
4420 };
4421
4422 let advertised = registration
4423 .control_ops
4424 .iter()
4425 .any(|op| op == MODULE_CONTROL_OP_HEALTH_CHECK);
4426 if !advertised {
4427 self.registered_connection = Some(registration.connection_id);
4428 self.advertised = false;
4429 self.next_probe_at = None;
4430 let _ = update_snapshot(snapshot, Some(&spec.module_id), |state| {
4431 if state.health.status != SupervisorHealthStatus::Unknown
4432 || state.health.consecutive_failures != 0
4433 || state.health.last_probe_ms.is_some()
4434 || state.health.detail.is_some()
4435 || state.health.metrics.is_some()
4436 {
4437 state.health.status = SupervisorHealthStatus::Unknown;
4438 state.health.consecutive_failures = 0;
4439 state.health.last_probe_ms = None;
4440 state.health.detail = None;
4441 state.health.metrics = None;
4442 }
4443 });
4444 return;
4445 }
4446
4447 let reregistered = self.registered_connection != Some(registration.connection_id);
4448 self.registered_connection = Some(registration.connection_id);
4449 self.advertised = true;
4450 if reregistered || self.next_probe_at.is_none() {
4451 self.probe_index = 0;
4452 self.next_probe_at = Some(
4453 Instant::now() + jittered_health_delay(&spec.module_id, 0, runtime.health.cadence),
4454 );
4455 let _ = update_snapshot(snapshot, Some(&spec.module_id), |state| {
4456 state.health.status = SupervisorHealthStatus::Unknown;
4457 state.health.consecutive_failures = 0;
4458 state.health.detail = None;
4459 state.health.metrics = None;
4460 });
4461 }
4462 }
4463
4464 fn wake_after(&self) -> Option<Duration> {
4465 if !self.advertised {
4466 return None;
4467 }
4468 self.next_probe_at
4469 .map(|next| next.saturating_duration_since(Instant::now()))
4470 }
4471
4472 fn due(&self) -> bool {
4473 self.advertised
4474 && self
4475 .next_probe_at
4476 .is_some_and(|next| Instant::now() >= next)
4477 }
4478
4479 fn schedule_next(&mut self, spec: &ModuleSpec, cadence: Duration) {
4480 self.probe_index = self.probe_index.wrapping_add(1);
4481 self.next_probe_at = Some(
4482 Instant::now() + jittered_health_delay(&spec.module_id, self.probe_index, cadence),
4483 );
4484 }
4485}
4486
4487#[derive(Debug)]
4522enum HealthProbeEvidence {
4523 LaneDead,
4525 NoAnswer,
4527 BadAnswer,
4529 Misconfigured,
4531}
4532
4533#[derive(Debug)]
4534struct HealthProbeError {
4535 evidence: HealthProbeEvidence,
4536 message: String,
4537}
4538
4539impl HealthProbeError {
4540 fn lane_dead(message: impl Into<String>) -> Self {
4541 Self::with(HealthProbeEvidence::LaneDead, message)
4542 }
4543
4544 fn no_answer(message: impl Into<String>) -> Self {
4545 Self::with(HealthProbeEvidence::NoAnswer, message)
4546 }
4547
4548 fn bad_answer(message: impl Into<String>) -> Self {
4549 Self::with(HealthProbeEvidence::BadAnswer, message)
4550 }
4551
4552 fn misconfigured(message: impl Into<String>) -> Self {
4553 Self::with(HealthProbeEvidence::Misconfigured, message)
4554 }
4555
4556 fn with(evidence: HealthProbeEvidence, message: impl Into<String>) -> Self {
4557 Self {
4558 evidence,
4559 message: message.into(),
4560 }
4561 }
4562
4563 #[allow(dead_code)]
4577 fn is_proof_of_death(&self) -> bool {
4578 matches!(self.evidence, HealthProbeEvidence::LaneDead)
4579 }
4580
4581 fn label(&self) -> &'static str {
4589 match self.evidence {
4590 HealthProbeEvidence::LaneDead => "lane-dead",
4591 HealthProbeEvidence::NoAnswer => "no-answer",
4592 HealthProbeEvidence::BadAnswer => "bad-answer",
4593 HealthProbeEvidence::Misconfigured => "daemon-misconfigured",
4594 }
4595 }
4596}
4597
4598impl fmt::Display for HealthProbeError {
4599 fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
4600 f.write_str(&self.message)
4601 }
4602}
4603
4604async fn run_health_probe_cycle(
4605 spec: &ModuleSpec,
4606 runtime: &SupervisorRuntimeConfig,
4607 registry: &Registry,
4608 process_liveness: &SupervisorProcessLiveness,
4609 snapshot: &SharedSnapshot,
4610 child: &mut Option<SupervisedChild>,
4611) {
4612 let now_ms = unix_ms_now();
4613 let http = (running_protocol(spec, snapshot) == ModuleProtocol::None)
4614 .then_some(runtime.health.http.as_deref())
4615 .flatten();
4616 let result = match http {
4617 Some(url) => probe_http_health(url, runtime.health.deadline).await,
4618 None => probe_module_health(&spec.module_id, runtime, None).await,
4619 };
4620 match result {
4621 Ok(report) => {
4622 handle_health_report(
4623 spec,
4624 runtime,
4625 registry,
4626 process_liveness,
4627 snapshot,
4628 child,
4629 report,
4630 now_ms,
4631 )
4632 .await;
4633 }
4634 Err(err) => {
4635 if http.is_some() {
4636 let _ = update_snapshot(snapshot, Some(&spec.module_id), |state| {
4637 state.health.status = SupervisorHealthStatus::Failing;
4638 });
4639 }
4640 handle_health_probe_failure(
4641 spec,
4642 runtime,
4643 registry,
4644 process_liveness,
4645 snapshot,
4646 child,
4647 err,
4648 now_ms,
4649 )
4650 .await;
4651 }
4652 }
4653}
4654
4655pub(crate) struct HttpProbeTarget<'a> {
4656 address: std::net::SocketAddr,
4657 localhost: bool,
4658 authority: &'a str,
4659 path: String,
4660}
4661
4662pub(crate) fn parse_http_probe_url(url: &str) -> Result<HttpProbeTarget<'_>, String> {
4665 if url.bytes().any(|b| b <= b' ' || b == 127) || url.contains('#') {
4666 return Err("must not contain whitespace, controls, or a fragment".into());
4667 }
4668 let rest = url
4669 .strip_prefix("http://")
4670 .ok_or("must use plain http://")?;
4671 let split = rest.find(['/', '?']).unwrap_or(rest.len());
4672 let (authority, suffix) = rest.split_at(split);
4673 let (host, port) = if let Some(rest) = authority.strip_prefix("[::1]") {
4674 ("::1", rest)
4675 } else {
4676 let split = authority.find(':').unwrap_or(authority.len());
4677 authority.split_at(split)
4678 };
4679 let ip: std::net::IpAddr = match host {
4680 "127.0.0.1" | "localhost" => std::net::Ipv4Addr::LOCALHOST.into(),
4681 "::1" => std::net::Ipv6Addr::LOCALHOST.into(),
4682 _ => return Err("host must be 127.0.0.1, [::1], or localhost".into()),
4683 };
4684 let port = if port.is_empty() {
4685 80
4686 } else {
4687 port.strip_prefix(':')
4688 .and_then(|p| p.parse::<u16>().ok())
4689 .filter(|p| *p > 0)
4690 .ok_or("must have a valid nonzero TCP port")?
4691 };
4692 let path = if suffix.is_empty() {
4693 "/".into()
4694 } else if suffix.starts_with('?') {
4695 format!("/{suffix}")
4696 } else {
4697 suffix.into()
4698 };
4699 Ok(HttpProbeTarget {
4700 address: std::net::SocketAddr::new(ip, port),
4701 localhost: host == "localhost",
4702 authority,
4703 path,
4704 })
4705}
4706
4707async fn probe_http_health(
4708 url: &str,
4709 deadline: Duration,
4710) -> Result<HealthReport, HealthProbeError> {
4711 use tokio::io::{AsyncReadExt, AsyncWriteExt, BufReader};
4712 let target = parse_http_probe_url(url).map_err(HealthProbeError::misconfigured)?;
4713 let mut response_status = String::new();
4716 let mut body = Vec::new();
4717 let probe = async {
4718 let connection = match tokio::net::TcpStream::connect(target.address).await {
4721 Err(_) if target.localhost => {
4722 tokio::net::TcpStream::connect((
4723 std::net::Ipv6Addr::LOCALHOST,
4724 target.address.port(),
4725 ))
4726 .await
4727 }
4728 result => result,
4729 };
4730 let mut stream = connection.map_err(|error| {
4731 HealthProbeError::no_answer(format!("HTTP connect failed: {error}"))
4732 })?;
4733 stream.write_all(format!("GET {} HTTP/1.1\r\nHost: {}\r\nAccept-Encoding: identity\r\nConnection: close\r\n\r\n", target.path, target.authority).as_bytes()).await
4734 .map_err(|error| HealthProbeError::no_answer(format!("HTTP write failed: {error}")))?;
4735 let mut reader = BufReader::new(stream);
4736 let mut budget = 16 * 1024;
4737 let status = http_line(&mut reader, &mut budget).await?;
4738 let mut words = status.split_ascii_whitespace();
4739 let version = words.next();
4740 let code = words
4741 .next()
4742 .filter(|word| word.len() == 3)
4743 .and_then(|word| word.parse::<u16>().ok());
4744 if !matches!(version, Some("HTTP/1.1" | "HTTP/1.0"))
4745 || !code.is_some_and(|code| (100..600).contains(&code))
4746 {
4747 return Err(HealthProbeError::bad_answer(format!(
4748 "invalid HTTP status: {status}"
4749 )));
4750 }
4751 let code = code.expect("validated status code");
4752 response_status = status.clone();
4753 let mut length = None;
4754 let mut chunked = false;
4755 loop {
4756 let line = http_line(&mut reader, &mut budget).await?;
4757 if line.is_empty() {
4758 break;
4759 }
4760 if let Some((name, value)) = line.split_once(':') {
4761 if name.eq_ignore_ascii_case("content-length") {
4762 length = Some(value.trim().parse::<u64>().map_err(|_| {
4763 HealthProbeError::bad_answer("invalid HTTP Content-Length")
4764 })?);
4765 } else if name.eq_ignore_ascii_case("transfer-encoding") {
4766 chunked = value.trim().eq_ignore_ascii_case("chunked");
4767 }
4768 }
4769 }
4770 if chunked {
4771 while body.len() < 200 {
4772 let line = http_line(&mut reader, &mut budget).await?;
4773 let size = u64::from_str_radix(line.split(';').next().unwrap_or("").trim(), 16)
4774 .map_err(|_| HealthProbeError::bad_answer("invalid HTTP chunk size"))?;
4775 if size == 0 {
4776 break;
4777 }
4778 let count = size.min((200 - body.len()) as u64) as usize;
4779 let start = body.len();
4780 (&mut reader)
4781 .take(count as u64)
4782 .read_to_end(&mut body)
4783 .await
4784 .map_err(|error| {
4785 HealthProbeError::no_answer(format!("HTTP body read failed: {error}"))
4786 })?;
4787 if body.len() - start != count {
4788 return Err(HealthProbeError::bad_answer("truncated HTTP chunk"));
4789 }
4790 if size > count as u64 || body.len() == 200 {
4791 break;
4792 }
4793 if !http_line(&mut reader, &mut budget).await?.is_empty() {
4794 return Err(HealthProbeError::bad_answer("invalid HTTP chunk delimiter"));
4795 }
4796 }
4797 } else {
4798 reader
4799 .take(length.unwrap_or(200).min(200))
4800 .read_to_end(&mut body)
4801 .await
4802 .map_err(|error| {
4803 HealthProbeError::no_answer(format!("HTTP body read failed: {error}"))
4804 })?;
4805 }
4806 if (200..300).contains(&code) {
4807 Ok(HealthReport::ok())
4808 } else {
4809 Err(HealthProbeError::bad_answer(
4810 "HTTP health endpoint returned non-2xx",
4811 ))
4812 }
4813 };
4814 let mut result = timeout(deadline, probe).await.unwrap_or_else(|_| {
4815 Err(HealthProbeError::no_answer(format!(
4816 "HTTP probe timed out after {deadline:?}"
4817 )))
4818 });
4819 if let Err(error) = &mut result {
4820 if !response_status.is_empty() {
4821 error.message = format!(
4822 "{}; {response_status}: {}",
4823 error.message,
4824 String::from_utf8_lossy(&body)
4825 );
4826 }
4827 }
4828 result
4829}
4830
4831async fn http_line(
4832 reader: &mut tokio::io::BufReader<tokio::net::TcpStream>,
4833 remaining: &mut usize,
4834) -> Result<String, HealthProbeError> {
4835 use tokio::io::{AsyncBufReadExt, AsyncReadExt};
4836 let mut line = Vec::new();
4837 (&mut *reader)
4838 .take(*remaining as u64)
4839 .read_until(b'\n', &mut line)
4840 .await
4841 .map_err(|error| {
4842 HealthProbeError::no_answer(format!("HTTP header read failed: {error}"))
4843 })?;
4844 *remaining -= line.len();
4845 if !line.ends_with(b"\r\n") {
4846 return Err(HealthProbeError::bad_answer(
4847 "HTTP headers are incomplete or exceed 16 KiB",
4848 ));
4849 }
4850 line.truncate(line.len() - 2);
4851 String::from_utf8(line).map_err(|_| HealthProbeError::bad_answer("HTTP header is not UTF-8"))
4852}
4853
4854async fn probe_module_health(
4855 module_id: &str,
4856 runtime: &SupervisorRuntimeConfig,
4857 drain_deadline: Option<Instant>,
4858) -> Result<HealthReport, HealthProbeError> {
4859 let Some(forwarding) = runtime.forwarding.as_ref() else {
4860 return Err(HealthProbeError::misconfigured(
4861 "supervisor was not configured with a forwarding table",
4862 ));
4863 };
4864 let probe_started_at = Instant::now();
4865 let mut deadline = probe_started_at + runtime.health.deadline;
4866 if let Some(drain_deadline) = drain_deadline {
4867 deadline = deadline.min(drain_deadline);
4868 }
4869 let pending = if drain_deadline.is_some() {
4870 forwarding.begin_drain_health_probe_rpc_for(
4871 module_id,
4872 MODULE_CONTROL_OP_HEALTH_CHECK,
4873 probe_started_at,
4874 deadline,
4875 )
4876 } else {
4877 forwarding.begin_health_probe_rpc_for(
4878 module_id,
4879 MODULE_CONTROL_OP_HEALTH_CHECK,
4880 probe_started_at,
4881 deadline,
4882 )
4883 }
4884 .map_err(|err| {
4885 HealthProbeError::lane_dead(format!("failed to begin health.check RPC: {err}"))
4888 })?;
4889 await_health_probe(forwarding, pending, deadline, runtime.health.deadline).await
4890}
4891
4892async fn probe_endpoint_health(
4899 endpoint: crate::ModuleEndpointId,
4900 runtime: &SupervisorRuntimeConfig,
4901 deadline_cap: Option<Instant>,
4902) -> Result<HealthReport, HealthProbeError> {
4903 let Some(forwarding) = runtime.forwarding.as_ref() else {
4904 return Err(HealthProbeError::misconfigured(
4905 "supervisor was not configured with a forwarding table",
4906 ));
4907 };
4908 let probe_started_at = Instant::now();
4909 let mut deadline = probe_started_at + runtime.health.deadline;
4910 if let Some(cap) = deadline_cap {
4911 deadline = deadline.min(cap);
4912 }
4913 let pending = forwarding
4914 .begin_endpoint_health_probe_rpc_for(
4915 endpoint,
4916 MODULE_CONTROL_OP_HEALTH_CHECK,
4917 probe_started_at,
4918 deadline,
4919 )
4920 .map_err(|err| {
4921 HealthProbeError::lane_dead(format!("failed to begin health.check RPC: {err}"))
4922 })?;
4923 await_health_probe(forwarding, pending, deadline, runtime.health.deadline).await
4924}
4925
4926async fn await_health_probe(
4928 forwarding: &ForwardingTable,
4929 pending: PendingModuleControlRpc,
4930 deadline: Instant,
4931 probe_budget: Duration,
4932) -> Result<HealthReport, HealthProbeError> {
4933 let PendingModuleControlRpc {
4934 endpoint,
4935 module_sink,
4936 negotiated_ver,
4937 corr,
4938 receiver,
4939 } = pending;
4940 let body = serde_json::to_vec(&ModuleControlRequest::HealthCheck {}).map_err(|err| {
4941 HealthProbeError::misconfigured(format!("failed to encode health.check: {err}"))
4942 })?;
4943 let frame = Frame::build_with_version(
4944 negotiated_ver,
4945 FrameType::Request,
4946 control_flags(),
4947 0,
4948 0,
4949 corr,
4950 body,
4951 )
4952 .map_err(|err| {
4953 HealthProbeError::misconfigured(format!("failed to build health.check frame: {err}"))
4954 })?;
4955
4956 match timeout_at(deadline, module_sink.send(frame)).await {
4962 Ok(Ok(())) => {}
4963 Ok(Err(err)) => {
4964 let _ = forwarding.cancel_module_control_rpc(endpoint, corr);
4965 return Err(HealthProbeError::lane_dead(format!(
4968 "failed to send health.check: {err}"
4969 )));
4970 }
4971 Err(_elapsed) => {
4972 let _ = forwarding.cancel_module_control_rpc(endpoint, corr);
4973 return Err(HealthProbeError::no_answer(
4977 "health.check send timed out before enqueue (module egress full)",
4978 ));
4979 }
4980 }
4981
4982 match timeout_at(deadline, receiver).await {
4983 Ok(Ok(ModuleControlRpcOutcome::Response(response))) => {
4987 response.health_report().ok_or_else(|| {
4988 HealthProbeError::bad_answer("health.check RPC returned a non-health response")
4989 })
4990 }
4991 Ok(Ok(ModuleControlRpcOutcome::Rejected(body))) => Err(HealthProbeError::bad_answer(
4992 format!("health.check rejected: {}", body.message),
4993 )),
4994 Ok(Ok(ModuleControlRpcOutcome::ModuleGone(message))) => {
4995 Err(HealthProbeError::lane_dead(message))
4996 }
4997 Ok(Ok(ModuleControlRpcOutcome::MalformedResponse(message))) => {
4998 Err(HealthProbeError::bad_answer(message))
4999 }
5000 Ok(Ok(ModuleControlRpcOutcome::UnexpectedOp { expected, actual })) => {
5001 Err(HealthProbeError::bad_answer(format!(
5002 "expected module-control op '{expected}', got '{actual}'"
5003 )))
5004 }
5005 Ok(Ok(ModuleControlRpcOutcome::DeadlineElapsed)) => Err(HealthProbeError::bad_answer(
5009 "module answered health.check after its daemon deadline",
5010 )),
5011 Ok(Err(_)) => Err(HealthProbeError::misconfigured(
5012 "health.check waiter was canceled before the module responded",
5013 )),
5014 Err(_) => {
5015 let _ = forwarding.tombstone_health_probe_rpc(endpoint, corr);
5016 Err(HealthProbeError::no_answer(format!(
5017 "module did not answer health.check within {probe_budget:?}"
5018 )))
5019 }
5020 }
5021}
5022
5023#[allow(clippy::too_many_arguments)]
5024async fn handle_health_report(
5025 spec: &ModuleSpec,
5026 runtime: &SupervisorRuntimeConfig,
5027 registry: &Registry,
5028 process_liveness: &SupervisorProcessLiveness,
5029 snapshot: &SharedSnapshot,
5030 child: &mut Option<SupervisedChild>,
5031 report: HealthReport,
5032 now_ms: u64,
5033) {
5034 let status = supervisor_health_status(report.status);
5035 let detail = report.detail.clone();
5036 let metrics = truncate_health_metrics(report.metrics);
5037 let _ = update_snapshot(snapshot, Some(&spec.module_id), |state| {
5038 state.health.status = status;
5039 state.health.last_probe_ms = Some(now_ms);
5040 state.health.detail = detail.clone();
5041 state.health.metrics = metrics.clone();
5042 state.health.consecutive_failures = 0;
5043 });
5044
5045 let action = match report.status {
5046 HealthStatus::Ok => return,
5047 HealthStatus::Degraded => runtime.health.on_degraded,
5048 HealthStatus::Failing => runtime.health.on_failing,
5049 };
5050 apply_l3_health_action(
5051 spec,
5052 runtime,
5053 registry,
5054 process_liveness,
5055 snapshot,
5056 child,
5057 status,
5058 detail.as_deref(),
5059 action,
5060 now_ms,
5061 )
5062 .await;
5063}
5064
5065#[allow(clippy::too_many_arguments)]
5066async fn handle_health_probe_failure(
5067 spec: &ModuleSpec,
5068 runtime: &SupervisorRuntimeConfig,
5069 registry: &Registry,
5070 process_liveness: &SupervisorProcessLiveness,
5071 snapshot: &SharedSnapshot,
5072 child: &mut Option<SupervisedChild>,
5073 err: HealthProbeError,
5074 now_ms: u64,
5075) {
5076 let threshold = runtime.health.failure_threshold.max(1);
5077 let mut failures = 0;
5078 let detail = format!("[{}] {err}", err.label());
5083 let _ = update_snapshot(snapshot, Some(&spec.module_id), |state| {
5084 if state.spawned_protocol.unwrap_or(spec.protocol) == ModuleProtocol::Subc {
5087 state.health.status = SupervisorHealthStatus::Unknown;
5088 }
5089 state.health.last_probe_ms = Some(now_ms);
5090 state.health.consecutive_failures = state.health.consecutive_failures.saturating_add(1);
5091 state.health.detail = Some(detail.clone());
5092 state.health.metrics = None;
5093 failures = state.health.consecutive_failures;
5094 });
5095
5096 if failures < threshold {
5097 warn!(
5098 module_id = %spec.module_id,
5099 consecutive_failures = failures,
5100 threshold,
5101 evidence = err.label(),
5102 detail = %detail,
5103 "health.check probe failed"
5104 );
5105 return;
5106 }
5107
5108 let _ = update_snapshot(snapshot, Some(&spec.module_id), |state| {
5109 state.state = ModuleState::Unresponsive;
5110 state.health.status = SupervisorHealthStatus::Unresponsive;
5111 });
5112 if runtime.health.critical {
5116 error!(
5117 module_id = %spec.module_id,
5118 status = "unresponsive",
5119 evidence = err.label(),
5120 detail = %detail,
5121 "critical module health alert"
5122 );
5123 } else {
5124 warn!(
5125 module_id = %spec.module_id,
5126 status = "unresponsive",
5127 evidence = err.label(),
5128 detail = %detail,
5129 "module health threshold breached"
5130 );
5131 }
5132 if let Err(err) = health_restart_child(
5133 spec,
5134 runtime,
5135 registry,
5136 process_liveness,
5137 snapshot,
5138 child,
5139 SupervisorHealthStatus::Unresponsive,
5140 Some(&detail),
5141 now_ms,
5142 )
5143 .await
5144 {
5145 error!(module_id = %spec.module_id, error = %err, "health-triggered restart failed");
5146 }
5147}
5148
5149#[allow(clippy::too_many_arguments)]
5150async fn apply_l3_health_action(
5151 spec: &ModuleSpec,
5152 runtime: &SupervisorRuntimeConfig,
5153 registry: &Registry,
5154 process_liveness: &SupervisorProcessLiveness,
5155 snapshot: &SharedSnapshot,
5156 child: &mut Option<SupervisedChild>,
5157 status: SupervisorHealthStatus,
5158 detail: Option<&str>,
5159 action: HealthAction,
5160 now_ms: u64,
5161) {
5162 record_health_action(snapshot, &spec.module_id, action.to_string(), now_ms);
5163 match action {
5164 HealthAction::Report => {
5165 info!(
5166 module_id = %spec.module_id,
5167 status = ?status,
5168 detail,
5169 "module reported non-ok health"
5170 );
5171 }
5172 HealthAction::Alert => {
5173 error!(
5174 module_id = %spec.module_id,
5175 status = ?status,
5176 detail,
5177 "module health alert"
5178 );
5179 }
5180 HealthAction::Restart => {
5181 if let Err(err) = health_restart_child(
5182 spec,
5183 runtime,
5184 registry,
5185 process_liveness,
5186 snapshot,
5187 child,
5188 status,
5189 detail,
5190 now_ms,
5191 )
5192 .await
5193 {
5194 error!(module_id = %spec.module_id, error = %err, "health-triggered restart failed");
5195 }
5196 }
5197 }
5198}
5199
5200#[allow(clippy::too_many_arguments)]
5201async fn health_restart_child(
5202 spec: &ModuleSpec,
5203 runtime: &SupervisorRuntimeConfig,
5204 registry: &Registry,
5205 process_liveness: &SupervisorProcessLiveness,
5206 snapshot: &SharedSnapshot,
5207 child: &mut Option<SupervisedChild>,
5208 status: SupervisorHealthStatus,
5209 detail: Option<&str>,
5210 now_ms: u64,
5211) -> Result<(), SuperviseError> {
5212 let (enabled, schedule) = {
5213 let mut state = lock_snapshot(snapshot)?;
5214 let enabled = state.enabled;
5215 let schedule = if enabled {
5216 state.next_crash_restart(&runtime.restart_policy, Instant::now())
5217 } else {
5218 None
5219 };
5220 (enabled, schedule)
5221 };
5222
5223 if !enabled {
5224 return Err(SuperviseError::Disabled {
5225 module_id: spec.module_id.clone(),
5226 });
5227 }
5228
5229 if schedule.is_none() {
5230 record_health_action(snapshot, &spec.module_id, "failed".to_string(), now_ms);
5231 error!(
5232 module_id = %spec.module_id,
5233 status = ?status,
5234 detail,
5235 max_restarts = runtime.restart_policy.max_restarts,
5236 window_secs = runtime.restart_policy.window.as_secs(),
5237 reason = %runtime.restart_policy.budget_exhausted_detail(),
5238 "health restart budget exhausted; marking module failed"
5239 );
5240 let stop_notice = begin_forwarding_drain_if_configured(
5241 spec,
5242 runtime,
5243 registry,
5244 snapshot,
5245 Some(true),
5246 RouteCloseReason::Disable,
5247 )
5248 .await?;
5249 update_snapshot(snapshot, Some(&spec.module_id), |state| {
5250 state.drain_disposition_detail = Some(runtime.restart_policy.budget_exhausted_detail());
5251 })?;
5252 drain_optional_child(
5253 &spec.module_id,
5254 spec.protocol,
5255 stop_notice,
5256 registry,
5257 runtime.forwarding.as_deref(),
5258 snapshot,
5259 &runtime.terminal_ring,
5260 &runtime.spawn_events,
5261 child,
5262 runtime.drain_timeout,
5263 ModuleState::Failed,
5264 Some(true),
5265 )
5266 .await?;
5267 process_liveness.untrack_if_current(&spec.module_id, snapshot);
5268 return Ok(());
5269 }
5270
5271 let schedule = schedule.expect("a health restart must have a crash-restart schedule");
5272 let mut restart_count = 0;
5273 update_snapshot(snapshot, Some(&spec.module_id), |state| {
5274 restart_count = state.crash_restarts.len();
5275 state.state = ModuleState::Unresponsive;
5276 state.health.status = status;
5277 state.health.last_action = Some(HealthAction::Restart.to_string());
5278 state.health.last_action_ms = Some(now_ms);
5279 })?;
5280 warn!(
5281 module_id = %spec.module_id,
5282 status = ?status,
5283 detail,
5284 restart_count,
5285 restart_in_window = schedule.restart_in_window,
5286 delay_ms = schedule.delay.as_millis() as u64,
5287 "health-triggered module restart"
5288 );
5289
5290 let stop_notice = begin_forwarding_drain_if_configured(
5291 spec,
5292 runtime,
5293 registry,
5294 snapshot,
5295 Some(true),
5296 RouteCloseReason::Restart,
5297 )
5298 .await?;
5299 drain_optional_child(
5300 &spec.module_id,
5301 spec.protocol,
5302 stop_notice,
5303 registry,
5304 runtime.forwarding.as_deref(),
5305 snapshot,
5306 &runtime.terminal_ring,
5307 &runtime.spawn_events,
5308 child,
5309 runtime.drain_timeout,
5310 ModuleState::Restarting,
5311 Some(true),
5312 )
5313 .await?;
5314 schedule_respawn(
5315 runtime,
5316 snapshot,
5317 &spec.module_id,
5318 schedule.delay,
5319 RespawnKind::Spawn,
5320 )
5321}
5322
5323fn cancel_deferred_reload(runtime: &SupervisorRuntimeConfig, module_id: &str, reason: &str) {
5324 if let Some(reply) = runtime
5325 .deferred_reload_reply
5326 .lock()
5327 .unwrap_or_else(|p| p.into_inner())
5328 .take()
5329 {
5330 let _ = reply.send(Err(SuperviseError::ReloadFailed {
5331 module_id: module_id.to_string(),
5332 reason: reason.to_string(),
5333 }));
5334 }
5335}
5336
5337fn schedule_respawn(
5338 runtime: &SupervisorRuntimeConfig,
5339 snapshot: &SharedSnapshot,
5340 module_id: &str,
5341 delay: Duration,
5342 kind: RespawnKind,
5343) -> Result<(), SuperviseError> {
5344 cancel_deferred_reload(runtime, module_id, "respawn superseded by another restart");
5345 update_snapshot(snapshot, Some(module_id), |state| {
5346 state.respawn_pending = true
5347 })?;
5348 *runtime
5349 .scheduled_respawn
5350 .lock()
5351 .unwrap_or_else(|p| p.into_inner()) = Some(PendingRespawn {
5352 deadline: Instant::now() + delay,
5353 kind,
5354 });
5355 Ok(())
5356}
5357
5358fn record_health_action(snapshot: &SharedSnapshot, module_id: &str, action: String, now_ms: u64) {
5359 let _ = update_snapshot(snapshot, Some(module_id), |state| {
5360 state.health.last_action = Some(action);
5361 state.health.last_action_ms = Some(now_ms);
5362 });
5363}
5364
5365fn supervisor_health_status(status: HealthStatus) -> SupervisorHealthStatus {
5366 match status {
5367 HealthStatus::Ok => SupervisorHealthStatus::Ok,
5368 HealthStatus::Degraded => SupervisorHealthStatus::Degraded,
5369 HealthStatus::Failing => SupervisorHealthStatus::Failing,
5370 }
5371}
5372
5373fn truncate_health_metrics(metrics: Option<Value>) -> Option<Value> {
5385 let metrics = metrics?;
5386 match serde_json::to_vec(&metrics) {
5387 Ok(encoded) if encoded.len() > MAX_HEALTH_METRICS_BYTES => Some(serde_json::json!({
5388 "truncated": true,
5389 "original_bytes": encoded.len(),
5390 })),
5391 Ok(_) | Err(_) => Some(metrics),
5392 }
5393}
5394
5395fn jittered_health_delay(module_id: &str, probe_index: u64, cadence: Duration) -> Duration {
5401 if cadence.is_zero() {
5402 return Duration::ZERO;
5403 }
5404 let cadence_ms = cadence.as_millis() as u64;
5405 if cadence_ms == 0 {
5421 return cadence;
5422 }
5423 let jitter_span = (cadence_ms / 10).max(1);
5438 let hash = module_id.as_bytes().iter().fold(
5439 probe_index.wrapping_mul(0x9E37_79B9_7F4A_7C15),
5440 |acc, byte| {
5441 acc.wrapping_mul(1099511628211)
5442 .wrapping_add(u64::from(*byte))
5443 },
5444 );
5445 cadence + Duration::from_millis(hash % jitter_span)
5446}
5447
5448#[cfg(test)]
5449mod tests {
5450 use super::*;
5451
5452 #[test]
5453 fn readding_a_module_clears_its_rescan_removal_tombstone() {
5454 let handle = SupervisorHandle::new();
5455 let module_id = "readded-tombstone";
5456 handle.record_rescan_removal(module_id);
5457 assert!(handle.removal_tombstone_age_ms(module_id).is_some());
5458
5459 handle.apply_identity_configuration(&ModuleSpec {
5460 module_id: module_id.to_string(),
5461 program: PathBuf::from("/test/module"),
5462 args: Vec::new(),
5463 env: Vec::new(),
5464 reserved: false,
5465 reserved_prefixes: Vec::new(),
5466 protocol: ModuleProtocol::Subc,
5467 overlap: Default::default(),
5468 });
5469
5470 assert!(
5471 handle.removal_tombstone_age_ms(module_id).is_none(),
5472 "a re-added module must not retain a stale removal tombstone"
5473 );
5474 }
5475
5476 #[derive(Debug, PartialEq, Eq)]
5479 struct OwnerInSpawnWindow {
5480 module_id: String,
5481 configured: bool,
5482 on_roster: bool,
5483 admission_refusal: Option<&'static str>,
5484 }
5485
5486 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
5497 async fn a_new_module_is_configured_before_its_first_process_can_register() {
5498 use crate::scopes::ScopeTable;
5499 use subc_protocol::{error_codes, scope::ScopeSelector, Principal};
5500
5501 let dir = cortexkit_test_support::ScratchDir::new("configured-before-spawn");
5502 let stub = |module_id: &str, program: PathBuf| ModuleSpec {
5503 module_id: module_id.to_string(),
5504 program,
5505 args: Vec::new(),
5506 env: ["XDG_DATA_HOME", "XDG_RUNTIME_DIR", "XDG_CONFIG_HOME"]
5507 .into_iter()
5508 .map(|key| (key.to_string(), dir.path().display().to_string()))
5509 .chain([("FAKE_AFT_NEVER_CONNECT".to_string(), "1".to_string())])
5510 .collect(),
5511 reserved: false,
5512 reserved_prefixes: Vec::new(),
5513 protocol: ModuleProtocol::Subc,
5514 overlap: Default::default(),
5515 };
5516 let live = super::terminal_history_tests::fake_aft_stub_path();
5517 let missing = dir.path().join("definitely-missing-module");
5518
5519 let handle = SupervisorHandle::new();
5520 let mut supervisor =
5521 Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default())
5522 .with_handle(handle.clone());
5523 let observed = Arc::new(Mutex::new(Vec::<OwnerInSpawnWindow>::new()));
5524 let hook_handle = handle.clone();
5525 let hook_observed = Arc::clone(&observed);
5526 supervisor.test_after_first_spawn = AfterFirstSpawnHook(Some(Arc::new(move |module_id| {
5527 let configured = hook_handle.is_configured(module_id);
5530 let selector = ScopeSelector::new(
5531 Principal::Reserved {
5532 module_id: module_id.to_string(),
5533 },
5534 "s".to_string(),
5535 Some(1),
5536 );
5537 let carrier = Principal::Reserved {
5538 module_id: "carrier".to_string(),
5539 };
5540 let admission_refusal = match ScopeTable::new(Vec::<String>::new())
5541 .admit(&carrier, module_id, &selector, configured)
5542 {
5543 Ok(_) => None,
5544 Err(refusal) => Some(refusal.code),
5545 };
5546 hook_observed.lock().unwrap().push(OwnerInSpawnWindow {
5547 module_id: module_id.to_string(),
5548 configured,
5549 on_roster: hook_handle.get(module_id).is_some(),
5550 admission_refusal,
5551 });
5552 })));
5553
5554 let plain = supervisor.spawn(stub("plain", live.clone())).unwrap();
5555 let configured = supervisor
5556 .supervise_configured(stub("configured", live.clone()), true)
5557 .unwrap();
5558 let with_health = supervisor
5559 .supervise_configured_with_health(
5560 stub("with-health", live.clone()),
5561 true,
5562 HealthConfig::default(),
5563 None,
5564 RestartPolicy::default(),
5565 )
5566 .unwrap();
5567 let failed = supervisor
5570 .supervise_configured_with_health(
5571 stub("failed-spawn", missing.clone()),
5572 true,
5573 HealthConfig::default(),
5574 None,
5575 RestartPolicy::default(),
5576 )
5577 .unwrap();
5578 assert!(supervisor.spawn(stub("spawn-error", missing)).is_err());
5581
5582 let expected = [
5583 "plain",
5584 "configured",
5585 "with-health",
5586 "failed-spawn",
5587 "spawn-error",
5588 ]
5589 .into_iter()
5590 .map(|module_id| OwnerInSpawnWindow {
5591 module_id: module_id.to_string(),
5592 configured: true,
5593 on_roster: !matches!(module_id, "plain" | "spawn-error"),
5594 admission_refusal: Some(error_codes::SCOPE_NOT_SYNCED),
5595 })
5596 .collect::<Vec<_>>();
5597 assert_eq!(*observed.lock().unwrap(), expected);
5598
5599 for module_id in ["plain", "configured", "with-health", "failed-spawn"] {
5600 assert!(
5601 handle.get(module_id).is_some(),
5602 "{module_id} is on the roster"
5603 );
5604 assert!(
5605 handle.is_configured(module_id),
5606 "{module_id} stays configured"
5607 );
5608 }
5609 assert!(handle.get("spawn-error").is_none());
5610 assert!(
5611 !handle.is_configured("spawn-error"),
5612 "a plain spawn that failed must not leave its module marked configured"
5613 );
5614
5615 handle.retire("failed-spawn");
5617 assert!(!handle.is_configured("failed-spawn"));
5618
5619 for module in [plain, configured, with_health] {
5620 module.stop().await.unwrap();
5621 }
5622 drop(failed);
5623 }
5624
5625 fn stale_process_snapshot(state: ModuleState, enabled: bool) -> SharedSnapshot {
5626 let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::new(state, enabled)));
5627 update_snapshot(&snapshot, Some("stale-process-facts"), |snapshot| {
5628 snapshot.process_alive = true;
5629 snapshot.pid = Some(41);
5630 snapshot.spawned_at_ms = Some(42);
5631 snapshot.spawned_from = Some(PathBuf::from("/spawned/module"));
5632 snapshot.spawned_file_identity = Some(SpawnedFileIdentity {
5633 device: 43,
5634 inode: 44,
5635 #[cfg(windows)]
5636 windows_image: None,
5637 });
5638 })
5639 .unwrap();
5640 snapshot
5641 }
5642
5643 fn assert_snapshot_process_facts_cleared(snapshot: &SharedSnapshot) {
5644 let snapshot = lock_snapshot(snapshot).unwrap();
5645 assert!(!snapshot.process_alive);
5646 assert_eq!(snapshot.pid, None);
5647 assert_eq!(snapshot.spawned_at_ms, None);
5648 assert_eq!(snapshot.spawned_from, None);
5649 assert_eq!(snapshot.spawned_file_identity, None);
5650 }
5651
5652 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
5653 async fn failed_enable_spawn_clears_preexisting_current_process_facts() {
5654 let supervisor =
5655 Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default());
5656 let mut runtime = supervisor.runtime_config();
5657 runtime.test_seed_stale_facts_before_enable_spawn = true;
5658 let snapshot = stale_process_snapshot(ModuleState::Disabled, false);
5659 let mut child = None;
5660 let spec = ModuleSpec {
5661 module_id: "failed-enable-clears-facts".to_string(),
5662 program: PathBuf::from("/definitely/missing/failed-enable-module"),
5663 args: Vec::new(),
5664 env: Vec::new(),
5665 reserved: false,
5666 reserved_prefixes: Vec::new(),
5667 protocol: ModuleProtocol::Subc,
5668 overlap: Default::default(),
5669 };
5670
5671 let result = set_child_enabled(
5672 &spec,
5673 &runtime,
5674 &supervisor.registry,
5675 &supervisor.process_liveness,
5676 &snapshot,
5677 &mut child,
5678 true,
5679 )
5680 .await;
5681
5682 assert!(matches!(result, Err(SuperviseError::Spawn { .. })));
5683 assert_eq!(lock_snapshot(&snapshot).unwrap().state, ModuleState::Failed);
5684 assert_snapshot_process_facts_cleared(&snapshot);
5685 }
5686
5687 #[tokio::test]
5688 async fn start_revives_stranded_restarting_but_not_pending_backoff() {
5689 let supervisor =
5690 Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default());
5691 let runtime = supervisor.runtime_config();
5692 let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::new(
5693 ModuleState::Restarting,
5694 true,
5695 )));
5696 let spec = ModuleSpec {
5697 module_id: "start-stranded-restarting".to_string(),
5698 program: super::terminal_history_tests::fake_aft_stub_path(),
5699 args: Vec::new(),
5700 env: vec![("FAKE_AFT_NEVER_CONNECT".to_string(), "1".to_string())],
5701 reserved: false,
5702 reserved_prefixes: Vec::new(),
5703 protocol: ModuleProtocol::None,
5704 overlap: Default::default(),
5705 };
5706 let mut child = None;
5707 lock_snapshot(&snapshot).unwrap().respawn_pending = true;
5708 assert!(!super::set_child_enabled(
5709 &spec,
5710 &runtime,
5711 &Registry::default(),
5712 &supervisor.process_liveness,
5713 &snapshot,
5714 &mut child,
5715 true
5716 )
5717 .await
5718 .unwrap());
5719 assert!(child.is_none());
5720 lock_snapshot(&snapshot).unwrap().respawn_pending = false;
5721 assert!(super::set_child_enabled(
5722 &spec,
5723 &runtime,
5724 &Registry::default(),
5725 &supervisor.process_liveness,
5726 &snapshot,
5727 &mut child,
5728 true
5729 )
5730 .await
5731 .unwrap());
5732 assert_eq!(
5733 lock_snapshot(&snapshot).unwrap().state,
5734 ModuleState::Running
5735 );
5736 let mut child = child.unwrap();
5737 child.start_kill().unwrap();
5738 child.wait().await.unwrap();
5739 }
5740
5741 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
5742 async fn failed_reload_spawn_clears_current_process_facts() {
5743 let supervisor =
5744 Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default());
5745 let mut runtime = supervisor.runtime_config();
5746 runtime.restart_policy = RestartPolicy::new(0, Duration::ZERO);
5747 let snapshot = stale_process_snapshot(ModuleState::Running, true);
5748 let mut child = None;
5749 let spec = ModuleSpec {
5750 module_id: "failed-reload-clears-facts".to_string(),
5751 program: PathBuf::from("/unused/failed-reload-module"),
5752 args: Vec::new(),
5753 env: Vec::new(),
5754 reserved: false,
5755 reserved_prefixes: Vec::new(),
5756 protocol: ModuleProtocol::Subc,
5757 overlap: Default::default(),
5758 };
5759
5760 let result = handle_reload_spawn_failure(
5761 &spec,
5762 &runtime,
5763 &supervisor.process_liveness,
5764 &snapshot,
5765 &mut child,
5766 "forced reload spawn failure".to_string(),
5767 )
5768 .await;
5769
5770 assert!(matches!(result, Err(SuperviseError::ReloadFailed { .. })));
5771 assert_eq!(lock_snapshot(&snapshot).unwrap().state, ModuleState::Failed);
5772 assert_snapshot_process_facts_cleared(&snapshot);
5773 }
5774
5775 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
5776 async fn dropping_a_module_with_an_active_monitor_clears_current_process_facts() {
5777 let supervisor =
5778 Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default());
5779 let snapshot = stale_process_snapshot(ModuleState::Running, true);
5780 let module = supervisor.supervised_module(
5781 ModuleSpec {
5782 module_id: "drop-clears-facts".to_string(),
5783 program: PathBuf::from("/unused/drop-module"),
5784 args: Vec::new(),
5785 env: Vec::new(),
5786 reserved: false,
5787 reserved_prefixes: Vec::new(),
5788 protocol: ModuleProtocol::Subc,
5789 overlap: Default::default(),
5790 },
5791 supervisor.runtime_config(),
5792 Arc::clone(&snapshot),
5793 None,
5794 );
5795 assert!(!module
5796 .inner
5797 .monitor
5798 .lock()
5799 .unwrap()
5800 .as_ref()
5801 .unwrap()
5802 .is_finished());
5803
5804 drop(module);
5805
5806 assert_eq!(
5807 lock_snapshot(&snapshot).unwrap().state,
5808 ModuleState::Stopped
5809 );
5810 assert_snapshot_process_facts_cleared(&snapshot);
5811 }
5812
5813 #[cfg(unix)]
5814 #[tokio::test]
5815 async fn rescan_preserves_running_protocol_until_respawn() {
5816 let dir = cortexkit_test_support::ScratchDir::new("rescan-protocol");
5817 let initial = ModuleSpec {
5818 module_id: "rescan-protocol".into(),
5819 program: PathBuf::from("/bin/sleep"),
5820 args: vec!["60".into()],
5821 env: ["XDG_DATA_HOME", "XDG_RUNTIME_DIR", "XDG_CONFIG_HOME"]
5822 .into_iter()
5823 .map(|key| (key.into(), dir.path().to_string_lossy().into_owned()))
5824 .collect(),
5825 reserved: false,
5826 reserved_prefixes: vec![],
5827 protocol: ModuleProtocol::None,
5828 overlap: Default::default(),
5829 };
5830 let supervisor =
5831 Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default());
5832 let module = supervisor.spawn(initial.clone()).unwrap();
5833 assert!(module.status().unwrap().live);
5834 let mut next = initial;
5835 next.protocol = ModuleProtocol::Subc;
5836 module
5837 .update_configuration(next.clone(), HealthConfig::default(), None)
5838 .await
5839 .unwrap();
5840 assert!(
5841 module.status().unwrap().live,
5842 "rescan must not require HELLO from the old non-wire process"
5843 );
5844 let runtime = supervisor.runtime_config();
5845 let action = on_child_exit(
5846 &next,
5847 RestartPolicy::default(),
5848 &supervisor.registry,
5849 &module.inner.snapshot,
5850 &runtime.terminal_ring,
5851 &runtime.spawn_events,
5852 &runtime.child_roster,
5853 ExitReport {
5854 kind: ExitKind::Clean,
5855 code: Some(0),
5856 signal: None,
5857 at_ms: unix_ms_now(),
5858 },
5859 )
5860 .await;
5861 assert!(
5862 matches!(action, NextAction::Restart { .. }),
5863 "the old non-wire process's clean exit must restart"
5864 );
5865 module.drain().await.unwrap();
5866 }
5867
5868 #[cfg(unix)]
5869 #[tokio::test]
5870 async fn orphan_identity_matches_path_names_and_shebang_interpreters() {
5871 use std::os::unix::fs::PermissionsExt;
5872 let dir = cortexkit_test_support::ScratchDir::new("spawn-image-identity");
5873 let script = dir.join("module.sh");
5874 std::fs::write(&script, "#!/bin/sh\nwhile :; do sleep 0.1; done\n").unwrap();
5875 std::fs::set_permissions(&script, std::fs::Permissions::from_mode(0o700)).unwrap();
5876 let record_path = dir.join("live-children.json");
5877 let supervisor =
5878 Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default())
5879 .with_live_children_record(&record_path);
5880 for (program, args) in [
5881 (PathBuf::from("sleep"), vec!["60".into()]),
5882 (script, vec![]),
5883 ] {
5884 let spec = ModuleSpec {
5885 module_id: "image-identity".into(),
5886 program: program.clone(),
5887 args,
5888 env: ["XDG_DATA_HOME", "XDG_RUNTIME_DIR", "XDG_CONFIG_HOME"]
5889 .into_iter()
5890 .map(|key| (key.into(), dir.to_string_lossy().into_owned()))
5891 .collect(),
5892 reserved: false,
5893 reserved_prefixes: vec![],
5894 protocol: ModuleProtocol::None,
5895 overlap: Default::default(),
5896 };
5897 let module = supervisor.spawn(spec).unwrap();
5898 #[cfg(target_os = "macos")]
5899 {
5900 let deadline = Instant::now() + Duration::from_secs(5);
5903 while crate::live_children::read_record(&record_path)
5904 .unwrap()
5905 .iter()
5906 .all(|entry| entry.executable.is_none())
5907 {
5908 assert!(Instant::now() < deadline, "module image was not confirmed");
5909 tokio::time::sleep(Duration::from_millis(5)).await;
5910 }
5911 }
5912 let entry = crate::live_children::read_record(&record_path)
5913 .unwrap()
5914 .pop()
5915 .unwrap();
5916 let observed = subc_os::Process::open(entry.pid)
5917 .unwrap()
5918 .unwrap()
5919 .observe()
5920 .unwrap();
5921 let verdict = crate::live_children::identity_verdict(&entry, entry.pid, &observed);
5922 module.drain().await.unwrap();
5923 assert_eq!(
5924 verdict,
5925 crate::live_children::IdentityVerdict::Matches,
5926 "program {program:?}: recorded {entry:?}, observed {observed:?}"
5927 );
5928 }
5929 }
5930
5931 #[cfg(unix)]
5932 fn http_fixture(
5933 dir: &std::path::Path,
5934 url: &str,
5935 threshold: u32,
5936 ) -> crate::daemon_config::ConfiguredModule {
5937 let path = dir.join("subc.jsonc");
5938 std::fs::write(&path, serde_json::json!({"version": 1, "modules": {"http-process": {
5939 "program": "/bin/sleep", "args": ["60"], "protocol": "none",
5940 "env": {"XDG_DATA_HOME":dir,"XDG_RUNTIME_DIR":dir,"XDG_CONFIG_HOME":dir},
5941 "health": {"http":url,"cadence_ms":20,"deadline_ms":200,"failure_threshold":threshold},
5942 "restart": {"backoff_ms":1,"max_backoff_ms":1}, "drain_timeout_ms":10
5943 }}}).to_string()).unwrap();
5944 crate::daemon_config::load(&path)
5945 .unwrap()
5946 .unwrap()
5947 .modules
5948 .pop()
5949 .unwrap()
5950 }
5951
5952 #[cfg(unix)]
5953 async fn wait_http_health(module: &SupervisedModule, status: SupervisorHealthStatus) {
5954 timeout(Duration::from_secs(5), async {
5955 loop {
5956 if module.status().unwrap().health.status == status {
5957 break;
5958 }
5959 sleep(Duration::from_millis(5)).await;
5960 }
5961 })
5962 .await
5963 .unwrap_or_else(|_| {
5964 panic!(
5965 "expected {status:?}, got {:?}",
5966 module.status().unwrap().health
5967 )
5968 });
5969 }
5970
5971 #[cfg(unix)]
5972 #[tokio::test]
5973 async fn http_health_status_flips_ok_failing_ok() {
5974 use tokio::io::{AsyncReadExt, AsyncWriteExt};
5975 let dir = cortexkit_test_support::ScratchDir::new("http-status-flips");
5976 let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
5977 let status = Arc::new(std::sync::atomic::AtomicU16::new(200));
5978 let serving_status = status.clone();
5979 let url = format!("http://{}/healthz", listener.local_addr().unwrap());
5980 let server = tokio::spawn(async move {
5981 loop {
5982 let (mut stream, _) = listener.accept().await.unwrap();
5983 let mut request = [0u8; 2048];
5984 let count = stream.read(&mut request).await.unwrap();
5985 assert!(count > 0, "a probe must send an HTTP request");
5986 let code = serving_status.load(std::sync::atomic::Ordering::SeqCst);
5987 let body = if code == 200 {
5988 "ready"
5989 } else {
5990 "{\"status\":\"unavailable\",\"error\":\"scratch failure\"}"
5991 };
5992 let response = format!("HTTP/1.1 {code} scratch\r\nContent-Length: {}\r\nConnection: close\r\n\r\n{body}", body.len());
5993 let _ = stream.write_all(response.as_bytes()).await;
5994 }
5995 });
5996 let configured = http_fixture(&dir, &url, 1000);
5997 let module =
5998 Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default())
5999 .supervise_configured_with_health(
6000 configured.module_spec(),
6001 true,
6002 configured.health,
6003 configured.drain_timeout_ms,
6004 configured.restart,
6005 )
6006 .unwrap();
6007 wait_http_health(&module, SupervisorHealthStatus::Ok).await;
6008 status.store(503, std::sync::atomic::Ordering::SeqCst);
6009 wait_http_health(&module, SupervisorHealthStatus::Failing).await;
6010 assert!(module
6011 .status()
6012 .unwrap()
6013 .health
6014 .detail
6015 .unwrap()
6016 .contains("scratch failure"));
6017 status.store(200, std::sync::atomic::Ordering::SeqCst);
6018 wait_http_health(&module, SupervisorHealthStatus::Ok).await;
6019 assert_eq!(module.status().unwrap().health.consecutive_failures, 0);
6020 let before = module.status().unwrap();
6021 let (spec, mut health) = module.configuration().unwrap();
6022 health.http = None;
6023 module
6024 .update_configuration(spec.clone(), health.clone(), Some(10))
6025 .await
6026 .unwrap();
6027 wait_http_health(&module, SupervisorHealthStatus::Unknown).await;
6028 health.http = Some(url);
6029 module
6030 .update_configuration(spec, health, Some(10))
6031 .await
6032 .unwrap();
6033 wait_http_health(&module, SupervisorHealthStatus::Ok).await;
6034 assert_eq!(
6035 module.status().unwrap().pid,
6036 before.pid,
6037 "changing a probe must apply live, not restart its process"
6038 );
6039 let (spec, mut health) = module.configuration().unwrap();
6040 health.failure_threshold = 2;
6041 module
6042 .update_configuration(spec, health, Some(10))
6043 .await
6044 .unwrap();
6045 status.store(503, std::sync::atomic::Ordering::SeqCst);
6046 timeout(Duration::from_secs(5), async {
6047 while module.status().unwrap().spawn_generation == before.spawn_generation {
6048 sleep(Duration::from_millis(5)).await;
6049 }
6050 })
6051 .await
6052 .expect("sustained HTTP 503 responses must trigger the health restart policy");
6053 module.drain().await.unwrap();
6054 server.abort();
6055 }
6056
6057 #[cfg(unix)]
6058 #[tokio::test]
6059 async fn http_health_no_listener_restarts_after_consecutive_failures() {
6060 let dir = cortexkit_test_support::ScratchDir::new("http-refused");
6061 let unused = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
6062 let url = format!("http://{}/healthz", unused.local_addr().unwrap());
6063 drop(unused);
6064 let configured = http_fixture(&dir, &url, 2);
6065 let module =
6066 Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default())
6067 .supervise_configured_with_health(
6068 configured.module_spec(),
6069 true,
6070 configured.health,
6071 configured.drain_timeout_ms,
6072 configured.restart,
6073 )
6074 .unwrap();
6075 let before = module.status().unwrap().spawn_generation;
6076 timeout(Duration::from_secs(5), async {
6077 loop {
6078 let status = module.status().unwrap();
6079 if status.spawn_generation > before {
6080 assert!(status.lifetime_restarts > 0);
6081 break;
6082 }
6083 sleep(Duration::from_millis(5)).await;
6084 }
6085 })
6086 .await
6087 .expect("sustained HTTP refusal must trigger the health restart policy");
6088 module.drain().await.unwrap();
6089 }
6090
6091 #[cfg(unix)]
6092 #[tokio::test]
6093 async fn http_health_timeout_honours_deadline() {
6094 let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
6095 let url = format!("http://{}/healthz", listener.local_addr().unwrap());
6096 let server = tokio::spawn(async move {
6097 let _held = listener.accept().await.unwrap();
6098 std::future::pending::<()>().await;
6099 });
6100 let error = timeout(
6101 Duration::from_secs(1),
6102 probe_http_health(&url, Duration::from_millis(10)),
6103 )
6104 .await
6105 .expect("the probe must enforce its own deadline")
6106 .unwrap_err();
6107 server.abort();
6108 assert!(error.to_string().contains("timed out"));
6109 }
6110
6111 #[cfg(unix)]
6112 #[tokio::test]
6113 async fn http_health_localhost_reaches_an_ipv6_only_listener() {
6114 use tokio::io::{AsyncReadExt, AsyncWriteExt};
6115 let listener = tokio::net::TcpListener::bind("[::1]:0").await.unwrap();
6116 let url = format!(
6117 "http://localhost:{}/healthz",
6118 listener.local_addr().unwrap().port()
6119 );
6120 let server = tokio::spawn(async move {
6121 let (mut stream, _) = listener.accept().await.unwrap();
6122 let mut request = [0u8; 2048];
6123 assert!(stream.read(&mut request).await.unwrap() > 0);
6124 stream
6125 .write_all(b"HTTP/1.1 200 OK\r\nContent-Length: 2\r\n\r\nok")
6126 .await
6127 .unwrap();
6128 });
6129 assert_eq!(
6134 probe_http_health(&url, Duration::from_secs(10))
6135 .await
6136 .unwrap()
6137 .status,
6138 HealthStatus::Ok
6139 );
6140 server.await.unwrap();
6141 }
6142
6143 #[cfg(unix)]
6144 #[tokio::test]
6145 async fn http_health_timeout_keeps_partial_status_and_body() {
6146 use tokio::io::{AsyncReadExt, AsyncWriteExt};
6147 let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
6148 let url = format!("http://{}/healthz", listener.local_addr().unwrap());
6149 let server = tokio::spawn(async move {
6150 let (mut stream, _) = listener.accept().await.unwrap();
6151 let mut request = [0u8; 2048];
6152 assert!(stream.read(&mut request).await.unwrap() > 0);
6153 stream
6154 .write_all(
6155 b"HTTP/1.1 503 Unavailable\r\nContent-Length: 100\r\n\r\npartial diagnostic",
6156 )
6157 .await
6158 .unwrap();
6159 std::future::pending::<()>().await;
6160 });
6161 let error = probe_http_health(&url, Duration::from_secs(1))
6162 .await
6163 .unwrap_err()
6164 .to_string();
6165 server.abort();
6166 assert!(
6167 error.contains("timed out")
6168 && error.contains("503 Unavailable")
6169 && error.contains("partial diagnostic"),
6170 "{error}"
6171 );
6172 }
6173
6174 #[cfg(unix)]
6175 #[tokio::test]
6176 async fn http_health_diagnostic_body_is_bounded_to_200_bytes() {
6177 use tokio::io::{AsyncReadExt, AsyncWriteExt};
6178 let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
6179 let url = format!("http://{}/healthz", listener.local_addr().unwrap());
6180 let server = tokio::spawn(async move {
6181 let (mut stream, _) = listener.accept().await.unwrap();
6182 let mut request = [0u8; 2048];
6183 assert!(stream.read(&mut request).await.unwrap() > 0);
6184 let body = format!("{}not-in-diagnostic", "x".repeat(400));
6185 let response = format!(
6186 "HTTP/1.1 503 Unavailable\r\nContent-Length: {}\r\n\r\n{body}",
6187 body.len()
6188 );
6189 stream.write_all(response.as_bytes()).await.unwrap();
6190 });
6191 let error = probe_http_health(&url, Duration::from_secs(1))
6192 .await
6193 .unwrap_err()
6194 .to_string();
6195 server.await.unwrap();
6196 assert!(error.ends_with(&"x".repeat(200)), "{error}");
6197 assert_eq!(error.split(": ").last().unwrap().len(), 200);
6198 assert!(!error.contains("not-in-diagnostic"));
6199 }
6200
6201 #[cfg(unix)]
6202 #[tokio::test]
6203 async fn http_health_real_nats_server_monitoring() {
6204 if std::process::Command::new("nats-server")
6205 .arg("--version")
6206 .env("XDG_DATA_HOME", std::env::temp_dir())
6207 .env("XDG_RUNTIME_DIR", std::env::temp_dir())
6208 .env("XDG_CONFIG_HOME", std::env::temp_dir())
6209 .output()
6210 .is_err()
6211 {
6212 eprintln!(
6213 "skipped: http_health_real_nats_server_monitoring (nats-server is not on PATH)"
6214 );
6215 return;
6216 }
6217 let dir = cortexkit_test_support::ScratchDir::new("nats-http-monitoring");
6218 let monitor = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
6219 let port = monitor.local_addr().unwrap().port();
6220 let client = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
6221 let client_port = client.local_addr().unwrap().port();
6222 let config = dir.join("server.conf");
6223 std::fs::write(&config, format!("listen: 127.0.0.1:{client_port}\nhttp: 127.0.0.1:{port}\njetstream {{ store_dir: \"{}\" }}\n", dir.join("jetstream").display())).unwrap();
6224 let mut configured = http_fixture(&dir, &format!("http://127.0.0.1:{port}/healthz"), 1000);
6225 configured.program = PathBuf::from("nats-server");
6226 configured.args = vec!["-c".into(), config.to_string_lossy().into_owned()];
6227 drop(monitor);
6228 drop(client);
6229 let module =
6230 Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default())
6231 .supervise_configured_with_health(
6232 configured.module_spec(),
6233 true,
6234 configured.health,
6235 configured.drain_timeout_ms,
6236 configured.restart,
6237 )
6238 .unwrap();
6239 wait_http_health(&module, SupervisorHealthStatus::Ok).await;
6240 module.drain().await.unwrap();
6241 }
6242
6243 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
6244 async fn configuration_update_does_not_replace_captured_running_process_facts() {
6245 let supervisor =
6246 Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default());
6247 let snapshot = stale_process_snapshot(ModuleState::Running, true);
6248 let initial = ModuleSpec {
6249 module_id: "rescan-preserves-spawn-facts".to_string(),
6250 program: PathBuf::from("/spawned/module"),
6251 args: Vec::new(),
6252 env: Vec::new(),
6253 reserved: false,
6254 reserved_prefixes: Vec::new(),
6255 protocol: ModuleProtocol::Subc,
6256 overlap: Default::default(),
6257 };
6258 let module = supervisor.supervised_module(
6259 initial.clone(),
6260 supervisor.runtime_config(),
6261 snapshot,
6262 None,
6263 );
6264 let before = module.status().unwrap();
6265 let mut replacement = initial;
6266 replacement.program = PathBuf::from("/rescanned/replacement-module");
6267
6268 module
6269 .update_configuration(replacement, HealthConfig::default(), None)
6270 .await
6271 .unwrap();
6272
6273 let after = module.status().unwrap();
6274 assert_eq!(after.pid, before.pid);
6275 assert_eq!(after.spawned_at_ms, before.spawned_at_ms);
6276 assert_eq!(after.spawned_from, before.spawned_from);
6277 drop(module);
6278 }
6279}
6280
6281fn unix_ms_now() -> u64 {
6282 SystemTime::now()
6283 .duration_since(UNIX_EPOCH)
6284 .map(|duration| duration.as_millis().min(u128::from(u64::MAX)) as u64)
6285 .unwrap_or(0)
6286}
6287
6288async fn supervise_loop(
6289 mut spec: ModuleSpec,
6290 mut runtime: SupervisorRuntimeConfig,
6291 registry: Arc<Registry>,
6292 process_liveness: Arc<SupervisorProcessLiveness>,
6293 snapshot: SharedSnapshot,
6294 mut child: Option<SupervisedChild>,
6295 mut commands: mpsc::Receiver<SupervisorCommand>,
6296) {
6297 let mut health_probe = HealthProbeRuntime::default();
6298 let mut registration_changes = match registry.subscribe_module_changes(&spec.module_id) {
6302 Ok(changes) => Some(changes),
6303 Err(err) => {
6304 warn!(module_id = %spec.module_id, error = %err, "health prober could not subscribe to registry");
6307 None
6308 }
6309 };
6310 let mut pending_respawn: Option<PendingRespawn> = None;
6314 let mut requeued: VecDeque<SupervisorCommand> = VecDeque::new();
6317 let mut previous_instance_recheck = tokio::time::interval_at(
6318 Instant::now() + Duration::from_secs(5),
6319 Duration::from_secs(5),
6320 );
6321 previous_instance_recheck.set_missed_tick_behavior(tokio::time::MissedTickBehavior::Skip);
6322 loop {
6323 #[cfg(test)]
6324 {
6325 lock_snapshot(&snapshot).unwrap().actor_select = None;
6326 }
6327 #[cfg(target_os = "macos")]
6328 if let Some(active) = child.as_mut() {
6329 active.confirm_privacy_exec().await;
6330 }
6331 if let Some(scheduled) = runtime
6332 .scheduled_respawn
6333 .lock()
6334 .unwrap_or_else(|p| p.into_inner())
6335 .take()
6336 {
6337 pending_respawn = Some(scheduled);
6338 }
6339 if pending_respawn.is_some() && (child.is_some() || !respawn_still_pending(&snapshot)) {
6340 pending_respawn = None;
6341 cancel_deferred_reload(
6342 &runtime,
6343 &spec.module_id,
6344 "respawn cancelled by a supervisor command",
6345 );
6346 }
6347 if child.is_none()
6348 && pending_respawn.is_none()
6349 && lock_snapshot(&snapshot).is_ok_and(|state| state.waiting_for_previous_pid.is_none())
6350 {
6351 cancel_deferred_reload(
6352 &runtime,
6353 &spec.module_id,
6354 "respawn cancelled before a replacement was spawned",
6355 );
6356 let _ = update_snapshot(&snapshot, Some(&spec.module_id), |state| {
6357 state.respawn_pending = false;
6358 state.coalesced_restart_pending = false;
6359 if matches!(
6360 state.state,
6361 ModuleState::Restarting
6362 | ModuleState::Starting
6363 | ModuleState::Draining
6364 | ModuleState::Unresponsive
6365 ) {
6366 error!(module_id = %spec.module_id, state = ?state.state,
6367 "supervision operation ended without a child or pending respawn; marking failed so start can retry");
6368 state.state = ModuleState::Failed;
6369 clear_current_process_facts(state);
6370 }
6371 });
6372 }
6373 if let Some(command) = requeued.pop_front() {
6374 if !handle_supervisor_command(
6375 command,
6376 &mut spec,
6377 &mut runtime,
6378 ®istry,
6379 &process_liveness,
6380 &snapshot,
6381 &mut child,
6382 &mut commands,
6383 &mut requeued,
6384 )
6385 .await
6386 {
6387 return;
6388 }
6389 if child.is_some() || !respawn_still_pending(&snapshot) {
6390 pending_respawn = None;
6391 let _ = update_snapshot(&snapshot, Some(&spec.module_id), |state| {
6392 state.respawn_pending = false
6393 });
6394 }
6395 continue;
6396 }
6397 if child.is_some() {
6398 #[cfg(test)]
6399 {
6400 lock_snapshot(&snapshot).unwrap().actor_turns += 1;
6401 }
6402 health_probe.refresh_registration(&spec, &runtime, ®istry, &snapshot);
6403 let wake_after = health_probe.wake_after();
6404 let probe_sleep = async move {
6405 match wake_after {
6406 Some(delay) => sleep(delay).await,
6407 None => std::future::pending().await,
6408 }
6409 };
6410 tokio::pin!(probe_sleep);
6411 let wire_child = running_protocol(&spec, &snapshot) == ModuleProtocol::Subc;
6416 #[cfg(test)]
6417 {
6418 let registration_pending = wire_child
6419 && registration_changes
6420 .as_ref()
6421 .is_some_and(|changes| changes.has_changed().unwrap());
6422 let mut state = lock_snapshot(&snapshot).unwrap();
6423 state.actor_select = (!registration_pending).then_some(ActorSelectCheckpoint {
6424 generation: state.spawn_generation,
6425 turn: state.actor_turns,
6426 registered_connection: health_probe.registered_connection,
6427 next_probe_at: health_probe.next_probe_at,
6428 wake_after,
6429 });
6430 }
6431 let active_child = child.as_mut().expect("child checked above");
6432 tokio::select! {
6433 wait_result = active_child.wait() => {
6434 #[cfg(test)]
6435 { lock_snapshot(&snapshot).unwrap().actor_select = None; }
6436 let exit_report = match wait_result {
6445 Ok(status) => classify_reaped_child_exit(&snapshot, active_child, &status),
6446 Err(err) => {
6447 active_child.drain_stderr(&spec.module_id).await;
6448 fail_after_wait_error(
6449 &spec.module_id,
6450 &snapshot,
6451 &runtime.terminal_ring,
6452 &runtime.spawn_events,
6453 format!("failed to wait for supervised module: {err}"),
6454 );
6455 untrack_if_registration_released(
6456 &process_liveness,
6457 ®istry,
6458 &spec.module_id,
6459 &snapshot,
6460 );
6461 error!(module_id = %spec.module_id, error = %err, "failed to wait for supervised module");
6462 child = None;
6463 continue;
6464 }
6465 };
6466 active_child.drain_stderr(&spec.module_id).await;
6467
6468 let next = on_child_exit(
6469 &spec,
6470 runtime.restart_policy,
6471 ®istry,
6472 &snapshot,
6473 &runtime.terminal_ring,
6474 &runtime.spawn_events,
6475 &runtime.child_roster,
6476 exit_report,
6477 ).await;
6478 runtime.start_requester = None;
6479 runtime.pending_activity_restart = false;
6480 active_child.release_roster();
6483 match next {
6484 NextAction::Stop { registration_released } => {
6485 if registration_released {
6486 process_liveness.untrack_if_current(&spec.module_id, &snapshot);
6487 }
6488 child = None;
6489 }
6490 NextAction::Restart { schedule } => {
6491 let delay = schedule.map_or(
6492 runtime.restart_policy.delay_for_restart(0),
6493 |schedule| schedule.delay,
6494 );
6495 if let Some(schedule) = schedule {
6496 log_crash_respawn(&spec.module_id, schedule);
6497 }
6498 child = None;
6506 pending_respawn = Some(PendingRespawn { deadline: Instant::now() + delay, kind: RespawnKind::Spawn });
6507 let _ = update_snapshot(&snapshot, Some(&spec.module_id), |state| state.respawn_pending = true);
6508 }
6509 }
6510 }
6511 command = commands.recv() => {
6512 #[cfg(test)]
6513 { lock_snapshot(&snapshot).unwrap().actor_select = None; }
6514 let Some(command) = command else {
6515 return;
6516 };
6517 if !handle_supervisor_command(
6518 command,
6519 &mut spec,
6520 &mut runtime,
6521 ®istry,
6522 &process_liveness,
6523 &snapshot,
6524 &mut child,
6525 &mut commands,
6526 &mut requeued,
6527 ).await {
6528 return;
6529 }
6530 }
6531 _ = async {
6532 match registration_changes.as_mut() {
6533 Some(changes) => { let _ = changes.changed().await; }
6534 None => std::future::pending().await,
6535 }
6536 }, if wire_child => {
6537 #[cfg(test)]
6538 { lock_snapshot(&snapshot).unwrap().actor_select = None; }
6539 }
6540 _ = &mut probe_sleep => {
6541 #[cfg(test)]
6542 { lock_snapshot(&snapshot).unwrap().actor_select = None; }
6543 if health_probe.due() {
6544 run_health_probe_cycle(
6545 &spec,
6546 &runtime,
6547 ®istry,
6548 &process_liveness,
6549 &snapshot,
6550 &mut child,
6551 ).await;
6552 if child.is_none() {
6553 runtime.start_requester = None;
6554 runtime.pending_activity_restart = false;
6555 }
6556 if child.is_some() {
6557 health_probe.schedule_next(&spec, runtime.health.cadence);
6558 }
6559 }
6560 }
6561 }
6562 } else if let Some(pending) = pending_respawn {
6563 tokio::select! {
6564 _ = sleep_until(pending.deadline) => {
6565 pending_respawn = None;
6566 let _ = update_snapshot(&snapshot, Some(&spec.module_id), |state| state.respawn_pending = false);
6567 if !respawn_still_pending(&snapshot) {
6571 continue;
6572 }
6573 if runtime.child_roster.is_closed() {
6578 let _ = update_snapshot(&snapshot, Some(&spec.module_id), |state| {
6579 state.state = ModuleState::Stopped;
6580 });
6581 debug!(module_id = %spec.module_id, "crash respawn cancelled by daemon shutdown");
6582 continue;
6583 }
6584 if let Err(err) = release_dead_registration(
6585 ®istry,
6586 runtime.forwarding.as_deref(),
6587 &snapshot,
6588 &spec.module_id,
6589 ).await {
6590 fail_snapshot(&snapshot, Some(&spec.module_id), None);
6591 error!(module_id = %spec.module_id, error = %err, "registration did not release before restart");
6592 continue;
6593 }
6594
6595 if matches!(pending.kind, RespawnKind::Reload) {
6596 let reply = runtime.deferred_reload_reply.lock().unwrap_or_else(|p| p.into_inner()).take();
6597 let result = finish_reload_child(&spec, &runtime, ®istry, &process_liveness, &snapshot, &mut child).await;
6598 if result.is_ok() && runtime.pending_activity_restart {
6599 runtime.record_activity_transition("subc.module_restarted", &spec.module_id, runtime.start_requester.clone());
6600 }
6601 runtime.pending_activity_restart = false;
6602 if let Some(reply) = reply { let _ = reply.send(result); }
6603 continue;
6604 }
6605 process_liveness.track(spec.module_id.clone(), Arc::clone(&snapshot));
6606 match spawn_and_mark_running(&spec, &runtime, &snapshot) {
6607 Ok(next_child) => {
6608 child = Some(next_child);
6609 if runtime.pending_activity_restart {
6610 runtime.record_activity_transition("subc.module_restarted", &spec.module_id, runtime.start_requester.clone());
6611 runtime.pending_activity_restart = false;
6612 }
6613 debug!(module_id = %spec.module_id, "supervised module restarted after crash");
6614 }
6615 Err(err) => {
6616 fail_snapshot(&snapshot, Some(&spec.module_id), None);
6617 process_liveness.untrack_if_current(&spec.module_id, &snapshot);
6618 error!(module_id = %spec.module_id, error = %err, "failed to restart supervised module");
6619 }
6620 }
6621 }
6622 command = commands.recv() => {
6623 let Some(command) = command else {
6624 return;
6625 };
6626 if !handle_supervisor_command(
6627 command,
6628 &mut spec,
6629 &mut runtime,
6630 ®istry,
6631 &process_liveness,
6632 &snapshot,
6633 &mut child,
6634 &mut commands,
6635 &mut requeued,
6636 ).await {
6637 return;
6638 }
6639 if child.is_some() || !respawn_still_pending(&snapshot) {
6644 pending_respawn = None;
6645 let _ = update_snapshot(&snapshot, Some(&spec.module_id), |state| state.respawn_pending = false);
6646 }
6647 }
6648 }
6649 } else {
6650 let command = if lock_snapshot(&snapshot)
6651 .is_ok_and(|state| state.waiting_for_previous_pid.is_some())
6652 {
6653 tokio::select! {
6654 command = commands.recv() => command,
6655 _ = previous_instance_recheck.tick() => {
6656 if runtime.child_roster.previous_instance_pid(&spec.module_id).is_none() {
6657 let enabled = lock_snapshot(&snapshot).is_ok_and(|state| state.enabled);
6658 if enabled && !runtime.child_roster.is_closed() {
6659 process_liveness.track(spec.module_id.clone(), Arc::clone(&snapshot));
6660 match spawn_and_mark_running(&spec, &runtime, &snapshot) {
6661 Ok(next_child) => child = Some(next_child),
6662 Err(error) => {
6663 fail_snapshot(&snapshot, Some(&spec.module_id), None);
6664 error!(module_id = %spec.module_id, %error, "failed to start module after its previous instance exited");
6665 }
6666 }
6667 } else {
6668 let _ = update_snapshot(&snapshot, Some(&spec.module_id), |state| {
6669 state.waiting_for_previous_pid = None;
6670 state.waiting_for_previous_detail = None;
6671 });
6672 }
6673 }
6674 continue;
6675 }
6676 }
6677 } else {
6678 commands.recv().await
6679 };
6680 let Some(command) = command else {
6681 return;
6682 };
6683 if !handle_supervisor_command(
6684 command,
6685 &mut spec,
6686 &mut runtime,
6687 ®istry,
6688 &process_liveness,
6689 &snapshot,
6690 &mut child,
6691 &mut commands,
6692 &mut requeued,
6693 )
6694 .await
6695 {
6696 return;
6697 }
6698 }
6699 }
6700}
6701
6702fn log_crash_respawn(module_id: &str, schedule: CrashRestartSchedule) {
6703 info!(
6704 module_id,
6705 restart_in_window = schedule.restart_in_window,
6706 delay_ms = schedule.delay.as_millis() as u64,
6707 "respawning after crash"
6708 );
6709}
6710
6711fn respawn_still_pending(snapshot: &SharedSnapshot) -> bool {
6717 matches!(
6718 lock_snapshot(snapshot),
6719 Ok(state) if state.enabled && state.state == ModuleState::Restarting
6720 )
6721}
6722
6723enum NextAction {
6724 Stop {
6725 registration_released: bool,
6726 },
6727 Restart {
6728 schedule: Option<CrashRestartSchedule>,
6729 },
6730}
6731
6732#[allow(clippy::too_many_arguments)]
6733async fn handle_supervisor_command(
6734 command: SupervisorCommand,
6735 spec: &mut ModuleSpec,
6736 runtime: &mut SupervisorRuntimeConfig,
6737 registry: &Arc<Registry>,
6738 process_liveness: &SupervisorProcessLiveness,
6739 snapshot: &SharedSnapshot,
6740 child: &mut Option<SupervisedChild>,
6741 commands: &mut mpsc::Receiver<SupervisorCommand>,
6742 requeued: &mut VecDeque<SupervisorCommand>,
6743) -> bool {
6744 let command =
6745 if let Some((pid, reason)) = runtime.child_roster.previous_instance(&spec.module_id) {
6746 match command {
6747 SupervisorCommand::Restart { reply, .. }
6748 | SupervisorCommand::Reload { reply, .. }
6749 | SupervisorCommand::Swap { reply, .. } => {
6750 let _ = reply.send(Err(previous_instance_error(spec, pid, reason)));
6751 return true;
6752 }
6753 SupervisorCommand::SetEnabled {
6754 enabled: true,
6755 reply,
6756 ..
6757 } => {
6758 let _ = reply.send(Err(previous_instance_error(spec, pid, reason)));
6759 return true;
6760 }
6761 other => other,
6762 }
6763 } else {
6764 command
6765 };
6766 match command {
6767 SupervisorCommand::Drain { requester, reply } => {
6768 let result = drain_optional_child(
6771 &spec.module_id,
6772 spec.protocol,
6773 StopNotice::NotSent,
6774 registry,
6775 runtime.forwarding.as_deref(),
6776 snapshot,
6777 &runtime.terminal_ring,
6778 &runtime.spawn_events,
6779 child,
6780 runtime.drain_timeout,
6781 ModuleState::Stopped,
6782 None,
6783 )
6784 .await;
6785 let registration_released = result.is_ok();
6786 if registration_released {
6787 runtime.record_activity_transition(
6788 "subc.module_stopped",
6789 &spec.module_id,
6790 requester,
6791 );
6792 }
6793 let _ = reply.send(result);
6794 if registration_released {
6795 process_liveness.untrack_if_current(&spec.module_id, snapshot);
6796 }
6797 false
6798 }
6799 SupervisorCommand::Retire { requester, reply } => {
6800 let result = async {
6801 let stop_notice = begin_forwarding_drain_if_configured(
6802 spec,
6803 runtime,
6804 registry,
6805 snapshot,
6806 None,
6807 RouteCloseReason::Disable,
6808 )
6809 .await?;
6810 drain_optional_child(
6811 &spec.module_id,
6812 spec.protocol,
6813 stop_notice,
6814 registry,
6815 runtime.forwarding.as_deref(),
6816 snapshot,
6817 &runtime.terminal_ring,
6818 &runtime.spawn_events,
6819 child,
6820 runtime.drain_timeout,
6821 ModuleState::Stopped,
6822 None,
6823 )
6824 .await
6825 }
6826 .await;
6827 let registration_released = result.is_ok();
6828 if registration_released {
6829 runtime.record_activity_transition(
6830 "subc.module_stopped",
6831 &spec.module_id,
6832 requester,
6833 );
6834 }
6835 let _ = reply.send(result);
6836 if registration_released {
6837 process_liveness.untrack_if_current(&spec.module_id, snapshot);
6838 }
6839 false
6840 }
6841 SupervisorCommand::Restart {
6842 requester,
6843 drain_timeout_ms,
6844 received_at_generation,
6845 queued_at,
6846 reply,
6847 } => {
6848 info!(
6852 module_id = %spec.module_id,
6853 queued_ms = u64::try_from(queued_at.elapsed().as_millis()).unwrap_or(u64::MAX),
6854 "restart command dequeued"
6855 );
6856 let validation = match lock_snapshot(snapshot) {
6868 Ok(state) if !state.enabled => Err(SuperviseError::Disabled {
6869 module_id: spec.module_id.clone(),
6870 }),
6871 Ok(_) => Ok(()),
6872 Err(err) => Err(err),
6873 };
6874 let initiated = validation.is_ok();
6875 let _ = reply.send(validation);
6876 let satisfied_by_generation = if initiated && child.is_some() {
6887 lock_snapshot(snapshot).ok().and_then(|state| {
6888 (state.spawn_generation > received_at_generation
6889 && !state.configuration_updated_since_spawn)
6890 .then_some(state.spawn_generation)
6891 })
6892 } else {
6893 None
6894 };
6895 let satisfied_by_pending = initiated
6896 && child.is_none()
6897 && lock_snapshot(snapshot).ok().is_some_and(|mut state| {
6898 let pending = state.respawn_pending && !state.configuration_updated_since_spawn;
6899 if pending {
6900 state.coalesced_restart_pending = true;
6901 }
6902 pending
6903 });
6904 if satisfied_by_pending {
6905 debug!(module_id = %spec.module_id, "restart coalesced into the pending replacement");
6906 } else if let Some(generation) = satisfied_by_generation {
6907 info!(
6908 module_id = %spec.module_id,
6909 received_at_generation,
6910 "restart already satisfied by generation {generation}; not restarting again"
6911 );
6912 } else if initiated {
6913 let drain_timeout = drain_timeout_ms
6916 .map(Duration::from_millis)
6917 .unwrap_or(runtime.drain_timeout);
6918 if let Err(err) = restart_child(
6919 spec,
6920 runtime,
6921 registry,
6922 process_liveness,
6923 snapshot,
6924 child,
6925 drain_timeout,
6926 )
6927 .await
6928 {
6929 warn!(
6930 module_id = %spec.module_id,
6931 error = %err,
6932 "operator restart failed after initiation ack; module state carries the outcome"
6933 );
6934 let _ = update_snapshot(snapshot, Some(&spec.module_id), |state| {
6935 state.state = ModuleState::Failed;
6936 clear_current_process_facts(state);
6937 });
6938 } else {
6939 runtime.start_requester = requester;
6940 runtime.pending_activity_restart = true;
6941 }
6942 }
6943 true
6944 }
6945 SupervisorCommand::Reload { requester, reply } => {
6946 let result =
6947 reload_child(spec, runtime, registry, process_liveness, snapshot, child).await;
6948 if result.is_ok() {
6949 runtime.start_requester = requester;
6950 runtime.pending_activity_restart = true;
6951 }
6952 if result.is_ok()
6953 && runtime
6954 .scheduled_respawn
6955 .lock()
6956 .unwrap_or_else(|p| p.into_inner())
6957 .as_ref()
6958 .is_some_and(|pending| matches!(pending.kind, RespawnKind::Reload))
6959 {
6960 *runtime
6961 .deferred_reload_reply
6962 .lock()
6963 .unwrap_or_else(|p| p.into_inner()) = Some(reply);
6964 } else {
6965 let _ = reply.send(result);
6966 }
6967 true
6968 }
6969 SupervisorCommand::SetEnabled {
6970 enabled,
6971 requester,
6972 reply,
6973 } => {
6974 let previous_requester = runtime.start_requester.clone();
6975 if enabled {
6976 runtime.start_requester = requester.clone();
6977 }
6978 let result = set_child_enabled(
6979 spec,
6980 runtime,
6981 registry,
6982 process_liveness,
6983 snapshot,
6984 child,
6985 enabled,
6986 )
6987 .await;
6988 if matches!(result, Ok(true)) {
6989 runtime.pending_activity_restart = false;
6990 if enabled {
6991 runtime.start_requester = requester;
6992 } else {
6993 runtime.start_requester = None;
6994 runtime.record_activity_transition(
6995 "subc.module_disabled",
6996 &spec.module_id,
6997 requester,
6998 );
6999 }
7000 } else {
7001 runtime.start_requester = previous_requester;
7002 }
7003 let _ = reply.send(result);
7004 true
7005 }
7006 SupervisorCommand::UpdateConfiguration {
7007 spec: next_spec,
7008 health,
7009 drain_timeout_ms,
7010 reply,
7011 } => {
7012 if let Some(handle) = &runtime.supervisor_handle {
7013 handle.apply_identity_configuration(&next_spec);
7014 }
7015 *spec = next_spec;
7016 let _ = update_snapshot(snapshot, Some(&spec.module_id), |state| {
7017 state.configuration_updated_since_spawn = true;
7018 });
7019 let health_changed = runtime.health != health;
7020 runtime.health = health;
7021 if health_changed {
7024 let _ = update_snapshot(snapshot, Some(&spec.module_id), |state| {
7025 state.health = ModuleHealthStatus::default();
7026 });
7027 }
7028 runtime.drain_timeout = drain_timeout_ms
7029 .map(Duration::from_millis)
7030 .unwrap_or(runtime.default_drain_timeout);
7031 *runtime
7032 .effective_drain_timeout
7033 .lock()
7034 .unwrap_or_else(|poisoned| poisoned.into_inner()) = runtime.drain_timeout;
7035 let _ = reply.send(());
7036 true
7037 }
7038 SupervisorCommand::Swap {
7039 requester,
7040 ready_timeout,
7041 reply,
7042 } => {
7043 let mut swap_runtime = runtime.clone();
7044 swap_runtime.start_requester = requester;
7045 let before = lock_snapshot(snapshot)
7046 .ok()
7047 .map(|state| state.spawn_generation);
7048 let end = swap::run_swap(
7049 spec,
7050 &swap_runtime,
7051 registry,
7052 process_liveness,
7053 snapshot,
7054 child,
7055 commands,
7056 ready_timeout.unwrap_or(DEFAULT_SWAP_READY_TIMEOUT),
7057 reply,
7058 )
7059 .await;
7060 if before
7061 != lock_snapshot(snapshot)
7062 .ok()
7063 .map(|state| state.spawn_generation)
7064 {
7065 runtime.start_requester = swap_runtime.start_requester;
7066 }
7067 requeued.extend(end.requeue);
7068 true
7069 }
7070 }
7071}
7072
7073async fn restart_child(
7074 spec: &ModuleSpec,
7075 runtime: &SupervisorRuntimeConfig,
7076 registry: &Registry,
7077 process_liveness: &SupervisorProcessLiveness,
7078 snapshot: &SharedSnapshot,
7079 child: &mut Option<SupervisedChild>,
7080 drain_timeout: Duration,
7081) -> Result<(), SuperviseError> {
7082 if !lock_snapshot(snapshot)?.enabled {
7084 return Err(SuperviseError::Disabled {
7085 module_id: spec.module_id.clone(),
7086 });
7087 }
7088 let stop_notice = begin_forwarding_drain_with_timeout(
7089 spec,
7090 runtime,
7091 registry,
7092 snapshot,
7093 None,
7094 RouteCloseReason::Restart,
7095 drain_timeout,
7096 )
7097 .await?;
7098
7099 if child.is_some() {
7100 drain_optional_child(
7101 &spec.module_id,
7102 spec.protocol,
7103 stop_notice,
7104 registry,
7105 runtime.forwarding.as_deref(),
7106 snapshot,
7107 &runtime.terminal_ring,
7108 &runtime.spawn_events,
7109 child,
7110 drain_timeout,
7111 ModuleState::Restarting,
7112 Some(true),
7113 )
7114 .await?;
7115 } else {
7116 update_snapshot(snapshot, Some(&spec.module_id), |state| {
7117 state.enabled = true;
7118 state.state = ModuleState::Restarting;
7119 clear_current_process_facts(state);
7120 })?;
7121 release_dead_registration(
7122 registry,
7123 runtime.forwarding.as_deref(),
7124 snapshot,
7125 &spec.module_id,
7126 )
7127 .await?;
7128 }
7129
7130 reset_restart_count(snapshot, &spec.module_id)?;
7131 process_liveness.track(spec.module_id.clone(), Arc::clone(snapshot));
7132 schedule_respawn(
7133 runtime,
7134 snapshot,
7135 &spec.module_id,
7136 runtime.restart_policy.backoff,
7137 RespawnKind::Spawn,
7138 )
7139}
7140
7141async fn reload_child(
7142 spec: &ModuleSpec,
7143 runtime: &SupervisorRuntimeConfig,
7144 registry: &Registry,
7145 process_liveness: &SupervisorProcessLiveness,
7146 snapshot: &SharedSnapshot,
7147 child: &mut Option<SupervisedChild>,
7148) -> Result<(), SuperviseError> {
7149 if !lock_snapshot(snapshot)?.enabled {
7151 return Err(SuperviseError::Disabled {
7152 module_id: spec.module_id.clone(),
7153 });
7154 }
7155 let stop_notice = begin_forwarding_drain(
7156 spec,
7157 runtime,
7158 registry,
7159 snapshot,
7160 Some(true),
7161 RouteCloseReason::Reload,
7162 )
7163 .await?;
7164
7165 if child.is_some() {
7166 drain_optional_child(
7167 &spec.module_id,
7168 spec.protocol,
7169 stop_notice,
7170 registry,
7171 runtime.forwarding.as_deref(),
7172 snapshot,
7173 &runtime.terminal_ring,
7174 &runtime.spawn_events,
7175 child,
7176 runtime.drain_timeout,
7177 ModuleState::Restarting,
7178 Some(true),
7179 )
7180 .await?;
7181 } else {
7182 update_snapshot(snapshot, Some(&spec.module_id), |state| {
7183 state.enabled = true;
7184 state.state = ModuleState::Restarting;
7185 clear_current_process_facts(state);
7186 })?;
7187 release_dead_registration(
7188 registry,
7189 runtime.forwarding.as_deref(),
7190 snapshot,
7191 &spec.module_id,
7192 )
7193 .await?;
7194 }
7195
7196 reset_restart_count(snapshot, &spec.module_id)?;
7197 process_liveness.track(spec.module_id.clone(), Arc::clone(snapshot));
7198 schedule_respawn(
7199 runtime,
7200 snapshot,
7201 &spec.module_id,
7202 runtime.restart_policy.backoff,
7203 RespawnKind::Reload,
7204 )
7205}
7206
7207async fn finish_reload_child(
7208 spec: &ModuleSpec,
7209 runtime: &SupervisorRuntimeConfig,
7210 registry: &Registry,
7211 process_liveness: &SupervisorProcessLiveness,
7212 snapshot: &SharedSnapshot,
7213 child: &mut Option<SupervisedChild>,
7214) -> Result<(), SuperviseError> {
7215 process_liveness.track(spec.module_id.clone(), Arc::clone(snapshot));
7216 let next_child = match spawn_and_mark_running(spec, runtime, snapshot) {
7217 Ok(next_child) => next_child,
7218 Err(err) => {
7219 return handle_reload_spawn_failure(
7220 spec,
7221 runtime,
7222 process_liveness,
7223 snapshot,
7224 child,
7225 format!("new child failed to spawn: {err}"),
7226 )
7227 .await;
7228 }
7229 };
7230 *child = Some(next_child);
7231
7232 let wait_outcome = {
7233 let active_child = child.as_mut().expect("new reload child was just stored");
7234 wait_for_registration_after_reload(
7235 registry,
7236 &spec.module_id,
7237 snapshot,
7238 active_child,
7239 REGISTRY_RELEASE_TIMEOUT,
7240 )
7241 .await?
7242 };
7243
7244 match wait_outcome {
7245 RegistrationWaitOutcome::Registered => {
7246 debug!(module_id = %spec.module_id, "supervised module reloaded and registered");
7247 Ok(())
7248 }
7249 RegistrationWaitOutcome::Exited(exit_report) => {
7250 if let Some(active_child) = child.as_mut() {
7251 active_child.drain_stderr(&spec.module_id).await;
7252 }
7253 let mut exited_child = child.take().expect("exited reload child is still stored");
7256 #[cfg(test)]
7257 if let Some(gate) = &runtime.test_reload_exit_record_gate {
7258 gate.reached.notify_one();
7259 gate.resume.notified().await;
7260 }
7261 let result = handle_reload_child_registration_failure(
7262 spec,
7263 runtime,
7264 registry,
7265 process_liveness,
7266 snapshot,
7267 child,
7268 ReloadRegistrationFailure {
7269 exit_report: registration_failure_exit_report(exit_report),
7270 reason: exited_child
7271 .spawn_failure
7272 .clone()
7273 .unwrap_or_else(|| "new child exited before registering".to_string()),
7274 },
7275 )
7276 .await;
7277 exited_child.release_roster();
7278 result
7279 }
7280 RegistrationWaitOutcome::TimedOut => {
7281 let mut timed_out_child = child
7282 .take()
7283 .expect("timed-out reload child is still running");
7284 timed_out_child
7285 .start_kill()
7286 .map_err(|source| SuperviseError::Kill {
7287 module_id: spec.module_id.clone(),
7288 source,
7289 })?;
7290 let status = timed_out_child
7291 .wait()
7292 .await
7293 .map_err(|source| SuperviseError::Wait {
7294 module_id: spec.module_id.clone(),
7295 source,
7296 })?;
7297 timed_out_child.drain_stderr(&spec.module_id).await;
7298 handle_reload_child_registration_failure(
7299 spec,
7300 runtime,
7301 registry,
7302 process_liveness,
7303 snapshot,
7304 child,
7305 ReloadRegistrationFailure {
7306 exit_report: registration_failure_exit_report(classify_reaped_child_exit(
7307 snapshot,
7308 &timed_out_child,
7309 &status,
7310 )),
7311 reason: format!(
7312 "new child did not register within {:?}",
7313 REGISTRY_RELEASE_TIMEOUT
7314 ),
7315 },
7316 )
7317 .await
7318 }
7319 }
7320}
7321
7322async fn set_child_enabled(
7323 spec: &ModuleSpec,
7324 runtime: &SupervisorRuntimeConfig,
7325 registry: &Registry,
7326 process_liveness: &SupervisorProcessLiveness,
7327 snapshot: &SharedSnapshot,
7328 child: &mut Option<SupervisedChild>,
7329 enabled: bool,
7330) -> Result<bool, SuperviseError> {
7331 let (current_enabled, current_state, respawn_pending) = {
7332 let state = lock_snapshot(snapshot)?;
7333 (state.enabled, state.state, state.respawn_pending)
7334 };
7335 let revive_terminal = enabled
7343 && current_enabled
7344 && child.is_none()
7345 && (matches!(current_state, ModuleState::Failed | ModuleState::Stopped)
7346 || (current_state == ModuleState::Restarting && !respawn_pending));
7347 if current_enabled == enabled && !revive_terminal {
7348 return Ok(false);
7349 }
7350
7351 if enabled {
7352 update_snapshot(snapshot, Some(&spec.module_id), |state| {
7353 state.enabled = true;
7354 state.state = ModuleState::Starting;
7355 clear_current_process_facts(state);
7356 })?;
7357 #[cfg(test)]
7358 if runtime.test_seed_stale_facts_before_enable_spawn {
7359 update_snapshot(snapshot, Some(&spec.module_id), |state| {
7360 state.process_alive = true;
7361 state.pid = Some(41);
7362 state.spawned_at_ms = Some(42);
7363 state.spawned_from = Some(PathBuf::from("/spawned/module"));
7364 state.spawned_file_identity = Some(SpawnedFileIdentity {
7365 device: 43,
7366 inode: 44,
7367 #[cfg(windows)]
7368 windows_image: None,
7369 });
7370 })?;
7371 }
7372 release_dead_registration(
7373 registry,
7374 runtime.forwarding.as_deref(),
7375 snapshot,
7376 &spec.module_id,
7377 )
7378 .await?;
7379 reset_restart_count(snapshot, &spec.module_id)?;
7380 process_liveness.track(spec.module_id.clone(), Arc::clone(snapshot));
7381 let next_child = match spawn_and_mark_running(spec, runtime, snapshot) {
7382 Ok(next_child) => next_child,
7383 Err(err) => {
7384 if let Err(state_err) = update_snapshot(snapshot, Some(&spec.module_id), |state| {
7385 state.state = ModuleState::Failed;
7386 clear_current_process_facts(state);
7387 }) {
7388 error!(module_id = %spec.module_id, error = %state_err, "failed to record enable spawn failure");
7389 }
7390 process_liveness.untrack_if_current(&spec.module_id, snapshot);
7391 return Err(err);
7392 }
7393 };
7394 *child = Some(next_child);
7395 debug!(module_id = %spec.module_id, "supervised module enabled");
7396 Ok(true)
7397 } else {
7398 let stop_notice = begin_forwarding_drain_if_configured(
7399 spec,
7400 runtime,
7401 registry,
7402 snapshot,
7403 Some(false),
7404 RouteCloseReason::Disable,
7405 )
7406 .await?;
7407 drain_optional_child(
7408 &spec.module_id,
7409 spec.protocol,
7410 stop_notice,
7411 registry,
7412 runtime.forwarding.as_deref(),
7413 snapshot,
7414 &runtime.terminal_ring,
7415 &runtime.spawn_events,
7416 child,
7417 runtime.drain_timeout,
7418 ModuleState::Disabled,
7419 Some(false),
7420 )
7421 .await?;
7422 debug!(module_id = %spec.module_id, "supervised module disabled");
7423 Ok(true)
7424 }
7425}
7426
7427#[allow(clippy::too_many_arguments)]
7428async fn on_child_exit(
7429 spec: &ModuleSpec,
7430 policy: RestartPolicy,
7431 registry: &Registry,
7432 snapshot: &SharedSnapshot,
7433 terminal_ring: &Arc<Mutex<TerminalRing>>,
7434 spawn_events: &SpawnEventFeed,
7435 roster: &ChildRoster,
7436 exit_report: ExitReport,
7437) -> NextAction {
7438 if roster.is_closed() {
7444 return on_child_exit_during_daemon_shutdown(
7445 spec,
7446 registry,
7447 snapshot,
7448 terminal_ring,
7449 spawn_events,
7450 exit_report,
7451 )
7452 .await;
7453 }
7454 let unrequested_clean_exit_of_protocol_none = exit_report.kind == ExitKind::Clean
7469 && running_protocol(spec, snapshot) == ModuleProtocol::None;
7470 match exit_report.kind {
7471 ExitKind::Clean if !unrequested_clean_exit_of_protocol_none => {
7472 info!(
7473 module_id = %spec.module_id,
7474 exit_code = ?exit_report.code,
7475 exit_signal = ?exit_report.signal,
7476 "supervised module exited cleanly"
7477 );
7478 if let Err(err) = publish_after_terminal_record(
7479 &spec.module_id,
7480 || {
7481 record_terminal(
7482 &spec.module_id,
7483 terminal_ring,
7484 spawn_events,
7485 &exit_report,
7486 TerminalDisposition::Stopped,
7487 )
7488 },
7489 || {
7490 update_snapshot(snapshot, Some(&spec.module_id), |state| {
7491 state.state = ModuleState::Stopped;
7492 clear_current_process_facts(state);
7493 state.last_exit = Some(exit_report.clone());
7494 })
7495 },
7496 ) {
7497 error!(module_id = %spec.module_id, error = %err, "failed to publish clean module exit");
7498 }
7499 let registration_released = match wait_for_registration_release(
7500 registry,
7501 &spec.module_id,
7502 REGISTRY_RELEASE_TIMEOUT,
7503 )
7504 .await
7505 {
7506 Ok(()) => true,
7507 Err(err) => {
7508 warn!(module_id = %spec.module_id, error = %err, "registration still active after clean exit");
7509 false
7510 }
7511 };
7512 NextAction::Stop {
7513 registration_released,
7514 }
7515 }
7516 ExitKind::Clean | ExitKind::Crash => {
7517 if unrequested_clean_exit_of_protocol_none {
7518 warn!(
7519 module_id = %spec.module_id,
7520 exit_code = ?exit_report.code,
7521 exit_signal = ?exit_report.signal,
7522 "protocol-none module exited cleanly without a stop request; handling it as a crash"
7523 );
7524 } else {
7525 warn!(
7526 module_id = %spec.module_id,
7527 exit_code = ?exit_report.code,
7528 exit_signal = ?exit_report.signal,
7529 "supervised module exited abnormally (crash)"
7530 );
7531 }
7532 let mut restart_schedule = None;
7533 let mut disposition = TerminalDisposition::Disabled;
7534 let mut final_state = ModuleState::Disabled;
7535 let mut disposition_detail = lock_snapshot(snapshot)
7539 .ok()
7540 .and_then(|mut state| state.spawn_failure.take());
7541 let now = Instant::now();
7542 if let Err(err) = update_snapshot(snapshot, Some(&spec.module_id), |state| {
7543 if state.enabled {
7544 if let Some(schedule) = state.next_crash_restart(&policy, now) {
7545 state.respawn_pending = true;
7551 final_state = ModuleState::Restarting;
7552 restart_schedule = Some(schedule);
7553 disposition = TerminalDisposition::Restarting;
7554 } else {
7555 final_state = ModuleState::Failed;
7556 disposition = TerminalDisposition::Failed;
7557 let budget = policy.budget_exhausted_detail();
7558 disposition_detail =
7559 Some(disposition_detail.take().map_or_else(
7560 || budget.clone(),
7561 |cause| format!("{cause}; {budget}"),
7562 ));
7563 }
7564 } else {
7565 disposition = TerminalDisposition::Disabled;
7566 }
7567 }) {
7568 error!(module_id = %spec.module_id, error = %err, "failed to record crashed module exit");
7569 return NextAction::Stop {
7570 registration_released: false,
7571 };
7572 }
7573 if disposition == TerminalDisposition::Failed {
7574 error!(
7579 module_id = %spec.module_id,
7580 max_restarts = policy.max_restarts,
7581 window_secs = policy.window.as_secs(),
7582 "module stopped: {}",
7583 policy.budget_exhausted_detail()
7584 );
7585 }
7586 if let Err(err) = publish_after_terminal_record(
7587 &spec.module_id,
7588 || {
7589 record_terminal_with_detail(
7590 &spec.module_id,
7591 terminal_ring,
7592 spawn_events,
7593 &exit_report,
7594 disposition,
7595 disposition_detail,
7596 )
7597 },
7598 || {
7599 update_snapshot(snapshot, Some(&spec.module_id), |state| {
7600 state.state = final_state;
7601 clear_current_process_facts(state);
7602 state.last_exit = Some(exit_report.clone());
7603 })
7604 },
7605 ) {
7606 error!(module_id = %spec.module_id, error = %err, "failed to publish crashed module exit");
7607 }
7608
7609 if let Some(schedule) = restart_schedule {
7610 NextAction::Restart {
7611 schedule: Some(schedule),
7612 }
7613 } else {
7614 let registration_released = match wait_for_registration_release(
7615 registry,
7616 &spec.module_id,
7617 REGISTRY_RELEASE_TIMEOUT,
7618 )
7619 .await
7620 {
7621 Ok(()) => true,
7622 Err(err) => {
7623 warn!(module_id = %spec.module_id, error = %err, "registration still active after failed module");
7624 false
7625 }
7626 };
7627 NextAction::Stop {
7628 registration_released,
7629 }
7630 }
7631 }
7632 ExitKind::DeliberateSeverance => {
7633 warn!(
7634 module_id = %spec.module_id,
7635 exit_code = ?exit_report.code,
7636 exit_signal = ?exit_report.signal,
7637 "supervised module exited after deliberate connection severance"
7638 );
7639 let should_restart = match lock_snapshot(snapshot) {
7640 Ok(state) => state.enabled,
7641 Err(err) => {
7642 error!(module_id = %spec.module_id, error = %err, "failed to read deliberately severed module state");
7643 return NextAction::Stop {
7644 registration_released: false,
7645 };
7646 }
7647 };
7648 let final_state = if should_restart {
7649 ModuleState::Restarting
7650 } else {
7651 ModuleState::Disabled
7652 };
7653 if let Err(err) = publish_after_terminal_record(
7654 &spec.module_id,
7655 || {
7656 record_terminal(
7657 &spec.module_id,
7658 terminal_ring,
7659 spawn_events,
7660 &exit_report,
7661 terminal_disposition(final_state),
7662 )
7663 },
7664 || {
7665 update_snapshot(snapshot, Some(&spec.module_id), |state| {
7666 state.state = final_state;
7667 clear_current_process_facts(state);
7668 state.last_exit = Some(exit_report.clone());
7669 state.lifetime_restarts += 1;
7670 })
7671 },
7672 ) {
7673 error!(module_id = %spec.module_id, error = %err, "failed to publish deliberately severed module exit");
7674 return NextAction::Stop {
7675 registration_released: false,
7676 };
7677 }
7678
7679 if should_restart {
7680 NextAction::Restart { schedule: None }
7681 } else {
7682 let registration_released = match wait_for_registration_release(
7683 registry,
7684 &spec.module_id,
7685 REGISTRY_RELEASE_TIMEOUT,
7686 )
7687 .await
7688 {
7689 Ok(()) => true,
7690 Err(err) => {
7691 warn!(module_id = %spec.module_id, error = %err, "registration still active after deliberately severed module exit");
7692 false
7693 }
7694 };
7695 NextAction::Stop {
7696 registration_released,
7697 }
7698 }
7699 }
7700 }
7701}
7702
7703async fn on_child_exit_during_daemon_shutdown(
7704 spec: &ModuleSpec,
7705 registry: &Registry,
7706 snapshot: &SharedSnapshot,
7707 terminal_ring: &Arc<Mutex<TerminalRing>>,
7708 spawn_events: &SpawnEventFeed,
7709 exit_report: ExitReport,
7710) -> NextAction {
7711 info!(
7712 module_id = %spec.module_id,
7713 exit_code = ?exit_report.code,
7714 exit_signal = ?exit_report.signal,
7715 exit_kind = ?exit_report.kind,
7716 "supervised module exited during daemon shutdown; not restarting it"
7717 );
7718 if let Err(err) = publish_after_terminal_record(
7719 &spec.module_id,
7720 || {
7721 record_terminal(
7722 &spec.module_id,
7723 terminal_ring,
7724 spawn_events,
7725 &exit_report,
7726 TerminalDisposition::DaemonShutdown,
7727 )
7728 },
7729 || {
7730 update_snapshot(snapshot, Some(&spec.module_id), |state| {
7731 state.state = ModuleState::Stopped;
7732 clear_current_process_facts(state);
7733 state.last_exit = Some(exit_report.clone());
7734 })
7735 },
7736 ) {
7737 error!(module_id = %spec.module_id, error = %err, "failed to publish module exit during daemon shutdown");
7738 }
7739 let registration_released =
7740 wait_for_registration_release(registry, &spec.module_id, REGISTRY_RELEASE_TIMEOUT)
7741 .await
7742 .is_ok();
7743 NextAction::Stop {
7744 registration_released,
7745 }
7746}
7747
7748fn publish_after_terminal_record<T>(
7753 module_id: &str,
7754 record: impl FnOnce(),
7755 publish: impl FnOnce() -> T,
7756) -> T {
7757 let recorded = std::panic::catch_unwind(std::panic::AssertUnwindSafe(record));
7758 let published = publish();
7759 if let Err(panic) = recorded {
7760 error!(
7761 module_id,
7762 "terminal recording panicked; published final state before resuming unwind"
7763 );
7764 std::panic::resume_unwind(panic);
7765 }
7766 published
7767}
7768
7769fn fail_after_wait_error(
7770 module_id: &str,
7771 snapshot: &SharedSnapshot,
7772 terminal_ring: &Arc<Mutex<TerminalRing>>,
7773 spawn_events: &SpawnEventFeed,
7774 detail: String,
7775) {
7776 let report = wait_error_exit_report();
7777 publish_after_terminal_record(
7778 module_id,
7779 || {
7780 record_terminal_with_detail(
7781 module_id,
7782 terminal_ring,
7783 spawn_events,
7784 &report,
7785 TerminalDisposition::Failed,
7786 Some(detail),
7787 )
7788 },
7789 || fail_snapshot(snapshot, Some(module_id), Some(report.clone())),
7790 );
7791}
7792
7793#[cfg(test)]
7794fn record_wait_error_terminal(
7795 module_id: &str,
7796 terminal_ring: &Arc<Mutex<TerminalRing>>,
7797 spawn_events: &SpawnEventFeed,
7798) {
7799 record_terminal(
7800 module_id,
7801 terminal_ring,
7802 spawn_events,
7803 &wait_error_exit_report(),
7804 TerminalDisposition::Failed,
7805 );
7806}
7807
7808fn record_terminal(
7809 module_id: &str,
7810 terminal_ring: &Arc<Mutex<TerminalRing>>,
7811 spawn_events: &SpawnEventFeed,
7812 exit_report: &ExitReport,
7813 disposition: TerminalDisposition,
7814) {
7815 record_terminal_with_detail(
7816 module_id,
7817 terminal_ring,
7818 spawn_events,
7819 exit_report,
7820 disposition,
7821 None,
7822 );
7823}
7824
7825fn durable_terminal_history_of(
7829 terminal_ring: &Mutex<TerminalRing>,
7830 module_id: &str,
7831) -> subc_control::TerminalHistory {
7832 let read = terminal_ring
7833 .lock()
7834 .unwrap_or_else(|p| p.into_inner())
7835 .capture_durable_history();
7836 read.read(module_id)
7837}
7838
7839fn record_terminal_with_detail(
7840 module_id: &str,
7841 terminal_ring: &Arc<Mutex<TerminalRing>>,
7842 spawn_events: &SpawnEventFeed,
7843 exit_report: &ExitReport,
7844 disposition: TerminalDisposition,
7845 disposition_detail: Option<String>,
7846) {
7847 #[cfg(test)]
7848 terminal_history_tests::before_terminal_record(spawn_events);
7849 spawn_events.emit_exited(module_id, exit_report.code, exit_report.signal);
7850 let record = TerminalRecord {
7851 exit_code: exit_report.code,
7852 exit_signal: exit_report.signal,
7853 at_ms: exit_report.at_ms,
7854 disposition,
7855 exit_kind: exit_report.kind.into(),
7856 disposition_detail,
7857 };
7858 terminal_ring
7859 .lock()
7860 .unwrap_or_else(|poisoned| poisoned.into_inner())
7861 .record_exit(module_id, record);
7862}
7863
7864fn untrack_if_registration_released(
7865 process_liveness: &SupervisorProcessLiveness,
7866 registry: &Registry,
7867 module_id: &str,
7868 snapshot: &SharedSnapshot,
7869) {
7870 match registry.get_module(module_id) {
7871 Ok(None) => process_liveness.untrack_if_current(module_id, snapshot),
7872 Ok(Some(_)) => {}
7873 Err(err) => {
7874 warn!(module_id, error = %err, "could not determine whether supervisor liveness can be untracked");
7875 }
7876 }
7877}
7878
7879#[cfg(test)]
7893fn apply_wire_spawn_args(
7894 command: &mut Command,
7895 spec: &ModuleSpec,
7896 connection_file_path: Option<&std::path::Path>,
7897 handle: Option<&SupervisorHandle>,
7898) -> Result<Option<NonceHandoff>, SuperviseError> {
7899 apply_wire_spawn_args_for_role(
7900 command,
7901 spec,
7902 connection_file_path,
7903 handle,
7904 SpawnRole::Plain,
7905 )
7906}
7907
7908#[cfg(unix)]
7913type NonceHandoff = subc_os::LaunchNonceHandoff;
7914#[cfg(windows)]
7915type NonceHandoff = subc_os::LaunchNoncePipeHandoff;
7916#[cfg(not(any(unix, windows)))]
7917type NonceHandoff = std::convert::Infallible;
7918
7919fn apply_wire_spawn_args_for_role(
7940 command: &mut Command,
7941 spec: &ModuleSpec,
7942 connection_file_path: Option<&std::path::Path>,
7943 handle: Option<&SupervisorHandle>,
7944 role: SpawnRole,
7945) -> Result<Option<NonceHandoff>, SuperviseError> {
7946 command.env(SUBC_MODULE_ID_ENV, &spec.module_id);
7947 command.env_remove(subc_os::LAUNCH_NONCE_FD_ENV);
7952 #[cfg(windows)]
7953 {
7954 command.env_remove(subc_os::LAUNCH_NONCE_PIPE_ENV);
7955 command.env_remove(subc_os::LAUNCH_NONCE_PIPE_FALLBACK_ENV);
7956 }
7957 command.env_remove(SUBC_LAUNCH_NONCE_ENV);
7959 if spec.protocol == ModuleProtocol::None {
7960 return Ok(None);
7961 }
7962 if let Some(connection_file_path) = connection_file_path {
7963 command.arg(SUBC_ARG).arg(connection_file_path);
7964 }
7965
7966 let nonce = generate_launch_nonce()?;
7970 if let Some(handle) = handle {
7971 match role {
7972 SpawnRole::Plain => {
7973 handle.set_spawn_nonce(&spec.module_id, nonce.clone());
7974 if spec.reserved {
7975 handle.set_reserved_nonce(&spec.module_id, nonce.clone());
7976 }
7977 }
7978 SpawnRole::SwapCandidate => handle.open_swap(&spec.module_id, nonce.clone()),
7979 }
7980 }
7981 #[cfg(unix)]
7982 let handoff = {
7983 let handoff =
7984 subc_os::LaunchNonceHandoff::new(&nonce).map_err(|source| SuperviseError::Spawn {
7985 program: spec.program.clone(),
7986 source,
7987 cgroup_path: None,
7988 })?;
7989 command.env(subc_os::LAUNCH_NONCE_FD_ENV, handoff.fd_env_value());
7990 Some(handoff)
7991 };
7992 #[cfg(windows)]
7993 let handoff = prepare_windows_nonce_pipe(command, spec, &nonce);
7994 #[cfg(not(any(unix, windows)))]
7995 let handoff = None;
7996 #[cfg(not(unix))]
8000 command.env(SUBC_LAUNCH_NONCE_ENV, nonce);
8001 Ok(handoff)
8002}
8003
8004fn apply_child_env(command: &mut Command, spec: &ModuleSpec) {
8005 command.env_remove(CK_LOG_ENV);
8006 command.env_remove(SUBC_SPAWN_ROLE_ENV);
8013 for (key, value) in &spec.env {
8014 if matches!(
8018 key.as_str(),
8019 CAPTURE_MAX_FILE_MB_ENV | CAPTURE_KEEP_ENV | CAPTURE_MAX_AGE_DAYS_ENV
8020 ) || key == SUBC_SPAWN_ROLE_ENV
8021 {
8022 continue;
8023 }
8024 command.env(key, value);
8025 }
8026}
8027
8028#[cfg(windows)]
8029fn prepare_windows_nonce_pipe(
8030 command: &mut Command,
8031 spec: &ModuleSpec,
8032 nonce: &str,
8033) -> Option<NonceHandoff> {
8034 match subc_os::LaunchNoncePipeHandoff::new(nonce) {
8035 Ok(handoff) => {
8036 command.env(subc_os::LAUNCH_NONCE_PIPE_ENV, handoff.name());
8037 command.env(subc_os::LAUNCH_NONCE_PIPE_FALLBACK_ENV, "env");
8042 Some(handoff)
8043 }
8044 Err(error) => {
8045 warn!(module_id = %spec.module_id, %error, "could not prepare launch nonce pipe; retaining the environment handoff");
8046 None
8047 }
8048 }
8049}
8050
8051#[cfg(windows)]
8052fn start_windows_nonce_pipe(
8053 handoff: Option<NonceHandoff>,
8054 pid: u32,
8055 spec: &ModuleSpec,
8056) -> Option<subc_os::LaunchNoncePipeDelivery> {
8057 handoff.and_then(|handoff| match handoff.serve(pid, REGISTRY_RELEASE_TIMEOUT) {
8061 Ok(delivery) => Some(delivery),
8062 Err(error) => {
8063 warn!(module_id = %spec.module_id, %error, "could not serve launch nonce pipe; retaining the environment handoff");
8064 None
8065 }
8066 })
8067}
8068
8069fn observed_launch_nonce_source(snapshot: &SupervisorSnapshot) -> Option<String> {
8070 #[cfg(windows)]
8071 if snapshot.process_alive && snapshot.spawned_protocol == Some(ModuleProtocol::Subc) {
8072 return Some(
8073 if snapshot
8074 .pipe_consumed
8075 .as_ref()
8076 .is_some_and(|consumed| consumed.get().is_some())
8077 {
8078 "pipe"
8079 } else {
8080 "env"
8081 }
8082 .to_owned(),
8083 );
8084 }
8085 let _ = snapshot;
8086 None
8087}
8088
8089#[derive(Debug, Clone, Copy, PartialEq, Eq)]
8092enum SpawnRole {
8093 Plain,
8094 SwapCandidate,
8095}
8096
8097fn apply_spawn_role(command: &mut Command, role: SpawnRole) {
8100 if role == SpawnRole::SwapCandidate {
8101 command.env(SUBC_SPAWN_ROLE_ENV, SPAWN_ROLE_SWAP_CANDIDATE);
8102 }
8103}
8104
8105fn previous_instance_error(
8106 spec: &ModuleSpec,
8107 pid: u32,
8108 reason: crate::live_children::RetainedReason,
8109) -> SuperviseError {
8110 SuperviseError::Spawn {
8111 program: spec.program.clone(),
8112 source: io::Error::new(io::ErrorKind::WouldBlock, reason.waiting_message(pid)),
8113 cgroup_path: None,
8114 }
8115}
8116
8117fn spawn_child(
8118 spec: &ModuleSpec,
8119 connection_file_path: Option<&std::path::Path>,
8120 handle: Option<&SupervisorHandle>,
8121 ring: &Arc<Mutex<StderrRing>>,
8122 capture_logs_dir: Option<&std::path::Path>,
8123 roster: &ChildRoster,
8124 #[cfg(target_os = "linux")] cgroup_placement: Option<&subc_cgroup::Placement>,
8125) -> Result<SupervisedChild, SuperviseError> {
8126 spawn_child_in_slot(
8127 spec,
8128 connection_file_path,
8129 handle,
8130 ring,
8131 capture_logs_dir,
8132 roster,
8133 #[cfg(target_os = "linux")]
8134 cgroup_placement,
8135 SpawnRole::Plain,
8136 false,
8137 )
8138}
8139
8140#[allow(clippy::too_many_arguments)]
8153fn spawn_child_in_slot(
8154 spec: &ModuleSpec,
8155 connection_file_path: Option<&std::path::Path>,
8156 handle: Option<&SupervisorHandle>,
8157 ring: &Arc<Mutex<StderrRing>>,
8158 capture_logs_dir: Option<&std::path::Path>,
8159 roster: &ChildRoster,
8160 #[cfg(target_os = "linux")] cgroup_placement: Option<&subc_cgroup::Placement>,
8161 role: SpawnRole,
8162 alternate_slot: bool,
8163) -> Result<SupervisedChild, SuperviseError> {
8164 if let Some((pid, reason)) = roster.previous_instance(&spec.module_id) {
8165 return Err(previous_instance_error(spec, pid, reason));
8166 }
8167 if roster.is_closed() {
8168 return Err(SuperviseError::Spawn {
8169 program: spec.program.clone(),
8170 source: io::Error::other("the daemon is shutting down; not starting a new process"),
8171 cgroup_path: None,
8172 });
8173 }
8174 #[cfg(target_os = "linux")]
8175 let cgroup_name = {
8176 if cgroup_placement.is_none() {
8181 swap::cgroup_name(&spec.module_id, alternate_slot)
8182 } else {
8183 let nonce = generate_launch_nonce()?;
8184 let suffix = u128::from_str_radix(&nonce[..32], 16).expect("hex launch nonce");
8185 let mut end = spec.module_id.len().min(64);
8188 while !spec.module_id.is_char_boundary(end) {
8189 end -= 1;
8190 }
8191 format!(
8192 "{}_{suffix}",
8193 swap::cgroup_name(&spec.module_id[..end], alternate_slot)
8194 )
8195 }
8196 };
8197 #[cfg(not(target_os = "linux"))]
8198 let _ = alternate_slot;
8199 #[cfg(target_os = "macos")]
8200 let (mut command, privacy_exec, exec_ack) = privacy_command(spec, roster)?;
8201 #[cfg(windows)]
8202 let image_capture = windows_spawn_capture(spec).map_err(|source| SuperviseError::Spawn {
8203 program: spec.program.clone(),
8204 source,
8205 cgroup_path: None,
8206 })?;
8207 #[cfg(windows)]
8208 let mut command = Command::new(image_capture.path());
8209 #[cfg(not(any(target_os = "macos", windows)))]
8210 let mut command = Command::new(&spec.program);
8211 command.args(&spec.args);
8212 apply_child_env(&mut command, spec);
8242 apply_spawn_role(&mut command, role);
8243 let nonce_handoff =
8244 apply_wire_spawn_args_for_role(&mut command, spec, connection_file_path, handle, role)?;
8245
8246 #[cfg(target_os = "linux")]
8247 let cgroup_path = cgroup_placement
8248 .map(|placement| placement.module_path(&cgroup_name))
8249 .transpose()
8250 .map_err(|source| SuperviseError::Cgroup {
8251 module_id: spec.module_id.clone(),
8252 source,
8253 })?;
8254 #[cfg(not(target_os = "linux"))]
8255 let cgroup_path: Option<PathBuf> = None;
8256 #[cfg(target_os = "linux")]
8257 if let Some(path) = &cgroup_path {
8258 if let Err(error) = apply_cgroup_placement(&mut command, spec, path) {
8259 if let Some(placement) = cgroup_placement {
8260 remove_module_cgroup(placement, &cgroup_name);
8261 }
8262 return Err(error);
8263 }
8264 }
8265
8266 let output_sink = if let Some(logs_dir) = capture_logs_dir {
8267 let path = logs_dir.join(format!("{}.stderr.log", spec.module_id));
8268 match ChildOutputSink::open(&path, capture_retention(spec)) {
8269 Ok(sink) => sink,
8270 Err(error) => {
8271 warn!(
8272 module_id = %spec.module_id,
8273 path = %path.display(),
8274 error = %error,
8275 "could not open child output capture file; forwarding to stderr"
8276 );
8277 ChildOutputSink::Stderr
8278 }
8279 }
8280 } else {
8281 ChildOutputSink::Stderr
8282 };
8283
8284 command.stdout(Stdio::piped());
8285 command.stderr(Stdio::piped());
8286 command.kill_on_drop(true);
8287 #[cfg(unix)]
8304 command.process_group(0);
8305 command.stdin(Stdio::null());
8306 #[cfg(unix)]
8310 if let Some(handoff) = nonce_handoff {
8311 handoff.install_last(command.as_std_mut());
8312 }
8313 #[cfg(not(any(unix, windows)))]
8314 let _ = nonce_handoff;
8315
8316 #[cfg(windows)]
8321 {
8322 subc_jobobject::ensure_root_job().map_err(|source| SuperviseError::Spawn {
8323 program: spec.program.clone(),
8324 source,
8325 cgroup_path: None,
8326 })?;
8327 subc_jobobject::suspend_on_create_async(&mut command);
8328 }
8329 let mut child = match command.spawn() {
8330 Ok(child) => child,
8331 Err(source) => {
8332 #[cfg(target_os = "linux")]
8333 if let Some(placement) = cgroup_placement {
8334 remove_module_cgroup(placement, &cgroup_name);
8335 }
8336 return Err(SuperviseError::Spawn {
8337 program: spec.program.clone(),
8338 source,
8339 cgroup_path,
8340 });
8341 }
8342 };
8343 #[cfg(test)]
8344 spawn_record_tests::after_create(&child, spec);
8345 #[cfg(target_os = "macos")]
8350 drop(exec_ack);
8351
8352 #[cfg(windows)]
8357 let windows_image = match image_capture.bind_tokio(&child) {
8358 Ok(image) => Arc::new(image),
8359 Err(source) => {
8360 kill_and_reap_rejected_child(&mut child);
8361 return Err(SuperviseError::Spawn {
8362 program: spec.program.clone(),
8363 source,
8364 cgroup_path: None,
8365 });
8366 }
8367 };
8368 #[cfg(windows)]
8371 let job = contain_spawned_child(&mut child, spec)?;
8372 let spawned_at_ms = unix_ms_now();
8373 #[cfg(not(windows))]
8374 let spawned_from = spec.program.clone();
8375 #[cfg(windows)]
8376 let spawned_from = windows_image.path().to_path_buf();
8377 #[cfg(not(windows))]
8378 let spawned_file_identity = spawned_file_identity(&spawned_from);
8379 #[cfg(windows)]
8380 let spawned_file_identity = Some(SpawnedFileIdentity {
8381 device: 0,
8382 inode: 0,
8383 windows_image: Some(Arc::clone(&windows_image)),
8384 });
8385 let pid = child.id().ok_or_else(|| SuperviseError::Spawn {
8386 program: spec.program.clone(),
8387 source: io::Error::other("spawned child exposed no live pid"),
8388 cgroup_path: cgroup_path.clone(),
8389 })?;
8390 #[cfg(windows)]
8391 let nonce_delivery = start_windows_nonce_pipe(nonce_handoff, pid, spec);
8392 #[cfg(not(windows))]
8393 let process_start_time = crate::provenance::process_start_time(pid);
8394 #[cfg(windows)]
8395 let process_start_time = Some(windows_image.start_time());
8396 let process_identity = process_start_time.map(|start_time| ProcessIdentity { pid, start_time });
8397 #[cfg(all(test, target_os = "macos"))]
8398 privacy_exec_boundary_tests::before_image_sample(spec, pid);
8399 #[cfg(not(windows))]
8404 let recorded_image = observe_spawned_image(pid);
8405 #[cfg(target_os = "macos")]
8409 let recorded_image = if privacy_exec.is_some() {
8410 None
8411 } else {
8412 recorded_image
8413 };
8414 #[cfg(target_os = "linux")]
8415 let recorded_cgroup_name = cgroup_path.as_ref().map(|_| cgroup_name.clone());
8416 #[cfg(not(target_os = "linux"))]
8417 let recorded_cgroup_name = None;
8418 let roster_guard = roster.admit(
8419 spec.module_id.clone(),
8420 pid,
8421 spec.protocol,
8422 process_start_time,
8423 crate::child_roster::RecordedIdentity {
8424 #[cfg(not(windows))]
8425 start_time: recorded_image.map(|image| image.start_time),
8426 #[cfg(windows)]
8427 start_time: process_start_time,
8428 #[cfg(not(windows))]
8429 executable: recorded_image
8430 .and_then(|image| image.executable)
8431 .map(crate::live_children::ExecutableIdentity::from),
8432 #[cfg(windows)]
8433 executable: Some(crate::live_children::ExecutableIdentity::from(
8434 windows_image.identity(),
8435 )),
8436 cgroup_name: recorded_cgroup_name,
8437 #[cfg(target_os = "linux")]
8438 cgroup_placement: cgroup_placement.cloned(),
8439 },
8440 );
8441 let roster_guard = match roster_guard {
8442 Ok(guard) => guard,
8443 Err(source) => {
8444 #[cfg(windows)]
8445 if let Some(job) = &job {
8446 let _ = job.terminate();
8447 }
8448 #[cfg(target_os = "linux")]
8449 kill_module_cgroup(cgroup_placement, &cgroup_name);
8450 kill_and_reap_rejected_child(&mut child);
8451 #[cfg(target_os = "linux")]
8452 if let Some(placement) = cgroup_placement {
8453 remove_module_cgroup(placement, &cgroup_name);
8454 }
8455 return Err(SuperviseError::Spawn {
8456 program: spec.program.clone(),
8457 source: io::Error::other(format!(
8458 "could not durably record module child: {source}"
8459 )),
8460 cgroup_path,
8461 });
8462 }
8463 };
8464 if roster.is_closed() {
8473 #[cfg(target_os = "linux")]
8475 kill_module_cgroup(cgroup_placement, &cgroup_name);
8476 if let Err(error) = child.start_kill() {
8477 debug!(module_id = %spec.module_id, pid, %error, "kill of a process spawned during daemon shutdown failed; it may already have exited");
8478 }
8479 #[cfg(target_os = "linux")]
8480 if let Some(placement) = cgroup_placement {
8481 while matches!(child.try_wait(), Ok(None)) {
8485 std::thread::yield_now();
8486 }
8487 if matches!(
8488 subc_cgroup::kill_module(Some(placement), &cgroup_name),
8489 subc_cgroup::KillOutcome::Killed
8490 ) {
8491 if let Ok(path) = placement.module_path(&cgroup_name) {
8492 while std::fs::read_to_string(path.join("cgroup.events"))
8493 .ok()
8494 .is_some_and(|events| events.lines().any(|line| line == "populated 1"))
8495 {
8496 std::thread::yield_now();
8497 }
8498 }
8499 }
8500 remove_module_cgroup(placement, &cgroup_name);
8501 }
8502 drop(roster_guard);
8503 return Err(SuperviseError::Spawn {
8504 program: spec.program.clone(),
8505 source: io::Error::other(
8506 "the daemon began shutting down while this process was starting; ended it",
8507 ),
8508 cgroup_path,
8509 });
8510 }
8511
8512 #[cfg(windows)]
8513 if let Err(error) = resume_suspended_child(pid, spec) {
8514 kill_and_reap_rejected_child(&mut child);
8515 return Err(error);
8516 }
8517
8518 let stdout_pump = match child.stdout.take() {
8519 Some(stdout) => Some(tokio::spawn(pump_stdout_to(stdout, output_sink.clone()))),
8520 None => {
8521 warn!(
8522 module_id = %spec.module_id,
8523 "spawned child exposed no stdout pipe; file capture will be incomplete"
8524 );
8525 None
8526 }
8527 };
8528 let stderr_pump = match child.stderr.take() {
8529 Some(stderr) => {
8530 let generation = ring
8531 .lock()
8532 .unwrap_or_else(|poisoned| poisoned.into_inner())
8533 .begin_process();
8534 Some(StderrPump {
8535 task: tokio::spawn(pump_stderr_to(
8536 stderr,
8537 Arc::clone(ring),
8538 generation,
8539 output_sink,
8540 )),
8541 generation,
8542 })
8543 }
8544 None => {
8545 ring.lock()
8549 .unwrap_or_else(|poisoned| poisoned.into_inner())
8550 .mark_not_captured("stderr pipe was not available on spawn");
8551 warn!(
8552 module_id = %spec.module_id,
8553 "spawned child exposed no stderr pipe; tail will be unavailable"
8554 );
8555 None
8556 }
8557 };
8558
8559 let exec = Arc::new(crate::activity::ExecOutcome::new());
8560 #[cfg(target_os = "macos")]
8561 let pending_exec = privacy_exec.is_some();
8562 #[cfg(not(target_os = "macos"))]
8563 let pending_exec = false;
8564 if !pending_exec {
8565 exec.finish(crate::activity::ExecState::CommittedExec);
8566 }
8567 Ok(SupervisedChild {
8568 child,
8569 exec: crate::activity::ExecOwner(exec),
8570 #[cfg(windows)]
8571 nonce_delivery,
8572 protocol: spec.protocol,
8573 #[cfg(target_os = "linux")]
8574 module_id: cgroup_name,
8575 #[cfg(target_os = "linux")]
8576 cgroup_placement: cgroup_placement.cloned(),
8577 #[cfg(windows)]
8578 job,
8579 stdout_pump,
8580 stderr_pump,
8581 stderr_ring: Arc::clone(ring),
8582 spawned_at_ms,
8583 spawned_from,
8584 spawned_file_identity,
8585 process_start_time,
8586 process_identity,
8587 pid,
8588 roster_guard: Some(roster_guard),
8589 #[cfg(target_os = "macos")]
8590 privacy_exec,
8591 #[cfg(target_os = "macos")]
8592 report_ready: Arc::new(OnceLock::new()),
8593 spawn_failure: None,
8594 })
8595}
8596
8597#[cfg(test)]
8598mod spawn_record_tests {
8599 use super::*;
8600
8601 thread_local! {
8602 static LAST_PID: std::cell::Cell<Option<u32>> = const { std::cell::Cell::new(None) };
8603 }
8604
8605 pub(super) fn after_create(child: &Child, _spec: &ModuleSpec) {
8606 LAST_PID.set(child.id());
8607 #[cfg(windows)]
8608 if let Some((_, address)) = _spec
8609 .env
8610 .iter()
8611 .find(|(key, _)| key == "SUBC_TEST_AFTER_CREATE")
8612 {
8613 use std::io::{Read, Write};
8614 let mut stream = std::net::TcpStream::connect(address).unwrap();
8615 stream
8616 .write_all(&child.id().unwrap().to_ne_bytes())
8617 .unwrap();
8618 let mut release = [0];
8619 stream.read_exact(&mut release).unwrap();
8620 }
8621 }
8622
8623 #[cfg(windows)]
8624 pub(super) fn assignment_failpoint(spec: &ModuleSpec) -> io::Result<()> {
8625 if spec
8626 .env
8627 .iter()
8628 .any(|(key, _)| key == "SUBC_TEST_ASSIGNMENT_FAILURE")
8629 {
8630 Err(io::Error::other("injected module job assignment failure"))
8631 } else {
8632 Ok(())
8633 }
8634 }
8635
8636 fn spec() -> ModuleSpec {
8637 ModuleSpec {
8638 module_id: "spawn-record-worker".into(),
8639 #[cfg(unix)]
8640 program: "/bin/sleep".into(),
8641 #[cfg(windows)]
8642 program: std::env::current_exe().unwrap(),
8643 #[cfg(unix)]
8644 args: vec!["60".into()],
8645 #[cfg(windows)]
8646 args: vec![
8647 "--exact".into(),
8648 "supervise::spawn_record_tests::windows_store_touch_fixture".into(),
8649 "--nocapture".into(),
8650 ],
8651 env: vec![],
8652 reserved: false,
8653 reserved_prefixes: vec![],
8654 protocol: ModuleProtocol::None,
8655 overlap: Default::default(),
8656 }
8657 }
8658
8659 #[tokio::test]
8660 async fn record_sync_failure_fails_spawn_and_reaps_child() {
8661 let home = cortexkit_test_support::ScratchDir::new("record-sync-failure");
8662 let supervisor =
8663 Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default())
8664 .with_live_children_record(home.join("live-children.json"));
8665 crate::live_children::fail_next_file_sync();
8666 let result = supervisor.spawn(spec());
8667 let failed = result.is_err();
8668 if let Ok(module) = result {
8669 module.stop().await.unwrap();
8670 }
8671 assert!(failed, "record failure must fail spawn");
8672 let pid = LAST_PID.get().expect("the real OS child was created");
8673 assert!(
8674 subc_os::Process::open(pid).unwrap().is_none(),
8675 "child {pid} survived rejected admission or was not reaped"
8676 );
8677 }
8678
8679 #[cfg(target_os = "macos")]
8680 #[test]
8681 #[ignore = "manual on-device latency measurement"]
8682 fn durable_record_write_latency_50() {
8683 let home = cortexkit_test_support::ScratchDir::new("record-latency");
8684 let path = home.join("live-children.json");
8685 let children = vec![crate::live_children::LiveChild {
8686 module_id: "latency-worker".into(),
8687 pid: std::process::id(),
8688 protocol: ModuleProtocol::None,
8689 start_time: Some(1),
8690 executable: None,
8691 cgroup_name: None,
8692 }];
8693 let mut samples = Vec::new();
8694 for _ in 0..50 {
8695 let start = std::time::Instant::now();
8696 crate::live_children::write_record(&path, &children).unwrap();
8697 samples.push(start.elapsed());
8698 }
8699 samples.sort();
8700 eprintln!(
8701 "durable_record_write_latency_50: n=50 median={:?} p95={:?}",
8702 (samples[24] + samples[25]) / 2,
8703 samples[47]
8704 );
8705 }
8706
8707 #[cfg(windows)]
8708 #[test]
8709 fn windows_store_touch_fixture() {
8710 if let Some(path) = std::env::var_os("SUBC_TEST_STORE_TOUCH") {
8711 std::fs::write(path, b"module ran").unwrap();
8712 }
8713 }
8714
8715 #[cfg(windows)]
8716 #[tokio::test]
8717 async fn windows_daemon_fixture() {
8718 let Ok(address) = std::env::var("SUBC_TEST_DAEMON_BARRIER") else {
8719 return;
8720 };
8721 let mut spec = spec();
8722 spec.env.push(("SUBC_TEST_AFTER_CREATE".into(), address));
8723 let supervisor =
8724 Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default());
8725 let _module = supervisor.spawn(spec).unwrap();
8726 }
8727
8728 #[cfg(windows)]
8729 #[test]
8730 fn windows_daemon_death_before_module_job_assignment_leaves_no_child() {
8731 use std::io::Read;
8732 let listener = std::net::TcpListener::bind("127.0.0.1:0").unwrap();
8733 let mut command = std::process::Command::new(std::env::current_exe().unwrap());
8736 command
8737 .args([
8738 "--exact",
8739 "supervise::spawn_record_tests::windows_daemon_fixture",
8740 "--nocapture",
8741 ])
8742 .env(
8743 "SUBC_TEST_DAEMON_BARRIER",
8744 listener.local_addr().unwrap().to_string(),
8745 );
8746 let mut daemon = subc_jobobject::spawn_contained(&mut command).unwrap();
8747 listener.set_nonblocking(true).unwrap();
8748 let deadline = std::time::Instant::now() + Duration::from_secs(30);
8749 let mut stream = loop {
8750 if let Ok((stream, _)) = listener.accept() {
8751 break stream;
8752 }
8753 assert!(
8754 daemon.child.try_wait().unwrap().is_none(),
8755 "daemon failed before rendezvous"
8756 );
8757 assert!(
8758 std::time::Instant::now() < deadline,
8759 "daemon did not reach CreateProcess failpoint"
8760 );
8761 std::thread::yield_now();
8762 };
8763 stream
8764 .set_read_timeout(Some(Duration::from_secs(30)))
8765 .unwrap();
8766 let mut pid = [0; 4];
8767 stream.read_exact(&mut pid).unwrap();
8768 let pid = u32::from_ne_bytes(pid);
8769 assert!(subc_jobobject::process_exists(pid));
8770 daemon.child.kill().unwrap();
8773 daemon.child.wait().unwrap();
8774 let exited = subc_jobobject::wait_for_process_exit(pid, Duration::from_secs(10));
8775 if !exited {
8776 daemon.job.terminate().unwrap();
8777 }
8778 assert!(exited, "suspended child {pid} outlived hard-killed daemon");
8779 }
8780
8781 #[cfg(windows)]
8782 #[tokio::test]
8783 async fn windows_containment_failure_never_resumes() {
8784 let home = cortexkit_test_support::ScratchDir::new("assignment-refusal");
8785 let sentinel = home.join("store-touched");
8786 let mut spec = spec();
8787 spec.env
8788 .push(("SUBC_TEST_ASSIGNMENT_FAILURE".into(), "1".into()));
8789 spec.env.push((
8790 "SUBC_TEST_STORE_TOUCH".into(),
8791 sentinel.display().to_string(),
8792 ));
8793 let supervisor =
8794 Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default());
8795 let result = supervisor.spawn(spec);
8796 if let Ok(module) = result {
8799 let _ = tokio::time::timeout(Duration::from_secs(10), async {
8800 while !sentinel.exists() && module.state().unwrap() == ModuleState::Running {
8801 tokio::task::yield_now().await;
8802 }
8803 })
8804 .await;
8805 let _ = module.stop().await;
8806 panic!(
8807 "assignment failure resumed a module; store touched={}",
8808 sentinel.exists()
8809 );
8810 }
8811 assert!(
8812 !sentinel.exists(),
8813 "refused module executed store-touch sentinel"
8814 );
8815 assert!(!subc_jobobject::process_exists(LAST_PID.get().unwrap()));
8816 }
8817
8818 #[cfg(windows)]
8819 #[tokio::test]
8820 async fn windows_nested_daemon_fixture() {
8821 let Some(sentinel) = std::env::var_os("SUBC_TEST_NESTED_STORE") else {
8822 return;
8823 };
8824 let mut spec = spec();
8825 spec.env.push((
8826 "SUBC_TEST_STORE_TOUCH".into(),
8827 sentinel.to_str().unwrap().into(),
8828 ));
8829 let supervisor =
8830 Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default());
8831 let module = supervisor
8832 .spawn(spec)
8833 .expect("per-module job must nest below daemon and host jobs");
8834 tokio::time::timeout(Duration::from_secs(10), async {
8835 while !PathBuf::from(&sentinel).exists() {
8836 tokio::task::yield_now().await;
8837 }
8838 })
8839 .await
8840 .unwrap();
8841 module.stop().await.unwrap();
8842 }
8843
8844 #[cfg(windows)]
8845 #[test]
8846 fn windows_module_job_nests_under_host_job() {
8847 let home = cortexkit_test_support::ScratchDir::new("nested-jobs");
8848 let sentinel = home.join("nested-store-touched");
8849 let mut command = std::process::Command::new(std::env::current_exe().unwrap());
8850 command
8851 .args([
8852 "--exact",
8853 "supervise::spawn_record_tests::windows_nested_daemon_fixture",
8854 "--nocapture",
8855 ])
8856 .env("SUBC_TEST_NESTED_STORE", &sentinel);
8857 let mut daemon = subc_jobobject::spawn_contained(&mut command).unwrap();
8858 assert!(daemon.child.wait().unwrap().success());
8859 assert!(sentinel.exists(), "nested protocol:none module never ran");
8860 }
8861}
8862
8863#[cfg(target_os = "linux")]
8864pub(crate) fn kill_module_cgroup(placement: Option<&subc_cgroup::Placement>, module_id: &str) {
8865 use subc_cgroup::KillOutcome;
8866 match subc_cgroup::kill_module(placement, module_id) {
8867 KillOutcome::Killed => {}
8868 KillOutcome::NotPlaced | KillOutcome::Unsupported => {
8869 debug!(
8870 module_id,
8871 "cgroup tree kill unavailable; using direct-child kill"
8872 );
8873 }
8874 KillOutcome::IoError { path, error } => {
8875 warn!(module_id, path = %path.display(), %error, "cgroup tree kill failed; using direct-child kill");
8876 }
8877 }
8878}
8879
8880#[cfg(windows)]
8888fn contain_spawned_child(
8889 child: &mut Child,
8890 spec: &ModuleSpec,
8891) -> Result<Option<subc_jobobject::JobObject>, SuperviseError> {
8892 let assignment = || {
8893 let job = subc_jobobject::JobObject::new()?;
8894 #[cfg(test)]
8895 spawn_record_tests::assignment_failpoint(spec)?;
8896 job.assign(child)?;
8897 Ok::<_, io::Error>(job)
8898 };
8899 match assignment() {
8900 Ok(job) => Ok(Some(job)),
8901 Err(source) => {
8902 kill_and_reap_rejected_child(child);
8903 Err(SuperviseError::Spawn {
8904 program: spec.program.clone(),
8905 source: io::Error::other(format!(
8906 "Windows module job containment failed: {source}"
8907 )),
8908 cgroup_path: None,
8909 })
8910 }
8911 }
8912}
8913
8914fn kill_and_reap_rejected_child(child: &mut Child) {
8917 #[cfg(unix)]
8918 if let Some(pid) = child
8919 .id()
8920 .and_then(|pid| rustix::process::Pid::from_raw(pid as i32))
8921 {
8922 let _ = rustix::process::kill_process_group(pid, rustix::process::Signal::KILL);
8923 }
8924 let _ = child.start_kill();
8925 while matches!(child.try_wait(), Ok(None)) {
8926 std::thread::yield_now();
8927 }
8928}
8929
8930#[cfg(windows)]
8935fn resume_suspended_child(pid: u32, spec: &ModuleSpec) -> Result<(), SuperviseError> {
8936 if let Err(source) = subc_jobobject::resume_main_thread(pid) {
8937 let _ = std::process::Command::new("taskkill.exe")
8941 .args(["/PID", &pid.to_string(), "/T", "/F"])
8942 .stdin(Stdio::null())
8943 .stdout(Stdio::null())
8944 .stderr(Stdio::null())
8945 .status();
8946 return Err(SuperviseError::Spawn {
8947 program: spec.program.clone(),
8948 source,
8949 cgroup_path: None,
8950 });
8951 }
8952 Ok(())
8953}
8954
8955#[cfg(target_os = "linux")]
8956fn remove_module_cgroup(placement: &subc_cgroup::Placement, module_id: &str) {
8957 match placement.remove_module(module_id) {
8958 Ok(()) => debug!(module_id, "removed module cgroup after process exit"),
8959 Err(error) => warn!(
8960 module_id,
8961 error = %error,
8962 "could not remove module cgroup after process exit; continuing teardown"
8963 ),
8964 }
8965}
8966
8967#[cfg(target_os = "linux")]
8968async fn cleanup_reaped_cgroup(placement: &subc_cgroup::Placement, module_id: &str) {
8969 if matches!(
8973 subc_cgroup::kill_module(Some(placement), module_id),
8974 subc_cgroup::KillOutcome::Killed
8975 ) {
8976 if let Ok(path) = placement.module_path(module_id) {
8977 while std::fs::read_to_string(path.join("cgroup.events"))
8978 .ok()
8979 .is_some_and(|events| events.lines().any(|line| line == "populated 1"))
8980 {
8981 sleep(Duration::from_millis(1)).await;
8982 }
8983 }
8984 }
8985 remove_module_cgroup(placement, module_id);
8986}
8987
8988#[cfg(target_os = "linux")]
8989fn apply_cgroup_placement(
8990 command: &mut Command,
8991 spec: &ModuleSpec,
8992 path: &std::path::Path,
8993) -> Result<(), SuperviseError> {
8994 subc_cgroup::apply(command, path).map_err(|source| SuperviseError::Cgroup {
8995 module_id: spec.module_id.clone(),
8996 source,
8997 })
8998}
8999
9000fn capture_retention(spec: &ModuleSpec) -> Retention {
9001 let defaults = Retention::default();
9002 let value = |name: &str| {
9003 spec.env
9004 .iter()
9005 .rev()
9006 .find_map(|(key, value)| (key == name).then_some(value.as_str()))
9007 };
9008 Retention {
9009 max_file_mb: value(CAPTURE_MAX_FILE_MB_ENV)
9010 .and_then(|value| value.parse().ok())
9011 .unwrap_or(defaults.max_file_mb),
9012 keep: value(CAPTURE_KEEP_ENV)
9013 .and_then(|value| value.parse().ok())
9014 .unwrap_or(defaults.keep),
9015 max_age_days: value(CAPTURE_MAX_AGE_DAYS_ENV)
9016 .and_then(|value| value.parse().ok())
9017 .unwrap_or(defaults.max_age_days),
9018 }
9019}
9020
9021fn generate_launch_nonce() -> Result<String, SuperviseError> {
9024 let mut bytes = [0u8; 32];
9025 getrandom::getrandom(&mut bytes).map_err(|source| SuperviseError::LaunchNonce {
9026 reason: source.to_string(),
9027 })?;
9028 let mut hex = String::with_capacity(64);
9029 for b in bytes {
9030 use std::fmt::Write;
9031 let _ = write!(hex, "{b:02x}");
9032 }
9033 Ok(hex)
9034}
9035
9036fn constant_time_eq(a: &[u8], b: &[u8]) -> bool {
9039 if a.len() != b.len() {
9040 return false;
9041 }
9042 let mut diff = 0u8;
9043 for (x, y) in a.iter().zip(b.iter()) {
9044 diff |= x ^ y;
9045 }
9046 diff == 0
9047}
9048
9049#[cfg(not(windows))]
9053fn observe_spawned_image(pid: u32) -> Option<subc_os::Observation> {
9054 let process = subc_os::Process::open(pid).ok().flatten()?;
9055 #[cfg(target_os = "linux")]
9056 return sample_after_linux_exec(
9057 || {
9058 let stat = std::fs::read_to_string(format!("/proc/{pid}/stat")).ok()?;
9059 let committed = linux_exec_committed(&stat);
9060 #[cfg(test)]
9061 spawn_image_tests::report_exec_state(committed);
9062 committed
9063 },
9064 || process.observe(),
9065 );
9066 #[cfg(not(target_os = "linux"))]
9067 process.observe()
9068}
9069
9070#[cfg(target_os = "linux")]
9074fn linux_exec_committed(stat: &str) -> Option<bool> {
9075 const PF_FORKNOEXEC: u64 = 0x40;
9076 let flags = stat.rsplit_once(')')?.1.split_whitespace().nth(6)?;
9077 Some(flags.parse::<u64>().ok()? & PF_FORKNOEXEC == 0)
9078}
9079
9080#[cfg(target_os = "linux")]
9081fn sample_after_linux_exec(
9082 mut exec_committed: impl FnMut() -> Option<bool>,
9083 observe: impl FnOnce() -> Option<subc_os::Observation>,
9084) -> Option<subc_os::Observation> {
9085 let deadline = std::time::Instant::now() + Duration::from_secs(1);
9086 while !exec_committed()? {
9087 if std::time::Instant::now() >= deadline {
9091 return None;
9092 }
9093 std::thread::sleep(Duration::from_micros(50));
9094 }
9095 observe()
9096}
9097
9098#[cfg(all(test, target_os = "linux"))]
9099mod spawn_image_tests {
9100 use super::*;
9101 use std::cell::{Cell, RefCell};
9102
9103 enum ExecEvent {
9104 Uncommitted,
9105 Image(Option<subc_os::Observation>),
9106 }
9107
9108 thread_local! {
9109 static EXEC_EVENTS: RefCell<Option<std::sync::mpsc::Sender<ExecEvent>>> = const { RefCell::new(None) };
9110 }
9111
9112 pub(super) fn report_exec_state(committed: Option<bool>) {
9113 if committed == Some(false) {
9114 EXEC_EVENTS.with(|events| {
9115 if let Some(events) = events.borrow_mut().take() {
9116 let _ = events.send(ExecEvent::Uncommitted);
9117 }
9118 });
9119 }
9120 }
9121
9122 #[test]
9123 fn linux_spawn_observation_waits_for_a_paused_fork_to_exec() {
9124 use std::io::{Read, Write};
9125 use std::os::fd::AsRawFd;
9126
9127 let descriptor = std::fs::File::open("/dev/null").unwrap();
9128 let mut command = std::process::Command::new("/bin/sleep");
9129 command.arg("30");
9130 let (mut ready, mut release) =
9131 subc_os::fork_exec_test::pause_before_exec(&mut command, descriptor.as_raw_fd())
9132 .unwrap();
9133 let spawn = std::thread::spawn(move || command.spawn());
9134 let mut message = [0u8; 8];
9135 ready.read_exact(&mut message).unwrap();
9136 let pid = i32::from_ne_bytes(message[..4].try_into().unwrap()) as u32;
9137 let before = std::fs::read_to_string(format!("/proc/{pid}/stat"));
9138 let inherited = subc_os::Process::open(pid).unwrap().unwrap().observe();
9139 let (events, received) = std::sync::mpsc::channel();
9140 let observer = std::thread::spawn(move || {
9141 EXEC_EVENTS.with(|slot| *slot.borrow_mut() = Some(events.clone()));
9142 let image = observe_spawned_image(pid);
9143 let _ = events.send(ExecEvent::Image(image));
9144 });
9145 let first = received.recv_timeout(Duration::from_secs(5));
9149 release.write_all(&[1]).unwrap();
9150 let mut child = spawn.join().unwrap().unwrap();
9151 let after = match first {
9152 Ok(ExecEvent::Uncommitted) => match received.recv_timeout(Duration::from_secs(5)) {
9153 Ok(ExecEvent::Image(image)) => image,
9154 _ => None,
9155 },
9156 Ok(ExecEvent::Image(image)) => image,
9157 Err(_) => None,
9158 };
9159 observer.join().unwrap();
9160 child.kill().unwrap();
9161 child.wait().unwrap();
9162
9163 assert_eq!(linux_exec_committed(&before.unwrap()), Some(false));
9164 assert_eq!(
9165 inherited.unwrap().executable,
9166 subc_os::file_identity(&std::env::current_exe().unwrap())
9167 );
9168 assert_eq!(
9169 after.unwrap().executable,
9170 subc_os::file_identity(std::path::Path::new("/bin/sleep"))
9171 );
9172 }
9173
9174 #[test]
9175 fn spawned_image_is_not_sampled_until_linux_exec_commits() {
9176 let committed = Cell::new(false);
9177 let inherited = subc_os::Observation {
9178 start_time: 23,
9179 executable: Some(subc_os::FileIdentity {
9180 device: 29,
9181 inode: 31,
9182 }),
9183 };
9184 let module = subc_os::Observation {
9185 start_time: 23,
9186 executable: Some(subc_os::FileIdentity {
9187 device: 29,
9188 inode: 37,
9189 }),
9190 };
9191 let polls = Cell::new(0);
9192 let image = sample_after_linux_exec(
9193 || {
9194 polls.set(polls.get() + 1);
9195 committed.set(polls.get() == 2);
9198 Some(committed.get())
9199 },
9200 || Some(if committed.get() { module } else { inherited }),
9201 );
9202 assert_eq!(image, Some(module));
9203 assert_eq!(polls.get(), 2);
9204 }
9205
9206 #[test]
9207 fn linux_exec_flag_is_read_from_flags_after_the_last_parenthesis() {
9208 assert_eq!(
9209 linux_exec_committed("17 (a ) b) R 1 2 3 4 5 4194368"),
9210 Some(false)
9211 );
9212 assert_eq!(
9213 linux_exec_committed("17 (a ) b) R 1 2 3 4 5 4194304"),
9214 Some(true)
9215 );
9216 assert_eq!(linux_exec_committed("17 (a) R 1 2 3 4 5 invalid"), None);
9217 assert_eq!(linux_exec_committed("17 (a) R"), None);
9218 }
9219
9220 #[test]
9221 fn unreadable_exec_state_never_samples_an_inherited_image() {
9222 assert_eq!(
9223 sample_after_linux_exec(
9224 || None,
9225 || panic!("must not sample before exec confirmation")
9226 ),
9227 None
9228 );
9229 }
9230
9231 #[test]
9232 fn uncommitted_exec_times_out_without_sampling_an_inherited_image() {
9233 assert_eq!(
9234 sample_after_linux_exec(
9235 || Some(false),
9236 || panic!("must not sample after an exec timeout")
9237 ),
9238 None
9239 );
9240 }
9241}
9242
9243#[cfg(windows)]
9246fn windows_spawn_capture(spec: &ModuleSpec) -> io::Result<subc_os::ExecutableCapture> {
9247 let mut program = spec.program.clone();
9248 if program.is_absolute() || program.components().count() > 1 {
9249 match subc_os::ExecutableCapture::open(&program) {
9252 Ok(capture) => return Ok(capture),
9253 Err(error)
9254 if error.kind() == io::ErrorKind::NotFound && program.extension().is_none() => {}
9255 Err(error) => return Err(error),
9256 }
9257 program.set_extension("exe");
9258 return subc_os::ExecutableCapture::open(&program);
9259 }
9260 if program.extension().is_none() {
9261 program.set_extension("exe");
9262 }
9263 let custom_path = spec
9264 .env
9265 .iter()
9266 .rev()
9267 .find(|(key, _)| key.eq_ignore_ascii_case("PATH"));
9268 let mut directories = Vec::new();
9269 if let Some(parent) = std::env::current_exe()?.parent() {
9272 directories.push(parent.to_path_buf());
9273 }
9274 directories.push(std::env::current_dir()?);
9275 if let Some(windows) = std::env::var_os("SystemRoot") {
9276 let windows = PathBuf::from(windows);
9277 directories.push(windows.join("System32"));
9278 directories.push(windows.clone());
9279 }
9280 let path = custom_path
9281 .map(|(_, value)| std::ffi::OsString::from(value))
9282 .or_else(|| std::env::var_os("PATH"));
9283 if let Some(path) = path {
9284 directories.extend(std::env::split_paths(&path));
9285 }
9286 for directory in directories {
9287 match subc_os::ExecutableCapture::open(&directory.join(&program)) {
9288 Ok(capture) => return Ok(capture),
9289 Err(error) if error.kind() == io::ErrorKind::NotFound => continue,
9290 Err(error) => return Err(error),
9291 }
9292 }
9293 Err(io::Error::new(
9294 io::ErrorKind::NotFound,
9295 "resolved executable not found",
9296 ))
9297}
9298
9299fn spawn_and_mark_running(
9300 spec: &ModuleSpec,
9301 runtime: &SupervisorRuntimeConfig,
9302 snapshot: &SharedSnapshot,
9303) -> Result<SupervisedChild, SuperviseError> {
9304 let activity_spawn = runtime.begin_activity_spawn(&spec.module_id);
9305 let child = spawn_child(
9306 spec,
9307 runtime.connection_file_path.as_deref(),
9308 runtime.supervisor_handle.as_ref(),
9309 &runtime.stderr_ring,
9310 runtime.capture_logs_dir.as_deref(),
9311 &runtime.child_roster,
9312 #[cfg(target_os = "linux")]
9313 runtime.cgroup_placement.as_ref(),
9314 )?;
9315 set_running(snapshot, &child, &spec.module_id, &runtime.spawn_events)?;
9316 if let Some(spawn) = activity_spawn {
9317 spawn.complete(snapshot);
9318 }
9319 Ok(child)
9320}
9321
9322enum RegistrationWaitOutcome {
9323 Registered,
9324 Exited(ExitReport),
9325 TimedOut,
9326}
9327
9328struct ReloadRegistrationFailure {
9329 exit_report: ExitReport,
9330 reason: String,
9331}
9332
9333#[derive(Debug, Clone, Copy, PartialEq, Eq)]
9334enum BusyGaugeObservation {
9335 Quiescent,
9336 Busy,
9337 Omitted,
9338}
9339
9340fn busy_gauge_observation(metrics: Option<&Value>, gauges: &[String]) -> BusyGaugeObservation {
9341 let Some(metrics) = metrics.and_then(Value::as_object) else {
9342 return BusyGaugeObservation::Omitted;
9343 };
9344 let mut sum = 0u128;
9345 for gauge in gauges {
9346 let Some(value) = metrics.get(gauge) else {
9347 return BusyGaugeObservation::Omitted;
9348 };
9349 let Some(value) = value.as_u64() else {
9350 return BusyGaugeObservation::Busy;
9351 };
9352 sum = sum.saturating_add(u128::from(value));
9353 }
9354 if sum == 0 {
9355 BusyGaugeObservation::Quiescent
9356 } else {
9357 BusyGaugeObservation::Busy
9358 }
9359}
9360
9361fn declared_busy_gauges(
9362 registry: &Registry,
9363 module_id: &str,
9364) -> Result<Vec<String>, SuperviseError> {
9365 busy_gauges_of(
9366 registry
9367 .get_module(module_id)
9368 .map_err(SuperviseError::Registry)?,
9369 )
9370}
9371
9372fn declared_busy_gauges_for_connection(
9376 registry: &Registry,
9377 connection_id: ConnectionId,
9378) -> Result<Vec<String>, SuperviseError> {
9379 busy_gauges_of(
9380 registry
9381 .get_module_by_connection(connection_id)
9382 .map_err(SuperviseError::Registry)?,
9383 )
9384}
9385
9386fn busy_gauges_of(
9387 registration: Option<crate::registry::ModuleRegistration>,
9388) -> Result<Vec<String>, SuperviseError> {
9389 let Some(registration) = registration else {
9390 return Ok(Vec::new());
9391 };
9392 let Some(self_signals) = registration.manifest.self_signals else {
9393 return Ok(Vec::new());
9394 };
9395
9396 let mut gauges = Vec::new();
9397 for declaration in self_signals {
9398 if declaration.kind != SelfSignalKind::Busy {
9399 continue;
9400 }
9401 match declaration.anchored_to {
9402 SignalAnchor::HealthGauges { gauges: declared } if !declared.is_empty() => {
9403 gauges.extend(declared)
9404 }
9405 _ => {
9406 gauges.push(String::new());
9409 }
9410 }
9411 }
9412 Ok(gauges)
9413}
9414
9415async fn wait_for_forwarding_quiescence(
9420 forwarding: &ForwardingTable,
9421 module_id: &str,
9422 runtime: &SupervisorRuntimeConfig,
9423 endpoint: crate::ModuleEndpointId,
9424 deadline: Instant,
9425 busy_gauges: &[String],
9426 scope: DrainScope,
9427) -> Result<bool, SuperviseError> {
9428 let mut gauges_quiescent = busy_gauges.is_empty();
9429 let mut next_probe_at = Instant::now();
9430 let mut omission_counted = false;
9431
9432 loop {
9433 let now = Instant::now();
9434 if !busy_gauges.is_empty() && now >= next_probe_at && now < deadline {
9435 let report = match scope {
9436 DrainScope::Active => probe_module_health(module_id, runtime, Some(deadline)).await,
9437 DrainScope::Endpoint(endpoint) => {
9438 probe_endpoint_health(endpoint, runtime, Some(deadline)).await
9439 }
9440 };
9441 gauges_quiescent = match report {
9442 Ok(report) => match busy_gauge_observation(report.metrics.as_ref(), busy_gauges) {
9443 BusyGaugeObservation::Quiescent => true,
9444 BusyGaugeObservation::Busy => false,
9445 BusyGaugeObservation::Omitted => {
9446 if !omission_counted {
9447 forwarding
9448 .counters()
9449 .increment_drains_with_undeclared_gauge();
9450 omission_counted = true;
9451 }
9452 false
9453 }
9454 },
9455 Err(err) => {
9456 warn!(
9457 module_id,
9458 error = %err,
9459 "drain health.check did not produce declared busy gauges; treating module as busy"
9460 );
9461 false
9462 }
9463 };
9464 next_probe_at = Instant::now() + runtime.health.cadence.max(REGISTRY_RELEASE_POLL);
9465 }
9466
9467 let in_flight = forwarding
9468 .endpoint_in_flight_count(endpoint)
9469 .map_err(SuperviseError::Forwarding)?;
9470 if in_flight == 0 && gauges_quiescent {
9471 return Ok(true);
9472 }
9473
9474 let now = Instant::now();
9475 if now >= deadline {
9476 return Ok(false);
9477 }
9478 let mut wait = deadline
9479 .saturating_duration_since(now)
9480 .min(REGISTRY_RELEASE_POLL);
9481 if !busy_gauges.is_empty() {
9482 wait = wait.min(next_probe_at.saturating_duration_since(now));
9483 }
9484 sleep(wait).await;
9485 }
9486}
9487
9488fn drained_after_quiescence_wait(wait_result: &Result<bool, SuperviseError>) -> bool {
9496 match wait_result {
9497 Ok(drained) => *drained,
9498 Err(_) => false,
9499 }
9500}
9501
9502fn send_route_goodbyes(forwarding: &ForwardingTable, released_routes: Vec<GoodbyeTarget>) {
9503 for released in released_routes {
9504 let frame = match Frame::build_with_version(
9505 released.negotiated_ver,
9506 FrameType::Goodbye,
9507 control_flags(),
9508 released.channel,
9509 released.epoch,
9510 0,
9511 Vec::new(),
9512 ) {
9513 Ok(frame) => frame,
9514 Err(err) => {
9515 warn!(
9516 route_channel = released.channel,
9517 error = %err,
9518 "failed to build supervisor drain route GOODBYE frame"
9519 );
9520 continue;
9521 }
9522 };
9523 if !released.close_on_delivery_failure() {
9524 crate::forwarding::send_module_route_goodbye(
9525 &forwarding.counters(),
9526 &released.sink,
9527 frame,
9528 released.module_id.as_deref(),
9529 "supervisor drain",
9530 );
9531 continue;
9532 }
9533 if let Err(err) = released.sink.try_send(frame) {
9534 warn!(
9535 target_connection_id = released.connection_id.get(),
9536 route_channel = released.channel,
9537 error = %err,
9538 "supervisor drain route GOODBYE was not delivered to client; closing target connection"
9539 );
9540 let _ = forwarding.escalate_client_delivery_failure(
9541 released.connection_id,
9542 released.channel,
9543 released.epoch,
9544 CloseReason::new(
9545 "route_goodbye_delivery_failed",
9546 format!(
9547 "failed to enqueue supervisor drain route GOODBYE for channel {}: {err}",
9548 released.channel
9549 ),
9550 ),
9551 crate::forwarding::UndeliveredFrame {
9552 module_id: released.module_id.as_deref(),
9553 sink: &released.sink,
9554 },
9555 );
9556 }
9557 }
9558}
9559
9560fn send_module_draining(
9561 module_id: &str,
9562 reason: RouteCloseReason,
9563 deadline_ms: u64,
9564 target: &ModuleDrainTarget,
9565) {
9566 let body = match serde_json::to_vec(&ModuleControlCommand::Draining {
9567 reason,
9568 deadline_ms,
9569 }) {
9570 Ok(body) => body,
9571 Err(err) => {
9572 warn!(
9573 module_id,
9574 error = %err,
9575 "failed to encode module draining command"
9576 );
9577 return;
9578 }
9579 };
9580 let frame = match Frame::build_with_version(
9581 target.negotiated_ver,
9582 FrameType::Push,
9583 control_flags(),
9584 0,
9585 0,
9586 0,
9587 body,
9588 ) {
9589 Ok(frame) => frame,
9590 Err(err) => {
9591 warn!(
9592 module_id,
9593 error = %err,
9594 "failed to build module draining command frame"
9595 );
9596 return;
9597 }
9598 };
9599 if let Err(err) = target.sink.try_send(frame) {
9600 warn!(
9601 module_id,
9602 target_connection_id = target.endpoint.connection_id.get(),
9603 error = %err,
9604 "module draining command was not delivered to peer"
9605 );
9606 }
9607}
9608
9609fn module_goodbye_frame(module_id: &str, negotiated_ver: u8) -> Option<Frame> {
9611 match Frame::build_with_version(
9612 negotiated_ver,
9613 FrameType::Goodbye,
9614 control_flags(),
9615 0,
9616 0,
9617 0,
9618 Vec::new(),
9619 ) {
9620 Ok(frame) => Some(frame),
9621 Err(err) => {
9622 warn!(
9623 module_id,
9624 error = %err,
9625 "failed to build module GOODBYE frame"
9626 );
9627 None
9628 }
9629 }
9630}
9631
9632#[cfg(unix)]
9646async fn send_module_goodbyes_for_daemon_shutdown(
9647 forwarding: &Arc<ForwardingTable>,
9648 reason: &CloseReason,
9649 wait_for_flush: bool,
9650) {
9651 const GOODBYE_BUDGET: Duration = Duration::from_millis(500);
9652 let targets = match forwarding.module_connections() {
9653 Ok(targets) => targets,
9654 Err(err) => {
9655 warn!(error = %err, "could not list module connections for shutdown GOODBYE");
9656 return;
9657 }
9658 };
9659 let deadline = Instant::now() + GOODBYE_BUDGET;
9660 let mut sends = tokio::task::JoinSet::new();
9661 for target in targets {
9662 let Some(frame) = module_goodbye_frame(&target.module_id, target.negotiated_ver) else {
9663 continue;
9664 };
9665 if !wait_for_flush {
9666 if let Err(err) = target.sink.try_send(frame) {
9667 debug!(
9668 module_id = %target.module_id,
9669 error = %err,
9670 "shutdown module GOODBYE was not queued"
9671 );
9672 }
9673 continue;
9674 }
9675 let forwarding = Arc::clone(forwarding);
9676 let reason = reason.clone();
9677 sends.spawn(async move {
9678 match timeout_at(deadline, target.sink.send_flushed(frame)).await {
9679 Ok(Ok(())) => {}
9680 Ok(Err(err)) => debug!(
9681 module_id = %target.module_id,
9682 error = %err,
9683 "module connection closed before its shutdown GOODBYE was written"
9684 ),
9685 Err(_) => warn!(
9686 module_id = %target.module_id,
9687 budget = ?GOODBYE_BUDGET,
9688 "shutdown module GOODBYE was not written within its budget; closing anyway"
9689 ),
9690 }
9691 forwarding.request_connection_close(target.endpoint.connection_id, reason);
9692 });
9693 }
9694 while sends.join_next().await.is_some() {}
9696}
9697
9698fn send_module_goodbye(module_id: &str, forwarding: &ForwardingTable, target: &ModuleDrainTarget) {
9699 let Some(frame) = module_goodbye_frame(module_id, target.negotiated_ver) else {
9700 return;
9701 };
9702 if let Err(err) = target.sink.try_send(frame) {
9703 warn!(
9704 module_id,
9705 target_connection_id = target.endpoint.connection_id.get(),
9706 error = %err,
9707 "supervisor drain module GOODBYE was not delivered to peer; closing module connection"
9708 );
9709 forwarding.request_connection_close(
9710 target.endpoint.connection_id,
9711 CloseReason::new(
9712 "module_goodbye_delivery_failed",
9713 format!("failed to enqueue supervisor drain module GOODBYE for module '{module_id}': {err}"),
9714 ),
9715 );
9716 }
9717}
9718
9719#[derive(Clone, Copy)]
9720struct ForwardingDrainContext<'a> {
9721 spec: &'a ModuleSpec,
9722 runtime: &'a SupervisorRuntimeConfig,
9723 registry: &'a Registry,
9724 scope: DrainScope,
9725}
9726
9727#[derive(Debug, Clone, Copy, PartialEq, Eq)]
9729enum DrainScope {
9730 Active,
9733 Endpoint(crate::ModuleEndpointId),
9738}
9739
9740#[derive(Debug, Clone, Copy, PartialEq, Eq)]
9748enum StopNotice {
9749 SentOverConnection,
9752 NoConnection,
9756 NotSent,
9760}
9761
9762async fn begin_forwarding_drain(
9763 spec: &ModuleSpec,
9764 runtime: &SupervisorRuntimeConfig,
9765 registry: &Registry,
9766 snapshot: &SharedSnapshot,
9767 enabled: Option<bool>,
9768 reason: RouteCloseReason,
9769) -> Result<StopNotice, SuperviseError> {
9770 let Some(forwarding) = runtime.forwarding.as_ref() else {
9771 return Err(SuperviseError::ReloadUnavailable {
9772 module_id: spec.module_id.clone(),
9773 reason: "supervisor was not configured with a forwarding table".to_string(),
9774 });
9775 };
9776
9777 begin_forwarding_drain_with(
9778 forwarding,
9779 ForwardingDrainContext {
9780 spec,
9781 runtime,
9782 registry,
9783 scope: DrainScope::Active,
9784 },
9785 snapshot,
9786 enabled,
9787 reason,
9788 runtime.drain_timeout,
9789 )
9790 .await
9791}
9792
9793async fn begin_forwarding_drain_if_configured(
9794 spec: &ModuleSpec,
9795 runtime: &SupervisorRuntimeConfig,
9796 registry: &Registry,
9797 snapshot: &SharedSnapshot,
9798 enabled: Option<bool>,
9799 reason: RouteCloseReason,
9800) -> Result<StopNotice, SuperviseError> {
9801 begin_forwarding_drain_with_timeout(
9802 spec,
9803 runtime,
9804 registry,
9805 snapshot,
9806 enabled,
9807 reason,
9808 runtime.drain_timeout,
9809 )
9810 .await
9811}
9812
9813async fn begin_forwarding_drain_with_timeout(
9817 spec: &ModuleSpec,
9818 runtime: &SupervisorRuntimeConfig,
9819 registry: &Registry,
9820 snapshot: &SharedSnapshot,
9821 enabled: Option<bool>,
9822 reason: RouteCloseReason,
9823 drain_timeout: Duration,
9824) -> Result<StopNotice, SuperviseError> {
9825 let Some(forwarding) = runtime.forwarding.as_ref() else {
9826 return Ok(StopNotice::NotSent);
9827 };
9828
9829 begin_forwarding_drain_with(
9830 forwarding,
9831 ForwardingDrainContext {
9832 spec,
9833 runtime,
9834 registry,
9835 scope: DrainScope::Active,
9836 },
9837 snapshot,
9838 enabled,
9839 reason,
9840 drain_timeout,
9841 )
9842 .await
9843}
9844
9845async fn begin_forwarding_drain_with(
9846 forwarding: &ForwardingTable,
9847 context: ForwardingDrainContext<'_>,
9848 snapshot: &SharedSnapshot,
9849 enabled: Option<bool>,
9850 reason: RouteCloseReason,
9851 drain_timeout: Duration,
9852) -> Result<StopNotice, SuperviseError> {
9853 let ForwardingDrainContext {
9854 spec,
9855 runtime,
9856 registry,
9857 scope,
9858 } = context;
9859 debug_assert_ne!(reason, RouteCloseReason::Crash);
9860 let terminal = matches!(reason, RouteCloseReason::Disable);
9861 let drain_started_at = Instant::now();
9862 let drain_deadline = drain_started_at + drain_timeout;
9863 let deadline_ms =
9864 unix_ms_now().saturating_add(u64::try_from(drain_timeout.as_millis()).unwrap_or(u64::MAX));
9865 let busy_gauges = match scope {
9866 DrainScope::Active => declared_busy_gauges(registry, &spec.module_id)?,
9867 DrainScope::Endpoint(endpoint) => {
9868 declared_busy_gauges_for_connection(registry, endpoint.connection_id)?
9869 }
9870 };
9871
9872 let gate_started = Instant::now();
9875 let drain_target = match scope {
9876 DrainScope::Active => forwarding.begin_module_drain(&spec.module_id, reason),
9877 DrainScope::Endpoint(endpoint) => forwarding.begin_endpoint_drain(endpoint, reason),
9878 }
9879 .map_err(SuperviseError::Forwarding)?;
9880 info!(
9885 module_id = %spec.module_id,
9886 ?reason,
9887 gate_ms = u64::try_from(gate_started.elapsed().as_millis()).unwrap_or(u64::MAX),
9888 connected = drain_target.is_some(),
9889 "module drain began; route admission closed"
9890 );
9891 if scope == DrainScope::Active {
9892 update_snapshot(snapshot, Some(&spec.module_id), |state| {
9893 state.state = ModuleState::Draining;
9894 state.draining_to_replace =
9895 matches!(reason, RouteCloseReason::Restart | RouteCloseReason::Reload);
9896 if let Some(enabled) = enabled {
9897 state.enabled = enabled;
9898 }
9899 })?;
9900 }
9901
9902 let Some(target) = drain_target.as_ref() else {
9903 return Ok(StopNotice::NoConnection);
9907 };
9908 {
9909 send_module_draining(&spec.module_id, reason, deadline_ms, target);
9910 #[cfg(feature = "test-support")]
9911 forwarding.wait_before_route_closing_for_test().await;
9912 let routes = forwarding
9913 .endpoint_routes(target.endpoint)
9914 .map_err(SuperviseError::Forwarding)?;
9915 let routes_notified = routes.len();
9916 crate::control::send_route_control_pushes(
9917 forwarding,
9918 routes.clone(),
9919 ClientControlPush::RouteClosing {
9920 module_id: spec.module_id.clone(),
9921 channels: Vec::new(),
9922 reason,
9923 },
9924 );
9925 send_route_goodbyes(forwarding, target.abandoned_bindings.clone());
9926
9927 let wait_result = wait_for_forwarding_quiescence(
9933 forwarding,
9934 &spec.module_id,
9935 runtime,
9936 target.endpoint,
9937 drain_deadline,
9938 &busy_gauges,
9939 scope,
9940 )
9941 .await;
9942 let drained = drained_after_quiescence_wait(&wait_result);
9943 if let Err(err) = &wait_result {
9944 error!(
9945 module_id = %spec.module_id,
9946 ?reason,
9947 error = %err,
9948 "forwarding quiescence wait failed after route.closing; forcing route.closed(drained: false) so the client is not left waiting on an unfulfilled promise"
9949 );
9950 } else if !drained {
9951 let holdouts = forwarding
9957 .endpoint_drain_holdouts(target.endpoint)
9958 .unwrap_or_default();
9959 warn!(
9960 module_id = %spec.module_id,
9961 waited = ?drain_timeout,
9962 ?reason,
9963 held_requests = holdouts.requests,
9964 held_routes = holdouts.routes,
9965 total_routes = holdouts.total_routes,
9966 top_connections = ?holdouts.top_connections,
9967 held = %holdouts
9970 .held
9971 .iter()
9972 .map(|(channel, corr)| format!("{channel}:{corr}"))
9973 .collect::<Vec<_>>()
9974 .join(","),
9975 "route drain timed out before request quiescence; forcing teardown"
9976 );
9977 }
9978 crate::control::send_route_control_pushes(
9979 forwarding,
9980 routes,
9981 ClientControlPush::RouteClosed {
9982 module_id: spec.module_id.clone(),
9983 channels: Vec::new(),
9984 reason,
9985 drained,
9986 abandoned: target.abandoned_bindings.len() as u32,
9987 excluded_subscriptions: target.excluded_subscriptions,
9988 terminal: Some(terminal),
9989 },
9990 );
9991 wait_result?;
9992
9993 let released_routes = match forwarding.release_module_endpoint_routes(target.endpoint) {
9999 Ok(routes) => routes,
10000 Err(err) => {
10001 warn!(
10002 module_id = %spec.module_id,
10003 ?reason,
10004 error = %err,
10005 "failed to release module endpoint routes after route.closed; module GOODBYE will still be sent"
10006 );
10007 send_module_goodbye(&spec.module_id, forwarding, target);
10008 return Err(SuperviseError::Forwarding(err));
10009 }
10010 };
10011 let route_goodbye_count = released_routes.len();
10012 send_route_goodbyes(forwarding, released_routes);
10013 send_module_goodbye(&spec.module_id, forwarding, target);
10014
10015 info!(
10021 module_id = %spec.module_id,
10022 ?reason,
10023 routes_notified,
10024 route_goodbyes = route_goodbye_count,
10025 abandoned_reservations = target.abandoned_bindings.len(),
10026 excluded_subscriptions = target.excluded_subscriptions,
10027 drained,
10028 "module drain complete; consumers notified via route.closing/route.closed pushes and per-route GOODBYE frames"
10029 );
10030 }
10031
10032 Ok(StopNotice::SentOverConnection)
10033}
10034
10035async fn wait_for_registration_after_reload(
10038 registry: &Registry,
10039 module_id: &str,
10040 snapshot: &SharedSnapshot,
10041 child: &mut SupervisedChild,
10042 wait: Duration,
10043) -> Result<RegistrationWaitOutcome, SuperviseError> {
10044 wait_for_slot_registration(
10045 registry,
10046 crate::registry::RegistrationSlot::Active(module_id),
10047 module_id,
10048 snapshot,
10049 child,
10050 wait,
10051 )
10052 .await
10053}
10054
10055async fn wait_for_slot_registration(
10063 registry: &Registry,
10064 slot: crate::registry::RegistrationSlot<'_>,
10065 module_id: &str,
10066 snapshot: &SharedSnapshot,
10067 child: &mut SupervisedChild,
10068 wait: Duration,
10069) -> Result<RegistrationWaitOutcome, SuperviseError> {
10070 let deadline = Instant::now() + wait;
10071 loop {
10072 if registry
10073 .registration(slot)
10074 .map_err(SuperviseError::Registry)?
10075 .is_some()
10076 {
10077 return Ok(RegistrationWaitOutcome::Registered);
10078 }
10079
10080 let now = Instant::now();
10081 if now >= deadline {
10082 return Ok(RegistrationWaitOutcome::TimedOut);
10083 }
10084 let remaining = deadline.saturating_duration_since(now);
10085 let poll = remaining.min(REGISTRY_RELEASE_POLL);
10086
10087 tokio::select! {
10088 wait_result = child.wait() => {
10089 let status = wait_result.map_err(|source| SuperviseError::Wait {
10090 module_id: module_id.to_string(),
10091 source,
10092 })?;
10093 return Ok(RegistrationWaitOutcome::Exited(classify_reaped_child_exit(
10094 snapshot,
10095 child,
10096 &status,
10097 )));
10098 }
10099 _ = sleep(poll) => {}
10100 }
10101 }
10102}
10103
10104fn registration_failure_exit_report(mut exit_report: ExitReport) -> ExitReport {
10105 if exit_report.kind != ExitKind::DeliberateSeverance {
10108 exit_report.kind = ExitKind::Crash;
10109 }
10110 exit_report
10111}
10112
10113async fn handle_reload_child_registration_failure(
10114 spec: &ModuleSpec,
10115 runtime: &SupervisorRuntimeConfig,
10116 registry: &Registry,
10117 process_liveness: &SupervisorProcessLiveness,
10118 snapshot: &SharedSnapshot,
10119 _child: &mut Option<SupervisedChild>,
10120 failure: ReloadRegistrationFailure,
10121) -> Result<(), SuperviseError> {
10122 let ReloadRegistrationFailure {
10123 exit_report,
10124 reason,
10125 } = failure;
10126 match on_child_exit(
10127 spec,
10128 runtime.restart_policy,
10129 registry,
10130 snapshot,
10131 &runtime.terminal_ring,
10132 &runtime.spawn_events,
10133 &runtime.child_roster,
10134 exit_report,
10135 )
10136 .await
10137 {
10138 NextAction::Stop {
10139 registration_released,
10140 } => {
10141 if registration_released {
10142 process_liveness.untrack_if_current(&spec.module_id, snapshot);
10143 }
10144 }
10145 NextAction::Restart { schedule } => {
10146 let delay = schedule.map_or(runtime.restart_policy.delay_for_restart(0), |schedule| {
10147 schedule.delay
10148 });
10149 if let Some(schedule) = schedule {
10150 log_crash_respawn(&spec.module_id, schedule);
10151 }
10152 schedule_respawn(
10153 runtime,
10154 snapshot,
10155 &spec.module_id,
10156 delay,
10157 RespawnKind::Spawn,
10158 )?;
10159 }
10160 }
10161 Err(SuperviseError::ReloadFailed {
10162 module_id: spec.module_id.clone(),
10163 reason,
10164 })
10165}
10166
10167async fn handle_reload_spawn_failure(
10168 spec: &ModuleSpec,
10169 runtime: &SupervisorRuntimeConfig,
10170 process_liveness: &SupervisorProcessLiveness,
10171 snapshot: &SharedSnapshot,
10172 _child: &mut Option<SupervisedChild>,
10173 reason: String,
10174) -> Result<(), SuperviseError> {
10175 let now = Instant::now();
10176 let mut schedule = None;
10177 update_snapshot(snapshot, Some(&spec.module_id), |state| {
10178 clear_current_process_facts(state);
10179 if state.enabled {
10180 schedule = state.next_crash_restart(&runtime.restart_policy, now);
10181 state.state = if schedule.is_some() {
10182 ModuleState::Restarting
10183 } else {
10184 ModuleState::Failed
10185 };
10186 } else {
10187 state.state = ModuleState::Disabled;
10188 }
10189 })?;
10190 if let Some(schedule) = schedule {
10191 schedule_respawn(
10192 runtime,
10193 snapshot,
10194 &spec.module_id,
10195 schedule.delay,
10196 RespawnKind::Spawn,
10197 )?;
10198 } else {
10199 process_liveness.untrack_if_current(&spec.module_id, snapshot);
10200 }
10201 Err(SuperviseError::ReloadFailed {
10202 module_id: spec.module_id.clone(),
10203 reason,
10204 })
10205}
10206
10207fn control_flags() -> Flags {
10208 Flags::new(false, Priority::Passive, false)
10209}
10210
10211#[allow(clippy::too_many_arguments)]
10212async fn drain_optional_child(
10213 module_id: &str,
10214 protocol: ModuleProtocol,
10215 stop_notice: StopNotice,
10216 registry: &Registry,
10217 forwarding: Option<&ForwardingTable>,
10218 snapshot: &SharedSnapshot,
10219 terminal_ring: &Arc<Mutex<TerminalRing>>,
10220 spawn_events: &SpawnEventFeed,
10221 child: &mut Option<SupervisedChild>,
10222 drain_timeout: Duration,
10223 final_state: ModuleState,
10224 enabled: Option<bool>,
10225) -> Result<(), SuperviseError> {
10226 if let Some(child) = child.take() {
10227 drain_child_to_state(
10228 module_id,
10229 protocol,
10230 stop_notice,
10231 registry,
10232 forwarding,
10233 snapshot,
10234 terminal_ring,
10235 spawn_events,
10236 child,
10237 drain_timeout,
10238 final_state,
10239 enabled,
10240 )
10241 .await
10242 } else {
10243 update_snapshot(snapshot, Some(module_id), |state| {
10244 state.state = final_state;
10245 if let Some(enabled) = enabled {
10246 state.enabled = enabled;
10247 }
10248 clear_current_process_facts(state);
10249 })?;
10250 release_dead_registration(registry, forwarding, snapshot, module_id).await
10251 }
10252}
10253
10254#[allow(clippy::too_many_arguments)]
10255async fn drain_child_to_state(
10256 module_id: &str,
10257 _protocol: ModuleProtocol,
10258 stop_notice: StopNotice,
10259 registry: &Registry,
10260 forwarding: Option<&ForwardingTable>,
10261 snapshot: &SharedSnapshot,
10262 terminal_ring: &Arc<Mutex<TerminalRing>>,
10263 spawn_events: &SpawnEventFeed,
10264 mut child: SupervisedChild,
10265 drain_timeout: Duration,
10266 final_state: ModuleState,
10267 enabled: Option<bool>,
10268) -> Result<(), SuperviseError> {
10269 let protocol = child.protocol;
10270 update_snapshot(snapshot, Some(module_id), |state| {
10271 state.state = ModuleState::Draining;
10272 state.draining_to_replace = final_state == ModuleState::Restarting;
10273 if let Some(enabled) = enabled {
10274 state.enabled = enabled;
10275 }
10276 })?;
10277
10278 if stop_notice != StopNotice::SentOverConnection {
10289 if protocol == ModuleProtocol::Subc && stop_notice == StopNotice::NoConnection {
10290 info!(
10291 module_id,
10292 pid = child.pid,
10293 budget_ms = u64::try_from(drain_timeout.as_millis()).unwrap_or(u64::MAX),
10294 "module has no connection yet; requesting stop by signal"
10295 );
10296 }
10297 request_graceful_stop(module_id, &child);
10298 }
10299
10300 let exit_report = match timeout(drain_timeout, child.wait()).await {
10301 Ok(Ok(status)) => classify_reaped_child_exit(snapshot, &child, &status),
10302 Ok(Err(source)) => {
10303 fail_after_wait_error(
10304 module_id,
10305 snapshot,
10306 terminal_ring,
10307 spawn_events,
10308 format!("failed to wait during drain: {source}"),
10309 );
10310 return Err(SuperviseError::Wait {
10311 module_id: module_id.to_string(),
10312 source,
10313 });
10314 }
10315 Err(_) => {
10316 warn!(
10329 module_id,
10330 pid = child.pid,
10331 budget_ms = u64::try_from(drain_timeout.as_millis()).unwrap_or(u64::MAX),
10332 reason = ?final_state,
10333 ?stop_notice,
10334 "drain budget expired before the module exited; killing it"
10335 );
10336 child.start_kill().map_err(|source| {
10337 fail_after_wait_error(
10338 module_id,
10339 snapshot,
10340 terminal_ring,
10341 spawn_events,
10342 format!("failed to kill during drain: {source}"),
10343 );
10344 SuperviseError::Kill {
10345 module_id: module_id.to_string(),
10346 source,
10347 }
10348 })?;
10349 let status = child.wait().await.map_err(|source| {
10350 fail_after_wait_error(
10351 module_id,
10352 snapshot,
10353 terminal_ring,
10354 spawn_events,
10355 format!("failed to wait after drain kill: {source}"),
10356 );
10357 SuperviseError::Wait {
10358 module_id: module_id.to_string(),
10359 source,
10360 }
10361 })?;
10362 classify_reaped_child_exit(snapshot, &child, &status)
10363 }
10364 };
10365
10366 let detail = lock_snapshot(snapshot)?.drain_disposition_detail.take();
10367 publish_after_terminal_record(
10368 module_id,
10369 || {
10370 record_terminal_with_detail(
10371 module_id,
10372 terminal_ring,
10373 spawn_events,
10374 &exit_report,
10375 terminal_disposition(final_state),
10376 detail,
10377 )
10378 },
10379 || {
10380 update_snapshot(snapshot, Some(module_id), |state| {
10381 state.state = final_state;
10382 if let Some(enabled) = enabled {
10383 state.enabled = enabled;
10384 }
10385 clear_current_process_facts(state);
10386 state.last_exit = Some(exit_report.clone());
10387 if exit_report.kind == ExitKind::DeliberateSeverance {
10388 state.lifetime_restarts += 1;
10389 }
10390 })
10391 },
10392 )?;
10393 child.drain_stderr(module_id).await;
10394
10395 release_dead_registration(registry, forwarding, snapshot, module_id).await
10396}
10397
10398#[cfg(unix)]
10418fn request_graceful_stop(module_id: &str, child: &SupervisedChild) {
10419 let Some(pid) = child
10420 .id()
10421 .and_then(|pid| i32::try_from(pid).ok())
10422 .and_then(rustix::process::Pid::from_raw)
10423 else {
10424 debug!(
10425 module_id,
10426 "no pid to signal for teardown; falling through to the drain wait"
10427 );
10428 return;
10429 };
10430 match rustix::process::kill_process(pid, rustix::process::Signal::TERM) {
10431 Ok(()) => debug!(
10432 module_id,
10433 "sent SIGTERM to a module nothing else asked to stop"
10434 ),
10435 Err(err) => debug!(
10436 module_id,
10437 error = %err,
10438 "SIGTERM to module failed; the drain wait and kill still apply"
10439 ),
10440 }
10441}
10442
10443#[cfg(not(unix))]
10451fn request_graceful_stop(module_id: &str, _child: &SupervisedChild) {
10452 debug!(
10453 module_id,
10454 "no graceful stop signal exists on this platform; teardown of a module nothing asked to stop waits, then kills"
10455 );
10456}
10457
10458fn terminal_disposition(final_state: ModuleState) -> TerminalDisposition {
10459 match final_state {
10460 ModuleState::Stopped => TerminalDisposition::Stopped,
10461 ModuleState::Disabled => TerminalDisposition::Disabled,
10462 ModuleState::Restarting => TerminalDisposition::Restarting,
10463 ModuleState::Failed => TerminalDisposition::Failed,
10464 ModuleState::Starting
10465 | ModuleState::Running
10466 | ModuleState::Unresponsive
10467 | ModuleState::Draining => {
10468 unreachable!("terminal exits only finish in terminal or restarting states")
10469 }
10470 }
10471}
10472
10473async fn release_dead_registration(
10481 registry: &Registry,
10482 forwarding: Option<&ForwardingTable>,
10483 snapshot: &SharedSnapshot,
10484 module_id: &str,
10485) -> Result<(), SuperviseError> {
10486 let result = async {
10487 let registration = registry
10488 .get_module(module_id)
10489 .map_err(SuperviseError::Registry)?;
10490 match wait_for_registration_release(registry, module_id, REGISTRY_RELEASE_TIMEOUT).await {
10491 Ok(()) => return Ok(()),
10492 Err(SuperviseError::RegistrationStillActive { .. }) => {}
10493 Err(err) => return Err(err),
10494 }
10495 let pid = lock_snapshot(snapshot)?.reaped_pid;
10496 if let (Some(registration), Some(forwarding), Some(pid)) = (registration, forwarding, pid) {
10497 warn!(
10498 module_id,
10499 pid,
10500 connection_id = registration.connection_id.get(),
10501 "reaped module registration outlived release grace; closing dead connection"
10502 );
10503 forwarding.request_connection_close(
10504 registration.connection_id,
10505 CloseReason::new(
10506 "supervised_process_reaped",
10507 format!("module '{module_id}' pid {pid} exited"),
10508 ),
10509 );
10510 wait_for_slot_registration_release(
10511 registry,
10512 crate::registry::RegistrationSlot::Connection(registration.connection_id),
10513 REGISTRY_RELEASE_TIMEOUT,
10514 )
10515 .await?;
10516 }
10517 wait_for_registration_release(registry, module_id, Duration::ZERO).await
10518 }
10519 .await;
10520 if let Err(err) = &result {
10521 fail_snapshot(snapshot, Some(module_id), None);
10522 error!(module_id, error = %err, "registration release failed after child exit; module is failed and start can retry");
10523 }
10524 result
10525}
10526
10527async fn wait_for_registration_release(
10530 registry: &Registry,
10531 module_id: &str,
10532 wait: Duration,
10533) -> Result<(), SuperviseError> {
10534 wait_for_slot_registration_release(
10535 registry,
10536 crate::registry::RegistrationSlot::Active(module_id),
10537 wait,
10538 )
10539 .await
10540}
10541
10542async fn wait_for_slot_registration_release(
10550 registry: &Registry,
10551 slot: crate::registry::RegistrationSlot<'_>,
10552 wait: Duration,
10553) -> Result<(), SuperviseError> {
10554 let deadline = Instant::now() + wait;
10555 let mut release_events = registration_release_events().subscribe();
10556 let still_active = |registration: &crate::registry::ModuleRegistration| {
10557 SuperviseError::RegistrationStillActive {
10558 module_id: registration.manifest.module_id.clone(),
10559 waited: wait,
10560 }
10561 };
10562 loop {
10563 let _observed_generation = *release_events.borrow_and_update();
10564 let Some(registration) = registry
10565 .registration(slot)
10566 .map_err(SuperviseError::Registry)?
10567 else {
10568 return Ok(());
10569 };
10570
10571 let now = Instant::now();
10572 if now >= deadline {
10573 return Err(still_active(®istration));
10574 }
10575
10576 let remaining = deadline.saturating_duration_since(now);
10577 match timeout(remaining, release_events.changed()).await {
10578 Ok(Ok(())) | Ok(Err(_)) => {}
10579 Err(_) => return Err(still_active(®istration)),
10580 }
10581 }
10582}
10583
10584#[cfg(test)]
10585mod slot_registration_wait_tests {
10586 use super::*;
10587 use crate::registry::{ConnectionId, RegistrationSlot};
10588 use subc_protocol::manifest::ModuleManifest;
10589
10590 #[tokio::test]
10591 async fn enable_release_failure_is_failed_and_a_second_enable_retries() {
10592 let registry = Arc::new(Registry::default());
10593 let supervisor = Supervisor::new_for_test(Arc::clone(®istry), RestartPolicy::default());
10594 let runtime = supervisor.runtime_config();
10595 let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::disabled()));
10596 let spec = ModuleSpec {
10597 module_id: "enable-stale-registration".to_string(),
10598 program: PathBuf::from("/missing/enable-retry-test"),
10599 args: Vec::new(),
10600 env: Vec::new(),
10601 reserved: false,
10602 reserved_prefixes: Vec::new(),
10603 protocol: ModuleProtocol::Subc,
10604 overlap: Default::default(),
10605 };
10606 let connection = ConnectionId::new(90);
10607 registry
10608 .register_with_control_ops(
10609 ModuleManifest::builder(&spec.module_id, "0.1.0").build(),
10610 1,
10611 connection,
10612 Vec::new(),
10613 )
10614 .unwrap();
10615 let mut child = None;
10616 let err = set_child_enabled(
10617 &spec,
10618 &runtime,
10619 ®istry,
10620 &supervisor.process_liveness,
10621 &snapshot,
10622 &mut child,
10623 true,
10624 )
10625 .await
10626 .unwrap_err();
10627 assert!(matches!(
10628 err,
10629 SuperviseError::RegistrationStillActive { .. }
10630 ));
10631 assert_eq!(lock_snapshot(&snapshot).unwrap().state, ModuleState::Failed);
10632 assert!(child.is_none());
10633 registry.deregister_connection(connection).unwrap();
10634 let err = set_child_enabled(
10635 &spec,
10636 &runtime,
10637 ®istry,
10638 &supervisor.process_liveness,
10639 &snapshot,
10640 &mut child,
10641 true,
10642 )
10643 .await
10644 .unwrap_err();
10645 assert!(
10646 matches!(err, SuperviseError::Spawn { .. }),
10647 "second enable must attempt a spawn: {err}"
10648 );
10649 assert_eq!(lock_snapshot(&snapshot).unwrap().state, ModuleState::Failed);
10650 }
10651
10652 const INCUMBENT: u64 = 1;
10653 const CANDIDATE: u64 = 2;
10654
10655 fn swapped_registry() -> Arc<Registry> {
10656 let registry = Arc::new(Registry::default());
10657 let manifest = ModuleManifest::builder("m", "0.1.0").build();
10658 registry
10659 .register_with_control_ops(
10660 manifest.clone(),
10661 1,
10662 ConnectionId::new(INCUMBENT),
10663 Vec::new(),
10664 )
10665 .unwrap();
10666 registry
10667 .register_candidate_with_control_ops(
10668 manifest,
10669 1,
10670 ConnectionId::new(CANDIDATE),
10671 Vec::new(),
10672 )
10673 .unwrap();
10674 registry
10675 }
10676
10677 #[tokio::test]
10681 async fn incumbent_release_is_awaited_by_connection_not_by_module_id() {
10682 let registry = swapped_registry();
10683 registry.promote_candidate("m").unwrap().unwrap();
10684
10685 assert!(matches!(
10686 wait_for_registration_release(®istry, "m", Duration::from_millis(50)).await,
10687 Err(SuperviseError::RegistrationStillActive { .. })
10688 ));
10689
10690 assert!(matches!(
10692 wait_for_slot_registration_release(
10693 ®istry,
10694 RegistrationSlot::Connection(ConnectionId::new(INCUMBENT)),
10695 Duration::from_millis(50),
10696 )
10697 .await,
10698 Err(SuperviseError::RegistrationStillActive { .. })
10699 ));
10700
10701 let releaser = Arc::clone(®istry);
10702 let release = tokio::spawn(async move {
10703 sleep(Duration::from_millis(20)).await;
10704 releaser
10705 .deregister_connection(ConnectionId::new(INCUMBENT))
10706 .unwrap();
10707 notify_registration_release();
10708 });
10709 wait_for_slot_registration_release(
10710 ®istry,
10711 RegistrationSlot::Connection(ConnectionId::new(INCUMBENT)),
10712 Duration::from_secs(5),
10713 )
10714 .await
10715 .expect("the incumbent's own registration is released");
10716 release.await.unwrap();
10717 assert!(registry.get_module("m").unwrap().is_some());
10718 }
10719
10720 #[tokio::test]
10723 async fn candidate_slot_wait_ignores_the_incumbents_registration() {
10724 let registry = swapped_registry();
10725 assert!(matches!(
10726 wait_for_slot_registration_release(
10727 ®istry,
10728 RegistrationSlot::Candidate("m"),
10729 Duration::from_millis(50),
10730 )
10731 .await,
10732 Err(SuperviseError::RegistrationStillActive { .. })
10733 ));
10734 registry
10735 .deregister_connection(ConnectionId::new(CANDIDATE))
10736 .unwrap();
10737 wait_for_slot_registration_release(
10738 ®istry,
10739 RegistrationSlot::Candidate("m"),
10740 Duration::from_millis(50),
10741 )
10742 .await
10743 .expect("a candidate slot with no candidate is released");
10744 assert!(registry
10745 .registration(RegistrationSlot::Active("m"))
10746 .unwrap()
10747 .is_some());
10748 }
10749}
10750
10751fn classify_exit(status: &ExitStatus) -> ExitReport {
10752 ExitReport {
10753 kind: if status.success() {
10754 ExitKind::Clean
10755 } else {
10756 ExitKind::Crash
10757 },
10758 code: status.code(),
10759 signal: exit_signal(status),
10760 at_ms: unix_ms_now(),
10761 }
10762}
10763
10764fn wait_error_exit_report() -> ExitReport {
10770 ExitReport {
10771 kind: ExitKind::Crash,
10772 code: None,
10773 signal: None,
10774 at_ms: unix_ms_now(),
10775 }
10776}
10777
10778#[cfg(unix)]
10779fn exit_signal(status: &ExitStatus) -> Option<i32> {
10780 use std::os::unix::process::ExitStatusExt;
10781
10782 status.signal()
10783}
10784
10785#[cfg(not(unix))]
10786fn exit_signal(_status: &ExitStatus) -> Option<i32> {
10787 None
10788}
10789
10790fn reset_restart_count(snapshot: &SharedSnapshot, module_id: &str) -> Result<(), SuperviseError> {
10796 update_snapshot(snapshot, Some(module_id), |state| {
10797 state.clear_crash_restarts();
10798 })
10799}
10800
10801fn set_running(
10802 snapshot: &SharedSnapshot,
10803 child: &SupervisedChild,
10804 module_id: &str,
10805 spawn_events: &SpawnEventFeed,
10806) -> Result<(), SuperviseError> {
10807 let mut state = snapshot.lock().map_err(|_| SuperviseError::StatePoisoned {
10808 module_id: Some(module_id.to_string()),
10809 })?;
10810 state.spawn_generation = spawn_events.emit_spawned(module_id, child.pid, child.spawned_at_ms);
10811 if std::mem::take(&mut state.coalesced_restart_pending) {
10812 let generation = state.spawn_generation;
10813 info!(module_id, "restart already satisfied by generation {generation}; coalesced pending request completed");
10814 }
10815 state.drain_disposition_detail = None;
10816 state.spawn_failure = None;
10817 state.waiting_for_previous_pid = None;
10818 state.waiting_for_previous_detail = None;
10819 state.in_alternate_slot = false;
10822 state.configuration_updated_since_spawn = false;
10823 state.spawned_protocol = Some(child.protocol);
10824 state.state = ModuleState::Running;
10825 state.enabled = true;
10826 state.process_alive = true;
10827 state.pid = child.id();
10828 #[cfg(windows)]
10829 {
10830 state.pipe_consumed = child
10831 .nonce_delivery
10832 .as_ref()
10833 .map(|delivery| delivery.consumed());
10834 }
10835 #[cfg(target_os = "macos")]
10836 {
10837 state.report_ready = Some(Arc::clone(&child.report_ready));
10838 }
10839 state.spawned_at_ms = Some(child.spawned_at_ms);
10840 state.exec_outcome = Some(Arc::clone(&child.exec.0));
10841 state.spawned_from = Some(child.spawned_from.clone());
10842 state.spawned_file_identity = child.spawned_file_identity.clone();
10843 state.process_start_time = child.process_start_time;
10844 Ok(())
10845}
10846
10847fn clear_current_process_facts(state: &mut SupervisorSnapshot) {
10848 state.waiting_for_previous_pid = None;
10849 state.waiting_for_previous_detail = None;
10850 state.process_alive = false;
10851 state.spawned_protocol = None;
10852 state.pid = None;
10853 #[cfg(windows)]
10854 {
10855 state.pipe_consumed = None;
10856 }
10857 #[cfg(target_os = "macos")]
10858 {
10859 state.report_ready = None;
10860 }
10861 state.spawned_at_ms = None;
10862 state.exec_outcome = None;
10863 state.spawned_from = None;
10864 state.spawned_file_identity = None;
10865 state.process_start_time = None;
10866 state.deliberate_severance = None;
10867}
10868
10869#[cfg(test)]
10870fn record_deliberate_severance(
10871 snapshot: &SharedSnapshot,
10872 identity: ProcessIdentity,
10873) -> Result<(), SuperviseError> {
10874 update_snapshot(snapshot, None, |state| {
10875 state.deliberate_severance = Some(identity);
10876 })
10877}
10878
10879fn apply_deliberate_severance_marker(
10880 snapshot: &SharedSnapshot,
10881 exited_identity: Option<ProcessIdentity>,
10882 mut exit_report: ExitReport,
10883) -> ExitReport {
10884 let marker = lock_snapshot(snapshot)
10885 .ok()
10886 .and_then(|mut state| state.deliberate_severance.take());
10887 if marker.is_some() && marker == exited_identity {
10888 exit_report.kind = ExitKind::DeliberateSeverance;
10889 }
10890 exit_report
10891}
10892
10893fn classify_reaped_child_exit(
10894 snapshot: &SharedSnapshot,
10895 child: &SupervisedChild,
10896 status: &ExitStatus,
10897) -> ExitReport {
10898 let _ = update_snapshot(snapshot, None, |state| {
10899 state.reaped_pid = Some(child.pid);
10900 state.spawn_failure = child.spawn_failure.clone();
10901 });
10902 apply_deliberate_severance_marker(snapshot, child.process_identity(), classify_exit(status))
10903}
10904
10905fn fail_snapshot(
10906 snapshot: &SharedSnapshot,
10907 module_id: Option<&str>,
10908 last_exit: Option<ExitReport>,
10909) {
10910 if let Err(err) = update_snapshot(snapshot, module_id, |state| {
10911 state.state = ModuleState::Failed;
10912 clear_current_process_facts(state);
10913 if let Some(last_exit) = last_exit {
10914 state.last_exit = Some(last_exit);
10915 }
10916 }) {
10917 error!(error = %err, "failed to mark supervisor state failed");
10918 }
10919}
10920
10921fn update_snapshot(
10922 snapshot: &SharedSnapshot,
10923 module_id: Option<&str>,
10924 update: impl FnOnce(&mut SupervisorSnapshot),
10925) -> Result<(), SuperviseError> {
10926 let mut state = snapshot.lock().map_err(|_| SuperviseError::StatePoisoned {
10927 module_id: module_id.map(ToOwned::to_owned),
10928 })?;
10929 update(&mut state);
10930 Ok(())
10931}
10932
10933const SLOW_SNAPSHOT_LOCK_THRESHOLD: Duration = Duration::from_millis(250);
10934
10935fn lock_snapshot_for_control<'a>(
10936 snapshot: &'a SharedSnapshot,
10937 module_id: &str,
10938 caller: &'static str,
10939) -> Result<std::sync::MutexGuard<'a, SupervisorSnapshot>, SuperviseError> {
10940 let started_at = Instant::now();
10941 let guard = lock_snapshot(snapshot)?;
10942 let waited = started_at.elapsed();
10943 if waited >= SLOW_SNAPSHOT_LOCK_THRESHOLD {
10944 warn!(
10945 module_id = %module_id,
10946 waited_ms = waited.as_millis() as u64,
10947 caller = %caller,
10948 "slow snapshot lock"
10949 );
10950 }
10951 Ok(guard)
10952}
10953
10954fn lock_snapshot(
10955 snapshot: &SharedSnapshot,
10956) -> Result<std::sync::MutexGuard<'_, SupervisorSnapshot>, SuperviseError> {
10957 snapshot
10958 .lock()
10959 .map_err(|_| SuperviseError::StatePoisoned { module_id: None })
10960}
10961
10962#[cfg(test)]
10963mod terminal_history_tests {
10964 use std::{
10965 path::PathBuf,
10966 sync::Arc,
10967 time::{Duration, Instant},
10968 };
10969
10970 use tokio::{sync::mpsc, time::sleep};
10971
10972 use super::{
10973 apply_deliberate_severance_marker, daemon_will_restart, drain_child_to_state,
10974 drained_after_quiescence_wait, handle_reload_spawn_failure, health_restart_child,
10975 lock_snapshot, on_child_exit, record_deliberate_severance, record_wait_error_terminal,
10976 reset_restart_count, spawn_and_mark_running, update_snapshot, wait_error_exit_report,
10977 ExitKind, ExitReport, ModuleProtocol, ModuleSpec, ModuleState, NextAction, ProcessIdentity,
10978 RestartPolicy, SpawnEventKind, StopNotice, SuperviseError, SupervisedConfiguration,
10979 SupervisedModule, SupervisedModuleInner, Supervisor, SupervisorHandle,
10980 SupervisorHealthStatus, SupervisorSnapshot,
10981 };
10982 use super::Instant as ClockInstant;
10987 use crate::{
10988 registry::Registry,
10989 terminal_ring::{TerminalRing, TerminalRingConfig},
10990 };
10991 use std::sync::Mutex;
10992 use subc_control::TerminalDisposition;
10993
10994 pub(super) fn fake_aft_stub_path() -> PathBuf {
10999 let mut path = std::env::current_exe().expect("current_exe available in tests");
11000 path.pop();
11001 path.pop();
11002 path.push(if cfg!(windows) {
11003 "fake-aft-stub.exe"
11004 } else {
11005 "fake-aft-stub"
11006 });
11007 assert!(
11008 path.exists(),
11009 "fake-aft-stub not built at {}: run `cargo test -p subc-core` (which builds \
11010 [[bin]] targets) rather than `cargo test -p subc-core --lib` (which does not)",
11011 path.display()
11012 );
11013 path
11014 }
11015
11016 #[test]
11017 fn reserved_never_spawned_refuses_every_hello() {
11018 let supervisor = SupervisorHandle::default();
11023 supervisor.apply_identity_configuration(&ModuleSpec {
11024 module_id: "never-spawned".to_string(),
11025 program: PathBuf::from("/usr/bin/false"),
11026 args: Vec::new(),
11027 env: Vec::new(),
11028 reserved: true,
11029 reserved_prefixes: Vec::new(),
11030 protocol: ModuleProtocol::Subc,
11031 overlap: Default::default(),
11032 });
11033 assert!(
11034 supervisor
11035 .reserved_hello_rejection("never-spawned", Some("any-forged-nonce"))
11036 .is_some(),
11037 "forged nonce must refuse on a reserved never-spawned id"
11038 );
11039 assert!(
11040 supervisor
11041 .reserved_hello_rejection("never-spawned", None)
11042 .is_some(),
11043 "absent nonce must refuse on a reserved never-spawned id"
11044 );
11045 supervisor.set_spawn_nonce("never-spawned", "minted".to_string());
11047 supervisor.apply_identity_configuration(&ModuleSpec {
11048 module_id: "never-spawned".to_string(),
11049 program: PathBuf::from("/usr/bin/false"),
11050 args: Vec::new(),
11051 env: Vec::new(),
11052 reserved: true,
11053 reserved_prefixes: Vec::new(),
11054 protocol: ModuleProtocol::Subc,
11055 overlap: Default::default(),
11056 });
11057 assert!(supervisor
11058 .reserved_hello_rejection("never-spawned", Some("minted"))
11059 .is_none());
11060 assert!(supervisor
11061 .reserved_hello_rejection("never-spawned", Some("forged"))
11062 .is_some());
11063 }
11064
11065 fn seed_crash_restarts(state: &mut SupervisorSnapshot, count: u32) {
11068 let now = ClockInstant::now();
11069 for _ in 0..count {
11070 state.crash_restarts.push_back(now);
11071 }
11072 }
11073
11074 fn age_oldest_crash_restart_out_of_window(state: &mut SupervisorSnapshot, window: Duration) {
11078 let aged = state
11079 .crash_restarts
11080 .front()
11081 .expect("a crash restart must be recorded before it can be aged")
11082 .checked_sub(window + Duration::from_secs(1))
11083 .expect("the test clock is far enough from its origin to age an instant");
11084 state.crash_restarts[0] = aged;
11085 }
11086
11087 fn snapshot_with_restarts(enabled: bool, count: u32) -> SupervisorSnapshot {
11088 let mut state = SupervisorSnapshot::new(ModuleState::Running, enabled);
11089 seed_crash_restarts(&mut state, count);
11090 state
11091 }
11092
11093 #[test]
11094 fn daemon_owned_recovery_predicate_uses_the_pre_increment_budget() {
11095 let policy = RestartPolicy::new(3, Duration::ZERO);
11096 let now = ClockInstant::now();
11097 assert!(daemon_will_restart(
11098 &mut snapshot_with_restarts(true, 2),
11099 &policy,
11100 now
11101 ));
11102 assert!(!daemon_will_restart(
11103 &mut snapshot_with_restarts(true, 3),
11104 &policy,
11105 now
11106 ));
11107 assert!(!daemon_will_restart(
11108 &mut snapshot_with_restarts(false, 0),
11109 &policy,
11110 now
11111 ));
11112 }
11113
11114 #[test]
11115 fn crash_restart_backoff_escalates_with_in_window_count() {
11116 let policy = RestartPolicy::new(4, Duration::from_millis(100))
11117 .with_max_backoff(Duration::from_secs(30));
11118 let now = ClockInstant::now();
11119 let mut state = SupervisorSnapshot::new(ModuleState::Running, true);
11120 let schedules = (0..4)
11121 .map(|_| {
11122 state
11123 .next_crash_restart(&policy, now)
11124 .expect("the test policy allows four crash restarts")
11125 })
11126 .collect::<Vec<_>>();
11127
11128 assert_eq!(
11129 schedules
11130 .iter()
11131 .map(|schedule| schedule.restart_in_window)
11132 .collect::<Vec<_>>(),
11133 vec![0, 1, 2, 3]
11134 );
11135 assert_eq!(
11136 schedules
11137 .iter()
11138 .map(|schedule| schedule.delay)
11139 .collect::<Vec<_>>(),
11140 vec![
11141 Duration::from_millis(100),
11142 Duration::from_secs(1),
11143 Duration::from_secs(10),
11144 Duration::from_secs(30),
11145 ]
11146 );
11147 }
11148
11149 #[test]
11150 fn crash_restart_backoff_resets_after_ring_clear() {
11151 let policy = RestartPolicy::new(3, Duration::from_millis(100));
11152 let now = ClockInstant::now();
11153 let mut state = SupervisorSnapshot::new(ModuleState::Running, true);
11154 assert_eq!(
11155 state.next_crash_restart(&policy, now).unwrap().delay,
11156 Duration::from_millis(100)
11157 );
11158 assert_eq!(
11159 state.next_crash_restart(&policy, now).unwrap().delay,
11160 Duration::from_secs(1)
11161 );
11162
11163 state.clear_crash_restarts();
11164 let schedule = state
11165 .next_crash_restart(&policy, now)
11166 .expect("a cleared ring must allow another restart");
11167 assert_eq!(schedule.restart_in_window, 0);
11168 assert_eq!(schedule.delay, Duration::from_millis(100));
11169 }
11170
11171 #[test]
11172 fn crash_restart_backoff_ignores_aged_restarts() {
11173 let policy = RestartPolicy::new(3, Duration::from_millis(100));
11174 let now = ClockInstant::now();
11175 let mut state = SupervisorSnapshot::new(ModuleState::Running, true);
11176 state
11177 .next_crash_restart(&policy, now)
11178 .expect("the first restart is allowed");
11179 state
11180 .next_crash_restart(&policy, now)
11181 .expect("the second restart is allowed");
11182 state.crash_restarts[0] = now
11183 .checked_sub(policy.window + Duration::from_secs(1))
11184 .expect("the fake clock can age a restart past the window");
11185
11186 let schedule = state
11187 .next_crash_restart(&policy, now)
11188 .expect("an aged restart must release its slot");
11189 assert_eq!(schedule.restart_in_window, 1);
11190 assert_eq!(schedule.delay, Duration::from_secs(1));
11191 assert_eq!(state.crash_restarts.len(), 2);
11192 }
11193
11194 #[test]
11198 fn a_budget_spent_before_the_window_no_longer_refuses() {
11199 let policy = RestartPolicy::new(3, Duration::ZERO);
11200 let mut state = snapshot_with_restarts(true, 3);
11201 let now = ClockInstant::now();
11202 assert!(!daemon_will_restart(&mut state, &policy, now));
11203
11204 assert!(daemon_will_restart(
11205 &mut state,
11206 &policy,
11207 now + policy.window + Duration::from_secs(1)
11208 ));
11209 assert!(
11210 state.crash_restarts.is_empty(),
11211 "reading the budget must drop the instants that left the window"
11212 );
11213 }
11214
11215 fn module_with_recovery_snapshot(
11216 state: ModuleState,
11217 enabled: bool,
11218 restart_count: u32,
11219 ) -> SupervisedModule {
11220 let registry = Arc::new(Registry::default());
11221 let supervisor =
11222 Supervisor::new_for_test(Arc::clone(®istry), RestartPolicy::new(3, Duration::ZERO));
11223 let runtime = supervisor.runtime_config();
11224 let spec = ModuleSpec {
11225 module_id: "recovery-snapshot".to_string(),
11226 program: fake_aft_stub_path(),
11227 args: Vec::new(),
11228 env: Vec::new(),
11229 reserved: false,
11230 reserved_prefixes: Vec::new(),
11231 protocol: ModuleProtocol::Subc,
11232 overlap: Default::default(),
11233 };
11234 let mut snapshot = SupervisorSnapshot::new(state, enabled);
11235 seed_crash_restarts(&mut snapshot, restart_count);
11236 let (commands, _rx) = mpsc::channel(4);
11239 SupervisedModule {
11240 inner: Arc::new(SupervisedModuleInner {
11241 module_id: spec.module_id.clone(),
11242 registry,
11243 snapshot: Arc::new(Mutex::new(snapshot)),
11244 configuration: Arc::new(Mutex::new(SupervisedConfiguration {
11245 spec,
11246 health: runtime.health,
11247 })),
11248 stderr_ring: runtime.stderr_ring,
11249 terminal_ring: runtime.terminal_ring,
11250 commands,
11251 monitor: Mutex::new(None),
11252 restart_policy: runtime.restart_policy,
11253 effective_drain_timeout: runtime.effective_drain_timeout,
11254 provenance_probe: supervisor.provenance_probe.clone(),
11255 }),
11256 }
11257 }
11258
11259 #[cfg(target_os = "linux")]
11260 #[tokio::test]
11261 async fn no_cgroup_placement_does_not_block_fake_aft_stub_spawn() {
11262 let supervisor =
11263 Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default())
11264 .with_cgroup_placement(None);
11265 let result = supervisor.spawn(ModuleSpec {
11266 module_id: "no-cgroup-placement".to_string(),
11267 program: fake_aft_stub_path(),
11268 args: Vec::new(),
11269 env: Vec::new(),
11270 reserved: false,
11271 reserved_prefixes: Vec::new(),
11272 protocol: ModuleProtocol::Subc,
11273 overlap: Default::default(),
11274 });
11275
11276 assert!(
11277 result.is_ok(),
11278 "no delegation must not turn an otherwise valid spawn into a failure: {result:?}"
11279 );
11280 }
11281
11282 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
11283 async fn undecided_snapshot_uses_shared_restart_predicate() {
11284 assert!(module_with_recovery_snapshot(ModuleState::Running, true, 2)
11285 .will_recover_after_connection_loss()
11286 .unwrap());
11287 assert!(
11288 !module_with_recovery_snapshot(ModuleState::Running, true, 3)
11289 .will_recover_after_connection_loss()
11290 .unwrap()
11291 );
11292 }
11293
11294 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
11295 async fn restarting_snapshot_at_exhausted_budget_is_non_terminal() {
11296 assert!(
11297 module_with_recovery_snapshot(ModuleState::Restarting, true, 3)
11298 .will_recover_after_connection_loss()
11299 .unwrap()
11300 );
11301 }
11302
11303 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
11304 async fn terminal_phase_snapshots_are_terminal_before_budget_exhaustion() {
11305 assert!(!module_with_recovery_snapshot(ModuleState::Failed, true, 0)
11306 .will_recover_after_connection_loss()
11307 .unwrap());
11308 assert!(
11309 !module_with_recovery_snapshot(ModuleState::Disabled, true, 0)
11310 .will_recover_after_connection_loss()
11311 .unwrap()
11312 );
11313 }
11314
11315 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
11316 async fn warming_snapshot_is_limited_to_startup_phases() {
11317 for state in [
11318 ModuleState::Starting,
11319 ModuleState::Running,
11320 ModuleState::Restarting,
11321 ] {
11322 assert!(
11323 module_with_recovery_snapshot(state, true, 0)
11324 .is_warming()
11325 .unwrap(),
11326 "{state:?} should be warming"
11327 );
11328 }
11329 for state in [
11330 ModuleState::Unresponsive,
11331 ModuleState::Draining,
11332 ModuleState::Stopped,
11333 ModuleState::Failed,
11334 ModuleState::Disabled,
11335 ] {
11336 assert!(
11337 !module_with_recovery_snapshot(state, true, 0)
11338 .is_warming()
11339 .unwrap(),
11340 "{state:?} should not be warming"
11341 );
11342 }
11343 }
11344
11345 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
11346 async fn terminal_history_survives_respawn_and_keeps_both_crashes_in_order() {
11347 let registry = Arc::new(Registry::default());
11348 let supervisor =
11349 Supervisor::new_for_test(Arc::clone(®istry), RestartPolicy::new(1, Duration::ZERO));
11350 let module = supervisor
11351 .spawn(ModuleSpec {
11352 module_id: "terminal-history".to_string(),
11353 program: fake_aft_stub_path(),
11354 args: Vec::new(),
11355 env: vec![("FAKE_AFT_EXIT_CODE".to_string(), "23".to_string())],
11356 reserved: false,
11357 reserved_prefixes: Vec::new(),
11358 protocol: ModuleProtocol::Subc,
11359 overlap: Default::default(),
11360 })
11361 .unwrap();
11362
11363 let deadline = Instant::now() + Duration::from_secs(5);
11364 loop {
11365 let history = module.terminal_history();
11366 if history.entries.len() == 2 {
11367 assert_eq!(module.status().unwrap().state, ModuleState::Failed);
11368 assert_eq!(history.dropped, 0);
11369 assert_eq!(
11370 history
11371 .entries
11372 .iter()
11373 .map(|entry| entry.exit_code)
11374 .collect::<Vec<_>>(),
11375 vec![Some(23), Some(23)]
11376 );
11377 assert!(history.entries[0].at_ms <= history.entries[1].at_ms);
11378 return;
11379 }
11380 assert!(
11381 Instant::now() < deadline,
11382 "module did not retain two terminal exits: {history:?}"
11383 );
11384 sleep(Duration::from_millis(10)).await;
11385 }
11386 }
11387
11388 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
11392 async fn disable_during_crash_backoff_cancels_pending_respawn() {
11393 let backoff = Duration::from_secs(2);
11394 let supervisor = Supervisor::new_for_test(
11395 Arc::new(Registry::default()),
11396 RestartPolicy::new(10, backoff),
11397 );
11398 let module = supervisor
11399 .spawn(ModuleSpec {
11400 module_id: "disable-during-backoff".to_string(),
11401 program: fake_aft_stub_path(),
11402 args: Vec::new(),
11403 env: vec![("FAKE_AFT_EXIT_CODE".to_string(), "23".to_string())],
11404 reserved: false,
11405 reserved_prefixes: Vec::new(),
11406 protocol: ModuleProtocol::Subc,
11407 overlap: Default::default(),
11408 })
11409 .unwrap();
11410
11411 let deadline = Instant::now() + Duration::from_secs(5);
11413 loop {
11414 if module.status().unwrap().state == ModuleState::Restarting {
11415 break;
11416 }
11417 assert!(
11418 Instant::now() < deadline,
11419 "module never entered the crash backoff"
11420 );
11421 sleep(Duration::from_millis(10)).await;
11422 }
11423
11424 let started = Instant::now();
11425 module.set_enabled(false).await.unwrap();
11426 let waited = started.elapsed();
11427
11428 assert!(
11429 waited < backoff / 2,
11430 "disable waited {waited:?} behind the {backoff:?} crash backoff; the operator command must preempt the pending respawn"
11431 );
11432 assert_eq!(module.status().unwrap().state, ModuleState::Disabled);
11433
11434 sleep(backoff + Duration::from_millis(500)).await;
11436 let status = module.status().unwrap();
11437 assert_eq!(status.state, ModuleState::Disabled);
11438 assert_eq!(
11439 status.spawn_generation, 1,
11440 "module respawned after the operator disabled it"
11441 );
11442 }
11443
11444 #[cfg(unix)]
11447 fn protocol_none_sigterm_exits_clean_spec(
11448 module_id: &str,
11449 dir: &std::path::Path,
11450 ) -> (ModuleSpec, PathBuf, PathBuf) {
11451 let ready = dir.join("ready");
11452 let marker = dir.join("sigterm");
11453 let spec = ModuleSpec {
11454 module_id: module_id.to_string(),
11455 program: fake_aft_stub_path(),
11456 args: Vec::new(),
11457 env: vec![
11458 ("FAKE_AFT_NEVER_CONNECT".to_string(), "1".to_string()),
11459 (
11460 "FAKE_AFT_SIGTERM_MARKER_PATH".to_string(),
11461 marker.display().to_string(),
11462 ),
11463 (
11464 "FAKE_AFT_NEVER_CONNECT_READY_PATH".to_string(),
11465 ready.display().to_string(),
11466 ),
11467 ],
11468 reserved: false,
11469 reserved_prefixes: Vec::new(),
11470 protocol: ModuleProtocol::None,
11471 overlap: Default::default(),
11472 };
11473 (spec, ready, marker)
11474 }
11475
11476 #[cfg(unix)]
11480 async fn wait_for_file(path: &std::path::Path) {
11481 let deadline = Instant::now() + Duration::from_secs(10);
11482 while !path.exists() {
11483 assert!(
11484 Instant::now() < deadline,
11485 "{} never appeared",
11486 path.display()
11487 );
11488 sleep(Duration::from_millis(10)).await;
11489 }
11490 }
11491
11492 #[cfg(unix)]
11496 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
11497 async fn protocol_none_unrequested_clean_exit_restarts_as_a_crash() {
11498 let dir = cortexkit_test_support::ScratchDir::new("none-unrequested-clean-exit");
11499 let (spec, ready, marker) =
11500 protocol_none_sigterm_exits_clean_spec("none-unrequested-clean-exit", dir.path());
11501 let supervisor = Supervisor::new_for_test(
11502 Arc::new(Registry::default()),
11503 RestartPolicy::new(3, Duration::ZERO),
11504 );
11505 let module = supervisor.spawn(spec).unwrap();
11506 wait_for_file(&ready).await;
11507 let deadline = Instant::now() + Duration::from_secs(10);
11511 let first_pid = loop {
11512 let status = module.status().unwrap();
11513 if status.state == ModuleState::Running {
11514 if let Some(pid) = status.pid {
11515 break pid;
11516 }
11517 }
11518 assert!(
11519 Instant::now() < deadline,
11520 "a running module must report its pid after exec confirmation: {status:?}"
11521 );
11522 sleep(Duration::from_millis(10)).await;
11523 };
11524
11525 rustix::process::kill_process(
11526 rustix::process::Pid::from_raw(i32::try_from(first_pid).unwrap()).unwrap(),
11527 rustix::process::Signal::TERM,
11528 )
11529 .unwrap();
11530
11531 let deadline = Instant::now() + Duration::from_secs(10);
11532 let respawned = loop {
11533 let status = module.status().unwrap();
11534 if status.state == ModuleState::Running
11535 && status.pid.is_some_and(|pid| pid != first_pid)
11536 {
11537 break status;
11538 }
11539 assert!(
11540 Instant::now() < deadline,
11541 "protocol-none module was not respawned after an unrequested clean exit: {status:?}"
11542 );
11543 sleep(Duration::from_millis(10)).await;
11544 };
11545 assert_eq!(respawned.spawn_generation, 2);
11546 assert!(
11547 marker.exists(),
11548 "the child must have exited through its SIGTERM handler (exit 0), or this proves nothing about clean exits"
11549 );
11550
11551 let history = module.terminal_history();
11552 assert_eq!(history.entries.len(), 1, "{history:?}");
11553 let entry = &history.entries[0];
11554 assert_eq!(entry.exit_code, Some(0));
11555 assert_eq!(entry.exit_kind, subc_control::TerminalExitKind::Clean);
11556 assert_eq!(entry.disposition, TerminalDisposition::Restarting);
11557
11558 module.stop().await.unwrap();
11559 }
11560
11561 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
11565 async fn protocol_none_repeated_clean_exits_exhaust_the_restart_budget() {
11566 let supervisor = Supervisor::new_for_test(
11567 Arc::new(Registry::default()),
11568 RestartPolicy::new(1, Duration::ZERO),
11569 );
11570 let module = supervisor
11571 .spawn(ModuleSpec {
11572 module_id: "none-clean-exit-budget".to_string(),
11573 program: fake_aft_stub_path(),
11574 args: Vec::new(),
11575 env: vec![("FAKE_AFT_EXIT_CODE".to_string(), "0".to_string())],
11576 reserved: false,
11577 reserved_prefixes: Vec::new(),
11578 protocol: ModuleProtocol::None,
11579 overlap: Default::default(),
11580 })
11581 .unwrap();
11582
11583 let deadline = Instant::now() + Duration::from_secs(10);
11586 loop {
11587 let status = module.status().unwrap();
11588 if status.state == ModuleState::Failed {
11589 break;
11590 }
11591 assert!(
11592 Instant::now() < deadline,
11593 "module never exhausted its budget: {status:?} {:?}",
11594 module.terminal_history()
11595 );
11596 sleep(Duration::from_millis(10)).await;
11597 }
11598 let history = module.terminal_history();
11599 assert_eq!(
11600 history
11601 .entries
11602 .iter()
11603 .map(|entry| (entry.exit_code, entry.disposition.clone()))
11604 .collect::<Vec<_>>(),
11605 vec![
11606 (Some(0), TerminalDisposition::Restarting),
11607 (Some(0), TerminalDisposition::Failed),
11608 ]
11609 );
11610 let detail = history.entries[1]
11611 .disposition_detail
11612 .as_deref()
11613 .expect("a budget failure names the budget");
11614 assert!(detail.contains("max_restarts=1"), "{detail}");
11615 assert_eq!(module.status().unwrap().spawn_generation, 2);
11616 }
11617
11618 #[test]
11619 fn restart_budget_failure_is_published_after_its_terminal_record() {
11620 use std::io::{Read, Write};
11621 let supervisor = Supervisor::new_for_test(
11622 Arc::new(Registry::default()),
11623 RestartPolicy::new(0, Duration::ZERO),
11624 );
11625 let runtime = supervisor.runtime_config();
11626 let spec = ModuleSpec {
11627 protocol: ModuleProtocol::None,
11628 ..windowed_crash_spec("budget-publication-order")
11629 };
11630 let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::new(
11631 ModuleState::Running,
11632 true,
11633 )));
11634 lock_snapshot(&snapshot).unwrap().process_alive = true;
11635 let reaped_snapshot = snapshot.clone();
11636 let mut barrier = terminal_record_barrier(&runtime.spawn_events);
11637 let history = runtime.terminal_ring.clone();
11638 let reap = std::thread::spawn(move || {
11639 tokio::runtime::Builder::new_current_thread()
11640 .enable_all()
11641 .build()
11642 .unwrap()
11643 .block_on(on_child_exit(
11644 &spec,
11645 runtime.restart_policy,
11646 &supervisor.registry,
11647 &reaped_snapshot,
11648 &runtime.terminal_ring,
11649 &runtime.spawn_events,
11650 &runtime.child_roster,
11651 ExitReport {
11652 kind: ExitKind::Clean,
11653 code: Some(0),
11654 signal: None,
11655 at_ms: 1,
11656 },
11657 ))
11658 });
11659 let mut reached = [0];
11660 barrier.read_exact(&mut reached).unwrap();
11661 assert_eq!(&reached, b"R");
11662 let (before_state, before_alive) = {
11663 let state = lock_snapshot(&snapshot).unwrap();
11664 (state.state, state.process_alive)
11665 };
11666 let before_history = history.lock().unwrap().snapshot();
11667 barrier.write_all(b"X").unwrap();
11668 assert!(matches!(reap.join().unwrap(), NextAction::Stop { .. }));
11669 assert!(before_history.entries.is_empty());
11670 assert_ne!(
11671 before_state,
11672 ModuleState::Failed,
11673 "Failed was visible before its terminal record could be written"
11674 );
11675 assert!(before_alive);
11676 assert_eq!(lock_snapshot(&snapshot).unwrap().state, ModuleState::Failed);
11677 let history = history.lock().unwrap().snapshot();
11678 assert_eq!(history.entries.len(), 1);
11679 assert_eq!(history.entries[0].exit_code, Some(0));
11680 assert_eq!(history.entries[0].disposition, TerminalDisposition::Failed);
11681 }
11682
11683 #[test]
11684 fn crash_restart_at_budget_limit_remains_recoverable_while_recording() {
11685 use std::io::{Read, Write};
11686
11687 let supervisor = Supervisor::new_for_test(
11688 Arc::new(Registry::default()),
11689 RestartPolicy::new(3, Duration::ZERO),
11690 );
11691 let mut runtime = supervisor.runtime_config();
11692 let module = module_with_recovery_snapshot(ModuleState::Running, true, 2);
11693 let snapshot = module.inner.snapshot.clone();
11694 lock_snapshot(&snapshot).unwrap().process_alive = true;
11695 runtime.terminal_ring = module.inner.terminal_ring.clone();
11696 let spec = windowed_crash_spec(module.module_id());
11697 let mut barrier = terminal_record_barrier(&runtime.spawn_events);
11698 let reaped_snapshot = snapshot.clone();
11699 let reap = std::thread::spawn(move || {
11700 tokio::runtime::Builder::new_current_thread()
11701 .enable_all()
11702 .build()
11703 .unwrap()
11704 .block_on(on_child_exit(
11705 &spec,
11706 runtime.restart_policy,
11707 &supervisor.registry,
11708 &reaped_snapshot,
11709 &runtime.terminal_ring,
11710 &runtime.spawn_events,
11711 &runtime.child_roster,
11712 ExitReport {
11713 kind: ExitKind::Crash,
11714 code: Some(2),
11715 signal: None,
11716 at_ms: 1,
11717 },
11718 ))
11719 });
11720
11721 let mut reached = [0];
11725 barrier.read_exact(&mut reached).unwrap();
11726 assert_eq!(&reached, b"R");
11727 let before_status = module.status().unwrap();
11728 let before_history = module.terminal_history();
11729 let before_recovers = module.will_recover_after_connection_loss().unwrap();
11730 barrier.write_all(b"X").unwrap();
11731 assert!(matches!(reap.join().unwrap(), NextAction::Restart { .. }));
11732
11733 assert_eq!(before_status.state, ModuleState::Running);
11734 assert!(before_status.process_alive);
11735 assert_eq!(before_status.restart_count, 3);
11736 assert!(before_history.entries.is_empty());
11737 assert!(
11738 before_recovers,
11739 "connection cleanup would report a terminal close for a scheduled crash restart"
11740 );
11741 let restarting = module.status().unwrap();
11742 assert_eq!(restarting.state, ModuleState::Restarting);
11743 assert_eq!(restarting.restart_count, 3);
11744 assert!(module.will_recover_after_connection_loss().unwrap());
11745 let history = module.terminal_history();
11746 assert_eq!(history.entries.len(), 1);
11747 assert_eq!(
11748 history.entries[0].disposition,
11749 TerminalDisposition::Restarting
11750 );
11751
11752 update_snapshot(&snapshot, Some(module.module_id()), |state| {
11755 state.state = ModuleState::Running;
11756 state.respawn_pending = false;
11757 })
11758 .unwrap();
11759 assert!(!module.will_recover_after_connection_loss().unwrap());
11760 update_snapshot(&snapshot, Some(module.module_id()), |state| {
11761 state.enabled = false;
11762 state.respawn_pending = true;
11763 })
11764 .unwrap();
11765 assert!(!module.will_recover_after_connection_loss().unwrap());
11766 }
11767
11768 fn terminal_record_barrier(events: &super::SpawnEventFeed) -> std::net::TcpStream {
11769 use std::net::{TcpListener, TcpStream};
11770 let listener = TcpListener::bind("127.0.0.1:0").unwrap();
11771 let writer = TcpStream::connect(listener.local_addr().unwrap()).unwrap();
11772 let (barrier, _) = listener.accept().unwrap();
11773 for stream in [&writer, &barrier] {
11774 stream
11775 .set_read_timeout(Some(Duration::from_secs(10)))
11776 .unwrap();
11777 stream
11778 .set_write_timeout(Some(Duration::from_secs(10)))
11779 .unwrap();
11780 }
11781 *events.1.lock().unwrap() = Some(writer);
11782 barrier
11783 }
11784
11785 pub(super) fn before_terminal_record(events: &super::SpawnEventFeed) {
11786 use std::io::{Read, Write};
11787 let barrier = events.1.lock().unwrap().take();
11788 if let Some(mut barrier) = barrier {
11789 barrier.write_all(b"R").unwrap();
11790 let mut release = [0];
11791 barrier.read_exact(&mut release).unwrap();
11792 assert_eq!(&release, b"X");
11793 }
11794 }
11795
11796 #[test]
11797 fn health_restart_budget_failure_is_published_after_its_terminal_record() {
11798 use std::io::{Read, Write};
11799
11800 let supervisor = Supervisor::new_for_test(
11801 Arc::new(Registry::default()),
11802 RestartPolicy::new(0, Duration::ZERO),
11803 );
11804 let mut runtime = supervisor.runtime_config();
11805 let module = module_with_recovery_snapshot(ModuleState::Running, true, 0);
11806 let snapshot = module.inner.snapshot.clone();
11807 runtime.terminal_ring = module.inner.terminal_ring.clone();
11808 let spec = ModuleSpec {
11809 module_id: module.module_id().into(),
11810 program: fake_aft_stub_path(),
11811 protocol: ModuleProtocol::None,
11812 env: vec![("FAKE_AFT_EXIT_CODE".into(), "17".into())],
11813 ..windowed_crash_spec(module.module_id())
11814 };
11815 let mut barrier = terminal_record_barrier(&runtime.spawn_events);
11816 let reap = std::thread::spawn(move || {
11817 tokio::runtime::Builder::new_current_thread()
11818 .enable_all()
11819 .build()
11820 .unwrap()
11821 .block_on(async {
11822 let mut child =
11823 Some(spawn_and_mark_running(&spec, &runtime, &snapshot).unwrap());
11824 health_restart_child(
11825 &spec,
11826 &runtime,
11827 &supervisor.registry,
11828 &supervisor.process_liveness,
11829 &snapshot,
11830 &mut child,
11831 SupervisorHealthStatus::Failing,
11832 Some("health probe did not answer"),
11833 1,
11834 )
11835 .await
11836 .unwrap();
11837 assert!(child.is_none());
11838 });
11839 });
11840 let mut reached = [0];
11845 barrier.read_exact(&mut reached).unwrap();
11846 assert_eq!(&reached, b"R");
11847 let before_status = module.status().unwrap();
11848 let before_history = module.terminal_history();
11849 barrier.write_all(b"X").unwrap();
11850 reap.join().unwrap();
11851 assert!(before_history.entries.is_empty());
11852 assert_ne!(
11853 before_status.state,
11854 ModuleState::Failed,
11855 "Failed was visible before its terminal record could be written"
11856 );
11857 assert!(
11858 before_status.process_alive,
11859 "process death was visible before its terminal record could be written"
11860 );
11861 let failed = module.status().unwrap();
11862 assert_eq!(failed.state, ModuleState::Failed);
11863 assert!(!failed.process_alive);
11864 let history = module.terminal_history();
11865 assert_eq!(history.entries.len(), 1);
11866 assert_eq!(history.entries[0].disposition, TerminalDisposition::Failed);
11867 assert_eq!(
11868 history.entries[0].disposition_detail.as_deref(),
11869 Some("crash budget exhausted: max_restarts=0 within window_secs=600")
11870 );
11871 }
11872
11873 #[test]
11874 fn observed_exits_publish_process_death_after_terminal_history() {
11875 use std::io::{Read, Write};
11876 for (kind, enabled, shutdown, wait_error, expected_state) in [
11877 (ExitKind::Clean, true, false, false, ModuleState::Stopped),
11878 (ExitKind::Crash, true, false, false, ModuleState::Restarting),
11879 (ExitKind::Crash, false, false, false, ModuleState::Disabled),
11880 (
11881 ExitKind::DeliberateSeverance,
11882 true,
11883 false,
11884 false,
11885 ModuleState::Restarting,
11886 ),
11887 #[cfg(unix)]
11888 (ExitKind::Crash, true, true, false, ModuleState::Stopped),
11889 (ExitKind::Crash, true, false, true, ModuleState::Failed),
11890 ] {
11891 let supervisor = Supervisor::new_for_test(
11892 Arc::new(Registry::default()),
11893 RestartPolicy::new(1, Duration::ZERO),
11894 );
11895 let runtime = supervisor.runtime_config();
11896 #[cfg(unix)]
11897 if shutdown {
11898 runtime.child_roster.close();
11899 }
11900 #[cfg(not(unix))]
11901 let _ = shutdown;
11902 let spec = windowed_crash_spec("exit-publication-order");
11903 let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::new(
11904 ModuleState::Running,
11905 enabled,
11906 )));
11907 lock_snapshot(&snapshot).unwrap().process_alive = true;
11908 let reaped_snapshot = snapshot.clone();
11909 let history = runtime.terminal_ring.clone();
11910 let mut barrier = terminal_record_barrier(&runtime.spawn_events);
11911 let reap = std::thread::spawn(move || {
11912 if wait_error {
11913 super::fail_after_wait_error(
11914 &spec.module_id,
11915 &reaped_snapshot,
11916 &runtime.terminal_ring,
11917 &runtime.spawn_events,
11918 "wait failed".into(),
11919 );
11920 } else {
11921 tokio::runtime::Builder::new_current_thread()
11922 .enable_all()
11923 .build()
11924 .unwrap()
11925 .block_on(on_child_exit(
11926 &spec,
11927 runtime.restart_policy,
11928 &supervisor.registry,
11929 &reaped_snapshot,
11930 &runtime.terminal_ring,
11931 &runtime.spawn_events,
11932 &runtime.child_roster,
11933 ExitReport {
11934 kind,
11935 code: Some(0),
11936 signal: None,
11937 at_ms: 1,
11938 },
11939 ));
11940 }
11941 });
11942 let mut reached = [0];
11943 barrier.read_exact(&mut reached).unwrap();
11944 assert_eq!(&reached, b"R");
11945 let before_alive = lock_snapshot(&snapshot).unwrap().process_alive;
11946 let before_history = history.lock().unwrap().snapshot();
11947 barrier.write_all(b"X").unwrap();
11948 reap.join().unwrap();
11949 assert!(
11950 before_alive,
11951 "{kind:?}: death was published before recording"
11952 );
11953 assert!(before_history.entries.is_empty());
11954 assert_eq!(lock_snapshot(&snapshot).unwrap().state, expected_state);
11955 assert!(!lock_snapshot(&snapshot).unwrap().process_alive);
11956 assert_eq!(history.lock().unwrap().snapshot().entries.len(), 1);
11957 }
11958 }
11959
11960 #[tokio::test]
11961 async fn health_restart_budget_failure_remains_failed_when_journal_append_fails() {
11962 let dir = cortexkit_test_support::ScratchDir::new("health-budget-journal-failure");
11963 let path = dir.join("terminals.jsonl");
11964 std::fs::create_dir(&path).unwrap();
11965 let supervisor = Supervisor::new_for_test(
11966 Arc::new(Registry::default()),
11967 RestartPolicy::new(0, Duration::ZERO),
11968 )
11969 .with_terminal_journal(path, "health-budget-journal-failure".into());
11970 let runtime = supervisor.runtime_config();
11971 let spec = ModuleSpec {
11972 program: fake_aft_stub_path(),
11973 protocol: ModuleProtocol::None,
11974 env: vec![("FAKE_AFT_EXIT_CODE".into(), "17".into())],
11975 ..windowed_crash_spec("health-budget-journal-failure")
11976 };
11977 let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::starting()));
11978 let mut child = Some(spawn_and_mark_running(&spec, &runtime, &snapshot).unwrap());
11979 health_restart_child(
11980 &spec,
11981 &runtime,
11982 &supervisor.registry,
11983 &supervisor.process_liveness,
11984 &snapshot,
11985 &mut child,
11986 SupervisorHealthStatus::Failing,
11987 None,
11988 1,
11989 )
11990 .await
11991 .unwrap();
11992 assert_eq!(lock_snapshot(&snapshot).unwrap().state, ModuleState::Failed);
11993 assert!(!lock_snapshot(&snapshot).unwrap().process_alive);
11994 let history = runtime
11995 .terminal_ring
11996 .lock()
11997 .unwrap()
11998 .durable_history(&spec.module_id);
11999 assert!(history.journal_write_failures > 0);
12000 assert_eq!(history.entries.len(), 1);
12001 assert_eq!(history.entries[0].disposition, TerminalDisposition::Failed);
12002 }
12003
12004 #[test]
12005 fn health_restart_budget_failure_remains_failed_when_exit_recording_panics() {
12006 let supervisor = Supervisor::new_for_test(
12007 Arc::new(Registry::default()),
12008 RestartPolicy::new(0, Duration::ZERO),
12009 );
12010 let runtime = supervisor.runtime_config();
12011 let spec = ModuleSpec {
12012 program: fake_aft_stub_path(),
12013 protocol: ModuleProtocol::None,
12014 env: vec![("FAKE_AFT_EXIT_CODE".into(), "17".into())],
12015 ..windowed_crash_spec("health-budget-recording-panic")
12016 };
12017 let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::starting()));
12018 let result = std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| {
12019 tokio::runtime::Builder::new_current_thread()
12020 .enable_all()
12021 .build()
12022 .unwrap()
12023 .block_on(async {
12024 let mut child =
12025 Some(spawn_and_mark_running(&spec, &runtime, &snapshot).unwrap());
12026 runtime.spawn_events.0.lock().unwrap().seq = u64::MAX;
12027 health_restart_child(
12028 &spec,
12029 &runtime,
12030 &supervisor.registry,
12031 &supervisor.process_liveness,
12032 &snapshot,
12033 &mut child,
12034 SupervisorHealthStatus::Failing,
12035 None,
12036 1,
12037 )
12038 .await
12039 .unwrap();
12040 });
12041 }));
12042 let panic = result.expect_err("recording must still unwind");
12043 assert_eq!(
12044 panic.downcast_ref::<String>().map(String::as_str),
12045 Some("spawn event sequence exhausted")
12046 );
12047 let state = lock_snapshot(&snapshot).unwrap();
12048 assert_eq!(state.state, ModuleState::Failed);
12049 assert!(!state.process_alive);
12050 assert!(state.last_exit.is_some());
12051 }
12052
12053 #[tokio::test]
12054 async fn restart_budget_failure_remains_failed_when_journal_append_fails() {
12055 let dir = cortexkit_test_support::ScratchDir::new("budget-journal-failure");
12056 let path = dir.join("terminals.jsonl");
12057 std::fs::create_dir(&path).unwrap();
12058 let supervisor = Supervisor::new_for_test(
12059 Arc::new(Registry::default()),
12060 RestartPolicy::new(0, Duration::ZERO),
12061 )
12062 .with_terminal_journal(path, "budget-journal-failure".into());
12063 let runtime = supervisor.runtime_config();
12064 let spec = windowed_crash_spec("budget-journal-failure");
12065 let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::starting()));
12066 assert!(matches!(
12067 on_child_exit(
12068 &spec,
12069 runtime.restart_policy,
12070 &supervisor.registry,
12071 &snapshot,
12072 &runtime.terminal_ring,
12073 &runtime.spawn_events,
12074 &runtime.child_roster,
12075 crash_exit_report(1),
12076 )
12077 .await,
12078 NextAction::Stop { .. }
12079 ));
12080 assert_eq!(lock_snapshot(&snapshot).unwrap().state, ModuleState::Failed);
12081 let history = runtime
12082 .terminal_ring
12083 .lock()
12084 .unwrap()
12085 .durable_history(&spec.module_id);
12086 assert!(history.journal_write_failures > 0);
12087 assert_eq!(history.entries.len(), 1);
12088 assert_eq!(history.entries[0].disposition, TerminalDisposition::Failed);
12089 }
12090
12091 #[test]
12092 fn restart_budget_failure_remains_failed_when_exit_recording_panics() {
12093 let supervisor = Supervisor::new_for_test(
12094 Arc::new(Registry::default()),
12095 RestartPolicy::new(0, Duration::ZERO),
12096 );
12097 let runtime = supervisor.runtime_config();
12098 let spec = windowed_crash_spec("budget-recording-panic");
12099 let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::starting()));
12100 runtime.spawn_events.emit_spawned(&spec.module_id, 1, 1);
12101 runtime.spawn_events.0.lock().unwrap().seq = u64::MAX;
12104 let result = std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| {
12105 tokio::runtime::Builder::new_current_thread()
12106 .enable_all()
12107 .build()
12108 .unwrap()
12109 .block_on(on_child_exit(
12110 &spec,
12111 runtime.restart_policy,
12112 &supervisor.registry,
12113 &snapshot,
12114 &runtime.terminal_ring,
12115 &runtime.spawn_events,
12116 &runtime.child_roster,
12117 crash_exit_report(1),
12118 ))
12119 }));
12120 let panic = result.err().expect("recording must still unwind");
12121 assert_eq!(
12122 panic.downcast_ref::<String>().map(String::as_str),
12123 Some("spawn event sequence exhausted")
12124 );
12125 assert_eq!(lock_snapshot(&snapshot).unwrap().state, ModuleState::Failed);
12126 }
12127
12128 #[cfg(unix)]
12131 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
12132 async fn protocol_none_requested_stop_and_disable_do_not_respawn() {
12133 for disable in [false, true] {
12134 let label = if disable {
12135 "none-requested-disable"
12136 } else {
12137 "none-requested-stop"
12138 };
12139 let dir = cortexkit_test_support::ScratchDir::new(label);
12140 let (spec, ready, marker) = protocol_none_sigterm_exits_clean_spec(label, dir.path());
12141 let supervisor = Supervisor::new_for_test(
12142 Arc::new(Registry::default()),
12143 RestartPolicy::new(3, Duration::ZERO),
12144 );
12145 let module = supervisor.spawn(spec).unwrap();
12146 wait_for_file(&ready).await;
12147
12148 if disable {
12149 module.set_enabled(false).await.unwrap();
12150 } else {
12151 module.stop().await.unwrap();
12152 }
12153 assert!(
12154 marker.exists(),
12155 "{label}: the child must have left through its SIGTERM handler with exit 0"
12156 );
12157
12158 sleep(Duration::from_millis(500)).await;
12161 let status = module.status().unwrap();
12162 let expected = if disable {
12163 ModuleState::Disabled
12164 } else {
12165 ModuleState::Stopped
12166 };
12167 assert_eq!(status.state, expected, "{label}");
12168 assert_eq!(
12169 status.spawn_generation, 1,
12170 "{label}: respawned after a requested stop"
12171 );
12172 let history = module.terminal_history();
12173 assert_eq!(history.entries.len(), 1, "{label}: {history:?}");
12174 assert_eq!(history.entries[0].exit_code, Some(0), "{label}");
12175 assert_ne!(
12176 history.entries[0].disposition,
12177 TerminalDisposition::Restarting,
12178 "{label}"
12179 );
12180 }
12181 }
12182
12183 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
12186 async fn subc_wire_clean_exit_is_still_a_stop() {
12187 let supervisor = Supervisor::new_for_test(
12188 Arc::new(Registry::default()),
12189 RestartPolicy::new(3, Duration::ZERO),
12190 );
12191 let module = supervisor
12192 .spawn(ModuleSpec {
12193 module_id: "wire-clean-exit".to_string(),
12194 program: fake_aft_stub_path(),
12195 args: Vec::new(),
12196 env: vec![("FAKE_AFT_EXIT_CODE".to_string(), "0".to_string())],
12197 reserved: false,
12198 reserved_prefixes: Vec::new(),
12199 protocol: ModuleProtocol::Subc,
12200 overlap: Default::default(),
12201 })
12202 .unwrap();
12203
12204 let deadline = Instant::now() + Duration::from_secs(10);
12205 while module.terminal_history().entries.is_empty() {
12206 assert!(Instant::now() < deadline, "module never exited");
12207 sleep(Duration::from_millis(10)).await;
12208 }
12209 sleep(Duration::from_millis(500)).await;
12211 let status = module.status().unwrap();
12212 assert_eq!(status.state, ModuleState::Stopped);
12213 assert_eq!(status.spawn_generation, 1);
12214 let history = module.terminal_history();
12215 assert_eq!(history.entries.len(), 1, "{history:?}");
12216 assert_eq!(history.entries[0].exit_code, Some(0));
12217 assert_eq!(history.entries[0].disposition, TerminalDisposition::Stopped);
12218 }
12219
12220 #[cfg(unix)]
12221 #[tokio::test]
12222 async fn reload_exit_keeps_roster_until_terminal_is_recorded_during_shutdown() {
12223 let dir = cortexkit_test_support::ScratchDir::new("reload-roster-terminal-order");
12224 let record = dir.join("live-children.json");
12225 let supervisor = Supervisor::new_for_test(
12226 Arc::new(Registry::default()),
12227 RestartPolicy::new(0, Duration::ZERO),
12228 );
12229 let mut runtime = supervisor.runtime_config();
12230 runtime.child_roster.record_to(record.clone());
12231 let gate = Arc::new(super::ReloadExitRecordGate::default());
12232 runtime.test_reload_exit_record_gate = Some(Arc::clone(&gate));
12233 let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::starting()));
12234 let spec = ModuleSpec {
12235 module_id: "reload-exit-roster".into(),
12236 program: fake_aft_stub_path(),
12237 args: Vec::new(),
12238 env: vec![("FAKE_AFT_EXIT_CODE".into(), "23".into())],
12239 reserved: false,
12240 reserved_prefixes: Vec::new(),
12241 protocol: ModuleProtocol::Subc,
12242 overlap: Default::default(),
12243 };
12244 let mut child = None;
12245 let reload = super::finish_reload_child(
12246 &spec,
12247 &runtime,
12248 &supervisor.registry,
12249 &supervisor.process_liveness,
12250 &snapshot,
12251 &mut child,
12252 );
12253 tokio::pin!(reload);
12254 tokio::select! {
12255 result = &mut reload => panic!("reload missed the before-record gate: {result:?}"),
12256 _ = gate.reached.notified() => {}
12257 }
12258 assert!(runtime
12259 .terminal_ring
12260 .lock()
12261 .unwrap()
12262 .snapshot()
12263 .entries
12264 .is_empty());
12265 assert_eq!(
12266 crate::live_children::read_record(&record).unwrap().len(),
12267 1,
12268 "shutdown must still wait for the reaped child until its terminal record exists"
12269 );
12270 runtime.child_roster.close();
12271 gate.resume.notify_one();
12272 assert!(reload.await.is_err());
12273 assert!(crate::live_children::read_record(&record)
12274 .unwrap()
12275 .is_empty());
12276 let history = runtime.terminal_ring.lock().unwrap().snapshot();
12277 assert_eq!(history.entries.len(), 1);
12278 assert_eq!(
12279 history.entries[0].disposition,
12280 TerminalDisposition::DaemonShutdown
12281 );
12282 }
12283
12284 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
12288 async fn every_restart_increment_path_advances_lifetime_count() {
12289 let supervisor = Supervisor::new_for_test(
12290 Arc::new(Registry::default()),
12291 RestartPolicy::new(1, Duration::ZERO),
12292 );
12293 let runtime = supervisor.runtime_config();
12294 let spec = ModuleSpec {
12295 module_id: "lifetime-increment-path".to_string(),
12296 program: PathBuf::from("/unused/lifetime-increment-path"),
12297 args: Vec::new(),
12298 env: Vec::new(),
12299 reserved: false,
12300 reserved_prefixes: Vec::new(),
12301 protocol: ModuleProtocol::Subc,
12302 overlap: Default::default(),
12303 };
12304
12305 let crash_snapshot = Arc::new(Mutex::new(SupervisorSnapshot::starting()));
12306 assert!(matches!(
12307 on_child_exit(
12308 &spec,
12309 runtime.restart_policy,
12310 &supervisor.registry,
12311 &crash_snapshot,
12312 &runtime.terminal_ring,
12313 &runtime.spawn_events,
12314 &runtime.child_roster,
12315 ExitReport {
12316 kind: ExitKind::Crash,
12317 code: Some(1),
12318 signal: None,
12319 at_ms: 1,
12320 },
12321 )
12322 .await,
12323 NextAction::Restart { schedule: _ }
12324 ));
12325 let (crash_restarts, crash_lifetime) = {
12326 let state = lock_snapshot(&crash_snapshot).unwrap();
12327 (state.crash_restarts.len(), state.lifetime_restarts)
12328 };
12329 assert_eq!(crash_restarts, 1);
12330 assert_eq!(crash_lifetime, 1);
12331
12332 let health_snapshot = Arc::new(Mutex::new(SupervisorSnapshot::starting()));
12333 let mut health_child = None;
12334 assert!(matches!(
12335 health_restart_child(
12336 &spec,
12337 &runtime,
12338 &supervisor.registry,
12339 &supervisor.process_liveness,
12340 &health_snapshot,
12341 &mut health_child,
12342 SupervisorHealthStatus::Failing,
12343 None,
12344 2,
12345 )
12346 .await,
12347 Ok(())
12348 ));
12349 assert!(health_child.is_none());
12350 assert!(lock_snapshot(&health_snapshot).unwrap().respawn_pending);
12351 assert!(runtime.scheduled_respawn.lock().unwrap().take().is_some());
12352 let (health_restarts, health_lifetime) = {
12353 let state = lock_snapshot(&health_snapshot).unwrap();
12354 (state.crash_restarts.len(), state.lifetime_restarts)
12355 };
12356 assert_eq!(health_restarts, 1);
12357 assert_eq!(health_lifetime, 1);
12358
12359 let reload_snapshot = Arc::new(Mutex::new(SupervisorSnapshot::starting()));
12360 let mut reload_child = None;
12361 assert!(matches!(
12362 handle_reload_spawn_failure(
12363 &spec,
12364 &runtime,
12365 &supervisor.process_liveness,
12366 &reload_snapshot,
12367 &mut reload_child,
12368 "forced reload spawn failure".to_string(),
12369 )
12370 .await,
12371 Err(SuperviseError::ReloadFailed { .. })
12372 ));
12373 let (reload_restarts, reload_lifetime) = {
12374 let state = lock_snapshot(&reload_snapshot).unwrap();
12375 (state.crash_restarts.len(), state.lifetime_restarts)
12376 };
12377 assert_eq!(reload_restarts, 1);
12378 assert_eq!(reload_lifetime, 1);
12379 }
12380
12381 #[tokio::test]
12382 async fn deliberately_severed_live_child_records_lifetime_without_spending_restart_budget() {
12383 let supervisor = Supervisor::new_for_test(
12384 Arc::new(Registry::default()),
12385 RestartPolicy::new(3, Duration::ZERO),
12386 );
12387 let runtime = supervisor.runtime_config();
12388 let spec = ModuleSpec {
12389 module_id: "deliberately-severed".to_string(),
12390 program: PathBuf::from("/unused/deliberately-severed"),
12391 args: Vec::new(),
12392 env: Vec::new(),
12393 reserved: false,
12394 reserved_prefixes: Vec::new(),
12395 protocol: ModuleProtocol::Subc,
12396 overlap: Default::default(),
12397 };
12398 let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::starting()));
12399 let process = ProcessIdentity {
12400 pid: 41,
12401 start_time: 101,
12402 };
12403 record_deliberate_severance(&snapshot, process).unwrap();
12404 let exit_report = apply_deliberate_severance_marker(
12405 &snapshot,
12406 Some(process),
12407 ExitReport {
12408 kind: ExitKind::Crash,
12409 code: Some(1),
12410 signal: None,
12411 at_ms: 1,
12412 },
12413 );
12414 assert_eq!(exit_report.kind, ExitKind::DeliberateSeverance);
12415
12416 assert!(matches!(
12417 on_child_exit(
12418 &spec,
12419 runtime.restart_policy,
12420 &supervisor.registry,
12421 &snapshot,
12422 &runtime.terminal_ring,
12423 &runtime.spawn_events,
12424 &runtime.child_roster,
12425 exit_report,
12426 )
12427 .await,
12428 NextAction::Restart { schedule: _ }
12429 ));
12430 let state = lock_snapshot(&snapshot).unwrap();
12431 assert_eq!(state.lifetime_restarts, 1);
12432 assert_eq!(state.crash_restarts.len(), 0);
12433 }
12434
12435 #[tokio::test]
12436 async fn genuine_crash_spends_restart_budget_and_records_lifetime() {
12437 let supervisor = Supervisor::new_for_test(
12438 Arc::new(Registry::default()),
12439 RestartPolicy::new(3, Duration::ZERO),
12440 );
12441 let runtime = supervisor.runtime_config();
12442 let spec = ModuleSpec {
12443 module_id: "genuine-crash".to_string(),
12444 program: PathBuf::from("/unused/genuine-crash"),
12445 args: Vec::new(),
12446 env: Vec::new(),
12447 reserved: false,
12448 reserved_prefixes: Vec::new(),
12449 protocol: ModuleProtocol::Subc,
12450 overlap: Default::default(),
12451 };
12452 let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::starting()));
12453
12454 assert!(matches!(
12455 on_child_exit(
12456 &spec,
12457 runtime.restart_policy,
12458 &supervisor.registry,
12459 &snapshot,
12460 &runtime.terminal_ring,
12461 &runtime.spawn_events,
12462 &runtime.child_roster,
12463 ExitReport {
12464 kind: ExitKind::Crash,
12465 code: Some(1),
12466 signal: None,
12467 at_ms: 1,
12468 },
12469 )
12470 .await,
12471 NextAction::Restart { schedule: _ }
12472 ));
12473 let state = lock_snapshot(&snapshot).unwrap();
12474 assert_eq!(state.lifetime_restarts, 1);
12475 assert_eq!(state.crash_restarts.len(), 1);
12476 }
12477
12478 fn crash_exit_report(at_ms: u64) -> ExitReport {
12479 ExitReport {
12480 kind: ExitKind::Crash,
12481 code: Some(1),
12482 signal: None,
12483 at_ms,
12484 }
12485 }
12486
12487 fn windowed_crash_spec(module_id: &str) -> ModuleSpec {
12488 ModuleSpec {
12489 module_id: module_id.to_string(),
12490 program: PathBuf::from("/unused").join(module_id),
12491 args: Vec::new(),
12492 env: Vec::new(),
12493 reserved: false,
12494 reserved_prefixes: Vec::new(),
12495 protocol: ModuleProtocol::Subc,
12496 overlap: Default::default(),
12497 }
12498 }
12499
12500 #[tokio::test]
12506 async fn three_crashes_inside_the_window_stop_the_module_and_name_the_window() {
12507 let (logs, _guard) = crate::router::test_log::log_capture(tracing::Level::ERROR);
12508 let supervisor = Supervisor::new_for_test(
12509 Arc::new(Registry::default()),
12510 RestartPolicy::new(2, Duration::ZERO),
12511 );
12512 let runtime = supervisor.runtime_config();
12513 let spec = windowed_crash_spec("crash-loop-in-window");
12514 let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::starting()));
12515
12516 for attempt in 1..=2 {
12517 assert!(
12518 matches!(
12519 on_child_exit(
12520 &spec,
12521 runtime.restart_policy,
12522 &supervisor.registry,
12523 &snapshot,
12524 &runtime.terminal_ring,
12525 &runtime.spawn_events,
12526 &runtime.child_roster,
12527 crash_exit_report(attempt),
12528 )
12529 .await,
12530 NextAction::Restart { schedule: _ }
12531 ),
12532 "crash {attempt} is inside the budget and must respawn"
12533 );
12534 }
12535
12536 assert!(matches!(
12537 on_child_exit(
12538 &spec,
12539 runtime.restart_policy,
12540 &supervisor.registry,
12541 &snapshot,
12542 &runtime.terminal_ring,
12543 &runtime.spawn_events,
12544 &runtime.child_roster,
12545 crash_exit_report(3),
12546 )
12547 .await,
12548 NextAction::Stop { .. }
12549 ));
12550
12551 {
12552 let state = lock_snapshot(&snapshot).unwrap();
12553 assert_eq!(state.state, ModuleState::Failed);
12554 assert_eq!(state.crash_restarts.len(), 2);
12555 assert_eq!(state.lifetime_restarts, 2);
12556 }
12557
12558 let history = runtime
12559 .terminal_ring
12560 .lock()
12561 .expect("terminal ring is not poisoned")
12562 .snapshot();
12563 let last = history
12564 .entries
12565 .last()
12566 .expect("the refused crash is retained");
12567 assert_eq!(last.disposition, TerminalDisposition::Failed);
12568 assert_eq!(
12569 last.disposition_detail.as_deref(),
12570 Some("crash budget exhausted: max_restarts=2 within window_secs=600")
12571 );
12572
12573 let captured = crate::router::test_log::captured_logs(&logs);
12574 assert!(
12575 captured.contains("crash budget exhausted: max_restarts=2 within window_secs=600"),
12576 "the stop must be logged with its window: {captured}"
12577 );
12578 }
12579
12580 #[tokio::test]
12588 async fn a_crash_older_than_the_window_frees_its_slot_for_a_later_crash() {
12589 let supervisor = Supervisor::new_for_test(
12590 Arc::new(Registry::default()),
12591 RestartPolicy::new(2, Duration::ZERO),
12592 );
12593 let runtime = supervisor.runtime_config();
12594 let spec = windowed_crash_spec("crash-across-windows");
12595 let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::starting()));
12596
12597 for attempt in 1..=2 {
12598 assert!(matches!(
12599 on_child_exit(
12600 &spec,
12601 runtime.restart_policy,
12602 &supervisor.registry,
12603 &snapshot,
12604 &runtime.terminal_ring,
12605 &runtime.spawn_events,
12606 &runtime.child_roster,
12607 crash_exit_report(attempt),
12608 )
12609 .await,
12610 NextAction::Restart { schedule: _ }
12611 ));
12612 }
12613
12614 update_snapshot(&snapshot, Some(&spec.module_id), |state| {
12617 age_oldest_crash_restart_out_of_window(state, runtime.restart_policy.window);
12618 })
12619 .unwrap();
12620
12621 assert!(
12622 matches!(
12623 on_child_exit(
12624 &spec,
12625 runtime.restart_policy,
12626 &supervisor.registry,
12627 &snapshot,
12628 &runtime.terminal_ring,
12629 &runtime.spawn_events,
12630 &runtime.child_roster,
12631 crash_exit_report(3),
12632 )
12633 .await,
12634 NextAction::Restart { schedule: _ }
12635 ),
12636 "a crash older than the window must not hold a budget slot"
12637 );
12638
12639 let state = lock_snapshot(&snapshot).unwrap();
12640 assert_eq!(state.state, ModuleState::Restarting);
12641 assert_eq!(
12642 state.crash_restarts.len(),
12643 2,
12644 "the aged instant is dropped and the new one takes its place"
12645 );
12646 assert_eq!(
12647 state.lifetime_restarts, 3,
12648 "the ledger counts every restart, including the ones the window forgot"
12649 );
12650 }
12651
12652 #[tokio::test]
12657 async fn an_operator_restart_clears_the_ring_and_leaves_the_ledger_alone() {
12658 let supervisor = Supervisor::new_for_test(
12659 Arc::new(Registry::default()),
12660 RestartPolicy::new(2, Duration::ZERO),
12661 );
12662 let runtime = supervisor.runtime_config();
12663 let spec = windowed_crash_spec("operator-cleared-budget");
12664 let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::starting()));
12665
12666 for attempt in 1..=2 {
12667 assert!(matches!(
12668 on_child_exit(
12669 &spec,
12670 runtime.restart_policy,
12671 &supervisor.registry,
12672 &snapshot,
12673 &runtime.terminal_ring,
12674 &runtime.spawn_events,
12675 &runtime.child_roster,
12676 crash_exit_report(attempt),
12677 )
12678 .await,
12679 NextAction::Restart { schedule: _ }
12680 ));
12681 }
12682
12683 reset_restart_count(&snapshot, &spec.module_id).unwrap();
12684 {
12685 let state = lock_snapshot(&snapshot).unwrap();
12686 assert!(
12687 state.crash_restarts.is_empty(),
12688 "an operator restart returns the full budget"
12689 );
12690 assert_eq!(
12691 state.lifetime_restarts, 2,
12692 "clearing the budget must not unmake the crashes"
12693 );
12694 }
12695
12696 assert!(
12697 matches!(
12698 on_child_exit(
12699 &spec,
12700 runtime.restart_policy,
12701 &supervisor.registry,
12702 &snapshot,
12703 &runtime.terminal_ring,
12704 &runtime.spawn_events,
12705 &runtime.child_roster,
12706 crash_exit_report(3),
12707 )
12708 .await,
12709 NextAction::Restart { schedule: _ }
12710 ),
12711 "the cleared budget must be spendable again"
12712 );
12713 let state = lock_snapshot(&snapshot).unwrap();
12714 assert_eq!(state.crash_restarts.len(), 1);
12715 assert_eq!(state.lifetime_restarts, 3);
12716 }
12717
12718 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
12719 async fn severance_marker_for_a_dead_child_does_not_label_its_successor() {
12720 let severed = ProcessIdentity {
12721 pid: 41,
12722 start_time: 101,
12723 };
12724 let successor = ProcessIdentity {
12725 pid: 41,
12726 start_time: 202,
12727 };
12728 let module = module_with_recovery_snapshot(ModuleState::Running, true, 0);
12729 update_snapshot(&module.inner.snapshot, Some("recovery-snapshot"), |state| {
12730 state.pid = Some(successor.pid);
12731 state.process_start_time = Some(successor.start_time);
12732 })
12733 .unwrap();
12734 assert!(!module.record_deliberate_severance(severed).unwrap());
12735
12736 let exit_report = apply_deliberate_severance_marker(
12737 &module.inner.snapshot,
12738 Some(successor),
12739 ExitReport {
12740 kind: ExitKind::Crash,
12741 code: Some(1),
12742 signal: None,
12743 at_ms: 1,
12744 },
12745 );
12746
12747 assert_eq!(exit_report.kind, ExitKind::Crash);
12748 }
12749
12750 #[tokio::test]
12751 async fn drain_reap_marks_deliberate_severance_and_records_lifetime_without_budget() {
12752 let registry = Registry::default();
12753 let supervisor = Supervisor::new_for_test(
12754 Arc::new(Registry::default()),
12755 RestartPolicy::new(3, Duration::ZERO),
12756 );
12757 let runtime = supervisor.runtime_config();
12758 let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::starting()));
12759 let spec = ModuleSpec {
12760 module_id: "drain-deliberate-severance".to_string(),
12761 program: fake_aft_stub_path(),
12762 args: Vec::new(),
12763 env: vec![("FAKE_AFT_EXIT_CODE".to_string(), "23".to_string())],
12764 reserved: false,
12765 reserved_prefixes: Vec::new(),
12766 protocol: ModuleProtocol::Subc,
12767 overlap: Default::default(),
12768 };
12769 let mut child = spawn_and_mark_running(&spec, &runtime, &snapshot).unwrap();
12770 let process = ProcessIdentity {
12771 pid: 41,
12772 start_time: 101,
12773 };
12774 child.process_identity = Some(process);
12775 update_snapshot(&snapshot, Some(&spec.module_id), |state| {
12776 state.pid = Some(process.pid);
12777 state.process_start_time = Some(process.start_time);
12778 })
12779 .unwrap();
12780 record_deliberate_severance(&snapshot, process).unwrap();
12781
12782 drain_child_to_state(
12783 &spec.module_id,
12784 spec.protocol,
12785 StopNotice::SentOverConnection,
12788 ®istry,
12789 None,
12790 &snapshot,
12791 &runtime.terminal_ring,
12792 &runtime.spawn_events,
12793 child,
12794 Duration::from_secs(1),
12795 ModuleState::Stopped,
12796 Some(false),
12797 )
12798 .await
12799 .unwrap();
12800
12801 let state = lock_snapshot(&snapshot).unwrap();
12802 assert_eq!(
12803 state.last_exit.as_ref().map(|exit| exit.kind),
12804 Some(ExitKind::DeliberateSeverance)
12805 );
12806 assert_eq!(state.lifetime_restarts, 1);
12807 assert_eq!(state.crash_restarts.len(), 0);
12808 drop(state);
12809 let history = runtime.terminal_ring.lock().unwrap().snapshot();
12810 assert_eq!(
12811 history.entries[0].exit_kind,
12812 subc_control::TerminalExitKind::DeliberateSeverance
12813 );
12814 }
12815
12816 #[tokio::test]
12817 async fn ordinary_drain_reap_does_not_record_a_lifetime_restart() {
12818 let registry = Registry::default();
12819 let supervisor = Supervisor::new_for_test(
12820 Arc::new(Registry::default()),
12821 RestartPolicy::new(3, Duration::ZERO),
12822 );
12823 let runtime = supervisor.runtime_config();
12824 let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::starting()));
12825 let spec = ModuleSpec {
12826 module_id: "ordinary-drain".to_string(),
12827 program: fake_aft_stub_path(),
12828 args: Vec::new(),
12829 env: vec![("FAKE_AFT_EXIT_CODE".to_string(), "23".to_string())],
12830 reserved: false,
12831 reserved_prefixes: Vec::new(),
12832 protocol: ModuleProtocol::Subc,
12833 overlap: Default::default(),
12834 };
12835 let child = spawn_and_mark_running(&spec, &runtime, &snapshot).unwrap();
12836
12837 drain_child_to_state(
12838 &spec.module_id,
12839 spec.protocol,
12840 StopNotice::SentOverConnection,
12843 ®istry,
12844 None,
12845 &snapshot,
12846 &runtime.terminal_ring,
12847 &runtime.spawn_events,
12848 child,
12849 Duration::from_secs(1),
12850 ModuleState::Stopped,
12851 Some(false),
12852 )
12853 .await
12854 .unwrap();
12855
12856 let state = lock_snapshot(&snapshot).unwrap();
12857 assert_eq!(
12858 state.last_exit.as_ref().map(|exit| exit.kind),
12859 Some(ExitKind::Crash)
12860 );
12861 assert_eq!(state.lifetime_restarts, 0);
12862 assert_eq!(state.crash_restarts.len(), 0);
12863 }
12864
12865 #[test]
12866 fn fatal_connection_teardown_cannot_arm_a_marker_for_a_surviving_process() {
12867 assert!(!include_str!("server.rs")
12873 .contains("router.record_deliberate_connection_severance(ctx.connection_id)"));
12874 }
12875
12876 #[test]
12883 fn drained_after_quiescence_wait_passes_ok_through_and_forces_false_on_err() {
12884 assert!(drained_after_quiescence_wait(&Ok(true)));
12885 assert!(!drained_after_quiescence_wait(&Ok(false)));
12886 assert!(!drained_after_quiescence_wait(&Err(
12887 SuperviseError::StatePoisoned { module_id: None }
12888 )));
12889 }
12890
12891 #[test]
12900 fn wait_error_exit_report_records_a_failed_terminal_with_no_code_or_signal() {
12901 let ring = Arc::new(Mutex::new(TerminalRing::new(
12902 TerminalRingConfig::default(),
12903 0,
12904 )));
12905 record_wait_error_terminal("wait-error", &ring, &super::SpawnEventFeed::default());
12906
12907 let snapshot = ring.lock().unwrap().snapshot();
12908 assert_eq!(snapshot.entries.len(), 1);
12909 let entry = &snapshot.entries[0];
12910 assert_eq!(entry.exit_code, None);
12911 assert_eq!(entry.exit_signal, None);
12912 assert_eq!(entry.disposition, TerminalDisposition::Failed);
12913 }
12914
12915 #[test]
12916 fn wait_error_exit_path_preserves_spawn_event_density() {
12917 let feed = super::SpawnEventFeed::default();
12918 feed.configure_incarnation("wait-error-density".to_string());
12919 feed.emit_spawned("wait-error", 41, 1);
12920 let ring = Arc::new(Mutex::new(TerminalRing::new(
12921 TerminalRingConfig::default(),
12922 0,
12923 )));
12924
12925 record_wait_error_terminal("wait-error", &ring, &feed);
12926 feed.emit_spawned("after-wait-error", 42, 2);
12927
12928 let state = feed.0.lock().unwrap();
12929 let sequences = state
12930 .events
12931 .iter()
12932 .map(|event| event.cursor.seq)
12933 .collect::<Vec<_>>();
12934 assert_eq!(sequences, vec![1, 2, 3]);
12935 assert_eq!(state.events[1].kind, SpawnEventKind::Exited);
12936 assert_eq!(state.events[1].exit_code, None);
12937 assert_eq!(state.events[1].exit_signal, None);
12938 }
12939
12940 #[test]
12944 fn wait_error_exit_report_is_classified_as_a_crash() {
12945 assert_eq!(wait_error_exit_report().kind, ExitKind::Crash);
12946 }
12947}
12948
12949#[cfg(all(test, unix))]
12950mod health_event_tests {
12951 use super::*;
12952 use cortexkit_test_support::ScratchDir;
12953 use subc_protocol::{manifest::Concurrency, session::ModuleControlResponse};
12954
12955 struct Actor {
12956 module: SupervisedModule,
12957 registry: Arc<Registry>,
12958 forwarding: Arc<ForwardingTable>,
12959 spec: ModuleSpec,
12960 health: HealthConfig,
12961 rx: mpsc::Receiver<crate::router::OutboundFrame>,
12962 _home: ScratchDir,
12963 }
12964
12965 async fn wait_for<T>(reason: &str, mut observe: impl FnMut() -> Option<T>) -> T {
12966 let deadline = std::time::Instant::now() + Duration::from_secs(10);
12970 let tick = Instant::now();
12971 loop {
12972 if let Some(value) = observe() {
12973 assert_eq!(Instant::now(), tick, "{reason} must not wait for a timer");
12974 return value;
12975 }
12976 assert!(
12977 std::time::Instant::now() < deadline,
12978 "timed out waiting for {reason}"
12979 );
12980 tokio::task::yield_now().await;
12981 }
12982 }
12983
12984 impl Actor {
12985 async fn start(protocol: ModuleProtocol, cadence: Duration) -> Self {
12986 let home = ScratchDir::new("health-events");
12987 let registry = Arc::new(Registry::default());
12988 let forwarding = Arc::new(ForwardingTable::default());
12989 let health = HealthConfig {
12990 cadence,
12991 deadline: Duration::from_secs(3600),
12992 ..HealthConfig::default()
12993 };
12994 let supervisor =
12995 Supervisor::new_for_test(registry.clone(), RestartPolicy::new(3, Duration::ZERO))
12996 .with_forwarding(forwarding.clone())
12997 .with_health_config(health.clone());
12998 let spec = ModuleSpec {
12999 module_id: "event-health-child".into(),
13000 program: PathBuf::from("/bin/sh"),
13001 args: vec!["-c".into(), "exec sleep 600".into()],
13004 env: ["XDG_DATA_HOME", "XDG_RUNTIME_DIR", "XDG_CONFIG_HOME"]
13005 .into_iter()
13006 .map(|key| (key.into(), home.path().display().to_string()))
13007 .collect(),
13008 reserved: false,
13009 reserved_prefixes: vec![],
13010 protocol,
13011 overlap: Default::default(),
13012 };
13013 let module = supervisor.spawn(spec.clone()).unwrap();
13014 let (_, rx) = mpsc::channel(8);
13015 let actor = Self {
13016 module,
13017 registry,
13018 forwarding,
13019 spec,
13020 health,
13021 rx,
13022 _home: home,
13023 };
13024 actor
13025 .wait_for_select(|checkpoint| checkpoint.generation > 0)
13026 .await;
13027 actor
13028 }
13029
13030 fn turns(&self) -> u64 {
13031 lock_snapshot(&self.module.inner.snapshot)
13032 .unwrap()
13033 .actor_turns
13034 }
13035
13036 async fn wait_for_select(
13037 &self,
13038 expected: impl Fn(&ActorSelectCheckpoint) -> bool,
13039 ) -> ActorSelectCheckpoint {
13040 wait_for("actor select checkpoint", || {
13041 let state = lock_snapshot(&self.module.inner.snapshot).unwrap();
13042 state.actor_select.filter(|checkpoint| {
13043 checkpoint.generation == state.spawn_generation
13044 && state.state == ModuleState::Running
13045 && state.process_alive
13046 && state.reported_pid().is_some()
13047 && expected(checkpoint)
13048 })
13049 })
13050 .await
13051 }
13052
13053 fn hello(&mut self, connection: u64, health: bool) {
13054 let manifest =
13055 subc_protocol::manifest::ModuleManifest::builder(&self.spec.module_id, "1.0.0")
13056 .protocol_ver(subc_protocol::PROTOCOL_VERSION)
13057 .build();
13058 let connection = ConnectionId::new(connection);
13059 let (tx, rx) = mpsc::channel(8);
13060 self.rx = rx;
13061 self.forwarding
13062 .register_module_connection(
13063 connection,
13064 self.spec.module_id.clone(),
13065 subc_protocol::PROTOCOL_VERSION,
13066 Concurrency::ModuleManaged,
13067 FrameSink::new(tx),
13068 )
13069 .unwrap();
13070 self.registry
13071 .register_with_control_ops(
13072 manifest,
13073 subc_protocol::PROTOCOL_VERSION,
13074 connection,
13075 if health {
13076 vec![MODULE_CONTROL_OP_HEALTH_CHECK.into()]
13077 } else {
13078 vec![]
13079 },
13080 )
13081 .unwrap();
13082 }
13083
13084 async fn probe(&mut self) -> crate::router::OutboundFrame {
13085 let frame = wait_for("outbound health probe", || match self.rx.try_recv() {
13086 Ok(frame) => Some(frame),
13087 Err(mpsc::error::TryRecvError::Empty) => None,
13088 Err(mpsc::error::TryRecvError::Disconnected) => panic!("health peer disconnected"),
13089 })
13090 .await;
13091 let body: Value = serde_json::from_slice(&frame.body).unwrap();
13092 assert_eq!(body["op"], MODULE_CONTROL_OP_HEALTH_CHECK);
13093 frame
13094 }
13095
13096 fn no_probe(&mut self) {
13097 assert!(matches!(
13098 self.rx.try_recv(),
13099 Err(mpsc::error::TryRecvError::Empty)
13100 ));
13101 }
13102
13103 fn answer(&self, connection: u64, frame: crate::router::OutboundFrame) {
13104 self.forwarding
13105 .complete_module_control_rpc(
13106 ConnectionId::new(connection),
13107 frame.header.corr,
13108 Some(MODULE_CONTROL_OP_HEALTH_CHECK),
13109 ModuleControlRpcOutcome::Response(ModuleControlResponse::HealthCheck {
13110 status: HealthStatus::Ok,
13111 detail: None,
13112 metrics: None,
13113 }),
13114 )
13115 .unwrap();
13116 }
13117 }
13118
13119 #[tokio::test(start_paused = true)]
13120 async fn no_health_children_park_for_sixty_seconds() {
13121 let none = Actor::start(ModuleProtocol::None, Duration::from_secs(30)).await;
13122 let unregistered = Actor::start(ModuleProtocol::Subc, Duration::from_secs(30)).await;
13123 let mut unadvertised = Actor::start(ModuleProtocol::Subc, Duration::from_secs(30)).await;
13124 unadvertised.hello(1, false);
13125 for _ in 0..6000 {
13130 tokio::time::advance(Duration::from_millis(10)).await;
13131 tokio::task::yield_now().await;
13132 }
13133 for actor in [&none, &unregistered, &unadvertised] {
13134 assert!(
13135 actor.turns() <= 5,
13136 "no-health actor woke {} times",
13137 actor.turns()
13138 );
13139 assert_eq!(actor.module.state().unwrap(), ModuleState::Running);
13140 }
13141 }
13142
13143 #[tokio::test(start_paused = true)]
13144 async fn late_hello_starts_a_due_probe_in_the_notification_tick() {
13145 let mut actor = Actor::start(ModuleProtocol::Subc, Duration::ZERO).await;
13149 let tick = Instant::now();
13150 actor.hello(1, true);
13151 actor.probe().await;
13152 assert_eq!(Instant::now(), tick, "HELLO must not wait for a poll");
13153 }
13154
13155 #[tokio::test(start_paused = true)]
13156 async fn hello_keeps_the_jittered_cadence_and_catalog_updates_do_not_reset_it() {
13157 let mut actor = Actor::start(ModuleProtocol::Subc, Duration::from_secs(30)).await;
13158 let hello_tick = Instant::now();
13159 actor.hello(1, true);
13160 let armed = actor
13161 .wait_for_select(|checkpoint| {
13162 checkpoint.registered_connection == Some(ConnectionId::new(1))
13163 && checkpoint.next_probe_at.is_some()
13164 })
13165 .await;
13166 let delay = jittered_health_delay(&actor.spec.module_id, 0, actor.health.cadence);
13167 assert!((Duration::from_secs(30)..Duration::from_secs(33)).contains(&delay));
13168 let first_deadline = hello_tick + delay;
13169 assert_eq!(armed.next_probe_at, Some(first_deadline));
13170 tokio::time::advance(Duration::from_secs(29)).await;
13171 let turns = actor.turns();
13172 actor
13173 .registry
13174 .replace_catalog_for_connection(ConnectionId::new(1), vec![], None, Some(true))
13175 .unwrap();
13176 let updated = actor
13177 .wait_for_select(|checkpoint| checkpoint.turn > turns)
13178 .await;
13179 assert_eq!(
13180 updated.next_probe_at,
13181 Some(first_deadline),
13182 "catalog update must not reset cadence"
13183 );
13184 actor.no_probe();
13185 tokio::time::advance(first_deadline - Instant::now()).await;
13186 let frame = actor.probe().await;
13187 actor.answer(1, frame);
13188 let next = jittered_health_delay(&actor.spec.module_id, 1, actor.health.cadence);
13189 let next_deadline = Instant::now() + next;
13190 let rearmed = actor
13191 .wait_for_select(|checkpoint| {
13192 checkpoint
13193 .next_probe_at
13194 .is_some_and(|deadline| deadline > first_deadline)
13195 })
13196 .await;
13197 assert_eq!(rearmed.next_probe_at, Some(next_deadline));
13198 tokio::time::advance(next - Duration::from_nanos(1)).await;
13199 actor.no_probe();
13200 tokio::time::advance(Duration::from_nanos(1)).await;
13201 actor.probe().await;
13202 }
13203
13204 #[tokio::test(start_paused = true)]
13205 async fn disconnect_disarms_and_reconnect_rearms_health() {
13206 let mut actor = Actor::start(ModuleProtocol::Subc, Duration::from_secs(30)).await;
13207 actor.hello(1, true);
13208 actor
13209 .wait_for_select(|checkpoint| {
13210 checkpoint.registered_connection == Some(ConnectionId::new(1))
13211 && checkpoint.next_probe_at.is_some()
13212 })
13213 .await;
13214 let turns = actor.turns();
13215 let tick = Instant::now();
13216 actor
13217 .registry
13218 .deregister_connection(ConnectionId::new(1))
13219 .unwrap();
13220 let disarmed = actor
13221 .wait_for_select(|checkpoint| {
13222 checkpoint.turn > turns && checkpoint.registered_connection.is_none()
13223 })
13224 .await;
13225 assert_eq!(disarmed.next_probe_at, None);
13226 assert_eq!(disarmed.wake_after, None);
13227 assert_eq!(
13228 Instant::now(),
13229 tick,
13230 "disconnect must disarm in the notification tick"
13231 );
13232 tokio::time::advance(Duration::from_secs(60)).await;
13233 actor.no_probe();
13234 let reconnect_tick = Instant::now();
13235 actor.hello(2, true);
13236 let rearmed = actor
13237 .wait_for_select(|checkpoint| {
13238 checkpoint.registered_connection == Some(ConnectionId::new(2))
13239 && checkpoint.next_probe_at.is_some()
13240 })
13241 .await;
13242 let delay = jittered_health_delay(&actor.spec.module_id, 0, actor.health.cadence);
13243 assert_eq!(rearmed.next_probe_at, Some(reconnect_tick + delay));
13244 tokio::time::advance(delay).await;
13245 actor.probe().await;
13246 }
13247
13248 #[tokio::test(start_paused = true)]
13249 async fn reregistration_without_health_disarms_the_old_schedule() {
13250 let mut actor = Actor::start(ModuleProtocol::Subc, Duration::from_secs(30)).await;
13251 actor.hello(1, true);
13252 actor
13253 .wait_for_select(|checkpoint| {
13254 checkpoint.registered_connection == Some(ConnectionId::new(1))
13255 && checkpoint.next_probe_at.is_some()
13256 })
13257 .await;
13258 actor
13259 .registry
13260 .deregister_connection(ConnectionId::new(1))
13261 .unwrap();
13262 actor.hello(2, false);
13263 let disarmed = actor
13264 .wait_for_select(|checkpoint| {
13265 checkpoint.registered_connection == Some(ConnectionId::new(2))
13266 })
13267 .await;
13268 assert_eq!(disarmed.next_probe_at, None);
13269 assert_eq!(disarmed.wake_after, None);
13270 tokio::time::advance(Duration::from_secs(60)).await;
13271 actor.no_probe();
13272 }
13273
13274 #[tokio::test(start_paused = true)]
13275 async fn restart_waits_for_its_own_hello_before_rearming_health() {
13276 let mut actor = Actor::start(ModuleProtocol::Subc, Duration::from_secs(30)).await;
13277 actor.hello(1, true);
13278 let armed = actor
13279 .wait_for_select(|checkpoint| {
13280 checkpoint.registered_connection == Some(ConnectionId::new(1))
13281 && checkpoint.next_probe_at.is_some()
13282 })
13283 .await;
13284 actor
13288 .registry
13289 .deregister_connection(ConnectionId::new(1))
13290 .unwrap();
13291 actor.module.restart(Some(0)).await.unwrap();
13292 let parked = actor
13293 .wait_for_select(|checkpoint| {
13294 checkpoint.generation > armed.generation
13295 && checkpoint.registered_connection.is_none()
13296 && checkpoint.next_probe_at.is_none()
13297 })
13298 .await;
13299 assert_eq!(parked.wake_after, None);
13300 actor.rx.close();
13301 while actor.rx.try_recv().is_ok() {}
13304 let turns = actor.turns();
13305 tokio::time::advance(Duration::from_secs(60)).await;
13306 let after = actor.turns();
13307 eprintln!(
13308 "restart park turns: before={turns} after={after} delta={}",
13309 after - turns
13310 );
13311 assert_eq!(after, turns, "replacement must park before HELLO");
13312 let hello_tick = Instant::now();
13313 actor.hello(2, true);
13314 let rearmed = actor
13315 .wait_for_select(|checkpoint| {
13316 checkpoint.registered_connection == Some(ConnectionId::new(2))
13317 && checkpoint.next_probe_at.is_some()
13318 })
13319 .await;
13320 let delay = jittered_health_delay(&actor.spec.module_id, 0, actor.health.cadence);
13321 assert_eq!(rearmed.next_probe_at, Some(hello_tick + delay));
13322 tokio::time::advance(delay).await;
13323 actor.probe().await;
13324 }
13325
13326 #[tokio::test(start_paused = true)]
13327 async fn rescan_adds_and_removes_http_health_without_polling() {
13328 use tokio::{
13329 io::{AsyncReadExt, AsyncWriteExt},
13330 net::TcpListener,
13331 };
13332 let mut actor = Actor::start(ModuleProtocol::None, Duration::from_secs(30)).await;
13333 let listener = TcpListener::bind("127.0.0.1:0").await.unwrap();
13334 actor.health.http = Some(format!("http://{}/health", listener.local_addr().unwrap()));
13335 actor.health.cadence = Duration::from_millis(1);
13337 let (started_tx, mut started_rx) = mpsc::channel(8);
13338 let peer = tokio::spawn(async move {
13339 loop {
13340 let (mut socket, _) = listener.accept().await.unwrap();
13341 let mut request = [0; 1024];
13342 assert!(socket.read(&mut request).await.unwrap() > 0);
13343 started_tx.send(()).await.unwrap();
13344 socket
13345 .write_all(b"HTTP/1.1 200 OK\r\nContent-Length: 0\r\n\r\n")
13346 .await
13347 .unwrap();
13348 }
13349 });
13350 let tick = Instant::now();
13351 let turns = actor.turns();
13352 actor
13353 .module
13354 .update_configuration(actor.spec.clone(), actor.health.clone(), None)
13355 .await
13356 .unwrap();
13357 let armed = actor
13358 .wait_for_select(|checkpoint| {
13359 checkpoint.turn > turns && checkpoint.next_probe_at.is_some()
13360 })
13361 .await;
13362 assert_eq!(armed.next_probe_at, Some(tick + Duration::from_millis(1)));
13363 assert_eq!(Instant::now(), tick);
13364 tokio::time::advance(Duration::from_millis(1)).await;
13365 let mut started = false;
13366 wait_for("successful HTTP probe", || {
13367 started |= started_rx.try_recv().is_ok();
13368 (started && actor.module.status().unwrap().health.status == SupervisorHealthStatus::Ok)
13369 .then_some(())
13370 })
13371 .await;
13372 assert!(started, "added HTTP check must start at its first deadline");
13373 assert_eq!(
13374 actor.module.status().unwrap().health.status,
13375 SupervisorHealthStatus::Ok
13376 );
13377 assert_eq!(Instant::now(), tick + Duration::from_millis(1));
13378 actor.health.http = None;
13379 let turns = actor.turns();
13380 actor
13381 .module
13382 .update_configuration(actor.spec.clone(), actor.health.clone(), None)
13383 .await
13384 .unwrap();
13385 let parked = actor
13386 .wait_for_select(|checkpoint| {
13387 checkpoint.turn > turns && checkpoint.next_probe_at.is_none()
13388 })
13389 .await;
13390 assert_eq!(parked.wake_after, None);
13391 let turns = actor.turns();
13392 tokio::time::advance(Duration::from_secs(60)).await;
13393 assert_eq!(
13394 actor.turns(),
13395 turns,
13396 "removed HTTP check must park the actor"
13397 );
13398 assert!(
13399 started_rx.try_recv().is_err(),
13400 "removed HTTP check must stay disarmed"
13401 );
13402 assert_eq!(
13403 actor.module.status().unwrap().health,
13404 ModuleHealthStatus::default()
13405 );
13406 peer.abort();
13407 }
13408}
13409
13410#[cfg(test)]
13411mod health_evidence_tests {
13412 use super::{HealthProbeError, HealthProbeEvidence};
13413 use std::collections::HashSet;
13414
13415 #[test]
13423 fn only_a_dead_lane_is_proof_of_death() {
13424 assert!(HealthProbeError::lane_dead("gone").is_proof_of_death());
13425 assert!(!HealthProbeError::no_answer("timed out").is_proof_of_death());
13429 assert!(!HealthProbeError::bad_answer("garbage").is_proof_of_death());
13430 assert!(!HealthProbeError::misconfigured("no table").is_proof_of_death());
13431 }
13432
13433 #[test]
13439 fn every_evidence_class_has_a_distinct_label() {
13440 let labels = [
13441 HealthProbeError::lane_dead("").label(),
13442 HealthProbeError::no_answer("").label(),
13443 HealthProbeError::bad_answer("").label(),
13444 HealthProbeError::misconfigured("").label(),
13445 ];
13446 let unique: HashSet<_> = labels.iter().collect();
13447 assert_eq!(unique.len(), labels.len(), "labels collided: {labels:?}");
13448 }
13449
13450 #[test]
13456 fn classification_preserves_the_original_message() {
13457 let err = HealthProbeError::no_answer("module did not answer within 5s");
13458 assert_eq!(err.to_string(), "module did not answer within 5s");
13459 assert!(matches!(err.evidence, HealthProbeEvidence::NoAnswer));
13460 }
13461}
13462
13463#[cfg(test)]
13464mod health_tombstone_tests {
13465 use std::{path::PathBuf, sync::Arc, time::Duration};
13466
13467 use subc_protocol::{
13468 manifest::Concurrency,
13469 session::{HealthStatus, ModuleControlResponse},
13470 };
13471 use tokio::sync::mpsc;
13472
13473 use super::{
13474 probe_module_health, HealthAction, HealthConfig, HealthProbeEvidence, ModuleProtocol,
13475 ModuleSpec, RestartPolicy, Supervisor, SupervisorRuntimeConfig,
13476 };
13477 use crate::{
13478 control::ControlHandler,
13479 forwarding::{ForwardingTable, ModuleControlRpcCompletion, ModuleControlRpcOutcome},
13480 registry::{ConnectionId, Registry},
13481 router::FrameSink,
13482 };
13483
13484 struct ProbeHarness {
13485 spec: ModuleSpec,
13486 runtime: SupervisorRuntimeConfig,
13487 forwarding: Arc<ForwardingTable>,
13488 module_connection: ConnectionId,
13489 module_rx: mpsc::Receiver<crate::router::OutboundFrame>,
13490 handler: ControlHandler,
13491 module: super::SupervisedModule,
13492 }
13493
13494 fn probe_harness() -> ProbeHarness {
13495 let registry = Arc::new(Registry::default());
13496 let forwarding = Arc::new(ForwardingTable::default());
13497 let supervisor_handle = super::SupervisorHandle::new();
13498 let health = HealthConfig {
13499 http: None,
13500 cadence: Duration::from_secs(30),
13501 deadline: Duration::from_secs(5),
13502 failure_threshold: 3,
13503 on_degraded: HealthAction::Report,
13504 on_failing: HealthAction::Report,
13505 critical: false,
13506 };
13507 let supervisor = Supervisor::new_for_test(Arc::clone(®istry), RestartPolicy::default())
13508 .with_forwarding(Arc::clone(&forwarding))
13509 .with_handle(supervisor_handle.clone())
13510 .with_health_config(health);
13511 let spec = ModuleSpec {
13512 module_id: "late-health-module".to_string(),
13513 program: PathBuf::from("disabled-module"),
13514 args: Vec::new(),
13515 env: Vec::new(),
13516 reserved: false,
13517 reserved_prefixes: Vec::new(),
13518 protocol: ModuleProtocol::Subc,
13519 overlap: Default::default(),
13520 };
13521 let module = supervisor
13522 .supervise_configured(spec.clone(), false)
13523 .unwrap();
13524 let runtime = supervisor.runtime_config();
13525 let handler = ControlHandler::with_forwarding(registry, Arc::clone(&forwarding))
13526 .with_supervisor(supervisor_handle);
13527 let module_connection = ConnectionId::new(700);
13528 let (module_tx, module_rx) = mpsc::channel(8);
13529 forwarding
13530 .register_module_connection(
13531 module_connection,
13532 spec.module_id.clone(),
13533 subc_protocol::PROTOCOL_VERSION,
13534 Concurrency::ModuleManaged,
13535 FrameSink::new(module_tx),
13536 )
13537 .unwrap();
13538
13539 ProbeHarness {
13540 spec,
13541 runtime,
13542 forwarding,
13543 module_connection,
13544 module_rx,
13545 handler,
13546 module,
13547 }
13548 }
13549
13550 async fn finish_after(
13551 harness: &mut ProbeHarness,
13552 stall: Duration,
13553 ) -> ModuleControlRpcCompletion {
13554 assert!(stall > harness.runtime.health.deadline);
13555 let deadline = harness.runtime.health.deadline;
13556 let probe = probe_module_health(&harness.spec.module_id, &harness.runtime, None);
13557 let answer = async {
13558 let frame = harness.module_rx.recv().await.expect("health.check frame");
13559 tokio::time::advance(deadline).await;
13560 tokio::task::yield_now().await;
13561 tokio::time::advance(stall - deadline).await;
13562 harness
13563 .forwarding
13564 .complete_module_control_rpc(
13565 harness.module_connection,
13566 frame.header.corr,
13567 Some("health.check"),
13568 ModuleControlRpcOutcome::Response(ModuleControlResponse::HealthCheck {
13569 status: HealthStatus::Ok,
13570 detail: None,
13571 metrics: None,
13572 }),
13573 )
13574 .unwrap()
13575 };
13576 let (probe_result, completion) = tokio::join!(probe, answer);
13577 let err = probe_result.expect_err("probe must miss its deadline");
13578 assert!(matches!(err.evidence, HealthProbeEvidence::NoAnswer));
13579 completion
13580 }
13581
13582 async fn time_out_without_answer(harness: &mut ProbeHarness) {
13583 let deadline = harness.runtime.health.deadline;
13584 let probe = probe_module_health(&harness.spec.module_id, &harness.runtime, None);
13585 let exhaust_deadline = async {
13586 let _frame = harness.module_rx.recv().await.expect("health.check frame");
13587 tokio::time::advance(deadline).await;
13588 tokio::task::yield_now().await;
13589 };
13590 let (probe_result, ()) = tokio::join!(probe, exhaust_deadline);
13591 let err = probe_result.expect_err("probe must miss its deadline");
13592 assert!(matches!(err.evidence, HealthProbeEvidence::NoAnswer));
13593 }
13594
13595 async fn run_probe_cycle(harness: &mut ProbeHarness, answer: bool) {
13596 let registry = Arc::clone(&harness.module.inner.registry);
13597 let snapshot = Arc::clone(&harness.module.inner.snapshot);
13598 let process_liveness = super::SupervisorProcessLiveness::default();
13599 let mut child = None;
13600 let cycle = super::run_health_probe_cycle(
13601 &harness.spec,
13602 &harness.runtime,
13603 ®istry,
13604 &process_liveness,
13605 &snapshot,
13606 &mut child,
13607 );
13608 let peer = async {
13609 let frame = harness.module_rx.recv().await.expect("health.check frame");
13610 if answer {
13611 harness
13612 .forwarding
13613 .complete_module_control_rpc(
13614 harness.module_connection,
13615 frame.header.corr,
13616 Some("health.check"),
13617 ModuleControlRpcOutcome::Response(ModuleControlResponse::HealthCheck {
13618 status: HealthStatus::Ok,
13619 detail: None,
13620 metrics: Some(serde_json::json!({"ready": true})),
13621 }),
13622 )
13623 .unwrap();
13624 } else {
13625 tokio::time::advance(harness.runtime.health.deadline).await;
13626 tokio::task::yield_now().await;
13627 }
13628 };
13629 tokio::join!(cycle, peer);
13630 }
13631
13632 #[tokio::test(start_paused = true)]
13633 async fn unanswered_probe_is_unknown_until_threshold_and_ok_report_recovers() {
13634 let mut harness = probe_harness();
13635 let monitor = harness.module.inner.monitor.lock().unwrap().take().unwrap();
13639 monitor.abort();
13640 let _ = monitor.await;
13641 super::update_snapshot(&harness.module.inner.snapshot, None, |state| {
13642 state.enabled = true;
13643 state.state = super::ModuleState::Running;
13644 state.process_alive = true;
13645 })
13646 .unwrap();
13647
13648 run_probe_cycle(&mut harness, true).await;
13649 assert_eq!(
13650 harness.module.status().unwrap().health.status,
13651 super::SupervisorHealthStatus::Ok
13652 );
13653
13654 for failures in 1..harness.runtime.health.failure_threshold {
13655 run_probe_cycle(&mut harness, false).await;
13656 let status = harness.module.status().unwrap();
13657 assert_eq!(status.health.status, super::SupervisorHealthStatus::Unknown);
13658 assert_eq!(status.health.consecutive_failures, failures);
13659 assert!(status.health.last_probe_ms.is_some());
13660 assert!(status.health.detail.unwrap().starts_with("[no-answer]"));
13661 assert!(status.health.metrics.is_none());
13662 assert_eq!(status.state, super::ModuleState::Running);
13663 assert!(status.process_alive);
13664 assert_eq!(status.restart_count, 0);
13665 assert_eq!(status.lifetime_restarts, 0);
13666 assert!(status.health.last_action.is_none());
13667 assert!(harness.runtime.scheduled_respawn.lock().unwrap().is_none());
13668 }
13669
13670 run_probe_cycle(&mut harness, true).await;
13671 let recovered = harness.module.status().unwrap();
13672 assert_eq!(recovered.health.status, super::SupervisorHealthStatus::Ok);
13673 assert_eq!(recovered.health.consecutive_failures, 0);
13674 assert!(recovered.health.detail.is_none());
13675 assert_eq!(
13676 recovered.health.metrics,
13677 Some(serde_json::json!({"ready": true}))
13678 );
13679 assert_eq!(recovered.lifetime_restarts, 0);
13680
13681 for failures in 1..=harness.runtime.health.failure_threshold {
13682 run_probe_cycle(&mut harness, false).await;
13683 let status = harness.module.status().unwrap();
13684 assert_eq!(status.health.consecutive_failures, failures);
13685 if failures < harness.runtime.health.failure_threshold {
13686 assert_eq!(status.health.status, super::SupervisorHealthStatus::Unknown);
13687 assert_eq!(status.state, super::ModuleState::Running);
13688 assert_eq!(status.lifetime_restarts, 0);
13689 assert!(harness.runtime.scheduled_respawn.lock().unwrap().is_none());
13690 } else {
13691 assert_eq!(
13692 status.health.status,
13693 super::SupervisorHealthStatus::Unresponsive
13694 );
13695 assert_eq!(status.state, super::ModuleState::Restarting);
13696 assert_eq!(status.restart_count, 1);
13697 assert_eq!(status.lifetime_restarts, 1);
13698 assert_eq!(status.health.last_action.as_deref(), Some("restart"));
13699 assert!(harness.runtime.scheduled_respawn.lock().unwrap().is_some());
13700 }
13701 }
13702 }
13703
13704 #[tokio::test(start_paused = true)]
13705 async fn late_health_answers_record_start_anchored_latency_for_two_stalls() {
13706 let mut harness = probe_harness();
13707
13708 let first = finish_after(&mut harness, Duration::from_secs(8)).await;
13709 let first_latency = match &first {
13710 ModuleControlRpcCompletion::LateHealthAnswer { latency, .. } => *latency,
13711 other => panic!("late answer was not retained: {other:?}"),
13712 };
13713 assert!(harness.handler.observe_module_control_completion(first));
13714
13715 let second = finish_after(&mut harness, Duration::from_secs(11)).await;
13716 let second_latency = match &second {
13717 ModuleControlRpcCompletion::LateHealthAnswer { latency, .. } => *latency,
13718 other => panic!("late answer was not retained: {other:?}"),
13719 };
13720 assert!(harness.handler.observe_module_control_completion(second));
13721
13722 assert_eq!(first_latency, Duration::from_secs(8));
13723 assert_eq!(
13724 second_latency - first_latency,
13725 Duration::from_secs(3),
13726 "latency must grow linearly with the additional stall"
13727 );
13728 let health = harness.module.status().unwrap().health;
13729 assert_eq!(health.late_answer_count, 2);
13730 assert_eq!(health.last_late_answer_latency_ms, Some(11_000));
13731 }
13732
13733 #[tokio::test(start_paused = true)]
13741 async fn late_answer_clears_the_consecutive_failure_streak() {
13742 let mut harness = probe_harness();
13743
13744 time_out_without_answer(&mut harness).await;
13746 harness
13747 .module
13748 .record_health_probe_failure_for_test("[no-answer] test miss")
13749 .unwrap();
13750 assert_eq!(
13751 harness.module.status().unwrap().health.consecutive_failures,
13752 1,
13753 "precondition: the miss must be on the streak before the late answer"
13754 );
13755
13756 let late = finish_after(&mut harness, Duration::from_secs(9)).await;
13758 assert!(matches!(
13759 late,
13760 ModuleControlRpcCompletion::LateHealthAnswer { .. }
13761 ));
13762 assert!(harness.handler.observe_module_control_completion(late));
13763
13764 let health = harness.module.status().unwrap().health;
13765 assert_eq!(
13766 health.consecutive_failures, 0,
13767 "a late answer is an answer: the streak must reset"
13768 );
13769 assert_eq!(health.late_answer_count, 1);
13770 }
13771
13772 #[tokio::test(start_paused = true)]
13773 async fn repeated_serial_probe_cycles_keep_one_tombstone_per_endpoint() {
13774 let mut harness = probe_harness();
13775
13776 for _ in 0..20 {
13777 time_out_without_answer(&mut harness).await;
13778 assert_eq!(
13779 harness.forwarding.health_probe_tombstone_count().unwrap(),
13780 1
13781 );
13782 }
13783 }
13784}
13785
13786#[cfg(test)]
13787mod child_env_tests {
13788 #[cfg(windows)]
13789 use super::{
13790 clear_current_process_facts, observed_launch_nonce_source, ModuleState, SupervisorSnapshot,
13791 };
13792 #[cfg(windows)]
13793 use std::sync::{Arc, OnceLock};
13794 #[cfg(windows)]
13795 #[test]
13796 fn windows_wire_spawn_prepares_pipe_and_keeps_phase_one_env() {
13797 let spec = spec(Vec::new());
13798 let mut command = Command::new("unused");
13799 let handoff = apply_wire_spawn_args(&mut command, &spec, None, None)
13800 .unwrap()
13801 .unwrap();
13802 let name = command
13803 .as_std()
13804 .get_envs()
13805 .find(|(key, _)| *key == OsStr::new(subc_os::launch_nonce::LAUNCH_NONCE_PIPE_ENV))
13806 .and_then(|(_, value)| value)
13807 .unwrap();
13808 assert_eq!(name, OsStr::new(handoff.name()));
13809 assert!(handoff.name().starts_with(r"\\.\pipe\subc-launch-"));
13810 assert!(command.as_std().get_envs().any(|(key, value)| key
13811 == OsStr::new(subc_os::LAUNCH_NONCE_PIPE_FALLBACK_ENV)
13812 && value == Some(OsStr::new("env"))));
13813 assert!(command
13814 .as_std()
13815 .get_envs()
13816 .any(|(key, value)| key == OsStr::new(SUBC_LAUNCH_NONCE_ENV) && value.is_some()));
13817 }
13818
13819 #[cfg(windows)]
13820 #[test]
13821 fn windows_source_reporting_tracks_consumption_and_resets_per_spawn() {
13822 let mut snapshot = SupervisorSnapshot::new(ModuleState::Running, true);
13823 snapshot.process_alive = true;
13824 snapshot.spawned_protocol = Some(ModuleProtocol::Subc);
13825 assert_eq!(
13826 observed_launch_nonce_source(&snapshot).as_deref(),
13827 Some("env")
13828 );
13829 let consumed = Arc::new(OnceLock::new());
13830 snapshot.pipe_consumed = Some(consumed.clone());
13831 assert_eq!(
13832 observed_launch_nonce_source(&snapshot).as_deref(),
13833 Some("env")
13834 );
13835 consumed.set(()).unwrap();
13836 assert_eq!(
13837 observed_launch_nonce_source(&snapshot).as_deref(),
13838 Some("pipe")
13839 );
13840 snapshot.pipe_consumed = Some(Arc::new(OnceLock::new()));
13841 assert_eq!(
13842 observed_launch_nonce_source(&snapshot).as_deref(),
13843 Some("env")
13844 );
13845 clear_current_process_facts(&mut snapshot);
13846 assert_eq!(observed_launch_nonce_source(&snapshot), None);
13847 }
13848 use super::{
13849 apply_child_env, apply_spawn_role, apply_wire_spawn_args, ModuleProtocol, ModuleSpec,
13850 SpawnRole, SupervisorHandle, SPAWN_ROLE_SWAP_CANDIDATE, SUBC_ARG, SUBC_LAUNCH_NONCE_ENV,
13851 SUBC_MODULE_ID_ENV, SUBC_SPAWN_ROLE_ENV,
13852 };
13853 use std::{ffi::OsStr, path::PathBuf};
13854 use tokio::process::Command;
13855
13856 fn spec(env: Vec<(String, String)>) -> ModuleSpec {
13857 ModuleSpec {
13858 module_id: "env-plan".to_string(),
13859 program: PathBuf::from("/nonexistent"),
13860 args: Vec::new(),
13861 env,
13862 reserved: false,
13863 reserved_prefixes: Vec::new(),
13864 protocol: ModuleProtocol::Subc,
13865 overlap: Default::default(),
13866 }
13867 }
13868
13869 #[test]
13883 fn ambient_ck_log_is_removed_and_a_configured_one_survives() {
13884 let mut command = Command::new("/nonexistent");
13885 apply_child_env(&mut command, &spec(Vec::new()));
13886 let removed = command
13887 .as_std()
13888 .get_envs()
13889 .any(|(key, value)| key == OsStr::new("CK_LOG") && value.is_none());
13890 assert!(
13891 removed,
13892 "ambient CK_LOG must be explicitly removed for an unconfigured module"
13893 );
13894
13895 let mut configured = Command::new("/nonexistent");
13896 apply_child_env(
13897 &mut configured,
13898 &spec(vec![("CK_LOG".to_string(), "debug".to_string())]),
13899 );
13900 let effective = configured
13901 .as_std()
13902 .get_envs()
13903 .filter(|(key, _)| *key == OsStr::new("CK_LOG"))
13904 .last()
13905 .map(|(_, value)| value.map(|v| v.to_string_lossy().into_owned()));
13906 assert_eq!(
13907 effective,
13908 Some(Some("debug".to_string())),
13909 "a module's configured CK_LOG must survive the ambient removal"
13910 );
13911 }
13912
13913 #[test]
13922 fn protocol_none_spawn_carries_no_subc_argument_and_no_nonce() {
13923 let connection_file = std::path::Path::new("/run/subc-connection.json");
13924 let handle = SupervisorHandle::new();
13925
13926 let mut none_spec = spec(Vec::new());
13927 none_spec.protocol = ModuleProtocol::None;
13928 let mut none = Command::new("/nonexistent");
13929 let none_handoff =
13930 apply_wire_spawn_args(&mut none, &none_spec, Some(connection_file), Some(&handle))
13931 .expect("protocol-none spawn args apply");
13932 assert!(
13933 none_handoff.is_none(),
13934 "protocol:none spawn must not receive a nonce descriptor"
13935 );
13936 #[cfg(windows)]
13937 assert!(!none.as_std().get_envs().any(|(key, value)| key
13938 == OsStr::new(subc_os::LAUNCH_NONCE_PIPE_FALLBACK_ENV)
13939 && value.is_some()));
13940 assert!(
13941 !none.as_std().get_envs().any(|(key, value)| key
13942 == OsStr::new(subc_os::LAUNCH_NONCE_FD_ENV)
13943 && value.is_some()),
13944 "protocol:none spawn must not name a nonce descriptor"
13945 );
13946 let none_args: Vec<String> = none
13947 .as_std()
13948 .get_args()
13949 .map(|a| a.to_string_lossy().into_owned())
13950 .collect();
13951 assert!(
13952 !none_args.iter().any(|a| a == SUBC_ARG),
13953 "protocol:none argv must not carry --subc; got {none_args:?}"
13954 );
13955 let none_has_nonce = none
13956 .as_std()
13957 .get_envs()
13958 .any(|(key, value)| key == OsStr::new(SUBC_LAUNCH_NONCE_ENV) && value.is_some());
13959 assert!(
13960 !none_has_nonce,
13961 "protocol:none spawn must not receive a launch nonce"
13962 );
13963 let none_has_module_id = none
13964 .as_std()
13965 .get_envs()
13966 .any(|(key, value)| key == OsStr::new(SUBC_MODULE_ID_ENV) && value.is_some());
13967 assert!(
13968 none_has_module_id,
13969 "SUBC_MODULE_ID is inert and stays on every path"
13970 );
13971 assert!(
13972 handle.spawn_nonce(&none_spec.module_id).is_none(),
13973 "no nonce record for a process that will never present one"
13974 );
13975
13976 let wire_spec = spec(Vec::new());
13978 let mut wire = Command::new("/nonexistent");
13979 let wire_handoff =
13980 apply_wire_spawn_args(&mut wire, &wire_spec, Some(connection_file), Some(&handle))
13981 .expect("subc-wire spawn args apply");
13982 let wire_fd_env = wire
13983 .as_std()
13984 .get_envs()
13985 .find(|(key, _)| *key == OsStr::new(subc_os::LAUNCH_NONCE_FD_ENV))
13986 .and_then(|(_, value)| value.map(|v| v.to_string_lossy().into_owned()));
13987 #[cfg(unix)]
13988 assert_eq!(
13989 wire_fd_env,
13990 Some(wire_handoff.expect("a descriptor handoff").fd_env_value()),
13991 "a subc-wire spawn names the pipe it will receive at descriptor 3"
13992 );
13993 #[cfg(windows)]
13994 assert!(wire_handoff.is_some() && wire_fd_env.is_none());
13995 #[cfg(not(any(unix, windows)))]
13996 assert!(wire_handoff.is_none() && wire_fd_env.is_none());
13997 let wire_args: Vec<String> = wire
13998 .as_std()
13999 .get_args()
14000 .map(|a| a.to_string_lossy().into_owned())
14001 .collect();
14002 assert_eq!(
14003 wire_args,
14004 vec![
14005 SUBC_ARG.to_string(),
14006 connection_file.to_string_lossy().into_owned()
14007 ],
14008 "a subc-wire spawn still carries --subc <path>"
14009 );
14010 assert_eq!(
14011 wire.as_std()
14012 .get_envs()
14013 .any(|(key, value)| key == OsStr::new(SUBC_LAUNCH_NONCE_ENV) && value.is_some()),
14014 !cfg!(unix),
14015 "only Windows supplies the environment nonce"
14016 );
14017 assert!(handle.spawn_nonce(&wire_spec.module_id).is_some());
14018 }
14019
14020 #[test]
14030 fn plain_spawn_removes_the_spawn_role_even_when_the_spec_sets_it() {
14031 let role = |command: &Command| {
14032 command
14033 .as_std()
14034 .get_envs()
14035 .filter(|(key, _)| *key == OsStr::new(SUBC_SPAWN_ROLE_ENV))
14036 .last()
14037 .map(|(_, value)| value.map(|v| v.to_string_lossy().into_owned()))
14038 };
14039 let forged = spec(vec![(
14040 SUBC_SPAWN_ROLE_ENV.to_string(),
14041 SPAWN_ROLE_SWAP_CANDIDATE.to_string(),
14042 )]);
14043
14044 let mut plain = Command::new("/nonexistent");
14045 apply_child_env(&mut plain, &forged);
14046 apply_spawn_role(&mut plain, SpawnRole::Plain);
14047 assert_eq!(
14048 role(&plain),
14049 Some(None),
14050 "a plain spawn must remove SUBC_SPAWN_ROLE, whatever the spec says"
14051 );
14052
14053 let mut candidate = Command::new("/nonexistent");
14054 apply_child_env(&mut candidate, &spec(Vec::new()));
14055 apply_spawn_role(&mut candidate, SpawnRole::SwapCandidate);
14056 assert_eq!(
14057 role(&candidate),
14058 Some(Some(SPAWN_ROLE_SWAP_CANDIDATE.to_string()))
14059 );
14060 }
14061
14062 #[test]
14068 fn daemon_private_capture_keys_are_not_passed_to_the_child() {
14069 let mut command = Command::new("/nonexistent");
14070 apply_child_env(
14071 &mut command,
14072 &spec(vec![
14073 (super::CAPTURE_KEEP_ENV.to_string(), "5".to_string()),
14074 ("KEPT".to_string(), "yes".to_string()),
14075 ]),
14076 );
14077 let keys: Vec<String> = command
14078 .as_std()
14079 .get_envs()
14080 .filter(|(_, value)| value.is_some())
14081 .map(|(key, _)| key.to_string_lossy().into_owned())
14082 .collect();
14083 assert!(keys.contains(&"KEPT".to_string()), "got {keys:?}");
14084 assert!(
14085 !keys.contains(&super::CAPTURE_KEEP_ENV.to_string()),
14086 "daemon-private capture key leaked to the child: {keys:?}"
14087 );
14088 }
14089}
14090
14091#[cfg(test)]
14092mod jitter_tests {
14093 use super::jittered_health_delay;
14094 use std::{collections::HashSet, time::Duration};
14095
14096 const FLEET: [&str; 14] = [
14105 "aft",
14106 "alfonso-core",
14107 "magic-context",
14108 "broca",
14109 "thalamus",
14110 "quota",
14111 "engram",
14112 "plexus",
14113 "cerebellum",
14114 "astrocyte",
14115 "synapse",
14116 "subc-mcp",
14117 "cortexkit-credentials",
14118 "subc-federation",
14119 ];
14120
14121 #[test]
14129 fn probe_delays_disperse_across_the_fleet() {
14130 let cadence = Duration::from_secs(30);
14131 let delays: HashSet<Duration> = FLEET
14132 .iter()
14133 .map(|id| jittered_health_delay(id, 0, cadence))
14134 .collect();
14135 assert_eq!(
14136 delays.len(),
14137 FLEET.len(),
14138 "every supervised module must land on its own probe offset"
14139 );
14140 }
14141
14142 #[test]
14148 fn jitter_only_delays_and_stays_within_one_tenth_of_cadence() {
14149 let cadence = Duration::from_secs(30);
14150 let span = cadence / 10;
14151 for id in FLEET {
14152 for probe_index in 0..8 {
14153 let delay = jittered_health_delay(id, probe_index, cadence);
14154 assert!(
14155 delay >= cadence,
14156 "{id}#{probe_index}: jitter must not shorten the cadence"
14157 );
14158 assert!(
14159 delay < cadence + span,
14160 "{id}#{probe_index}: jitter must stay inside one tenth of the cadence"
14161 );
14162 }
14163 }
14164 }
14165
14166 #[test]
14172 fn a_module_offset_is_stable_across_restarts() {
14173 let cadence = Duration::from_secs(30);
14174 for id in FLEET {
14175 assert_eq!(
14176 jittered_health_delay(id, 0, cadence),
14177 jittered_health_delay(id, 0, cadence),
14178 "{id}: the same module and probe index must produce the same offset"
14179 );
14180 }
14181 }
14182
14183 #[test]
14185 fn zero_cadence_yields_zero_delay() {
14186 assert_eq!(
14187 jittered_health_delay("aft", 0, Duration::ZERO),
14188 Duration::ZERO
14189 );
14190 }
14191}
14192
14193#[cfg(all(test, target_os = "linux"))]
14194mod cgroup_placement_tests {
14195 use super::{
14196 apply_cgroup_placement, remove_module_cgroup, ModuleProtocol, ModuleSpec, SuperviseError,
14197 SupervisedChild,
14198 };
14199 use crate::stderr_tail::{StderrRing, StderrTailConfig};
14200 use cortexkit_test_support::ScratchDir;
14201 use std::{
14202 fs, io,
14203 path::{Path, PathBuf},
14204 sync::{Arc, Mutex},
14205 };
14206 use tokio::process::Command;
14207
14208 #[tokio::test]
14209 async fn unique_spawn_cgroups_clean_up_across_restarts_and_shutdown() {
14210 use super::*;
14211 let dir = ScratchDir::new("unique-spawn-cgroups");
14212 let root = PathBuf::from(format!(
14213 "/sys/fs/cgroup/subc-unique-test-{}-{}",
14214 std::process::id(),
14215 unix_ms_now()
14216 ));
14217 if let Err(error) = fs::create_dir(&root) {
14218 assert!(
14219 std::env::var_os("SUBC_REQUIRE_CGROUP_TEST").is_none(),
14220 "required cgroup test cannot execute: {error}"
14221 );
14222 eprintln!("SKIP unique_spawn_cgroups_clean_up_across_restarts_and_shutdown: {error}");
14223 return;
14224 }
14225 let placement = subc_cgroup::prepare_at(&root).unwrap().unwrap();
14226 let group_count = || {
14227 fs::read_dir(root.join("subc-modules"))
14228 .unwrap()
14229 .map(|entry| entry.unwrap().file_type().unwrap())
14230 .filter(|kind| kind.is_dir())
14231 .count()
14232 };
14233 let supervisor =
14234 Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default())
14235 .with_cgroup_placement(Some(placement.clone()));
14236 let runtime = supervisor.runtime_config();
14237 let mut spec = ModuleSpec {
14238 module_id: "unique-spawn".into(),
14239 program: PathBuf::from("/bin/sleep"),
14240 args: vec!["60".into()],
14241 env: ["XDG_DATA_HOME", "XDG_RUNTIME_DIR", "XDG_CONFIG_HOME"]
14242 .into_iter()
14243 .map(|key| (key.into(), dir.to_string_lossy().into_owned()))
14244 .collect(),
14245 reserved: false,
14246 reserved_prefixes: vec![],
14247 protocol: ModuleProtocol::None,
14248 overlap: Default::default(),
14249 };
14250 let spawn = |spec: &ModuleSpec| {
14251 spawn_child(
14252 spec,
14253 None,
14254 None,
14255 &runtime.stderr_ring,
14256 None,
14257 &runtime.child_roster,
14258 Some(&placement),
14259 )
14260 .unwrap()
14261 };
14262 let mut live = spawn(&spec);
14263 for _ in 0..3 {
14264 let next = spawn(&spec);
14266 assert_ne!(live.module_id, next.module_id);
14267 live.start_kill().unwrap();
14268 live.wait().await.unwrap();
14269 live = next;
14270 assert!(
14271 live.child.try_wait().unwrap().is_none(),
14272 "retiring the old slot must not kill the replacement"
14273 );
14274 assert_eq!(
14275 group_count(),
14276 1,
14277 "only the live spawn's cgroup should remain"
14278 );
14279 }
14280 supervisor.begin_daemon_shutdown();
14281 let reap = tokio::spawn(async move {
14282 live.wait().await.unwrap();
14283 });
14284 supervisor
14285 .end_children_for_daemon_shutdown(false, std::future::pending())
14286 .await;
14287 reap.await.unwrap();
14288 assert_eq!(group_count(), 0);
14289 spec.program = PathBuf::from("/bin/true");
14291 spec.args.clear();
14292 let fresh_roster = ChildRoster::default();
14293 let mut short = spawn_child(
14294 &spec,
14295 None,
14296 None,
14297 &runtime.stderr_ring,
14298 None,
14299 &fresh_roster,
14300 Some(&placement),
14301 )
14302 .unwrap();
14303 short.wait().await.unwrap();
14304 assert_eq!(group_count(), 0);
14305 spec.module_id = "_".repeat(255);
14306 let mut long_id = spawn_child(
14307 &spec,
14308 None,
14309 None,
14310 &runtime.stderr_ring,
14311 None,
14312 &fresh_roster,
14313 Some(&placement),
14314 )
14315 .unwrap();
14316 long_id.wait().await.unwrap();
14317 assert_eq!(
14318 group_count(),
14319 0,
14320 "valid long module IDs must not exceed cgroup NAME_MAX"
14321 );
14322 fs::remove_dir(root.join("subc-modules")).unwrap();
14323 fs::remove_dir(root).unwrap();
14324 eprintln!("EXECUTED unique_spawn_cgroups_clean_up_across_restarts_and_shutdown");
14325 }
14326
14327 #[test]
14328 fn failed_parent_cgroup_open_is_a_cgroup_supervision_error() {
14329 let path = Path::new("/definitely-missing-subc-cgroup");
14330 let mut command = Command::new("true");
14331 let error = apply_cgroup_placement(
14332 &mut command,
14333 &ModuleSpec {
14334 module_id: "broken-cgroup".to_string(),
14335 program: PathBuf::from("true"),
14336 args: Vec::new(),
14337 env: Vec::new(),
14338 reserved: false,
14339 reserved_prefixes: Vec::new(),
14340 protocol: ModuleProtocol::Subc,
14341 overlap: Default::default(),
14342 },
14343 path,
14344 )
14345 .expect_err("a parent cgroup open failure must reject the supervised spawn");
14346 let reason = error.to_string();
14347
14348 assert!(
14349 matches!(error, SuperviseError::Cgroup { .. }),
14350 "parent cgroup open must be reported as a cgroup supervision error: {reason}"
14351 );
14352 assert!(
14353 reason.contains("/definitely-missing-subc-cgroup/cgroup.procs"),
14354 "parent cgroup open failure must name cgroup.procs: {reason}"
14355 );
14356 }
14357
14358 #[tokio::test]
14359 async fn reaping_a_child_removes_its_empty_module_cgroup() {
14360 let root = ScratchDir::new("supervisor-reap-cgroup");
14361 fs::write(root.join("cgroup.procs"), b"").expect("write scratch cgroup marker");
14362 let placement = subc_cgroup::prepare_at(&root)
14363 .expect("prepare scratch cgroup root")
14364 .expect("scratch root has a cgroup.procs marker");
14365 let module_id = "reaped-module";
14366 let module = placement
14367 .module_path(module_id)
14368 .expect("create scratch module cgroup");
14369 let child = Command::new("true")
14370 .env("XDG_DATA_HOME", root.path())
14371 .env("XDG_RUNTIME_DIR", root.path())
14372 .env("XDG_CONFIG_HOME", root.path())
14373 .spawn()
14374 .expect("spawn short-lived child");
14375 let pid = child.id().expect("spawned child has pid");
14376 let mut child = SupervisedChild {
14377 child,
14378 exec: crate::activity::ExecOwner(Arc::new(crate::activity::ExecOutcome::new())),
14379 protocol: ModuleProtocol::Subc,
14380 module_id: module_id.to_string(),
14381 cgroup_placement: Some(placement),
14382 stdout_pump: None,
14383 stderr_pump: None,
14384 stderr_ring: Arc::new(Mutex::new(StderrRing::new(StderrTailConfig::default()))),
14385 spawned_at_ms: 0,
14386 spawned_from: PathBuf::from("true"),
14387 spawned_file_identity: None,
14388 process_start_time: None,
14389 process_identity: None,
14390 pid,
14391 roster_guard: None,
14392 #[cfg(target_os = "macos")]
14393 privacy_exec: None,
14394 spawn_failure: None,
14395 };
14396
14397 child.wait().await.expect("reap short-lived child");
14398
14399 assert!(
14400 !module.exists(),
14401 "reaping the supervised child must remove its empty cgroup"
14402 );
14403 }
14404
14405 #[test]
14406 fn non_empty_cgroup_removal_is_reported_without_blocking_teardown() {
14407 let root = ScratchDir::new("supervisor-non-empty-cgroup");
14408 fs::write(root.join("cgroup.procs"), b"").expect("write scratch cgroup marker");
14409 let placement = subc_cgroup::prepare_at(&root)
14410 .expect("prepare scratch cgroup root")
14411 .expect("scratch root has a cgroup.procs marker");
14412 let module = placement
14413 .module_path("surviving-module")
14414 .expect("create scratch module cgroup");
14415 fs::write(module.join("surviving-process"), b"still present")
14416 .expect("make scratch cgroup non-empty");
14417 let (logs, _guard) = crate::router::test_log::log_capture(tracing::Level::WARN);
14418
14419 remove_module_cgroup(&placement, "surviving-module");
14420
14421 let logs = crate::router::test_log::captured_logs(&logs);
14422 assert!(
14423 module.exists(),
14424 "failed removal must leave the cgroup intact"
14425 );
14426 assert!(
14427 logs.contains("could not remove module cgroup after process exit; continuing teardown")
14428 && logs.contains("surviving-module"),
14429 "best-effort removal must report the failure without returning it: {logs}"
14430 );
14431 }
14432
14433 #[test]
14434 fn cgroup_pre_exec_spawn_failure_names_the_cgroup_path() {
14435 let cgroup_path = PathBuf::from("/sys/fs/cgroup/subc-modules/broken-module");
14436 let reason = SuperviseError::Spawn {
14437 program: PathBuf::from("/bin/true"),
14438 source: io::Error::from_raw_os_error(13),
14439 cgroup_path: Some(cgroup_path.clone()),
14440 }
14441 .to_string();
14442
14443 assert!(
14444 reason.contains(&cgroup_path.display().to_string()),
14445 "a pre_exec spawn failure must name the cgroup path: {reason}"
14446 );
14447 }
14448}
14449
14450#[cfg(test)]
14451mod spawn_subscriber_lag_tests {
14452 use super::*;
14453
14454 #[tokio::test]
14459 async fn lagged_spawn_subscriber_receives_a_terminal_lagged_error_after_its_queued_frames() {
14460 let feed = SpawnEventFeed::default();
14461 feed.configure_incarnation("lag-incarnation".to_string());
14462 let (tx, mut rx) = mpsc::channel(1);
14465 feed.subscribe(ConnectionId::new(1), 7, 1, None, FrameSink::new(tx))
14466 .expect("subscribe");
14467 let emitted = SPAWN_SUBSCRIBER_BUFFER + 16;
14468 for index in 0..emitted {
14469 feed.emit_spawned(&format!("lag-module-{index}"), 1000, 0);
14470 tokio::task::yield_now().await;
14473 }
14474 assert_eq!(
14475 feed.subscriber_count(),
14476 0,
14477 "the lagged subscriber must be removed"
14478 );
14479
14480 let mut data = Vec::new();
14481 let mut last = None;
14482 loop {
14483 let next = tokio::time::timeout(Duration::from_secs(5), rx.recv())
14484 .await
14485 .expect("the forwarder must finish once the subscriber is dropped");
14486 let Some(outbound) = next else { break };
14487 let frame = outbound.frame;
14488 if frame.header.ty == FrameType::StreamData {
14489 assert!(last.is_none(), "no data may follow the terminal frame");
14490 let event: SpawnEvent = serde_json::from_slice(&frame.body).unwrap();
14491 data.push(event.cursor.seq);
14492 } else {
14493 assert!(last.is_none(), "exactly one terminal frame");
14494 last = Some(frame);
14495 }
14496 }
14497 assert!(!data.is_empty(), "queued frames drain before the terminal");
14498 for pair in data.windows(2) {
14499 assert_eq!(
14500 pair[1],
14501 pair[0] + 1,
14502 "queued frames arrive dense and in order"
14503 );
14504 }
14505 let terminal = last.expect("a lagged subscriber must receive a terminal frame");
14506 assert_eq!(terminal.header.ty, FrameType::Error);
14507 assert_eq!(terminal.header.corr, 7);
14508 let body: subc_protocol::ErrorBody = serde_json::from_slice(&terminal.body).unwrap();
14509 assert_eq!(body.code, SPAWN_SUBSCRIBER_LAGGED_CODE);
14510 let detail = body.detail.expect("lagged error carries detail");
14511 assert_eq!(
14512 detail["first_undelivered_cursor"]["seq"],
14513 data.last().unwrap() + 1,
14514 "the named cursor is the first event the subscriber did not receive"
14515 );
14516 assert_eq!(
14517 detail["first_undelivered_cursor"]["daemon_incarnation"],
14518 "lag-incarnation"
14519 );
14520 }
14521}
14522
14523#[cfg(test)]
14524mod terminal_history_read_concurrency_tests {
14525 use super::*;
14526 use crate::terminal_journal::read_pause;
14527 use cortexkit_test_support::ScratchDir;
14528 use std::sync::mpsc as std_mpsc;
14529
14530 fn journaled_ring(
14531 journal: &Arc<crate::terminal_journal::TerminalJournal>,
14532 ) -> Arc<Mutex<TerminalRing>> {
14533 Arc::new(Mutex::new(
14534 TerminalRing::new(TerminalRingConfig::default(), 1)
14535 .with_journal(Some(Arc::clone(journal))),
14536 ))
14537 }
14538
14539 fn crash(at_ms: u64) -> ExitReport {
14540 ExitReport {
14541 kind: ExitKind::Crash,
14542 code: Some(1),
14543 signal: None,
14544 at_ms,
14545 }
14546 }
14547
14548 fn record_within(
14551 module_id: &'static str,
14552 ring: &Arc<Mutex<TerminalRing>>,
14553 at_ms: u64,
14554 bound: Duration,
14555 ) -> bool {
14556 let ring = Arc::clone(ring);
14557 let (done, done_rx) = std_mpsc::channel();
14558 std::thread::spawn(move || {
14559 record_terminal(
14560 module_id,
14561 &ring,
14562 &SpawnEventFeed::default(),
14563 &crash(at_ms),
14564 TerminalDisposition::Restarting,
14565 );
14566 let _ = done.send(());
14567 });
14568 done_rx.recv_timeout(bound).is_ok()
14569 }
14570
14571 #[test]
14576 fn exits_recorded_during_a_paused_history_read_are_not_blocked_or_half_merged() {
14577 let dir = ScratchDir::new("terminal-history-concurrent-read");
14578 let path = dir.join("terminals.jsonl");
14579 let journal = Arc::new(crate::terminal_journal::TerminalJournal::open(
14580 path.clone(),
14581 "daemon".into(),
14582 ));
14583 let reader_ring = journaled_ring(&journal);
14584 let other_ring = journaled_ring(&journal);
14585 assert!(record_within(
14586 "reader-module",
14587 &reader_ring,
14588 10,
14589 Duration::from_secs(5)
14590 ));
14591
14592 let (started, release) = read_pause::install(&path);
14593 let reading = {
14594 let ring = Arc::clone(&reader_ring);
14595 std::thread::spawn(move || durable_terminal_history_of(&ring, "reader-module"))
14596 };
14597 started
14598 .recv_timeout(Duration::from_secs(5))
14599 .expect("the history read reached its pause");
14600
14601 let bound = Duration::from_secs(1);
14602 assert!(
14603 record_within("other-module", &other_ring, 20, bound),
14604 "another module's exit waited on a history read (journal writer held)"
14605 );
14606 assert!(
14607 record_within("reader-module", &reader_ring, 30, bound),
14608 "the read module's own exit waited on its history read (ring held)"
14609 );
14610
14611 drop(release);
14612 let paused = reading.join().unwrap();
14613 assert_eq!(
14614 paused.entries.iter().map(|e| e.at_ms).collect::<Vec<_>>(),
14615 vec![10],
14616 "an exit recorded after the read began lands in neither half of it"
14617 );
14618 assert_eq!(paused.journal_skipped_lines, 0);
14619 assert_eq!(paused.journal_read_errors, 0);
14620
14621 let after = durable_terminal_history_of(&reader_ring, "reader-module");
14622 assert_eq!(
14623 after.entries.iter().map(|e| e.at_ms).collect::<Vec<_>>(),
14624 vec![10, 30],
14625 "the next read merges ring and journal with no duplicate"
14626 );
14627 assert_eq!(after.journal_skipped_lines, 0);
14628 }
14629}
14630
14631#[cfg(test)]
14636mod stderr_settle_tests {
14637 use std::{
14638 future::Future,
14639 io,
14640 pin::Pin,
14641 sync::{Arc, Mutex},
14642 task::{Context, Poll},
14643 time::Duration,
14644 };
14645
14646 use tokio::{
14647 io::{AsyncRead, ReadBuf},
14648 sync::oneshot,
14649 time::Instant,
14650 };
14651
14652 use super::{settle_stderr_pump, settle_stdout_pump, StderrPump};
14653 use crate::stderr_tail::{
14654 pump_stderr_to, pump_stdout_to, split_capture_stamp, untimed, CaptureState,
14655 ChildOutputSink, OutputSink, StderrRing, StderrTailConfig, TailEntry,
14656 };
14657
14658 const BOUND: Duration = Duration::from_millis(250);
14659
14660 struct HeldReader {
14664 before: Option<Vec<u8>>,
14665 gate: Option<oneshot::Receiver<()>>,
14666 after: io::Cursor<Vec<u8>>,
14667 }
14668
14669 impl AsyncRead for HeldReader {
14670 fn poll_read(
14671 mut self: Pin<&mut Self>,
14672 cx: &mut Context<'_>,
14673 buf: &mut ReadBuf<'_>,
14674 ) -> Poll<io::Result<()>> {
14675 if let Some(bytes) = self.before.take() {
14676 buf.put_slice(&bytes);
14677 return Poll::Ready(Ok(()));
14678 }
14679 if let Some(gate) = self.gate.as_mut() {
14680 match Pin::new(gate).poll(cx) {
14681 Poll::Pending => return Poll::Pending,
14682 Poll::Ready(_) => self.gate = None,
14683 }
14684 }
14685 Pin::new(&mut self.after).poll_read(cx, buf)
14686 }
14687 }
14688
14689 struct DiscardSink;
14690
14691 impl OutputSink for DiscardSink {
14692 fn write_line(&mut self, _line: &[u8]) {}
14693 }
14694
14695 fn line(text: &str) -> TailEntry {
14696 TailEntry::Line {
14697 text: text.to_string(),
14698 truncated: false,
14699 at_ms: None,
14700 }
14701 }
14702
14703 fn lock(ring: &Arc<Mutex<StderrRing>>) -> std::sync::MutexGuard<'_, StderrRing> {
14704 ring.lock().unwrap()
14705 }
14706
14707 fn held_pump(
14711 ring: &Arc<Mutex<StderrRing>>,
14712 before: &str,
14713 after: &str,
14714 ) -> (StderrPump, oneshot::Sender<()>) {
14715 let generation = lock(ring).begin_process();
14716 let (release, gate) = oneshot::channel();
14717 let reader = HeldReader {
14718 before: Some(before.as_bytes().to_vec()),
14719 gate: Some(gate),
14720 after: io::Cursor::new(after.as_bytes().to_vec()),
14721 };
14722 let task = tokio::spawn(pump_stderr_to(
14723 reader,
14724 Arc::clone(ring),
14725 generation,
14726 DiscardSink,
14727 ));
14728 (StderrPump { task, generation }, release)
14729 }
14730
14731 async fn wait_until(ring: &Arc<Mutex<StderrRing>>, done: impl Fn(&StderrRing) -> bool) {
14732 for _ in 0..1000 {
14733 if done(&lock(ring)) {
14734 return;
14735 }
14736 tokio::time::sleep(Duration::from_millis(1)).await;
14737 }
14738 panic!(
14739 "ring never reached the expected state: {:?}",
14740 lock(ring).snapshot(None, None)
14741 );
14742 }
14743
14744 #[tokio::test(start_paused = true)]
14745 async fn a_crash_line_the_reader_had_not_reached_by_the_bound_is_kept_before_the_restart() {
14746 let ring = Arc::new(Mutex::new(StderrRing::new(StderrTailConfig::default())));
14747 let (pump, release) = held_pump(&ring, "booting\n", "config error: missing storage\n");
14748
14749 settle_stderr_pump("crasher", &ring, pump, BOUND).await;
14750 let before_release = lock(&ring).snapshot(None, None);
14751 assert!(
14752 matches!(before_release.capture, CaptureState::Incomplete { .. }),
14753 "a reader that has not reached EOF cannot claim a whole tail: {before_release:?}"
14754 );
14755
14756 let next = lock(&ring).begin_process();
14759 lock(&ring).push_line_from(next, "next process booting");
14760 release.send(()).unwrap();
14761 wait_until(&ring, |ring| {
14762 ring.snapshot(None, None).capture == CaptureState::Captured
14763 })
14764 .await;
14765
14766 assert_eq!(
14767 untimed(lock(&ring).snapshot(None, None).entries),
14768 vec![
14769 line("booting"),
14770 line("config error: missing storage"),
14771 TailEntry::ProcessStart,
14772 line("next process booting"),
14773 ],
14774 "the crash's last line must survive a slow reader and stay in the crashed process's section"
14775 );
14776 }
14777
14778 #[tokio::test(start_paused = true)]
14779 async fn a_pipe_held_open_by_a_descendant_reads_incomplete_without_delaying_the_restart_past_the_bound(
14780 ) {
14781 let ring = Arc::new(Mutex::new(StderrRing::new(StderrTailConfig::default())));
14782 let (pump, _held) = held_pump(&ring, "parent exiting\n", "");
14785
14786 let started = Instant::now();
14787 settle_stderr_pump("orphaning", &ring, pump, BOUND).await;
14788 assert_eq!(
14789 started.elapsed(),
14790 BOUND,
14791 "the restart must wait exactly the bound for a pipe that stays open, no longer"
14792 );
14793
14794 let next = lock(&ring).begin_process();
14795 lock(&ring).push_line_from(next, "next process booting");
14796 tokio::time::sleep(Duration::from_secs(60)).await;
14797
14798 let snapshot = lock(&ring).snapshot(None, None);
14799 match &snapshot.capture {
14800 CaptureState::Incomplete { reason } => assert!(
14801 reason.contains("had not reached EOF") && reason.contains("250ms"),
14802 "the reason must say what is missing and after how long: {reason}"
14803 ),
14804 other => panic!("expected Incomplete while the pipe is held open, got {other:?}"),
14805 }
14806 assert_eq!(
14807 untimed(snapshot.entries),
14808 vec![
14809 line("parent exiting"),
14810 TailEntry::ProcessStart,
14811 line("next process booting"),
14812 ]
14813 );
14814 }
14815
14816 #[tokio::test(start_paused = true)]
14817 async fn a_reader_that_reaches_eof_within_the_bound_leaves_the_tail_captured() {
14818 let ring = Arc::new(Mutex::new(StderrRing::new(StderrTailConfig::default())));
14819 let (pump, release) = held_pump(&ring, "one\n", "two\n");
14820 release.send(()).unwrap();
14821
14822 settle_stderr_pump("clean", &ring, pump, BOUND).await;
14823
14824 let snapshot = lock(&ring).snapshot(None, None);
14825 assert_eq!(snapshot.capture, CaptureState::Captured);
14826 assert_eq!(untimed(snapshot.entries), vec![line("one"), line("two")]);
14827 }
14828
14829 #[tokio::test(start_paused = true)]
14830 async fn stdout_buffered_past_the_exit_drain_bound_reaches_the_capture_file() {
14831 const BURST: usize = 150;
14832 let capture = cortexkit_test_support::ScratchDir::new("stdout-exit-drain");
14833 let path = capture.join("two-pipe.stderr.log");
14834 let sink = ChildOutputSink::open(&path, cortexkit_log::Retention::default()).unwrap();
14835 let ring = Arc::new(Mutex::new(StderrRing::new(StderrTailConfig::default())));
14836 let generation = lock(&ring).begin_process();
14837 let stdout = (0..BURST)
14838 .map(|i| format!("out-{i:04}-{}\n", "o".repeat(64)))
14839 .collect::<String>();
14840 let stderr = (0..BURST)
14841 .map(|i| format!("err-{i:04}-{}\n", "e".repeat(64)))
14842 .collect::<String>();
14843 let first_chunk = stdout.find('\n').unwrap() + 1 + "out-0001-".len();
14848 let (release, gate) = oneshot::channel();
14849 let reader = HeldReader {
14850 before: Some(stdout.as_bytes()[..first_chunk].to_vec()),
14851 gate: Some(gate),
14852 after: io::Cursor::new(stdout.as_bytes()[first_chunk..stdout.len() - 1].to_vec()),
14853 };
14854 let (finished, completion) = oneshot::channel();
14855 let stdout_sink = sink.clone();
14856 let pump = tokio::spawn(async move {
14857 pump_stdout_to(reader, stdout_sink).await;
14858 finished.send(()).unwrap();
14859 });
14860 pump_stderr_to(io::Cursor::new(stderr.into_bytes()), ring, generation, sink).await;
14861
14862 let started = Instant::now();
14863 settle_stdout_pump("two-pipe", pump, BOUND).await;
14864 assert_eq!(
14865 started.elapsed(),
14866 BOUND,
14867 "a late reader must not hold up restart"
14868 );
14869 assert_eq!(
14870 std::fs::read_to_string(&path).unwrap().lines().count(),
14871 BURST + 1
14872 );
14873
14874 release
14875 .send(())
14876 .expect("stdout reader must survive the exit drain bound");
14877 tokio::time::timeout(Duration::from_secs(1), completion)
14878 .await
14879 .expect("released stdout reader must reach EOF")
14880 .expect("stdout capture must finish at EOF, not be cancelled");
14881 let contents = std::fs::read_to_string(&path).unwrap();
14882 let mut lines = contents
14883 .lines()
14884 .map(|line| split_capture_stamp(line).expect("capture stamp").1)
14885 .collect::<Vec<_>>();
14886 let mut expected = (0..BURST)
14887 .flat_map(|i| {
14888 [
14889 format!("out-{i:04}-{}", "o".repeat(64)),
14890 format!("err-{i:04}-{}", "e".repeat(64)),
14891 ]
14892 })
14893 .collect::<Vec<_>>();
14894 lines.sort_unstable();
14895 expected.sort_unstable();
14896 assert_eq!(
14897 lines, expected,
14898 "both pipes must retain every complete line exactly once"
14899 );
14900 }
14901}
14902
14903#[cfg(all(test, windows))]
14917mod job_containment_tests {
14918 use super::*;
14919 use cortexkit_test_support::ScratchDir;
14920 use std::{
14921 path::{Path, PathBuf},
14922 sync::{Arc, Mutex},
14923 time::{Duration, Instant},
14924 };
14925
14926 fn stub_path() -> PathBuf {
14932 let mut path = std::env::current_exe().expect("current_exe available in tests");
14933 path.pop();
14934 path.pop();
14935 path.push("fake-aft-stub.exe");
14936 assert!(
14937 path.exists(),
14938 "fake-aft-stub not built at {}: run `cargo test -p subc-core` (which builds \
14939 [[bin]] targets) rather than `cargo test -p subc-core --lib` (which does not)",
14940 path.display()
14941 );
14942 path
14943 }
14944
14945 fn read_grandchild_pid(path: &Path) -> u32 {
14947 let deadline = Instant::now() + Duration::from_secs(10);
14948 loop {
14949 if let Ok(contents) = std::fs::read_to_string(path) {
14950 if let Ok(pid) = contents.trim().parse() {
14951 return pid;
14952 }
14953 }
14954 assert!(
14955 Instant::now() < deadline,
14956 "the stub never recorded a grandchild pid at {}",
14957 path.display()
14958 );
14959 std::thread::sleep(Duration::from_millis(10));
14960 }
14961 }
14962
14963 struct Fixture {
14966 _dir: ScratchDir,
14967 module_id: String,
14968 grandchild: u32,
14969 child: Option<SupervisedChild>,
14970 registry: Arc<Registry>,
14971 snapshot: Arc<Mutex<SupervisorSnapshot>>,
14972 terminal_ring: Arc<Mutex<TerminalRing>>,
14973 spawn_events: SpawnEventFeed,
14974 }
14975
14976 fn fixture(label: &str, module_id: &str) -> Fixture {
14977 let dir = ScratchDir::new(label);
14978 let pid_file = dir.join("grandchild.pid");
14979 let supervisor = Supervisor::new_for_test(
14980 Arc::new(Registry::default()),
14981 RestartPolicy::new(3, Duration::ZERO),
14982 );
14983 let runtime = supervisor.runtime_config();
14984 let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::starting()));
14985 let spec = ModuleSpec {
14986 module_id: module_id.to_string(),
14987 program: stub_path(),
14988 args: Vec::new(),
14992 env: vec![
14993 ("FAKE_AFT_NEVER_CONNECT".to_string(), "1".to_string()),
14994 (
14995 "FAKE_AFT_GRANDCHILD_PID_FILE".to_string(),
14996 pid_file.display().to_string(),
14997 ),
14998 ],
14999 reserved: false,
15000 reserved_prefixes: Vec::new(),
15001 protocol: ModuleProtocol::Subc,
15002 overlap: Default::default(),
15003 };
15004 let child = spawn_and_mark_running(&spec, &runtime, &snapshot)
15005 .expect("spawn the supervised fixture");
15006 let grandchild = read_grandchild_pid(&pid_file);
15007 Fixture {
15008 _dir: dir,
15009 module_id: module_id.to_string(),
15010 grandchild,
15011 child: Some(child),
15012 registry: Arc::new(Registry::default()),
15013 snapshot,
15014 terminal_ring: Arc::clone(&runtime.terminal_ring),
15015 spawn_events: SpawnEventFeed::default(),
15016 }
15017 }
15018
15019 impl Fixture {
15020 async fn drain(&mut self) {
15022 let child = self
15023 .child
15024 .take()
15025 .expect("the fixture child is still present");
15026 drain_child_to_state(
15027 &self.module_id,
15028 ModuleProtocol::Subc,
15029 StopNotice::NotSent,
15032 &self.registry,
15033 None,
15034 &self.snapshot,
15035 &self.terminal_ring,
15036 &self.spawn_events,
15037 child,
15038 Duration::from_millis(500),
15039 ModuleState::Stopped,
15040 Some(false),
15041 )
15042 .await
15043 .expect("drain the supervised fixture");
15044 }
15045 }
15046
15047 #[tokio::test]
15053 async fn teardown_reaps_the_grandchild() {
15054 let mut fixture = fixture("teardown-grandchild", "tree-teardown");
15055 let grandchild = fixture.grandchild;
15056
15057 assert!(
15058 subc_jobobject::process_exists(grandchild),
15059 "grandchild {grandchild} must be alive before teardown, or this proves nothing"
15060 );
15061
15062 fixture.drain().await;
15063
15064 assert!(
15065 subc_jobobject::wait_for_process_exit(grandchild, Duration::from_secs(10)),
15066 "grandchild {grandchild} outlived module teardown: the tree was not contained"
15067 );
15068 }
15069
15070 #[test]
15083 fn an_uncontained_grandchild_survives_a_direct_child_kill() {
15084 let dir = ScratchDir::new("teardown-uncontained");
15085 let pid_file = dir.join("grandchild.pid");
15086 let mut child = std::process::Command::new(stub_path())
15087 .env("FAKE_AFT_NEVER_CONNECT", "1")
15088 .env(
15089 "FAKE_AFT_GRANDCHILD_PID_FILE",
15090 pid_file.display().to_string(),
15091 )
15092 .stdin(std::process::Stdio::null())
15093 .stdout(std::process::Stdio::null())
15094 .stderr(std::process::Stdio::null())
15095 .spawn()
15096 .expect("spawn the uncontained fixture");
15097 let grandchild = read_grandchild_pid(&pid_file);
15098
15099 child.kill().expect("kill the direct child");
15101 let _ = child.wait();
15102
15103 assert!(
15104 subc_jobobject::process_exists(grandchild),
15105 "grandchild {grandchild} died with the direct child, so this control no longer \
15106 distinguishes contained from uncontained teardown and the regression test is \
15107 passing vacuously"
15108 );
15109
15110 kill_tree(grandchild);
15113 }
15114
15115 #[tokio::test]
15124 async fn dropping_containment_reaps_the_grandchild() {
15125 let mut fixture = fixture("drop-containment", "tree-drop");
15126 let grandchild = fixture.grandchild;
15127
15128 assert!(subc_jobobject::process_exists(grandchild));
15129
15130 fixture.child.as_mut().expect("child present").job = None;
15132
15133 assert!(
15134 subc_jobobject::wait_for_process_exit(grandchild, Duration::from_secs(10)),
15135 "grandchild {grandchild} survived the containment handle closing, so a daemon \
15136 crash would leave the tree behind"
15137 );
15138 }
15139
15140 fn kill_tree(pid: u32) {
15142 let _ = std::process::Command::new("taskkill.exe")
15143 .args(["/PID", &pid.to_string(), "/T", "/F"])
15144 .stdin(std::process::Stdio::null())
15145 .stdout(std::process::Stdio::null())
15146 .stderr(std::process::Stdio::null())
15147 .status();
15148 assert!(
15149 subc_jobobject::wait_for_process_exit(pid, Duration::from_secs(10)),
15150 "could not clean up grandchild {pid}"
15151 );
15152 }
15153}
15154
15155#[cfg(test)]
15156mod privacy_trampoline_configuration_tests {
15157 #[tokio::test]
15158 #[cfg_attr(not(target_os = "macos"), ignore = "requires macOS privacy trampoline")]
15159 async fn macos_spawn_without_a_configured_trampoline_refuses_by_name() {
15160 #[cfg(target_os = "macos")]
15161 {
15162 let supervisor = super::Supervisor::new(
15163 std::sync::Arc::new(crate::Registry::default()),
15164 super::RestartPolicy::default(),
15165 );
15166 let error = supervisor.spawn(spec()).unwrap_err();
15167 assert!(
15168 error
15169 .to_string()
15170 .contains("no privacy trampoline configured"),
15171 "{error}"
15172 );
15173 }
15174 }
15175
15176 #[tokio::test]
15177 #[cfg_attr(not(target_os = "macos"), ignore = "requires macOS privacy trampoline")]
15178 async fn macos_wrong_trampoline_is_rejected_by_the_startup_probe() {
15179 #[cfg(target_os = "macos")]
15180 {
15181 let supervisor = super::Supervisor::new(
15182 std::sync::Arc::new(crate::Registry::default()),
15183 super::RestartPolicy::default(),
15184 )
15185 .with_privacy_trampoline(std::env::current_exe().unwrap());
15186 let error = supervisor.spawn(spec()).unwrap_err();
15187 assert!(
15188 error
15189 .to_string()
15190 .contains("binary does not implement the privacy trampoline protocol"),
15191 "{error}"
15192 );
15193 }
15194 }
15195
15196 #[cfg(target_os = "macos")]
15197 fn spec() -> super::ModuleSpec {
15198 super::ModuleSpec {
15199 module_id: "privacy-configuration".into(),
15200 program: "/bin/sleep".into(),
15201 args: vec!["30".into()],
15202 env: vec![],
15203 reserved: false,
15204 reserved_prefixes: vec![],
15205 protocol: subc_control::ModuleProtocol::None,
15206 overlap: super::ModuleOverlap::Exclusive,
15207 }
15208 }
15209}
15210
15211#[cfg(test)]
15212mod privacy_exec_boundary_tests {
15213 #[cfg(target_os = "macos")]
15214 use super::*;
15215 #[cfg(target_os = "macos")]
15216 use std::{
15217 io::{Read, Write},
15218 net::{TcpListener, TcpStream},
15219 };
15220
15221 #[cfg(target_os = "macos")]
15224 pub(super) fn before_image_sample(spec: &ModuleSpec, pid: u32) {
15225 if let Some((_, path)) = spec
15226 .env
15227 .iter()
15228 .find(|(key, _)| key == "SUBC_TEST_PRIVACY_SAMPLE_BARRIER")
15229 {
15230 let mut barrier = TcpStream::connect(path).unwrap();
15231 barrier
15232 .set_read_timeout(Some(Duration::from_secs(30)))
15233 .unwrap();
15234 barrier.write_all(&pid.to_ne_bytes()).unwrap();
15235 let mut release = [0];
15236 barrier.read_exact(&mut release).unwrap();
15237 assert_eq!(&release, b"X");
15238 }
15239 }
15240
15241 #[cfg(target_os = "macos")]
15242 fn spec(program: &str, args: &[&str]) -> ModuleSpec {
15243 ModuleSpec {
15244 module_id: "privacy-boundary".into(),
15245 program: program.into(),
15246 args: args.iter().map(|arg| (*arg).into()).collect(),
15247 env: vec![],
15248 reserved: false,
15249 reserved_prefixes: vec![],
15250 protocol: ModuleProtocol::None,
15251 overlap: ModuleOverlap::Exclusive,
15252 }
15253 }
15254
15255 #[cfg(target_os = "macos")]
15256 async fn accept(listener: TcpListener) -> TcpStream {
15257 let listener = tokio::net::TcpListener::from_std({
15259 listener.set_nonblocking(true).unwrap();
15260 listener
15261 })
15262 .unwrap();
15263 let (stream, _) = tokio::time::timeout(Duration::from_secs(30), listener.accept())
15264 .await
15265 .unwrap()
15266 .unwrap();
15267 let stream = stream.into_std().unwrap();
15268 stream.set_nonblocking(false).unwrap();
15269 stream
15270 .set_read_timeout(Some(Duration::from_secs(30)))
15271 .unwrap();
15272 stream
15273 }
15274
15275 #[tokio::test]
15276 #[cfg_attr(not(target_os = "macos"), ignore = "requires macOS privacy trampoline")]
15277 async fn macos_roster_withholds_a_nonnull_trampoline_image_until_exec_confirmation() {
15278 #[cfg(target_os = "macos")]
15279 {
15280 let root = cortexkit_test_support::ScratchDir::new("privacy-roster-barrier");
15281 let exec_listener = TcpListener::bind("127.0.0.1:0").unwrap();
15284 let sample_listener = TcpListener::bind("127.0.0.1:0").unwrap();
15285 let record = root.join("live-children.json");
15286 let supervisor =
15287 Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default())
15288 .with_live_children_record(&record);
15289 let runtime = supervisor.runtime_config();
15290 let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::starting()));
15291 let mut spec = spec("/bin/sleep", &["30"]);
15292 spec.env = vec![
15293 (
15294 "SUBC_TEST_PRIVACY_EXEC_BARRIER".into(),
15295 exec_listener.local_addr().unwrap().to_string(),
15296 ),
15297 (
15298 "SUBC_TEST_PRIVACY_SAMPLE_BARRIER".into(),
15299 sample_listener.local_addr().unwrap().to_string(),
15300 ),
15301 ];
15302 let spawn = tokio::task::spawn_blocking(move || {
15303 spawn_and_mark_running(&spec, &runtime, &snapshot).unwrap()
15304 });
15305 let mut sample = accept(sample_listener).await;
15306 let mut pid = [0; 4];
15307 sample.read_exact(&mut pid).unwrap();
15308 let pid = u32::from_ne_bytes(pid);
15309 let mut exec = accept(exec_listener).await;
15310 let mut ready = [0];
15311 exec.read_exact(&mut ready).unwrap();
15312 assert_eq!(&ready, b"R");
15313 let trampoline = subc_os::file_identity(&test_privacy_trampoline()).unwrap();
15316 assert_eq!(
15317 observe_spawned_image(pid).unwrap().executable,
15318 Some(trampoline)
15319 );
15320 sample.write_all(b"X").unwrap();
15321 let mut child = spawn.await.unwrap();
15322 let early = crate::live_children::read_record(&record).unwrap();
15323 assert_eq!(early.len(), 1);
15324 assert_eq!(early[0].pid, pid);
15325 assert_eq!(
15326 early[0].executable, None,
15327 "unconfirmed trampoline image entered the roster"
15328 );
15329 assert!(child.report_ready.get().is_none());
15330 child.privacy_exec.as_mut().unwrap().deadline =
15333 tokio::time::Instant::now() + Duration::from_secs(30);
15334 exec.write_all(b"X").unwrap();
15335 child.confirm_privacy_exec().await;
15336 assert_eq!(child.spawn_failure, None);
15337 assert!(child.report_ready.get().is_some());
15338 let confirmed = crate::live_children::read_record(&record).unwrap();
15339 let module = subc_os::file_identity(std::path::Path::new("/bin/sleep")).unwrap();
15340 assert_ne!(module, trampoline);
15341 assert_eq!(confirmed[0].executable, Some(module.into()));
15342 child.start_kill().unwrap();
15343 child.wait().await.unwrap();
15344 child.release_roster();
15345 }
15346 }
15347
15348 #[tokio::test]
15349 #[cfg_attr(not(target_os = "macos"), ignore = "requires macOS privacy trampoline")]
15350 async fn macos_already_exited_121_in_try_wait_is_a_module_exit_not_a_trampoline_refusal() {
15351 #[cfg(target_os = "macos")]
15352 {
15353 let registry = Arc::new(Registry::default());
15354 let policy = RestartPolicy::new(0, Duration::ZERO);
15355 let supervisor = Supervisor::new_for_test(Arc::clone(®istry), policy);
15356 let runtime = supervisor.runtime_config();
15357 let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::starting()));
15358 let spec = spec("/bin/sh", &["-c", "exit 121"]);
15359 let mut child = spawn_and_mark_running(&spec, &runtime, &snapshot).unwrap();
15363 let pid = child.pid;
15364 tokio::task::spawn_blocking(move || {
15365 subc_os::fork_exec_test::wait_for_child_exit_without_reaping(pid)
15366 })
15367 .await
15368 .unwrap()
15369 .unwrap();
15370 child.privacy_exec.as_mut().unwrap().deadline =
15371 tokio::time::Instant::now() + Duration::from_secs(30);
15372 let status = child.wait().await.unwrap();
15373 assert_eq!(status.code(), Some(121));
15374 assert!(child.privacy_exec.is_none());
15375 assert!(
15376 child.report_ready.get().is_none(),
15377 "an exited module must not publish a live pid"
15378 );
15379 let report = classify_reaped_child_exit(&snapshot, &child, &status);
15380 on_child_exit(
15381 &spec,
15382 policy,
15383 ®istry,
15384 &snapshot,
15385 &runtime.terminal_ring,
15386 &runtime.spawn_events,
15387 &runtime.child_roster,
15388 report,
15389 )
15390 .await;
15391 let state = lock_snapshot(&snapshot).unwrap();
15392 assert_eq!(state.state, ModuleState::Failed);
15393 assert_eq!(state.last_exit.as_ref().unwrap().code, Some(121));
15394 assert_eq!(state.reported_pid(), None);
15395 drop(state);
15396 let history = runtime.terminal_ring.lock().unwrap().snapshot();
15397 assert_eq!(history.entries.len(), 1);
15398 let terminal = &history.entries[0];
15399 assert_eq!(terminal.exit_code, Some(121));
15400 assert_eq!(terminal.exit_kind, subc_control::TerminalExitKind::Crash);
15401 assert_eq!(terminal.disposition, TerminalDisposition::Failed);
15402 assert_eq!(
15403 terminal.disposition_detail.as_deref(),
15404 Some(policy.budget_exhausted_detail().as_str()),
15405 "module exit 121 was classified as a trampoline refusal: {terminal:?}"
15406 );
15407 assert_eq!(child.spawn_failure, None);
15408 child.release_roster();
15409 }
15410 }
15411}
15412
15413#[cfg(all(test, unix))]
15417mod launch_nonce_descriptor_tests {
15418 use super::{ChildRoster, ModuleProtocol, ModuleSpec, SupervisorHandle};
15419 use crate::stderr_tail::{StderrRing, StderrTailConfig};
15420 use cortexkit_test_support::ScratchDir;
15421 use std::{
15422 path::PathBuf,
15423 sync::{Arc, Mutex},
15424 time::{Duration, Instant},
15425 };
15426
15427 async fn probe(role: super::SpawnRole) {
15428 let scratch = ScratchDir::new("launch-nonce-descriptor");
15429 let fd_copy = scratch.join("from-descriptor");
15430 let env_copy = scratch.join("environment");
15431 let script = format!(
15432 "env > '{env}.tmp' && mv '{env}.tmp' '{env}'; cat <&3 > '{fd}.tmp' && mv '{fd}.tmp' '{fd}'; sleep 30",
15433 fd = fd_copy.display(), env = env_copy.display(),
15434 );
15435 let xdg = |name: &str| (name.to_string(), scratch.join(name).display().to_string());
15436 let spec = ModuleSpec {
15437 module_id: "nonce-descriptor-probe".to_string(),
15438 program: PathBuf::from("/bin/sh"),
15439 args: vec!["-c".to_string(), script],
15440 env: vec![
15441 xdg("XDG_DATA_HOME"),
15442 xdg("XDG_RUNTIME_DIR"),
15443 xdg("XDG_CONFIG_HOME"),
15444 ("SUBC_LAUNCH_NONCE".to_string(), "must-not-leak".to_string()),
15445 ],
15446 reserved: true,
15447 reserved_prefixes: Vec::new(),
15448 protocol: ModuleProtocol::Subc,
15449 overlap: Default::default(),
15450 };
15451 let handle = SupervisorHandle::new();
15452 let ring = Arc::new(Mutex::new(StderrRing::new(StderrTailConfig::default())));
15453 let roster = ChildRoster::default();
15454 #[cfg(target_os = "macos")]
15455 {
15456 let path = super::test_privacy_trampoline();
15457 roster.set_privacy_trampoline(super::probe_privacy_trampoline(&path).map(|()| path));
15458 }
15459 let child = super::spawn_child_in_slot(
15460 &spec,
15461 None,
15462 Some(&handle),
15463 &ring,
15464 None,
15465 &roster,
15466 #[cfg(target_os = "linux")]
15467 None,
15468 role,
15469 matches!(role, super::SpawnRole::SwapCandidate),
15470 )
15471 .expect("spawn probe");
15472 let deadline = Instant::now() + Duration::from_secs(10);
15473 while !(fd_copy.exists() && env_copy.exists()) {
15474 assert!(Instant::now() < deadline, "probe never wrote its copies");
15475 tokio::time::sleep(Duration::from_millis(20)).await;
15476 }
15477 let nonce = std::fs::read_to_string(fd_copy).unwrap();
15478 assert!(!nonce.is_empty());
15479 let environment = std::fs::read_to_string(env_copy).unwrap();
15480 assert!(environment
15481 .lines()
15482 .any(|line| line.starts_with("SUBC_LAUNCH_NONCE_FD=3:")));
15483 let copy = environment
15484 .lines()
15485 .find_map(|line| line.strip_prefix("SUBC_LAUNCH_NONCE="));
15486 assert_eq!(
15487 copy, None,
15488 "Unix children must never receive the environment nonce"
15489 );
15490 if matches!(role, super::SpawnRole::Plain) {
15491 assert_eq!(
15492 handle.spawn_nonce(&spec.module_id).as_deref(),
15493 Some(nonce.as_str())
15494 );
15495 }
15496 drop(child);
15497 }
15498
15499 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
15500 async fn a_spawned_module_receives_its_nonce_only_on_descriptor_3() {
15501 probe(super::SpawnRole::Plain).await;
15502 }
15503
15504 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
15505 async fn swap_candidate_receives_its_nonce_only_on_descriptor_3() {
15506 probe(super::SpawnRole::SwapCandidate).await;
15507 }
15508}
15509
15510#[cfg(all(test, target_os = "linux"))]
15511mod cgroup_containment_tests {
15512 use super::*;
15513 use cortexkit_test_support::ScratchDir;
15514
15515 fn running(pid: u32) -> bool {
15516 std::fs::read_to_string(format!("/proc/{pid}/stat"))
15518 .ok()
15519 .and_then(|stat| {
15520 stat.rsplit_once(") ")
15521 .map(|(_, rest)| rest.starts_with('Z'))
15522 })
15523 .is_some_and(|zombie| !zombie)
15524 }
15525
15526 #[tokio::test]
15527 async fn linux_teardown_reaps_the_grandchild() {
15528 teardown_tree("linux_teardown_reaps_the_grandchild", false).await;
15529 }
15530
15531 #[tokio::test]
15532 async fn linux_shutdown_straggler_reaps_the_grandchild() {
15533 teardown_tree("linux_shutdown_straggler_reaps_the_grandchild", true).await;
15534 }
15535
15536 async fn teardown_tree(test_name: &str, shutdown: bool) {
15537 let dir = ScratchDir::new(test_name);
15538 let root = PathBuf::from(format!(
15539 "/sys/fs/cgroup/subc-tree-test-{test_name}-{}-{}",
15540 std::process::id(),
15541 unix_ms_now()
15542 ));
15543 if let Err(error) = std::fs::create_dir(&root) {
15544 assert!(
15545 std::env::var_os("SUBC_REQUIRE_CGROUP_TEST").is_none(),
15546 "required cgroup test cannot execute: {error}"
15547 );
15548 eprintln!(
15549 "SKIP {test_name}: no writable delegated cgroup at {}: {error}",
15550 root.display()
15551 );
15552 return;
15553 }
15554 let placement = subc_cgroup::prepare_at(&root)
15555 .expect("prepare isolated kernel cgroup")
15556 .expect("isolated cgroup is delegated");
15557 let module_id = "tree-teardown";
15558 let module = placement
15559 .module_path(module_id)
15560 .expect("create isolated module cgroup");
15561 if !module.join("cgroup.kill").exists() {
15562 std::fs::remove_dir(&module).unwrap();
15563 std::fs::remove_dir(root.join("subc-modules")).unwrap();
15564 std::fs::remove_dir(&root).unwrap();
15565 assert!(
15566 std::env::var_os("SUBC_REQUIRE_CGROUP_TEST").is_none(),
15567 "required cgroup.kill interface unavailable"
15568 );
15569 eprintln!("SKIP {test_name}: cgroup.kill unavailable (kernel < 5.14)");
15570 return;
15571 }
15572 let supervisor = Supervisor::new_for_test(
15573 Arc::new(Registry::default()),
15574 RestartPolicy::new(3, Duration::ZERO),
15575 )
15576 .with_cgroup_placement(Some(placement));
15577 let mut runtime = supervisor.runtime_config();
15578 runtime.child_roster = runtime
15579 .child_roster
15580 .for_module(Arc::new(Mutex::new(Duration::from_millis(100))));
15581 let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::starting()));
15582 let pid_file = dir.join("grandchild.pid");
15583 let spec = ModuleSpec {
15584 module_id: module_id.to_string(),
15585 program: PathBuf::from("/bin/sh"),
15586 args: vec![
15587 "-c".into(),
15588 "trap '' TERM; sleep 600 & echo $! > \"$1\"; wait".into(),
15589 "fixture".into(),
15590 pid_file.display().to_string(),
15591 ],
15592 env: ["XDG_DATA_HOME", "XDG_RUNTIME_DIR", "XDG_CONFIG_HOME"]
15593 .into_iter()
15594 .map(|key| (key.to_string(), dir.display().to_string()))
15595 .collect(),
15596 reserved: false,
15597 reserved_prefixes: Vec::new(),
15598 protocol: ModuleProtocol::None,
15599 overlap: Default::default(),
15600 };
15601 let child =
15602 spawn_and_mark_running(&spec, &runtime, &snapshot).expect("spawn supervised tree");
15603 let deadline = tokio::time::Instant::now() + Duration::from_secs(10);
15604 let grandchild: u32 = loop {
15605 if let Ok(contents) = std::fs::read_to_string(&pid_file) {
15606 if let Ok(pid) = contents.trim().parse() {
15607 break pid;
15608 }
15609 }
15610 assert!(
15611 tokio::time::Instant::now() < deadline,
15612 "grandchild pid was not recorded"
15613 );
15614 tokio::time::sleep(Duration::from_millis(10)).await;
15615 };
15616 assert!(
15617 running(grandchild),
15618 "grandchild must be alive before teardown"
15619 );
15620 if shutdown {
15621 let mut child = child;
15622 crate::child_roster::end_children_for_daemon_shutdown(
15623 &runtime.child_roster,
15624 false,
15625 std::future::pending(),
15626 )
15627 .await;
15628 child.wait().await.expect("reap shutdown straggler");
15629 } else {
15630 drain_child_to_state(
15631 module_id,
15632 ModuleProtocol::None,
15633 StopNotice::NotSent,
15634 &Registry::default(),
15635 None,
15636 &snapshot,
15637 &runtime.terminal_ring,
15638 &SpawnEventFeed::default(),
15639 child,
15640 Duration::from_millis(100),
15641 ModuleState::Stopped,
15642 Some(false),
15643 )
15644 .await
15645 .expect("real supervisor teardown");
15646 }
15647 let deadline = tokio::time::Instant::now() + Duration::from_secs(5);
15648 while running(grandchild) && tokio::time::Instant::now() < deadline {
15649 tokio::time::sleep(Duration::from_millis(10)).await;
15650 }
15651 let survived = running(grandchild);
15652 if survived {
15654 let pid = rustix::process::Pid::from_raw(grandchild as i32).unwrap();
15655 let _ = rustix::process::kill_process(pid, rustix::process::Signal::KILL);
15656 tokio::time::sleep(Duration::from_millis(100)).await;
15657 }
15658 if module.exists() {
15659 std::fs::remove_dir(&module).expect("remove empty module cgroup");
15660 }
15661 std::fs::remove_dir(root.join("subc-modules")).unwrap();
15662 std::fs::remove_dir(&root).unwrap();
15663 assert!(
15664 !survived,
15665 "grandchild {grandchild} outlived module teardown"
15666 );
15667 eprintln!("EXECUTED {test_name}: grandchild {grandchild} killed in isolated cgroup");
15668 }
15669}