1#[cfg(all(test, target_os = "linux"))]
6#[path = "supervise/audit_identity_tests.rs"]
7mod audit_identity_tests;
8mod resource_launch;
9pub(crate) use resource_launch::{ResourceCommitError, ResourceLaunch};
10
11use std::{
12 collections::{HashMap, HashSet, VecDeque},
13 error::Error,
14 fmt, io,
15 path::PathBuf,
16 process::{ExitStatus, Stdio},
17 sync::{Arc, Mutex, OnceLock},
18 time::{Duration, SystemTime, UNIX_EPOCH},
19};
20
21use cortexkit_log::Retention;
22use serde_json::Value;
23use sha2::{Digest, Sha256};
24use subc_control::{
25 ClientControlPush, LiveSpawn, ModuleProtocol, RouteCloseReason, SpawnCursor, SpawnEvent,
26 SpawnEventKind, SpawnSnapshot, SupervisorHealthStatus, TerminalDisposition, TerminalExitKind,
27};
28use subc_protocol::{
29 manifest::{SelfSignalKind, SignalAnchor},
30 session::{
31 HealthReport, HealthStatus, ModuleControlCommand, ModuleControlRequest,
32 MODULE_CONTROL_OP_HEALTH_CHECK,
33 },
34 Flags, FrameType, Principal, Priority, SUBC_LAUNCH_NONCE_ENV, SUBC_MODULE_ID_ENV,
35};
36use tokio::{
37 process::{Child, Command},
38 sync::{mpsc, oneshot, watch, Mutex as AsyncMutex},
39 task::JoinHandle,
40 time::{sleep, sleep_until, timeout, timeout_at, Instant},
41};
42use tracing::{debug, error, info, warn};
43
44use crate::{
45 child_roster::ChildRoster,
46 daemon_config::{
47 CAPTURE_KEEP_ENV, CAPTURE_MAX_AGE_DAYS_ENV, CAPTURE_MAX_FILE_MB_ENV, CK_LOG_ENV,
48 },
49 forwarding::{
50 CloseReason, ForwardingError, ForwardingTable, GoodbyeTarget, ModuleControlRpcOutcome,
51 ModuleDrainTarget, PendingModuleControlRpc,
52 },
53 provenance::{ExecutableIdentityProbe, SpawnedFileIdentity},
54 registry::{ConnectionId, RegistrationEndReason, RegistryError},
55 stderr_tail::{
56 pump_stderr_to, pump_stdout_to, ChildOutputSink, StderrRing, StderrTailConfig,
57 StderrTailSnapshot,
58 },
59 terminal_ring::{TerminalHistorySnapshot, TerminalRecord, TerminalRing, TerminalRingConfig},
60 Frame, FrameSink, Registry,
61};
62
63#[cfg(not(windows))]
64use crate::provenance::spawned_file_identity;
65
66#[path = "supervise_swap.rs"]
67mod swap;
68
69#[path = "supervise_activity.rs"]
70mod activity;
71
72#[cfg(unix)]
73#[path = "supervise/admission.rs"]
74mod admission;
75mod admission_exit;
76#[cfg(all(test, target_os = "linux"))]
77mod async_spawn_tests;
78#[cfg(all(test, windows))]
79mod carried_job_tests;
80mod config_delivery;
81mod content_binding;
82mod program_binding;
83mod registration;
84pub(crate) mod script;
85type NonceLaunches = HashMap<String, (String, std::sync::Weak<RegistrationSpawn>)>;
87pub(crate) use registration::NonceAttestation;
88#[cfg(all(test, unix))]
89pub(crate) use registration::RegistrationProof;
90pub(crate) use registration::RegistrationSpawn;
91pub(crate) use registration::REGISTRATION_CHANNEL_CLOSED;
92#[cfg(all(test, unix))]
93pub(crate) mod unix_channel_placement_tests;
94use program_binding::ProgramBindingPublication;
95mod config_exit_failure;
96#[cfg(all(test, any(target_os = "linux", windows)))]
97mod missing_path_tests;
98#[cfg(all(test, unix))]
99mod program_binding_tests;
100#[cfg(all(test, windows))]
101mod program_binding_windows_tests;
102#[cfg(all(test, windows))]
103mod windows_channel_tests;
104pub use config_exit_failure::ModuleStopReason;
105#[cfg(all(test, unix))]
106mod config_exit_hold_tests;
107pub(crate) mod containment;
108#[cfg(all(test, unix))]
109mod containment_tests;
110mod slow_retry;
111use slow_retry::{FailedRetry, FailureContext};
112mod host_health;
113use host_health::HostHealth;
114
115#[cfg(all(test, unix))]
116mod mark_readiness_tests;
117#[cfg(all(test, unix))]
118mod slow_retry_tests;
119
120pub const SUBC_ARG: &str = "--subc";
126
127const DEFAULT_MAX_RESTARTS: u32 = 3;
128const DEFAULT_BACKOFF: Duration = Duration::from_millis(100);
129const DEFAULT_MAX_BACKOFF: Duration = Duration::from_secs(30);
130const DEFAULT_RESTART_WINDOW: Duration = Duration::from_secs(600);
134pub const DEFAULT_DRAIN_TIMEOUT: Duration = Duration::from_secs(30);
145pub(crate) const DEFAULT_REGISTER_TIMEOUT: Duration = Duration::from_secs(120);
146const REGISTRY_RELEASE_TIMEOUT: Duration = Duration::from_secs(1);
147const REGISTRY_RELEASE_POLL: Duration = Duration::from_millis(10);
148const DRAIN_BUSY_REPROBE_INTERVAL: Duration = Duration::from_millis(250);
160const STDERR_PUMP_DRAIN_TIMEOUT: Duration = Duration::from_millis(250);
183pub const SPAWN_EVENT_RING_CAPACITY: usize = 4096;
185const SPAWN_SUBSCRIBER_BUFFER: usize = SPAWN_EVENT_RING_CAPACITY + 1;
186pub(crate) const SPAWN_SUBSCRIBER_LAGGED_CODE: &str = "spawn_subscriber_lagged";
191
192#[cfg(target_os = "macos")]
193struct PrivacyExec {
194 content_check: Option<tokio::task::JoinHandle<Result<(), crate::config_gate::GateError>>>,
195 gated: bool,
196 admission_binding: Option<ProgramBindingPublication>,
197 reader: tokio::io::unix::AsyncFd<std::io::PipeReader>,
198 deadline: tokio::time::Instant,
199 expected: Option<subc_os::FileIdentity>,
200 trampoline: Option<subc_os::FileIdentity>,
201 signature: Option<Arc<subc_os::privacy_identity::SuspendedSignature>>,
202 module_id: String,
203 #[cfg(test)]
204 confirmation_pause: Option<(String, Option<tokio::net::TcpStream>)>,
205 #[cfg(test)]
206 image_probe: Option<fn(u32) -> Option<subc_os::Observation>>,
207 #[cfg(test)]
208 image_read_delay: Option<Duration>,
209}
210
211#[cfg(all(test, target_os = "macos"))]
212pub(crate) fn test_privacy_trampoline() -> PathBuf {
213 let path = cortexkit_test_support::cargo_profile_bin("privacy-trampoline-fixture");
216 assert!(
220 path.exists(),
221 "privacy-trampoline-fixture not built at {}: run `cargo build -p subc-daemon \
222 --bins --features test-support` or `cargo test -p subc-daemon` first",
223 path.display()
224 );
225 path
226}
227
228#[cfg(target_os = "macos")]
229fn probe_privacy_trampoline(path: &std::path::Path) -> Result<(), String> {
230 use std::io::Read;
231 let mut probe = subc_os::child_spawn::spawn(
232 std::process::Command::new(path)
233 .args(["__disclaim-exec", "--probe-suspended"])
234 .stdin(Stdio::null())
235 .stdout(Stdio::piped())
236 .stderr(Stdio::piped()),
237 )
238 .map_err(|error| {
239 format!(
240 "privacy trampoline probe failed for {}: {error}",
241 path.display()
242 )
243 })?;
244 let deadline = std::time::Instant::now() + Duration::from_secs(5);
245 let status = loop {
246 match probe.try_wait() {
247 Ok(Some(status)) => break status,
248 Ok(None) if std::time::Instant::now() < deadline => {
249 std::thread::sleep(Duration::from_millis(5))
250 }
251 result => {
252 let _ = probe.kill();
253 let _ = probe.wait();
254 return Err(format!(
255 "privacy trampoline probe failed or timed out for {}: {result:?}",
256 path.display()
257 ));
258 }
259 }
260 };
261 let mut answer = String::new();
262 if let Some(stdout) = probe.stdout.take() {
263 let _ = stdout.take(256).read_to_string(&mut answer);
264 }
265 if status.success() && answer.trim() == subc_os::privacy_identity::SUSPENDED_TRAMPOLINE_PROBE {
266 return Ok(());
267 }
268 let mut diagnostic = String::new();
269 if let Some(stderr) = probe.stderr.take() {
270 let _ = stderr.take(1024).read_to_string(&mut diagnostic);
271 }
272 let cause = diagnostic
273 .trim()
274 .strip_prefix("ck-subc: own privacy identity refused: ")
275 .unwrap_or("binary does not implement the privacy trampoline protocol");
276 Err(format!(
277 "{cause}: probe of {} exited {status}",
278 path.display()
279 ))
280}
281
282#[cfg(target_os = "macos")]
283fn privacy_command(
284 spec: &ModuleSpec,
285 binding: &SpawnProgramBinding,
286 roster: &ChildRoster,
287 gated: bool,
288) -> Result<
289 (
290 Command,
291 Option<PrivacyExec>,
292 subc_os::privacy_identity::ExecAcknowledgement,
293 ),
294 SuperviseError,
295> {
296 let failure = |cause: String| {
297 warn!(module_id = %spec.module_id, %cause, "privacy identity trampoline refused module spawn");
298 SuperviseError::Spawn {
299 program: spec.program.clone(),
300 source: io::Error::other(cause),
301 cgroup_path: None,
302 }
303 };
304 let trampoline = roster.privacy_trampoline().map_err(failure)?;
305 let expected = Some(binding.identity);
306 let trampoline_image = subc_os::file_identity(&trampoline);
307 let signature = gated
308 .then(|| {
309 subc_os::privacy_identity::SuspendedSignature::capture(&binding._file, &binding.path)
310 .map(Arc::new)
311 })
312 .transpose()
313 .map_err(|error| failure(error.to_string()))?;
314 if expected == trampoline_image {
315 return Err(failure(
316 "privacy identity module executable is missing or is the trampoline itself".to_string(),
317 ));
318 }
319 let (reader, ack) = subc_os::privacy_identity::ExecAcknowledgement::pipe()
320 .map_err(|error| failure(error.to_string()))?;
321 let reader =
322 tokio::io::unix::AsyncFd::new(reader).map_err(|error| failure(error.to_string()))?;
323 let mut command = Command::new(&trampoline);
324 command
325 .arg(if gated {
326 "__disclaim-exec-suspended"
327 } else {
328 "__disclaim-exec"
329 })
330 .arg(ack.fd().to_string())
331 .arg(&binding.path);
332 Ok((
333 command,
334 Some(PrivacyExec {
335 gated,
336 admission_binding: None,
337 content_check: None,
338 reader,
339 deadline: tokio::time::Instant::now() + Duration::from_secs(5),
340 expected,
341 trampoline: trampoline_image,
342 signature,
343 module_id: spec.module_id.clone(),
344 #[cfg(test)]
345 image_probe: None,
346 #[cfg(test)]
347 image_read_delay: spec
348 .env
349 .iter()
350 .find(|(key, _)| key == "SUBC_TEST_PRIVACY_IMAGE_DELAY_MS")
351 .map(|(_, millis)| Duration::from_millis(millis.parse().unwrap())),
352 #[cfg(test)]
353 confirmation_pause: spec
354 .env
355 .iter()
356 .find(|(key, _)| key == "SUBC_TEST_BEFORE_EXEC_CONFIRM")
357 .map(|(_, address)| (address.clone(), None)),
358 }),
359 ack,
360 ))
361}
362
363#[cfg(unix)]
364type SpawnRegistrationChannel = std::os::unix::net::UnixStream;
365#[cfg(windows)]
366type SpawnRegistrationChannel = subc_os::registration_channel::RegistrationPipe;
367#[cfg(test)]
368type CapturedRegistrationChannels =
369 Arc<Mutex<Option<Vec<(Arc<RegistrationSpawn>, CapturedRegistrationEndpoint)>>>>;
370#[cfg(test)]
371struct CapturedRegistrationEndpoint {
372 _channel: SpawnRegistrationChannel,
373}
374#[cfg(test)]
375impl fmt::Debug for CapturedRegistrationEndpoint {
376 fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
377 f.write_str("CapturedRegistrationEndpoint")
378 }
379}
380
381struct SupervisedChild {
382 registration_spawn: Option<Arc<RegistrationSpawn>>,
383 #[cfg(any(unix, windows))]
385 registration_channel: Option<SpawnRegistrationChannel>,
386 live_program_binding: Option<Arc<content_binding::LiveProgramBinding>>,
387 registered_config_connection: Option<ConnectionId>,
388 admission_base: Option<subc_config::Digest>,
389 registration_declaration: Option<String>,
390 _program_binding: Option<ProgramBindingPublication>,
391 subtree_target: crate::subtree_resources::Target,
392 child: containment::ProcessChild,
393 exec: crate::activity::ExecOwner,
394 #[cfg(windows)]
395 nonce_delivery: Option<subc_os::LaunchNoncePipeDelivery>,
396 #[cfg(target_os = "macos")]
397 privacy_exec: Option<PrivacyExec>,
398 #[cfg(target_os = "macos")]
405 report_ready: Arc<OnceLock<()>>,
406 spawn_failure: Option<String>,
409 admission_kill_sent: bool,
412 protocol: ModuleProtocol,
417 #[cfg(target_os = "linux")]
424 module_id: String,
425 #[cfg(target_os = "linux")]
426 cgroup_placement: Option<subc_cgroup::Placement>,
427 #[cfg(windows)]
450 job: Option<Arc<Mutex<subc_jobobject::JobObject>>>,
451 stdout_pump: Option<JoinHandle<()>>,
452 stderr_pump: Option<StderrPump>,
453 stderr_ring: Arc<Mutex<StderrRing>>,
454 spawned_at: Instant,
457 config_delivery: Option<crate::config_gate::runtime::DeliverySpawn>,
458 registration_deadline: Option<Instant>,
461 registration_watch: Option<RegistrationWatch>,
462 spawned_at_ms: u64,
463 spawned_from: PathBuf,
464 launch_spec_digest: LaunchSpecDigest,
465 spawned_file_identity: Option<SpawnedFileIdentity>,
466 process_start_time: Option<u64>,
467 process_identity: Option<ProcessIdentity>,
468 pid: u32,
469 roster_guard: Option<crate::child_roster::RosterGuard>,
472}
473
474impl SupervisedChild {
475 fn retire_registration(&self) {
476 if let Some(spawn) = &self.registration_spawn {
481 spawn.retire();
482 } else if let Some(binding) = &self._program_binding {
483 binding.retire();
484 }
485 #[cfg(unix)]
486 if let Some(channel) = &self.registration_channel {
487 let _ = channel.shutdown(std::net::Shutdown::Both);
488 }
489 #[cfg(windows)]
490 if let Some(channel) = &self.registration_channel {
491 if let Ok(shutdown) = channel.shutdown_handle() {
492 shutdown.shutdown();
493 }
494 }
495 }
496 fn registered_since_spawn(&self) -> bool {
497 self.registration_watch
498 .as_ref()
499 .is_some_and(|watch| *watch.changes.borrow() != watch.baseline)
500 }
501
502 fn awaiting_registration(&self) -> bool {
503 let admission_failed = self.admission_kill_sent && self.spawn_failure.is_some();
507 !admission_failed && !self.registered_since_spawn()
508 }
509
510 fn id(&self) -> Option<u32> {
511 Some(self.pid)
512 }
513
514 fn process_identity(&self) -> Option<ProcessIdentity> {
515 self.process_identity
516 }
517
518 fn try_wait(&mut self) -> io::Result<Option<ExitStatus>> {
519 let result = self.child.try_wait();
520 if !matches!(result, Ok(None)) {
521 self.retire_registration();
522 #[cfg(any(unix, windows))]
523 self.registration_channel.take();
524 if let Some(binding) = &self._program_binding {
525 binding.retire();
526 }
527 }
528 result
529 }
530
531 async fn wait(&mut self) -> io::Result<ExitStatus> {
532 #[cfg(target_os = "macos")]
533 self.confirm_privacy_exec().await;
534 let result = self.child.wait().await;
542 self.retire_registration();
543 #[cfg(any(unix, windows))]
544 self.registration_channel.take();
545 if let Some(binding) = &self._program_binding {
548 binding.retire();
549 }
550 #[cfg(all(test, target_os = "linux"))]
551 if let Some(binding) = &self._program_binding {
552 binding.pause_after_wait_for_test().await;
553 }
554 #[cfg(target_os = "linux")]
555 if result.is_ok() && self.child.container().is_none() {
556 if let Some(placement) = self.cgroup_placement.as_ref() {
557 cleanup_reaped_cgroup(placement, &self.module_id).await;
561 self.cgroup_placement = None;
562 }
563 }
564 result
565 }
566
567 async fn finish_exit(&mut self, carry: bool) -> io::Result<()> {
568 self.retire_registration();
569 if let Some(binding) = self._program_binding.take() {
571 binding.retire();
572 }
573 if self.child.container().is_some() {
574 if !carry {
575 self.child.terminate_container().await;
576 }
577 } else {
578 #[cfg(target_os = "linux")]
579 if let Some(placement) = self.cgroup_placement.as_ref() {
580 cleanup_reaped_cgroup(placement, &self.module_id).await;
581 self.cgroup_placement = None;
582 }
583 }
584 self.child.reap().await?;
585 self.release_roster();
586 Ok(())
587 }
588
589 #[cfg(target_os = "macos")]
590 async fn confirm_privacy_exec(&mut self) {
591 if self
592 .privacy_exec
593 .as_ref()
594 .is_some_and(|pending| pending.content_check.is_some())
595 {
596 self.finish_privacy_content_check().await;
597 return;
598 }
599 let Some(pending) = &mut self.privacy_exec else {
600 return;
601 };
602 let result = tokio::time::timeout_at(pending.deadline, async {
603 let mut record = Vec::new();
604 loop {
605 let mut ready = pending.reader.readable().await?;
606 let read = ready.try_io(|reader| {
607 use std::io::Read;
608 let mut reader = reader.get_ref();
609 let mut buffer = [0u8; 256];
610 reader.read(&mut buffer).map(|count| (count, buffer))
611 });
612 match read {
613 Ok(Ok((0, _))) => return Ok::<_, io::Error>(record),
614 Ok(Ok((count, buffer))) => {
615 if record.len() + count > 1024 {
616 return Err(io::Error::other(
617 "privacy exec refusal record is too long",
618 ));
619 }
620 record.extend_from_slice(&buffer[..count]);
621 }
622 Ok(Err(error)) => return Err(error),
623 Err(_) => continue,
624 }
625 }
626 })
627 .await;
628 #[cfg(test)]
629 let result = if let Some((address, stream)) = &mut pending.confirmation_pause {
630 use tokio::io::{AsyncReadExt, AsyncWriteExt};
631 if stream.is_none() {
632 *stream = Some(
633 tokio::net::TcpStream::connect(address.as_str())
634 .await
635 .unwrap(),
636 );
637 stream.as_mut().unwrap().write_all(b"R").await.unwrap();
638 stream
639 .as_mut()
640 .unwrap()
641 .write_all(&self.pid.to_ne_bytes())
642 .await
643 .unwrap();
644 }
645 let mut reply = [0];
646 stream
647 .as_mut()
648 .unwrap()
649 .read_exact(&mut reply)
650 .await
651 .unwrap();
652 pending.confirmation_pause = None;
653 if &reply == b"F" {
654 Ok(Err(io::Error::other("injected late exec refusal")))
655 } else {
656 assert_eq!(&reply, b"X");
657 result
658 }
659 } else {
660 result
661 };
662 let waited = if matches!(&result, Ok(Ok(record)) if record.is_empty()) {
667 Some(self.try_wait())
668 } else {
669 None
670 };
671 let pending = self.privacy_exec.as_ref().expect("pending exec");
672 let cause = match result {
673 Err(_) => Some("privacy identity trampoline did not exec within 5s".to_string()),
674 Ok(Err(error)) => Some(format!(
675 "privacy identity exec acknowledgement failed: {error}"
676 )),
677 Ok(Ok(record)) if !record.is_empty() => Some(
678 std::str::from_utf8(&record)
679 .ok()
680 .and_then(|record| {
681 record
682 .trim()
683 .strip_prefix(subc_os::privacy_identity::EXEC_REFUSAL_TAG)
684 })
685 .filter(|cause| !cause.is_empty())
686 .unwrap_or("invalid privacy exec refusal record")
687 .to_string(),
688 ),
689 Ok(Ok(_)) if !pending.gated => match waited.expect("exec EOF checked child") {
690 Ok(Some(_)) => None,
691 Err(error) => Some(format!("privacy identity trampoline wait failed: {error}")),
692 Ok(None) => {
693 #[cfg(test)]
694 if let Some(delay) = pending.image_read_delay {
695 tokio::time::sleep(delay).await;
696 }
697 let image = observe_spawned_image(self.pid);
698 #[cfg(test)]
699 let image = pending.image_probe.map_or(image, |probe| probe(self.pid));
700 if let Some(image) = image.filter(|image| {
704 image.executable.is_some() && image.executable != pending.trampoline
705 }) {
706 match self
707 .roster_guard
708 .as_ref()
709 .map(|guard| guard.confirm_executable(image))
710 .transpose()
711 {
712 Err(error) => Some(format!(
713 "could not durably record confirmed module image: {error}"
714 )),
715 Ok(_) => {
716 let _ = self.report_ready.set(());
717 None
718 }
719 }
720 } else if image.is_none()
721 || image.is_some_and(|image| image.executable.is_none())
722 {
723 match tokio::time::timeout_at(pending.deadline, self.child.wait()).await {
726 Ok(result) => {
727 self.retire_registration();
728 self.registration_channel.take();
729 if let Some(binding) = &self._program_binding {
730 binding.retire();
731 }
732 result.err().map(|error| format!("privacy identity trampoline wait failed: {error}"))
733 }
734 Err(_) => Some("privacy identity module executable remained unreadable until the 5s exec deadline".to_string()),
735 }
736 } else {
737 Some("privacy identity trampoline did not exec the module".to_string())
738 }
739 }
740 },
741 Ok(Ok(_)) => match waited.expect("exec EOF checked child") {
742 Ok(Some(_)) => {
743 Some("privacy identity target exited before suspended admission".to_string())
744 }
745 Err(error) => Some(format!("privacy identity trampoline wait failed: {error}")),
746 Ok(None) => {
747 let stopped = tokio::time::timeout_at(pending.deadline, async {
750 loop {
751 if subc_os::privacy_identity::SuspendedSignature::is_suspended(
752 self.pid,
753 )? {
754 return Ok::<_, io::Error>(());
755 }
756 tokio::time::sleep(Duration::from_millis(1)).await;
757 }
758 })
759 .await;
760 match stopped {
761 Err(_) => {
762 Some("privacy identity target was not suspended within 5s".to_string())
763 }
764 Ok(Err(error)) => Some(format!(
765 "privacy identity suspended target unavailable: {error}"
766 )),
767 Ok(Ok(())) => {
768 let image = observe_spawned_image(self.pid);
769 if image.is_some_and(|image| {
770 image.executable.is_some()
771 && image.executable != pending.trampoline
772 && image.executable == pending.expected
773 }) {
774 None
775 } else {
776 Some("privacy identity trampoline executable mismatch: expected the suspended module program".to_string())
777 }
778 }
779 }
780 }
781 },
782 };
783 #[cfg(test)]
784 if let Some(binding) = &self._program_binding {
785 binding.pause_after_wait_for_test().await;
786 }
787 if cause.is_none() && pending.gated {
788 let publication = pending.admission_binding.clone();
789 let live = self.live_program_binding.clone();
790 let signature = Arc::clone(pending.signature.as_ref().expect("gated signature"));
791 let pid = self.pid;
792 let gated = pending.gated;
793 self.privacy_exec
794 .as_mut()
795 .expect("pending exec")
796 .content_check = Some(tokio::task::spawn_blocking(move || {
797 signature.verify(pid).map_err(|error| {
798 crate::config_gate::GateError::new("program_refused", error)
799 })?;
800 if !gated {
801 return Ok(());
802 }
803 match (publication, live) {
804 (Some(publication), _) => publication.verify(),
805 (None, Some(live)) => live.check(),
806 _ => Err(crate::config_gate::GateError::new(
807 "config_delivery_deferred",
808 "live program witness is unavailable",
809 )),
810 }
811 }));
812 self.finish_privacy_content_check().await;
813 } else {
814 self.finish_privacy_exec(cause);
815 }
816 }
817
818 #[cfg(target_os = "macos")]
819 async fn finish_privacy_content_check(&mut self) {
820 let result = self
821 .privacy_exec
822 .as_mut()
823 .expect("pending exec")
824 .content_check
825 .as_mut()
826 .expect("pending content check")
827 .await;
828 let cause = match result {
829 Ok(Ok(())) => None,
830 Ok(Err(error)) => Some(format!("{}: {}", error.code, error.detail)),
831 Err(error) => Some(format!(
832 "config_delivery_deferred: content check stopped: {error}"
833 )),
834 };
835 self.finish_privacy_exec(cause);
836 }
837
838 #[cfg(target_os = "macos")]
839 fn finish_privacy_exec(&mut self, mut cause: Option<String>) {
840 let mut pending = self.privacy_exec.take().expect("pending exec");
841 if cause.is_none() && pending.gated {
842 let admission = (|| -> io::Result<()> {
843 let image = observe_spawned_image(self.pid)
844 .filter(|image| image.executable == pending.expected)
845 .ok_or_else(|| io::Error::other("suspended executable mismatch"))?;
846 if let Some(guard) = &self.roster_guard {
847 guard.confirm_executable(image)?;
848 }
849 pending
850 .signature
851 .as_ref()
852 .expect("gated signature")
853 .resume(self.pid)?;
854 let _ = self.report_ready.set(());
855 info!(module_id = %pending.module_id, pid = self.pid,
856 "verified suspended module resumed with own privacy identity");
857 Ok(())
858 })();
859 if let Err(error) = admission {
860 cause = Some(format!("program_refused: {error}"));
861 }
862 }
863 if cause.is_some() {
864 if let Some(spawn) = &self.registration_spawn {
865 spawn.refuse();
866 }
867 if let Some(binding) = &mut pending.admission_binding {
868 binding.reject();
869 }
870 } else if let Some(spawn) = &self.registration_spawn {
871 spawn.ready();
872 }
873 self.exec.0.finish(if cause.is_some() {
876 crate::activity::ExecState::Refused
877 } else {
878 crate::activity::ExecState::CommittedExec
879 });
880 if let Some(cause) = cause {
881 warn!(module_id = %pending.module_id, pid = self.pid, %cause, "privacy identity trampoline refused module spawn");
882 self.spawn_failure = Some(
883 if pending.gated
884 && (cause.contains("executable mismatch")
885 || cause.starts_with("program_refused:"))
886 {
887 format!("program_refused: {cause}")
888 } else if pending.gated {
889 format!("config_delivery_deferred: {cause}")
890 } else {
891 cause
892 },
893 );
894 let _ = self.start_admission_kill();
897 }
898 }
899
900 fn release_roster(&mut self) {
904 self.roster_guard = None;
905 }
906
907 fn start_kill(&mut self) -> io::Result<()> {
920 self.retire_registration();
921 if let Some(container) = self.child.container() {
922 container.hard_kill();
923 return self.kill_leader();
924 }
925 #[cfg(windows)]
926 if let Some(job) = &self.job {
927 if let Err(error) = job.lock().unwrap_or_else(|p| p.into_inner()).terminate() {
928 debug!(
929 error = %error,
930 "job termination failed; the direct-child kill still owns the outcome"
931 );
932 }
933 }
934 #[cfg(target_os = "linux")]
935 kill_module_cgroup(self.cgroup_placement.as_ref(), &self.module_id);
936 self.child.start_kill()
937 }
938
939 fn kill_leader(&mut self) -> io::Result<()> {
940 self.retire_registration();
941 self.child.start_kill()
942 }
943
944 fn start_admission_kill(&mut self) -> io::Result<()> {
945 self.retire_registration();
946 #[cfg(windows)]
947 let result = {
948 let result = self
951 .spawned_file_identity
952 .as_ref()
953 .and_then(|identity| identity.windows_image.as_ref())
954 .ok_or_else(|| io::Error::other("admission child handle unavailable"))
955 .and_then(|image| {
956 image.process().terminate_with_exit_code(
957 admission_exit::WINDOWS_ADMISSION_KILL_EXIT_CODE as u32,
958 )
959 });
960 if let Some(container) = self.child.container() {
961 container.hard_kill();
962 } else if let Some(job) = &self.job {
963 if let Err(error) = job.lock().unwrap_or_else(|p| p.into_inner()).terminate() {
964 debug!(%error, "job termination failed after admission leader termination");
965 }
966 }
967 result
968 };
969 #[cfg(not(windows))]
970 let result = self.start_kill().map(|()| true);
971 #[cfg(unix)]
972 let result = {
973 let group_sent = self.child.container().is_none()
976 && rustix::process::Pid::from_raw(self.pid as i32).is_some_and(|group| {
977 rustix::process::kill_process_group(group, rustix::process::Signal::KILL)
978 .is_ok()
979 });
980 if group_sent {
981 Ok(true)
982 } else {
983 result
984 }
985 };
986 self.admission_kill_sent |= result?;
987 Ok(())
988 }
989
990 async fn drain_stderr(&mut self, module_id: &str) {
991 if let Some(pump) = self.stdout_pump.take() {
992 settle_stdout_pump(module_id, pump, STDERR_PUMP_DRAIN_TIMEOUT).await;
993 }
994
995 let Some(pump) = self.stderr_pump.take() else {
996 return;
997 };
998 settle_stderr_pump(
999 module_id,
1000 &self.stderr_ring,
1001 pump,
1002 STDERR_PUMP_DRAIN_TIMEOUT,
1003 )
1004 .await;
1005 }
1006}
1007
1008async fn settle_stdout_pump(module_id: &str, mut pump: JoinHandle<()>, bound: Duration) {
1012 match timeout(bound, &mut pump).await {
1013 Ok(Ok(())) => {}
1014 Ok(Err(error)) => {
1015 warn!(module_id, error = %error, "stdout pump ended unexpectedly");
1016 }
1017 Err(_) => {
1018 drop(pump);
1019 warn!(
1020 module_id,
1021 waited = ?bound,
1022 "stdout pipe of the exited process has not drained; its reader keeps running without delaying the restart"
1023 );
1024 }
1025 }
1026}
1027
1028struct StderrPump {
1031 task: JoinHandle<()>,
1032 generation: u64,
1033}
1034
1035async fn settle_stderr_pump(
1041 module_id: &str,
1042 ring: &Arc<Mutex<StderrRing>>,
1043 pump: StderrPump,
1044 bound: Duration,
1045) {
1046 let lock = || ring.lock().unwrap_or_else(|poisoned| poisoned.into_inner());
1047 let StderrPump {
1048 mut task,
1049 generation,
1050 } = pump;
1051 lock().retire_pump(generation);
1052 match timeout(bound, &mut task).await {
1053 Ok(Ok(())) => {}
1054 Ok(Err(err)) => {
1055 let mut ring = lock();
1056 ring.mark_incomplete_from(generation, format!("stderr pump ended unexpectedly: {err}"));
1057 ring.finish_pump(generation);
1058 warn!(module_id, error = %err, "stderr pump ended before clean EOF");
1059 }
1060 Err(_) => {
1061 drop(task);
1063 lock().mark_pump_late(
1064 generation,
1065 format!(
1066 "stderr of the exited process had not reached EOF {bound:?} after it was \
1067 retired (a descendant may still hold the pipe open); lines it still \
1068 writes are kept in that process's section"
1069 ),
1070 );
1071 warn!(
1072 module_id,
1073 waited = ?bound,
1074 "stderr pipe of the exited process is still open; its reader keeps running without delaying the restart"
1075 );
1076 }
1077 }
1078}
1079
1080fn registration_release_events() -> &'static watch::Sender<u64> {
1081 static EVENTS: OnceLock<watch::Sender<u64>> = OnceLock::new();
1082 EVENTS.get_or_init(|| {
1083 let (sender, _receiver) = watch::channel(0);
1084 sender
1085 })
1086}
1087
1088pub(crate) fn notify_registration_release() {
1089 let events = registration_release_events();
1090 let next_generation = (*events.borrow()).wrapping_add(1);
1091 events.send_replace(next_generation);
1092}
1093
1094#[derive(Debug, Clone, Copy, PartialEq, Eq)]
1096#[non_exhaustive]
1097pub enum ScriptRuntime {
1098 Bun,
1099 Python,
1100}
1101
1102impl ScriptRuntime {
1103 pub const fn as_str(self) -> &'static str {
1104 match self {
1105 Self::Bun => "bun",
1106 Self::Python => "python",
1107 }
1108 }
1109}
1110
1111#[derive(Debug, Clone, PartialEq, Eq)]
1113#[non_exhaustive]
1114pub struct ScriptEntry {
1115 pub runtime: ScriptRuntime,
1116 pub path: PathBuf,
1117}
1118
1119impl ScriptEntry {
1120 pub fn new(runtime: ScriptRuntime, path: PathBuf) -> Self {
1121 Self { runtime, path }
1122 }
1123}
1124
1125#[derive(Debug, Clone, PartialEq, Eq)]
1127pub struct ModuleSpec {
1128 pub module_id: String,
1129 pub program: PathBuf,
1130 pub script: Option<ScriptEntry>,
1132 pub args: Vec<String>,
1133 pub env: Vec<(String, String)>,
1134 pub reserved: bool,
1139 pub reserved_prefixes: Vec<String>,
1144 pub protocol: ModuleProtocol,
1163 pub overlap: ModuleOverlap,
1168 pub carry_containment: bool,
1172}
1173
1174#[derive(Clone, PartialEq, Eq)]
1177pub(crate) struct LaunchSpecDigest([u8; 32]);
1178
1179impl fmt::Debug for LaunchSpecDigest {
1180 fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
1181 f.write_str("LaunchSpecDigest([redacted])")
1182 }
1183}
1184
1185impl LaunchSpecDigest {
1186 pub(crate) fn from_spec(spec: &ModuleSpec) -> Self {
1187 fn field(hash: &mut Sha256, bytes: &[u8]) {
1188 hash.update((bytes.len() as u64).to_le_bytes());
1189 hash.update(bytes);
1190 }
1191 let mut hash = Sha256::new();
1192 if let Some(script) = &spec.script {
1193 field(
1194 &mut hash,
1195 match script.runtime {
1196 ScriptRuntime::Bun => b"bun-script",
1197 ScriptRuntime::Python => b"python-script",
1198 },
1199 );
1200 field(&mut hash, script.path.as_os_str().as_encoded_bytes());
1201 }
1202 if let Some(program) = spec.program.to_str() {
1203 field(&mut hash, b"utf8-program");
1204 field(&mut hash, program.as_bytes());
1205 } else {
1206 #[cfg(unix)]
1209 {
1210 use std::os::unix::ffi::OsStrExt;
1211 field(&mut hash, b"unix-program");
1212 field(&mut hash, spec.program.as_os_str().as_bytes());
1213 }
1214 #[cfg(windows)]
1215 {
1216 use std::os::windows::ffi::OsStrExt;
1217 field(&mut hash, b"windows-program");
1218 let bytes = spec
1219 .program
1220 .as_os_str()
1221 .encode_wide()
1222 .flat_map(u16::to_le_bytes)
1223 .collect::<Vec<_>>();
1224 field(&mut hash, &bytes);
1225 }
1226 #[cfg(not(any(unix, windows)))]
1227 field(&mut hash, spec.program.to_string_lossy().as_bytes());
1228 }
1229 hash.update((spec.args.len() as u64).to_le_bytes());
1230 for arg in &spec.args {
1231 field(&mut hash, arg.as_bytes());
1232 }
1233 let mut env = spec.env.iter().collect::<Vec<_>>();
1234 env.sort_unstable();
1235 hash.update((env.len() as u64).to_le_bytes());
1236 for (key, value) in env {
1237 field(&mut hash, key.as_bytes());
1238 field(&mut hash, value.as_bytes());
1239 }
1240 field(
1241 &mut hash,
1242 match spec.protocol {
1243 ModuleProtocol::Subc => b"subc",
1244 ModuleProtocol::None => b"none",
1245 },
1246 );
1247 Self(hash.finalize().into())
1248 }
1249}
1250
1251#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)]
1258pub enum ModuleOverlap {
1259 #[default]
1261 Exclusive,
1262 Safe,
1274}
1275
1276impl ModuleOverlap {
1277 pub fn as_str(self) -> &'static str {
1278 match self {
1279 Self::Exclusive => "exclusive",
1280 Self::Safe => "safe",
1281 }
1282 }
1283}
1284
1285pub const SUBC_SPAWN_ROLE_ENV: &str = "SUBC_SPAWN_ROLE";
1295pub const SPAWN_ROLE_SWAP_CANDIDATE: &str = "swap_candidate";
1297pub const DEFAULT_SWAP_READY_TIMEOUT: Duration = Duration::from_secs(100);
1302
1303#[derive(Debug, Clone, Copy, PartialEq, Eq)]
1321pub struct RestartPolicy {
1322 pub max_restarts: u32,
1323 pub backoff: Duration,
1326 pub max_backoff: Duration,
1328 pub window: Duration,
1332}
1333
1334impl RestartPolicy {
1335 pub fn new(max_restarts: u32, backoff: Duration) -> Self {
1339 Self {
1340 max_restarts,
1341 backoff,
1342 max_backoff: DEFAULT_MAX_BACKOFF,
1343 window: DEFAULT_RESTART_WINDOW,
1344 }
1345 }
1346
1347 pub fn with_max_backoff(mut self, max_backoff: Duration) -> Self {
1348 self.max_backoff = max_backoff;
1349 self
1350 }
1351
1352 pub fn with_window(mut self, window: Duration) -> Self {
1353 self.window = window;
1354 self
1355 }
1356
1357 fn delay_for_restart(&self, restart_in_window: u32) -> Duration {
1362 if self.backoff.is_zero() || self.max_backoff.is_zero() {
1363 return Duration::ZERO;
1364 }
1365
1366 let mut delay = self.backoff;
1367 for _ in 0..restart_in_window {
1368 if delay >= self.max_backoff {
1369 return self.max_backoff;
1370 }
1371 delay = delay
1372 .checked_mul(10)
1373 .unwrap_or(self.max_backoff)
1374 .min(self.max_backoff);
1375 }
1376 delay.min(self.max_backoff)
1377 }
1378
1379 fn budget_exhausted_detail(&self) -> String {
1384 format!(
1385 "crash budget exhausted: max_restarts={} within window_secs={}",
1386 self.max_restarts,
1387 self.window.as_secs()
1388 )
1389 }
1390}
1391
1392impl Default for RestartPolicy {
1393 fn default() -> Self {
1394 Self {
1395 max_restarts: DEFAULT_MAX_RESTARTS,
1396 backoff: DEFAULT_BACKOFF,
1397 max_backoff: DEFAULT_MAX_BACKOFF,
1398 window: DEFAULT_RESTART_WINDOW,
1399 }
1400 }
1401}
1402
1403#[derive(Debug, Clone, Copy, PartialEq, Eq)]
1404struct CrashRestartSchedule {
1405 restart_in_window: u32,
1406 delay: Duration,
1407}
1408
1409fn daemon_will_restart(
1416 state: &mut SupervisorSnapshot,
1417 policy: &RestartPolicy,
1418 now: Instant,
1419) -> bool {
1420 state.enabled && state.crash_restarts_in_window(policy.window, now) < policy.max_restarts
1421}
1422
1423const DEFAULT_HEALTH_CADENCE: Duration = Duration::from_secs(30);
1424const DEFAULT_HEALTH_DEADLINE: Duration = Duration::from_secs(5);
1425const DEFAULT_HEALTH_FAILURE_THRESHOLD: u32 = 3;
1426const MAX_HEALTH_METRICS_BYTES: usize = 16 * 1024;
1427
1428#[derive(Debug, Clone, Copy, PartialEq, Eq)]
1429pub enum HealthAction {
1430 Report,
1431 Restart,
1432 Alert,
1433}
1434
1435impl fmt::Display for HealthAction {
1436 fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
1437 f.write_str(match self {
1438 Self::Report => "report",
1439 Self::Restart => "restart",
1440 Self::Alert => "alert",
1441 })
1442 }
1443}
1444
1445#[derive(Debug, Clone, PartialEq, Eq)]
1446pub struct HealthConfig {
1447 pub http: Option<String>,
1451 pub cadence: Duration,
1452 pub deadline: Duration,
1453 pub failure_threshold: u32,
1454 pub on_degraded: HealthAction,
1455 pub on_failing: HealthAction,
1456 pub critical: bool,
1457}
1458
1459impl Default for HealthConfig {
1460 fn default() -> Self {
1461 Self {
1462 http: None,
1463 cadence: DEFAULT_HEALTH_CADENCE,
1464 deadline: DEFAULT_HEALTH_DEADLINE,
1465 failure_threshold: DEFAULT_HEALTH_FAILURE_THRESHOLD,
1466 on_degraded: HealthAction::Report,
1467 on_failing: HealthAction::Report,
1468 critical: false,
1469 }
1470 }
1471}
1472
1473#[derive(Debug, Clone, PartialEq)]
1491pub struct ModuleHealthStatus {
1492 pub status: SupervisorHealthStatus,
1493 pub last_probe_ms: Option<u64>,
1494 pub detail: Option<String>,
1495 pub metrics: Option<Value>,
1496 pub consecutive_failures: u32,
1497 pub late_answer_count: u64,
1500 pub last_late_answer_latency_ms: Option<u64>,
1502 pub last_action: Option<String>,
1503 pub last_action_ms: Option<u64>,
1507}
1508
1509impl Default for ModuleHealthStatus {
1510 fn default() -> Self {
1511 Self {
1512 status: SupervisorHealthStatus::Unknown,
1513 last_probe_ms: None,
1514 detail: None,
1515 metrics: None,
1516 consecutive_failures: 0,
1517 late_answer_count: 0,
1518 last_late_answer_latency_ms: None,
1519 last_action: None,
1520 last_action_ms: None,
1521 }
1522 }
1523}
1524
1525#[derive(Debug, Clone, Copy, PartialEq, Eq)]
1527pub enum ModuleState {
1528 Starting,
1529 Running,
1530 Unresponsive,
1531 Restarting,
1532 Draining,
1533 Stopped,
1534 Failed,
1535 Disabled,
1536}
1537
1538impl fmt::Display for ModuleState {
1539 fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
1540 f.write_str(match self {
1541 Self::Starting => "starting",
1542 Self::Running => "running",
1543 Self::Unresponsive => "unresponsive",
1544 Self::Restarting => "restarting",
1545 Self::Draining => "draining",
1546 Self::Stopped => "stopped",
1547 Self::Failed => "failed",
1548 Self::Disabled => "disabled",
1549 })
1550 }
1551}
1552
1553#[derive(Debug, Clone, Copy, PartialEq, Eq)]
1555pub enum ExitKind {
1556 Clean,
1557 Crash,
1558 DeliberateSeverance,
1559}
1560
1561impl From<ExitKind> for TerminalExitKind {
1562 fn from(kind: ExitKind) -> Self {
1563 match kind {
1564 ExitKind::Clean => Self::Clean,
1565 ExitKind::Crash => Self::Crash,
1566 ExitKind::DeliberateSeverance => Self::DeliberateSeverance,
1567 }
1568 }
1569}
1570
1571#[derive(Debug, Clone, Copy, PartialEq, Eq)]
1574pub(crate) struct ProcessIdentity {
1575 pub(crate) pid: u32,
1576 pub(crate) start_time: u64,
1577}
1578
1579#[derive(Debug, Clone, PartialEq, Eq)]
1581pub struct ExitReport {
1582 pub kind: ExitKind,
1583 pub code: Option<i32>,
1584 pub signal: Option<i32>,
1585 pub at_ms: u64,
1586}
1587
1588#[derive(Debug, Clone, PartialEq)]
1591pub struct ModuleStatus {
1592 pub(crate) admission_refused: Option<String>,
1593 pub(crate) legacy_registration_spawn: bool,
1594 pub(crate) registration_declaration: Option<String>,
1595 pub module_id: String,
1596 pub stop_reason: Option<ModuleStopReason>,
1599 pub waiting_for_previous_pid: Option<u32>,
1601 pub waiting_for_previous_detail: Option<String>,
1603 pub launch_nonce_source: Option<String>,
1608 pub state: ModuleState,
1609 pub enabled: bool,
1610 pub process_alive: bool,
1611 pub registration_active: bool,
1612 pub protocol: ModuleProtocol,
1617 pub live: bool,
1628 pub restart_count: u32,
1632 pub lifetime_restarts: u32,
1636 pub spawn_generation: u64,
1637 pub swap_outcome: Option<SwapOutcome>,
1639 pub max_restarts: u32,
1644 pub restart_window: Duration,
1648 pub drain_timeout: Duration,
1652 pub restart_backoff: Duration,
1653 pub restart_max_backoff: Duration,
1654 pub pid: Option<u32>,
1659 pub spawned_at_ms: Option<u64>,
1660 pub spawned_from: Option<PathBuf>,
1661 pub(crate) launch_spec_digest: Option<LaunchSpecDigest>,
1662 pub process_start_time: Option<u64>,
1663 pub last_exit: Option<ExitReport>,
1664 pub health: ModuleHealthStatus,
1665}
1666
1667#[cfg(test)]
1668#[derive(Debug, Clone, Copy, PartialEq)]
1669struct ActorSelectCheckpoint {
1670 generation: u64,
1671 turn: u64,
1672 registered_connection: Option<ConnectionId>,
1673 next_probe_at: Option<Instant>,
1674 wake_after: Option<Duration>,
1675}
1676
1677#[derive(Debug, Clone, PartialEq)]
1678struct SupervisorSnapshot {
1679 admission_refused: Option<String>,
1680 config_hold_parked: bool,
1682 config_hold_daemon_initiated: bool,
1685 config_hold_approval: Option<crate::config_gate::runtime::ApprovalIdentity>,
1687 approved_start_failure: Option<(crate::config_gate::runtime::ApprovalIdentity, String)>,
1689 legacy_registration_spawn: bool,
1690 registration_declaration: Option<String>,
1691 stop_reason: Option<ModuleStopReason>,
1692 config_exit_failed_approval: Option<crate::config_gate::runtime::ApprovalIdentity>,
1693 subtree_target: crate::subtree_resources::Target,
1694 failed_retry: FailedRetry,
1695 exec_outcome: Option<Arc<crate::activity::ExecOutcome>>,
1696 #[cfg(windows)]
1697 pipe_consumed: Option<Arc<OnceLock<()>>>,
1698 #[cfg(test)]
1700 actor_turns: u64,
1701 #[cfg(test)]
1704 actor_select: Option<ActorSelectCheckpoint>,
1705 state: ModuleState,
1706 enabled: bool,
1707 process_alive: bool,
1708 spawned_protocol: Option<ModuleProtocol>,
1709 spawn_failure: Option<String>,
1710 waiting_for_previous_pid: Option<u32>,
1711 waiting_for_previous_detail: Option<String>,
1712 crash_restarts: VecDeque<Instant>,
1718 lifetime_restarts: u32,
1719 spawn_generation: u64,
1728 swap_outcome: Option<SwapOutcome>,
1729 pid: Option<u32>,
1730 #[cfg(target_os = "macos")]
1731 report_ready: Option<Arc<OnceLock<()>>>,
1732 reaped_pid: Option<u32>,
1734 respawn_pending: bool,
1736 coalesced_restart_pending: bool,
1738 spawned_at_ms: Option<u64>,
1739 spawned_from: Option<PathBuf>,
1740 launch_spec_digest: Option<LaunchSpecDigest>,
1741 spawned_file_identity: Option<SpawnedFileIdentity>,
1742 process_start_time: Option<u64>,
1743 deliberate_severance: Option<ProcessIdentity>,
1744 last_exit: Option<ExitReport>,
1745 drain_disposition_detail: Option<String>,
1747 health: ModuleHealthStatus,
1748 last_probe_evidence: Option<HealthProbeEvidence>,
1749 in_alternate_slot: bool,
1754 draining_to_replace: bool,
1761 configuration_updated_since_spawn: bool,
1767}
1768
1769impl SupervisorSnapshot {
1770 fn reported_pid(&self) -> Option<u32> {
1775 #[cfg(target_os = "macos")]
1776 if self
1777 .report_ready
1778 .as_ref()
1779 .is_some_and(|ready| ready.get().is_none())
1780 {
1781 return None;
1782 }
1783 self.pid
1784 }
1785
1786 fn starting() -> Self {
1787 Self::new(ModuleState::Starting, true)
1788 }
1789
1790 fn disabled() -> Self {
1791 Self::new(ModuleState::Disabled, false)
1792 }
1793
1794 fn crash_restarts_in_window(&mut self, window: Duration, now: Instant) -> u32 {
1798 while let Some(oldest) = self.crash_restarts.front() {
1799 if now.duration_since(*oldest) > window {
1800 self.crash_restarts.pop_front();
1801 } else {
1802 break;
1803 }
1804 }
1805 u32::try_from(self.crash_restarts.len()).unwrap_or(u32::MAX)
1806 }
1807
1808 fn record_crash_restart(&mut self, policy: &RestartPolicy, now: Instant) {
1814 self.crash_restarts.push_back(now);
1815 while self.crash_restarts.len() > policy.max_restarts as usize {
1816 self.crash_restarts.pop_front();
1817 }
1818 self.lifetime_restarts += 1;
1819 }
1820
1821 fn next_crash_restart(
1825 &mut self,
1826 policy: &RestartPolicy,
1827 now: Instant,
1828 ) -> Option<CrashRestartSchedule> {
1829 let restart_in_window = self.crash_restarts_in_window(policy.window, now);
1830 if restart_in_window >= policy.max_restarts {
1831 return None;
1832 }
1833 self.record_crash_restart(policy, now);
1834 Some(CrashRestartSchedule {
1835 restart_in_window,
1836 delay: policy.delay_for_restart(restart_in_window),
1837 })
1838 }
1839
1840 fn clear_crash_restarts(&mut self) {
1845 self.crash_restarts.clear();
1846 }
1847
1848 fn new(state: ModuleState, enabled: bool) -> Self {
1849 Self {
1850 admission_refused: None,
1851 config_hold_parked: false,
1852 config_hold_daemon_initiated: false,
1853 config_hold_approval: None,
1854 approved_start_failure: None,
1855 legacy_registration_spawn: false,
1856 registration_declaration: None,
1857 stop_reason: None,
1858 config_exit_failed_approval: None,
1859 subtree_target: crate::subtree_resources::Target::Unavailable("not_running"),
1860 failed_retry: FailedRetry::default(),
1861 exec_outcome: None,
1862 #[cfg(test)]
1863 actor_turns: 0,
1864 #[cfg(test)]
1865 actor_select: None,
1866 state,
1867 enabled,
1868 process_alive: false,
1869 #[cfg(windows)]
1870 pipe_consumed: None,
1871 spawned_protocol: None,
1872 spawn_failure: None,
1873 waiting_for_previous_pid: None,
1874 waiting_for_previous_detail: None,
1875 crash_restarts: VecDeque::new(),
1876 lifetime_restarts: 0,
1877 spawn_generation: 0,
1878 swap_outcome: None,
1879 pid: None,
1880 #[cfg(target_os = "macos")]
1881 report_ready: None,
1882 reaped_pid: None,
1883 respawn_pending: false,
1884 coalesced_restart_pending: false,
1885 spawned_at_ms: None,
1886 spawned_from: None,
1887 launch_spec_digest: None,
1888 spawned_file_identity: None,
1889 process_start_time: None,
1890 deliberate_severance: None,
1891 last_exit: None,
1892 drain_disposition_detail: None,
1893 health: ModuleHealthStatus::default(),
1894 last_probe_evidence: None,
1895 in_alternate_slot: false,
1896 draining_to_replace: false,
1897 configuration_updated_since_spawn: false,
1898 }
1899 }
1900}
1901
1902type SharedSnapshot = Arc<Mutex<SupervisorSnapshot>>;
1903
1904type SpawnSubscriberKey = (ConnectionId, u64);
1905
1906#[derive(Debug)]
1907struct SpawnSubscriber {
1908 version: u8,
1909 frames: mpsc::Sender<Frame>,
1910 lagged: Option<oneshot::Sender<SpawnCursor>>,
1914}
1915
1916#[derive(Debug)]
1917struct SpawnEventState {
1918 daemon_incarnation: String,
1919 seq: u64,
1920 capacity: usize,
1921 live: HashMap<String, LiveSpawn>,
1922 generations: HashMap<String, u64>,
1923 events: VecDeque<SpawnEvent>,
1924 subscribers: HashMap<SpawnSubscriberKey, SpawnSubscriber>,
1925 process_changes: HashMap<String, tokio::sync::watch::Sender<u64>>,
1926}
1927
1928impl Default for SpawnEventState {
1929 fn default() -> Self {
1930 Self {
1931 daemon_incarnation: "unconfigured".to_string(),
1932 seq: 0,
1933 capacity: SPAWN_EVENT_RING_CAPACITY,
1934 live: HashMap::new(),
1935 generations: HashMap::new(),
1936 events: VecDeque::new(),
1937 subscribers: HashMap::new(),
1938 process_changes: HashMap::new(),
1939 }
1940 }
1941}
1942
1943#[derive(Debug, Clone, Default)]
1944struct SpawnEventFeed(
1945 Arc<Mutex<SpawnEventState>>,
1946 #[cfg(test)] Arc<Mutex<Option<std::net::TcpStream>>>,
1947);
1948
1949#[derive(Debug, Clone, PartialEq, Eq)]
1950pub(crate) enum SpawnSubscribeRefusal {
1951 ForeignIncarnation { current: String },
1952 TooOld { oldest: SpawnCursor },
1953 Frame(String),
1954}
1955
1956impl SpawnEventFeed {
1957 fn configure_incarnation(&self, daemon_incarnation: String) {
1958 let mut state = self.0.lock().unwrap_or_else(|p| p.into_inner());
1959 state.daemon_incarnation = daemon_incarnation;
1960 state.seq = 0;
1961 state.live.clear();
1962 state.generations.clear();
1963 state.events.clear();
1964 state.subscribers.clear();
1965 for changed in state.process_changes.values() {
1966 changed.send_replace(0);
1967 }
1968 }
1969
1970 fn cursor(state: &SpawnEventState) -> SpawnCursor {
1971 SpawnCursor {
1972 daemon_incarnation: state.daemon_incarnation.clone(),
1973 seq: state.seq,
1974 }
1975 }
1976
1977 fn snapshot(&self) -> SpawnSnapshot {
1978 let state = self.0.lock().unwrap_or_else(|p| p.into_inner());
1979 let mut live = state.live.values().cloned().collect::<Vec<_>>();
1980 live.sort_by(|left, right| left.module_id.cmp(&right.module_id));
1981 SpawnSnapshot {
1982 cursor: Self::cursor(&state),
1983 ring_bound: state.capacity as u64,
1984 live,
1985 }
1986 }
1987
1988 fn allocate_generation_locked(state: &mut SpawnEventState, module_id: &str) -> u64 {
1989 let generation = state
1990 .generations
1991 .get(module_id)
1992 .copied()
1993 .unwrap_or(0)
1994 .checked_add(1)
1995 .expect("spawn generation exhausted");
1996 state.generations.insert(module_id.to_string(), generation);
1997 generation
1998 }
1999
2000 fn allocate_generation(&self, module_id: &str) -> (String, u64) {
2005 let mut state = self.0.lock().unwrap_or_else(|p| p.into_inner());
2006 let generation = Self::allocate_generation_locked(&mut state, module_id);
2007 (state.daemon_incarnation.clone(), generation)
2008 }
2009
2010 fn emit_spawned(
2014 &self,
2015 module_id: &str,
2016 allocated: Option<u64>,
2017 pid: u32,
2018 spawned_at_ms: u64,
2019 ) -> u64 {
2020 let mut state = self.0.lock().unwrap_or_else(|p| p.into_inner());
2021 let generation = match allocated {
2022 Some(generation) => generation,
2023 None => Self::allocate_generation_locked(&mut state, module_id),
2024 };
2025 let live = LiveSpawn {
2026 module_id: module_id.to_string(),
2027 spawn_generation: generation,
2028 pid,
2029 spawned_at_ms,
2030 };
2031 state.live.insert(module_id.to_string(), live);
2032 Self::emit_locked(
2033 &mut state,
2034 SpawnEventKind::Spawned,
2035 module_id.to_string(),
2036 generation,
2037 pid,
2038 None,
2039 None,
2040 );
2041 generation
2042 }
2043
2044 fn emit_exited(&self, module_id: &str, exit_code: Option<i32>, exit_signal: Option<i32>) {
2045 let mut state = self.0.lock().unwrap_or_else(|p| p.into_inner());
2046 let Some(live) = state.live.remove(module_id) else {
2047 warn!(
2048 module_id,
2049 "terminal record had no live spawn event identity"
2050 );
2051 return;
2052 };
2053 Self::emit_locked(
2054 &mut state,
2055 SpawnEventKind::Exited,
2056 module_id.to_string(),
2057 live.spawn_generation,
2058 live.pid,
2059 exit_code,
2060 exit_signal,
2061 );
2062 }
2063
2064 fn emit_superseded_exited(
2071 &self,
2072 module_id: &str,
2073 spawn_generation: u64,
2074 pid: u32,
2075 exit_code: Option<i32>,
2076 exit_signal: Option<i32>,
2077 ) {
2078 let mut state = self.0.lock().unwrap_or_else(|p| p.into_inner());
2079 if state
2080 .live
2081 .get(module_id)
2082 .is_some_and(|live| live.spawn_generation == spawn_generation)
2083 {
2084 state.live.remove(module_id);
2085 }
2086 Self::emit_locked(
2087 &mut state,
2088 SpawnEventKind::Exited,
2089 module_id.to_string(),
2090 spawn_generation,
2091 pid,
2092 exit_code,
2093 exit_signal,
2094 );
2095 }
2096
2097 #[allow(clippy::too_many_arguments)]
2098 fn emit_locked(
2099 state: &mut SpawnEventState,
2100 kind: SpawnEventKind,
2101 module_id: String,
2102 spawn_generation: u64,
2103 pid: u32,
2104 exit_code: Option<i32>,
2105 exit_signal: Option<i32>,
2106 ) {
2107 state.seq = state
2108 .seq
2109 .checked_add(1)
2110 .expect("spawn event sequence exhausted");
2111 let event = SpawnEvent {
2112 cursor: Self::cursor(state),
2113 kind,
2114 module_id,
2115 spawn_generation,
2116 pid,
2117 exit_code,
2118 exit_signal,
2119 };
2120 state
2121 .process_changes
2122 .retain(|_, changed| changed.receiver_count() > 0);
2123 if let Some(changed) = state.process_changes.get(&event.module_id) {
2124 changed.send_replace(state.seq);
2125 }
2126 state.events.push_back(event.clone());
2127 while state.events.len() > state.capacity {
2128 state.events.pop_front();
2129 }
2130 let body = match serde_json::to_vec(&event) {
2131 Ok(body) => body,
2132 Err(error) => {
2133 error!(%error, "failed to serialize supervisor spawn event");
2134 return;
2135 }
2136 };
2137 state.subscribers.retain(|(connection_id, corr), subscriber| {
2138 let frame = Frame::build_with_version(
2139 subscriber.version,
2140 FrameType::StreamData,
2141 control_flags(),
2142 0,
2143 0,
2144 *corr,
2145 body.clone(),
2146 );
2147 match frame {
2148 Ok(frame) => {
2149 if subscriber.frames.try_send(frame).is_ok() {
2150 true
2151 } else {
2152 warn!(connection_id = connection_id.get(), corr, "dropping lagged supervisor spawn subscriber");
2153 if let Some(lagged) = subscriber.lagged.take() {
2154 let _ = lagged.send(event.cursor.clone());
2155 }
2156 false
2157 }
2158 }
2159 Err(error) => {
2160 warn!(connection_id = connection_id.get(), corr, %error, "dropping supervisor spawn subscriber after frame build failure");
2161 false
2162 }
2163 }
2164 });
2165 }
2166
2167 fn subscribe(
2168 &self,
2169 connection_id: ConnectionId,
2170 corr: u64,
2171 version: u8,
2172 since: Option<SpawnCursor>,
2173 sink: FrameSink,
2174 ) -> Result<(), SpawnSubscribeRefusal> {
2175 let (frames, mut receiver) = mpsc::channel(SPAWN_SUBSCRIBER_BUFFER);
2176 let (lagged, mut lagged_rx) = oneshot::channel::<SpawnCursor>();
2177 {
2178 let mut state = self.0.lock().unwrap_or_else(|p| p.into_inner());
2179 let replay = if let Some(since) = since {
2180 if since.daemon_incarnation != state.daemon_incarnation {
2181 return Err(SpawnSubscribeRefusal::ForeignIncarnation {
2182 current: state.daemon_incarnation.clone(),
2183 });
2184 }
2185 if let Some(oldest) = state.events.front().map(|event| event.cursor.clone()) {
2186 if since.seq < oldest.seq.saturating_sub(1) {
2187 return Err(SpawnSubscribeRefusal::TooOld { oldest });
2188 }
2189 }
2190 state
2191 .events
2192 .iter()
2193 .filter(|event| event.cursor.seq > since.seq)
2194 .cloned()
2195 .collect::<Vec<_>>()
2196 } else {
2197 Vec::new()
2198 };
2199 for event in replay {
2200 let body = serde_json::to_vec(&event)
2201 .map_err(|error| SpawnSubscribeRefusal::Frame(error.to_string()))?;
2202 let frame = Frame::build_with_version(
2203 version,
2204 FrameType::StreamData,
2205 control_flags(),
2206 0,
2207 0,
2208 corr,
2209 body,
2210 )
2211 .map_err(|error| SpawnSubscribeRefusal::Frame(error.to_string()))?;
2212 frames
2213 .try_send(frame)
2214 .map_err(|error| SpawnSubscribeRefusal::Frame(error.to_string()))?;
2215 }
2216 state.subscribers.insert(
2217 (connection_id, corr),
2218 SpawnSubscriber {
2219 version,
2220 frames,
2221 lagged: Some(lagged),
2222 },
2223 );
2224 }
2225 tokio::spawn(async move {
2236 while let Some(frame) = receiver.recv().await {
2237 if sink.send(frame).await.is_err() {
2238 return;
2239 }
2240 }
2241 let Ok(first_undelivered) = lagged_rx.try_recv() else {
2242 return;
2243 };
2244 match spawn_subscriber_lagged_frame(version, corr, first_undelivered) {
2245 Ok(frame) => {
2246 let _ = sink.send(frame).await;
2247 }
2248 Err(error) => {
2249 error!(%error, corr, "failed to build lagged spawn subscriber terminal frame");
2250 }
2251 }
2252 });
2253 Ok(())
2254 }
2255
2256 fn cancel(&self, connection_id: ConnectionId, corr: u64) -> bool {
2257 let Some(subscriber) = self
2258 .0
2259 .lock()
2260 .unwrap_or_else(|p| p.into_inner())
2261 .subscribers
2262 .remove(&(connection_id, corr))
2263 else {
2264 return false;
2265 };
2266 if let Ok(frame) = Frame::build_with_version(
2267 subscriber.version,
2268 FrameType::StreamEnd,
2269 control_flags(),
2270 0,
2271 0,
2272 corr,
2273 Vec::new(),
2274 ) {
2275 tokio::spawn(async move {
2276 let _ = subscriber.frames.send(frame).await;
2277 });
2278 }
2279 true
2280 }
2281
2282 fn remove_connection(&self, connection_id: ConnectionId) {
2283 self.0
2284 .lock()
2285 .unwrap_or_else(|p| p.into_inner())
2286 .subscribers
2287 .retain(|(subscriber_connection, _), _| *subscriber_connection != connection_id);
2288 }
2289
2290 #[cfg(any(test, feature = "test-support"))]
2291 fn set_capacity(&self, capacity: usize) {
2292 self.0.lock().unwrap_or_else(|p| p.into_inner()).capacity = capacity;
2293 }
2294
2295 #[cfg(any(test, feature = "test-support"))]
2296 fn subscriber_count(&self) -> usize {
2297 self.0
2298 .lock()
2299 .unwrap_or_else(|p| p.into_inner())
2300 .subscribers
2301 .len()
2302 }
2303}
2304
2305fn spawn_subscriber_lagged_frame(
2308 version: u8,
2309 corr: u64,
2310 first_undelivered: SpawnCursor,
2311) -> Result<Frame, String> {
2312 let body = serde_json::to_vec(&subc_protocol::ErrorBody {
2314 code: SPAWN_SUBSCRIBER_LAGGED_CODE.to_string(),
2315 message: "spawn subscriber fell behind and was dropped; resubscribe from the last cursor received"
2316 .to_string(),
2317 detail: Some(serde_json::json!({
2318 "first_undelivered_cursor": first_undelivered
2319 })),
2320 })
2321 .map_err(|error| error.to_string())?;
2322 Frame::build_with_version(
2323 version,
2324 FrameType::Error,
2325 crate::router::daemon_error_flags(),
2326 0,
2327 0,
2328 corr,
2329 body,
2330 )
2331 .map_err(|error| error.to_string())
2332}
2333
2334pub trait ModuleProcessLiveness: Send + Sync {
2335 fn process_live(&self, module_id: &str) -> Option<bool>;
2336
2337 fn process_replacing(&self, _module_id: &str) -> bool {
2343 false
2344 }
2345}
2346
2347#[derive(Debug, Clone, Default)]
2349pub struct SupervisorProcessLiveness {
2350 snapshots: Arc<Mutex<HashMap<String, SharedSnapshot>>>,
2351}
2352
2353impl SupervisorProcessLiveness {
2354 pub fn new() -> Self {
2355 Self::default()
2356 }
2357
2358 fn track(&self, module_id: String, snapshot: SharedSnapshot) {
2359 let mut snapshots = self
2360 .snapshots
2361 .lock()
2362 .unwrap_or_else(|poisoned| poisoned.into_inner());
2363 snapshots.insert(module_id, snapshot);
2364 }
2365
2366 fn untrack_if_current(&self, module_id: &str, snapshot: &SharedSnapshot) {
2367 let mut snapshots = self
2368 .snapshots
2369 .lock()
2370 .unwrap_or_else(|poisoned| poisoned.into_inner());
2371 let is_current = snapshots
2372 .get(module_id)
2373 .map(|tracked| Arc::ptr_eq(tracked, snapshot))
2374 .unwrap_or(false);
2375 if is_current {
2376 snapshots.remove(module_id);
2377 }
2378 }
2379}
2380
2381impl ModuleProcessLiveness for SupervisorProcessLiveness {
2382 fn process_live(&self, module_id: &str) -> Option<bool> {
2383 let snapshot = {
2384 let snapshots = self
2385 .snapshots
2386 .lock()
2387 .unwrap_or_else(|poisoned| poisoned.into_inner());
2388 snapshots.get(module_id).cloned()
2389 }?;
2390 let snapshot = snapshot
2391 .lock()
2392 .unwrap_or_else(|poisoned| poisoned.into_inner());
2393 Some(snapshot.state == ModuleState::Running && snapshot.process_alive)
2394 }
2395
2396 fn process_replacing(&self, module_id: &str) -> bool {
2397 let Some(snapshot) = self
2398 .snapshots
2399 .lock()
2400 .unwrap_or_else(|poisoned| poisoned.into_inner())
2401 .get(module_id)
2402 .cloned()
2403 else {
2404 return false;
2405 };
2406 let snapshot = snapshot
2407 .lock()
2408 .unwrap_or_else(|poisoned| poisoned.into_inner());
2409 snapshot.enabled
2410 && match snapshot.state {
2411 ModuleState::Restarting => true,
2412 ModuleState::Draining => snapshot.draining_to_replace,
2413 ModuleState::Starting
2414 | ModuleState::Running
2415 | ModuleState::Unresponsive
2416 | ModuleState::Stopped
2417 | ModuleState::Failed
2418 | ModuleState::Disabled => false,
2419 }
2420 }
2421}
2422
2423#[cfg(test)]
2424#[derive(Debug, Default)]
2425struct ReloadExitRecordGate {
2426 reached: tokio::sync::Notify,
2427 resume: tokio::sync::Notify,
2428}
2429
2430#[derive(Debug, Clone, Copy)]
2431enum RespawnKind {
2432 Spawn,
2433 Reload,
2434}
2435
2436#[derive(Debug, Clone)]
2437struct PendingRespawn {
2438 deadline: Instant,
2439 kind: RespawnKind,
2440 daemon_initiated: bool,
2441 approval: Option<crate::config_gate::runtime::ApprovalIdentity>,
2442}
2443
2444impl PendingRespawn {
2445 fn failure_context(&self, runtime: &SupervisorRuntimeConfig) -> FailureContext {
2446 FailureContext {
2447 approval: self.approval.clone(),
2448 ..runtime.failure_context(self.daemon_initiated)
2449 }
2450 }
2451}
2452
2453type ReloadReply = oneshot::Sender<Result<(), SuperviseError>>;
2454
2455#[derive(Debug, Clone)]
2456struct SupervisorRuntimeConfig {
2457 registry: Arc<Registry>,
2458 config_delivery: Option<config_delivery::ConfigDelivery>,
2459 recovery_is_daemon: bool,
2461 start_requester: Option<Principal>,
2462 pending_activity_restart: bool,
2463 provenance_probe: ExecutableIdentityProbe,
2464 scheduled_respawn: Arc<Mutex<Option<PendingRespawn>>>,
2466 deferred_reload_reply: Arc<Mutex<Option<ReloadReply>>>,
2468 restart_policy: RestartPolicy,
2469 register_timeout: Duration,
2470 drain_timeout: Duration,
2473 effective_drain_timeout: Arc<Mutex<Duration>>,
2476 default_drain_timeout: Duration,
2479 health: HealthConfig,
2480 host_health: HostHealth,
2481 connection_file_path: Option<PathBuf>,
2482 capture_logs_dir: Option<PathBuf>,
2483 forwarding: Option<Arc<ForwardingTable>>,
2484 supervisor_handle: Option<SupervisorHandle>,
2487 stderr_ring: Arc<Mutex<StderrRing>>,
2494 terminal_ring: Arc<Mutex<TerminalRing>>,
2495 spawn_events: SpawnEventFeed,
2496 child_roster: ChildRoster,
2497 #[cfg(target_os = "linux")]
2498 cgroup_placement: Option<subc_cgroup::Placement>,
2499 #[cfg(test)]
2500 test_seed_stale_facts_before_enable_spawn: bool,
2501 #[cfg(test)]
2502 test_reload_exit_record_gate: Option<Arc<ReloadExitRecordGate>>,
2503}
2504
2505impl SupervisorRuntimeConfig {
2506 fn failure_context(&self, daemon_initiated: bool) -> FailureContext {
2507 FailureContext {
2508 daemon_initiated,
2509 approval: None,
2510 shutdown: self.child_roster.shutdown_flag(),
2511 }
2512 }
2513}
2514
2515#[derive(Debug, Clone, PartialEq, Eq)]
2516struct SupervisedConfiguration {
2517 spec: ModuleSpec,
2518 enabled: bool,
2519 health: HealthConfig,
2520 register_timeout: Duration,
2521 restart_policy: RestartPolicy,
2522}
2523
2524#[derive(Debug, Clone, Default)]
2530pub struct SupervisorHandle {
2531 #[cfg(test)]
2532 resource_commit_pause: Arc<Mutex<Option<resource_launch::ResourceCommitPause>>>,
2533 #[cfg(test)]
2534 registration_test_channels: CapturedRegistrationChannels,
2535 registration_service: Arc<Mutex<Option<registration::RegistrationService>>>,
2536 registration_connections: Arc<Mutex<HashMap<ConnectionId, Arc<RegistrationSpawn>>>>,
2537 registration_launches: Arc<Mutex<Vec<std::sync::Weak<RegistrationSpawn>>>>,
2538 gate_runtime: Arc<Mutex<Option<std::sync::Weak<crate::config_gate::runtime::RuntimeOwner>>>>,
2539 program_bindings: program_binding::ProgramBindings,
2540 #[cfg(test)]
2541 identity_sample_pause: Arc<Mutex<Option<IdentitySamplePause>>>,
2542 #[cfg(all(test, unix))]
2543 plain_identity_policy_pause: Arc<Mutex<Option<(String, IdentitySamplePause)>>>,
2544 host_health: HostHealth,
2545 restart_intent: Arc<Mutex<DaemonRestartIntent>>,
2546 activity: Arc<Mutex<Option<crate::activity::ActivityLedger>>>,
2547 activity_images: Arc<Mutex<HashMap<String, activity::ActivityImage>>>,
2548 modules: Arc<Mutex<HashMap<String, SupervisedModule>>>,
2549 configured_ids: Arc<Mutex<HashSet<String>>>,
2561 spawn_events: SpawnEventFeed,
2562 reserved_nonces: Arc<Mutex<HashMap<String, Option<String>>>>,
2573 implicit_protocol_none: Arc<Mutex<HashSet<String>>>,
2577 removal_tombstones: Arc<Mutex<HashMap<String, u64>>>,
2583 spawn_nonces: Arc<Mutex<HashMap<String, String>>>,
2587 nonce_launches: Arc<Mutex<NonceLaunches>>,
2593 reserved_prefix_owners: Arc<Mutex<HashMap<String, String>>>,
2601 swaps: Arc<Mutex<HashMap<String, OpenSwap>>>,
2607 promotion_observer: PromotionObserverSlot,
2609 operation_lock: Arc<AsyncMutex<()>>,
2613}
2614
2615#[cfg(test)]
2616#[derive(Debug)]
2617struct IdentitySamplePause {
2618 reached: std::sync::mpsc::Sender<()>,
2619 release: std::sync::mpsc::Receiver<()>,
2620}
2621
2622pub(crate) trait SwapPromotionObserver: Send + Sync {
2631 fn swap_promoted(&self, registration: &crate::registry::ModuleRegistration);
2632}
2633
2634#[derive(Clone, Default)]
2638struct PromotionObserverSlot(Arc<Mutex<Option<std::sync::Weak<dyn SwapPromotionObserver>>>>);
2639
2640impl fmt::Debug for PromotionObserverSlot {
2641 fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
2642 f.write_str("PromotionObserverSlot")
2643 }
2644}
2645
2646#[derive(Debug, Clone)]
2648struct OpenSwap {
2649 candidate_nonce: String,
2652 incumbent_nonce: Option<String>,
2655 candidate_admitted: bool,
2659}
2660
2661#[derive(Debug, Clone, Copy, PartialEq, Eq)]
2664pub(crate) enum SwapHelloAdmission {
2665 NotSwapping,
2668 Candidate,
2670 Refused,
2673 ProtocolNone,
2676 ChannelRequired,
2679}
2680
2681#[derive(Debug, Clone, PartialEq, Eq)]
2682pub(crate) enum ReservedHelloRejection {
2683 Exact {
2684 module_id: String,
2685 },
2686 Prefix {
2687 prefix: String,
2688 owner_module_id: String,
2689 },
2690}
2691
2692#[derive(Debug, Default)]
2694pub(crate) struct ReservedHelloVerification {
2695 pub(crate) nonce_verified: bool,
2696 pub(crate) prefix_owner: Option<String>,
2697}
2698
2699#[derive(Debug)]
2700struct DaemonRestartIntent {
2701 expires: Option<Duration>,
2702 window: Duration,
2703}
2704
2705impl Default for DaemonRestartIntent {
2706 fn default() -> Self {
2707 Self {
2708 expires: None,
2709 window: Duration::from_secs(120),
2710 }
2711 }
2712}
2713
2714impl SupervisorHandle {
2715 pub(crate) fn module_is_gated(&self, module: &str) -> bool {
2716 let runtime = {
2717 let Ok(runtime) = self.gate_runtime.lock() else {
2718 return true;
2719 };
2720 runtime.clone()
2721 };
2722 runtime.is_some_and(|runtime| {
2723 runtime
2724 .upgrade()
2725 .is_none_or(|owner| owner.module_is_gated(module))
2726 })
2727 }
2728 pub(crate) fn announce_daemon_restart(&self) -> u64 {
2729 let mut intent = self
2730 .restart_intent
2731 .lock()
2732 .unwrap_or_else(|p| p.into_inner());
2733 intent.expires = Some(subc_uptime::suspend_inclusive_now() + intent.window);
2734 unix_ms_now().saturating_add(intent.window.as_millis() as u64)
2735 }
2736
2737 #[cfg(feature = "test-support")]
2738 pub(crate) fn set_shutdown_intent_window(&self, window: Duration) {
2739 self.restart_intent
2740 .lock()
2741 .unwrap_or_else(|p| p.into_inner())
2742 .window = window;
2743 }
2744
2745 #[cfg(unix)]
2746 fn take_daemon_shutdown_reason(&self) -> RouteCloseReason {
2747 let expiry = self
2748 .restart_intent
2749 .lock()
2750 .unwrap_or_else(|p| p.into_inner())
2751 .expires
2752 .take();
2753 if expiry.is_some_and(|expiry| subc_uptime::suspend_inclusive_now() < expiry) {
2754 RouteCloseReason::Restart
2755 } else {
2756 RouteCloseReason::Disable
2757 }
2758 }
2759
2760 pub fn new() -> Self {
2761 Self::default()
2762 }
2763
2764 pub(crate) fn spawn_snapshot(&self) -> SpawnSnapshot {
2765 self.spawn_events.snapshot()
2766 }
2767
2768 pub(crate) fn subscribe_process_changes(
2771 &self,
2772 module: &str,
2773 ) -> tokio::sync::watch::Receiver<u64> {
2774 let mut state = self
2775 .spawn_events
2776 .0
2777 .lock()
2778 .unwrap_or_else(|p| p.into_inner());
2779 state
2780 .process_changes
2781 .retain(|_, changed| changed.receiver_count() > 0);
2782 let seq = state.seq;
2783 state
2784 .process_changes
2785 .entry(module.into())
2786 .or_insert_with(|| tokio::sync::watch::channel(seq).0)
2787 .subscribe()
2788 }
2789
2790 pub(crate) fn current_spawn(&self, module: &str) -> Option<LiveSpawn> {
2791 self.spawn_events
2792 .0
2793 .lock()
2794 .unwrap_or_else(|p| p.into_inner())
2795 .live
2796 .get(module)
2797 .cloned()
2798 }
2799
2800 pub(crate) fn subscribe_spawns(
2801 &self,
2802 connection_id: ConnectionId,
2803 corr: u64,
2804 version: u8,
2805 since: Option<SpawnCursor>,
2806 sink: FrameSink,
2807 ) -> Result<(), SpawnSubscribeRefusal> {
2808 self.spawn_events
2809 .subscribe(connection_id, corr, version, since, sink)
2810 }
2811
2812 pub(crate) fn cancel_spawn_subscription(&self, connection_id: ConnectionId, corr: u64) -> bool {
2813 self.spawn_events.cancel(connection_id, corr)
2814 }
2815
2816 pub(crate) fn remove_spawn_subscribers(&self, connection_id: ConnectionId) {
2817 self.spawn_events.remove_connection(connection_id);
2818 }
2819
2820 #[cfg(any(test, feature = "test-support"))]
2821 pub fn set_spawn_event_capacity_for_test(&self, capacity: usize) {
2822 assert!(capacity > 0, "spawn event capacity must be non-zero");
2823 self.spawn_events.set_capacity(capacity);
2824 }
2825
2826 #[cfg(any(test, feature = "test-support"))]
2827 pub fn spawn_subscriber_count_for_test(&self) -> usize {
2828 self.spawn_events.subscriber_count()
2829 }
2830
2831 pub fn set_spawn_nonce(&self, module_id: &str, nonce: String) {
2834 self.spawn_nonces
2835 .lock()
2836 .unwrap_or_else(|poisoned| poisoned.into_inner())
2837 .insert(module_id.to_string(), nonce);
2838 self.refresh_program_binding_nonces(module_id);
2839 }
2840
2841 pub fn set_reserved_nonce(&self, module_id: &str, nonce: String) {
2844 {
2845 let mut reserved = self
2846 .reserved_nonces
2847 .lock()
2848 .unwrap_or_else(|p| p.into_inner());
2849 self.implicit_protocol_none
2850 .lock()
2851 .unwrap_or_else(|p| p.into_inner())
2852 .remove(module_id);
2853 reserved.insert(module_id.to_string(), Some(nonce));
2854 }
2855 self.refresh_program_binding_nonces(module_id);
2856 }
2857
2858 pub fn set_reserved_prefixes(&self, owner_module_id: &str, prefixes: &[String]) {
2860 let mut owners = self
2861 .reserved_prefix_owners
2862 .lock()
2863 .unwrap_or_else(|poisoned| poisoned.into_inner());
2864 owners.retain(|_, owner| owner != owner_module_id);
2865 for prefix in prefixes {
2866 owners.insert(prefix.clone(), owner_module_id.to_string());
2867 }
2868 }
2869
2870 pub(crate) fn spawn_nonce(&self, module_id: &str) -> Option<String> {
2872 if self.module_is_gated(module_id) {
2873 return None;
2874 }
2875 self.spawn_nonces
2876 .lock()
2877 .unwrap_or_else(|poisoned| poisoned.into_inner())
2878 .get(module_id)
2879 .cloned()
2880 }
2881
2882 #[cfg(all(test, unix))]
2883 pub(crate) fn legacy_nonce_present_for_test(&self, module: &str) -> bool {
2884 self.spawn_nonces.lock().unwrap().contains_key(module)
2885 }
2886
2887 pub(crate) fn apply_identity_configuration(&self, spec: &ModuleSpec) {
2888 self.set_reserved_prefixes(&spec.module_id, &spec.reserved_prefixes);
2891 let protocol = self
2892 .get(&spec.module_id)
2893 .and_then(|module| {
2894 let snapshot = module.inner.snapshot.lock().ok()?;
2895 snapshot
2896 .process_alive
2897 .then_some(snapshot.spawned_protocol.unwrap_or(spec.protocol))
2898 })
2899 .unwrap_or(spec.protocol);
2900 self.apply_identity_policy(spec, protocol);
2901 }
2902
2903 #[cfg(all(test, unix))]
2904 pub(crate) fn pause_plain_identity_policy_for_test(
2905 &self,
2906 module: &str,
2907 ) -> (std::sync::mpsc::Receiver<()>, std::sync::mpsc::Sender<()>) {
2908 let (reached, waiting) = std::sync::mpsc::channel();
2909 let (release, paused) = std::sync::mpsc::channel();
2910 *self.plain_identity_policy_pause.lock().unwrap() = Some((
2911 module.to_owned(),
2912 IdentitySamplePause {
2913 reached,
2914 release: paused,
2915 },
2916 ));
2917 (waiting, release)
2918 }
2919
2920 fn apply_identity_policy(&self, spec: &ModuleSpec, protocol: ModuleProtocol) {
2921 let channel_only = self.module_is_gated(&spec.module_id);
2922 let mut current_nonces = self
2923 .spawn_nonces
2924 .lock()
2925 .unwrap_or_else(|poisoned| poisoned.into_inner());
2926 if protocol == ModuleProtocol::None || channel_only {
2927 current_nonces.remove(&spec.module_id);
2928 }
2929 let spawn_nonce = current_nonces.get(&spec.module_id).cloned();
2930 #[cfg(test)]
2931 if let Some(pause) = self.identity_sample_pause.lock().unwrap().take() {
2932 pause.reached.send(()).unwrap();
2933 pause.release.recv().unwrap();
2934 }
2935 let mut reserved_nonces = self
2936 .reserved_nonces
2937 .lock()
2938 .unwrap_or_else(|poisoned| poisoned.into_inner());
2939 let mut implicit = self
2940 .implicit_protocol_none
2941 .lock()
2942 .unwrap_or_else(|p| p.into_inner());
2943 if spec.reserved {
2944 implicit.remove(&spec.module_id);
2945 } else if protocol == ModuleProtocol::None {
2946 if !reserved_nonces.contains_key(&spec.module_id) {
2947 implicit.insert(spec.module_id.clone());
2948 }
2949 } else if implicit.remove(&spec.module_id) {
2950 reserved_nonces.remove(&spec.module_id);
2951 }
2952 if spec.reserved || protocol == ModuleProtocol::None {
2953 reserved_nonces.insert(spec.module_id.clone(), spawn_nonce);
2958 }
2959 drop(implicit);
2960 drop(reserved_nonces);
2961 drop(current_nonces);
2962 self.refresh_program_binding_nonces(&spec.module_id);
2963 self.removal_tombstones
2967 .lock()
2968 .unwrap_or_else(|poisoned| poisoned.into_inner())
2969 .remove(&spec.module_id);
2970 }
2971
2972 pub fn reserved_hello_authorized(&self, module_id: &str, presented: Option<&str>) -> bool {
2977 self.reserved_hello_verification(module_id, presented)
2978 .is_ok()
2979 }
2980
2981 pub(crate) fn reserved_hello_verification(
2984 &self,
2985 module_id: &str,
2986 presented: Option<&str>,
2987 ) -> Result<ReservedHelloVerification, ReservedHelloRejection> {
2988 if self.module_is_gated(module_id) {
2989 return Err(ReservedHelloRejection::Exact {
2990 module_id: module_id.into(),
2991 });
2992 }
2993 self.hello_verification(module_id, presented, false)
2994 }
2995
2996 pub(crate) fn channel_hello_verification(
2997 &self,
2998 module_id: &str,
2999 ) -> Result<ReservedHelloVerification, ReservedHelloRejection> {
3000 if self
3001 .get(module_id)
3002 .is_some_and(|module| module.declared_protocol().ok() == Some(ModuleProtocol::None))
3003 {
3004 return Err(ReservedHelloRejection::Exact {
3005 module_id: module_id.into(),
3006 });
3007 }
3008 let mut verification = self.hello_verification(module_id, None, true)?;
3009 verification.nonce_verified = true;
3010 Ok(verification)
3011 }
3012
3013 pub(crate) fn prefix_claim_authorized(&self, owner: &str, module_id: &str) -> bool {
3014 self.prefix_claim_publication(owner, module_id).is_some()
3015 }
3016
3017 #[cfg(all(test, unix))]
3018 pub(crate) fn prefix_publication_locked_for_test(&self) -> bool {
3019 matches!(
3020 self.reserved_prefix_owners.try_lock(),
3021 Err(std::sync::TryLockError::WouldBlock)
3022 )
3023 }
3024
3025 pub(crate) fn prefix_claim_publication(
3029 &self,
3030 owner: &str,
3031 module_id: &str,
3032 ) -> Option<std::sync::MutexGuard<'_, HashMap<String, String>>> {
3033 if subc_protocol::manifest::validate_module_id(module_id).is_err()
3034 || self.module_is_gated(module_id)
3035 || self
3036 .reserved_nonces
3037 .lock()
3038 .unwrap_or_else(|p| p.into_inner())
3039 .contains_key(module_id)
3040 {
3041 return None;
3042 }
3043 let owners = self
3044 .reserved_prefix_owners
3045 .lock()
3046 .unwrap_or_else(|p| p.into_inner());
3047 let authorized = owners
3048 .iter()
3049 .filter(|(prefix, _)| module_id.starts_with(prefix.as_str()))
3050 .max_by_key(|(prefix, _)| prefix.len())
3051 .is_some_and(|(prefix, holder)| holder == owner && module_id.len() > prefix.len());
3052 authorized.then_some(owners)
3053 }
3054
3055 fn hello_verification(
3056 &self,
3057 module_id: &str,
3058 presented: Option<&str>,
3059 channel: bool,
3060 ) -> Result<ReservedHelloVerification, ReservedHelloRejection> {
3061 let nonces = self
3062 .reserved_nonces
3063 .lock()
3064 .unwrap_or_else(|poisoned| poisoned.into_inner());
3065 if let Some(expected) = nonces.get(module_id) {
3066 let authorized = channel
3070 || match expected {
3071 Some(expected) => presented.is_some_and(|p| {
3072 !p.is_empty() && constant_time_eq(expected.as_bytes(), p.as_bytes())
3073 }),
3074 None => false,
3075 };
3076 if authorized {
3077 return Ok(ReservedHelloVerification {
3078 nonce_verified: true,
3079 prefix_owner: None,
3080 });
3081 }
3082 return Err(ReservedHelloRejection::Exact {
3083 module_id: module_id.to_string(),
3084 });
3085 }
3086 drop(nonces);
3087
3088 let matched_prefix = self
3089 .reserved_prefix_owners
3090 .lock()
3091 .unwrap_or_else(|poisoned| poisoned.into_inner())
3092 .iter()
3093 .filter(|(prefix, _)| module_id.starts_with(prefix.as_str()))
3094 .max_by_key(|(prefix, _)| prefix.len())
3095 .map(|(prefix, owner)| (prefix.clone(), owner.clone()));
3096 let Some((prefix, owner_module_id)) = matched_prefix else {
3097 return Ok(ReservedHelloVerification::default());
3098 };
3099
3100 if module_id.len() == prefix.len()
3101 || (!channel
3102 && (self.module_is_gated(&owner_module_id) || self.protocol_none(&owner_module_id)))
3103 {
3104 return Err(ReservedHelloRejection::Prefix {
3105 prefix,
3106 owner_module_id,
3107 });
3108 }
3109
3110 let authorized = if channel {
3111 owner_module_id == module_id
3112 } else {
3113 presented.is_some_and(|presented| {
3114 self.spawn_nonces
3115 .lock()
3116 .unwrap_or_else(|poisoned| poisoned.into_inner())
3117 .get(&owner_module_id)
3118 .is_some_and(|expected| !presented.is_empty() && constant_time_eq(expected.as_bytes(), presented.as_bytes()))
3119 || self.swap_nonce_matches(&owner_module_id, presented)
3122 })
3123 };
3124 if authorized {
3125 Ok(ReservedHelloVerification {
3126 nonce_verified: true,
3127 prefix_owner: (owner_module_id != module_id).then_some(owner_module_id),
3128 })
3129 } else {
3130 Err(ReservedHelloRejection::Prefix {
3131 prefix,
3132 owner_module_id,
3133 })
3134 }
3135 }
3136
3137 pub fn spawned_consumer_authorized(&self, module_id: &str, presented: &str) -> bool {
3142 if self.module_is_gated(module_id) || self.protocol_none(module_id) {
3143 return false;
3144 }
3145 let nonces = self
3146 .spawn_nonces
3147 .lock()
3148 .unwrap_or_else(|poisoned| poisoned.into_inner());
3149 let current = nonces.get(module_id).is_some_and(|expected| {
3150 !presented.is_empty() && constant_time_eq(expected.as_bytes(), presented.as_bytes())
3151 });
3152 drop(nonces);
3153 current || self.swap_nonce_matches(module_id, presented)
3156 }
3157
3158 fn swap_nonce_matches(&self, module_id: &str, presented: &str) -> bool {
3160 if self.module_is_gated(module_id) || self.protocol_none(module_id) || presented.is_empty()
3161 {
3162 return false;
3163 }
3164 let swaps = self
3165 .swaps
3166 .lock()
3167 .unwrap_or_else(|poisoned| poisoned.into_inner());
3168 swaps.get(module_id).is_some_and(|swap| {
3169 constant_time_eq(swap.candidate_nonce.as_bytes(), presented.as_bytes())
3170 || swap.incumbent_nonce.as_deref().is_some_and(|incumbent| {
3171 constant_time_eq(incumbent.as_bytes(), presented.as_bytes())
3172 })
3173 })
3174 }
3175
3176 pub(crate) fn open_swap(&self, module_id: &str, candidate_nonce: String) {
3179 let incumbent_nonce = self
3180 .spawn_nonces
3181 .lock()
3182 .unwrap_or_else(|poisoned| poisoned.into_inner())
3183 .get(module_id)
3184 .cloned();
3185 self.swaps
3186 .lock()
3187 .unwrap_or_else(|poisoned| poisoned.into_inner())
3188 .insert(
3189 module_id.to_string(),
3190 OpenSwap {
3191 candidate_nonce,
3192 incumbent_nonce,
3193 candidate_admitted: false,
3194 },
3195 );
3196 self.refresh_program_binding_nonces(module_id);
3197 }
3198
3199 pub(crate) fn close_swap(&self, module_id: &str) {
3202 self.swaps
3203 .lock()
3204 .unwrap_or_else(|poisoned| poisoned.into_inner())
3205 .remove(module_id);
3206 self.refresh_program_binding_nonces(module_id);
3207 }
3208
3209 pub(crate) fn set_swap_promotion_observer(
3212 &self,
3213 observer: std::sync::Weak<dyn SwapPromotionObserver>,
3214 ) {
3215 *self
3216 .promotion_observer
3217 .0
3218 .lock()
3219 .unwrap_or_else(|poisoned| poisoned.into_inner()) = Some(observer);
3220 }
3221
3222 fn notify_swap_promoted(&self, registration: &crate::registry::ModuleRegistration) {
3225 let observer = self
3226 .promotion_observer
3227 .0
3228 .lock()
3229 .unwrap_or_else(|poisoned| poisoned.into_inner())
3230 .as_ref()
3231 .and_then(std::sync::Weak::upgrade);
3232 if let Some(observer) = observer {
3233 observer.swap_promoted(registration);
3234 }
3235 }
3236
3237 pub(crate) fn swap_open(&self, module_id: &str) -> bool {
3239 self.swaps
3240 .lock()
3241 .unwrap_or_else(|poisoned| poisoned.into_inner())
3242 .contains_key(module_id)
3243 }
3244
3245 fn promote_swap_nonce(&self, module_id: &str, reserved: bool) {
3250 let candidate_nonce = self
3251 .swaps
3252 .lock()
3253 .unwrap_or_else(|poisoned| poisoned.into_inner())
3254 .get_mut(module_id)
3255 .map(|swap| {
3256 swap.incumbent_nonce = None;
3257 swap.candidate_nonce.clone()
3258 });
3259 self.spawn_nonces
3260 .lock()
3261 .unwrap_or_else(|p| p.into_inner())
3262 .remove(module_id);
3263 {
3264 let mut nonces = self
3265 .reserved_nonces
3266 .lock()
3267 .unwrap_or_else(|p| p.into_inner());
3268 if reserved || nonces.contains_key(module_id) {
3269 nonces.insert(module_id.into(), None);
3270 }
3271 }
3272 if self.module_is_gated(module_id) {
3273 return;
3274 }
3275 let Some(nonce) = candidate_nonce.filter(|nonce| !nonce.is_empty()) else {
3276 return;
3277 };
3278 self.set_spawn_nonce(module_id, nonce.clone());
3279 if reserved {
3280 self.set_reserved_nonce(module_id, nonce);
3281 }
3282 }
3283
3284 pub(crate) fn swap_hello_admission(
3299 &self,
3300 module_id: &str,
3301 presented: Option<&str>,
3302 ) -> SwapHelloAdmission {
3303 if self.protocol_none(module_id) {
3304 return SwapHelloAdmission::ProtocolNone;
3305 }
3306 if self.module_is_gated(module_id) {
3307 return SwapHelloAdmission::ChannelRequired;
3308 }
3309 let swaps = self
3310 .swaps
3311 .lock()
3312 .unwrap_or_else(|poisoned| poisoned.into_inner());
3313 let Some(swap) = swaps.get(module_id) else {
3314 return SwapHelloAdmission::NotSwapping;
3315 };
3316 let Some(presented) = presented.filter(|nonce| !nonce.is_empty()) else {
3317 return SwapHelloAdmission::Refused;
3318 };
3319 if constant_time_eq(swap.candidate_nonce.as_bytes(), presented.as_bytes()) {
3320 return if swap.candidate_admitted {
3321 SwapHelloAdmission::Refused
3322 } else {
3323 SwapHelloAdmission::Candidate
3324 };
3325 }
3326 if swap
3327 .incumbent_nonce
3328 .as_deref()
3329 .is_some_and(|incumbent| constant_time_eq(incumbent.as_bytes(), presented.as_bytes()))
3330 {
3331 return SwapHelloAdmission::NotSwapping;
3332 }
3333 SwapHelloAdmission::Refused
3334 }
3335
3336 fn protocol_none(&self, module: &str) -> bool {
3337 self.get(module)
3338 .is_some_and(|module| module.declared_protocol().ok() == Some(ModuleProtocol::None))
3339 }
3340
3341 pub(crate) fn with_swap_candidate_admission<T>(
3343 &self,
3344 module_id: &str,
3345 publish: impl FnOnce() -> (T, bool),
3346 ) -> Option<T> {
3347 let mut swaps = self.swaps.lock().unwrap_or_else(|p| p.into_inner());
3348 let swap = swaps
3349 .get_mut(module_id)
3350 .filter(|swap| !swap.candidate_admitted)?;
3351 let (result, admitted) = publish();
3352 swap.candidate_admitted = admitted;
3353 Some(result)
3354 }
3355
3356 pub fn spawn_launch_nonce_for(&self, module_id: &str) -> Option<String> {
3358 self.spawn_nonce(module_id)
3359 }
3360
3361 pub fn reserved_launch_nonce_for(&self, module_id: &str) -> Option<String> {
3363 if self.module_is_gated(module_id) {
3364 return None;
3365 }
3366 self.reserved_nonces
3367 .lock()
3368 .unwrap_or_else(|poisoned| poisoned.into_inner())
3369 .get(module_id)
3370 .cloned()
3371 .flatten()
3372 }
3373
3374 pub fn insert(&self, module: SupervisedModule) -> Option<SupervisedModule> {
3375 self.mark_configured(module.module_id());
3379 let mut modules = self
3380 .modules
3381 .lock()
3382 .unwrap_or_else(|poisoned| poisoned.into_inner());
3383 modules.insert(module.module_id().to_string(), module)
3384 }
3385
3386 fn mark_configured(&self, module_id: &str) {
3390 self.update_configured_ids(|ids| {
3391 ids.insert(module_id.to_string());
3392 });
3393 }
3394
3395 fn publish_configured_ids(&self, ids: Option<&HashSet<String>>) {
3396 let owner = self
3397 .gate_runtime
3398 .lock()
3399 .unwrap_or_else(|p| p.into_inner())
3400 .clone()
3401 .and_then(|owner| owner.upgrade());
3402 if let Some(owner) = owner {
3403 owner.observe_supervisor_configured_ids(ids, Arc::downgrade(&self.configured_ids));
3404 }
3405 }
3406
3407 fn update_configured_ids(&self, update: impl FnOnce(&mut HashSet<String>)) {
3408 match self.configured_ids.lock() {
3409 Ok(mut ids) => {
3410 update(&mut ids);
3411 self.publish_configured_ids(Some(&ids));
3414 }
3415 Err(poisoned) => {
3416 update(&mut poisoned.into_inner());
3417 self.publish_configured_ids(None);
3418 }
3419 }
3420 }
3421
3422 fn unmark_configured_unless_rostered(&self, module_id: &str) {
3426 let modules = self
3427 .modules
3428 .lock()
3429 .unwrap_or_else(|poisoned| poisoned.into_inner());
3430 if !modules.contains_key(module_id) {
3431 self.update_configured_ids(|ids| {
3432 ids.remove(module_id);
3433 });
3434 }
3435 }
3436
3437 pub(crate) fn is_configured(&self, module_id: &str) -> bool {
3445 self.configured_ids
3446 .lock()
3447 .unwrap_or_else(|poisoned| poisoned.into_inner())
3448 .contains(module_id)
3449 }
3450
3451 pub fn get(&self, module_id: &str) -> Option<SupervisedModule> {
3452 let modules = self
3453 .modules
3454 .lock()
3455 .unwrap_or_else(|poisoned| poisoned.into_inner());
3456 modules.get(module_id).cloned()
3457 }
3458
3459 pub(crate) fn record_late_health_answer(
3460 &self,
3461 module_id: &str,
3462 latency_ms: u64,
3463 ) -> Result<bool, SuperviseError> {
3464 let Some(module) = self.get(module_id) else {
3465 return Ok(false);
3466 };
3467 self.host_health.record_late_module(module_id);
3468 update_snapshot(&module.inner.snapshot, Some(module_id), |state| {
3469 state.health.late_answer_count = state.health.late_answer_count.saturating_add(1);
3470 state.health.last_late_answer_latency_ms = Some(latency_ms);
3471 state.health.consecutive_failures = 0;
3479 })?;
3480 Ok(true)
3481 }
3482
3483 pub(crate) fn spawn_host_health_monitor(&self) -> JoinHandle<()> {
3484 self.host_health.spawn_monitor()
3485 }
3486
3487 pub fn record_deliberate_severance(&self, module_id: &str) -> Result<bool, SuperviseError> {
3493 let Some(module) = self.get(module_id) else {
3494 return Ok(false);
3495 };
3496 let snapshot = lock_snapshot(&module.inner.snapshot)?;
3497 let Some((pid, start_time)) = snapshot.pid.zip(snapshot.process_start_time) else {
3498 return Ok(false);
3499 };
3500 drop(snapshot);
3501 module.record_deliberate_severance(ProcessIdentity { pid, start_time })
3502 }
3503
3504 pub fn list(&self) -> Vec<SupervisedModule> {
3505 let modules = self
3506 .modules
3507 .lock()
3508 .unwrap_or_else(|poisoned| poisoned.into_inner());
3509 let mut modules = modules.values().cloned().collect::<Vec<_>>();
3510 modules.sort_by(|left, right| left.module_id().cmp(right.module_id()));
3511 modules
3512 }
3513
3514 pub(crate) fn retire(&self, module_id: &str) -> Option<SupervisedModule> {
3515 self.activity_images
3516 .lock()
3517 .unwrap_or_else(|p| p.into_inner())
3518 .remove(module_id);
3519 self.spawn_nonces
3520 .lock()
3521 .unwrap_or_else(|poisoned| poisoned.into_inner())
3522 .remove(module_id);
3523 self.close_swap(module_id);
3524 let mut reserved_nonces = self
3525 .reserved_nonces
3526 .lock()
3527 .unwrap_or_else(|poisoned| poisoned.into_inner());
3528 if reserved_nonces.contains_key(module_id) {
3529 reserved_nonces.insert(module_id.to_string(), None);
3532 }
3533 drop(reserved_nonces);
3534 self.reserved_prefix_owners
3535 .lock()
3536 .unwrap_or_else(|poisoned| poisoned.into_inner())
3537 .retain(|_, owner| owner != module_id);
3538 let removed = self
3539 .modules
3540 .lock()
3541 .unwrap_or_else(|poisoned| poisoned.into_inner())
3542 .remove(module_id);
3543 self.update_configured_ids(|ids| {
3544 ids.remove(module_id);
3545 });
3546 self.remove_program_binding_records(module_id);
3547 removed
3548 }
3549
3550 pub(crate) fn record_rescan_removal(&self, module_id: &str) {
3553 self.removal_tombstones
3554 .lock()
3555 .unwrap_or_else(|poisoned| poisoned.into_inner())
3556 .insert(module_id.to_string(), unix_ms_now());
3557 }
3558
3559 pub(crate) fn removal_tombstone_age_ms(&self, module_id: &str) -> Option<u64> {
3561 self.removal_tombstones
3562 .lock()
3563 .unwrap_or_else(|poisoned| poisoned.into_inner())
3564 .get(module_id)
3565 .copied()
3566 .map(|removed_at_ms| unix_ms_now().saturating_sub(removed_at_ms))
3567 }
3568
3569 pub(crate) fn release_retained_reserved_gate(&self, module_id: &str) -> bool {
3574 if self.get(module_id).is_some() {
3575 return false;
3576 }
3577 let mut reserved_nonces = self
3578 .reserved_nonces
3579 .lock()
3580 .unwrap_or_else(|poisoned| poisoned.into_inner());
3581 if !matches!(reserved_nonces.get(module_id), Some(None)) {
3582 return false;
3583 }
3584 reserved_nonces.remove(module_id);
3585 self.implicit_protocol_none
3586 .lock()
3587 .unwrap_or_else(|p| p.into_inner())
3588 .remove(module_id);
3589 true
3590 }
3591
3592 pub(crate) fn operation_lock(&self) -> Arc<AsyncMutex<()>> {
3593 Arc::clone(&self.operation_lock)
3594 }
3595}
3596
3597#[derive(Debug, Clone)]
3599pub struct Supervisor {
3600 registry: Arc<Registry>,
3601 config_delivery: Option<config_delivery::ConfigDelivery>,
3602 restart_policy: RestartPolicy,
3603 drain_timeout: Duration,
3604 connection_file_path: Option<PathBuf>,
3605 capture_logs_dir: Option<PathBuf>,
3606 forwarding: Option<Arc<ForwardingTable>>,
3607 process_liveness: Arc<SupervisorProcessLiveness>,
3608 supervisor_handle: Option<SupervisorHandle>,
3609 health: HealthConfig,
3610 host_health: HostHealth,
3611 daemon_start_clock: crate::clock::StartClock,
3612 terminal_journal: Option<Arc<crate::terminal_journal::TerminalJournal>>,
3613 spawn_events: SpawnEventFeed,
3614 provenance_probe: ExecutableIdentityProbe,
3615 child_roster: ChildRoster,
3618 #[cfg(target_os = "linux")]
3619 cgroup_placement: Option<subc_cgroup::Placement>,
3620 #[cfg(test)]
3621 test_after_first_spawn: AfterFirstSpawnHook,
3622}
3623
3624pub(crate) struct RegisteredConfiguredModule {
3628 spec: ModuleSpec,
3629 runtime: SupervisorRuntimeConfig,
3630 module: SupervisedModule,
3631 commands: mpsc::Receiver<SupervisorCommand>,
3632}
3633
3634#[cfg(test)]
3640#[derive(Clone, Default)]
3641struct AfterFirstSpawnHook(Option<AfterFirstSpawnFn>);
3642
3643#[cfg(test)]
3644type AfterFirstSpawnFn = Arc<dyn Fn(&str) + Send + Sync>;
3645
3646#[cfg(test)]
3647impl fmt::Debug for AfterFirstSpawnHook {
3648 fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
3649 f.write_str("AfterFirstSpawnHook")
3650 }
3651}
3652
3653#[cfg(test)]
3654impl AfterFirstSpawnHook {
3655 fn run(&self, module_id: &str) {
3656 if let Some(hook) = &self.0 {
3657 hook(module_id);
3658 }
3659 }
3660}
3661
3662impl Supervisor {
3663 #[cfg(all(test, target_os = "linux"))]
3664 pub(crate) fn set_admission_hook_for_test(
3665 &self,
3666 hook: impl Fn(&str, &'static str, Option<u32>) -> bool + Send + Sync + 'static,
3667 ) {
3668 self.child_roster.set_admission_hook_for_test(hook);
3669 }
3670
3671 #[cfg(test)]
3672 pub(crate) fn new_for_test(registry: Arc<Registry>, policy: RestartPolicy) -> Self {
3673 let supervisor = Self::new(registry, policy);
3674 #[cfg(target_os = "macos")]
3675 let supervisor = supervisor.with_privacy_trampoline(test_privacy_trampoline());
3676 supervisor
3677 }
3678 pub fn with_privacy_trampoline(self, path: impl Into<PathBuf>) -> Self {
3683 let path = path.into();
3684 #[cfg(target_os = "macos")]
3685 {
3686 let result = probe_privacy_trampoline(&path).map(|()| path);
3687 if let Err(cause) = &result {
3688 error!(%cause, "macOS privacy identity unavailable; supervised launches will refuse");
3689 }
3690 self.child_roster.set_privacy_trampoline(result);
3691 }
3692 #[cfg(not(target_os = "macos"))]
3693 let _ = path;
3694 self
3695 }
3696 pub(crate) fn daemon_shutdown_flag(&self) -> crate::child_roster::DaemonShutdownFlag {
3699 self.child_roster.shutdown_flag()
3700 }
3701
3702 #[cfg(unix)]
3713 pub(crate) fn begin_daemon_shutdown(&self) {
3714 self.child_roster.close();
3715 if let Some(journal) = &self.terminal_journal {
3716 journal.stamp_shutdown();
3717 }
3718 }
3719
3720 #[cfg(unix)]
3725 pub(crate) async fn drain_for_daemon_shutdown(&self) -> Result<(), SuperviseError> {
3726 const NOTICE_BUDGET: Duration = Duration::from_millis(500);
3727 const DRAIN_BUDGET: Duration = Duration::from_secs(2);
3728 let Some(forwarding) = &self.forwarding else {
3729 return Ok(());
3730 };
3731 let notice_started = Instant::now();
3732 let modules = forwarding
3733 .begin_daemon_drain()
3734 .map_err(SuperviseError::Forwarding)?;
3735 let deadline_ms =
3736 unix_ms_now().saturating_add((NOTICE_BUDGET + DRAIN_BUDGET).as_millis() as u64);
3737 let shutdown_reason = self
3738 .supervisor_handle
3739 .as_ref()
3740 .map(SupervisorHandle::take_daemon_shutdown_reason)
3741 .unwrap_or(RouteCloseReason::Disable);
3742 let mut modules_told = 0;
3743 let mut notices = tokio::task::JoinSet::new();
3744 let mut drains = Vec::new();
3745 for connection in forwarding
3746 .module_connections()
3747 .map_err(SuperviseError::Forwarding)?
3748 {
3749 let target = ModuleDrainTarget {
3750 endpoint: connection.endpoint,
3751 sink: connection.sink,
3752 negotiated_ver: connection.negotiated_ver,
3753 abandoned_bindings: Vec::new(),
3754 excluded_subscriptions: 0,
3755 };
3756 notices.spawn(async move {
3757 (
3758 true,
3759 send_module_draining(
3760 &connection.module_id,
3761 shutdown_reason,
3762 deadline_ms,
3763 &target,
3764 true,
3765 )
3766 .await,
3767 )
3768 });
3769 }
3770 for (module_id, endpoint) in modules {
3771 let Some(target) = forwarding
3772 .begin_endpoint_drain(endpoint, RouteCloseReason::Restart)
3773 .map_err(SuperviseError::Forwarding)?
3774 else {
3775 continue;
3776 };
3777 let routes = forwarding
3778 .endpoint_routes(target.endpoint)
3779 .map_err(SuperviseError::Forwarding)?;
3780 let mut recipients = Vec::new();
3783 let mut clients: Vec<(crate::forwarding::GoodbyeTarget, Vec<u16>)> = Vec::new();
3784 for route in routes {
3785 let client = route.goodbye_target;
3786 if let Some((_, channels)) = clients
3787 .iter_mut()
3788 .find(|(existing, _)| existing.connection_id == client.connection_id)
3789 {
3790 channels.push(client.channel);
3791 } else {
3792 let channel = client.channel;
3793 clients.push((client, vec![channel]));
3794 }
3795 }
3796 for (client, mut channels) in clients {
3797 channels.sort_unstable();
3798 channels.dedup();
3799 let closing = serde_json::to_vec(&ClientControlPush::RouteClosing {
3800 module_id: module_id.clone(),
3801 channels,
3802 reason: RouteCloseReason::Restart,
3803 })
3804 .expect("route closing serializes");
3805 recipients.push((client.sink, client.negotiated_ver, closing));
3806 }
3807 for (sink, version, body) in recipients {
3808 notices.spawn(async move {
3809 let frame = Frame::build_with_version(
3810 version,
3811 FrameType::Push,
3812 control_flags(),
3813 0,
3814 0,
3815 0,
3816 body,
3817 )
3818 .expect("bounded lifecycle notice frame builds");
3819 (false, sink.send_flushed(frame).await.is_ok())
3820 });
3821 }
3822 let gauges = declared_busy_gauges(&self.registry, &module_id)?;
3823 drains.push((module_id, target.endpoint, gauges));
3824 }
3825 let notice_deadline = Instant::now() + NOTICE_BUDGET;
3828 while let Ok(Some(result)) = timeout_at(notice_deadline, notices.join_next()).await {
3829 match result {
3830 Ok((true, true)) => modules_told += 1,
3831 Ok((false, true)) => {}
3832 result => warn!(?result, "daemon shutdown notice delivery failed"),
3833 }
3834 }
3835 notices.abort_all();
3836 info!(
3837 ?shutdown_reason,
3838 modules_told, "daemon shutdown module notices delivered"
3839 );
3840 let notice_elapsed = notice_started.elapsed();
3841 let drain_started = Instant::now();
3842 let deadline = drain_started + DRAIN_BUDGET;
3843 let mut waits = tokio::task::JoinSet::new();
3844 for (module_id, endpoint, gauges) in drains {
3845 let forwarding = Arc::clone(forwarding);
3846 let mut runtime = self.runtime_config();
3847 runtime.health.cadence = Duration::from_millis(100);
3848 waits.spawn(async move {
3849 let wait_result = wait_for_forwarding_quiescence(
3850 &forwarding,
3851 &module_id,
3852 &runtime,
3853 endpoint,
3854 deadline,
3855 &gauges,
3856 DrainScope::Endpoint(endpoint),
3857 &mut DrainGaugeTrace::new(&gauges),
3858 )
3859 .await;
3860 forwarding
3864 .record_daemon_drain_result(
3865 endpoint,
3866 drained_after_quiescence_wait(&wait_result),
3867 )
3868 .map_err(SuperviseError::Forwarding)?;
3869 wait_result
3870 });
3871 }
3872 while let Ok(Some(result)) = timeout_at(deadline, waits.join_next()).await {
3873 if !matches!(result, Ok(Ok(true))) {
3874 warn!(?result, "daemon shutdown drain did not reach quiescence");
3875 }
3876 }
3877 info!(
3880 notice_elapsed_ms = notice_elapsed.as_millis() as u64,
3881 drain_elapsed_ms = drain_started.elapsed().as_millis() as u64,
3882 "daemon shutdown forwarding drain complete"
3883 );
3884 Ok(())
3885 }
3886
3887 #[cfg(unix)]
3899 pub(crate) async fn end_children_for_daemon_shutdown(
3900 &self,
3901 already_escalated: bool,
3902 escalate: impl std::future::Future<Output = ()>,
3903 ) {
3904 tokio::pin!(escalate);
3905 let mut escalated = already_escalated;
3906 if let Some(forwarding) = &self.forwarding {
3907 let reason = CloseReason::new(
3908 "daemon_shutdown",
3909 "the daemon is exiting after its shutdown notice and drain",
3910 );
3911 if escalated {
3912 send_module_goodbyes_for_daemon_shutdown(forwarding, &reason, false).await;
3915 } else {
3916 tokio::select! {
3917 biased;
3918 _ = escalate.as_mut() => {
3919 info!("second SIGTERM: abandoning module GOODBYE delivery");
3920 escalated = true;
3921 }
3922 _ = send_module_goodbyes_for_daemon_shutdown(forwarding, &reason, true) => {}
3923 }
3924 }
3925 let closed = forwarding.close_all_connections(&reason);
3926 debug!(closed, "closed established connections for daemon shutdown");
3927 }
3928 let escalated_here = escalated && !already_escalated;
3932 let remaining_escalate = async move {
3933 if escalated_here {
3934 std::future::pending::<()>().await;
3935 } else {
3936 escalate.await;
3937 }
3938 };
3939 crate::child_roster::end_children_for_daemon_shutdown(
3940 &self.child_roster,
3941 escalated,
3942 remaining_escalate,
3943 )
3944 .await;
3945 }
3946
3947 pub fn new(registry: Arc<Registry>, restart_policy: RestartPolicy) -> Self {
3948 Self {
3949 registry,
3950 config_delivery: None,
3951 restart_policy,
3952 drain_timeout: DEFAULT_DRAIN_TIMEOUT,
3953 connection_file_path: None,
3954 capture_logs_dir: None,
3955 forwarding: None,
3956 process_liveness: Arc::new(SupervisorProcessLiveness::default()),
3957 supervisor_handle: None,
3958 health: HealthConfig::default(),
3959 host_health: HostHealth::default(),
3960 daemon_start_clock: crate::clock::StartClock::capture(),
3961 terminal_journal: None,
3962 spawn_events: SpawnEventFeed::default(),
3963 provenance_probe: ExecutableIdentityProbe::default(),
3964 child_roster: ChildRoster::default(),
3965 #[cfg(target_os = "linux")]
3966 cgroup_placement: None,
3967 #[cfg(test)]
3968 test_after_first_spawn: AfterFirstSpawnHook::default(),
3969 }
3970 }
3971
3972 pub fn with_drain_timeout(mut self, drain_timeout: Duration) -> Self {
3973 self.drain_timeout = drain_timeout;
3974 self
3975 }
3976
3977 pub(crate) fn default_drain_timeout(&self) -> Duration {
3979 self.drain_timeout
3980 }
3981
3982 pub fn with_process_liveness(
3983 mut self,
3984 process_liveness: Arc<SupervisorProcessLiveness>,
3985 ) -> Self {
3986 self.process_liveness = process_liveness;
3987 self
3988 }
3989
3990 pub fn with_config_runtime(
3992 mut self,
3993 owner: Arc<crate::config_gate::runtime::RuntimeOwner>,
3994 ) -> Self {
3995 if let Some(handle) = &self.supervisor_handle {
3996 handle.install_binding_retirement(&owner);
3997 }
3998 self.config_delivery = Some(config_delivery::ConfigDelivery(owner));
3999 self
4000 }
4001
4002 pub fn with_connection_file_path(mut self, connection_file_path: impl Into<PathBuf>) -> Self {
4003 self.connection_file_path = Some(connection_file_path.into());
4004 self
4005 }
4006
4007 pub fn with_capture_logs_dir(mut self, logs_dir: impl Into<PathBuf>) -> Self {
4009 self.capture_logs_dir = Some(logs_dir.into());
4010 self
4011 }
4012
4013 pub fn with_daemon_incarnation(self, daemon_incarnation: String) -> Self {
4016 self.spawn_events.configure_incarnation(daemon_incarnation);
4020 self
4021 }
4022
4023 pub fn with_terminal_journal(self, path: PathBuf, daemon_incarnation: String) -> Self {
4026 let mut this = self.with_daemon_incarnation(daemon_incarnation.clone());
4027 this.terminal_journal = Some(Arc::new(crate::terminal_journal::TerminalJournal::open(
4028 path,
4029 daemon_incarnation,
4030 )));
4031 this
4032 }
4033
4034 pub fn with_forwarding(mut self, forwarding: Arc<ForwardingTable>) -> Self {
4035 self.child_roster.counters = forwarding.counters();
4036 self.forwarding = Some(forwarding);
4037 self
4038 }
4039
4040 pub fn with_handle(mut self, supervisor_handle: SupervisorHandle) -> Self {
4041 if let Some(delivery) = &self.config_delivery {
4042 supervisor_handle.install_binding_retirement(&delivery.0);
4043 }
4044 self.host_health = supervisor_handle.host_health.clone();
4045 self.spawn_events = supervisor_handle.spawn_events.clone();
4046 self.supervisor_handle = Some(supervisor_handle);
4047 self
4048 }
4049
4050 pub fn with_health_config(mut self, health: HealthConfig) -> Self {
4051 self.health = health;
4052 self
4053 }
4054
4055 pub fn with_live_children_record(self, path: impl Into<PathBuf>) -> Self {
4059 self.child_roster.record_to(path.into());
4060 self
4061 }
4062
4063 #[cfg(target_os = "linux")]
4064 pub fn with_cgroup_placement(
4065 mut self,
4066 cgroup_placement: Option<subc_cgroup::Placement>,
4067 ) -> Self {
4068 self.child_roster
4069 .set_cgroup_placement(cgroup_placement.clone());
4070 self.cgroup_placement = cgroup_placement;
4071 self
4072 }
4073
4074 pub fn spawn(&self, spec: ModuleSpec) -> Result<SupervisedModule, SuperviseError> {
4083 validate_spec(&spec)?;
4084 self.establish_identity(&spec);
4085
4086 let runtime = self.runtime_config();
4087 let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::starting()));
4088 let activity_spawn = runtime.begin_activity_spawn(&spec.module_id);
4089 let registration_watch = watch_registration(&self.registry, &spec);
4090 let spawned = spawn_child(
4091 &spec,
4092 runtime.connection_file_path.as_deref(),
4093 self.supervisor_handle.as_ref(),
4094 &runtime.stderr_ring,
4095 runtime.capture_logs_dir.as_deref(),
4096 &runtime.child_roster,
4097 runtime.spawn_delivery(),
4098 #[cfg(target_os = "linux")]
4099 runtime.cgroup_placement.as_ref(),
4100 runtime.register_timeout,
4101 );
4102 #[cfg(test)]
4103 self.test_after_first_spawn.run(&spec.module_id);
4104 let mut child = match spawned {
4105 Ok(child) => child,
4106 Err(SuperviseError::ConfigDeliveryDeferred { .. }) => {
4107 defer_config_spawn(
4108 &spec,
4109 &runtime,
4110 &snapshot,
4111 RespawnKind::Spawn,
4112 &runtime.failure_context(runtime.recovery_is_daemon),
4113 )?;
4114 return Ok(self.supervised_module(spec, runtime, snapshot, None));
4115 }
4116 Err(SuperviseError::ConfigHeld { .. }) => {
4117 hold_config_spawn(
4118 &spec,
4119 &runtime,
4120 &snapshot,
4121 &runtime.failure_context(runtime.recovery_is_daemon),
4122 )?;
4123 return Ok(self.supervised_module(spec, runtime, snapshot, None));
4124 }
4125 Err(err) => {
4126 self.abandon_unrostered(&spec.module_id);
4129 return Err(err);
4130 }
4131 };
4132 arm_registration_deadline(&mut child, runtime.register_timeout, registration_watch);
4133 self.mark_first_process_running(&spec, &runtime, &snapshot, &mut child)?;
4134 if let Some(spawn) = activity_spawn {
4135 spawn.complete(&snapshot);
4136 }
4137
4138 Ok(self.supervised_module(spec, runtime, snapshot, Some(child)))
4139 }
4140
4141 pub async fn spawn_async(&self, spec: ModuleSpec) -> Result<SupervisedModule, SuperviseError> {
4144 validate_spec(&spec)?;
4145 self.establish_identity(&spec);
4146 let mut identity = AsyncInitialIdentity {
4147 supervisor: self,
4148 module: spec.module_id.clone(),
4149 committed: false,
4150 };
4151 let runtime = self.runtime_config();
4152 let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::starting()));
4153 let activity_spawn = runtime.begin_activity_spawn(&spec.module_id);
4154 let registration_watch = watch_registration(&self.registry, &spec);
4155 let spawned = spawn_child_async(&spec, &runtime).await;
4156 #[cfg(test)]
4157 self.test_after_first_spawn.run(&spec.module_id);
4158 let mut child = match spawned {
4159 Ok(child) => child,
4160 Err(SuperviseError::ConfigDeliveryDeferred { .. }) => {
4161 defer_config_spawn(
4162 &spec,
4163 &runtime,
4164 &snapshot,
4165 RespawnKind::Spawn,
4166 &runtime.failure_context(runtime.recovery_is_daemon),
4167 )?;
4168 identity.committed = true;
4169 return Ok(self.supervised_module(spec, runtime, snapshot, None));
4170 }
4171 Err(SuperviseError::ConfigHeld { .. }) => {
4172 hold_config_spawn(
4173 &spec,
4174 &runtime,
4175 &snapshot,
4176 &runtime.failure_context(runtime.recovery_is_daemon),
4177 )?;
4178 identity.committed = true;
4179 return Ok(self.supervised_module(spec, runtime, snapshot, None));
4180 }
4181 Err(err) => {
4182 self.abandon_unrostered(&spec.module_id);
4183 return Err(err);
4184 }
4185 };
4186 arm_registration_deadline(&mut child, runtime.register_timeout, registration_watch);
4187 self.mark_first_process_running(&spec, &runtime, &snapshot, &mut child)?;
4188 if let Some(spawn) = activity_spawn {
4189 spawn.complete(&snapshot);
4190 }
4191 identity.committed = true;
4192 Ok(self.supervised_module(spec, runtime, snapshot, Some(child)))
4193 }
4194
4195 fn establish_identity(&self, spec: &ModuleSpec) {
4208 if let Some(supervisor_handle) = &self.supervisor_handle {
4209 supervisor_handle.apply_identity_configuration(spec);
4210 supervisor_handle.mark_configured(&spec.module_id);
4211 }
4212 }
4213
4214 fn abandon_unrostered(&self, module_id: &str) {
4217 if let Some(supervisor_handle) = &self.supervisor_handle {
4218 supervisor_handle.unmark_configured_unless_rostered(module_id);
4219 }
4220 }
4221
4222 fn mark_first_process_running(
4225 &self,
4226 spec: &ModuleSpec,
4227 runtime: &SupervisorRuntimeConfig,
4228 snapshot: &SharedSnapshot,
4229 child: &mut SupervisedChild,
4230 ) -> Result<(), SuperviseError> {
4231 if let Err(err) = set_running(snapshot, child, &spec.module_id, &runtime.spawn_events) {
4232 containment::terminate_blocking(&runtime.child_roster.containment);
4233 self.abandon_unrostered(&spec.module_id);
4234 return Err(err);
4235 }
4236 self.process_liveness
4237 .track(spec.module_id.clone(), Arc::clone(snapshot));
4238 Ok(())
4239 }
4240
4241 pub fn supervise_configured(
4247 &self,
4248 spec: ModuleSpec,
4249 enabled: bool,
4250 ) -> Result<SupervisedModule, SuperviseError> {
4251 self.supervise_configured_requested(
4252 spec,
4253 enabled,
4254 self.health.clone(),
4255 None,
4256 self.restart_policy,
4257 DEFAULT_REGISTER_TIMEOUT,
4258 None,
4259 )
4260 }
4261
4262 pub fn supervise_configured_with_health(
4268 &self,
4269 spec: ModuleSpec,
4270 enabled: bool,
4271 health: HealthConfig,
4272 drain_timeout_ms: Option<u64>,
4273 restart_policy: RestartPolicy,
4274 ) -> Result<SupervisedModule, SuperviseError> {
4275 self.supervise_configured_requested(
4276 spec,
4277 enabled,
4278 health,
4279 drain_timeout_ms,
4280 restart_policy,
4281 DEFAULT_REGISTER_TIMEOUT,
4282 None,
4283 )
4284 }
4285
4286 #[allow(clippy::too_many_arguments)]
4287 pub(crate) fn supervise_configured_requested(
4288 &self,
4289 spec: ModuleSpec,
4290 enabled: bool,
4291 health: HealthConfig,
4292 drain_timeout_ms: Option<u64>,
4293 restart_policy: RestartPolicy,
4294 register_timeout: Duration,
4295 requester: Option<Principal>,
4296 ) -> Result<SupervisedModule, SuperviseError> {
4297 let registered = self.register_configured_with_health(
4298 spec,
4299 enabled,
4300 health,
4301 drain_timeout_ms,
4302 restart_policy,
4303 register_timeout,
4304 )?;
4305 let daemon_initiated = requester.is_none();
4306 self.start_registered_configured(registered, requester, daemon_initiated)
4307 }
4308
4309 #[allow(clippy::too_many_arguments)]
4310 pub(crate) async fn supervise_configured_requested_async(
4311 &self,
4312 spec: ModuleSpec,
4313 enabled: bool,
4314 health: HealthConfig,
4315 drain_timeout_ms: Option<u64>,
4316 restart_policy: RestartPolicy,
4317 register_timeout: Duration,
4318 requester: Option<Principal>,
4319 ) -> Result<SupervisedModule, SuperviseError> {
4320 let registered = self.register_configured_with_health(
4321 spec,
4322 enabled,
4323 health,
4324 drain_timeout_ms,
4325 restart_policy,
4326 register_timeout,
4327 )?;
4328 let daemon_initiated = requester.is_none();
4329 self.start_registered_configured_async(registered, requester, daemon_initiated)
4330 .await
4331 }
4332
4333 pub(crate) fn register_configured_with_health(
4338 &self,
4339 spec: ModuleSpec,
4340 enabled: bool,
4341 health: HealthConfig,
4342 drain_timeout_ms: Option<u64>,
4343 restart_policy: RestartPolicy,
4344 register_timeout: Duration,
4345 ) -> Result<RegisteredConfiguredModule, SuperviseError> {
4346 validate_spec(&spec)?;
4347 self.establish_identity(&spec);
4348
4349 let mut runtime = self.runtime_config();
4350 runtime.health = health;
4351 runtime.restart_policy = restart_policy;
4352 runtime.register_timeout = register_timeout;
4353 if let Some(ms) = drain_timeout_ms {
4354 runtime.drain_timeout = Duration::from_millis(ms);
4355 *runtime
4356 .effective_drain_timeout
4357 .lock()
4358 .unwrap_or_else(|poisoned| poisoned.into_inner()) = runtime.drain_timeout;
4359 }
4360 let snapshot = if enabled {
4361 SupervisorSnapshot::starting()
4362 } else {
4363 SupervisorSnapshot::disabled()
4364 };
4365 let snapshot = {
4366 let mut snapshot = snapshot;
4367 if enabled {
4368 if let Some((pid, reason)) = runtime.child_roster.previous_instance(&spec.module_id)
4369 {
4370 snapshot.waiting_for_previous_pid = Some(pid);
4371 snapshot.waiting_for_previous_detail = Some(reason.waiting_message(pid));
4372 }
4373 }
4374 snapshot
4375 };
4376 let (module, commands) =
4377 self.register_module(&spec, &runtime, Arc::new(Mutex::new(snapshot)));
4378 Ok(RegisteredConfiguredModule {
4379 spec,
4380 runtime,
4381 module,
4382 commands,
4383 })
4384 }
4385
4386 pub(crate) fn start_registered_configured(
4389 &self,
4390 registered: RegisteredConfiguredModule,
4391 requester: Option<Principal>,
4392 daemon_initiated: bool,
4393 ) -> Result<SupervisedModule, SuperviseError> {
4394 let RegisteredConfiguredModule {
4395 spec,
4396 mut runtime,
4397 module,
4398 commands,
4399 } = registered;
4400 runtime.start_requester = requester;
4401 runtime.recovery_is_daemon = daemon_initiated;
4402 let snapshot = &module.inner.snapshot;
4403 let should_spawn = {
4404 let state = lock_snapshot(snapshot)?;
4405 state.enabled && state.waiting_for_previous_pid.is_none()
4406 };
4407 let child = if should_spawn {
4408 let activity_spawn = runtime.begin_activity_spawn(&spec.module_id);
4409 let registration_watch = watch_registration(&self.registry, &spec);
4410 let spawned = spawn_child(
4411 &spec,
4412 runtime.connection_file_path.as_deref(),
4413 self.supervisor_handle.as_ref(),
4414 &runtime.stderr_ring,
4415 runtime.capture_logs_dir.as_deref(),
4416 &runtime.child_roster,
4417 runtime.spawn_delivery(),
4418 #[cfg(target_os = "linux")]
4419 runtime.cgroup_placement.as_ref(),
4420 runtime.register_timeout,
4421 );
4422 #[cfg(test)]
4423 self.test_after_first_spawn.run(&spec.module_id);
4424 match spawned {
4425 Ok(mut child) => {
4426 arm_registration_deadline(
4427 &mut child,
4428 runtime.register_timeout,
4429 registration_watch,
4430 );
4431 self.mark_first_process_running(&spec, &runtime, snapshot, &mut child)?;
4432 if let Some(spawn) = activity_spawn {
4433 spawn.complete(snapshot);
4434 }
4435 Some(child)
4436 }
4437 Err(SuperviseError::ConfigDeliveryDeferred { .. }) => {
4438 defer_config_spawn(
4439 &spec,
4440 &runtime,
4441 snapshot,
4442 RespawnKind::Spawn,
4443 &runtime.failure_context(daemon_initiated),
4444 )?;
4445 None
4446 }
4447 Err(mut err) => {
4448 if matches!(err, SuperviseError::ConfigHeld { .. }) {
4449 match hold_config_spawn(
4450 &spec,
4451 &runtime,
4452 snapshot,
4453 &runtime.failure_context(daemon_initiated),
4454 ) {
4455 Ok(()) => {
4456 self.start_monitor(spec, runtime, &module, None, commands);
4457 return Ok(module);
4458 }
4459 Err(parking_error) => err = parking_error,
4460 }
4461 }
4462 if runtime.health.critical {
4463 error!(
4464 module_id = %spec.module_id,
4465 program = %spec.program.display(),
4466 error = %err,
4467 "critical configured module failed to spawn; marking failed and alerting"
4468 );
4469 } else {
4470 error!(
4471 module_id = %spec.module_id,
4472 program = %spec.program.display(),
4473 error = %err,
4474 "configured module failed to spawn; marking failed and continuing"
4475 );
4476 }
4477 fail_snapshot(
4478 snapshot,
4479 Some(&spec.module_id),
4480 None,
4481 &runtime.failure_context(daemon_initiated),
4482 );
4483 None
4484 }
4485 }
4486 } else {
4487 None
4488 };
4489 self.start_monitor(spec, runtime, &module, child, commands);
4490 Ok(module)
4491 }
4492
4493 pub(crate) async fn start_registered_configured_async(
4494 &self,
4495 registered: RegisteredConfiguredModule,
4496 requester: Option<Principal>,
4497 daemon_initiated: bool,
4498 ) -> Result<SupervisedModule, SuperviseError> {
4499 self.start_registered_with_failure_policy(registered, requester, daemon_initiated, None)
4500 .await
4501 }
4502
4503 pub(crate) async fn start_registered_approved_async(
4504 &self,
4505 registered: RegisteredConfiguredModule,
4506 requester: Principal,
4507 generation: crate::config_gate::runtime::ApprovalIdentity,
4508 ) -> Result<SupervisedModule, SuperviseError> {
4509 self.start_registered_with_failure_policy(
4510 registered,
4511 Some(requester),
4512 false,
4513 Some(generation),
4514 )
4515 .await
4516 }
4517
4518 async fn start_registered_with_failure_policy(
4519 &self,
4520 registered: RegisteredConfiguredModule,
4521 requester: Option<Principal>,
4522 daemon_initiated: bool,
4523 approved_generation: Option<crate::config_gate::runtime::ApprovalIdentity>,
4524 ) -> Result<SupervisedModule, SuperviseError> {
4525 let RegisteredConfiguredModule {
4526 spec,
4527 mut runtime,
4528 module,
4529 commands,
4530 } = registered;
4531 runtime.start_requester = requester;
4532 runtime.recovery_is_daemon = daemon_initiated;
4533 let snapshot = &module.inner.snapshot;
4534 let mut startup_error = None;
4535 let failure = FailureContext {
4536 approval: approved_generation.clone(),
4537 ..runtime.failure_context(daemon_initiated)
4538 };
4539 let should_spawn = {
4540 let state = lock_snapshot(snapshot)?;
4541 state.enabled && state.waiting_for_previous_pid.is_none()
4542 };
4543 let child = if should_spawn {
4544 let activity_spawn = runtime.begin_activity_spawn(&spec.module_id);
4545 let registration_watch = watch_registration(&self.registry, &spec);
4546 let spawned = spawn_child_async(&spec, &runtime).await;
4547 #[cfg(test)]
4548 self.test_after_first_spawn.run(&spec.module_id);
4549 match spawned {
4550 Ok(mut child) => {
4551 arm_registration_deadline(
4552 &mut child,
4553 runtime.register_timeout,
4554 registration_watch,
4555 );
4556 self.mark_first_process_running(&spec, &runtime, snapshot, &mut child)?;
4557 if let Some(spawn) = activity_spawn {
4558 spawn.complete(snapshot);
4559 }
4560 Some(child)
4561 }
4562 Err(SuperviseError::ConfigDeliveryDeferred { .. }) => {
4563 defer_config_spawn(&spec, &runtime, snapshot, RespawnKind::Spawn, &failure)?;
4564 None
4565 }
4566 Err(mut err) => {
4567 if matches!(err, SuperviseError::ConfigHeld { .. }) {
4568 match hold_config_spawn(&spec, &runtime, snapshot, &failure) {
4569 Ok(()) => {
4570 self.start_monitor(spec, runtime, &module, None, commands);
4571 return Ok(module);
4572 }
4573 Err(parking_error) => err = parking_error,
4574 }
4575 }
4576 if runtime.health.critical {
4577 error!(module_id = %spec.module_id, program = %spec.program.display(), error = %err,
4578 "critical configured module failed to spawn; marking failed and alerting");
4579 } else {
4580 error!(module_id = %spec.module_id, program = %spec.program.display(), error = %err,
4581 "configured module failed to spawn; marking failed and continuing");
4582 }
4583 fail_snapshot(snapshot, Some(&spec.module_id), None, &failure);
4584 if let Some(generation) = approved_generation {
4585 let _ = update_snapshot(snapshot, Some(&spec.module_id), |state| {
4586 state.approved_start_failure = Some((generation, err.to_string()));
4587 });
4588 startup_error = Some(err);
4589 }
4590 None
4591 }
4592 }
4593 } else {
4594 None
4595 };
4596 self.start_monitor(spec, runtime, &module, child, commands);
4597 startup_error.map_or(Ok(module), Err)
4601 }
4602
4603 fn runtime_config(&self) -> SupervisorRuntimeConfig {
4604 let effective_drain_timeout = Arc::new(Mutex::new(self.drain_timeout));
4605 SupervisorRuntimeConfig {
4606 registry: self.registry.clone(),
4607 config_delivery: self.config_delivery.clone(),
4608 recovery_is_daemon: true,
4609 start_requester: None,
4610 pending_activity_restart: false,
4611 provenance_probe: self.provenance_probe.clone(),
4612 scheduled_respawn: Arc::default(),
4613 deferred_reload_reply: Arc::default(),
4614 restart_policy: self.restart_policy,
4615 register_timeout: DEFAULT_REGISTER_TIMEOUT,
4616 drain_timeout: self.drain_timeout,
4617 child_roster: self
4620 .child_roster
4621 .for_module(Arc::clone(&effective_drain_timeout)),
4622 effective_drain_timeout,
4623 default_drain_timeout: self.drain_timeout,
4624 health: self.health.clone(),
4625 host_health: self.host_health.clone(),
4626 connection_file_path: self.connection_file_path.clone(),
4627 capture_logs_dir: self.capture_logs_dir.clone(),
4628 forwarding: self.forwarding.clone(),
4629 supervisor_handle: self.supervisor_handle.clone(),
4630 stderr_ring: Arc::new(Mutex::new(StderrRing::new(StderrTailConfig::default()))),
4631 terminal_ring: Arc::new(Mutex::new(
4632 TerminalRing::new(
4633 TerminalRingConfig::default(),
4634 self.daemon_start_clock.started_at_ms(),
4635 )
4636 .with_start_clock(self.daemon_start_clock)
4637 .with_journal(self.terminal_journal.clone())
4638 .with_daemon_shutdown(self.child_roster.shutdown_flag()),
4639 )),
4640 spawn_events: self.spawn_events.clone(),
4641 #[cfg(target_os = "linux")]
4642 cgroup_placement: self.cgroup_placement.clone(),
4643 #[cfg(test)]
4644 test_seed_stale_facts_before_enable_spawn: false,
4645 #[cfg(test)]
4646 test_reload_exit_record_gate: None,
4647 }
4648 }
4649
4650 fn supervised_module(
4651 &self,
4652 spec: ModuleSpec,
4653 runtime: SupervisorRuntimeConfig,
4654 snapshot: SharedSnapshot,
4655 child: Option<SupervisedChild>,
4656 ) -> SupervisedModule {
4657 let (module, commands) = self.register_module(&spec, &runtime, snapshot);
4658 self.start_monitor(spec, runtime, &module, child, commands);
4659 module
4660 }
4661
4662 fn register_module(
4663 &self,
4664 spec: &ModuleSpec,
4665 runtime: &SupervisorRuntimeConfig,
4666 snapshot: SharedSnapshot,
4667 ) -> (SupervisedModule, mpsc::Receiver<SupervisorCommand>) {
4668 runtime.host_health.track(spec, &snapshot);
4669 let configuration = Arc::new(Mutex::new(SupervisedConfiguration {
4670 spec: spec.clone(),
4671 enabled: snapshot.lock().unwrap_or_else(|p| p.into_inner()).enabled,
4672 health: runtime.health.clone(),
4673 register_timeout: runtime.register_timeout,
4674 restart_policy: runtime.restart_policy,
4675 }));
4676 let stderr_ring = Arc::clone(&runtime.stderr_ring);
4677 let terminal_ring = Arc::clone(&runtime.terminal_ring);
4678 let effective_drain_timeout = Arc::clone(&runtime.effective_drain_timeout);
4679 let (tx, rx) = mpsc::channel(4);
4680 let module_id = spec.module_id.clone();
4681 let module = SupervisedModule {
4682 inner: Arc::new(SupervisedModuleInner {
4683 module_id: module_id.clone(),
4684 registry: Arc::clone(&self.registry),
4685 snapshot,
4686 configuration,
4687 stderr_ring,
4688 terminal_ring,
4689 commands: tx,
4690 monitor: Mutex::new(None),
4691 effective_drain_timeout,
4692 provenance_probe: self.provenance_probe.clone(),
4693 retained: runtime.child_roster.retained_view(),
4694 }),
4695 };
4696 if let Some(supervisor_handle) = &self.supervisor_handle {
4699 supervisor_handle.insert(module.clone());
4700 }
4701 (module, rx)
4702 }
4703
4704 fn start_monitor(
4705 &self,
4706 spec: ModuleSpec,
4707 runtime: SupervisorRuntimeConfig,
4708 module: &SupervisedModule,
4709 child: Option<SupervisedChild>,
4710 commands: mpsc::Receiver<SupervisorCommand>,
4711 ) {
4712 let monitor = tokio::spawn(supervise_loop(
4713 spec,
4714 runtime,
4715 Arc::clone(&self.registry),
4716 Arc::clone(&self.process_liveness),
4717 Arc::clone(&module.inner.snapshot),
4718 child,
4719 commands,
4720 ));
4721 *module
4722 .inner
4723 .monitor
4724 .lock()
4725 .unwrap_or_else(|p| p.into_inner()) = Some(monitor);
4726 }
4727}
4728
4729impl Default for Supervisor {
4730 fn default() -> Self {
4731 Self::new(Arc::new(Registry::default()), RestartPolicy::default())
4732 }
4733}
4734
4735#[derive(Clone)]
4737pub struct SupervisedModule {
4738 inner: Arc<SupervisedModuleInner>,
4739}
4740
4741struct SupervisedModuleInner {
4742 retained: crate::child_roster::RetainedView,
4743 module_id: String,
4744 registry: Arc<Registry>,
4745 snapshot: SharedSnapshot,
4746 configuration: Arc<Mutex<SupervisedConfiguration>>,
4747 stderr_ring: Arc<Mutex<StderrRing>>,
4748 terminal_ring: Arc<Mutex<TerminalRing>>,
4749 commands: mpsc::Sender<SupervisorCommand>,
4750 monitor: Mutex<Option<JoinHandle<()>>>,
4751 effective_drain_timeout: Arc<Mutex<Duration>>,
4752 provenance_probe: ExecutableIdentityProbe,
4753}
4754
4755impl fmt::Debug for SupervisedModule {
4756 fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
4757 f.debug_struct("SupervisedModule")
4758 .field("module_id", &self.inner.module_id)
4759 .field("status", &self.status())
4760 .finish_non_exhaustive()
4761 }
4762}
4763
4764impl SupervisedModule {
4765 pub(crate) fn check_approved_start_failure(
4766 &self,
4767 generation: &crate::config_gate::runtime::ApprovalIdentity,
4768 ) -> Result<bool, SuperviseError> {
4769 let state = lock_snapshot(&self.inner.snapshot)?;
4770 if let Some((failed_generation, detail)) = &state.approved_start_failure {
4771 if failed_generation == generation {
4772 return Err(SuperviseError::ReloadFailed {
4773 module_id: self.inner.module_id.clone(),
4774 reason: detail.clone(),
4775 });
4776 }
4777 }
4778 Ok(state.approved_start_failure.is_some())
4779 }
4780
4781 pub(crate) fn has_approved_start_failure(
4782 &self,
4783 identity: &crate::config_gate::runtime::ApprovalIdentity,
4784 ) -> bool {
4785 lock_snapshot(&self.inner.snapshot).is_ok_and(|state| {
4786 state
4787 .approved_start_failure
4788 .as_ref()
4789 .is_some_and(|(failed, _)| failed == identity)
4790 })
4791 }
4792
4793 pub(crate) fn needs_approved_gate_restart(
4794 &self,
4795 approval: &crate::config_gate::runtime::ApprovalIdentity,
4796 ) -> bool {
4797 lock_snapshot(&self.inner.snapshot)
4798 .is_ok_and(|state| state.needs_approved_gate_restart(approval))
4799 }
4800
4801 #[cfg(all(test, unix))]
4802 pub(crate) fn config_hold_origin_for_test(
4803 &self,
4804 ) -> (bool, Option<crate::config_gate::runtime::ApprovalIdentity>) {
4805 let state = lock_snapshot(&self.inner.snapshot).unwrap();
4806 (
4807 state.config_hold_daemon_initiated,
4808 state.config_hold_approval.clone(),
4809 )
4810 }
4811
4812 pub(crate) async fn config_policy_probe(
4813 &self,
4814 connection: crate::ConnectionId,
4815 generation: u64,
4816 attempt: u64,
4817 ) -> Result<(), SuperviseError> {
4818 self.inner
4819 .commands
4820 .send(SupervisorCommand::ConfigPolicyProbe {
4821 connection,
4822 generation,
4823 attempt,
4824 })
4825 .await
4826 .map_err(|_| SuperviseError::CommandClosed {
4827 module_id: self.inner.module_id.clone(),
4828 })
4829 }
4830 pub(crate) async fn config_exit_hold(&self) -> Result<(), SuperviseError> {
4831 let (reply, result) = oneshot::channel();
4832 self.inner
4833 .commands
4834 .send(SupervisorCommand::ConfigExitHold { reply })
4835 .await
4836 .map_err(|_| SuperviseError::CommandClosed {
4837 module_id: self.module_id().to_string(),
4838 })?;
4839 result.await.map_err(|_| SuperviseError::CommandClosed {
4840 module_id: self.module_id().to_string(),
4841 })?
4842 }
4843 pub(crate) async fn config_delivery_failed(
4844 &self,
4845 connection: crate::ConnectionId,
4846 hold: bool,
4847 ) -> Result<(), SuperviseError> {
4848 self.inner
4849 .commands
4850 .send(SupervisorCommand::ConfigDeliveryFailed { connection, hold })
4851 .await
4852 .map_err(|_| SuperviseError::CommandClosed {
4853 module_id: self.module_id().to_string(),
4854 })
4855 }
4856
4857 pub(crate) async fn config_apply_restart(&self, generation: u64) -> Result<(), SuperviseError> {
4858 self.inner
4859 .commands
4860 .send(SupervisorCommand::ConfigApplyRestart {
4861 generation,
4862 supersedes_swap: false,
4863 })
4864 .await
4865 .map_err(|_| SuperviseError::CommandClosed {
4866 module_id: self.inner.module_id.clone(),
4867 })
4868 }
4869 pub fn module_id(&self) -> &str {
4870 &self.inner.module_id
4871 }
4872
4873 #[cfg(test)]
4877 pub(crate) fn record_health_probe_failure_for_test(
4878 &self,
4879 detail: &str,
4880 ) -> Result<(), SuperviseError> {
4881 update_snapshot(&self.inner.snapshot, Some(&self.inner.module_id), |state| {
4882 state.health.consecutive_failures = state.health.consecutive_failures.saturating_add(1);
4883 state.health.detail = Some(detail.to_string());
4884 })
4885 }
4886
4887 pub fn state(&self) -> Result<ModuleState, SuperviseError> {
4888 Ok(lock_snapshot(&self.inner.snapshot)?.state)
4889 }
4890
4891 pub(crate) fn registration_end_reason(
4892 &self,
4893 ) -> Result<Option<RegistrationEndReason>, SuperviseError> {
4894 let snapshot = lock_snapshot(&self.inner.snapshot)?;
4895 Ok(match snapshot.state {
4896 ModuleState::Draining if snapshot.draining_to_replace => {
4897 Some(RegistrationEndReason::SupervisorRestart)
4898 }
4899 ModuleState::Draining => Some(RegistrationEndReason::SupervisorStop),
4900 ModuleState::Restarting if snapshot.draining_to_replace => {
4901 Some(RegistrationEndReason::SupervisorRestart)
4902 }
4903 _ => None,
4904 })
4905 }
4906
4907 pub fn stderr_tail(
4914 &self,
4915 max_lines: Option<usize>,
4916 max_bytes: Option<usize>,
4917 ) -> StderrTailSnapshot {
4918 self.inner
4919 .stderr_ring
4920 .lock()
4921 .unwrap_or_else(|poisoned| poisoned.into_inner())
4922 .snapshot(max_lines, max_bytes)
4923 }
4924
4925 pub fn terminal_history(&self) -> TerminalHistorySnapshot {
4930 self.inner
4931 .terminal_ring
4932 .lock()
4933 .unwrap_or_else(|poisoned| poisoned.into_inner())
4934 .snapshot()
4935 }
4936
4937 pub fn durable_terminal_history(&self) -> subc_control::TerminalHistory {
4942 durable_terminal_history_of(&self.inner.terminal_ring, &self.inner.module_id)
4943 }
4944
4945 pub(crate) async fn read_durable_terminal_history(
4950 &self,
4951 ) -> Result<subc_control::TerminalHistory, tokio::task::JoinError> {
4952 let terminal_ring = Arc::clone(&self.inner.terminal_ring);
4953 let module_id = self.inner.module_id.clone();
4954 tokio::task::spawn_blocking(move || durable_terminal_history_of(&terminal_ring, &module_id))
4955 .await
4956 }
4957
4958 pub fn status(&self) -> Result<ModuleStatus, SuperviseError> {
4959 self.status_with_snapshot_lock(&self.inner.snapshot, None)
4960 .map(|(status, _)| status)
4961 }
4962
4963 pub(crate) fn next_retry_at_ms(&self) -> Result<Option<u64>, SuperviseError> {
4964 let state = lock_snapshot(&self.inner.snapshot)?;
4965 Ok(state
4966 .failed_retry
4967 .pending
4968 .filter(|_| state.state == ModuleState::Failed && state.enabled)
4969 .map(|retry| {
4970 unix_ms_now().saturating_add(
4971 u64::try_from(
4972 retry
4973 .deadline
4974 .saturating_duration_since(Instant::now())
4975 .as_millis(),
4976 )
4977 .unwrap_or(u64::MAX),
4978 )
4979 }))
4980 }
4981
4982 #[cfg(all(test, unix))]
4983 pub(crate) fn retry_deadline_for_test(&self) -> Option<Instant> {
4984 lock_snapshot(&self.inner.snapshot)
4985 .unwrap()
4986 .failed_retry
4987 .pending
4988 .map(|retry| retry.deadline)
4989 }
4990
4991 pub(crate) fn record_deliberate_severance(
4992 &self,
4993 identity: ProcessIdentity,
4994 ) -> Result<bool, SuperviseError> {
4995 let mut snapshot = lock_snapshot(&self.inner.snapshot)?;
4996 if snapshot.pid != Some(identity.pid)
4997 || snapshot.process_start_time != Some(identity.start_time)
4998 {
4999 return Ok(false);
5000 }
5001 snapshot.deliberate_severance = Some(identity);
5002 Ok(true)
5003 }
5004
5005 pub(crate) fn status_for_control(
5010 &self,
5011 caller: &'static str,
5012 ) -> Result<ModuleStatus, SuperviseError> {
5013 self.status_with_snapshot_lock(&self.inner.snapshot, Some(caller))
5014 .map(|(status, _)| status)
5015 }
5016
5017 fn status_with_snapshot_lock(
5018 &self,
5019 snapshot: &SharedSnapshot,
5020 caller: Option<&'static str>,
5021 ) -> Result<(ModuleStatus, Option<SpawnedFileIdentity>), SuperviseError> {
5022 let restart_policy = self
5023 .inner
5024 .configuration
5025 .lock()
5026 .map_err(|_| SuperviseError::StatePoisoned {
5027 module_id: Some(self.inner.module_id.clone()),
5028 })?
5029 .restart_policy;
5030 let mut guard = match caller {
5031 Some(caller) => lock_snapshot_for_control(snapshot, &self.inner.module_id, caller)?,
5032 None => lock_snapshot(snapshot)?,
5033 };
5034 let restart_count = guard.crash_restarts_in_window(restart_policy.window, Instant::now());
5037 let snapshot = guard.clone();
5038 drop(guard);
5039 let drain_timeout = *self.inner.effective_drain_timeout.lock().map_err(|_| {
5040 SuperviseError::StatePoisoned {
5041 module_id: Some(self.inner.module_id.clone()),
5042 }
5043 })?;
5044 let registration_active = self
5045 .inner
5046 .registry
5047 .get_module(&self.inner.module_id)
5048 .map_err(SuperviseError::Registry)?
5049 .is_some();
5050 let protocol = snapshot
5051 .spawned_protocol
5052 .unwrap_or(self.declared_protocol()?);
5053 let running_process =
5054 snapshot.enabled && snapshot.state == ModuleState::Running && snapshot.process_alive;
5055 let live = match protocol {
5061 ModuleProtocol::Subc => running_process && registration_active,
5062 ModuleProtocol::None => running_process,
5063 };
5064 let retained = self.inner.retained.waiting_detail(&self.inner.module_id);
5067 let carry_configured = self
5068 .inner
5069 .configuration
5070 .lock()
5071 .map_err(|_| SuperviseError::StatePoisoned {
5072 module_id: Some(self.inner.module_id.clone()),
5073 })?
5074 .spec
5075 .carry_containment;
5076 let recovery_block = carries_containment(
5077 carry_configured,
5078 self.inner
5079 .retained
5080 .has_carried_container(&self.inner.module_id),
5081 )
5082 .then(|| self.inner.retained.recovery_block())
5083 .flatten();
5084 let launch_nonce_source = observed_launch_nonce_source(&snapshot);
5085 let pid = snapshot.reported_pid();
5086 let mut health = snapshot.health;
5087 if let Some(detail) = recovery_block
5088 .as_ref()
5089 .or_else(|| retained.as_ref().map(|(_, detail)| detail))
5090 {
5091 health.detail = Some(match health.detail {
5092 Some(previous) => format!("{previous}; {detail}"),
5093 None => detail.clone(),
5094 });
5095 }
5096
5097 Ok((
5098 ModuleStatus {
5099 admission_refused: snapshot.admission_refused,
5100 legacy_registration_spawn: snapshot.legacy_registration_spawn,
5101 registration_declaration: snapshot.registration_declaration.clone(),
5102 stop_reason: snapshot.stop_reason,
5103 waiting_for_previous_pid: retained
5104 .as_ref()
5105 .map(|(pid, _)| *pid)
5106 .filter(|pid| *pid > 0)
5107 .or(snapshot.waiting_for_previous_pid),
5108 waiting_for_previous_detail: recovery_block
5109 .or_else(|| retained.map(|(_, detail)| detail))
5110 .or(snapshot.waiting_for_previous_detail),
5111 module_id: self.inner.module_id.clone(),
5112 launch_nonce_source,
5113 state: snapshot.state,
5114 enabled: snapshot.enabled,
5115 process_alive: snapshot.process_alive,
5116 registration_active,
5117 protocol,
5118 live,
5119 restart_count,
5120 lifetime_restarts: snapshot.lifetime_restarts,
5121 spawn_generation: snapshot.spawn_generation,
5122 swap_outcome: snapshot.swap_outcome,
5123 max_restarts: restart_policy.max_restarts,
5124 restart_window: restart_policy.window,
5125 drain_timeout,
5126 restart_backoff: restart_policy.backoff,
5127 restart_max_backoff: restart_policy.max_backoff,
5128 pid,
5129 spawned_at_ms: snapshot.spawned_at_ms,
5130 spawned_from: snapshot.spawned_from,
5131 launch_spec_digest: snapshot.launch_spec_digest,
5132 process_start_time: snapshot.process_start_time,
5133 last_exit: snapshot.last_exit,
5134 health,
5135 },
5136 snapshot.spawned_file_identity.clone(),
5137 ))
5138 }
5139
5140 #[cfg(test)]
5141 pub(crate) fn hold_snapshot_for_test(
5142 &self,
5143 acquired: std::sync::mpsc::Sender<()>,
5144 hold: Duration,
5145 ) -> std::thread::JoinHandle<()> {
5146 let snapshot = Arc::clone(&self.inner.snapshot);
5147 std::thread::spawn(move || {
5148 let _guard = snapshot.lock().expect("test snapshot lock is not poisoned");
5149 acquired
5150 .send(())
5151 .expect("test receiver waits for snapshot lock");
5152 std::thread::sleep(hold);
5153 })
5154 }
5155
5156 pub(crate) async fn status_and_running_image_agreement(
5162 &self,
5163 ) -> Result<(ModuleStatus, subc_control::RunningImageAgreement), SuperviseError> {
5164 let (status, identity) = self.status_with_snapshot_lock(&self.inner.snapshot, None)?;
5165 let image = self
5166 .inner
5167 .provenance_probe
5168 .observe(
5169 status.pid,
5170 status.spawned_from.as_deref(),
5171 identity,
5172 status.process_start_time,
5173 )
5174 .await;
5175 Ok((status, image))
5176 }
5177
5178 pub(crate) async fn running_image_agreement(&self) -> subc_control::RunningImageAgreement {
5179 let snapshot = match lock_snapshot(&self.inner.snapshot) {
5180 Ok(snapshot) => snapshot.clone(),
5181 Err(_) => {
5182 return subc_control::RunningImageAgreement::Unavailable {
5183 reason: subc_control::RunningImageUnavailableReason::NotRunning,
5184 };
5185 }
5186 };
5187 self.inner
5188 .provenance_probe
5189 .observe(
5190 snapshot.reported_pid(),
5191 snapshot.spawned_from.as_deref(),
5192 snapshot.spawned_file_identity.clone(),
5193 snapshot.process_start_time,
5194 )
5195 .await
5196 }
5197
5198 pub(crate) fn child_resource_usage(&self) -> subc_control::ChildResourceUsage {
5201 #[cfg(windows)]
5202 {
5203 let image = lock_snapshot(&self.inner.snapshot)
5204 .ok()
5205 .and_then(|snapshot| {
5206 snapshot
5207 .spawned_file_identity
5208 .as_ref()
5209 .and_then(|identity| identity.windows_image.clone())
5210 });
5211 match image {
5212 Some(image) => {
5213 crate::child_resources::read_windows_handle(image.process(), image.start_time())
5214 }
5215 None => subc_control::ChildResourceUsage::Unavailable {
5216 reason: subc_control::ChildResourceUnavailableReason::NotRunning,
5217 },
5218 }
5219 }
5220 #[cfg(not(windows))]
5221 {
5222 let (pid, start_time) = match lock_snapshot(&self.inner.snapshot) {
5223 Ok(snapshot) => (snapshot.reported_pid(), snapshot.process_start_time),
5224 Err(_) => {
5225 return subc_control::ChildResourceUsage::Unavailable {
5226 reason: subc_control::ChildResourceUnavailableReason::Unreadable,
5227 }
5228 }
5229 };
5230 crate::child_resources::read(pid, start_time)
5231 }
5232 }
5233
5234 #[cfg(all(test, any(target_os = "linux", windows)))]
5235 pub(crate) fn subtree_target(&self) -> crate::subtree_resources::Target {
5236 self.inner
5237 .snapshot
5238 .try_lock()
5239 .map(|snapshot| snapshot.subtree_target.clone())
5240 .unwrap_or(crate::subtree_resources::Target::Unavailable("read_failed"))
5241 }
5242
5243 #[cfg(test)]
5244 pub(crate) fn set_subtree_target_for_test(&self, target: crate::subtree_resources::Target) {
5245 self.inner.snapshot.lock().unwrap().subtree_target = target;
5246 }
5247
5248 pub(crate) fn will_recover_after_connection_loss(&self) -> Result<bool, SuperviseError> {
5249 let mut snapshot = lock_snapshot(&self.inner.snapshot)?;
5250 Ok(match snapshot.state {
5251 ModuleState::Restarting => true,
5252 ModuleState::Failed | ModuleState::Disabled => false,
5253 _ if snapshot.enabled && snapshot.respawn_pending => true,
5254 _ => daemon_will_restart(
5255 &mut snapshot,
5256 &self
5257 .inner
5258 .configuration
5259 .lock()
5260 .map_err(|_| SuperviseError::StatePoisoned {
5261 module_id: Some(self.inner.module_id.clone()),
5262 })?
5263 .restart_policy,
5264 Instant::now(),
5265 ),
5266 })
5267 }
5268
5269 #[cfg(test)]
5270 pub(crate) fn is_warming(&self) -> Result<bool, SuperviseError> {
5271 self.is_warming_with_snapshot_lock(None)
5272 }
5273
5274 pub(crate) fn is_warming_for_control(
5275 &self,
5276 caller: &'static str,
5277 ) -> Result<bool, SuperviseError> {
5278 self.is_warming_with_snapshot_lock(Some(caller))
5279 }
5280
5281 fn is_warming_with_snapshot_lock(
5282 &self,
5283 caller: Option<&'static str>,
5284 ) -> Result<bool, SuperviseError> {
5285 let snapshot = match caller {
5286 Some(caller) => {
5287 lock_snapshot_for_control(&self.inner.snapshot, &self.inner.module_id, caller)?
5288 }
5289 None => lock_snapshot(&self.inner.snapshot)?,
5290 }
5291 .clone();
5292 Ok(matches!(
5293 snapshot.state,
5294 ModuleState::Starting | ModuleState::Running | ModuleState::Restarting
5295 ))
5296 }
5297
5298 pub async fn drain(&self) -> Result<(), SuperviseError> {
5300 self.stop().await
5301 }
5302
5303 pub(crate) async fn retire_requested(
5304 &self,
5305 requester: Option<Principal>,
5306 ) -> Result<(), SuperviseError> {
5307 match self.state()? {
5308 ModuleState::Stopped => return Ok(()),
5309 ModuleState::Failed
5310 if lock_snapshot(&self.inner.snapshot)?
5311 .failed_retry
5312 .pending
5313 .is_none() =>
5314 {
5315 return Ok(())
5316 }
5317 ModuleState::Failed
5318 | ModuleState::Starting
5319 | ModuleState::Running
5320 | ModuleState::Unresponsive
5321 | ModuleState::Restarting
5322 | ModuleState::Draining
5323 | ModuleState::Disabled => {}
5324 }
5325
5326 let (reply_tx, reply_rx) = oneshot::channel();
5327 self.inner
5328 .commands
5329 .send(SupervisorCommand::Retire {
5330 requester,
5331 reply: reply_tx,
5332 })
5333 .await
5334 .map_err(|_| SuperviseError::CommandClosed {
5335 module_id: self.inner.module_id.clone(),
5336 })?;
5337 reply_rx.await.map_err(|_| SuperviseError::CommandClosed {
5338 module_id: self.inner.module_id.clone(),
5339 })?
5340 }
5341
5342 pub async fn stop(&self) -> Result<(), SuperviseError> {
5343 self.stop_requested(None).await
5344 }
5345
5346 pub(crate) async fn stop_requested(
5347 &self,
5348 requester: Option<Principal>,
5349 ) -> Result<(), SuperviseError> {
5350 match self.state()? {
5351 ModuleState::Stopped => return Ok(()),
5352 ModuleState::Failed
5353 if lock_snapshot(&self.inner.snapshot)?
5354 .failed_retry
5355 .pending
5356 .is_none() =>
5357 {
5358 return Ok(())
5359 }
5360 ModuleState::Failed
5361 | ModuleState::Starting
5362 | ModuleState::Running
5363 | ModuleState::Unresponsive
5364 | ModuleState::Restarting
5365 | ModuleState::Draining
5366 | ModuleState::Disabled => {}
5367 }
5368
5369 let (reply_tx, reply_rx) = oneshot::channel();
5370 self.inner
5371 .commands
5372 .send(SupervisorCommand::Drain {
5373 requester,
5374 reply: reply_tx,
5375 })
5376 .await
5377 .map_err(|_| SuperviseError::CommandClosed {
5378 module_id: self.inner.module_id.clone(),
5379 })?;
5380 reply_rx.await.map_err(|_| SuperviseError::CommandClosed {
5381 module_id: self.inner.module_id.clone(),
5382 })?
5383 }
5384
5385 pub async fn restart(&self, drain_timeout_ms: Option<u64>) -> Result<(), SuperviseError> {
5386 self.restart_requested(drain_timeout_ms, None).await
5387 }
5388
5389 pub(crate) async fn restart_requested(
5390 &self,
5391 drain_timeout_ms: Option<u64>,
5392 requester: Option<Principal>,
5393 ) -> Result<(), SuperviseError> {
5394 let queued_at = Instant::now();
5395 let received_at_generation = lock_snapshot(&self.inner.snapshot)?.spawn_generation;
5396 let (reply_tx, reply_rx) = oneshot::channel();
5397 self.inner
5398 .commands
5399 .send(SupervisorCommand::Restart {
5400 requester,
5401 drain_timeout_ms,
5402 received_at_generation,
5403 queued_at,
5404 reply: reply_tx,
5405 })
5406 .await
5407 .map_err(|_| SuperviseError::CommandClosed {
5408 module_id: self.inner.module_id.clone(),
5409 })?;
5410 reply_rx.await.map_err(|_| SuperviseError::CommandClosed {
5411 module_id: self.inner.module_id.clone(),
5412 })?
5413 }
5414
5415 pub async fn swap(
5421 &self,
5422 ready_timeout: Option<Duration>,
5423 ) -> Result<SwapOutcome, SuperviseError> {
5424 self.swap_requested(ready_timeout, None).await
5425 }
5426
5427 pub(crate) async fn swap_requested(
5428 &self,
5429 ready_timeout: Option<Duration>,
5430 requester: Option<Principal>,
5431 ) -> Result<SwapOutcome, SuperviseError> {
5432 let (reply_tx, reply_rx) = oneshot::channel();
5433 self.inner
5434 .commands
5435 .send(SupervisorCommand::Swap {
5436 requester,
5437 ready_timeout,
5438 reply: reply_tx,
5439 })
5440 .await
5441 .map_err(|_| SuperviseError::CommandClosed {
5442 module_id: self.inner.module_id.clone(),
5443 })?;
5444 reply_rx.await.map_err(|_| SuperviseError::CommandClosed {
5445 module_id: self.inner.module_id.clone(),
5446 })?
5447 }
5448
5449 pub async fn reload(&self) -> Result<(), SuperviseError> {
5450 self.reload_requested(None).await
5451 }
5452
5453 pub(crate) async fn reload_requested(
5454 &self,
5455 requester: Option<Principal>,
5456 ) -> Result<(), SuperviseError> {
5457 let (reply_tx, reply_rx) = oneshot::channel();
5458 self.inner
5459 .commands
5460 .send(SupervisorCommand::Reload {
5461 requester,
5462 reply: reply_tx,
5463 })
5464 .await
5465 .map_err(|_| SuperviseError::CommandClosed {
5466 module_id: self.inner.module_id.clone(),
5467 })?;
5468 reply_rx.await.map_err(|_| SuperviseError::CommandClosed {
5469 module_id: self.inner.module_id.clone(),
5470 })?
5471 }
5472
5473 pub async fn set_enabled(&self, enabled: bool) -> Result<bool, SuperviseError> {
5474 self.set_enabled_requested(enabled, None).await
5475 }
5476
5477 pub(crate) async fn set_enabled_requested(
5478 &self,
5479 enabled: bool,
5480 requester: Option<Principal>,
5481 ) -> Result<bool, SuperviseError> {
5482 self.set_enabled_with_budget_policy(enabled, requester, true)
5483 .await
5484 }
5485
5486 pub(crate) async fn release_config_hold(&self) -> Result<bool, SuperviseError> {
5487 self.set_enabled_with_budget_policy(true, None, false).await
5488 }
5489
5490 #[cfg(all(test, unix))]
5491 pub(crate) fn seed_crash_budget_for_test(&self, count: u32) {
5492 let policy = self.inner.configuration.lock().unwrap().restart_policy;
5493 let mut state = self.inner.snapshot.lock().unwrap();
5494 for _ in 0..count {
5495 state.record_crash_restart(&policy, Instant::now());
5496 }
5497 }
5498
5499 async fn set_enabled_with_budget_policy(
5500 &self,
5501 enabled: bool,
5502 requester: Option<Principal>,
5503 reset_budget: bool,
5504 ) -> Result<bool, SuperviseError> {
5505 let (reply_tx, reply_rx) = oneshot::channel();
5506 self.inner
5507 .commands
5508 .send(SupervisorCommand::SetEnabled {
5509 requester,
5510 enabled,
5511 reset_budget,
5512 reply: reply_tx,
5513 })
5514 .await
5515 .map_err(|_| SuperviseError::CommandClosed {
5516 module_id: self.inner.module_id.clone(),
5517 })?;
5518 reply_rx.await.map_err(|_| SuperviseError::CommandClosed {
5519 module_id: self.inner.module_id.clone(),
5520 })?
5521 }
5522
5523 pub(crate) fn declared_protocol(&self) -> Result<ModuleProtocol, SuperviseError> {
5527 let configured = self
5528 .inner
5529 .configuration
5530 .lock()
5531 .map_err(|_| SuperviseError::StatePoisoned {
5532 module_id: Some(self.inner.module_id.clone()),
5533 })?
5534 .spec
5535 .protocol;
5536 let state = lock_snapshot(&self.inner.snapshot)?;
5537 Ok(state.spawned_protocol.unwrap_or(configured))
5538 }
5539
5540 pub(crate) fn configuration(&self) -> Result<(ModuleSpec, HealthConfig), SuperviseError> {
5541 let configuration =
5542 self.inner
5543 .configuration
5544 .lock()
5545 .map_err(|_| SuperviseError::StatePoisoned {
5546 module_id: Some(self.inner.module_id.clone()),
5547 })?;
5548 Ok((configuration.spec.clone(), configuration.health.clone()))
5549 }
5550
5551 pub(crate) fn configured_register_timeout(&self) -> Result<Duration, SuperviseError> {
5552 self.inner
5553 .configuration
5554 .lock()
5555 .map(|configuration| configuration.register_timeout)
5556 .map_err(|_| SuperviseError::StatePoisoned {
5557 module_id: Some(self.inner.module_id.clone()),
5558 })
5559 }
5560
5561 pub(crate) fn configured_enabled(&self) -> Result<bool, SuperviseError> {
5562 self.inner
5563 .configuration
5564 .lock()
5565 .map(|configuration| configuration.enabled)
5566 .map_err(|_| SuperviseError::StatePoisoned {
5567 module_id: Some(self.inner.module_id.clone()),
5568 })
5569 }
5570
5571 #[cfg(any(test, feature = "test-support"))]
5575 pub async fn update_spec_for_test(&self, spec: ModuleSpec) -> Result<(), SuperviseError> {
5576 let (_, health) = self.configuration()?;
5577 let drain_timeout_ms = u64::try_from(
5578 self.inner
5579 .effective_drain_timeout
5580 .lock()
5581 .unwrap_or_else(|poisoned| poisoned.into_inner())
5582 .as_millis(),
5583 )
5584 .ok();
5585 self.update_configuration(spec, health, drain_timeout_ms)
5586 .await
5587 }
5588
5589 pub(crate) async fn reconcile_approved_configuration(
5590 &self,
5591 approved: crate::config_gate::runtime::ApprovedDaemonConfiguration,
5592 ) -> Result<(), SuperviseError> {
5593 let entry = approved
5594 .module_entry(&self.inner.module_id)
5595 .map_err(|error| SuperviseError::InvalidSpec {
5596 reason: error.to_string(),
5597 })?;
5598 validate_spec(&entry.module_spec())?;
5599 let (reply, received) = oneshot::channel();
5600 self.inner
5601 .commands
5602 .send(SupervisorCommand::ApprovedConfiguration { approved, reply })
5603 .await
5604 .map_err(|_| SuperviseError::CommandClosed {
5605 module_id: self.inner.module_id.clone(),
5606 })?;
5607 received.await.map_err(|_| SuperviseError::CommandClosed {
5608 module_id: self.inner.module_id.clone(),
5609 })??;
5610 let mut configuration =
5611 self.inner
5612 .configuration
5613 .lock()
5614 .map_err(|_| SuperviseError::StatePoisoned {
5615 module_id: Some(self.inner.module_id.clone()),
5616 })?;
5617 configuration.spec = entry.module_spec();
5618 configuration.enabled = entry.enabled;
5619 configuration.health = entry.health;
5620 configuration.register_timeout = entry.register_timeout;
5621 configuration.restart_policy = entry.restart;
5622 Ok(())
5623 }
5624
5625 pub(crate) async fn update_configuration(
5626 &self,
5627 spec: ModuleSpec,
5628 health: HealthConfig,
5629 drain_timeout_ms: Option<u64>,
5630 ) -> Result<(), SuperviseError> {
5631 if spec.module_id != self.inner.module_id {
5632 return Err(SuperviseError::InvalidSpec {
5633 reason: "a supervised module's module_id cannot be changed".to_string(),
5634 });
5635 }
5636 validate_spec(&spec)?;
5637 let (reply_tx, reply_rx) = oneshot::channel();
5638 self.inner
5639 .commands
5640 .send(SupervisorCommand::UpdateConfiguration {
5641 spec: spec.clone(),
5642 health: health.clone(),
5643 drain_timeout_ms,
5644 reply: reply_tx,
5645 })
5646 .await
5647 .map_err(|_| SuperviseError::CommandClosed {
5648 module_id: self.inner.module_id.clone(),
5649 })?;
5650 reply_rx.await.map_err(|_| SuperviseError::CommandClosed {
5651 module_id: self.inner.module_id.clone(),
5652 })??;
5653 let mut configuration =
5654 self.inner
5655 .configuration
5656 .lock()
5657 .map_err(|_| SuperviseError::StatePoisoned {
5658 module_id: Some(self.inner.module_id.clone()),
5659 })?;
5660 configuration.spec = spec;
5661 configuration.health = health;
5662 Ok(())
5663 }
5664}
5665
5666impl Drop for SupervisedModuleInner {
5667 fn drop(&mut self) {
5668 let Ok(mut monitor) = self.monitor.lock() else {
5669 return;
5670 };
5671 if let Some(monitor) = monitor.as_ref().filter(|monitor| !monitor.is_finished()) {
5672 let _ = update_snapshot(&self.snapshot, Some(&self.module_id), |state| {
5673 state.state = ModuleState::Stopped;
5674 clear_current_process_facts(state);
5675 });
5676 monitor.abort();
5677 }
5678 let _ = monitor.take();
5679 }
5680}
5681
5682#[derive(Debug)]
5683enum SupervisorCommand {
5684 ApprovedConfiguration {
5685 approved: crate::config_gate::runtime::ApprovedDaemonConfiguration,
5686 reply: oneshot::Sender<Result<(), SuperviseError>>,
5687 },
5688 ConfigPolicyProbe {
5689 connection: crate::ConnectionId,
5690 generation: u64,
5691 attempt: u64,
5692 },
5693 ConfigApplyRestart {
5694 generation: u64,
5695 supersedes_swap: bool,
5696 },
5697 ConfigExitHold {
5698 reply: oneshot::Sender<Result<(), SuperviseError>>,
5699 },
5700 ConfigDeliveryFailed {
5701 connection: crate::ConnectionId,
5702 hold: bool,
5703 },
5704 Drain {
5705 requester: Option<Principal>,
5706 reply: oneshot::Sender<Result<(), SuperviseError>>,
5707 },
5708 Retire {
5709 requester: Option<Principal>,
5710 reply: oneshot::Sender<Result<(), SuperviseError>>,
5711 },
5712 Restart {
5713 requester: Option<Principal>,
5714 drain_timeout_ms: Option<u64>,
5719 received_at_generation: u64,
5723 queued_at: Instant,
5726 reply: oneshot::Sender<Result<(), SuperviseError>>,
5727 },
5728 Reload {
5729 requester: Option<Principal>,
5730 reply: oneshot::Sender<Result<(), SuperviseError>>,
5731 },
5732 SetEnabled {
5733 reset_budget: bool,
5734 requester: Option<Principal>,
5735 enabled: bool,
5736 reply: oneshot::Sender<Result<bool, SuperviseError>>,
5737 },
5738 UpdateConfiguration {
5739 spec: ModuleSpec,
5740 health: HealthConfig,
5741 drain_timeout_ms: Option<u64>,
5744 reply: oneshot::Sender<Result<(), SuperviseError>>,
5745 },
5746 Swap {
5747 requester: Option<Principal>,
5748 ready_timeout: Option<Duration>,
5751 reply: oneshot::Sender<Result<SwapOutcome, SuperviseError>>,
5753 },
5754}
5755
5756#[derive(Clone, Copy, Debug, PartialEq, Eq)]
5760pub enum SwapOutcome {
5761 Cutover,
5762 SupersededByConfigRestart,
5763}
5764
5765impl SwapOutcome {
5766 pub fn as_str(self) -> &'static str {
5767 match self {
5768 Self::Cutover => "cutover",
5769 Self::SupersededByConfigRestart => "superseded_by_config_restart",
5770 }
5771 }
5772}
5773
5774#[derive(Debug)]
5775pub enum SuperviseError {
5776 ConfigHeld {
5777 module_id: String,
5778 code: String,
5779 detail: String,
5780 },
5781 ConfigDeliveryDeferred {
5782 module_id: String,
5783 },
5784 InvalidSpec {
5785 reason: String,
5786 },
5787 Spawn {
5788 program: PathBuf,
5789 source: io::Error,
5790 cgroup_path: Option<PathBuf>,
5791 },
5792 Cgroup {
5793 module_id: String,
5794 source: io::Error,
5795 },
5796 LaunchNonce {
5799 reason: String,
5800 },
5801 Wait {
5802 module_id: String,
5803 source: io::Error,
5804 },
5805 Kill {
5806 module_id: String,
5807 source: io::Error,
5808 },
5809 Forwarding(ForwardingError),
5810 Registry(RegistryError),
5811 ReloadUnavailable {
5812 module_id: String,
5813 reason: String,
5814 },
5815 Disabled {
5820 module_id: String,
5821 },
5822 ReloadFailed {
5823 module_id: String,
5824 reason: String,
5825 },
5826 RegistrationStillActive {
5827 module_id: String,
5828 waited: Duration,
5829 },
5830 StatePoisoned {
5831 module_id: Option<String>,
5832 },
5833 CommandClosed {
5834 module_id: String,
5835 },
5836 SwapInProgress {
5840 module_id: String,
5841 },
5842 SwapRefused {
5844 module_id: String,
5845 reason: SwapRefusal,
5846 },
5847 SwapFailed {
5851 module_id: String,
5852 arm: SwapFailureArm,
5853 detail: String,
5854 candidate_exit: Option<ExitReport>,
5857 },
5858}
5859
5860#[derive(Debug, Clone, Copy, PartialEq, Eq)]
5862pub enum SwapRefusal {
5863 OverlapExclusive,
5865 NotRegistered,
5868 ProtocolNone,
5871 NotConfigured,
5874 AlreadySwapping,
5876}
5877
5878impl SwapRefusal {
5879 pub fn as_str(self) -> &'static str {
5880 match self {
5881 Self::OverlapExclusive => "overlap_exclusive",
5882 Self::NotRegistered => "not_registered",
5883 Self::ProtocolNone => "protocol_none",
5884 Self::NotConfigured => "not_configured",
5885 Self::AlreadySwapping => "already_swapping",
5886 }
5887 }
5888}
5889
5890#[derive(Debug, Clone, Copy, PartialEq, Eq)]
5893pub enum SwapFailureArm {
5894 SpawnFailed,
5896 NeverRegistered,
5898 NeverReady,
5900 CandidateExited,
5902 CandidateUnhealthy,
5904 Interrupted,
5908 CutoverLost,
5913}
5914
5915impl SwapFailureArm {
5916 pub fn as_str(self) -> &'static str {
5917 match self {
5918 Self::SpawnFailed => "spawn_failed",
5919 Self::NeverRegistered => "never_registered",
5920 Self::NeverReady => "never_ready",
5921 Self::CandidateExited => "candidate_exited",
5922 Self::CandidateUnhealthy => "candidate_unhealthy",
5923 Self::Interrupted => "interrupted",
5924 Self::CutoverLost => "cutover_lost",
5925 }
5926 }
5927}
5928
5929impl SuperviseError {
5930 pub(crate) fn is_config_gate_error(&self) -> bool {
5931 match self {
5932 Self::ConfigHeld { .. } => true,
5933 Self::Spawn { source, .. } => source
5934 .get_ref()
5935 .is_some_and(|cause| cause.is::<crate::config_gate::GateError>()),
5936 _ => false,
5937 }
5938 }
5939}
5940
5941impl fmt::Display for SuperviseError {
5942 fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
5943 match self {
5944 Self::ConfigHeld { module_id, code, detail } => write!(f, "{code}: module '{module_id}' held: {detail}"),
5945 Self::ConfigDeliveryDeferred { module_id } => write!(f, "config_gated: module '{module_id}' is waiting for its exited delivery target's failure record"),
5946 Self::InvalidSpec { reason } => write!(f, "invalid module spec: {reason}"),
5947 Self::Spawn {
5948 program,
5949 source,
5950 cgroup_path: Some(cgroup_path),
5951 } => write!(
5952 f,
5953 "failed to place module in cgroup '{}' while spawning '{}': {source}",
5954 cgroup_path.display(),
5955 program.display()
5956 ),
5957 Self::Spawn {
5958 program,
5959 source,
5960 cgroup_path: None,
5961 } => write!(
5962 f,
5963 "failed to spawn module '{}': {source}",
5964 program.display()
5965 ),
5966 Self::Cgroup { module_id, source } => {
5967 write!(
5968 f,
5969 "failed to prepare cgroup for module '{module_id}': {source}"
5970 )
5971 }
5972 Self::LaunchNonce { reason } => {
5973 write!(
5974 f,
5975 "failed to generate reserved-module launch nonce: {reason}"
5976 )
5977 }
5978 Self::Wait { module_id, source } => {
5979 write!(f, "failed to wait for module '{module_id}': {source}")
5980 }
5981 Self::Kill { module_id, source } => {
5982 write!(f, "failed to kill module '{module_id}': {source}")
5983 }
5984 Self::Forwarding(err) => write!(f, "forwarding error: {err}"),
5985 Self::Registry(err) => write!(f, "registry error: {err}"),
5986 Self::ReloadUnavailable { module_id, reason } => {
5987 write!(f, "reload unavailable for module '{module_id}': {reason}")
5988 }
5989 Self::Disabled { module_id } => {
5990 write!(
5991 f,
5992 "module '{module_id}' is disabled; enable it before restart or reload"
5993 )
5994 }
5995 Self::ReloadFailed { module_id, reason } => {
5996 write!(f, "reload failed for module '{module_id}': {reason}")
5997 }
5998 Self::RegistrationStillActive { module_id, waited } => write!(
5999 f,
6000 "module '{module_id}' registration remained active after waiting {waited:?}"
6001 ),
6002 Self::StatePoisoned { module_id } => match module_id {
6003 Some(module_id) => {
6004 write!(f, "supervisor state for module '{module_id}' was poisoned")
6005 }
6006 None => write!(f, "supervisor state was poisoned"),
6007 },
6008 Self::CommandClosed { module_id } => {
6009 write!(
6010 f,
6011 "supervisor command channel for module '{module_id}' is closed"
6012 )
6013 }
6014 Self::SwapInProgress { module_id } => write!(
6015 f,
6016 "module '{module_id}' is being swapped; retry once the swap has cut over or failed, or stop the module to abort the swap"
6017 ),
6018 Self::SwapRefused { module_id, reason } => match reason {
6019 SwapRefusal::OverlapExclusive => write!(
6020 f,
6021 "module '{module_id}' is declared overlap: \"exclusive\" (the default): two processes of it must not run at once, so it cannot be swapped; use a plain restart, or declare overlap: \"safe\" in its config if it really tolerates a second process"
6022 ),
6023 SwapRefusal::NotRegistered => write!(
6024 f,
6025 "module '{module_id}' is not registered, so there is no serving process to keep while a replacement warms; use a plain restart"
6026 ),
6027 SwapRefusal::ProtocolNone => write!(
6028 f,
6029 "module '{module_id}' is protocol: \"none\" and never registers, so a swap could never see its replacement become ready; use a plain restart"
6030 ),
6031 SwapRefusal::NotConfigured => write!(
6032 f,
6033 "module '{module_id}' cannot be swapped: the supervisor was built without the forwarding table or shared handle a swap needs"
6034 ),
6035 SwapRefusal::AlreadySwapping => {
6036 write!(f, "module '{module_id}' is already being swapped")
6037 }
6038 },
6039 Self::SwapFailed {
6040 module_id,
6041 arm,
6042 detail,
6043 ..
6044 } => write!(
6045 f,
6046 "swap of module '{module_id}' failed ({}): {detail}; the running process was left serving",
6047 arm.as_str()
6048 ),
6049 }
6050 }
6051}
6052
6053impl Error for SuperviseError {
6054 fn source(&self) -> Option<&(dyn Error + 'static)> {
6055 match self {
6056 Self::Spawn { source, .. }
6057 | Self::Cgroup { source, .. }
6058 | Self::Wait { source, .. }
6059 | Self::Kill { source, .. } => Some(source),
6060 Self::Forwarding(err) => Some(err),
6061 Self::Registry(err) => Some(err),
6062 Self::LaunchNonce { .. }
6063 | Self::ConfigDeliveryDeferred { .. }
6064 | Self::ConfigHeld { .. }
6065 | Self::InvalidSpec { .. }
6066 | Self::ReloadUnavailable { .. }
6067 | Self::Disabled { .. }
6068 | Self::ReloadFailed { .. }
6069 | Self::RegistrationStillActive { .. }
6070 | Self::StatePoisoned { .. }
6071 | Self::CommandClosed { .. }
6072 | Self::SwapInProgress { .. }
6073 | Self::SwapRefused { .. }
6074 | Self::SwapFailed { .. } => None,
6075 }
6076 }
6077}
6078
6079pub(crate) fn validate_spec(spec: &ModuleSpec) -> Result<(), SuperviseError> {
6080 if spec.carry_containment && spec.overlap == ModuleOverlap::Safe {
6081 return Err(SuperviseError::InvalidSpec {
6082 reason: "carry_containment cannot be combined with overlap: safe".to_owned(),
6083 });
6084 }
6085 if spec.module_id.trim().is_empty() {
6086 return Err(SuperviseError::InvalidSpec {
6087 reason: "module_id must not be empty".to_string(),
6088 });
6089 }
6090
6091 Ok(())
6092}
6093
6094#[derive(Debug, Default)]
6095struct HealthProbeRuntime {
6096 spawn_generation: Option<u64>,
6097 configured_health: Option<HealthConfig>,
6098 registered_connection: Option<crate::ConnectionId>,
6099 advertised: bool,
6100 next_probe_at: Option<Instant>,
6101 probe_index: u64,
6102}
6103
6104fn running_protocol(spec: &ModuleSpec, snapshot: &SharedSnapshot) -> ModuleProtocol {
6105 lock_snapshot(snapshot)
6106 .ok()
6107 .and_then(|state| state.spawned_protocol)
6108 .unwrap_or(spec.protocol)
6109}
6110
6111impl HealthProbeRuntime {
6112 fn refresh_registration(
6113 &mut self,
6114 spec: &ModuleSpec,
6115 runtime: &SupervisorRuntimeConfig,
6116 registry: &Registry,
6117 snapshot: &SharedSnapshot,
6118 ) {
6119 let generation = lock_snapshot(snapshot)
6120 .ok()
6121 .map(|state| state.spawn_generation);
6122 if self.spawn_generation != generation {
6123 *self = Self {
6127 spawn_generation: generation,
6128 ..Self::default()
6129 };
6130 }
6131 if self.configured_health.as_ref() != Some(&runtime.health) {
6132 self.configured_health = Some(runtime.health.clone());
6133 self.next_probe_at = None;
6134 self.registered_connection = None;
6135 self.probe_index = 0;
6136 }
6137 if running_protocol(spec, snapshot) == ModuleProtocol::None {
6141 self.registered_connection = None;
6142 self.advertised = runtime.health.http.is_some();
6143 if !self.advertised {
6144 self.next_probe_at = None;
6145 let _ = update_snapshot(snapshot, Some(&spec.module_id), |state| {
6146 let unknown = ModuleHealthStatus::default();
6147 if state.health != unknown {
6148 state.health = unknown;
6149 state.last_probe_evidence = None;
6150 }
6151 });
6152 } else if self.next_probe_at.is_none() {
6153 self.next_probe_at = Some(
6154 Instant::now()
6155 + jittered_health_delay(&spec.module_id, 0, runtime.health.cadence),
6156 );
6157 }
6158 return;
6159 }
6160
6161 let registration = match registry.get_module(&spec.module_id) {
6162 Ok(registration) => registration,
6163 Err(err) => {
6164 warn!(module_id = %spec.module_id, error = %err, "health prober could not read registry");
6165 self.advertised = false;
6166 self.next_probe_at = None;
6167 return;
6168 }
6169 };
6170
6171 let Some(registration) = registration else {
6172 self.registered_connection = None;
6173 self.advertised = false;
6174 self.next_probe_at = None;
6175 return;
6176 };
6177
6178 let advertised = registration
6179 .control_ops
6180 .iter()
6181 .any(|op| op == MODULE_CONTROL_OP_HEALTH_CHECK);
6182 if !advertised {
6183 self.registered_connection = Some(registration.connection_id);
6184 self.advertised = false;
6185 self.next_probe_at = None;
6186 let _ = update_snapshot(snapshot, Some(&spec.module_id), |state| {
6187 if state.health.status != SupervisorHealthStatus::Unknown
6188 || state.health.consecutive_failures != 0
6189 || state.health.last_probe_ms.is_some()
6190 || state.health.detail.is_some()
6191 || state.health.metrics.is_some()
6192 {
6193 state.health.status = SupervisorHealthStatus::Unknown;
6194 state.last_probe_evidence = None;
6195 state.health.consecutive_failures = 0;
6196 state.health.last_probe_ms = None;
6197 state.health.detail = None;
6198 state.health.metrics = None;
6199 }
6200 });
6201 return;
6202 }
6203
6204 let reregistered = self.registered_connection != Some(registration.connection_id);
6205 self.registered_connection = Some(registration.connection_id);
6206 self.advertised = true;
6207 if reregistered || self.next_probe_at.is_none() {
6208 self.probe_index = 0;
6209 self.next_probe_at = Some(
6210 Instant::now() + jittered_health_delay(&spec.module_id, 0, runtime.health.cadence),
6211 );
6212 let _ = update_snapshot(snapshot, Some(&spec.module_id), |state| {
6213 state.health.status = SupervisorHealthStatus::Unknown;
6214 state.last_probe_evidence = None;
6215 state.health.consecutive_failures = 0;
6216 state.health.detail = None;
6217 state.health.metrics = None;
6218 });
6219 }
6220 }
6221
6222 fn wake_at(&self) -> Option<Instant> {
6223 if !self.advertised {
6224 return None;
6225 }
6226 self.next_probe_at
6227 }
6228
6229 fn due(&self) -> bool {
6230 self.advertised
6231 && self
6232 .next_probe_at
6233 .is_some_and(|next| Instant::now() >= next)
6234 }
6235
6236 fn schedule_next(&mut self, spec: &ModuleSpec, cadence: Duration) {
6237 self.probe_index = self.probe_index.wrapping_add(1);
6238 self.next_probe_at = Some(
6239 Instant::now() + jittered_health_delay(&spec.module_id, self.probe_index, cadence),
6240 );
6241 }
6242}
6243
6244#[derive(Debug, Clone, Copy, PartialEq, Eq)]
6248enum HealthProbeEvidence {
6249 LaneDead,
6251 NoAnswer,
6253 BadAnswer,
6255 Misconfigured,
6257}
6258
6259#[derive(Debug)]
6260struct HealthProbeError {
6261 evidence: HealthProbeEvidence,
6262 message: String,
6263}
6264
6265impl HealthProbeError {
6266 fn lane_dead(message: impl Into<String>) -> Self {
6267 Self::with(HealthProbeEvidence::LaneDead, message)
6268 }
6269
6270 fn no_answer(message: impl Into<String>) -> Self {
6271 Self::with(HealthProbeEvidence::NoAnswer, message)
6272 }
6273
6274 fn bad_answer(message: impl Into<String>) -> Self {
6275 Self::with(HealthProbeEvidence::BadAnswer, message)
6276 }
6277
6278 fn misconfigured(message: impl Into<String>) -> Self {
6279 Self::with(HealthProbeEvidence::Misconfigured, message)
6280 }
6281
6282 fn with(evidence: HealthProbeEvidence, message: impl Into<String>) -> Self {
6283 Self {
6284 evidence,
6285 message: message.into(),
6286 }
6287 }
6288
6289 #[allow(dead_code)]
6298 fn is_proof_of_death(&self) -> bool {
6299 matches!(self.evidence, HealthProbeEvidence::LaneDead)
6300 }
6301
6302 fn label(&self) -> &'static str {
6310 match self.evidence {
6311 HealthProbeEvidence::LaneDead => "lane-dead",
6312 HealthProbeEvidence::NoAnswer => "no-answer",
6313 HealthProbeEvidence::BadAnswer => "bad-answer",
6314 HealthProbeEvidence::Misconfigured => "daemon-misconfigured",
6315 }
6316 }
6317}
6318
6319impl fmt::Display for HealthProbeError {
6320 fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
6321 f.write_str(&self.message)
6322 }
6323}
6324
6325async fn run_health_probe_cycle(
6326 spec: &ModuleSpec,
6327 runtime: &SupervisorRuntimeConfig,
6328 registry: &Registry,
6329 process_liveness: &SupervisorProcessLiveness,
6330 snapshot: &SharedSnapshot,
6331 child: &mut Option<SupervisedChild>,
6332) {
6333 let now_ms = unix_ms_now();
6334 let delivery_probe = match &runtime.config_delivery {
6335 Some(delivery) => {
6336 delivery
6337 .health_probe(&spec.module_id, runtime.forwarding.as_deref(), registry)
6338 .await
6339 }
6340 None => None,
6341 };
6342 let http = (running_protocol(spec, snapshot) == ModuleProtocol::None)
6343 .then_some(runtime.health.http.as_deref())
6344 .flatten();
6345 let result = match http {
6346 Some(url) => probe_http_health(url, runtime.health.deadline)
6347 .await
6348 .map(|report| (report, None)),
6349 None => probe_module_health_from(&spec.module_id, runtime, None)
6350 .await
6351 .map(|(report, probed)| (report, Some(probed))),
6352 };
6353 match result {
6354 Ok((report, probed)) => {
6355 handle_health_report(
6356 spec,
6357 runtime,
6358 registry,
6359 process_liveness,
6360 snapshot,
6361 child,
6362 report,
6363 now_ms,
6364 delivery_probe,
6365 probed,
6366 )
6367 .await;
6368 }
6369 Err(err) => {
6370 if http.is_some() {
6371 let _ = update_snapshot(snapshot, Some(&spec.module_id), |state| {
6372 state.health.status = SupervisorHealthStatus::Failing;
6373 });
6374 }
6375 handle_health_probe_failure(
6376 spec,
6377 runtime,
6378 registry,
6379 process_liveness,
6380 snapshot,
6381 child,
6382 err,
6383 now_ms,
6384 delivery_probe,
6385 )
6386 .await;
6387 }
6388 }
6389}
6390
6391pub(crate) struct HttpProbeTarget<'a> {
6392 address: std::net::SocketAddr,
6393 localhost: bool,
6394 authority: &'a str,
6395 path: String,
6396}
6397
6398pub(crate) fn parse_http_probe_url(url: &str) -> Result<HttpProbeTarget<'_>, String> {
6401 if url.bytes().any(|b| b <= b' ' || b == 127) || url.contains('#') {
6402 return Err("must not contain whitespace, controls, or a fragment".into());
6403 }
6404 let rest = url
6405 .strip_prefix("http://")
6406 .ok_or("must use plain http://")?;
6407 let split = rest.find(['/', '?']).unwrap_or(rest.len());
6408 let (authority, suffix) = rest.split_at(split);
6409 let (host, port) = if let Some(rest) = authority.strip_prefix("[::1]") {
6410 ("::1", rest)
6411 } else {
6412 let split = authority.find(':').unwrap_or(authority.len());
6413 authority.split_at(split)
6414 };
6415 let ip: std::net::IpAddr = match host {
6416 "127.0.0.1" | "localhost" => std::net::Ipv4Addr::LOCALHOST.into(),
6417 "::1" => std::net::Ipv6Addr::LOCALHOST.into(),
6418 _ => return Err("host must be 127.0.0.1, [::1], or localhost".into()),
6419 };
6420 let port = if port.is_empty() {
6421 80
6422 } else {
6423 port.strip_prefix(':')
6424 .and_then(|p| p.parse::<u16>().ok())
6425 .filter(|p| *p > 0)
6426 .ok_or("must have a valid nonzero TCP port")?
6427 };
6428 let path = if suffix.is_empty() {
6429 "/".into()
6430 } else if suffix.starts_with('?') {
6431 format!("/{suffix}")
6432 } else {
6433 suffix.into()
6434 };
6435 Ok(HttpProbeTarget {
6436 address: std::net::SocketAddr::new(ip, port),
6437 localhost: host == "localhost",
6438 authority,
6439 path,
6440 })
6441}
6442
6443async fn probe_http_health(
6444 url: &str,
6445 deadline: Duration,
6446) -> Result<HealthReport, HealthProbeError> {
6447 use tokio::io::{AsyncReadExt, AsyncWriteExt, BufReader};
6448 let target = parse_http_probe_url(url).map_err(HealthProbeError::misconfigured)?;
6449 let mut response_status = String::new();
6452 let mut body = Vec::new();
6453 let probe = async {
6454 let connection = match tokio::net::TcpStream::connect(target.address).await {
6457 Err(_) if target.localhost => {
6458 tokio::net::TcpStream::connect((
6459 std::net::Ipv6Addr::LOCALHOST,
6460 target.address.port(),
6461 ))
6462 .await
6463 }
6464 result => result,
6465 };
6466 let mut stream = connection.map_err(|error| {
6467 HealthProbeError::no_answer(format!("HTTP connect failed: {error}"))
6468 })?;
6469 stream.write_all(format!("GET {} HTTP/1.1\r\nHost: {}\r\nAccept-Encoding: identity\r\nConnection: close\r\n\r\n", target.path, target.authority).as_bytes()).await
6470 .map_err(|error| HealthProbeError::no_answer(format!("HTTP write failed: {error}")))?;
6471 let mut reader = BufReader::new(stream);
6472 let mut budget = 16 * 1024;
6473 let status = http_line(&mut reader, &mut budget).await?;
6474 let mut words = status.split_ascii_whitespace();
6475 let version = words.next();
6476 let code = words
6477 .next()
6478 .filter(|word| word.len() == 3)
6479 .and_then(|word| word.parse::<u16>().ok());
6480 if !matches!(version, Some("HTTP/1.1" | "HTTP/1.0"))
6481 || !code.is_some_and(|code| (100..600).contains(&code))
6482 {
6483 return Err(HealthProbeError::bad_answer(format!(
6484 "invalid HTTP status: {status}"
6485 )));
6486 }
6487 let code = code.expect("validated status code");
6488 response_status = status.clone();
6489 let mut length = None;
6490 let mut chunked = false;
6491 loop {
6492 let line = http_line(&mut reader, &mut budget).await?;
6493 if line.is_empty() {
6494 break;
6495 }
6496 if let Some((name, value)) = line.split_once(':') {
6497 if name.eq_ignore_ascii_case("content-length") {
6498 length = Some(value.trim().parse::<u64>().map_err(|_| {
6499 HealthProbeError::bad_answer("invalid HTTP Content-Length")
6500 })?);
6501 } else if name.eq_ignore_ascii_case("transfer-encoding") {
6502 chunked = value.trim().eq_ignore_ascii_case("chunked");
6503 }
6504 }
6505 }
6506 if chunked {
6507 while body.len() < 200 {
6508 let line = http_line(&mut reader, &mut budget).await?;
6509 let size = u64::from_str_radix(line.split(';').next().unwrap_or("").trim(), 16)
6510 .map_err(|_| HealthProbeError::bad_answer("invalid HTTP chunk size"))?;
6511 if size == 0 {
6512 break;
6513 }
6514 let count = size.min((200 - body.len()) as u64) as usize;
6515 let start = body.len();
6516 (&mut reader)
6517 .take(count as u64)
6518 .read_to_end(&mut body)
6519 .await
6520 .map_err(|error| {
6521 HealthProbeError::no_answer(format!("HTTP body read failed: {error}"))
6522 })?;
6523 if body.len() - start != count {
6524 return Err(HealthProbeError::bad_answer("truncated HTTP chunk"));
6525 }
6526 if size > count as u64 || body.len() == 200 {
6527 break;
6528 }
6529 if !http_line(&mut reader, &mut budget).await?.is_empty() {
6530 return Err(HealthProbeError::bad_answer("invalid HTTP chunk delimiter"));
6531 }
6532 }
6533 } else {
6534 reader
6535 .take(length.unwrap_or(200).min(200))
6536 .read_to_end(&mut body)
6537 .await
6538 .map_err(|error| {
6539 HealthProbeError::no_answer(format!("HTTP body read failed: {error}"))
6540 })?;
6541 }
6542 if (200..300).contains(&code) {
6543 Ok(HealthReport::ok())
6544 } else {
6545 Err(HealthProbeError::bad_answer(
6546 "HTTP health endpoint returned non-2xx",
6547 ))
6548 }
6549 };
6550 let mut result = timeout(deadline, probe).await.unwrap_or_else(|_| {
6551 Err(HealthProbeError::no_answer(format!(
6552 "HTTP probe timed out after {deadline:?}"
6553 )))
6554 });
6555 if let Err(error) = &mut result {
6556 if !response_status.is_empty() {
6557 error.message = format!(
6558 "{}; {response_status}: {}",
6559 error.message,
6560 String::from_utf8_lossy(&body)
6561 );
6562 }
6563 }
6564 result
6565}
6566
6567async fn http_line(
6568 reader: &mut tokio::io::BufReader<tokio::net::TcpStream>,
6569 remaining: &mut usize,
6570) -> Result<String, HealthProbeError> {
6571 use tokio::io::{AsyncBufReadExt, AsyncReadExt};
6572 let mut line = Vec::new();
6573 (&mut *reader)
6574 .take(*remaining as u64)
6575 .read_until(b'\n', &mut line)
6576 .await
6577 .map_err(|error| {
6578 HealthProbeError::no_answer(format!("HTTP header read failed: {error}"))
6579 })?;
6580 *remaining -= line.len();
6581 if !line.ends_with(b"\r\n") {
6582 return Err(HealthProbeError::bad_answer(
6583 "HTTP headers are incomplete or exceed 16 KiB",
6584 ));
6585 }
6586 line.truncate(line.len() - 2);
6587 String::from_utf8(line).map_err(|_| HealthProbeError::bad_answer("HTTP header is not UTF-8"))
6588}
6589
6590async fn probe_module_health(
6591 module_id: &str,
6592 runtime: &SupervisorRuntimeConfig,
6593 drain_deadline: Option<Instant>,
6594) -> Result<HealthReport, HealthProbeError> {
6595 probe_module_health_from(module_id, runtime, drain_deadline)
6596 .await
6597 .map(|(report, _)| report)
6598}
6599
6600async fn probe_module_health_from(
6603 module_id: &str,
6604 runtime: &SupervisorRuntimeConfig,
6605 drain_deadline: Option<Instant>,
6606) -> Result<(HealthReport, ConnectionId), HealthProbeError> {
6607 let Some(forwarding) = runtime.forwarding.as_ref() else {
6608 return Err(HealthProbeError::misconfigured(
6609 "supervisor was not configured with a forwarding table",
6610 ));
6611 };
6612 let probe_started_at = Instant::now();
6613 let mut deadline = probe_started_at + runtime.health.deadline;
6614 if let Some(drain_deadline) = drain_deadline {
6615 deadline = deadline.min(drain_deadline);
6616 }
6617 let pending = if drain_deadline.is_some() {
6618 forwarding.begin_drain_health_probe_rpc_for(
6619 module_id,
6620 MODULE_CONTROL_OP_HEALTH_CHECK,
6621 probe_started_at,
6622 deadline,
6623 )
6624 } else {
6625 forwarding.begin_health_probe_rpc_for(
6626 module_id,
6627 MODULE_CONTROL_OP_HEALTH_CHECK,
6628 probe_started_at,
6629 deadline,
6630 )
6631 }
6632 .map_err(|err| {
6633 HealthProbeError::lane_dead(format!("failed to begin health.check RPC: {err}"))
6636 })?;
6637 let probed = pending.endpoint.connection_id;
6638 await_health_probe(forwarding, pending, deadline, runtime.health.deadline)
6639 .await
6640 .map(|report| (report, probed))
6641}
6642
6643async fn probe_endpoint_health(
6650 endpoint: crate::ModuleEndpointId,
6651 runtime: &SupervisorRuntimeConfig,
6652 deadline_cap: Option<Instant>,
6653) -> Result<HealthReport, HealthProbeError> {
6654 let Some(forwarding) = runtime.forwarding.as_ref() else {
6655 return Err(HealthProbeError::misconfigured(
6656 "supervisor was not configured with a forwarding table",
6657 ));
6658 };
6659 let probe_started_at = Instant::now();
6660 let mut deadline = probe_started_at + runtime.health.deadline;
6661 if let Some(cap) = deadline_cap {
6662 deadline = deadline.min(cap);
6663 }
6664 let pending = forwarding
6665 .begin_endpoint_health_probe_rpc_for(
6666 endpoint,
6667 MODULE_CONTROL_OP_HEALTH_CHECK,
6668 probe_started_at,
6669 deadline,
6670 )
6671 .map_err(|err| {
6672 HealthProbeError::lane_dead(format!("failed to begin health.check RPC: {err}"))
6673 })?;
6674 await_health_probe(forwarding, pending, deadline, runtime.health.deadline).await
6675}
6676
6677async fn await_health_probe(
6679 forwarding: &ForwardingTable,
6680 pending: PendingModuleControlRpc,
6681 deadline: Instant,
6682 probe_budget: Duration,
6683) -> Result<HealthReport, HealthProbeError> {
6684 let PendingModuleControlRpc {
6685 endpoint,
6686 module_sink,
6687 negotiated_ver,
6688 corr,
6689 receiver,
6690 } = pending;
6691 let body = serde_json::to_vec(&ModuleControlRequest::HealthCheck {}).map_err(|err| {
6692 HealthProbeError::misconfigured(format!("failed to encode health.check: {err}"))
6693 })?;
6694 let frame = Frame::build_with_version(
6695 negotiated_ver,
6696 FrameType::Request,
6697 control_flags(),
6698 0,
6699 0,
6700 corr,
6701 body,
6702 )
6703 .map_err(|err| {
6704 HealthProbeError::misconfigured(format!("failed to build health.check frame: {err}"))
6705 })?;
6706
6707 match timeout_at(deadline, module_sink.send(frame)).await {
6713 Ok(Ok(())) => {}
6714 Ok(Err(err)) => {
6715 let _ = forwarding.cancel_module_control_rpc(endpoint, corr);
6716 return Err(HealthProbeError::lane_dead(format!(
6719 "failed to send health.check: {err}"
6720 )));
6721 }
6722 Err(_elapsed) => {
6723 let _ = forwarding.cancel_module_control_rpc(endpoint, corr);
6724 return Err(HealthProbeError::no_answer(
6728 "health.check send timed out before enqueue (module egress full)",
6729 ));
6730 }
6731 }
6732
6733 match timeout_at(deadline, receiver).await {
6734 Ok(Ok(ModuleControlRpcOutcome::RawResponse(_))) => Err(HealthProbeError::bad_answer(
6735 "health.check returned an unexpected config response",
6736 )),
6737 Ok(Ok(ModuleControlRpcOutcome::Response(response))) => {
6741 response.health_report().ok_or_else(|| {
6742 HealthProbeError::bad_answer("health.check RPC returned a non-health response")
6743 })
6744 }
6745 Ok(Ok(ModuleControlRpcOutcome::Rejected(body))) => Err(HealthProbeError::bad_answer(
6746 format!("health.check rejected: {}", body.message),
6747 )),
6748 Ok(Ok(ModuleControlRpcOutcome::ModuleGone(message))) => {
6749 Err(HealthProbeError::lane_dead(message))
6750 }
6751 Ok(Ok(ModuleControlRpcOutcome::MalformedResponse(message))) => {
6752 Err(HealthProbeError::bad_answer(message))
6753 }
6754 Ok(Ok(ModuleControlRpcOutcome::UnexpectedOp { expected, actual })) => {
6755 Err(HealthProbeError::bad_answer(format!(
6756 "expected module-control op '{expected}', got '{actual}'"
6757 )))
6758 }
6759 Ok(Ok(ModuleControlRpcOutcome::DeadlineElapsed)) => Err(HealthProbeError::bad_answer(
6763 "module answered health.check after its daemon deadline",
6764 )),
6765 Ok(Err(_)) => Err(HealthProbeError::misconfigured(
6766 "health.check waiter was canceled before the module responded",
6767 )),
6768 Err(_) => {
6769 let _ = forwarding.tombstone_health_probe_rpc(endpoint, corr);
6770 Err(HealthProbeError::no_answer(format!(
6771 "module did not answer health.check within {probe_budget:?}"
6772 )))
6773 }
6774 }
6775}
6776
6777#[allow(clippy::too_many_arguments)]
6781async fn handle_health_report(
6782 spec: &ModuleSpec,
6783 runtime: &SupervisorRuntimeConfig,
6784 registry: &Registry,
6785 process_liveness: &SupervisorProcessLiveness,
6786 snapshot: &SharedSnapshot,
6787 child: &mut Option<SupervisedChild>,
6788 report: HealthReport,
6789 now_ms: u64,
6790 delivery_probe: Option<crate::config_gate::runtime::DeliveryProbe>,
6791 probed: Option<ConnectionId>,
6792) {
6793 let delivery_failed = match (&runtime.config_delivery, delivery_probe) {
6794 (Some(delivery), Some(probe)) => {
6795 delivery
6796 .health_result(
6797 &spec.module_id,
6798 runtime.forwarding.as_deref(),
6799 registry,
6800 probe,
6801 report.status == HealthStatus::Ok,
6802 )
6803 .await
6804 }
6805 _ => false,
6806 };
6807 if let (Some(delivery), Some(spawn)) = (
6808 &runtime.config_delivery,
6809 child
6810 .as_ref()
6811 .and_then(|child| child.config_delivery.as_ref()),
6812 ) {
6813 delivery
6814 .mark_readiness(
6815 &spec.module_id,
6816 runtime.forwarding.as_deref(),
6817 probed,
6818 spawn,
6819 report.status == HealthStatus::Ok,
6820 )
6821 .await;
6822 }
6823 let status = supervisor_health_status(report.status);
6824 let detail = report.detail.clone();
6825 let metrics = truncate_health_metrics(report.metrics);
6826 let _ = update_snapshot(snapshot, Some(&spec.module_id), |state| {
6827 state.health.status = status;
6828 state.last_probe_evidence = None;
6829 state.health.last_probe_ms = Some(now_ms);
6830 state.health.detail = detail.clone();
6831 state.health.metrics = metrics.clone();
6832 state.health.consecutive_failures = 0;
6833 state.failed_retry.observe_health(
6834 report.status == HealthStatus::Ok,
6835 runtime.restart_policy.window,
6836 );
6837 });
6838
6839 if delivery_failed {
6840 return;
6841 }
6842 let action = match report.status {
6843 HealthStatus::Ok => return,
6844 HealthStatus::Degraded => runtime.health.on_degraded,
6845 HealthStatus::Failing => runtime.health.on_failing,
6846 };
6847 apply_l3_health_action(
6848 spec,
6849 runtime,
6850 registry,
6851 process_liveness,
6852 snapshot,
6853 child,
6854 status,
6855 detail.as_deref(),
6856 action,
6857 now_ms,
6858 )
6859 .await;
6860}
6861
6862#[allow(clippy::too_many_arguments)]
6863async fn handle_health_probe_failure(
6864 spec: &ModuleSpec,
6865 runtime: &SupervisorRuntimeConfig,
6866 registry: &Registry,
6867 process_liveness: &SupervisorProcessLiveness,
6868 snapshot: &SharedSnapshot,
6869 child: &mut Option<SupervisedChild>,
6870 err: HealthProbeError,
6871 now_ms: u64,
6872 delivery_probe: Option<crate::config_gate::runtime::DeliveryProbe>,
6873) {
6874 let delivery_failed = match (&runtime.config_delivery, delivery_probe) {
6875 (Some(delivery), Some(probe)) => {
6876 delivery
6877 .health_result(
6878 &spec.module_id,
6879 runtime.forwarding.as_deref(),
6880 registry,
6881 probe,
6882 false,
6883 )
6884 .await
6885 }
6886 _ => false,
6887 };
6888 let threshold = runtime.health.failure_threshold.max(1);
6889 let held = if err.evidence == HealthProbeEvidence::NoAnswer {
6890 runtime.host_health.record_late_module(&spec.module_id)
6891 } else {
6892 false
6893 };
6894 let mut failures = 0;
6895 let detail = if held {
6900 format!("[{}] {err}; host overloaded", err.label())
6901 } else {
6902 format!("[{}] {err}", err.label())
6903 };
6904 let _ = update_snapshot(snapshot, Some(&spec.module_id), |state| {
6905 state.last_probe_evidence = Some(err.evidence);
6906 state
6907 .failed_retry
6908 .observe_health(false, runtime.restart_policy.window);
6909 if held {
6912 state.health.status = SupervisorHealthStatus::Degraded;
6913 } else if state.spawned_protocol.unwrap_or(spec.protocol) == ModuleProtocol::Subc {
6914 state.health.status = SupervisorHealthStatus::Unknown;
6915 }
6916 state.health.last_probe_ms = Some(now_ms);
6917 state.health.consecutive_failures = if held {
6920 0
6921 } else {
6922 state.health.consecutive_failures.saturating_add(1)
6923 };
6924 state.health.detail = Some(detail.clone());
6925 state.health.metrics = None;
6926 failures = state.health.consecutive_failures;
6927 });
6928
6929 if delivery_failed {
6930 return;
6931 }
6932 if held || failures < threshold {
6933 warn!(
6934 module_id = %spec.module_id,
6935 consecutive_failures = failures,
6936 threshold,
6937 evidence = err.label(),
6938 detail = %detail,
6939 "health.check probe failed"
6940 );
6941 return;
6942 }
6943
6944 let _ = update_snapshot(snapshot, Some(&spec.module_id), |state| {
6945 state.state = ModuleState::Unresponsive;
6946 state.health.status = SupervisorHealthStatus::Unresponsive;
6947 });
6948 if runtime.health.critical {
6952 error!(
6953 module_id = %spec.module_id,
6954 status = "unresponsive",
6955 evidence = err.label(),
6956 detail = %detail,
6957 "critical module health alert"
6958 );
6959 } else {
6960 warn!(
6961 module_id = %spec.module_id,
6962 status = "unresponsive",
6963 evidence = err.label(),
6964 detail = %detail,
6965 "module health threshold breached"
6966 );
6967 }
6968 if let Err(err) = health_restart_child(
6969 spec,
6970 runtime,
6971 registry,
6972 process_liveness,
6973 snapshot,
6974 child,
6975 SupervisorHealthStatus::Unresponsive,
6976 Some(&detail),
6977 now_ms,
6978 )
6979 .await
6980 {
6981 error!(module_id = %spec.module_id, error = %err, "health-triggered restart failed");
6982 }
6983}
6984
6985#[allow(clippy::too_many_arguments)]
6986async fn apply_l3_health_action(
6987 spec: &ModuleSpec,
6988 runtime: &SupervisorRuntimeConfig,
6989 registry: &Registry,
6990 process_liveness: &SupervisorProcessLiveness,
6991 snapshot: &SharedSnapshot,
6992 child: &mut Option<SupervisedChild>,
6993 status: SupervisorHealthStatus,
6994 detail: Option<&str>,
6995 action: HealthAction,
6996 now_ms: u64,
6997) {
6998 record_health_action(snapshot, &spec.module_id, action.to_string(), now_ms);
6999 match action {
7000 HealthAction::Report => {
7001 info!(
7002 module_id = %spec.module_id,
7003 status = ?status,
7004 detail,
7005 "module reported non-ok health"
7006 );
7007 }
7008 HealthAction::Alert => {
7009 error!(
7010 module_id = %spec.module_id,
7011 status = ?status,
7012 detail,
7013 "module health alert"
7014 );
7015 }
7016 HealthAction::Restart => {
7017 if let Err(err) = health_restart_child(
7018 spec,
7019 runtime,
7020 registry,
7021 process_liveness,
7022 snapshot,
7023 child,
7024 status,
7025 detail,
7026 now_ms,
7027 )
7028 .await
7029 {
7030 error!(module_id = %spec.module_id, error = %err, "health-triggered restart failed");
7031 }
7032 }
7033 }
7034}
7035
7036#[allow(clippy::too_many_arguments)]
7037async fn health_restart_child(
7038 spec: &ModuleSpec,
7039 runtime: &SupervisorRuntimeConfig,
7040 registry: &Registry,
7041 process_liveness: &SupervisorProcessLiveness,
7042 snapshot: &SharedSnapshot,
7043 child: &mut Option<SupervisedChild>,
7044 status: SupervisorHealthStatus,
7045 detail: Option<&str>,
7046 now_ms: u64,
7047) -> Result<(), SuperviseError> {
7048 budgeted_restart_child(
7049 spec,
7050 runtime,
7051 registry,
7052 process_liveness,
7053 snapshot,
7054 child,
7055 status,
7056 detail,
7057 now_ms,
7058 true,
7059 )
7060 .await
7061}
7062
7063#[allow(clippy::too_many_arguments)]
7064async fn budgeted_restart_child(
7065 spec: &ModuleSpec,
7066 runtime: &SupervisorRuntimeConfig,
7067 registry: &Registry,
7068 process_liveness: &SupervisorProcessLiveness,
7069 snapshot: &SharedSnapshot,
7070 child: &mut Option<SupervisedChild>,
7071 status: SupervisorHealthStatus,
7072 detail: Option<&str>,
7073 now_ms: u64,
7074 health_failure: bool,
7075) -> Result<(), SuperviseError> {
7076 if let Some(error) = runtime
7077 .config_delivery
7078 .as_ref()
7079 .and_then(|delivery| delivery.0.check_module_hold(&spec.module_id).err())
7080 .filter(|error| error.code != "config_delivery_deferred")
7081 {
7082 return Err(config_delivery::gate_hold_error(
7083 spec,
7084 error,
7085 runtime.config_delivery.as_ref(),
7086 ));
7087 }
7088 let recovery_reason = if health_failure {
7089 "health_check"
7090 } else {
7091 "configuration_delivery"
7092 };
7093 let (enabled, schedule) = {
7094 let mut state = lock_snapshot(snapshot)?;
7095 let enabled = state.enabled;
7096 let schedule = if enabled {
7097 state.next_crash_restart(&runtime.restart_policy, Instant::now())
7098 } else {
7099 None
7100 };
7101 (enabled, schedule)
7102 };
7103
7104 if !enabled {
7105 return Err(SuperviseError::Disabled {
7106 module_id: spec.module_id.clone(),
7107 });
7108 }
7109
7110 if schedule.is_none() {
7111 if health_failure {
7112 record_health_action(snapshot, &spec.module_id, "failed".to_string(), now_ms);
7113 }
7114 error!(
7115 module_id = %spec.module_id,
7116 status = ?status,
7117 detail,
7118 max_restarts = runtime.restart_policy.max_restarts,
7119 window_secs = runtime.restart_policy.window.as_secs(),
7120 reason = %runtime.restart_policy.budget_exhausted_detail(),
7121 recovery_reason,
7122 "automatic restart budget exhausted; marking module failed"
7123 );
7124 let stop_notice = begin_forwarding_drain_if_configured(
7125 spec,
7126 runtime,
7127 registry,
7128 snapshot,
7129 Some(true),
7130 RouteCloseReason::Disable,
7131 )
7132 .await?;
7133 update_snapshot(snapshot, Some(&spec.module_id), |state| {
7134 state.drain_disposition_detail = Some(runtime.restart_policy.budget_exhausted_detail());
7135 })?;
7136 let result = drain_optional_child(
7137 &spec.module_id,
7138 spec.protocol,
7139 stop_notice,
7140 registry,
7141 runtime.forwarding.as_deref(),
7142 snapshot,
7143 &runtime.terminal_ring,
7144 &runtime.spawn_events,
7145 child,
7146 runtime.drain_timeout,
7147 ModuleState::Failed,
7148 Some(true),
7149 &runtime.failure_context(true),
7150 )
7151 .await;
7152 update_snapshot(snapshot, Some(&spec.module_id), |state| {
7153 if state.state == ModuleState::Failed && !runtime.child_roster.is_closed() {
7154 state.failed_retry.schedule(&spec.module_id);
7155 }
7156 })?;
7157 result?;
7158 process_liveness.untrack_if_current(&spec.module_id, snapshot);
7159 return Ok(());
7160 }
7161
7162 let schedule = schedule.expect("a health restart must have a crash-restart schedule");
7163 let mut restart_count = 0;
7164 update_snapshot(snapshot, Some(&spec.module_id), |state| {
7165 restart_count = state.crash_restarts.len();
7166 state.state = ModuleState::Unresponsive;
7167 if health_failure {
7168 state.health.status = status;
7169 state.health.last_action = Some(HealthAction::Restart.to_string());
7170 state.health.last_action_ms = Some(now_ms);
7171 }
7172 })?;
7173 warn!(
7174 module_id = %spec.module_id,
7175 status = ?status,
7176 detail,
7177 restart_count,
7178 restart_in_window = schedule.restart_in_window,
7179 delay_ms = schedule.delay.as_millis() as u64,
7180 recovery_reason,
7181 "automatic module restart"
7182 );
7183
7184 let stop_notice = begin_forwarding_drain_if_configured(
7185 spec,
7186 runtime,
7187 registry,
7188 snapshot,
7189 Some(true),
7190 RouteCloseReason::Restart,
7191 )
7192 .await?;
7193 drain_optional_child(
7194 &spec.module_id,
7195 spec.protocol,
7196 stop_notice,
7197 registry,
7198 runtime.forwarding.as_deref(),
7199 snapshot,
7200 &runtime.terminal_ring,
7201 &runtime.spawn_events,
7202 child,
7203 runtime.drain_timeout,
7204 ModuleState::Restarting,
7205 Some(true),
7206 &runtime.failure_context(true),
7207 )
7208 .await?;
7209 schedule_respawn(
7210 runtime,
7211 snapshot,
7212 &spec.module_id,
7213 schedule.delay,
7214 RespawnKind::Spawn,
7215 true,
7216 )
7217}
7218
7219fn cancel_deferred_reload(runtime: &SupervisorRuntimeConfig, module_id: &str, reason: &str) {
7220 if let Some(reply) = runtime
7221 .deferred_reload_reply
7222 .lock()
7223 .unwrap_or_else(|p| p.into_inner())
7224 .take()
7225 {
7226 let _ = reply.send(Err(SuperviseError::ReloadFailed {
7227 module_id: module_id.to_string(),
7228 reason: reason.to_string(),
7229 }));
7230 }
7231}
7232
7233fn park_config_spawn(
7234 spec: &ModuleSpec,
7235 runtime: &SupervisorRuntimeConfig,
7236 snapshot: &SharedSnapshot,
7237 failure: &FailureContext,
7238 admission_refused: Option<String>,
7239) -> Result<(), SuperviseError> {
7240 *runtime
7241 .scheduled_respawn
7242 .lock()
7243 .unwrap_or_else(|poisoned| poisoned.into_inner()) = None;
7244 if let Some(reply) = runtime
7245 .deferred_reload_reply
7246 .lock()
7247 .unwrap_or_else(|poisoned| poisoned.into_inner())
7248 .take()
7249 {
7250 let _ = reply.send(Err(SuperviseError::ConfigHeld {
7251 module_id: spec.module_id.clone(),
7252 code: "config_gated".into(),
7253 detail: "module is held by configuration admission".into(),
7254 }));
7255 }
7256 update_snapshot(snapshot, Some(&spec.module_id), |state| {
7257 state.admission_refused = admission_refused;
7258 state.config_hold_daemon_initiated = failure.daemon_initiated;
7259 state.config_hold_approval = failure.approval.clone();
7260 state.config_hold_parked = true;
7261 state.state = ModuleState::Stopped;
7262 state.respawn_pending = false;
7263 state.failed_retry.pending = None;
7264 state.failed_retry.failed_at = None;
7267 clear_current_process_facts(state);
7268 })
7269}
7270
7271fn try_park_config_attempt(
7272 spec: &ModuleSpec,
7273 runtime: &SupervisorRuntimeConfig,
7274 snapshot: &SharedSnapshot,
7275 failure: &FailureContext,
7276 program_base: Option<&subc_config::Digest>,
7277) -> Result<bool, SuperviseError> {
7278 let result = (|| {
7279 if !lock_snapshot(snapshot)?.enabled {
7280 return Ok(false);
7281 }
7282 #[cfg(test)]
7283 if let Some(delivery) = &runtime.config_delivery {
7284 delivery.0.pause_before_config_park_for_test();
7285 }
7286 let Some(delivery) = &runtime.config_delivery else {
7287 return Ok(false);
7288 };
7289 let parked = delivery
7290 .0
7291 .with_recorded_admission_hold(&spec.module_id, program_base, |refusal| {
7292 park_config_spawn(spec, runtime, snapshot, failure, refusal)
7293 })
7294 .map_err(|error| config_delivery::gate_hold_error(spec, error, Some(delivery)))?;
7295 if let Some(result) = parked {
7296 result?;
7297 Ok(true)
7298 } else {
7299 Ok(false)
7300 }
7301 })();
7302 if let Err(error) = &result {
7303 fail_snapshot(snapshot, Some(&spec.module_id), None, failure);
7307 record_approved_attempt_failure(spec, snapshot, failure, error);
7308 }
7309 result
7310}
7311
7312fn record_approved_attempt_failure(
7313 spec: &ModuleSpec,
7314 snapshot: &SharedSnapshot,
7315 failure: &FailureContext,
7316 error: &SuperviseError,
7317) {
7318 let Some(approval) = &failure.approval else {
7319 return;
7320 };
7321 let _ = update_snapshot(snapshot, Some(&spec.module_id), |state| {
7322 if state.enabled
7323 && state.state == ModuleState::Failed
7324 && state
7325 .approved_start_failure
7326 .as_ref()
7327 .is_none_or(|(identity, _)| identity != approval)
7328 {
7329 state.approved_start_failure = Some((approval.clone(), error.to_string()));
7330 }
7331 });
7332}
7333
7334fn hold_config_spawn(
7335 spec: &ModuleSpec,
7336 runtime: &SupervisorRuntimeConfig,
7337 snapshot: &SharedSnapshot,
7338 failure: &FailureContext,
7339) -> Result<(), SuperviseError> {
7340 if !lock_snapshot(snapshot)?.enabled {
7341 return Ok(());
7342 }
7343 if !try_park_config_attempt(spec, runtime, snapshot, failure, None)? {
7344 defer_config_spawn(spec, runtime, snapshot, RespawnKind::Spawn, failure)?;
7347 }
7348 Ok(())
7349}
7350
7351fn schedule_respawn(
7352 runtime: &SupervisorRuntimeConfig,
7353 snapshot: &SharedSnapshot,
7354 module_id: &str,
7355 delay: Duration,
7356 kind: RespawnKind,
7357 daemon_initiated: bool,
7358) -> Result<(), SuperviseError> {
7359 cancel_deferred_reload(runtime, module_id, "respawn superseded by another restart");
7360 update_snapshot(snapshot, Some(module_id), |state| {
7361 state.respawn_pending = true
7362 })?;
7363 *runtime
7364 .scheduled_respawn
7365 .lock()
7366 .unwrap_or_else(|p| p.into_inner()) = Some(PendingRespawn {
7367 deadline: Instant::now() + delay,
7368 kind,
7369 daemon_initiated,
7370 approval: None,
7371 });
7372 Ok(())
7373}
7374
7375const CONFIG_ADMISSION_RETRY_DELAY: Duration = Duration::from_millis(100);
7376
7377fn defer_config_spawn(
7378 spec: &ModuleSpec,
7379 runtime: &SupervisorRuntimeConfig,
7380 snapshot: &SharedSnapshot,
7381 kind: RespawnKind,
7382 failure: &FailureContext,
7383) -> Result<(), SuperviseError> {
7384 update_snapshot(snapshot, Some(&spec.module_id), |state| {
7385 state.state = ModuleState::Restarting;
7386 state.respawn_pending = true;
7387 clear_current_process_facts(state);
7388 })?;
7389 *runtime
7390 .scheduled_respawn
7391 .lock()
7392 .unwrap_or_else(|p| p.into_inner()) = Some(PendingRespawn {
7393 deadline: Instant::now() + CONFIG_ADMISSION_RETRY_DELAY,
7394 kind,
7395 daemon_initiated: failure.daemon_initiated,
7396 approval: failure.approval.clone(),
7397 });
7398 Ok(())
7399}
7400
7401fn record_health_action(snapshot: &SharedSnapshot, module_id: &str, action: String, now_ms: u64) {
7402 let _ = update_snapshot(snapshot, Some(module_id), |state| {
7403 state.health.last_action = Some(action);
7404 state.health.last_action_ms = Some(now_ms);
7405 });
7406}
7407
7408fn supervisor_health_status(status: HealthStatus) -> SupervisorHealthStatus {
7409 match status {
7410 HealthStatus::Ok => SupervisorHealthStatus::Ok,
7411 HealthStatus::Degraded => SupervisorHealthStatus::Degraded,
7412 HealthStatus::Failing => SupervisorHealthStatus::Failing,
7413 }
7414}
7415
7416fn truncate_health_metrics(metrics: Option<Value>) -> Option<Value> {
7428 let metrics = metrics?;
7429 match serde_json::to_vec(&metrics) {
7430 Ok(encoded) if encoded.len() > MAX_HEALTH_METRICS_BYTES => Some(serde_json::json!({
7431 "truncated": true,
7432 "original_bytes": encoded.len(),
7433 })),
7434 Ok(_) | Err(_) => Some(metrics),
7435 }
7436}
7437
7438fn jittered_health_delay(module_id: &str, probe_index: u64, cadence: Duration) -> Duration {
7444 if cadence.is_zero() {
7445 return Duration::ZERO;
7446 }
7447 let cadence_ms = cadence.as_millis() as u64;
7448 if cadence_ms == 0 {
7464 return cadence;
7465 }
7466 let jitter_span = (cadence_ms / 10).max(1);
7481 let hash = module_id.as_bytes().iter().fold(
7482 probe_index.wrapping_mul(0x9E37_79B9_7F4A_7C15),
7483 |acc, byte| {
7484 acc.wrapping_mul(1099511628211)
7485 .wrapping_add(u64::from(*byte))
7486 },
7487 );
7488 cadence + Duration::from_millis(hash % jitter_span)
7489}
7490
7491#[cfg(test)]
7492mod tests {
7493 use super::*;
7494
7495 #[test]
7496 fn registration_retirement_self_prefixed_owner_is_not_a_prefix_submodule() {
7497 let handle = SupervisorHandle::new();
7498 handle.set_spawn_nonce("prefix:owner", "owner-nonce".into());
7499 handle.set_reserved_prefixes("prefix:owner", &["prefix:".into()]);
7500 let owner = handle
7501 .reserved_hello_verification("prefix:owner", Some("owner-nonce"))
7502 .unwrap();
7503 assert!(owner.nonce_verified);
7504 assert!(
7505 owner.prefix_owner.is_none(),
7506 "an unreserved programmatic owner is not its own prefix child"
7507 );
7508 let child = handle
7509 .reserved_hello_verification("prefix:child", Some("owner-nonce"))
7510 .unwrap();
7511 assert!(child.nonce_verified);
7512 assert_eq!(child.prefix_owner.as_deref(), Some("prefix:owner"));
7513 }
7514
7515 #[test]
7516 fn readding_a_module_clears_its_rescan_removal_tombstone() {
7517 let handle = SupervisorHandle::new();
7518 let module_id = "readded-tombstone";
7519 handle.record_rescan_removal(module_id);
7520 assert!(handle.removal_tombstone_age_ms(module_id).is_some());
7521
7522 handle.apply_identity_configuration(&ModuleSpec {
7523 script: None,
7524 module_id: module_id.to_string(),
7525 program: PathBuf::from("/test/module"),
7526 args: Vec::new(),
7527 env: Vec::new(),
7528 reserved: false,
7529 reserved_prefixes: Vec::new(),
7530 protocol: ModuleProtocol::Subc,
7531 overlap: Default::default(),
7532 carry_containment: false,
7533 });
7534
7535 assert!(
7536 handle.removal_tombstone_age_ms(module_id).is_none(),
7537 "a re-added module must not retain a stale removal tombstone"
7538 );
7539 }
7540
7541 #[derive(Debug, PartialEq, Eq)]
7544 struct OwnerInSpawnWindow {
7545 module_id: String,
7546 configured: bool,
7547 on_roster: bool,
7548 admission_refusal: Option<&'static str>,
7549 }
7550
7551 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
7562 async fn a_new_module_is_configured_before_its_first_process_can_register() {
7563 use crate::scopes::ScopeTable;
7564 use subc_protocol::{error_codes, scope::ScopeSelector, Principal};
7565
7566 let dir = cortexkit_test_support::ScratchDir::new("configured-before-spawn");
7567 let stub = |module_id: &str, program: PathBuf| ModuleSpec {
7568 script: None,
7569 module_id: module_id.to_string(),
7570 program,
7571 args: Vec::new(),
7572 env: ["XDG_DATA_HOME", "XDG_RUNTIME_DIR", "XDG_CONFIG_HOME"]
7573 .into_iter()
7574 .map(|key| (key.to_string(), dir.path().display().to_string()))
7575 .chain([("FAKE_AFT_NEVER_CONNECT".to_string(), "1".to_string())])
7576 .collect(),
7577 reserved: false,
7578 reserved_prefixes: Vec::new(),
7579 protocol: ModuleProtocol::Subc,
7580 overlap: Default::default(),
7581 carry_containment: false,
7582 };
7583 let live = super::terminal_history_tests::fake_aft_stub_path();
7584 let missing = dir.path().join("definitely-missing-module");
7585
7586 let handle = SupervisorHandle::new();
7587 let mut supervisor =
7588 Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default())
7589 .with_handle(handle.clone());
7590 let observed = Arc::new(Mutex::new(Vec::<OwnerInSpawnWindow>::new()));
7591 let hook_handle = handle.clone();
7592 let hook_observed = Arc::clone(&observed);
7593 supervisor.test_after_first_spawn = AfterFirstSpawnHook(Some(Arc::new(move |module_id| {
7594 let configured = hook_handle.is_configured(module_id);
7597 let selector = ScopeSelector::new(
7598 Principal::Reserved {
7599 module_id: module_id.to_string(),
7600 },
7601 "s".to_string(),
7602 Some(1),
7603 );
7604 let carrier = Principal::Reserved {
7605 module_id: "carrier".to_string(),
7606 };
7607 let admission_refusal = match ScopeTable::new(Vec::<String>::new())
7608 .admit(&carrier, module_id, &selector, configured)
7609 {
7610 Ok(_) => None,
7611 Err(refusal) => Some(refusal.code),
7612 };
7613 hook_observed.lock().unwrap().push(OwnerInSpawnWindow {
7614 module_id: module_id.to_string(),
7615 configured,
7616 on_roster: hook_handle.get(module_id).is_some(),
7617 admission_refusal,
7618 });
7619 })));
7620
7621 let plain = supervisor.spawn(stub("plain", live.clone())).unwrap();
7622 let configured = supervisor
7623 .supervise_configured(stub("configured", live.clone()), true)
7624 .unwrap();
7625 let with_health = supervisor
7626 .supervise_configured_with_health(
7627 stub("with-health", live.clone()),
7628 true,
7629 HealthConfig::default(),
7630 None,
7631 RestartPolicy::default(),
7632 )
7633 .unwrap();
7634 let failed = supervisor
7637 .supervise_configured_with_health(
7638 stub("failed-spawn", missing.clone()),
7639 true,
7640 HealthConfig::default(),
7641 None,
7642 RestartPolicy::default(),
7643 )
7644 .unwrap();
7645 assert!(supervisor.spawn(stub("spawn-error", missing)).is_err());
7648
7649 let expected = [
7650 "plain",
7651 "configured",
7652 "with-health",
7653 "failed-spawn",
7654 "spawn-error",
7655 ]
7656 .into_iter()
7657 .map(|module_id| OwnerInSpawnWindow {
7658 module_id: module_id.to_string(),
7659 configured: true,
7660 on_roster: !matches!(module_id, "plain" | "spawn-error"),
7661 admission_refusal: Some(error_codes::SCOPE_NOT_SYNCED),
7662 })
7663 .collect::<Vec<_>>();
7664 assert_eq!(*observed.lock().unwrap(), expected);
7665
7666 for module_id in ["plain", "configured", "with-health", "failed-spawn"] {
7667 assert!(
7668 handle.get(module_id).is_some(),
7669 "{module_id} is on the roster"
7670 );
7671 assert!(
7672 handle.is_configured(module_id),
7673 "{module_id} stays configured"
7674 );
7675 }
7676 assert!(handle.get("spawn-error").is_none());
7677 assert!(
7678 !handle.is_configured("spawn-error"),
7679 "a plain spawn that failed must not leave its module marked configured"
7680 );
7681
7682 handle.retire("failed-spawn");
7684 assert!(!handle.is_configured("failed-spawn"));
7685
7686 for module in [plain, configured, with_health] {
7687 module.stop().await.unwrap();
7688 }
7689 drop(failed);
7690 }
7691
7692 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
7699 async fn a_configured_module_reads_starting_while_its_first_process_already_runs() {
7700 let dir = cortexkit_test_support::ScratchDir::new("starting-while-spawned");
7701 let spec = ModuleSpec {
7702 script: None,
7703 module_id: "spawned-not-admitted".to_string(),
7704 program: super::terminal_history_tests::fake_aft_stub_path(),
7705 args: Vec::new(),
7706 env: ["XDG_DATA_HOME", "XDG_RUNTIME_DIR", "XDG_CONFIG_HOME"]
7707 .into_iter()
7708 .map(|key| (key.to_string(), dir.path().display().to_string()))
7709 .chain([("FAKE_AFT_NEVER_CONNECT".to_string(), "1".to_string())])
7710 .collect(),
7711 reserved: false,
7712 reserved_prefixes: Vec::new(),
7713 protocol: ModuleProtocol::Subc,
7714 overlap: Default::default(),
7715 carry_containment: false,
7716 };
7717 let handle = SupervisorHandle::new();
7718 let mut supervisor =
7719 Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default())
7720 .with_handle(handle.clone());
7721 let observed = Arc::new(Mutex::new(None));
7722 let hook_handle = handle.clone();
7723 let hook_observed = Arc::clone(&observed);
7724 supervisor.test_after_first_spawn = AfterFirstSpawnHook(Some(Arc::new(move |module_id| {
7725 let state = hook_handle
7726 .get(module_id)
7727 .expect("a configured module is on the roster before it is spawned")
7728 .state()
7729 .unwrap();
7730 *hook_observed.lock().unwrap() = Some(state);
7731 })));
7732
7733 let module = supervisor.supervise_configured(spec, true).unwrap();
7734 assert_eq!(
7735 *observed.lock().unwrap(),
7736 Some(ModuleState::Starting),
7737 "between the spawn and its admission the supervisor still reports starting"
7738 );
7739 assert_eq!(module.state().unwrap(), ModuleState::Running);
7740 module.stop().await.unwrap();
7741 }
7742
7743 fn stale_process_snapshot(state: ModuleState, enabled: bool) -> SharedSnapshot {
7744 let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::new(state, enabled)));
7745 update_snapshot(&snapshot, Some("stale-process-facts"), |snapshot| {
7746 snapshot.process_alive = true;
7747 snapshot.pid = Some(41);
7748 snapshot.spawned_at_ms = Some(42);
7749 snapshot.spawned_from = Some(PathBuf::from("/spawned/module"));
7750 snapshot.launch_spec_digest = Some(LaunchSpecDigest([0xA5; 32]));
7751 snapshot.spawned_file_identity = Some(SpawnedFileIdentity {
7752 device: 43,
7753 inode: 44,
7754 #[cfg(windows)]
7755 windows_image: None,
7756 });
7757 })
7758 .unwrap();
7759 snapshot
7760 }
7761
7762 fn assert_snapshot_process_facts_cleared(snapshot: &SharedSnapshot) {
7763 let snapshot = lock_snapshot(snapshot).unwrap();
7764 assert!(!snapshot.process_alive);
7765 assert_eq!(snapshot.pid, None);
7766 assert_eq!(snapshot.spawned_at_ms, None);
7767 assert_eq!(snapshot.spawned_from, None);
7768 assert_eq!(snapshot.launch_spec_digest, None);
7769 assert_eq!(snapshot.spawned_file_identity, None);
7770 }
7771
7772 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
7773 async fn failed_enable_spawn_clears_preexisting_current_process_facts() {
7774 let supervisor =
7775 Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default());
7776 let mut runtime = supervisor.runtime_config();
7777 runtime.test_seed_stale_facts_before_enable_spawn = true;
7778 let snapshot = stale_process_snapshot(ModuleState::Disabled, false);
7779 let mut child = None;
7780 let spec = ModuleSpec {
7781 script: None,
7782 module_id: "failed-enable-clears-facts".to_string(),
7783 program: PathBuf::from("/definitely/missing/failed-enable-module"),
7784 args: Vec::new(),
7785 env: Vec::new(),
7786 reserved: false,
7787 reserved_prefixes: Vec::new(),
7788 protocol: ModuleProtocol::Subc,
7789 overlap: Default::default(),
7790 carry_containment: false,
7791 };
7792
7793 let result = set_child_enabled(
7794 &spec,
7795 &runtime,
7796 &supervisor.registry,
7797 &supervisor.process_liveness,
7798 &snapshot,
7799 &mut child,
7800 true,
7801 &runtime.failure_context(false),
7802 )
7803 .await;
7804
7805 assert!(matches!(result, Err(SuperviseError::Spawn { .. })));
7806 assert_eq!(lock_snapshot(&snapshot).unwrap().state, ModuleState::Failed);
7807 assert_snapshot_process_facts_cleared(&snapshot);
7808 }
7809
7810 #[tokio::test]
7811 async fn start_revives_stranded_restarting_but_not_pending_backoff() {
7812 let supervisor =
7813 Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default());
7814 let runtime = supervisor.runtime_config();
7815 let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::new(
7816 ModuleState::Restarting,
7817 true,
7818 )));
7819 let spec = ModuleSpec {
7820 script: None,
7821 module_id: "start-stranded-restarting".to_string(),
7822 program: super::terminal_history_tests::fake_aft_stub_path(),
7823 args: Vec::new(),
7824 env: vec![("FAKE_AFT_NEVER_CONNECT".to_string(), "1".to_string())],
7825 reserved: false,
7826 reserved_prefixes: Vec::new(),
7827 protocol: ModuleProtocol::None,
7828 overlap: Default::default(),
7829 carry_containment: false,
7830 };
7831 let mut child = None;
7832 lock_snapshot(&snapshot).unwrap().respawn_pending = true;
7833 assert!(!super::set_child_enabled(
7834 &spec,
7835 &runtime,
7836 &Registry::default(),
7837 &supervisor.process_liveness,
7838 &snapshot,
7839 &mut child,
7840 true,
7841 &runtime.failure_context(false),
7842 )
7843 .await
7844 .unwrap());
7845 assert!(child.is_none());
7846 lock_snapshot(&snapshot).unwrap().respawn_pending = false;
7847 assert!(super::set_child_enabled(
7848 &spec,
7849 &runtime,
7850 &Registry::default(),
7851 &supervisor.process_liveness,
7852 &snapshot,
7853 &mut child,
7854 true,
7855 &runtime.failure_context(false),
7856 )
7857 .await
7858 .unwrap());
7859 assert_eq!(
7860 lock_snapshot(&snapshot).unwrap().state,
7861 ModuleState::Running
7862 );
7863 let mut child = child.unwrap();
7864 child.start_kill().unwrap();
7865 child.wait().await.unwrap();
7866 }
7867
7868 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
7869 async fn failed_reload_spawn_clears_current_process_facts() {
7870 let supervisor =
7871 Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default());
7872 let mut runtime = supervisor.runtime_config();
7873 runtime.restart_policy = RestartPolicy::new(0, Duration::ZERO);
7874 let snapshot = stale_process_snapshot(ModuleState::Running, true);
7875 let mut child = None;
7876 let spec = ModuleSpec {
7877 script: None,
7878 module_id: "failed-reload-clears-facts".to_string(),
7879 program: PathBuf::from("/unused/failed-reload-module"),
7880 args: Vec::new(),
7881 env: Vec::new(),
7882 reserved: false,
7883 reserved_prefixes: Vec::new(),
7884 protocol: ModuleProtocol::Subc,
7885 overlap: Default::default(),
7886 carry_containment: false,
7887 };
7888
7889 let result = handle_reload_spawn_failure(
7890 &spec,
7891 &runtime,
7892 &supervisor.process_liveness,
7893 &snapshot,
7894 &mut child,
7895 "forced reload spawn failure".to_string(),
7896 )
7897 .await;
7898
7899 assert!(matches!(result, Err(SuperviseError::ReloadFailed { .. })));
7900 assert_eq!(lock_snapshot(&snapshot).unwrap().state, ModuleState::Failed);
7901 assert_snapshot_process_facts_cleared(&snapshot);
7902 }
7903
7904 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
7905 async fn dropping_a_module_with_an_active_monitor_clears_current_process_facts() {
7906 let supervisor =
7907 Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default());
7908 let snapshot = stale_process_snapshot(ModuleState::Running, true);
7909 let module = supervisor.supervised_module(
7910 ModuleSpec {
7911 script: None,
7912 module_id: "drop-clears-facts".to_string(),
7913 program: PathBuf::from("/unused/drop-module"),
7914 args: Vec::new(),
7915 env: Vec::new(),
7916 reserved: false,
7917 reserved_prefixes: Vec::new(),
7918 protocol: ModuleProtocol::Subc,
7919 overlap: Default::default(),
7920 carry_containment: false,
7921 },
7922 supervisor.runtime_config(),
7923 Arc::clone(&snapshot),
7924 None,
7925 );
7926 assert!(!module
7927 .inner
7928 .monitor
7929 .lock()
7930 .unwrap()
7931 .as_ref()
7932 .unwrap()
7933 .is_finished());
7934
7935 drop(module);
7936
7937 assert_eq!(
7938 lock_snapshot(&snapshot).unwrap().state,
7939 ModuleState::Stopped
7940 );
7941 assert_snapshot_process_facts_cleared(&snapshot);
7942 }
7943
7944 #[cfg(unix)]
7945 #[tokio::test]
7946 async fn rescan_preserves_running_protocol_until_respawn() {
7947 let dir = cortexkit_test_support::ScratchDir::new("rescan-protocol");
7948 let initial = ModuleSpec {
7949 script: None,
7950 module_id: "rescan-protocol".into(),
7951 program: PathBuf::from("/bin/sleep"),
7952 args: vec!["60".into()],
7953 env: ["XDG_DATA_HOME", "XDG_RUNTIME_DIR", "XDG_CONFIG_HOME"]
7954 .into_iter()
7955 .map(|key| (key.into(), dir.path().to_string_lossy().into_owned()))
7956 .collect(),
7957 reserved: false,
7958 reserved_prefixes: vec![],
7959 protocol: ModuleProtocol::None,
7960 overlap: Default::default(),
7961 carry_containment: false,
7962 };
7963 let supervisor =
7964 Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default());
7965 let module = supervisor.spawn(initial.clone()).unwrap();
7966 assert!(module.status().unwrap().live);
7967 let mut next = initial;
7968 next.protocol = ModuleProtocol::Subc;
7969 module
7970 .update_configuration(next.clone(), HealthConfig::default(), None)
7971 .await
7972 .unwrap();
7973 assert!(
7974 module.status().unwrap().live,
7975 "rescan must not require HELLO from the old non-wire process"
7976 );
7977 let runtime = supervisor.runtime_config();
7978 let action = on_child_exit(
7979 &next,
7980 RestartPolicy::default(),
7981 &supervisor.registry,
7982 &module.inner.snapshot,
7983 &runtime.terminal_ring,
7984 &runtime.spawn_events,
7985 &runtime.child_roster,
7986 ExitReport {
7987 kind: ExitKind::Clean,
7988 code: Some(0),
7989 signal: None,
7990 at_ms: unix_ms_now(),
7991 },
7992 true,
7993 )
7994 .await;
7995 assert!(
7996 matches!(action, NextAction::Restart { .. }),
7997 "the old non-wire process's clean exit must restart"
7998 );
7999 module.drain().await.unwrap();
8000 }
8001
8002 #[cfg(unix)]
8003 #[tokio::test]
8004 async fn orphan_identity_matches_path_names_and_shebang_interpreters() {
8005 use std::os::unix::fs::PermissionsExt;
8006 let dir = cortexkit_test_support::ScratchDir::new("spawn-image-identity");
8007 let script = dir.join("module.sh");
8008 std::fs::write(&script, "#!/bin/sh\nwhile :; do sleep 0.1; done\n").unwrap();
8009 std::fs::set_permissions(&script, std::fs::Permissions::from_mode(0o700)).unwrap();
8010 let record_path = dir.join("live-children.json");
8011 let supervisor =
8012 Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default())
8013 .with_live_children_record(&record_path);
8014 for (program, args) in [
8015 (PathBuf::from("sleep"), vec!["60".into()]),
8016 (script, vec![]),
8017 ] {
8018 let spec = ModuleSpec {
8019 script: None,
8020 module_id: "image-identity".into(),
8021 program: program.clone(),
8022 args,
8023 env: ["XDG_DATA_HOME", "XDG_RUNTIME_DIR", "XDG_CONFIG_HOME"]
8024 .into_iter()
8025 .map(|key| (key.into(), dir.to_string_lossy().into_owned()))
8026 .collect(),
8027 reserved: false,
8028 reserved_prefixes: vec![],
8029 protocol: ModuleProtocol::None,
8030 overlap: Default::default(),
8031 carry_containment: false,
8032 };
8033 let module = supervisor.spawn(spec).unwrap();
8034 #[cfg(target_os = "macos")]
8035 {
8036 let deadline = Instant::now() + Duration::from_secs(5);
8039 while crate::live_children::read_record(&record_path)
8040 .unwrap()
8041 .iter()
8042 .all(|entry| entry.executable.is_none())
8043 {
8044 assert!(Instant::now() < deadline, "module image was not confirmed");
8045 tokio::time::sleep(Duration::from_millis(5)).await;
8046 }
8047 }
8048 let entry = crate::live_children::read_record(&record_path)
8049 .unwrap()
8050 .pop()
8051 .unwrap();
8052 let observed = subc_os::Process::open(entry.pid)
8053 .unwrap()
8054 .unwrap()
8055 .observe()
8056 .unwrap();
8057 let verdict = crate::live_children::identity_verdict(&entry, entry.pid, &observed);
8058 module.drain().await.unwrap();
8059 assert_eq!(
8060 verdict,
8061 crate::live_children::IdentityVerdict::Matches,
8062 "program {program:?}: recorded {entry:?}, observed {observed:?}"
8063 );
8064 }
8065 }
8066
8067 #[cfg(unix)]
8068 fn http_fixture(
8069 dir: &std::path::Path,
8070 url: &str,
8071 threshold: u32,
8072 ) -> crate::daemon_config::ConfiguredModule {
8073 let path = dir.join("subc.jsonc");
8074 std::fs::write(&path, serde_json::json!({"version": 1, "modules": {"http-process": {
8075 "program": "/bin/sleep", "args": ["60"], "protocol": "none",
8076 "env": {"XDG_DATA_HOME":dir,"XDG_RUNTIME_DIR":dir,"XDG_CONFIG_HOME":dir},
8077 "health": {"http":url,"cadence_ms":20,"deadline_ms":200,"failure_threshold":threshold},
8078 "restart": {"backoff_ms":1,"max_backoff_ms":1}, "drain_timeout_ms":10
8079 }}}).to_string()).unwrap();
8080 crate::daemon_config::load(&path)
8081 .unwrap()
8082 .unwrap()
8083 .modules
8084 .pop()
8085 .unwrap()
8086 }
8087
8088 #[cfg(unix)]
8089 async fn wait_http_health(module: &SupervisedModule, status: SupervisorHealthStatus) {
8090 timeout(Duration::from_secs(5), async {
8091 loop {
8092 if module.status().unwrap().health.status == status {
8093 break;
8094 }
8095 sleep(Duration::from_millis(5)).await;
8096 }
8097 })
8098 .await
8099 .unwrap_or_else(|_| {
8100 panic!(
8101 "expected {status:?}, got {:?}",
8102 module.status().unwrap().health
8103 )
8104 });
8105 }
8106
8107 #[cfg(unix)]
8108 #[tokio::test]
8109 async fn http_health_status_flips_ok_failing_ok() {
8110 use tokio::io::{AsyncReadExt, AsyncWriteExt};
8111 let dir = cortexkit_test_support::ScratchDir::new("http-status-flips");
8112 let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
8113 let status = Arc::new(std::sync::atomic::AtomicU16::new(200));
8114 let serving_status = status.clone();
8115 let url = format!("http://{}/healthz", listener.local_addr().unwrap());
8116 let server = tokio::spawn(async move {
8117 loop {
8118 let (mut stream, _) = listener.accept().await.unwrap();
8119 let mut request = [0u8; 2048];
8120 let count = stream.read(&mut request).await.unwrap();
8121 assert!(count > 0, "a probe must send an HTTP request");
8122 let code = serving_status.load(std::sync::atomic::Ordering::SeqCst);
8123 let body = if code == 200 {
8124 "ready"
8125 } else {
8126 "{\"status\":\"unavailable\",\"error\":\"scratch failure\"}"
8127 };
8128 let response = format!("HTTP/1.1 {code} scratch\r\nContent-Length: {}\r\nConnection: close\r\n\r\n{body}", body.len());
8129 let _ = stream.write_all(response.as_bytes()).await;
8130 }
8131 });
8132 let configured = http_fixture(&dir, &url, 1000);
8133 let module =
8134 Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default())
8135 .supervise_configured_with_health(
8136 configured.module_spec(),
8137 true,
8138 configured.health,
8139 configured.drain_timeout_ms,
8140 configured.restart,
8141 )
8142 .unwrap();
8143 wait_http_health(&module, SupervisorHealthStatus::Ok).await;
8144 status.store(503, std::sync::atomic::Ordering::SeqCst);
8145 wait_http_health(&module, SupervisorHealthStatus::Failing).await;
8146 assert!(module
8147 .status()
8148 .unwrap()
8149 .health
8150 .detail
8151 .unwrap()
8152 .contains("scratch failure"));
8153 status.store(200, std::sync::atomic::Ordering::SeqCst);
8154 wait_http_health(&module, SupervisorHealthStatus::Ok).await;
8155 assert_eq!(module.status().unwrap().health.consecutive_failures, 0);
8156 let before = module.status().unwrap();
8157 let (spec, mut health) = module.configuration().unwrap();
8158 health.http = None;
8159 module
8160 .update_configuration(spec.clone(), health.clone(), Some(10))
8161 .await
8162 .unwrap();
8163 wait_http_health(&module, SupervisorHealthStatus::Unknown).await;
8164 health.http = Some(url);
8165 module
8166 .update_configuration(spec, health, Some(10))
8167 .await
8168 .unwrap();
8169 wait_http_health(&module, SupervisorHealthStatus::Ok).await;
8170 assert_eq!(
8171 module.status().unwrap().pid,
8172 before.pid,
8173 "changing a probe must apply live, not restart its process"
8174 );
8175 let (spec, mut health) = module.configuration().unwrap();
8176 health.failure_threshold = 2;
8177 module
8178 .update_configuration(spec, health, Some(10))
8179 .await
8180 .unwrap();
8181 status.store(503, std::sync::atomic::Ordering::SeqCst);
8182 timeout(Duration::from_secs(5), async {
8183 while module.status().unwrap().spawn_generation == before.spawn_generation {
8184 sleep(Duration::from_millis(5)).await;
8185 }
8186 })
8187 .await
8188 .expect("sustained HTTP 503 responses must trigger the health restart policy");
8189 module.drain().await.unwrap();
8190 server.abort();
8191 }
8192
8193 #[cfg(unix)]
8194 #[tokio::test]
8195 async fn http_health_replacement_recovers_after_one_restart() {
8196 use tokio::io::{AsyncReadExt, AsyncWriteExt};
8197 let dir = cortexkit_test_support::ScratchDir::new("http-replacement-recovers");
8198 let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
8199 let url = format!("http://{}/healthz", listener.local_addr().unwrap());
8200 let mut configured = http_fixture(&dir, &url, 2);
8201 configured.restart.max_restarts = 2;
8202 let module =
8203 Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default())
8204 .supervise_configured_with_health(
8205 configured.module_spec(),
8206 true,
8207 configured.health,
8208 configured.drain_timeout_ms,
8209 configured.restart,
8210 )
8211 .unwrap();
8212 let first_generation = module.status().unwrap().spawn_generation;
8213 let observed = module.clone();
8214 let (healthy_tx, mut healthy_rx) = mpsc::channel(8);
8215 let server = tokio::spawn(async move {
8216 let mut generation = first_generation;
8217 let mut probes = 0;
8218 loop {
8219 let (mut stream, _) = listener.accept().await.unwrap();
8220 let mut request = [0; 2048];
8221 assert!(stream.read(&mut request).await.unwrap() > 0);
8222 let current = observed.status().unwrap().spawn_generation;
8223 if current != generation {
8224 generation = current;
8225 probes = 0;
8226 }
8227 probes += 1;
8228 let healthy = generation != first_generation && probes > 1;
8231 let code = if healthy { 200 } else { 503 };
8232 let response = format!(
8233 "HTTP/1.1 {code} scratch\r\nContent-Length: 0\r\nConnection: close\r\n\r\n"
8234 );
8235 let _ = stream.write_all(response.as_bytes()).await;
8236 if healthy {
8237 healthy_tx.send(()).await.unwrap();
8238 }
8239 }
8240 });
8241 timeout(Duration::from_secs(5), async {
8242 let mut healthy_probes = 0;
8243 loop {
8244 if module.state().unwrap() == ModuleState::Failed {
8245 break;
8246 }
8247 while healthy_rx.try_recv().is_ok() {
8248 healthy_probes += 1;
8249 }
8250 if healthy_probes >= 3
8251 && module.status().unwrap().health.status == SupervisorHealthStatus::Ok
8252 {
8253 break;
8254 }
8255 sleep(Duration::from_millis(5)).await;
8256 }
8257 })
8258 .await
8259 .expect("the replacement must settle healthy without exhausting its restart budget");
8260 let status = module.status().unwrap();
8261 module.drain().await.unwrap();
8262 server.abort();
8263 assert_eq!(status.state, ModuleState::Running);
8264 assert_eq!(status.health.status, SupervisorHealthStatus::Ok);
8265 assert_eq!(status.health.consecutive_failures, 0);
8266 assert_eq!(status.lifetime_restarts, 1);
8267 assert_eq!(status.restart_count, 1);
8268 }
8269
8270 #[cfg(unix)]
8271 #[tokio::test]
8272 async fn http_health_sustained_failure_exhausts_restart_budget() {
8273 use std::collections::BTreeMap;
8274 use tokio::io::{AsyncReadExt, AsyncWriteExt};
8275 let dir = cortexkit_test_support::ScratchDir::new("http-health-budget");
8276 let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
8277 let url = format!("http://{}/healthz", listener.local_addr().unwrap());
8278 let mut configured = http_fixture(&dir, &url, 2);
8279 configured.restart.max_restarts = 2;
8280 let module =
8281 Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default())
8282 .supervise_configured_with_health(
8283 configured.module_spec(),
8284 true,
8285 configured.health,
8286 configured.drain_timeout_ms,
8287 configured.restart,
8288 )
8289 .unwrap();
8290 let observed = module.clone();
8291 let probes = Arc::new(Mutex::new(BTreeMap::<u64, u32>::new()));
8292 let served = probes.clone();
8293 let server = tokio::spawn(async move {
8294 loop {
8295 let (mut stream, _) = listener.accept().await.unwrap();
8296 let mut request = [0; 2048];
8297 assert!(stream.read(&mut request).await.unwrap() > 0);
8298 *served
8299 .lock()
8300 .unwrap()
8301 .entry(observed.status().unwrap().spawn_generation)
8302 .or_default() += 1;
8303 let _ = stream
8304 .write_all(b"HTTP/1.1 503 Unavailable\r\nContent-Length: 0\r\nConnection: close\r\n\r\n")
8305 .await;
8306 }
8307 });
8308 timeout(Duration::from_secs(5), async {
8309 while module.state().unwrap() != ModuleState::Failed {
8310 sleep(Duration::from_millis(5)).await;
8311 }
8312 })
8313 .await
8314 .expect("sustained failures must still exhaust the health restart budget");
8315 let status = module.status().unwrap();
8316 server.abort();
8317 assert_eq!(status.restart_count, 2);
8318 assert_eq!(status.lifetime_restarts, 2);
8319 assert!(!status.process_alive);
8320 assert_eq!(
8321 probes.lock().unwrap().values().copied().collect::<Vec<_>>(),
8322 vec![2, 2, 2]
8323 );
8324 }
8325
8326 #[cfg(unix)]
8327 #[tokio::test]
8328 async fn spawn_clears_the_previous_process_health() {
8329 let dir = cortexkit_test_support::ScratchDir::new("spawn-fresh-health");
8330 let mut configured = http_fixture(&dir, "http://127.0.0.1:1/healthz", 2);
8331 configured.health.cadence = Duration::from_secs(3600);
8332 let module =
8333 Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default())
8334 .supervise_configured_with_health(
8335 configured.module_spec(),
8336 false,
8337 configured.health,
8338 configured.drain_timeout_ms,
8339 configured.restart,
8340 )
8341 .unwrap();
8342 update_snapshot(&module.inner.snapshot, Some(module.module_id()), |state| {
8343 state.health.status = SupervisorHealthStatus::Unresponsive;
8344 state.health.consecutive_failures = 3;
8345 state.health.detail = Some("old process refused connections".into());
8346 state.health.metrics = Some(serde_json::json!({"old": true}));
8347 state.health.last_probe_ms = Some(123);
8348 state.health.last_action = Some("restart".into());
8349 state.health.last_action_ms = Some(124);
8350 })
8351 .unwrap();
8352 module.set_enabled(true).await.unwrap();
8353 let health = module.status().unwrap().health;
8354 module.drain().await.unwrap();
8355 assert_eq!(health.consecutive_failures, 0);
8356 assert_eq!(health.status, SupervisorHealthStatus::Unknown);
8357 assert_eq!(health.detail, None);
8358 assert_eq!(health.metrics, None);
8359 assert_eq!(health.last_probe_ms, None);
8360 assert_eq!(health.last_action.as_deref(), Some("restart"));
8361 assert_eq!(health.last_action_ms, Some(124));
8362 }
8363
8364 #[cfg(unix)]
8365 #[tokio::test]
8366 async fn http_health_no_listener_restarts_after_consecutive_failures() {
8367 let dir = cortexkit_test_support::ScratchDir::new("http-refused");
8368 let unused = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
8369 let url = format!("http://{}/healthz", unused.local_addr().unwrap());
8370 drop(unused);
8371 let configured = http_fixture(&dir, &url, 2);
8372 let module =
8373 Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default())
8374 .supervise_configured_with_health(
8375 configured.module_spec(),
8376 true,
8377 configured.health,
8378 configured.drain_timeout_ms,
8379 configured.restart,
8380 )
8381 .unwrap();
8382 let before = module.status().unwrap().spawn_generation;
8383 timeout(Duration::from_secs(5), async {
8384 loop {
8385 let status = module.status().unwrap();
8386 if status.spawn_generation > before {
8387 assert!(status.lifetime_restarts > 0);
8388 break;
8389 }
8390 sleep(Duration::from_millis(5)).await;
8391 }
8392 })
8393 .await
8394 .expect("sustained HTTP refusal must trigger the health restart policy");
8395 module.drain().await.unwrap();
8396 }
8397
8398 #[cfg(unix)]
8399 #[tokio::test]
8400 async fn http_health_timeout_honours_deadline() {
8401 let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
8402 let url = format!("http://{}/healthz", listener.local_addr().unwrap());
8403 let server = tokio::spawn(async move {
8404 let _held = listener.accept().await.unwrap();
8405 std::future::pending::<()>().await;
8406 });
8407 let error = timeout(
8408 Duration::from_secs(1),
8409 probe_http_health(&url, Duration::from_millis(10)),
8410 )
8411 .await
8412 .expect("the probe must enforce its own deadline")
8413 .unwrap_err();
8414 server.abort();
8415 assert!(error.to_string().contains("timed out"));
8416 }
8417
8418 #[cfg(unix)]
8419 #[tokio::test]
8420 async fn http_health_localhost_reaches_an_ipv6_only_listener() {
8421 use tokio::io::{AsyncReadExt, AsyncWriteExt};
8422 let listener = tokio::net::TcpListener::bind("[::1]:0").await.unwrap();
8423 let url = format!(
8424 "http://localhost:{}/healthz",
8425 listener.local_addr().unwrap().port()
8426 );
8427 let server = tokio::spawn(async move {
8428 let (mut stream, _) = listener.accept().await.unwrap();
8429 let mut request = [0u8; 2048];
8430 assert!(stream.read(&mut request).await.unwrap() > 0);
8431 stream
8432 .write_all(b"HTTP/1.1 200 OK\r\nContent-Length: 2\r\n\r\nok")
8433 .await
8434 .unwrap();
8435 });
8436 assert_eq!(
8441 probe_http_health(&url, Duration::from_secs(10))
8442 .await
8443 .unwrap()
8444 .status,
8445 HealthStatus::Ok
8446 );
8447 server.await.unwrap();
8448 }
8449
8450 #[cfg(unix)]
8451 #[tokio::test]
8452 async fn http_health_timeout_keeps_partial_status_and_body() {
8453 use tokio::io::{AsyncReadExt, AsyncWriteExt};
8454 let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
8455 let url = format!("http://{}/healthz", listener.local_addr().unwrap());
8456 let server = tokio::spawn(async move {
8457 let (mut stream, _) = listener.accept().await.unwrap();
8458 let mut request = [0u8; 2048];
8459 assert!(stream.read(&mut request).await.unwrap() > 0);
8460 stream
8461 .write_all(
8462 b"HTTP/1.1 503 Unavailable\r\nContent-Length: 100\r\n\r\npartial diagnostic",
8463 )
8464 .await
8465 .unwrap();
8466 std::future::pending::<()>().await;
8467 });
8468 let error = probe_http_health(&url, Duration::from_secs(1))
8469 .await
8470 .unwrap_err()
8471 .to_string();
8472 server.abort();
8473 assert!(
8474 error.contains("timed out")
8475 && error.contains("503 Unavailable")
8476 && error.contains("partial diagnostic"),
8477 "{error}"
8478 );
8479 }
8480
8481 #[cfg(unix)]
8482 #[tokio::test]
8483 async fn http_health_diagnostic_body_is_bounded_to_200_bytes() {
8484 use tokio::io::{AsyncReadExt, AsyncWriteExt};
8485 let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
8486 let url = format!("http://{}/healthz", listener.local_addr().unwrap());
8487 let server = tokio::spawn(async move {
8488 let (mut stream, _) = listener.accept().await.unwrap();
8489 let mut request = [0u8; 2048];
8490 assert!(stream.read(&mut request).await.unwrap() > 0);
8491 let body = format!("{}not-in-diagnostic", "x".repeat(400));
8492 let response = format!(
8493 "HTTP/1.1 503 Unavailable\r\nContent-Length: {}\r\n\r\n{body}",
8494 body.len()
8495 );
8496 stream.write_all(response.as_bytes()).await.unwrap();
8497 });
8498 let error = probe_http_health(&url, Duration::from_secs(1))
8499 .await
8500 .unwrap_err()
8501 .to_string();
8502 server.await.unwrap();
8503 assert!(error.ends_with(&"x".repeat(200)), "{error}");
8504 assert_eq!(error.split(": ").last().unwrap().len(), 200);
8505 assert!(!error.contains("not-in-diagnostic"));
8506 }
8507
8508 #[cfg(unix)]
8509 #[tokio::test]
8510 async fn http_health_real_nats_server_monitoring() {
8511 if std::process::Command::new("nats-server")
8512 .arg("--version")
8513 .env("XDG_DATA_HOME", std::env::temp_dir())
8514 .env("XDG_RUNTIME_DIR", std::env::temp_dir())
8515 .env("XDG_CONFIG_HOME", std::env::temp_dir())
8516 .output()
8517 .is_err()
8518 {
8519 eprintln!(
8520 "skipped: http_health_real_nats_server_monitoring (nats-server is not on PATH)"
8521 );
8522 return;
8523 }
8524 let dir = cortexkit_test_support::ScratchDir::new("nats-http-monitoring");
8525 let monitor = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
8526 let port = monitor.local_addr().unwrap().port();
8527 let client = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
8528 let client_port = client.local_addr().unwrap().port();
8529 let config = dir.join("server.conf");
8530 std::fs::write(&config, format!("listen: 127.0.0.1:{client_port}\nhttp: 127.0.0.1:{port}\njetstream {{ store_dir: \"{}\" }}\n", dir.join("jetstream").display())).unwrap();
8531 let mut configured = http_fixture(&dir, &format!("http://127.0.0.1:{port}/healthz"), 1000);
8532 configured.program = PathBuf::from("nats-server");
8533 configured.args = vec!["-c".into(), config.to_string_lossy().into_owned()];
8534 drop(monitor);
8535 drop(client);
8536 let module =
8537 Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default())
8538 .supervise_configured_with_health(
8539 configured.module_spec(),
8540 true,
8541 configured.health,
8542 configured.drain_timeout_ms,
8543 configured.restart,
8544 )
8545 .unwrap();
8546 wait_http_health(&module, SupervisorHealthStatus::Ok).await;
8547 module.drain().await.unwrap();
8548 }
8549
8550 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
8551 async fn configuration_update_does_not_replace_captured_running_process_facts() {
8552 let supervisor =
8553 Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default());
8554 let snapshot = stale_process_snapshot(ModuleState::Running, true);
8555 let initial = ModuleSpec {
8556 script: None,
8557 module_id: "rescan-preserves-spawn-facts".to_string(),
8558 program: PathBuf::from("/spawned/module"),
8559 args: Vec::new(),
8560 env: Vec::new(),
8561 reserved: false,
8562 reserved_prefixes: Vec::new(),
8563 protocol: ModuleProtocol::Subc,
8564 overlap: Default::default(),
8565 carry_containment: false,
8566 };
8567 let module = supervisor.supervised_module(
8568 initial.clone(),
8569 supervisor.runtime_config(),
8570 snapshot,
8571 None,
8572 );
8573 let before = module.status().unwrap();
8574 let mut replacement = initial;
8575 replacement.program = PathBuf::from("/rescanned/replacement-module");
8576
8577 module
8578 .update_configuration(replacement, HealthConfig::default(), None)
8579 .await
8580 .unwrap();
8581
8582 let after = module.status().unwrap();
8583 assert_eq!(after.pid, before.pid);
8584 assert_eq!(after.spawned_at_ms, before.spawned_at_ms);
8585 assert_eq!(after.spawned_from, before.spawned_from);
8586 drop(module);
8587 }
8588}
8589
8590fn unix_ms_now() -> u64 {
8591 SystemTime::now()
8592 .duration_since(UNIX_EPOCH)
8593 .map(|duration| duration.as_millis().min(u128::from(u64::MAX)) as u64)
8594 .unwrap_or(0)
8595}
8596
8597async fn supervise_loop(
8598 mut spec: ModuleSpec,
8599 mut runtime: SupervisorRuntimeConfig,
8600 registry: Arc<Registry>,
8601 process_liveness: Arc<SupervisorProcessLiveness>,
8602 snapshot: SharedSnapshot,
8603 mut child: Option<SupervisedChild>,
8604 mut commands: mpsc::Receiver<SupervisorCommand>,
8605) {
8606 let mut health_probe = HealthProbeRuntime::default();
8607 let mut registration_changes = match registry.subscribe_module_changes(&spec.module_id) {
8611 Ok(changes) => Some(changes),
8612 Err(err) => {
8613 warn!(module_id = %spec.module_id, error = %err, "health prober could not subscribe to registry");
8616 None
8617 }
8618 };
8619 let mut pending_respawn: Option<PendingRespawn> = None;
8623 let mut requeued: VecDeque<SupervisorCommand> = VecDeque::new();
8626 let mut previous_instance_recheck = tokio::time::interval_at(
8627 Instant::now() + Duration::from_secs(5),
8628 Duration::from_secs(5),
8629 );
8630 previous_instance_recheck.set_missed_tick_behavior(tokio::time::MissedTickBehavior::Skip);
8631 'supervision: loop {
8632 #[cfg(test)]
8633 {
8634 lock_snapshot(&snapshot).unwrap().actor_select = None;
8635 }
8636 #[cfg(target_os = "macos")]
8637 if let Some(active) = child.as_mut() {
8638 active.confirm_privacy_exec().await;
8639 }
8640 if let Some(scheduled) = runtime
8641 .scheduled_respawn
8642 .lock()
8643 .unwrap_or_else(|p| p.into_inner())
8644 .take()
8645 {
8646 runtime.recovery_is_daemon = scheduled.daemon_initiated;
8647 pending_respawn = Some(scheduled);
8648 }
8649 if pending_respawn.is_some() && (child.is_some() || !respawn_still_pending(&snapshot)) {
8650 pending_respawn = None;
8651 cancel_deferred_reload(
8652 &runtime,
8653 &spec.module_id,
8654 "respawn cancelled by a supervisor command",
8655 );
8656 }
8657 if child.is_none()
8658 && pending_respawn.is_none()
8659 && lock_snapshot(&snapshot).is_ok_and(|state| state.waiting_for_previous_pid.is_none())
8660 {
8661 cancel_deferred_reload(
8662 &runtime,
8663 &spec.module_id,
8664 "respawn cancelled before a replacement was spawned",
8665 );
8666 let mut stranded = false;
8667 let _ = update_snapshot(&snapshot, Some(&spec.module_id), |state| {
8668 state.respawn_pending = false;
8669 state.coalesced_restart_pending = false;
8670 if matches!(
8671 state.state,
8672 ModuleState::Restarting
8673 | ModuleState::Starting
8674 | ModuleState::Draining
8675 | ModuleState::Unresponsive
8676 ) {
8677 error!(module_id = %spec.module_id, state = ?state.state,
8678 "supervision operation ended without a child or pending respawn; marking failed so start can retry");
8679 stranded = true;
8680 }
8681 });
8682 if stranded {
8683 fail_snapshot(
8684 &snapshot,
8685 Some(&spec.module_id),
8686 None,
8687 &runtime.failure_context(runtime.recovery_is_daemon),
8688 );
8689 }
8690 }
8691 if child.is_none()
8692 && (runtime.child_roster.is_closed()
8693 || pending_respawn.is_none()
8694 || !lock_snapshot(&snapshot)
8695 .is_ok_and(|state| state.enabled && state.state == ModuleState::Restarting))
8696 {
8697 containment::terminate(&runtime.child_roster.containment).await;
8698 }
8699 if let Some(command) = requeued.pop_front() {
8700 if !handle_supervisor_command(
8701 command,
8702 &mut spec,
8703 &mut runtime,
8704 ®istry,
8705 &process_liveness,
8706 &snapshot,
8707 &mut child,
8708 &mut commands,
8709 &mut requeued,
8710 )
8711 .await
8712 {
8713 containment::terminate(&runtime.child_roster.containment).await;
8714 return;
8715 }
8716 if child.is_some() || !respawn_still_pending(&snapshot) {
8717 pending_respawn = None;
8718 let _ = update_snapshot(&snapshot, Some(&spec.module_id), |state| {
8719 state.respawn_pending = false
8720 });
8721 }
8722 continue;
8723 }
8724 if child.is_some() {
8725 runtime.recovery_is_daemon = true;
8726 #[cfg(test)]
8727 {
8728 lock_snapshot(&snapshot).unwrap().actor_turns += 1;
8729 }
8730 health_probe.refresh_registration(&spec, &runtime, ®istry, &snapshot);
8731 let active = child.as_mut().expect("child checked above");
8732 if active.registered_since_spawn() {
8733 if let Some(delivery) = &runtime.config_delivery {
8734 delivery.queue_registered(
8735 &spec.module_id,
8736 runtime.forwarding.as_ref(),
8737 ®istry,
8738 active,
8739 );
8740 }
8741 }
8742 if runtime.child_roster.is_closed() || !active.awaiting_registration() {
8743 active.registration_deadline = None;
8744 }
8745 let registration_deadline = active.registration_deadline;
8746 let no_checks = runtime.health.http.is_none()
8747 && (running_protocol(&spec, &snapshot) == ModuleProtocol::None
8748 || (health_probe.registered_connection.is_some() && !health_probe.advertised));
8749 let uptime_reset = lock_snapshot(&snapshot).ok().and_then(|mut state| {
8750 state
8751 .failed_retry
8752 .uptime_reset_at(no_checks, runtime.restart_policy.window)
8753 });
8754 let wake_at = match (health_probe.wake_at(), uptime_reset) {
8755 (probe, Some(reset)) => Some(probe.map_or(reset, |probe| probe.min(reset))),
8756 (probe, None) => probe,
8757 };
8758 let wake_at = match (wake_at, registration_deadline) {
8759 (Some(wake), Some(register)) => Some(wake.min(register)),
8760 (wake, register) => wake.or(register),
8761 };
8762 #[cfg(test)]
8763 let wake_after = wake_at.map(|at| at.saturating_duration_since(Instant::now()));
8764 let probe_sleep = supervisor_sleep(wake_at);
8765 tokio::pin!(probe_sleep);
8766 let wire_child = running_protocol(&spec, &snapshot) == ModuleProtocol::Subc;
8771 #[cfg(test)]
8772 {
8773 let registration_pending = wire_child
8774 && registration_changes
8775 .as_ref()
8776 .is_some_and(|changes| changes.has_changed().unwrap());
8777 let mut state = lock_snapshot(&snapshot).unwrap();
8778 state.actor_select = (!registration_pending).then_some(ActorSelectCheckpoint {
8779 generation: state.spawn_generation,
8780 turn: state.actor_turns,
8781 registered_connection: health_probe.registered_connection,
8782 next_probe_at: health_probe.next_probe_at,
8783 wake_after,
8784 });
8785 }
8786 let registration = active
8787 .awaiting_registration()
8788 .then(|| active.registration_spawn.clone())
8789 .flatten();
8790 #[cfg(all(test, target_os = "linux"))]
8791 let registration_before_exit = !runtime.child_roster.admission_event_for_test(
8792 &spec.module_id,
8793 "observe_registration_before_exit",
8794 Some(active.pid),
8795 ) && registration.is_some();
8796 #[cfg(not(all(test, target_os = "linux")))]
8797 let registration_before_exit = false;
8798 #[cfg(all(test, target_os = "linux"))]
8799 let registration_child_is_running = |child: &mut SupervisedChild| {
8800 registration_before_exit || registration_child_is_running(child)
8803 };
8804 let registration_failed = async {
8805 match ®istration {
8806 Some(spawn) => spawn.wait_registration_failure().await,
8807 None => std::future::pending().await,
8808 }
8809 };
8810 let active_child = child.as_mut().expect("child checked above");
8811 tokio::select! {
8812 _ = registration_failed => {
8813 if !active_child.registered_since_spawn() && registration_child_is_running(active_child) {
8814 active_child.registration_deadline = None;
8815 active_child.spawn_failure = Some("registration_channel_closed".into());
8816 if let Err(error) = stop_unregistered_child(active_child).await {
8817 fail_after_wait_error(&spec.module_id, &snapshot, &runtime.terminal_ring,
8818 &runtime.spawn_events, format!("registration_channel_closed: {error}"),
8819 &runtime.failure_context(true));
8820 child = None;
8821 }
8822 }
8823 continue;
8824 }
8825 wait_result = active_child.wait(), if !registration_before_exit => {
8826 if let Ok(mut state) = lock_snapshot(&snapshot) {
8827 state.failed_retry.uptime_reset_at(no_checks, runtime.restart_policy.window);
8828 }
8829 #[cfg(test)]
8830 { lock_snapshot(&snapshot).unwrap().actor_select = None; }
8831 let exit_report = match wait_result {
8840 Ok(status) => classify_reaped_child_exit(&snapshot, active_child, &status),
8841 Err(err) => {
8842 active_child.drain_stderr(&spec.module_id).await;
8843 fail_after_wait_error(
8844 &spec.module_id,
8845 &snapshot,
8846 &runtime.terminal_ring,
8847 &runtime.spawn_events,
8848 format!("failed to wait for supervised module: {err}"),
8849 &runtime.failure_context(true),
8850 );
8851 untrack_if_registration_released(
8852 &process_liveness,
8853 ®istry,
8854 &spec.module_id,
8855 &snapshot,
8856 );
8857 error!(module_id = %spec.module_id, error = %err, "failed to wait for supervised module");
8858 child = None;
8859 continue;
8860 }
8861 };
8862 active_child.drain_stderr(&spec.module_id).await;
8863
8864 let delivery=runtime.config_delivery.clone();
8865 let exited_spawn=active_child.spawned_at;
8866 let spawn=active_child.config_delivery.clone();
8867 let refusal=active_child.spawn_failure.as_deref().filter(|detail|detail.starts_with("program_refused:") && !active_child.live_program_binding.as_ref().is_some_and(|binding|binding.is_refused())).zip(active_child.admission_base.clone()).map(|(detail,base)|(base,subc_config::raw_digest(detail.as_bytes())));
8868 let module_id=spec.module_id.clone();
8869 let observed=async move {
8873 if let Some(delivery)=delivery {
8874 delivery.0.on_blocking(move |owner| {
8875 if let Some(spawn)=spawn { owner.delivery_child_exited(&spawn)?; }
8876 if let Some((base,identity))=refusal {owner.record_program_refusal(&module_id,&base,identity)?;}
8877 Ok(())
8878 }).await
8879 } else {Ok(())}
8880 };
8881 tokio::pin!(observed);
8882 loop {
8883 tokio::select! {
8884 result=&mut observed=>{
8885 if let Err(error)=result {warn!(module_id=%spec.module_id,%error,"configuration exit observation failed");}
8886 break;
8887 }
8888 command=commands.recv()=>{
8889 let Some(command)=command else {containment::terminate(&runtime.child_roster.containment).await;return;};
8890 let deferred=||SuperviseError::ConfigDeliveryDeferred {module_id:spec.module_id.clone()};
8891 let command=match command {
8892 SupervisorCommand::Restart {reply,..}|SupervisorCommand::Reload {reply,..}=>{let _=reply.send(Err(deferred()));continue;}
8893 SupervisorCommand::Swap {reply,..}=>{let _=reply.send(Err(deferred()));continue;}
8894 command @ (SupervisorCommand::ConfigApplyRestart {..}|SupervisorCommand::ConfigPolicyProbe {..}|SupervisorCommand::ConfigDeliveryFailed {..})=>{requeued.push_back(command);continue;}
8895 command=>command,
8896 };
8897 if !handle_supervisor_command(command,&mut spec,&mut runtime,®istry,&process_liveness,&snapshot,&mut child,&mut commands,&mut requeued).await {
8898 containment::terminate(&runtime.child_roster.containment).await;return;
8899 }
8900 if child.as_ref().is_none_or(|current|current.spawned_at!=exited_spawn) {continue 'supervision;}
8901 }
8902 }
8903 }
8904 let active_child=child.as_mut().expect("exit observation retained its child");
8905 let admission_kill = admission_exit::is_admission_kill(active_child.admission_kill_sent, &exit_report);
8909 let deferred_spawn=admission_kill && active_child.spawn_failure.as_deref().is_some_and(|detail|detail.starts_with("config_delivery_deferred:"));
8910 let refused_program = active_child.live_program_binding.as_ref().is_some_and(|binding| binding.is_refused());
8911 let exit_held = !deferred_spawn && (refused_program || runtime.config_delivery.as_ref().is_some_and(|delivery| delivery.0.exit_is_held(&spec.module_id)));
8912 let parked = exit_held && (!refused_program || active_child.admission_base.is_some())
8913 && try_park_config_attempt(&spec, &runtime, &snapshot, &runtime.failure_context(true),
8914 if refused_program { active_child.admission_base.as_ref() } else { None })
8915 .unwrap_or_else(|error| { warn!(module_id=%spec.module_id, %error, "exit hold could not be recorded; using ordinary exit accounting"); false });
8916 let next = if deferred_spawn {
8917 record_terminal_with_detail(&spec.module_id,&runtime.terminal_ring,&runtime.spawn_events,&exit_report,TerminalDisposition::Restarting,active_child.spawn_failure.clone());
8918 let _=update_snapshot(&snapshot,Some(&spec.module_id),|state| {
8919 state.state=ModuleState::Restarting;
8920 clear_current_process_facts(state);
8921 state.last_exit=Some(exit_report.clone());
8922 });
8923 NextAction::Restart {schedule:None}
8924 } else if parked {
8925 record_terminal_with_detail(&spec.module_id, &runtime.terminal_ring, &runtime.spawn_events, &exit_report, TerminalDisposition::Stopped, active_child.spawn_failure.clone().or_else(|| Some("config_gated".into())));
8926 NextAction::Stop { registration_released: registry.get_module(&spec.module_id).ok().flatten().is_none() }
8927 } else { on_child_exit(
8928 &spec,
8929 runtime.restart_policy,
8930 ®istry,
8931 &snapshot,
8932 &runtime.terminal_ring,
8933 &runtime.spawn_events,
8934 &runtime.child_roster,
8935 exit_report,
8936 true,
8937 ).await };
8938 runtime.start_requester = None;
8939 runtime.pending_activity_restart = false;
8940 let carry = matches!(next, NextAction::Restart { .. })
8943 && active_child.spawn_failure.is_none()
8944 && !runtime.child_roster.is_closed();
8945 if let Err(error) = active_child.finish_exit(carry).await {
8946 containment::terminate(&runtime.child_roster.containment).await;
8947 fail_after_wait_error(&spec.module_id, &snapshot, &runtime.terminal_ring,
8948 &runtime.spawn_events, format!("failed to reap classified exit: {error}"),
8949 &runtime.failure_context(true));
8950 child = None;
8951 continue;
8952 }
8953 match next {
8954 NextAction::Stop { registration_released } => {
8955 if registration_released {
8956 process_liveness.untrack_if_current(&spec.module_id, &snapshot);
8957 }
8958 child = None;
8959 }
8960 NextAction::Restart { schedule } => {
8961 let delay = if deferred_spawn {CONFIG_ADMISSION_RETRY_DELAY} else {schedule.map_or(
8962 runtime.restart_policy.delay_for_restart(0),
8963 |schedule| schedule.delay,
8964 )};
8965 if let Some(schedule) = schedule {
8966 log_crash_respawn(&spec.module_id, schedule);
8967 }
8968 child = None;
8976 pending_respawn = Some(PendingRespawn { deadline: Instant::now() + delay, kind: RespawnKind::Spawn, daemon_initiated: true, approval: None });
8977 let _ = update_snapshot(&snapshot, Some(&spec.module_id), |state| state.respawn_pending = true);
8978 }
8979 }
8980 }
8981 command = commands.recv() => {
8982 #[cfg(test)]
8983 { lock_snapshot(&snapshot).unwrap().actor_select = None; }
8984 let Some(command) = command else {
8985 containment::terminate(&runtime.child_roster.containment).await;
8986 return;
8987 };
8988 if !handle_supervisor_command(
8989 command,
8990 &mut spec,
8991 &mut runtime,
8992 ®istry,
8993 &process_liveness,
8994 &snapshot,
8995 &mut child,
8996 &mut commands,
8997 &mut requeued,
8998 ).await {
8999 containment::terminate(&runtime.child_roster.containment).await;
9000 return;
9001 }
9002 }
9003 _ = async {
9004 match registration_changes.as_mut() {
9005 Some(changes) => { let _ = changes.changed().await; }
9006 None => std::future::pending().await,
9007 }
9008 }, if wire_child => {
9009 #[cfg(test)]
9010 { lock_snapshot(&snapshot).unwrap().actor_select = None; }
9011 }
9012 _ = &mut probe_sleep => {
9013 #[cfg(test)]
9014 { lock_snapshot(&snapshot).unwrap().actor_select = None; }
9015 if registration_deadline.is_some_and(|deadline| Instant::now() >= deadline) {
9016 active_child.registration_deadline = None;
9019 if !runtime.child_roster.is_closed()
9020 && !active_child.registered_since_spawn()
9021 && registration_child_is_running(active_child)
9022 {
9023 warn!(module_id = %spec.module_id, pid = active_child.pid,
9024 wait_secs = runtime.register_timeout.as_secs(),
9025 "supervised module registration_timeout: first HELLO deadline expired; stopping child");
9026 active_child.spawn_failure = Some("registration_timeout".into());
9027 if let Err(error) = stop_unregistered_child(active_child).await {
9028 error!(module_id = %spec.module_id, %error, "failed to stop unregistered child");
9029 fail_after_wait_error(&spec.module_id, &snapshot, &runtime.terminal_ring,
9030 &runtime.spawn_events, format!("registration_timeout: {error}"),
9031 &runtime.failure_context(true));
9032 child = None;
9033 }
9034 continue;
9037 }
9038 }
9039 if health_probe.due() {
9040 run_health_probe_cycle(
9041 &spec,
9042 &runtime,
9043 ®istry,
9044 &process_liveness,
9045 &snapshot,
9046 &mut child,
9047 ).await;
9048 if child.is_none() {
9049 runtime.start_requester = None;
9050 runtime.pending_activity_restart = false;
9051 }
9052 if child.is_some() {
9053 health_probe.schedule_next(&spec, runtime.health.cadence);
9054 }
9055 }
9056 }
9057 }
9058 } else if let Some(pending) = pending_respawn.clone() {
9059 tokio::select! {
9060 _ = runtime.child_roster.closed() => {
9061 pending_respawn = None;
9062 let _ = update_snapshot(&snapshot, Some(&spec.module_id), |state| {
9063 state.state = ModuleState::Stopped;
9064 state.respawn_pending = false;
9065 });
9066 containment::terminate(&runtime.child_roster.containment).await;
9067 }
9068 _ = async {
9069 sleep_until(pending.deadline).await;
9070 if let Some(delivery) = &runtime.config_delivery {
9071 delivery.unfenced(&spec.module_id).await;
9072 }
9073 } => {
9074 pending_respawn = None;
9075 let failure = pending.failure_context(&runtime);
9076 let _ = update_snapshot(&snapshot, Some(&spec.module_id), |state| state.respawn_pending = false);
9077 if !respawn_still_pending(&snapshot) {
9081 continue;
9082 }
9083 if runtime.child_roster.is_closed() {
9088 let _ = update_snapshot(&snapshot, Some(&spec.module_id), |state| {
9089 state.state = ModuleState::Stopped;
9090 });
9091 debug!(module_id = %spec.module_id, "crash respawn cancelled by daemon shutdown");
9092 continue;
9093 }
9094 if let Err(err) = release_dead_registration(
9095 ®istry,
9096 runtime.forwarding.as_deref(),
9097 &snapshot,
9098 &spec.module_id,
9099 &failure,
9100 ).await {
9101 record_approved_attempt_failure(&spec, &snapshot, &failure, &err);
9102 error!(module_id = %spec.module_id, error = %err, "registration did not release before restart");
9103 continue;
9104 }
9105
9106 if matches!(pending.kind, RespawnKind::Reload) {
9107 let reply = runtime.deferred_reload_reply.lock().unwrap_or_else(|p| p.into_inner()).take();
9108 let result = finish_reload_child(&spec, &runtime, ®istry, &process_liveness, &snapshot, &mut child).await;
9109 if matches!(result, Err(SuperviseError::ConfigDeliveryDeferred { .. })) {
9110 *runtime.deferred_reload_reply.lock().unwrap_or_else(|p| p.into_inner()) = reply;
9111 continue;
9112 }
9113 if result.is_ok() && runtime.pending_activity_restart {
9114 runtime.record_activity_transition("subc.module_restarted", &spec.module_id, runtime.start_requester.clone());
9115 }
9116 runtime.pending_activity_restart = false;
9117 if let Some(reply) = reply { let _ = reply.send(result); }
9118 continue;
9119 }
9120 process_liveness.track(spec.module_id.clone(), Arc::clone(&snapshot));
9121 match spawn_and_mark_running_async(&spec, &runtime, &snapshot).await {
9122 Ok(next_child) => {
9123 child = Some(next_child);
9124 if runtime.pending_activity_restart {
9125 runtime.record_activity_transition("subc.module_restarted", &spec.module_id, runtime.start_requester.clone());
9126 runtime.pending_activity_restart = false;
9127 }
9128 debug!(module_id = %spec.module_id, "supervised module restarted after crash");
9129 }
9130 Err(SuperviseError::ConfigDeliveryDeferred { .. }) => {
9131 pending_respawn = Some(PendingRespawn { deadline:Instant::now()+CONFIG_ADMISSION_RETRY_DELAY, ..pending });
9132 let _ = update_snapshot(&snapshot, Some(&spec.module_id), |state| state.respawn_pending = true);
9133 }
9134 Err(SuperviseError::ConfigHeld { .. }) => {
9135 let _ = hold_config_spawn(&spec, &runtime, &snapshot, &failure);
9136 runtime.pending_activity_restart = false;
9137 process_liveness.untrack_if_current(&spec.module_id, &snapshot);
9138 }
9139 Err(err) => {
9140 fail_snapshot(&snapshot, Some(&spec.module_id), None, &failure);
9141 record_approved_attempt_failure(&spec, &snapshot, &failure, &err);
9142 process_liveness.untrack_if_current(&spec.module_id, &snapshot);
9143 error!(module_id = %spec.module_id, error = %err, "failed to restart supervised module");
9144 }
9145 }
9146 }
9147 command = commands.recv() => {
9148 let Some(command) = command else {
9149 containment::terminate(&runtime.child_roster.containment).await;
9150 return;
9151 };
9152 if !handle_supervisor_command(
9153 command,
9154 &mut spec,
9155 &mut runtime,
9156 ®istry,
9157 &process_liveness,
9158 &snapshot,
9159 &mut child,
9160 &mut commands,
9161 &mut requeued,
9162 ).await {
9163 containment::terminate(&runtime.child_roster.containment).await;
9164 return;
9165 }
9166 if child.is_some() || !respawn_still_pending(&snapshot) {
9171 pending_respawn = None;
9172 let _ = update_snapshot(&snapshot, Some(&spec.module_id), |state| state.respawn_pending = false);
9173 }
9174 }
9175 }
9176 } else {
9177 let slow_retry = lock_snapshot(&snapshot).ok().and_then(|state| {
9178 (state.enabled && state.state == ModuleState::Failed)
9179 .then_some(state.failed_retry.pending)
9180 .flatten()
9181 });
9182 let command = if lock_snapshot(&snapshot)
9183 .is_ok_and(|state| state.waiting_for_previous_pid.is_some())
9184 {
9185 tokio::select! {
9186 command = commands.recv() => command,
9187 _ = previous_instance_recheck.tick() => {
9188 if runtime.child_roster.previous_instance_pid(&spec.module_id).is_none() {
9189 let enabled = lock_snapshot(&snapshot).is_ok_and(|state| state.enabled);
9190 if enabled && !runtime.child_roster.is_closed() {
9191 process_liveness.track(spec.module_id.clone(), Arc::clone(&snapshot));
9192 match spawn_and_mark_running_async(&spec, &runtime, &snapshot).await {
9193 Ok(next_child) => child = Some(next_child),
9194 Err(SuperviseError::ConfigDeliveryDeferred { .. }) => {
9195 let _ = defer_config_spawn(&spec, &runtime, &snapshot, RespawnKind::Spawn, &runtime.failure_context(true));
9196 }
9197 Err(SuperviseError::ConfigHeld { .. }) => {
9198 let _ = hold_config_spawn(&spec, &runtime, &snapshot, &runtime.failure_context(true));
9199 }
9200 Err(error) => {
9201 fail_snapshot(&snapshot, Some(&spec.module_id), None, &runtime.failure_context(true));
9202 error!(module_id = %spec.module_id, %error, "failed to start module after its previous instance exited");
9203 }
9204 }
9205 } else {
9206 let _ = update_snapshot(&snapshot, Some(&spec.module_id), |state| {
9207 state.waiting_for_previous_pid = None;
9208 state.waiting_for_previous_detail = None;
9209 });
9210 }
9211 }
9212 continue;
9213 }
9214 }
9215 } else {
9216 tokio::select! {
9217 biased;
9218 command = commands.recv() => command,
9219 _ = async {
9220 match slow_retry {
9221 Some(retry) => sleep_until(retry.deadline).await,
9222 None => std::future::pending().await,
9223 }
9224 } => {
9225 let retry = slow_retry.expect("only a scheduled slow retry can wake");
9226 let _ = update_snapshot(&snapshot, Some(&spec.module_id), |state| {
9227 state.failed_retry.pending = None;
9228 });
9229 if runtime.child_roster.is_closed() {
9230 continue;
9231 }
9232 info!(module_id = %spec.module_id, attempt = retry.attempt,
9233 delay_secs = retry.delay.as_secs(), "starting failed module retry");
9234 if let Err(error) = set_child_enabled(
9235 &spec, &runtime, ®istry, &process_liveness, &snapshot, &mut child, true,
9236 &runtime.failure_context(true),
9237 ).await {
9238 error!(module_id = %spec.module_id, %error, "failed module retry could not start");
9239 }
9240 continue;
9241 }
9242 }
9243 };
9244 let Some(command) = command else {
9245 containment::terminate(&runtime.child_roster.containment).await;
9246 return;
9247 };
9248 if !handle_supervisor_command(
9249 command,
9250 &mut spec,
9251 &mut runtime,
9252 ®istry,
9253 &process_liveness,
9254 &snapshot,
9255 &mut child,
9256 &mut commands,
9257 &mut requeued,
9258 )
9259 .await
9260 {
9261 containment::terminate(&runtime.child_roster.containment).await;
9262 return;
9263 }
9264 }
9265 }
9266}
9267
9268async fn supervisor_sleep(wake_at: Option<Instant>) {
9271 match wake_at {
9272 Some(deadline) => sleep_until(deadline).await,
9273 None => std::future::pending().await,
9274 }
9275}
9276
9277fn log_crash_respawn(module_id: &str, schedule: CrashRestartSchedule) {
9278 info!(
9279 module_id,
9280 restart_in_window = schedule.restart_in_window,
9281 delay_ms = schedule.delay.as_millis() as u64,
9282 "respawning after crash"
9283 );
9284}
9285
9286fn respawn_still_pending(snapshot: &SharedSnapshot) -> bool {
9292 matches!(
9293 lock_snapshot(snapshot),
9294 Ok(state) if state.enabled && state.state == ModuleState::Restarting
9295 )
9296}
9297
9298enum NextAction {
9299 Stop {
9300 registration_released: bool,
9301 },
9302 Restart {
9303 schedule: Option<CrashRestartSchedule>,
9304 },
9305}
9306
9307#[allow(clippy::too_many_arguments)]
9308async fn handle_supervisor_command(
9309 command: SupervisorCommand,
9310 spec: &mut ModuleSpec,
9311 runtime: &mut SupervisorRuntimeConfig,
9312 registry: &Arc<Registry>,
9313 process_liveness: &SupervisorProcessLiveness,
9314 snapshot: &SharedSnapshot,
9315 child: &mut Option<SupervisedChild>,
9316 commands: &mut mpsc::Receiver<SupervisorCommand>,
9317 requeued: &mut VecDeque<SupervisorCommand>,
9318) -> bool {
9319 if let SupervisorCommand::SetEnabled {
9320 enabled: true,
9321 reset_budget: false,
9322 reply,
9323 ..
9324 } = command
9325 {
9326 let parked = lock_snapshot(snapshot).ok().and_then(|state| {
9327 (state.config_hold_parked
9328 && state.enabled
9329 && !state.process_alive
9330 && child.is_none()
9331 && state.state == ModuleState::Stopped)
9332 .then(|| {
9333 (
9334 state.config_hold_daemon_initiated,
9335 state.config_hold_approval.clone(),
9336 )
9337 })
9338 });
9339 let result = if let Some((daemon_initiated, approval)) = parked {
9343 runtime.recovery_is_daemon = daemon_initiated;
9344 let failure = FailureContext {
9345 approval,
9346 ..runtime.failure_context(daemon_initiated)
9347 };
9348 let result = set_child_enabled_with_budget_policy(
9349 spec,
9350 runtime,
9351 registry,
9352 process_liveness,
9353 snapshot,
9354 child,
9355 true,
9356 &failure,
9357 false,
9358 )
9359 .await;
9360 if let Err(error) = &result {
9361 record_approved_attempt_failure(spec, snapshot, &failure, error);
9362 }
9363 result
9364 } else {
9365 Ok(false)
9366 };
9367 let _ = reply.send(result);
9368 return true;
9369 }
9370 let blocked = runtime
9371 .config_delivery
9372 .as_ref()
9373 .and_then(|delivery| delivery.0.check_module_hold(&spec.module_id).err())
9374 .filter(|error| error.code != "config_delivery_deferred")
9375 .map(|error| {
9376 config_delivery::gate_hold_error(spec, error, runtime.config_delivery.as_ref())
9377 })
9378 .or_else(|| recovery_block_error(spec, &runtime.child_roster))
9379 .or_else(|| {
9380 runtime
9381 .child_roster
9382 .previous_instance(&spec.module_id)
9383 .map(|(pid, reason)| previous_instance_error(spec, pid, reason))
9384 });
9385 let command = if let Some(error) = blocked {
9386 match command {
9387 SupervisorCommand::Restart { reply, .. } | SupervisorCommand::Reload { reply, .. } => {
9388 let _ = reply.send(Err(error));
9389 return true;
9390 }
9391 SupervisorCommand::Swap { reply, .. } => {
9392 let _ = reply.send(Err(error));
9393 return true;
9394 }
9395 SupervisorCommand::ConfigApplyRestart { .. } => return true,
9396 SupervisorCommand::SetEnabled {
9397 enabled: true,
9398 requester,
9399 reply,
9400 ..
9401 } => {
9402 if matches!(error, SuperviseError::ConfigHeld { .. })
9403 && lock_snapshot(snapshot).is_ok_and(|state| !state.enabled)
9404 && runtime
9405 .config_delivery
9406 .as_ref()
9407 .is_some_and(|delivery| delivery.0.can_stage_unstamp_enable())
9408 {
9409 let mut changed = false;
9413 let result = update_snapshot(snapshot, Some(&spec.module_id), |state| {
9414 changed = !state.enabled;
9415 state.enabled = true;
9416 if child.is_none() && state.waiting_for_previous_pid.is_none() {
9417 state.state = ModuleState::Starting;
9418 state.failed_retry.cancel();
9419 }
9420 })
9421 .and_then(|_| {
9422 if child.is_none()
9423 && lock_snapshot(snapshot)?.waiting_for_previous_pid.is_none()
9424 {
9425 hold_config_spawn(
9426 spec,
9427 runtime,
9428 snapshot,
9429 &runtime.failure_context(false),
9430 )?;
9431 }
9432 Ok(changed)
9433 });
9434 if result.is_ok() && changed {
9435 runtime.start_requester = requester;
9436 }
9437 let _ = reply.send(result);
9438 return true;
9439 }
9440 let _ = reply.send(Err(error));
9441 return true;
9442 }
9443 other => other,
9444 }
9445 } else {
9446 command
9447 };
9448 if matches!(
9451 command,
9452 SupervisorCommand::Drain { .. }
9453 | SupervisorCommand::Retire { .. }
9454 | SupervisorCommand::SetEnabled { .. }
9455 | SupervisorCommand::Restart { .. }
9456 ) {
9457 let _ = update_snapshot(snapshot, Some(&spec.module_id), |state| {
9458 state.failed_retry.cancel();
9459 });
9460 }
9461 match command {
9462 SupervisorCommand::ApprovedConfiguration { approved, reply } => {
9463 let result = async {
9464 let next = approved.module_entry(&spec.module_id).map_err(|error| {
9465 SuperviseError::InvalidSpec { reason: error.to_string() }
9466 })?;
9467 let retry_approved_failure = {
9468 let mut state = lock_snapshot(snapshot)?;
9469 if let Some((generation, detail)) = &state.approved_start_failure {
9470 if *generation == approved.identity() && next.enabled {
9471 return Err(SuperviseError::Spawn {
9472 program: spec.program.clone(),
9473 source: std::io::Error::other(detail.clone()),
9474 cgroup_path: None,
9475 });
9476 }
9477 }
9478 let retry = state.approved_start_failure.is_some();
9479 state.approved_start_failure = None;
9480 retry
9481 };
9482 stage_supervisor_configuration(
9483 spec, runtime, snapshot, next.module_spec(), next.health,
9484 next.drain_timeout_ms,
9485 );
9486 runtime.restart_policy = next.restart;
9487 runtime.register_timeout = next.register_timeout;
9488 if let Some(active) = child.as_mut()
9489 .filter(|child| child.registration_deadline.is_some())
9490 {
9491 active.registration_deadline = Some(active.spawned_at + next.register_timeout);
9492 }
9493 let enabled = lock_snapshot(snapshot)?.enabled;
9494 let retry_gate_stopped = lock_snapshot(snapshot)?
9501 .needs_approved_gate_restart(&approved.identity());
9502 let retry_failed = enabled && child.is_none()
9503 && ((lock_snapshot(snapshot)?.state == ModuleState::Failed
9504 && (retry_approved_failure || approved.changes_entry(&spec.module_id)))
9505 || retry_gate_stopped);
9506 if retry_failed {
9507 update_snapshot(snapshot, Some(&spec.module_id), |state| {
9510 state.failed_retry.cancel();
9511 if retry_gate_stopped {
9512 state.stop_reason = None;
9513 state.config_exit_failed_approval = None;
9514 }
9515 })?;
9516 }
9517 if enabled != next.enabled || retry_failed {
9518 let failure = FailureContext {
9519 approval: next.enabled.then(|| approved.identity()),
9520 ..runtime.failure_context(false)
9521 };
9522 let held = next.enabled && runtime.config_delivery.as_ref()
9523 .is_some_and(|delivery| delivery.0.has_recorded_admission_hold(&spec.module_id)
9524 || delivery.0.check_module_hold(&spec.module_id).is_err_and(|error| error.code == "config_delivery_deferred"));
9525 if held {
9526 reset_restart_count(snapshot, &spec.module_id)?;
9529 update_snapshot(snapshot, Some(&spec.module_id), |state| {
9530 state.enabled = true;
9531 if child.is_none() && state.waiting_for_previous_pid.is_none() {
9532 state.state = ModuleState::Starting;
9533 }
9534 })?;
9535 if child.is_none() && lock_snapshot(snapshot)?.waiting_for_previous_pid.is_none() {
9536 hold_config_spawn(spec, runtime, snapshot, &failure)?;
9537 }
9538 } else {
9539 match set_child_enabled_with_budget_policy(
9540 spec, runtime, registry, process_liveness, snapshot, child,
9541 next.enabled, &failure, true,
9542 ).await {
9543 Ok(_) => {},
9544 Err(SuperviseError::ConfigHeld { .. }) if next.enabled => {},
9545 Err(error) => return Err(error),
9546 }
9547 if !next.enabled {
9548 containment::terminate(&runtime.child_roster.containment).await;
9549 }
9550 }
9551 }
9552 if (child.as_ref().is_some_and(|child| child.registration_spawn.is_some() && child.admission_base.is_none())
9553 || (child.is_none() && lock_snapshot(snapshot)?.state == ModuleState::Restarting
9554 && runtime.config_delivery.as_ref().is_some_and(|delivery| delivery.0.stamp_channel_cutover_pending())))
9555 && runtime.config_delivery.as_ref().is_some_and(|delivery| delivery.0.module_is_gated(&spec.module_id))
9556 {
9557 let failure = FailureContext {
9558 approval: Some(approved.identity()),
9559 ..runtime.failure_context(false)
9560 };
9561 if child.is_some() {
9565 restart_child_with_budget_policy(spec, runtime, registry,
9566 process_liveness, snapshot, child, runtime.drain_timeout, false).await?;
9567 }
9568 update_snapshot(snapshot, Some(&spec.module_id), |state| state.respawn_pending = false)?;
9571 match spawn_and_mark_running_async(spec, runtime, snapshot).await {
9572 Ok(replacement) => *child = Some(replacement),
9573 Err(SuperviseError::ConfigDeliveryDeferred { .. }) => {
9574 defer_config_spawn(spec, runtime, snapshot, RespawnKind::Spawn, &failure)?;
9575 }
9576 Err(SuperviseError::ConfigHeld { .. }) => hold_config_spawn(spec, runtime, snapshot, &failure)?,
9577 Err(error) => {
9578 fail_snapshot(snapshot, Some(&spec.module_id), None, &failure);
9579 return Err(error);
9580 }
9581 }
9582 }
9583 info!(module_id=%spec.module_id, config_generation=approved.generation(), "approved daemon entry reconciled");
9584 Ok(())
9585 }
9586 .await;
9587 if let Err(error) = &result {
9588 let _ = update_snapshot(snapshot, Some(&spec.module_id), |state| {
9589 if state.enabled
9590 && state.state == ModuleState::Failed
9591 && state
9592 .approved_start_failure
9593 .as_ref()
9594 .is_none_or(|(generation, _)| *generation != approved.identity())
9595 {
9596 state.approved_start_failure =
9597 Some((approved.identity(), error.to_string()));
9598 }
9599 });
9600 }
9601 let _ = reply.send(result);
9602 true
9603 }
9604 SupervisorCommand::ConfigPolicyProbe {
9605 connection,
9606 generation,
9607 attempt,
9608 } => {
9609 let Some(delivery) = &runtime.config_delivery else {
9610 return true;
9611 };
9612 let Ok(gate) = delivery.0.module_gate(&spec.module_id) else {
9613 return true;
9614 };
9615 let guard = gate.lock().await;
9616 let pending = delivery
9617 .0
9618 .on_blocking(move |owner| {
9619 owner.policy_probe_pending(connection, generation, attempt)
9620 })
9621 .await
9622 .unwrap_or(false)
9623 && child.is_some()
9624 && registry
9625 .get_module(&spec.module_id)
9626 .ok()
9627 .flatten()
9628 .is_some_and(|registration| registration.connection_id == connection);
9629 drop(guard);
9630 if pending {
9631 run_health_probe_cycle(spec, runtime, registry, process_liveness, snapshot, child)
9632 .await;
9633 }
9634 true
9635 }
9636 SupervisorCommand::ConfigApplyRestart {
9637 generation,
9638 supersedes_swap,
9639 } => {
9640 let Some(owner) = runtime
9641 .config_delivery
9642 .as_ref()
9643 .map(|delivery| delivery.0.clone())
9644 else {
9645 return true;
9646 };
9647 let Ok(gate) = owner.module_gate(&spec.module_id) else {
9648 return true;
9649 };
9650 let _guard = gate.lock().await;
9651 let spawn = child
9652 .as_ref()
9653 .and_then(|child| child.config_delivery.clone());
9654 let module = spec.module_id.clone();
9655 let current = owner
9656 .on_blocking(move |owner| {
9657 Ok((supersedes_swap
9658 || owner
9659 .pending_config_restarts()?
9660 .contains(&(module.clone(), generation)))
9661 && owner.requires_fresher_spawn(&module, spawn.as_ref())?
9662 == Some(generation))
9663 })
9664 .await
9665 .unwrap_or(false);
9666 if !current
9667 || !lock_snapshot(snapshot).is_ok_and(|state| {
9668 state.enabled && !state.respawn_pending && state.process_alive
9669 })
9670 {
9671 return true;
9672 }
9673 drop(_guard);
9674 info!(module_id=%spec.module_id, config_generation=generation, "approved configuration restart preserves the crash budget");
9675 if let Err(error) = restart_child_with_budget_policy(
9676 spec,
9677 runtime,
9678 registry,
9679 process_liveness,
9680 snapshot,
9681 child,
9682 runtime.drain_timeout,
9683 false,
9684 )
9685 .await
9686 {
9687 warn!(module_id=%spec.module_id, %error, "approved configuration restart could not be scheduled");
9688 }
9689 true
9690 }
9691 SupervisorCommand::ConfigExitHold { reply } => {
9692 let idle_start = child.is_none()
9696 && lock_snapshot(snapshot).is_ok_and(|state| {
9697 state.enabled
9698 && state.state == ModuleState::Starting
9699 && !state.process_alive
9700 && !state.respawn_pending
9701 && state.waiting_for_previous_pid.is_none()
9702 && state.waiting_for_previous_detail.is_none()
9703 && state.failed_retry.pending.is_none()
9704 })
9705 && runtime
9706 .scheduled_respawn
9707 .lock()
9708 .unwrap_or_else(|p| p.into_inner())
9709 .is_none()
9710 && runtime
9711 .deferred_reload_reply
9712 .lock()
9713 .unwrap_or_else(|p| p.into_inner())
9714 .is_none();
9715 let result = if idle_start {
9716 try_park_config_attempt(
9717 spec,
9718 runtime,
9719 snapshot,
9720 &runtime.failure_context(runtime.recovery_is_daemon),
9721 None,
9722 )
9723 .map(|_| ())
9724 } else {
9725 Ok(())
9726 };
9727 let _ = reply.send(result);
9728 true
9729 }
9730 SupervisorCommand::ConfigDeliveryFailed { connection, hold } => {
9731 if registry
9732 .get_module(&spec.module_id)
9733 .ok()
9734 .flatten()
9735 .is_none_or(|registration| registration.connection_id != connection)
9736 {
9737 return true;
9738 }
9739 runtime.recovery_is_daemon = true;
9740 if hold {
9744 let notice = begin_forwarding_drain_if_configured(
9745 spec,
9746 runtime,
9747 registry,
9748 snapshot,
9749 Some(true),
9750 RouteCloseReason::Disable,
9751 )
9752 .await;
9753 if let Ok(notice) = notice {
9754 let drained = drain_optional_child(
9755 &spec.module_id,
9756 spec.protocol,
9757 notice,
9758 registry,
9759 runtime.forwarding.as_deref(),
9760 snapshot,
9761 &runtime.terminal_ring,
9762 &runtime.spawn_events,
9763 child,
9764 runtime.drain_timeout,
9765 ModuleState::Failed,
9766 Some(true),
9767 &runtime.failure_context(true),
9768 )
9769 .await;
9770 if drained.is_ok() {
9771 match try_park_config_attempt(
9772 spec,
9773 runtime,
9774 snapshot,
9775 &runtime.failure_context(true),
9776 None,
9777 ) {
9778 Ok(true) => {}
9779 Ok(false) => {
9780 if let Err(error) = budgeted_restart_child(
9781 spec,
9782 runtime,
9783 registry,
9784 process_liveness,
9785 snapshot,
9786 child,
9787 SupervisorHealthStatus::Unknown,
9788 Some("configuration delivery failed after its hold ended"),
9789 unix_ms_now(),
9790 false,
9791 )
9792 .await
9793 {
9794 warn!(module_id=%spec.module_id, %error, "configuration failure restart could not be scheduled");
9795 }
9796 }
9797 Err(error) => {
9798 warn!(module_id=%spec.module_id, %error, "configuration failure could not be parked")
9799 }
9800 }
9801 }
9802 }
9803 } else if let Err(error) = budgeted_restart_child(
9804 spec,
9805 runtime,
9806 registry,
9807 process_liveness,
9808 snapshot,
9809 child,
9810 SupervisorHealthStatus::Unknown,
9811 Some("configuration delivery failed"),
9812 unix_ms_now(),
9813 false,
9814 )
9815 .await
9816 {
9817 warn!(module_id=%spec.module_id, %error, "configuration failure restart could not be scheduled");
9818 }
9819 true
9820 }
9821 SupervisorCommand::Drain { requester, reply } => {
9822 let result = drain_optional_child(
9825 &spec.module_id,
9826 spec.protocol,
9827 StopNotice::NotSent,
9828 registry,
9829 runtime.forwarding.as_deref(),
9830 snapshot,
9831 &runtime.terminal_ring,
9832 &runtime.spawn_events,
9833 child,
9834 runtime.drain_timeout,
9835 ModuleState::Stopped,
9836 None,
9837 &FailureContext::operator(),
9838 )
9839 .await;
9840 containment::terminate(&runtime.child_roster.containment).await;
9841 let registration_released = result.is_ok();
9842 if registration_released {
9843 runtime.record_activity_transition(
9844 "subc.module_stopped",
9845 &spec.module_id,
9846 requester,
9847 );
9848 }
9849 let _ = reply.send(result);
9850 if registration_released {
9851 process_liveness.untrack_if_current(&spec.module_id, snapshot);
9852 }
9853 false
9854 }
9855 SupervisorCommand::Retire { requester, reply } => {
9856 let result = async {
9857 let stop_notice = begin_forwarding_drain_if_configured(
9858 spec,
9859 runtime,
9860 registry,
9861 snapshot,
9862 None,
9863 RouteCloseReason::Disable,
9864 )
9865 .await?;
9866 drain_optional_child(
9867 &spec.module_id,
9868 spec.protocol,
9869 stop_notice,
9870 registry,
9871 runtime.forwarding.as_deref(),
9872 snapshot,
9873 &runtime.terminal_ring,
9874 &runtime.spawn_events,
9875 child,
9876 runtime.drain_timeout,
9877 ModuleState::Stopped,
9878 None,
9879 &FailureContext::operator(),
9880 )
9881 .await
9882 }
9883 .await;
9884 containment::terminate(&runtime.child_roster.containment).await;
9885 let registration_released = result.is_ok();
9886 if registration_released {
9887 runtime.record_activity_transition(
9888 "subc.module_stopped",
9889 &spec.module_id,
9890 requester,
9891 );
9892 }
9893 let _ = reply.send(result);
9894 if registration_released {
9895 process_liveness.untrack_if_current(&spec.module_id, snapshot);
9896 }
9897 false
9898 }
9899 SupervisorCommand::Restart {
9900 requester,
9901 drain_timeout_ms,
9902 received_at_generation,
9903 queued_at,
9904 reply,
9905 } => {
9906 info!(
9910 module_id = %spec.module_id,
9911 queued_ms = u64::try_from(queued_at.elapsed().as_millis()).unwrap_or(u64::MAX),
9912 "restart command dequeued"
9913 );
9914 let validation = match lock_snapshot(snapshot) {
9926 Ok(state) if !state.enabled => Err(SuperviseError::Disabled {
9927 module_id: spec.module_id.clone(),
9928 }),
9929 Ok(_) => Ok(()),
9930 Err(err) => Err(err),
9931 };
9932 let initiated = validation.is_ok();
9933 let _ = reply.send(validation);
9934 let satisfied_by_generation = if initiated
9948 && child
9949 .as_ref()
9950 .is_some_and(|child| child.spawned_at > queued_at)
9951 {
9952 lock_snapshot(snapshot).ok().and_then(|state| {
9953 (!state.configuration_updated_since_spawn).then_some(state.spawn_generation)
9954 })
9955 } else {
9956 None
9957 };
9958 let satisfied_by_pending = initiated
9959 && child.is_none()
9960 && lock_snapshot(snapshot).ok().is_some_and(|mut state| {
9961 let pending = state.respawn_pending && !state.configuration_updated_since_spawn;
9965 if pending {
9966 state.coalesced_restart_pending = true;
9967 }
9968 pending
9969 });
9970 if satisfied_by_pending {
9971 debug!(module_id = %spec.module_id, "restart coalesced into the pending replacement");
9972 } else if let Some(generation) = satisfied_by_generation {
9973 info!(
9974 module_id = %spec.module_id,
9975 received_at_generation,
9976 "restart already satisfied by generation {generation}; not restarting again"
9977 );
9978 } else if initiated {
9979 let drain_timeout = drain_timeout_ms
9982 .map(Duration::from_millis)
9983 .unwrap_or(runtime.drain_timeout);
9984 if let Err(err) = restart_child(
9985 spec,
9986 runtime,
9987 registry,
9988 process_liveness,
9989 snapshot,
9990 child,
9991 drain_timeout,
9992 )
9993 .await
9994 {
9995 warn!(
9996 module_id = %spec.module_id,
9997 error = %err,
9998 "operator restart failed after initiation ack; module state carries the outcome"
9999 );
10000 let _ = update_snapshot(snapshot, Some(&spec.module_id), |state| {
10001 state.state = ModuleState::Failed;
10002 clear_current_process_facts(state);
10003 });
10004 } else {
10005 runtime.start_requester = requester;
10006 runtime.pending_activity_restart = true;
10007 }
10008 }
10009 true
10010 }
10011 SupervisorCommand::Reload { requester, reply } => {
10012 let result =
10013 reload_child(spec, runtime, registry, process_liveness, snapshot, child).await;
10014 if result.is_ok() {
10015 runtime.start_requester = requester;
10016 runtime.pending_activity_restart = true;
10017 }
10018 if result.is_ok()
10019 && runtime
10020 .scheduled_respawn
10021 .lock()
10022 .unwrap_or_else(|p| p.into_inner())
10023 .as_ref()
10024 .is_some_and(|pending| matches!(pending.kind, RespawnKind::Reload))
10025 {
10026 *runtime
10027 .deferred_reload_reply
10028 .lock()
10029 .unwrap_or_else(|p| p.into_inner()) = Some(reply);
10030 } else {
10031 let _ = reply.send(result);
10032 }
10033 true
10034 }
10035 SupervisorCommand::SetEnabled {
10036 enabled,
10037 requester,
10038 reply,
10039 reset_budget,
10040 } => {
10041 let previous_requester = runtime.start_requester.clone();
10042 if enabled {
10043 runtime.start_requester = requester.clone();
10044 }
10045 let result = set_child_enabled_with_budget_policy(
10046 spec,
10047 runtime,
10048 registry,
10049 process_liveness,
10050 snapshot,
10051 child,
10052 enabled,
10053 &runtime.failure_context(false),
10054 reset_budget,
10055 )
10056 .await;
10057 if !enabled {
10058 containment::terminate(&runtime.child_roster.containment).await;
10059 }
10060 if matches!(result, Ok(true)) {
10061 runtime.pending_activity_restart = false;
10062 if enabled {
10063 runtime.start_requester = requester;
10064 } else {
10065 runtime.start_requester = None;
10066 runtime.record_activity_transition(
10067 "subc.module_disabled",
10068 &spec.module_id,
10069 requester,
10070 );
10071 }
10072 } else {
10073 runtime.start_requester = previous_requester;
10074 }
10075 let _ = reply.send(result);
10076 true
10077 }
10078 SupervisorCommand::UpdateConfiguration {
10079 spec: next_spec,
10080 health,
10081 drain_timeout_ms,
10082 reply,
10083 } => {
10084 if runtime
10085 .config_delivery
10086 .as_ref()
10087 .is_some_and(|delivery| delivery.0.file_configuration_is_gated(&spec.module_id))
10088 {
10089 let _ = reply.send(Err(SuperviseError::ConfigHeld {
10090 module_id: spec.module_id.clone(),
10091 code: "config_gated".into(),
10092 detail: "live file reconfiguration cannot replace stamped authority".into(),
10093 }));
10094 return true;
10095 }
10096 stage_supervisor_configuration(
10097 spec,
10098 runtime,
10099 snapshot,
10100 next_spec,
10101 health,
10102 drain_timeout_ms,
10103 );
10104 let _ = update_snapshot(snapshot, Some(&spec.module_id), |state| {
10105 state.configuration_updated_since_spawn = true;
10106 });
10107 let _ = reply.send(Ok(()));
10108 true
10109 }
10110 SupervisorCommand::Swap {
10111 requester,
10112 ready_timeout,
10113 reply,
10114 } => {
10115 if spec.carry_containment
10116 || containment::current(&runtime.child_roster.containment).is_some()
10117 {
10118 let _ = reply.send(Err(SuperviseError::InvalidSpec {
10119 reason: "a carried container cannot overlap a swap candidate".to_owned(),
10120 }));
10121 return true;
10122 }
10123 let mut swap_runtime = runtime.clone();
10124 swap_runtime.start_requester = requester;
10125 let before = lock_snapshot(snapshot)
10126 .ok()
10127 .map(|state| state.spawn_generation);
10128 let end = swap::run_swap(
10129 spec,
10130 &swap_runtime,
10131 registry,
10132 process_liveness,
10133 snapshot,
10134 child,
10135 commands,
10136 ready_timeout.unwrap_or(DEFAULT_SWAP_READY_TIMEOUT),
10137 reply,
10138 )
10139 .await;
10140 if before
10141 != lock_snapshot(snapshot)
10142 .ok()
10143 .map(|state| state.spawn_generation)
10144 {
10145 runtime.start_requester = swap_runtime.start_requester;
10146 }
10147 requeued.extend(end.requeue);
10148 true
10149 }
10150 }
10151}
10152
10153fn stage_supervisor_configuration(
10154 spec: &mut ModuleSpec,
10155 runtime: &mut SupervisorRuntimeConfig,
10156 snapshot: &SharedSnapshot,
10157 next_spec: ModuleSpec,
10158 health: HealthConfig,
10159 drain_timeout_ms: Option<u64>,
10160) {
10161 if let Some(handle) = &runtime.supervisor_handle {
10162 handle.apply_identity_configuration(&next_spec);
10163 }
10164 let launch_changed = *spec != next_spec;
10165 *spec = next_spec;
10166 if launch_changed {
10167 let _ = update_snapshot(snapshot, Some(&spec.module_id), |state| {
10168 state.configuration_updated_since_spawn = true
10169 });
10170 }
10171 let health_changed = runtime.health != health;
10172 runtime.health = health;
10173 if health_changed {
10176 let _ = update_snapshot(snapshot, Some(&spec.module_id), |state| {
10177 state.health = ModuleHealthStatus::default();
10178 state.last_probe_evidence = None;
10179 });
10180 }
10181 runtime.drain_timeout = drain_timeout_ms
10182 .map(Duration::from_millis)
10183 .unwrap_or(runtime.default_drain_timeout);
10184 *runtime
10185 .effective_drain_timeout
10186 .lock()
10187 .unwrap_or_else(|p| p.into_inner()) = runtime.drain_timeout;
10188}
10189
10190async fn restart_child(
10191 spec: &ModuleSpec,
10192 runtime: &SupervisorRuntimeConfig,
10193 registry: &Registry,
10194 process_liveness: &SupervisorProcessLiveness,
10195 snapshot: &SharedSnapshot,
10196 child: &mut Option<SupervisedChild>,
10197 drain_timeout: Duration,
10198) -> Result<(), SuperviseError> {
10199 restart_child_with_budget_policy(
10200 spec,
10201 runtime,
10202 registry,
10203 process_liveness,
10204 snapshot,
10205 child,
10206 drain_timeout,
10207 true,
10208 )
10209 .await
10210}
10211
10212#[allow(clippy::too_many_arguments)]
10213async fn restart_child_with_budget_policy(
10214 spec: &ModuleSpec,
10215 runtime: &SupervisorRuntimeConfig,
10216 registry: &Registry,
10217 process_liveness: &SupervisorProcessLiveness,
10218 snapshot: &SharedSnapshot,
10219 child: &mut Option<SupervisedChild>,
10220 drain_timeout: Duration,
10221 reset_budget: bool,
10222) -> Result<(), SuperviseError> {
10223 if !lock_snapshot(snapshot)?.enabled {
10225 return Err(SuperviseError::Disabled {
10226 module_id: spec.module_id.clone(),
10227 });
10228 }
10229 let stop_notice = begin_forwarding_drain_with_timeout(
10230 spec,
10231 runtime,
10232 registry,
10233 snapshot,
10234 None,
10235 RouteCloseReason::Restart,
10236 drain_timeout,
10237 )
10238 .await?;
10239
10240 if child.is_some() {
10241 drain_optional_child(
10242 &spec.module_id,
10243 spec.protocol,
10244 stop_notice,
10245 registry,
10246 runtime.forwarding.as_deref(),
10247 snapshot,
10248 &runtime.terminal_ring,
10249 &runtime.spawn_events,
10250 child,
10251 drain_timeout,
10252 ModuleState::Restarting,
10253 Some(true),
10254 &FailureContext::operator(),
10255 )
10256 .await?;
10257 } else {
10258 update_snapshot(snapshot, Some(&spec.module_id), |state| {
10259 state.enabled = true;
10260 state.state = ModuleState::Restarting;
10261 clear_current_process_facts(state);
10262 })?;
10263 release_dead_registration(
10264 registry,
10265 runtime.forwarding.as_deref(),
10266 snapshot,
10267 &spec.module_id,
10268 &runtime.failure_context(false),
10269 )
10270 .await?;
10271 }
10272
10273 if reset_budget {
10274 reset_restart_count(snapshot, &spec.module_id)?;
10275 }
10276 process_liveness.track(spec.module_id.clone(), Arc::clone(snapshot));
10277 schedule_respawn(
10278 runtime,
10279 snapshot,
10280 &spec.module_id,
10281 runtime.restart_policy.backoff,
10282 RespawnKind::Spawn,
10283 false,
10284 )
10285}
10286
10287async fn reload_child(
10288 spec: &ModuleSpec,
10289 runtime: &SupervisorRuntimeConfig,
10290 registry: &Registry,
10291 process_liveness: &SupervisorProcessLiveness,
10292 snapshot: &SharedSnapshot,
10293 child: &mut Option<SupervisedChild>,
10294) -> Result<(), SuperviseError> {
10295 if !lock_snapshot(snapshot)?.enabled {
10297 return Err(SuperviseError::Disabled {
10298 module_id: spec.module_id.clone(),
10299 });
10300 }
10301 let stop_notice = begin_forwarding_drain(
10302 spec,
10303 runtime,
10304 registry,
10305 snapshot,
10306 Some(true),
10307 RouteCloseReason::Reload,
10308 )
10309 .await?;
10310
10311 if child.is_some() {
10312 drain_optional_child(
10313 &spec.module_id,
10314 spec.protocol,
10315 stop_notice,
10316 registry,
10317 runtime.forwarding.as_deref(),
10318 snapshot,
10319 &runtime.terminal_ring,
10320 &runtime.spawn_events,
10321 child,
10322 runtime.drain_timeout,
10323 ModuleState::Restarting,
10324 Some(true),
10325 &FailureContext::operator(),
10326 )
10327 .await?;
10328 } else {
10329 update_snapshot(snapshot, Some(&spec.module_id), |state| {
10330 state.enabled = true;
10331 state.state = ModuleState::Restarting;
10332 clear_current_process_facts(state);
10333 })?;
10334 release_dead_registration(
10335 registry,
10336 runtime.forwarding.as_deref(),
10337 snapshot,
10338 &spec.module_id,
10339 &runtime.failure_context(false),
10340 )
10341 .await?;
10342 }
10343
10344 reset_restart_count(snapshot, &spec.module_id)?;
10345 process_liveness.track(spec.module_id.clone(), Arc::clone(snapshot));
10346 schedule_respawn(
10347 runtime,
10348 snapshot,
10349 &spec.module_id,
10350 runtime.restart_policy.backoff,
10351 RespawnKind::Reload,
10352 false,
10353 )
10354}
10355
10356async fn finish_reload_child(
10357 spec: &ModuleSpec,
10358 runtime: &SupervisorRuntimeConfig,
10359 registry: &Registry,
10360 process_liveness: &SupervisorProcessLiveness,
10361 snapshot: &SharedSnapshot,
10362 child: &mut Option<SupervisedChild>,
10363) -> Result<(), SuperviseError> {
10364 process_liveness.track(spec.module_id.clone(), Arc::clone(snapshot));
10365 let next_child = match spawn_and_mark_running_async(spec, runtime, snapshot).await {
10366 Ok(next_child) => next_child,
10367 Err(error @ SuperviseError::ConfigDeliveryDeferred { .. }) => {
10368 defer_config_spawn(
10369 spec,
10370 runtime,
10371 snapshot,
10372 RespawnKind::Reload,
10373 &runtime.failure_context(runtime.recovery_is_daemon),
10374 )?;
10375 return Err(error);
10376 }
10377 Err(error @ SuperviseError::ConfigHeld { .. }) => {
10378 let failure = runtime.failure_context(runtime.recovery_is_daemon);
10379 if try_park_config_attempt(spec, runtime, snapshot, &failure, None)? {
10380 return Err(error);
10381 }
10382 defer_config_spawn(spec, runtime, snapshot, RespawnKind::Reload, &failure)?;
10383 return Err(SuperviseError::ConfigDeliveryDeferred {
10384 module_id: spec.module_id.clone(),
10385 });
10386 }
10387 Err(err) => {
10388 return handle_reload_spawn_failure(
10389 spec,
10390 runtime,
10391 process_liveness,
10392 snapshot,
10393 child,
10394 format!("new child failed to spawn: {err}"),
10395 )
10396 .await;
10397 }
10398 };
10399 *child = Some(next_child);
10400
10401 let wait_outcome = {
10402 let active_child = child.as_mut().expect("new reload child was just stored");
10403 wait_for_registration_after_reload(
10404 registry,
10405 &spec.module_id,
10406 snapshot,
10407 active_child,
10408 REGISTRY_RELEASE_TIMEOUT,
10409 )
10410 .await?
10411 };
10412
10413 match wait_outcome {
10414 RegistrationWaitOutcome::Registered => {
10415 child
10416 .as_mut()
10417 .expect("registered reload child")
10418 .registration_deadline = None;
10419 debug!(module_id = %spec.module_id, "supervised module reloaded and registered");
10420 Ok(())
10421 }
10422 RegistrationWaitOutcome::Exited(exit_report) => {
10423 if let Some(active_child) = child.as_mut() {
10424 active_child.drain_stderr(&spec.module_id).await;
10425 }
10426 let mut exited_child = child.take().expect("exited reload child is still stored");
10429 #[cfg(test)]
10430 if let Some(gate) = &runtime.test_reload_exit_record_gate {
10431 gate.reached.notify_one();
10432 gate.resume.notified().await;
10433 }
10434 let result = handle_reload_child_registration_failure(
10435 spec,
10436 runtime,
10437 registry,
10438 process_liveness,
10439 snapshot,
10440 child,
10441 ReloadRegistrationFailure {
10442 exit_report: registration_failure_exit_report(exit_report),
10443 reason: exited_child
10444 .spawn_failure
10445 .clone()
10446 .unwrap_or_else(|| "new child exited before registering".to_string()),
10447 },
10448 )
10449 .await;
10450 exited_child
10451 .finish_exit(false)
10452 .await
10453 .map_err(|source| SuperviseError::Wait {
10454 module_id: spec.module_id.clone(),
10455 source,
10456 })?;
10457 result
10458 }
10459 RegistrationWaitOutcome::TimedOut => {
10460 let mut timed_out_child = child
10461 .take()
10462 .expect("timed-out reload child is still running");
10463 timed_out_child
10464 .start_kill()
10465 .map_err(|source| SuperviseError::Kill {
10466 module_id: spec.module_id.clone(),
10467 source,
10468 })?;
10469 let status = timed_out_child
10470 .wait()
10471 .await
10472 .map_err(|source| SuperviseError::Wait {
10473 module_id: spec.module_id.clone(),
10474 source,
10475 })?;
10476 timed_out_child.drain_stderr(&spec.module_id).await;
10477 let result = handle_reload_child_registration_failure(
10478 spec,
10479 runtime,
10480 registry,
10481 process_liveness,
10482 snapshot,
10483 child,
10484 ReloadRegistrationFailure {
10485 exit_report: registration_failure_exit_report(classify_reaped_child_exit(
10486 snapshot,
10487 &timed_out_child,
10488 &status,
10489 )),
10490 reason: format!(
10491 "new child did not register within {:?}",
10492 REGISTRY_RELEASE_TIMEOUT
10493 ),
10494 },
10495 )
10496 .await;
10497 timed_out_child
10498 .finish_exit(false)
10499 .await
10500 .map_err(|source| SuperviseError::Wait {
10501 module_id: spec.module_id.clone(),
10502 source,
10503 })?;
10504 result
10505 }
10506 }
10507}
10508
10509#[allow(clippy::too_many_arguments)]
10510async fn set_child_enabled(
10511 spec: &ModuleSpec,
10512 runtime: &SupervisorRuntimeConfig,
10513 registry: &Registry,
10514 process_liveness: &SupervisorProcessLiveness,
10515 snapshot: &SharedSnapshot,
10516 child: &mut Option<SupervisedChild>,
10517 enabled: bool,
10518 failure: &FailureContext,
10519) -> Result<bool, SuperviseError> {
10520 set_child_enabled_with_budget_policy(
10521 spec,
10522 runtime,
10523 registry,
10524 process_liveness,
10525 snapshot,
10526 child,
10527 enabled,
10528 failure,
10529 true,
10530 )
10531 .await
10532}
10533
10534#[allow(clippy::too_many_arguments)]
10535async fn set_child_enabled_with_budget_policy(
10536 spec: &ModuleSpec,
10537 runtime: &SupervisorRuntimeConfig,
10538 registry: &Registry,
10539 process_liveness: &SupervisorProcessLiveness,
10540 snapshot: &SharedSnapshot,
10541 child: &mut Option<SupervisedChild>,
10542 enabled: bool,
10543 failure: &FailureContext,
10544 reset_budget: bool,
10545) -> Result<bool, SuperviseError> {
10546 let (current_enabled, current_state, respawn_pending) = {
10547 let state = lock_snapshot(snapshot)?;
10548 (state.enabled, state.state, state.respawn_pending)
10549 };
10550 let revive_terminal = enabled
10558 && current_enabled
10559 && child.is_none()
10560 && (matches!(current_state, ModuleState::Failed | ModuleState::Stopped)
10561 || (current_state == ModuleState::Restarting && !respawn_pending));
10562 if current_enabled == enabled && !revive_terminal {
10563 return Ok(false);
10564 }
10565
10566 if enabled {
10567 update_snapshot(snapshot, Some(&spec.module_id), |state| {
10568 state.enabled = true;
10569 state.state = ModuleState::Starting;
10570 clear_current_process_facts(state);
10571 })?;
10572 #[cfg(test)]
10573 if runtime.test_seed_stale_facts_before_enable_spawn {
10574 update_snapshot(snapshot, Some(&spec.module_id), |state| {
10575 state.process_alive = true;
10576 state.pid = Some(41);
10577 state.spawned_at_ms = Some(42);
10578 state.spawned_from = Some(PathBuf::from("/spawned/module"));
10579 state.spawned_file_identity = Some(SpawnedFileIdentity {
10580 device: 43,
10581 inode: 44,
10582 #[cfg(windows)]
10583 windows_image: None,
10584 });
10585 })?;
10586 }
10587 release_dead_registration(
10588 registry,
10589 runtime.forwarding.as_deref(),
10590 snapshot,
10591 &spec.module_id,
10592 failure,
10593 )
10594 .await?;
10595 if reset_budget {
10596 reset_restart_count(snapshot, &spec.module_id)?;
10597 }
10598 process_liveness.track(spec.module_id.clone(), Arc::clone(snapshot));
10599 let next_child = match spawn_and_mark_running_async(spec, runtime, snapshot).await {
10600 Ok(next_child) => next_child,
10601 Err(SuperviseError::ConfigDeliveryDeferred { .. }) => {
10602 defer_config_spawn(spec, runtime, snapshot, RespawnKind::Spawn, failure)?;
10603 return Ok(true);
10604 }
10605 Err(error @ SuperviseError::ConfigHeld { .. }) => {
10606 if let Err(parking_error) = hold_config_spawn(spec, runtime, snapshot, failure) {
10607 process_liveness.untrack_if_current(&spec.module_id, snapshot);
10608 return Err(parking_error);
10609 }
10610 return Err(error);
10611 }
10612 Err(err) => {
10613 fail_snapshot(snapshot, Some(&spec.module_id), None, failure);
10614 process_liveness.untrack_if_current(&spec.module_id, snapshot);
10615 return Err(err);
10616 }
10617 };
10618 *child = Some(next_child);
10619 debug!(module_id = %spec.module_id, "supervised module enabled");
10620 Ok(true)
10621 } else {
10622 update_snapshot(snapshot, Some(&spec.module_id), |state| {
10623 state.config_hold_parked = false;
10624 state.config_hold_daemon_initiated = false;
10625 state.config_hold_approval = None;
10626 })?;
10627 let stop_notice = begin_forwarding_drain_if_configured(
10628 spec,
10629 runtime,
10630 registry,
10631 snapshot,
10632 Some(false),
10633 RouteCloseReason::Disable,
10634 )
10635 .await?;
10636 drain_optional_child(
10637 &spec.module_id,
10638 spec.protocol,
10639 stop_notice,
10640 registry,
10641 runtime.forwarding.as_deref(),
10642 snapshot,
10643 &runtime.terminal_ring,
10644 &runtime.spawn_events,
10645 child,
10646 runtime.drain_timeout,
10647 ModuleState::Disabled,
10648 Some(false),
10649 failure,
10650 )
10651 .await?;
10652 debug!(module_id = %spec.module_id, "supervised module disabled");
10653 Ok(true)
10654 }
10655}
10656
10657#[allow(clippy::too_many_arguments)]
10658async fn on_child_exit(
10659 spec: &ModuleSpec,
10660 policy: RestartPolicy,
10661 registry: &Registry,
10662 snapshot: &SharedSnapshot,
10663 terminal_ring: &Arc<Mutex<TerminalRing>>,
10664 spawn_events: &SpawnEventFeed,
10665 roster: &ChildRoster,
10666 exit_report: ExitReport,
10667 daemon_initiated: bool,
10668) -> NextAction {
10669 if roster.is_closed() {
10675 return on_child_exit_during_daemon_shutdown(
10676 spec,
10677 registry,
10678 snapshot,
10679 terminal_ring,
10680 spawn_events,
10681 exit_report,
10682 )
10683 .await;
10684 }
10685 let unrequested_clean_exit_of_protocol_none = exit_report.kind == ExitKind::Clean
10701 && running_protocol(spec, snapshot) == ModuleProtocol::None;
10702 match exit_report.kind {
10703 ExitKind::Clean if !unrequested_clean_exit_of_protocol_none => {
10704 info!(
10705 module_id = %spec.module_id,
10706 exit_code = ?exit_report.code,
10707 exit_signal = ?exit_report.signal,
10708 "supervised module exited cleanly"
10709 );
10710 if let Err(err) = publish_after_terminal_record(
10711 &spec.module_id,
10712 || {
10713 record_terminal(
10714 &spec.module_id,
10715 terminal_ring,
10716 spawn_events,
10717 &exit_report,
10718 TerminalDisposition::Stopped,
10719 )
10720 },
10721 || {
10722 update_snapshot(snapshot, Some(&spec.module_id), |state| {
10723 state.state = ModuleState::Stopped;
10724 clear_current_process_facts(state);
10725 state.last_exit = Some(exit_report.clone());
10726 })
10727 },
10728 ) {
10729 error!(module_id = %spec.module_id, error = %err, "failed to publish clean module exit");
10730 }
10731 let registration_released = match wait_for_registration_release(
10732 registry,
10733 &spec.module_id,
10734 REGISTRY_RELEASE_TIMEOUT,
10735 )
10736 .await
10737 {
10738 Ok(()) => true,
10739 Err(err) => {
10740 warn!(module_id = %spec.module_id, error = %err, "registration still active after clean exit");
10741 false
10742 }
10743 };
10744 NextAction::Stop {
10745 registration_released,
10746 }
10747 }
10748 ExitKind::Clean | ExitKind::Crash => {
10749 if unrequested_clean_exit_of_protocol_none {
10750 warn!(
10751 module_id = %spec.module_id,
10752 exit_code = ?exit_report.code,
10753 exit_signal = ?exit_report.signal,
10754 "protocol-none module exited cleanly without a stop request; handling it as a crash"
10755 );
10756 } else {
10757 warn!(
10758 module_id = %spec.module_id,
10759 exit_code = ?exit_report.code,
10760 exit_signal = ?exit_report.signal,
10761 "supervised module exited abnormally (crash)"
10762 );
10763 }
10764 let mut restart_schedule = None;
10765 let mut disposition = TerminalDisposition::Disabled;
10766 let mut final_state = ModuleState::Disabled;
10767 let mut disposition_detail = lock_snapshot(snapshot)
10771 .ok()
10772 .and_then(|mut state| state.spawn_failure.take());
10773 let now = Instant::now();
10774 if let Err(err) = update_snapshot(snapshot, Some(&spec.module_id), |state| {
10775 if state.enabled {
10776 if let Some(schedule) = state.next_crash_restart(&policy, now) {
10777 state.respawn_pending = true;
10783 final_state = ModuleState::Restarting;
10784 restart_schedule = Some(schedule);
10785 disposition = TerminalDisposition::Restarting;
10786 } else {
10787 final_state = ModuleState::Failed;
10788 disposition = TerminalDisposition::Failed;
10789 let budget = policy.budget_exhausted_detail();
10790 disposition_detail =
10791 Some(disposition_detail.take().map_or_else(
10792 || budget.clone(),
10793 |cause| format!("{cause}; {budget}"),
10794 ));
10795 }
10796 } else {
10797 disposition = TerminalDisposition::Disabled;
10798 }
10799 }) {
10800 error!(module_id = %spec.module_id, error = %err, "failed to record crashed module exit");
10801 return NextAction::Stop {
10802 registration_released: false,
10803 };
10804 }
10805 if disposition == TerminalDisposition::Failed {
10806 error!(
10811 module_id = %spec.module_id,
10812 max_restarts = policy.max_restarts,
10813 window_secs = policy.window.as_secs(),
10814 "module stopped: {}",
10815 policy.budget_exhausted_detail()
10816 );
10817 }
10818 if let Err(err) = publish_after_terminal_record(
10819 &spec.module_id,
10820 || {
10821 record_terminal_with_detail(
10822 &spec.module_id,
10823 terminal_ring,
10824 spawn_events,
10825 &exit_report,
10826 disposition,
10827 disposition_detail,
10828 )
10829 },
10830 || {
10831 update_snapshot(snapshot, Some(&spec.module_id), |state| {
10832 state.state = final_state;
10833 clear_current_process_facts(state);
10834 if final_state == ModuleState::Failed
10835 && daemon_initiated
10836 && !roster.is_closed()
10837 {
10838 state.failed_retry.schedule(&spec.module_id);
10839 }
10840 state.last_exit = Some(exit_report.clone());
10841 })
10842 },
10843 ) {
10844 error!(module_id = %spec.module_id, error = %err, "failed to publish crashed module exit");
10845 }
10846
10847 if let Some(schedule) = restart_schedule {
10848 NextAction::Restart {
10849 schedule: Some(schedule),
10850 }
10851 } else {
10852 let registration_released = match wait_for_registration_release(
10853 registry,
10854 &spec.module_id,
10855 REGISTRY_RELEASE_TIMEOUT,
10856 )
10857 .await
10858 {
10859 Ok(()) => true,
10860 Err(err) => {
10861 warn!(module_id = %spec.module_id, error = %err, "registration still active after failed module");
10862 false
10863 }
10864 };
10865 NextAction::Stop {
10866 registration_released,
10867 }
10868 }
10869 }
10870 ExitKind::DeliberateSeverance => {
10871 warn!(
10872 module_id = %spec.module_id,
10873 exit_code = ?exit_report.code,
10874 exit_signal = ?exit_report.signal,
10875 "supervised module exited after deliberate connection severance"
10876 );
10877 let should_restart = match lock_snapshot(snapshot) {
10878 Ok(state) => state.enabled,
10879 Err(err) => {
10880 error!(module_id = %spec.module_id, error = %err, "failed to read deliberately severed module state");
10881 return NextAction::Stop {
10882 registration_released: false,
10883 };
10884 }
10885 };
10886 let final_state = if should_restart {
10887 ModuleState::Restarting
10888 } else {
10889 ModuleState::Disabled
10890 };
10891 if let Err(err) = publish_after_terminal_record(
10892 &spec.module_id,
10893 || {
10894 record_terminal(
10895 &spec.module_id,
10896 terminal_ring,
10897 spawn_events,
10898 &exit_report,
10899 terminal_disposition(final_state),
10900 )
10901 },
10902 || {
10903 update_snapshot(snapshot, Some(&spec.module_id), |state| {
10904 state.state = final_state;
10905 clear_current_process_facts(state);
10906 state.last_exit = Some(exit_report.clone());
10907 state.lifetime_restarts += 1;
10908 })
10909 },
10910 ) {
10911 error!(module_id = %spec.module_id, error = %err, "failed to publish deliberately severed module exit");
10912 return NextAction::Stop {
10913 registration_released: false,
10914 };
10915 }
10916
10917 if should_restart {
10918 NextAction::Restart { schedule: None }
10919 } else {
10920 let registration_released = match wait_for_registration_release(
10921 registry,
10922 &spec.module_id,
10923 REGISTRY_RELEASE_TIMEOUT,
10924 )
10925 .await
10926 {
10927 Ok(()) => true,
10928 Err(err) => {
10929 warn!(module_id = %spec.module_id, error = %err, "registration still active after deliberately severed module exit");
10930 false
10931 }
10932 };
10933 NextAction::Stop {
10934 registration_released,
10935 }
10936 }
10937 }
10938 }
10939}
10940
10941async fn on_child_exit_during_daemon_shutdown(
10942 spec: &ModuleSpec,
10943 registry: &Registry,
10944 snapshot: &SharedSnapshot,
10945 terminal_ring: &Arc<Mutex<TerminalRing>>,
10946 spawn_events: &SpawnEventFeed,
10947 exit_report: ExitReport,
10948) -> NextAction {
10949 info!(
10950 module_id = %spec.module_id,
10951 exit_code = ?exit_report.code,
10952 exit_signal = ?exit_report.signal,
10953 exit_kind = ?exit_report.kind,
10954 "supervised module exited during daemon shutdown; not restarting it"
10955 );
10956 if let Err(err) = publish_after_terminal_record(
10957 &spec.module_id,
10958 || {
10959 record_terminal(
10960 &spec.module_id,
10961 terminal_ring,
10962 spawn_events,
10963 &exit_report,
10964 TerminalDisposition::DaemonShutdown,
10965 )
10966 },
10967 || {
10968 update_snapshot(snapshot, Some(&spec.module_id), |state| {
10969 state.state = ModuleState::Stopped;
10970 clear_current_process_facts(state);
10971 state.last_exit = Some(exit_report.clone());
10972 })
10973 },
10974 ) {
10975 error!(module_id = %spec.module_id, error = %err, "failed to publish module exit during daemon shutdown");
10976 }
10977 let registration_released =
10978 wait_for_registration_release(registry, &spec.module_id, REGISTRY_RELEASE_TIMEOUT)
10979 .await
10980 .is_ok();
10981 NextAction::Stop {
10982 registration_released,
10983 }
10984}
10985
10986fn publish_after_terminal_record<T>(
10991 module_id: &str,
10992 record: impl FnOnce(),
10993 publish: impl FnOnce() -> T,
10994) -> T {
10995 let recorded = std::panic::catch_unwind(std::panic::AssertUnwindSafe(record));
10996 let published = publish();
10997 if let Err(panic) = recorded {
10998 error!(
10999 module_id,
11000 "terminal recording panicked; published final state before resuming unwind"
11001 );
11002 std::panic::resume_unwind(panic);
11003 }
11004 published
11005}
11006
11007fn fail_after_wait_error(
11008 module_id: &str,
11009 snapshot: &SharedSnapshot,
11010 terminal_ring: &Arc<Mutex<TerminalRing>>,
11011 spawn_events: &SpawnEventFeed,
11012 detail: String,
11013 failure: &FailureContext,
11014) {
11015 let report = wait_error_exit_report();
11016 publish_after_terminal_record(
11017 module_id,
11018 || {
11019 record_terminal_with_detail(
11020 module_id,
11021 terminal_ring,
11022 spawn_events,
11023 &report,
11024 TerminalDisposition::Failed,
11025 Some(detail),
11026 )
11027 },
11028 || fail_snapshot(snapshot, Some(module_id), Some(report.clone()), failure),
11029 );
11030}
11031
11032#[cfg(test)]
11033fn record_wait_error_terminal(
11034 module_id: &str,
11035 terminal_ring: &Arc<Mutex<TerminalRing>>,
11036 spawn_events: &SpawnEventFeed,
11037) {
11038 record_terminal(
11039 module_id,
11040 terminal_ring,
11041 spawn_events,
11042 &wait_error_exit_report(),
11043 TerminalDisposition::Failed,
11044 );
11045}
11046
11047fn record_terminal(
11048 module_id: &str,
11049 terminal_ring: &Arc<Mutex<TerminalRing>>,
11050 spawn_events: &SpawnEventFeed,
11051 exit_report: &ExitReport,
11052 disposition: TerminalDisposition,
11053) {
11054 record_terminal_with_detail(
11055 module_id,
11056 terminal_ring,
11057 spawn_events,
11058 exit_report,
11059 disposition,
11060 None,
11061 );
11062}
11063
11064fn durable_terminal_history_of(
11068 terminal_ring: &Mutex<TerminalRing>,
11069 module_id: &str,
11070) -> subc_control::TerminalHistory {
11071 let read = terminal_ring
11072 .lock()
11073 .unwrap_or_else(|p| p.into_inner())
11074 .capture_durable_history();
11075 read.read(module_id)
11076}
11077
11078fn record_terminal_with_detail(
11079 module_id: &str,
11080 terminal_ring: &Arc<Mutex<TerminalRing>>,
11081 spawn_events: &SpawnEventFeed,
11082 exit_report: &ExitReport,
11083 disposition: TerminalDisposition,
11084 disposition_detail: Option<String>,
11085) {
11086 #[cfg(test)]
11087 terminal_history_tests::before_terminal_record(spawn_events);
11088 spawn_events.emit_exited(module_id, exit_report.code, exit_report.signal);
11089 let record = TerminalRecord {
11090 exit_code: exit_report.code,
11091 exit_signal: exit_report.signal,
11092 at_ms: exit_report.at_ms,
11093 disposition,
11094 exit_kind: exit_report.kind.into(),
11095 disposition_detail,
11096 };
11097 terminal_ring
11098 .lock()
11099 .unwrap_or_else(|poisoned| poisoned.into_inner())
11100 .record_exit(module_id, record);
11101}
11102
11103fn untrack_if_registration_released(
11104 process_liveness: &SupervisorProcessLiveness,
11105 registry: &Registry,
11106 module_id: &str,
11107 snapshot: &SharedSnapshot,
11108) {
11109 match registry.get_module(module_id) {
11110 Ok(None) => process_liveness.untrack_if_current(module_id, snapshot),
11111 Ok(Some(_)) => {}
11112 Err(err) => {
11113 warn!(module_id, error = %err, "could not determine whether supervisor liveness can be untracked");
11114 }
11115 }
11116}
11117
11118#[cfg(test)]
11132fn apply_wire_spawn_args(
11133 command: &mut Command,
11134 spec: &ModuleSpec,
11135 connection_file_path: Option<&std::path::Path>,
11136 handle: Option<&SupervisorHandle>,
11137) -> Result<Option<NonceHandoff>, SuperviseError> {
11138 apply_wire_spawn_args_for_role(
11139 command,
11140 spec,
11141 connection_file_path,
11142 handle,
11143 SpawnRole::Plain,
11144 )
11145}
11146
11147#[cfg(unix)]
11152type NonceHandoff = subc_os::LaunchNonceHandoff;
11153#[cfg(windows)]
11154type NonceHandoff = subc_os::LaunchNoncePipeHandoff;
11155#[cfg(not(any(unix, windows)))]
11156type NonceHandoff = std::convert::Infallible;
11157
11158#[cfg(test)]
11176fn apply_wire_spawn_args_for_role(
11177 command: &mut Command,
11178 spec: &ModuleSpec,
11179 connection_file_path: Option<&std::path::Path>,
11180 handle: Option<&SupervisorHandle>,
11181 role: SpawnRole,
11182) -> Result<Option<NonceHandoff>, SuperviseError> {
11183 apply_wire_spawn_args_for_admission(command, spec, connection_file_path, handle, role, None)
11184}
11185
11186fn apply_wire_spawn_args_for_admission(
11187 command: &mut Command,
11188 spec: &ModuleSpec,
11189 connection_file_path: Option<&std::path::Path>,
11190 handle: Option<&SupervisorHandle>,
11191 role: SpawnRole,
11192 admission: Option<&crate::config_gate::runtime::SpawnAdmission>,
11193) -> Result<Option<NonceHandoff>, SuperviseError> {
11194 command.env(SUBC_MODULE_ID_ENV, &spec.module_id);
11195 command.env_remove(subc_os::LAUNCH_NONCE_FD_ENV);
11200 #[cfg(windows)]
11201 {
11202 command.env_remove(subc_os::LAUNCH_NONCE_PIPE_ENV);
11203 command.env_remove(subc_os::LAUNCH_NONCE_PIPE_FALLBACK_ENV);
11204 }
11205 command.env_remove(SUBC_LAUNCH_NONCE_ENV);
11207 command.env_remove(subc_os::registration_channel::REGISTRATION_FD_ENV);
11208 command.env_remove(subc_os::registration_channel::REGISTRATION_PIPE_ENV);
11209 command.env_remove(subc_os::registration_channel::DAEMON_PID_ENV);
11210 if role == SpawnRole::Plain {
11211 if let Some(handle) = handle {
11212 #[cfg(all(test, unix))]
11213 {
11214 let pause = {
11215 let mut slot = handle.plain_identity_policy_pause.lock().unwrap();
11216 if slot
11217 .as_ref()
11218 .is_some_and(|(module, _)| module == &spec.module_id)
11219 {
11220 slot.take().map(|(_, pause)| pause)
11221 } else {
11222 None
11223 }
11224 };
11225 if let Some(pause) = pause {
11226 pause.reached.send(()).unwrap();
11227 pause.release.recv().unwrap();
11228 }
11229 }
11230 handle.apply_identity_policy(spec, spec.protocol);
11234 }
11235 }
11236 if spec.protocol == ModuleProtocol::None {
11237 return Ok(None);
11238 }
11239 if let Some(connection_file_path) = connection_file_path {
11240 command.arg(SUBC_ARG).arg(connection_file_path);
11241 }
11242
11243 if admission.is_some_and(|admission| admission.launch.is_some()) {
11244 if let Some(handle) = handle {
11245 match role {
11246 SpawnRole::Plain => {
11247 handle
11248 .spawn_nonces
11249 .lock()
11250 .unwrap_or_else(|p| p.into_inner())
11251 .remove(&spec.module_id);
11252 if spec.reserved {
11253 handle
11254 .reserved_nonces
11255 .lock()
11256 .unwrap_or_else(|p| p.into_inner())
11257 .insert(spec.module_id.clone(), None);
11258 }
11259 }
11260 SpawnRole::SwapCandidate => handle.open_swap(&spec.module_id, String::new()),
11263 }
11264 }
11265 return Ok(None);
11266 }
11267 let nonce = generate_launch_nonce()?;
11271 if let Some(handle) = handle {
11272 match role {
11273 SpawnRole::Plain => {
11274 handle.set_spawn_nonce(&spec.module_id, nonce.clone());
11275 if spec.reserved {
11276 handle.set_reserved_nonce(&spec.module_id, nonce.clone());
11277 }
11278 }
11279 SpawnRole::SwapCandidate => handle.open_swap(&spec.module_id, nonce.clone()),
11280 }
11281 }
11282 #[cfg(unix)]
11283 let handoff = {
11284 let handoff =
11285 subc_os::LaunchNonceHandoff::new(&nonce).map_err(|source| SuperviseError::Spawn {
11286 program: spec.program.clone(),
11287 source,
11288 cgroup_path: None,
11289 })?;
11290 command.env(subc_os::LAUNCH_NONCE_FD_ENV, handoff.fd_env_value());
11291 Some(handoff)
11292 };
11293 #[cfg(windows)]
11294 let handoff = prepare_windows_nonce_pipe(command, spec, &nonce);
11295 #[cfg(not(any(unix, windows)))]
11296 let handoff = None;
11297 #[cfg(not(unix))]
11301 command.env(SUBC_LAUNCH_NONCE_ENV, nonce);
11302 Ok(handoff)
11303}
11304
11305pub(crate) fn apply_child_env(command: &mut Command, spec: &ModuleSpec) {
11306 command.env_remove(CK_LOG_ENV);
11307 command.env_remove(SUBC_SPAWN_ROLE_ENV);
11314 for (key, value) in &spec.env {
11315 if matches!(
11319 key.as_str(),
11320 CAPTURE_MAX_FILE_MB_ENV | CAPTURE_KEEP_ENV | CAPTURE_MAX_AGE_DAYS_ENV
11321 ) || key == SUBC_SPAWN_ROLE_ENV
11322 {
11323 continue;
11324 }
11325 command.env(key, value);
11326 }
11327}
11328
11329#[cfg(windows)]
11330fn prepare_windows_registration_pipe(
11331 command: &mut Command,
11332 spec: &ModuleSpec,
11333) -> Result<Option<SpawnRegistrationChannel>, SuperviseError> {
11334 if spec.protocol == ModuleProtocol::None {
11335 return Ok(None);
11336 }
11337 let pipe = subc_os::registration_channel::RegistrationPipe::new().map_err(|source| {
11338 SuperviseError::Spawn {
11339 program: spec.program.clone(),
11340 source,
11341 cgroup_path: None,
11342 }
11343 })?;
11344 command
11345 .env(
11346 subc_os::registration_channel::REGISTRATION_PIPE_ENV,
11347 pipe.name(),
11348 )
11349 .env(
11350 subc_os::registration_channel::DAEMON_PID_ENV,
11351 std::process::id().to_string(),
11352 );
11353 Ok(Some(pipe))
11354}
11355
11356#[cfg(windows)]
11357fn prepare_windows_nonce_pipe(
11358 command: &mut Command,
11359 spec: &ModuleSpec,
11360 nonce: &str,
11361) -> Option<NonceHandoff> {
11362 match subc_os::LaunchNoncePipeHandoff::new(nonce) {
11363 Ok(handoff) => {
11364 command.env(subc_os::LAUNCH_NONCE_PIPE_ENV, handoff.name());
11365 command.env(subc_os::LAUNCH_NONCE_PIPE_FALLBACK_ENV, "env");
11370 Some(handoff)
11371 }
11372 Err(error) => {
11373 warn!(module_id = %spec.module_id, %error, "could not prepare launch nonce pipe; retaining the environment handoff");
11374 None
11375 }
11376 }
11377}
11378
11379#[cfg(windows)]
11380fn start_windows_nonce_pipe(
11381 handoff: Option<NonceHandoff>,
11382 pid: u32,
11383 spec: &ModuleSpec,
11384) -> Option<subc_os::LaunchNoncePipeDelivery> {
11385 handoff.and_then(|handoff| match handoff.serve(pid, REGISTRY_RELEASE_TIMEOUT) {
11389 Ok(delivery) => Some(delivery),
11390 Err(error) => {
11391 warn!(module_id = %spec.module_id, %error, "could not serve launch nonce pipe; retaining the environment handoff");
11392 None
11393 }
11394 })
11395}
11396
11397fn observed_launch_nonce_source(snapshot: &SupervisorSnapshot) -> Option<String> {
11398 #[cfg(windows)]
11399 if snapshot.process_alive && snapshot.spawned_protocol == Some(ModuleProtocol::Subc) {
11400 return Some(
11401 if snapshot
11402 .pipe_consumed
11403 .as_ref()
11404 .is_some_and(|consumed| consumed.get().is_some())
11405 {
11406 "pipe"
11407 } else {
11408 "env"
11409 }
11410 .to_owned(),
11411 );
11412 }
11413 let _ = snapshot;
11414 None
11415}
11416
11417#[derive(Debug, Clone, Copy, PartialEq, Eq)]
11420enum SpawnRole {
11421 Plain,
11422 SwapCandidate,
11423}
11424
11425fn apply_spawn_role(command: &mut Command, role: SpawnRole) {
11428 if role == SpawnRole::SwapCandidate {
11429 command.env(SUBC_SPAWN_ROLE_ENV, SPAWN_ROLE_SWAP_CANDIDATE);
11430 }
11431}
11432
11433fn previous_instance_error(
11434 spec: &ModuleSpec,
11435 pid: u32,
11436 reason: crate::live_children::RetainedReason,
11437) -> SuperviseError {
11438 SuperviseError::Spawn {
11439 program: spec.program.clone(),
11440 source: io::Error::new(io::ErrorKind::WouldBlock, reason.waiting_message(pid)),
11441 cgroup_path: None,
11442 }
11443}
11444
11445fn carries_containment(configured: bool, has_container: bool) -> bool {
11446 configured || has_container
11447}
11448
11449fn recovery_block_error(spec: &ModuleSpec, roster: &ChildRoster) -> Option<SuperviseError> {
11450 if !carries_containment(
11451 spec.carry_containment,
11452 containment::current(&roster.containment).is_some(),
11453 ) {
11454 return None;
11455 }
11456 Some(SuperviseError::Spawn {
11457 program: spec.program.clone(),
11458 source: io::Error::new(io::ErrorKind::WouldBlock, roster.recovery_block()?),
11459 cgroup_path: None,
11460 })
11461}
11462
11463#[allow(clippy::too_many_arguments)]
11464fn spawn_child(
11465 spec: &ModuleSpec,
11466 connection_file_path: Option<&std::path::Path>,
11467 handle: Option<&SupervisorHandle>,
11468 ring: &Arc<Mutex<StderrRing>>,
11469 capture_logs_dir: Option<&std::path::Path>,
11470 roster: &ChildRoster,
11471 config_delivery: Option<config_delivery::SpawnDelivery<'_>>,
11472 #[cfg(target_os = "linux")] cgroup_placement: Option<&subc_cgroup::Placement>,
11473 register_timeout: Duration,
11474) -> Result<SupervisedChild, SuperviseError> {
11475 #[cfg(all(test, unix))]
11476 if let Some(source) = containment_tests::join_failure(spec, roster) {
11477 containment::terminate_blocking(&roster.containment);
11478 return Err(SuperviseError::Spawn {
11479 program: spec.program.clone(),
11480 source,
11481 cgroup_path: None,
11482 });
11483 }
11484 let result = spawn_child_in_slot(
11485 spec,
11486 connection_file_path,
11487 handle,
11488 ring,
11489 capture_logs_dir,
11490 roster,
11491 config_delivery,
11492 #[cfg(target_os = "linux")]
11493 cgroup_placement,
11494 SpawnRole::Plain,
11495 false,
11496 register_timeout,
11497 );
11498 if result
11499 .as_ref()
11500 .is_err_and(|error| !is_launch_deferral(error))
11501 {
11502 if roster.admission_pending(&spec.module_id) {
11503 containment::terminate_deferred(&roster.containment);
11504 } else {
11505 containment::terminate_blocking(&roster.containment);
11506 }
11507 }
11508 result
11509}
11510
11511fn is_launch_deferral(error: &SuperviseError) -> bool {
11517 matches!(error, SuperviseError::ConfigDeliveryDeferred { .. })
11518}
11519
11520#[allow(clippy::too_many_arguments)]
11533fn spawn_child_in_slot(
11534 spec: &ModuleSpec,
11535 connection_file_path: Option<&std::path::Path>,
11536 handle: Option<&SupervisorHandle>,
11537 ring: &Arc<Mutex<StderrRing>>,
11538 capture_logs_dir: Option<&std::path::Path>,
11539 roster: &ChildRoster,
11540 config_delivery: Option<config_delivery::SpawnDelivery<'_>>,
11541 #[cfg(target_os = "linux")] cgroup_placement: Option<&subc_cgroup::Placement>,
11542 role: SpawnRole,
11543 alternate_slot: bool,
11544 register_timeout: Duration,
11545) -> Result<SupervisedChild, SuperviseError> {
11546 let prepared = prepare_child_in_slot(
11547 spec,
11548 connection_file_path,
11549 handle,
11550 capture_logs_dir,
11551 roster,
11552 config_delivery,
11553 #[cfg(target_os = "linux")]
11554 cgroup_placement,
11555 role,
11556 alternate_slot,
11557 false,
11558 )?;
11559 let gate = config_delivery
11560 .map(|delivery| delivery.module_gate(spec))
11561 .transpose()?;
11562 let _gate = gate
11563 .as_ref()
11564 .map(|gate| {
11565 gate.try_lock()
11566 .map_err(|_| SuperviseError::ConfigDeliveryDeferred {
11567 module_id: spec.module_id.clone(),
11568 })
11569 })
11570 .transpose()?;
11571 if let (Some(delivery), Some(admission)) = (config_delivery, &prepared.spawn_admission) {
11572 delivery.check(spec, admission)?;
11573 }
11574 let checked_base = prepared
11575 .spawn_admission
11576 .as_ref()
11577 .and_then(|admission| admission.approved_base())
11578 .cloned();
11579 let mut result = launch_child_in_slot(spec, ring, roster, prepared, register_timeout);
11580 drop(_gate);
11581 if let (Some(delivery), Err(SuperviseError::ConfigHeld { code, detail, .. })) =
11582 (config_delivery, &result)
11583 {
11584 if let Some(base) = checked_base.as_ref().filter(|_| code == "program_refused") {
11585 if !delivery.record_refusal(spec, base, detail)? {
11586 result = Err(spawn_plan_stale(spec));
11587 }
11588 }
11589 }
11590 result
11591}
11592
11593struct AsyncInitialIdentity<'a> {
11594 supervisor: &'a Supervisor,
11595 module: String,
11596 committed: bool,
11597}
11598impl Drop for AsyncInitialIdentity<'_> {
11599 fn drop(&mut self) {
11600 if !self.committed {
11601 self.supervisor.abandon_unrostered(&self.module);
11602 }
11603 }
11604}
11605
11606struct PreparedChild {
11610 script_delivery: Option<script::Delivery>,
11611 registration_spawn: Option<Arc<RegistrationSpawn>>,
11612 registration_handle: Option<SupervisorHandle>,
11613 #[cfg(any(unix, windows))]
11614 registration_channel: Option<SpawnRegistrationChannel>,
11615 config_context: Option<config_delivery::ConfigDelivery>,
11616 binding_publication: Option<ProgramBindingPublication>,
11617 launch_spec: ModuleSpec,
11618 spawn_admission: Option<crate::config_gate::runtime::SpawnAdmission>,
11619 command: Command,
11620 carrying: bool,
11621 carried: Option<Arc<containment::CarriedContainment>>,
11622 #[cfg(unix)]
11623 launch_identity: Option<containment::LaunchIdentity>,
11624 #[cfg(unix)]
11625 group_target: containment::GroupTarget,
11626 #[cfg(target_os = "linux")]
11627 cgroup_placement: Option<subc_cgroup::Placement>,
11628 #[cfg(target_os = "linux")]
11629 cgroup_name: String,
11630 #[cfg(target_os = "linux")]
11631 cgroup_cleanup: PreparedCgroup,
11632 cgroup_path: Option<PathBuf>,
11633 #[cfg(unix)]
11634 admission_gate: Option<admission::Gate>,
11635 output_sink: ChildOutputSink,
11636 #[cfg(target_os = "macos")]
11637 privacy_exec: Option<PrivacyExec>,
11638 #[cfg(windows)]
11639 image_capture: subc_os::ExecutableCapture,
11640 #[cfg(windows)]
11641 nonce_handoff: Option<NonceHandoff>,
11642 spawned_at: Instant,
11643 config_delivery: Option<crate::config_gate::runtime::DeliverySpawn>,
11644 delivery_registration: Option<config_delivery::RegistrationGuard>,
11645 program_binding: Option<SpawnProgramBinding>,
11646}
11647
11648#[cfg(target_os = "linux")]
11649struct PreparedCgroup(Option<(subc_cgroup::Placement, String)>);
11650#[cfg(target_os = "linux")]
11651impl Drop for PreparedCgroup {
11652 fn drop(&mut self) {
11653 if let Some((placement, name)) = &self.0 {
11654 remove_module_cgroup(placement, name);
11655 }
11656 }
11657}
11658
11659pub(crate) struct SpawnProgramBinding {
11660 pub(crate) path: PathBuf,
11661 _file: std::fs::File,
11664 pub(crate) identity: subc_os::RetainedFileIdentity,
11665}
11666
11667impl SpawnProgramBinding {
11668 pub(crate) fn capture(spec: &ModuleSpec) -> Result<Self, SuperviseError> {
11669 Self::capture_after_resolution(spec, |_| {})
11670 }
11671
11672 fn capture_after_resolution(
11673 spec: &ModuleSpec,
11674 before_open: impl FnOnce(&std::path::Path),
11675 ) -> Result<Self, SuperviseError> {
11676 require_program_search_path(spec)?;
11677 #[cfg(windows)]
11678 let windows_capture =
11679 windows_spawn_capture(spec).map_err(|source| SuperviseError::Spawn {
11680 program: spec.program.clone(),
11681 source,
11682 cgroup_path: None,
11683 })?;
11684 #[cfg(windows)]
11685 let path = windows_capture.path().to_path_buf();
11686 #[cfg(not(windows))]
11687 let path = if spec.program.is_absolute() || spec.program.components().count() > 1 {
11688 spec.program.clone()
11689 } else {
11690 let path = spec
11691 .env
11692 .iter()
11693 .rev()
11694 .find(|(key, _)| key == "PATH")
11695 .map(|(_, path)| std::ffi::OsString::from(path))
11696 .or_else(|| std::env::var_os("PATH"))
11697 .unwrap_or_else(|| {
11698 if cfg!(target_os = "macos") {
11699 "/usr/bin:/bin".into()
11702 } else {
11703 Default::default()
11704 }
11705 });
11706 std::env::split_paths(&path)
11707 .map(|dir| dir.join(&spec.program))
11708 .find(|path| {
11709 if !path.is_file() {
11710 return false;
11711 }
11712 #[cfg(unix)]
11713 {
11714 rustix::fs::accessat(
11715 rustix::fs::CWD,
11716 path,
11717 rustix::fs::Access::EXEC_OK,
11718 rustix::fs::AtFlags::EACCESS,
11719 )
11720 .is_ok()
11721 }
11722 #[cfg(not(unix))]
11723 {
11724 true
11725 }
11726 })
11727 .ok_or_else(|| SuperviseError::Spawn {
11728 program: spec.program.clone(),
11729 source: io::Error::new(io::ErrorKind::NotFound, "executable not found in PATH"),
11730 cgroup_path: None,
11731 })?
11732 };
11733 let resolved = std::fs::canonicalize(&path).map_err(|source| SuperviseError::Spawn {
11734 program: spec.program.clone(),
11735 source,
11736 cgroup_path: None,
11737 })?;
11738 before_open(&resolved);
11739 let file =
11742 subc_os::open_regular_nofollow(&resolved).map_err(|source| SuperviseError::Spawn {
11743 program: spec.program.clone(),
11744 source: io::Error::other(source),
11745 cgroup_path: None,
11746 })?;
11747 let identity =
11748 subc_os::retained_file_identity(&file).map_err(|source| SuperviseError::Spawn {
11749 program: spec.program.clone(),
11750 source,
11751 cgroup_path: None,
11752 })?;
11753 Ok(Self {
11754 path,
11755 _file: file,
11756 identity,
11757 })
11758 }
11759 fn current(&self) -> bool {
11760 #[cfg(unix)]
11761 {
11762 subc_os::file_identity(&self.path) == Some(self.identity)
11763 }
11764 #[cfg(windows)]
11765 {
11766 std::fs::canonicalize(&self.path)
11767 .ok()
11768 .and_then(|path| subc_os::open_regular_nofollow(&path).ok())
11769 .and_then(|file| subc_os::retained_file_identity(&file).ok())
11770 == Some(self.identity)
11771 }
11772 }
11773}
11774
11775fn spawn_plan_stale(spec: &ModuleSpec) -> SuperviseError {
11776 SuperviseError::ConfigDeliveryDeferred {
11777 module_id: spec.module_id.clone(),
11778 }
11779}
11780
11781fn require_program_search_path(spec: &ModuleSpec) -> Result<(), SuperviseError> {
11782 if !cfg!(target_os = "macos")
11783 && !spec.program.is_absolute()
11784 && spec.program.components().count() == 1
11785 && !spec.env.iter().any(|(key, _)| {
11786 if cfg!(windows) {
11787 key.eq_ignore_ascii_case("PATH")
11788 } else {
11789 key == "PATH"
11790 }
11791 })
11792 && std::env::var_os("PATH").is_none()
11793 {
11794 return Err(SuperviseError::Spawn {
11797 program: spec.program.clone(),
11798 source: io::Error::new(io::ErrorKind::InvalidInput, "bare program requires PATH"),
11799 cgroup_path: None,
11800 });
11801 }
11802 Ok(())
11803}
11804
11805async fn spawn_child_async(
11806 spec: &ModuleSpec,
11807 runtime: &SupervisorRuntimeConfig,
11808) -> Result<SupervisedChild, SuperviseError> {
11809 #[cfg(all(test, unix))]
11810 if let Some(source) = containment_tests::join_failure(spec, &runtime.child_roster) {
11811 containment::terminate_blocking(&runtime.child_roster.containment);
11812 return Err(SuperviseError::Spawn {
11813 program: spec.program.clone(),
11814 source,
11815 cgroup_path: None,
11816 });
11817 }
11818 let result = spawn_child_in_slot_async(spec, runtime, SpawnRole::Plain, false).await;
11819 if result
11820 .as_ref()
11821 .is_err_and(|error| !is_launch_deferral(error))
11822 {
11823 if runtime.child_roster.admission_pending(&spec.module_id) {
11824 containment::terminate_deferred(&runtime.child_roster.containment);
11825 } else {
11826 containment::terminate_blocking(&runtime.child_roster.containment);
11827 }
11828 }
11829 result
11830}
11831
11832async fn spawn_child_in_slot_async(
11833 spec: &ModuleSpec,
11834 runtime: &SupervisorRuntimeConfig,
11835 role: SpawnRole,
11836 alternate_slot: bool,
11837) -> Result<SupervisedChild, SuperviseError> {
11838 let gate = runtime
11839 .config_delivery
11840 .as_ref()
11841 .map(|delivery| delivery.0.module_gate(&spec.module_id))
11842 .transpose()
11843 .map_err(|error| SuperviseError::InvalidSpec {
11844 reason: error.to_string(),
11845 })?;
11846 if gate.as_ref().is_some_and(|gate| gate.try_lock().is_err()) {
11847 return Err(SuperviseError::ConfigDeliveryDeferred {
11848 module_id: spec.module_id.clone(),
11849 });
11850 }
11851 let prepared_spec = spec.clone();
11852 let prepared_runtime = runtime.clone();
11853 let prepared = tokio::task::spawn_blocking(move || {
11854 let prepared = prepare_child_in_slot(
11855 &prepared_spec,
11856 prepared_runtime.connection_file_path.as_deref(),
11857 prepared_runtime.supervisor_handle.as_ref(),
11858 prepared_runtime.capture_logs_dir.as_deref(),
11859 &prepared_runtime.child_roster,
11860 prepared_runtime.spawn_delivery(),
11861 #[cfg(target_os = "linux")]
11862 prepared_runtime.cgroup_placement.as_ref(),
11863 role,
11864 alternate_slot,
11865 true,
11866 );
11867 prepared
11868 })
11869 .await
11870 .map_err(|_| SuperviseError::InvalidSpec {
11871 reason: "configuration launch preparation stopped".into(),
11872 })??;
11873 if let Some(registration) = &prepared.delivery_registration {
11874 if !registration.launch_current()? {
11875 return Err(spawn_plan_stale(spec));
11876 }
11877 }
11878 let _gate = match gate {
11879 Some(gate) => {
11880 Some(
11881 gate.try_lock_owned()
11882 .map_err(|_| SuperviseError::ConfigDeliveryDeferred {
11883 module_id: spec.module_id.clone(),
11884 })?,
11885 )
11886 }
11887 None => None,
11888 };
11889 if let (Some(delivery), Some(admission)) = (runtime.spawn_delivery(), &prepared.spawn_admission)
11890 {
11891 delivery.check(spec, admission)?;
11892 }
11893 if prepared
11894 .spawn_admission
11895 .as_ref()
11896 .is_none_or(|admission| admission.launch.is_none())
11897 && prepared
11898 .program_binding
11899 .as_ref()
11900 .is_some_and(|binding| !binding.current())
11901 {
11902 return Err(spawn_plan_stale(spec));
11903 }
11904 let checked_base = prepared
11906 .spawn_admission
11907 .as_ref()
11908 .and_then(|admission| admission.approved_base())
11909 .cloned();
11910 let mut result = launch_child_in_slot(
11911 spec,
11912 &runtime.stderr_ring,
11913 &runtime.child_roster,
11914 prepared,
11915 runtime.register_timeout,
11916 );
11917 drop(_gate);
11918 if let (Some(delivery), Err(SuperviseError::ConfigHeld { code, detail, .. })) =
11919 (runtime.config_delivery.as_ref(), &result)
11920 {
11921 if let Some(base) = checked_base.filter(|_| code == "program_refused") {
11922 let module = spec.module_id.clone();
11923 let identity = subc_config::raw_digest(detail.as_bytes());
11924 let recorded = delivery
11925 .0
11926 .on_blocking(move |owner| owner.record_program_refusal(&module, &base, identity))
11927 .await;
11928 match recorded {
11929 Ok(true) => {}
11930 Ok(false) => result = Err(spawn_plan_stale(spec)),
11931 Err(error) => {
11932 result = Err(config_delivery::gate_hold_error(
11933 spec,
11934 error,
11935 Some(delivery),
11936 ))
11937 }
11938 }
11939 }
11940 }
11941 result
11942}
11943
11944fn validate_spawn_program(
11945 spec: &ModuleSpec,
11946 admission: Option<&crate::config_gate::runtime::SpawnAdmission>,
11947 delivery: Option<config_delivery::SpawnDelivery<'_>>,
11948) -> Result<(), SuperviseError> {
11949 if let Some(admission) = admission {
11950 if let Some(program) = &admission.launch {
11951 if let Err(error) = program.validate_for_spawn() {
11952 if let (Some(context), Some(base)) = (delivery, admission.approved_base()) {
11956 if error.code == "program_refused"
11957 && !context.record_refusal(spec, base, &error.detail)?
11958 {
11959 return Err(spawn_plan_stale(spec));
11960 }
11961 }
11962 return Err(config_delivery::gate_hold_error(
11963 spec,
11964 error,
11965 delivery.map(|context| context.runtime()).as_ref(),
11966 ));
11967 }
11968 }
11969 }
11970 Ok(())
11971}
11972
11973#[allow(clippy::too_many_arguments)]
11974fn prepare_child_in_slot(
11975 spec: &ModuleSpec,
11976 connection_file_path: Option<&std::path::Path>,
11977 handle: Option<&SupervisorHandle>,
11978 capture_logs_dir: Option<&std::path::Path>,
11979 roster: &ChildRoster,
11980 config_delivery: Option<config_delivery::SpawnDelivery<'_>>,
11981 #[cfg(target_os = "linux")] cgroup_placement: Option<&subc_cgroup::Placement>,
11982 role: SpawnRole,
11983 alternate_slot: bool,
11984 asynchronous: bool,
11985) -> Result<PreparedChild, SuperviseError> {
11986 require_program_search_path(spec)?;
11987 let spawn_admission = config_delivery
11988 .map(|delivery| delivery.prepare(spec))
11989 .transpose()?;
11990 #[cfg(target_os = "macos")]
11991 let gated = spawn_admission
11992 .as_ref()
11993 .is_some_and(|admission| admission.approved_base().is_some());
11994 let mut launch_spec = spawn_admission
11995 .as_ref()
11996 .and_then(|admission| admission.launch.as_ref())
11997 .map(|launch| launch.spec.clone())
11998 .unwrap_or_else(|| spec.clone());
11999 if let Some(script) = &launch_spec.script {
12000 launch_spec.program =
12001 script::runner_path(script.runtime).map_err(|source| SuperviseError::Spawn {
12002 program: launch_spec.program.clone(),
12003 source,
12004 cgroup_path: None,
12005 })?;
12006 }
12007 let script_content = if let Some(entry) = &launch_spec.script {
12008 Some(
12009 spawn_admission
12010 .as_ref()
12011 .and_then(|admission| admission.launch.as_ref())
12012 .and_then(|launch| launch.script.clone())
12013 .map_or_else(|| script::ScriptBytes::read(&entry.path), Ok)
12014 .map_err(|error| config_delivery::gate_hold_error(&launch_spec, error, None))?,
12015 )
12016 } else {
12017 None
12018 };
12019 let spec = &launch_spec;
12020 if let Some(error) = recovery_block_error(spec, roster) {
12021 return Err(error);
12022 }
12023 if let Some((pid, reason)) = roster.previous_instance(&spec.module_id) {
12024 return Err(previous_instance_error(spec, pid, reason));
12025 }
12026 if roster.is_closed() {
12027 return Err(SuperviseError::Spawn {
12028 program: spec.program.clone(),
12029 source: io::Error::other("the daemon is shutting down; not starting a new process"),
12030 cgroup_path: None,
12031 });
12032 }
12033 let carried = containment::current(&roster.containment);
12034 let carrying = carries_containment(spec.carry_containment, carried.is_some());
12035 if role != SpawnRole::Plain && carrying {
12036 return Err(SuperviseError::InvalidSpec {
12037 reason: "a carried container cannot overlap a swap candidate".to_owned(),
12038 });
12039 }
12040 #[cfg(unix)]
12041 let launch_identity = if carrying && carried.is_none() {
12042 Some(
12043 containment::LaunchIdentity::capture().map_err(|source| SuperviseError::Spawn {
12044 program: spec.program.clone(),
12045 source,
12046 cgroup_path: None,
12047 })?,
12048 )
12049 } else {
12050 None
12051 };
12052 #[cfg(target_os = "linux")]
12053 let cgroup_placement = match &carried {
12054 Some(container) => container.cgroup().map(|(placement, _)| placement),
12055 None => cgroup_placement,
12056 };
12057 #[cfg(unix)]
12058 let group_target = if let Some(container) = carried
12059 .as_ref()
12060 .filter(|container| container.group().is_some())
12061 {
12062 containment::GroupTarget::Pinned(Arc::clone(container))
12063 } else {
12064 #[cfg(target_os = "linux")]
12065 let cgroup_container = carrying && cgroup_placement.is_some();
12066 #[cfg(not(target_os = "linux"))]
12067 let cgroup_container = false;
12068 if cgroup_container {
12069 containment::GroupTarget::NoGroup
12070 } else {
12071 containment::GroupTarget::OwnLeader
12072 }
12073 };
12074 #[cfg(target_os = "linux")]
12075 let cgroup_name = {
12076 if let Some((_, name)) = carried.as_ref().and_then(|container| container.cgroup()) {
12081 name.to_owned()
12082 } else if cgroup_placement.is_none() {
12083 swap::cgroup_name(&spec.module_id, alternate_slot)
12084 } else {
12085 let nonce = generate_launch_nonce()?;
12086 let suffix = u128::from_str_radix(&nonce[..32], 16).expect("hex launch nonce");
12087 let mut end = spec.module_id.len().min(64);
12090 while !spec.module_id.is_char_boundary(end) {
12091 end -= 1;
12092 }
12093 format!(
12094 "{}_{suffix}",
12095 swap::cgroup_name(&spec.module_id[..end], alternate_slot)
12096 )
12097 }
12098 };
12099 #[cfg(not(target_os = "linux"))]
12100 let _ = alternate_slot;
12101 #[cfg(windows)]
12102 let image_capture = windows_spawn_capture(spec).map_err(|source| SuperviseError::Spawn {
12103 program: spec.program.clone(),
12104 source,
12105 cgroup_path: None,
12106 })?;
12107 #[cfg(windows)]
12108 let program_binding = if asynchronous {
12109 let mut resolved = spec.clone();
12110 resolved.program = image_capture.path().to_path_buf();
12111 Some(SpawnProgramBinding::capture(&resolved)?)
12112 } else {
12113 None
12114 };
12115 #[cfg(not(windows))]
12116 let program_binding = (asynchronous || cfg!(target_os = "macos"))
12117 .then(|| SpawnProgramBinding::capture(spec))
12118 .transpose()?;
12119 #[cfg(target_os = "macos")]
12120 validate_spawn_program(spec, spawn_admission.as_ref(), config_delivery)?;
12121 #[cfg(target_os = "macos")]
12122 let (mut command, privacy_exec, exec_ack) = privacy_command(
12123 spec,
12124 program_binding
12125 .as_ref()
12126 .expect("macOS captures every launch"),
12127 roster,
12128 gated,
12132 )?;
12133 #[cfg(windows)]
12134 let mut command = Command::new(image_capture.path());
12135 #[cfg(not(any(target_os = "macos", windows)))]
12136 let mut command = Command::new(
12137 program_binding
12138 .as_ref()
12139 .map_or(spec.program.as_path(), |binding| binding.path.as_path()),
12140 );
12141 command.args(&spec.args);
12142 apply_child_env(&mut command, spec);
12172 if spec.script.is_some() {
12173 script::remove_runtime_env(&mut command, spec);
12174 }
12175 let origin_witness = crate::program_origin::LaunchWitness::capture_command(spec, &command)
12176 .map_err(|source| SuperviseError::Spawn {
12177 program: spec.program.clone(),
12178 source,
12179 cgroup_path: None,
12180 })?;
12181 command.current_dir(&origin_witness.working_directory);
12182 apply_spawn_role(&mut command, role);
12183 let nonce_handoff = apply_wire_spawn_args_for_admission(
12184 &mut command,
12185 spec,
12186 connection_file_path,
12187 handle,
12188 role,
12189 spawn_admission.as_ref(),
12190 )?;
12191 #[cfg(unix)]
12192 let mut slots = subc_os::registration_channel::ChildDescriptorSlots::default();
12193 command.env_remove(subc_os::script_pipe::SCRIPT_FD_ENV);
12194 let script_delivery = script_content
12195 .map(|content| {
12196 script::Delivery::prepare(
12197 content,
12198 &mut command,
12199 #[cfg(unix)]
12200 &mut slots,
12201 )
12202 })
12203 .transpose()
12204 .map_err(|source| SuperviseError::Spawn {
12205 program: spec.program.clone(),
12206 source,
12207 cgroup_path: None,
12208 })?;
12209 #[cfg(unix)]
12210 let registration_channel = if spec.protocol == ModuleProtocol::Subc {
12211 let (parent, child, metadata) = subc_os::registration_channel::registration_socketpair()
12212 .map_err(|source| SuperviseError::Spawn {
12213 program: spec.program.clone(),
12214 source,
12215 cgroup_path: None,
12216 })?;
12217 slots
12218 .channel(child)
12219 .map_err(|source| SuperviseError::Spawn {
12220 program: spec.program.clone(),
12221 source,
12222 cgroup_path: None,
12223 })?;
12224 command
12225 .env(subc_os::registration_channel::REGISTRATION_FD_ENV, metadata)
12226 .env(
12227 subc_os::registration_channel::DAEMON_PID_ENV,
12228 std::process::id().to_string(),
12229 );
12230 Some(parent)
12231 } else {
12232 None
12233 };
12234 #[cfg(windows)]
12235 let registration_channel = prepare_windows_registration_pipe(&mut command, spec)?;
12236 let binding_publication = if spawn_admission
12237 .as_ref()
12238 .is_some_and(|admission| admission.launch.is_some())
12239 {
12240 handle.and_then(|handle| handle.begin_program_binding(&spec.module_id, role))
12241 } else {
12242 None
12243 };
12244 let registration_spawn = (spec.protocol == ModuleProtocol::Subc)
12245 .then(|| {
12246 handle.map(|handle| {
12247 let spawn = RegistrationSpawn::new_with_binding_requirement(
12248 handle,
12249 &spec.module_id,
12250 role,
12251 binding_publication.clone(),
12252 spawn_admission
12253 .as_ref()
12254 .is_some_and(|admission| admission.launch.is_some()),
12255 );
12256 if let Some(admission) = &spawn_admission {
12257 admission.initialize_origin(&spawn.authority, &origin_witness);
12258 }
12259 handle.record_nonce_launch(&spawn);
12260 spawn
12261 })
12262 })
12263 .flatten();
12264
12265 #[cfg(target_os = "linux")]
12266 let cgroup_path = cgroup_placement
12267 .map(|placement| {
12268 if let Some(container) = &carried {
12269 let path = container
12270 .cgroup_path()
12271 .ok_or_else(|| io::Error::other("carried container has no cgroup path"))?;
12272 placement.existing_module_path(path)?.ok_or_else(|| {
12273 io::Error::new(
12274 io::ErrorKind::NotFound,
12275 "carried cgroup disappeared before replacement",
12276 )
12277 })
12278 } else {
12279 if carrying {
12280 containment::prepare_cgroup_path(placement, &cgroup_name)
12281 } else {
12282 placement.module_path(&cgroup_name)
12283 }
12284 }
12285 })
12286 .transpose()
12287 .map_err(|source| SuperviseError::Cgroup {
12288 module_id: spec.module_id.clone(),
12289 source,
12290 })?;
12291 #[cfg(not(target_os = "linux"))]
12292 let cgroup_path: Option<PathBuf> = None;
12293 #[cfg(target_os = "linux")]
12296 let cgroup_cleanup = PreparedCgroup(
12297 cgroup_placement
12298 .filter(|_| carried.is_none())
12299 .map(|placement| (placement.clone(), cgroup_name.clone())),
12300 );
12301 #[cfg(unix)]
12302 let mut admission_gate = if carrying {
12303 Some(
12304 admission::Gate::install(&mut command, spec, roster).map_err(|source| {
12305 SuperviseError::Spawn {
12306 program: spec.program.clone(),
12307 source,
12308 cgroup_path: cgroup_path.clone(),
12309 }
12310 })?,
12311 )
12312 } else {
12313 None
12314 };
12315 #[cfg(target_os = "linux")]
12316 if let Some(path) = &cgroup_path {
12317 apply_cgroup_placement(&mut command, spec, path)?;
12318 }
12319
12320 let output_sink = if let Some(logs_dir) = capture_logs_dir {
12321 let path = logs_dir.join(format!("{}.stderr.log", spec.module_id));
12322 match ChildOutputSink::open(&path, capture_retention(spec)) {
12323 Ok(sink) => sink,
12324 Err(error) => {
12325 warn!(
12326 module_id = %spec.module_id,
12327 path = %path.display(),
12328 error = %error,
12329 "could not open child output capture file; forwarding to stderr"
12330 );
12331 ChildOutputSink::Stderr
12332 }
12333 }
12334 } else {
12335 ChildOutputSink::Stderr
12336 };
12337
12338 command.stdout(Stdio::piped());
12339 command.stderr(Stdio::piped());
12340 command.kill_on_drop(!carrying);
12341 #[cfg(unix)]
12358 command.process_group(
12359 carried
12360 .as_ref()
12361 .map(|container| container.verified_join_group())
12362 .transpose()
12363 .map_err(|source| SuperviseError::Spawn {
12364 program: spec.program.clone(),
12365 source,
12366 cgroup_path: cgroup_path.clone(),
12367 })?
12368 .flatten()
12369 .unwrap_or(0) as i32,
12370 );
12371 command.stdin(Stdio::null());
12372 #[cfg(unix)]
12373 if let Some(gate) = &mut admission_gate {
12374 gate.install_placement(&mut command);
12375 }
12376 #[cfg(unix)]
12379 {
12380 if let Some(handoff) = nonce_handoff {
12381 slots.nonce(handoff);
12382 }
12383 #[cfg(target_os = "macos")]
12384 slots
12385 .acknowledgement(exec_ack.into_descriptor())
12386 .map_err(|source| SuperviseError::Spawn {
12387 program: spec.program.clone(),
12388 source,
12389 cgroup_path: cgroup_path.clone(),
12390 })?;
12391 slots.install(command.as_std_mut());
12392 }
12393 #[cfg(not(any(unix, windows)))]
12394 let _ = nonce_handoff;
12395
12396 #[cfg(windows)]
12401 {
12402 subc_jobobject::ensure_root_job().map_err(|source| SuperviseError::Spawn {
12403 program: spec.program.clone(),
12404 source,
12405 cgroup_path: None,
12406 })?;
12407 subc_jobobject::suspend_on_create_async(&mut command);
12408 }
12409 let spawned_at = Instant::now();
12413 let delivery_context = config_delivery;
12414 let config_delivery = config_delivery
12415 .map(|delivery| {
12416 delivery.spawned(
12417 spec,
12418 spawn_admission
12419 .as_ref()
12420 .expect("delivery prepares spawn admission"),
12421 spawned_at,
12422 role,
12423 registration_spawn
12424 .as_ref()
12425 .map(|spawn| spawn.authority.clone()),
12426 )
12427 })
12428 .transpose()?
12429 .flatten();
12430 let mut delivery_registration =
12431 delivery_context.map(|context| context.registration_guard(config_delivery.as_ref(), role));
12432 if let Some(registration) = &mut delivery_registration {
12433 registration.defer_cleanup();
12434 }
12435 if asynchronous {
12436 #[cfg(test)]
12437 if let Some(context) = delivery_context {
12438 context.pause_preflight_for_test();
12439 }
12440 }
12441 validate_spawn_program(spec, spawn_admission.as_ref(), delivery_context)?;
12442 Ok(PreparedChild {
12443 script_delivery,
12444 registration_spawn,
12445 registration_handle: handle.cloned(),
12446 #[cfg(any(unix, windows))]
12447 registration_channel,
12448 config_context: delivery_context.map(|delivery| delivery.runtime()),
12449 binding_publication,
12450 launch_spec,
12451 spawn_admission,
12452 command,
12453 carrying,
12454 carried: carried.clone(),
12455 #[cfg(unix)]
12456 launch_identity,
12457 #[cfg(unix)]
12458 group_target,
12459 #[cfg(target_os = "linux")]
12460 cgroup_placement: cgroup_placement.cloned(),
12461 #[cfg(target_os = "linux")]
12462 cgroup_cleanup,
12463 #[cfg(target_os = "linux")]
12464 cgroup_name,
12465 cgroup_path,
12466 #[cfg(unix)]
12467 admission_gate,
12468 output_sink,
12469 #[cfg(target_os = "macos")]
12470 privacy_exec,
12471 #[cfg(windows)]
12472 image_capture,
12473 #[cfg(windows)]
12474 nonce_handoff,
12475 spawned_at,
12476 config_delivery,
12477 delivery_registration,
12478 program_binding,
12479 })
12480}
12481
12482fn launch_child_in_slot(
12483 _spec: &ModuleSpec,
12484 ring: &Arc<Mutex<StderrRing>>,
12485 roster: &ChildRoster,
12486 prepared: PreparedChild,
12487 register_timeout: Duration,
12488) -> Result<SupervisedChild, SuperviseError> {
12489 let launch_spec = prepared.launch_spec.clone();
12490 let registration_declaration = prepared
12491 .spawn_admission
12492 .as_ref()
12493 .and_then(|admission| admission.registration_declaration(&launch_spec.module_id));
12494 let admission_base = prepared
12495 .spawn_admission
12496 .as_ref()
12497 .and_then(|admission| admission.approved_base())
12498 .cloned();
12499 let spec = &launch_spec;
12500 let expected_image = prepared
12501 .spawn_admission
12502 .as_ref()
12503 .and_then(|admission| admission.launch.as_ref())
12504 .map(|launch| launch.image);
12505 let program_launch = prepared
12506 .spawn_admission
12507 .and_then(|admission| admission.launch);
12508 let config_owner = prepared
12509 .config_context
12510 .as_ref()
12511 .map(|context| context.0.clone());
12512 let binding_publication = prepared.binding_publication;
12513 #[cfg(test)]
12514 if let Some(context) = &prepared.config_context {
12515 context.0.before_spawn_exec_for_test();
12516 }
12517 #[cfg(feature = "test-support")]
12518 if let Some(context) = &prepared.config_context {
12519 context
12520 .0
12521 .pause_spawn_before_exec()
12522 .map_err(|error| config_delivery::gate_hold_error(spec, error, Some(context)))?;
12523 }
12524 let mut command = prepared.command;
12525 let carrying = prepared.carrying;
12526 let carried = prepared.carried;
12527 #[cfg(unix)]
12528 let launch_identity = prepared.launch_identity;
12529 #[cfg(unix)]
12530 let group_target = prepared.group_target;
12531 #[cfg(target_os = "linux")]
12532 let cgroup_placement = prepared.cgroup_placement.as_ref();
12533 #[cfg(target_os = "linux")]
12534 let cgroup_name = prepared.cgroup_name;
12535 #[cfg(target_os = "linux")]
12536 let mut cgroup_cleanup = prepared.cgroup_cleanup;
12537 #[cfg(target_os = "linux")]
12538 let _ = cgroup_cleanup.0.take();
12539 let cgroup_path = prepared.cgroup_path;
12540 #[cfg(unix)]
12541 let admission_gate = prepared.admission_gate;
12542 let output_sink = prepared.output_sink;
12543 #[cfg(target_os = "macos")]
12544 let mut privacy_exec = prepared.privacy_exec;
12545 #[cfg(target_os = "macos")]
12546 if let Some(pending) = &mut privacy_exec {
12547 pending.admission_binding = binding_publication.clone();
12548 if expected_image.is_some() {
12549 pending.expected = expected_image;
12550 }
12551 }
12552 #[cfg(any(unix, windows))]
12553 let registration_channel = prepared.registration_channel;
12554 let registration_spawn = prepared.registration_spawn;
12555 let registration_handle = prepared.registration_handle;
12556 #[cfg(windows)]
12557 let image_capture = prepared.image_capture;
12558 #[cfg(windows)]
12559 let nonce_handoff = prepared.nonce_handoff;
12560 let spawned_at = prepared.spawned_at;
12561 let config_delivery = prepared.config_delivery;
12562 let delivery_registration = prepared.delivery_registration;
12563 let _program_binding = prepared.program_binding;
12564 #[cfg(all(test, unix))]
12565 containment_tests::record_spawn_attempt(spec, &mut command);
12566 #[cfg(unix)]
12567 let spawned = match admission_gate {
12568 Some(gate) => gate
12569 .spawn(
12570 command,
12571 spec,
12572 roster,
12573 launch_identity,
12574 #[cfg(target_os = "linux")]
12575 cgroup_placement
12576 .zip(cgroup_path.as_ref())
12577 .map(|(placement, path)| {
12578 (placement.clone(), cgroup_name.clone(), path.clone())
12579 }),
12580 )
12581 .map(|(child, admitted)| (child, Some(admitted))),
12582 None => {
12583 let spawned =
12584 subc_os::child_spawn::tokio_spawn(&mut command).map(|child| (child, None));
12585 drop(command);
12588 spawned
12589 }
12590 };
12591 #[cfg(not(unix))]
12592 let spawned = subc_os::child_spawn::tokio_spawn(&mut command);
12593 let spawned = match spawned {
12594 Ok(child) => child,
12595 Err(source) => {
12596 if carried
12597 .as_ref()
12598 .and_then(|container| container.group())
12599 .is_some()
12600 {
12601 containment::terminate_blocking(&roster.containment);
12602 }
12603 #[cfg(target_os = "linux")]
12604 if let Some(placement) = cgroup_placement.filter(|_| carried.is_none()) {
12605 remove_module_cgroup(placement, &cgroup_name);
12606 }
12607 return Err(SuperviseError::Spawn {
12608 program: spec.program.clone(),
12609 source,
12610 cgroup_path,
12611 });
12612 }
12613 };
12614 #[cfg(unix)]
12615 let (mut child, mut admitted) = spawned;
12616 #[cfg(unix)]
12617 let group_target = admitted
12618 .as_ref()
12619 .map(|attempt| attempt.group_target())
12620 .unwrap_or(group_target);
12621 #[cfg(not(unix))]
12622 let mut child = spawned;
12623 if let Some(delivery) = prepared.script_delivery {
12624 delivery.start(child.id().expect("spawned child has a pid"));
12625 }
12626 #[cfg(test)]
12627 spawn_record_tests::after_create(&child, spec);
12628
12629 #[cfg(windows)]
12634 let windows_image = match image_capture.bind_tokio(&child) {
12635 Ok(image) => Arc::new(image),
12636 Err(source) => {
12637 kill_and_reap_rejected_child(&mut child);
12638 return Err(SuperviseError::Spawn {
12639 program: spec.program.clone(),
12640 source,
12641 cgroup_path: None,
12642 });
12643 }
12644 };
12645 #[cfg(windows)]
12648 let job = contain_spawned_child(&mut child, spec, carried.as_ref())?;
12649 let spawned_at_ms = unix_ms_now();
12650 #[cfg(not(windows))]
12651 let spawned_from = spec.program.clone();
12652 #[cfg(windows)]
12653 let spawned_from = windows_image.path().to_path_buf();
12654 #[cfg(not(windows))]
12655 let spawned_file_identity = spawned_file_identity(&spawned_from);
12656 #[cfg(windows)]
12657 let spawned_file_identity = Some(SpawnedFileIdentity {
12658 device: 0,
12659 inode: 0,
12660 windows_image: Some(Arc::clone(&windows_image)),
12661 });
12662 let pid = child.id().ok_or_else(|| SuperviseError::Spawn {
12663 program: spec.program.clone(),
12664 source: io::Error::other("spawned child exposed no live pid"),
12665 cgroup_path: cgroup_path.clone(),
12666 })?;
12667 #[cfg(windows)]
12668 let nonce_delivery = start_windows_nonce_pipe(nonce_handoff, pid, spec);
12669 #[cfg(not(windows))]
12670 let process_start_time = admitted
12671 .as_ref()
12672 .map(|admitted| admitted.birth)
12673 .or_else(|| crate::provenance::process_start_time(pid));
12674 #[cfg(windows)]
12675 let process_start_time = Some(windows_image.start_time());
12676 let process_identity = process_start_time.map(|start_time| ProcessIdentity { pid, start_time });
12677 #[cfg(all(test, target_os = "macos"))]
12678 privacy_exec_boundary_tests::before_image_sample(spec, pid);
12679 #[cfg(all(test, target_os = "linux"))]
12680 assert!(roster.admission_event_for_test(&spec.module_id, "before_image_binding", Some(pid)));
12681 #[cfg(not(windows))]
12686 let recorded_image = observe_spawned_image(pid);
12687 #[cfg(all(test, target_os = "linux"))]
12688 let recorded_image = if roster.admission_event_for_test(
12689 &spec.module_id,
12690 "after_spawn_image_sample",
12691 Some(pid),
12692 ) {
12693 recorded_image
12694 } else {
12695 None
12696 };
12697 #[cfg(target_os = "linux")]
12698 let exited_before_binding = expected_image.is_some()
12701 && recorded_image.and_then(|image| image.executable) != expected_image
12702 && child.try_wait().ok().flatten().is_some();
12703 #[cfg(target_os = "linux")]
12704 let unavailable_initial_image = expected_image.is_some()
12705 && recorded_image.and_then(|image| image.executable).is_none()
12706 && !exited_before_binding;
12707 #[cfg(not(target_os = "linux"))]
12708 let unavailable_initial_image = false;
12709 #[cfg(target_os = "linux")]
12710 if expected_image
12711 .is_some_and(|expected| recorded_image.and_then(|image| image.executable) != Some(expected))
12712 && !exited_before_binding
12713 && !unavailable_initial_image
12714 {
12715 kill_module_cgroup(cgroup_placement, &cgroup_name);
12716 kill_and_reap_rejected_child(&mut child, group_target.clone());
12717 return Err(SuperviseError::ConfigHeld {
12718 module_id: spec.module_id.clone(),
12719 code: "program_refused".into(),
12720 detail: format!(
12721 "spawned executable {:?} does not match hashed program {:?}",
12722 recorded_image.and_then(|image| image.executable),
12723 expected_image
12724 ),
12725 });
12726 }
12727 #[cfg(not(target_os = "linux"))]
12728 let _ = expected_image;
12729 #[cfg(target_os = "macos")]
12730 let binding_ready = privacy_exec.is_none();
12731 #[cfg(target_os = "linux")]
12732 let binding_ready = !exited_before_binding;
12733 #[cfg(not(any(target_os = "macos", target_os = "linux")))]
12734 let binding_ready = cfg!(windows);
12735 #[cfg(target_os = "macos")]
12739 let recorded_image = if privacy_exec.is_some() {
12740 None
12741 } else {
12742 recorded_image
12743 };
12744 #[cfg(target_os = "linux")]
12745 let recorded_cgroup_name = cgroup_path.as_ref().map(|_| cgroup_name.clone());
12746 #[cfg(not(target_os = "linux"))]
12747 let recorded_cgroup_name = None;
12748 #[cfg(unix)]
12749 let preadmitted = admitted
12750 .as_mut()
12751 .and_then(|admitted| admitted.leader.take());
12752 #[cfg(not(unix))]
12753 let preadmitted: Option<crate::child_roster::RosterGuard> = None;
12754 let roster_guard = if let Some(guard) = preadmitted {
12755 Ok(guard)
12756 } else {
12757 roster.admit(
12758 spec.module_id.clone(),
12759 pid,
12760 spec.protocol,
12761 process_start_time,
12762 crate::child_roster::RecordedIdentity {
12763 container: None,
12764 #[cfg(unix)]
12765 carried: None,
12766 #[cfg(not(windows))]
12767 start_time: recorded_image.map(|image| image.start_time),
12768 #[cfg(windows)]
12769 start_time: process_start_time,
12770 #[cfg(not(windows))]
12771 executable: recorded_image
12772 .and_then(|image| image.executable)
12773 .map(crate::live_children::ExecutableIdentity::from),
12774 #[cfg(windows)]
12775 executable: Some(crate::live_children::ExecutableIdentity::from(
12776 windows_image.identity(),
12777 )),
12778 cgroup_name: recorded_cgroup_name,
12779 #[cfg(target_os = "linux")]
12780 cgroup_placement: cgroup_placement.cloned(),
12781 },
12782 )
12783 };
12784 let roster_guard = match roster_guard {
12785 Ok(guard) => guard,
12786 Err(failure) => {
12787 return failure.with_cleanup(|source| {
12788 #[cfg(windows)]
12789 if let Some(job) = &job {
12790 let _ = job.lock().unwrap_or_else(|p| p.into_inner()).terminate();
12791 }
12792 #[cfg(target_os = "linux")]
12793 kill_module_cgroup(cgroup_placement, &cgroup_name);
12794 #[cfg(unix)]
12795 let rejected_group = group_target.clone();
12796 kill_and_reap_rejected_child(
12797 &mut child,
12798 #[cfg(unix)]
12799 rejected_group,
12800 );
12801 #[cfg(target_os = "linux")]
12802 if let Some(placement) = cgroup_placement.filter(|_| carried.is_none()) {
12803 remove_module_cgroup(placement, &cgroup_name);
12804 }
12805 Err(SuperviseError::Spawn {
12806 program: spec.program.clone(),
12807 source: io::Error::other(format!(
12808 "could not durably record module child: {source}"
12809 )),
12810 cgroup_path,
12811 })
12812 });
12813 }
12814 };
12815 #[cfg(unix)]
12816 if admitted.is_some() {
12817 if let Some(image) = recorded_image {
12818 if let Err(source) = roster_guard.confirm_executable(image) {
12819 kill_and_reap_rejected_child(&mut child, group_target.clone());
12820 return Err(SuperviseError::Spawn {
12821 program: spec.program.clone(),
12822 source,
12823 cgroup_path,
12824 });
12825 }
12826 }
12827 }
12828 if roster.is_closed() {
12833 if carrying {
12834 #[cfg(unix)]
12835 let rejected_group = admitted
12836 .as_ref()
12837 .map(|admitted| admitted.rejection_target())
12838 .unwrap_or_else(|| group_target.clone());
12839 #[cfg(windows)]
12840 if let Some(job) = &job {
12841 let _ = job.lock().unwrap_or_else(|p| p.into_inner()).terminate();
12842 }
12843 kill_and_reap_rejected_child(
12844 &mut child,
12845 #[cfg(unix)]
12846 rejected_group,
12847 );
12848 #[cfg(all(test, unix))]
12849 let _ =
12850 roster.admission_event_for_test(&spec.module_id, "after_rejected_child", Some(pid));
12851 drop(roster_guard);
12852 return Err(SuperviseError::Spawn {
12853 program: spec.program.clone(),
12854 source: io::Error::other(
12855 "the daemon began shutting down during carried admission; ended the container",
12856 ),
12857 cgroup_path,
12858 });
12859 }
12860 #[cfg(target_os = "linux")]
12862 kill_module_cgroup(cgroup_placement, &cgroup_name);
12863 if let Err(error) = child.start_kill() {
12864 debug!(module_id = %spec.module_id, pid, %error, "kill of a process spawned during daemon shutdown failed; it may already have exited");
12865 }
12866 #[cfg(target_os = "linux")]
12867 if let Some(placement) = cgroup_placement.filter(|_| carried.is_none()) {
12868 while matches!(child.try_wait(), Ok(None)) {
12872 std::thread::yield_now();
12873 }
12874 if matches!(
12875 subc_cgroup::kill_module(Some(placement), &cgroup_name),
12876 subc_cgroup::KillOutcome::Killed
12877 ) {
12878 if let Ok(path) = placement.module_path(&cgroup_name) {
12879 while std::fs::read_to_string(path.join("cgroup.events"))
12880 .ok()
12881 .is_some_and(|events| events.lines().any(|line| line == "populated 1"))
12882 {
12883 std::thread::yield_now();
12884 }
12885 }
12886 }
12887 remove_module_cgroup(placement, &cgroup_name);
12888 }
12889 drop(roster_guard);
12890 return Err(SuperviseError::Spawn {
12891 program: spec.program.clone(),
12892 source: io::Error::other(
12893 "the daemon began shutting down while this process was starting; ended it",
12894 ),
12895 cgroup_path,
12896 });
12897 }
12898
12899 let stdout_pump = match child.stdout.take() {
12900 Some(stdout) => Some(tokio::spawn(pump_stdout_to(stdout, output_sink.clone()))),
12901 None => {
12902 warn!(
12903 module_id = %spec.module_id,
12904 "spawned child exposed no stdout pipe; file capture will be incomplete"
12905 );
12906 None
12907 }
12908 };
12909 let stderr_pump = match child.stderr.take() {
12910 Some(stderr) => {
12911 let generation = ring
12912 .lock()
12913 .unwrap_or_else(|poisoned| poisoned.into_inner())
12914 .begin_process();
12915 Some(StderrPump {
12916 task: tokio::spawn(pump_stderr_to(
12917 stderr,
12918 Arc::clone(ring),
12919 generation,
12920 output_sink,
12921 )),
12922 generation,
12923 })
12924 }
12925 None => {
12926 ring.lock()
12930 .unwrap_or_else(|poisoned| poisoned.into_inner())
12931 .mark_not_captured("stderr pipe was not available on spawn");
12932 warn!(
12933 module_id = %spec.module_id,
12934 "spawned child exposed no stderr pipe; tail will be unavailable"
12935 );
12936 None
12937 }
12938 };
12939
12940 let exec = Arc::new(crate::activity::ExecOutcome::new());
12941 #[cfg(target_os = "macos")]
12942 let pending_exec = privacy_exec.is_some();
12943 #[cfg(not(target_os = "macos"))]
12944 let pending_exec = false;
12945 if !pending_exec {
12946 exec.finish(crate::activity::ExecState::CommittedExec);
12947 }
12948 #[cfg(windows)]
12949 let subtree_target = job
12950 .as_ref()
12951 .map(|job| crate::subtree_resources::Target::Job(Arc::downgrade(job)))
12952 .unwrap_or(crate::subtree_resources::Target::Unavailable("no_job"));
12953 #[cfg(target_os = "linux")]
12954 let subtree_target = cgroup_path
12955 .map(crate::subtree_resources::Target::Cgroup)
12956 .unwrap_or(crate::subtree_resources::Target::Unavailable("no_cgroup"));
12957 #[cfg(not(any(target_os = "linux", windows)))]
12958 let subtree_target = crate::subtree_resources::Target::Unavailable("unsupported_platform");
12959 let mut supervised = SupervisedChild {
12960 registration_spawn,
12961 #[cfg(any(unix, windows))]
12962 registration_channel,
12963 live_program_binding: None,
12964 registered_config_connection: None,
12965 admission_base,
12966 registration_declaration,
12967 _program_binding: binding_publication,
12968 subtree_target,
12969 child: containment::ProcessChild::plain(child, pid),
12970 exec: crate::activity::ExecOwner(exec),
12971 #[cfg(windows)]
12972 nonce_delivery,
12973 protocol: spec.protocol,
12974 #[cfg(target_os = "linux")]
12975 module_id: cgroup_name,
12976 #[cfg(target_os = "linux")]
12977 cgroup_placement: cgroup_placement.cloned(),
12978 #[cfg(windows)]
12979 job,
12980 stdout_pump,
12981 stderr_pump,
12982 stderr_ring: Arc::clone(ring),
12983 spawned_at,
12984 config_delivery,
12985 registration_deadline: None,
12986 registration_watch: None,
12987 spawned_at_ms,
12988 spawned_from,
12989 launch_spec_digest: LaunchSpecDigest::from_spec(spec),
12990 spawned_file_identity,
12991 process_start_time,
12992 process_identity,
12993 pid,
12994 roster_guard: Some(roster_guard),
12995 #[cfg(target_os = "macos")]
12996 privacy_exec,
12997 #[cfg(target_os = "macos")]
12998 report_ready: Arc::new(OnceLock::new()),
12999 spawn_failure: None,
13000 admission_kill_sent: false,
13001 };
13002 if let Err(error) = containment::adopt(&mut supervised, spec, roster) {
13003 supervised.child.reject(
13004 #[cfg(unix)]
13005 group_target,
13006 );
13007 #[cfg(all(test, unix))]
13008 let _ = roster.admission_event_for_test(&spec.module_id, "after_rejected_child", Some(pid));
13009 return Err(error);
13010 }
13011 if let (Some(launch), Some(owner), Some(base)) = (
13012 program_launch,
13013 config_owner,
13014 supervised.admission_base.clone(),
13015 ) {
13016 supervised.live_program_binding =
13017 content_binding::LiveProgramBinding::new(launch, &supervised, owner, base);
13018 if let (Some(publication), Some(live)) = (
13019 &supervised._program_binding,
13020 &supervised.live_program_binding,
13021 ) {
13022 publication.attach(live);
13023 }
13024 }
13025 #[cfg(unix)]
13026 if let Some(admitted) = admitted.take() {
13027 admitted.disarm();
13028 }
13029 #[cfg(windows)]
13030 if let Err(error) = resume_suspended_child(pid, spec) {
13031 containment::terminate_blocking(&roster.containment);
13032 let _ = supervised.child.start_kill();
13033 return Err(error);
13034 }
13035 if binding_ready {
13036 let checked = if unavailable_initial_image {
13040 Err(crate::config_gate::GateError::new(
13041 "config_delivery_deferred",
13042 "initial process image is unavailable",
13043 ))
13044 } else {
13045 match (
13046 &supervised._program_binding,
13047 &supervised.live_program_binding,
13048 ) {
13049 (Some(binding), _) => binding.verify(),
13050 (None, Some(live)) => live.check(),
13051 _ if supervised.admission_base.is_some() => {
13052 Err(crate::config_gate::GateError::new(
13053 "config_delivery_deferred",
13054 "live program witness is unavailable",
13055 ))
13056 }
13057 _ => Ok(()),
13058 }
13059 };
13060 #[cfg(all(test, target_os = "linux"))]
13061 let checked = if roster.admission_event_for_test(
13062 &spec.module_id,
13063 "after_content_witness_check",
13064 Some(pid),
13065 ) {
13066 checked
13067 } else {
13068 Err(crate::config_gate::GateError::new(
13069 "config_delivery_deferred",
13070 "current process image is unavailable",
13071 ))
13072 };
13073 if let Err(error) = checked {
13074 if let Some(binding) = &mut supervised._program_binding {
13075 binding.reject();
13076 }
13077 if error.code == "program_refused" || supervised.try_wait().ok().flatten().is_none() {
13080 supervised.spawn_failure = Some(format!("{}: {}", error.code, error.detail));
13081 #[cfg(all(test, target_os = "linux"))]
13082 assert!(roster.admission_event_for_test(
13083 &spec.module_id,
13084 if unavailable_initial_image {
13085 "before_initial_image_kill"
13086 } else {
13087 "before_content_witness_kill"
13088 },
13089 Some(pid),
13090 ));
13091 let _ = supervised.start_admission_kill();
13092 #[cfg(all(test, target_os = "linux"))]
13093 if supervised.admission_kill_sent {
13094 assert!(roster.admission_event_for_test(
13095 &spec.module_id,
13096 "after_admission_kill_sent",
13097 Some(pid),
13098 ));
13099 }
13100 }
13101 }
13102 }
13103 if let Some(registration) = delivery_registration {
13104 registration.commit();
13105 }
13106 if let Some(spawn) = &supervised.registration_spawn {
13107 if let Some(identity) = supervised.process_identity {
13108 let _ = spawn.identity.set(identity);
13109 }
13110 if supervised.spawn_failure.is_some() {
13111 spawn.refuse();
13112 } else if binding_ready {
13113 spawn.ready();
13114 }
13115 #[cfg(any(unix, windows))]
13116 if let (Some(handle), Some(channel)) =
13117 (registration_handle, supervised.registration_channel.take())
13118 {
13119 handle.serve_registration(channel, spawn.clone(), pid, spawned_at + register_timeout);
13120 }
13121 }
13122 Ok(supervised)
13123}
13124
13125#[cfg(test)]
13126mod spawn_record_tests {
13127 use super::*;
13128
13129 thread_local! {
13130 static LAST_PID: std::cell::Cell<Option<u32>> = const { std::cell::Cell::new(None) };
13131 }
13132
13133 pub(super) fn after_create(child: &Child, _spec: &ModuleSpec) {
13134 LAST_PID.set(child.id());
13135 #[cfg(windows)]
13136 if let Some((_, address)) = _spec
13137 .env
13138 .iter()
13139 .find(|(key, _)| key == "SUBC_TEST_AFTER_CREATE")
13140 {
13141 use std::io::{Read, Write};
13142 let mut stream = std::net::TcpStream::connect(address).unwrap();
13143 stream
13144 .write_all(&child.id().unwrap().to_ne_bytes())
13145 .unwrap();
13146 let mut release = [0];
13147 stream.read_exact(&mut release).unwrap();
13148 }
13149 }
13150
13151 #[cfg(windows)]
13152 pub(super) fn assignment_failpoint(spec: &ModuleSpec) -> io::Result<()> {
13153 if spec
13154 .env
13155 .iter()
13156 .any(|(key, _)| key == "SUBC_TEST_ASSIGNMENT_FAILURE")
13157 {
13158 Err(io::Error::other("injected module job assignment failure"))
13159 } else {
13160 Ok(())
13161 }
13162 }
13163
13164 fn spec() -> ModuleSpec {
13165 ModuleSpec {
13166 script: None,
13167 module_id: "spawn-record-worker".into(),
13168 #[cfg(unix)]
13169 program: "/bin/sleep".into(),
13170 #[cfg(windows)]
13171 program: std::env::current_exe().unwrap(),
13172 #[cfg(unix)]
13173 args: vec!["60".into()],
13174 #[cfg(windows)]
13175 args: vec![
13176 "--exact".into(),
13177 "supervise::spawn_record_tests::windows_store_touch_fixture".into(),
13178 "--nocapture".into(),
13179 ],
13180 env: vec![],
13181 reserved: false,
13182 reserved_prefixes: vec![],
13183 protocol: ModuleProtocol::None,
13184 overlap: Default::default(),
13185 carry_containment: false,
13186 }
13187 }
13188
13189 #[tokio::test]
13190 async fn record_sync_failure_fails_spawn_and_reaps_child() {
13191 let home = cortexkit_test_support::ScratchDir::new("record-sync-failure");
13192 let supervisor =
13193 Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default())
13194 .with_live_children_record(home.join("live-children.json"));
13195 crate::live_children::fail_next_file_sync();
13196 let result = supervisor.spawn(spec());
13197 let failed = result.is_err();
13198 if let Ok(module) = result {
13199 module.stop().await.unwrap();
13200 }
13201 assert!(failed, "record failure must fail spawn");
13202 let pid = LAST_PID.get().expect("the real OS child was created");
13203 assert!(
13204 subc_os::Process::open(pid).unwrap().is_none(),
13205 "child {pid} survived rejected admission or was not reaped"
13206 );
13207 }
13208
13209 #[cfg(target_os = "macos")]
13210 #[test]
13211 #[ignore = "manual on-device latency measurement"]
13212 fn durable_record_write_latency_50() {
13213 let home = cortexkit_test_support::ScratchDir::new("record-latency");
13214 let path = home.join("live-children.json");
13215 let children = vec![crate::live_children::LiveChild {
13216 module_id: "latency-worker".into(),
13217 pid: std::process::id(),
13218 protocol: ModuleProtocol::None,
13219 start_time: Some(1),
13220 executable: None,
13221 cgroup_name: None,
13222 container: None,
13223 }];
13224 let mut samples = Vec::new();
13225 for _ in 0..50 {
13226 let start = std::time::Instant::now();
13227 crate::live_children::write_record(&path, &children).unwrap();
13228 samples.push(start.elapsed());
13229 }
13230 samples.sort();
13231 eprintln!(
13232 "durable_record_write_latency_50: n=50 median={:?} p95={:?}",
13233 (samples[24] + samples[25]) / 2,
13234 samples[47]
13235 );
13236 }
13237
13238 #[cfg(windows)]
13239 #[test]
13240 fn windows_store_touch_fixture() {
13241 if let Some(path) = std::env::var_os("SUBC_TEST_STORE_TOUCH") {
13242 std::fs::write(path, b"module ran").unwrap();
13243 }
13244 }
13245
13246 #[cfg(windows)]
13247 #[tokio::test]
13248 async fn windows_daemon_fixture() {
13249 let Ok(address) = std::env::var("SUBC_TEST_DAEMON_BARRIER") else {
13250 return;
13251 };
13252 let mut spec = spec();
13253 spec.env.push(("SUBC_TEST_AFTER_CREATE".into(), address));
13254 let supervisor =
13255 Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default());
13256 let _module = supervisor.spawn(spec).unwrap();
13257 }
13258
13259 #[cfg(windows)]
13260 #[test]
13261 fn windows_daemon_death_before_module_job_assignment_leaves_no_child() {
13262 use std::io::Read;
13263 let listener = std::net::TcpListener::bind("127.0.0.1:0").unwrap();
13264 let mut command = std::process::Command::new(std::env::current_exe().unwrap());
13267 command
13268 .args([
13269 "--exact",
13270 "supervise::spawn_record_tests::windows_daemon_fixture",
13271 "--nocapture",
13272 ])
13273 .env(
13274 "SUBC_TEST_DAEMON_BARRIER",
13275 listener.local_addr().unwrap().to_string(),
13276 );
13277 let mut daemon = subc_jobobject::spawn_contained(&mut command).unwrap();
13278 listener.set_nonblocking(true).unwrap();
13279 let deadline = std::time::Instant::now() + Duration::from_secs(30);
13280 let mut stream = loop {
13281 if let Ok((stream, _)) = listener.accept() {
13282 break stream;
13283 }
13284 assert!(
13285 daemon.child.try_wait().unwrap().is_none(),
13286 "daemon failed before rendezvous"
13287 );
13288 assert!(
13289 std::time::Instant::now() < deadline,
13290 "daemon did not reach CreateProcess failpoint"
13291 );
13292 std::thread::yield_now();
13293 };
13294 stream
13295 .set_read_timeout(Some(Duration::from_secs(30)))
13296 .unwrap();
13297 let mut pid = [0; 4];
13298 stream.read_exact(&mut pid).unwrap();
13299 let pid = u32::from_ne_bytes(pid);
13300 assert!(subc_jobobject::process_exists(pid));
13301 daemon.child.kill().unwrap();
13304 daemon.child.wait().unwrap();
13305 let exited = subc_jobobject::wait_for_process_exit(pid, Duration::from_secs(10));
13306 if !exited {
13307 daemon.job.terminate().unwrap();
13308 }
13309 assert!(exited, "suspended child {pid} outlived hard-killed daemon");
13310 }
13311
13312 #[cfg(windows)]
13313 #[tokio::test]
13314 async fn windows_containment_failure_never_resumes() {
13315 let home = cortexkit_test_support::ScratchDir::new("assignment-refusal");
13316 let sentinel = home.join("store-touched");
13317 let mut spec = spec();
13318 spec.env
13319 .push(("SUBC_TEST_ASSIGNMENT_FAILURE".into(), "1".into()));
13320 spec.env.push((
13321 "SUBC_TEST_STORE_TOUCH".into(),
13322 sentinel.display().to_string(),
13323 ));
13324 let supervisor =
13325 Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default());
13326 let result = supervisor.spawn(spec);
13327 if let Ok(module) = result {
13330 let _ = tokio::time::timeout(Duration::from_secs(10), async {
13331 while !sentinel.exists() && module.state().unwrap() == ModuleState::Running {
13332 tokio::task::yield_now().await;
13333 }
13334 })
13335 .await;
13336 let _ = module.stop().await;
13337 panic!(
13338 "assignment failure resumed a module; store touched={}",
13339 sentinel.exists()
13340 );
13341 }
13342 assert!(
13343 !sentinel.exists(),
13344 "refused module executed store-touch sentinel"
13345 );
13346 assert!(!subc_jobobject::process_exists(LAST_PID.get().unwrap()));
13347 }
13348
13349 #[cfg(windows)]
13350 #[tokio::test]
13351 async fn windows_nested_daemon_fixture() {
13352 let Some(sentinel) = std::env::var_os("SUBC_TEST_NESTED_STORE") else {
13353 return;
13354 };
13355 let mut spec = spec();
13356 spec.env.push((
13357 "SUBC_TEST_STORE_TOUCH".into(),
13358 sentinel.to_str().unwrap().into(),
13359 ));
13360 let supervisor =
13361 Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default());
13362 let module = supervisor
13363 .spawn(spec)
13364 .expect("per-module job must nest below daemon and host jobs");
13365 tokio::time::timeout(Duration::from_secs(10), async {
13366 while !PathBuf::from(&sentinel).exists() {
13367 tokio::task::yield_now().await;
13368 }
13369 })
13370 .await
13371 .unwrap();
13372 module.stop().await.unwrap();
13373 }
13374
13375 #[cfg(windows)]
13376 #[test]
13377 fn windows_module_job_nests_under_host_job() {
13378 let home = cortexkit_test_support::ScratchDir::new("nested-jobs");
13379 let sentinel = home.join("nested-store-touched");
13380 let mut command = std::process::Command::new(std::env::current_exe().unwrap());
13381 command
13382 .args([
13383 "--exact",
13384 "supervise::spawn_record_tests::windows_nested_daemon_fixture",
13385 "--nocapture",
13386 ])
13387 .env("SUBC_TEST_NESTED_STORE", &sentinel);
13388 let mut daemon = subc_jobobject::spawn_contained(&mut command).unwrap();
13389 assert!(daemon.child.wait().unwrap().success());
13390 assert!(sentinel.exists(), "nested protocol:none module never ran");
13391 }
13392}
13393
13394#[cfg(target_os = "linux")]
13395pub(crate) fn kill_module_cgroup(placement: Option<&subc_cgroup::Placement>, module_id: &str) {
13396 use subc_cgroup::KillOutcome;
13397 match subc_cgroup::kill_module(placement, module_id) {
13398 KillOutcome::Killed => {}
13399 KillOutcome::NotPlaced | KillOutcome::Unsupported => {
13400 debug!(
13401 module_id,
13402 "cgroup tree kill unavailable; using direct-child kill"
13403 );
13404 }
13405 KillOutcome::IoError { path, error } => {
13406 warn!(module_id, path = %path.display(), %error, "cgroup tree kill failed; using direct-child kill");
13407 }
13408 }
13409}
13410
13411#[cfg(windows)]
13419fn contain_spawned_child(
13420 child: &mut Child,
13421 spec: &ModuleSpec,
13422 carried: Option<&Arc<containment::CarriedContainment>>,
13423) -> Result<Option<Arc<Mutex<subc_jobobject::JobObject>>>, SuperviseError> {
13424 let assignment = || {
13425 let job = match carried {
13426 Some(container) => container.job(),
13427 None => Arc::new(Mutex::new(subc_jobobject::JobObject::new()?)),
13428 };
13429 #[cfg(test)]
13430 spawn_record_tests::assignment_failpoint(spec)?;
13431 job.lock()
13432 .unwrap_or_else(|p| p.into_inner())
13433 .assign(child)?;
13434 Ok::<_, io::Error>(job)
13435 };
13436 match assignment() {
13437 Ok(job) => Ok(Some(job)),
13438 Err(source) => {
13439 kill_and_reap_rejected_child(child);
13440 Err(SuperviseError::Spawn {
13441 program: spec.program.clone(),
13442 source: io::Error::other(format!(
13443 "Windows module job containment failed: {source}"
13444 )),
13445 cgroup_path: None,
13446 })
13447 }
13448 }
13449}
13450
13451fn kill_and_reap_rejected_child(child: &mut Child, #[cfg(unix)] group: containment::GroupTarget) {
13454 #[cfg(unix)]
13455 group.signal_rejected(child);
13456 let _ = child.start_kill();
13457 while matches!(child.try_wait(), Ok(None)) {
13458 std::thread::yield_now();
13459 }
13460}
13461
13462#[cfg(windows)]
13467fn resume_suspended_child(pid: u32, spec: &ModuleSpec) -> Result<(), SuperviseError> {
13468 if let Err(source) = subc_jobobject::resume_main_thread(pid) {
13469 let _ = subc_os::child_spawn::status(
13473 std::process::Command::new("taskkill.exe")
13474 .args(["/PID", &pid.to_string(), "/T", "/F"])
13475 .stdin(Stdio::null())
13476 .stdout(Stdio::null())
13477 .stderr(Stdio::null()),
13478 );
13479 return Err(SuperviseError::Spawn {
13480 program: spec.program.clone(),
13481 source,
13482 cgroup_path: None,
13483 });
13484 }
13485 Ok(())
13486}
13487
13488#[cfg(target_os = "linux")]
13489fn remove_module_cgroup(placement: &subc_cgroup::Placement, module_id: &str) {
13490 match placement.remove_module(module_id) {
13491 Ok(()) => debug!(module_id, "removed module cgroup after process exit"),
13492 Err(error) => warn!(
13493 module_id,
13494 error = %error,
13495 "could not remove module cgroup after process exit; continuing teardown"
13496 ),
13497 }
13498}
13499
13500#[cfg(target_os = "linux")]
13501const REAPED_CGROUP_TERM_GRACE: Duration = Duration::from_secs(2);
13502#[cfg(target_os = "linux")]
13503const REAPED_CGROUP_FREEZE_TIMEOUT: Duration = Duration::from_millis(100);
13504
13505#[cfg(target_os = "linux")]
13506async fn cleanup_reaped_cgroup(placement: &subc_cgroup::Placement, module_id: &str) {
13507 let path = placement.module_path(module_id).ok();
13508 if path.as_ref().is_some_and(|path| {
13509 std::fs::read_to_string(path.join("cgroup.events"))
13510 .ok()
13511 .is_some_and(|events| events.lines().any(|line| line == "populated 0"))
13512 }) {
13513 remove_module_cgroup(placement, module_id);
13514 return;
13515 }
13516
13517 if terminate_reaped_cgroup(placement, module_id).await {
13520 if let Some(path) = &path {
13521 let deadline = Instant::now() + REAPED_CGROUP_TERM_GRACE;
13522 while cgroup_is_populated(path) && Instant::now() < deadline {
13523 sleep_until((Instant::now() + Duration::from_millis(1)).min(deadline)).await;
13524 }
13525 }
13526 }
13527
13528 if matches!(
13532 subc_cgroup::kill_module(Some(placement), module_id),
13533 subc_cgroup::KillOutcome::Killed
13534 ) {
13535 if let Some(path) = &path {
13536 while cgroup_is_populated(path) {
13537 sleep(Duration::from_millis(1)).await;
13538 }
13539 }
13540 }
13541 remove_module_cgroup(placement, module_id);
13542}
13543
13544#[cfg(target_os = "linux")]
13545fn cgroup_is_populated(path: &std::path::Path) -> bool {
13546 std::fs::read_to_string(path.join("cgroup.events"))
13547 .ok()
13548 .is_some_and(|events| events.lines().any(|line| line == "populated 1"))
13549}
13550
13551#[cfg(target_os = "linux")]
13552async fn terminate_reaped_cgroup(placement: &subc_cgroup::Placement, module_id: &str) -> bool {
13553 let frozen =
13554 match subc_cgroup::freeze_module(Some(placement), module_id, REAPED_CGROUP_FREEZE_TIMEOUT)
13555 .await
13556 {
13557 subc_cgroup::FreezeOutcome::Frozen(frozen) => frozen,
13558 outcome => {
13559 tracing::debug!(
13560 module_id,
13561 ?outcome,
13562 "cgroup freeze failed; skipping SIGTERM grace"
13563 );
13564 return false;
13565 }
13566 };
13567 let listed = match frozen.processes() {
13568 subc_cgroup::ProcsOutcome::Listed(pids) => {
13569 for pid in pids.into_iter().filter(|pid| *pid > 1) {
13570 if let Some(pid) = rustix::process::Pid::from_raw(pid) {
13571 if let Err(error) =
13572 rustix::process::kill_process(pid, rustix::process::Signal::TERM)
13573 {
13574 tracing::debug!(module_id, %pid, %error, "leftover cgroup SIGTERM failed");
13575 }
13576 }
13577 }
13578 true
13579 }
13580 outcome => {
13581 tracing::debug!(
13582 module_id,
13583 ?outcome,
13584 "cgroup process listing failed; skipping SIGTERM grace"
13585 );
13586 false
13587 }
13588 };
13589 match frozen.thaw() {
13590 subc_cgroup::ThawOutcome::Thawed => listed,
13591 outcome => {
13592 tracing::debug!(
13593 module_id,
13594 ?outcome,
13595 "cgroup thaw failed; proceeding to cgroup.kill"
13596 );
13597 false
13598 }
13599 }
13600}
13601
13602#[cfg(target_os = "linux")]
13603fn apply_cgroup_placement(
13604 command: &mut Command,
13605 spec: &ModuleSpec,
13606 path: &std::path::Path,
13607) -> Result<(), SuperviseError> {
13608 subc_cgroup::apply(command, path).map_err(|source| SuperviseError::Cgroup {
13609 module_id: spec.module_id.clone(),
13610 source,
13611 })
13612}
13613
13614fn capture_retention(spec: &ModuleSpec) -> Retention {
13615 let defaults = Retention::default();
13616 let value = |name: &str| {
13617 spec.env
13618 .iter()
13619 .rev()
13620 .find_map(|(key, value)| (key == name).then_some(value.as_str()))
13621 };
13622 Retention {
13623 max_file_mb: value(CAPTURE_MAX_FILE_MB_ENV)
13624 .and_then(|value| value.parse().ok())
13625 .unwrap_or(defaults.max_file_mb),
13626 keep: value(CAPTURE_KEEP_ENV)
13627 .and_then(|value| value.parse().ok())
13628 .unwrap_or(defaults.keep),
13629 max_age_days: value(CAPTURE_MAX_AGE_DAYS_ENV)
13630 .and_then(|value| value.parse().ok())
13631 .unwrap_or(defaults.max_age_days),
13632 }
13633}
13634
13635fn generate_launch_nonce() -> Result<String, SuperviseError> {
13638 let mut bytes = [0u8; 32];
13639 getrandom::getrandom(&mut bytes).map_err(|source| SuperviseError::LaunchNonce {
13640 reason: source.to_string(),
13641 })?;
13642 let mut hex = String::with_capacity(64);
13643 for b in bytes {
13644 use std::fmt::Write;
13645 let _ = write!(hex, "{b:02x}");
13646 }
13647 Ok(hex)
13648}
13649
13650fn constant_time_eq(a: &[u8], b: &[u8]) -> bool {
13653 if a.len() != b.len() {
13654 return false;
13655 }
13656 let mut diff = 0u8;
13657 for (x, y) in a.iter().zip(b.iter()) {
13658 diff |= x ^ y;
13659 }
13660 diff == 0
13661}
13662
13663#[cfg(not(windows))]
13667fn observe_spawned_image(pid: u32) -> Option<subc_os::Observation> {
13668 let process = subc_os::Process::open(pid).ok().flatten()?;
13669 #[cfg(target_os = "linux")]
13670 return sample_after_linux_exec(
13671 || {
13672 let stat = std::fs::read_to_string(format!("/proc/{pid}/stat")).ok()?;
13673 let committed = linux_exec_committed(&stat);
13674 #[cfg(test)]
13675 spawn_image_tests::report_exec_state(committed);
13676 committed
13677 },
13678 || process.observe(),
13679 );
13680 #[cfg(not(target_os = "linux"))]
13681 process.observe()
13682}
13683
13684#[cfg(target_os = "linux")]
13688fn linux_exec_committed(stat: &str) -> Option<bool> {
13689 const PF_FORKNOEXEC: u64 = 0x40;
13690 let flags = stat.rsplit_once(')')?.1.split_whitespace().nth(6)?;
13691 Some(flags.parse::<u64>().ok()? & PF_FORKNOEXEC == 0)
13692}
13693
13694#[cfg(target_os = "linux")]
13695fn sample_after_linux_exec(
13696 mut exec_committed: impl FnMut() -> Option<bool>,
13697 observe: impl FnOnce() -> Option<subc_os::Observation>,
13698) -> Option<subc_os::Observation> {
13699 let deadline = std::time::Instant::now() + Duration::from_secs(1);
13700 while !exec_committed()? {
13701 if std::time::Instant::now() >= deadline {
13705 return None;
13706 }
13707 std::thread::sleep(Duration::from_micros(50));
13708 }
13709 observe()
13710}
13711
13712#[cfg(all(test, target_os = "linux"))]
13713mod spawn_image_tests {
13714 use super::*;
13715 use std::cell::{Cell, RefCell};
13716
13717 enum ExecEvent {
13718 Uncommitted,
13719 Image(Option<subc_os::Observation>),
13720 }
13721
13722 thread_local! {
13723 static EXEC_EVENTS: RefCell<Option<std::sync::mpsc::Sender<ExecEvent>>> = const { RefCell::new(None) };
13724 }
13725
13726 pub(super) fn report_exec_state(committed: Option<bool>) {
13727 if committed == Some(false) {
13728 EXEC_EVENTS.with(|events| {
13729 if let Some(events) = events.borrow_mut().take() {
13730 let _ = events.send(ExecEvent::Uncommitted);
13731 }
13732 });
13733 }
13734 }
13735
13736 #[test]
13737 fn linux_spawn_observation_waits_for_a_paused_fork_to_exec() {
13738 use std::io::{Read, Write};
13739 use std::os::fd::AsRawFd;
13740
13741 let descriptor = std::fs::File::open("/dev/null").unwrap();
13742 let mut command = std::process::Command::new("/bin/sleep");
13743 command.arg("30");
13744 let (mut ready, mut release) =
13745 subc_os::fork_exec_test::pause_before_exec(&mut command, descriptor.as_raw_fd())
13746 .unwrap();
13747 let spawn = std::thread::spawn(move || command.spawn());
13748 let mut message = [0u8; 8];
13749 ready.read_exact(&mut message).unwrap();
13750 let pid = i32::from_ne_bytes(message[..4].try_into().unwrap()) as u32;
13751 let before = std::fs::read_to_string(format!("/proc/{pid}/stat"));
13752 let inherited = subc_os::Process::open(pid).unwrap().unwrap().observe();
13753 let (events, received) = std::sync::mpsc::channel();
13754 let observer = std::thread::spawn(move || {
13755 EXEC_EVENTS.with(|slot| *slot.borrow_mut() = Some(events.clone()));
13756 let image = observe_spawned_image(pid);
13757 let _ = events.send(ExecEvent::Image(image));
13758 });
13759 let first = received.recv_timeout(Duration::from_secs(5));
13763 release.write_all(&[1]).unwrap();
13764 let mut child = spawn.join().unwrap().unwrap();
13765 let after = match first {
13766 Ok(ExecEvent::Uncommitted) => match received.recv_timeout(Duration::from_secs(5)) {
13767 Ok(ExecEvent::Image(image)) => image,
13768 _ => None,
13769 },
13770 Ok(ExecEvent::Image(image)) => image,
13771 Err(_) => None,
13772 };
13773 observer.join().unwrap();
13774 child.kill().unwrap();
13775 child.wait().unwrap();
13776
13777 assert_eq!(linux_exec_committed(&before.unwrap()), Some(false));
13778 assert_eq!(
13779 inherited.unwrap().executable,
13780 subc_os::file_identity(&std::env::current_exe().unwrap())
13781 );
13782 assert_eq!(
13783 after.unwrap().executable,
13784 subc_os::file_identity(std::path::Path::new("/bin/sleep"))
13785 );
13786 }
13787
13788 #[test]
13789 fn spawned_image_is_not_sampled_until_linux_exec_commits() {
13790 let committed = Cell::new(false);
13791 let inherited = subc_os::Observation {
13792 start_time: 23,
13793 executable: Some(subc_os::FileIdentity {
13794 device: 29,
13795 inode: 31,
13796 }),
13797 };
13798 let module = subc_os::Observation {
13799 start_time: 23,
13800 executable: Some(subc_os::FileIdentity {
13801 device: 29,
13802 inode: 37,
13803 }),
13804 };
13805 let polls = Cell::new(0);
13806 let image = sample_after_linux_exec(
13807 || {
13808 polls.set(polls.get() + 1);
13809 committed.set(polls.get() == 2);
13812 Some(committed.get())
13813 },
13814 || Some(if committed.get() { module } else { inherited }),
13815 );
13816 assert_eq!(image, Some(module));
13817 assert_eq!(polls.get(), 2);
13818 }
13819
13820 #[test]
13821 fn linux_exec_flag_is_read_from_flags_after_the_last_parenthesis() {
13822 assert_eq!(
13823 linux_exec_committed("17 (a ) b) R 1 2 3 4 5 4194368"),
13824 Some(false)
13825 );
13826 assert_eq!(
13827 linux_exec_committed("17 (a ) b) R 1 2 3 4 5 4194304"),
13828 Some(true)
13829 );
13830 assert_eq!(linux_exec_committed("17 (a) R 1 2 3 4 5 invalid"), None);
13831 assert_eq!(linux_exec_committed("17 (a) R"), None);
13832 }
13833
13834 #[test]
13835 fn unreadable_exec_state_never_samples_an_inherited_image() {
13836 assert_eq!(
13837 sample_after_linux_exec(
13838 || None,
13839 || panic!("must not sample before exec confirmation")
13840 ),
13841 None
13842 );
13843 }
13844
13845 #[test]
13846 fn uncommitted_exec_times_out_without_sampling_an_inherited_image() {
13847 assert_eq!(
13848 sample_after_linux_exec(
13849 || Some(false),
13850 || panic!("must not sample after an exec timeout")
13851 ),
13852 None
13853 );
13854 }
13855}
13856
13857#[cfg(windows)]
13860pub(crate) fn windows_spawn_capture(spec: &ModuleSpec) -> io::Result<subc_os::ExecutableCapture> {
13861 if let Err(error) = require_program_search_path(spec) {
13862 return Err(match error {
13863 SuperviseError::Spawn { source, .. } => source,
13864 error => io::Error::other(error),
13865 });
13866 }
13867 let mut program = spec.program.clone();
13868 if program.is_absolute() || program.components().count() > 1 {
13869 match subc_os::ExecutableCapture::open(&program) {
13872 Ok(capture) => return Ok(capture),
13873 Err(error)
13874 if error.kind() == io::ErrorKind::NotFound && program.extension().is_none() => {}
13875 Err(error) => return Err(error),
13876 }
13877 program.set_extension("exe");
13878 return subc_os::ExecutableCapture::open(&program);
13879 }
13880 if program.extension().is_none() {
13881 program.set_extension("exe");
13882 }
13883 let custom_path = spec
13884 .env
13885 .iter()
13886 .rev()
13887 .find(|(key, _)| key.eq_ignore_ascii_case("PATH"));
13888 let mut directories = Vec::new();
13889 if let Some(parent) = std::env::current_exe()?.parent() {
13892 directories.push(parent.to_path_buf());
13893 }
13894 directories.push(std::env::current_dir()?);
13895 if let Some(windows) = std::env::var_os("SystemRoot") {
13896 let windows = PathBuf::from(windows);
13897 directories.push(windows.join("System32"));
13898 directories.push(windows.clone());
13899 }
13900 let path = custom_path
13901 .map(|(_, value)| std::ffi::OsString::from(value))
13902 .or_else(|| std::env::var_os("PATH"));
13903 if let Some(path) = path {
13904 directories.extend(std::env::split_paths(&path));
13905 }
13906 for directory in directories {
13907 match subc_os::ExecutableCapture::open(&directory.join(&program)) {
13908 Ok(capture) => return Ok(capture),
13909 Err(error) if error.kind() == io::ErrorKind::NotFound => continue,
13910 Err(error) => return Err(error),
13911 }
13912 }
13913 Err(io::Error::new(
13914 io::ErrorKind::NotFound,
13915 "resolved executable not found",
13916 ))
13917}
13918
13919async fn spawn_and_mark_running_async(
13920 spec: &ModuleSpec,
13921 runtime: &SupervisorRuntimeConfig,
13922 snapshot: &SharedSnapshot,
13923) -> Result<SupervisedChild, SuperviseError> {
13924 let activity_spawn = runtime.begin_activity_spawn(&spec.module_id);
13925 let registration_watch = watch_registration(&runtime.registry, spec);
13926 #[cfg(unix)]
13927 let joining_group = containment::current(&runtime.child_roster.containment)
13928 .is_some_and(|container| container.group().is_some());
13929 let result = spawn_child_async(spec, runtime).await;
13930 #[cfg(unix)]
13931 let result = match result {
13932 Err(error)
13933 if joining_group
13934 && !is_launch_deferral(&error)
13935 && !runtime.child_roster.is_closed()
13936 && lock_snapshot(snapshot).is_ok_and(|state| state.enabled)
13937 && runtime
13938 .child_roster
13939 .previous_instance(&spec.module_id)
13940 .is_none() =>
13941 {
13942 warn!(module_id = %spec.module_id, %error, "carried process-group spawn failed; trying one fresh container");
13945 spawn_child_async(spec, runtime).await
13946 }
13947 other => other,
13948 };
13949 let mut child = result?;
13950 arm_registration_deadline(&mut child, runtime.register_timeout, registration_watch);
13951 if let Err(error) = set_running(snapshot, &child, &spec.module_id, &runtime.spawn_events) {
13952 containment::terminate_blocking(&runtime.child_roster.containment);
13953 return Err(error);
13954 }
13955 if let Some(spawn) = activity_spawn {
13956 spawn.complete(snapshot);
13957 }
13958 Ok(child)
13959}
13960
13961#[cfg(test)]
13962fn spawn_and_mark_running(
13963 spec: &ModuleSpec,
13964 runtime: &SupervisorRuntimeConfig,
13965 snapshot: &SharedSnapshot,
13966) -> Result<SupervisedChild, SuperviseError> {
13967 let activity_spawn = runtime.begin_activity_spawn(&spec.module_id);
13968 let registration_watch = watch_registration(&runtime.registry, spec);
13969 #[cfg(unix)]
13970 let joining_group = containment::current(&runtime.child_roster.containment)
13971 .is_some_and(|container| container.group().is_some());
13972 let spawn = || {
13973 spawn_child(
13974 spec,
13975 runtime.connection_file_path.as_deref(),
13976 runtime.supervisor_handle.as_ref(),
13977 &runtime.stderr_ring,
13978 runtime.capture_logs_dir.as_deref(),
13979 &runtime.child_roster,
13980 runtime.spawn_delivery(),
13981 #[cfg(target_os = "linux")]
13982 runtime.cgroup_placement.as_ref(),
13983 runtime.register_timeout,
13984 )
13985 };
13986 let result = spawn();
13987 #[cfg(unix)]
13988 let result = match result {
13989 Err(error)
13990 if joining_group
13991 && !is_launch_deferral(&error)
13992 && !runtime.child_roster.is_closed()
13993 && lock_snapshot(snapshot).is_ok_and(|state| state.enabled)
13994 && runtime
13995 .child_roster
13996 .previous_instance(&spec.module_id)
13997 .is_none() =>
13998 {
13999 warn!(module_id = %spec.module_id, %error, "carried process-group spawn failed; trying one fresh container");
14003 spawn()
14004 }
14005 other => other,
14006 };
14007 let child = result?;
14008 let mut child = child;
14009 arm_registration_deadline(&mut child, runtime.register_timeout, registration_watch);
14010 if let Err(error) = set_running(snapshot, &child, &spec.module_id, &runtime.spawn_events) {
14011 containment::terminate_blocking(&runtime.child_roster.containment);
14012 return Err(error);
14013 }
14014 if let Some(spawn) = activity_spawn {
14015 spawn.complete(snapshot);
14016 }
14017 Ok(child)
14018}
14019
14020struct RegistrationWatch {
14021 changes: watch::Receiver<u64>,
14022 baseline: u64,
14023}
14024
14025fn watch_registration(registry: &Registry, spec: &ModuleSpec) -> Option<RegistrationWatch> {
14029 if spec.protocol != ModuleProtocol::Subc {
14030 return None;
14031 }
14032 let changes = registry.subscribe_module_changes(&spec.module_id).ok()?;
14033 let baseline = *changes.borrow();
14034 Some(RegistrationWatch { changes, baseline })
14035}
14036
14037fn arm_registration_deadline(
14038 child: &mut SupervisedChild,
14039 budget: Duration,
14040 registration_watch: Option<RegistrationWatch>,
14041) {
14042 child.registration_watch = registration_watch;
14043 child.registration_deadline = (child.protocol == ModuleProtocol::Subc
14044 && child.awaiting_registration())
14045 .then_some(child.spawned_at + budget);
14046}
14047
14048async fn stop_unregistered_child(child: &mut SupervisedChild) -> io::Result<()> {
14049 child.retire_registration();
14050 if let Some(container) = child.child.container() {
14051 container.stop_members().await;
14052 child.kill_leader()?;
14053 child.wait().await?;
14054 return Ok(());
14055 }
14056 #[cfg(unix)]
14057 {
14058 let group = unreaped_registration_group(child)?;
14061 let _ = rustix::process::kill_process_group(group, rustix::process::Signal::TERM);
14062 sleep(crate::child_roster::TERM_TO_KILL).await;
14067 let group = unreaped_registration_group(child)?;
14068 let _ = rustix::process::kill_process_group(group, rustix::process::Signal::KILL);
14069 }
14070 child.start_kill()?;
14073 child.wait().await?;
14074 Ok(())
14075}
14076
14077#[cfg(unix)]
14078fn unreaped_registration_group(child: &SupervisedChild) -> io::Result<rustix::process::Pid> {
14079 if let Some(container) = child.child.container() {
14080 return container
14081 .group()
14082 .and_then(|pid| rustix::process::Pid::from_raw(pid as i32))
14083 .ok_or_else(|| io::Error::other("a carried cgroup has no signalable process group"));
14084 }
14085 let group = i32::try_from(child.pid)
14088 .ok()
14089 .filter(|pid| *pid > 1)
14090 .and_then(rustix::process::Pid::from_raw)
14091 .ok_or_else(|| {
14092 io::Error::new(
14093 io::ErrorKind::InvalidInput,
14094 "invalid supervised process group",
14095 )
14096 })?;
14097 if child.child.id() != Some(child.pid) {
14098 return Err(io::Error::new(
14099 io::ErrorKind::InvalidInput,
14100 "refusing process-group signal after supervised child was reaped",
14101 ));
14102 }
14103 Ok(group)
14104}
14105
14106enum RegistrationWaitOutcome {
14107 Registered,
14108 Exited(ExitReport),
14109 TimedOut,
14110}
14111
14112struct ReloadRegistrationFailure {
14113 exit_report: ExitReport,
14114 reason: String,
14115}
14116
14117#[derive(Debug, Clone, Copy, PartialEq, Eq)]
14118enum BusyGaugeObservation {
14119 Quiescent,
14120 Busy,
14121 Omitted,
14122}
14123
14124#[derive(Debug, Clone, PartialEq, Eq)]
14126enum GaugeVerdict {
14127 Zero,
14129 NonZero(u64),
14131 NotACount(String),
14135 Missing,
14137 ProbeFailed(&'static str),
14140}
14141
14142impl GaugeVerdict {
14143 fn reads_quiet(&self) -> bool {
14144 matches!(self, Self::Zero)
14145 }
14146}
14147
14148impl fmt::Display for GaugeVerdict {
14149 fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
14150 match self {
14151 Self::Zero => f.write_str("zero"),
14152 Self::NonZero(value) => write!(f, "non_zero({value})"),
14153 Self::NotACount(value) => write!(f, "not_a_count({value})"),
14154 Self::Missing => f.write_str("missing_from_health_body"),
14155 Self::ProbeFailed(kind) => write!(f, "probe_failed({kind})"),
14156 }
14157 }
14158}
14159
14160const GAUGE_VALUE_TEXT_LIMIT: usize = 32;
14163
14164fn gauge_verdicts(metrics: Option<&Value>, gauges: &[String]) -> Vec<GaugeVerdict> {
14166 let metrics = metrics.and_then(Value::as_object);
14167 gauges
14168 .iter()
14169 .map(
14170 |gauge| match metrics.and_then(|metrics| metrics.get(gauge)) {
14171 None => GaugeVerdict::Missing,
14172 Some(value) => match value.as_u64() {
14173 Some(0) => GaugeVerdict::Zero,
14174 Some(count) => GaugeVerdict::NonZero(count),
14175 None => {
14176 let mut text = value.to_string();
14177 if text.len() > GAUGE_VALUE_TEXT_LIMIT {
14178 let mut end = GAUGE_VALUE_TEXT_LIMIT;
14179 while !text.is_char_boundary(end) {
14180 end -= 1;
14181 }
14182 text.truncate(end);
14183 text.push_str("...");
14184 }
14185 GaugeVerdict::NotACount(text)
14186 }
14187 },
14188 },
14189 )
14190 .collect()
14191}
14192
14193fn busy_gauge_observation(verdicts: &[GaugeVerdict]) -> BusyGaugeObservation {
14196 if verdicts
14197 .iter()
14198 .any(|verdict| matches!(verdict, GaugeVerdict::Missing))
14199 {
14200 BusyGaugeObservation::Omitted
14201 } else if verdicts.iter().all(GaugeVerdict::reads_quiet) {
14202 BusyGaugeObservation::Quiescent
14203 } else {
14204 BusyGaugeObservation::Busy
14205 }
14206}
14207
14208#[derive(Debug, Clone, Default, PartialEq, Eq)]
14213struct GaugeTrace {
14214 latest: Option<(GaugeVerdict, Duration)>,
14215 last_value: Option<(GaugeVerdict, Duration)>,
14216}
14217
14218#[derive(Debug, Clone, Default, PartialEq, Eq)]
14224struct DrainGaugeTrace {
14225 probes: u32,
14227 gauges: Vec<(String, GaugeTrace)>,
14229}
14230
14231impl DrainGaugeTrace {
14232 fn new(gauges: &[String]) -> Self {
14233 Self {
14234 probes: 0,
14235 gauges: gauges
14236 .iter()
14237 .map(|gauge| (gauge.clone(), GaugeTrace::default()))
14238 .collect(),
14239 }
14240 }
14241
14242 fn record(&mut self, verdicts: &[GaugeVerdict], at: Duration) {
14244 self.probes = self.probes.saturating_add(1);
14245 for ((_, trace), verdict) in self.gauges.iter_mut().zip(verdicts) {
14246 trace.latest = Some((verdict.clone(), at));
14247 if matches!(
14248 verdict,
14249 GaugeVerdict::Zero | GaugeVerdict::NonZero(_) | GaugeVerdict::NotACount(_)
14250 ) {
14251 trace.last_value = Some((verdict.clone(), at));
14252 }
14253 }
14254 }
14255}
14256
14257impl fmt::Display for DrainGaugeTrace {
14262 fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
14263 if self.gauges.is_empty() {
14264 return f.write_str("none declared");
14265 }
14266 for (index, (name, trace)) in self.gauges.iter().enumerate() {
14267 if index > 0 {
14268 f.write_str(" | ")?;
14269 }
14270 write!(f, "{name}: ")?;
14271 match &trace.latest {
14272 Some((verdict, at)) => write!(f, "{verdict} @{}ms", at.as_millis())?,
14273 None => f.write_str("never probed")?,
14274 }
14275 match &trace.last_value {
14276 Some((value, at)) => write!(f, "; last value {value} @{}ms", at.as_millis())?,
14277 None => f.write_str("; no value ever read")?,
14278 }
14279 }
14280 Ok(())
14281 }
14282}
14283
14284pub(crate) fn declared_busy_gauges(
14285 registry: &Registry,
14286 module_id: &str,
14287) -> Result<Vec<String>, SuperviseError> {
14288 busy_gauges_of(
14289 registry
14290 .get_module(module_id)
14291 .map_err(SuperviseError::Registry)?,
14292 )
14293}
14294
14295fn declared_busy_gauges_for_connection(
14299 registry: &Registry,
14300 connection_id: ConnectionId,
14301) -> Result<Vec<String>, SuperviseError> {
14302 busy_gauges_of(
14303 registry
14304 .get_module_by_connection(connection_id)
14305 .map_err(SuperviseError::Registry)?,
14306 )
14307}
14308
14309fn busy_gauges_of(
14310 registration: Option<crate::registry::ModuleRegistration>,
14311) -> Result<Vec<String>, SuperviseError> {
14312 let Some(registration) = registration else {
14313 return Ok(Vec::new());
14314 };
14315 let Some(self_signals) = registration.manifest.self_signals else {
14316 return Ok(Vec::new());
14317 };
14318
14319 let mut gauges = Vec::new();
14320 for declaration in self_signals {
14321 if declaration.kind != SelfSignalKind::Busy {
14322 continue;
14323 }
14324 match declaration.anchored_to {
14325 SignalAnchor::HealthGauges { gauges: declared } if !declared.is_empty() => {
14326 gauges.extend(declared)
14327 }
14328 _ => {
14329 gauges.push(String::new());
14332 }
14333 }
14334 }
14335 Ok(gauges)
14336}
14337
14338#[allow(clippy::too_many_arguments)]
14347async fn wait_for_forwarding_quiescence(
14348 forwarding: &ForwardingTable,
14349 module_id: &str,
14350 runtime: &SupervisorRuntimeConfig,
14351 endpoint: crate::ModuleEndpointId,
14352 deadline: Instant,
14353 busy_gauges: &[String],
14354 scope: DrainScope,
14355 trace: &mut DrainGaugeTrace,
14356) -> Result<bool, SuperviseError> {
14357 let wait_started = Instant::now();
14358 let mut gauges_quiescent = busy_gauges.is_empty();
14359 let mut next_probe_at = wait_started;
14360 let mut omission_reported = false;
14361
14362 loop {
14363 let now = Instant::now();
14364 if !busy_gauges.is_empty() && now >= next_probe_at && now < deadline {
14365 let report = match scope {
14366 DrainScope::Active => probe_module_health(module_id, runtime, Some(deadline)).await,
14367 DrainScope::Endpoint(endpoint) => {
14368 probe_endpoint_health(endpoint, runtime, Some(deadline)).await
14369 }
14370 };
14371 let verdicts = match &report {
14372 Ok(report) => gauge_verdicts(report.metrics.as_ref(), busy_gauges),
14373 Err(err) => vec![GaugeVerdict::ProbeFailed(err.label()); busy_gauges.len()],
14374 };
14375 trace.record(&verdicts, wait_started.elapsed());
14376 gauges_quiescent = match report {
14377 Ok(_) => match busy_gauge_observation(&verdicts) {
14378 BusyGaugeObservation::Quiescent => true,
14379 BusyGaugeObservation::Busy => false,
14380 BusyGaugeObservation::Omitted => {
14381 if !omission_reported {
14384 forwarding
14385 .counters()
14386 .increment_drains_with_undeclared_gauge();
14387 let missing = busy_gauges
14388 .iter()
14389 .zip(&verdicts)
14390 .filter(|(_, verdict)| matches!(verdict, GaugeVerdict::Missing))
14391 .map(|(gauge, _)| gauge.as_str())
14392 .collect::<Vec<_>>()
14393 .join(",");
14394 warn!(
14395 module_id,
14396 missing_gauges = %missing,
14397 "drain health.check answer lacks a declared busy gauge; the drain treats the module as busy until it reports it"
14398 );
14399 omission_reported = true;
14400 }
14401 false
14402 }
14403 },
14404 Err(err) => {
14405 warn!(
14406 module_id,
14407 error = %err,
14408 "drain health.check did not produce declared busy gauges; treating module as busy"
14409 );
14410 false
14411 }
14412 };
14413 next_probe_at = Instant::now() + drain_busy_reprobe_interval();
14416 }
14417
14418 let in_flight = forwarding
14419 .endpoint_in_flight_count(endpoint)
14420 .map_err(SuperviseError::Forwarding)?;
14421 if in_flight == 0 && gauges_quiescent {
14422 return Ok(true);
14423 }
14424
14425 let now = Instant::now();
14426 if now >= deadline {
14427 return Ok(false);
14428 }
14429 let mut wait = deadline
14430 .saturating_duration_since(now)
14431 .min(REGISTRY_RELEASE_POLL);
14432 if !busy_gauges.is_empty() {
14433 wait = wait.min(next_probe_at.saturating_duration_since(now));
14434 }
14435 sleep(wait).await;
14436 }
14437}
14438
14439fn drain_busy_reprobe_interval() -> Duration {
14443 DRAIN_BUSY_REPROBE_INTERVAL.max(REGISTRY_RELEASE_POLL)
14444}
14445
14446fn warn_drain_timed_out(
14461 module_id: &str,
14462 drain_timeout: Duration,
14463 reason: RouteCloseReason,
14464 holdouts: &crate::forwarding::DrainHoldouts,
14465 gauges: &DrainGaugeTrace,
14466) {
14467 warn!(
14468 module_id = %module_id,
14469 waited = ?drain_timeout,
14470 ?reason,
14471 held_requests = holdouts.requests,
14472 held_routes = holdouts.routes,
14473 total_routes = holdouts.total_routes,
14474 top_connections = ?holdouts.top_connections,
14475 held = %holdouts
14478 .held
14479 .iter()
14480 .map(|(channel, corr)| format!("{channel}:{corr}"))
14481 .collect::<Vec<_>>()
14482 .join(","),
14483 gauge_probes = gauges.probes,
14484 busy_gauges = %gauges,
14485 "route drain timed out before request quiescence; forcing teardown"
14486 );
14487}
14488
14489fn drained_after_quiescence_wait(wait_result: &Result<bool, SuperviseError>) -> bool {
14497 match wait_result {
14498 Ok(drained) => *drained,
14499 Err(_) => false,
14500 }
14501}
14502
14503fn send_route_goodbyes(forwarding: &ForwardingTable, released_routes: Vec<GoodbyeTarget>) {
14504 for released in released_routes {
14505 let frame = match Frame::build_with_version(
14506 released.negotiated_ver,
14507 FrameType::Goodbye,
14508 control_flags(),
14509 released.channel,
14510 released.epoch,
14511 0,
14512 Vec::new(),
14513 ) {
14514 Ok(frame) => frame,
14515 Err(err) => {
14516 warn!(
14517 route_channel = released.channel,
14518 error = %err,
14519 "failed to build supervisor drain route GOODBYE frame"
14520 );
14521 continue;
14522 }
14523 };
14524 if !released.close_on_delivery_failure() {
14525 crate::forwarding::send_module_route_goodbye(
14526 &forwarding.counters(),
14527 &released.sink,
14528 frame,
14529 released.module_id.as_deref(),
14530 "supervisor drain",
14531 );
14532 continue;
14533 }
14534 if let Err(err) = released.sink.try_send(frame) {
14535 warn!(
14536 target_connection_id = released.connection_id.get(),
14537 route_channel = released.channel,
14538 error = %err,
14539 "supervisor drain route GOODBYE was not delivered to client; closing target connection"
14540 );
14541 let _ = forwarding.escalate_client_delivery_failure(
14542 released.connection_id,
14543 released.channel,
14544 released.epoch,
14545 CloseReason::new(
14546 "route_goodbye_delivery_failed",
14547 format!(
14548 "failed to enqueue supervisor drain route GOODBYE for channel {}: {err}",
14549 released.channel
14550 ),
14551 ),
14552 crate::forwarding::UndeliveredFrame {
14553 module_id: released.module_id.as_deref(),
14554 sink: &released.sink,
14555 },
14556 );
14557 }
14558 }
14559}
14560
14561async fn send_module_draining(
14562 module_id: &str,
14563 reason: RouteCloseReason,
14564 deadline_ms: u64,
14565 target: &ModuleDrainTarget,
14566 _wait_for_flush: bool,
14567) -> bool {
14568 let body = match serde_json::to_vec(&ModuleControlCommand::Draining {
14569 reason,
14570 deadline_ms,
14571 }) {
14572 Ok(body) => body,
14573 Err(err) => {
14574 warn!(
14575 module_id,
14576 error = %err,
14577 "failed to encode module draining command"
14578 );
14579 return false;
14580 }
14581 };
14582 let frame = match Frame::build_with_version(
14583 target.negotiated_ver,
14584 FrameType::Push,
14585 control_flags(),
14586 0,
14587 0,
14588 0,
14589 body,
14590 ) {
14591 Ok(frame) => frame,
14592 Err(err) => {
14593 warn!(
14594 module_id,
14595 error = %err,
14596 "failed to build module draining command frame"
14597 );
14598 return false;
14599 }
14600 };
14601 #[cfg(unix)]
14602 if _wait_for_flush {
14603 return target.sink.send_flushed(frame).await.is_ok();
14604 }
14605 if let Err(err) = target.sink.try_send(frame) {
14606 warn!(
14607 module_id,
14608 target_connection_id = target.endpoint.connection_id.get(),
14609 error = %err,
14610 "module draining command was not delivered to peer"
14611 );
14612 return false;
14613 }
14614 true
14615}
14616
14617fn module_goodbye_frame(module_id: &str, negotiated_ver: u8) -> Option<Frame> {
14619 match Frame::build_with_version(
14620 negotiated_ver,
14621 FrameType::Goodbye,
14622 control_flags(),
14623 0,
14624 0,
14625 0,
14626 Vec::new(),
14627 ) {
14628 Ok(frame) => Some(frame),
14629 Err(err) => {
14630 warn!(
14631 module_id,
14632 error = %err,
14633 "failed to build module GOODBYE frame"
14634 );
14635 None
14636 }
14637 }
14638}
14639
14640#[cfg(unix)]
14654async fn send_module_goodbyes_for_daemon_shutdown(
14655 forwarding: &Arc<ForwardingTable>,
14656 reason: &CloseReason,
14657 wait_for_flush: bool,
14658) {
14659 const GOODBYE_BUDGET: Duration = Duration::from_millis(500);
14660 let targets = match forwarding.module_connections() {
14661 Ok(targets) => targets,
14662 Err(err) => {
14663 warn!(error = %err, "could not list module connections for shutdown GOODBYE");
14664 return;
14665 }
14666 };
14667 let deadline = Instant::now() + GOODBYE_BUDGET;
14668 let mut sends = tokio::task::JoinSet::new();
14669 for target in targets {
14670 let Some(frame) = module_goodbye_frame(&target.module_id, target.negotiated_ver) else {
14671 continue;
14672 };
14673 if !wait_for_flush {
14674 if let Err(err) = target.sink.try_send(frame) {
14675 debug!(
14676 module_id = %target.module_id,
14677 error = %err,
14678 "shutdown module GOODBYE was not queued"
14679 );
14680 }
14681 continue;
14682 }
14683 let forwarding = Arc::clone(forwarding);
14684 let reason = reason.clone();
14685 sends.spawn(async move {
14686 match timeout_at(deadline, target.sink.send_flushed(frame)).await {
14687 Ok(Ok(())) => {}
14688 Ok(Err(err)) => debug!(
14689 module_id = %target.module_id,
14690 error = %err,
14691 "module connection closed before its shutdown GOODBYE was written"
14692 ),
14693 Err(_) => warn!(
14694 module_id = %target.module_id,
14695 budget = ?GOODBYE_BUDGET,
14696 "shutdown module GOODBYE was not written within its budget; closing anyway"
14697 ),
14698 }
14699 forwarding.request_connection_close(target.endpoint.connection_id, reason);
14700 });
14701 }
14702 while sends.join_next().await.is_some() {}
14704}
14705
14706async fn send_module_goodbye(
14707 module_id: &str,
14708 forwarding: &ForwardingTable,
14709 target: &ModuleDrainTarget,
14710 wait_for_flush: bool,
14711) {
14712 let Some(frame) = module_goodbye_frame(module_id, target.negotiated_ver) else {
14713 return;
14714 };
14715 let sent = if wait_for_flush {
14718 timeout(Duration::from_millis(500), target.sink.send_flushed(frame))
14719 .await
14720 .map_err(|_| "module GOODBYE flush exceeded 500ms".to_owned())
14721 .and_then(|result| result.map_err(|err| err.to_string()))
14722 } else {
14723 target.sink.try_send(frame).map_err(|err| err.to_string())
14724 };
14725 if let Err(err) = sent {
14726 warn!(
14727 module_id,
14728 target_connection_id = target.endpoint.connection_id.get(),
14729 error = %err,
14730 "supervisor drain module GOODBYE was not delivered to peer; closing module connection"
14731 );
14732 forwarding.request_connection_close(
14733 target.endpoint.connection_id,
14734 CloseReason::new(
14735 "module_goodbye_delivery_failed",
14736 format!("failed to enqueue supervisor drain module GOODBYE for module '{module_id}': {err}"),
14737 ),
14738 );
14739 }
14740}
14741
14742#[derive(Clone, Copy)]
14743struct ForwardingDrainContext<'a> {
14744 spec: &'a ModuleSpec,
14745 runtime: &'a SupervisorRuntimeConfig,
14746 registry: &'a Registry,
14747 scope: DrainScope,
14748}
14749
14750#[derive(Debug, Clone, Copy, PartialEq, Eq)]
14752enum DrainScope {
14753 Active,
14756 Endpoint(crate::ModuleEndpointId),
14761}
14762
14763#[derive(Debug, Clone, Copy, PartialEq, Eq)]
14771enum StopNotice {
14772 SentOverConnection,
14775 NoConnection,
14779 NotSent,
14783}
14784
14785async fn begin_forwarding_drain(
14786 spec: &ModuleSpec,
14787 runtime: &SupervisorRuntimeConfig,
14788 registry: &Registry,
14789 snapshot: &SharedSnapshot,
14790 enabled: Option<bool>,
14791 reason: RouteCloseReason,
14792) -> Result<StopNotice, SuperviseError> {
14793 let Some(forwarding) = runtime.forwarding.as_ref() else {
14794 return Err(SuperviseError::ReloadUnavailable {
14795 module_id: spec.module_id.clone(),
14796 reason: "supervisor was not configured with a forwarding table".to_string(),
14797 });
14798 };
14799
14800 begin_forwarding_drain_with(
14801 forwarding,
14802 ForwardingDrainContext {
14803 spec,
14804 runtime,
14805 registry,
14806 scope: DrainScope::Active,
14807 },
14808 snapshot,
14809 enabled,
14810 reason,
14811 runtime.drain_timeout,
14812 )
14813 .await
14814}
14815
14816async fn begin_forwarding_drain_if_configured(
14817 spec: &ModuleSpec,
14818 runtime: &SupervisorRuntimeConfig,
14819 registry: &Registry,
14820 snapshot: &SharedSnapshot,
14821 enabled: Option<bool>,
14822 reason: RouteCloseReason,
14823) -> Result<StopNotice, SuperviseError> {
14824 begin_forwarding_drain_with_timeout(
14825 spec,
14826 runtime,
14827 registry,
14828 snapshot,
14829 enabled,
14830 reason,
14831 runtime.drain_timeout,
14832 )
14833 .await
14834}
14835
14836async fn begin_forwarding_drain_with_timeout(
14840 spec: &ModuleSpec,
14841 runtime: &SupervisorRuntimeConfig,
14842 registry: &Registry,
14843 snapshot: &SharedSnapshot,
14844 enabled: Option<bool>,
14845 reason: RouteCloseReason,
14846 drain_timeout: Duration,
14847) -> Result<StopNotice, SuperviseError> {
14848 let Some(forwarding) = runtime.forwarding.as_ref() else {
14849 return Ok(StopNotice::NotSent);
14850 };
14851
14852 begin_forwarding_drain_with(
14853 forwarding,
14854 ForwardingDrainContext {
14855 spec,
14856 runtime,
14857 registry,
14858 scope: DrainScope::Active,
14859 },
14860 snapshot,
14861 enabled,
14862 reason,
14863 drain_timeout,
14864 )
14865 .await
14866}
14867
14868async fn begin_forwarding_drain_with(
14869 forwarding: &ForwardingTable,
14870 context: ForwardingDrainContext<'_>,
14871 snapshot: &SharedSnapshot,
14872 enabled: Option<bool>,
14873 reason: RouteCloseReason,
14874 drain_timeout: Duration,
14875) -> Result<StopNotice, SuperviseError> {
14876 let ForwardingDrainContext {
14877 spec,
14878 runtime,
14879 registry,
14880 scope,
14881 } = context;
14882 debug_assert_ne!(reason, RouteCloseReason::Crash);
14883 let terminal = matches!(reason, RouteCloseReason::Disable);
14884 let drain_started_at = Instant::now();
14885 let drain_deadline = drain_started_at + drain_timeout;
14886 let deadline_ms =
14887 unix_ms_now().saturating_add(u64::try_from(drain_timeout.as_millis()).unwrap_or(u64::MAX));
14888 let busy_gauges = match scope {
14889 DrainScope::Active => declared_busy_gauges(registry, &spec.module_id)?,
14890 DrainScope::Endpoint(endpoint) => {
14891 declared_busy_gauges_for_connection(registry, endpoint.connection_id)?
14892 }
14893 };
14894
14895 let gate_started = Instant::now();
14898 let drain_target = match scope {
14899 DrainScope::Active => forwarding.begin_module_drain(&spec.module_id, reason),
14900 DrainScope::Endpoint(endpoint) => forwarding.begin_endpoint_drain(endpoint, reason),
14901 }
14902 .map_err(SuperviseError::Forwarding)?;
14903 info!(
14908 module_id = %spec.module_id,
14909 ?reason,
14910 gate_ms = u64::try_from(gate_started.elapsed().as_millis()).unwrap_or(u64::MAX),
14911 connected = drain_target.is_some(),
14912 "module drain began; route admission closed"
14913 );
14914 if scope == DrainScope::Active {
14915 update_snapshot(snapshot, Some(&spec.module_id), |state| {
14916 state.state = ModuleState::Draining;
14917 state.draining_to_replace =
14918 matches!(reason, RouteCloseReason::Restart | RouteCloseReason::Reload);
14919 if let Some(enabled) = enabled {
14920 state.enabled = enabled;
14921 }
14922 })?;
14923 }
14924
14925 let Some(target) = drain_target.as_ref() else {
14926 return Ok(StopNotice::NoConnection);
14930 };
14931 let flush_goodbye = runtime.supervisor_handle.as_ref().is_some_and(|handle| {
14932 handle
14933 .registration_spawn(target.endpoint.connection_id)
14934 .is_some()
14935 });
14936 {
14937 send_module_draining(&spec.module_id, reason, deadline_ms, target, false).await;
14938 #[cfg(feature = "test-support")]
14939 forwarding.wait_before_route_closing_for_test().await;
14940 let routes = forwarding
14941 .endpoint_routes(target.endpoint)
14942 .map_err(SuperviseError::Forwarding)?;
14943 let routes_notified = routes.len();
14944 crate::control::send_route_control_pushes(
14945 forwarding,
14946 routes.clone(),
14947 ClientControlPush::RouteClosing {
14948 module_id: spec.module_id.clone(),
14949 channels: Vec::new(),
14950 reason,
14951 },
14952 );
14953 send_route_goodbyes(forwarding, target.abandoned_bindings.clone());
14954
14955 let mut gauge_trace = DrainGaugeTrace::new(&busy_gauges);
14961 let wait_result = wait_for_forwarding_quiescence(
14962 forwarding,
14963 &spec.module_id,
14964 runtime,
14965 target.endpoint,
14966 drain_deadline,
14967 &busy_gauges,
14968 scope,
14969 &mut gauge_trace,
14970 )
14971 .await;
14972 let drained = drained_after_quiescence_wait(&wait_result);
14973 if let Err(err) = &wait_result {
14974 error!(
14975 module_id = %spec.module_id,
14976 ?reason,
14977 error = %err,
14978 "forwarding quiescence wait failed after route.closing; forcing route.closed(drained: false) so the client is not left waiting on an unfulfilled promise"
14979 );
14980 } else if !drained {
14981 let holdouts = forwarding
14983 .endpoint_drain_holdouts(target.endpoint)
14984 .unwrap_or_default();
14985 warn_drain_timed_out(
14986 &spec.module_id,
14987 drain_timeout,
14988 reason,
14989 &holdouts,
14990 &gauge_trace,
14991 );
14992 }
14993 crate::control::send_route_control_pushes(
14994 forwarding,
14995 routes,
14996 ClientControlPush::RouteClosed {
14997 module_id: spec.module_id.clone(),
14998 channels: Vec::new(),
14999 reason,
15000 drained,
15001 abandoned: target.abandoned_bindings.len() as u32,
15002 excluded_subscriptions: target.excluded_subscriptions,
15003 terminal: Some(terminal),
15004 },
15005 );
15006 wait_result?;
15007
15008 let released_routes = match forwarding.release_module_endpoint_routes(target.endpoint) {
15014 Ok(routes) => routes,
15015 Err(err) => {
15016 warn!(
15017 module_id = %spec.module_id,
15018 ?reason,
15019 error = %err,
15020 "failed to release module endpoint routes after route.closed; module GOODBYE will still be sent"
15021 );
15022 send_module_goodbye(&spec.module_id, forwarding, target, flush_goodbye).await;
15023 return Err(SuperviseError::Forwarding(err));
15024 }
15025 };
15026 let route_goodbye_count = released_routes.len();
15027 send_route_goodbyes(forwarding, released_routes);
15028 send_module_goodbye(&spec.module_id, forwarding, target, flush_goodbye).await;
15029
15030 info!(
15036 module_id = %spec.module_id,
15037 ?reason,
15038 routes_notified,
15039 route_goodbyes = route_goodbye_count,
15040 abandoned_reservations = target.abandoned_bindings.len(),
15041 excluded_subscriptions = target.excluded_subscriptions,
15042 drained,
15043 "module drain complete; consumers notified via route.closing/route.closed pushes and per-route GOODBYE frames"
15044 );
15045 }
15046
15047 Ok(StopNotice::SentOverConnection)
15048}
15049
15050async fn wait_for_registration_after_reload(
15053 registry: &Registry,
15054 module_id: &str,
15055 snapshot: &SharedSnapshot,
15056 child: &mut SupervisedChild,
15057 wait: Duration,
15058) -> Result<RegistrationWaitOutcome, SuperviseError> {
15059 wait_for_slot_registration(
15060 registry,
15061 crate::registry::RegistrationSlot::Active(module_id),
15062 module_id,
15063 snapshot,
15064 child,
15065 wait,
15066 )
15067 .await
15068}
15069
15070async fn wait_for_slot_registration(
15078 registry: &Registry,
15079 slot: crate::registry::RegistrationSlot<'_>,
15080 module_id: &str,
15081 snapshot: &SharedSnapshot,
15082 child: &mut SupervisedChild,
15083 wait: Duration,
15084) -> Result<RegistrationWaitOutcome, SuperviseError> {
15085 let deadline = Instant::now() + wait;
15086 loop {
15087 if registry
15088 .registration(slot)
15089 .map_err(SuperviseError::Registry)?
15090 .is_some()
15091 {
15092 return Ok(RegistrationWaitOutcome::Registered);
15093 }
15094
15095 let now = Instant::now();
15096 if now >= deadline {
15097 return Ok(RegistrationWaitOutcome::TimedOut);
15098 }
15099 let remaining = deadline.saturating_duration_since(now);
15100 let poll = remaining.min(REGISTRY_RELEASE_POLL);
15101
15102 tokio::select! {
15103 wait_result = child.wait() => {
15104 let status = wait_result.map_err(|source| SuperviseError::Wait {
15105 module_id: module_id.to_string(),
15106 source,
15107 })?;
15108 return Ok(RegistrationWaitOutcome::Exited(classify_reaped_child_exit(
15109 snapshot,
15110 child,
15111 &status,
15112 )));
15113 }
15114 _ = sleep(poll) => {}
15115 }
15116 }
15117}
15118
15119fn registration_failure_exit_report(mut exit_report: ExitReport) -> ExitReport {
15120 if exit_report.kind != ExitKind::DeliberateSeverance {
15124 exit_report.kind = ExitKind::Crash;
15125 }
15126 exit_report
15127}
15128
15129async fn handle_reload_child_registration_failure(
15130 spec: &ModuleSpec,
15131 runtime: &SupervisorRuntimeConfig,
15132 registry: &Registry,
15133 process_liveness: &SupervisorProcessLiveness,
15134 snapshot: &SharedSnapshot,
15135 _child: &mut Option<SupervisedChild>,
15136 failure: ReloadRegistrationFailure,
15137) -> Result<(), SuperviseError> {
15138 let ReloadRegistrationFailure {
15139 exit_report,
15140 reason,
15141 } = failure;
15142 match on_child_exit(
15143 spec,
15144 runtime.restart_policy,
15145 registry,
15146 snapshot,
15147 &runtime.terminal_ring,
15148 &runtime.spawn_events,
15149 &runtime.child_roster,
15150 exit_report,
15151 runtime.recovery_is_daemon,
15152 )
15153 .await
15154 {
15155 NextAction::Stop {
15156 registration_released,
15157 } => {
15158 if registration_released {
15159 process_liveness.untrack_if_current(&spec.module_id, snapshot);
15160 }
15161 }
15162 NextAction::Restart { schedule } => {
15163 let delay = schedule.map_or(runtime.restart_policy.delay_for_restart(0), |schedule| {
15164 schedule.delay
15165 });
15166 if let Some(schedule) = schedule {
15167 log_crash_respawn(&spec.module_id, schedule);
15168 }
15169 schedule_respawn(
15170 runtime,
15171 snapshot,
15172 &spec.module_id,
15173 delay,
15174 RespawnKind::Spawn,
15175 true,
15176 )?;
15177 }
15178 }
15179 Err(SuperviseError::ReloadFailed {
15180 module_id: spec.module_id.clone(),
15181 reason,
15182 })
15183}
15184
15185async fn handle_reload_spawn_failure(
15186 spec: &ModuleSpec,
15187 runtime: &SupervisorRuntimeConfig,
15188 process_liveness: &SupervisorProcessLiveness,
15189 snapshot: &SharedSnapshot,
15190 _child: &mut Option<SupervisedChild>,
15191 reason: String,
15192) -> Result<(), SuperviseError> {
15193 let now = Instant::now();
15194 let mut schedule = None;
15195 update_snapshot(snapshot, Some(&spec.module_id), |state| {
15196 clear_current_process_facts(state);
15197 if state.enabled {
15198 schedule = state.next_crash_restart(&runtime.restart_policy, now);
15199 state.state = if schedule.is_some() {
15200 ModuleState::Restarting
15201 } else {
15202 if runtime
15203 .failure_context(runtime.recovery_is_daemon)
15204 .may_retry()
15205 {
15206 state.failed_retry.schedule(&spec.module_id);
15207 }
15208 ModuleState::Failed
15209 };
15210 } else {
15211 state.state = ModuleState::Disabled;
15212 }
15213 })?;
15214 if let Some(schedule) = schedule {
15215 schedule_respawn(
15216 runtime,
15217 snapshot,
15218 &spec.module_id,
15219 schedule.delay,
15220 RespawnKind::Spawn,
15221 true,
15222 )?;
15223 } else {
15224 process_liveness.untrack_if_current(&spec.module_id, snapshot);
15225 }
15226 Err(SuperviseError::ReloadFailed {
15227 module_id: spec.module_id.clone(),
15228 reason,
15229 })
15230}
15231
15232fn control_flags() -> Flags {
15233 Flags::new(false, Priority::Passive, false)
15234}
15235
15236#[allow(clippy::too_many_arguments)]
15237async fn drain_optional_child(
15238 module_id: &str,
15239 protocol: ModuleProtocol,
15240 stop_notice: StopNotice,
15241 registry: &Registry,
15242 forwarding: Option<&ForwardingTable>,
15243 snapshot: &SharedSnapshot,
15244 terminal_ring: &Arc<Mutex<TerminalRing>>,
15245 spawn_events: &SpawnEventFeed,
15246 child: &mut Option<SupervisedChild>,
15247 drain_timeout: Duration,
15248 final_state: ModuleState,
15249 enabled: Option<bool>,
15250 failure: &FailureContext,
15251) -> Result<(), SuperviseError> {
15252 if let Some(child) = child.take() {
15253 drain_child_to_state(
15254 module_id,
15255 protocol,
15256 stop_notice,
15257 registry,
15258 forwarding,
15259 snapshot,
15260 terminal_ring,
15261 spawn_events,
15262 child,
15263 drain_timeout,
15264 final_state,
15265 enabled,
15266 failure,
15267 )
15268 .await
15269 } else {
15270 update_snapshot(snapshot, Some(module_id), |state| {
15271 state.state = final_state;
15272 if final_state == ModuleState::Failed {
15273 state.failed_retry.failed_at = Some(Instant::now());
15274 }
15275 if let Some(enabled) = enabled {
15276 state.enabled = enabled;
15277 }
15278 clear_current_process_facts(state);
15279 })?;
15280 release_dead_registration(registry, forwarding, snapshot, module_id, failure).await
15281 }
15282}
15283
15284#[allow(clippy::too_many_arguments)]
15285async fn drain_child_to_state(
15286 module_id: &str,
15287 _protocol: ModuleProtocol,
15288 stop_notice: StopNotice,
15289 registry: &Registry,
15290 forwarding: Option<&ForwardingTable>,
15291 snapshot: &SharedSnapshot,
15292 terminal_ring: &Arc<Mutex<TerminalRing>>,
15293 spawn_events: &SpawnEventFeed,
15294 mut child: SupervisedChild,
15295 drain_timeout: Duration,
15296 final_state: ModuleState,
15297 enabled: Option<bool>,
15298 failure: &FailureContext,
15299) -> Result<(), SuperviseError> {
15300 let protocol = child.protocol;
15301 child.retire_registration();
15302 update_snapshot(snapshot, Some(module_id), |state| {
15303 state.state = ModuleState::Draining;
15304 state.draining_to_replace = final_state == ModuleState::Restarting;
15305 if let Some(enabled) = enabled {
15306 state.enabled = enabled;
15307 }
15308 })?;
15309
15310 if stop_notice != StopNotice::SentOverConnection {
15321 if protocol == ModuleProtocol::Subc && stop_notice == StopNotice::NoConnection {
15322 info!(
15323 module_id,
15324 pid = child.pid,
15325 budget_ms = u64::try_from(drain_timeout.as_millis()).unwrap_or(u64::MAX),
15326 "module has no connection yet; requesting stop by signal"
15327 );
15328 }
15329 request_graceful_stop(module_id, &child);
15330 }
15331
15332 let exit_report = match timeout(drain_timeout, child.wait()).await {
15333 Ok(Ok(status)) => classify_reaped_child_exit(snapshot, &child, &status),
15334 Ok(Err(source)) => {
15335 fail_after_wait_error(
15336 module_id,
15337 snapshot,
15338 terminal_ring,
15339 spawn_events,
15340 format!("failed to wait during drain: {source}"),
15341 failure,
15342 );
15343 return Err(SuperviseError::Wait {
15344 module_id: module_id.to_string(),
15345 source,
15346 });
15347 }
15348 Err(_) => {
15349 warn!(
15362 module_id,
15363 pid = child.pid,
15364 budget_ms = u64::try_from(drain_timeout.as_millis()).unwrap_or(u64::MAX),
15365 reason = ?final_state,
15366 ?stop_notice,
15367 "drain budget expired before the module exited; killing it"
15368 );
15369 let kill =
15370 if final_state == ModuleState::Restarting && child.child.container().is_some() {
15371 child.kill_leader()
15372 } else {
15373 child.start_kill()
15374 };
15375 kill.map_err(|source| {
15376 fail_after_wait_error(
15377 module_id,
15378 snapshot,
15379 terminal_ring,
15380 spawn_events,
15381 format!("failed to kill during drain: {source}"),
15382 failure,
15383 );
15384 SuperviseError::Kill {
15385 module_id: module_id.to_string(),
15386 source,
15387 }
15388 })?;
15389 let status = child.wait().await.map_err(|source| {
15390 fail_after_wait_error(
15391 module_id,
15392 snapshot,
15393 terminal_ring,
15394 spawn_events,
15395 format!("failed to wait after drain kill: {source}"),
15396 failure,
15397 );
15398 SuperviseError::Wait {
15399 module_id: module_id.to_string(),
15400 source,
15401 }
15402 })?;
15403 classify_reaped_child_exit(snapshot, &child, &status)
15404 }
15405 };
15406
15407 let detail = lock_snapshot(snapshot)?.drain_disposition_detail.take();
15408 publish_after_terminal_record(
15409 module_id,
15410 || {
15411 record_terminal_with_detail(
15412 module_id,
15413 terminal_ring,
15414 spawn_events,
15415 &exit_report,
15416 terminal_disposition(final_state),
15417 detail,
15418 )
15419 },
15420 || {
15421 update_snapshot(snapshot, Some(module_id), |state| {
15422 state.state = final_state;
15423 if final_state == ModuleState::Failed {
15424 state.failed_retry.failed_at = Some(Instant::now());
15425 }
15426 if let Some(enabled) = enabled {
15427 state.enabled = enabled;
15428 }
15429 clear_current_process_facts(state);
15430 state.last_exit = Some(exit_report.clone());
15431 if exit_report.kind == ExitKind::DeliberateSeverance {
15432 state.lifetime_restarts += 1;
15433 }
15434 })
15435 },
15436 )?;
15437 child.drain_stderr(module_id).await;
15438
15439 let released =
15440 release_dead_registration(registry, forwarding, snapshot, module_id, failure).await;
15441 child
15442 .finish_exit(
15443 released.is_ok()
15444 && final_state == ModuleState::Restarting
15445 && child.spawn_failure.is_none(),
15446 )
15447 .await
15448 .map_err(|source| SuperviseError::Wait {
15449 module_id: module_id.to_owned(),
15450 source,
15451 })?;
15452 released
15453}
15454
15455#[cfg(unix)]
15475fn request_graceful_stop(module_id: &str, child: &SupervisedChild) {
15476 let Some(pid) = child
15477 .id()
15478 .and_then(|pid| i32::try_from(pid).ok())
15479 .and_then(rustix::process::Pid::from_raw)
15480 else {
15481 debug!(
15482 module_id,
15483 "no pid to signal for teardown; falling through to the drain wait"
15484 );
15485 return;
15486 };
15487 match rustix::process::kill_process(pid, rustix::process::Signal::TERM) {
15488 Ok(()) => debug!(
15489 module_id,
15490 "sent SIGTERM to a module nothing else asked to stop"
15491 ),
15492 Err(err) => debug!(
15493 module_id,
15494 error = %err,
15495 "SIGTERM to module failed; the drain wait and kill still apply"
15496 ),
15497 }
15498}
15499
15500#[cfg(not(unix))]
15508fn request_graceful_stop(module_id: &str, _child: &SupervisedChild) {
15509 debug!(
15510 module_id,
15511 "no graceful stop signal exists on this platform; teardown of a module nothing asked to stop waits, then kills"
15512 );
15513}
15514
15515fn terminal_disposition(final_state: ModuleState) -> TerminalDisposition {
15516 match final_state {
15517 ModuleState::Stopped => TerminalDisposition::Stopped,
15518 ModuleState::Disabled => TerminalDisposition::Disabled,
15519 ModuleState::Restarting => TerminalDisposition::Restarting,
15520 ModuleState::Failed => TerminalDisposition::Failed,
15521 ModuleState::Starting
15522 | ModuleState::Running
15523 | ModuleState::Unresponsive
15524 | ModuleState::Draining => {
15525 unreachable!("terminal exits only finish in terminal or restarting states")
15526 }
15527 }
15528}
15529
15530async fn release_dead_registration(
15538 registry: &Registry,
15539 forwarding: Option<&ForwardingTable>,
15540 snapshot: &SharedSnapshot,
15541 module_id: &str,
15542 failure: &FailureContext,
15543) -> Result<(), SuperviseError> {
15544 let result = async {
15545 let registration = registry
15546 .get_module(module_id)
15547 .map_err(SuperviseError::Registry)?;
15548 match wait_for_registration_release(registry, module_id, REGISTRY_RELEASE_TIMEOUT).await {
15549 Ok(()) => return Ok(()),
15550 Err(SuperviseError::RegistrationStillActive { .. }) => {}
15551 Err(err) => return Err(err),
15552 }
15553 let pid = lock_snapshot(snapshot)?.reaped_pid;
15554 if let (Some(registration), Some(forwarding), Some(pid)) = (registration, forwarding, pid) {
15555 warn!(
15556 module_id,
15557 pid,
15558 connection_id = registration.connection_id.get(),
15559 "reaped module registration outlived release grace; closing dead connection"
15560 );
15561 forwarding.request_connection_close(
15562 registration.connection_id,
15563 CloseReason::new(
15564 "supervised_process_reaped",
15565 format!("module '{module_id}' pid {pid} exited"),
15566 ),
15567 );
15568 wait_for_slot_registration_release(
15569 registry,
15570 crate::registry::RegistrationSlot::Connection(registration.connection_id),
15571 REGISTRY_RELEASE_TIMEOUT,
15572 )
15573 .await?;
15574 }
15575 wait_for_registration_release(registry, module_id, Duration::ZERO).await
15576 }
15577 .await;
15578 if let Err(err) = &result {
15579 fail_snapshot(snapshot, Some(module_id), None, failure);
15580 error!(module_id, error = %err, "registration release failed after child exit; module is failed and start can retry");
15581 }
15582 result
15583}
15584
15585async fn wait_for_registration_release(
15588 registry: &Registry,
15589 module_id: &str,
15590 wait: Duration,
15591) -> Result<(), SuperviseError> {
15592 wait_for_slot_registration_release(
15593 registry,
15594 crate::registry::RegistrationSlot::Active(module_id),
15595 wait,
15596 )
15597 .await
15598}
15599
15600async fn wait_for_slot_registration_release(
15608 registry: &Registry,
15609 slot: crate::registry::RegistrationSlot<'_>,
15610 wait: Duration,
15611) -> Result<(), SuperviseError> {
15612 let deadline = Instant::now() + wait;
15613 let mut release_events = registration_release_events().subscribe();
15614 let still_active = |registration: &crate::registry::ModuleRegistration| {
15615 SuperviseError::RegistrationStillActive {
15616 module_id: registration.manifest.module_id.clone(),
15617 waited: wait,
15618 }
15619 };
15620 loop {
15621 let _observed_generation = *release_events.borrow_and_update();
15622 let Some(registration) = registry
15623 .registration(slot)
15624 .map_err(SuperviseError::Registry)?
15625 else {
15626 return Ok(());
15627 };
15628
15629 let now = Instant::now();
15630 if now >= deadline {
15631 return Err(still_active(®istration));
15632 }
15633
15634 let remaining = deadline.saturating_duration_since(now);
15635 match timeout(remaining, release_events.changed()).await {
15636 Ok(Ok(())) | Ok(Err(_)) => {}
15637 Err(_) => return Err(still_active(®istration)),
15638 }
15639 }
15640}
15641
15642#[cfg(test)]
15643mod slot_registration_wait_tests {
15644 use super::*;
15645 use crate::registry::{ConnectionId, RegistrationSlot};
15646 use subc_protocol::manifest::ModuleManifest;
15647
15648 #[tokio::test]
15649 async fn enable_release_failure_is_failed_and_a_second_enable_retries() {
15650 let registry = Arc::new(Registry::default());
15651 let supervisor = Supervisor::new_for_test(Arc::clone(®istry), RestartPolicy::default());
15652 let runtime = supervisor.runtime_config();
15653 let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::disabled()));
15654 let spec = ModuleSpec {
15655 script: None,
15656 module_id: "enable-stale-registration".to_string(),
15657 program: PathBuf::from("/missing/enable-retry-test"),
15658 args: Vec::new(),
15659 env: Vec::new(),
15660 reserved: false,
15661 reserved_prefixes: Vec::new(),
15662 protocol: ModuleProtocol::Subc,
15663 overlap: Default::default(),
15664 carry_containment: false,
15665 };
15666 let connection = ConnectionId::new(90);
15667 registry
15668 .register_with_control_ops(
15669 ModuleManifest::builder(&spec.module_id, "0.1.0").build(),
15670 1,
15671 connection,
15672 Vec::new(),
15673 )
15674 .unwrap();
15675 let mut child = None;
15676 let err = set_child_enabled(
15677 &spec,
15678 &runtime,
15679 ®istry,
15680 &supervisor.process_liveness,
15681 &snapshot,
15682 &mut child,
15683 true,
15684 &runtime.failure_context(false),
15685 )
15686 .await
15687 .unwrap_err();
15688 assert!(matches!(
15689 err,
15690 SuperviseError::RegistrationStillActive { .. }
15691 ));
15692 assert_eq!(lock_snapshot(&snapshot).unwrap().state, ModuleState::Failed);
15693 assert!(child.is_none());
15694 registry.deregister_connection(connection).unwrap();
15695 let err = set_child_enabled(
15696 &spec,
15697 &runtime,
15698 ®istry,
15699 &supervisor.process_liveness,
15700 &snapshot,
15701 &mut child,
15702 true,
15703 &runtime.failure_context(false),
15704 )
15705 .await
15706 .unwrap_err();
15707 assert!(
15708 matches!(err, SuperviseError::Spawn { .. }),
15709 "second enable must attempt a spawn: {err}"
15710 );
15711 assert_eq!(lock_snapshot(&snapshot).unwrap().state, ModuleState::Failed);
15712 }
15713
15714 const INCUMBENT: u64 = 1;
15715 const CANDIDATE: u64 = 2;
15716
15717 fn swapped_registry() -> Arc<Registry> {
15718 let registry = Arc::new(Registry::default());
15719 let manifest = ModuleManifest::builder("m", "0.1.0").build();
15720 registry
15721 .register_with_control_ops(
15722 manifest.clone(),
15723 1,
15724 ConnectionId::new(INCUMBENT),
15725 Vec::new(),
15726 )
15727 .unwrap();
15728 registry
15729 .register_candidate_with_control_ops(
15730 manifest,
15731 1,
15732 ConnectionId::new(CANDIDATE),
15733 Vec::new(),
15734 )
15735 .unwrap();
15736 registry
15737 }
15738
15739 #[tokio::test]
15743 async fn incumbent_release_is_awaited_by_connection_not_by_module_id() {
15744 let registry = swapped_registry();
15745 registry.promote_candidate("m").unwrap().unwrap();
15746
15747 assert!(matches!(
15748 wait_for_registration_release(®istry, "m", Duration::from_millis(50)).await,
15749 Err(SuperviseError::RegistrationStillActive { .. })
15750 ));
15751
15752 assert!(matches!(
15754 wait_for_slot_registration_release(
15755 ®istry,
15756 RegistrationSlot::Connection(ConnectionId::new(INCUMBENT)),
15757 Duration::from_millis(50),
15758 )
15759 .await,
15760 Err(SuperviseError::RegistrationStillActive { .. })
15761 ));
15762
15763 let releaser = Arc::clone(®istry);
15764 let release = tokio::spawn(async move {
15765 sleep(Duration::from_millis(20)).await;
15766 releaser
15767 .deregister_connection(ConnectionId::new(INCUMBENT))
15768 .unwrap();
15769 notify_registration_release();
15770 });
15771 wait_for_slot_registration_release(
15772 ®istry,
15773 RegistrationSlot::Connection(ConnectionId::new(INCUMBENT)),
15774 Duration::from_secs(5),
15775 )
15776 .await
15777 .expect("the incumbent's own registration is released");
15778 release.await.unwrap();
15779 assert!(registry.get_module("m").unwrap().is_some());
15780 }
15781
15782 #[tokio::test]
15785 async fn candidate_slot_wait_ignores_the_incumbents_registration() {
15786 let registry = swapped_registry();
15787 assert!(matches!(
15788 wait_for_slot_registration_release(
15789 ®istry,
15790 RegistrationSlot::Candidate("m"),
15791 Duration::from_millis(50),
15792 )
15793 .await,
15794 Err(SuperviseError::RegistrationStillActive { .. })
15795 ));
15796 registry
15797 .deregister_connection(ConnectionId::new(CANDIDATE))
15798 .unwrap();
15799 wait_for_slot_registration_release(
15800 ®istry,
15801 RegistrationSlot::Candidate("m"),
15802 Duration::from_millis(50),
15803 )
15804 .await
15805 .expect("a candidate slot with no candidate is released");
15806 assert!(registry
15807 .registration(RegistrationSlot::Active("m"))
15808 .unwrap()
15809 .is_some());
15810 }
15811}
15812
15813fn classify_exit(status: &ExitStatus) -> ExitReport {
15814 ExitReport {
15815 kind: if status.success() {
15816 ExitKind::Clean
15817 } else {
15818 ExitKind::Crash
15819 },
15820 code: status.code(),
15821 signal: exit_signal(status),
15822 at_ms: unix_ms_now(),
15823 }
15824}
15825
15826fn wait_error_exit_report() -> ExitReport {
15832 ExitReport {
15833 kind: ExitKind::Crash,
15834 code: None,
15835 signal: None,
15836 at_ms: unix_ms_now(),
15837 }
15838}
15839
15840#[cfg(unix)]
15841fn exit_signal(status: &ExitStatus) -> Option<i32> {
15842 use std::os::unix::process::ExitStatusExt;
15843
15844 status.signal()
15845}
15846
15847#[cfg(not(unix))]
15848fn exit_signal(_status: &ExitStatus) -> Option<i32> {
15849 None
15850}
15851
15852fn reset_restart_count(snapshot: &SharedSnapshot, module_id: &str) -> Result<(), SuperviseError> {
15858 update_snapshot(snapshot, Some(module_id), |state| {
15859 state.clear_crash_restarts();
15860 })
15861}
15862
15863fn set_running(
15864 snapshot: &SharedSnapshot,
15865 child: &SupervisedChild,
15866 module_id: &str,
15867 spawn_events: &SpawnEventFeed,
15868) -> Result<(), SuperviseError> {
15869 let mut state = snapshot.lock().map_err(|_| SuperviseError::StatePoisoned {
15870 module_id: Some(module_id.to_string()),
15871 })?;
15872 state.spawn_generation = spawn_events.emit_spawned(
15873 module_id,
15874 child
15875 .registration_spawn
15876 .as_ref()
15877 .map(|spawn| spawn.authority.spawn_number()),
15878 child.pid,
15879 child.spawned_at_ms,
15880 );
15881 if std::mem::take(&mut state.coalesced_restart_pending) {
15882 let generation = state.spawn_generation;
15883 info!(module_id, "restart already satisfied by generation {generation}; coalesced pending request completed");
15884 }
15885 state.drain_disposition_detail = None;
15886 state.spawn_failure = None;
15887 state.admission_refused = None;
15888 state.config_hold_parked = false;
15889 state.config_hold_daemon_initiated = false;
15890 state.config_hold_approval = None;
15891 state.legacy_registration_spawn =
15892 child.registration_spawn.is_some() && child.admission_base.is_none();
15893 state.registration_declaration = child.registration_declaration.clone();
15894 state.stop_reason = None;
15895 state.config_exit_failed_approval = None;
15896 state.waiting_for_previous_pid = None;
15897 state.waiting_for_previous_detail = None;
15898 state.in_alternate_slot = false;
15901 state.configuration_updated_since_spawn = false;
15902 state.spawned_protocol = Some(child.protocol);
15903 state.state = ModuleState::Running;
15904 state.approved_start_failure = None;
15905 state.enabled = true;
15906 state.process_alive = true;
15907 state.pid = child.id();
15908 state.subtree_target = child.subtree_target.clone();
15909 state.health.status = SupervisorHealthStatus::Unknown;
15912 state.last_probe_evidence = None;
15913 state.health.consecutive_failures = 0;
15914 state.health.detail = None;
15915 state.health.metrics = None;
15916 state.health.last_probe_ms = None;
15917 state.failed_retry.healthy_since = None;
15918 state.failed_retry.failed_at = None;
15919 state.failed_retry.running_since = Some(child.spawned_at);
15920 #[cfg(windows)]
15921 {
15922 state.pipe_consumed = child
15923 .nonce_delivery
15924 .as_ref()
15925 .map(|delivery| delivery.consumed());
15926 }
15927 #[cfg(target_os = "macos")]
15928 {
15929 state.report_ready = Some(Arc::clone(&child.report_ready));
15930 }
15931 state.spawned_at_ms = Some(child.spawned_at_ms);
15932 state.exec_outcome = Some(Arc::clone(&child.exec.0));
15933 state.spawned_from = Some(child.spawned_from.clone());
15934 state.launch_spec_digest = Some(child.launch_spec_digest.clone());
15935 state.spawned_file_identity = child.spawned_file_identity.clone();
15936 state.process_start_time = child.process_start_time;
15937 Ok(())
15938}
15939
15940fn clear_current_process_facts(state: &mut SupervisorSnapshot) {
15941 state.legacy_registration_spawn = false;
15942 state.registration_declaration = None;
15943 state.subtree_target = crate::subtree_resources::Target::Unavailable("not_running");
15944 state.failed_retry.healthy_since = None;
15945 state.failed_retry.running_since = None;
15946 state.waiting_for_previous_pid = None;
15947 state.waiting_for_previous_detail = None;
15948 state.process_alive = false;
15949 state.spawned_protocol = None;
15950 state.pid = None;
15951 #[cfg(windows)]
15952 {
15953 state.pipe_consumed = None;
15954 }
15955 #[cfg(target_os = "macos")]
15956 {
15957 state.report_ready = None;
15958 }
15959 state.spawned_at_ms = None;
15960 state.exec_outcome = None;
15961 state.spawned_from = None;
15962 state.launch_spec_digest = None;
15963 state.spawned_file_identity = None;
15964 state.process_start_time = None;
15965 state.deliberate_severance = None;
15966}
15967
15968#[cfg(test)]
15969fn record_deliberate_severance(
15970 snapshot: &SharedSnapshot,
15971 identity: ProcessIdentity,
15972) -> Result<(), SuperviseError> {
15973 update_snapshot(snapshot, None, |state| {
15974 state.deliberate_severance = Some(identity);
15975 })
15976}
15977
15978fn apply_deliberate_severance_marker(
15979 snapshot: &SharedSnapshot,
15980 exited_identity: Option<ProcessIdentity>,
15981 mut exit_report: ExitReport,
15982) -> ExitReport {
15983 let marker = lock_snapshot(snapshot)
15984 .ok()
15985 .and_then(|mut state| state.deliberate_severance.take());
15986 if marker.is_some() && marker == exited_identity {
15987 exit_report.kind = ExitKind::DeliberateSeverance;
15988 }
15989 exit_report
15990}
15991
15992fn classify_reaped_child_exit(
15993 snapshot: &SharedSnapshot,
15994 child: &SupervisedChild,
15995 status: &ExitStatus,
15996) -> ExitReport {
15997 let _ = update_snapshot(snapshot, None, |state| {
15998 state.reaped_pid = Some(child.pid);
15999 state.spawn_failure = child.spawn_failure.clone();
16000 });
16001 if matches!(
16002 child.spawn_failure.as_deref(),
16003 Some("registration_timeout" | "registration_channel_closed")
16004 ) {
16005 registration_failure_exit_report(classify_exit(status))
16006 } else {
16007 apply_deliberate_severance_marker(snapshot, child.process_identity(), classify_exit(status))
16008 }
16009}
16010
16011fn fail_snapshot(
16012 snapshot: &SharedSnapshot,
16013 module_id: Option<&str>,
16014 last_exit: Option<ExitReport>,
16015 failure: &FailureContext,
16016) {
16017 if let Err(err) = update_snapshot(snapshot, module_id, |state| {
16018 state.state = ModuleState::Failed;
16019 state.config_hold_parked = false;
16020 state.config_hold_daemon_initiated = false;
16021 state.config_hold_approval = None;
16022 clear_current_process_facts(state);
16023 if failure.may_retry() && state.enabled {
16024 state.failed_retry.schedule(module_id.unwrap_or("unknown"));
16025 } else if failure.shutdown.is_set() {
16026 state.failed_retry.cancel();
16027 }
16028 if let Some(last_exit) = last_exit {
16029 state.last_exit = Some(last_exit);
16030 }
16031 }) {
16032 error!(error = %err, "failed to mark supervisor state failed");
16033 }
16034}
16035
16036fn update_snapshot(
16037 snapshot: &SharedSnapshot,
16038 module_id: Option<&str>,
16039 update: impl FnOnce(&mut SupervisorSnapshot),
16040) -> Result<(), SuperviseError> {
16041 let mut state = snapshot.lock().map_err(|_| SuperviseError::StatePoisoned {
16042 module_id: module_id.map(ToOwned::to_owned),
16043 })?;
16044 update(&mut state);
16045 Ok(())
16046}
16047
16048const SLOW_SNAPSHOT_LOCK_THRESHOLD: Duration = Duration::from_millis(250);
16049
16050fn lock_snapshot_for_control<'a>(
16051 snapshot: &'a SharedSnapshot,
16052 module_id: &str,
16053 caller: &'static str,
16054) -> Result<std::sync::MutexGuard<'a, SupervisorSnapshot>, SuperviseError> {
16055 let started_at = Instant::now();
16056 let guard = lock_snapshot(snapshot)?;
16057 let waited = started_at.elapsed();
16058 if waited >= SLOW_SNAPSHOT_LOCK_THRESHOLD {
16059 warn!(
16060 module_id = %module_id,
16061 waited_ms = waited.as_millis() as u64,
16062 caller = %caller,
16063 "slow snapshot lock"
16064 );
16065 }
16066 Ok(guard)
16067}
16068
16069fn lock_snapshot(
16070 snapshot: &SharedSnapshot,
16071) -> Result<std::sync::MutexGuard<'_, SupervisorSnapshot>, SuperviseError> {
16072 snapshot
16073 .lock()
16074 .map_err(|_| SuperviseError::StatePoisoned { module_id: None })
16075}
16076
16077#[cfg(test)]
16078mod terminal_history_tests {
16079 include!("supervise_route_open_recovery_tests.rs");
16080 use std::{
16081 path::PathBuf,
16082 sync::Arc,
16083 time::{Duration, Instant},
16084 };
16085
16086 use tokio::{sync::mpsc, time::sleep};
16087
16088 use super::{
16089 apply_deliberate_severance_marker, daemon_will_restart, drain_child_to_state,
16090 drained_after_quiescence_wait, handle_reload_spawn_failure, health_restart_child,
16091 lock_snapshot, on_child_exit, record_deliberate_severance, record_wait_error_terminal,
16092 reset_restart_count, spawn_and_mark_running, update_snapshot, wait_error_exit_report,
16093 ExitKind, ExitReport, ModuleProtocol, ModuleSpec, ModuleState, NextAction, ProcessIdentity,
16094 RestartPolicy, SpawnEventKind, StopNotice, SuperviseError, SupervisedConfiguration,
16095 SupervisedModule, SupervisedModuleInner, Supervisor, SupervisorHandle,
16096 SupervisorHealthStatus, SupervisorSnapshot,
16097 };
16098 use super::Instant as ClockInstant;
16103 use crate::{
16104 registry::Registry,
16105 terminal_ring::{TerminalRing, TerminalRingConfig},
16106 };
16107 use std::sync::Mutex;
16108 use subc_control::TerminalDisposition;
16109
16110 pub(super) fn fake_aft_stub_path() -> PathBuf {
16115 let path = cortexkit_test_support::cargo_profile_bin("fake-aft-stub");
16116 assert!(
16117 path.exists(),
16118 "fake-aft-stub not built at {}: run `cargo test -p subc-core` (which builds \
16119 [[bin]] targets) rather than `cargo test -p subc-core --lib` (which does not)",
16120 path.display()
16121 );
16122 path
16123 }
16124
16125 #[test]
16126 fn reserved_never_spawned_refuses_every_hello() {
16127 let supervisor = SupervisorHandle::default();
16132 supervisor.apply_identity_configuration(&ModuleSpec {
16133 script: None,
16134 module_id: "never-spawned".to_string(),
16135 program: PathBuf::from("/usr/bin/false"),
16136 args: Vec::new(),
16137 env: Vec::new(),
16138 reserved: true,
16139 reserved_prefixes: Vec::new(),
16140 protocol: ModuleProtocol::Subc,
16141 overlap: Default::default(),
16142 carry_containment: false,
16143 });
16144 assert!(
16145 supervisor
16146 .reserved_hello_verification("never-spawned", Some("any-forged-nonce"))
16147 .is_err(),
16148 "forged nonce must refuse on a reserved never-spawned id"
16149 );
16150 assert!(
16151 supervisor
16152 .reserved_hello_verification("never-spawned", None)
16153 .is_err(),
16154 "absent nonce must refuse on a reserved never-spawned id"
16155 );
16156 supervisor.set_spawn_nonce("never-spawned", "minted".to_string());
16158 supervisor.apply_identity_configuration(&ModuleSpec {
16159 script: None,
16160 module_id: "never-spawned".to_string(),
16161 program: PathBuf::from("/usr/bin/false"),
16162 args: Vec::new(),
16163 env: Vec::new(),
16164 reserved: true,
16165 reserved_prefixes: Vec::new(),
16166 protocol: ModuleProtocol::Subc,
16167 overlap: Default::default(),
16168 carry_containment: false,
16169 });
16170 assert!(supervisor
16171 .reserved_hello_verification("never-spawned", Some("minted"))
16172 .is_ok());
16173 assert!(supervisor
16174 .reserved_hello_verification("never-spawned", Some("forged"))
16175 .is_err());
16176 }
16177
16178 fn seed_crash_restarts(state: &mut SupervisorSnapshot, count: u32) {
16181 let now = ClockInstant::now();
16182 for _ in 0..count {
16183 state.crash_restarts.push_back(now);
16184 }
16185 }
16186
16187 fn age_oldest_crash_restart_out_of_window(state: &mut SupervisorSnapshot, window: Duration) {
16191 let aged = state
16192 .crash_restarts
16193 .front()
16194 .expect("a crash restart must be recorded before it can be aged")
16195 .checked_sub(window + Duration::from_secs(1))
16196 .expect("the test clock is far enough from its origin to age an instant");
16197 state.crash_restarts[0] = aged;
16198 }
16199
16200 fn snapshot_with_restarts(enabled: bool, count: u32) -> SupervisorSnapshot {
16201 let mut state = SupervisorSnapshot::new(ModuleState::Running, enabled);
16202 seed_crash_restarts(&mut state, count);
16203 state
16204 }
16205
16206 #[test]
16207 fn daemon_owned_recovery_predicate_uses_the_pre_increment_budget() {
16208 let policy = RestartPolicy::new(3, Duration::ZERO);
16209 let now = ClockInstant::now();
16210 assert!(daemon_will_restart(
16211 &mut snapshot_with_restarts(true, 2),
16212 &policy,
16213 now
16214 ));
16215 assert!(!daemon_will_restart(
16216 &mut snapshot_with_restarts(true, 3),
16217 &policy,
16218 now
16219 ));
16220 assert!(!daemon_will_restart(
16221 &mut snapshot_with_restarts(false, 0),
16222 &policy,
16223 now
16224 ));
16225 }
16226
16227 #[test]
16228 fn crash_restart_backoff_escalates_with_in_window_count() {
16229 let policy = RestartPolicy::new(4, Duration::from_millis(100))
16230 .with_max_backoff(Duration::from_secs(30));
16231 let now = ClockInstant::now();
16232 let mut state = SupervisorSnapshot::new(ModuleState::Running, true);
16233 let schedules = (0..4)
16234 .map(|_| {
16235 state
16236 .next_crash_restart(&policy, now)
16237 .expect("the test policy allows four crash restarts")
16238 })
16239 .collect::<Vec<_>>();
16240
16241 assert_eq!(
16242 schedules
16243 .iter()
16244 .map(|schedule| schedule.restart_in_window)
16245 .collect::<Vec<_>>(),
16246 vec![0, 1, 2, 3]
16247 );
16248 assert_eq!(
16249 schedules
16250 .iter()
16251 .map(|schedule| schedule.delay)
16252 .collect::<Vec<_>>(),
16253 vec![
16254 Duration::from_millis(100),
16255 Duration::from_secs(1),
16256 Duration::from_secs(10),
16257 Duration::from_secs(30),
16258 ]
16259 );
16260 }
16261
16262 #[test]
16263 fn crash_restart_backoff_resets_after_ring_clear() {
16264 let policy = RestartPolicy::new(3, Duration::from_millis(100));
16265 let now = ClockInstant::now();
16266 let mut state = SupervisorSnapshot::new(ModuleState::Running, true);
16267 assert_eq!(
16268 state.next_crash_restart(&policy, now).unwrap().delay,
16269 Duration::from_millis(100)
16270 );
16271 assert_eq!(
16272 state.next_crash_restart(&policy, now).unwrap().delay,
16273 Duration::from_secs(1)
16274 );
16275
16276 state.clear_crash_restarts();
16277 let schedule = state
16278 .next_crash_restart(&policy, now)
16279 .expect("a cleared ring must allow another restart");
16280 assert_eq!(schedule.restart_in_window, 0);
16281 assert_eq!(schedule.delay, Duration::from_millis(100));
16282 }
16283
16284 #[test]
16285 fn crash_restart_backoff_ignores_aged_restarts() {
16286 let policy = RestartPolicy::new(3, Duration::from_millis(100));
16287 let now = ClockInstant::now();
16288 let mut state = SupervisorSnapshot::new(ModuleState::Running, true);
16289 state
16290 .next_crash_restart(&policy, now)
16291 .expect("the first restart is allowed");
16292 state
16293 .next_crash_restart(&policy, now)
16294 .expect("the second restart is allowed");
16295 state.crash_restarts[0] = now
16296 .checked_sub(policy.window + Duration::from_secs(1))
16297 .expect("the fake clock can age a restart past the window");
16298
16299 let schedule = state
16300 .next_crash_restart(&policy, now)
16301 .expect("an aged restart must release its slot");
16302 assert_eq!(schedule.restart_in_window, 1);
16303 assert_eq!(schedule.delay, Duration::from_secs(1));
16304 assert_eq!(state.crash_restarts.len(), 2);
16305 }
16306
16307 #[test]
16311 fn a_budget_spent_before_the_window_no_longer_refuses() {
16312 let policy = RestartPolicy::new(3, Duration::ZERO);
16313 let mut state = snapshot_with_restarts(true, 3);
16314 let now = ClockInstant::now();
16315 assert!(!daemon_will_restart(&mut state, &policy, now));
16316
16317 assert!(daemon_will_restart(
16318 &mut state,
16319 &policy,
16320 now + policy.window + Duration::from_secs(1)
16321 ));
16322 assert!(
16323 state.crash_restarts.is_empty(),
16324 "reading the budget must drop the instants that left the window"
16325 );
16326 }
16327
16328 fn module_with_recovery_snapshot(
16329 state: ModuleState,
16330 enabled: bool,
16331 restart_count: u32,
16332 ) -> SupervisedModule {
16333 let registry = Arc::new(Registry::default());
16334 let supervisor =
16335 Supervisor::new_for_test(Arc::clone(®istry), RestartPolicy::new(3, Duration::ZERO));
16336 let runtime = supervisor.runtime_config();
16337 let spec = ModuleSpec {
16338 script: None,
16339 module_id: "recovery-snapshot".to_string(),
16340 program: fake_aft_stub_path(),
16341 args: Vec::new(),
16342 env: Vec::new(),
16343 reserved: false,
16344 reserved_prefixes: Vec::new(),
16345 protocol: ModuleProtocol::Subc,
16346 overlap: Default::default(),
16347 carry_containment: false,
16348 };
16349 let mut snapshot = SupervisorSnapshot::new(state, enabled);
16350 seed_crash_restarts(&mut snapshot, restart_count);
16351 let (commands, _rx) = mpsc::channel(4);
16354 let configured_enabled = snapshot.enabled;
16355 SupervisedModule {
16356 inner: Arc::new(SupervisedModuleInner {
16357 module_id: spec.module_id.clone(),
16358 registry,
16359 snapshot: Arc::new(Mutex::new(snapshot)),
16360 configuration: Arc::new(Mutex::new(SupervisedConfiguration {
16361 spec,
16362 enabled: configured_enabled,
16363 health: runtime.health,
16364 register_timeout: runtime.register_timeout,
16365 restart_policy: runtime.restart_policy,
16366 })),
16367 stderr_ring: runtime.stderr_ring,
16368 terminal_ring: runtime.terminal_ring,
16369 commands,
16370 monitor: Mutex::new(None),
16371 effective_drain_timeout: runtime.effective_drain_timeout,
16372 provenance_probe: supervisor.provenance_probe.clone(),
16373 retained: runtime.child_roster.retained_view(),
16374 }),
16375 }
16376 }
16377
16378 #[cfg(target_os = "linux")]
16379 #[tokio::test]
16380 async fn no_cgroup_placement_does_not_block_fake_aft_stub_spawn() {
16381 let supervisor =
16382 Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default())
16383 .with_cgroup_placement(None);
16384 let result = supervisor.spawn(ModuleSpec {
16385 script: None,
16386 module_id: "no-cgroup-placement".to_string(),
16387 program: fake_aft_stub_path(),
16388 args: Vec::new(),
16389 env: Vec::new(),
16390 reserved: false,
16391 reserved_prefixes: Vec::new(),
16392 protocol: ModuleProtocol::Subc,
16393 overlap: Default::default(),
16394 carry_containment: false,
16395 });
16396
16397 assert!(
16398 result.is_ok(),
16399 "no delegation must not turn an otherwise valid spawn into a failure: {result:?}"
16400 );
16401 }
16402
16403 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
16404 async fn undecided_snapshot_uses_shared_restart_predicate() {
16405 assert!(module_with_recovery_snapshot(ModuleState::Running, true, 2)
16406 .will_recover_after_connection_loss()
16407 .unwrap());
16408 assert!(
16409 !module_with_recovery_snapshot(ModuleState::Running, true, 3)
16410 .will_recover_after_connection_loss()
16411 .unwrap()
16412 );
16413 }
16414
16415 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
16416 async fn restarting_snapshot_at_exhausted_budget_is_non_terminal() {
16417 assert!(
16418 module_with_recovery_snapshot(ModuleState::Restarting, true, 3)
16419 .will_recover_after_connection_loss()
16420 .unwrap()
16421 );
16422 }
16423
16424 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
16425 async fn terminal_phase_snapshots_are_terminal_before_budget_exhaustion() {
16426 assert!(!module_with_recovery_snapshot(ModuleState::Failed, true, 0)
16427 .will_recover_after_connection_loss()
16428 .unwrap());
16429 assert!(
16430 !module_with_recovery_snapshot(ModuleState::Disabled, true, 0)
16431 .will_recover_after_connection_loss()
16432 .unwrap()
16433 );
16434 }
16435
16436 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
16437 async fn warming_snapshot_is_limited_to_startup_phases() {
16438 for state in [
16439 ModuleState::Starting,
16440 ModuleState::Running,
16441 ModuleState::Restarting,
16442 ] {
16443 assert!(
16444 module_with_recovery_snapshot(state, true, 0)
16445 .is_warming()
16446 .unwrap(),
16447 "{state:?} should be warming"
16448 );
16449 }
16450 for state in [
16451 ModuleState::Unresponsive,
16452 ModuleState::Draining,
16453 ModuleState::Stopped,
16454 ModuleState::Failed,
16455 ModuleState::Disabled,
16456 ] {
16457 assert!(
16458 !module_with_recovery_snapshot(state, true, 0)
16459 .is_warming()
16460 .unwrap(),
16461 "{state:?} should not be warming"
16462 );
16463 }
16464 }
16465
16466 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
16467 async fn terminal_history_survives_respawn_and_keeps_both_crashes_in_order() {
16468 let registry = Arc::new(Registry::default());
16469 let supervisor =
16470 Supervisor::new_for_test(Arc::clone(®istry), RestartPolicy::new(1, Duration::ZERO));
16471 let module = supervisor
16472 .spawn(ModuleSpec {
16473 script: None,
16474 module_id: "terminal-history".to_string(),
16475 program: fake_aft_stub_path(),
16476 args: Vec::new(),
16477 env: vec![("FAKE_AFT_EXIT_CODE".to_string(), "23".to_string())],
16478 reserved: false,
16479 reserved_prefixes: Vec::new(),
16480 protocol: ModuleProtocol::Subc,
16481 overlap: Default::default(),
16482 carry_containment: false,
16483 })
16484 .unwrap();
16485
16486 let deadline = Instant::now() + Duration::from_secs(5);
16487 loop {
16488 let history = module.terminal_history();
16489 if history.entries.len() == 2 {
16490 assert_eq!(module.status().unwrap().state, ModuleState::Failed);
16491 assert_eq!(history.dropped, 0);
16492 assert_eq!(
16493 history
16494 .entries
16495 .iter()
16496 .map(|entry| entry.exit_code)
16497 .collect::<Vec<_>>(),
16498 vec![Some(23), Some(23)]
16499 );
16500 assert!(history.entries[0].at_ms <= history.entries[1].at_ms);
16501 return;
16502 }
16503 assert!(
16504 Instant::now() < deadline,
16505 "module did not retain two terminal exits: {history:?}"
16506 );
16507 sleep(Duration::from_millis(10)).await;
16508 }
16509 }
16510
16511 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
16515 async fn disable_during_crash_backoff_cancels_pending_respawn() {
16516 let backoff = Duration::from_secs(2);
16517 let supervisor = Supervisor::new_for_test(
16518 Arc::new(Registry::default()),
16519 RestartPolicy::new(10, backoff),
16520 );
16521 let module = supervisor
16522 .spawn(ModuleSpec {
16523 script: None,
16524 module_id: "disable-during-backoff".to_string(),
16525 program: fake_aft_stub_path(),
16526 args: Vec::new(),
16527 env: vec![("FAKE_AFT_EXIT_CODE".to_string(), "23".to_string())],
16528 reserved: false,
16529 reserved_prefixes: Vec::new(),
16530 protocol: ModuleProtocol::Subc,
16531 overlap: Default::default(),
16532 carry_containment: false,
16533 })
16534 .unwrap();
16535
16536 let deadline = Instant::now() + Duration::from_secs(5);
16538 loop {
16539 if module.status().unwrap().state == ModuleState::Restarting {
16540 break;
16541 }
16542 assert!(
16543 Instant::now() < deadline,
16544 "module never entered the crash backoff"
16545 );
16546 sleep(Duration::from_millis(10)).await;
16547 }
16548
16549 let started = Instant::now();
16550 module.set_enabled(false).await.unwrap();
16551 let waited = started.elapsed();
16552
16553 assert!(
16554 waited < backoff / 2,
16555 "disable waited {waited:?} behind the {backoff:?} crash backoff; the operator command must preempt the pending respawn"
16556 );
16557 assert_eq!(module.status().unwrap().state, ModuleState::Disabled);
16558
16559 sleep(backoff + Duration::from_millis(500)).await;
16561 let status = module.status().unwrap();
16562 assert_eq!(status.state, ModuleState::Disabled);
16563 assert_eq!(
16564 status.spawn_generation, 1,
16565 "module respawned after the operator disabled it"
16566 );
16567 }
16568
16569 #[cfg(unix)]
16572 fn protocol_none_sigterm_exits_clean_spec(
16573 module_id: &str,
16574 dir: &std::path::Path,
16575 ) -> (ModuleSpec, PathBuf, PathBuf) {
16576 let ready = dir.join("ready");
16577 let marker = dir.join("sigterm");
16578 let spec = ModuleSpec {
16579 script: None,
16580 module_id: module_id.to_string(),
16581 program: fake_aft_stub_path(),
16582 args: Vec::new(),
16583 env: vec![
16584 ("FAKE_AFT_NEVER_CONNECT".to_string(), "1".to_string()),
16585 (
16586 "FAKE_AFT_SIGTERM_MARKER_PATH".to_string(),
16587 marker.display().to_string(),
16588 ),
16589 (
16590 "FAKE_AFT_NEVER_CONNECT_READY_PATH".to_string(),
16591 ready.display().to_string(),
16592 ),
16593 ],
16594 reserved: false,
16595 reserved_prefixes: Vec::new(),
16596 protocol: ModuleProtocol::None,
16597 overlap: Default::default(),
16598 carry_containment: false,
16599 };
16600 (spec, ready, marker)
16601 }
16602
16603 #[cfg(unix)]
16607 async fn wait_for_file(path: &std::path::Path) {
16608 let deadline = Instant::now() + Duration::from_secs(10);
16609 while !path.exists() {
16610 assert!(
16611 Instant::now() < deadline,
16612 "{} never appeared",
16613 path.display()
16614 );
16615 sleep(Duration::from_millis(10)).await;
16616 }
16617 }
16618
16619 #[cfg(unix)]
16623 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
16624 async fn protocol_none_unrequested_clean_exit_restarts_as_a_crash() {
16625 let dir = cortexkit_test_support::ScratchDir::new("none-unrequested-clean-exit");
16626 let (spec, ready, marker) =
16627 protocol_none_sigterm_exits_clean_spec("none-unrequested-clean-exit", dir.path());
16628 let supervisor = Supervisor::new_for_test(
16629 Arc::new(Registry::default()),
16630 RestartPolicy::new(3, Duration::ZERO),
16631 );
16632 let module = supervisor.spawn(spec).unwrap();
16633 wait_for_file(&ready).await;
16634 let deadline = Instant::now() + Duration::from_secs(10);
16638 let first_pid = loop {
16639 let status = module.status().unwrap();
16640 if status.state == ModuleState::Running {
16641 if let Some(pid) = status.pid {
16642 break pid;
16643 }
16644 }
16645 assert!(
16646 Instant::now() < deadline,
16647 "a running module must report its pid after exec confirmation: {status:?}"
16648 );
16649 sleep(Duration::from_millis(10)).await;
16650 };
16651
16652 rustix::process::kill_process(
16653 rustix::process::Pid::from_raw(i32::try_from(first_pid).unwrap()).unwrap(),
16654 rustix::process::Signal::TERM,
16655 )
16656 .unwrap();
16657
16658 let deadline = Instant::now() + Duration::from_secs(10);
16659 let respawned = loop {
16660 let status = module.status().unwrap();
16661 if status.state == ModuleState::Running
16662 && status.pid.is_some_and(|pid| pid != first_pid)
16663 {
16664 break status;
16665 }
16666 assert!(
16667 Instant::now() < deadline,
16668 "protocol-none module was not respawned after an unrequested clean exit: {status:?}"
16669 );
16670 sleep(Duration::from_millis(10)).await;
16671 };
16672 assert_eq!(respawned.spawn_generation, 2);
16673 assert!(
16674 marker.exists(),
16675 "the child must have exited through its SIGTERM handler (exit 0), or this proves nothing about clean exits"
16676 );
16677
16678 let history = module.terminal_history();
16679 assert_eq!(history.entries.len(), 1, "{history:?}");
16680 let entry = &history.entries[0];
16681 assert_eq!(entry.exit_code, Some(0));
16682 assert_eq!(entry.exit_kind, subc_control::TerminalExitKind::Clean);
16683 assert_eq!(entry.disposition, TerminalDisposition::Restarting);
16684
16685 module.stop().await.unwrap();
16686 }
16687
16688 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
16692 async fn protocol_none_repeated_clean_exits_exhaust_the_restart_budget() {
16693 let supervisor = Supervisor::new_for_test(
16694 Arc::new(Registry::default()),
16695 RestartPolicy::new(1, Duration::ZERO),
16696 );
16697 let module = supervisor
16698 .spawn(ModuleSpec {
16699 script: None,
16700 module_id: "none-clean-exit-budget".to_string(),
16701 program: fake_aft_stub_path(),
16702 args: Vec::new(),
16703 env: vec![("FAKE_AFT_EXIT_CODE".to_string(), "0".to_string())],
16704 reserved: false,
16705 reserved_prefixes: Vec::new(),
16706 protocol: ModuleProtocol::None,
16707 overlap: Default::default(),
16708 carry_containment: false,
16709 })
16710 .unwrap();
16711
16712 let deadline = Instant::now() + Duration::from_secs(10);
16715 loop {
16716 let status = module.status().unwrap();
16717 if status.state == ModuleState::Failed {
16718 break;
16719 }
16720 assert!(
16721 Instant::now() < deadline,
16722 "module never exhausted its budget: {status:?} {:?}",
16723 module.terminal_history()
16724 );
16725 sleep(Duration::from_millis(10)).await;
16726 }
16727 let history = module.terminal_history();
16728 assert_eq!(
16729 history
16730 .entries
16731 .iter()
16732 .map(|entry| (entry.exit_code, entry.disposition.clone()))
16733 .collect::<Vec<_>>(),
16734 vec![
16735 (Some(0), TerminalDisposition::Restarting),
16736 (Some(0), TerminalDisposition::Failed),
16737 ]
16738 );
16739 let detail = history.entries[1]
16740 .disposition_detail
16741 .as_deref()
16742 .expect("a budget failure names the budget");
16743 assert!(detail.contains("max_restarts=1"), "{detail}");
16744 assert_eq!(module.status().unwrap().spawn_generation, 2);
16745 }
16746
16747 #[test]
16748 fn restart_budget_failure_is_published_after_its_terminal_record() {
16749 use std::io::{Read, Write};
16750 let supervisor = Supervisor::new_for_test(
16751 Arc::new(Registry::default()),
16752 RestartPolicy::new(0, Duration::ZERO),
16753 );
16754 let runtime = supervisor.runtime_config();
16755 let spec = ModuleSpec {
16756 script: None,
16757 protocol: ModuleProtocol::None,
16758 ..windowed_crash_spec("budget-publication-order")
16759 };
16760 let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::new(
16761 ModuleState::Running,
16762 true,
16763 )));
16764 lock_snapshot(&snapshot).unwrap().process_alive = true;
16765 let reaped_snapshot = snapshot.clone();
16766 let mut barrier = terminal_record_barrier(&runtime.spawn_events);
16767 let history = runtime.terminal_ring.clone();
16768 let reap = std::thread::spawn(move || {
16769 tokio::runtime::Builder::new_current_thread()
16770 .enable_all()
16771 .build()
16772 .unwrap()
16773 .block_on(on_child_exit(
16774 &spec,
16775 runtime.restart_policy,
16776 &supervisor.registry,
16777 &reaped_snapshot,
16778 &runtime.terminal_ring,
16779 &runtime.spawn_events,
16780 &runtime.child_roster,
16781 ExitReport {
16782 kind: ExitKind::Clean,
16783 code: Some(0),
16784 signal: None,
16785 at_ms: 1,
16786 },
16787 true,
16788 ))
16789 });
16790 let mut reached = [0];
16791 barrier.read_exact(&mut reached).unwrap();
16792 assert_eq!(&reached, b"R");
16793 let (before_state, before_alive) = {
16794 let state = lock_snapshot(&snapshot).unwrap();
16795 (state.state, state.process_alive)
16796 };
16797 let before_history = history.lock().unwrap().snapshot();
16798 barrier.write_all(b"X").unwrap();
16799 assert!(matches!(reap.join().unwrap(), NextAction::Stop { .. }));
16800 assert!(before_history.entries.is_empty());
16801 assert_ne!(
16802 before_state,
16803 ModuleState::Failed,
16804 "Failed was visible before its terminal record could be written"
16805 );
16806 assert!(before_alive);
16807 assert_eq!(lock_snapshot(&snapshot).unwrap().state, ModuleState::Failed);
16808 let history = history.lock().unwrap().snapshot();
16809 assert_eq!(history.entries.len(), 1);
16810 assert_eq!(history.entries[0].exit_code, Some(0));
16811 assert_eq!(history.entries[0].disposition, TerminalDisposition::Failed);
16812 }
16813
16814 #[test]
16815 fn crash_restart_at_budget_limit_remains_recoverable_while_recording() {
16816 use std::io::{Read, Write};
16817
16818 let supervisor = Supervisor::new_for_test(
16819 Arc::new(Registry::default()),
16820 RestartPolicy::new(3, Duration::ZERO),
16821 );
16822 let mut runtime = supervisor.runtime_config();
16823 let module = module_with_recovery_snapshot(ModuleState::Running, true, 2);
16824 let snapshot = module.inner.snapshot.clone();
16825 lock_snapshot(&snapshot).unwrap().process_alive = true;
16826 runtime.terminal_ring = module.inner.terminal_ring.clone();
16827 let spec = windowed_crash_spec(module.module_id());
16828 let mut barrier = terminal_record_barrier(&runtime.spawn_events);
16829 let reaped_snapshot = snapshot.clone();
16830 let reap = std::thread::spawn(move || {
16831 tokio::runtime::Builder::new_current_thread()
16832 .enable_all()
16833 .build()
16834 .unwrap()
16835 .block_on(on_child_exit(
16836 &spec,
16837 runtime.restart_policy,
16838 &supervisor.registry,
16839 &reaped_snapshot,
16840 &runtime.terminal_ring,
16841 &runtime.spawn_events,
16842 &runtime.child_roster,
16843 ExitReport {
16844 kind: ExitKind::Crash,
16845 code: Some(2),
16846 signal: None,
16847 at_ms: 1,
16848 },
16849 true,
16850 ))
16851 });
16852
16853 let mut reached = [0];
16857 barrier.read_exact(&mut reached).unwrap();
16858 assert_eq!(&reached, b"R");
16859 let before_status = module.status().unwrap();
16860 let before_history = module.terminal_history();
16861 let before_recovers = module.will_recover_after_connection_loss().unwrap();
16862 barrier.write_all(b"X").unwrap();
16863 assert!(matches!(reap.join().unwrap(), NextAction::Restart { .. }));
16864
16865 assert_eq!(before_status.state, ModuleState::Running);
16866 assert!(before_status.process_alive);
16867 assert_eq!(before_status.restart_count, 3);
16868 assert!(before_history.entries.is_empty());
16869 assert!(
16870 before_recovers,
16871 "connection cleanup would report a terminal close for a scheduled crash restart"
16872 );
16873 let restarting = module.status().unwrap();
16874 assert_eq!(restarting.state, ModuleState::Restarting);
16875 assert_eq!(restarting.restart_count, 3);
16876 assert!(module.will_recover_after_connection_loss().unwrap());
16877 let history = module.terminal_history();
16878 assert_eq!(history.entries.len(), 1);
16879 assert_eq!(
16880 history.entries[0].disposition,
16881 TerminalDisposition::Restarting
16882 );
16883
16884 update_snapshot(&snapshot, Some(module.module_id()), |state| {
16887 state.state = ModuleState::Running;
16888 state.respawn_pending = false;
16889 })
16890 .unwrap();
16891 assert!(!module.will_recover_after_connection_loss().unwrap());
16892 update_snapshot(&snapshot, Some(module.module_id()), |state| {
16893 state.enabled = false;
16894 state.respawn_pending = true;
16895 })
16896 .unwrap();
16897 assert!(!module.will_recover_after_connection_loss().unwrap());
16898 }
16899
16900 fn terminal_record_barrier(events: &super::SpawnEventFeed) -> std::net::TcpStream {
16901 use std::net::{TcpListener, TcpStream};
16902 let listener = TcpListener::bind("127.0.0.1:0").unwrap();
16903 let writer = TcpStream::connect(listener.local_addr().unwrap()).unwrap();
16904 let (barrier, _) = listener.accept().unwrap();
16905 for stream in [&writer, &barrier] {
16906 stream
16907 .set_read_timeout(Some(Duration::from_secs(10)))
16908 .unwrap();
16909 stream
16910 .set_write_timeout(Some(Duration::from_secs(10)))
16911 .unwrap();
16912 }
16913 *events.1.lock().unwrap() = Some(writer);
16914 barrier
16915 }
16916
16917 pub(super) fn before_terminal_record(events: &super::SpawnEventFeed) {
16918 use std::io::{Read, Write};
16919 let barrier = events.1.lock().unwrap().take();
16920 if let Some(mut barrier) = barrier {
16921 barrier.write_all(b"R").unwrap();
16922 let mut release = [0];
16923 barrier.read_exact(&mut release).unwrap();
16924 assert_eq!(&release, b"X");
16925 }
16926 }
16927
16928 #[test]
16929 fn health_restart_budget_failure_is_published_after_its_terminal_record() {
16930 use std::io::{Read, Write};
16931
16932 let supervisor = Supervisor::new_for_test(
16933 Arc::new(Registry::default()),
16934 RestartPolicy::new(0, Duration::ZERO),
16935 );
16936 let mut runtime = supervisor.runtime_config();
16937 let module = module_with_recovery_snapshot(ModuleState::Running, true, 0);
16938 let snapshot = module.inner.snapshot.clone();
16939 runtime.terminal_ring = module.inner.terminal_ring.clone();
16940 let spec = ModuleSpec {
16941 script: None,
16942 module_id: module.module_id().into(),
16943 program: fake_aft_stub_path(),
16944 protocol: ModuleProtocol::None,
16945 env: vec![("FAKE_AFT_EXIT_CODE".into(), "17".into())],
16946 ..windowed_crash_spec(module.module_id())
16947 };
16948 let mut barrier = terminal_record_barrier(&runtime.spawn_events);
16949 let reap = std::thread::spawn(move || {
16950 tokio::runtime::Builder::new_current_thread()
16951 .enable_all()
16952 .build()
16953 .unwrap()
16954 .block_on(async {
16955 let mut child =
16956 Some(spawn_and_mark_running(&spec, &runtime, &snapshot).unwrap());
16957 health_restart_child(
16958 &spec,
16959 &runtime,
16960 &supervisor.registry,
16961 &supervisor.process_liveness,
16962 &snapshot,
16963 &mut child,
16964 SupervisorHealthStatus::Failing,
16965 Some("health probe did not answer"),
16966 1,
16967 )
16968 .await
16969 .unwrap();
16970 assert!(child.is_none());
16971 });
16972 });
16973 let mut reached = [0];
16978 barrier.read_exact(&mut reached).unwrap();
16979 assert_eq!(&reached, b"R");
16980 let before_status = module.status().unwrap();
16981 let before_history = module.terminal_history();
16982 barrier.write_all(b"X").unwrap();
16983 reap.join().unwrap();
16984 assert!(before_history.entries.is_empty());
16985 assert_ne!(
16986 before_status.state,
16987 ModuleState::Failed,
16988 "Failed was visible before its terminal record could be written"
16989 );
16990 assert!(
16991 before_status.process_alive,
16992 "process death was visible before its terminal record could be written"
16993 );
16994 let failed = module.status().unwrap();
16995 assert_eq!(failed.state, ModuleState::Failed);
16996 assert!(!failed.process_alive);
16997 let history = module.terminal_history();
16998 assert_eq!(history.entries.len(), 1);
16999 assert_eq!(history.entries[0].disposition, TerminalDisposition::Failed);
17000 assert_eq!(
17001 history.entries[0].disposition_detail.as_deref(),
17002 Some("crash budget exhausted: max_restarts=0 within window_secs=600")
17003 );
17004 }
17005
17006 #[test]
17007 fn observed_exits_publish_process_death_after_terminal_history() {
17008 use std::io::{Read, Write};
17009 for (kind, enabled, shutdown, wait_error, expected_state) in [
17010 (ExitKind::Clean, true, false, false, ModuleState::Stopped),
17011 (ExitKind::Crash, true, false, false, ModuleState::Restarting),
17012 (ExitKind::Crash, false, false, false, ModuleState::Disabled),
17013 (
17014 ExitKind::DeliberateSeverance,
17015 true,
17016 false,
17017 false,
17018 ModuleState::Restarting,
17019 ),
17020 #[cfg(unix)]
17021 (ExitKind::Crash, true, true, false, ModuleState::Stopped),
17022 (ExitKind::Crash, true, false, true, ModuleState::Failed),
17023 ] {
17024 let supervisor = Supervisor::new_for_test(
17025 Arc::new(Registry::default()),
17026 RestartPolicy::new(1, Duration::ZERO),
17027 );
17028 let runtime = supervisor.runtime_config();
17029 #[cfg(unix)]
17030 if shutdown {
17031 runtime.child_roster.close();
17032 }
17033 #[cfg(not(unix))]
17034 let _ = shutdown;
17035 let spec = windowed_crash_spec("exit-publication-order");
17036 let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::new(
17037 ModuleState::Running,
17038 enabled,
17039 )));
17040 lock_snapshot(&snapshot).unwrap().process_alive = true;
17041 let reaped_snapshot = snapshot.clone();
17042 let history = runtime.terminal_ring.clone();
17043 let mut barrier = terminal_record_barrier(&runtime.spawn_events);
17044 let reap = std::thread::spawn(move || {
17045 if wait_error {
17046 super::fail_after_wait_error(
17047 &spec.module_id,
17048 &reaped_snapshot,
17049 &runtime.terminal_ring,
17050 &runtime.spawn_events,
17051 "wait failed".into(),
17052 &runtime.failure_context(true),
17053 );
17054 } else {
17055 tokio::runtime::Builder::new_current_thread()
17056 .enable_all()
17057 .build()
17058 .unwrap()
17059 .block_on(on_child_exit(
17060 &spec,
17061 runtime.restart_policy,
17062 &supervisor.registry,
17063 &reaped_snapshot,
17064 &runtime.terminal_ring,
17065 &runtime.spawn_events,
17066 &runtime.child_roster,
17067 ExitReport {
17068 kind,
17069 code: Some(0),
17070 signal: None,
17071 at_ms: 1,
17072 },
17073 true,
17074 ));
17075 }
17076 });
17077 let mut reached = [0];
17078 barrier.read_exact(&mut reached).unwrap();
17079 assert_eq!(&reached, b"R");
17080 let before_alive = lock_snapshot(&snapshot).unwrap().process_alive;
17081 let before_history = history.lock().unwrap().snapshot();
17082 barrier.write_all(b"X").unwrap();
17083 reap.join().unwrap();
17084 assert!(
17085 before_alive,
17086 "{kind:?}: death was published before recording"
17087 );
17088 assert!(before_history.entries.is_empty());
17089 assert_eq!(lock_snapshot(&snapshot).unwrap().state, expected_state);
17090 assert!(!lock_snapshot(&snapshot).unwrap().process_alive);
17091 assert_eq!(history.lock().unwrap().snapshot().entries.len(), 1);
17092 }
17093 }
17094
17095 #[tokio::test]
17096 async fn health_restart_budget_failure_remains_failed_when_journal_append_fails() {
17097 let dir = cortexkit_test_support::ScratchDir::new("health-budget-journal-failure");
17098 let path = dir.join("terminals.jsonl");
17099 std::fs::create_dir(&path).unwrap();
17100 let supervisor = Supervisor::new_for_test(
17101 Arc::new(Registry::default()),
17102 RestartPolicy::new(0, Duration::ZERO),
17103 )
17104 .with_terminal_journal(path, "health-budget-journal-failure".into());
17105 let runtime = supervisor.runtime_config();
17106 let spec = ModuleSpec {
17107 script: None,
17108 program: fake_aft_stub_path(),
17109 protocol: ModuleProtocol::None,
17110 env: vec![("FAKE_AFT_EXIT_CODE".into(), "17".into())],
17111 ..windowed_crash_spec("health-budget-journal-failure")
17112 };
17113 let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::starting()));
17114 let mut child = Some(spawn_and_mark_running(&spec, &runtime, &snapshot).unwrap());
17115 health_restart_child(
17116 &spec,
17117 &runtime,
17118 &supervisor.registry,
17119 &supervisor.process_liveness,
17120 &snapshot,
17121 &mut child,
17122 SupervisorHealthStatus::Failing,
17123 None,
17124 1,
17125 )
17126 .await
17127 .unwrap();
17128 assert_eq!(lock_snapshot(&snapshot).unwrap().state, ModuleState::Failed);
17129 assert!(!lock_snapshot(&snapshot).unwrap().process_alive);
17130 let history = runtime
17131 .terminal_ring
17132 .lock()
17133 .unwrap()
17134 .durable_history(&spec.module_id);
17135 assert!(history.journal_write_failures > 0);
17136 assert_eq!(history.entries.len(), 1);
17137 assert_eq!(history.entries[0].disposition, TerminalDisposition::Failed);
17138 }
17139
17140 #[test]
17141 fn health_restart_budget_failure_remains_failed_when_exit_recording_panics() {
17142 let supervisor = Supervisor::new_for_test(
17143 Arc::new(Registry::default()),
17144 RestartPolicy::new(0, Duration::ZERO),
17145 );
17146 let runtime = supervisor.runtime_config();
17147 let spec = ModuleSpec {
17148 script: None,
17149 program: fake_aft_stub_path(),
17150 protocol: ModuleProtocol::None,
17151 env: vec![("FAKE_AFT_EXIT_CODE".into(), "17".into())],
17152 ..windowed_crash_spec("health-budget-recording-panic")
17153 };
17154 let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::starting()));
17155 let result = std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| {
17156 tokio::runtime::Builder::new_current_thread()
17157 .enable_all()
17158 .build()
17159 .unwrap()
17160 .block_on(async {
17161 let mut child =
17162 Some(spawn_and_mark_running(&spec, &runtime, &snapshot).unwrap());
17163 runtime.spawn_events.0.lock().unwrap().seq = u64::MAX;
17164 health_restart_child(
17165 &spec,
17166 &runtime,
17167 &supervisor.registry,
17168 &supervisor.process_liveness,
17169 &snapshot,
17170 &mut child,
17171 SupervisorHealthStatus::Failing,
17172 None,
17173 1,
17174 )
17175 .await
17176 .unwrap();
17177 });
17178 }));
17179 let panic = result.expect_err("recording must still unwind");
17180 assert_eq!(
17181 panic.downcast_ref::<String>().map(String::as_str),
17182 Some("spawn event sequence exhausted")
17183 );
17184 let state = lock_snapshot(&snapshot).unwrap();
17185 assert_eq!(state.state, ModuleState::Failed);
17186 assert!(!state.process_alive);
17187 assert!(state.last_exit.is_some());
17188 }
17189
17190 #[tokio::test]
17191 async fn restart_budget_failure_remains_failed_when_journal_append_fails() {
17192 let dir = cortexkit_test_support::ScratchDir::new("budget-journal-failure");
17193 let path = dir.join("terminals.jsonl");
17194 std::fs::create_dir(&path).unwrap();
17195 let supervisor = Supervisor::new_for_test(
17196 Arc::new(Registry::default()),
17197 RestartPolicy::new(0, Duration::ZERO),
17198 )
17199 .with_terminal_journal(path, "budget-journal-failure".into());
17200 let runtime = supervisor.runtime_config();
17201 let spec = windowed_crash_spec("budget-journal-failure");
17202 let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::starting()));
17203 assert!(matches!(
17204 on_child_exit(
17205 &spec,
17206 runtime.restart_policy,
17207 &supervisor.registry,
17208 &snapshot,
17209 &runtime.terminal_ring,
17210 &runtime.spawn_events,
17211 &runtime.child_roster,
17212 crash_exit_report(1),
17213 true,
17214 )
17215 .await,
17216 NextAction::Stop { .. }
17217 ));
17218 assert_eq!(lock_snapshot(&snapshot).unwrap().state, ModuleState::Failed);
17219 let history = runtime
17220 .terminal_ring
17221 .lock()
17222 .unwrap()
17223 .durable_history(&spec.module_id);
17224 assert!(history.journal_write_failures > 0);
17225 assert_eq!(history.entries.len(), 1);
17226 assert_eq!(history.entries[0].disposition, TerminalDisposition::Failed);
17227 }
17228
17229 #[test]
17230 fn restart_budget_failure_remains_failed_when_exit_recording_panics() {
17231 let supervisor = Supervisor::new_for_test(
17232 Arc::new(Registry::default()),
17233 RestartPolicy::new(0, Duration::ZERO),
17234 );
17235 let runtime = supervisor.runtime_config();
17236 let spec = windowed_crash_spec("budget-recording-panic");
17237 let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::starting()));
17238 runtime
17239 .spawn_events
17240 .emit_spawned(&spec.module_id, None, 1, 1);
17241 runtime.spawn_events.0.lock().unwrap().seq = u64::MAX;
17244 let result = std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| {
17245 tokio::runtime::Builder::new_current_thread()
17246 .enable_all()
17247 .build()
17248 .unwrap()
17249 .block_on(on_child_exit(
17250 &spec,
17251 runtime.restart_policy,
17252 &supervisor.registry,
17253 &snapshot,
17254 &runtime.terminal_ring,
17255 &runtime.spawn_events,
17256 &runtime.child_roster,
17257 crash_exit_report(1),
17258 true,
17259 ))
17260 }));
17261 let panic = result.err().expect("recording must still unwind");
17262 assert_eq!(
17263 panic.downcast_ref::<String>().map(String::as_str),
17264 Some("spawn event sequence exhausted")
17265 );
17266 assert_eq!(lock_snapshot(&snapshot).unwrap().state, ModuleState::Failed);
17267 }
17268
17269 #[cfg(unix)]
17272 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
17273 async fn protocol_none_requested_stop_and_disable_do_not_respawn() {
17274 for disable in [false, true] {
17275 let label = if disable {
17276 "none-requested-disable"
17277 } else {
17278 "none-requested-stop"
17279 };
17280 let dir = cortexkit_test_support::ScratchDir::new(label);
17281 let (spec, ready, marker) = protocol_none_sigterm_exits_clean_spec(label, dir.path());
17282 let supervisor = Supervisor::new_for_test(
17283 Arc::new(Registry::default()),
17284 RestartPolicy::new(3, Duration::ZERO),
17285 );
17286 let module = supervisor.spawn(spec).unwrap();
17287 wait_for_file(&ready).await;
17288
17289 if disable {
17290 module.set_enabled(false).await.unwrap();
17291 } else {
17292 module.stop().await.unwrap();
17293 }
17294 assert!(
17295 marker.exists(),
17296 "{label}: the child must have left through its SIGTERM handler with exit 0"
17297 );
17298
17299 sleep(Duration::from_millis(500)).await;
17302 let status = module.status().unwrap();
17303 let expected = if disable {
17304 ModuleState::Disabled
17305 } else {
17306 ModuleState::Stopped
17307 };
17308 assert_eq!(status.state, expected, "{label}");
17309 assert_eq!(
17310 status.spawn_generation, 1,
17311 "{label}: respawned after a requested stop"
17312 );
17313 let history = module.terminal_history();
17314 assert_eq!(history.entries.len(), 1, "{label}: {history:?}");
17315 assert_eq!(history.entries[0].exit_code, Some(0), "{label}");
17316 assert_ne!(
17317 history.entries[0].disposition,
17318 TerminalDisposition::Restarting,
17319 "{label}"
17320 );
17321 }
17322 }
17323
17324 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
17327 async fn subc_wire_clean_exit_is_still_a_stop() {
17328 let supervisor = Supervisor::new_for_test(
17329 Arc::new(Registry::default()),
17330 RestartPolicy::new(3, Duration::ZERO),
17331 );
17332 let module = supervisor
17333 .spawn(ModuleSpec {
17334 script: None,
17335 module_id: "wire-clean-exit".to_string(),
17336 program: fake_aft_stub_path(),
17337 args: Vec::new(),
17338 env: vec![("FAKE_AFT_EXIT_CODE".to_string(), "0".to_string())],
17339 reserved: false,
17340 reserved_prefixes: Vec::new(),
17341 protocol: ModuleProtocol::Subc,
17342 overlap: Default::default(),
17343 carry_containment: false,
17344 })
17345 .unwrap();
17346
17347 let deadline = Instant::now() + Duration::from_secs(10);
17348 while module.terminal_history().entries.is_empty() {
17349 assert!(Instant::now() < deadline, "module never exited");
17350 sleep(Duration::from_millis(10)).await;
17351 }
17352 sleep(Duration::from_millis(500)).await;
17354 let status = module.status().unwrap();
17355 assert_eq!(status.state, ModuleState::Stopped);
17356 assert_eq!(status.spawn_generation, 1);
17357 let history = module.terminal_history();
17358 assert_eq!(history.entries.len(), 1, "{history:?}");
17359 assert_eq!(history.entries[0].exit_code, Some(0));
17360 assert_eq!(history.entries[0].disposition, TerminalDisposition::Stopped);
17361 }
17362
17363 #[cfg(unix)]
17364 #[tokio::test]
17365 async fn reload_exit_keeps_roster_until_terminal_is_recorded_during_shutdown() {
17366 let dir = cortexkit_test_support::ScratchDir::new("reload-roster-terminal-order");
17367 let record = dir.join("live-children.json");
17368 let supervisor = Supervisor::new_for_test(
17369 Arc::new(Registry::default()),
17370 RestartPolicy::new(0, Duration::ZERO),
17371 );
17372 let mut runtime = supervisor.runtime_config();
17373 runtime.child_roster.record_to(record.clone());
17374 let gate = Arc::new(super::ReloadExitRecordGate::default());
17375 runtime.test_reload_exit_record_gate = Some(Arc::clone(&gate));
17376 let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::starting()));
17377 let spec = ModuleSpec {
17378 script: None,
17379 module_id: "reload-exit-roster".into(),
17380 program: fake_aft_stub_path(),
17381 args: Vec::new(),
17382 env: vec![("FAKE_AFT_EXIT_CODE".into(), "23".into())],
17383 reserved: false,
17384 reserved_prefixes: Vec::new(),
17385 protocol: ModuleProtocol::Subc,
17386 overlap: Default::default(),
17387 carry_containment: false,
17388 };
17389 let mut child = None;
17390 let reload = super::finish_reload_child(
17391 &spec,
17392 &runtime,
17393 &supervisor.registry,
17394 &supervisor.process_liveness,
17395 &snapshot,
17396 &mut child,
17397 );
17398 tokio::pin!(reload);
17399 tokio::select! {
17400 result = &mut reload => panic!("reload missed the before-record gate: {result:?}"),
17401 _ = gate.reached.notified() => {}
17402 }
17403 assert!(runtime
17404 .terminal_ring
17405 .lock()
17406 .unwrap()
17407 .snapshot()
17408 .entries
17409 .is_empty());
17410 assert_eq!(
17411 crate::live_children::read_record(&record).unwrap().len(),
17412 1,
17413 "shutdown must still wait for the reaped child until its terminal record exists"
17414 );
17415 runtime.child_roster.close();
17416 gate.resume.notify_one();
17417 assert!(reload.await.is_err());
17418 assert!(crate::live_children::read_record(&record)
17419 .unwrap()
17420 .is_empty());
17421 let history = runtime.terminal_ring.lock().unwrap().snapshot();
17422 assert_eq!(history.entries.len(), 1);
17423 assert_eq!(
17424 history.entries[0].disposition,
17425 TerminalDisposition::DaemonShutdown
17426 );
17427 }
17428
17429 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
17433 async fn every_restart_increment_path_advances_lifetime_count() {
17434 let supervisor = Supervisor::new_for_test(
17435 Arc::new(Registry::default()),
17436 RestartPolicy::new(1, Duration::ZERO),
17437 );
17438 let runtime = supervisor.runtime_config();
17439 let spec = ModuleSpec {
17440 script: None,
17441 module_id: "lifetime-increment-path".to_string(),
17442 program: PathBuf::from("/unused/lifetime-increment-path"),
17443 args: Vec::new(),
17444 env: Vec::new(),
17445 reserved: false,
17446 reserved_prefixes: Vec::new(),
17447 protocol: ModuleProtocol::Subc,
17448 overlap: Default::default(),
17449 carry_containment: false,
17450 };
17451
17452 let crash_snapshot = Arc::new(Mutex::new(SupervisorSnapshot::starting()));
17453 assert!(matches!(
17454 on_child_exit(
17455 &spec,
17456 runtime.restart_policy,
17457 &supervisor.registry,
17458 &crash_snapshot,
17459 &runtime.terminal_ring,
17460 &runtime.spawn_events,
17461 &runtime.child_roster,
17462 ExitReport {
17463 kind: ExitKind::Crash,
17464 code: Some(1),
17465 signal: None,
17466 at_ms: 1,
17467 },
17468 true,
17469 )
17470 .await,
17471 NextAction::Restart { schedule: _ }
17472 ));
17473 let (crash_restarts, crash_lifetime) = {
17474 let state = lock_snapshot(&crash_snapshot).unwrap();
17475 (state.crash_restarts.len(), state.lifetime_restarts)
17476 };
17477 assert_eq!(crash_restarts, 1);
17478 assert_eq!(crash_lifetime, 1);
17479
17480 let health_snapshot = Arc::new(Mutex::new(SupervisorSnapshot::starting()));
17481 let mut health_child = None;
17482 assert!(matches!(
17483 health_restart_child(
17484 &spec,
17485 &runtime,
17486 &supervisor.registry,
17487 &supervisor.process_liveness,
17488 &health_snapshot,
17489 &mut health_child,
17490 SupervisorHealthStatus::Failing,
17491 None,
17492 2,
17493 )
17494 .await,
17495 Ok(())
17496 ));
17497 assert!(health_child.is_none());
17498 assert!(lock_snapshot(&health_snapshot).unwrap().respawn_pending);
17499 assert!(runtime.scheduled_respawn.lock().unwrap().take().is_some());
17500 let (health_restarts, health_lifetime) = {
17501 let state = lock_snapshot(&health_snapshot).unwrap();
17502 (state.crash_restarts.len(), state.lifetime_restarts)
17503 };
17504 assert_eq!(health_restarts, 1);
17505 assert_eq!(health_lifetime, 1);
17506
17507 let reload_snapshot = Arc::new(Mutex::new(SupervisorSnapshot::starting()));
17508 let mut reload_child = None;
17509 assert!(matches!(
17510 handle_reload_spawn_failure(
17511 &spec,
17512 &runtime,
17513 &supervisor.process_liveness,
17514 &reload_snapshot,
17515 &mut reload_child,
17516 "forced reload spawn failure".to_string(),
17517 )
17518 .await,
17519 Err(SuperviseError::ReloadFailed { .. })
17520 ));
17521 let (reload_restarts, reload_lifetime) = {
17522 let state = lock_snapshot(&reload_snapshot).unwrap();
17523 (state.crash_restarts.len(), state.lifetime_restarts)
17524 };
17525 assert_eq!(reload_restarts, 1);
17526 assert_eq!(reload_lifetime, 1);
17527 }
17528
17529 #[tokio::test]
17530 async fn deliberately_severed_live_child_records_lifetime_without_spending_restart_budget() {
17531 let supervisor = Supervisor::new_for_test(
17532 Arc::new(Registry::default()),
17533 RestartPolicy::new(3, Duration::ZERO),
17534 );
17535 let runtime = supervisor.runtime_config();
17536 let spec = ModuleSpec {
17537 script: None,
17538 module_id: "deliberately-severed".to_string(),
17539 program: PathBuf::from("/unused/deliberately-severed"),
17540 args: Vec::new(),
17541 env: Vec::new(),
17542 reserved: false,
17543 reserved_prefixes: Vec::new(),
17544 protocol: ModuleProtocol::Subc,
17545 overlap: Default::default(),
17546 carry_containment: false,
17547 };
17548 let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::starting()));
17549 let process = ProcessIdentity {
17550 pid: 41,
17551 start_time: 101,
17552 };
17553 record_deliberate_severance(&snapshot, process).unwrap();
17554 let exit_report = apply_deliberate_severance_marker(
17555 &snapshot,
17556 Some(process),
17557 ExitReport {
17558 kind: ExitKind::Crash,
17559 code: Some(1),
17560 signal: None,
17561 at_ms: 1,
17562 },
17563 );
17564 assert_eq!(exit_report.kind, ExitKind::DeliberateSeverance);
17565
17566 assert!(matches!(
17567 on_child_exit(
17568 &spec,
17569 runtime.restart_policy,
17570 &supervisor.registry,
17571 &snapshot,
17572 &runtime.terminal_ring,
17573 &runtime.spawn_events,
17574 &runtime.child_roster,
17575 exit_report,
17576 true,
17577 )
17578 .await,
17579 NextAction::Restart { schedule: _ }
17580 ));
17581 let state = lock_snapshot(&snapshot).unwrap();
17582 assert_eq!(state.lifetime_restarts, 1);
17583 assert_eq!(state.crash_restarts.len(), 0);
17584 }
17585
17586 #[tokio::test]
17587 async fn genuine_crash_spends_restart_budget_and_records_lifetime() {
17588 let supervisor = Supervisor::new_for_test(
17589 Arc::new(Registry::default()),
17590 RestartPolicy::new(3, Duration::ZERO),
17591 );
17592 let runtime = supervisor.runtime_config();
17593 let spec = ModuleSpec {
17594 script: None,
17595 module_id: "genuine-crash".to_string(),
17596 program: PathBuf::from("/unused/genuine-crash"),
17597 args: Vec::new(),
17598 env: Vec::new(),
17599 reserved: false,
17600 reserved_prefixes: Vec::new(),
17601 protocol: ModuleProtocol::Subc,
17602 overlap: Default::default(),
17603 carry_containment: false,
17604 };
17605 let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::starting()));
17606
17607 assert!(matches!(
17608 on_child_exit(
17609 &spec,
17610 runtime.restart_policy,
17611 &supervisor.registry,
17612 &snapshot,
17613 &runtime.terminal_ring,
17614 &runtime.spawn_events,
17615 &runtime.child_roster,
17616 ExitReport {
17617 kind: ExitKind::Crash,
17618 code: Some(1),
17619 signal: None,
17620 at_ms: 1,
17621 },
17622 true,
17623 )
17624 .await,
17625 NextAction::Restart { schedule: _ }
17626 ));
17627 let state = lock_snapshot(&snapshot).unwrap();
17628 assert_eq!(state.lifetime_restarts, 1);
17629 assert_eq!(state.crash_restarts.len(), 1);
17630 }
17631
17632 fn crash_exit_report(at_ms: u64) -> ExitReport {
17633 ExitReport {
17634 kind: ExitKind::Crash,
17635 code: Some(1),
17636 signal: None,
17637 at_ms,
17638 }
17639 }
17640
17641 fn windowed_crash_spec(module_id: &str) -> ModuleSpec {
17642 ModuleSpec {
17643 script: None,
17644 module_id: module_id.to_string(),
17645 program: PathBuf::from("/unused").join(module_id),
17646 args: Vec::new(),
17647 env: Vec::new(),
17648 reserved: false,
17649 reserved_prefixes: Vec::new(),
17650 protocol: ModuleProtocol::Subc,
17651 overlap: Default::default(),
17652 carry_containment: false,
17653 }
17654 }
17655
17656 #[tokio::test]
17662 async fn three_crashes_inside_the_window_stop_the_module_and_name_the_window() {
17663 let (logs, _guard) = crate::router::test_log::log_capture(tracing::Level::ERROR);
17664 let supervisor = Supervisor::new_for_test(
17665 Arc::new(Registry::default()),
17666 RestartPolicy::new(2, Duration::ZERO),
17667 );
17668 let runtime = supervisor.runtime_config();
17669 let spec = windowed_crash_spec("crash-loop-in-window");
17670 let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::starting()));
17671
17672 for attempt in 1..=2 {
17673 assert!(
17674 matches!(
17675 on_child_exit(
17676 &spec,
17677 runtime.restart_policy,
17678 &supervisor.registry,
17679 &snapshot,
17680 &runtime.terminal_ring,
17681 &runtime.spawn_events,
17682 &runtime.child_roster,
17683 crash_exit_report(attempt),
17684 true,
17685 )
17686 .await,
17687 NextAction::Restart { schedule: _ }
17688 ),
17689 "crash {attempt} is inside the budget and must respawn"
17690 );
17691 }
17692
17693 assert!(matches!(
17694 on_child_exit(
17695 &spec,
17696 runtime.restart_policy,
17697 &supervisor.registry,
17698 &snapshot,
17699 &runtime.terminal_ring,
17700 &runtime.spawn_events,
17701 &runtime.child_roster,
17702 crash_exit_report(3),
17703 true,
17704 )
17705 .await,
17706 NextAction::Stop { .. }
17707 ));
17708
17709 {
17710 let state = lock_snapshot(&snapshot).unwrap();
17711 assert_eq!(state.state, ModuleState::Failed);
17712 assert_eq!(state.crash_restarts.len(), 2);
17713 assert_eq!(state.lifetime_restarts, 2);
17714 }
17715
17716 let history = runtime
17717 .terminal_ring
17718 .lock()
17719 .expect("terminal ring is not poisoned")
17720 .snapshot();
17721 let last = history
17722 .entries
17723 .last()
17724 .expect("the refused crash is retained");
17725 assert_eq!(last.disposition, TerminalDisposition::Failed);
17726 assert_eq!(
17727 last.disposition_detail.as_deref(),
17728 Some("crash budget exhausted: max_restarts=2 within window_secs=600")
17729 );
17730
17731 let captured = crate::router::test_log::captured_logs(&logs);
17732 assert!(
17733 captured.contains("crash budget exhausted: max_restarts=2 within window_secs=600"),
17734 "the stop must be logged with its window: {captured}"
17735 );
17736 }
17737
17738 #[tokio::test]
17746 async fn a_crash_older_than_the_window_frees_its_slot_for_a_later_crash() {
17747 let supervisor = Supervisor::new_for_test(
17748 Arc::new(Registry::default()),
17749 RestartPolicy::new(2, Duration::ZERO),
17750 );
17751 let runtime = supervisor.runtime_config();
17752 let spec = windowed_crash_spec("crash-across-windows");
17753 let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::starting()));
17754
17755 for attempt in 1..=2 {
17756 assert!(matches!(
17757 on_child_exit(
17758 &spec,
17759 runtime.restart_policy,
17760 &supervisor.registry,
17761 &snapshot,
17762 &runtime.terminal_ring,
17763 &runtime.spawn_events,
17764 &runtime.child_roster,
17765 crash_exit_report(attempt),
17766 true,
17767 )
17768 .await,
17769 NextAction::Restart { schedule: _ }
17770 ));
17771 }
17772
17773 update_snapshot(&snapshot, Some(&spec.module_id), |state| {
17776 age_oldest_crash_restart_out_of_window(state, runtime.restart_policy.window);
17777 })
17778 .unwrap();
17779
17780 assert!(
17781 matches!(
17782 on_child_exit(
17783 &spec,
17784 runtime.restart_policy,
17785 &supervisor.registry,
17786 &snapshot,
17787 &runtime.terminal_ring,
17788 &runtime.spawn_events,
17789 &runtime.child_roster,
17790 crash_exit_report(3),
17791 true,
17792 )
17793 .await,
17794 NextAction::Restart { schedule: _ }
17795 ),
17796 "a crash older than the window must not hold a budget slot"
17797 );
17798
17799 let state = lock_snapshot(&snapshot).unwrap();
17800 assert_eq!(state.state, ModuleState::Restarting);
17801 assert_eq!(
17802 state.crash_restarts.len(),
17803 2,
17804 "the aged instant is dropped and the new one takes its place"
17805 );
17806 assert_eq!(
17807 state.lifetime_restarts, 3,
17808 "the ledger counts every restart, including the ones the window forgot"
17809 );
17810 }
17811
17812 #[tokio::test]
17817 async fn an_operator_restart_clears_the_ring_and_leaves_the_ledger_alone() {
17818 let supervisor = Supervisor::new_for_test(
17819 Arc::new(Registry::default()),
17820 RestartPolicy::new(2, Duration::ZERO),
17821 );
17822 let runtime = supervisor.runtime_config();
17823 let spec = windowed_crash_spec("operator-cleared-budget");
17824 let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::starting()));
17825
17826 for attempt in 1..=2 {
17827 assert!(matches!(
17828 on_child_exit(
17829 &spec,
17830 runtime.restart_policy,
17831 &supervisor.registry,
17832 &snapshot,
17833 &runtime.terminal_ring,
17834 &runtime.spawn_events,
17835 &runtime.child_roster,
17836 crash_exit_report(attempt),
17837 true,
17838 )
17839 .await,
17840 NextAction::Restart { schedule: _ }
17841 ));
17842 }
17843
17844 reset_restart_count(&snapshot, &spec.module_id).unwrap();
17845 {
17846 let state = lock_snapshot(&snapshot).unwrap();
17847 assert!(
17848 state.crash_restarts.is_empty(),
17849 "an operator restart returns the full budget"
17850 );
17851 assert_eq!(
17852 state.lifetime_restarts, 2,
17853 "clearing the budget must not unmake the crashes"
17854 );
17855 }
17856
17857 assert!(
17858 matches!(
17859 on_child_exit(
17860 &spec,
17861 runtime.restart_policy,
17862 &supervisor.registry,
17863 &snapshot,
17864 &runtime.terminal_ring,
17865 &runtime.spawn_events,
17866 &runtime.child_roster,
17867 crash_exit_report(3),
17868 true,
17869 )
17870 .await,
17871 NextAction::Restart { schedule: _ }
17872 ),
17873 "the cleared budget must be spendable again"
17874 );
17875 let state = lock_snapshot(&snapshot).unwrap();
17876 assert_eq!(state.crash_restarts.len(), 1);
17877 assert_eq!(state.lifetime_restarts, 3);
17878 }
17879
17880 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
17881 async fn severance_marker_for_a_dead_child_does_not_label_its_successor() {
17882 let severed = ProcessIdentity {
17883 pid: 41,
17884 start_time: 101,
17885 };
17886 let successor = ProcessIdentity {
17887 pid: 41,
17888 start_time: 202,
17889 };
17890 let module = module_with_recovery_snapshot(ModuleState::Running, true, 0);
17891 update_snapshot(&module.inner.snapshot, Some("recovery-snapshot"), |state| {
17892 state.pid = Some(successor.pid);
17893 state.process_start_time = Some(successor.start_time);
17894 })
17895 .unwrap();
17896 assert!(!module.record_deliberate_severance(severed).unwrap());
17897
17898 let exit_report = apply_deliberate_severance_marker(
17899 &module.inner.snapshot,
17900 Some(successor),
17901 ExitReport {
17902 kind: ExitKind::Crash,
17903 code: Some(1),
17904 signal: None,
17905 at_ms: 1,
17906 },
17907 );
17908
17909 assert_eq!(exit_report.kind, ExitKind::Crash);
17910 }
17911
17912 #[tokio::test]
17913 async fn drain_reap_marks_deliberate_severance_and_records_lifetime_without_budget() {
17914 let registry = Registry::default();
17915 let supervisor = Supervisor::new_for_test(
17916 Arc::new(Registry::default()),
17917 RestartPolicy::new(3, Duration::ZERO),
17918 );
17919 let runtime = supervisor.runtime_config();
17920 let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::starting()));
17921 let spec = ModuleSpec {
17922 script: None,
17923 module_id: "drain-deliberate-severance".to_string(),
17924 program: fake_aft_stub_path(),
17925 args: Vec::new(),
17926 env: vec![("FAKE_AFT_EXIT_CODE".to_string(), "23".to_string())],
17927 reserved: false,
17928 reserved_prefixes: Vec::new(),
17929 protocol: ModuleProtocol::Subc,
17930 overlap: Default::default(),
17931 carry_containment: false,
17932 };
17933 let mut child = spawn_and_mark_running(&spec, &runtime, &snapshot).unwrap();
17934 let process = ProcessIdentity {
17935 pid: 41,
17936 start_time: 101,
17937 };
17938 child.process_identity = Some(process);
17939 update_snapshot(&snapshot, Some(&spec.module_id), |state| {
17940 state.pid = Some(process.pid);
17941 state.process_start_time = Some(process.start_time);
17942 })
17943 .unwrap();
17944 record_deliberate_severance(&snapshot, process).unwrap();
17945
17946 drain_child_to_state(
17947 &spec.module_id,
17948 spec.protocol,
17949 StopNotice::SentOverConnection,
17952 ®istry,
17953 None,
17954 &snapshot,
17955 &runtime.terminal_ring,
17956 &runtime.spawn_events,
17957 child,
17958 Duration::from_secs(1),
17959 ModuleState::Stopped,
17960 Some(false),
17961 &super::FailureContext::operator(),
17962 )
17963 .await
17964 .unwrap();
17965
17966 let state = lock_snapshot(&snapshot).unwrap();
17967 assert_eq!(
17968 state.last_exit.as_ref().map(|exit| exit.kind),
17969 Some(ExitKind::DeliberateSeverance)
17970 );
17971 assert_eq!(state.lifetime_restarts, 1);
17972 assert_eq!(state.crash_restarts.len(), 0);
17973 drop(state);
17974 let history = runtime.terminal_ring.lock().unwrap().snapshot();
17975 assert_eq!(
17976 history.entries[0].exit_kind,
17977 subc_control::TerminalExitKind::DeliberateSeverance
17978 );
17979 }
17980
17981 #[tokio::test]
17982 async fn ordinary_drain_reap_does_not_record_a_lifetime_restart() {
17983 let registry = Registry::default();
17984 let supervisor = Supervisor::new_for_test(
17985 Arc::new(Registry::default()),
17986 RestartPolicy::new(3, Duration::ZERO),
17987 );
17988 let runtime = supervisor.runtime_config();
17989 let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::starting()));
17990 let spec = ModuleSpec {
17991 script: None,
17992 module_id: "ordinary-drain".to_string(),
17993 program: fake_aft_stub_path(),
17994 args: Vec::new(),
17995 env: vec![("FAKE_AFT_EXIT_CODE".to_string(), "23".to_string())],
17996 reserved: false,
17997 reserved_prefixes: Vec::new(),
17998 protocol: ModuleProtocol::Subc,
17999 overlap: Default::default(),
18000 carry_containment: false,
18001 };
18002 let child = spawn_and_mark_running(&spec, &runtime, &snapshot).unwrap();
18003
18004 drain_child_to_state(
18005 &spec.module_id,
18006 spec.protocol,
18007 StopNotice::SentOverConnection,
18010 ®istry,
18011 None,
18012 &snapshot,
18013 &runtime.terminal_ring,
18014 &runtime.spawn_events,
18015 child,
18016 Duration::from_secs(1),
18017 ModuleState::Stopped,
18018 Some(false),
18019 &super::FailureContext::operator(),
18020 )
18021 .await
18022 .unwrap();
18023
18024 let state = lock_snapshot(&snapshot).unwrap();
18025 assert_eq!(
18026 state.last_exit.as_ref().map(|exit| exit.kind),
18027 Some(ExitKind::Crash)
18028 );
18029 assert_eq!(state.lifetime_restarts, 0);
18030 assert_eq!(state.crash_restarts.len(), 0);
18031 }
18032
18033 #[test]
18034 fn fatal_connection_teardown_cannot_arm_a_marker_for_a_surviving_process() {
18035 assert!(!include_str!("server.rs")
18041 .contains("router.record_deliberate_connection_severance(ctx.connection_id)"));
18042 }
18043
18044 #[test]
18051 fn drained_after_quiescence_wait_passes_ok_through_and_forces_false_on_err() {
18052 assert!(drained_after_quiescence_wait(&Ok(true)));
18053 assert!(!drained_after_quiescence_wait(&Ok(false)));
18054 assert!(!drained_after_quiescence_wait(&Err(
18055 SuperviseError::StatePoisoned { module_id: None }
18056 )));
18057 }
18058
18059 #[test]
18068 fn wait_error_exit_report_records_a_failed_terminal_with_no_code_or_signal() {
18069 let ring = Arc::new(Mutex::new(TerminalRing::new(
18070 TerminalRingConfig::default(),
18071 0,
18072 )));
18073 record_wait_error_terminal("wait-error", &ring, &super::SpawnEventFeed::default());
18074
18075 let snapshot = ring.lock().unwrap().snapshot();
18076 assert_eq!(snapshot.entries.len(), 1);
18077 let entry = &snapshot.entries[0];
18078 assert_eq!(entry.exit_code, None);
18079 assert_eq!(entry.exit_signal, None);
18080 assert_eq!(entry.disposition, TerminalDisposition::Failed);
18081 }
18082
18083 #[test]
18086 fn spawn_generation_of_a_failed_launch_is_never_reused() {
18087 let feed = super::SpawnEventFeed::default();
18088 feed.configure_incarnation("launch-numbering".to_string());
18089 let (incarnation, failed) = feed.allocate_generation("relaunched");
18090 assert_eq!(incarnation, "launch-numbering");
18091 let (_, next) = feed.allocate_generation("relaunched");
18092 assert!(next > failed);
18093 assert_eq!(feed.emit_spawned("relaunched", Some(next), 7, 1), next);
18094 assert!(feed.emit_spawned("relaunched", None, 8, 2) > next);
18096 assert_eq!(
18097 feed.allocate_generation("other").1,
18098 1,
18099 "numbering is per module"
18100 );
18101 }
18102
18103 #[test]
18104 fn wait_error_exit_path_preserves_spawn_event_density() {
18105 let feed = super::SpawnEventFeed::default();
18106 feed.configure_incarnation("wait-error-density".to_string());
18107 feed.emit_spawned("wait-error", None, 41, 1);
18108 let ring = Arc::new(Mutex::new(TerminalRing::new(
18109 TerminalRingConfig::default(),
18110 0,
18111 )));
18112
18113 record_wait_error_terminal("wait-error", &ring, &feed);
18114 feed.emit_spawned("after-wait-error", None, 42, 2);
18115
18116 let state = feed.0.lock().unwrap();
18117 let sequences = state
18118 .events
18119 .iter()
18120 .map(|event| event.cursor.seq)
18121 .collect::<Vec<_>>();
18122 assert_eq!(sequences, vec![1, 2, 3]);
18123 assert_eq!(state.events[1].kind, SpawnEventKind::Exited);
18124 assert_eq!(state.events[1].exit_code, None);
18125 assert_eq!(state.events[1].exit_signal, None);
18126 }
18127
18128 #[test]
18132 fn wait_error_exit_report_is_classified_as_a_crash() {
18133 assert_eq!(wait_error_exit_report().kind, ExitKind::Crash);
18134 }
18135}
18136
18137#[cfg(all(test, unix))]
18138mod restart_ordering_tests {
18139 use super::*;
18140 use std::io::Write;
18141 use tokio::io::AsyncReadExt;
18142
18143 #[test]
18144 #[ignore = "child process fixture"]
18145 fn serving_child() {
18146 let Ok(address_file) = std::env::var("RESTART_TEST_ADDRESS_FILE") else {
18147 return;
18148 };
18149 let listener = std::net::TcpListener::bind("127.0.0.1:0").unwrap();
18150 std::fs::write(address_file, listener.local_addr().unwrap().to_string()).unwrap();
18151 for stream in listener.incoming() {
18152 writeln!(stream.unwrap(), "{}", std::process::id()).unwrap();
18153 }
18154 }
18155
18156 async fn restart_before_publication(requests: usize) {
18157 let home = cortexkit_test_support::ScratchDir::new("restart-before-publication");
18158 let address_file = home.path().join("address");
18159 let spec = ModuleSpec {
18160 script: None,
18161 module_id: "restart-serving-child".into(),
18162 program: std::env::current_exe().unwrap(),
18163 args: vec![
18164 "--exact".into(),
18165 "supervise::restart_ordering_tests::serving_child".into(),
18166 "--ignored".into(),
18167 "--nocapture".into(),
18168 ],
18169 env: vec![(
18170 "RESTART_TEST_ADDRESS_FILE".into(),
18171 address_file.display().to_string(),
18172 )],
18173 reserved: false,
18174 reserved_prefixes: vec![],
18175 protocol: ModuleProtocol::None,
18176 overlap: Default::default(),
18177 carry_containment: false,
18178 };
18179 let mut supervisor =
18180 Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default());
18181 let (spawned_tx, spawned_rx) = oneshot::channel();
18182 let spawned_tx = Mutex::new(Some(spawned_tx));
18183 let (release_tx, release_rx) = std::sync::mpsc::channel();
18184 let release_rx = Mutex::new(release_rx);
18185 supervisor.test_after_first_spawn = AfterFirstSpawnHook(Some(Arc::new(move |_| {
18188 spawned_tx.lock().unwrap().take().unwrap().send(()).unwrap();
18189 release_rx
18190 .lock()
18191 .unwrap()
18192 .recv_timeout(Duration::from_secs(20))
18193 .unwrap();
18194 })));
18195 let registered = supervisor
18196 .register_configured_with_health(
18197 spec.clone(),
18198 true,
18199 HealthConfig::default(),
18200 None,
18201 RestartPolicy::new(3, Duration::ZERO),
18202 DEFAULT_REGISTER_TIMEOUT,
18203 )
18204 .unwrap();
18205 let module = registered.module.clone();
18206 let startup = tokio::task::spawn_blocking(move || {
18207 supervisor.start_registered_configured(registered, None, true)
18208 });
18209 spawned_rx.await.unwrap();
18210 let old_pid = tokio::time::timeout(Duration::from_secs(10), async {
18211 let address = loop {
18212 if let Ok(address) = std::fs::read_to_string(&address_file) {
18213 if !address.is_empty() {
18214 break address;
18215 }
18216 }
18217 tokio::time::sleep(Duration::from_millis(1)).await;
18218 };
18219 let mut stream = tokio::net::TcpStream::connect(address).await.unwrap();
18220 let mut reply = String::new();
18221 stream.read_to_string(&mut reply).await.unwrap();
18222 reply.trim().parse::<u32>().unwrap()
18223 })
18224 .await
18225 .expect("child must serve before generation publication");
18226 assert_eq!(
18227 lock_snapshot(&module.inner.snapshot)
18228 .unwrap()
18229 .spawn_generation,
18230 0
18231 );
18232 let mut replies = Vec::new();
18233 for _ in 0..requests {
18234 let mut restart = Box::pin(module.restart(Some(0)));
18235 std::future::poll_fn(|cx| {
18238 assert!(std::future::Future::poll(restart.as_mut(), cx).is_pending());
18239 std::task::Poll::Ready(())
18240 })
18241 .await;
18242 replies.push(restart);
18243 }
18244 release_tx.send(()).unwrap();
18245 startup.await.unwrap().unwrap();
18246 for reply in replies {
18247 reply.await.unwrap();
18248 }
18249 let replacement = tokio::time::timeout(Duration::from_secs(10), async {
18252 loop {
18253 if lock_snapshot(&module.inner.snapshot)
18254 .unwrap()
18255 .spawn_generation
18256 > 1
18257 {
18258 break;
18259 }
18260 tokio::time::sleep(Duration::from_millis(1)).await;
18261 }
18262 })
18263 .await;
18264 module.update_spec_for_test(spec).await.unwrap();
18268 let (generation, pid) = {
18269 let state = lock_snapshot(&module.inner.snapshot).unwrap();
18270 (state.spawn_generation, state.pid)
18271 };
18272 module.stop().await.unwrap();
18273 replacement.expect("restart must publish a replacement for the already-serving child");
18274 assert_ne!(
18275 pid,
18276 Some(old_pid),
18277 "a serving child predating the restart must be replaced"
18278 );
18279 assert_eq!(generation, 2, "queued restarts must share one replacement");
18280 }
18281
18282 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
18283 async fn restart_received_after_serving_before_generation_publication_replaces_child() {
18284 restart_before_publication(1).await;
18285 }
18286
18287 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
18288 async fn restart_queued_behind_restart_does_not_kill_its_replacement() {
18289 restart_before_publication(2).await;
18290 }
18291
18292 #[tokio::test]
18293 async fn restart_queued_before_replacement_spawn_keeps_the_new_child() {
18294 let supervisor =
18295 Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default());
18296 let registered = supervisor
18297 .register_configured_with_health(
18298 ModuleSpec {
18299 script: None,
18300 module_id: "queued-before-replacement".into(),
18301 program: "/bin/sleep".into(),
18306 args: vec!["600".into()],
18307 env: vec![],
18308 reserved: false,
18309 reserved_prefixes: vec![],
18310 protocol: ModuleProtocol::None,
18311 overlap: Default::default(),
18312 carry_containment: false,
18313 },
18314 true,
18315 HealthConfig::default(),
18316 None,
18317 RestartPolicy::default(),
18318 DEFAULT_REGISTER_TIMEOUT,
18319 )
18320 .unwrap();
18321 let RegisteredConfiguredModule {
18322 mut spec,
18323 mut runtime,
18324 module,
18325 mut commands,
18326 } = registered;
18327 let snapshot = &module.inner.snapshot;
18328 let mut restart = Box::pin(module.restart(Some(0)));
18329 std::future::poll_fn(|cx| {
18330 assert!(std::future::Future::poll(restart.as_mut(), cx).is_pending());
18331 std::task::Poll::Ready(())
18332 })
18333 .await;
18334 let mut child = Some(spawn_and_mark_running(&spec, &runtime, snapshot).unwrap());
18337 let replacement_pid = child.as_ref().unwrap().id();
18338 let command = commands.recv().await.unwrap();
18339 assert!(
18340 handle_supervisor_command(
18341 command,
18342 &mut spec,
18343 &mut runtime,
18344 &supervisor.registry,
18345 &supervisor.process_liveness,
18346 snapshot,
18347 &mut child,
18348 &mut commands,
18349 &mut VecDeque::new(),
18350 )
18351 .await
18352 );
18353 restart.await.unwrap();
18354 let kept_pid = child.as_ref().and_then(SupervisedChild::id);
18355 let pending = lock_snapshot(snapshot).unwrap().respawn_pending;
18356 if let Some(mut child) = child {
18357 child.start_kill().unwrap();
18358 child.wait().await.unwrap();
18359 }
18360 assert_eq!(
18361 kept_pid, replacement_pid,
18362 "a newer replacement must survive"
18363 );
18364 assert!(
18365 !pending,
18366 "a satisfied restart must not schedule another spawn"
18367 );
18368 }
18369}
18370
18371#[cfg(all(test, unix))]
18372mod health_event_tests {
18373 use super::*;
18374 use cortexkit_test_support::ScratchDir;
18375 use subc_protocol::{manifest::Concurrency, session::ModuleControlResponse};
18376
18377 struct Actor {
18378 module: SupervisedModule,
18379 registry: Arc<Registry>,
18380 forwarding: Arc<ForwardingTable>,
18381 spec: ModuleSpec,
18382 health: HealthConfig,
18383 rx: mpsc::Receiver<crate::router::OutboundFrame>,
18384 roster: ChildRoster,
18385 _home: ScratchDir,
18386 }
18387
18388 async fn wait_for<T>(reason: &str, mut observe: impl FnMut() -> Option<T>) -> T {
18389 let deadline = std::time::Instant::now() + Duration::from_secs(10);
18393 let tick = Instant::now();
18394 loop {
18395 if let Some(value) = observe() {
18396 assert_eq!(Instant::now(), tick, "{reason} must not wait for a timer");
18397 return value;
18398 }
18399 assert!(
18400 std::time::Instant::now() < deadline,
18401 "timed out waiting for {reason}"
18402 );
18403 tokio::task::yield_now().await;
18404 }
18405 }
18406
18407 impl Actor {
18408 async fn start(protocol: ModuleProtocol, cadence: Duration) -> Self {
18409 Self::start_with_policy(
18410 protocol,
18411 cadence,
18412 DEFAULT_REGISTER_TIMEOUT,
18413 RestartPolicy::new(3, Duration::ZERO),
18414 )
18415 .await
18416 }
18417
18418 async fn start_with_policy(
18419 protocol: ModuleProtocol,
18420 cadence: Duration,
18421 register_timeout: Duration,
18422 policy: RestartPolicy,
18423 ) -> Self {
18424 let home = ScratchDir::new("health-events");
18425 let registry = Arc::new(Registry::default());
18426 let forwarding = Arc::new(ForwardingTable::default());
18427 let health = HealthConfig {
18428 cadence,
18429 deadline: Duration::from_secs(3600),
18430 ..HealthConfig::default()
18431 };
18432 let supervisor = Supervisor::new_for_test(registry.clone(), policy)
18433 .with_forwarding(forwarding.clone())
18434 .with_health_config(health.clone());
18435 let spec = ModuleSpec {
18436 script: None,
18437 module_id: "event-health-child".into(),
18438 program: PathBuf::from("/bin/sleep"),
18446 args: vec!["600".into()],
18449 env: ["XDG_DATA_HOME", "XDG_RUNTIME_DIR", "XDG_CONFIG_HOME"]
18450 .into_iter()
18451 .map(|key| (key.into(), home.path().display().to_string()))
18452 .collect(),
18453 reserved: false,
18454 reserved_prefixes: vec![],
18455 protocol,
18456 overlap: Default::default(),
18457 carry_containment: false,
18458 };
18459 let module = supervisor
18460 .supervise_configured_requested(
18461 spec.clone(),
18462 true,
18463 health.clone(),
18464 Some(0),
18465 policy,
18466 register_timeout,
18467 None,
18468 )
18469 .unwrap();
18470 let (_, rx) = mpsc::channel(8);
18471 let actor = Self {
18472 module,
18473 registry,
18474 forwarding,
18475 spec,
18476 health,
18477 rx,
18478 roster: supervisor.child_roster.clone(),
18479 _home: home,
18480 };
18481 actor
18482 .wait_for_select(|checkpoint| checkpoint.generation > 0)
18483 .await;
18484 actor
18485 }
18486
18487 async fn expire_registration_wait(&self, remaining: Duration) {
18488 tokio::time::advance(remaining).await;
18489 wait_for("registration timeout TERM grace", || {
18490 lock_snapshot(&self.module.inner.snapshot)
18491 .unwrap()
18492 .actor_select
18493 .is_none()
18494 .then_some(())
18495 })
18496 .await;
18497 tokio::time::advance(crate::child_roster::TERM_TO_KILL).await;
18498 }
18499
18500 fn turns(&self) -> u64 {
18501 lock_snapshot(&self.module.inner.snapshot)
18502 .unwrap()
18503 .actor_turns
18504 }
18505
18506 async fn wait_for_select(
18507 &self,
18508 expected: impl Fn(&ActorSelectCheckpoint) -> bool,
18509 ) -> ActorSelectCheckpoint {
18510 wait_for("actor select checkpoint", || {
18511 let state = lock_snapshot(&self.module.inner.snapshot).unwrap();
18512 state.actor_select.filter(|checkpoint| {
18513 checkpoint.generation == state.spawn_generation
18514 && state.state == ModuleState::Running
18515 && state.process_alive
18516 && state.reported_pid().is_some()
18517 && expected(checkpoint)
18518 })
18519 })
18520 .await
18521 }
18522
18523 fn hello(&mut self, connection: u64, health: bool) {
18524 let manifest =
18525 subc_protocol::manifest::ModuleManifest::builder(&self.spec.module_id, "1.0.0")
18526 .protocol_ver(subc_protocol::PROTOCOL_VERSION)
18527 .build();
18528 let connection = ConnectionId::new(connection);
18529 let (tx, rx) = mpsc::channel(8);
18530 self.rx = rx;
18531 self.forwarding
18532 .register_module_connection(
18533 connection,
18534 self.spec.module_id.clone(),
18535 subc_protocol::PROTOCOL_VERSION,
18536 Concurrency::ModuleManaged,
18537 FrameSink::new(tx),
18538 )
18539 .unwrap();
18540 self.registry
18541 .register_with_control_ops(
18542 manifest,
18543 subc_protocol::PROTOCOL_VERSION,
18544 connection,
18545 if health {
18546 vec![MODULE_CONTROL_OP_HEALTH_CHECK.into()]
18547 } else {
18548 vec![]
18549 },
18550 )
18551 .unwrap();
18552 }
18553
18554 async fn probe(&mut self) -> crate::router::OutboundFrame {
18555 let frame = wait_for("outbound health probe", || match self.rx.try_recv() {
18556 Ok(frame) => Some(frame),
18557 Err(mpsc::error::TryRecvError::Empty) => None,
18558 Err(mpsc::error::TryRecvError::Disconnected) => panic!("health peer disconnected"),
18559 })
18560 .await;
18561 let body: Value = serde_json::from_slice(&frame.body).unwrap();
18562 assert_eq!(body["op"], MODULE_CONTROL_OP_HEALTH_CHECK);
18563 frame
18564 }
18565
18566 fn no_probe(&mut self) {
18567 assert!(matches!(
18568 self.rx.try_recv(),
18569 Err(mpsc::error::TryRecvError::Empty)
18570 ));
18571 }
18572
18573 fn answer(&self, connection: u64, frame: crate::router::OutboundFrame) {
18574 self.forwarding
18575 .complete_module_control_rpc(
18576 ConnectionId::new(connection),
18577 frame.header.corr,
18578 Some(MODULE_CONTROL_OP_HEALTH_CHECK),
18579 ModuleControlRpcOutcome::Response(ModuleControlResponse::HealthCheck {
18580 status: HealthStatus::Ok,
18581 detail: None,
18582 metrics: None,
18583 }),
18584 )
18585 .unwrap();
18586 }
18587 }
18588
18589 #[tokio::test(start_paused = true)]
18590 async fn registration_timeout_kills_at_120_seconds_and_respawns() {
18591 let actor = Actor::start(ModuleProtocol::Subc, Duration::from_secs(30)).await;
18592 let initial = actor.module.status().unwrap();
18593 tokio::time::advance(Duration::from_secs(119)).await;
18594 tokio::task::yield_now().await;
18595 assert_eq!(actor.module.status().unwrap().pid, initial.pid);
18596 assert!(actor.module.terminal_history().entries.is_empty());
18597 actor.expire_registration_wait(Duration::from_secs(1)).await;
18598 actor
18599 .wait_for_select(|at| at.generation > initial.spawn_generation)
18600 .await;
18601 let status = actor.module.status().unwrap();
18602 assert_ne!(status.pid, initial.pid);
18603 assert!(!cortexkit_test_support::process_alive(
18604 initial.pid.unwrap() as i32
18605 ));
18606 assert_eq!(status.restart_count, 1);
18607 assert_eq!(status.last_exit.unwrap().kind, ExitKind::Crash);
18608 let entries = actor.module.terminal_history().entries;
18609 assert_eq!(entries.len(), 1);
18610 assert_eq!(
18611 entries[0].disposition_detail.as_deref(),
18612 Some("registration_timeout")
18613 );
18614 actor.module.stop().await.unwrap();
18615 }
18616
18617 #[tokio::test(start_paused = true)]
18618 async fn registration_timeout_is_cancelled_by_hello_at_119_seconds() {
18619 let mut actor = Actor::start(ModuleProtocol::Subc, Duration::from_secs(30)).await;
18620 let pid = actor.module.status().unwrap().pid;
18621 tokio::time::advance(Duration::from_secs(119)).await;
18622 actor.hello(1, false);
18623 let registered = actor
18624 .wait_for_select(|at| at.registered_connection.is_some())
18625 .await;
18626 assert_eq!(
18627 registered.wake_after, None,
18628 "HELLO must cancel the deadline"
18629 );
18630 tokio::time::advance(Duration::from_secs(300)).await;
18631 tokio::task::yield_now().await;
18632 assert_eq!(actor.module.status().unwrap().pid, pid);
18633 assert!(actor.module.terminal_history().entries.is_empty());
18634 actor
18636 .registry
18637 .deregister_connection(ConnectionId::new(1))
18638 .unwrap();
18639 actor
18640 .wait_for_select(|at| at.registered_connection.is_none())
18641 .await;
18642 tokio::time::advance(Duration::from_secs(300)).await;
18643 tokio::task::yield_now().await;
18644 assert_eq!(actor.module.status().unwrap().pid, pid);
18645 assert!(actor.module.terminal_history().entries.is_empty());
18646 actor.module.stop().await.unwrap();
18647
18648 let mut transient = Actor::start(ModuleProtocol::Subc, Duration::from_secs(30)).await;
18649 let pid = transient.module.status().unwrap().pid;
18650 tokio::time::advance(Duration::from_secs(119)).await;
18651 let turns = transient.turns();
18652 transient.hello(1, false);
18655 transient
18656 .registry
18657 .deregister_connection(ConnectionId::new(1))
18658 .unwrap();
18659 let parked = transient.wait_for_select(|at| at.turn > turns).await;
18660 assert_eq!(
18661 parked.wake_after, None,
18662 "transient HELLO must cancel the deadline"
18663 );
18664 tokio::time::advance(Duration::from_secs(300)).await;
18665 tokio::task::yield_now().await;
18666 assert_eq!(transient.module.status().unwrap().pid, pid);
18667 assert!(transient.module.terminal_history().entries.is_empty());
18668 transient.module.stop().await.unwrap();
18669 }
18670
18671 #[tokio::test(start_paused = true)]
18672 async fn registration_timeouts_exhaust_budget_and_schedule_slow_retry() {
18673 let actor = Actor::start_with_policy(
18674 ModuleProtocol::Subc,
18675 Duration::from_secs(30),
18676 Duration::from_secs(10),
18677 RestartPolicy::new(1, Duration::ZERO),
18678 )
18679 .await;
18680 let generation = actor.module.status().unwrap().spawn_generation;
18681 actor
18682 .expire_registration_wait(Duration::from_secs(10))
18683 .await;
18684 actor.wait_for_select(|at| at.generation > generation).await;
18685 actor
18686 .expire_registration_wait(Duration::from_secs(10))
18687 .await;
18688 let retry = wait_for("failed retry schedule", || {
18689 let state = lock_snapshot(&actor.module.inner.snapshot).unwrap();
18690 (state.state == ModuleState::Failed)
18691 .then_some(state.failed_retry.pending)
18692 .flatten()
18693 })
18694 .await;
18695 assert_eq!(retry.delay, Duration::from_secs(600));
18696 assert_eq!(retry.attempt, 1);
18697 let entries = actor.module.terminal_history().entries;
18698 assert_eq!(entries.len(), 2);
18699 assert_eq!(entries[1].disposition, TerminalDisposition::Failed);
18700 assert!(entries[1]
18701 .disposition_detail
18702 .as_deref()
18703 .unwrap()
18704 .contains("registration_timeout"));
18705 tokio::time::advance(Duration::from_secs(599)).await;
18706 tokio::task::yield_now().await;
18707 assert_eq!(actor.module.state().unwrap(), ModuleState::Failed);
18708 tokio::time::advance(Duration::from_secs(1)).await;
18709 actor
18710 .wait_for_select(|at| at.generation > generation + 1)
18711 .await;
18712 actor
18714 .expire_registration_wait(Duration::from_secs(10))
18715 .await;
18716 wait_for("slow retry registration timeout", || {
18717 (actor.module.terminal_history().entries.len() == 3).then_some(())
18718 })
18719 .await;
18720 actor.module.stop().await.unwrap();
18721 }
18722
18723 #[tokio::test(start_paused = true)]
18724 async fn registration_timeout_honours_ten_second_override() {
18725 let actor = Actor::start_with_policy(
18726 ModuleProtocol::Subc,
18727 Duration::from_secs(30),
18728 Duration::from_secs(10),
18729 RestartPolicy::new(3, Duration::ZERO),
18730 )
18731 .await;
18732 let generation = actor.module.status().unwrap().spawn_generation;
18733 tokio::time::advance(Duration::from_secs(9)).await;
18734 tokio::task::yield_now().await;
18735 assert!(actor.module.terminal_history().entries.is_empty());
18736 actor.expire_registration_wait(Duration::from_secs(1)).await;
18737 actor.wait_for_select(|at| at.generation > generation).await;
18738 assert_eq!(actor.module.terminal_history().entries.len(), 1);
18739 actor.module.stop().await.unwrap();
18740 }
18741
18742 #[tokio::test]
18743 async fn registration_timeout_kills_term_ignoring_helper_before_reaping_leader() {
18744 use std::os::unix::process::{CommandExt, ExitStatusExt};
18745
18746 struct Helper(std::process::Child);
18747 impl Drop for Helper {
18748 fn drop(&mut self) {
18749 if matches!(self.0.try_wait(), Ok(None)) {
18752 let _ = self.0.kill();
18753 let _ = self.0.wait();
18754 }
18755 }
18756 }
18757
18758 let home = ScratchDir::new("registration-timeout-helper");
18759 let supervisor = Supervisor::new_for_test(
18760 Arc::new(Registry::default()),
18761 RestartPolicy::new(0, Duration::ZERO),
18762 );
18763 let spec = ModuleSpec {
18764 script: None,
18765 module_id: "registration-timeout-group".into(),
18766 program: "/bin/sleep".into(),
18767 args: vec!["120".into()],
18768 env: ["XDG_DATA_HOME", "XDG_RUNTIME_DIR", "XDG_CONFIG_HOME"]
18769 .into_iter()
18770 .map(|key| (key.into(), home.path().display().to_string()))
18771 .collect(),
18772 reserved: false,
18773 reserved_prefixes: vec![],
18774 protocol: ModuleProtocol::Subc,
18775 overlap: Default::default(),
18776 carry_containment: false,
18777 };
18778 let module = supervisor
18779 .supervise_configured_requested(
18780 spec,
18781 true,
18782 HealthConfig::default(),
18783 None,
18784 RestartPolicy::new(0, Duration::ZERO),
18785 Duration::from_secs(10),
18786 None,
18787 )
18788 .unwrap();
18789 let leader = timeout(Duration::from_secs(5), async {
18790 loop {
18791 if let Some(pid) = module.status().unwrap().pid {
18792 break pid;
18793 }
18794 sleep(Duration::from_millis(5)).await;
18795 }
18796 })
18797 .await
18798 .expect("supervised group leader must be reported before the deadline");
18799 assert!(leader > 1);
18800 let ready = home.join("helper.ready");
18801 let mut helper = Helper(
18804 std::process::Command::new("/bin/sh")
18805 .args([
18806 "-c",
18807 "trap '' TERM; : > \"$1\"; exec /bin/sleep 120",
18808 "helper",
18809 ])
18810 .arg(&ready)
18811 .process_group(i32::try_from(leader).unwrap())
18812 .stdout(std::process::Stdio::null())
18813 .stderr(std::process::Stdio::null())
18814 .spawn()
18815 .unwrap(),
18816 );
18817 timeout(Duration::from_secs(5), async {
18818 while !ready.exists() {
18819 assert!(helper.0.try_wait().unwrap().is_none());
18820 sleep(Duration::from_millis(5)).await;
18821 }
18822 })
18823 .await
18824 .expect("helper must install TERM ignore before the registration deadline");
18825 let helper_pid =
18826 rustix::process::Pid::from_raw(i32::try_from(helper.0.id()).unwrap()).unwrap();
18827 rustix::process::kill_process(helper_pid, rustix::process::Signal::TERM).unwrap();
18828 sleep(Duration::from_millis(20)).await;
18829 assert!(
18830 helper.0.try_wait().unwrap().is_none(),
18831 "helper must ignore TERM"
18832 );
18833
18834 timeout(Duration::from_secs(20), async {
18835 while module.state().unwrap() != ModuleState::Failed {
18836 sleep(Duration::from_millis(5)).await;
18837 }
18838 })
18839 .await
18840 .expect("registration timeout must stop and reap the group leader");
18841 let helper_exit = timeout(Duration::from_secs(3), async {
18842 loop {
18843 if let Some(status) = helper.0.try_wait().unwrap() {
18844 break status;
18845 }
18846 sleep(Duration::from_millis(5)).await;
18847 }
18848 })
18849 .await
18850 .expect("TERM-ignoring helper survived the registration timeout's group KILL");
18851 assert_eq!(helper_exit.signal(), Some(9));
18852 assert!(!cortexkit_test_support::process_alive(
18853 i32::try_from(leader).unwrap()
18854 ));
18855 assert_eq!(
18856 module.status().unwrap().last_exit.unwrap().kind,
18857 ExitKind::Crash
18858 );
18859 assert!(module.terminal_history().entries[0]
18860 .disposition_detail
18861 .as_deref()
18862 .unwrap()
18863 .contains("registration_timeout"));
18864 module.stop().await.unwrap();
18865 }
18866
18867 #[tokio::test(start_paused = true)]
18868 async fn registration_timeout_does_not_count_an_earlier_exit() {
18869 let actor = Actor::start_with_policy(
18870 ModuleProtocol::Subc,
18871 Duration::from_secs(30),
18872 DEFAULT_REGISTER_TIMEOUT,
18873 RestartPolicy::new(0, Duration::ZERO),
18874 )
18875 .await;
18876 let pid = actor.module.status().unwrap().pid.unwrap();
18877 rustix::process::kill_process(
18878 rustix::process::Pid::from_raw(pid as i32).unwrap(),
18879 rustix::process::Signal::TERM,
18880 )
18881 .unwrap();
18882 wait_for("early exit", || {
18883 (actor.module.state().unwrap() == ModuleState::Failed).then_some(())
18884 })
18885 .await;
18886 tokio::time::advance(Duration::from_secs(121)).await;
18887 tokio::task::yield_now().await;
18888 let entries = actor.module.terminal_history().entries;
18889 assert_eq!(entries.len(), 1);
18890 assert!(!entries[0]
18891 .disposition_detail
18892 .as_deref()
18893 .unwrap()
18894 .contains("registration_timeout"));
18895 assert_eq!(actor.module.status().unwrap().lifetime_restarts, 0);
18896 actor.module.stop().await.unwrap();
18897 }
18898
18899 #[tokio::test(start_paused = true)]
18900 async fn registration_timeout_is_cancelled_by_daemon_shutdown() {
18901 let actor = Actor::start(ModuleProtocol::Subc, Duration::from_secs(30)).await;
18902 let pid = actor.module.status().unwrap().pid;
18903 tokio::time::advance(Duration::from_secs(119)).await;
18904 actor.roster.close();
18905 let turns = actor.turns();
18906 tokio::time::advance(Duration::from_secs(300)).await;
18907 let parked = actor.wait_for_select(|at| at.turn > turns).await;
18908 assert_eq!(parked.wake_after, None);
18909 assert_eq!(actor.module.status().unwrap().pid, pid);
18910 assert!(actor.module.terminal_history().entries.is_empty());
18911 actor.module.stop().await.unwrap();
18912 let status = actor.module.status().unwrap();
18913 assert_eq!(status.restart_count, 0);
18914 assert_eq!(
18915 actor.module.terminal_history().entries[0].disposition,
18916 TerminalDisposition::DaemonShutdown
18917 );
18918 }
18919
18920 #[tokio::test(start_paused = true)]
18921 async fn no_health_children_park_for_sixty_seconds() {
18922 let none = Actor::start(ModuleProtocol::None, Duration::from_secs(30)).await;
18923 let unregistered = Actor::start(ModuleProtocol::Subc, Duration::from_secs(30)).await;
18924 let mut unadvertised = Actor::start(ModuleProtocol::Subc, Duration::from_secs(30)).await;
18925 unadvertised.hello(1, false);
18926 for _ in 0..6000 {
18931 tokio::time::advance(Duration::from_millis(10)).await;
18932 tokio::task::yield_now().await;
18933 }
18934 for actor in [&none, &unregistered, &unadvertised] {
18935 assert!(
18936 actor.turns() <= 5,
18937 "no-health actor woke {} times",
18938 actor.turns()
18939 );
18940 assert_eq!(actor.module.state().unwrap(), ModuleState::Running);
18941 }
18942 }
18943
18944 #[tokio::test(start_paused = true)]
18945 async fn late_hello_starts_a_due_probe_in_the_notification_tick() {
18946 let mut actor = Actor::start(ModuleProtocol::Subc, Duration::ZERO).await;
18950 let tick = Instant::now();
18951 actor.hello(1, true);
18952 actor.probe().await;
18953 assert_eq!(Instant::now(), tick, "HELLO must not wait for a poll");
18954 }
18955
18956 #[tokio::test(start_paused = true)]
18957 async fn hello_keeps_the_jittered_cadence_and_catalog_updates_do_not_reset_it() {
18958 let mut actor = Actor::start(ModuleProtocol::Subc, Duration::from_secs(30)).await;
18959 let hello_tick = Instant::now();
18960 actor.hello(1, true);
18961 let armed = actor
18962 .wait_for_select(|checkpoint| {
18963 checkpoint.registered_connection == Some(ConnectionId::new(1))
18964 && checkpoint.next_probe_at.is_some()
18965 })
18966 .await;
18967 let delay = jittered_health_delay(&actor.spec.module_id, 0, actor.health.cadence);
18968 assert!((Duration::from_secs(30)..Duration::from_secs(33)).contains(&delay));
18969 let first_deadline = hello_tick + delay;
18970 assert_eq!(armed.next_probe_at, Some(first_deadline));
18971 tokio::time::advance(Duration::from_secs(29)).await;
18972 let turns = actor.turns();
18973 actor
18974 .registry
18975 .replace_catalog_for_connection(ConnectionId::new(1), vec![], None, Some(true))
18976 .unwrap();
18977 let updated = actor
18978 .wait_for_select(|checkpoint| checkpoint.turn > turns)
18979 .await;
18980 assert_eq!(
18981 updated.next_probe_at,
18982 Some(first_deadline),
18983 "catalog update must not reset cadence"
18984 );
18985 actor.no_probe();
18986 tokio::time::advance(first_deadline - Instant::now()).await;
18987 let frame = actor.probe().await;
18988 actor.answer(1, frame);
18989 let next = jittered_health_delay(&actor.spec.module_id, 1, actor.health.cadence);
18990 let next_deadline = Instant::now() + next;
18991 let rearmed = actor
18992 .wait_for_select(|checkpoint| {
18993 checkpoint
18994 .next_probe_at
18995 .is_some_and(|deadline| deadline > first_deadline)
18996 })
18997 .await;
18998 assert_eq!(rearmed.next_probe_at, Some(next_deadline));
18999 tokio::time::advance(next - Duration::from_nanos(1)).await;
19000 actor.no_probe();
19001 tokio::time::advance(Duration::from_nanos(1)).await;
19002 actor.probe().await;
19003 }
19004
19005 #[tokio::test(start_paused = true)]
19006 async fn disconnect_disarms_and_reconnect_rearms_health() {
19007 let mut actor = Actor::start(ModuleProtocol::Subc, Duration::from_secs(30)).await;
19008 actor.hello(1, true);
19009 actor
19010 .wait_for_select(|checkpoint| {
19011 checkpoint.registered_connection == Some(ConnectionId::new(1))
19012 && checkpoint.next_probe_at.is_some()
19013 })
19014 .await;
19015 let turns = actor.turns();
19016 let tick = Instant::now();
19017 actor
19018 .registry
19019 .deregister_connection(ConnectionId::new(1))
19020 .unwrap();
19021 let disarmed = actor
19022 .wait_for_select(|checkpoint| {
19023 checkpoint.turn > turns && checkpoint.registered_connection.is_none()
19024 })
19025 .await;
19026 assert_eq!(disarmed.next_probe_at, None);
19027 assert_eq!(disarmed.wake_after, None);
19028 assert_eq!(
19029 Instant::now(),
19030 tick,
19031 "disconnect must disarm in the notification tick"
19032 );
19033 tokio::time::advance(Duration::from_secs(60)).await;
19034 actor.no_probe();
19035 let reconnect_tick = Instant::now();
19036 actor.hello(2, true);
19037 let rearmed = actor
19038 .wait_for_select(|checkpoint| {
19039 checkpoint.registered_connection == Some(ConnectionId::new(2))
19040 && checkpoint.next_probe_at.is_some()
19041 })
19042 .await;
19043 let delay = jittered_health_delay(&actor.spec.module_id, 0, actor.health.cadence);
19044 assert_eq!(rearmed.next_probe_at, Some(reconnect_tick + delay));
19045 tokio::time::advance(delay).await;
19046 actor.probe().await;
19047 }
19048
19049 #[tokio::test(start_paused = true)]
19050 async fn reregistration_without_health_disarms_the_old_schedule() {
19051 let mut actor = Actor::start(ModuleProtocol::Subc, Duration::from_secs(30)).await;
19052 actor.hello(1, true);
19053 actor
19054 .wait_for_select(|checkpoint| {
19055 checkpoint.registered_connection == Some(ConnectionId::new(1))
19056 && checkpoint.next_probe_at.is_some()
19057 })
19058 .await;
19059 actor
19060 .registry
19061 .deregister_connection(ConnectionId::new(1))
19062 .unwrap();
19063 actor.hello(2, false);
19064 let disarmed = actor
19065 .wait_for_select(|checkpoint| {
19066 checkpoint.registered_connection == Some(ConnectionId::new(2))
19067 })
19068 .await;
19069 assert_eq!(disarmed.next_probe_at, None);
19070 assert_eq!(disarmed.wake_after, None);
19071 tokio::time::advance(Duration::from_secs(60)).await;
19072 actor.no_probe();
19073 }
19074
19075 #[tokio::test(start_paused = true)]
19076 async fn restart_waits_for_its_own_hello_before_rearming_health() {
19077 let mut actor = Actor::start(ModuleProtocol::Subc, Duration::from_secs(30)).await;
19078 actor.hello(1, true);
19079 let armed = actor
19080 .wait_for_select(|checkpoint| {
19081 checkpoint.registered_connection == Some(ConnectionId::new(1))
19082 && checkpoint.next_probe_at.is_some()
19083 })
19084 .await;
19085 actor
19089 .registry
19090 .deregister_connection(ConnectionId::new(1))
19091 .unwrap();
19092 actor.module.restart(Some(0)).await.unwrap();
19093 let parked = actor
19094 .wait_for_select(|checkpoint| {
19095 checkpoint.generation > armed.generation
19096 && checkpoint.registered_connection.is_none()
19097 && checkpoint.next_probe_at.is_none()
19098 })
19099 .await;
19100 assert_eq!(parked.wake_after, Some(DEFAULT_REGISTER_TIMEOUT));
19101 actor.rx.close();
19102 while actor.rx.try_recv().is_ok() {}
19105 let turns = actor.turns();
19106 tokio::time::advance(Duration::from_secs(60)).await;
19107 let after = actor.turns();
19108 eprintln!(
19109 "restart park turns: before={turns} after={after} delta={}",
19110 after - turns
19111 );
19112 assert_eq!(after, turns, "replacement must park before HELLO");
19113 let hello_tick = Instant::now();
19114 actor.hello(2, true);
19115 let rearmed = actor
19116 .wait_for_select(|checkpoint| {
19117 checkpoint.registered_connection == Some(ConnectionId::new(2))
19118 && checkpoint.next_probe_at.is_some()
19119 })
19120 .await;
19121 let delay = jittered_health_delay(&actor.spec.module_id, 0, actor.health.cadence);
19122 assert_eq!(rearmed.next_probe_at, Some(hello_tick + delay));
19123 tokio::time::advance(delay).await;
19124 actor.probe().await;
19125 }
19126
19127 #[tokio::test(start_paused = true)]
19128 async fn http_probe_waits_a_full_cadence_for_each_spawn() {
19129 use tokio::{
19130 io::{AsyncReadExt, AsyncWriteExt},
19131 net::TcpListener,
19132 };
19133 let mut actor = Actor::start(ModuleProtocol::None, Duration::from_secs(30)).await;
19134 let listener = TcpListener::bind("127.0.0.1:0").await.unwrap();
19135 actor.health.http = Some(format!("http://{}/healthz", listener.local_addr().unwrap()));
19136 let (started_tx, mut started_rx) = mpsc::channel(8);
19137 let peer = tokio::spawn(async move {
19138 loop {
19139 let (mut socket, _) = listener.accept().await.unwrap();
19140 let mut request = [0; 1024];
19141 assert!(socket.read(&mut request).await.unwrap() > 0);
19142 started_tx.send(Instant::now()).await.unwrap();
19143 socket
19144 .write_all(b"HTTP/1.1 200 OK\r\nContent-Length: 0\r\n\r\n")
19145 .await
19146 .unwrap();
19147 }
19148 });
19149 actor
19150 .module
19151 .update_configuration(actor.spec.clone(), actor.health.clone(), None)
19152 .await
19153 .unwrap();
19154 let first_tick = Instant::now();
19155 let first = actor
19156 .wait_for_select(|checkpoint| checkpoint.next_probe_at.is_some())
19157 .await;
19158 assert!(first.next_probe_at.unwrap() >= first_tick + actor.health.cadence);
19159 assert!(started_rx.try_recv().is_err());
19160
19161 tokio::time::advance(
19164 first.next_probe_at.unwrap() - Instant::now() - Duration::from_secs(1),
19165 )
19166 .await;
19167 actor.module.restart(Some(0)).await.unwrap();
19168 let spawn_tick = Instant::now();
19169 let replacement = actor
19170 .wait_for_select(|checkpoint| checkpoint.generation > first.generation)
19171 .await;
19172 let deadline = replacement.next_probe_at.unwrap();
19173 assert!(
19174 deadline >= spawn_tick + actor.health.cadence,
19175 "a replacement must not inherit its predecessor's probe deadline"
19176 );
19177 tokio::time::advance(actor.health.cadence - Duration::from_nanos(1)).await;
19178 assert!(
19179 started_rx.try_recv().is_err(),
19180 "no HTTP probe during startup grace"
19181 );
19182 tokio::time::advance(deadline - Instant::now()).await;
19183 let landed = wait_for("first replacement HTTP probe", || {
19184 started_rx.try_recv().ok()
19185 })
19186 .await;
19187 assert!(landed >= spawn_tick + actor.health.cadence);
19188 wait_for("healthy replacement", || {
19189 (actor.module.status().unwrap().health.status == SupervisorHealthStatus::Ok)
19190 .then_some(())
19191 })
19192 .await;
19193 peer.abort();
19194 }
19195
19196 #[tokio::test(start_paused = true)]
19197 async fn rescan_adds_and_removes_http_health_without_polling() {
19198 use tokio::{
19199 io::{AsyncReadExt, AsyncWriteExt},
19200 net::TcpListener,
19201 };
19202 let mut actor = Actor::start(ModuleProtocol::None, Duration::from_secs(30)).await;
19203 let listener = TcpListener::bind("127.0.0.1:0").await.unwrap();
19204 actor.health.http = Some(format!("http://{}/health", listener.local_addr().unwrap()));
19205 actor.health.cadence = Duration::from_millis(1);
19207 let (started_tx, mut started_rx) = mpsc::channel(8);
19208 let peer = tokio::spawn(async move {
19209 loop {
19210 let (mut socket, _) = listener.accept().await.unwrap();
19211 let mut request = [0; 1024];
19212 assert!(socket.read(&mut request).await.unwrap() > 0);
19213 started_tx.send(()).await.unwrap();
19214 socket
19215 .write_all(b"HTTP/1.1 200 OK\r\nContent-Length: 0\r\n\r\n")
19216 .await
19217 .unwrap();
19218 }
19219 });
19220 let tick = Instant::now();
19221 let turns = actor.turns();
19222 actor
19223 .module
19224 .update_configuration(actor.spec.clone(), actor.health.clone(), None)
19225 .await
19226 .unwrap();
19227 let armed = actor
19228 .wait_for_select(|checkpoint| {
19229 checkpoint.turn > turns && checkpoint.next_probe_at.is_some()
19230 })
19231 .await;
19232 assert_eq!(armed.next_probe_at, Some(tick + Duration::from_millis(1)));
19233 assert_eq!(Instant::now(), tick);
19234 tokio::time::advance(Duration::from_millis(1)).await;
19235 let mut started = false;
19236 wait_for("successful HTTP probe", || {
19237 started |= started_rx.try_recv().is_ok();
19238 (started && actor.module.status().unwrap().health.status == SupervisorHealthStatus::Ok)
19239 .then_some(())
19240 })
19241 .await;
19242 assert!(started, "added HTTP check must start at its first deadline");
19243 assert_eq!(
19244 actor.module.status().unwrap().health.status,
19245 SupervisorHealthStatus::Ok
19246 );
19247 assert_eq!(Instant::now(), tick + Duration::from_millis(1));
19248 actor.health.http = None;
19249 let turns = actor.turns();
19250 actor
19251 .module
19252 .update_configuration(actor.spec.clone(), actor.health.clone(), None)
19253 .await
19254 .unwrap();
19255 let parked = actor
19256 .wait_for_select(|checkpoint| {
19257 checkpoint.turn > turns && checkpoint.next_probe_at.is_none()
19258 })
19259 .await;
19260 assert_eq!(parked.wake_after, None);
19261 let turns = actor.turns();
19262 tokio::time::advance(Duration::from_secs(60)).await;
19263 assert_eq!(
19264 actor.turns(),
19265 turns,
19266 "removed HTTP check must park the actor"
19267 );
19268 assert!(
19269 started_rx.try_recv().is_err(),
19270 "removed HTTP check must stay disarmed"
19271 );
19272 assert_eq!(
19273 actor.module.status().unwrap().health,
19274 ModuleHealthStatus::default()
19275 );
19276 peer.abort();
19277 }
19278}
19279
19280#[cfg(test)]
19281mod drain_busy_reprobe_tests {
19282 use super::*;
19286 use std::collections::BTreeMap;
19287 use std::sync::Mutex as StdMutex;
19288 use subc_protocol::{manifest::Concurrency, session::ModuleControlResponse};
19289
19290 const MODULE_ID: &str = "drain-reprobe-module";
19291 const CONNECTION: u64 = 7;
19292 const GAUGE: &str = "runs_in_flight";
19293 const CADENCE: Duration = Duration::from_secs(30);
19296 const CEILING: Duration = Duration::from_secs(30);
19297
19298 #[derive(Clone, Copy)]
19300 enum Answer {
19301 BusyUntil { until: Instant, latency: Duration },
19304 OmitsGauge,
19306 Silent,
19308 }
19309
19310 struct Drain {
19311 forwarding: Arc<ForwardingTable>,
19312 runtime: SupervisorRuntimeConfig,
19313 endpoint: crate::ModuleEndpointId,
19314 probes: Arc<StdMutex<Vec<Instant>>>,
19316 responder: tokio::task::JoinHandle<()>,
19317 }
19318
19319 impl Drain {
19320 fn start(answer: Answer) -> Self {
19321 let registry = Arc::new(Registry::default());
19322 let forwarding = Arc::new(ForwardingTable::default());
19323 let health = HealthConfig {
19324 cadence: CADENCE,
19325 ..HealthConfig::default()
19326 };
19327 let runtime = Supervisor::new_for_test(registry, RestartPolicy::new(3, Duration::ZERO))
19328 .with_forwarding(forwarding.clone())
19329 .with_health_config(health)
19330 .runtime_config();
19331 let (tx, mut rx) = mpsc::channel::<crate::router::OutboundFrame>(8);
19332 let endpoint = forwarding
19333 .register_module_connection(
19334 ConnectionId::new(CONNECTION),
19335 MODULE_ID.to_string(),
19336 subc_protocol::PROTOCOL_VERSION,
19337 Concurrency::ModuleManaged,
19338 FrameSink::new(tx),
19339 )
19340 .unwrap();
19341 let probes = Arc::new(StdMutex::new(Vec::new()));
19342 let responder = tokio::spawn({
19343 let forwarding = forwarding.clone();
19344 let probes = probes.clone();
19345 async move {
19346 while let Some(frame) = rx.recv().await {
19347 let body: Value = serde_json::from_slice(&frame.body).unwrap();
19348 assert_eq!(body["op"], MODULE_CONTROL_OP_HEALTH_CHECK);
19349 probes.lock().unwrap().push(Instant::now());
19350 let metrics = match answer {
19351 Answer::BusyUntil { until, latency } => {
19352 sleep(latency).await;
19353 let busy = u64::from(Instant::now() < until);
19354 serde_json::json!({ GAUGE: busy })
19355 }
19356 Answer::OmitsGauge => serde_json::json!({ "queue_depth": 0 }),
19357 Answer::Silent => continue,
19358 };
19359 let _ = forwarding.complete_module_control_rpc(
19362 ConnectionId::new(CONNECTION),
19363 frame.header.corr,
19364 Some(MODULE_CONTROL_OP_HEALTH_CHECK),
19365 ModuleControlRpcOutcome::Response(ModuleControlResponse::HealthCheck {
19366 status: HealthStatus::Ok,
19367 detail: None,
19368 metrics: Some(metrics),
19369 }),
19370 );
19371 }
19372 }
19373 });
19374 Self {
19375 forwarding,
19376 runtime,
19377 endpoint,
19378 probes,
19379 responder,
19380 }
19381 }
19382
19383 async fn run(&self, ceiling: Duration) -> (bool, Duration) {
19386 let (drained, elapsed, _) = self.run_traced(ceiling).await;
19387 (drained, elapsed)
19388 }
19389
19390 async fn run_traced(&self, ceiling: Duration) -> (bool, Duration, DrainGaugeTrace) {
19392 let started = Instant::now();
19393 let mut trace = DrainGaugeTrace::new(&[GAUGE.to_string()]);
19394 let drained = wait_for_forwarding_quiescence(
19395 &self.forwarding,
19396 MODULE_ID,
19397 &self.runtime,
19398 self.endpoint,
19399 started + ceiling,
19400 &[GAUGE.to_string()],
19401 DrainScope::Active,
19402 &mut trace,
19403 )
19404 .await
19405 .unwrap();
19406 (drained, started.elapsed(), trace)
19407 }
19408
19409 fn probes(&self) -> Vec<Instant> {
19410 self.probes.lock().unwrap().clone()
19411 }
19412 }
19413
19414 impl Drop for Drain {
19415 fn drop(&mut self) {
19416 self.responder.abort();
19417 }
19418 }
19419
19420 #[tokio::test(start_paused = true)]
19421 async fn busy_at_first_probe_then_quiet_drains_long_before_the_ceiling() {
19422 let drain = Drain::start(Answer::BusyUntil {
19423 until: Instant::now() + Duration::from_secs(1),
19424 latency: Duration::ZERO,
19425 });
19426 let (drained, elapsed) = drain.run(CEILING).await;
19427 assert!(
19428 drained,
19429 "gauges that read zero during the drain must let it finish drained, \
19430 not wait out the {CADENCE:?} health cadence (elapsed {elapsed:?})"
19431 );
19432 assert!(
19433 elapsed < Duration::from_secs(3),
19434 "drain took {elapsed:?} after the gauges went quiet at 1s"
19435 );
19436 let probes = drain.probes();
19437 assert!(
19438 probes.len() >= 2,
19439 "the module was busy at the first probe, so a quiet verdict needs a second: {probes:?}"
19440 );
19441 }
19442
19443 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
19448 async fn an_instant_busy_first_answer_does_not_stop_later_probes_in_real_time() {
19449 let drain = Drain::start(Answer::BusyUntil {
19450 until: Instant::now() + Duration::from_millis(600),
19451 latency: Duration::ZERO,
19452 });
19453 let (drained, elapsed, trace) = drain.run_traced(CEILING).await;
19454 assert!(drained, "the gauge fell at 600ms: {trace}");
19455 assert!(
19456 elapsed < Duration::from_secs(3),
19457 "drain took {elapsed:?} after the gauge fell at 600ms: {trace}"
19458 );
19459 let probes = drain.probes();
19460 assert!(
19461 probes.len() >= 2 && trace.probes as usize == probes.len(),
19462 "the drain must re-probe after the first busy answer: {probes:?}, {trace}"
19463 );
19464 }
19465
19466 #[tokio::test(start_paused = true)]
19467 async fn busy_throughout_ends_undrained_at_the_ceiling_with_bounded_probes() {
19468 let started = Instant::now();
19469 let drain = Drain::start(Answer::BusyUntil {
19470 until: started + Duration::from_secs(3600),
19471 latency: Duration::ZERO,
19472 });
19473 let (drained, elapsed) = drain.run(CEILING).await;
19474 assert!(!drained, "a module busy throughout must not drain");
19475 assert!(
19476 elapsed >= CEILING && elapsed < CEILING + drain_busy_reprobe_interval(),
19477 "a busy module is held until the ceiling and no longer: {elapsed:?}"
19478 );
19479 let probes = drain.probes();
19482 let max_probes = (CEILING.as_millis() / drain_busy_reprobe_interval().as_millis()) + 1;
19483 assert!(
19484 probes.len() as u128 <= max_probes,
19485 "{} probes in a {CEILING:?} drain exceeds the {max_probes} the interval allows",
19486 probes.len()
19487 );
19488 for pair in probes.windows(2) {
19489 assert!(
19490 pair[1] - pair[0] >= drain_busy_reprobe_interval(),
19491 "consecutive probes {:?} apart",
19492 pair[1] - pair[0]
19493 );
19494 }
19495 }
19496
19497 #[tokio::test(start_paused = true)]
19498 async fn a_slow_probe_is_never_overlapped_by_the_next() {
19499 let latency = Duration::from_secs(1);
19502 let drain = Drain::start(Answer::BusyUntil {
19503 until: Instant::now() + Duration::from_secs(3600),
19504 latency,
19505 });
19506 let (drained, _) = drain.run(Duration::from_secs(5)).await;
19507 assert!(!drained);
19508 let probes = drain.probes();
19509 assert!(probes.len() >= 2, "the drain must re-probe: {probes:?}");
19510 for pair in probes.windows(2) {
19511 assert!(
19512 pair[1] - pair[0] >= latency + drain_busy_reprobe_interval(),
19513 "a probe started {:?} after the previous one, before it could finish",
19514 pair[1] - pair[0]
19515 );
19516 }
19517 }
19518
19519 #[tokio::test(start_paused = true)]
19520 async fn an_unanswered_probe_cannot_hold_the_drain_past_its_ceiling() {
19521 let drain = Drain::start(Answer::Silent);
19522 let ceiling = Duration::from_secs(2);
19523 let (drained, elapsed) = drain.run(ceiling).await;
19524 assert!(!drained, "an unanswered probe counts as busy");
19525 assert!(
19526 elapsed >= ceiling && elapsed < ceiling + drain_busy_reprobe_interval(),
19527 "the probe deadline is capped at the drain ceiling: {elapsed:?}"
19528 );
19529 assert_eq!(
19530 drain.probes().len(),
19531 1,
19532 "the unanswered probe holds the only slot until the ceiling"
19533 );
19534 }
19535
19536 #[derive(Clone, Default)]
19538 struct LogCapture {
19539 events: Arc<StdMutex<Vec<BTreeMap<String, String>>>>,
19540 }
19541
19542 impl<S: tracing::Subscriber> tracing_subscriber::Layer<S> for LogCapture {
19543 fn on_event(
19544 &self,
19545 event: &tracing::Event<'_>,
19546 _context: tracing_subscriber::layer::Context<'_, S>,
19547 ) {
19548 struct Fields(BTreeMap<String, String>);
19549 impl tracing::field::Visit for Fields {
19550 fn record_debug(&mut self, field: &tracing::field::Field, value: &dyn fmt::Debug) {
19551 self.0
19552 .insert(field.name().to_string(), format!("{value:?}"));
19553 }
19554 }
19555 let mut fields = Fields(BTreeMap::new());
19556 event.record(&mut fields);
19557 self.events.lock().unwrap().push(fields.0);
19558 }
19559 }
19560
19561 impl LogCapture {
19562 fn install(&self) -> tracing::subscriber::DefaultGuard {
19563 use tracing_subscriber::layer::SubscriberExt;
19564 tracing::subscriber::set_default(tracing_subscriber::registry().with(self.clone()))
19565 }
19566
19567 fn with_message(&self, needle: &str) -> Vec<BTreeMap<String, String>> {
19568 self.events
19569 .lock()
19570 .unwrap()
19571 .iter()
19572 .filter(|fields| fields.get("message").is_some_and(|m| m.contains(needle)))
19573 .cloned()
19574 .collect()
19575 }
19576 }
19577
19578 #[tokio::test(start_paused = true)]
19583 async fn a_missing_declared_gauge_is_named_on_the_drain_timeout_line() {
19584 let capture = LogCapture::default();
19585 let _guard = capture.install();
19586 let drain = Drain::start(Answer::OmitsGauge);
19587 let (drained, _, trace) = drain.run_traced(Duration::from_secs(2)).await;
19588 assert!(!drained, "a gauge never reported cannot read quiet");
19589 assert!(trace.probes >= 2, "the drain must re-probe: {trace:?}");
19590
19591 let missing = capture.with_message("lacks a declared busy gauge");
19592 assert_eq!(missing.len(), 1, "one warning per drain: {missing:?}");
19593 assert_eq!(
19594 missing[0].get("missing_gauges").map(String::as_str),
19595 Some(GAUGE)
19596 );
19597
19598 warn_drain_timed_out(
19599 MODULE_ID,
19600 Duration::from_secs(2),
19601 RouteCloseReason::Restart,
19602 &Default::default(),
19603 &trace,
19604 );
19605 let timeouts = capture.with_message("route drain timed out");
19606 assert_eq!(timeouts.len(), 1, "{timeouts:?}");
19607 let gauges = &timeouts[0]["busy_gauges"];
19608 assert!(
19609 gauges.starts_with(&format!("{GAUGE}: missing_from_health_body @")),
19610 "the timeout line must name the gauge and why it read busy: {gauges}"
19611 );
19612 assert!(
19613 gauges.ends_with("; no value ever read"),
19614 "no value was ever read for the gauge: {gauges}"
19615 );
19616 assert_eq!(timeouts[0]["gauge_probes"], trace.probes.to_string());
19617 }
19618
19619 #[tokio::test(start_paused = true)]
19623 async fn the_drain_trace_names_a_busy_value_and_an_unanswered_probe() {
19624 let busy = Drain::start(Answer::BusyUntil {
19625 until: Instant::now() + Duration::from_secs(3600),
19626 latency: Duration::ZERO,
19627 });
19628 let (_, _, trace) = busy.run_traced(Duration::from_secs(1)).await;
19629 let rendered = trace.to_string();
19630 assert!(
19631 rendered.starts_with(&format!("{GAUGE}: non_zero(1) @"))
19632 && rendered.contains("; last value non_zero(1) @"),
19633 "{rendered}"
19634 );
19635
19636 let silent = Drain::start(Answer::Silent);
19637 let (_, _, trace) = silent.run_traced(Duration::from_secs(1)).await;
19638 assert_eq!(
19641 trace.to_string(),
19642 format!("{GAUGE}: probe_failed(no-answer) @1000ms; no value ever read")
19643 );
19644 assert_eq!(trace.probes, 1);
19645 }
19646
19647 #[test]
19649 fn a_gauge_that_is_not_a_count_is_busy_and_shown_as_sent() {
19650 let gauges = [GAUGE.to_string()];
19651 let verdicts = gauge_verdicts(Some(&serde_json::json!({ GAUGE: 0.0 })), &gauges);
19652 assert_eq!(verdicts, vec![GaugeVerdict::NotACount("0.0".to_string())]);
19653 assert_eq!(
19654 busy_gauge_observation(&verdicts),
19655 BusyGaugeObservation::Busy
19656 );
19657 let zero = gauge_verdicts(Some(&serde_json::json!({ GAUGE: 0 })), &gauges);
19658 assert_eq!(
19659 busy_gauge_observation(&zero),
19660 BusyGaugeObservation::Quiescent
19661 );
19662 let none = gauge_verdicts(None, &gauges);
19663 assert_eq!(busy_gauge_observation(&none), BusyGaugeObservation::Omitted);
19664 }
19665}
19666
19667#[cfg(test)]
19668mod health_evidence_tests {
19669 use super::{HealthProbeError, HealthProbeEvidence};
19670 use std::collections::HashSet;
19671
19672 #[test]
19680 fn only_a_dead_lane_is_proof_of_death() {
19681 assert!(HealthProbeError::lane_dead("gone").is_proof_of_death());
19682 assert!(!HealthProbeError::no_answer("timed out").is_proof_of_death());
19686 assert!(!HealthProbeError::bad_answer("garbage").is_proof_of_death());
19687 assert!(!HealthProbeError::misconfigured("no table").is_proof_of_death());
19688 }
19689
19690 #[test]
19696 fn every_evidence_class_has_a_distinct_label() {
19697 let labels = [
19698 HealthProbeError::lane_dead("").label(),
19699 HealthProbeError::no_answer("").label(),
19700 HealthProbeError::bad_answer("").label(),
19701 HealthProbeError::misconfigured("").label(),
19702 ];
19703 let unique: HashSet<_> = labels.iter().collect();
19704 assert_eq!(unique.len(), labels.len(), "labels collided: {labels:?}");
19705 }
19706
19707 #[test]
19713 fn classification_preserves_the_original_message() {
19714 let err = HealthProbeError::no_answer("module did not answer within 5s");
19715 assert_eq!(err.to_string(), "module did not answer within 5s");
19716 assert!(matches!(err.evidence, HealthProbeEvidence::NoAnswer));
19717 }
19718}
19719
19720#[cfg(test)]
19721mod health_tombstone_tests {
19722 use std::{path::PathBuf, sync::Arc, time::Duration};
19723
19724 use subc_protocol::{
19725 manifest::Concurrency,
19726 session::{HealthStatus, ModuleControlResponse},
19727 };
19728 use tokio::sync::mpsc;
19729
19730 use super::{
19731 probe_module_health, HealthAction, HealthConfig, HealthProbeEvidence, ModuleProtocol,
19732 ModuleSpec, RestartPolicy, Supervisor, SupervisorRuntimeConfig,
19733 };
19734 use crate::{
19735 control::ControlHandler,
19736 forwarding::{ForwardingTable, ModuleControlRpcCompletion, ModuleControlRpcOutcome},
19737 registry::{ConnectionId, Registry},
19738 router::FrameSink,
19739 };
19740
19741 struct ProbeHarness {
19742 spec: ModuleSpec,
19743 runtime: SupervisorRuntimeConfig,
19744 forwarding: Arc<ForwardingTable>,
19745 module_connection: ConnectionId,
19746 module_rx: mpsc::Receiver<crate::router::OutboundFrame>,
19747 handler: ControlHandler,
19748 module: super::SupervisedModule,
19749 }
19750
19751 fn probe_harness() -> ProbeHarness {
19752 let registry = Arc::new(Registry::default());
19753 let forwarding = Arc::new(ForwardingTable::default());
19754 let supervisor_handle = super::SupervisorHandle::new();
19755 let health = HealthConfig {
19756 http: None,
19757 cadence: Duration::from_secs(30),
19758 deadline: Duration::from_secs(5),
19759 failure_threshold: 3,
19760 on_degraded: HealthAction::Report,
19761 on_failing: HealthAction::Report,
19762 critical: false,
19763 };
19764 let supervisor = Supervisor::new_for_test(Arc::clone(®istry), RestartPolicy::default())
19765 .with_forwarding(Arc::clone(&forwarding))
19766 .with_handle(supervisor_handle.clone())
19767 .with_health_config(health);
19768 let spec = ModuleSpec {
19769 script: None,
19770 module_id: "late-health-module".to_string(),
19771 program: PathBuf::from("disabled-module"),
19772 args: Vec::new(),
19773 env: Vec::new(),
19774 reserved: false,
19775 reserved_prefixes: Vec::new(),
19776 protocol: ModuleProtocol::Subc,
19777 overlap: Default::default(),
19778 carry_containment: false,
19779 };
19780 let module = supervisor
19781 .supervise_configured(spec.clone(), false)
19782 .unwrap();
19783 let runtime = supervisor.runtime_config();
19784 let handler = ControlHandler::with_forwarding(registry, Arc::clone(&forwarding))
19785 .with_supervisor(supervisor_handle);
19786 let module_connection = ConnectionId::new(700);
19787 let (module_tx, module_rx) = mpsc::channel(8);
19788 forwarding
19789 .register_module_connection(
19790 module_connection,
19791 spec.module_id.clone(),
19792 subc_protocol::PROTOCOL_VERSION,
19793 Concurrency::ModuleManaged,
19794 FrameSink::new(module_tx),
19795 )
19796 .unwrap();
19797
19798 ProbeHarness {
19799 spec,
19800 runtime,
19801 forwarding,
19802 module_connection,
19803 module_rx,
19804 handler,
19805 module,
19806 }
19807 }
19808
19809 async fn finish_after(
19810 harness: &mut ProbeHarness,
19811 stall: Duration,
19812 ) -> ModuleControlRpcCompletion {
19813 assert!(stall > harness.runtime.health.deadline);
19814 let deadline = harness.runtime.health.deadline;
19815 let probe = probe_module_health(&harness.spec.module_id, &harness.runtime, None);
19816 let answer = async {
19817 let frame = harness.module_rx.recv().await.expect("health.check frame");
19818 tokio::time::advance(deadline).await;
19819 tokio::task::yield_now().await;
19820 tokio::time::advance(stall - deadline).await;
19821 harness
19822 .forwarding
19823 .complete_module_control_rpc(
19824 harness.module_connection,
19825 frame.header.corr,
19826 Some("health.check"),
19827 ModuleControlRpcOutcome::Response(ModuleControlResponse::HealthCheck {
19828 status: HealthStatus::Ok,
19829 detail: None,
19830 metrics: None,
19831 }),
19832 )
19833 .unwrap()
19834 };
19835 let (probe_result, completion) = tokio::join!(probe, answer);
19836 let err = probe_result.expect_err("probe must miss its deadline");
19837 assert!(matches!(err.evidence, HealthProbeEvidence::NoAnswer));
19838 completion
19839 }
19840
19841 async fn time_out_without_answer(harness: &mut ProbeHarness) {
19842 let deadline = harness.runtime.health.deadline;
19843 let probe = probe_module_health(&harness.spec.module_id, &harness.runtime, None);
19844 let exhaust_deadline = async {
19845 let _frame = harness.module_rx.recv().await.expect("health.check frame");
19846 tokio::time::advance(deadline).await;
19847 tokio::task::yield_now().await;
19848 };
19849 let (probe_result, ()) = tokio::join!(probe, exhaust_deadline);
19850 let err = probe_result.expect_err("probe must miss its deadline");
19851 assert!(matches!(err.evidence, HealthProbeEvidence::NoAnswer));
19852 }
19853
19854 async fn run_probe_cycle(harness: &mut ProbeHarness, answer: bool) {
19855 let registry = Arc::clone(&harness.module.inner.registry);
19856 let snapshot = Arc::clone(&harness.module.inner.snapshot);
19857 let process_liveness = super::SupervisorProcessLiveness::default();
19858 let mut child = None;
19859 let cycle = super::run_health_probe_cycle(
19860 &harness.spec,
19861 &harness.runtime,
19862 ®istry,
19863 &process_liveness,
19864 &snapshot,
19865 &mut child,
19866 );
19867 let peer = async {
19868 let frame = harness.module_rx.recv().await.expect("health.check frame");
19869 if answer {
19870 harness
19871 .forwarding
19872 .complete_module_control_rpc(
19873 harness.module_connection,
19874 frame.header.corr,
19875 Some("health.check"),
19876 ModuleControlRpcOutcome::Response(ModuleControlResponse::HealthCheck {
19877 status: HealthStatus::Ok,
19878 detail: None,
19879 metrics: Some(serde_json::json!({"ready": true})),
19880 }),
19881 )
19882 .unwrap();
19883 } else {
19884 tokio::time::advance(harness.runtime.health.deadline).await;
19885 tokio::task::yield_now().await;
19886 }
19887 };
19888 tokio::join!(cycle, peer);
19889 }
19890
19891 #[tokio::test(start_paused = true)]
19892 async fn unanswered_probe_is_unknown_until_threshold_and_ok_report_recovers() {
19893 let mut harness = probe_harness();
19894 let monitor = harness.module.inner.monitor.lock().unwrap().take().unwrap();
19898 monitor.abort();
19899 let _ = monitor.await;
19900 super::update_snapshot(&harness.module.inner.snapshot, None, |state| {
19901 state.enabled = true;
19902 state.state = super::ModuleState::Running;
19903 state.process_alive = true;
19904 })
19905 .unwrap();
19906
19907 run_probe_cycle(&mut harness, true).await;
19908 assert_eq!(
19909 harness.module.status().unwrap().health.status,
19910 super::SupervisorHealthStatus::Ok
19911 );
19912
19913 for failures in 1..harness.runtime.health.failure_threshold {
19914 run_probe_cycle(&mut harness, false).await;
19915 let status = harness.module.status().unwrap();
19916 assert_eq!(status.health.status, super::SupervisorHealthStatus::Unknown);
19917 assert_eq!(status.health.consecutive_failures, failures);
19918 assert!(status.health.last_probe_ms.is_some());
19919 assert!(status.health.detail.unwrap().starts_with("[no-answer]"));
19920 assert!(status.health.metrics.is_none());
19921 assert_eq!(status.state, super::ModuleState::Running);
19922 assert!(status.process_alive);
19923 assert_eq!(status.restart_count, 0);
19924 assert_eq!(status.lifetime_restarts, 0);
19925 assert!(status.health.last_action.is_none());
19926 assert!(harness.runtime.scheduled_respawn.lock().unwrap().is_none());
19927 }
19928
19929 run_probe_cycle(&mut harness, true).await;
19930 let recovered = harness.module.status().unwrap();
19931 assert_eq!(recovered.health.status, super::SupervisorHealthStatus::Ok);
19932 assert_eq!(recovered.health.consecutive_failures, 0);
19933 assert!(recovered.health.detail.is_none());
19934 assert_eq!(
19935 recovered.health.metrics,
19936 Some(serde_json::json!({"ready": true}))
19937 );
19938 assert_eq!(recovered.lifetime_restarts, 0);
19939
19940 for failures in 1..=harness.runtime.health.failure_threshold {
19941 run_probe_cycle(&mut harness, false).await;
19942 let status = harness.module.status().unwrap();
19943 assert_eq!(status.health.consecutive_failures, failures);
19944 if failures < harness.runtime.health.failure_threshold {
19945 assert_eq!(status.health.status, super::SupervisorHealthStatus::Unknown);
19946 assert_eq!(status.state, super::ModuleState::Running);
19947 assert_eq!(status.lifetime_restarts, 0);
19948 assert!(harness.runtime.scheduled_respawn.lock().unwrap().is_none());
19949 } else {
19950 assert_eq!(
19951 status.health.status,
19952 super::SupervisorHealthStatus::Unresponsive
19953 );
19954 assert_eq!(status.state, super::ModuleState::Restarting);
19955 assert_eq!(status.restart_count, 1);
19956 assert_eq!(status.lifetime_restarts, 1);
19957 assert_eq!(status.health.last_action.as_deref(), Some("restart"));
19958 assert!(harness.runtime.scheduled_respawn.lock().unwrap().is_some());
19959 }
19960 }
19961 }
19962
19963 #[tokio::test(start_paused = true)]
19964 async fn late_health_answers_record_start_anchored_latency_for_two_stalls() {
19965 let mut harness = probe_harness();
19966
19967 let first = finish_after(&mut harness, Duration::from_secs(8)).await;
19968 let first_latency = match &first {
19969 ModuleControlRpcCompletion::LateHealthAnswer { latency, .. } => *latency,
19970 other => panic!("late answer was not retained: {other:?}"),
19971 };
19972 assert!(harness.handler.observe_module_control_completion(first));
19973
19974 let second = finish_after(&mut harness, Duration::from_secs(11)).await;
19975 let second_latency = match &second {
19976 ModuleControlRpcCompletion::LateHealthAnswer { latency, .. } => *latency,
19977 other => panic!("late answer was not retained: {other:?}"),
19978 };
19979 assert!(harness.handler.observe_module_control_completion(second));
19980
19981 assert_eq!(first_latency, Duration::from_secs(8));
19982 assert_eq!(
19983 second_latency - first_latency,
19984 Duration::from_secs(3),
19985 "latency must grow linearly with the additional stall"
19986 );
19987 let health = harness.module.status().unwrap().health;
19988 assert_eq!(health.late_answer_count, 2);
19989 assert_eq!(health.last_late_answer_latency_ms, Some(11_000));
19990 }
19991
19992 #[tokio::test(start_paused = true)]
20000 async fn late_answer_clears_the_consecutive_failure_streak() {
20001 let mut harness = probe_harness();
20002
20003 time_out_without_answer(&mut harness).await;
20005 harness
20006 .module
20007 .record_health_probe_failure_for_test("[no-answer] test miss")
20008 .unwrap();
20009 assert_eq!(
20010 harness.module.status().unwrap().health.consecutive_failures,
20011 1,
20012 "precondition: the miss must be on the streak before the late answer"
20013 );
20014
20015 let late = finish_after(&mut harness, Duration::from_secs(9)).await;
20017 assert!(matches!(
20018 late,
20019 ModuleControlRpcCompletion::LateHealthAnswer { .. }
20020 ));
20021 assert!(harness.handler.observe_module_control_completion(late));
20022
20023 let health = harness.module.status().unwrap().health;
20024 assert_eq!(
20025 health.consecutive_failures, 0,
20026 "a late answer is an answer: the streak must reset"
20027 );
20028 assert_eq!(health.late_answer_count, 1);
20029 }
20030
20031 #[tokio::test(start_paused = true)]
20032 async fn repeated_serial_probe_cycles_keep_one_tombstone_per_endpoint() {
20033 let mut harness = probe_harness();
20034
20035 for _ in 0..20 {
20036 time_out_without_answer(&mut harness).await;
20037 assert_eq!(
20038 harness.forwarding.health_probe_tombstone_count().unwrap(),
20039 1
20040 );
20041 }
20042 }
20043}
20044
20045#[cfg(test)]
20046mod child_env_tests {
20047 #[cfg(windows)]
20048 use super::{
20049 clear_current_process_facts, observed_launch_nonce_source, ModuleState, SupervisorSnapshot,
20050 };
20051 #[cfg(windows)]
20052 use std::sync::{Arc, OnceLock};
20053 #[cfg(windows)]
20054 #[test]
20055 fn windows_wire_spawn_prepares_pipe_and_keeps_phase_one_env() {
20056 let spec = spec(Vec::new());
20057 let mut command = Command::new("unused");
20058 let handoff = apply_wire_spawn_args(&mut command, &spec, None, None)
20059 .unwrap()
20060 .unwrap();
20061 let name = command
20062 .as_std()
20063 .get_envs()
20064 .find(|(key, _)| *key == OsStr::new(subc_os::launch_nonce::LAUNCH_NONCE_PIPE_ENV))
20065 .and_then(|(_, value)| value)
20066 .unwrap();
20067 assert_eq!(name, OsStr::new(handoff.name()));
20068 assert!(handoff.name().starts_with(r"\\.\pipe\subc-launch-"));
20069 assert!(command.as_std().get_envs().any(|(key, value)| key
20070 == OsStr::new(subc_os::LAUNCH_NONCE_PIPE_FALLBACK_ENV)
20071 && value == Some(OsStr::new("env"))));
20072 assert!(command
20073 .as_std()
20074 .get_envs()
20075 .any(|(key, value)| key == OsStr::new(SUBC_LAUNCH_NONCE_ENV) && value.is_some()));
20076 }
20077
20078 #[cfg(windows)]
20079 #[test]
20080 fn windows_source_reporting_tracks_consumption_and_resets_per_spawn() {
20081 let mut snapshot = SupervisorSnapshot::new(ModuleState::Running, true);
20082 snapshot.process_alive = true;
20083 snapshot.spawned_protocol = Some(ModuleProtocol::Subc);
20084 assert_eq!(
20085 observed_launch_nonce_source(&snapshot).as_deref(),
20086 Some("env")
20087 );
20088 let consumed = Arc::new(OnceLock::new());
20089 snapshot.pipe_consumed = Some(consumed.clone());
20090 assert_eq!(
20091 observed_launch_nonce_source(&snapshot).as_deref(),
20092 Some("env")
20093 );
20094 consumed.set(()).unwrap();
20095 assert_eq!(
20096 observed_launch_nonce_source(&snapshot).as_deref(),
20097 Some("pipe")
20098 );
20099 snapshot.pipe_consumed = Some(Arc::new(OnceLock::new()));
20100 assert_eq!(
20101 observed_launch_nonce_source(&snapshot).as_deref(),
20102 Some("env")
20103 );
20104 clear_current_process_facts(&mut snapshot);
20105 assert_eq!(observed_launch_nonce_source(&snapshot), None);
20106 }
20107 use super::{
20108 apply_child_env, apply_spawn_role, apply_wire_spawn_args, LaunchSpecDigest, ModuleProtocol,
20109 ModuleSpec, SpawnRole, SupervisorHandle, SPAWN_ROLE_SWAP_CANDIDATE, SUBC_ARG,
20110 SUBC_LAUNCH_NONCE_ENV, SUBC_MODULE_ID_ENV, SUBC_SPAWN_ROLE_ENV,
20111 };
20112 use std::{ffi::OsStr, path::PathBuf};
20113 use tokio::process::Command;
20114
20115 fn spec(env: Vec<(String, String)>) -> ModuleSpec {
20116 ModuleSpec {
20117 script: None,
20118 module_id: "env-plan".to_string(),
20119 program: PathBuf::from("/nonexistent"),
20120 args: Vec::new(),
20121 env,
20122 reserved: false,
20123 reserved_prefixes: Vec::new(),
20124 protocol: ModuleProtocol::Subc,
20125 overlap: Default::default(),
20126 carry_containment: false,
20127 }
20128 }
20129
20130 #[test]
20131 fn configured_launch_spec_digest_is_canonical_and_excludes_injected_inputs() {
20132 let base = spec(vec![("A".into(), "B=C".into()), ("Z".into(), "v".into())]);
20133 let digest = LaunchSpecDigest::from_spec(&base);
20134 let mut reordered = base.clone();
20135 reordered.env.reverse();
20136 reordered.module_id = "different-id".into();
20137 reordered.reserved = true;
20138 reordered.overlap = Default::default();
20139 assert_eq!(digest, LaunchSpecDigest::from_spec(&reordered));
20140 let mut delimiter_collision = base.clone();
20141 delimiter_collision.env[0] = ("A=B".into(), "C".into());
20142 assert_ne!(digest, LaunchSpecDigest::from_spec(&delimiter_collision));
20143 let mut changed = base.clone();
20144 changed.protocol = ModuleProtocol::None;
20145 assert_ne!(digest, LaunchSpecDigest::from_spec(&changed));
20146 changed = base.clone();
20147 changed.args.push("--flag".into());
20148 assert_ne!(digest, LaunchSpecDigest::from_spec(&changed));
20149 changed = base.clone();
20150 changed.program = PathBuf::from("/other");
20151 assert_ne!(digest, LaunchSpecDigest::from_spec(&changed));
20152 changed = base.clone();
20153 changed.env[1].1 = "other".into();
20154 assert_ne!(digest, LaunchSpecDigest::from_spec(&changed));
20155 assert!(!format!("{digest:?}").contains("B=C"));
20156 assert!(!format!("{digest:?}").contains("[u8"));
20157 }
20158
20159 #[cfg(unix)]
20160 #[test]
20161 fn native_program_bytes_do_not_collapse_through_lossy_utf8() {
20162 use std::os::unix::ffi::OsStringExt;
20163 let mut left = spec(Vec::new());
20164 left.program = std::ffi::OsString::from_vec(vec![b'/', 0xff]).into();
20165 let mut right = left.clone();
20166 right.program = std::ffi::OsString::from_vec(vec![b'/', 0xfe]).into();
20167 assert_eq!(
20168 left.program.to_string_lossy(),
20169 right.program.to_string_lossy()
20170 );
20171 assert_ne!(
20172 LaunchSpecDigest::from_spec(&left),
20173 LaunchSpecDigest::from_spec(&right)
20174 );
20175 }
20176
20177 #[test]
20191 fn ambient_ck_log_is_removed_and_a_configured_one_survives() {
20192 let mut command = Command::new("/nonexistent");
20193 apply_child_env(&mut command, &spec(Vec::new()));
20194 let removed = command
20195 .as_std()
20196 .get_envs()
20197 .any(|(key, value)| key == OsStr::new("CK_LOG") && value.is_none());
20198 assert!(
20199 removed,
20200 "ambient CK_LOG must be explicitly removed for an unconfigured module"
20201 );
20202
20203 let mut configured = Command::new("/nonexistent");
20204 apply_child_env(
20205 &mut configured,
20206 &spec(vec![("CK_LOG".to_string(), "debug".to_string())]),
20207 );
20208 let effective = configured
20209 .as_std()
20210 .get_envs()
20211 .filter(|(key, _)| *key == OsStr::new("CK_LOG"))
20212 .last()
20213 .map(|(_, value)| value.map(|v| v.to_string_lossy().into_owned()));
20214 assert_eq!(
20215 effective,
20216 Some(Some("debug".to_string())),
20217 "a module's configured CK_LOG must survive the ambient removal"
20218 );
20219 }
20220
20221 #[test]
20230 fn protocol_none_spawn_carries_no_subc_argument_and_no_nonce() {
20231 let connection_file = std::path::Path::new("/run/subc-connection.json");
20232 let handle = SupervisorHandle::new();
20233
20234 let mut none_spec = spec(Vec::new());
20235 none_spec.protocol = ModuleProtocol::None;
20236 let mut none = Command::new("/nonexistent");
20237 let none_handoff =
20238 apply_wire_spawn_args(&mut none, &none_spec, Some(connection_file), Some(&handle))
20239 .expect("protocol-none spawn args apply");
20240 assert!(
20241 none_handoff.is_none(),
20242 "protocol:none spawn must not receive a nonce descriptor"
20243 );
20244 #[cfg(windows)]
20245 assert!(!none.as_std().get_envs().any(|(key, value)| key
20246 == OsStr::new(subc_os::LAUNCH_NONCE_PIPE_FALLBACK_ENV)
20247 && value.is_some()));
20248 assert!(
20249 !none.as_std().get_envs().any(|(key, value)| key
20250 == OsStr::new(subc_os::LAUNCH_NONCE_FD_ENV)
20251 && value.is_some()),
20252 "protocol:none spawn must not name a nonce descriptor"
20253 );
20254 let none_args: Vec<String> = none
20255 .as_std()
20256 .get_args()
20257 .map(|a| a.to_string_lossy().into_owned())
20258 .collect();
20259 assert!(
20260 !none_args.iter().any(|a| a == SUBC_ARG),
20261 "protocol:none argv must not carry --subc; got {none_args:?}"
20262 );
20263 let none_has_nonce = none
20264 .as_std()
20265 .get_envs()
20266 .any(|(key, value)| key == OsStr::new(SUBC_LAUNCH_NONCE_ENV) && value.is_some());
20267 assert!(
20268 !none_has_nonce,
20269 "protocol:none spawn must not receive a launch nonce"
20270 );
20271 let none_has_module_id = none
20272 .as_std()
20273 .get_envs()
20274 .any(|(key, value)| key == OsStr::new(SUBC_MODULE_ID_ENV) && value.is_some());
20275 assert!(
20276 none_has_module_id,
20277 "SUBC_MODULE_ID is inert and stays on every path"
20278 );
20279 assert!(
20280 handle.spawn_nonce(&none_spec.module_id).is_none(),
20281 "no nonce record for a process that will never present one"
20282 );
20283
20284 let wire_spec = spec(Vec::new());
20286 let mut wire = Command::new("/nonexistent");
20287 let wire_handoff =
20288 apply_wire_spawn_args(&mut wire, &wire_spec, Some(connection_file), Some(&handle))
20289 .expect("subc-wire spawn args apply");
20290 let wire_fd_env = wire
20291 .as_std()
20292 .get_envs()
20293 .find(|(key, _)| *key == OsStr::new(subc_os::LAUNCH_NONCE_FD_ENV))
20294 .and_then(|(_, value)| value.map(|v| v.to_string_lossy().into_owned()));
20295 #[cfg(unix)]
20296 assert_eq!(
20297 wire_fd_env,
20298 Some(wire_handoff.expect("a descriptor handoff").fd_env_value()),
20299 "a subc-wire spawn names the pipe it will receive at descriptor 3"
20300 );
20301 #[cfg(windows)]
20302 assert!(wire_handoff.is_some() && wire_fd_env.is_none());
20303 #[cfg(not(any(unix, windows)))]
20304 assert!(wire_handoff.is_none() && wire_fd_env.is_none());
20305 let wire_args: Vec<String> = wire
20306 .as_std()
20307 .get_args()
20308 .map(|a| a.to_string_lossy().into_owned())
20309 .collect();
20310 assert_eq!(
20311 wire_args,
20312 vec![
20313 SUBC_ARG.to_string(),
20314 connection_file.to_string_lossy().into_owned()
20315 ],
20316 "a subc-wire spawn still carries --subc <path>"
20317 );
20318 assert_eq!(
20319 wire.as_std()
20320 .get_envs()
20321 .any(|(key, value)| key == OsStr::new(SUBC_LAUNCH_NONCE_ENV) && value.is_some()),
20322 !cfg!(unix),
20323 "only Windows supplies the environment nonce"
20324 );
20325 assert!(handle.spawn_nonce(&wire_spec.module_id).is_some());
20326 }
20327
20328 #[test]
20338 fn plain_spawn_removes_the_spawn_role_even_when_the_spec_sets_it() {
20339 let role = |command: &Command| {
20340 command
20341 .as_std()
20342 .get_envs()
20343 .filter(|(key, _)| *key == OsStr::new(SUBC_SPAWN_ROLE_ENV))
20344 .last()
20345 .map(|(_, value)| value.map(|v| v.to_string_lossy().into_owned()))
20346 };
20347 let forged = spec(vec![(
20348 SUBC_SPAWN_ROLE_ENV.to_string(),
20349 SPAWN_ROLE_SWAP_CANDIDATE.to_string(),
20350 )]);
20351
20352 let mut plain = Command::new("/nonexistent");
20353 apply_child_env(&mut plain, &forged);
20354 apply_spawn_role(&mut plain, SpawnRole::Plain);
20355 assert_eq!(
20356 role(&plain),
20357 Some(None),
20358 "a plain spawn must remove SUBC_SPAWN_ROLE, whatever the spec says"
20359 );
20360
20361 let mut candidate = Command::new("/nonexistent");
20362 apply_child_env(&mut candidate, &spec(Vec::new()));
20363 apply_spawn_role(&mut candidate, SpawnRole::SwapCandidate);
20364 assert_eq!(
20365 role(&candidate),
20366 Some(Some(SPAWN_ROLE_SWAP_CANDIDATE.to_string()))
20367 );
20368 }
20369
20370 #[test]
20376 fn daemon_private_capture_keys_are_not_passed_to_the_child() {
20377 let mut command = Command::new("/nonexistent");
20378 apply_child_env(
20379 &mut command,
20380 &spec(vec![
20381 (super::CAPTURE_KEEP_ENV.to_string(), "5".to_string()),
20382 ("KEPT".to_string(), "yes".to_string()),
20383 ]),
20384 );
20385 let keys: Vec<String> = command
20386 .as_std()
20387 .get_envs()
20388 .filter(|(_, value)| value.is_some())
20389 .map(|(key, _)| key.to_string_lossy().into_owned())
20390 .collect();
20391 assert!(keys.contains(&"KEPT".to_string()), "got {keys:?}");
20392 assert!(
20393 !keys.contains(&super::CAPTURE_KEEP_ENV.to_string()),
20394 "daemon-private capture key leaked to the child: {keys:?}"
20395 );
20396 }
20397}
20398
20399#[cfg(test)]
20400mod jitter_tests {
20401 use super::jittered_health_delay;
20402 use std::{collections::HashSet, time::Duration};
20403
20404 const FLEET: [&str; 14] = [
20413 "aft",
20414 "alfonso-core",
20415 "magic-context",
20416 "broca",
20417 "thalamus",
20418 "quota",
20419 "engram",
20420 "plexus",
20421 "cerebellum",
20422 "astrocyte",
20423 "synapse",
20424 "subc-mcp",
20425 "cortexkit-credentials",
20426 "subc-federation",
20427 ];
20428
20429 #[test]
20437 fn probe_delays_disperse_across_the_fleet() {
20438 let cadence = Duration::from_secs(30);
20439 let delays: HashSet<Duration> = FLEET
20440 .iter()
20441 .map(|id| jittered_health_delay(id, 0, cadence))
20442 .collect();
20443 assert_eq!(
20444 delays.len(),
20445 FLEET.len(),
20446 "every supervised module must land on its own probe offset"
20447 );
20448 }
20449
20450 #[test]
20456 fn jitter_only_delays_and_stays_within_one_tenth_of_cadence() {
20457 let cadence = Duration::from_secs(30);
20458 let span = cadence / 10;
20459 for id in FLEET {
20460 for probe_index in 0..8 {
20461 let delay = jittered_health_delay(id, probe_index, cadence);
20462 assert!(
20463 delay >= cadence,
20464 "{id}#{probe_index}: jitter must not shorten the cadence"
20465 );
20466 assert!(
20467 delay < cadence + span,
20468 "{id}#{probe_index}: jitter must stay inside one tenth of the cadence"
20469 );
20470 }
20471 }
20472 }
20473
20474 #[test]
20480 fn a_module_offset_is_stable_across_restarts() {
20481 let cadence = Duration::from_secs(30);
20482 for id in FLEET {
20483 assert_eq!(
20484 jittered_health_delay(id, 0, cadence),
20485 jittered_health_delay(id, 0, cadence),
20486 "{id}: the same module and probe index must produce the same offset"
20487 );
20488 }
20489 }
20490
20491 #[test]
20493 fn zero_cadence_yields_zero_delay() {
20494 assert_eq!(
20495 jittered_health_delay("aft", 0, Duration::ZERO),
20496 Duration::ZERO
20497 );
20498 }
20499}
20500
20501#[cfg(all(test, target_os = "linux"))]
20502mod cgroup_placement_tests {
20503 use super::{
20504 apply_cgroup_placement, remove_module_cgroup, LaunchSpecDigest, ModuleProtocol, ModuleSpec,
20505 SuperviseError, SupervisedChild,
20506 };
20507 use crate::stderr_tail::{StderrRing, StderrTailConfig};
20508 use cortexkit_test_support::ScratchDir;
20509 use std::{
20510 fs, io,
20511 path::{Path, PathBuf},
20512 sync::{Arc, Mutex},
20513 };
20514 use tokio::process::Command;
20515
20516 #[tokio::test]
20517 async fn unique_spawn_cgroups_clean_up_across_restarts_and_shutdown() {
20518 use super::*;
20519 let dir = ScratchDir::new("unique-spawn-cgroups");
20520 let root = PathBuf::from(format!(
20521 "/sys/fs/cgroup/subc-unique-test-{}-{}",
20522 std::process::id(),
20523 unix_ms_now()
20524 ));
20525 if let Err(error) = fs::create_dir(&root) {
20526 assert!(
20527 std::env::var_os("SUBC_REQUIRE_CGROUP_TEST").is_none(),
20528 "required cgroup test cannot execute: {error}"
20529 );
20530 eprintln!("SKIP unique_spawn_cgroups_clean_up_across_restarts_and_shutdown: {error}");
20531 return;
20532 }
20533 let placement = subc_cgroup::prepare_at(&root).unwrap().unwrap();
20534 let group_count = || {
20535 fs::read_dir(root.join("subc-modules"))
20536 .unwrap()
20537 .map(|entry| entry.unwrap().file_type().unwrap())
20538 .filter(|kind| kind.is_dir())
20539 .count()
20540 };
20541 let supervisor =
20542 Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default())
20543 .with_cgroup_placement(Some(placement.clone()));
20544 let runtime = supervisor.runtime_config();
20545 let mut spec = ModuleSpec {
20546 script: None,
20547 module_id: "unique-spawn".into(),
20548 program: PathBuf::from("/bin/sleep"),
20549 args: vec!["60".into()],
20550 env: ["XDG_DATA_HOME", "XDG_RUNTIME_DIR", "XDG_CONFIG_HOME"]
20551 .into_iter()
20552 .map(|key| (key.into(), dir.to_string_lossy().into_owned()))
20553 .collect(),
20554 reserved: false,
20555 reserved_prefixes: vec![],
20556 protocol: ModuleProtocol::None,
20557 overlap: Default::default(),
20558 carry_containment: false,
20559 };
20560 let spawn = |spec: &ModuleSpec| {
20561 spawn_child(
20562 spec,
20563 None,
20564 None,
20565 &runtime.stderr_ring,
20566 None,
20567 &runtime.child_roster,
20568 runtime.spawn_delivery(),
20569 Some(&placement),
20570 runtime.register_timeout,
20571 )
20572 .unwrap()
20573 };
20574 let mut live = spawn(&spec);
20575 for _ in 0..3 {
20576 let next = spawn(&spec);
20578 assert_ne!(live.module_id, next.module_id);
20579 live.start_kill().unwrap();
20580 live.wait().await.unwrap();
20581 live = next;
20582 assert!(
20583 live.child.try_wait().unwrap().is_none(),
20584 "retiring the old slot must not kill the replacement"
20585 );
20586 assert_eq!(
20587 group_count(),
20588 1,
20589 "only the live spawn's cgroup should remain"
20590 );
20591 }
20592 supervisor.begin_daemon_shutdown();
20593 let reap = tokio::spawn(async move {
20594 live.wait().await.unwrap();
20595 });
20596 supervisor
20597 .end_children_for_daemon_shutdown(false, std::future::pending())
20598 .await;
20599 reap.await.unwrap();
20600 assert_eq!(group_count(), 0);
20601 spec.program = PathBuf::from("/bin/true");
20603 spec.args.clear();
20604 let fresh_roster = ChildRoster::default();
20605 let mut short = spawn_child(
20606 &spec,
20607 None,
20608 None,
20609 &runtime.stderr_ring,
20610 None,
20611 &fresh_roster,
20612 None,
20613 Some(&placement),
20614 runtime.register_timeout,
20615 )
20616 .unwrap();
20617 short.wait().await.unwrap();
20618 assert_eq!(group_count(), 0);
20619 spec.module_id = "_".repeat(255);
20620 let mut long_id = spawn_child(
20621 &spec,
20622 None,
20623 None,
20624 &runtime.stderr_ring,
20625 None,
20626 &fresh_roster,
20627 None,
20628 Some(&placement),
20629 runtime.register_timeout,
20630 )
20631 .unwrap();
20632 long_id.wait().await.unwrap();
20633 assert_eq!(
20634 group_count(),
20635 0,
20636 "valid long module IDs must not exceed cgroup NAME_MAX"
20637 );
20638 fs::remove_dir(root.join("subc-modules")).unwrap();
20639 fs::remove_dir(root).unwrap();
20640 eprintln!("EXECUTED unique_spawn_cgroups_clean_up_across_restarts_and_shutdown");
20641 }
20642
20643 #[test]
20644 fn failed_parent_cgroup_open_is_a_cgroup_supervision_error() {
20645 let path = Path::new("/definitely-missing-subc-cgroup");
20646 let mut command = Command::new("true");
20647 let error = apply_cgroup_placement(
20648 &mut command,
20649 &ModuleSpec {
20650 script: None,
20651 module_id: "broken-cgroup".to_string(),
20652 program: PathBuf::from("true"),
20653 args: Vec::new(),
20654 env: Vec::new(),
20655 reserved: false,
20656 reserved_prefixes: Vec::new(),
20657 protocol: ModuleProtocol::Subc,
20658 overlap: Default::default(),
20659 carry_containment: false,
20660 },
20661 path,
20662 )
20663 .expect_err("a parent cgroup open failure must reject the supervised spawn");
20664 let reason = error.to_string();
20665
20666 assert!(
20667 matches!(error, SuperviseError::Cgroup { .. }),
20668 "parent cgroup open must be reported as a cgroup supervision error: {reason}"
20669 );
20670 assert!(
20671 reason.contains("/definitely-missing-subc-cgroup/cgroup.procs"),
20672 "parent cgroup open failure must name cgroup.procs: {reason}"
20673 );
20674 }
20675
20676 #[tokio::test]
20677 async fn reaping_a_child_removes_its_empty_module_cgroup() {
20678 let root = ScratchDir::new("supervisor-reap-cgroup");
20679 fs::write(root.join("cgroup.procs"), b"").expect("write scratch cgroup marker");
20680 let placement = subc_cgroup::prepare_at(&root)
20681 .expect("prepare scratch cgroup root")
20682 .expect("scratch root has a cgroup.procs marker");
20683 let module_id = "reaped-module";
20684 let module = placement
20685 .module_path(module_id)
20686 .expect("create scratch module cgroup");
20687 let child = Command::new("true")
20688 .env("XDG_DATA_HOME", root.path())
20689 .env("XDG_RUNTIME_DIR", root.path())
20690 .env("XDG_CONFIG_HOME", root.path())
20691 .spawn()
20692 .expect("spawn short-lived child");
20693 let pid = child.id().expect("spawned child has pid");
20694 let mut child = SupervisedChild {
20695 registration_channel: None,
20696 registration_spawn: None,
20697 live_program_binding: None,
20698 registered_config_connection: None,
20699 admission_base: None,
20700 registration_declaration: None,
20701 _program_binding: None,
20702 subtree_target: crate::subtree_resources::Target::Unavailable("not_running"),
20703 child: crate::supervise::containment::ProcessChild::plain(child, pid),
20704 exec: crate::activity::ExecOwner(Arc::new(crate::activity::ExecOutcome::new())),
20705 protocol: ModuleProtocol::Subc,
20706 module_id: module_id.to_string(),
20707 cgroup_placement: Some(placement),
20708 stdout_pump: None,
20709 stderr_pump: None,
20710 stderr_ring: Arc::new(Mutex::new(StderrRing::new(StderrTailConfig::default()))),
20711 spawned_at: tokio::time::Instant::now(),
20712 config_delivery: None,
20713 registration_deadline: None,
20714 registration_watch: None,
20715 spawned_at_ms: 0,
20716 spawned_from: PathBuf::from("true"),
20717 launch_spec_digest: LaunchSpecDigest([0; 32]),
20718 spawned_file_identity: None,
20719 process_start_time: None,
20720 process_identity: None,
20721 pid,
20722 roster_guard: None,
20723 #[cfg(target_os = "macos")]
20724 privacy_exec: None,
20725 spawn_failure: None,
20726 admission_kill_sent: false,
20727 };
20728
20729 child.wait().await.expect("reap short-lived child");
20730
20731 assert!(
20732 !module.exists(),
20733 "waiting for a non-carried child must remove its empty cgroup before returning"
20734 );
20735 child.finish_exit(false).await.unwrap();
20736 }
20737
20738 #[test]
20739 fn non_empty_cgroup_removal_is_reported_without_blocking_teardown() {
20740 let root = ScratchDir::new("supervisor-non-empty-cgroup");
20741 fs::write(root.join("cgroup.procs"), b"").expect("write scratch cgroup marker");
20742 let placement = subc_cgroup::prepare_at(&root)
20743 .expect("prepare scratch cgroup root")
20744 .expect("scratch root has a cgroup.procs marker");
20745 let module = placement
20746 .module_path("surviving-module")
20747 .expect("create scratch module cgroup");
20748 fs::write(module.join("surviving-process"), b"still present")
20749 .expect("make scratch cgroup non-empty");
20750 let (logs, _guard) = crate::router::test_log::log_capture(tracing::Level::WARN);
20751
20752 remove_module_cgroup(&placement, "surviving-module");
20753
20754 let logs = crate::router::test_log::captured_logs(&logs);
20755 assert!(
20756 module.exists(),
20757 "failed removal must leave the cgroup intact"
20758 );
20759 assert!(
20760 logs.contains("could not remove module cgroup after process exit; continuing teardown")
20761 && logs.contains("surviving-module"),
20762 "best-effort removal must report the failure without returning it: {logs}"
20763 );
20764 }
20765
20766 #[test]
20767 fn cgroup_pre_exec_spawn_failure_names_the_cgroup_path() {
20768 let cgroup_path = PathBuf::from("/sys/fs/cgroup/subc-modules/broken-module");
20769 let reason = SuperviseError::Spawn {
20770 program: PathBuf::from("/bin/true"),
20771 source: io::Error::from_raw_os_error(13),
20772 cgroup_path: Some(cgroup_path.clone()),
20773 }
20774 .to_string();
20775
20776 assert!(
20777 reason.contains(&cgroup_path.display().to_string()),
20778 "a pre_exec spawn failure must name the cgroup path: {reason}"
20779 );
20780 }
20781}
20782
20783#[cfg(test)]
20784mod spawn_subscriber_lag_tests {
20785 use super::*;
20786
20787 #[tokio::test]
20792 async fn lagged_spawn_subscriber_receives_a_terminal_lagged_error_after_its_queued_frames() {
20793 let feed = SpawnEventFeed::default();
20794 feed.configure_incarnation("lag-incarnation".to_string());
20795 let (tx, mut rx) = mpsc::channel(1);
20798 feed.subscribe(ConnectionId::new(1), 7, 1, None, FrameSink::new(tx))
20799 .expect("subscribe");
20800 let emitted = SPAWN_SUBSCRIBER_BUFFER + 16;
20801 for index in 0..emitted {
20802 feed.emit_spawned(&format!("lag-module-{index}"), None, 1000, 0);
20803 tokio::task::yield_now().await;
20806 }
20807 assert_eq!(
20808 feed.subscriber_count(),
20809 0,
20810 "the lagged subscriber must be removed"
20811 );
20812
20813 let mut data = Vec::new();
20814 let mut last = None;
20815 loop {
20816 let next = tokio::time::timeout(Duration::from_secs(5), rx.recv())
20817 .await
20818 .expect("the forwarder must finish once the subscriber is dropped");
20819 let Some(outbound) = next else { break };
20820 let frame = outbound.frame;
20821 if frame.header.ty == FrameType::StreamData {
20822 assert!(last.is_none(), "no data may follow the terminal frame");
20823 let event: SpawnEvent = serde_json::from_slice(&frame.body).unwrap();
20824 data.push(event.cursor.seq);
20825 } else {
20826 assert!(last.is_none(), "exactly one terminal frame");
20827 last = Some(frame);
20828 }
20829 }
20830 assert!(!data.is_empty(), "queued frames drain before the terminal");
20831 for pair in data.windows(2) {
20832 assert_eq!(
20833 pair[1],
20834 pair[0] + 1,
20835 "queued frames arrive dense and in order"
20836 );
20837 }
20838 let terminal = last.expect("a lagged subscriber must receive a terminal frame");
20839 assert_eq!(terminal.header.ty, FrameType::Error);
20840 assert_eq!(terminal.header.corr, 7);
20841 let body: subc_protocol::ErrorBody = serde_json::from_slice(&terminal.body).unwrap();
20842 assert_eq!(body.code, SPAWN_SUBSCRIBER_LAGGED_CODE);
20843 let detail = body.detail.expect("lagged error carries detail");
20844 assert_eq!(
20845 detail["first_undelivered_cursor"]["seq"],
20846 data.last().unwrap() + 1,
20847 "the named cursor is the first event the subscriber did not receive"
20848 );
20849 assert_eq!(
20850 detail["first_undelivered_cursor"]["daemon_incarnation"],
20851 "lag-incarnation"
20852 );
20853 }
20854}
20855
20856#[cfg(test)]
20857mod terminal_history_read_concurrency_tests {
20858 use super::*;
20859 use crate::terminal_journal::read_pause;
20860 use cortexkit_test_support::ScratchDir;
20861 use std::sync::mpsc as std_mpsc;
20862
20863 fn journaled_ring(
20864 journal: &Arc<crate::terminal_journal::TerminalJournal>,
20865 ) -> Arc<Mutex<TerminalRing>> {
20866 Arc::new(Mutex::new(
20867 TerminalRing::new(TerminalRingConfig::default(), 1)
20868 .with_journal(Some(Arc::clone(journal))),
20869 ))
20870 }
20871
20872 fn crash(at_ms: u64) -> ExitReport {
20873 ExitReport {
20874 kind: ExitKind::Crash,
20875 code: Some(1),
20876 signal: None,
20877 at_ms,
20878 }
20879 }
20880
20881 fn record_within(
20884 module_id: &'static str,
20885 ring: &Arc<Mutex<TerminalRing>>,
20886 at_ms: u64,
20887 bound: Duration,
20888 ) -> bool {
20889 let ring = Arc::clone(ring);
20890 let (done, done_rx) = std_mpsc::channel();
20891 std::thread::spawn(move || {
20892 record_terminal(
20893 module_id,
20894 &ring,
20895 &SpawnEventFeed::default(),
20896 &crash(at_ms),
20897 TerminalDisposition::Restarting,
20898 );
20899 let _ = done.send(());
20900 });
20901 done_rx.recv_timeout(bound).is_ok()
20902 }
20903
20904 #[test]
20909 fn exits_recorded_during_a_paused_history_read_are_not_blocked_or_half_merged() {
20910 let dir = ScratchDir::new("terminal-history-concurrent-read");
20911 let path = dir.join("terminals.jsonl");
20912 let journal = Arc::new(crate::terminal_journal::TerminalJournal::open(
20913 path.clone(),
20914 "daemon".into(),
20915 ));
20916 let reader_ring = journaled_ring(&journal);
20917 let other_ring = journaled_ring(&journal);
20918 assert!(record_within(
20919 "reader-module",
20920 &reader_ring,
20921 10,
20922 Duration::from_secs(5)
20923 ));
20924
20925 let (started, release) = read_pause::install(&path);
20926 let reading = {
20927 let ring = Arc::clone(&reader_ring);
20928 std::thread::spawn(move || durable_terminal_history_of(&ring, "reader-module"))
20929 };
20930 started
20931 .recv_timeout(Duration::from_secs(5))
20932 .expect("the history read reached its pause");
20933
20934 let bound = Duration::from_secs(1);
20935 assert!(
20936 record_within("other-module", &other_ring, 20, bound),
20937 "another module's exit waited on a history read (journal writer held)"
20938 );
20939 assert!(
20940 record_within("reader-module", &reader_ring, 30, bound),
20941 "the read module's own exit waited on its history read (ring held)"
20942 );
20943
20944 drop(release);
20945 let paused = reading.join().unwrap();
20946 assert_eq!(
20947 paused.entries.iter().map(|e| e.at_ms).collect::<Vec<_>>(),
20948 vec![10],
20949 "an exit recorded after the read began lands in neither half of it"
20950 );
20951 assert_eq!(paused.journal_skipped_lines, 0);
20952 assert_eq!(paused.journal_read_errors, 0);
20953
20954 let after = durable_terminal_history_of(&reader_ring, "reader-module");
20955 assert_eq!(
20956 after.entries.iter().map(|e| e.at_ms).collect::<Vec<_>>(),
20957 vec![10, 30],
20958 "the next read merges ring and journal with no duplicate"
20959 );
20960 assert_eq!(after.journal_skipped_lines, 0);
20961 }
20962}
20963
20964#[cfg(test)]
20969mod stderr_settle_tests {
20970 use std::{
20971 future::Future,
20972 io,
20973 pin::Pin,
20974 sync::{Arc, Mutex},
20975 task::{Context, Poll},
20976 time::Duration,
20977 };
20978
20979 use tokio::{
20980 io::{AsyncRead, ReadBuf},
20981 sync::oneshot,
20982 time::Instant,
20983 };
20984
20985 use super::{settle_stderr_pump, settle_stdout_pump, StderrPump};
20986 use crate::stderr_tail::{
20987 pump_stderr_to, pump_stdout_to, split_capture_stamp, untimed, CaptureState,
20988 ChildOutputSink, OutputSink, StderrRing, StderrTailConfig, TailEntry,
20989 };
20990
20991 const BOUND: Duration = Duration::from_millis(250);
20992
20993 struct HeldReader {
20997 before: Option<Vec<u8>>,
20998 gate: Option<oneshot::Receiver<()>>,
20999 after: io::Cursor<Vec<u8>>,
21000 }
21001
21002 impl AsyncRead for HeldReader {
21003 fn poll_read(
21004 mut self: Pin<&mut Self>,
21005 cx: &mut Context<'_>,
21006 buf: &mut ReadBuf<'_>,
21007 ) -> Poll<io::Result<()>> {
21008 if let Some(bytes) = self.before.take() {
21009 buf.put_slice(&bytes);
21010 return Poll::Ready(Ok(()));
21011 }
21012 if let Some(gate) = self.gate.as_mut() {
21013 match Pin::new(gate).poll(cx) {
21014 Poll::Pending => return Poll::Pending,
21015 Poll::Ready(_) => self.gate = None,
21016 }
21017 }
21018 Pin::new(&mut self.after).poll_read(cx, buf)
21019 }
21020 }
21021
21022 struct DiscardSink;
21023
21024 impl OutputSink for DiscardSink {
21025 fn write_line(&mut self, _line: &[u8]) {}
21026 }
21027
21028 fn line(text: &str) -> TailEntry {
21029 TailEntry::Line {
21030 text: text.to_string(),
21031 truncated: false,
21032 at_ms: None,
21033 }
21034 }
21035
21036 fn lock(ring: &Arc<Mutex<StderrRing>>) -> std::sync::MutexGuard<'_, StderrRing> {
21037 ring.lock().unwrap()
21038 }
21039
21040 fn held_pump(
21044 ring: &Arc<Mutex<StderrRing>>,
21045 before: &str,
21046 after: &str,
21047 ) -> (StderrPump, oneshot::Sender<()>) {
21048 let generation = lock(ring).begin_process();
21049 let (release, gate) = oneshot::channel();
21050 let reader = HeldReader {
21051 before: Some(before.as_bytes().to_vec()),
21052 gate: Some(gate),
21053 after: io::Cursor::new(after.as_bytes().to_vec()),
21054 };
21055 let task = tokio::spawn(pump_stderr_to(
21056 reader,
21057 Arc::clone(ring),
21058 generation,
21059 DiscardSink,
21060 ));
21061 (StderrPump { task, generation }, release)
21062 }
21063
21064 async fn wait_until(ring: &Arc<Mutex<StderrRing>>, done: impl Fn(&StderrRing) -> bool) {
21065 for _ in 0..1000 {
21066 if done(&lock(ring)) {
21067 return;
21068 }
21069 tokio::time::sleep(Duration::from_millis(1)).await;
21070 }
21071 panic!(
21072 "ring never reached the expected state: {:?}",
21073 lock(ring).snapshot(None, None)
21074 );
21075 }
21076
21077 #[tokio::test(start_paused = true)]
21078 async fn a_crash_line_the_reader_had_not_reached_by_the_bound_is_kept_before_the_restart() {
21079 let ring = Arc::new(Mutex::new(StderrRing::new(StderrTailConfig::default())));
21080 let (pump, release) = held_pump(&ring, "booting\n", "config error: missing storage\n");
21081
21082 settle_stderr_pump("crasher", &ring, pump, BOUND).await;
21083 let before_release = lock(&ring).snapshot(None, None);
21084 assert!(
21085 matches!(before_release.capture, CaptureState::Incomplete { .. }),
21086 "a reader that has not reached EOF cannot claim a whole tail: {before_release:?}"
21087 );
21088
21089 let next = lock(&ring).begin_process();
21092 lock(&ring).push_line_from(next, "next process booting");
21093 release.send(()).unwrap();
21094 wait_until(&ring, |ring| {
21095 ring.snapshot(None, None).capture == CaptureState::Captured
21096 })
21097 .await;
21098
21099 assert_eq!(
21100 untimed(lock(&ring).snapshot(None, None).entries),
21101 vec![
21102 line("booting"),
21103 line("config error: missing storage"),
21104 TailEntry::ProcessStart,
21105 line("next process booting"),
21106 ],
21107 "the crash's last line must survive a slow reader and stay in the crashed process's section"
21108 );
21109 }
21110
21111 #[tokio::test(start_paused = true)]
21112 async fn a_pipe_held_open_by_a_descendant_reads_incomplete_without_delaying_the_restart_past_the_bound(
21113 ) {
21114 let ring = Arc::new(Mutex::new(StderrRing::new(StderrTailConfig::default())));
21115 let (pump, _held) = held_pump(&ring, "parent exiting\n", "");
21118
21119 let started = Instant::now();
21120 settle_stderr_pump("orphaning", &ring, pump, BOUND).await;
21121 assert_eq!(
21122 started.elapsed(),
21123 BOUND,
21124 "the restart must wait exactly the bound for a pipe that stays open, no longer"
21125 );
21126
21127 let next = lock(&ring).begin_process();
21128 lock(&ring).push_line_from(next, "next process booting");
21129 tokio::time::sleep(Duration::from_secs(60)).await;
21130
21131 let snapshot = lock(&ring).snapshot(None, None);
21132 match &snapshot.capture {
21133 CaptureState::Incomplete { reason } => assert!(
21134 reason.contains("had not reached EOF") && reason.contains("250ms"),
21135 "the reason must say what is missing and after how long: {reason}"
21136 ),
21137 other => panic!("expected Incomplete while the pipe is held open, got {other:?}"),
21138 }
21139 assert_eq!(
21140 untimed(snapshot.entries),
21141 vec![
21142 line("parent exiting"),
21143 TailEntry::ProcessStart,
21144 line("next process booting"),
21145 ]
21146 );
21147 }
21148
21149 #[tokio::test(start_paused = true)]
21150 async fn a_reader_that_reaches_eof_within_the_bound_leaves_the_tail_captured() {
21151 let ring = Arc::new(Mutex::new(StderrRing::new(StderrTailConfig::default())));
21152 let (pump, release) = held_pump(&ring, "one\n", "two\n");
21153 release.send(()).unwrap();
21154
21155 settle_stderr_pump("clean", &ring, pump, BOUND).await;
21156
21157 let snapshot = lock(&ring).snapshot(None, None);
21158 assert_eq!(snapshot.capture, CaptureState::Captured);
21159 assert_eq!(untimed(snapshot.entries), vec![line("one"), line("two")]);
21160 }
21161
21162 #[tokio::test(start_paused = true)]
21163 async fn stdout_buffered_past_the_exit_drain_bound_reaches_the_capture_file() {
21164 const BURST: usize = 150;
21165 let capture = cortexkit_test_support::ScratchDir::new("stdout-exit-drain");
21166 let path = capture.join("two-pipe.stderr.log");
21167 let sink = ChildOutputSink::open(&path, cortexkit_log::Retention::default()).unwrap();
21168 let ring = Arc::new(Mutex::new(StderrRing::new(StderrTailConfig::default())));
21169 let generation = lock(&ring).begin_process();
21170 let stdout = (0..BURST)
21171 .map(|i| format!("out-{i:04}-{}\n", "o".repeat(64)))
21172 .collect::<String>();
21173 let stderr = (0..BURST)
21174 .map(|i| format!("err-{i:04}-{}\n", "e".repeat(64)))
21175 .collect::<String>();
21176 let first_chunk = stdout.find('\n').unwrap() + 1 + "out-0001-".len();
21181 let (release, gate) = oneshot::channel();
21182 let reader = HeldReader {
21183 before: Some(stdout.as_bytes()[..first_chunk].to_vec()),
21184 gate: Some(gate),
21185 after: io::Cursor::new(stdout.as_bytes()[first_chunk..stdout.len() - 1].to_vec()),
21186 };
21187 let (finished, completion) = oneshot::channel();
21188 let stdout_sink = sink.clone();
21189 let pump = tokio::spawn(async move {
21190 pump_stdout_to(reader, stdout_sink).await;
21191 finished.send(()).unwrap();
21192 });
21193 pump_stderr_to(io::Cursor::new(stderr.into_bytes()), ring, generation, sink).await;
21194
21195 let started = Instant::now();
21196 settle_stdout_pump("two-pipe", pump, BOUND).await;
21197 assert_eq!(
21198 started.elapsed(),
21199 BOUND,
21200 "a late reader must not hold up restart"
21201 );
21202 assert_eq!(
21203 std::fs::read_to_string(&path).unwrap().lines().count(),
21204 BURST + 1
21205 );
21206
21207 release
21208 .send(())
21209 .expect("stdout reader must survive the exit drain bound");
21210 tokio::time::timeout(Duration::from_secs(1), completion)
21211 .await
21212 .expect("released stdout reader must reach EOF")
21213 .expect("stdout capture must finish at EOF, not be cancelled");
21214 let contents = std::fs::read_to_string(&path).unwrap();
21215 let mut lines = contents
21216 .lines()
21217 .map(|line| split_capture_stamp(line).expect("capture stamp").1)
21218 .collect::<Vec<_>>();
21219 let mut expected = (0..BURST)
21220 .flat_map(|i| {
21221 [
21222 format!("out-{i:04}-{}", "o".repeat(64)),
21223 format!("err-{i:04}-{}", "e".repeat(64)),
21224 ]
21225 })
21226 .collect::<Vec<_>>();
21227 lines.sort_unstable();
21228 expected.sort_unstable();
21229 assert_eq!(
21230 lines, expected,
21231 "both pipes must retain every complete line exactly once"
21232 );
21233 }
21234}
21235
21236#[cfg(all(test, windows))]
21250mod job_containment_tests {
21251 use super::*;
21252 use cortexkit_test_support::ScratchDir;
21253 use std::{
21254 path::{Path, PathBuf},
21255 sync::{Arc, Mutex},
21256 time::{Duration, Instant},
21257 };
21258
21259 fn stub_path() -> PathBuf {
21265 let path = cortexkit_test_support::cargo_profile_bin("fake-aft-stub");
21266 assert!(
21267 path.exists(),
21268 "fake-aft-stub not built at {}: run `cargo test -p subc-core` (which builds \
21269 [[bin]] targets) rather than `cargo test -p subc-core --lib` (which does not)",
21270 path.display()
21271 );
21272 path
21273 }
21274
21275 fn read_grandchild_pid(path: &Path) -> u32 {
21277 let deadline = Instant::now() + Duration::from_secs(10);
21278 loop {
21279 if let Ok(contents) = std::fs::read_to_string(path) {
21280 if let Ok(pid) = contents.trim().parse() {
21281 return pid;
21282 }
21283 }
21284 assert!(
21285 Instant::now() < deadline,
21286 "the stub never recorded a grandchild pid at {}",
21287 path.display()
21288 );
21289 std::thread::sleep(Duration::from_millis(10));
21290 }
21291 }
21292
21293 struct Fixture {
21296 _dir: ScratchDir,
21297 _roster: ChildRoster,
21298 module_id: String,
21299 grandchild: u32,
21300 child: Option<SupervisedChild>,
21301 registry: Arc<Registry>,
21302 snapshot: Arc<Mutex<SupervisorSnapshot>>,
21303 terminal_ring: Arc<Mutex<TerminalRing>>,
21304 spawn_events: SpawnEventFeed,
21305 }
21306
21307 fn fixture(label: &str, module_id: &str) -> Fixture {
21308 fixture_with_carry(label, module_id, false)
21309 }
21310
21311 fn fixture_with_carry(label: &str, module_id: &str, carry_containment: bool) -> Fixture {
21312 let dir = ScratchDir::new(label);
21313 let pid_file = dir.join("grandchild.pid");
21314 let supervisor = Supervisor::new_for_test(
21315 Arc::new(Registry::default()),
21316 RestartPolicy::new(3, Duration::ZERO),
21317 );
21318 let runtime = supervisor.runtime_config();
21319 let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::starting()));
21320 let spec = ModuleSpec {
21321 script: None,
21322 module_id: module_id.to_string(),
21323 program: stub_path(),
21324 args: Vec::new(),
21328 env: vec![
21329 ("FAKE_AFT_NEVER_CONNECT".to_string(), "1".to_string()),
21330 (
21331 "FAKE_AFT_GRANDCHILD_PID_FILE".to_string(),
21332 pid_file.display().to_string(),
21333 ),
21334 ],
21335 reserved: false,
21336 reserved_prefixes: Vec::new(),
21337 protocol: ModuleProtocol::Subc,
21338 overlap: Default::default(),
21339 carry_containment,
21340 };
21341 let child = spawn_and_mark_running(&spec, &runtime, &snapshot)
21342 .expect("spawn the supervised fixture");
21343 let grandchild = read_grandchild_pid(&pid_file);
21344 Fixture {
21345 _dir: dir,
21346 _roster: runtime.child_roster.clone(),
21347 module_id: module_id.to_string(),
21348 grandchild,
21349 child: Some(child),
21350 registry: Arc::new(Registry::default()),
21351 snapshot,
21352 terminal_ring: Arc::clone(&runtime.terminal_ring),
21353 spawn_events: SpawnEventFeed::default(),
21354 }
21355 }
21356
21357 async fn assert_admission_kill_reaps_grandchild(carry: bool) {
21358 let label = if carry {
21359 "admission-kill-carried-tree"
21360 } else {
21361 "admission-kill-plain-tree"
21362 };
21363 let mut fixture = fixture_with_carry(label, label, carry);
21364 let descendant = subc_os::Process::open(fixture.grandchild)
21365 .unwrap()
21366 .expect("the stub descendant must be alive before admission kill");
21367 assert!(descendant.observe().is_some());
21368 let child = fixture.child.as_mut().unwrap();
21369 assert_eq!(child.child.container().is_some(), carry);
21370 assert_eq!(child.job.is_some(), !carry);
21371
21372 child.start_admission_kill().unwrap();
21373 assert!(child.admission_kill_sent);
21374 let status = timeout(Duration::from_secs(10), child.wait())
21375 .await
21376 .unwrap()
21377 .unwrap();
21378 assert_eq!(
21379 status.code(),
21380 Some(admission_exit::WINDOWS_ADMISSION_KILL_EXIT_CODE),
21381 "containment teardown must preserve the leader's distinctive status"
21382 );
21383 assert!(
21386 descendant.wait_for_exit(Duration::from_secs(10)).unwrap(),
21387 "admission kill left a descendant running before exit finalization"
21388 );
21389 assert_eq!(child.child.container().is_some(), carry);
21390 assert_eq!(child.job.is_some(), !carry);
21391 child.finish_exit(false).await.unwrap();
21392 }
21393
21394 #[tokio::test]
21395 async fn admission_kill_reaps_grandchild_with_plain_job() {
21396 assert_admission_kill_reaps_grandchild(false).await;
21397 }
21398
21399 #[tokio::test]
21400 async fn admission_kill_reaps_grandchild_with_carried_job() {
21401 assert_admission_kill_reaps_grandchild(true).await;
21402 }
21403
21404 #[tokio::test]
21405 async fn admission_kill_retires_registration_before_wait() {
21406 let mut fixture = fixture("admission-registration", "admission-registration");
21407 let handle = SupervisorHandle::new();
21408 let spawn =
21409 RegistrationSpawn::new(&handle, "admission-registration", SpawnRole::Plain, None);
21410 spawn.ready();
21411 let child = fixture.child.as_mut().unwrap();
21412 child.registration_spawn = Some(spawn.clone());
21413 child.start_admission_kill().unwrap();
21414 assert!(
21415 !spawn.launch_ready(),
21416 "admission kill must retire before its caller can await the child"
21417 );
21418 timeout(Duration::from_secs(10), child.wait())
21419 .await
21420 .unwrap()
21421 .unwrap();
21422 child.finish_exit(false).await.unwrap();
21423 }
21424
21425 impl Fixture {
21426 async fn drain(&mut self) {
21428 let child = self
21429 .child
21430 .take()
21431 .expect("the fixture child is still present");
21432 drain_child_to_state(
21433 &self.module_id,
21434 ModuleProtocol::Subc,
21435 StopNotice::NotSent,
21438 &self.registry,
21439 None,
21440 &self.snapshot,
21441 &self.terminal_ring,
21442 &self.spawn_events,
21443 child,
21444 Duration::from_millis(500),
21445 ModuleState::Stopped,
21446 Some(false),
21447 &FailureContext::operator(),
21448 )
21449 .await
21450 .expect("drain the supervised fixture");
21451 }
21452 }
21453
21454 #[tokio::test]
21460 async fn teardown_reaps_the_grandchild() {
21461 let mut fixture = fixture("teardown-grandchild", "tree-teardown");
21462 let grandchild = fixture.grandchild;
21463
21464 assert!(
21465 subc_jobobject::process_exists(grandchild),
21466 "grandchild {grandchild} must be alive before teardown, or this proves nothing"
21467 );
21468
21469 fixture.drain().await;
21470
21471 assert!(
21472 subc_jobobject::wait_for_process_exit(grandchild, Duration::from_secs(10)),
21473 "grandchild {grandchild} outlived module teardown: the tree was not contained"
21474 );
21475 }
21476
21477 #[test]
21490 fn an_uncontained_grandchild_survives_a_direct_child_kill() {
21491 let dir = ScratchDir::new("teardown-uncontained");
21492 let pid_file = dir.join("grandchild.pid");
21493 let mut child = std::process::Command::new(stub_path())
21494 .env("FAKE_AFT_NEVER_CONNECT", "1")
21495 .env(
21496 "FAKE_AFT_GRANDCHILD_PID_FILE",
21497 pid_file.display().to_string(),
21498 )
21499 .stdin(std::process::Stdio::null())
21500 .stdout(std::process::Stdio::null())
21501 .stderr(std::process::Stdio::null())
21502 .spawn()
21503 .expect("spawn the uncontained fixture");
21504 let grandchild = read_grandchild_pid(&pid_file);
21505
21506 child.kill().expect("kill the direct child");
21508 let _ = child.wait();
21509
21510 assert!(
21511 subc_jobobject::process_exists(grandchild),
21512 "grandchild {grandchild} died with the direct child, so this control no longer \
21513 distinguishes contained from uncontained teardown and the regression test is \
21514 passing vacuously"
21515 );
21516
21517 kill_tree(grandchild);
21520 }
21521
21522 #[tokio::test]
21531 async fn dropping_containment_reaps_the_grandchild() {
21532 let mut fixture = fixture("drop-containment", "tree-drop");
21533 let grandchild = fixture.grandchild;
21534
21535 assert!(subc_jobobject::process_exists(grandchild));
21536
21537 fixture.child.as_mut().expect("child present").job = None;
21539
21540 assert!(
21541 subc_jobobject::wait_for_process_exit(grandchild, Duration::from_secs(10)),
21542 "grandchild {grandchild} survived the containment handle closing, so a daemon \
21543 crash would leave the tree behind"
21544 );
21545 }
21546
21547 fn kill_tree(pid: u32) {
21549 let _ = std::process::Command::new("taskkill.exe")
21550 .args(["/PID", &pid.to_string(), "/T", "/F"])
21551 .stdin(std::process::Stdio::null())
21552 .stdout(std::process::Stdio::null())
21553 .stderr(std::process::Stdio::null())
21554 .status();
21555 assert!(
21556 subc_jobobject::wait_for_process_exit(pid, Duration::from_secs(10)),
21557 "could not clean up grandchild {pid}"
21558 );
21559 }
21560}
21561
21562#[cfg(test)]
21563mod privacy_trampoline_configuration_tests {
21564 #[tokio::test]
21565 #[cfg_attr(not(target_os = "macos"), ignore = "requires macOS privacy trampoline")]
21566 async fn macos_spawn_without_a_configured_trampoline_refuses_by_name() {
21567 #[cfg(target_os = "macos")]
21568 {
21569 let supervisor = super::Supervisor::new(
21570 std::sync::Arc::new(crate::Registry::default()),
21571 super::RestartPolicy::default(),
21572 );
21573 let error = supervisor.spawn(spec()).unwrap_err();
21574 assert!(
21575 error
21576 .to_string()
21577 .contains("no privacy trampoline configured"),
21578 "{error}"
21579 );
21580 }
21581 }
21582
21583 #[tokio::test]
21584 #[cfg_attr(not(target_os = "macos"), ignore = "requires macOS privacy trampoline")]
21585 async fn macos_wrong_trampoline_is_rejected_by_the_startup_probe() {
21586 #[cfg(target_os = "macos")]
21587 {
21588 let supervisor = super::Supervisor::new(
21589 std::sync::Arc::new(crate::Registry::default()),
21590 super::RestartPolicy::default(),
21591 )
21592 .with_privacy_trampoline(std::env::current_exe().unwrap());
21593 let error = supervisor.spawn(spec()).unwrap_err();
21594 assert!(
21595 error
21596 .to_string()
21597 .contains("binary does not implement the privacy trampoline protocol"),
21598 "{error}"
21599 );
21600 }
21601 }
21602
21603 #[cfg(target_os = "macos")]
21604 fn spec() -> super::ModuleSpec {
21605 super::ModuleSpec {
21606 module_id: "privacy-configuration".into(),
21607 program: "/bin/sleep".into(),
21608 args: vec!["30".into()],
21609 env: vec![],
21610 reserved: false,
21611 reserved_prefixes: vec![],
21612 protocol: subc_control::ModuleProtocol::None,
21613 overlap: super::ModuleOverlap::Exclusive,
21614 carry_containment: false,
21615 script: None,
21616 }
21617 }
21618}
21619
21620#[cfg(test)]
21621mod privacy_exec_boundary_tests {
21622 #[cfg(target_os = "macos")]
21623 use super::*;
21624 #[cfg(target_os = "macos")]
21625 use std::{
21626 io::{Read, Write},
21627 net::{TcpListener, TcpStream},
21628 };
21629
21630 #[cfg(target_os = "macos")]
21633 pub(super) fn before_image_sample(spec: &ModuleSpec, pid: u32) {
21634 if let Some((_, path)) = spec
21635 .env
21636 .iter()
21637 .find(|(key, _)| key == "SUBC_TEST_PRIVACY_SAMPLE_BARRIER")
21638 {
21639 let mut barrier = TcpStream::connect(path).unwrap();
21640 barrier
21641 .set_read_timeout(Some(Duration::from_secs(30)))
21642 .unwrap();
21643 barrier.write_all(&pid.to_ne_bytes()).unwrap();
21644 let mut release = [0];
21645 barrier.read_exact(&mut release).unwrap();
21646 assert_eq!(&release, b"X");
21647 }
21648 }
21649
21650 #[cfg(target_os = "macos")]
21651 fn spec(program: &str, args: &[&str]) -> ModuleSpec {
21652 ModuleSpec {
21653 script: None,
21654 module_id: "privacy-boundary".into(),
21655 program: program.into(),
21656 args: args.iter().map(|arg| (*arg).into()).collect(),
21657 env: vec![],
21658 reserved: false,
21659 reserved_prefixes: vec![],
21660 protocol: ModuleProtocol::None,
21661 overlap: ModuleOverlap::Exclusive,
21662 carry_containment: false,
21663 }
21664 }
21665
21666 #[cfg(target_os = "macos")]
21667 async fn accept(listener: TcpListener) -> TcpStream {
21668 let listener = tokio::net::TcpListener::from_std({
21670 listener.set_nonblocking(true).unwrap();
21671 listener
21672 })
21673 .unwrap();
21674 let (stream, _) = tokio::time::timeout(Duration::from_secs(30), listener.accept())
21675 .await
21676 .unwrap()
21677 .unwrap();
21678 let stream = stream.into_std().unwrap();
21679 stream.set_nonblocking(false).unwrap();
21680 stream
21681 .set_read_timeout(Some(Duration::from_secs(30)))
21682 .unwrap();
21683 stream
21684 }
21685
21686 #[tokio::test]
21687 #[cfg_attr(not(target_os = "macos"), ignore = "requires macOS privacy trampoline")]
21688 async fn macos_roster_withholds_a_nonnull_trampoline_image_until_exec_confirmation() {
21689 #[cfg(target_os = "macos")]
21690 {
21691 let root = cortexkit_test_support::ScratchDir::new("privacy-roster-barrier");
21692 let exec_listener = TcpListener::bind("127.0.0.1:0").unwrap();
21695 let sample_listener = TcpListener::bind("127.0.0.1:0").unwrap();
21696 let record = root.join("live-children.json");
21697 let supervisor =
21698 Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default())
21699 .with_live_children_record(&record);
21700 let runtime = supervisor.runtime_config();
21701 let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::starting()));
21702 let mut spec = spec("/bin/sleep", &["30"]);
21703 spec.env = vec![
21704 (
21705 "SUBC_TEST_PRIVACY_EXEC_BARRIER".into(),
21706 exec_listener.local_addr().unwrap().to_string(),
21707 ),
21708 (
21709 "SUBC_TEST_PRIVACY_SAMPLE_BARRIER".into(),
21710 sample_listener.local_addr().unwrap().to_string(),
21711 ),
21712 ];
21713 let spawn = tokio::task::spawn_blocking(move || {
21714 spawn_and_mark_running(&spec, &runtime, &snapshot).unwrap()
21715 });
21716 let mut sample = accept(sample_listener).await;
21717 let mut pid = [0; 4];
21718 sample.read_exact(&mut pid).unwrap();
21719 let pid = u32::from_ne_bytes(pid);
21720 let mut exec = accept(exec_listener).await;
21721 let mut ready = [0];
21722 exec.read_exact(&mut ready).unwrap();
21723 assert_eq!(&ready, b"R");
21724 let trampoline = subc_os::file_identity(&test_privacy_trampoline()).unwrap();
21727 assert_eq!(
21728 observe_spawned_image(pid).unwrap().executable,
21729 Some(trampoline)
21730 );
21731 sample.write_all(b"X").unwrap();
21732 let mut child = spawn.await.unwrap();
21733 let early = crate::live_children::read_record(&record).unwrap();
21734 assert_eq!(early.len(), 1);
21735 assert_eq!(early[0].pid, pid);
21736 assert_eq!(
21737 early[0].executable, None,
21738 "unconfirmed trampoline image entered the roster"
21739 );
21740 assert!(child.report_ready.get().is_none());
21741 child.privacy_exec.as_mut().unwrap().deadline =
21744 tokio::time::Instant::now() + Duration::from_secs(30);
21745 exec.write_all(b"X").unwrap();
21746 child.confirm_privacy_exec().await;
21747 assert_eq!(child.spawn_failure, None);
21748 assert!(child.report_ready.get().is_some());
21749 let confirmed = crate::live_children::read_record(&record).unwrap();
21750 let module = subc_os::file_identity(std::path::Path::new("/bin/sleep")).unwrap();
21751 assert_ne!(module, trampoline);
21752 assert_eq!(confirmed[0].executable, Some(module.into()));
21753 child.start_kill().unwrap();
21754 child.wait().await.unwrap();
21755 child.release_roster();
21756 }
21757 }
21758
21759 #[tokio::test]
21760 #[cfg_attr(not(target_os = "macos"), ignore = "requires macOS privacy trampoline")]
21761 async fn macos_unstamped_reexec_records_the_observed_native_image() {
21762 #[cfg(target_os = "macos")]
21763 {
21764 let root = cortexkit_test_support::ScratchDir::new("privacy-reexec");
21765 let record = root.join("live-children.json");
21766 let supervisor =
21767 Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default())
21768 .with_live_children_record(&record);
21769 let runtime = supervisor.runtime_config();
21770 let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::starting()));
21771 let fixture = crate::config_gate::program::native_fixture_for_test();
21772 let mut spec = spec(
21773 "/bin/sh",
21774 &["-c", "exec \"$1\" 600", "reexec", fixture.to_str().unwrap()],
21775 );
21776 spec.env
21777 .push(("SUBC_TEST_PRIVACY_IMAGE_DELAY_MS".into(), "5".into()));
21778 let mut child = spawn_and_mark_running(&spec, &runtime, &snapshot).unwrap();
21779 let native_image = subc_os::file_identity(&fixture).unwrap();
21780 let reexec = tokio::time::timeout(Duration::from_secs(5), async {
21783 while observe_spawned_image(child.pid).and_then(|image| image.executable)
21784 != Some(native_image)
21785 {
21786 tokio::time::sleep(Duration::from_millis(1)).await;
21787 }
21788 })
21789 .await;
21790 child.confirm_privacy_exec().await;
21791 let failure = child.spawn_failure.clone();
21792 let ready = child.report_ready.get().is_some();
21793 let confirmed = crate::live_children::read_record(&record).unwrap();
21794 child.start_kill().unwrap();
21795 tokio::time::timeout(Duration::from_secs(5), child.wait())
21796 .await
21797 .unwrap()
21798 .unwrap();
21799 child.release_roster();
21800
21801 reexec.expect("the shell must re-exec the native fixture");
21802 assert_eq!(
21803 failure, None,
21804 "an unstamped program may re-exec before observation"
21805 );
21806 assert!(ready);
21807 assert_eq!(confirmed.len(), 1);
21808 assert_eq!(confirmed[0].executable, Some(native_image.into()));
21809 }
21810 }
21811
21812 #[tokio::test]
21813 #[cfg_attr(not(target_os = "macos"), ignore = "requires macOS privacy trampoline")]
21814 async fn macos_unstamped_ack_without_exec_still_refuses_the_trampoline() {
21815 #[cfg(target_os = "macos")]
21816 {
21817 let listener = TcpListener::bind("127.0.0.1:0").unwrap();
21818 let supervisor =
21819 Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default());
21820 let runtime = supervisor.runtime_config();
21821 let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::starting()));
21822 let fixture = crate::config_gate::program::native_fixture_for_test();
21823 let mut spec = spec(fixture.to_str().unwrap(), &["600"]);
21824 spec.env.push((
21825 "SUBC_TEST_PRIVACY_EXEC_BARRIER".into(),
21826 listener.local_addr().unwrap().to_string(),
21827 ));
21828 let mut child = spawn_and_mark_running(&spec, &runtime, &snapshot).unwrap();
21829 let mut exec = accept(listener).await;
21830 let mut ready = [0];
21831 exec.read_exact(&mut ready).unwrap();
21832 assert_eq!(&ready, b"R");
21833 assert_eq!(
21834 observe_spawned_image(child.pid).unwrap().executable,
21835 subc_os::file_identity(&test_privacy_trampoline())
21836 );
21837 let (reader, writer) = std::io::pipe().unwrap();
21840 drop(writer);
21841 child.privacy_exec.as_mut().unwrap().reader =
21842 tokio::io::unix::AsyncFd::new(reader).unwrap();
21843 child.confirm_privacy_exec().await;
21844 let failure = child.spawn_failure.clone();
21845 let ready = child.report_ready.get().is_some();
21846 child.start_kill().unwrap();
21847 tokio::time::timeout(Duration::from_secs(5), child.wait())
21848 .await
21849 .unwrap()
21850 .unwrap();
21851 child.release_roster();
21852 assert_eq!(
21853 failure.as_deref(),
21854 Some("privacy identity trampoline did not exec the module")
21855 );
21856 assert!(!ready);
21857 }
21858 }
21859
21860 #[tokio::test]
21861 #[cfg_attr(not(target_os = "macos"), ignore = "requires macOS privacy trampoline")]
21862 async fn macos_resumed_121_is_a_module_exit_not_a_trampoline_refusal() {
21863 #[cfg(target_os = "macos")]
21864 {
21865 let registry = Arc::new(Registry::default());
21866 let policy = RestartPolicy::new(0, Duration::ZERO);
21867 let fixture = crate::config_gate::program::native_fixture_for_test();
21868 let spec = spec(fixture.to_str().unwrap(), &["exit121"]);
21869 let scratch = cortexkit_test_support::ScratchDir::new("resumed-exit");
21870 let supervisor = Supervisor::new_for_test(Arc::clone(®istry), policy)
21871 .with_config_runtime(suspended_spawn_tests::gated_owner(&scratch, &spec, true));
21872 let runtime = supervisor.runtime_config();
21873 let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::starting()));
21874 let mut child = spawn_and_mark_running(&spec, &runtime, &snapshot).unwrap();
21877 child.confirm_privacy_exec().await;
21878 assert_eq!(child.spawn_failure, None);
21879 assert!(child.report_ready.get().is_some());
21880 let pid = child.pid;
21881 tokio::task::spawn_blocking(move || {
21882 subc_os::fork_exec_test::wait_for_child_exit_without_reaping(pid)
21883 })
21884 .await
21885 .unwrap()
21886 .unwrap();
21887 let status = child.wait().await.unwrap();
21888 assert_eq!(status.code(), Some(121));
21889 assert!(child.privacy_exec.is_none());
21890 let report = classify_reaped_child_exit(&snapshot, &child, &status);
21891 on_child_exit(
21892 &spec,
21893 policy,
21894 ®istry,
21895 &snapshot,
21896 &runtime.terminal_ring,
21897 &runtime.spawn_events,
21898 &runtime.child_roster,
21899 report,
21900 true,
21901 )
21902 .await;
21903 let state = lock_snapshot(&snapshot).unwrap();
21904 assert_eq!(state.state, ModuleState::Failed);
21905 assert_eq!(state.last_exit.as_ref().unwrap().code, Some(121));
21906 assert_eq!(state.reported_pid(), None);
21907 drop(state);
21908 let history = runtime.terminal_ring.lock().unwrap().snapshot();
21909 assert_eq!(history.entries.len(), 1);
21910 let terminal = &history.entries[0];
21911 assert_eq!(terminal.exit_code, Some(121));
21912 assert_eq!(terminal.exit_kind, subc_control::TerminalExitKind::Crash);
21913 assert_eq!(terminal.disposition, TerminalDisposition::Failed);
21914 assert_eq!(
21915 terminal.disposition_detail.as_deref(),
21916 Some(policy.budget_exhausted_detail().as_str()),
21917 "module exit 121 was classified as a trampoline refusal: {terminal:?}"
21918 );
21919 assert_eq!(child.spawn_failure, None);
21920 child.release_roster();
21921 }
21922 }
21923}
21924
21925#[cfg(all(test, target_os = "macos"))]
21926#[path = "supervise/suspended_spawn_tests.rs"]
21927mod suspended_spawn_tests;
21928
21929#[cfg(all(test, unix))]
21933mod launch_nonce_descriptor_tests {
21934 use super::{ChildRoster, ModuleProtocol, ModuleSpec, SupervisorHandle};
21935 use crate::stderr_tail::{StderrRing, StderrTailConfig};
21936 use cortexkit_test_support::ScratchDir;
21937 use std::{
21938 path::PathBuf,
21939 sync::{Arc, Mutex},
21940 time::{Duration, Instant},
21941 };
21942
21943 async fn probe(role: super::SpawnRole) {
21944 let scratch = ScratchDir::new("launch-nonce-descriptor");
21945 let fd_copy = scratch.join("from-descriptor");
21946 let env_copy = scratch.join("environment");
21947 let script = format!(
21948 "env > '{env}.tmp' && mv '{env}.tmp' '{env}'; cat <&3 > '{fd}.tmp' && mv '{fd}.tmp' '{fd}'; sleep 30",
21949 fd = fd_copy.display(), env = env_copy.display(),
21950 );
21951 let xdg = |name: &str| (name.to_string(), scratch.join(name).display().to_string());
21952 let spec = ModuleSpec {
21953 script: None,
21954 module_id: "nonce-descriptor-probe".to_string(),
21955 program: PathBuf::from("/bin/sh"),
21956 args: vec!["-c".to_string(), script],
21957 env: vec![
21958 xdg("XDG_DATA_HOME"),
21959 xdg("XDG_RUNTIME_DIR"),
21960 xdg("XDG_CONFIG_HOME"),
21961 ("SUBC_LAUNCH_NONCE".to_string(), "must-not-leak".to_string()),
21962 ],
21963 reserved: true,
21964 reserved_prefixes: Vec::new(),
21965 protocol: ModuleProtocol::Subc,
21966 overlap: Default::default(),
21967 carry_containment: false,
21968 };
21969 let handle = SupervisorHandle::new();
21970 let ring = Arc::new(Mutex::new(StderrRing::new(StderrTailConfig::default())));
21971 let roster = ChildRoster::default();
21972 #[cfg(target_os = "macos")]
21973 {
21974 let path = super::test_privacy_trampoline();
21975 roster.set_privacy_trampoline(super::probe_privacy_trampoline(&path).map(|()| path));
21976 }
21977 let child = super::spawn_child_in_slot(
21978 &spec,
21979 None,
21980 Some(&handle),
21981 &ring,
21982 None,
21983 &roster,
21984 None,
21985 #[cfg(target_os = "linux")]
21986 None,
21987 role,
21988 matches!(role, super::SpawnRole::SwapCandidate),
21989 super::DEFAULT_REGISTER_TIMEOUT,
21990 )
21991 .expect("spawn probe");
21992 let deadline = Instant::now() + Duration::from_secs(10);
21993 while !(fd_copy.exists() && env_copy.exists()) {
21994 assert!(Instant::now() < deadline, "probe never wrote its copies");
21995 tokio::time::sleep(Duration::from_millis(20)).await;
21996 }
21997 let nonce = std::fs::read_to_string(fd_copy).unwrap();
21998 assert!(!nonce.is_empty());
21999 let environment = std::fs::read_to_string(env_copy).unwrap();
22000 assert!(environment
22001 .lines()
22002 .any(|line| line.starts_with("SUBC_LAUNCH_NONCE_FD=3:")));
22003 let copy = environment
22004 .lines()
22005 .find_map(|line| line.strip_prefix("SUBC_LAUNCH_NONCE="));
22006 assert_eq!(
22007 copy, None,
22008 "Unix children must never receive the environment nonce"
22009 );
22010 if matches!(role, super::SpawnRole::Plain) {
22011 assert_eq!(
22012 handle.spawn_nonce(&spec.module_id).as_deref(),
22013 Some(nonce.as_str())
22014 );
22015 }
22016 drop(child);
22017 }
22018
22019 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
22020 async fn a_spawned_module_receives_its_nonce_only_on_descriptor_3() {
22021 probe(super::SpawnRole::Plain).await;
22022 }
22023
22024 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
22025 async fn swap_candidate_receives_its_nonce_only_on_descriptor_3() {
22026 probe(super::SpawnRole::SwapCandidate).await;
22027 }
22028}
22029
22030#[cfg(all(test, target_os = "linux"))]
22031#[path = "supervise/cgroup_reap_tests.rs"]
22032mod cgroup_reap_tests;
22033
22034#[cfg(all(test, target_os = "linux"))]
22035mod cgroup_containment_tests {
22036 use super::*;
22037 use cortexkit_test_support::ScratchDir;
22038
22039 #[tokio::test]
22040 async fn linux_subtree_memory_covers_module_children() {
22041 let dir = ScratchDir::new("subtree-memory");
22042 let root = PathBuf::from(format!(
22043 "/sys/fs/cgroup/subc-memory-test-{}-{}",
22044 std::process::id(),
22045 unix_ms_now()
22046 ));
22047 if let Err(error) = std::fs::create_dir(&root) {
22048 assert!(
22049 std::env::var_os("SUBC_REQUIRE_CGROUP_TEST").is_none(),
22050 "required cgroup test cannot execute: {error}"
22051 );
22052 eprintln!("SKIP linux_subtree_memory_covers_module_children: {error}");
22053 return;
22054 }
22055 let placement = subc_cgroup::prepare_at(&root)
22056 .unwrap()
22057 .expect("delegated cgroup");
22058 let supervisor = Supervisor::new_for_test(
22059 Arc::new(Registry::default()),
22060 RestartPolicy::new(3, Duration::ZERO),
22061 )
22062 .with_cgroup_placement(Some(placement));
22063 let ready = dir.join("ready");
22064 let module = supervisor
22065 .spawn(ModuleSpec {
22066 script: None,
22067 module_id: "memory-tree".into(),
22068 program: PathBuf::from("/bin/sh"),
22069 args: vec![
22070 "-c".into(),
22071 "sleep 600 & sleep 600 & sleep 600 & echo ready > \"$1\"; wait".into(),
22072 "fixture".into(),
22073 ready.display().to_string(),
22074 ],
22075 env: ["XDG_DATA_HOME", "XDG_RUNTIME_DIR", "XDG_CONFIG_HOME"]
22076 .into_iter()
22077 .map(|key| (key.into(), dir.display().to_string()))
22078 .collect(),
22079 reserved: false,
22080 reserved_prefixes: Vec::new(),
22081 protocol: ModuleProtocol::None,
22082 overlap: Default::default(),
22083 carry_containment: false,
22084 })
22085 .unwrap();
22086 let deadline = tokio::time::Instant::now() + Duration::from_secs(10);
22087 while !ready.exists() {
22088 assert!(
22089 tokio::time::Instant::now() < deadline,
22090 "children never started"
22091 );
22092 tokio::time::sleep(Duration::from_millis(10)).await;
22093 }
22094 let readings = crate::subtree_resources::read(vec![module.subtree_target()]).await;
22095 let reading = &readings[0];
22096 assert!(
22097 reading.process_count.is_some_and(|count| count >= 4),
22098 "{reading:?}"
22099 );
22100 assert!(
22101 reading.memory_bytes.is_some_and(|bytes| bytes > 0),
22102 "{reading:?}"
22103 );
22104 module.stop().await.unwrap();
22105 drop(module);
22106 std::fs::remove_dir(root.join("subc-modules")).unwrap();
22107 std::fs::remove_dir(&root).unwrap();
22108 eprintln!("EXECUTED linux_subtree_memory_covers_module_children: {reading:?}");
22109 }
22110
22111 fn running(pid: u32) -> bool {
22112 std::fs::read_to_string(format!("/proc/{pid}/stat"))
22114 .ok()
22115 .and_then(|stat| {
22116 stat.rsplit_once(") ")
22117 .map(|(_, rest)| rest.starts_with('Z'))
22118 })
22119 .is_some_and(|zombie| !zombie)
22120 }
22121
22122 #[tokio::test]
22123 async fn linux_teardown_reaps_the_grandchild() {
22124 teardown_tree("linux_teardown_reaps_the_grandchild", false).await;
22125 }
22126
22127 #[tokio::test]
22128 async fn linux_shutdown_straggler_reaps_the_grandchild() {
22129 teardown_tree("linux_shutdown_straggler_reaps_the_grandchild", true).await;
22130 }
22131
22132 async fn teardown_tree(test_name: &str, shutdown: bool) {
22133 let dir = ScratchDir::new(test_name);
22134 let root = PathBuf::from(format!(
22135 "/sys/fs/cgroup/subc-tree-test-{test_name}-{}-{}",
22136 std::process::id(),
22137 unix_ms_now()
22138 ));
22139 if let Err(error) = std::fs::create_dir(&root) {
22140 assert!(
22141 std::env::var_os("SUBC_REQUIRE_CGROUP_TEST").is_none(),
22142 "required cgroup test cannot execute: {error}"
22143 );
22144 eprintln!(
22145 "SKIP {test_name}: no writable delegated cgroup at {}: {error}",
22146 root.display()
22147 );
22148 return;
22149 }
22150 let placement = subc_cgroup::prepare_at(&root)
22151 .expect("prepare isolated kernel cgroup")
22152 .expect("isolated cgroup is delegated");
22153 let module_id = "tree-teardown";
22154 let module = placement
22155 .module_path(module_id)
22156 .expect("create isolated module cgroup");
22157 if !module.join("cgroup.kill").exists() {
22158 std::fs::remove_dir(&module).unwrap();
22159 std::fs::remove_dir(root.join("subc-modules")).unwrap();
22160 std::fs::remove_dir(&root).unwrap();
22161 assert!(
22162 std::env::var_os("SUBC_REQUIRE_CGROUP_TEST").is_none(),
22163 "required cgroup.kill interface unavailable"
22164 );
22165 eprintln!("SKIP {test_name}: cgroup.kill unavailable (kernel < 5.14)");
22166 return;
22167 }
22168 let supervisor = Supervisor::new_for_test(
22169 Arc::new(Registry::default()),
22170 RestartPolicy::new(3, Duration::ZERO),
22171 )
22172 .with_cgroup_placement(Some(placement));
22173 let mut runtime = supervisor.runtime_config();
22174 runtime.child_roster = runtime
22175 .child_roster
22176 .for_module(Arc::new(Mutex::new(Duration::from_millis(100))));
22177 let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::starting()));
22178 let pid_file = dir.join("grandchild.pid");
22179 let spec = ModuleSpec {
22180 script: None,
22181 module_id: module_id.to_string(),
22182 program: PathBuf::from("/bin/sh"),
22183 args: vec![
22184 "-c".into(),
22185 "trap '' TERM; sleep 600 & echo $! > \"$1\"; wait".into(),
22186 "fixture".into(),
22187 pid_file.display().to_string(),
22188 ],
22189 env: ["XDG_DATA_HOME", "XDG_RUNTIME_DIR", "XDG_CONFIG_HOME"]
22190 .into_iter()
22191 .map(|key| (key.to_string(), dir.display().to_string()))
22192 .collect(),
22193 reserved: false,
22194 reserved_prefixes: Vec::new(),
22195 protocol: ModuleProtocol::None,
22196 overlap: Default::default(),
22197 carry_containment: false,
22198 };
22199 let child =
22200 spawn_and_mark_running(&spec, &runtime, &snapshot).expect("spawn supervised tree");
22201 let deadline = tokio::time::Instant::now() + Duration::from_secs(10);
22202 let grandchild: u32 = loop {
22203 if let Ok(contents) = std::fs::read_to_string(&pid_file) {
22204 if let Ok(pid) = contents.trim().parse() {
22205 break pid;
22206 }
22207 }
22208 assert!(
22209 tokio::time::Instant::now() < deadline,
22210 "grandchild pid was not recorded"
22211 );
22212 tokio::time::sleep(Duration::from_millis(10)).await;
22213 };
22214 assert!(
22215 running(grandchild),
22216 "grandchild must be alive before teardown"
22217 );
22218 if shutdown {
22219 let mut child = child;
22220 crate::child_roster::end_children_for_daemon_shutdown(
22221 &runtime.child_roster,
22222 false,
22223 std::future::pending(),
22224 )
22225 .await;
22226 child.wait().await.expect("reap shutdown straggler");
22227 } else {
22228 drain_child_to_state(
22229 module_id,
22230 ModuleProtocol::None,
22231 StopNotice::NotSent,
22232 &Registry::default(),
22233 None,
22234 &snapshot,
22235 &runtime.terminal_ring,
22236 &SpawnEventFeed::default(),
22237 child,
22238 Duration::from_millis(100),
22239 ModuleState::Stopped,
22240 Some(false),
22241 &FailureContext::operator(),
22242 )
22243 .await
22244 .expect("real supervisor teardown");
22245 }
22246 let deadline = tokio::time::Instant::now() + Duration::from_secs(5);
22247 while running(grandchild) && tokio::time::Instant::now() < deadline {
22248 tokio::time::sleep(Duration::from_millis(10)).await;
22249 }
22250 let survived = running(grandchild);
22251 if survived {
22253 let pid = rustix::process::Pid::from_raw(grandchild as i32).unwrap();
22254 let _ = rustix::process::kill_process(pid, rustix::process::Signal::KILL);
22255 tokio::time::sleep(Duration::from_millis(100)).await;
22256 }
22257 if module.exists() {
22258 std::fs::remove_dir(&module).expect("remove empty module cgroup");
22259 }
22260 std::fs::remove_dir(root.join("subc-modules")).unwrap();
22261 std::fs::remove_dir(&root).unwrap();
22262 assert!(
22263 !survived,
22264 "grandchild {grandchild} outlived module teardown"
22265 );
22266 eprintln!("EXECUTED {test_name}: grandchild {grandchild} killed in isolated cgroup");
22267 }
22268}
22269
22270#[cfg(all(test, unix))]
22273mod record_write_stall_tests {
22274 use super::*;
22275 use std::sync::atomic::{AtomicBool, Ordering};
22276 use subc_transport::{authenticate_client, ConnectionInfo, Endpoint, SCHEMA_VERSION};
22277
22278 const MODULE_AUTH_DEADLINE: Duration = Duration::from_secs(2);
22281 const HELD_RECORD_WRITE: Duration = Duration::from_secs(3);
22285
22286 fn sleeping_spec() -> ModuleSpec {
22298 ModuleSpec {
22299 script: None,
22300 module_id: "record-write-stall".into(),
22301 program: "/bin/sleep".into(),
22302 args: vec!["600".into()],
22303 env: vec![],
22304 reserved: false,
22305 reserved_prefixes: vec![],
22306 protocol: ModuleProtocol::None,
22307 overlap: Default::default(),
22308 carry_containment: false,
22309 }
22310 }
22311
22312 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
22313 async fn module_auth_completes_while_the_restart_waits_on_the_record_write() {
22314 let home = cortexkit_test_support::ScratchDir::new("record-write-stall");
22315 let supervisor =
22316 Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default())
22317 .with_live_children_record(home.join("live-children.json"));
22318 let module = supervisor.spawn(sleeping_spec()).unwrap();
22319 let old_pid = tokio::time::timeout(Duration::from_secs(10), async {
22324 loop {
22325 if let Some(pid) = lock_snapshot(&module.inner.snapshot)
22326 .unwrap()
22327 .reported_pid()
22328 {
22329 return pid;
22330 }
22331 tokio::time::sleep(Duration::from_millis(10)).await;
22332 }
22333 })
22334 .await
22335 .expect("the first process was admitted as the module");
22336
22337 let key = vec![0x42; 32];
22340 let daemon_id = [0x24; 16];
22341 let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
22342 let port = listener.local_addr().unwrap().port();
22343 let server = tokio::spawn(crate::server::serve_listener(
22344 listener,
22345 Arc::new(crate::router::Router::with_default_self_handler()),
22346 crate::server::ServerAuth::with_limits(
22347 key.clone(),
22348 daemon_id,
22349 "test",
22350 crate::server::DEFAULT_AUTH_DEADLINE,
22351 4,
22352 ),
22353 ));
22354
22355 let (held_tx, held_rx) = std::sync::mpsc::channel();
22358 let armed = AtomicBool::new(true);
22359 supervisor
22360 .child_roster
22361 .set_record_write_hook_for_test(move |live| {
22362 let admitting_replacement = live.values().any(|entry| entry.pid != old_pid);
22363 if admitting_replacement && armed.swap(false, Ordering::SeqCst) {
22364 let _ = held_tx.send(());
22365 std::thread::sleep(HELD_RECORD_WRITE);
22366 }
22367 });
22368
22369 let module_process = std::thread::spawn(move || {
22372 held_rx
22373 .recv_timeout(Duration::from_secs(30))
22374 .expect("the restart reached the replacement's record write");
22375 let conn = ConnectionInfo {
22376 schema: SCHEMA_VERSION,
22377 wire_version: None,
22378 endpoints: vec![Endpoint {
22379 host: "127.0.0.1".to_owned(),
22380 port,
22381 }],
22382 key,
22383 daemon_id,
22384 pid: std::process::id(),
22385 daemon_ver: "test".to_owned(),
22386 };
22387 tokio::runtime::Builder::new_current_thread()
22388 .enable_all()
22389 .build()
22390 .unwrap()
22391 .block_on(async {
22392 let started = Instant::now();
22393 let mut stream = tokio::net::TcpStream::connect(("127.0.0.1", port))
22394 .await
22395 .unwrap();
22396 let result =
22397 authenticate_client(&mut stream, &conn, MODULE_AUTH_DEADLINE).await;
22398 (result, started.elapsed())
22399 })
22400 });
22401
22402 module.restart(Some(0)).await.unwrap();
22403 let (auth, elapsed) = tokio::task::spawn_blocking(move || module_process.join().unwrap())
22404 .await
22405 .unwrap();
22406 assert!(
22407 auth.is_ok(),
22408 "the daemon must answer a module's handshake while a restart waits on the \
22409 live-children record write; got {auth:?} after {elapsed:?}"
22410 );
22411
22412 tokio::time::timeout(Duration::from_secs(20), async {
22413 while lock_snapshot(&module.inner.snapshot)
22414 .unwrap()
22415 .reported_pid()
22416 .is_none_or(|pid| pid == old_pid)
22417 {
22418 tokio::time::sleep(Duration::from_millis(10)).await;
22419 }
22420 })
22421 .await
22422 .expect("the restart published its replacement");
22423 module.stop().await.unwrap();
22424 server.abort();
22425 }
22426}
22427
22428#[cfg(all(test, unix))]
22429mod retained_status_tests {
22430 use super::*;
22431
22432 #[tokio::test]
22433 async fn cached_container_block_is_visible_in_status_and_health_without_rechecking() {
22434 use crate::live_children::{ContainerIdentity, RetainedReason};
22435 for (reason, carried, expected) in [
22436 (
22437 RetainedReason::ContainerSurvivors(vec![4242]),
22438 true,
22439 Some("waiting: container survivors (pids [4242])".to_owned()),
22440 ),
22441 (
22442 RetainedReason::ContainerUnverifiable("member inspection denied".into()),
22443 true,
22444 Some(
22445 "waiting: container survivors could not be verified: member inspection denied"
22446 .to_owned(),
22447 ),
22448 ),
22449 (RetainedReason::Survived, false, None),
22450 ] {
22451 let supervisor =
22452 Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default());
22453 let spec = ModuleSpec {
22454 script: None,
22455 module_id: "retained-status-fixture".into(),
22456 program: PathBuf::from("/not-spawned"),
22457 args: Vec::new(),
22458 env: Vec::new(),
22459 reserved: false,
22460 reserved_prefixes: Vec::new(),
22461 protocol: ModuleProtocol::None,
22462 overlap: ModuleOverlap::Exclusive,
22463 carry_containment: carried,
22464 };
22465 let module = supervisor
22466 .supervise_configured(spec.clone(), false)
22467 .unwrap();
22468 let guard = supervisor
22472 .child_roster
22473 .admit(
22474 spec.module_id,
22475 0,
22476 ModuleProtocol::None,
22477 None,
22478 crate::child_roster::RecordedIdentity {
22479 container: carried.then_some(ContainerIdentity {
22480 pgid: Some(0),
22481 created_start_time: Some(0),
22482 cgroup_path: None,
22483 ..ContainerIdentity::test_metadata()
22484 }),
22485 ..crate::child_roster::RecordedIdentity::default()
22486 },
22487 )
22488 .unwrap();
22489 guard.retain(reason);
22490 let status = module.status().unwrap();
22491 assert_eq!(status.waiting_for_previous_detail, expected.clone());
22492 assert_eq!(status.health.detail, expected);
22493 assert!(!status.process_alive);
22494 }
22495 }
22496}
22497
22498fn registration_child_is_running(child: &mut SupervisedChild) -> bool {
22499 child.try_wait().is_ok_and(|status| status.is_none())
22500}