1#[cfg(all(test, target_os = "linux"))]
6#[path = "supervise/audit_identity_tests.rs"]
7mod audit_identity_tests;
8mod resource_launch;
9pub(crate) use resource_launch::{ResourceCommitError, ResourceLaunch};
10
11use std::{
12 collections::{HashMap, HashSet, VecDeque},
13 error::Error,
14 fmt, io,
15 path::PathBuf,
16 process::{ExitStatus, Stdio},
17 sync::{Arc, Mutex, OnceLock},
18 time::{Duration, SystemTime, UNIX_EPOCH},
19};
20
21use cortexkit_log::Retention;
22use serde_json::Value;
23use sha2::{Digest, Sha256};
24use subc_control::{
25 ClientControlPush, LiveSpawn, ModuleProtocol, RouteCloseReason, SpawnCursor, SpawnEvent,
26 SpawnEventKind, SpawnSnapshot, SupervisorHealthStatus, TerminalDisposition, TerminalExitKind,
27};
28use subc_protocol::{
29 manifest::{SelfSignalKind, SignalAnchor},
30 session::{
31 HealthReport, HealthStatus, ModuleControlCommand, ModuleControlRequest,
32 MODULE_CONTROL_OP_HEALTH_CHECK,
33 },
34 Flags, FrameType, Principal, Priority, SUBC_LAUNCH_NONCE_ENV, SUBC_MODULE_ID_ENV,
35};
36use tokio::{
37 process::{Child, Command},
38 sync::{mpsc, oneshot, watch, Mutex as AsyncMutex},
39 task::JoinHandle,
40 time::{sleep, sleep_until, timeout, timeout_at, Instant},
41};
42use tracing::{debug, error, info, warn};
43
44use crate::{
45 child_roster::ChildRoster,
46 daemon_config::{
47 CAPTURE_KEEP_ENV, CAPTURE_MAX_AGE_DAYS_ENV, CAPTURE_MAX_FILE_MB_ENV, CK_LOG_ENV,
48 },
49 forwarding::{
50 CloseReason, ForwardingError, ForwardingTable, GoodbyeTarget, ModuleControlRpcOutcome,
51 ModuleDrainTarget, PendingModuleControlRpc,
52 },
53 provenance::{ExecutableIdentityProbe, SpawnedFileIdentity},
54 registry::{ConnectionId, RegistrationEndReason, RegistryError},
55 stderr_tail::{
56 pump_stderr_to, pump_stdout_to, ChildOutputSink, StderrRing, StderrTailConfig,
57 StderrTailSnapshot,
58 },
59 terminal_ring::{TerminalHistorySnapshot, TerminalRecord, TerminalRing, TerminalRingConfig},
60 Frame, FrameSink, Registry,
61};
62
63#[cfg(not(windows))]
64use crate::provenance::spawned_file_identity;
65
66#[path = "supervise_swap.rs"]
67mod swap;
68
69#[path = "supervise_activity.rs"]
70mod activity;
71
72#[cfg(unix)]
73#[path = "supervise/admission.rs"]
74mod admission;
75mod admission_exit;
76#[cfg(all(test, target_os = "linux"))]
77mod async_spawn_tests;
78#[cfg(all(test, windows))]
79mod carried_job_tests;
80mod config_delivery;
81mod content_binding;
82mod program_binding;
83mod registration;
84pub(crate) mod script;
85type NonceLaunches = HashMap<String, (String, std::sync::Weak<RegistrationSpawn>)>;
87pub(crate) use registration::NonceAttestation;
88#[cfg(all(test, unix))]
89pub(crate) use registration::RegistrationProof;
90pub(crate) use registration::RegistrationSpawn;
91pub(crate) use registration::REGISTRATION_CHANNEL_CLOSED;
92#[cfg(all(test, unix))]
93pub(crate) mod unix_channel_placement_tests;
94use program_binding::ProgramBindingPublication;
95mod config_exit_failure;
96#[cfg(all(test, any(target_os = "linux", windows)))]
97mod missing_path_tests;
98#[cfg(all(test, unix))]
99mod program_binding_tests;
100#[cfg(all(test, windows))]
101mod program_binding_windows_tests;
102#[cfg(all(test, windows))]
103mod windows_channel_tests;
104pub use config_exit_failure::ModuleStopReason;
105#[cfg(all(test, unix))]
106mod config_exit_hold_tests;
107pub(crate) mod containment;
108#[cfg(all(test, unix))]
109mod containment_tests;
110mod slow_retry;
111use slow_retry::{FailedRetry, FailureContext};
112mod host_health;
113use host_health::HostHealth;
114
115#[cfg(all(test, unix))]
116mod mark_readiness_tests;
117#[cfg(all(test, unix))]
118mod slow_retry_tests;
119
120pub const SUBC_ARG: &str = "--subc";
126
127const DEFAULT_MAX_RESTARTS: u32 = 3;
128const DEFAULT_BACKOFF: Duration = Duration::from_millis(100);
129const DEFAULT_MAX_BACKOFF: Duration = Duration::from_secs(30);
130const DEFAULT_RESTART_WINDOW: Duration = Duration::from_secs(600);
134pub const DEFAULT_DRAIN_TIMEOUT: Duration = Duration::from_secs(30);
145pub(crate) const DEFAULT_REGISTER_TIMEOUT: Duration = Duration::from_secs(120);
146const REGISTRY_RELEASE_TIMEOUT: Duration = Duration::from_secs(1);
147const REGISTRY_RELEASE_POLL: Duration = Duration::from_millis(10);
148const DRAIN_BUSY_REPROBE_INTERVAL: Duration = Duration::from_millis(250);
160const STDERR_PUMP_DRAIN_TIMEOUT: Duration = Duration::from_millis(250);
183pub const SPAWN_EVENT_RING_CAPACITY: usize = 4096;
185const SPAWN_SUBSCRIBER_BUFFER: usize = SPAWN_EVENT_RING_CAPACITY + 1;
186pub(crate) const SPAWN_SUBSCRIBER_LAGGED_CODE: &str = "spawn_subscriber_lagged";
191
192#[cfg(target_os = "macos")]
193struct PrivacyExec {
194 content_check: Option<tokio::task::JoinHandle<Result<(), crate::config_gate::GateError>>>,
195 gated: bool,
196 admission_binding: Option<ProgramBindingPublication>,
197 reader: tokio::io::unix::AsyncFd<std::io::PipeReader>,
198 deadline: tokio::time::Instant,
199 expected: Option<subc_os::FileIdentity>,
200 trampoline: Option<subc_os::FileIdentity>,
201 signature: Option<Arc<subc_os::privacy_identity::SuspendedSignature>>,
202 module_id: String,
203 #[cfg(test)]
204 confirmation_pause: Option<(String, Option<tokio::net::TcpStream>)>,
205 #[cfg(test)]
206 image_probe: Option<fn(u32) -> Option<subc_os::Observation>>,
207 #[cfg(test)]
208 image_read_delay: Option<Duration>,
209}
210
211#[cfg(all(test, target_os = "macos"))]
212pub(crate) fn test_privacy_trampoline() -> PathBuf {
213 let path = cortexkit_test_support::cargo_profile_bin("privacy-trampoline-fixture");
216 assert!(
220 path.exists(),
221 "privacy-trampoline-fixture not built at {}: run `cargo build -p subc-daemon \
222 --bins --features test-support` or `cargo test -p subc-daemon` first",
223 path.display()
224 );
225 path
226}
227
228#[cfg(target_os = "macos")]
229fn probe_privacy_trampoline(path: &std::path::Path) -> Result<(), String> {
230 use std::io::Read;
231 let mut probe = subc_os::child_spawn::spawn(
232 std::process::Command::new(path)
233 .args(["__disclaim-exec", "--probe-suspended"])
234 .stdin(Stdio::null())
235 .stdout(Stdio::piped())
236 .stderr(Stdio::piped()),
237 )
238 .map_err(|error| {
239 format!(
240 "privacy trampoline probe failed for {}: {error}",
241 path.display()
242 )
243 })?;
244 let deadline = std::time::Instant::now() + Duration::from_secs(5);
245 let status = loop {
246 match probe.try_wait() {
247 Ok(Some(status)) => break status,
248 Ok(None) if std::time::Instant::now() < deadline => {
249 std::thread::sleep(Duration::from_millis(5))
250 }
251 result => {
252 let _ = probe.kill();
253 let _ = probe.wait();
254 return Err(format!(
255 "privacy trampoline probe failed or timed out for {}: {result:?}",
256 path.display()
257 ));
258 }
259 }
260 };
261 let mut answer = String::new();
262 if let Some(stdout) = probe.stdout.take() {
263 let _ = stdout.take(256).read_to_string(&mut answer);
264 }
265 if status.success() && answer.trim() == subc_os::privacy_identity::SUSPENDED_TRAMPOLINE_PROBE {
266 return Ok(());
267 }
268 let mut diagnostic = String::new();
269 if let Some(stderr) = probe.stderr.take() {
270 let _ = stderr.take(1024).read_to_string(&mut diagnostic);
271 }
272 let cause = diagnostic
273 .trim()
274 .strip_prefix("ck-subc: own privacy identity refused: ")
275 .unwrap_or("binary does not implement the privacy trampoline protocol");
276 Err(format!(
277 "{cause}: probe of {} exited {status}",
278 path.display()
279 ))
280}
281
282#[cfg(target_os = "macos")]
283fn privacy_command(
284 spec: &ModuleSpec,
285 binding: &SpawnProgramBinding,
286 roster: &ChildRoster,
287 gated: bool,
288) -> Result<
289 (
290 Command,
291 Option<PrivacyExec>,
292 subc_os::privacy_identity::ExecAcknowledgement,
293 ),
294 SuperviseError,
295> {
296 let failure = |cause: String| {
297 warn!(module_id = %spec.module_id, %cause, "privacy identity trampoline refused module spawn");
298 SuperviseError::Spawn {
299 program: spec.program.clone(),
300 source: io::Error::other(cause),
301 cgroup_path: None,
302 }
303 };
304 let trampoline = roster.privacy_trampoline().map_err(failure)?;
305 let expected = Some(binding.identity);
306 let trampoline_image = subc_os::file_identity(&trampoline);
307 let signature = gated
308 .then(|| {
309 subc_os::privacy_identity::SuspendedSignature::capture(&binding._file, &binding.path)
310 .map(Arc::new)
311 })
312 .transpose()
313 .map_err(|error| failure(error.to_string()))?;
314 if expected == trampoline_image {
315 return Err(failure(
316 "privacy identity module executable is missing or is the trampoline itself".to_string(),
317 ));
318 }
319 let (reader, ack) = subc_os::privacy_identity::ExecAcknowledgement::pipe()
320 .map_err(|error| failure(error.to_string()))?;
321 let reader =
322 tokio::io::unix::AsyncFd::new(reader).map_err(|error| failure(error.to_string()))?;
323 let mut command = Command::new(&trampoline);
324 command
325 .arg(if gated {
326 "__disclaim-exec-suspended"
327 } else {
328 "__disclaim-exec"
329 })
330 .arg(ack.fd().to_string())
331 .arg(&binding.path);
332 Ok((
333 command,
334 Some(PrivacyExec {
335 gated,
336 admission_binding: None,
337 content_check: None,
338 reader,
339 deadline: tokio::time::Instant::now() + Duration::from_secs(5),
340 expected,
341 trampoline: trampoline_image,
342 signature,
343 module_id: spec.module_id.clone(),
344 #[cfg(test)]
345 image_probe: None,
346 #[cfg(test)]
347 image_read_delay: spec
348 .env
349 .iter()
350 .find(|(key, _)| key == "SUBC_TEST_PRIVACY_IMAGE_DELAY_MS")
351 .map(|(_, millis)| Duration::from_millis(millis.parse().unwrap())),
352 #[cfg(test)]
353 confirmation_pause: spec
354 .env
355 .iter()
356 .find(|(key, _)| key == "SUBC_TEST_BEFORE_EXEC_CONFIRM")
357 .map(|(_, address)| (address.clone(), None)),
358 }),
359 ack,
360 ))
361}
362
363#[cfg(unix)]
364type SpawnRegistrationChannel = std::os::unix::net::UnixStream;
365#[cfg(windows)]
366type SpawnRegistrationChannel = subc_os::registration_channel::RegistrationPipe;
367#[cfg(test)]
368type CapturedRegistrationChannels =
369 Arc<Mutex<Option<Vec<(Arc<RegistrationSpawn>, CapturedRegistrationEndpoint)>>>>;
370#[cfg(test)]
371struct CapturedRegistrationEndpoint {
372 _channel: SpawnRegistrationChannel,
373}
374#[cfg(test)]
375impl fmt::Debug for CapturedRegistrationEndpoint {
376 fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
377 f.write_str("CapturedRegistrationEndpoint")
378 }
379}
380
381struct SupervisedChild {
382 registration_spawn: Option<Arc<RegistrationSpawn>>,
383 #[cfg(any(unix, windows))]
385 registration_channel: Option<SpawnRegistrationChannel>,
386 live_program_binding: Option<Arc<content_binding::LiveProgramBinding>>,
387 registered_config_connection: Option<ConnectionId>,
388 admission_base: Option<subc_config::Digest>,
389 registration_declaration: Option<String>,
390 _program_binding: Option<ProgramBindingPublication>,
391 subtree_target: crate::subtree_resources::Target,
392 child: containment::ProcessChild,
393 exec: crate::activity::ExecOwner,
394 #[cfg(windows)]
395 nonce_delivery: Option<subc_os::LaunchNoncePipeDelivery>,
396 #[cfg(target_os = "macos")]
397 privacy_exec: Option<PrivacyExec>,
398 #[cfg(target_os = "macos")]
405 report_ready: Arc<OnceLock<()>>,
406 spawn_failure: Option<String>,
409 admission_kill_sent: bool,
412 protocol: ModuleProtocol,
417 #[cfg(target_os = "linux")]
424 module_id: String,
425 #[cfg(target_os = "linux")]
426 cgroup_placement: Option<subc_cgroup::Placement>,
427 #[cfg(windows)]
450 job: Option<Arc<Mutex<subc_jobobject::JobObject>>>,
451 stdout_pump: Option<JoinHandle<()>>,
452 stderr_pump: Option<StderrPump>,
453 stderr_ring: Arc<Mutex<StderrRing>>,
454 spawned_at: Instant,
457 config_delivery: Option<crate::config_gate::runtime::DeliverySpawn>,
458 registration_deadline: Option<Instant>,
461 registration_watch: Option<RegistrationWatch>,
462 spawned_at_ms: u64,
463 spawned_from: PathBuf,
464 launch_spec_digest: LaunchSpecDigest,
465 spawned_file_identity: Option<SpawnedFileIdentity>,
466 process_start_time: Option<u64>,
467 process_identity: Option<ProcessIdentity>,
468 pid: u32,
469 roster_guard: Option<crate::child_roster::RosterGuard>,
472}
473
474impl SupervisedChild {
475 fn retire_registration(&self) {
476 if let Some(spawn) = &self.registration_spawn {
481 spawn.retire();
482 } else if let Some(binding) = &self._program_binding {
483 binding.retire();
484 }
485 #[cfg(unix)]
486 if let Some(channel) = &self.registration_channel {
487 let _ = channel.shutdown(std::net::Shutdown::Both);
488 }
489 #[cfg(windows)]
490 if let Some(channel) = &self.registration_channel {
491 if let Ok(shutdown) = channel.shutdown_handle() {
492 shutdown.shutdown();
493 }
494 }
495 }
496 fn registered_since_spawn(&self) -> bool {
497 self.registration_watch
498 .as_ref()
499 .is_some_and(|watch| *watch.changes.borrow() != watch.baseline)
500 }
501
502 fn awaiting_registration(&self) -> bool {
503 let admission_failed = self.admission_kill_sent && self.spawn_failure.is_some();
507 !admission_failed && !self.registered_since_spawn()
508 }
509
510 fn id(&self) -> Option<u32> {
511 Some(self.pid)
512 }
513
514 fn process_identity(&self) -> Option<ProcessIdentity> {
515 self.process_identity
516 }
517
518 fn try_wait(&mut self) -> io::Result<Option<ExitStatus>> {
519 let result = self.child.try_wait();
520 if !matches!(result, Ok(None)) {
521 self.retire_registration();
522 #[cfg(any(unix, windows))]
523 self.registration_channel.take();
524 if let Some(binding) = &self._program_binding {
525 binding.retire();
526 }
527 }
528 result
529 }
530
531 async fn wait(&mut self) -> io::Result<ExitStatus> {
532 #[cfg(target_os = "macos")]
533 self.confirm_privacy_exec().await;
534 let result = self.child.wait().await;
542 self.retire_registration();
543 #[cfg(any(unix, windows))]
544 self.registration_channel.take();
545 if let Some(binding) = &self._program_binding {
548 binding.retire();
549 }
550 #[cfg(all(test, target_os = "linux"))]
551 if let Some(binding) = &self._program_binding {
552 binding.pause_after_wait_for_test().await;
553 }
554 #[cfg(target_os = "linux")]
555 if result.is_ok() && self.child.container().is_none() {
556 if let Some(placement) = self.cgroup_placement.as_ref() {
557 cleanup_reaped_cgroup(placement, &self.module_id).await;
561 self.cgroup_placement = None;
562 }
563 }
564 result
565 }
566
567 async fn finish_exit(&mut self, carry: bool) -> io::Result<()> {
568 self.retire_registration();
569 if let Some(binding) = self._program_binding.take() {
571 binding.retire();
572 }
573 if self.child.container().is_some() {
574 if !carry {
575 self.child.terminate_container().await;
576 }
577 } else {
578 #[cfg(target_os = "linux")]
579 if let Some(placement) = self.cgroup_placement.as_ref() {
580 cleanup_reaped_cgroup(placement, &self.module_id).await;
581 self.cgroup_placement = None;
582 }
583 }
584 self.child.reap().await?;
585 self.release_roster();
586 Ok(())
587 }
588
589 #[cfg(target_os = "macos")]
590 async fn confirm_privacy_exec(&mut self) {
591 if self
592 .privacy_exec
593 .as_ref()
594 .is_some_and(|pending| pending.content_check.is_some())
595 {
596 self.finish_privacy_content_check().await;
597 return;
598 }
599 let Some(pending) = &mut self.privacy_exec else {
600 return;
601 };
602 let result = tokio::time::timeout_at(pending.deadline, async {
603 let mut record = Vec::new();
604 loop {
605 let mut ready = pending.reader.readable().await?;
606 let read = ready.try_io(|reader| {
607 use std::io::Read;
608 let mut reader = reader.get_ref();
609 let mut buffer = [0u8; 256];
610 reader.read(&mut buffer).map(|count| (count, buffer))
611 });
612 match read {
613 Ok(Ok((0, _))) => return Ok::<_, io::Error>(record),
614 Ok(Ok((count, buffer))) => {
615 if record.len() + count > 1024 {
616 return Err(io::Error::other(
617 "privacy exec refusal record is too long",
618 ));
619 }
620 record.extend_from_slice(&buffer[..count]);
621 }
622 Ok(Err(error)) => return Err(error),
623 Err(_) => continue,
624 }
625 }
626 })
627 .await;
628 #[cfg(test)]
629 let result = if let Some((address, stream)) = &mut pending.confirmation_pause {
630 use tokio::io::{AsyncReadExt, AsyncWriteExt};
631 if stream.is_none() {
632 *stream = Some(
633 tokio::net::TcpStream::connect(address.as_str())
634 .await
635 .unwrap(),
636 );
637 stream.as_mut().unwrap().write_all(b"R").await.unwrap();
638 stream
639 .as_mut()
640 .unwrap()
641 .write_all(&self.pid.to_ne_bytes())
642 .await
643 .unwrap();
644 }
645 let mut reply = [0];
646 stream
647 .as_mut()
648 .unwrap()
649 .read_exact(&mut reply)
650 .await
651 .unwrap();
652 pending.confirmation_pause = None;
653 if &reply == b"F" {
654 Ok(Err(io::Error::other("injected late exec refusal")))
655 } else {
656 assert_eq!(&reply, b"X");
657 result
658 }
659 } else {
660 result
661 };
662 let waited = if matches!(&result, Ok(Ok(record)) if record.is_empty()) {
667 Some(self.try_wait())
668 } else {
669 None
670 };
671 let pending = self.privacy_exec.as_ref().expect("pending exec");
672 let cause = match result {
673 Err(_) => Some("privacy identity trampoline did not exec within 5s".to_string()),
674 Ok(Err(error)) => Some(format!(
675 "privacy identity exec acknowledgement failed: {error}"
676 )),
677 Ok(Ok(record)) if !record.is_empty() => Some(
678 std::str::from_utf8(&record)
679 .ok()
680 .and_then(|record| {
681 record
682 .trim()
683 .strip_prefix(subc_os::privacy_identity::EXEC_REFUSAL_TAG)
684 })
685 .filter(|cause| !cause.is_empty())
686 .unwrap_or("invalid privacy exec refusal record")
687 .to_string(),
688 ),
689 Ok(Ok(_)) if !pending.gated => match waited.expect("exec EOF checked child") {
690 Ok(Some(_)) => None,
691 Err(error) => Some(format!("privacy identity trampoline wait failed: {error}")),
692 Ok(None) => {
693 #[cfg(test)]
694 if let Some(delay) = pending.image_read_delay {
695 tokio::time::sleep(delay).await;
696 }
697 let image = observe_spawned_image(self.pid);
698 #[cfg(test)]
699 let image = pending.image_probe.map_or(image, |probe| probe(self.pid));
700 if let Some(image) = image.filter(|image| {
704 image.executable.is_some() && image.executable != pending.trampoline
705 }) {
706 match self
707 .roster_guard
708 .as_ref()
709 .map(|guard| guard.confirm_executable(image))
710 .transpose()
711 {
712 Err(error) => Some(format!(
713 "could not durably record confirmed module image: {error}"
714 )),
715 Ok(_) => {
716 let _ = self.report_ready.set(());
717 None
718 }
719 }
720 } else if image.is_none()
721 || image.is_some_and(|image| image.executable.is_none())
722 {
723 match tokio::time::timeout_at(pending.deadline, self.child.wait()).await {
726 Ok(result) => {
727 self.retire_registration();
728 self.registration_channel.take();
729 if let Some(binding) = &self._program_binding {
730 binding.retire();
731 }
732 result.err().map(|error| format!("privacy identity trampoline wait failed: {error}"))
733 }
734 Err(_) => Some("privacy identity module executable remained unreadable until the 5s exec deadline".to_string()),
735 }
736 } else {
737 Some("privacy identity trampoline did not exec the module".to_string())
738 }
739 }
740 },
741 Ok(Ok(_)) => match waited.expect("exec EOF checked child") {
742 Ok(Some(_)) => {
743 Some("privacy identity target exited before suspended admission".to_string())
744 }
745 Err(error) => Some(format!("privacy identity trampoline wait failed: {error}")),
746 Ok(None) => {
747 let stopped = tokio::time::timeout_at(pending.deadline, async {
750 loop {
751 if subc_os::privacy_identity::SuspendedSignature::is_suspended(
752 self.pid,
753 )? {
754 return Ok::<_, io::Error>(());
755 }
756 tokio::time::sleep(Duration::from_millis(1)).await;
757 }
758 })
759 .await;
760 match stopped {
761 Err(_) => {
762 Some("privacy identity target was not suspended within 5s".to_string())
763 }
764 Ok(Err(error)) => Some(format!(
765 "privacy identity suspended target unavailable: {error}"
766 )),
767 Ok(Ok(())) => {
768 let image = observe_spawned_image(self.pid);
769 if image.is_some_and(|image| {
770 image.executable.is_some()
771 && image.executable != pending.trampoline
772 && image.executable == pending.expected
773 }) {
774 None
775 } else {
776 Some("privacy identity trampoline executable mismatch: expected the suspended module program".to_string())
777 }
778 }
779 }
780 }
781 },
782 };
783 #[cfg(test)]
784 if let Some(binding) = &self._program_binding {
785 binding.pause_after_wait_for_test().await;
786 }
787 if cause.is_none() && pending.gated {
788 let publication = pending.admission_binding.clone();
789 let live = self.live_program_binding.clone();
790 let signature = Arc::clone(pending.signature.as_ref().expect("gated signature"));
791 let pid = self.pid;
792 let gated = pending.gated;
793 self.privacy_exec
794 .as_mut()
795 .expect("pending exec")
796 .content_check = Some(tokio::task::spawn_blocking(move || {
797 signature.verify(pid).map_err(|error| {
798 crate::config_gate::GateError::new("program_refused", error)
799 })?;
800 if !gated {
801 return Ok(());
802 }
803 match (publication, live) {
804 (Some(publication), _) => publication.verify(),
805 (None, Some(live)) => live.check(),
806 _ => Err(crate::config_gate::GateError::new(
807 "config_delivery_deferred",
808 "live program witness is unavailable",
809 )),
810 }
811 }));
812 self.finish_privacy_content_check().await;
813 } else {
814 self.finish_privacy_exec(cause);
815 }
816 }
817
818 #[cfg(target_os = "macos")]
819 async fn finish_privacy_content_check(&mut self) {
820 let result = self
821 .privacy_exec
822 .as_mut()
823 .expect("pending exec")
824 .content_check
825 .as_mut()
826 .expect("pending content check")
827 .await;
828 let cause = match result {
829 Ok(Ok(())) => None,
830 Ok(Err(error)) => Some(format!("{}: {}", error.code, error.detail)),
831 Err(error) => Some(format!(
832 "config_delivery_deferred: content check stopped: {error}"
833 )),
834 };
835 self.finish_privacy_exec(cause);
836 }
837
838 #[cfg(target_os = "macos")]
839 fn finish_privacy_exec(&mut self, mut cause: Option<String>) {
840 let mut pending = self.privacy_exec.take().expect("pending exec");
841 if cause.is_none() && pending.gated {
842 let admission = (|| -> io::Result<()> {
843 let image = observe_spawned_image(self.pid)
844 .filter(|image| image.executable == pending.expected)
845 .ok_or_else(|| io::Error::other("suspended executable mismatch"))?;
846 if let Some(guard) = &self.roster_guard {
847 guard.confirm_executable(image)?;
848 }
849 pending
850 .signature
851 .as_ref()
852 .expect("gated signature")
853 .resume(self.pid)?;
854 let _ = self.report_ready.set(());
855 info!(module_id = %pending.module_id, pid = self.pid,
856 "verified suspended module resumed with own privacy identity");
857 Ok(())
858 })();
859 if let Err(error) = admission {
860 cause = Some(format!("program_refused: {error}"));
861 }
862 }
863 if cause.is_some() {
864 if let Some(spawn) = &self.registration_spawn {
865 spawn.refuse();
866 }
867 if let Some(binding) = &mut pending.admission_binding {
868 binding.reject();
869 }
870 } else if let Some(spawn) = &self.registration_spawn {
871 spawn.ready();
872 }
873 self.exec.0.finish(if cause.is_some() {
876 crate::activity::ExecState::Refused
877 } else {
878 crate::activity::ExecState::CommittedExec
879 });
880 if let Some(cause) = cause {
881 warn!(module_id = %pending.module_id, pid = self.pid, %cause, "privacy identity trampoline refused module spawn");
882 self.spawn_failure = Some(
883 if pending.gated
884 && (cause.contains("executable mismatch")
885 || cause.starts_with("program_refused:"))
886 {
887 format!("program_refused: {cause}")
888 } else if pending.gated {
889 format!("config_delivery_deferred: {cause}")
890 } else {
891 cause
892 },
893 );
894 let _ = self.start_admission_kill();
897 }
898 }
899
900 fn release_roster(&mut self) {
904 self.roster_guard = None;
905 }
906
907 fn start_kill(&mut self) -> io::Result<()> {
920 self.retire_registration();
921 if let Some(container) = self.child.container() {
922 container.hard_kill();
923 return self.kill_leader();
924 }
925 #[cfg(windows)]
926 if let Some(job) = &self.job {
927 if let Err(error) = job.lock().unwrap_or_else(|p| p.into_inner()).terminate() {
928 debug!(
929 error = %error,
930 "job termination failed; the direct-child kill still owns the outcome"
931 );
932 }
933 }
934 #[cfg(target_os = "linux")]
935 kill_module_cgroup(self.cgroup_placement.as_ref(), &self.module_id);
936 self.child.start_kill()
937 }
938
939 fn kill_leader(&mut self) -> io::Result<()> {
940 self.retire_registration();
941 self.child.start_kill()
942 }
943
944 fn start_admission_kill(&mut self) -> io::Result<()> {
945 self.retire_registration();
946 #[cfg(windows)]
947 let result = {
948 let result = self
951 .spawned_file_identity
952 .as_ref()
953 .and_then(|identity| identity.windows_image.as_ref())
954 .ok_or_else(|| io::Error::other("admission child handle unavailable"))
955 .and_then(|image| {
956 image.process().terminate_with_exit_code(
957 admission_exit::WINDOWS_ADMISSION_KILL_EXIT_CODE as u32,
958 )
959 });
960 if let Some(container) = self.child.container() {
961 container.hard_kill();
962 } else if let Some(job) = &self.job {
963 if let Err(error) = job.lock().unwrap_or_else(|p| p.into_inner()).terminate() {
964 debug!(%error, "job termination failed after admission leader termination");
965 }
966 }
967 result
968 };
969 #[cfg(not(windows))]
970 let result = self.start_kill().map(|()| true);
971 #[cfg(unix)]
972 let result = {
973 let group_sent = self.child.container().is_none()
976 && rustix::process::Pid::from_raw(self.pid as i32).is_some_and(|group| {
977 rustix::process::kill_process_group(group, rustix::process::Signal::KILL)
978 .is_ok()
979 });
980 if group_sent {
981 Ok(true)
982 } else {
983 result
984 }
985 };
986 self.admission_kill_sent |= result?;
987 Ok(())
988 }
989
990 async fn drain_stderr(&mut self, module_id: &str) {
991 if let Some(pump) = self.stdout_pump.take() {
992 settle_stdout_pump(module_id, pump, STDERR_PUMP_DRAIN_TIMEOUT).await;
993 }
994
995 let Some(pump) = self.stderr_pump.take() else {
996 return;
997 };
998 settle_stderr_pump(
999 module_id,
1000 &self.stderr_ring,
1001 pump,
1002 STDERR_PUMP_DRAIN_TIMEOUT,
1003 )
1004 .await;
1005 }
1006}
1007
1008async fn settle_stdout_pump(module_id: &str, mut pump: JoinHandle<()>, bound: Duration) {
1012 match timeout(bound, &mut pump).await {
1013 Ok(Ok(())) => {}
1014 Ok(Err(error)) => {
1015 warn!(module_id, error = %error, "stdout pump ended unexpectedly");
1016 }
1017 Err(_) => {
1018 drop(pump);
1019 warn!(
1020 module_id,
1021 waited = ?bound,
1022 "stdout pipe of the exited process has not drained; its reader keeps running without delaying the restart"
1023 );
1024 }
1025 }
1026}
1027
1028struct StderrPump {
1031 task: JoinHandle<()>,
1032 generation: u64,
1033}
1034
1035async fn settle_stderr_pump(
1041 module_id: &str,
1042 ring: &Arc<Mutex<StderrRing>>,
1043 pump: StderrPump,
1044 bound: Duration,
1045) {
1046 let lock = || ring.lock().unwrap_or_else(|poisoned| poisoned.into_inner());
1047 let StderrPump {
1048 mut task,
1049 generation,
1050 } = pump;
1051 lock().retire_pump(generation);
1052 match timeout(bound, &mut task).await {
1053 Ok(Ok(())) => {}
1054 Ok(Err(err)) => {
1055 let mut ring = lock();
1056 ring.mark_incomplete_from(generation, format!("stderr pump ended unexpectedly: {err}"));
1057 ring.finish_pump(generation);
1058 warn!(module_id, error = %err, "stderr pump ended before clean EOF");
1059 }
1060 Err(_) => {
1061 drop(task);
1063 lock().mark_pump_late(
1064 generation,
1065 format!(
1066 "stderr of the exited process had not reached EOF {bound:?} after it was \
1067 retired (a descendant may still hold the pipe open); lines it still \
1068 writes are kept in that process's section"
1069 ),
1070 );
1071 warn!(
1072 module_id,
1073 waited = ?bound,
1074 "stderr pipe of the exited process is still open; its reader keeps running without delaying the restart"
1075 );
1076 }
1077 }
1078}
1079
1080fn registration_release_events() -> &'static watch::Sender<u64> {
1081 static EVENTS: OnceLock<watch::Sender<u64>> = OnceLock::new();
1082 EVENTS.get_or_init(|| {
1083 let (sender, _receiver) = watch::channel(0);
1084 sender
1085 })
1086}
1087
1088pub(crate) fn notify_registration_release() {
1089 let events = registration_release_events();
1090 let next_generation = (*events.borrow()).wrapping_add(1);
1091 events.send_replace(next_generation);
1092}
1093
1094#[derive(Debug, Clone, PartialEq, Eq)]
1096pub struct ModuleSpec {
1097 pub module_id: String,
1098 pub program: PathBuf,
1099 pub script: Option<PathBuf>,
1101 pub args: Vec<String>,
1102 pub env: Vec<(String, String)>,
1103 pub reserved: bool,
1108 pub reserved_prefixes: Vec<String>,
1113 pub protocol: ModuleProtocol,
1132 pub overlap: ModuleOverlap,
1137 pub carry_containment: bool,
1141}
1142
1143#[derive(Clone, PartialEq, Eq)]
1146pub(crate) struct LaunchSpecDigest([u8; 32]);
1147
1148impl fmt::Debug for LaunchSpecDigest {
1149 fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
1150 f.write_str("LaunchSpecDigest([redacted])")
1151 }
1152}
1153
1154impl LaunchSpecDigest {
1155 pub(crate) fn from_spec(spec: &ModuleSpec) -> Self {
1156 fn field(hash: &mut Sha256, bytes: &[u8]) {
1157 hash.update((bytes.len() as u64).to_le_bytes());
1158 hash.update(bytes);
1159 }
1160 let mut hash = Sha256::new();
1161 if let Some(script) = &spec.script {
1162 field(&mut hash, b"bun-script");
1163 field(&mut hash, script.as_os_str().as_encoded_bytes());
1164 }
1165 if let Some(program) = spec.program.to_str() {
1166 field(&mut hash, b"utf8-program");
1167 field(&mut hash, program.as_bytes());
1168 } else {
1169 #[cfg(unix)]
1172 {
1173 use std::os::unix::ffi::OsStrExt;
1174 field(&mut hash, b"unix-program");
1175 field(&mut hash, spec.program.as_os_str().as_bytes());
1176 }
1177 #[cfg(windows)]
1178 {
1179 use std::os::windows::ffi::OsStrExt;
1180 field(&mut hash, b"windows-program");
1181 let bytes = spec
1182 .program
1183 .as_os_str()
1184 .encode_wide()
1185 .flat_map(u16::to_le_bytes)
1186 .collect::<Vec<_>>();
1187 field(&mut hash, &bytes);
1188 }
1189 #[cfg(not(any(unix, windows)))]
1190 field(&mut hash, spec.program.to_string_lossy().as_bytes());
1191 }
1192 hash.update((spec.args.len() as u64).to_le_bytes());
1193 for arg in &spec.args {
1194 field(&mut hash, arg.as_bytes());
1195 }
1196 let mut env = spec.env.iter().collect::<Vec<_>>();
1197 env.sort_unstable();
1198 hash.update((env.len() as u64).to_le_bytes());
1199 for (key, value) in env {
1200 field(&mut hash, key.as_bytes());
1201 field(&mut hash, value.as_bytes());
1202 }
1203 field(
1204 &mut hash,
1205 match spec.protocol {
1206 ModuleProtocol::Subc => b"subc",
1207 ModuleProtocol::None => b"none",
1208 },
1209 );
1210 Self(hash.finalize().into())
1211 }
1212}
1213
1214#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)]
1221pub enum ModuleOverlap {
1222 #[default]
1224 Exclusive,
1225 Safe,
1237}
1238
1239impl ModuleOverlap {
1240 pub fn as_str(self) -> &'static str {
1241 match self {
1242 Self::Exclusive => "exclusive",
1243 Self::Safe => "safe",
1244 }
1245 }
1246}
1247
1248pub const SUBC_SPAWN_ROLE_ENV: &str = "SUBC_SPAWN_ROLE";
1258pub const SPAWN_ROLE_SWAP_CANDIDATE: &str = "swap_candidate";
1260pub const DEFAULT_SWAP_READY_TIMEOUT: Duration = Duration::from_secs(100);
1265
1266#[derive(Debug, Clone, Copy, PartialEq, Eq)]
1284pub struct RestartPolicy {
1285 pub max_restarts: u32,
1286 pub backoff: Duration,
1289 pub max_backoff: Duration,
1291 pub window: Duration,
1295}
1296
1297impl RestartPolicy {
1298 pub fn new(max_restarts: u32, backoff: Duration) -> Self {
1302 Self {
1303 max_restarts,
1304 backoff,
1305 max_backoff: DEFAULT_MAX_BACKOFF,
1306 window: DEFAULT_RESTART_WINDOW,
1307 }
1308 }
1309
1310 pub fn with_max_backoff(mut self, max_backoff: Duration) -> Self {
1311 self.max_backoff = max_backoff;
1312 self
1313 }
1314
1315 pub fn with_window(mut self, window: Duration) -> Self {
1316 self.window = window;
1317 self
1318 }
1319
1320 fn delay_for_restart(&self, restart_in_window: u32) -> Duration {
1325 if self.backoff.is_zero() || self.max_backoff.is_zero() {
1326 return Duration::ZERO;
1327 }
1328
1329 let mut delay = self.backoff;
1330 for _ in 0..restart_in_window {
1331 if delay >= self.max_backoff {
1332 return self.max_backoff;
1333 }
1334 delay = delay
1335 .checked_mul(10)
1336 .unwrap_or(self.max_backoff)
1337 .min(self.max_backoff);
1338 }
1339 delay.min(self.max_backoff)
1340 }
1341
1342 fn budget_exhausted_detail(&self) -> String {
1347 format!(
1348 "crash budget exhausted: max_restarts={} within window_secs={}",
1349 self.max_restarts,
1350 self.window.as_secs()
1351 )
1352 }
1353}
1354
1355impl Default for RestartPolicy {
1356 fn default() -> Self {
1357 Self {
1358 max_restarts: DEFAULT_MAX_RESTARTS,
1359 backoff: DEFAULT_BACKOFF,
1360 max_backoff: DEFAULT_MAX_BACKOFF,
1361 window: DEFAULT_RESTART_WINDOW,
1362 }
1363 }
1364}
1365
1366#[derive(Debug, Clone, Copy, PartialEq, Eq)]
1367struct CrashRestartSchedule {
1368 restart_in_window: u32,
1369 delay: Duration,
1370}
1371
1372fn daemon_will_restart(
1379 state: &mut SupervisorSnapshot,
1380 policy: &RestartPolicy,
1381 now: Instant,
1382) -> bool {
1383 state.enabled && state.crash_restarts_in_window(policy.window, now) < policy.max_restarts
1384}
1385
1386const DEFAULT_HEALTH_CADENCE: Duration = Duration::from_secs(30);
1387const DEFAULT_HEALTH_DEADLINE: Duration = Duration::from_secs(5);
1388const DEFAULT_HEALTH_FAILURE_THRESHOLD: u32 = 3;
1389const MAX_HEALTH_METRICS_BYTES: usize = 16 * 1024;
1390
1391#[derive(Debug, Clone, Copy, PartialEq, Eq)]
1392pub enum HealthAction {
1393 Report,
1394 Restart,
1395 Alert,
1396}
1397
1398impl fmt::Display for HealthAction {
1399 fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
1400 f.write_str(match self {
1401 Self::Report => "report",
1402 Self::Restart => "restart",
1403 Self::Alert => "alert",
1404 })
1405 }
1406}
1407
1408#[derive(Debug, Clone, PartialEq, Eq)]
1409pub struct HealthConfig {
1410 pub http: Option<String>,
1414 pub cadence: Duration,
1415 pub deadline: Duration,
1416 pub failure_threshold: u32,
1417 pub on_degraded: HealthAction,
1418 pub on_failing: HealthAction,
1419 pub critical: bool,
1420}
1421
1422impl Default for HealthConfig {
1423 fn default() -> Self {
1424 Self {
1425 http: None,
1426 cadence: DEFAULT_HEALTH_CADENCE,
1427 deadline: DEFAULT_HEALTH_DEADLINE,
1428 failure_threshold: DEFAULT_HEALTH_FAILURE_THRESHOLD,
1429 on_degraded: HealthAction::Report,
1430 on_failing: HealthAction::Report,
1431 critical: false,
1432 }
1433 }
1434}
1435
1436#[derive(Debug, Clone, PartialEq)]
1454pub struct ModuleHealthStatus {
1455 pub status: SupervisorHealthStatus,
1456 pub last_probe_ms: Option<u64>,
1457 pub detail: Option<String>,
1458 pub metrics: Option<Value>,
1459 pub consecutive_failures: u32,
1460 pub late_answer_count: u64,
1463 pub last_late_answer_latency_ms: Option<u64>,
1465 pub last_action: Option<String>,
1466 pub last_action_ms: Option<u64>,
1470}
1471
1472impl Default for ModuleHealthStatus {
1473 fn default() -> Self {
1474 Self {
1475 status: SupervisorHealthStatus::Unknown,
1476 last_probe_ms: None,
1477 detail: None,
1478 metrics: None,
1479 consecutive_failures: 0,
1480 late_answer_count: 0,
1481 last_late_answer_latency_ms: None,
1482 last_action: None,
1483 last_action_ms: None,
1484 }
1485 }
1486}
1487
1488#[derive(Debug, Clone, Copy, PartialEq, Eq)]
1490pub enum ModuleState {
1491 Starting,
1492 Running,
1493 Unresponsive,
1494 Restarting,
1495 Draining,
1496 Stopped,
1497 Failed,
1498 Disabled,
1499}
1500
1501impl fmt::Display for ModuleState {
1502 fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
1503 f.write_str(match self {
1504 Self::Starting => "starting",
1505 Self::Running => "running",
1506 Self::Unresponsive => "unresponsive",
1507 Self::Restarting => "restarting",
1508 Self::Draining => "draining",
1509 Self::Stopped => "stopped",
1510 Self::Failed => "failed",
1511 Self::Disabled => "disabled",
1512 })
1513 }
1514}
1515
1516#[derive(Debug, Clone, Copy, PartialEq, Eq)]
1518pub enum ExitKind {
1519 Clean,
1520 Crash,
1521 DeliberateSeverance,
1522}
1523
1524impl From<ExitKind> for TerminalExitKind {
1525 fn from(kind: ExitKind) -> Self {
1526 match kind {
1527 ExitKind::Clean => Self::Clean,
1528 ExitKind::Crash => Self::Crash,
1529 ExitKind::DeliberateSeverance => Self::DeliberateSeverance,
1530 }
1531 }
1532}
1533
1534#[derive(Debug, Clone, Copy, PartialEq, Eq)]
1537pub(crate) struct ProcessIdentity {
1538 pub(crate) pid: u32,
1539 pub(crate) start_time: u64,
1540}
1541
1542#[derive(Debug, Clone, PartialEq, Eq)]
1544pub struct ExitReport {
1545 pub kind: ExitKind,
1546 pub code: Option<i32>,
1547 pub signal: Option<i32>,
1548 pub at_ms: u64,
1549}
1550
1551#[derive(Debug, Clone, PartialEq)]
1554pub struct ModuleStatus {
1555 pub(crate) admission_refused: Option<String>,
1556 pub(crate) legacy_registration_spawn: bool,
1557 pub(crate) registration_declaration: Option<String>,
1558 pub module_id: String,
1559 pub stop_reason: Option<ModuleStopReason>,
1562 pub waiting_for_previous_pid: Option<u32>,
1564 pub waiting_for_previous_detail: Option<String>,
1566 pub launch_nonce_source: Option<String>,
1571 pub state: ModuleState,
1572 pub enabled: bool,
1573 pub process_alive: bool,
1574 pub registration_active: bool,
1575 pub protocol: ModuleProtocol,
1580 pub live: bool,
1591 pub restart_count: u32,
1595 pub lifetime_restarts: u32,
1599 pub spawn_generation: u64,
1600 pub swap_outcome: Option<SwapOutcome>,
1602 pub max_restarts: u32,
1607 pub restart_window: Duration,
1611 pub drain_timeout: Duration,
1615 pub restart_backoff: Duration,
1616 pub restart_max_backoff: Duration,
1617 pub pid: Option<u32>,
1622 pub spawned_at_ms: Option<u64>,
1623 pub spawned_from: Option<PathBuf>,
1624 pub(crate) launch_spec_digest: Option<LaunchSpecDigest>,
1625 pub process_start_time: Option<u64>,
1626 pub last_exit: Option<ExitReport>,
1627 pub health: ModuleHealthStatus,
1628}
1629
1630#[cfg(test)]
1631#[derive(Debug, Clone, Copy, PartialEq)]
1632struct ActorSelectCheckpoint {
1633 generation: u64,
1634 turn: u64,
1635 registered_connection: Option<ConnectionId>,
1636 next_probe_at: Option<Instant>,
1637 wake_after: Option<Duration>,
1638}
1639
1640#[derive(Debug, Clone, PartialEq)]
1641struct SupervisorSnapshot {
1642 admission_refused: Option<String>,
1643 config_hold_parked: bool,
1645 config_hold_daemon_initiated: bool,
1648 config_hold_approval: Option<crate::config_gate::runtime::ApprovalIdentity>,
1650 approved_start_failure: Option<(crate::config_gate::runtime::ApprovalIdentity, String)>,
1652 legacy_registration_spawn: bool,
1653 registration_declaration: Option<String>,
1654 stop_reason: Option<ModuleStopReason>,
1655 config_exit_failed_approval: Option<crate::config_gate::runtime::ApprovalIdentity>,
1656 subtree_target: crate::subtree_resources::Target,
1657 failed_retry: FailedRetry,
1658 exec_outcome: Option<Arc<crate::activity::ExecOutcome>>,
1659 #[cfg(windows)]
1660 pipe_consumed: Option<Arc<OnceLock<()>>>,
1661 #[cfg(test)]
1663 actor_turns: u64,
1664 #[cfg(test)]
1667 actor_select: Option<ActorSelectCheckpoint>,
1668 state: ModuleState,
1669 enabled: bool,
1670 process_alive: bool,
1671 spawned_protocol: Option<ModuleProtocol>,
1672 spawn_failure: Option<String>,
1673 waiting_for_previous_pid: Option<u32>,
1674 waiting_for_previous_detail: Option<String>,
1675 crash_restarts: VecDeque<Instant>,
1681 lifetime_restarts: u32,
1682 spawn_generation: u64,
1691 swap_outcome: Option<SwapOutcome>,
1692 pid: Option<u32>,
1693 #[cfg(target_os = "macos")]
1694 report_ready: Option<Arc<OnceLock<()>>>,
1695 reaped_pid: Option<u32>,
1697 respawn_pending: bool,
1699 coalesced_restart_pending: bool,
1701 spawned_at_ms: Option<u64>,
1702 spawned_from: Option<PathBuf>,
1703 launch_spec_digest: Option<LaunchSpecDigest>,
1704 spawned_file_identity: Option<SpawnedFileIdentity>,
1705 process_start_time: Option<u64>,
1706 deliberate_severance: Option<ProcessIdentity>,
1707 last_exit: Option<ExitReport>,
1708 drain_disposition_detail: Option<String>,
1710 health: ModuleHealthStatus,
1711 last_probe_evidence: Option<HealthProbeEvidence>,
1712 in_alternate_slot: bool,
1717 draining_to_replace: bool,
1724 configuration_updated_since_spawn: bool,
1730}
1731
1732impl SupervisorSnapshot {
1733 fn reported_pid(&self) -> Option<u32> {
1738 #[cfg(target_os = "macos")]
1739 if self
1740 .report_ready
1741 .as_ref()
1742 .is_some_and(|ready| ready.get().is_none())
1743 {
1744 return None;
1745 }
1746 self.pid
1747 }
1748
1749 fn starting() -> Self {
1750 Self::new(ModuleState::Starting, true)
1751 }
1752
1753 fn disabled() -> Self {
1754 Self::new(ModuleState::Disabled, false)
1755 }
1756
1757 fn crash_restarts_in_window(&mut self, window: Duration, now: Instant) -> u32 {
1761 while let Some(oldest) = self.crash_restarts.front() {
1762 if now.duration_since(*oldest) > window {
1763 self.crash_restarts.pop_front();
1764 } else {
1765 break;
1766 }
1767 }
1768 u32::try_from(self.crash_restarts.len()).unwrap_or(u32::MAX)
1769 }
1770
1771 fn record_crash_restart(&mut self, policy: &RestartPolicy, now: Instant) {
1777 self.crash_restarts.push_back(now);
1778 while self.crash_restarts.len() > policy.max_restarts as usize {
1779 self.crash_restarts.pop_front();
1780 }
1781 self.lifetime_restarts += 1;
1782 }
1783
1784 fn next_crash_restart(
1788 &mut self,
1789 policy: &RestartPolicy,
1790 now: Instant,
1791 ) -> Option<CrashRestartSchedule> {
1792 let restart_in_window = self.crash_restarts_in_window(policy.window, now);
1793 if restart_in_window >= policy.max_restarts {
1794 return None;
1795 }
1796 self.record_crash_restart(policy, now);
1797 Some(CrashRestartSchedule {
1798 restart_in_window,
1799 delay: policy.delay_for_restart(restart_in_window),
1800 })
1801 }
1802
1803 fn clear_crash_restarts(&mut self) {
1808 self.crash_restarts.clear();
1809 }
1810
1811 fn new(state: ModuleState, enabled: bool) -> Self {
1812 Self {
1813 admission_refused: None,
1814 config_hold_parked: false,
1815 config_hold_daemon_initiated: false,
1816 config_hold_approval: None,
1817 approved_start_failure: None,
1818 legacy_registration_spawn: false,
1819 registration_declaration: None,
1820 stop_reason: None,
1821 config_exit_failed_approval: None,
1822 subtree_target: crate::subtree_resources::Target::Unavailable("not_running"),
1823 failed_retry: FailedRetry::default(),
1824 exec_outcome: None,
1825 #[cfg(test)]
1826 actor_turns: 0,
1827 #[cfg(test)]
1828 actor_select: None,
1829 state,
1830 enabled,
1831 process_alive: false,
1832 #[cfg(windows)]
1833 pipe_consumed: None,
1834 spawned_protocol: None,
1835 spawn_failure: None,
1836 waiting_for_previous_pid: None,
1837 waiting_for_previous_detail: None,
1838 crash_restarts: VecDeque::new(),
1839 lifetime_restarts: 0,
1840 spawn_generation: 0,
1841 swap_outcome: None,
1842 pid: None,
1843 #[cfg(target_os = "macos")]
1844 report_ready: None,
1845 reaped_pid: None,
1846 respawn_pending: false,
1847 coalesced_restart_pending: false,
1848 spawned_at_ms: None,
1849 spawned_from: None,
1850 launch_spec_digest: None,
1851 spawned_file_identity: None,
1852 process_start_time: None,
1853 deliberate_severance: None,
1854 last_exit: None,
1855 drain_disposition_detail: None,
1856 health: ModuleHealthStatus::default(),
1857 last_probe_evidence: None,
1858 in_alternate_slot: false,
1859 draining_to_replace: false,
1860 configuration_updated_since_spawn: false,
1861 }
1862 }
1863}
1864
1865type SharedSnapshot = Arc<Mutex<SupervisorSnapshot>>;
1866
1867type SpawnSubscriberKey = (ConnectionId, u64);
1868
1869#[derive(Debug)]
1870struct SpawnSubscriber {
1871 version: u8,
1872 frames: mpsc::Sender<Frame>,
1873 lagged: Option<oneshot::Sender<SpawnCursor>>,
1877}
1878
1879#[derive(Debug)]
1880struct SpawnEventState {
1881 daemon_incarnation: String,
1882 seq: u64,
1883 capacity: usize,
1884 live: HashMap<String, LiveSpawn>,
1885 generations: HashMap<String, u64>,
1886 events: VecDeque<SpawnEvent>,
1887 subscribers: HashMap<SpawnSubscriberKey, SpawnSubscriber>,
1888 process_changes: HashMap<String, tokio::sync::watch::Sender<u64>>,
1889}
1890
1891impl Default for SpawnEventState {
1892 fn default() -> Self {
1893 Self {
1894 daemon_incarnation: "unconfigured".to_string(),
1895 seq: 0,
1896 capacity: SPAWN_EVENT_RING_CAPACITY,
1897 live: HashMap::new(),
1898 generations: HashMap::new(),
1899 events: VecDeque::new(),
1900 subscribers: HashMap::new(),
1901 process_changes: HashMap::new(),
1902 }
1903 }
1904}
1905
1906#[derive(Debug, Clone, Default)]
1907struct SpawnEventFeed(
1908 Arc<Mutex<SpawnEventState>>,
1909 #[cfg(test)] Arc<Mutex<Option<std::net::TcpStream>>>,
1910);
1911
1912#[derive(Debug, Clone, PartialEq, Eq)]
1913pub(crate) enum SpawnSubscribeRefusal {
1914 ForeignIncarnation { current: String },
1915 TooOld { oldest: SpawnCursor },
1916 Frame(String),
1917}
1918
1919impl SpawnEventFeed {
1920 fn configure_incarnation(&self, daemon_incarnation: String) {
1921 let mut state = self.0.lock().unwrap_or_else(|p| p.into_inner());
1922 state.daemon_incarnation = daemon_incarnation;
1923 state.seq = 0;
1924 state.live.clear();
1925 state.generations.clear();
1926 state.events.clear();
1927 state.subscribers.clear();
1928 for changed in state.process_changes.values() {
1929 changed.send_replace(0);
1930 }
1931 }
1932
1933 fn cursor(state: &SpawnEventState) -> SpawnCursor {
1934 SpawnCursor {
1935 daemon_incarnation: state.daemon_incarnation.clone(),
1936 seq: state.seq,
1937 }
1938 }
1939
1940 fn snapshot(&self) -> SpawnSnapshot {
1941 let state = self.0.lock().unwrap_or_else(|p| p.into_inner());
1942 let mut live = state.live.values().cloned().collect::<Vec<_>>();
1943 live.sort_by(|left, right| left.module_id.cmp(&right.module_id));
1944 SpawnSnapshot {
1945 cursor: Self::cursor(&state),
1946 ring_bound: state.capacity as u64,
1947 live,
1948 }
1949 }
1950
1951 fn allocate_generation_locked(state: &mut SpawnEventState, module_id: &str) -> u64 {
1952 let generation = state
1953 .generations
1954 .get(module_id)
1955 .copied()
1956 .unwrap_or(0)
1957 .checked_add(1)
1958 .expect("spawn generation exhausted");
1959 state.generations.insert(module_id.to_string(), generation);
1960 generation
1961 }
1962
1963 fn allocate_generation(&self, module_id: &str) -> (String, u64) {
1968 let mut state = self.0.lock().unwrap_or_else(|p| p.into_inner());
1969 let generation = Self::allocate_generation_locked(&mut state, module_id);
1970 (state.daemon_incarnation.clone(), generation)
1971 }
1972
1973 fn emit_spawned(
1977 &self,
1978 module_id: &str,
1979 allocated: Option<u64>,
1980 pid: u32,
1981 spawned_at_ms: u64,
1982 ) -> u64 {
1983 let mut state = self.0.lock().unwrap_or_else(|p| p.into_inner());
1984 let generation = match allocated {
1985 Some(generation) => generation,
1986 None => Self::allocate_generation_locked(&mut state, module_id),
1987 };
1988 let live = LiveSpawn {
1989 module_id: module_id.to_string(),
1990 spawn_generation: generation,
1991 pid,
1992 spawned_at_ms,
1993 };
1994 state.live.insert(module_id.to_string(), live);
1995 Self::emit_locked(
1996 &mut state,
1997 SpawnEventKind::Spawned,
1998 module_id.to_string(),
1999 generation,
2000 pid,
2001 None,
2002 None,
2003 );
2004 generation
2005 }
2006
2007 fn emit_exited(&self, module_id: &str, exit_code: Option<i32>, exit_signal: Option<i32>) {
2008 let mut state = self.0.lock().unwrap_or_else(|p| p.into_inner());
2009 let Some(live) = state.live.remove(module_id) else {
2010 warn!(
2011 module_id,
2012 "terminal record had no live spawn event identity"
2013 );
2014 return;
2015 };
2016 Self::emit_locked(
2017 &mut state,
2018 SpawnEventKind::Exited,
2019 module_id.to_string(),
2020 live.spawn_generation,
2021 live.pid,
2022 exit_code,
2023 exit_signal,
2024 );
2025 }
2026
2027 fn emit_superseded_exited(
2034 &self,
2035 module_id: &str,
2036 spawn_generation: u64,
2037 pid: u32,
2038 exit_code: Option<i32>,
2039 exit_signal: Option<i32>,
2040 ) {
2041 let mut state = self.0.lock().unwrap_or_else(|p| p.into_inner());
2042 if state
2043 .live
2044 .get(module_id)
2045 .is_some_and(|live| live.spawn_generation == spawn_generation)
2046 {
2047 state.live.remove(module_id);
2048 }
2049 Self::emit_locked(
2050 &mut state,
2051 SpawnEventKind::Exited,
2052 module_id.to_string(),
2053 spawn_generation,
2054 pid,
2055 exit_code,
2056 exit_signal,
2057 );
2058 }
2059
2060 #[allow(clippy::too_many_arguments)]
2061 fn emit_locked(
2062 state: &mut SpawnEventState,
2063 kind: SpawnEventKind,
2064 module_id: String,
2065 spawn_generation: u64,
2066 pid: u32,
2067 exit_code: Option<i32>,
2068 exit_signal: Option<i32>,
2069 ) {
2070 state.seq = state
2071 .seq
2072 .checked_add(1)
2073 .expect("spawn event sequence exhausted");
2074 let event = SpawnEvent {
2075 cursor: Self::cursor(state),
2076 kind,
2077 module_id,
2078 spawn_generation,
2079 pid,
2080 exit_code,
2081 exit_signal,
2082 };
2083 state
2084 .process_changes
2085 .retain(|_, changed| changed.receiver_count() > 0);
2086 if let Some(changed) = state.process_changes.get(&event.module_id) {
2087 changed.send_replace(state.seq);
2088 }
2089 state.events.push_back(event.clone());
2090 while state.events.len() > state.capacity {
2091 state.events.pop_front();
2092 }
2093 let body = match serde_json::to_vec(&event) {
2094 Ok(body) => body,
2095 Err(error) => {
2096 error!(%error, "failed to serialize supervisor spawn event");
2097 return;
2098 }
2099 };
2100 state.subscribers.retain(|(connection_id, corr), subscriber| {
2101 let frame = Frame::build_with_version(
2102 subscriber.version,
2103 FrameType::StreamData,
2104 control_flags(),
2105 0,
2106 0,
2107 *corr,
2108 body.clone(),
2109 );
2110 match frame {
2111 Ok(frame) => {
2112 if subscriber.frames.try_send(frame).is_ok() {
2113 true
2114 } else {
2115 warn!(connection_id = connection_id.get(), corr, "dropping lagged supervisor spawn subscriber");
2116 if let Some(lagged) = subscriber.lagged.take() {
2117 let _ = lagged.send(event.cursor.clone());
2118 }
2119 false
2120 }
2121 }
2122 Err(error) => {
2123 warn!(connection_id = connection_id.get(), corr, %error, "dropping supervisor spawn subscriber after frame build failure");
2124 false
2125 }
2126 }
2127 });
2128 }
2129
2130 fn subscribe(
2131 &self,
2132 connection_id: ConnectionId,
2133 corr: u64,
2134 version: u8,
2135 since: Option<SpawnCursor>,
2136 sink: FrameSink,
2137 ) -> Result<(), SpawnSubscribeRefusal> {
2138 let (frames, mut receiver) = mpsc::channel(SPAWN_SUBSCRIBER_BUFFER);
2139 let (lagged, mut lagged_rx) = oneshot::channel::<SpawnCursor>();
2140 {
2141 let mut state = self.0.lock().unwrap_or_else(|p| p.into_inner());
2142 let replay = if let Some(since) = since {
2143 if since.daemon_incarnation != state.daemon_incarnation {
2144 return Err(SpawnSubscribeRefusal::ForeignIncarnation {
2145 current: state.daemon_incarnation.clone(),
2146 });
2147 }
2148 if let Some(oldest) = state.events.front().map(|event| event.cursor.clone()) {
2149 if since.seq < oldest.seq.saturating_sub(1) {
2150 return Err(SpawnSubscribeRefusal::TooOld { oldest });
2151 }
2152 }
2153 state
2154 .events
2155 .iter()
2156 .filter(|event| event.cursor.seq > since.seq)
2157 .cloned()
2158 .collect::<Vec<_>>()
2159 } else {
2160 Vec::new()
2161 };
2162 for event in replay {
2163 let body = serde_json::to_vec(&event)
2164 .map_err(|error| SpawnSubscribeRefusal::Frame(error.to_string()))?;
2165 let frame = Frame::build_with_version(
2166 version,
2167 FrameType::StreamData,
2168 control_flags(),
2169 0,
2170 0,
2171 corr,
2172 body,
2173 )
2174 .map_err(|error| SpawnSubscribeRefusal::Frame(error.to_string()))?;
2175 frames
2176 .try_send(frame)
2177 .map_err(|error| SpawnSubscribeRefusal::Frame(error.to_string()))?;
2178 }
2179 state.subscribers.insert(
2180 (connection_id, corr),
2181 SpawnSubscriber {
2182 version,
2183 frames,
2184 lagged: Some(lagged),
2185 },
2186 );
2187 }
2188 tokio::spawn(async move {
2199 while let Some(frame) = receiver.recv().await {
2200 if sink.send(frame).await.is_err() {
2201 return;
2202 }
2203 }
2204 let Ok(first_undelivered) = lagged_rx.try_recv() else {
2205 return;
2206 };
2207 match spawn_subscriber_lagged_frame(version, corr, first_undelivered) {
2208 Ok(frame) => {
2209 let _ = sink.send(frame).await;
2210 }
2211 Err(error) => {
2212 error!(%error, corr, "failed to build lagged spawn subscriber terminal frame");
2213 }
2214 }
2215 });
2216 Ok(())
2217 }
2218
2219 fn cancel(&self, connection_id: ConnectionId, corr: u64) -> bool {
2220 let Some(subscriber) = self
2221 .0
2222 .lock()
2223 .unwrap_or_else(|p| p.into_inner())
2224 .subscribers
2225 .remove(&(connection_id, corr))
2226 else {
2227 return false;
2228 };
2229 if let Ok(frame) = Frame::build_with_version(
2230 subscriber.version,
2231 FrameType::StreamEnd,
2232 control_flags(),
2233 0,
2234 0,
2235 corr,
2236 Vec::new(),
2237 ) {
2238 tokio::spawn(async move {
2239 let _ = subscriber.frames.send(frame).await;
2240 });
2241 }
2242 true
2243 }
2244
2245 fn remove_connection(&self, connection_id: ConnectionId) {
2246 self.0
2247 .lock()
2248 .unwrap_or_else(|p| p.into_inner())
2249 .subscribers
2250 .retain(|(subscriber_connection, _), _| *subscriber_connection != connection_id);
2251 }
2252
2253 #[cfg(any(test, feature = "test-support"))]
2254 fn set_capacity(&self, capacity: usize) {
2255 self.0.lock().unwrap_or_else(|p| p.into_inner()).capacity = capacity;
2256 }
2257
2258 #[cfg(any(test, feature = "test-support"))]
2259 fn subscriber_count(&self) -> usize {
2260 self.0
2261 .lock()
2262 .unwrap_or_else(|p| p.into_inner())
2263 .subscribers
2264 .len()
2265 }
2266}
2267
2268fn spawn_subscriber_lagged_frame(
2271 version: u8,
2272 corr: u64,
2273 first_undelivered: SpawnCursor,
2274) -> Result<Frame, String> {
2275 let body = serde_json::to_vec(&subc_protocol::ErrorBody {
2277 code: SPAWN_SUBSCRIBER_LAGGED_CODE.to_string(),
2278 message: "spawn subscriber fell behind and was dropped; resubscribe from the last cursor received"
2279 .to_string(),
2280 detail: Some(serde_json::json!({
2281 "first_undelivered_cursor": first_undelivered
2282 })),
2283 })
2284 .map_err(|error| error.to_string())?;
2285 Frame::build_with_version(
2286 version,
2287 FrameType::Error,
2288 crate::router::daemon_error_flags(),
2289 0,
2290 0,
2291 corr,
2292 body,
2293 )
2294 .map_err(|error| error.to_string())
2295}
2296
2297pub trait ModuleProcessLiveness: Send + Sync {
2298 fn process_live(&self, module_id: &str) -> Option<bool>;
2299
2300 fn process_replacing(&self, _module_id: &str) -> bool {
2306 false
2307 }
2308}
2309
2310#[derive(Debug, Clone, Default)]
2312pub struct SupervisorProcessLiveness {
2313 snapshots: Arc<Mutex<HashMap<String, SharedSnapshot>>>,
2314}
2315
2316impl SupervisorProcessLiveness {
2317 pub fn new() -> Self {
2318 Self::default()
2319 }
2320
2321 fn track(&self, module_id: String, snapshot: SharedSnapshot) {
2322 let mut snapshots = self
2323 .snapshots
2324 .lock()
2325 .unwrap_or_else(|poisoned| poisoned.into_inner());
2326 snapshots.insert(module_id, snapshot);
2327 }
2328
2329 fn untrack_if_current(&self, module_id: &str, snapshot: &SharedSnapshot) {
2330 let mut snapshots = self
2331 .snapshots
2332 .lock()
2333 .unwrap_or_else(|poisoned| poisoned.into_inner());
2334 let is_current = snapshots
2335 .get(module_id)
2336 .map(|tracked| Arc::ptr_eq(tracked, snapshot))
2337 .unwrap_or(false);
2338 if is_current {
2339 snapshots.remove(module_id);
2340 }
2341 }
2342}
2343
2344impl ModuleProcessLiveness for SupervisorProcessLiveness {
2345 fn process_live(&self, module_id: &str) -> Option<bool> {
2346 let snapshot = {
2347 let snapshots = self
2348 .snapshots
2349 .lock()
2350 .unwrap_or_else(|poisoned| poisoned.into_inner());
2351 snapshots.get(module_id).cloned()
2352 }?;
2353 let snapshot = snapshot
2354 .lock()
2355 .unwrap_or_else(|poisoned| poisoned.into_inner());
2356 Some(snapshot.state == ModuleState::Running && snapshot.process_alive)
2357 }
2358
2359 fn process_replacing(&self, module_id: &str) -> bool {
2360 let Some(snapshot) = self
2361 .snapshots
2362 .lock()
2363 .unwrap_or_else(|poisoned| poisoned.into_inner())
2364 .get(module_id)
2365 .cloned()
2366 else {
2367 return false;
2368 };
2369 let snapshot = snapshot
2370 .lock()
2371 .unwrap_or_else(|poisoned| poisoned.into_inner());
2372 snapshot.enabled
2373 && match snapshot.state {
2374 ModuleState::Restarting => true,
2375 ModuleState::Draining => snapshot.draining_to_replace,
2376 ModuleState::Starting
2377 | ModuleState::Running
2378 | ModuleState::Unresponsive
2379 | ModuleState::Stopped
2380 | ModuleState::Failed
2381 | ModuleState::Disabled => false,
2382 }
2383 }
2384}
2385
2386#[cfg(test)]
2387#[derive(Debug, Default)]
2388struct ReloadExitRecordGate {
2389 reached: tokio::sync::Notify,
2390 resume: tokio::sync::Notify,
2391}
2392
2393#[derive(Debug, Clone, Copy)]
2394enum RespawnKind {
2395 Spawn,
2396 Reload,
2397}
2398
2399#[derive(Debug, Clone)]
2400struct PendingRespawn {
2401 deadline: Instant,
2402 kind: RespawnKind,
2403 daemon_initiated: bool,
2404 approval: Option<crate::config_gate::runtime::ApprovalIdentity>,
2405}
2406
2407impl PendingRespawn {
2408 fn failure_context(&self, runtime: &SupervisorRuntimeConfig) -> FailureContext {
2409 FailureContext {
2410 approval: self.approval.clone(),
2411 ..runtime.failure_context(self.daemon_initiated)
2412 }
2413 }
2414}
2415
2416type ReloadReply = oneshot::Sender<Result<(), SuperviseError>>;
2417
2418#[derive(Debug, Clone)]
2419struct SupervisorRuntimeConfig {
2420 registry: Arc<Registry>,
2421 config_delivery: Option<config_delivery::ConfigDelivery>,
2422 recovery_is_daemon: bool,
2424 start_requester: Option<Principal>,
2425 pending_activity_restart: bool,
2426 provenance_probe: ExecutableIdentityProbe,
2427 scheduled_respawn: Arc<Mutex<Option<PendingRespawn>>>,
2429 deferred_reload_reply: Arc<Mutex<Option<ReloadReply>>>,
2431 restart_policy: RestartPolicy,
2432 register_timeout: Duration,
2433 drain_timeout: Duration,
2436 effective_drain_timeout: Arc<Mutex<Duration>>,
2439 default_drain_timeout: Duration,
2442 health: HealthConfig,
2443 host_health: HostHealth,
2444 connection_file_path: Option<PathBuf>,
2445 capture_logs_dir: Option<PathBuf>,
2446 forwarding: Option<Arc<ForwardingTable>>,
2447 supervisor_handle: Option<SupervisorHandle>,
2450 stderr_ring: Arc<Mutex<StderrRing>>,
2457 terminal_ring: Arc<Mutex<TerminalRing>>,
2458 spawn_events: SpawnEventFeed,
2459 child_roster: ChildRoster,
2460 #[cfg(target_os = "linux")]
2461 cgroup_placement: Option<subc_cgroup::Placement>,
2462 #[cfg(test)]
2463 test_seed_stale_facts_before_enable_spawn: bool,
2464 #[cfg(test)]
2465 test_reload_exit_record_gate: Option<Arc<ReloadExitRecordGate>>,
2466}
2467
2468impl SupervisorRuntimeConfig {
2469 fn failure_context(&self, daemon_initiated: bool) -> FailureContext {
2470 FailureContext {
2471 daemon_initiated,
2472 approval: None,
2473 shutdown: self.child_roster.shutdown_flag(),
2474 }
2475 }
2476}
2477
2478#[derive(Debug, Clone, PartialEq, Eq)]
2479struct SupervisedConfiguration {
2480 spec: ModuleSpec,
2481 enabled: bool,
2482 health: HealthConfig,
2483 register_timeout: Duration,
2484 restart_policy: RestartPolicy,
2485}
2486
2487#[derive(Debug, Clone, Default)]
2493pub struct SupervisorHandle {
2494 #[cfg(test)]
2495 resource_commit_pause: Arc<Mutex<Option<resource_launch::ResourceCommitPause>>>,
2496 #[cfg(test)]
2497 registration_test_channels: CapturedRegistrationChannels,
2498 registration_service: Arc<Mutex<Option<registration::RegistrationService>>>,
2499 registration_connections: Arc<Mutex<HashMap<ConnectionId, Arc<RegistrationSpawn>>>>,
2500 registration_launches: Arc<Mutex<Vec<std::sync::Weak<RegistrationSpawn>>>>,
2501 gate_runtime: Arc<Mutex<Option<std::sync::Weak<crate::config_gate::runtime::RuntimeOwner>>>>,
2502 program_bindings: program_binding::ProgramBindings,
2503 #[cfg(test)]
2504 identity_sample_pause: Arc<Mutex<Option<IdentitySamplePause>>>,
2505 #[cfg(all(test, unix))]
2506 plain_identity_policy_pause: Arc<Mutex<Option<(String, IdentitySamplePause)>>>,
2507 host_health: HostHealth,
2508 restart_intent: Arc<Mutex<DaemonRestartIntent>>,
2509 activity: Arc<Mutex<Option<crate::activity::ActivityLedger>>>,
2510 activity_images: Arc<Mutex<HashMap<String, activity::ActivityImage>>>,
2511 modules: Arc<Mutex<HashMap<String, SupervisedModule>>>,
2512 configured_ids: Arc<Mutex<HashSet<String>>>,
2524 spawn_events: SpawnEventFeed,
2525 reserved_nonces: Arc<Mutex<HashMap<String, Option<String>>>>,
2536 implicit_protocol_none: Arc<Mutex<HashSet<String>>>,
2540 removal_tombstones: Arc<Mutex<HashMap<String, u64>>>,
2546 spawn_nonces: Arc<Mutex<HashMap<String, String>>>,
2550 nonce_launches: Arc<Mutex<NonceLaunches>>,
2556 reserved_prefix_owners: Arc<Mutex<HashMap<String, String>>>,
2564 swaps: Arc<Mutex<HashMap<String, OpenSwap>>>,
2570 promotion_observer: PromotionObserverSlot,
2572 operation_lock: Arc<AsyncMutex<()>>,
2576}
2577
2578#[cfg(test)]
2579#[derive(Debug)]
2580struct IdentitySamplePause {
2581 reached: std::sync::mpsc::Sender<()>,
2582 release: std::sync::mpsc::Receiver<()>,
2583}
2584
2585pub(crate) trait SwapPromotionObserver: Send + Sync {
2594 fn swap_promoted(&self, registration: &crate::registry::ModuleRegistration);
2595}
2596
2597#[derive(Clone, Default)]
2601struct PromotionObserverSlot(Arc<Mutex<Option<std::sync::Weak<dyn SwapPromotionObserver>>>>);
2602
2603impl fmt::Debug for PromotionObserverSlot {
2604 fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
2605 f.write_str("PromotionObserverSlot")
2606 }
2607}
2608
2609#[derive(Debug, Clone)]
2611struct OpenSwap {
2612 candidate_nonce: String,
2615 incumbent_nonce: Option<String>,
2618 candidate_admitted: bool,
2622}
2623
2624#[derive(Debug, Clone, Copy, PartialEq, Eq)]
2627pub(crate) enum SwapHelloAdmission {
2628 NotSwapping,
2631 Candidate,
2633 Refused,
2636 ProtocolNone,
2639 ChannelRequired,
2642}
2643
2644#[derive(Debug, Clone, PartialEq, Eq)]
2645pub(crate) enum ReservedHelloRejection {
2646 Exact {
2647 module_id: String,
2648 },
2649 Prefix {
2650 prefix: String,
2651 owner_module_id: String,
2652 },
2653}
2654
2655#[derive(Debug, Default)]
2657pub(crate) struct ReservedHelloVerification {
2658 pub(crate) nonce_verified: bool,
2659 pub(crate) prefix_owner: Option<String>,
2660}
2661
2662#[derive(Debug)]
2663struct DaemonRestartIntent {
2664 expires: Option<Duration>,
2665 window: Duration,
2666}
2667
2668impl Default for DaemonRestartIntent {
2669 fn default() -> Self {
2670 Self {
2671 expires: None,
2672 window: Duration::from_secs(120),
2673 }
2674 }
2675}
2676
2677impl SupervisorHandle {
2678 pub(crate) fn module_is_gated(&self, module: &str) -> bool {
2679 let runtime = {
2680 let Ok(runtime) = self.gate_runtime.lock() else {
2681 return true;
2682 };
2683 runtime.clone()
2684 };
2685 runtime.is_some_and(|runtime| {
2686 runtime
2687 .upgrade()
2688 .is_none_or(|owner| owner.module_is_gated(module))
2689 })
2690 }
2691 pub(crate) fn announce_daemon_restart(&self) -> u64 {
2692 let mut intent = self
2693 .restart_intent
2694 .lock()
2695 .unwrap_or_else(|p| p.into_inner());
2696 intent.expires = Some(subc_uptime::suspend_inclusive_now() + intent.window);
2697 unix_ms_now().saturating_add(intent.window.as_millis() as u64)
2698 }
2699
2700 #[cfg(feature = "test-support")]
2701 pub(crate) fn set_shutdown_intent_window(&self, window: Duration) {
2702 self.restart_intent
2703 .lock()
2704 .unwrap_or_else(|p| p.into_inner())
2705 .window = window;
2706 }
2707
2708 #[cfg(unix)]
2709 fn take_daemon_shutdown_reason(&self) -> RouteCloseReason {
2710 let expiry = self
2711 .restart_intent
2712 .lock()
2713 .unwrap_or_else(|p| p.into_inner())
2714 .expires
2715 .take();
2716 if expiry.is_some_and(|expiry| subc_uptime::suspend_inclusive_now() < expiry) {
2717 RouteCloseReason::Restart
2718 } else {
2719 RouteCloseReason::Disable
2720 }
2721 }
2722
2723 pub fn new() -> Self {
2724 Self::default()
2725 }
2726
2727 pub(crate) fn spawn_snapshot(&self) -> SpawnSnapshot {
2728 self.spawn_events.snapshot()
2729 }
2730
2731 pub(crate) fn subscribe_process_changes(
2734 &self,
2735 module: &str,
2736 ) -> tokio::sync::watch::Receiver<u64> {
2737 let mut state = self
2738 .spawn_events
2739 .0
2740 .lock()
2741 .unwrap_or_else(|p| p.into_inner());
2742 state
2743 .process_changes
2744 .retain(|_, changed| changed.receiver_count() > 0);
2745 let seq = state.seq;
2746 state
2747 .process_changes
2748 .entry(module.into())
2749 .or_insert_with(|| tokio::sync::watch::channel(seq).0)
2750 .subscribe()
2751 }
2752
2753 pub(crate) fn current_spawn(&self, module: &str) -> Option<LiveSpawn> {
2754 self.spawn_events
2755 .0
2756 .lock()
2757 .unwrap_or_else(|p| p.into_inner())
2758 .live
2759 .get(module)
2760 .cloned()
2761 }
2762
2763 pub(crate) fn subscribe_spawns(
2764 &self,
2765 connection_id: ConnectionId,
2766 corr: u64,
2767 version: u8,
2768 since: Option<SpawnCursor>,
2769 sink: FrameSink,
2770 ) -> Result<(), SpawnSubscribeRefusal> {
2771 self.spawn_events
2772 .subscribe(connection_id, corr, version, since, sink)
2773 }
2774
2775 pub(crate) fn cancel_spawn_subscription(&self, connection_id: ConnectionId, corr: u64) -> bool {
2776 self.spawn_events.cancel(connection_id, corr)
2777 }
2778
2779 pub(crate) fn remove_spawn_subscribers(&self, connection_id: ConnectionId) {
2780 self.spawn_events.remove_connection(connection_id);
2781 }
2782
2783 #[cfg(any(test, feature = "test-support"))]
2784 pub fn set_spawn_event_capacity_for_test(&self, capacity: usize) {
2785 assert!(capacity > 0, "spawn event capacity must be non-zero");
2786 self.spawn_events.set_capacity(capacity);
2787 }
2788
2789 #[cfg(any(test, feature = "test-support"))]
2790 pub fn spawn_subscriber_count_for_test(&self) -> usize {
2791 self.spawn_events.subscriber_count()
2792 }
2793
2794 pub fn set_spawn_nonce(&self, module_id: &str, nonce: String) {
2797 self.spawn_nonces
2798 .lock()
2799 .unwrap_or_else(|poisoned| poisoned.into_inner())
2800 .insert(module_id.to_string(), nonce);
2801 self.refresh_program_binding_nonces(module_id);
2802 }
2803
2804 pub fn set_reserved_nonce(&self, module_id: &str, nonce: String) {
2807 {
2808 let mut reserved = self
2809 .reserved_nonces
2810 .lock()
2811 .unwrap_or_else(|p| p.into_inner());
2812 self.implicit_protocol_none
2813 .lock()
2814 .unwrap_or_else(|p| p.into_inner())
2815 .remove(module_id);
2816 reserved.insert(module_id.to_string(), Some(nonce));
2817 }
2818 self.refresh_program_binding_nonces(module_id);
2819 }
2820
2821 pub fn set_reserved_prefixes(&self, owner_module_id: &str, prefixes: &[String]) {
2823 let mut owners = self
2824 .reserved_prefix_owners
2825 .lock()
2826 .unwrap_or_else(|poisoned| poisoned.into_inner());
2827 owners.retain(|_, owner| owner != owner_module_id);
2828 for prefix in prefixes {
2829 owners.insert(prefix.clone(), owner_module_id.to_string());
2830 }
2831 }
2832
2833 pub(crate) fn spawn_nonce(&self, module_id: &str) -> Option<String> {
2835 if self.module_is_gated(module_id) {
2836 return None;
2837 }
2838 self.spawn_nonces
2839 .lock()
2840 .unwrap_or_else(|poisoned| poisoned.into_inner())
2841 .get(module_id)
2842 .cloned()
2843 }
2844
2845 #[cfg(all(test, unix))]
2846 pub(crate) fn legacy_nonce_present_for_test(&self, module: &str) -> bool {
2847 self.spawn_nonces.lock().unwrap().contains_key(module)
2848 }
2849
2850 pub(crate) fn apply_identity_configuration(&self, spec: &ModuleSpec) {
2851 self.set_reserved_prefixes(&spec.module_id, &spec.reserved_prefixes);
2854 let protocol = self
2855 .get(&spec.module_id)
2856 .and_then(|module| {
2857 let snapshot = module.inner.snapshot.lock().ok()?;
2858 snapshot
2859 .process_alive
2860 .then_some(snapshot.spawned_protocol.unwrap_or(spec.protocol))
2861 })
2862 .unwrap_or(spec.protocol);
2863 self.apply_identity_policy(spec, protocol);
2864 }
2865
2866 #[cfg(all(test, unix))]
2867 pub(crate) fn pause_plain_identity_policy_for_test(
2868 &self,
2869 module: &str,
2870 ) -> (std::sync::mpsc::Receiver<()>, std::sync::mpsc::Sender<()>) {
2871 let (reached, waiting) = std::sync::mpsc::channel();
2872 let (release, paused) = std::sync::mpsc::channel();
2873 *self.plain_identity_policy_pause.lock().unwrap() = Some((
2874 module.to_owned(),
2875 IdentitySamplePause {
2876 reached,
2877 release: paused,
2878 },
2879 ));
2880 (waiting, release)
2881 }
2882
2883 fn apply_identity_policy(&self, spec: &ModuleSpec, protocol: ModuleProtocol) {
2884 let channel_only = self.module_is_gated(&spec.module_id);
2885 let mut current_nonces = self
2886 .spawn_nonces
2887 .lock()
2888 .unwrap_or_else(|poisoned| poisoned.into_inner());
2889 if protocol == ModuleProtocol::None || channel_only {
2890 current_nonces.remove(&spec.module_id);
2891 }
2892 let spawn_nonce = current_nonces.get(&spec.module_id).cloned();
2893 #[cfg(test)]
2894 if let Some(pause) = self.identity_sample_pause.lock().unwrap().take() {
2895 pause.reached.send(()).unwrap();
2896 pause.release.recv().unwrap();
2897 }
2898 let mut reserved_nonces = self
2899 .reserved_nonces
2900 .lock()
2901 .unwrap_or_else(|poisoned| poisoned.into_inner());
2902 let mut implicit = self
2903 .implicit_protocol_none
2904 .lock()
2905 .unwrap_or_else(|p| p.into_inner());
2906 if spec.reserved {
2907 implicit.remove(&spec.module_id);
2908 } else if protocol == ModuleProtocol::None {
2909 if !reserved_nonces.contains_key(&spec.module_id) {
2910 implicit.insert(spec.module_id.clone());
2911 }
2912 } else if implicit.remove(&spec.module_id) {
2913 reserved_nonces.remove(&spec.module_id);
2914 }
2915 if spec.reserved || protocol == ModuleProtocol::None {
2916 reserved_nonces.insert(spec.module_id.clone(), spawn_nonce);
2921 }
2922 drop(implicit);
2923 drop(reserved_nonces);
2924 drop(current_nonces);
2925 self.refresh_program_binding_nonces(&spec.module_id);
2926 self.removal_tombstones
2930 .lock()
2931 .unwrap_or_else(|poisoned| poisoned.into_inner())
2932 .remove(&spec.module_id);
2933 }
2934
2935 pub fn reserved_hello_authorized(&self, module_id: &str, presented: Option<&str>) -> bool {
2940 self.reserved_hello_verification(module_id, presented)
2941 .is_ok()
2942 }
2943
2944 pub(crate) fn reserved_hello_verification(
2947 &self,
2948 module_id: &str,
2949 presented: Option<&str>,
2950 ) -> Result<ReservedHelloVerification, ReservedHelloRejection> {
2951 if self.module_is_gated(module_id) {
2952 return Err(ReservedHelloRejection::Exact {
2953 module_id: module_id.into(),
2954 });
2955 }
2956 self.hello_verification(module_id, presented, false)
2957 }
2958
2959 pub(crate) fn channel_hello_verification(
2960 &self,
2961 module_id: &str,
2962 ) -> Result<ReservedHelloVerification, ReservedHelloRejection> {
2963 if self
2964 .get(module_id)
2965 .is_some_and(|module| module.declared_protocol().ok() == Some(ModuleProtocol::None))
2966 {
2967 return Err(ReservedHelloRejection::Exact {
2968 module_id: module_id.into(),
2969 });
2970 }
2971 let mut verification = self.hello_verification(module_id, None, true)?;
2972 verification.nonce_verified = true;
2973 Ok(verification)
2974 }
2975
2976 pub(crate) fn prefix_claim_authorized(&self, owner: &str, module_id: &str) -> bool {
2977 self.prefix_claim_publication(owner, module_id).is_some()
2978 }
2979
2980 #[cfg(all(test, unix))]
2981 pub(crate) fn prefix_publication_locked_for_test(&self) -> bool {
2982 matches!(
2983 self.reserved_prefix_owners.try_lock(),
2984 Err(std::sync::TryLockError::WouldBlock)
2985 )
2986 }
2987
2988 pub(crate) fn prefix_claim_publication(
2992 &self,
2993 owner: &str,
2994 module_id: &str,
2995 ) -> Option<std::sync::MutexGuard<'_, HashMap<String, String>>> {
2996 if subc_protocol::manifest::validate_module_id(module_id).is_err()
2997 || self.module_is_gated(module_id)
2998 || self
2999 .reserved_nonces
3000 .lock()
3001 .unwrap_or_else(|p| p.into_inner())
3002 .contains_key(module_id)
3003 {
3004 return None;
3005 }
3006 let owners = self
3007 .reserved_prefix_owners
3008 .lock()
3009 .unwrap_or_else(|p| p.into_inner());
3010 let authorized = owners
3011 .iter()
3012 .filter(|(prefix, _)| module_id.starts_with(prefix.as_str()))
3013 .max_by_key(|(prefix, _)| prefix.len())
3014 .is_some_and(|(prefix, holder)| holder == owner && module_id.len() > prefix.len());
3015 authorized.then_some(owners)
3016 }
3017
3018 fn hello_verification(
3019 &self,
3020 module_id: &str,
3021 presented: Option<&str>,
3022 channel: bool,
3023 ) -> Result<ReservedHelloVerification, ReservedHelloRejection> {
3024 let nonces = self
3025 .reserved_nonces
3026 .lock()
3027 .unwrap_or_else(|poisoned| poisoned.into_inner());
3028 if let Some(expected) = nonces.get(module_id) {
3029 let authorized = channel
3033 || match expected {
3034 Some(expected) => presented.is_some_and(|p| {
3035 !p.is_empty() && constant_time_eq(expected.as_bytes(), p.as_bytes())
3036 }),
3037 None => false,
3038 };
3039 if authorized {
3040 return Ok(ReservedHelloVerification {
3041 nonce_verified: true,
3042 prefix_owner: None,
3043 });
3044 }
3045 return Err(ReservedHelloRejection::Exact {
3046 module_id: module_id.to_string(),
3047 });
3048 }
3049 drop(nonces);
3050
3051 let matched_prefix = self
3052 .reserved_prefix_owners
3053 .lock()
3054 .unwrap_or_else(|poisoned| poisoned.into_inner())
3055 .iter()
3056 .filter(|(prefix, _)| module_id.starts_with(prefix.as_str()))
3057 .max_by_key(|(prefix, _)| prefix.len())
3058 .map(|(prefix, owner)| (prefix.clone(), owner.clone()));
3059 let Some((prefix, owner_module_id)) = matched_prefix else {
3060 return Ok(ReservedHelloVerification::default());
3061 };
3062
3063 if module_id.len() == prefix.len()
3064 || (!channel
3065 && (self.module_is_gated(&owner_module_id) || self.protocol_none(&owner_module_id)))
3066 {
3067 return Err(ReservedHelloRejection::Prefix {
3068 prefix,
3069 owner_module_id,
3070 });
3071 }
3072
3073 let authorized = if channel {
3074 owner_module_id == module_id
3075 } else {
3076 presented.is_some_and(|presented| {
3077 self.spawn_nonces
3078 .lock()
3079 .unwrap_or_else(|poisoned| poisoned.into_inner())
3080 .get(&owner_module_id)
3081 .is_some_and(|expected| !presented.is_empty() && constant_time_eq(expected.as_bytes(), presented.as_bytes()))
3082 || self.swap_nonce_matches(&owner_module_id, presented)
3085 })
3086 };
3087 if authorized {
3088 Ok(ReservedHelloVerification {
3089 nonce_verified: true,
3090 prefix_owner: (owner_module_id != module_id).then_some(owner_module_id),
3091 })
3092 } else {
3093 Err(ReservedHelloRejection::Prefix {
3094 prefix,
3095 owner_module_id,
3096 })
3097 }
3098 }
3099
3100 pub fn spawned_consumer_authorized(&self, module_id: &str, presented: &str) -> bool {
3105 if self.module_is_gated(module_id) || self.protocol_none(module_id) {
3106 return false;
3107 }
3108 let nonces = self
3109 .spawn_nonces
3110 .lock()
3111 .unwrap_or_else(|poisoned| poisoned.into_inner());
3112 let current = nonces.get(module_id).is_some_and(|expected| {
3113 !presented.is_empty() && constant_time_eq(expected.as_bytes(), presented.as_bytes())
3114 });
3115 drop(nonces);
3116 current || self.swap_nonce_matches(module_id, presented)
3119 }
3120
3121 fn swap_nonce_matches(&self, module_id: &str, presented: &str) -> bool {
3123 if self.module_is_gated(module_id) || self.protocol_none(module_id) || presented.is_empty()
3124 {
3125 return false;
3126 }
3127 let swaps = self
3128 .swaps
3129 .lock()
3130 .unwrap_or_else(|poisoned| poisoned.into_inner());
3131 swaps.get(module_id).is_some_and(|swap| {
3132 constant_time_eq(swap.candidate_nonce.as_bytes(), presented.as_bytes())
3133 || swap.incumbent_nonce.as_deref().is_some_and(|incumbent| {
3134 constant_time_eq(incumbent.as_bytes(), presented.as_bytes())
3135 })
3136 })
3137 }
3138
3139 pub(crate) fn open_swap(&self, module_id: &str, candidate_nonce: String) {
3142 let incumbent_nonce = self
3143 .spawn_nonces
3144 .lock()
3145 .unwrap_or_else(|poisoned| poisoned.into_inner())
3146 .get(module_id)
3147 .cloned();
3148 self.swaps
3149 .lock()
3150 .unwrap_or_else(|poisoned| poisoned.into_inner())
3151 .insert(
3152 module_id.to_string(),
3153 OpenSwap {
3154 candidate_nonce,
3155 incumbent_nonce,
3156 candidate_admitted: false,
3157 },
3158 );
3159 self.refresh_program_binding_nonces(module_id);
3160 }
3161
3162 pub(crate) fn close_swap(&self, module_id: &str) {
3165 self.swaps
3166 .lock()
3167 .unwrap_or_else(|poisoned| poisoned.into_inner())
3168 .remove(module_id);
3169 self.refresh_program_binding_nonces(module_id);
3170 }
3171
3172 pub(crate) fn set_swap_promotion_observer(
3175 &self,
3176 observer: std::sync::Weak<dyn SwapPromotionObserver>,
3177 ) {
3178 *self
3179 .promotion_observer
3180 .0
3181 .lock()
3182 .unwrap_or_else(|poisoned| poisoned.into_inner()) = Some(observer);
3183 }
3184
3185 fn notify_swap_promoted(&self, registration: &crate::registry::ModuleRegistration) {
3188 let observer = self
3189 .promotion_observer
3190 .0
3191 .lock()
3192 .unwrap_or_else(|poisoned| poisoned.into_inner())
3193 .as_ref()
3194 .and_then(std::sync::Weak::upgrade);
3195 if let Some(observer) = observer {
3196 observer.swap_promoted(registration);
3197 }
3198 }
3199
3200 pub(crate) fn swap_open(&self, module_id: &str) -> bool {
3202 self.swaps
3203 .lock()
3204 .unwrap_or_else(|poisoned| poisoned.into_inner())
3205 .contains_key(module_id)
3206 }
3207
3208 fn promote_swap_nonce(&self, module_id: &str, reserved: bool) {
3213 let candidate_nonce = self
3214 .swaps
3215 .lock()
3216 .unwrap_or_else(|poisoned| poisoned.into_inner())
3217 .get_mut(module_id)
3218 .map(|swap| {
3219 swap.incumbent_nonce = None;
3220 swap.candidate_nonce.clone()
3221 });
3222 self.spawn_nonces
3223 .lock()
3224 .unwrap_or_else(|p| p.into_inner())
3225 .remove(module_id);
3226 {
3227 let mut nonces = self
3228 .reserved_nonces
3229 .lock()
3230 .unwrap_or_else(|p| p.into_inner());
3231 if reserved || nonces.contains_key(module_id) {
3232 nonces.insert(module_id.into(), None);
3233 }
3234 }
3235 if self.module_is_gated(module_id) {
3236 return;
3237 }
3238 let Some(nonce) = candidate_nonce.filter(|nonce| !nonce.is_empty()) else {
3239 return;
3240 };
3241 self.set_spawn_nonce(module_id, nonce.clone());
3242 if reserved {
3243 self.set_reserved_nonce(module_id, nonce);
3244 }
3245 }
3246
3247 pub(crate) fn swap_hello_admission(
3262 &self,
3263 module_id: &str,
3264 presented: Option<&str>,
3265 ) -> SwapHelloAdmission {
3266 if self.protocol_none(module_id) {
3267 return SwapHelloAdmission::ProtocolNone;
3268 }
3269 if self.module_is_gated(module_id) {
3270 return SwapHelloAdmission::ChannelRequired;
3271 }
3272 let swaps = self
3273 .swaps
3274 .lock()
3275 .unwrap_or_else(|poisoned| poisoned.into_inner());
3276 let Some(swap) = swaps.get(module_id) else {
3277 return SwapHelloAdmission::NotSwapping;
3278 };
3279 let Some(presented) = presented.filter(|nonce| !nonce.is_empty()) else {
3280 return SwapHelloAdmission::Refused;
3281 };
3282 if constant_time_eq(swap.candidate_nonce.as_bytes(), presented.as_bytes()) {
3283 return if swap.candidate_admitted {
3284 SwapHelloAdmission::Refused
3285 } else {
3286 SwapHelloAdmission::Candidate
3287 };
3288 }
3289 if swap
3290 .incumbent_nonce
3291 .as_deref()
3292 .is_some_and(|incumbent| constant_time_eq(incumbent.as_bytes(), presented.as_bytes()))
3293 {
3294 return SwapHelloAdmission::NotSwapping;
3295 }
3296 SwapHelloAdmission::Refused
3297 }
3298
3299 fn protocol_none(&self, module: &str) -> bool {
3300 self.get(module)
3301 .is_some_and(|module| module.declared_protocol().ok() == Some(ModuleProtocol::None))
3302 }
3303
3304 pub(crate) fn with_swap_candidate_admission<T>(
3306 &self,
3307 module_id: &str,
3308 publish: impl FnOnce() -> (T, bool),
3309 ) -> Option<T> {
3310 let mut swaps = self.swaps.lock().unwrap_or_else(|p| p.into_inner());
3311 let swap = swaps
3312 .get_mut(module_id)
3313 .filter(|swap| !swap.candidate_admitted)?;
3314 let (result, admitted) = publish();
3315 swap.candidate_admitted = admitted;
3316 Some(result)
3317 }
3318
3319 pub fn spawn_launch_nonce_for(&self, module_id: &str) -> Option<String> {
3321 self.spawn_nonce(module_id)
3322 }
3323
3324 pub fn reserved_launch_nonce_for(&self, module_id: &str) -> Option<String> {
3326 if self.module_is_gated(module_id) {
3327 return None;
3328 }
3329 self.reserved_nonces
3330 .lock()
3331 .unwrap_or_else(|poisoned| poisoned.into_inner())
3332 .get(module_id)
3333 .cloned()
3334 .flatten()
3335 }
3336
3337 pub fn insert(&self, module: SupervisedModule) -> Option<SupervisedModule> {
3338 self.mark_configured(module.module_id());
3342 let mut modules = self
3343 .modules
3344 .lock()
3345 .unwrap_or_else(|poisoned| poisoned.into_inner());
3346 modules.insert(module.module_id().to_string(), module)
3347 }
3348
3349 fn mark_configured(&self, module_id: &str) {
3353 self.update_configured_ids(|ids| {
3354 ids.insert(module_id.to_string());
3355 });
3356 }
3357
3358 fn publish_configured_ids(&self, ids: Option<&HashSet<String>>) {
3359 let owner = self
3360 .gate_runtime
3361 .lock()
3362 .unwrap_or_else(|p| p.into_inner())
3363 .clone()
3364 .and_then(|owner| owner.upgrade());
3365 if let Some(owner) = owner {
3366 owner.observe_supervisor_configured_ids(ids, Arc::downgrade(&self.configured_ids));
3367 }
3368 }
3369
3370 fn update_configured_ids(&self, update: impl FnOnce(&mut HashSet<String>)) {
3371 match self.configured_ids.lock() {
3372 Ok(mut ids) => {
3373 update(&mut ids);
3374 self.publish_configured_ids(Some(&ids));
3377 }
3378 Err(poisoned) => {
3379 update(&mut poisoned.into_inner());
3380 self.publish_configured_ids(None);
3381 }
3382 }
3383 }
3384
3385 fn unmark_configured_unless_rostered(&self, module_id: &str) {
3389 let modules = self
3390 .modules
3391 .lock()
3392 .unwrap_or_else(|poisoned| poisoned.into_inner());
3393 if !modules.contains_key(module_id) {
3394 self.update_configured_ids(|ids| {
3395 ids.remove(module_id);
3396 });
3397 }
3398 }
3399
3400 pub(crate) fn is_configured(&self, module_id: &str) -> bool {
3408 self.configured_ids
3409 .lock()
3410 .unwrap_or_else(|poisoned| poisoned.into_inner())
3411 .contains(module_id)
3412 }
3413
3414 pub fn get(&self, module_id: &str) -> Option<SupervisedModule> {
3415 let modules = self
3416 .modules
3417 .lock()
3418 .unwrap_or_else(|poisoned| poisoned.into_inner());
3419 modules.get(module_id).cloned()
3420 }
3421
3422 pub(crate) fn record_late_health_answer(
3423 &self,
3424 module_id: &str,
3425 latency_ms: u64,
3426 ) -> Result<bool, SuperviseError> {
3427 let Some(module) = self.get(module_id) else {
3428 return Ok(false);
3429 };
3430 self.host_health.record_late_module(module_id);
3431 update_snapshot(&module.inner.snapshot, Some(module_id), |state| {
3432 state.health.late_answer_count = state.health.late_answer_count.saturating_add(1);
3433 state.health.last_late_answer_latency_ms = Some(latency_ms);
3434 state.health.consecutive_failures = 0;
3442 })?;
3443 Ok(true)
3444 }
3445
3446 pub(crate) fn spawn_host_health_monitor(&self) -> JoinHandle<()> {
3447 self.host_health.spawn_monitor()
3448 }
3449
3450 pub fn record_deliberate_severance(&self, module_id: &str) -> Result<bool, SuperviseError> {
3456 let Some(module) = self.get(module_id) else {
3457 return Ok(false);
3458 };
3459 let snapshot = lock_snapshot(&module.inner.snapshot)?;
3460 let Some((pid, start_time)) = snapshot.pid.zip(snapshot.process_start_time) else {
3461 return Ok(false);
3462 };
3463 drop(snapshot);
3464 module.record_deliberate_severance(ProcessIdentity { pid, start_time })
3465 }
3466
3467 pub fn list(&self) -> Vec<SupervisedModule> {
3468 let modules = self
3469 .modules
3470 .lock()
3471 .unwrap_or_else(|poisoned| poisoned.into_inner());
3472 let mut modules = modules.values().cloned().collect::<Vec<_>>();
3473 modules.sort_by(|left, right| left.module_id().cmp(right.module_id()));
3474 modules
3475 }
3476
3477 pub(crate) fn retire(&self, module_id: &str) -> Option<SupervisedModule> {
3478 self.activity_images
3479 .lock()
3480 .unwrap_or_else(|p| p.into_inner())
3481 .remove(module_id);
3482 self.spawn_nonces
3483 .lock()
3484 .unwrap_or_else(|poisoned| poisoned.into_inner())
3485 .remove(module_id);
3486 self.close_swap(module_id);
3487 let mut reserved_nonces = self
3488 .reserved_nonces
3489 .lock()
3490 .unwrap_or_else(|poisoned| poisoned.into_inner());
3491 if reserved_nonces.contains_key(module_id) {
3492 reserved_nonces.insert(module_id.to_string(), None);
3495 }
3496 drop(reserved_nonces);
3497 self.reserved_prefix_owners
3498 .lock()
3499 .unwrap_or_else(|poisoned| poisoned.into_inner())
3500 .retain(|_, owner| owner != module_id);
3501 let removed = self
3502 .modules
3503 .lock()
3504 .unwrap_or_else(|poisoned| poisoned.into_inner())
3505 .remove(module_id);
3506 self.update_configured_ids(|ids| {
3507 ids.remove(module_id);
3508 });
3509 self.remove_program_binding_records(module_id);
3510 removed
3511 }
3512
3513 pub(crate) fn record_rescan_removal(&self, module_id: &str) {
3516 self.removal_tombstones
3517 .lock()
3518 .unwrap_or_else(|poisoned| poisoned.into_inner())
3519 .insert(module_id.to_string(), unix_ms_now());
3520 }
3521
3522 pub(crate) fn removal_tombstone_age_ms(&self, module_id: &str) -> Option<u64> {
3524 self.removal_tombstones
3525 .lock()
3526 .unwrap_or_else(|poisoned| poisoned.into_inner())
3527 .get(module_id)
3528 .copied()
3529 .map(|removed_at_ms| unix_ms_now().saturating_sub(removed_at_ms))
3530 }
3531
3532 pub(crate) fn release_retained_reserved_gate(&self, module_id: &str) -> bool {
3537 if self.get(module_id).is_some() {
3538 return false;
3539 }
3540 let mut reserved_nonces = self
3541 .reserved_nonces
3542 .lock()
3543 .unwrap_or_else(|poisoned| poisoned.into_inner());
3544 if !matches!(reserved_nonces.get(module_id), Some(None)) {
3545 return false;
3546 }
3547 reserved_nonces.remove(module_id);
3548 self.implicit_protocol_none
3549 .lock()
3550 .unwrap_or_else(|p| p.into_inner())
3551 .remove(module_id);
3552 true
3553 }
3554
3555 pub(crate) fn operation_lock(&self) -> Arc<AsyncMutex<()>> {
3556 Arc::clone(&self.operation_lock)
3557 }
3558}
3559
3560#[derive(Debug, Clone)]
3562pub struct Supervisor {
3563 registry: Arc<Registry>,
3564 config_delivery: Option<config_delivery::ConfigDelivery>,
3565 restart_policy: RestartPolicy,
3566 drain_timeout: Duration,
3567 connection_file_path: Option<PathBuf>,
3568 capture_logs_dir: Option<PathBuf>,
3569 forwarding: Option<Arc<ForwardingTable>>,
3570 process_liveness: Arc<SupervisorProcessLiveness>,
3571 supervisor_handle: Option<SupervisorHandle>,
3572 health: HealthConfig,
3573 host_health: HostHealth,
3574 daemon_start_clock: crate::clock::StartClock,
3575 terminal_journal: Option<Arc<crate::terminal_journal::TerminalJournal>>,
3576 spawn_events: SpawnEventFeed,
3577 provenance_probe: ExecutableIdentityProbe,
3578 child_roster: ChildRoster,
3581 #[cfg(target_os = "linux")]
3582 cgroup_placement: Option<subc_cgroup::Placement>,
3583 #[cfg(test)]
3584 test_after_first_spawn: AfterFirstSpawnHook,
3585}
3586
3587pub(crate) struct RegisteredConfiguredModule {
3591 spec: ModuleSpec,
3592 runtime: SupervisorRuntimeConfig,
3593 module: SupervisedModule,
3594 commands: mpsc::Receiver<SupervisorCommand>,
3595}
3596
3597#[cfg(test)]
3603#[derive(Clone, Default)]
3604struct AfterFirstSpawnHook(Option<AfterFirstSpawnFn>);
3605
3606#[cfg(test)]
3607type AfterFirstSpawnFn = Arc<dyn Fn(&str) + Send + Sync>;
3608
3609#[cfg(test)]
3610impl fmt::Debug for AfterFirstSpawnHook {
3611 fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
3612 f.write_str("AfterFirstSpawnHook")
3613 }
3614}
3615
3616#[cfg(test)]
3617impl AfterFirstSpawnHook {
3618 fn run(&self, module_id: &str) {
3619 if let Some(hook) = &self.0 {
3620 hook(module_id);
3621 }
3622 }
3623}
3624
3625impl Supervisor {
3626 #[cfg(all(test, target_os = "linux"))]
3627 pub(crate) fn set_admission_hook_for_test(
3628 &self,
3629 hook: impl Fn(&str, &'static str, Option<u32>) -> bool + Send + Sync + 'static,
3630 ) {
3631 self.child_roster.set_admission_hook_for_test(hook);
3632 }
3633
3634 #[cfg(test)]
3635 pub(crate) fn new_for_test(registry: Arc<Registry>, policy: RestartPolicy) -> Self {
3636 let supervisor = Self::new(registry, policy);
3637 #[cfg(target_os = "macos")]
3638 let supervisor = supervisor.with_privacy_trampoline(test_privacy_trampoline());
3639 supervisor
3640 }
3641 pub fn with_privacy_trampoline(self, path: impl Into<PathBuf>) -> Self {
3646 let path = path.into();
3647 #[cfg(target_os = "macos")]
3648 {
3649 let result = probe_privacy_trampoline(&path).map(|()| path);
3650 if let Err(cause) = &result {
3651 error!(%cause, "macOS privacy identity unavailable; supervised launches will refuse");
3652 }
3653 self.child_roster.set_privacy_trampoline(result);
3654 }
3655 #[cfg(not(target_os = "macos"))]
3656 let _ = path;
3657 self
3658 }
3659 pub(crate) fn daemon_shutdown_flag(&self) -> crate::child_roster::DaemonShutdownFlag {
3662 self.child_roster.shutdown_flag()
3663 }
3664
3665 #[cfg(unix)]
3676 pub(crate) fn begin_daemon_shutdown(&self) {
3677 self.child_roster.close();
3678 if let Some(journal) = &self.terminal_journal {
3679 journal.stamp_shutdown();
3680 }
3681 }
3682
3683 #[cfg(unix)]
3688 pub(crate) async fn drain_for_daemon_shutdown(&self) -> Result<(), SuperviseError> {
3689 const NOTICE_BUDGET: Duration = Duration::from_millis(500);
3690 const DRAIN_BUDGET: Duration = Duration::from_secs(2);
3691 let Some(forwarding) = &self.forwarding else {
3692 return Ok(());
3693 };
3694 let notice_started = Instant::now();
3695 let modules = forwarding
3696 .begin_daemon_drain()
3697 .map_err(SuperviseError::Forwarding)?;
3698 let deadline_ms =
3699 unix_ms_now().saturating_add((NOTICE_BUDGET + DRAIN_BUDGET).as_millis() as u64);
3700 let shutdown_reason = self
3701 .supervisor_handle
3702 .as_ref()
3703 .map(SupervisorHandle::take_daemon_shutdown_reason)
3704 .unwrap_or(RouteCloseReason::Disable);
3705 let mut modules_told = 0;
3706 let mut notices = tokio::task::JoinSet::new();
3707 let mut drains = Vec::new();
3708 for connection in forwarding
3709 .module_connections()
3710 .map_err(SuperviseError::Forwarding)?
3711 {
3712 let target = ModuleDrainTarget {
3713 endpoint: connection.endpoint,
3714 sink: connection.sink,
3715 negotiated_ver: connection.negotiated_ver,
3716 abandoned_bindings: Vec::new(),
3717 excluded_subscriptions: 0,
3718 };
3719 notices.spawn(async move {
3720 (
3721 true,
3722 send_module_draining(
3723 &connection.module_id,
3724 shutdown_reason,
3725 deadline_ms,
3726 &target,
3727 true,
3728 )
3729 .await,
3730 )
3731 });
3732 }
3733 for (module_id, endpoint) in modules {
3734 let Some(target) = forwarding
3735 .begin_endpoint_drain(endpoint, RouteCloseReason::Restart)
3736 .map_err(SuperviseError::Forwarding)?
3737 else {
3738 continue;
3739 };
3740 let routes = forwarding
3741 .endpoint_routes(target.endpoint)
3742 .map_err(SuperviseError::Forwarding)?;
3743 let mut recipients = Vec::new();
3746 let mut clients: Vec<(crate::forwarding::GoodbyeTarget, Vec<u16>)> = Vec::new();
3747 for route in routes {
3748 let client = route.goodbye_target;
3749 if let Some((_, channels)) = clients
3750 .iter_mut()
3751 .find(|(existing, _)| existing.connection_id == client.connection_id)
3752 {
3753 channels.push(client.channel);
3754 } else {
3755 let channel = client.channel;
3756 clients.push((client, vec![channel]));
3757 }
3758 }
3759 for (client, mut channels) in clients {
3760 channels.sort_unstable();
3761 channels.dedup();
3762 let closing = serde_json::to_vec(&ClientControlPush::RouteClosing {
3763 module_id: module_id.clone(),
3764 channels,
3765 reason: RouteCloseReason::Restart,
3766 })
3767 .expect("route closing serializes");
3768 recipients.push((client.sink, client.negotiated_ver, closing));
3769 }
3770 for (sink, version, body) in recipients {
3771 notices.spawn(async move {
3772 let frame = Frame::build_with_version(
3773 version,
3774 FrameType::Push,
3775 control_flags(),
3776 0,
3777 0,
3778 0,
3779 body,
3780 )
3781 .expect("bounded lifecycle notice frame builds");
3782 (false, sink.send_flushed(frame).await.is_ok())
3783 });
3784 }
3785 let gauges = declared_busy_gauges(&self.registry, &module_id)?;
3786 drains.push((module_id, target.endpoint, gauges));
3787 }
3788 let notice_deadline = Instant::now() + NOTICE_BUDGET;
3791 while let Ok(Some(result)) = timeout_at(notice_deadline, notices.join_next()).await {
3792 match result {
3793 Ok((true, true)) => modules_told += 1,
3794 Ok((false, true)) => {}
3795 result => warn!(?result, "daemon shutdown notice delivery failed"),
3796 }
3797 }
3798 notices.abort_all();
3799 info!(
3800 ?shutdown_reason,
3801 modules_told, "daemon shutdown module notices delivered"
3802 );
3803 let notice_elapsed = notice_started.elapsed();
3804 let drain_started = Instant::now();
3805 let deadline = drain_started + DRAIN_BUDGET;
3806 let mut waits = tokio::task::JoinSet::new();
3807 for (module_id, endpoint, gauges) in drains {
3808 let forwarding = Arc::clone(forwarding);
3809 let mut runtime = self.runtime_config();
3810 runtime.health.cadence = Duration::from_millis(100);
3811 waits.spawn(async move {
3812 let wait_result = wait_for_forwarding_quiescence(
3813 &forwarding,
3814 &module_id,
3815 &runtime,
3816 endpoint,
3817 deadline,
3818 &gauges,
3819 DrainScope::Endpoint(endpoint),
3820 &mut DrainGaugeTrace::new(&gauges),
3821 )
3822 .await;
3823 forwarding
3827 .record_daemon_drain_result(
3828 endpoint,
3829 drained_after_quiescence_wait(&wait_result),
3830 )
3831 .map_err(SuperviseError::Forwarding)?;
3832 wait_result
3833 });
3834 }
3835 while let Ok(Some(result)) = timeout_at(deadline, waits.join_next()).await {
3836 if !matches!(result, Ok(Ok(true))) {
3837 warn!(?result, "daemon shutdown drain did not reach quiescence");
3838 }
3839 }
3840 info!(
3843 notice_elapsed_ms = notice_elapsed.as_millis() as u64,
3844 drain_elapsed_ms = drain_started.elapsed().as_millis() as u64,
3845 "daemon shutdown forwarding drain complete"
3846 );
3847 Ok(())
3848 }
3849
3850 #[cfg(unix)]
3862 pub(crate) async fn end_children_for_daemon_shutdown(
3863 &self,
3864 already_escalated: bool,
3865 escalate: impl std::future::Future<Output = ()>,
3866 ) {
3867 tokio::pin!(escalate);
3868 let mut escalated = already_escalated;
3869 if let Some(forwarding) = &self.forwarding {
3870 let reason = CloseReason::new(
3871 "daemon_shutdown",
3872 "the daemon is exiting after its shutdown notice and drain",
3873 );
3874 if escalated {
3875 send_module_goodbyes_for_daemon_shutdown(forwarding, &reason, false).await;
3878 } else {
3879 tokio::select! {
3880 biased;
3881 _ = escalate.as_mut() => {
3882 info!("second SIGTERM: abandoning module GOODBYE delivery");
3883 escalated = true;
3884 }
3885 _ = send_module_goodbyes_for_daemon_shutdown(forwarding, &reason, true) => {}
3886 }
3887 }
3888 let closed = forwarding.close_all_connections(&reason);
3889 debug!(closed, "closed established connections for daemon shutdown");
3890 }
3891 let escalated_here = escalated && !already_escalated;
3895 let remaining_escalate = async move {
3896 if escalated_here {
3897 std::future::pending::<()>().await;
3898 } else {
3899 escalate.await;
3900 }
3901 };
3902 crate::child_roster::end_children_for_daemon_shutdown(
3903 &self.child_roster,
3904 escalated,
3905 remaining_escalate,
3906 )
3907 .await;
3908 }
3909
3910 pub fn new(registry: Arc<Registry>, restart_policy: RestartPolicy) -> Self {
3911 Self {
3912 registry,
3913 config_delivery: None,
3914 restart_policy,
3915 drain_timeout: DEFAULT_DRAIN_TIMEOUT,
3916 connection_file_path: None,
3917 capture_logs_dir: None,
3918 forwarding: None,
3919 process_liveness: Arc::new(SupervisorProcessLiveness::default()),
3920 supervisor_handle: None,
3921 health: HealthConfig::default(),
3922 host_health: HostHealth::default(),
3923 daemon_start_clock: crate::clock::StartClock::capture(),
3924 terminal_journal: None,
3925 spawn_events: SpawnEventFeed::default(),
3926 provenance_probe: ExecutableIdentityProbe::default(),
3927 child_roster: ChildRoster::default(),
3928 #[cfg(target_os = "linux")]
3929 cgroup_placement: None,
3930 #[cfg(test)]
3931 test_after_first_spawn: AfterFirstSpawnHook::default(),
3932 }
3933 }
3934
3935 pub fn with_drain_timeout(mut self, drain_timeout: Duration) -> Self {
3936 self.drain_timeout = drain_timeout;
3937 self
3938 }
3939
3940 pub(crate) fn default_drain_timeout(&self) -> Duration {
3942 self.drain_timeout
3943 }
3944
3945 pub fn with_process_liveness(
3946 mut self,
3947 process_liveness: Arc<SupervisorProcessLiveness>,
3948 ) -> Self {
3949 self.process_liveness = process_liveness;
3950 self
3951 }
3952
3953 pub fn with_config_runtime(
3955 mut self,
3956 owner: Arc<crate::config_gate::runtime::RuntimeOwner>,
3957 ) -> Self {
3958 if let Some(handle) = &self.supervisor_handle {
3959 handle.install_binding_retirement(&owner);
3960 }
3961 self.config_delivery = Some(config_delivery::ConfigDelivery(owner));
3962 self
3963 }
3964
3965 pub fn with_connection_file_path(mut self, connection_file_path: impl Into<PathBuf>) -> Self {
3966 self.connection_file_path = Some(connection_file_path.into());
3967 self
3968 }
3969
3970 pub fn with_capture_logs_dir(mut self, logs_dir: impl Into<PathBuf>) -> Self {
3972 self.capture_logs_dir = Some(logs_dir.into());
3973 self
3974 }
3975
3976 pub fn with_daemon_incarnation(self, daemon_incarnation: String) -> Self {
3979 self.spawn_events.configure_incarnation(daemon_incarnation);
3983 self
3984 }
3985
3986 pub fn with_terminal_journal(self, path: PathBuf, daemon_incarnation: String) -> Self {
3989 let mut this = self.with_daemon_incarnation(daemon_incarnation.clone());
3990 this.terminal_journal = Some(Arc::new(crate::terminal_journal::TerminalJournal::open(
3991 path,
3992 daemon_incarnation,
3993 )));
3994 this
3995 }
3996
3997 pub fn with_forwarding(mut self, forwarding: Arc<ForwardingTable>) -> Self {
3998 self.child_roster.counters = forwarding.counters();
3999 self.forwarding = Some(forwarding);
4000 self
4001 }
4002
4003 pub fn with_handle(mut self, supervisor_handle: SupervisorHandle) -> Self {
4004 if let Some(delivery) = &self.config_delivery {
4005 supervisor_handle.install_binding_retirement(&delivery.0);
4006 }
4007 self.host_health = supervisor_handle.host_health.clone();
4008 self.spawn_events = supervisor_handle.spawn_events.clone();
4009 self.supervisor_handle = Some(supervisor_handle);
4010 self
4011 }
4012
4013 pub fn with_health_config(mut self, health: HealthConfig) -> Self {
4014 self.health = health;
4015 self
4016 }
4017
4018 pub fn with_live_children_record(self, path: impl Into<PathBuf>) -> Self {
4022 self.child_roster.record_to(path.into());
4023 self
4024 }
4025
4026 #[cfg(target_os = "linux")]
4027 pub fn with_cgroup_placement(
4028 mut self,
4029 cgroup_placement: Option<subc_cgroup::Placement>,
4030 ) -> Self {
4031 self.child_roster
4032 .set_cgroup_placement(cgroup_placement.clone());
4033 self.cgroup_placement = cgroup_placement;
4034 self
4035 }
4036
4037 pub fn spawn(&self, spec: ModuleSpec) -> Result<SupervisedModule, SuperviseError> {
4046 validate_spec(&spec)?;
4047 self.establish_identity(&spec);
4048
4049 let runtime = self.runtime_config();
4050 let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::starting()));
4051 let activity_spawn = runtime.begin_activity_spawn(&spec.module_id);
4052 let registration_watch = watch_registration(&self.registry, &spec);
4053 let spawned = spawn_child(
4054 &spec,
4055 runtime.connection_file_path.as_deref(),
4056 self.supervisor_handle.as_ref(),
4057 &runtime.stderr_ring,
4058 runtime.capture_logs_dir.as_deref(),
4059 &runtime.child_roster,
4060 runtime.spawn_delivery(),
4061 #[cfg(target_os = "linux")]
4062 runtime.cgroup_placement.as_ref(),
4063 runtime.register_timeout,
4064 );
4065 #[cfg(test)]
4066 self.test_after_first_spawn.run(&spec.module_id);
4067 let mut child = match spawned {
4068 Ok(child) => child,
4069 Err(SuperviseError::ConfigDeliveryDeferred { .. }) => {
4070 defer_config_spawn(
4071 &spec,
4072 &runtime,
4073 &snapshot,
4074 RespawnKind::Spawn,
4075 &runtime.failure_context(runtime.recovery_is_daemon),
4076 )?;
4077 return Ok(self.supervised_module(spec, runtime, snapshot, None));
4078 }
4079 Err(SuperviseError::ConfigHeld { .. }) => {
4080 hold_config_spawn(
4081 &spec,
4082 &runtime,
4083 &snapshot,
4084 &runtime.failure_context(runtime.recovery_is_daemon),
4085 )?;
4086 return Ok(self.supervised_module(spec, runtime, snapshot, None));
4087 }
4088 Err(err) => {
4089 self.abandon_unrostered(&spec.module_id);
4092 return Err(err);
4093 }
4094 };
4095 arm_registration_deadline(&mut child, runtime.register_timeout, registration_watch);
4096 self.mark_first_process_running(&spec, &runtime, &snapshot, &mut child)?;
4097 if let Some(spawn) = activity_spawn {
4098 spawn.complete(&snapshot);
4099 }
4100
4101 Ok(self.supervised_module(spec, runtime, snapshot, Some(child)))
4102 }
4103
4104 pub async fn spawn_async(&self, spec: ModuleSpec) -> Result<SupervisedModule, SuperviseError> {
4107 validate_spec(&spec)?;
4108 self.establish_identity(&spec);
4109 let mut identity = AsyncInitialIdentity {
4110 supervisor: self,
4111 module: spec.module_id.clone(),
4112 committed: false,
4113 };
4114 let runtime = self.runtime_config();
4115 let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::starting()));
4116 let activity_spawn = runtime.begin_activity_spawn(&spec.module_id);
4117 let registration_watch = watch_registration(&self.registry, &spec);
4118 let spawned = spawn_child_async(&spec, &runtime).await;
4119 #[cfg(test)]
4120 self.test_after_first_spawn.run(&spec.module_id);
4121 let mut child = match spawned {
4122 Ok(child) => child,
4123 Err(SuperviseError::ConfigDeliveryDeferred { .. }) => {
4124 defer_config_spawn(
4125 &spec,
4126 &runtime,
4127 &snapshot,
4128 RespawnKind::Spawn,
4129 &runtime.failure_context(runtime.recovery_is_daemon),
4130 )?;
4131 identity.committed = true;
4132 return Ok(self.supervised_module(spec, runtime, snapshot, None));
4133 }
4134 Err(SuperviseError::ConfigHeld { .. }) => {
4135 hold_config_spawn(
4136 &spec,
4137 &runtime,
4138 &snapshot,
4139 &runtime.failure_context(runtime.recovery_is_daemon),
4140 )?;
4141 identity.committed = true;
4142 return Ok(self.supervised_module(spec, runtime, snapshot, None));
4143 }
4144 Err(err) => {
4145 self.abandon_unrostered(&spec.module_id);
4146 return Err(err);
4147 }
4148 };
4149 arm_registration_deadline(&mut child, runtime.register_timeout, registration_watch);
4150 self.mark_first_process_running(&spec, &runtime, &snapshot, &mut child)?;
4151 if let Some(spawn) = activity_spawn {
4152 spawn.complete(&snapshot);
4153 }
4154 identity.committed = true;
4155 Ok(self.supervised_module(spec, runtime, snapshot, Some(child)))
4156 }
4157
4158 fn establish_identity(&self, spec: &ModuleSpec) {
4171 if let Some(supervisor_handle) = &self.supervisor_handle {
4172 supervisor_handle.apply_identity_configuration(spec);
4173 supervisor_handle.mark_configured(&spec.module_id);
4174 }
4175 }
4176
4177 fn abandon_unrostered(&self, module_id: &str) {
4180 if let Some(supervisor_handle) = &self.supervisor_handle {
4181 supervisor_handle.unmark_configured_unless_rostered(module_id);
4182 }
4183 }
4184
4185 fn mark_first_process_running(
4188 &self,
4189 spec: &ModuleSpec,
4190 runtime: &SupervisorRuntimeConfig,
4191 snapshot: &SharedSnapshot,
4192 child: &mut SupervisedChild,
4193 ) -> Result<(), SuperviseError> {
4194 if let Err(err) = set_running(snapshot, child, &spec.module_id, &runtime.spawn_events) {
4195 containment::terminate_blocking(&runtime.child_roster.containment);
4196 self.abandon_unrostered(&spec.module_id);
4197 return Err(err);
4198 }
4199 self.process_liveness
4200 .track(spec.module_id.clone(), Arc::clone(snapshot));
4201 Ok(())
4202 }
4203
4204 pub fn supervise_configured(
4210 &self,
4211 spec: ModuleSpec,
4212 enabled: bool,
4213 ) -> Result<SupervisedModule, SuperviseError> {
4214 self.supervise_configured_requested(
4215 spec,
4216 enabled,
4217 self.health.clone(),
4218 None,
4219 self.restart_policy,
4220 DEFAULT_REGISTER_TIMEOUT,
4221 None,
4222 )
4223 }
4224
4225 pub fn supervise_configured_with_health(
4231 &self,
4232 spec: ModuleSpec,
4233 enabled: bool,
4234 health: HealthConfig,
4235 drain_timeout_ms: Option<u64>,
4236 restart_policy: RestartPolicy,
4237 ) -> Result<SupervisedModule, SuperviseError> {
4238 self.supervise_configured_requested(
4239 spec,
4240 enabled,
4241 health,
4242 drain_timeout_ms,
4243 restart_policy,
4244 DEFAULT_REGISTER_TIMEOUT,
4245 None,
4246 )
4247 }
4248
4249 #[allow(clippy::too_many_arguments)]
4250 pub(crate) fn supervise_configured_requested(
4251 &self,
4252 spec: ModuleSpec,
4253 enabled: bool,
4254 health: HealthConfig,
4255 drain_timeout_ms: Option<u64>,
4256 restart_policy: RestartPolicy,
4257 register_timeout: Duration,
4258 requester: Option<Principal>,
4259 ) -> Result<SupervisedModule, SuperviseError> {
4260 let registered = self.register_configured_with_health(
4261 spec,
4262 enabled,
4263 health,
4264 drain_timeout_ms,
4265 restart_policy,
4266 register_timeout,
4267 )?;
4268 let daemon_initiated = requester.is_none();
4269 self.start_registered_configured(registered, requester, daemon_initiated)
4270 }
4271
4272 #[allow(clippy::too_many_arguments)]
4273 pub(crate) async fn supervise_configured_requested_async(
4274 &self,
4275 spec: ModuleSpec,
4276 enabled: bool,
4277 health: HealthConfig,
4278 drain_timeout_ms: Option<u64>,
4279 restart_policy: RestartPolicy,
4280 register_timeout: Duration,
4281 requester: Option<Principal>,
4282 ) -> Result<SupervisedModule, SuperviseError> {
4283 let registered = self.register_configured_with_health(
4284 spec,
4285 enabled,
4286 health,
4287 drain_timeout_ms,
4288 restart_policy,
4289 register_timeout,
4290 )?;
4291 let daemon_initiated = requester.is_none();
4292 self.start_registered_configured_async(registered, requester, daemon_initiated)
4293 .await
4294 }
4295
4296 pub(crate) fn register_configured_with_health(
4301 &self,
4302 spec: ModuleSpec,
4303 enabled: bool,
4304 health: HealthConfig,
4305 drain_timeout_ms: Option<u64>,
4306 restart_policy: RestartPolicy,
4307 register_timeout: Duration,
4308 ) -> Result<RegisteredConfiguredModule, SuperviseError> {
4309 validate_spec(&spec)?;
4310 self.establish_identity(&spec);
4311
4312 let mut runtime = self.runtime_config();
4313 runtime.health = health;
4314 runtime.restart_policy = restart_policy;
4315 runtime.register_timeout = register_timeout;
4316 if let Some(ms) = drain_timeout_ms {
4317 runtime.drain_timeout = Duration::from_millis(ms);
4318 *runtime
4319 .effective_drain_timeout
4320 .lock()
4321 .unwrap_or_else(|poisoned| poisoned.into_inner()) = runtime.drain_timeout;
4322 }
4323 let snapshot = if enabled {
4324 SupervisorSnapshot::starting()
4325 } else {
4326 SupervisorSnapshot::disabled()
4327 };
4328 let snapshot = {
4329 let mut snapshot = snapshot;
4330 if enabled {
4331 if let Some((pid, reason)) = runtime.child_roster.previous_instance(&spec.module_id)
4332 {
4333 snapshot.waiting_for_previous_pid = Some(pid);
4334 snapshot.waiting_for_previous_detail = Some(reason.waiting_message(pid));
4335 }
4336 }
4337 snapshot
4338 };
4339 let (module, commands) =
4340 self.register_module(&spec, &runtime, Arc::new(Mutex::new(snapshot)));
4341 Ok(RegisteredConfiguredModule {
4342 spec,
4343 runtime,
4344 module,
4345 commands,
4346 })
4347 }
4348
4349 pub(crate) fn start_registered_configured(
4352 &self,
4353 registered: RegisteredConfiguredModule,
4354 requester: Option<Principal>,
4355 daemon_initiated: bool,
4356 ) -> Result<SupervisedModule, SuperviseError> {
4357 let RegisteredConfiguredModule {
4358 spec,
4359 mut runtime,
4360 module,
4361 commands,
4362 } = registered;
4363 runtime.start_requester = requester;
4364 runtime.recovery_is_daemon = daemon_initiated;
4365 let snapshot = &module.inner.snapshot;
4366 let should_spawn = {
4367 let state = lock_snapshot(snapshot)?;
4368 state.enabled && state.waiting_for_previous_pid.is_none()
4369 };
4370 let child = if should_spawn {
4371 let activity_spawn = runtime.begin_activity_spawn(&spec.module_id);
4372 let registration_watch = watch_registration(&self.registry, &spec);
4373 let spawned = spawn_child(
4374 &spec,
4375 runtime.connection_file_path.as_deref(),
4376 self.supervisor_handle.as_ref(),
4377 &runtime.stderr_ring,
4378 runtime.capture_logs_dir.as_deref(),
4379 &runtime.child_roster,
4380 runtime.spawn_delivery(),
4381 #[cfg(target_os = "linux")]
4382 runtime.cgroup_placement.as_ref(),
4383 runtime.register_timeout,
4384 );
4385 #[cfg(test)]
4386 self.test_after_first_spawn.run(&spec.module_id);
4387 match spawned {
4388 Ok(mut child) => {
4389 arm_registration_deadline(
4390 &mut child,
4391 runtime.register_timeout,
4392 registration_watch,
4393 );
4394 self.mark_first_process_running(&spec, &runtime, snapshot, &mut child)?;
4395 if let Some(spawn) = activity_spawn {
4396 spawn.complete(snapshot);
4397 }
4398 Some(child)
4399 }
4400 Err(SuperviseError::ConfigDeliveryDeferred { .. }) => {
4401 defer_config_spawn(
4402 &spec,
4403 &runtime,
4404 snapshot,
4405 RespawnKind::Spawn,
4406 &runtime.failure_context(daemon_initiated),
4407 )?;
4408 None
4409 }
4410 Err(mut err) => {
4411 if matches!(err, SuperviseError::ConfigHeld { .. }) {
4412 match hold_config_spawn(
4413 &spec,
4414 &runtime,
4415 snapshot,
4416 &runtime.failure_context(daemon_initiated),
4417 ) {
4418 Ok(()) => {
4419 self.start_monitor(spec, runtime, &module, None, commands);
4420 return Ok(module);
4421 }
4422 Err(parking_error) => err = parking_error,
4423 }
4424 }
4425 if runtime.health.critical {
4426 error!(
4427 module_id = %spec.module_id,
4428 program = %spec.program.display(),
4429 error = %err,
4430 "critical configured module failed to spawn; marking failed and alerting"
4431 );
4432 } else {
4433 error!(
4434 module_id = %spec.module_id,
4435 program = %spec.program.display(),
4436 error = %err,
4437 "configured module failed to spawn; marking failed and continuing"
4438 );
4439 }
4440 fail_snapshot(
4441 snapshot,
4442 Some(&spec.module_id),
4443 None,
4444 &runtime.failure_context(daemon_initiated),
4445 );
4446 None
4447 }
4448 }
4449 } else {
4450 None
4451 };
4452 self.start_monitor(spec, runtime, &module, child, commands);
4453 Ok(module)
4454 }
4455
4456 pub(crate) async fn start_registered_configured_async(
4457 &self,
4458 registered: RegisteredConfiguredModule,
4459 requester: Option<Principal>,
4460 daemon_initiated: bool,
4461 ) -> Result<SupervisedModule, SuperviseError> {
4462 self.start_registered_with_failure_policy(registered, requester, daemon_initiated, None)
4463 .await
4464 }
4465
4466 pub(crate) async fn start_registered_approved_async(
4467 &self,
4468 registered: RegisteredConfiguredModule,
4469 requester: Principal,
4470 generation: crate::config_gate::runtime::ApprovalIdentity,
4471 ) -> Result<SupervisedModule, SuperviseError> {
4472 self.start_registered_with_failure_policy(
4473 registered,
4474 Some(requester),
4475 false,
4476 Some(generation),
4477 )
4478 .await
4479 }
4480
4481 async fn start_registered_with_failure_policy(
4482 &self,
4483 registered: RegisteredConfiguredModule,
4484 requester: Option<Principal>,
4485 daemon_initiated: bool,
4486 approved_generation: Option<crate::config_gate::runtime::ApprovalIdentity>,
4487 ) -> Result<SupervisedModule, SuperviseError> {
4488 let RegisteredConfiguredModule {
4489 spec,
4490 mut runtime,
4491 module,
4492 commands,
4493 } = registered;
4494 runtime.start_requester = requester;
4495 runtime.recovery_is_daemon = daemon_initiated;
4496 let snapshot = &module.inner.snapshot;
4497 let mut startup_error = None;
4498 let failure = FailureContext {
4499 approval: approved_generation.clone(),
4500 ..runtime.failure_context(daemon_initiated)
4501 };
4502 let should_spawn = {
4503 let state = lock_snapshot(snapshot)?;
4504 state.enabled && state.waiting_for_previous_pid.is_none()
4505 };
4506 let child = if should_spawn {
4507 let activity_spawn = runtime.begin_activity_spawn(&spec.module_id);
4508 let registration_watch = watch_registration(&self.registry, &spec);
4509 let spawned = spawn_child_async(&spec, &runtime).await;
4510 #[cfg(test)]
4511 self.test_after_first_spawn.run(&spec.module_id);
4512 match spawned {
4513 Ok(mut child) => {
4514 arm_registration_deadline(
4515 &mut child,
4516 runtime.register_timeout,
4517 registration_watch,
4518 );
4519 self.mark_first_process_running(&spec, &runtime, snapshot, &mut child)?;
4520 if let Some(spawn) = activity_spawn {
4521 spawn.complete(snapshot);
4522 }
4523 Some(child)
4524 }
4525 Err(SuperviseError::ConfigDeliveryDeferred { .. }) => {
4526 defer_config_spawn(&spec, &runtime, snapshot, RespawnKind::Spawn, &failure)?;
4527 None
4528 }
4529 Err(mut err) => {
4530 if matches!(err, SuperviseError::ConfigHeld { .. }) {
4531 match hold_config_spawn(&spec, &runtime, snapshot, &failure) {
4532 Ok(()) => {
4533 self.start_monitor(spec, runtime, &module, None, commands);
4534 return Ok(module);
4535 }
4536 Err(parking_error) => err = parking_error,
4537 }
4538 }
4539 if runtime.health.critical {
4540 error!(module_id = %spec.module_id, program = %spec.program.display(), error = %err,
4541 "critical configured module failed to spawn; marking failed and alerting");
4542 } else {
4543 error!(module_id = %spec.module_id, program = %spec.program.display(), error = %err,
4544 "configured module failed to spawn; marking failed and continuing");
4545 }
4546 fail_snapshot(snapshot, Some(&spec.module_id), None, &failure);
4547 if let Some(generation) = approved_generation {
4548 let _ = update_snapshot(snapshot, Some(&spec.module_id), |state| {
4549 state.approved_start_failure = Some((generation, err.to_string()));
4550 });
4551 startup_error = Some(err);
4552 }
4553 None
4554 }
4555 }
4556 } else {
4557 None
4558 };
4559 self.start_monitor(spec, runtime, &module, child, commands);
4560 startup_error.map_or(Ok(module), Err)
4564 }
4565
4566 fn runtime_config(&self) -> SupervisorRuntimeConfig {
4567 let effective_drain_timeout = Arc::new(Mutex::new(self.drain_timeout));
4568 SupervisorRuntimeConfig {
4569 registry: self.registry.clone(),
4570 config_delivery: self.config_delivery.clone(),
4571 recovery_is_daemon: true,
4572 start_requester: None,
4573 pending_activity_restart: false,
4574 provenance_probe: self.provenance_probe.clone(),
4575 scheduled_respawn: Arc::default(),
4576 deferred_reload_reply: Arc::default(),
4577 restart_policy: self.restart_policy,
4578 register_timeout: DEFAULT_REGISTER_TIMEOUT,
4579 drain_timeout: self.drain_timeout,
4580 child_roster: self
4583 .child_roster
4584 .for_module(Arc::clone(&effective_drain_timeout)),
4585 effective_drain_timeout,
4586 default_drain_timeout: self.drain_timeout,
4587 health: self.health.clone(),
4588 host_health: self.host_health.clone(),
4589 connection_file_path: self.connection_file_path.clone(),
4590 capture_logs_dir: self.capture_logs_dir.clone(),
4591 forwarding: self.forwarding.clone(),
4592 supervisor_handle: self.supervisor_handle.clone(),
4593 stderr_ring: Arc::new(Mutex::new(StderrRing::new(StderrTailConfig::default()))),
4594 terminal_ring: Arc::new(Mutex::new(
4595 TerminalRing::new(
4596 TerminalRingConfig::default(),
4597 self.daemon_start_clock.started_at_ms(),
4598 )
4599 .with_start_clock(self.daemon_start_clock)
4600 .with_journal(self.terminal_journal.clone())
4601 .with_daemon_shutdown(self.child_roster.shutdown_flag()),
4602 )),
4603 spawn_events: self.spawn_events.clone(),
4604 #[cfg(target_os = "linux")]
4605 cgroup_placement: self.cgroup_placement.clone(),
4606 #[cfg(test)]
4607 test_seed_stale_facts_before_enable_spawn: false,
4608 #[cfg(test)]
4609 test_reload_exit_record_gate: None,
4610 }
4611 }
4612
4613 fn supervised_module(
4614 &self,
4615 spec: ModuleSpec,
4616 runtime: SupervisorRuntimeConfig,
4617 snapshot: SharedSnapshot,
4618 child: Option<SupervisedChild>,
4619 ) -> SupervisedModule {
4620 let (module, commands) = self.register_module(&spec, &runtime, snapshot);
4621 self.start_monitor(spec, runtime, &module, child, commands);
4622 module
4623 }
4624
4625 fn register_module(
4626 &self,
4627 spec: &ModuleSpec,
4628 runtime: &SupervisorRuntimeConfig,
4629 snapshot: SharedSnapshot,
4630 ) -> (SupervisedModule, mpsc::Receiver<SupervisorCommand>) {
4631 runtime.host_health.track(spec, &snapshot);
4632 let configuration = Arc::new(Mutex::new(SupervisedConfiguration {
4633 spec: spec.clone(),
4634 enabled: snapshot.lock().unwrap_or_else(|p| p.into_inner()).enabled,
4635 health: runtime.health.clone(),
4636 register_timeout: runtime.register_timeout,
4637 restart_policy: runtime.restart_policy,
4638 }));
4639 let stderr_ring = Arc::clone(&runtime.stderr_ring);
4640 let terminal_ring = Arc::clone(&runtime.terminal_ring);
4641 let effective_drain_timeout = Arc::clone(&runtime.effective_drain_timeout);
4642 let (tx, rx) = mpsc::channel(4);
4643 let module_id = spec.module_id.clone();
4644 let module = SupervisedModule {
4645 inner: Arc::new(SupervisedModuleInner {
4646 module_id: module_id.clone(),
4647 registry: Arc::clone(&self.registry),
4648 snapshot,
4649 configuration,
4650 stderr_ring,
4651 terminal_ring,
4652 commands: tx,
4653 monitor: Mutex::new(None),
4654 effective_drain_timeout,
4655 provenance_probe: self.provenance_probe.clone(),
4656 retained: runtime.child_roster.retained_view(),
4657 }),
4658 };
4659 if let Some(supervisor_handle) = &self.supervisor_handle {
4662 supervisor_handle.insert(module.clone());
4663 }
4664 (module, rx)
4665 }
4666
4667 fn start_monitor(
4668 &self,
4669 spec: ModuleSpec,
4670 runtime: SupervisorRuntimeConfig,
4671 module: &SupervisedModule,
4672 child: Option<SupervisedChild>,
4673 commands: mpsc::Receiver<SupervisorCommand>,
4674 ) {
4675 let monitor = tokio::spawn(supervise_loop(
4676 spec,
4677 runtime,
4678 Arc::clone(&self.registry),
4679 Arc::clone(&self.process_liveness),
4680 Arc::clone(&module.inner.snapshot),
4681 child,
4682 commands,
4683 ));
4684 *module
4685 .inner
4686 .monitor
4687 .lock()
4688 .unwrap_or_else(|p| p.into_inner()) = Some(monitor);
4689 }
4690}
4691
4692impl Default for Supervisor {
4693 fn default() -> Self {
4694 Self::new(Arc::new(Registry::default()), RestartPolicy::default())
4695 }
4696}
4697
4698#[derive(Clone)]
4700pub struct SupervisedModule {
4701 inner: Arc<SupervisedModuleInner>,
4702}
4703
4704struct SupervisedModuleInner {
4705 retained: crate::child_roster::RetainedView,
4706 module_id: String,
4707 registry: Arc<Registry>,
4708 snapshot: SharedSnapshot,
4709 configuration: Arc<Mutex<SupervisedConfiguration>>,
4710 stderr_ring: Arc<Mutex<StderrRing>>,
4711 terminal_ring: Arc<Mutex<TerminalRing>>,
4712 commands: mpsc::Sender<SupervisorCommand>,
4713 monitor: Mutex<Option<JoinHandle<()>>>,
4714 effective_drain_timeout: Arc<Mutex<Duration>>,
4715 provenance_probe: ExecutableIdentityProbe,
4716}
4717
4718impl fmt::Debug for SupervisedModule {
4719 fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
4720 f.debug_struct("SupervisedModule")
4721 .field("module_id", &self.inner.module_id)
4722 .field("status", &self.status())
4723 .finish_non_exhaustive()
4724 }
4725}
4726
4727impl SupervisedModule {
4728 pub(crate) fn check_approved_start_failure(
4729 &self,
4730 generation: &crate::config_gate::runtime::ApprovalIdentity,
4731 ) -> Result<bool, SuperviseError> {
4732 let state = lock_snapshot(&self.inner.snapshot)?;
4733 if let Some((failed_generation, detail)) = &state.approved_start_failure {
4734 if failed_generation == generation {
4735 return Err(SuperviseError::ReloadFailed {
4736 module_id: self.inner.module_id.clone(),
4737 reason: detail.clone(),
4738 });
4739 }
4740 }
4741 Ok(state.approved_start_failure.is_some())
4742 }
4743
4744 pub(crate) fn has_approved_start_failure(
4745 &self,
4746 identity: &crate::config_gate::runtime::ApprovalIdentity,
4747 ) -> bool {
4748 lock_snapshot(&self.inner.snapshot).is_ok_and(|state| {
4749 state
4750 .approved_start_failure
4751 .as_ref()
4752 .is_some_and(|(failed, _)| failed == identity)
4753 })
4754 }
4755
4756 pub(crate) fn needs_approved_gate_restart(
4757 &self,
4758 approval: &crate::config_gate::runtime::ApprovalIdentity,
4759 ) -> bool {
4760 lock_snapshot(&self.inner.snapshot)
4761 .is_ok_and(|state| state.needs_approved_gate_restart(approval))
4762 }
4763
4764 #[cfg(all(test, unix))]
4765 pub(crate) fn config_hold_origin_for_test(
4766 &self,
4767 ) -> (bool, Option<crate::config_gate::runtime::ApprovalIdentity>) {
4768 let state = lock_snapshot(&self.inner.snapshot).unwrap();
4769 (
4770 state.config_hold_daemon_initiated,
4771 state.config_hold_approval.clone(),
4772 )
4773 }
4774
4775 pub(crate) async fn config_policy_probe(
4776 &self,
4777 connection: crate::ConnectionId,
4778 generation: u64,
4779 attempt: u64,
4780 ) -> Result<(), SuperviseError> {
4781 self.inner
4782 .commands
4783 .send(SupervisorCommand::ConfigPolicyProbe {
4784 connection,
4785 generation,
4786 attempt,
4787 })
4788 .await
4789 .map_err(|_| SuperviseError::CommandClosed {
4790 module_id: self.inner.module_id.clone(),
4791 })
4792 }
4793 pub(crate) async fn config_exit_hold(&self) -> Result<(), SuperviseError> {
4794 let (reply, result) = oneshot::channel();
4795 self.inner
4796 .commands
4797 .send(SupervisorCommand::ConfigExitHold { reply })
4798 .await
4799 .map_err(|_| SuperviseError::CommandClosed {
4800 module_id: self.module_id().to_string(),
4801 })?;
4802 result.await.map_err(|_| SuperviseError::CommandClosed {
4803 module_id: self.module_id().to_string(),
4804 })?
4805 }
4806 pub(crate) async fn config_delivery_failed(
4807 &self,
4808 connection: crate::ConnectionId,
4809 hold: bool,
4810 ) -> Result<(), SuperviseError> {
4811 self.inner
4812 .commands
4813 .send(SupervisorCommand::ConfigDeliveryFailed { connection, hold })
4814 .await
4815 .map_err(|_| SuperviseError::CommandClosed {
4816 module_id: self.module_id().to_string(),
4817 })
4818 }
4819
4820 pub(crate) async fn config_apply_restart(&self, generation: u64) -> Result<(), SuperviseError> {
4821 self.inner
4822 .commands
4823 .send(SupervisorCommand::ConfigApplyRestart {
4824 generation,
4825 supersedes_swap: false,
4826 })
4827 .await
4828 .map_err(|_| SuperviseError::CommandClosed {
4829 module_id: self.inner.module_id.clone(),
4830 })
4831 }
4832 pub fn module_id(&self) -> &str {
4833 &self.inner.module_id
4834 }
4835
4836 #[cfg(test)]
4840 pub(crate) fn record_health_probe_failure_for_test(
4841 &self,
4842 detail: &str,
4843 ) -> Result<(), SuperviseError> {
4844 update_snapshot(&self.inner.snapshot, Some(&self.inner.module_id), |state| {
4845 state.health.consecutive_failures = state.health.consecutive_failures.saturating_add(1);
4846 state.health.detail = Some(detail.to_string());
4847 })
4848 }
4849
4850 pub fn state(&self) -> Result<ModuleState, SuperviseError> {
4851 Ok(lock_snapshot(&self.inner.snapshot)?.state)
4852 }
4853
4854 pub(crate) fn registration_end_reason(
4855 &self,
4856 ) -> Result<Option<RegistrationEndReason>, SuperviseError> {
4857 let snapshot = lock_snapshot(&self.inner.snapshot)?;
4858 Ok(match snapshot.state {
4859 ModuleState::Draining if snapshot.draining_to_replace => {
4860 Some(RegistrationEndReason::SupervisorRestart)
4861 }
4862 ModuleState::Draining => Some(RegistrationEndReason::SupervisorStop),
4863 ModuleState::Restarting if snapshot.draining_to_replace => {
4864 Some(RegistrationEndReason::SupervisorRestart)
4865 }
4866 _ => None,
4867 })
4868 }
4869
4870 pub fn stderr_tail(
4877 &self,
4878 max_lines: Option<usize>,
4879 max_bytes: Option<usize>,
4880 ) -> StderrTailSnapshot {
4881 self.inner
4882 .stderr_ring
4883 .lock()
4884 .unwrap_or_else(|poisoned| poisoned.into_inner())
4885 .snapshot(max_lines, max_bytes)
4886 }
4887
4888 pub fn terminal_history(&self) -> TerminalHistorySnapshot {
4893 self.inner
4894 .terminal_ring
4895 .lock()
4896 .unwrap_or_else(|poisoned| poisoned.into_inner())
4897 .snapshot()
4898 }
4899
4900 pub fn durable_terminal_history(&self) -> subc_control::TerminalHistory {
4905 durable_terminal_history_of(&self.inner.terminal_ring, &self.inner.module_id)
4906 }
4907
4908 pub(crate) async fn read_durable_terminal_history(
4913 &self,
4914 ) -> Result<subc_control::TerminalHistory, tokio::task::JoinError> {
4915 let terminal_ring = Arc::clone(&self.inner.terminal_ring);
4916 let module_id = self.inner.module_id.clone();
4917 tokio::task::spawn_blocking(move || durable_terminal_history_of(&terminal_ring, &module_id))
4918 .await
4919 }
4920
4921 pub fn status(&self) -> Result<ModuleStatus, SuperviseError> {
4922 self.status_with_snapshot_lock(&self.inner.snapshot, None)
4923 .map(|(status, _)| status)
4924 }
4925
4926 pub(crate) fn next_retry_at_ms(&self) -> Result<Option<u64>, SuperviseError> {
4927 let state = lock_snapshot(&self.inner.snapshot)?;
4928 Ok(state
4929 .failed_retry
4930 .pending
4931 .filter(|_| state.state == ModuleState::Failed && state.enabled)
4932 .map(|retry| {
4933 unix_ms_now().saturating_add(
4934 u64::try_from(
4935 retry
4936 .deadline
4937 .saturating_duration_since(Instant::now())
4938 .as_millis(),
4939 )
4940 .unwrap_or(u64::MAX),
4941 )
4942 }))
4943 }
4944
4945 #[cfg(all(test, unix))]
4946 pub(crate) fn retry_deadline_for_test(&self) -> Option<Instant> {
4947 lock_snapshot(&self.inner.snapshot)
4948 .unwrap()
4949 .failed_retry
4950 .pending
4951 .map(|retry| retry.deadline)
4952 }
4953
4954 pub(crate) fn record_deliberate_severance(
4955 &self,
4956 identity: ProcessIdentity,
4957 ) -> Result<bool, SuperviseError> {
4958 let mut snapshot = lock_snapshot(&self.inner.snapshot)?;
4959 if snapshot.pid != Some(identity.pid)
4960 || snapshot.process_start_time != Some(identity.start_time)
4961 {
4962 return Ok(false);
4963 }
4964 snapshot.deliberate_severance = Some(identity);
4965 Ok(true)
4966 }
4967
4968 pub(crate) fn status_for_control(
4973 &self,
4974 caller: &'static str,
4975 ) -> Result<ModuleStatus, SuperviseError> {
4976 self.status_with_snapshot_lock(&self.inner.snapshot, Some(caller))
4977 .map(|(status, _)| status)
4978 }
4979
4980 fn status_with_snapshot_lock(
4981 &self,
4982 snapshot: &SharedSnapshot,
4983 caller: Option<&'static str>,
4984 ) -> Result<(ModuleStatus, Option<SpawnedFileIdentity>), SuperviseError> {
4985 let restart_policy = self
4986 .inner
4987 .configuration
4988 .lock()
4989 .map_err(|_| SuperviseError::StatePoisoned {
4990 module_id: Some(self.inner.module_id.clone()),
4991 })?
4992 .restart_policy;
4993 let mut guard = match caller {
4994 Some(caller) => lock_snapshot_for_control(snapshot, &self.inner.module_id, caller)?,
4995 None => lock_snapshot(snapshot)?,
4996 };
4997 let restart_count = guard.crash_restarts_in_window(restart_policy.window, Instant::now());
5000 let snapshot = guard.clone();
5001 drop(guard);
5002 let drain_timeout = *self.inner.effective_drain_timeout.lock().map_err(|_| {
5003 SuperviseError::StatePoisoned {
5004 module_id: Some(self.inner.module_id.clone()),
5005 }
5006 })?;
5007 let registration_active = self
5008 .inner
5009 .registry
5010 .get_module(&self.inner.module_id)
5011 .map_err(SuperviseError::Registry)?
5012 .is_some();
5013 let protocol = snapshot
5014 .spawned_protocol
5015 .unwrap_or(self.declared_protocol()?);
5016 let running_process =
5017 snapshot.enabled && snapshot.state == ModuleState::Running && snapshot.process_alive;
5018 let live = match protocol {
5024 ModuleProtocol::Subc => running_process && registration_active,
5025 ModuleProtocol::None => running_process,
5026 };
5027 let retained = self.inner.retained.waiting_detail(&self.inner.module_id);
5030 let carry_configured = self
5031 .inner
5032 .configuration
5033 .lock()
5034 .map_err(|_| SuperviseError::StatePoisoned {
5035 module_id: Some(self.inner.module_id.clone()),
5036 })?
5037 .spec
5038 .carry_containment;
5039 let recovery_block = carries_containment(
5040 carry_configured,
5041 self.inner
5042 .retained
5043 .has_carried_container(&self.inner.module_id),
5044 )
5045 .then(|| self.inner.retained.recovery_block())
5046 .flatten();
5047 let launch_nonce_source = observed_launch_nonce_source(&snapshot);
5048 let pid = snapshot.reported_pid();
5049 let mut health = snapshot.health;
5050 if let Some(detail) = recovery_block
5051 .as_ref()
5052 .or_else(|| retained.as_ref().map(|(_, detail)| detail))
5053 {
5054 health.detail = Some(match health.detail {
5055 Some(previous) => format!("{previous}; {detail}"),
5056 None => detail.clone(),
5057 });
5058 }
5059
5060 Ok((
5061 ModuleStatus {
5062 admission_refused: snapshot.admission_refused,
5063 legacy_registration_spawn: snapshot.legacy_registration_spawn,
5064 registration_declaration: snapshot.registration_declaration.clone(),
5065 stop_reason: snapshot.stop_reason,
5066 waiting_for_previous_pid: retained
5067 .as_ref()
5068 .map(|(pid, _)| *pid)
5069 .filter(|pid| *pid > 0)
5070 .or(snapshot.waiting_for_previous_pid),
5071 waiting_for_previous_detail: recovery_block
5072 .or_else(|| retained.map(|(_, detail)| detail))
5073 .or(snapshot.waiting_for_previous_detail),
5074 module_id: self.inner.module_id.clone(),
5075 launch_nonce_source,
5076 state: snapshot.state,
5077 enabled: snapshot.enabled,
5078 process_alive: snapshot.process_alive,
5079 registration_active,
5080 protocol,
5081 live,
5082 restart_count,
5083 lifetime_restarts: snapshot.lifetime_restarts,
5084 spawn_generation: snapshot.spawn_generation,
5085 swap_outcome: snapshot.swap_outcome,
5086 max_restarts: restart_policy.max_restarts,
5087 restart_window: restart_policy.window,
5088 drain_timeout,
5089 restart_backoff: restart_policy.backoff,
5090 restart_max_backoff: restart_policy.max_backoff,
5091 pid,
5092 spawned_at_ms: snapshot.spawned_at_ms,
5093 spawned_from: snapshot.spawned_from,
5094 launch_spec_digest: snapshot.launch_spec_digest,
5095 process_start_time: snapshot.process_start_time,
5096 last_exit: snapshot.last_exit,
5097 health,
5098 },
5099 snapshot.spawned_file_identity.clone(),
5100 ))
5101 }
5102
5103 #[cfg(test)]
5104 pub(crate) fn hold_snapshot_for_test(
5105 &self,
5106 acquired: std::sync::mpsc::Sender<()>,
5107 hold: Duration,
5108 ) -> std::thread::JoinHandle<()> {
5109 let snapshot = Arc::clone(&self.inner.snapshot);
5110 std::thread::spawn(move || {
5111 let _guard = snapshot.lock().expect("test snapshot lock is not poisoned");
5112 acquired
5113 .send(())
5114 .expect("test receiver waits for snapshot lock");
5115 std::thread::sleep(hold);
5116 })
5117 }
5118
5119 pub(crate) async fn status_and_running_image_agreement(
5125 &self,
5126 ) -> Result<(ModuleStatus, subc_control::RunningImageAgreement), SuperviseError> {
5127 let (status, identity) = self.status_with_snapshot_lock(&self.inner.snapshot, None)?;
5128 let image = self
5129 .inner
5130 .provenance_probe
5131 .observe(
5132 status.pid,
5133 status.spawned_from.as_deref(),
5134 identity,
5135 status.process_start_time,
5136 )
5137 .await;
5138 Ok((status, image))
5139 }
5140
5141 pub(crate) async fn running_image_agreement(&self) -> subc_control::RunningImageAgreement {
5142 let snapshot = match lock_snapshot(&self.inner.snapshot) {
5143 Ok(snapshot) => snapshot.clone(),
5144 Err(_) => {
5145 return subc_control::RunningImageAgreement::Unavailable {
5146 reason: subc_control::RunningImageUnavailableReason::NotRunning,
5147 };
5148 }
5149 };
5150 self.inner
5151 .provenance_probe
5152 .observe(
5153 snapshot.reported_pid(),
5154 snapshot.spawned_from.as_deref(),
5155 snapshot.spawned_file_identity.clone(),
5156 snapshot.process_start_time,
5157 )
5158 .await
5159 }
5160
5161 pub(crate) fn child_resource_usage(&self) -> subc_control::ChildResourceUsage {
5164 #[cfg(windows)]
5165 {
5166 let image = lock_snapshot(&self.inner.snapshot)
5167 .ok()
5168 .and_then(|snapshot| {
5169 snapshot
5170 .spawned_file_identity
5171 .as_ref()
5172 .and_then(|identity| identity.windows_image.clone())
5173 });
5174 match image {
5175 Some(image) => {
5176 crate::child_resources::read_windows_handle(image.process(), image.start_time())
5177 }
5178 None => subc_control::ChildResourceUsage::Unavailable {
5179 reason: subc_control::ChildResourceUnavailableReason::NotRunning,
5180 },
5181 }
5182 }
5183 #[cfg(not(windows))]
5184 {
5185 let (pid, start_time) = match lock_snapshot(&self.inner.snapshot) {
5186 Ok(snapshot) => (snapshot.reported_pid(), snapshot.process_start_time),
5187 Err(_) => {
5188 return subc_control::ChildResourceUsage::Unavailable {
5189 reason: subc_control::ChildResourceUnavailableReason::Unreadable,
5190 }
5191 }
5192 };
5193 crate::child_resources::read(pid, start_time)
5194 }
5195 }
5196
5197 #[cfg(all(test, any(target_os = "linux", windows)))]
5198 pub(crate) fn subtree_target(&self) -> crate::subtree_resources::Target {
5199 self.inner
5200 .snapshot
5201 .try_lock()
5202 .map(|snapshot| snapshot.subtree_target.clone())
5203 .unwrap_or(crate::subtree_resources::Target::Unavailable("read_failed"))
5204 }
5205
5206 #[cfg(test)]
5207 pub(crate) fn set_subtree_target_for_test(&self, target: crate::subtree_resources::Target) {
5208 self.inner.snapshot.lock().unwrap().subtree_target = target;
5209 }
5210
5211 pub(crate) fn will_recover_after_connection_loss(&self) -> Result<bool, SuperviseError> {
5212 let mut snapshot = lock_snapshot(&self.inner.snapshot)?;
5213 Ok(match snapshot.state {
5214 ModuleState::Restarting => true,
5215 ModuleState::Failed | ModuleState::Disabled => false,
5216 _ if snapshot.enabled && snapshot.respawn_pending => true,
5217 _ => daemon_will_restart(
5218 &mut snapshot,
5219 &self
5220 .inner
5221 .configuration
5222 .lock()
5223 .map_err(|_| SuperviseError::StatePoisoned {
5224 module_id: Some(self.inner.module_id.clone()),
5225 })?
5226 .restart_policy,
5227 Instant::now(),
5228 ),
5229 })
5230 }
5231
5232 #[cfg(test)]
5233 pub(crate) fn is_warming(&self) -> Result<bool, SuperviseError> {
5234 self.is_warming_with_snapshot_lock(None)
5235 }
5236
5237 pub(crate) fn is_warming_for_control(
5238 &self,
5239 caller: &'static str,
5240 ) -> Result<bool, SuperviseError> {
5241 self.is_warming_with_snapshot_lock(Some(caller))
5242 }
5243
5244 fn is_warming_with_snapshot_lock(
5245 &self,
5246 caller: Option<&'static str>,
5247 ) -> Result<bool, SuperviseError> {
5248 let snapshot = match caller {
5249 Some(caller) => {
5250 lock_snapshot_for_control(&self.inner.snapshot, &self.inner.module_id, caller)?
5251 }
5252 None => lock_snapshot(&self.inner.snapshot)?,
5253 }
5254 .clone();
5255 Ok(matches!(
5256 snapshot.state,
5257 ModuleState::Starting | ModuleState::Running | ModuleState::Restarting
5258 ))
5259 }
5260
5261 pub async fn drain(&self) -> Result<(), SuperviseError> {
5263 self.stop().await
5264 }
5265
5266 pub(crate) async fn retire_requested(
5267 &self,
5268 requester: Option<Principal>,
5269 ) -> Result<(), SuperviseError> {
5270 match self.state()? {
5271 ModuleState::Stopped => return Ok(()),
5272 ModuleState::Failed
5273 if lock_snapshot(&self.inner.snapshot)?
5274 .failed_retry
5275 .pending
5276 .is_none() =>
5277 {
5278 return Ok(())
5279 }
5280 ModuleState::Failed
5281 | ModuleState::Starting
5282 | ModuleState::Running
5283 | ModuleState::Unresponsive
5284 | ModuleState::Restarting
5285 | ModuleState::Draining
5286 | ModuleState::Disabled => {}
5287 }
5288
5289 let (reply_tx, reply_rx) = oneshot::channel();
5290 self.inner
5291 .commands
5292 .send(SupervisorCommand::Retire {
5293 requester,
5294 reply: reply_tx,
5295 })
5296 .await
5297 .map_err(|_| SuperviseError::CommandClosed {
5298 module_id: self.inner.module_id.clone(),
5299 })?;
5300 reply_rx.await.map_err(|_| SuperviseError::CommandClosed {
5301 module_id: self.inner.module_id.clone(),
5302 })?
5303 }
5304
5305 pub async fn stop(&self) -> Result<(), SuperviseError> {
5306 self.stop_requested(None).await
5307 }
5308
5309 pub(crate) async fn stop_requested(
5310 &self,
5311 requester: Option<Principal>,
5312 ) -> Result<(), SuperviseError> {
5313 match self.state()? {
5314 ModuleState::Stopped => return Ok(()),
5315 ModuleState::Failed
5316 if lock_snapshot(&self.inner.snapshot)?
5317 .failed_retry
5318 .pending
5319 .is_none() =>
5320 {
5321 return Ok(())
5322 }
5323 ModuleState::Failed
5324 | ModuleState::Starting
5325 | ModuleState::Running
5326 | ModuleState::Unresponsive
5327 | ModuleState::Restarting
5328 | ModuleState::Draining
5329 | ModuleState::Disabled => {}
5330 }
5331
5332 let (reply_tx, reply_rx) = oneshot::channel();
5333 self.inner
5334 .commands
5335 .send(SupervisorCommand::Drain {
5336 requester,
5337 reply: reply_tx,
5338 })
5339 .await
5340 .map_err(|_| SuperviseError::CommandClosed {
5341 module_id: self.inner.module_id.clone(),
5342 })?;
5343 reply_rx.await.map_err(|_| SuperviseError::CommandClosed {
5344 module_id: self.inner.module_id.clone(),
5345 })?
5346 }
5347
5348 pub async fn restart(&self, drain_timeout_ms: Option<u64>) -> Result<(), SuperviseError> {
5349 self.restart_requested(drain_timeout_ms, None).await
5350 }
5351
5352 pub(crate) async fn restart_requested(
5353 &self,
5354 drain_timeout_ms: Option<u64>,
5355 requester: Option<Principal>,
5356 ) -> Result<(), SuperviseError> {
5357 let queued_at = Instant::now();
5358 let received_at_generation = lock_snapshot(&self.inner.snapshot)?.spawn_generation;
5359 let (reply_tx, reply_rx) = oneshot::channel();
5360 self.inner
5361 .commands
5362 .send(SupervisorCommand::Restart {
5363 requester,
5364 drain_timeout_ms,
5365 received_at_generation,
5366 queued_at,
5367 reply: reply_tx,
5368 })
5369 .await
5370 .map_err(|_| SuperviseError::CommandClosed {
5371 module_id: self.inner.module_id.clone(),
5372 })?;
5373 reply_rx.await.map_err(|_| SuperviseError::CommandClosed {
5374 module_id: self.inner.module_id.clone(),
5375 })?
5376 }
5377
5378 pub async fn swap(
5384 &self,
5385 ready_timeout: Option<Duration>,
5386 ) -> Result<SwapOutcome, SuperviseError> {
5387 self.swap_requested(ready_timeout, None).await
5388 }
5389
5390 pub(crate) async fn swap_requested(
5391 &self,
5392 ready_timeout: Option<Duration>,
5393 requester: Option<Principal>,
5394 ) -> Result<SwapOutcome, SuperviseError> {
5395 let (reply_tx, reply_rx) = oneshot::channel();
5396 self.inner
5397 .commands
5398 .send(SupervisorCommand::Swap {
5399 requester,
5400 ready_timeout,
5401 reply: reply_tx,
5402 })
5403 .await
5404 .map_err(|_| SuperviseError::CommandClosed {
5405 module_id: self.inner.module_id.clone(),
5406 })?;
5407 reply_rx.await.map_err(|_| SuperviseError::CommandClosed {
5408 module_id: self.inner.module_id.clone(),
5409 })?
5410 }
5411
5412 pub async fn reload(&self) -> Result<(), SuperviseError> {
5413 self.reload_requested(None).await
5414 }
5415
5416 pub(crate) async fn reload_requested(
5417 &self,
5418 requester: Option<Principal>,
5419 ) -> Result<(), SuperviseError> {
5420 let (reply_tx, reply_rx) = oneshot::channel();
5421 self.inner
5422 .commands
5423 .send(SupervisorCommand::Reload {
5424 requester,
5425 reply: reply_tx,
5426 })
5427 .await
5428 .map_err(|_| SuperviseError::CommandClosed {
5429 module_id: self.inner.module_id.clone(),
5430 })?;
5431 reply_rx.await.map_err(|_| SuperviseError::CommandClosed {
5432 module_id: self.inner.module_id.clone(),
5433 })?
5434 }
5435
5436 pub async fn set_enabled(&self, enabled: bool) -> Result<bool, SuperviseError> {
5437 self.set_enabled_requested(enabled, None).await
5438 }
5439
5440 pub(crate) async fn set_enabled_requested(
5441 &self,
5442 enabled: bool,
5443 requester: Option<Principal>,
5444 ) -> Result<bool, SuperviseError> {
5445 self.set_enabled_with_budget_policy(enabled, requester, true)
5446 .await
5447 }
5448
5449 pub(crate) async fn release_config_hold(&self) -> Result<bool, SuperviseError> {
5450 self.set_enabled_with_budget_policy(true, None, false).await
5451 }
5452
5453 #[cfg(all(test, unix))]
5454 pub(crate) fn seed_crash_budget_for_test(&self, count: u32) {
5455 let policy = self.inner.configuration.lock().unwrap().restart_policy;
5456 let mut state = self.inner.snapshot.lock().unwrap();
5457 for _ in 0..count {
5458 state.record_crash_restart(&policy, Instant::now());
5459 }
5460 }
5461
5462 async fn set_enabled_with_budget_policy(
5463 &self,
5464 enabled: bool,
5465 requester: Option<Principal>,
5466 reset_budget: bool,
5467 ) -> Result<bool, SuperviseError> {
5468 let (reply_tx, reply_rx) = oneshot::channel();
5469 self.inner
5470 .commands
5471 .send(SupervisorCommand::SetEnabled {
5472 requester,
5473 enabled,
5474 reset_budget,
5475 reply: reply_tx,
5476 })
5477 .await
5478 .map_err(|_| SuperviseError::CommandClosed {
5479 module_id: self.inner.module_id.clone(),
5480 })?;
5481 reply_rx.await.map_err(|_| SuperviseError::CommandClosed {
5482 module_id: self.inner.module_id.clone(),
5483 })?
5484 }
5485
5486 pub(crate) fn declared_protocol(&self) -> Result<ModuleProtocol, SuperviseError> {
5490 let configured = self
5491 .inner
5492 .configuration
5493 .lock()
5494 .map_err(|_| SuperviseError::StatePoisoned {
5495 module_id: Some(self.inner.module_id.clone()),
5496 })?
5497 .spec
5498 .protocol;
5499 let state = lock_snapshot(&self.inner.snapshot)?;
5500 Ok(state.spawned_protocol.unwrap_or(configured))
5501 }
5502
5503 pub(crate) fn configuration(&self) -> Result<(ModuleSpec, HealthConfig), SuperviseError> {
5504 let configuration =
5505 self.inner
5506 .configuration
5507 .lock()
5508 .map_err(|_| SuperviseError::StatePoisoned {
5509 module_id: Some(self.inner.module_id.clone()),
5510 })?;
5511 Ok((configuration.spec.clone(), configuration.health.clone()))
5512 }
5513
5514 pub(crate) fn configured_register_timeout(&self) -> Result<Duration, SuperviseError> {
5515 self.inner
5516 .configuration
5517 .lock()
5518 .map(|configuration| configuration.register_timeout)
5519 .map_err(|_| SuperviseError::StatePoisoned {
5520 module_id: Some(self.inner.module_id.clone()),
5521 })
5522 }
5523
5524 pub(crate) fn configured_enabled(&self) -> Result<bool, SuperviseError> {
5525 self.inner
5526 .configuration
5527 .lock()
5528 .map(|configuration| configuration.enabled)
5529 .map_err(|_| SuperviseError::StatePoisoned {
5530 module_id: Some(self.inner.module_id.clone()),
5531 })
5532 }
5533
5534 #[cfg(any(test, feature = "test-support"))]
5538 pub async fn update_spec_for_test(&self, spec: ModuleSpec) -> Result<(), SuperviseError> {
5539 let (_, health) = self.configuration()?;
5540 let drain_timeout_ms = u64::try_from(
5541 self.inner
5542 .effective_drain_timeout
5543 .lock()
5544 .unwrap_or_else(|poisoned| poisoned.into_inner())
5545 .as_millis(),
5546 )
5547 .ok();
5548 self.update_configuration(spec, health, drain_timeout_ms)
5549 .await
5550 }
5551
5552 pub(crate) async fn reconcile_approved_configuration(
5553 &self,
5554 approved: crate::config_gate::runtime::ApprovedDaemonConfiguration,
5555 ) -> Result<(), SuperviseError> {
5556 let entry = approved
5557 .module_entry(&self.inner.module_id)
5558 .map_err(|error| SuperviseError::InvalidSpec {
5559 reason: error.to_string(),
5560 })?;
5561 validate_spec(&entry.module_spec())?;
5562 let (reply, received) = oneshot::channel();
5563 self.inner
5564 .commands
5565 .send(SupervisorCommand::ApprovedConfiguration { approved, reply })
5566 .await
5567 .map_err(|_| SuperviseError::CommandClosed {
5568 module_id: self.inner.module_id.clone(),
5569 })?;
5570 received.await.map_err(|_| SuperviseError::CommandClosed {
5571 module_id: self.inner.module_id.clone(),
5572 })??;
5573 let mut configuration =
5574 self.inner
5575 .configuration
5576 .lock()
5577 .map_err(|_| SuperviseError::StatePoisoned {
5578 module_id: Some(self.inner.module_id.clone()),
5579 })?;
5580 configuration.spec = entry.module_spec();
5581 configuration.enabled = entry.enabled;
5582 configuration.health = entry.health;
5583 configuration.register_timeout = entry.register_timeout;
5584 configuration.restart_policy = entry.restart;
5585 Ok(())
5586 }
5587
5588 pub(crate) async fn update_configuration(
5589 &self,
5590 spec: ModuleSpec,
5591 health: HealthConfig,
5592 drain_timeout_ms: Option<u64>,
5593 ) -> Result<(), SuperviseError> {
5594 if spec.module_id != self.inner.module_id {
5595 return Err(SuperviseError::InvalidSpec {
5596 reason: "a supervised module's module_id cannot be changed".to_string(),
5597 });
5598 }
5599 validate_spec(&spec)?;
5600 let (reply_tx, reply_rx) = oneshot::channel();
5601 self.inner
5602 .commands
5603 .send(SupervisorCommand::UpdateConfiguration {
5604 spec: spec.clone(),
5605 health: health.clone(),
5606 drain_timeout_ms,
5607 reply: reply_tx,
5608 })
5609 .await
5610 .map_err(|_| SuperviseError::CommandClosed {
5611 module_id: self.inner.module_id.clone(),
5612 })?;
5613 reply_rx.await.map_err(|_| SuperviseError::CommandClosed {
5614 module_id: self.inner.module_id.clone(),
5615 })??;
5616 let mut configuration =
5617 self.inner
5618 .configuration
5619 .lock()
5620 .map_err(|_| SuperviseError::StatePoisoned {
5621 module_id: Some(self.inner.module_id.clone()),
5622 })?;
5623 configuration.spec = spec;
5624 configuration.health = health;
5625 Ok(())
5626 }
5627}
5628
5629impl Drop for SupervisedModuleInner {
5630 fn drop(&mut self) {
5631 let Ok(mut monitor) = self.monitor.lock() else {
5632 return;
5633 };
5634 if let Some(monitor) = monitor.as_ref().filter(|monitor| !monitor.is_finished()) {
5635 let _ = update_snapshot(&self.snapshot, Some(&self.module_id), |state| {
5636 state.state = ModuleState::Stopped;
5637 clear_current_process_facts(state);
5638 });
5639 monitor.abort();
5640 }
5641 let _ = monitor.take();
5642 }
5643}
5644
5645#[derive(Debug)]
5646enum SupervisorCommand {
5647 ApprovedConfiguration {
5648 approved: crate::config_gate::runtime::ApprovedDaemonConfiguration,
5649 reply: oneshot::Sender<Result<(), SuperviseError>>,
5650 },
5651 ConfigPolicyProbe {
5652 connection: crate::ConnectionId,
5653 generation: u64,
5654 attempt: u64,
5655 },
5656 ConfigApplyRestart {
5657 generation: u64,
5658 supersedes_swap: bool,
5659 },
5660 ConfigExitHold {
5661 reply: oneshot::Sender<Result<(), SuperviseError>>,
5662 },
5663 ConfigDeliveryFailed {
5664 connection: crate::ConnectionId,
5665 hold: bool,
5666 },
5667 Drain {
5668 requester: Option<Principal>,
5669 reply: oneshot::Sender<Result<(), SuperviseError>>,
5670 },
5671 Retire {
5672 requester: Option<Principal>,
5673 reply: oneshot::Sender<Result<(), SuperviseError>>,
5674 },
5675 Restart {
5676 requester: Option<Principal>,
5677 drain_timeout_ms: Option<u64>,
5682 received_at_generation: u64,
5686 queued_at: Instant,
5689 reply: oneshot::Sender<Result<(), SuperviseError>>,
5690 },
5691 Reload {
5692 requester: Option<Principal>,
5693 reply: oneshot::Sender<Result<(), SuperviseError>>,
5694 },
5695 SetEnabled {
5696 reset_budget: bool,
5697 requester: Option<Principal>,
5698 enabled: bool,
5699 reply: oneshot::Sender<Result<bool, SuperviseError>>,
5700 },
5701 UpdateConfiguration {
5702 spec: ModuleSpec,
5703 health: HealthConfig,
5704 drain_timeout_ms: Option<u64>,
5707 reply: oneshot::Sender<Result<(), SuperviseError>>,
5708 },
5709 Swap {
5710 requester: Option<Principal>,
5711 ready_timeout: Option<Duration>,
5714 reply: oneshot::Sender<Result<SwapOutcome, SuperviseError>>,
5716 },
5717}
5718
5719#[derive(Clone, Copy, Debug, PartialEq, Eq)]
5723pub enum SwapOutcome {
5724 Cutover,
5725 SupersededByConfigRestart,
5726}
5727
5728impl SwapOutcome {
5729 pub fn as_str(self) -> &'static str {
5730 match self {
5731 Self::Cutover => "cutover",
5732 Self::SupersededByConfigRestart => "superseded_by_config_restart",
5733 }
5734 }
5735}
5736
5737#[derive(Debug)]
5738pub enum SuperviseError {
5739 ConfigHeld {
5740 module_id: String,
5741 code: String,
5742 detail: String,
5743 },
5744 ConfigDeliveryDeferred {
5745 module_id: String,
5746 },
5747 InvalidSpec {
5748 reason: String,
5749 },
5750 Spawn {
5751 program: PathBuf,
5752 source: io::Error,
5753 cgroup_path: Option<PathBuf>,
5754 },
5755 Cgroup {
5756 module_id: String,
5757 source: io::Error,
5758 },
5759 LaunchNonce {
5762 reason: String,
5763 },
5764 Wait {
5765 module_id: String,
5766 source: io::Error,
5767 },
5768 Kill {
5769 module_id: String,
5770 source: io::Error,
5771 },
5772 Forwarding(ForwardingError),
5773 Registry(RegistryError),
5774 ReloadUnavailable {
5775 module_id: String,
5776 reason: String,
5777 },
5778 Disabled {
5783 module_id: String,
5784 },
5785 ReloadFailed {
5786 module_id: String,
5787 reason: String,
5788 },
5789 RegistrationStillActive {
5790 module_id: String,
5791 waited: Duration,
5792 },
5793 StatePoisoned {
5794 module_id: Option<String>,
5795 },
5796 CommandClosed {
5797 module_id: String,
5798 },
5799 SwapInProgress {
5803 module_id: String,
5804 },
5805 SwapRefused {
5807 module_id: String,
5808 reason: SwapRefusal,
5809 },
5810 SwapFailed {
5814 module_id: String,
5815 arm: SwapFailureArm,
5816 detail: String,
5817 candidate_exit: Option<ExitReport>,
5820 },
5821}
5822
5823#[derive(Debug, Clone, Copy, PartialEq, Eq)]
5825pub enum SwapRefusal {
5826 OverlapExclusive,
5828 NotRegistered,
5831 ProtocolNone,
5834 NotConfigured,
5837 AlreadySwapping,
5839}
5840
5841impl SwapRefusal {
5842 pub fn as_str(self) -> &'static str {
5843 match self {
5844 Self::OverlapExclusive => "overlap_exclusive",
5845 Self::NotRegistered => "not_registered",
5846 Self::ProtocolNone => "protocol_none",
5847 Self::NotConfigured => "not_configured",
5848 Self::AlreadySwapping => "already_swapping",
5849 }
5850 }
5851}
5852
5853#[derive(Debug, Clone, Copy, PartialEq, Eq)]
5856pub enum SwapFailureArm {
5857 SpawnFailed,
5859 NeverRegistered,
5861 NeverReady,
5863 CandidateExited,
5865 CandidateUnhealthy,
5867 Interrupted,
5871 CutoverLost,
5876}
5877
5878impl SwapFailureArm {
5879 pub fn as_str(self) -> &'static str {
5880 match self {
5881 Self::SpawnFailed => "spawn_failed",
5882 Self::NeverRegistered => "never_registered",
5883 Self::NeverReady => "never_ready",
5884 Self::CandidateExited => "candidate_exited",
5885 Self::CandidateUnhealthy => "candidate_unhealthy",
5886 Self::Interrupted => "interrupted",
5887 Self::CutoverLost => "cutover_lost",
5888 }
5889 }
5890}
5891
5892impl SuperviseError {
5893 pub(crate) fn is_config_gate_error(&self) -> bool {
5894 match self {
5895 Self::ConfigHeld { .. } => true,
5896 Self::Spawn { source, .. } => source
5897 .get_ref()
5898 .is_some_and(|cause| cause.is::<crate::config_gate::GateError>()),
5899 _ => false,
5900 }
5901 }
5902}
5903
5904impl fmt::Display for SuperviseError {
5905 fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
5906 match self {
5907 Self::ConfigHeld { module_id, code, detail } => write!(f, "{code}: module '{module_id}' held: {detail}"),
5908 Self::ConfigDeliveryDeferred { module_id } => write!(f, "config_gated: module '{module_id}' is waiting for its exited delivery target's failure record"),
5909 Self::InvalidSpec { reason } => write!(f, "invalid module spec: {reason}"),
5910 Self::Spawn {
5911 program,
5912 source,
5913 cgroup_path: Some(cgroup_path),
5914 } => write!(
5915 f,
5916 "failed to place module in cgroup '{}' while spawning '{}': {source}",
5917 cgroup_path.display(),
5918 program.display()
5919 ),
5920 Self::Spawn {
5921 program,
5922 source,
5923 cgroup_path: None,
5924 } => write!(
5925 f,
5926 "failed to spawn module '{}': {source}",
5927 program.display()
5928 ),
5929 Self::Cgroup { module_id, source } => {
5930 write!(
5931 f,
5932 "failed to prepare cgroup for module '{module_id}': {source}"
5933 )
5934 }
5935 Self::LaunchNonce { reason } => {
5936 write!(
5937 f,
5938 "failed to generate reserved-module launch nonce: {reason}"
5939 )
5940 }
5941 Self::Wait { module_id, source } => {
5942 write!(f, "failed to wait for module '{module_id}': {source}")
5943 }
5944 Self::Kill { module_id, source } => {
5945 write!(f, "failed to kill module '{module_id}': {source}")
5946 }
5947 Self::Forwarding(err) => write!(f, "forwarding error: {err}"),
5948 Self::Registry(err) => write!(f, "registry error: {err}"),
5949 Self::ReloadUnavailable { module_id, reason } => {
5950 write!(f, "reload unavailable for module '{module_id}': {reason}")
5951 }
5952 Self::Disabled { module_id } => {
5953 write!(
5954 f,
5955 "module '{module_id}' is disabled; enable it before restart or reload"
5956 )
5957 }
5958 Self::ReloadFailed { module_id, reason } => {
5959 write!(f, "reload failed for module '{module_id}': {reason}")
5960 }
5961 Self::RegistrationStillActive { module_id, waited } => write!(
5962 f,
5963 "module '{module_id}' registration remained active after waiting {waited:?}"
5964 ),
5965 Self::StatePoisoned { module_id } => match module_id {
5966 Some(module_id) => {
5967 write!(f, "supervisor state for module '{module_id}' was poisoned")
5968 }
5969 None => write!(f, "supervisor state was poisoned"),
5970 },
5971 Self::CommandClosed { module_id } => {
5972 write!(
5973 f,
5974 "supervisor command channel for module '{module_id}' is closed"
5975 )
5976 }
5977 Self::SwapInProgress { module_id } => write!(
5978 f,
5979 "module '{module_id}' is being swapped; retry once the swap has cut over or failed, or stop the module to abort the swap"
5980 ),
5981 Self::SwapRefused { module_id, reason } => match reason {
5982 SwapRefusal::OverlapExclusive => write!(
5983 f,
5984 "module '{module_id}' is declared overlap: \"exclusive\" (the default): two processes of it must not run at once, so it cannot be swapped; use a plain restart, or declare overlap: \"safe\" in its config if it really tolerates a second process"
5985 ),
5986 SwapRefusal::NotRegistered => write!(
5987 f,
5988 "module '{module_id}' is not registered, so there is no serving process to keep while a replacement warms; use a plain restart"
5989 ),
5990 SwapRefusal::ProtocolNone => write!(
5991 f,
5992 "module '{module_id}' is protocol: \"none\" and never registers, so a swap could never see its replacement become ready; use a plain restart"
5993 ),
5994 SwapRefusal::NotConfigured => write!(
5995 f,
5996 "module '{module_id}' cannot be swapped: the supervisor was built without the forwarding table or shared handle a swap needs"
5997 ),
5998 SwapRefusal::AlreadySwapping => {
5999 write!(f, "module '{module_id}' is already being swapped")
6000 }
6001 },
6002 Self::SwapFailed {
6003 module_id,
6004 arm,
6005 detail,
6006 ..
6007 } => write!(
6008 f,
6009 "swap of module '{module_id}' failed ({}): {detail}; the running process was left serving",
6010 arm.as_str()
6011 ),
6012 }
6013 }
6014}
6015
6016impl Error for SuperviseError {
6017 fn source(&self) -> Option<&(dyn Error + 'static)> {
6018 match self {
6019 Self::Spawn { source, .. }
6020 | Self::Cgroup { source, .. }
6021 | Self::Wait { source, .. }
6022 | Self::Kill { source, .. } => Some(source),
6023 Self::Forwarding(err) => Some(err),
6024 Self::Registry(err) => Some(err),
6025 Self::LaunchNonce { .. }
6026 | Self::ConfigDeliveryDeferred { .. }
6027 | Self::ConfigHeld { .. }
6028 | Self::InvalidSpec { .. }
6029 | Self::ReloadUnavailable { .. }
6030 | Self::Disabled { .. }
6031 | Self::ReloadFailed { .. }
6032 | Self::RegistrationStillActive { .. }
6033 | Self::StatePoisoned { .. }
6034 | Self::CommandClosed { .. }
6035 | Self::SwapInProgress { .. }
6036 | Self::SwapRefused { .. }
6037 | Self::SwapFailed { .. } => None,
6038 }
6039 }
6040}
6041
6042pub(crate) fn validate_spec(spec: &ModuleSpec) -> Result<(), SuperviseError> {
6043 if spec.carry_containment && spec.overlap == ModuleOverlap::Safe {
6044 return Err(SuperviseError::InvalidSpec {
6045 reason: "carry_containment cannot be combined with overlap: safe".to_owned(),
6046 });
6047 }
6048 if spec.module_id.trim().is_empty() {
6049 return Err(SuperviseError::InvalidSpec {
6050 reason: "module_id must not be empty".to_string(),
6051 });
6052 }
6053
6054 Ok(())
6055}
6056
6057#[derive(Debug, Default)]
6058struct HealthProbeRuntime {
6059 spawn_generation: Option<u64>,
6060 configured_health: Option<HealthConfig>,
6061 registered_connection: Option<crate::ConnectionId>,
6062 advertised: bool,
6063 next_probe_at: Option<Instant>,
6064 probe_index: u64,
6065}
6066
6067fn running_protocol(spec: &ModuleSpec, snapshot: &SharedSnapshot) -> ModuleProtocol {
6068 lock_snapshot(snapshot)
6069 .ok()
6070 .and_then(|state| state.spawned_protocol)
6071 .unwrap_or(spec.protocol)
6072}
6073
6074impl HealthProbeRuntime {
6075 fn refresh_registration(
6076 &mut self,
6077 spec: &ModuleSpec,
6078 runtime: &SupervisorRuntimeConfig,
6079 registry: &Registry,
6080 snapshot: &SharedSnapshot,
6081 ) {
6082 let generation = lock_snapshot(snapshot)
6083 .ok()
6084 .map(|state| state.spawn_generation);
6085 if self.spawn_generation != generation {
6086 *self = Self {
6090 spawn_generation: generation,
6091 ..Self::default()
6092 };
6093 }
6094 if self.configured_health.as_ref() != Some(&runtime.health) {
6095 self.configured_health = Some(runtime.health.clone());
6096 self.next_probe_at = None;
6097 self.registered_connection = None;
6098 self.probe_index = 0;
6099 }
6100 if running_protocol(spec, snapshot) == ModuleProtocol::None {
6104 self.registered_connection = None;
6105 self.advertised = runtime.health.http.is_some();
6106 if !self.advertised {
6107 self.next_probe_at = None;
6108 let _ = update_snapshot(snapshot, Some(&spec.module_id), |state| {
6109 let unknown = ModuleHealthStatus::default();
6110 if state.health != unknown {
6111 state.health = unknown;
6112 state.last_probe_evidence = None;
6113 }
6114 });
6115 } else if self.next_probe_at.is_none() {
6116 self.next_probe_at = Some(
6117 Instant::now()
6118 + jittered_health_delay(&spec.module_id, 0, runtime.health.cadence),
6119 );
6120 }
6121 return;
6122 }
6123
6124 let registration = match registry.get_module(&spec.module_id) {
6125 Ok(registration) => registration,
6126 Err(err) => {
6127 warn!(module_id = %spec.module_id, error = %err, "health prober could not read registry");
6128 self.advertised = false;
6129 self.next_probe_at = None;
6130 return;
6131 }
6132 };
6133
6134 let Some(registration) = registration else {
6135 self.registered_connection = None;
6136 self.advertised = false;
6137 self.next_probe_at = None;
6138 return;
6139 };
6140
6141 let advertised = registration
6142 .control_ops
6143 .iter()
6144 .any(|op| op == MODULE_CONTROL_OP_HEALTH_CHECK);
6145 if !advertised {
6146 self.registered_connection = Some(registration.connection_id);
6147 self.advertised = false;
6148 self.next_probe_at = None;
6149 let _ = update_snapshot(snapshot, Some(&spec.module_id), |state| {
6150 if state.health.status != SupervisorHealthStatus::Unknown
6151 || state.health.consecutive_failures != 0
6152 || state.health.last_probe_ms.is_some()
6153 || state.health.detail.is_some()
6154 || state.health.metrics.is_some()
6155 {
6156 state.health.status = SupervisorHealthStatus::Unknown;
6157 state.last_probe_evidence = None;
6158 state.health.consecutive_failures = 0;
6159 state.health.last_probe_ms = None;
6160 state.health.detail = None;
6161 state.health.metrics = None;
6162 }
6163 });
6164 return;
6165 }
6166
6167 let reregistered = self.registered_connection != Some(registration.connection_id);
6168 self.registered_connection = Some(registration.connection_id);
6169 self.advertised = true;
6170 if reregistered || self.next_probe_at.is_none() {
6171 self.probe_index = 0;
6172 self.next_probe_at = Some(
6173 Instant::now() + jittered_health_delay(&spec.module_id, 0, runtime.health.cadence),
6174 );
6175 let _ = update_snapshot(snapshot, Some(&spec.module_id), |state| {
6176 state.health.status = SupervisorHealthStatus::Unknown;
6177 state.last_probe_evidence = None;
6178 state.health.consecutive_failures = 0;
6179 state.health.detail = None;
6180 state.health.metrics = None;
6181 });
6182 }
6183 }
6184
6185 fn wake_at(&self) -> Option<Instant> {
6186 if !self.advertised {
6187 return None;
6188 }
6189 self.next_probe_at
6190 }
6191
6192 fn due(&self) -> bool {
6193 self.advertised
6194 && self
6195 .next_probe_at
6196 .is_some_and(|next| Instant::now() >= next)
6197 }
6198
6199 fn schedule_next(&mut self, spec: &ModuleSpec, cadence: Duration) {
6200 self.probe_index = self.probe_index.wrapping_add(1);
6201 self.next_probe_at = Some(
6202 Instant::now() + jittered_health_delay(&spec.module_id, self.probe_index, cadence),
6203 );
6204 }
6205}
6206
6207#[derive(Debug, Clone, Copy, PartialEq, Eq)]
6211enum HealthProbeEvidence {
6212 LaneDead,
6214 NoAnswer,
6216 BadAnswer,
6218 Misconfigured,
6220}
6221
6222#[derive(Debug)]
6223struct HealthProbeError {
6224 evidence: HealthProbeEvidence,
6225 message: String,
6226}
6227
6228impl HealthProbeError {
6229 fn lane_dead(message: impl Into<String>) -> Self {
6230 Self::with(HealthProbeEvidence::LaneDead, message)
6231 }
6232
6233 fn no_answer(message: impl Into<String>) -> Self {
6234 Self::with(HealthProbeEvidence::NoAnswer, message)
6235 }
6236
6237 fn bad_answer(message: impl Into<String>) -> Self {
6238 Self::with(HealthProbeEvidence::BadAnswer, message)
6239 }
6240
6241 fn misconfigured(message: impl Into<String>) -> Self {
6242 Self::with(HealthProbeEvidence::Misconfigured, message)
6243 }
6244
6245 fn with(evidence: HealthProbeEvidence, message: impl Into<String>) -> Self {
6246 Self {
6247 evidence,
6248 message: message.into(),
6249 }
6250 }
6251
6252 #[allow(dead_code)]
6261 fn is_proof_of_death(&self) -> bool {
6262 matches!(self.evidence, HealthProbeEvidence::LaneDead)
6263 }
6264
6265 fn label(&self) -> &'static str {
6273 match self.evidence {
6274 HealthProbeEvidence::LaneDead => "lane-dead",
6275 HealthProbeEvidence::NoAnswer => "no-answer",
6276 HealthProbeEvidence::BadAnswer => "bad-answer",
6277 HealthProbeEvidence::Misconfigured => "daemon-misconfigured",
6278 }
6279 }
6280}
6281
6282impl fmt::Display for HealthProbeError {
6283 fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
6284 f.write_str(&self.message)
6285 }
6286}
6287
6288async fn run_health_probe_cycle(
6289 spec: &ModuleSpec,
6290 runtime: &SupervisorRuntimeConfig,
6291 registry: &Registry,
6292 process_liveness: &SupervisorProcessLiveness,
6293 snapshot: &SharedSnapshot,
6294 child: &mut Option<SupervisedChild>,
6295) {
6296 let now_ms = unix_ms_now();
6297 let delivery_probe = match &runtime.config_delivery {
6298 Some(delivery) => {
6299 delivery
6300 .health_probe(&spec.module_id, runtime.forwarding.as_deref(), registry)
6301 .await
6302 }
6303 None => None,
6304 };
6305 let http = (running_protocol(spec, snapshot) == ModuleProtocol::None)
6306 .then_some(runtime.health.http.as_deref())
6307 .flatten();
6308 let result = match http {
6309 Some(url) => probe_http_health(url, runtime.health.deadline)
6310 .await
6311 .map(|report| (report, None)),
6312 None => probe_module_health_from(&spec.module_id, runtime, None)
6313 .await
6314 .map(|(report, probed)| (report, Some(probed))),
6315 };
6316 match result {
6317 Ok((report, probed)) => {
6318 handle_health_report(
6319 spec,
6320 runtime,
6321 registry,
6322 process_liveness,
6323 snapshot,
6324 child,
6325 report,
6326 now_ms,
6327 delivery_probe,
6328 probed,
6329 )
6330 .await;
6331 }
6332 Err(err) => {
6333 if http.is_some() {
6334 let _ = update_snapshot(snapshot, Some(&spec.module_id), |state| {
6335 state.health.status = SupervisorHealthStatus::Failing;
6336 });
6337 }
6338 handle_health_probe_failure(
6339 spec,
6340 runtime,
6341 registry,
6342 process_liveness,
6343 snapshot,
6344 child,
6345 err,
6346 now_ms,
6347 delivery_probe,
6348 )
6349 .await;
6350 }
6351 }
6352}
6353
6354pub(crate) struct HttpProbeTarget<'a> {
6355 address: std::net::SocketAddr,
6356 localhost: bool,
6357 authority: &'a str,
6358 path: String,
6359}
6360
6361pub(crate) fn parse_http_probe_url(url: &str) -> Result<HttpProbeTarget<'_>, String> {
6364 if url.bytes().any(|b| b <= b' ' || b == 127) || url.contains('#') {
6365 return Err("must not contain whitespace, controls, or a fragment".into());
6366 }
6367 let rest = url
6368 .strip_prefix("http://")
6369 .ok_or("must use plain http://")?;
6370 let split = rest.find(['/', '?']).unwrap_or(rest.len());
6371 let (authority, suffix) = rest.split_at(split);
6372 let (host, port) = if let Some(rest) = authority.strip_prefix("[::1]") {
6373 ("::1", rest)
6374 } else {
6375 let split = authority.find(':').unwrap_or(authority.len());
6376 authority.split_at(split)
6377 };
6378 let ip: std::net::IpAddr = match host {
6379 "127.0.0.1" | "localhost" => std::net::Ipv4Addr::LOCALHOST.into(),
6380 "::1" => std::net::Ipv6Addr::LOCALHOST.into(),
6381 _ => return Err("host must be 127.0.0.1, [::1], or localhost".into()),
6382 };
6383 let port = if port.is_empty() {
6384 80
6385 } else {
6386 port.strip_prefix(':')
6387 .and_then(|p| p.parse::<u16>().ok())
6388 .filter(|p| *p > 0)
6389 .ok_or("must have a valid nonzero TCP port")?
6390 };
6391 let path = if suffix.is_empty() {
6392 "/".into()
6393 } else if suffix.starts_with('?') {
6394 format!("/{suffix}")
6395 } else {
6396 suffix.into()
6397 };
6398 Ok(HttpProbeTarget {
6399 address: std::net::SocketAddr::new(ip, port),
6400 localhost: host == "localhost",
6401 authority,
6402 path,
6403 })
6404}
6405
6406async fn probe_http_health(
6407 url: &str,
6408 deadline: Duration,
6409) -> Result<HealthReport, HealthProbeError> {
6410 use tokio::io::{AsyncReadExt, AsyncWriteExt, BufReader};
6411 let target = parse_http_probe_url(url).map_err(HealthProbeError::misconfigured)?;
6412 let mut response_status = String::new();
6415 let mut body = Vec::new();
6416 let probe = async {
6417 let connection = match tokio::net::TcpStream::connect(target.address).await {
6420 Err(_) if target.localhost => {
6421 tokio::net::TcpStream::connect((
6422 std::net::Ipv6Addr::LOCALHOST,
6423 target.address.port(),
6424 ))
6425 .await
6426 }
6427 result => result,
6428 };
6429 let mut stream = connection.map_err(|error| {
6430 HealthProbeError::no_answer(format!("HTTP connect failed: {error}"))
6431 })?;
6432 stream.write_all(format!("GET {} HTTP/1.1\r\nHost: {}\r\nAccept-Encoding: identity\r\nConnection: close\r\n\r\n", target.path, target.authority).as_bytes()).await
6433 .map_err(|error| HealthProbeError::no_answer(format!("HTTP write failed: {error}")))?;
6434 let mut reader = BufReader::new(stream);
6435 let mut budget = 16 * 1024;
6436 let status = http_line(&mut reader, &mut budget).await?;
6437 let mut words = status.split_ascii_whitespace();
6438 let version = words.next();
6439 let code = words
6440 .next()
6441 .filter(|word| word.len() == 3)
6442 .and_then(|word| word.parse::<u16>().ok());
6443 if !matches!(version, Some("HTTP/1.1" | "HTTP/1.0"))
6444 || !code.is_some_and(|code| (100..600).contains(&code))
6445 {
6446 return Err(HealthProbeError::bad_answer(format!(
6447 "invalid HTTP status: {status}"
6448 )));
6449 }
6450 let code = code.expect("validated status code");
6451 response_status = status.clone();
6452 let mut length = None;
6453 let mut chunked = false;
6454 loop {
6455 let line = http_line(&mut reader, &mut budget).await?;
6456 if line.is_empty() {
6457 break;
6458 }
6459 if let Some((name, value)) = line.split_once(':') {
6460 if name.eq_ignore_ascii_case("content-length") {
6461 length = Some(value.trim().parse::<u64>().map_err(|_| {
6462 HealthProbeError::bad_answer("invalid HTTP Content-Length")
6463 })?);
6464 } else if name.eq_ignore_ascii_case("transfer-encoding") {
6465 chunked = value.trim().eq_ignore_ascii_case("chunked");
6466 }
6467 }
6468 }
6469 if chunked {
6470 while body.len() < 200 {
6471 let line = http_line(&mut reader, &mut budget).await?;
6472 let size = u64::from_str_radix(line.split(';').next().unwrap_or("").trim(), 16)
6473 .map_err(|_| HealthProbeError::bad_answer("invalid HTTP chunk size"))?;
6474 if size == 0 {
6475 break;
6476 }
6477 let count = size.min((200 - body.len()) as u64) as usize;
6478 let start = body.len();
6479 (&mut reader)
6480 .take(count as u64)
6481 .read_to_end(&mut body)
6482 .await
6483 .map_err(|error| {
6484 HealthProbeError::no_answer(format!("HTTP body read failed: {error}"))
6485 })?;
6486 if body.len() - start != count {
6487 return Err(HealthProbeError::bad_answer("truncated HTTP chunk"));
6488 }
6489 if size > count as u64 || body.len() == 200 {
6490 break;
6491 }
6492 if !http_line(&mut reader, &mut budget).await?.is_empty() {
6493 return Err(HealthProbeError::bad_answer("invalid HTTP chunk delimiter"));
6494 }
6495 }
6496 } else {
6497 reader
6498 .take(length.unwrap_or(200).min(200))
6499 .read_to_end(&mut body)
6500 .await
6501 .map_err(|error| {
6502 HealthProbeError::no_answer(format!("HTTP body read failed: {error}"))
6503 })?;
6504 }
6505 if (200..300).contains(&code) {
6506 Ok(HealthReport::ok())
6507 } else {
6508 Err(HealthProbeError::bad_answer(
6509 "HTTP health endpoint returned non-2xx",
6510 ))
6511 }
6512 };
6513 let mut result = timeout(deadline, probe).await.unwrap_or_else(|_| {
6514 Err(HealthProbeError::no_answer(format!(
6515 "HTTP probe timed out after {deadline:?}"
6516 )))
6517 });
6518 if let Err(error) = &mut result {
6519 if !response_status.is_empty() {
6520 error.message = format!(
6521 "{}; {response_status}: {}",
6522 error.message,
6523 String::from_utf8_lossy(&body)
6524 );
6525 }
6526 }
6527 result
6528}
6529
6530async fn http_line(
6531 reader: &mut tokio::io::BufReader<tokio::net::TcpStream>,
6532 remaining: &mut usize,
6533) -> Result<String, HealthProbeError> {
6534 use tokio::io::{AsyncBufReadExt, AsyncReadExt};
6535 let mut line = Vec::new();
6536 (&mut *reader)
6537 .take(*remaining as u64)
6538 .read_until(b'\n', &mut line)
6539 .await
6540 .map_err(|error| {
6541 HealthProbeError::no_answer(format!("HTTP header read failed: {error}"))
6542 })?;
6543 *remaining -= line.len();
6544 if !line.ends_with(b"\r\n") {
6545 return Err(HealthProbeError::bad_answer(
6546 "HTTP headers are incomplete or exceed 16 KiB",
6547 ));
6548 }
6549 line.truncate(line.len() - 2);
6550 String::from_utf8(line).map_err(|_| HealthProbeError::bad_answer("HTTP header is not UTF-8"))
6551}
6552
6553async fn probe_module_health(
6554 module_id: &str,
6555 runtime: &SupervisorRuntimeConfig,
6556 drain_deadline: Option<Instant>,
6557) -> Result<HealthReport, HealthProbeError> {
6558 probe_module_health_from(module_id, runtime, drain_deadline)
6559 .await
6560 .map(|(report, _)| report)
6561}
6562
6563async fn probe_module_health_from(
6566 module_id: &str,
6567 runtime: &SupervisorRuntimeConfig,
6568 drain_deadline: Option<Instant>,
6569) -> Result<(HealthReport, ConnectionId), HealthProbeError> {
6570 let Some(forwarding) = runtime.forwarding.as_ref() else {
6571 return Err(HealthProbeError::misconfigured(
6572 "supervisor was not configured with a forwarding table",
6573 ));
6574 };
6575 let probe_started_at = Instant::now();
6576 let mut deadline = probe_started_at + runtime.health.deadline;
6577 if let Some(drain_deadline) = drain_deadline {
6578 deadline = deadline.min(drain_deadline);
6579 }
6580 let pending = if drain_deadline.is_some() {
6581 forwarding.begin_drain_health_probe_rpc_for(
6582 module_id,
6583 MODULE_CONTROL_OP_HEALTH_CHECK,
6584 probe_started_at,
6585 deadline,
6586 )
6587 } else {
6588 forwarding.begin_health_probe_rpc_for(
6589 module_id,
6590 MODULE_CONTROL_OP_HEALTH_CHECK,
6591 probe_started_at,
6592 deadline,
6593 )
6594 }
6595 .map_err(|err| {
6596 HealthProbeError::lane_dead(format!("failed to begin health.check RPC: {err}"))
6599 })?;
6600 let probed = pending.endpoint.connection_id;
6601 await_health_probe(forwarding, pending, deadline, runtime.health.deadline)
6602 .await
6603 .map(|report| (report, probed))
6604}
6605
6606async fn probe_endpoint_health(
6613 endpoint: crate::ModuleEndpointId,
6614 runtime: &SupervisorRuntimeConfig,
6615 deadline_cap: Option<Instant>,
6616) -> Result<HealthReport, HealthProbeError> {
6617 let Some(forwarding) = runtime.forwarding.as_ref() else {
6618 return Err(HealthProbeError::misconfigured(
6619 "supervisor was not configured with a forwarding table",
6620 ));
6621 };
6622 let probe_started_at = Instant::now();
6623 let mut deadline = probe_started_at + runtime.health.deadline;
6624 if let Some(cap) = deadline_cap {
6625 deadline = deadline.min(cap);
6626 }
6627 let pending = forwarding
6628 .begin_endpoint_health_probe_rpc_for(
6629 endpoint,
6630 MODULE_CONTROL_OP_HEALTH_CHECK,
6631 probe_started_at,
6632 deadline,
6633 )
6634 .map_err(|err| {
6635 HealthProbeError::lane_dead(format!("failed to begin health.check RPC: {err}"))
6636 })?;
6637 await_health_probe(forwarding, pending, deadline, runtime.health.deadline).await
6638}
6639
6640async fn await_health_probe(
6642 forwarding: &ForwardingTable,
6643 pending: PendingModuleControlRpc,
6644 deadline: Instant,
6645 probe_budget: Duration,
6646) -> Result<HealthReport, HealthProbeError> {
6647 let PendingModuleControlRpc {
6648 endpoint,
6649 module_sink,
6650 negotiated_ver,
6651 corr,
6652 receiver,
6653 } = pending;
6654 let body = serde_json::to_vec(&ModuleControlRequest::HealthCheck {}).map_err(|err| {
6655 HealthProbeError::misconfigured(format!("failed to encode health.check: {err}"))
6656 })?;
6657 let frame = Frame::build_with_version(
6658 negotiated_ver,
6659 FrameType::Request,
6660 control_flags(),
6661 0,
6662 0,
6663 corr,
6664 body,
6665 )
6666 .map_err(|err| {
6667 HealthProbeError::misconfigured(format!("failed to build health.check frame: {err}"))
6668 })?;
6669
6670 match timeout_at(deadline, module_sink.send(frame)).await {
6676 Ok(Ok(())) => {}
6677 Ok(Err(err)) => {
6678 let _ = forwarding.cancel_module_control_rpc(endpoint, corr);
6679 return Err(HealthProbeError::lane_dead(format!(
6682 "failed to send health.check: {err}"
6683 )));
6684 }
6685 Err(_elapsed) => {
6686 let _ = forwarding.cancel_module_control_rpc(endpoint, corr);
6687 return Err(HealthProbeError::no_answer(
6691 "health.check send timed out before enqueue (module egress full)",
6692 ));
6693 }
6694 }
6695
6696 match timeout_at(deadline, receiver).await {
6697 Ok(Ok(ModuleControlRpcOutcome::RawResponse(_))) => Err(HealthProbeError::bad_answer(
6698 "health.check returned an unexpected config response",
6699 )),
6700 Ok(Ok(ModuleControlRpcOutcome::Response(response))) => {
6704 response.health_report().ok_or_else(|| {
6705 HealthProbeError::bad_answer("health.check RPC returned a non-health response")
6706 })
6707 }
6708 Ok(Ok(ModuleControlRpcOutcome::Rejected(body))) => Err(HealthProbeError::bad_answer(
6709 format!("health.check rejected: {}", body.message),
6710 )),
6711 Ok(Ok(ModuleControlRpcOutcome::ModuleGone(message))) => {
6712 Err(HealthProbeError::lane_dead(message))
6713 }
6714 Ok(Ok(ModuleControlRpcOutcome::MalformedResponse(message))) => {
6715 Err(HealthProbeError::bad_answer(message))
6716 }
6717 Ok(Ok(ModuleControlRpcOutcome::UnexpectedOp { expected, actual })) => {
6718 Err(HealthProbeError::bad_answer(format!(
6719 "expected module-control op '{expected}', got '{actual}'"
6720 )))
6721 }
6722 Ok(Ok(ModuleControlRpcOutcome::DeadlineElapsed)) => Err(HealthProbeError::bad_answer(
6726 "module answered health.check after its daemon deadline",
6727 )),
6728 Ok(Err(_)) => Err(HealthProbeError::misconfigured(
6729 "health.check waiter was canceled before the module responded",
6730 )),
6731 Err(_) => {
6732 let _ = forwarding.tombstone_health_probe_rpc(endpoint, corr);
6733 Err(HealthProbeError::no_answer(format!(
6734 "module did not answer health.check within {probe_budget:?}"
6735 )))
6736 }
6737 }
6738}
6739
6740#[allow(clippy::too_many_arguments)]
6744async fn handle_health_report(
6745 spec: &ModuleSpec,
6746 runtime: &SupervisorRuntimeConfig,
6747 registry: &Registry,
6748 process_liveness: &SupervisorProcessLiveness,
6749 snapshot: &SharedSnapshot,
6750 child: &mut Option<SupervisedChild>,
6751 report: HealthReport,
6752 now_ms: u64,
6753 delivery_probe: Option<crate::config_gate::runtime::DeliveryProbe>,
6754 probed: Option<ConnectionId>,
6755) {
6756 let delivery_failed = match (&runtime.config_delivery, delivery_probe) {
6757 (Some(delivery), Some(probe)) => {
6758 delivery
6759 .health_result(
6760 &spec.module_id,
6761 runtime.forwarding.as_deref(),
6762 registry,
6763 probe,
6764 report.status == HealthStatus::Ok,
6765 )
6766 .await
6767 }
6768 _ => false,
6769 };
6770 if let (Some(delivery), Some(spawn)) = (
6771 &runtime.config_delivery,
6772 child
6773 .as_ref()
6774 .and_then(|child| child.config_delivery.as_ref()),
6775 ) {
6776 delivery
6777 .mark_readiness(
6778 &spec.module_id,
6779 runtime.forwarding.as_deref(),
6780 probed,
6781 spawn,
6782 report.status == HealthStatus::Ok,
6783 )
6784 .await;
6785 }
6786 let status = supervisor_health_status(report.status);
6787 let detail = report.detail.clone();
6788 let metrics = truncate_health_metrics(report.metrics);
6789 let _ = update_snapshot(snapshot, Some(&spec.module_id), |state| {
6790 state.health.status = status;
6791 state.last_probe_evidence = None;
6792 state.health.last_probe_ms = Some(now_ms);
6793 state.health.detail = detail.clone();
6794 state.health.metrics = metrics.clone();
6795 state.health.consecutive_failures = 0;
6796 state.failed_retry.observe_health(
6797 report.status == HealthStatus::Ok,
6798 runtime.restart_policy.window,
6799 );
6800 });
6801
6802 if delivery_failed {
6803 return;
6804 }
6805 let action = match report.status {
6806 HealthStatus::Ok => return,
6807 HealthStatus::Degraded => runtime.health.on_degraded,
6808 HealthStatus::Failing => runtime.health.on_failing,
6809 };
6810 apply_l3_health_action(
6811 spec,
6812 runtime,
6813 registry,
6814 process_liveness,
6815 snapshot,
6816 child,
6817 status,
6818 detail.as_deref(),
6819 action,
6820 now_ms,
6821 )
6822 .await;
6823}
6824
6825#[allow(clippy::too_many_arguments)]
6826async fn handle_health_probe_failure(
6827 spec: &ModuleSpec,
6828 runtime: &SupervisorRuntimeConfig,
6829 registry: &Registry,
6830 process_liveness: &SupervisorProcessLiveness,
6831 snapshot: &SharedSnapshot,
6832 child: &mut Option<SupervisedChild>,
6833 err: HealthProbeError,
6834 now_ms: u64,
6835 delivery_probe: Option<crate::config_gate::runtime::DeliveryProbe>,
6836) {
6837 let delivery_failed = match (&runtime.config_delivery, delivery_probe) {
6838 (Some(delivery), Some(probe)) => {
6839 delivery
6840 .health_result(
6841 &spec.module_id,
6842 runtime.forwarding.as_deref(),
6843 registry,
6844 probe,
6845 false,
6846 )
6847 .await
6848 }
6849 _ => false,
6850 };
6851 let threshold = runtime.health.failure_threshold.max(1);
6852 let held = if err.evidence == HealthProbeEvidence::NoAnswer {
6853 runtime.host_health.record_late_module(&spec.module_id)
6854 } else {
6855 false
6856 };
6857 let mut failures = 0;
6858 let detail = if held {
6863 format!("[{}] {err}; host overloaded", err.label())
6864 } else {
6865 format!("[{}] {err}", err.label())
6866 };
6867 let _ = update_snapshot(snapshot, Some(&spec.module_id), |state| {
6868 state.last_probe_evidence = Some(err.evidence);
6869 state
6870 .failed_retry
6871 .observe_health(false, runtime.restart_policy.window);
6872 if held {
6875 state.health.status = SupervisorHealthStatus::Degraded;
6876 } else if state.spawned_protocol.unwrap_or(spec.protocol) == ModuleProtocol::Subc {
6877 state.health.status = SupervisorHealthStatus::Unknown;
6878 }
6879 state.health.last_probe_ms = Some(now_ms);
6880 state.health.consecutive_failures = if held {
6883 0
6884 } else {
6885 state.health.consecutive_failures.saturating_add(1)
6886 };
6887 state.health.detail = Some(detail.clone());
6888 state.health.metrics = None;
6889 failures = state.health.consecutive_failures;
6890 });
6891
6892 if delivery_failed {
6893 return;
6894 }
6895 if held || failures < threshold {
6896 warn!(
6897 module_id = %spec.module_id,
6898 consecutive_failures = failures,
6899 threshold,
6900 evidence = err.label(),
6901 detail = %detail,
6902 "health.check probe failed"
6903 );
6904 return;
6905 }
6906
6907 let _ = update_snapshot(snapshot, Some(&spec.module_id), |state| {
6908 state.state = ModuleState::Unresponsive;
6909 state.health.status = SupervisorHealthStatus::Unresponsive;
6910 });
6911 if runtime.health.critical {
6915 error!(
6916 module_id = %spec.module_id,
6917 status = "unresponsive",
6918 evidence = err.label(),
6919 detail = %detail,
6920 "critical module health alert"
6921 );
6922 } else {
6923 warn!(
6924 module_id = %spec.module_id,
6925 status = "unresponsive",
6926 evidence = err.label(),
6927 detail = %detail,
6928 "module health threshold breached"
6929 );
6930 }
6931 if let Err(err) = health_restart_child(
6932 spec,
6933 runtime,
6934 registry,
6935 process_liveness,
6936 snapshot,
6937 child,
6938 SupervisorHealthStatus::Unresponsive,
6939 Some(&detail),
6940 now_ms,
6941 )
6942 .await
6943 {
6944 error!(module_id = %spec.module_id, error = %err, "health-triggered restart failed");
6945 }
6946}
6947
6948#[allow(clippy::too_many_arguments)]
6949async fn apply_l3_health_action(
6950 spec: &ModuleSpec,
6951 runtime: &SupervisorRuntimeConfig,
6952 registry: &Registry,
6953 process_liveness: &SupervisorProcessLiveness,
6954 snapshot: &SharedSnapshot,
6955 child: &mut Option<SupervisedChild>,
6956 status: SupervisorHealthStatus,
6957 detail: Option<&str>,
6958 action: HealthAction,
6959 now_ms: u64,
6960) {
6961 record_health_action(snapshot, &spec.module_id, action.to_string(), now_ms);
6962 match action {
6963 HealthAction::Report => {
6964 info!(
6965 module_id = %spec.module_id,
6966 status = ?status,
6967 detail,
6968 "module reported non-ok health"
6969 );
6970 }
6971 HealthAction::Alert => {
6972 error!(
6973 module_id = %spec.module_id,
6974 status = ?status,
6975 detail,
6976 "module health alert"
6977 );
6978 }
6979 HealthAction::Restart => {
6980 if let Err(err) = health_restart_child(
6981 spec,
6982 runtime,
6983 registry,
6984 process_liveness,
6985 snapshot,
6986 child,
6987 status,
6988 detail,
6989 now_ms,
6990 )
6991 .await
6992 {
6993 error!(module_id = %spec.module_id, error = %err, "health-triggered restart failed");
6994 }
6995 }
6996 }
6997}
6998
6999#[allow(clippy::too_many_arguments)]
7000async fn health_restart_child(
7001 spec: &ModuleSpec,
7002 runtime: &SupervisorRuntimeConfig,
7003 registry: &Registry,
7004 process_liveness: &SupervisorProcessLiveness,
7005 snapshot: &SharedSnapshot,
7006 child: &mut Option<SupervisedChild>,
7007 status: SupervisorHealthStatus,
7008 detail: Option<&str>,
7009 now_ms: u64,
7010) -> Result<(), SuperviseError> {
7011 budgeted_restart_child(
7012 spec,
7013 runtime,
7014 registry,
7015 process_liveness,
7016 snapshot,
7017 child,
7018 status,
7019 detail,
7020 now_ms,
7021 true,
7022 )
7023 .await
7024}
7025
7026#[allow(clippy::too_many_arguments)]
7027async fn budgeted_restart_child(
7028 spec: &ModuleSpec,
7029 runtime: &SupervisorRuntimeConfig,
7030 registry: &Registry,
7031 process_liveness: &SupervisorProcessLiveness,
7032 snapshot: &SharedSnapshot,
7033 child: &mut Option<SupervisedChild>,
7034 status: SupervisorHealthStatus,
7035 detail: Option<&str>,
7036 now_ms: u64,
7037 health_failure: bool,
7038) -> Result<(), SuperviseError> {
7039 if let Some(error) = runtime
7040 .config_delivery
7041 .as_ref()
7042 .and_then(|delivery| delivery.0.check_module_hold(&spec.module_id).err())
7043 .filter(|error| error.code != "config_delivery_deferred")
7044 {
7045 return Err(config_delivery::gate_hold_error(
7046 spec,
7047 error,
7048 runtime.config_delivery.as_ref(),
7049 ));
7050 }
7051 let recovery_reason = if health_failure {
7052 "health_check"
7053 } else {
7054 "configuration_delivery"
7055 };
7056 let (enabled, schedule) = {
7057 let mut state = lock_snapshot(snapshot)?;
7058 let enabled = state.enabled;
7059 let schedule = if enabled {
7060 state.next_crash_restart(&runtime.restart_policy, Instant::now())
7061 } else {
7062 None
7063 };
7064 (enabled, schedule)
7065 };
7066
7067 if !enabled {
7068 return Err(SuperviseError::Disabled {
7069 module_id: spec.module_id.clone(),
7070 });
7071 }
7072
7073 if schedule.is_none() {
7074 if health_failure {
7075 record_health_action(snapshot, &spec.module_id, "failed".to_string(), now_ms);
7076 }
7077 error!(
7078 module_id = %spec.module_id,
7079 status = ?status,
7080 detail,
7081 max_restarts = runtime.restart_policy.max_restarts,
7082 window_secs = runtime.restart_policy.window.as_secs(),
7083 reason = %runtime.restart_policy.budget_exhausted_detail(),
7084 recovery_reason,
7085 "automatic restart budget exhausted; marking module failed"
7086 );
7087 let stop_notice = begin_forwarding_drain_if_configured(
7088 spec,
7089 runtime,
7090 registry,
7091 snapshot,
7092 Some(true),
7093 RouteCloseReason::Disable,
7094 )
7095 .await?;
7096 update_snapshot(snapshot, Some(&spec.module_id), |state| {
7097 state.drain_disposition_detail = Some(runtime.restart_policy.budget_exhausted_detail());
7098 })?;
7099 let result = drain_optional_child(
7100 &spec.module_id,
7101 spec.protocol,
7102 stop_notice,
7103 registry,
7104 runtime.forwarding.as_deref(),
7105 snapshot,
7106 &runtime.terminal_ring,
7107 &runtime.spawn_events,
7108 child,
7109 runtime.drain_timeout,
7110 ModuleState::Failed,
7111 Some(true),
7112 &runtime.failure_context(true),
7113 )
7114 .await;
7115 update_snapshot(snapshot, Some(&spec.module_id), |state| {
7116 if state.state == ModuleState::Failed && !runtime.child_roster.is_closed() {
7117 state.failed_retry.schedule(&spec.module_id);
7118 }
7119 })?;
7120 result?;
7121 process_liveness.untrack_if_current(&spec.module_id, snapshot);
7122 return Ok(());
7123 }
7124
7125 let schedule = schedule.expect("a health restart must have a crash-restart schedule");
7126 let mut restart_count = 0;
7127 update_snapshot(snapshot, Some(&spec.module_id), |state| {
7128 restart_count = state.crash_restarts.len();
7129 state.state = ModuleState::Unresponsive;
7130 if health_failure {
7131 state.health.status = status;
7132 state.health.last_action = Some(HealthAction::Restart.to_string());
7133 state.health.last_action_ms = Some(now_ms);
7134 }
7135 })?;
7136 warn!(
7137 module_id = %spec.module_id,
7138 status = ?status,
7139 detail,
7140 restart_count,
7141 restart_in_window = schedule.restart_in_window,
7142 delay_ms = schedule.delay.as_millis() as u64,
7143 recovery_reason,
7144 "automatic module restart"
7145 );
7146
7147 let stop_notice = begin_forwarding_drain_if_configured(
7148 spec,
7149 runtime,
7150 registry,
7151 snapshot,
7152 Some(true),
7153 RouteCloseReason::Restart,
7154 )
7155 .await?;
7156 drain_optional_child(
7157 &spec.module_id,
7158 spec.protocol,
7159 stop_notice,
7160 registry,
7161 runtime.forwarding.as_deref(),
7162 snapshot,
7163 &runtime.terminal_ring,
7164 &runtime.spawn_events,
7165 child,
7166 runtime.drain_timeout,
7167 ModuleState::Restarting,
7168 Some(true),
7169 &runtime.failure_context(true),
7170 )
7171 .await?;
7172 schedule_respawn(
7173 runtime,
7174 snapshot,
7175 &spec.module_id,
7176 schedule.delay,
7177 RespawnKind::Spawn,
7178 true,
7179 )
7180}
7181
7182fn cancel_deferred_reload(runtime: &SupervisorRuntimeConfig, module_id: &str, reason: &str) {
7183 if let Some(reply) = runtime
7184 .deferred_reload_reply
7185 .lock()
7186 .unwrap_or_else(|p| p.into_inner())
7187 .take()
7188 {
7189 let _ = reply.send(Err(SuperviseError::ReloadFailed {
7190 module_id: module_id.to_string(),
7191 reason: reason.to_string(),
7192 }));
7193 }
7194}
7195
7196fn park_config_spawn(
7197 spec: &ModuleSpec,
7198 runtime: &SupervisorRuntimeConfig,
7199 snapshot: &SharedSnapshot,
7200 failure: &FailureContext,
7201 admission_refused: Option<String>,
7202) -> Result<(), SuperviseError> {
7203 *runtime
7204 .scheduled_respawn
7205 .lock()
7206 .unwrap_or_else(|poisoned| poisoned.into_inner()) = None;
7207 if let Some(reply) = runtime
7208 .deferred_reload_reply
7209 .lock()
7210 .unwrap_or_else(|poisoned| poisoned.into_inner())
7211 .take()
7212 {
7213 let _ = reply.send(Err(SuperviseError::ConfigHeld {
7214 module_id: spec.module_id.clone(),
7215 code: "config_gated".into(),
7216 detail: "module is held by configuration admission".into(),
7217 }));
7218 }
7219 update_snapshot(snapshot, Some(&spec.module_id), |state| {
7220 state.admission_refused = admission_refused;
7221 state.config_hold_daemon_initiated = failure.daemon_initiated;
7222 state.config_hold_approval = failure.approval.clone();
7223 state.config_hold_parked = true;
7224 state.state = ModuleState::Stopped;
7225 state.respawn_pending = false;
7226 state.failed_retry.pending = None;
7227 state.failed_retry.failed_at = None;
7230 clear_current_process_facts(state);
7231 })
7232}
7233
7234fn try_park_config_attempt(
7235 spec: &ModuleSpec,
7236 runtime: &SupervisorRuntimeConfig,
7237 snapshot: &SharedSnapshot,
7238 failure: &FailureContext,
7239 program_base: Option<&subc_config::Digest>,
7240) -> Result<bool, SuperviseError> {
7241 let result = (|| {
7242 if !lock_snapshot(snapshot)?.enabled {
7243 return Ok(false);
7244 }
7245 #[cfg(test)]
7246 if let Some(delivery) = &runtime.config_delivery {
7247 delivery.0.pause_before_config_park_for_test();
7248 }
7249 let Some(delivery) = &runtime.config_delivery else {
7250 return Ok(false);
7251 };
7252 let parked = delivery
7253 .0
7254 .with_recorded_admission_hold(&spec.module_id, program_base, |refusal| {
7255 park_config_spawn(spec, runtime, snapshot, failure, refusal)
7256 })
7257 .map_err(|error| config_delivery::gate_hold_error(spec, error, Some(delivery)))?;
7258 if let Some(result) = parked {
7259 result?;
7260 Ok(true)
7261 } else {
7262 Ok(false)
7263 }
7264 })();
7265 if let Err(error) = &result {
7266 fail_snapshot(snapshot, Some(&spec.module_id), None, failure);
7270 record_approved_attempt_failure(spec, snapshot, failure, error);
7271 }
7272 result
7273}
7274
7275fn record_approved_attempt_failure(
7276 spec: &ModuleSpec,
7277 snapshot: &SharedSnapshot,
7278 failure: &FailureContext,
7279 error: &SuperviseError,
7280) {
7281 let Some(approval) = &failure.approval else {
7282 return;
7283 };
7284 let _ = update_snapshot(snapshot, Some(&spec.module_id), |state| {
7285 if state.enabled
7286 && state.state == ModuleState::Failed
7287 && state
7288 .approved_start_failure
7289 .as_ref()
7290 .is_none_or(|(identity, _)| identity != approval)
7291 {
7292 state.approved_start_failure = Some((approval.clone(), error.to_string()));
7293 }
7294 });
7295}
7296
7297fn hold_config_spawn(
7298 spec: &ModuleSpec,
7299 runtime: &SupervisorRuntimeConfig,
7300 snapshot: &SharedSnapshot,
7301 failure: &FailureContext,
7302) -> Result<(), SuperviseError> {
7303 if !lock_snapshot(snapshot)?.enabled {
7304 return Ok(());
7305 }
7306 if !try_park_config_attempt(spec, runtime, snapshot, failure, None)? {
7307 defer_config_spawn(spec, runtime, snapshot, RespawnKind::Spawn, failure)?;
7310 }
7311 Ok(())
7312}
7313
7314fn schedule_respawn(
7315 runtime: &SupervisorRuntimeConfig,
7316 snapshot: &SharedSnapshot,
7317 module_id: &str,
7318 delay: Duration,
7319 kind: RespawnKind,
7320 daemon_initiated: bool,
7321) -> Result<(), SuperviseError> {
7322 cancel_deferred_reload(runtime, module_id, "respawn superseded by another restart");
7323 update_snapshot(snapshot, Some(module_id), |state| {
7324 state.respawn_pending = true
7325 })?;
7326 *runtime
7327 .scheduled_respawn
7328 .lock()
7329 .unwrap_or_else(|p| p.into_inner()) = Some(PendingRespawn {
7330 deadline: Instant::now() + delay,
7331 kind,
7332 daemon_initiated,
7333 approval: None,
7334 });
7335 Ok(())
7336}
7337
7338const CONFIG_ADMISSION_RETRY_DELAY: Duration = Duration::from_millis(100);
7339
7340fn defer_config_spawn(
7341 spec: &ModuleSpec,
7342 runtime: &SupervisorRuntimeConfig,
7343 snapshot: &SharedSnapshot,
7344 kind: RespawnKind,
7345 failure: &FailureContext,
7346) -> Result<(), SuperviseError> {
7347 update_snapshot(snapshot, Some(&spec.module_id), |state| {
7348 state.state = ModuleState::Restarting;
7349 state.respawn_pending = true;
7350 clear_current_process_facts(state);
7351 })?;
7352 *runtime
7353 .scheduled_respawn
7354 .lock()
7355 .unwrap_or_else(|p| p.into_inner()) = Some(PendingRespawn {
7356 deadline: Instant::now() + CONFIG_ADMISSION_RETRY_DELAY,
7357 kind,
7358 daemon_initiated: failure.daemon_initiated,
7359 approval: failure.approval.clone(),
7360 });
7361 Ok(())
7362}
7363
7364fn record_health_action(snapshot: &SharedSnapshot, module_id: &str, action: String, now_ms: u64) {
7365 let _ = update_snapshot(snapshot, Some(module_id), |state| {
7366 state.health.last_action = Some(action);
7367 state.health.last_action_ms = Some(now_ms);
7368 });
7369}
7370
7371fn supervisor_health_status(status: HealthStatus) -> SupervisorHealthStatus {
7372 match status {
7373 HealthStatus::Ok => SupervisorHealthStatus::Ok,
7374 HealthStatus::Degraded => SupervisorHealthStatus::Degraded,
7375 HealthStatus::Failing => SupervisorHealthStatus::Failing,
7376 }
7377}
7378
7379fn truncate_health_metrics(metrics: Option<Value>) -> Option<Value> {
7391 let metrics = metrics?;
7392 match serde_json::to_vec(&metrics) {
7393 Ok(encoded) if encoded.len() > MAX_HEALTH_METRICS_BYTES => Some(serde_json::json!({
7394 "truncated": true,
7395 "original_bytes": encoded.len(),
7396 })),
7397 Ok(_) | Err(_) => Some(metrics),
7398 }
7399}
7400
7401fn jittered_health_delay(module_id: &str, probe_index: u64, cadence: Duration) -> Duration {
7407 if cadence.is_zero() {
7408 return Duration::ZERO;
7409 }
7410 let cadence_ms = cadence.as_millis() as u64;
7411 if cadence_ms == 0 {
7427 return cadence;
7428 }
7429 let jitter_span = (cadence_ms / 10).max(1);
7444 let hash = module_id.as_bytes().iter().fold(
7445 probe_index.wrapping_mul(0x9E37_79B9_7F4A_7C15),
7446 |acc, byte| {
7447 acc.wrapping_mul(1099511628211)
7448 .wrapping_add(u64::from(*byte))
7449 },
7450 );
7451 cadence + Duration::from_millis(hash % jitter_span)
7452}
7453
7454#[cfg(test)]
7455mod tests {
7456 use super::*;
7457
7458 #[test]
7459 fn registration_retirement_self_prefixed_owner_is_not_a_prefix_submodule() {
7460 let handle = SupervisorHandle::new();
7461 handle.set_spawn_nonce("prefix:owner", "owner-nonce".into());
7462 handle.set_reserved_prefixes("prefix:owner", &["prefix:".into()]);
7463 let owner = handle
7464 .reserved_hello_verification("prefix:owner", Some("owner-nonce"))
7465 .unwrap();
7466 assert!(owner.nonce_verified);
7467 assert!(
7468 owner.prefix_owner.is_none(),
7469 "an unreserved programmatic owner is not its own prefix child"
7470 );
7471 let child = handle
7472 .reserved_hello_verification("prefix:child", Some("owner-nonce"))
7473 .unwrap();
7474 assert!(child.nonce_verified);
7475 assert_eq!(child.prefix_owner.as_deref(), Some("prefix:owner"));
7476 }
7477
7478 #[test]
7479 fn readding_a_module_clears_its_rescan_removal_tombstone() {
7480 let handle = SupervisorHandle::new();
7481 let module_id = "readded-tombstone";
7482 handle.record_rescan_removal(module_id);
7483 assert!(handle.removal_tombstone_age_ms(module_id).is_some());
7484
7485 handle.apply_identity_configuration(&ModuleSpec {
7486 script: None,
7487 module_id: module_id.to_string(),
7488 program: PathBuf::from("/test/module"),
7489 args: Vec::new(),
7490 env: Vec::new(),
7491 reserved: false,
7492 reserved_prefixes: Vec::new(),
7493 protocol: ModuleProtocol::Subc,
7494 overlap: Default::default(),
7495 carry_containment: false,
7496 });
7497
7498 assert!(
7499 handle.removal_tombstone_age_ms(module_id).is_none(),
7500 "a re-added module must not retain a stale removal tombstone"
7501 );
7502 }
7503
7504 #[derive(Debug, PartialEq, Eq)]
7507 struct OwnerInSpawnWindow {
7508 module_id: String,
7509 configured: bool,
7510 on_roster: bool,
7511 admission_refusal: Option<&'static str>,
7512 }
7513
7514 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
7525 async fn a_new_module_is_configured_before_its_first_process_can_register() {
7526 use crate::scopes::ScopeTable;
7527 use subc_protocol::{error_codes, scope::ScopeSelector, Principal};
7528
7529 let dir = cortexkit_test_support::ScratchDir::new("configured-before-spawn");
7530 let stub = |module_id: &str, program: PathBuf| ModuleSpec {
7531 script: None,
7532 module_id: module_id.to_string(),
7533 program,
7534 args: Vec::new(),
7535 env: ["XDG_DATA_HOME", "XDG_RUNTIME_DIR", "XDG_CONFIG_HOME"]
7536 .into_iter()
7537 .map(|key| (key.to_string(), dir.path().display().to_string()))
7538 .chain([("FAKE_AFT_NEVER_CONNECT".to_string(), "1".to_string())])
7539 .collect(),
7540 reserved: false,
7541 reserved_prefixes: Vec::new(),
7542 protocol: ModuleProtocol::Subc,
7543 overlap: Default::default(),
7544 carry_containment: false,
7545 };
7546 let live = super::terminal_history_tests::fake_aft_stub_path();
7547 let missing = dir.path().join("definitely-missing-module");
7548
7549 let handle = SupervisorHandle::new();
7550 let mut supervisor =
7551 Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default())
7552 .with_handle(handle.clone());
7553 let observed = Arc::new(Mutex::new(Vec::<OwnerInSpawnWindow>::new()));
7554 let hook_handle = handle.clone();
7555 let hook_observed = Arc::clone(&observed);
7556 supervisor.test_after_first_spawn = AfterFirstSpawnHook(Some(Arc::new(move |module_id| {
7557 let configured = hook_handle.is_configured(module_id);
7560 let selector = ScopeSelector::new(
7561 Principal::Reserved {
7562 module_id: module_id.to_string(),
7563 },
7564 "s".to_string(),
7565 Some(1),
7566 );
7567 let carrier = Principal::Reserved {
7568 module_id: "carrier".to_string(),
7569 };
7570 let admission_refusal = match ScopeTable::new(Vec::<String>::new())
7571 .admit(&carrier, module_id, &selector, configured)
7572 {
7573 Ok(_) => None,
7574 Err(refusal) => Some(refusal.code),
7575 };
7576 hook_observed.lock().unwrap().push(OwnerInSpawnWindow {
7577 module_id: module_id.to_string(),
7578 configured,
7579 on_roster: hook_handle.get(module_id).is_some(),
7580 admission_refusal,
7581 });
7582 })));
7583
7584 let plain = supervisor.spawn(stub("plain", live.clone())).unwrap();
7585 let configured = supervisor
7586 .supervise_configured(stub("configured", live.clone()), true)
7587 .unwrap();
7588 let with_health = supervisor
7589 .supervise_configured_with_health(
7590 stub("with-health", live.clone()),
7591 true,
7592 HealthConfig::default(),
7593 None,
7594 RestartPolicy::default(),
7595 )
7596 .unwrap();
7597 let failed = supervisor
7600 .supervise_configured_with_health(
7601 stub("failed-spawn", missing.clone()),
7602 true,
7603 HealthConfig::default(),
7604 None,
7605 RestartPolicy::default(),
7606 )
7607 .unwrap();
7608 assert!(supervisor.spawn(stub("spawn-error", missing)).is_err());
7611
7612 let expected = [
7613 "plain",
7614 "configured",
7615 "with-health",
7616 "failed-spawn",
7617 "spawn-error",
7618 ]
7619 .into_iter()
7620 .map(|module_id| OwnerInSpawnWindow {
7621 module_id: module_id.to_string(),
7622 configured: true,
7623 on_roster: !matches!(module_id, "plain" | "spawn-error"),
7624 admission_refusal: Some(error_codes::SCOPE_NOT_SYNCED),
7625 })
7626 .collect::<Vec<_>>();
7627 assert_eq!(*observed.lock().unwrap(), expected);
7628
7629 for module_id in ["plain", "configured", "with-health", "failed-spawn"] {
7630 assert!(
7631 handle.get(module_id).is_some(),
7632 "{module_id} is on the roster"
7633 );
7634 assert!(
7635 handle.is_configured(module_id),
7636 "{module_id} stays configured"
7637 );
7638 }
7639 assert!(handle.get("spawn-error").is_none());
7640 assert!(
7641 !handle.is_configured("spawn-error"),
7642 "a plain spawn that failed must not leave its module marked configured"
7643 );
7644
7645 handle.retire("failed-spawn");
7647 assert!(!handle.is_configured("failed-spawn"));
7648
7649 for module in [plain, configured, with_health] {
7650 module.stop().await.unwrap();
7651 }
7652 drop(failed);
7653 }
7654
7655 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
7662 async fn a_configured_module_reads_starting_while_its_first_process_already_runs() {
7663 let dir = cortexkit_test_support::ScratchDir::new("starting-while-spawned");
7664 let spec = ModuleSpec {
7665 script: None,
7666 module_id: "spawned-not-admitted".to_string(),
7667 program: super::terminal_history_tests::fake_aft_stub_path(),
7668 args: Vec::new(),
7669 env: ["XDG_DATA_HOME", "XDG_RUNTIME_DIR", "XDG_CONFIG_HOME"]
7670 .into_iter()
7671 .map(|key| (key.to_string(), dir.path().display().to_string()))
7672 .chain([("FAKE_AFT_NEVER_CONNECT".to_string(), "1".to_string())])
7673 .collect(),
7674 reserved: false,
7675 reserved_prefixes: Vec::new(),
7676 protocol: ModuleProtocol::Subc,
7677 overlap: Default::default(),
7678 carry_containment: false,
7679 };
7680 let handle = SupervisorHandle::new();
7681 let mut supervisor =
7682 Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default())
7683 .with_handle(handle.clone());
7684 let observed = Arc::new(Mutex::new(None));
7685 let hook_handle = handle.clone();
7686 let hook_observed = Arc::clone(&observed);
7687 supervisor.test_after_first_spawn = AfterFirstSpawnHook(Some(Arc::new(move |module_id| {
7688 let state = hook_handle
7689 .get(module_id)
7690 .expect("a configured module is on the roster before it is spawned")
7691 .state()
7692 .unwrap();
7693 *hook_observed.lock().unwrap() = Some(state);
7694 })));
7695
7696 let module = supervisor.supervise_configured(spec, true).unwrap();
7697 assert_eq!(
7698 *observed.lock().unwrap(),
7699 Some(ModuleState::Starting),
7700 "between the spawn and its admission the supervisor still reports starting"
7701 );
7702 assert_eq!(module.state().unwrap(), ModuleState::Running);
7703 module.stop().await.unwrap();
7704 }
7705
7706 fn stale_process_snapshot(state: ModuleState, enabled: bool) -> SharedSnapshot {
7707 let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::new(state, enabled)));
7708 update_snapshot(&snapshot, Some("stale-process-facts"), |snapshot| {
7709 snapshot.process_alive = true;
7710 snapshot.pid = Some(41);
7711 snapshot.spawned_at_ms = Some(42);
7712 snapshot.spawned_from = Some(PathBuf::from("/spawned/module"));
7713 snapshot.launch_spec_digest = Some(LaunchSpecDigest([0xA5; 32]));
7714 snapshot.spawned_file_identity = Some(SpawnedFileIdentity {
7715 device: 43,
7716 inode: 44,
7717 #[cfg(windows)]
7718 windows_image: None,
7719 });
7720 })
7721 .unwrap();
7722 snapshot
7723 }
7724
7725 fn assert_snapshot_process_facts_cleared(snapshot: &SharedSnapshot) {
7726 let snapshot = lock_snapshot(snapshot).unwrap();
7727 assert!(!snapshot.process_alive);
7728 assert_eq!(snapshot.pid, None);
7729 assert_eq!(snapshot.spawned_at_ms, None);
7730 assert_eq!(snapshot.spawned_from, None);
7731 assert_eq!(snapshot.launch_spec_digest, None);
7732 assert_eq!(snapshot.spawned_file_identity, None);
7733 }
7734
7735 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
7736 async fn failed_enable_spawn_clears_preexisting_current_process_facts() {
7737 let supervisor =
7738 Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default());
7739 let mut runtime = supervisor.runtime_config();
7740 runtime.test_seed_stale_facts_before_enable_spawn = true;
7741 let snapshot = stale_process_snapshot(ModuleState::Disabled, false);
7742 let mut child = None;
7743 let spec = ModuleSpec {
7744 script: None,
7745 module_id: "failed-enable-clears-facts".to_string(),
7746 program: PathBuf::from("/definitely/missing/failed-enable-module"),
7747 args: Vec::new(),
7748 env: Vec::new(),
7749 reserved: false,
7750 reserved_prefixes: Vec::new(),
7751 protocol: ModuleProtocol::Subc,
7752 overlap: Default::default(),
7753 carry_containment: false,
7754 };
7755
7756 let result = set_child_enabled(
7757 &spec,
7758 &runtime,
7759 &supervisor.registry,
7760 &supervisor.process_liveness,
7761 &snapshot,
7762 &mut child,
7763 true,
7764 &runtime.failure_context(false),
7765 )
7766 .await;
7767
7768 assert!(matches!(result, Err(SuperviseError::Spawn { .. })));
7769 assert_eq!(lock_snapshot(&snapshot).unwrap().state, ModuleState::Failed);
7770 assert_snapshot_process_facts_cleared(&snapshot);
7771 }
7772
7773 #[tokio::test]
7774 async fn start_revives_stranded_restarting_but_not_pending_backoff() {
7775 let supervisor =
7776 Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default());
7777 let runtime = supervisor.runtime_config();
7778 let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::new(
7779 ModuleState::Restarting,
7780 true,
7781 )));
7782 let spec = ModuleSpec {
7783 script: None,
7784 module_id: "start-stranded-restarting".to_string(),
7785 program: super::terminal_history_tests::fake_aft_stub_path(),
7786 args: Vec::new(),
7787 env: vec![("FAKE_AFT_NEVER_CONNECT".to_string(), "1".to_string())],
7788 reserved: false,
7789 reserved_prefixes: Vec::new(),
7790 protocol: ModuleProtocol::None,
7791 overlap: Default::default(),
7792 carry_containment: false,
7793 };
7794 let mut child = None;
7795 lock_snapshot(&snapshot).unwrap().respawn_pending = true;
7796 assert!(!super::set_child_enabled(
7797 &spec,
7798 &runtime,
7799 &Registry::default(),
7800 &supervisor.process_liveness,
7801 &snapshot,
7802 &mut child,
7803 true,
7804 &runtime.failure_context(false),
7805 )
7806 .await
7807 .unwrap());
7808 assert!(child.is_none());
7809 lock_snapshot(&snapshot).unwrap().respawn_pending = false;
7810 assert!(super::set_child_enabled(
7811 &spec,
7812 &runtime,
7813 &Registry::default(),
7814 &supervisor.process_liveness,
7815 &snapshot,
7816 &mut child,
7817 true,
7818 &runtime.failure_context(false),
7819 )
7820 .await
7821 .unwrap());
7822 assert_eq!(
7823 lock_snapshot(&snapshot).unwrap().state,
7824 ModuleState::Running
7825 );
7826 let mut child = child.unwrap();
7827 child.start_kill().unwrap();
7828 child.wait().await.unwrap();
7829 }
7830
7831 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
7832 async fn failed_reload_spawn_clears_current_process_facts() {
7833 let supervisor =
7834 Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default());
7835 let mut runtime = supervisor.runtime_config();
7836 runtime.restart_policy = RestartPolicy::new(0, Duration::ZERO);
7837 let snapshot = stale_process_snapshot(ModuleState::Running, true);
7838 let mut child = None;
7839 let spec = ModuleSpec {
7840 script: None,
7841 module_id: "failed-reload-clears-facts".to_string(),
7842 program: PathBuf::from("/unused/failed-reload-module"),
7843 args: Vec::new(),
7844 env: Vec::new(),
7845 reserved: false,
7846 reserved_prefixes: Vec::new(),
7847 protocol: ModuleProtocol::Subc,
7848 overlap: Default::default(),
7849 carry_containment: false,
7850 };
7851
7852 let result = handle_reload_spawn_failure(
7853 &spec,
7854 &runtime,
7855 &supervisor.process_liveness,
7856 &snapshot,
7857 &mut child,
7858 "forced reload spawn failure".to_string(),
7859 )
7860 .await;
7861
7862 assert!(matches!(result, Err(SuperviseError::ReloadFailed { .. })));
7863 assert_eq!(lock_snapshot(&snapshot).unwrap().state, ModuleState::Failed);
7864 assert_snapshot_process_facts_cleared(&snapshot);
7865 }
7866
7867 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
7868 async fn dropping_a_module_with_an_active_monitor_clears_current_process_facts() {
7869 let supervisor =
7870 Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default());
7871 let snapshot = stale_process_snapshot(ModuleState::Running, true);
7872 let module = supervisor.supervised_module(
7873 ModuleSpec {
7874 script: None,
7875 module_id: "drop-clears-facts".to_string(),
7876 program: PathBuf::from("/unused/drop-module"),
7877 args: Vec::new(),
7878 env: Vec::new(),
7879 reserved: false,
7880 reserved_prefixes: Vec::new(),
7881 protocol: ModuleProtocol::Subc,
7882 overlap: Default::default(),
7883 carry_containment: false,
7884 },
7885 supervisor.runtime_config(),
7886 Arc::clone(&snapshot),
7887 None,
7888 );
7889 assert!(!module
7890 .inner
7891 .monitor
7892 .lock()
7893 .unwrap()
7894 .as_ref()
7895 .unwrap()
7896 .is_finished());
7897
7898 drop(module);
7899
7900 assert_eq!(
7901 lock_snapshot(&snapshot).unwrap().state,
7902 ModuleState::Stopped
7903 );
7904 assert_snapshot_process_facts_cleared(&snapshot);
7905 }
7906
7907 #[cfg(unix)]
7908 #[tokio::test]
7909 async fn rescan_preserves_running_protocol_until_respawn() {
7910 let dir = cortexkit_test_support::ScratchDir::new("rescan-protocol");
7911 let initial = ModuleSpec {
7912 script: None,
7913 module_id: "rescan-protocol".into(),
7914 program: PathBuf::from("/bin/sleep"),
7915 args: vec!["60".into()],
7916 env: ["XDG_DATA_HOME", "XDG_RUNTIME_DIR", "XDG_CONFIG_HOME"]
7917 .into_iter()
7918 .map(|key| (key.into(), dir.path().to_string_lossy().into_owned()))
7919 .collect(),
7920 reserved: false,
7921 reserved_prefixes: vec![],
7922 protocol: ModuleProtocol::None,
7923 overlap: Default::default(),
7924 carry_containment: false,
7925 };
7926 let supervisor =
7927 Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default());
7928 let module = supervisor.spawn(initial.clone()).unwrap();
7929 assert!(module.status().unwrap().live);
7930 let mut next = initial;
7931 next.protocol = ModuleProtocol::Subc;
7932 module
7933 .update_configuration(next.clone(), HealthConfig::default(), None)
7934 .await
7935 .unwrap();
7936 assert!(
7937 module.status().unwrap().live,
7938 "rescan must not require HELLO from the old non-wire process"
7939 );
7940 let runtime = supervisor.runtime_config();
7941 let action = on_child_exit(
7942 &next,
7943 RestartPolicy::default(),
7944 &supervisor.registry,
7945 &module.inner.snapshot,
7946 &runtime.terminal_ring,
7947 &runtime.spawn_events,
7948 &runtime.child_roster,
7949 ExitReport {
7950 kind: ExitKind::Clean,
7951 code: Some(0),
7952 signal: None,
7953 at_ms: unix_ms_now(),
7954 },
7955 true,
7956 )
7957 .await;
7958 assert!(
7959 matches!(action, NextAction::Restart { .. }),
7960 "the old non-wire process's clean exit must restart"
7961 );
7962 module.drain().await.unwrap();
7963 }
7964
7965 #[cfg(unix)]
7966 #[tokio::test]
7967 async fn orphan_identity_matches_path_names_and_shebang_interpreters() {
7968 use std::os::unix::fs::PermissionsExt;
7969 let dir = cortexkit_test_support::ScratchDir::new("spawn-image-identity");
7970 let script = dir.join("module.sh");
7971 std::fs::write(&script, "#!/bin/sh\nwhile :; do sleep 0.1; done\n").unwrap();
7972 std::fs::set_permissions(&script, std::fs::Permissions::from_mode(0o700)).unwrap();
7973 let record_path = dir.join("live-children.json");
7974 let supervisor =
7975 Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default())
7976 .with_live_children_record(&record_path);
7977 for (program, args) in [
7978 (PathBuf::from("sleep"), vec!["60".into()]),
7979 (script, vec![]),
7980 ] {
7981 let spec = ModuleSpec {
7982 script: None,
7983 module_id: "image-identity".into(),
7984 program: program.clone(),
7985 args,
7986 env: ["XDG_DATA_HOME", "XDG_RUNTIME_DIR", "XDG_CONFIG_HOME"]
7987 .into_iter()
7988 .map(|key| (key.into(), dir.to_string_lossy().into_owned()))
7989 .collect(),
7990 reserved: false,
7991 reserved_prefixes: vec![],
7992 protocol: ModuleProtocol::None,
7993 overlap: Default::default(),
7994 carry_containment: false,
7995 };
7996 let module = supervisor.spawn(spec).unwrap();
7997 #[cfg(target_os = "macos")]
7998 {
7999 let deadline = Instant::now() + Duration::from_secs(5);
8002 while crate::live_children::read_record(&record_path)
8003 .unwrap()
8004 .iter()
8005 .all(|entry| entry.executable.is_none())
8006 {
8007 assert!(Instant::now() < deadline, "module image was not confirmed");
8008 tokio::time::sleep(Duration::from_millis(5)).await;
8009 }
8010 }
8011 let entry = crate::live_children::read_record(&record_path)
8012 .unwrap()
8013 .pop()
8014 .unwrap();
8015 let observed = subc_os::Process::open(entry.pid)
8016 .unwrap()
8017 .unwrap()
8018 .observe()
8019 .unwrap();
8020 let verdict = crate::live_children::identity_verdict(&entry, entry.pid, &observed);
8021 module.drain().await.unwrap();
8022 assert_eq!(
8023 verdict,
8024 crate::live_children::IdentityVerdict::Matches,
8025 "program {program:?}: recorded {entry:?}, observed {observed:?}"
8026 );
8027 }
8028 }
8029
8030 #[cfg(unix)]
8031 fn http_fixture(
8032 dir: &std::path::Path,
8033 url: &str,
8034 threshold: u32,
8035 ) -> crate::daemon_config::ConfiguredModule {
8036 let path = dir.join("subc.jsonc");
8037 std::fs::write(&path, serde_json::json!({"version": 1, "modules": {"http-process": {
8038 "program": "/bin/sleep", "args": ["60"], "protocol": "none",
8039 "env": {"XDG_DATA_HOME":dir,"XDG_RUNTIME_DIR":dir,"XDG_CONFIG_HOME":dir},
8040 "health": {"http":url,"cadence_ms":20,"deadline_ms":200,"failure_threshold":threshold},
8041 "restart": {"backoff_ms":1,"max_backoff_ms":1}, "drain_timeout_ms":10
8042 }}}).to_string()).unwrap();
8043 crate::daemon_config::load(&path)
8044 .unwrap()
8045 .unwrap()
8046 .modules
8047 .pop()
8048 .unwrap()
8049 }
8050
8051 #[cfg(unix)]
8052 async fn wait_http_health(module: &SupervisedModule, status: SupervisorHealthStatus) {
8053 timeout(Duration::from_secs(5), async {
8054 loop {
8055 if module.status().unwrap().health.status == status {
8056 break;
8057 }
8058 sleep(Duration::from_millis(5)).await;
8059 }
8060 })
8061 .await
8062 .unwrap_or_else(|_| {
8063 panic!(
8064 "expected {status:?}, got {:?}",
8065 module.status().unwrap().health
8066 )
8067 });
8068 }
8069
8070 #[cfg(unix)]
8071 #[tokio::test]
8072 async fn http_health_status_flips_ok_failing_ok() {
8073 use tokio::io::{AsyncReadExt, AsyncWriteExt};
8074 let dir = cortexkit_test_support::ScratchDir::new("http-status-flips");
8075 let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
8076 let status = Arc::new(std::sync::atomic::AtomicU16::new(200));
8077 let serving_status = status.clone();
8078 let url = format!("http://{}/healthz", listener.local_addr().unwrap());
8079 let server = tokio::spawn(async move {
8080 loop {
8081 let (mut stream, _) = listener.accept().await.unwrap();
8082 let mut request = [0u8; 2048];
8083 let count = stream.read(&mut request).await.unwrap();
8084 assert!(count > 0, "a probe must send an HTTP request");
8085 let code = serving_status.load(std::sync::atomic::Ordering::SeqCst);
8086 let body = if code == 200 {
8087 "ready"
8088 } else {
8089 "{\"status\":\"unavailable\",\"error\":\"scratch failure\"}"
8090 };
8091 let response = format!("HTTP/1.1 {code} scratch\r\nContent-Length: {}\r\nConnection: close\r\n\r\n{body}", body.len());
8092 let _ = stream.write_all(response.as_bytes()).await;
8093 }
8094 });
8095 let configured = http_fixture(&dir, &url, 1000);
8096 let module =
8097 Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default())
8098 .supervise_configured_with_health(
8099 configured.module_spec(),
8100 true,
8101 configured.health,
8102 configured.drain_timeout_ms,
8103 configured.restart,
8104 )
8105 .unwrap();
8106 wait_http_health(&module, SupervisorHealthStatus::Ok).await;
8107 status.store(503, std::sync::atomic::Ordering::SeqCst);
8108 wait_http_health(&module, SupervisorHealthStatus::Failing).await;
8109 assert!(module
8110 .status()
8111 .unwrap()
8112 .health
8113 .detail
8114 .unwrap()
8115 .contains("scratch failure"));
8116 status.store(200, std::sync::atomic::Ordering::SeqCst);
8117 wait_http_health(&module, SupervisorHealthStatus::Ok).await;
8118 assert_eq!(module.status().unwrap().health.consecutive_failures, 0);
8119 let before = module.status().unwrap();
8120 let (spec, mut health) = module.configuration().unwrap();
8121 health.http = None;
8122 module
8123 .update_configuration(spec.clone(), health.clone(), Some(10))
8124 .await
8125 .unwrap();
8126 wait_http_health(&module, SupervisorHealthStatus::Unknown).await;
8127 health.http = Some(url);
8128 module
8129 .update_configuration(spec, health, Some(10))
8130 .await
8131 .unwrap();
8132 wait_http_health(&module, SupervisorHealthStatus::Ok).await;
8133 assert_eq!(
8134 module.status().unwrap().pid,
8135 before.pid,
8136 "changing a probe must apply live, not restart its process"
8137 );
8138 let (spec, mut health) = module.configuration().unwrap();
8139 health.failure_threshold = 2;
8140 module
8141 .update_configuration(spec, health, Some(10))
8142 .await
8143 .unwrap();
8144 status.store(503, std::sync::atomic::Ordering::SeqCst);
8145 timeout(Duration::from_secs(5), async {
8146 while module.status().unwrap().spawn_generation == before.spawn_generation {
8147 sleep(Duration::from_millis(5)).await;
8148 }
8149 })
8150 .await
8151 .expect("sustained HTTP 503 responses must trigger the health restart policy");
8152 module.drain().await.unwrap();
8153 server.abort();
8154 }
8155
8156 #[cfg(unix)]
8157 #[tokio::test]
8158 async fn http_health_replacement_recovers_after_one_restart() {
8159 use tokio::io::{AsyncReadExt, AsyncWriteExt};
8160 let dir = cortexkit_test_support::ScratchDir::new("http-replacement-recovers");
8161 let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
8162 let url = format!("http://{}/healthz", listener.local_addr().unwrap());
8163 let mut configured = http_fixture(&dir, &url, 2);
8164 configured.restart.max_restarts = 2;
8165 let module =
8166 Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default())
8167 .supervise_configured_with_health(
8168 configured.module_spec(),
8169 true,
8170 configured.health,
8171 configured.drain_timeout_ms,
8172 configured.restart,
8173 )
8174 .unwrap();
8175 let first_generation = module.status().unwrap().spawn_generation;
8176 let observed = module.clone();
8177 let (healthy_tx, mut healthy_rx) = mpsc::channel(8);
8178 let server = tokio::spawn(async move {
8179 let mut generation = first_generation;
8180 let mut probes = 0;
8181 loop {
8182 let (mut stream, _) = listener.accept().await.unwrap();
8183 let mut request = [0; 2048];
8184 assert!(stream.read(&mut request).await.unwrap() > 0);
8185 let current = observed.status().unwrap().spawn_generation;
8186 if current != generation {
8187 generation = current;
8188 probes = 0;
8189 }
8190 probes += 1;
8191 let healthy = generation != first_generation && probes > 1;
8194 let code = if healthy { 200 } else { 503 };
8195 let response = format!(
8196 "HTTP/1.1 {code} scratch\r\nContent-Length: 0\r\nConnection: close\r\n\r\n"
8197 );
8198 let _ = stream.write_all(response.as_bytes()).await;
8199 if healthy {
8200 healthy_tx.send(()).await.unwrap();
8201 }
8202 }
8203 });
8204 timeout(Duration::from_secs(5), async {
8205 let mut healthy_probes = 0;
8206 loop {
8207 if module.state().unwrap() == ModuleState::Failed {
8208 break;
8209 }
8210 while healthy_rx.try_recv().is_ok() {
8211 healthy_probes += 1;
8212 }
8213 if healthy_probes >= 3
8214 && module.status().unwrap().health.status == SupervisorHealthStatus::Ok
8215 {
8216 break;
8217 }
8218 sleep(Duration::from_millis(5)).await;
8219 }
8220 })
8221 .await
8222 .expect("the replacement must settle healthy without exhausting its restart budget");
8223 let status = module.status().unwrap();
8224 module.drain().await.unwrap();
8225 server.abort();
8226 assert_eq!(status.state, ModuleState::Running);
8227 assert_eq!(status.health.status, SupervisorHealthStatus::Ok);
8228 assert_eq!(status.health.consecutive_failures, 0);
8229 assert_eq!(status.lifetime_restarts, 1);
8230 assert_eq!(status.restart_count, 1);
8231 }
8232
8233 #[cfg(unix)]
8234 #[tokio::test]
8235 async fn http_health_sustained_failure_exhausts_restart_budget() {
8236 use std::collections::BTreeMap;
8237 use tokio::io::{AsyncReadExt, AsyncWriteExt};
8238 let dir = cortexkit_test_support::ScratchDir::new("http-health-budget");
8239 let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
8240 let url = format!("http://{}/healthz", listener.local_addr().unwrap());
8241 let mut configured = http_fixture(&dir, &url, 2);
8242 configured.restart.max_restarts = 2;
8243 let module =
8244 Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default())
8245 .supervise_configured_with_health(
8246 configured.module_spec(),
8247 true,
8248 configured.health,
8249 configured.drain_timeout_ms,
8250 configured.restart,
8251 )
8252 .unwrap();
8253 let observed = module.clone();
8254 let probes = Arc::new(Mutex::new(BTreeMap::<u64, u32>::new()));
8255 let served = probes.clone();
8256 let server = tokio::spawn(async move {
8257 loop {
8258 let (mut stream, _) = listener.accept().await.unwrap();
8259 let mut request = [0; 2048];
8260 assert!(stream.read(&mut request).await.unwrap() > 0);
8261 *served
8262 .lock()
8263 .unwrap()
8264 .entry(observed.status().unwrap().spawn_generation)
8265 .or_default() += 1;
8266 let _ = stream
8267 .write_all(b"HTTP/1.1 503 Unavailable\r\nContent-Length: 0\r\nConnection: close\r\n\r\n")
8268 .await;
8269 }
8270 });
8271 timeout(Duration::from_secs(5), async {
8272 while module.state().unwrap() != ModuleState::Failed {
8273 sleep(Duration::from_millis(5)).await;
8274 }
8275 })
8276 .await
8277 .expect("sustained failures must still exhaust the health restart budget");
8278 let status = module.status().unwrap();
8279 server.abort();
8280 assert_eq!(status.restart_count, 2);
8281 assert_eq!(status.lifetime_restarts, 2);
8282 assert!(!status.process_alive);
8283 assert_eq!(
8284 probes.lock().unwrap().values().copied().collect::<Vec<_>>(),
8285 vec![2, 2, 2]
8286 );
8287 }
8288
8289 #[cfg(unix)]
8290 #[tokio::test]
8291 async fn spawn_clears_the_previous_process_health() {
8292 let dir = cortexkit_test_support::ScratchDir::new("spawn-fresh-health");
8293 let mut configured = http_fixture(&dir, "http://127.0.0.1:1/healthz", 2);
8294 configured.health.cadence = Duration::from_secs(3600);
8295 let module =
8296 Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default())
8297 .supervise_configured_with_health(
8298 configured.module_spec(),
8299 false,
8300 configured.health,
8301 configured.drain_timeout_ms,
8302 configured.restart,
8303 )
8304 .unwrap();
8305 update_snapshot(&module.inner.snapshot, Some(module.module_id()), |state| {
8306 state.health.status = SupervisorHealthStatus::Unresponsive;
8307 state.health.consecutive_failures = 3;
8308 state.health.detail = Some("old process refused connections".into());
8309 state.health.metrics = Some(serde_json::json!({"old": true}));
8310 state.health.last_probe_ms = Some(123);
8311 state.health.last_action = Some("restart".into());
8312 state.health.last_action_ms = Some(124);
8313 })
8314 .unwrap();
8315 module.set_enabled(true).await.unwrap();
8316 let health = module.status().unwrap().health;
8317 module.drain().await.unwrap();
8318 assert_eq!(health.consecutive_failures, 0);
8319 assert_eq!(health.status, SupervisorHealthStatus::Unknown);
8320 assert_eq!(health.detail, None);
8321 assert_eq!(health.metrics, None);
8322 assert_eq!(health.last_probe_ms, None);
8323 assert_eq!(health.last_action.as_deref(), Some("restart"));
8324 assert_eq!(health.last_action_ms, Some(124));
8325 }
8326
8327 #[cfg(unix)]
8328 #[tokio::test]
8329 async fn http_health_no_listener_restarts_after_consecutive_failures() {
8330 let dir = cortexkit_test_support::ScratchDir::new("http-refused");
8331 let unused = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
8332 let url = format!("http://{}/healthz", unused.local_addr().unwrap());
8333 drop(unused);
8334 let configured = http_fixture(&dir, &url, 2);
8335 let module =
8336 Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default())
8337 .supervise_configured_with_health(
8338 configured.module_spec(),
8339 true,
8340 configured.health,
8341 configured.drain_timeout_ms,
8342 configured.restart,
8343 )
8344 .unwrap();
8345 let before = module.status().unwrap().spawn_generation;
8346 timeout(Duration::from_secs(5), async {
8347 loop {
8348 let status = module.status().unwrap();
8349 if status.spawn_generation > before {
8350 assert!(status.lifetime_restarts > 0);
8351 break;
8352 }
8353 sleep(Duration::from_millis(5)).await;
8354 }
8355 })
8356 .await
8357 .expect("sustained HTTP refusal must trigger the health restart policy");
8358 module.drain().await.unwrap();
8359 }
8360
8361 #[cfg(unix)]
8362 #[tokio::test]
8363 async fn http_health_timeout_honours_deadline() {
8364 let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
8365 let url = format!("http://{}/healthz", listener.local_addr().unwrap());
8366 let server = tokio::spawn(async move {
8367 let _held = listener.accept().await.unwrap();
8368 std::future::pending::<()>().await;
8369 });
8370 let error = timeout(
8371 Duration::from_secs(1),
8372 probe_http_health(&url, Duration::from_millis(10)),
8373 )
8374 .await
8375 .expect("the probe must enforce its own deadline")
8376 .unwrap_err();
8377 server.abort();
8378 assert!(error.to_string().contains("timed out"));
8379 }
8380
8381 #[cfg(unix)]
8382 #[tokio::test]
8383 async fn http_health_localhost_reaches_an_ipv6_only_listener() {
8384 use tokio::io::{AsyncReadExt, AsyncWriteExt};
8385 let listener = tokio::net::TcpListener::bind("[::1]:0").await.unwrap();
8386 let url = format!(
8387 "http://localhost:{}/healthz",
8388 listener.local_addr().unwrap().port()
8389 );
8390 let server = tokio::spawn(async move {
8391 let (mut stream, _) = listener.accept().await.unwrap();
8392 let mut request = [0u8; 2048];
8393 assert!(stream.read(&mut request).await.unwrap() > 0);
8394 stream
8395 .write_all(b"HTTP/1.1 200 OK\r\nContent-Length: 2\r\n\r\nok")
8396 .await
8397 .unwrap();
8398 });
8399 assert_eq!(
8404 probe_http_health(&url, Duration::from_secs(10))
8405 .await
8406 .unwrap()
8407 .status,
8408 HealthStatus::Ok
8409 );
8410 server.await.unwrap();
8411 }
8412
8413 #[cfg(unix)]
8414 #[tokio::test]
8415 async fn http_health_timeout_keeps_partial_status_and_body() {
8416 use tokio::io::{AsyncReadExt, AsyncWriteExt};
8417 let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
8418 let url = format!("http://{}/healthz", listener.local_addr().unwrap());
8419 let server = tokio::spawn(async move {
8420 let (mut stream, _) = listener.accept().await.unwrap();
8421 let mut request = [0u8; 2048];
8422 assert!(stream.read(&mut request).await.unwrap() > 0);
8423 stream
8424 .write_all(
8425 b"HTTP/1.1 503 Unavailable\r\nContent-Length: 100\r\n\r\npartial diagnostic",
8426 )
8427 .await
8428 .unwrap();
8429 std::future::pending::<()>().await;
8430 });
8431 let error = probe_http_health(&url, Duration::from_secs(1))
8432 .await
8433 .unwrap_err()
8434 .to_string();
8435 server.abort();
8436 assert!(
8437 error.contains("timed out")
8438 && error.contains("503 Unavailable")
8439 && error.contains("partial diagnostic"),
8440 "{error}"
8441 );
8442 }
8443
8444 #[cfg(unix)]
8445 #[tokio::test]
8446 async fn http_health_diagnostic_body_is_bounded_to_200_bytes() {
8447 use tokio::io::{AsyncReadExt, AsyncWriteExt};
8448 let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
8449 let url = format!("http://{}/healthz", listener.local_addr().unwrap());
8450 let server = tokio::spawn(async move {
8451 let (mut stream, _) = listener.accept().await.unwrap();
8452 let mut request = [0u8; 2048];
8453 assert!(stream.read(&mut request).await.unwrap() > 0);
8454 let body = format!("{}not-in-diagnostic", "x".repeat(400));
8455 let response = format!(
8456 "HTTP/1.1 503 Unavailable\r\nContent-Length: {}\r\n\r\n{body}",
8457 body.len()
8458 );
8459 stream.write_all(response.as_bytes()).await.unwrap();
8460 });
8461 let error = probe_http_health(&url, Duration::from_secs(1))
8462 .await
8463 .unwrap_err()
8464 .to_string();
8465 server.await.unwrap();
8466 assert!(error.ends_with(&"x".repeat(200)), "{error}");
8467 assert_eq!(error.split(": ").last().unwrap().len(), 200);
8468 assert!(!error.contains("not-in-diagnostic"));
8469 }
8470
8471 #[cfg(unix)]
8472 #[tokio::test]
8473 async fn http_health_real_nats_server_monitoring() {
8474 if std::process::Command::new("nats-server")
8475 .arg("--version")
8476 .env("XDG_DATA_HOME", std::env::temp_dir())
8477 .env("XDG_RUNTIME_DIR", std::env::temp_dir())
8478 .env("XDG_CONFIG_HOME", std::env::temp_dir())
8479 .output()
8480 .is_err()
8481 {
8482 eprintln!(
8483 "skipped: http_health_real_nats_server_monitoring (nats-server is not on PATH)"
8484 );
8485 return;
8486 }
8487 let dir = cortexkit_test_support::ScratchDir::new("nats-http-monitoring");
8488 let monitor = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
8489 let port = monitor.local_addr().unwrap().port();
8490 let client = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
8491 let client_port = client.local_addr().unwrap().port();
8492 let config = dir.join("server.conf");
8493 std::fs::write(&config, format!("listen: 127.0.0.1:{client_port}\nhttp: 127.0.0.1:{port}\njetstream {{ store_dir: \"{}\" }}\n", dir.join("jetstream").display())).unwrap();
8494 let mut configured = http_fixture(&dir, &format!("http://127.0.0.1:{port}/healthz"), 1000);
8495 configured.program = PathBuf::from("nats-server");
8496 configured.args = vec!["-c".into(), config.to_string_lossy().into_owned()];
8497 drop(monitor);
8498 drop(client);
8499 let module =
8500 Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default())
8501 .supervise_configured_with_health(
8502 configured.module_spec(),
8503 true,
8504 configured.health,
8505 configured.drain_timeout_ms,
8506 configured.restart,
8507 )
8508 .unwrap();
8509 wait_http_health(&module, SupervisorHealthStatus::Ok).await;
8510 module.drain().await.unwrap();
8511 }
8512
8513 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
8514 async fn configuration_update_does_not_replace_captured_running_process_facts() {
8515 let supervisor =
8516 Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default());
8517 let snapshot = stale_process_snapshot(ModuleState::Running, true);
8518 let initial = ModuleSpec {
8519 script: None,
8520 module_id: "rescan-preserves-spawn-facts".to_string(),
8521 program: PathBuf::from("/spawned/module"),
8522 args: Vec::new(),
8523 env: Vec::new(),
8524 reserved: false,
8525 reserved_prefixes: Vec::new(),
8526 protocol: ModuleProtocol::Subc,
8527 overlap: Default::default(),
8528 carry_containment: false,
8529 };
8530 let module = supervisor.supervised_module(
8531 initial.clone(),
8532 supervisor.runtime_config(),
8533 snapshot,
8534 None,
8535 );
8536 let before = module.status().unwrap();
8537 let mut replacement = initial;
8538 replacement.program = PathBuf::from("/rescanned/replacement-module");
8539
8540 module
8541 .update_configuration(replacement, HealthConfig::default(), None)
8542 .await
8543 .unwrap();
8544
8545 let after = module.status().unwrap();
8546 assert_eq!(after.pid, before.pid);
8547 assert_eq!(after.spawned_at_ms, before.spawned_at_ms);
8548 assert_eq!(after.spawned_from, before.spawned_from);
8549 drop(module);
8550 }
8551}
8552
8553fn unix_ms_now() -> u64 {
8554 SystemTime::now()
8555 .duration_since(UNIX_EPOCH)
8556 .map(|duration| duration.as_millis().min(u128::from(u64::MAX)) as u64)
8557 .unwrap_or(0)
8558}
8559
8560async fn supervise_loop(
8561 mut spec: ModuleSpec,
8562 mut runtime: SupervisorRuntimeConfig,
8563 registry: Arc<Registry>,
8564 process_liveness: Arc<SupervisorProcessLiveness>,
8565 snapshot: SharedSnapshot,
8566 mut child: Option<SupervisedChild>,
8567 mut commands: mpsc::Receiver<SupervisorCommand>,
8568) {
8569 let mut health_probe = HealthProbeRuntime::default();
8570 let mut registration_changes = match registry.subscribe_module_changes(&spec.module_id) {
8574 Ok(changes) => Some(changes),
8575 Err(err) => {
8576 warn!(module_id = %spec.module_id, error = %err, "health prober could not subscribe to registry");
8579 None
8580 }
8581 };
8582 let mut pending_respawn: Option<PendingRespawn> = None;
8586 let mut requeued: VecDeque<SupervisorCommand> = VecDeque::new();
8589 let mut previous_instance_recheck = tokio::time::interval_at(
8590 Instant::now() + Duration::from_secs(5),
8591 Duration::from_secs(5),
8592 );
8593 previous_instance_recheck.set_missed_tick_behavior(tokio::time::MissedTickBehavior::Skip);
8594 'supervision: loop {
8595 #[cfg(test)]
8596 {
8597 lock_snapshot(&snapshot).unwrap().actor_select = None;
8598 }
8599 #[cfg(target_os = "macos")]
8600 if let Some(active) = child.as_mut() {
8601 active.confirm_privacy_exec().await;
8602 }
8603 if let Some(scheduled) = runtime
8604 .scheduled_respawn
8605 .lock()
8606 .unwrap_or_else(|p| p.into_inner())
8607 .take()
8608 {
8609 runtime.recovery_is_daemon = scheduled.daemon_initiated;
8610 pending_respawn = Some(scheduled);
8611 }
8612 if pending_respawn.is_some() && (child.is_some() || !respawn_still_pending(&snapshot)) {
8613 pending_respawn = None;
8614 cancel_deferred_reload(
8615 &runtime,
8616 &spec.module_id,
8617 "respawn cancelled by a supervisor command",
8618 );
8619 }
8620 if child.is_none()
8621 && pending_respawn.is_none()
8622 && lock_snapshot(&snapshot).is_ok_and(|state| state.waiting_for_previous_pid.is_none())
8623 {
8624 cancel_deferred_reload(
8625 &runtime,
8626 &spec.module_id,
8627 "respawn cancelled before a replacement was spawned",
8628 );
8629 let mut stranded = false;
8630 let _ = update_snapshot(&snapshot, Some(&spec.module_id), |state| {
8631 state.respawn_pending = false;
8632 state.coalesced_restart_pending = false;
8633 if matches!(
8634 state.state,
8635 ModuleState::Restarting
8636 | ModuleState::Starting
8637 | ModuleState::Draining
8638 | ModuleState::Unresponsive
8639 ) {
8640 error!(module_id = %spec.module_id, state = ?state.state,
8641 "supervision operation ended without a child or pending respawn; marking failed so start can retry");
8642 stranded = true;
8643 }
8644 });
8645 if stranded {
8646 fail_snapshot(
8647 &snapshot,
8648 Some(&spec.module_id),
8649 None,
8650 &runtime.failure_context(runtime.recovery_is_daemon),
8651 );
8652 }
8653 }
8654 if child.is_none()
8655 && (runtime.child_roster.is_closed()
8656 || pending_respawn.is_none()
8657 || !lock_snapshot(&snapshot)
8658 .is_ok_and(|state| state.enabled && state.state == ModuleState::Restarting))
8659 {
8660 containment::terminate(&runtime.child_roster.containment).await;
8661 }
8662 if let Some(command) = requeued.pop_front() {
8663 if !handle_supervisor_command(
8664 command,
8665 &mut spec,
8666 &mut runtime,
8667 ®istry,
8668 &process_liveness,
8669 &snapshot,
8670 &mut child,
8671 &mut commands,
8672 &mut requeued,
8673 )
8674 .await
8675 {
8676 containment::terminate(&runtime.child_roster.containment).await;
8677 return;
8678 }
8679 if child.is_some() || !respawn_still_pending(&snapshot) {
8680 pending_respawn = None;
8681 let _ = update_snapshot(&snapshot, Some(&spec.module_id), |state| {
8682 state.respawn_pending = false
8683 });
8684 }
8685 continue;
8686 }
8687 if child.is_some() {
8688 runtime.recovery_is_daemon = true;
8689 #[cfg(test)]
8690 {
8691 lock_snapshot(&snapshot).unwrap().actor_turns += 1;
8692 }
8693 health_probe.refresh_registration(&spec, &runtime, ®istry, &snapshot);
8694 let active = child.as_mut().expect("child checked above");
8695 if active.registered_since_spawn() {
8696 if let Some(delivery) = &runtime.config_delivery {
8697 delivery.queue_registered(
8698 &spec.module_id,
8699 runtime.forwarding.as_ref(),
8700 ®istry,
8701 active,
8702 );
8703 }
8704 }
8705 if runtime.child_roster.is_closed() || !active.awaiting_registration() {
8706 active.registration_deadline = None;
8707 }
8708 let registration_deadline = active.registration_deadline;
8709 let no_checks = runtime.health.http.is_none()
8710 && (running_protocol(&spec, &snapshot) == ModuleProtocol::None
8711 || (health_probe.registered_connection.is_some() && !health_probe.advertised));
8712 let uptime_reset = lock_snapshot(&snapshot).ok().and_then(|mut state| {
8713 state
8714 .failed_retry
8715 .uptime_reset_at(no_checks, runtime.restart_policy.window)
8716 });
8717 let wake_at = match (health_probe.wake_at(), uptime_reset) {
8718 (probe, Some(reset)) => Some(probe.map_or(reset, |probe| probe.min(reset))),
8719 (probe, None) => probe,
8720 };
8721 let wake_at = match (wake_at, registration_deadline) {
8722 (Some(wake), Some(register)) => Some(wake.min(register)),
8723 (wake, register) => wake.or(register),
8724 };
8725 #[cfg(test)]
8726 let wake_after = wake_at.map(|at| at.saturating_duration_since(Instant::now()));
8727 let probe_sleep = supervisor_sleep(wake_at);
8728 tokio::pin!(probe_sleep);
8729 let wire_child = running_protocol(&spec, &snapshot) == ModuleProtocol::Subc;
8734 #[cfg(test)]
8735 {
8736 let registration_pending = wire_child
8737 && registration_changes
8738 .as_ref()
8739 .is_some_and(|changes| changes.has_changed().unwrap());
8740 let mut state = lock_snapshot(&snapshot).unwrap();
8741 state.actor_select = (!registration_pending).then_some(ActorSelectCheckpoint {
8742 generation: state.spawn_generation,
8743 turn: state.actor_turns,
8744 registered_connection: health_probe.registered_connection,
8745 next_probe_at: health_probe.next_probe_at,
8746 wake_after,
8747 });
8748 }
8749 let registration = active
8750 .awaiting_registration()
8751 .then(|| active.registration_spawn.clone())
8752 .flatten();
8753 #[cfg(all(test, target_os = "linux"))]
8754 let registration_before_exit = !runtime.child_roster.admission_event_for_test(
8755 &spec.module_id,
8756 "observe_registration_before_exit",
8757 Some(active.pid),
8758 ) && registration.is_some();
8759 #[cfg(not(all(test, target_os = "linux")))]
8760 let registration_before_exit = false;
8761 #[cfg(all(test, target_os = "linux"))]
8762 let registration_child_is_running = |child: &mut SupervisedChild| {
8763 registration_before_exit || registration_child_is_running(child)
8766 };
8767 let registration_failed = async {
8768 match ®istration {
8769 Some(spawn) => spawn.wait_registration_failure().await,
8770 None => std::future::pending().await,
8771 }
8772 };
8773 let active_child = child.as_mut().expect("child checked above");
8774 tokio::select! {
8775 _ = registration_failed => {
8776 if !active_child.registered_since_spawn() && registration_child_is_running(active_child) {
8777 active_child.registration_deadline = None;
8778 active_child.spawn_failure = Some("registration_channel_closed".into());
8779 if let Err(error) = stop_unregistered_child(active_child).await {
8780 fail_after_wait_error(&spec.module_id, &snapshot, &runtime.terminal_ring,
8781 &runtime.spawn_events, format!("registration_channel_closed: {error}"),
8782 &runtime.failure_context(true));
8783 child = None;
8784 }
8785 }
8786 continue;
8787 }
8788 wait_result = active_child.wait(), if !registration_before_exit => {
8789 if let Ok(mut state) = lock_snapshot(&snapshot) {
8790 state.failed_retry.uptime_reset_at(no_checks, runtime.restart_policy.window);
8791 }
8792 #[cfg(test)]
8793 { lock_snapshot(&snapshot).unwrap().actor_select = None; }
8794 let exit_report = match wait_result {
8803 Ok(status) => classify_reaped_child_exit(&snapshot, active_child, &status),
8804 Err(err) => {
8805 active_child.drain_stderr(&spec.module_id).await;
8806 fail_after_wait_error(
8807 &spec.module_id,
8808 &snapshot,
8809 &runtime.terminal_ring,
8810 &runtime.spawn_events,
8811 format!("failed to wait for supervised module: {err}"),
8812 &runtime.failure_context(true),
8813 );
8814 untrack_if_registration_released(
8815 &process_liveness,
8816 ®istry,
8817 &spec.module_id,
8818 &snapshot,
8819 );
8820 error!(module_id = %spec.module_id, error = %err, "failed to wait for supervised module");
8821 child = None;
8822 continue;
8823 }
8824 };
8825 active_child.drain_stderr(&spec.module_id).await;
8826
8827 let delivery=runtime.config_delivery.clone();
8828 let exited_spawn=active_child.spawned_at;
8829 let spawn=active_child.config_delivery.clone();
8830 let refusal=active_child.spawn_failure.as_deref().filter(|detail|detail.starts_with("program_refused:") && !active_child.live_program_binding.as_ref().is_some_and(|binding|binding.is_refused())).zip(active_child.admission_base.clone()).map(|(detail,base)|(base,subc_config::raw_digest(detail.as_bytes())));
8831 let module_id=spec.module_id.clone();
8832 let observed=async move {
8836 if let Some(delivery)=delivery {
8837 delivery.0.on_blocking(move |owner| {
8838 if let Some(spawn)=spawn { owner.delivery_child_exited(&spawn)?; }
8839 if let Some((base,identity))=refusal {owner.record_program_refusal(&module_id,&base,identity)?;}
8840 Ok(())
8841 }).await
8842 } else {Ok(())}
8843 };
8844 tokio::pin!(observed);
8845 loop {
8846 tokio::select! {
8847 result=&mut observed=>{
8848 if let Err(error)=result {warn!(module_id=%spec.module_id,%error,"configuration exit observation failed");}
8849 break;
8850 }
8851 command=commands.recv()=>{
8852 let Some(command)=command else {containment::terminate(&runtime.child_roster.containment).await;return;};
8853 let deferred=||SuperviseError::ConfigDeliveryDeferred {module_id:spec.module_id.clone()};
8854 let command=match command {
8855 SupervisorCommand::Restart {reply,..}|SupervisorCommand::Reload {reply,..}=>{let _=reply.send(Err(deferred()));continue;}
8856 SupervisorCommand::Swap {reply,..}=>{let _=reply.send(Err(deferred()));continue;}
8857 command @ (SupervisorCommand::ConfigApplyRestart {..}|SupervisorCommand::ConfigPolicyProbe {..}|SupervisorCommand::ConfigDeliveryFailed {..})=>{requeued.push_back(command);continue;}
8858 command=>command,
8859 };
8860 if !handle_supervisor_command(command,&mut spec,&mut runtime,®istry,&process_liveness,&snapshot,&mut child,&mut commands,&mut requeued).await {
8861 containment::terminate(&runtime.child_roster.containment).await;return;
8862 }
8863 if child.as_ref().is_none_or(|current|current.spawned_at!=exited_spawn) {continue 'supervision;}
8864 }
8865 }
8866 }
8867 let active_child=child.as_mut().expect("exit observation retained its child");
8868 let admission_kill = admission_exit::is_admission_kill(active_child.admission_kill_sent, &exit_report);
8872 let deferred_spawn=admission_kill && active_child.spawn_failure.as_deref().is_some_and(|detail|detail.starts_with("config_delivery_deferred:"));
8873 let refused_program = active_child.live_program_binding.as_ref().is_some_and(|binding| binding.is_refused());
8874 let exit_held = !deferred_spawn && (refused_program || runtime.config_delivery.as_ref().is_some_and(|delivery| delivery.0.exit_is_held(&spec.module_id)));
8875 let parked = exit_held && (!refused_program || active_child.admission_base.is_some())
8876 && try_park_config_attempt(&spec, &runtime, &snapshot, &runtime.failure_context(true),
8877 if refused_program { active_child.admission_base.as_ref() } else { None })
8878 .unwrap_or_else(|error| { warn!(module_id=%spec.module_id, %error, "exit hold could not be recorded; using ordinary exit accounting"); false });
8879 let next = if deferred_spawn {
8880 record_terminal_with_detail(&spec.module_id,&runtime.terminal_ring,&runtime.spawn_events,&exit_report,TerminalDisposition::Restarting,active_child.spawn_failure.clone());
8881 let _=update_snapshot(&snapshot,Some(&spec.module_id),|state| {
8882 state.state=ModuleState::Restarting;
8883 clear_current_process_facts(state);
8884 state.last_exit=Some(exit_report.clone());
8885 });
8886 NextAction::Restart {schedule:None}
8887 } else if parked {
8888 record_terminal_with_detail(&spec.module_id, &runtime.terminal_ring, &runtime.spawn_events, &exit_report, TerminalDisposition::Stopped, active_child.spawn_failure.clone().or_else(|| Some("config_gated".into())));
8889 NextAction::Stop { registration_released: registry.get_module(&spec.module_id).ok().flatten().is_none() }
8890 } else { on_child_exit(
8891 &spec,
8892 runtime.restart_policy,
8893 ®istry,
8894 &snapshot,
8895 &runtime.terminal_ring,
8896 &runtime.spawn_events,
8897 &runtime.child_roster,
8898 exit_report,
8899 true,
8900 ).await };
8901 runtime.start_requester = None;
8902 runtime.pending_activity_restart = false;
8903 let carry = matches!(next, NextAction::Restart { .. })
8906 && active_child.spawn_failure.is_none()
8907 && !runtime.child_roster.is_closed();
8908 if let Err(error) = active_child.finish_exit(carry).await {
8909 containment::terminate(&runtime.child_roster.containment).await;
8910 fail_after_wait_error(&spec.module_id, &snapshot, &runtime.terminal_ring,
8911 &runtime.spawn_events, format!("failed to reap classified exit: {error}"),
8912 &runtime.failure_context(true));
8913 child = None;
8914 continue;
8915 }
8916 match next {
8917 NextAction::Stop { registration_released } => {
8918 if registration_released {
8919 process_liveness.untrack_if_current(&spec.module_id, &snapshot);
8920 }
8921 child = None;
8922 }
8923 NextAction::Restart { schedule } => {
8924 let delay = if deferred_spawn {CONFIG_ADMISSION_RETRY_DELAY} else {schedule.map_or(
8925 runtime.restart_policy.delay_for_restart(0),
8926 |schedule| schedule.delay,
8927 )};
8928 if let Some(schedule) = schedule {
8929 log_crash_respawn(&spec.module_id, schedule);
8930 }
8931 child = None;
8939 pending_respawn = Some(PendingRespawn { deadline: Instant::now() + delay, kind: RespawnKind::Spawn, daemon_initiated: true, approval: None });
8940 let _ = update_snapshot(&snapshot, Some(&spec.module_id), |state| state.respawn_pending = true);
8941 }
8942 }
8943 }
8944 command = commands.recv() => {
8945 #[cfg(test)]
8946 { lock_snapshot(&snapshot).unwrap().actor_select = None; }
8947 let Some(command) = command else {
8948 containment::terminate(&runtime.child_roster.containment).await;
8949 return;
8950 };
8951 if !handle_supervisor_command(
8952 command,
8953 &mut spec,
8954 &mut runtime,
8955 ®istry,
8956 &process_liveness,
8957 &snapshot,
8958 &mut child,
8959 &mut commands,
8960 &mut requeued,
8961 ).await {
8962 containment::terminate(&runtime.child_roster.containment).await;
8963 return;
8964 }
8965 }
8966 _ = async {
8967 match registration_changes.as_mut() {
8968 Some(changes) => { let _ = changes.changed().await; }
8969 None => std::future::pending().await,
8970 }
8971 }, if wire_child => {
8972 #[cfg(test)]
8973 { lock_snapshot(&snapshot).unwrap().actor_select = None; }
8974 }
8975 _ = &mut probe_sleep => {
8976 #[cfg(test)]
8977 { lock_snapshot(&snapshot).unwrap().actor_select = None; }
8978 if registration_deadline.is_some_and(|deadline| Instant::now() >= deadline) {
8979 active_child.registration_deadline = None;
8982 if !runtime.child_roster.is_closed()
8983 && !active_child.registered_since_spawn()
8984 && registration_child_is_running(active_child)
8985 {
8986 warn!(module_id = %spec.module_id, pid = active_child.pid,
8987 wait_secs = runtime.register_timeout.as_secs(),
8988 "supervised module registration_timeout: first HELLO deadline expired; stopping child");
8989 active_child.spawn_failure = Some("registration_timeout".into());
8990 if let Err(error) = stop_unregistered_child(active_child).await {
8991 error!(module_id = %spec.module_id, %error, "failed to stop unregistered child");
8992 fail_after_wait_error(&spec.module_id, &snapshot, &runtime.terminal_ring,
8993 &runtime.spawn_events, format!("registration_timeout: {error}"),
8994 &runtime.failure_context(true));
8995 child = None;
8996 }
8997 continue;
9000 }
9001 }
9002 if health_probe.due() {
9003 run_health_probe_cycle(
9004 &spec,
9005 &runtime,
9006 ®istry,
9007 &process_liveness,
9008 &snapshot,
9009 &mut child,
9010 ).await;
9011 if child.is_none() {
9012 runtime.start_requester = None;
9013 runtime.pending_activity_restart = false;
9014 }
9015 if child.is_some() {
9016 health_probe.schedule_next(&spec, runtime.health.cadence);
9017 }
9018 }
9019 }
9020 }
9021 } else if let Some(pending) = pending_respawn.clone() {
9022 tokio::select! {
9023 _ = runtime.child_roster.closed() => {
9024 pending_respawn = None;
9025 let _ = update_snapshot(&snapshot, Some(&spec.module_id), |state| {
9026 state.state = ModuleState::Stopped;
9027 state.respawn_pending = false;
9028 });
9029 containment::terminate(&runtime.child_roster.containment).await;
9030 }
9031 _ = async {
9032 sleep_until(pending.deadline).await;
9033 if let Some(delivery) = &runtime.config_delivery {
9034 delivery.unfenced(&spec.module_id).await;
9035 }
9036 } => {
9037 pending_respawn = None;
9038 let failure = pending.failure_context(&runtime);
9039 let _ = update_snapshot(&snapshot, Some(&spec.module_id), |state| state.respawn_pending = false);
9040 if !respawn_still_pending(&snapshot) {
9044 continue;
9045 }
9046 if runtime.child_roster.is_closed() {
9051 let _ = update_snapshot(&snapshot, Some(&spec.module_id), |state| {
9052 state.state = ModuleState::Stopped;
9053 });
9054 debug!(module_id = %spec.module_id, "crash respawn cancelled by daemon shutdown");
9055 continue;
9056 }
9057 if let Err(err) = release_dead_registration(
9058 ®istry,
9059 runtime.forwarding.as_deref(),
9060 &snapshot,
9061 &spec.module_id,
9062 &failure,
9063 ).await {
9064 record_approved_attempt_failure(&spec, &snapshot, &failure, &err);
9065 error!(module_id = %spec.module_id, error = %err, "registration did not release before restart");
9066 continue;
9067 }
9068
9069 if matches!(pending.kind, RespawnKind::Reload) {
9070 let reply = runtime.deferred_reload_reply.lock().unwrap_or_else(|p| p.into_inner()).take();
9071 let result = finish_reload_child(&spec, &runtime, ®istry, &process_liveness, &snapshot, &mut child).await;
9072 if matches!(result, Err(SuperviseError::ConfigDeliveryDeferred { .. })) {
9073 *runtime.deferred_reload_reply.lock().unwrap_or_else(|p| p.into_inner()) = reply;
9074 continue;
9075 }
9076 if result.is_ok() && runtime.pending_activity_restart {
9077 runtime.record_activity_transition("subc.module_restarted", &spec.module_id, runtime.start_requester.clone());
9078 }
9079 runtime.pending_activity_restart = false;
9080 if let Some(reply) = reply { let _ = reply.send(result); }
9081 continue;
9082 }
9083 process_liveness.track(spec.module_id.clone(), Arc::clone(&snapshot));
9084 match spawn_and_mark_running_async(&spec, &runtime, &snapshot).await {
9085 Ok(next_child) => {
9086 child = Some(next_child);
9087 if runtime.pending_activity_restart {
9088 runtime.record_activity_transition("subc.module_restarted", &spec.module_id, runtime.start_requester.clone());
9089 runtime.pending_activity_restart = false;
9090 }
9091 debug!(module_id = %spec.module_id, "supervised module restarted after crash");
9092 }
9093 Err(SuperviseError::ConfigDeliveryDeferred { .. }) => {
9094 pending_respawn = Some(PendingRespawn { deadline:Instant::now()+CONFIG_ADMISSION_RETRY_DELAY, ..pending });
9095 let _ = update_snapshot(&snapshot, Some(&spec.module_id), |state| state.respawn_pending = true);
9096 }
9097 Err(SuperviseError::ConfigHeld { .. }) => {
9098 let _ = hold_config_spawn(&spec, &runtime, &snapshot, &failure);
9099 runtime.pending_activity_restart = false;
9100 process_liveness.untrack_if_current(&spec.module_id, &snapshot);
9101 }
9102 Err(err) => {
9103 fail_snapshot(&snapshot, Some(&spec.module_id), None, &failure);
9104 record_approved_attempt_failure(&spec, &snapshot, &failure, &err);
9105 process_liveness.untrack_if_current(&spec.module_id, &snapshot);
9106 error!(module_id = %spec.module_id, error = %err, "failed to restart supervised module");
9107 }
9108 }
9109 }
9110 command = commands.recv() => {
9111 let Some(command) = command else {
9112 containment::terminate(&runtime.child_roster.containment).await;
9113 return;
9114 };
9115 if !handle_supervisor_command(
9116 command,
9117 &mut spec,
9118 &mut runtime,
9119 ®istry,
9120 &process_liveness,
9121 &snapshot,
9122 &mut child,
9123 &mut commands,
9124 &mut requeued,
9125 ).await {
9126 containment::terminate(&runtime.child_roster.containment).await;
9127 return;
9128 }
9129 if child.is_some() || !respawn_still_pending(&snapshot) {
9134 pending_respawn = None;
9135 let _ = update_snapshot(&snapshot, Some(&spec.module_id), |state| state.respawn_pending = false);
9136 }
9137 }
9138 }
9139 } else {
9140 let slow_retry = lock_snapshot(&snapshot).ok().and_then(|state| {
9141 (state.enabled && state.state == ModuleState::Failed)
9142 .then_some(state.failed_retry.pending)
9143 .flatten()
9144 });
9145 let command = if lock_snapshot(&snapshot)
9146 .is_ok_and(|state| state.waiting_for_previous_pid.is_some())
9147 {
9148 tokio::select! {
9149 command = commands.recv() => command,
9150 _ = previous_instance_recheck.tick() => {
9151 if runtime.child_roster.previous_instance_pid(&spec.module_id).is_none() {
9152 let enabled = lock_snapshot(&snapshot).is_ok_and(|state| state.enabled);
9153 if enabled && !runtime.child_roster.is_closed() {
9154 process_liveness.track(spec.module_id.clone(), Arc::clone(&snapshot));
9155 match spawn_and_mark_running_async(&spec, &runtime, &snapshot).await {
9156 Ok(next_child) => child = Some(next_child),
9157 Err(SuperviseError::ConfigDeliveryDeferred { .. }) => {
9158 let _ = defer_config_spawn(&spec, &runtime, &snapshot, RespawnKind::Spawn, &runtime.failure_context(true));
9159 }
9160 Err(SuperviseError::ConfigHeld { .. }) => {
9161 let _ = hold_config_spawn(&spec, &runtime, &snapshot, &runtime.failure_context(true));
9162 }
9163 Err(error) => {
9164 fail_snapshot(&snapshot, Some(&spec.module_id), None, &runtime.failure_context(true));
9165 error!(module_id = %spec.module_id, %error, "failed to start module after its previous instance exited");
9166 }
9167 }
9168 } else {
9169 let _ = update_snapshot(&snapshot, Some(&spec.module_id), |state| {
9170 state.waiting_for_previous_pid = None;
9171 state.waiting_for_previous_detail = None;
9172 });
9173 }
9174 }
9175 continue;
9176 }
9177 }
9178 } else {
9179 tokio::select! {
9180 biased;
9181 command = commands.recv() => command,
9182 _ = async {
9183 match slow_retry {
9184 Some(retry) => sleep_until(retry.deadline).await,
9185 None => std::future::pending().await,
9186 }
9187 } => {
9188 let retry = slow_retry.expect("only a scheduled slow retry can wake");
9189 let _ = update_snapshot(&snapshot, Some(&spec.module_id), |state| {
9190 state.failed_retry.pending = None;
9191 });
9192 if runtime.child_roster.is_closed() {
9193 continue;
9194 }
9195 info!(module_id = %spec.module_id, attempt = retry.attempt,
9196 delay_secs = retry.delay.as_secs(), "starting failed module retry");
9197 if let Err(error) = set_child_enabled(
9198 &spec, &runtime, ®istry, &process_liveness, &snapshot, &mut child, true,
9199 &runtime.failure_context(true),
9200 ).await {
9201 error!(module_id = %spec.module_id, %error, "failed module retry could not start");
9202 }
9203 continue;
9204 }
9205 }
9206 };
9207 let Some(command) = command else {
9208 containment::terminate(&runtime.child_roster.containment).await;
9209 return;
9210 };
9211 if !handle_supervisor_command(
9212 command,
9213 &mut spec,
9214 &mut runtime,
9215 ®istry,
9216 &process_liveness,
9217 &snapshot,
9218 &mut child,
9219 &mut commands,
9220 &mut requeued,
9221 )
9222 .await
9223 {
9224 containment::terminate(&runtime.child_roster.containment).await;
9225 return;
9226 }
9227 }
9228 }
9229}
9230
9231async fn supervisor_sleep(wake_at: Option<Instant>) {
9234 match wake_at {
9235 Some(deadline) => sleep_until(deadline).await,
9236 None => std::future::pending().await,
9237 }
9238}
9239
9240fn log_crash_respawn(module_id: &str, schedule: CrashRestartSchedule) {
9241 info!(
9242 module_id,
9243 restart_in_window = schedule.restart_in_window,
9244 delay_ms = schedule.delay.as_millis() as u64,
9245 "respawning after crash"
9246 );
9247}
9248
9249fn respawn_still_pending(snapshot: &SharedSnapshot) -> bool {
9255 matches!(
9256 lock_snapshot(snapshot),
9257 Ok(state) if state.enabled && state.state == ModuleState::Restarting
9258 )
9259}
9260
9261enum NextAction {
9262 Stop {
9263 registration_released: bool,
9264 },
9265 Restart {
9266 schedule: Option<CrashRestartSchedule>,
9267 },
9268}
9269
9270#[allow(clippy::too_many_arguments)]
9271async fn handle_supervisor_command(
9272 command: SupervisorCommand,
9273 spec: &mut ModuleSpec,
9274 runtime: &mut SupervisorRuntimeConfig,
9275 registry: &Arc<Registry>,
9276 process_liveness: &SupervisorProcessLiveness,
9277 snapshot: &SharedSnapshot,
9278 child: &mut Option<SupervisedChild>,
9279 commands: &mut mpsc::Receiver<SupervisorCommand>,
9280 requeued: &mut VecDeque<SupervisorCommand>,
9281) -> bool {
9282 if let SupervisorCommand::SetEnabled {
9283 enabled: true,
9284 reset_budget: false,
9285 reply,
9286 ..
9287 } = command
9288 {
9289 let parked = lock_snapshot(snapshot).ok().and_then(|state| {
9290 (state.config_hold_parked
9291 && state.enabled
9292 && !state.process_alive
9293 && child.is_none()
9294 && state.state == ModuleState::Stopped)
9295 .then(|| {
9296 (
9297 state.config_hold_daemon_initiated,
9298 state.config_hold_approval.clone(),
9299 )
9300 })
9301 });
9302 let result = if let Some((daemon_initiated, approval)) = parked {
9306 runtime.recovery_is_daemon = daemon_initiated;
9307 let failure = FailureContext {
9308 approval,
9309 ..runtime.failure_context(daemon_initiated)
9310 };
9311 let result = set_child_enabled_with_budget_policy(
9312 spec,
9313 runtime,
9314 registry,
9315 process_liveness,
9316 snapshot,
9317 child,
9318 true,
9319 &failure,
9320 false,
9321 )
9322 .await;
9323 if let Err(error) = &result {
9324 record_approved_attempt_failure(spec, snapshot, &failure, error);
9325 }
9326 result
9327 } else {
9328 Ok(false)
9329 };
9330 let _ = reply.send(result);
9331 return true;
9332 }
9333 let blocked = runtime
9334 .config_delivery
9335 .as_ref()
9336 .and_then(|delivery| delivery.0.check_module_hold(&spec.module_id).err())
9337 .filter(|error| error.code != "config_delivery_deferred")
9338 .map(|error| {
9339 config_delivery::gate_hold_error(spec, error, runtime.config_delivery.as_ref())
9340 })
9341 .or_else(|| recovery_block_error(spec, &runtime.child_roster))
9342 .or_else(|| {
9343 runtime
9344 .child_roster
9345 .previous_instance(&spec.module_id)
9346 .map(|(pid, reason)| previous_instance_error(spec, pid, reason))
9347 });
9348 let command = if let Some(error) = blocked {
9349 match command {
9350 SupervisorCommand::Restart { reply, .. } | SupervisorCommand::Reload { reply, .. } => {
9351 let _ = reply.send(Err(error));
9352 return true;
9353 }
9354 SupervisorCommand::Swap { reply, .. } => {
9355 let _ = reply.send(Err(error));
9356 return true;
9357 }
9358 SupervisorCommand::ConfigApplyRestart { .. } => return true,
9359 SupervisorCommand::SetEnabled {
9360 enabled: true,
9361 requester,
9362 reply,
9363 ..
9364 } => {
9365 if matches!(error, SuperviseError::ConfigHeld { .. })
9366 && lock_snapshot(snapshot).is_ok_and(|state| !state.enabled)
9367 && runtime
9368 .config_delivery
9369 .as_ref()
9370 .is_some_and(|delivery| delivery.0.can_stage_unstamp_enable())
9371 {
9372 let mut changed = false;
9376 let result = update_snapshot(snapshot, Some(&spec.module_id), |state| {
9377 changed = !state.enabled;
9378 state.enabled = true;
9379 if child.is_none() && state.waiting_for_previous_pid.is_none() {
9380 state.state = ModuleState::Starting;
9381 state.failed_retry.cancel();
9382 }
9383 })
9384 .and_then(|_| {
9385 if child.is_none()
9386 && lock_snapshot(snapshot)?.waiting_for_previous_pid.is_none()
9387 {
9388 hold_config_spawn(
9389 spec,
9390 runtime,
9391 snapshot,
9392 &runtime.failure_context(false),
9393 )?;
9394 }
9395 Ok(changed)
9396 });
9397 if result.is_ok() && changed {
9398 runtime.start_requester = requester;
9399 }
9400 let _ = reply.send(result);
9401 return true;
9402 }
9403 let _ = reply.send(Err(error));
9404 return true;
9405 }
9406 other => other,
9407 }
9408 } else {
9409 command
9410 };
9411 if matches!(
9414 command,
9415 SupervisorCommand::Drain { .. }
9416 | SupervisorCommand::Retire { .. }
9417 | SupervisorCommand::SetEnabled { .. }
9418 | SupervisorCommand::Restart { .. }
9419 ) {
9420 let _ = update_snapshot(snapshot, Some(&spec.module_id), |state| {
9421 state.failed_retry.cancel();
9422 });
9423 }
9424 match command {
9425 SupervisorCommand::ApprovedConfiguration { approved, reply } => {
9426 let result = async {
9427 let next = approved.module_entry(&spec.module_id).map_err(|error| {
9428 SuperviseError::InvalidSpec { reason: error.to_string() }
9429 })?;
9430 let retry_approved_failure = {
9431 let mut state = lock_snapshot(snapshot)?;
9432 if let Some((generation, detail)) = &state.approved_start_failure {
9433 if *generation == approved.identity() && next.enabled {
9434 return Err(SuperviseError::Spawn {
9435 program: spec.program.clone(),
9436 source: std::io::Error::other(detail.clone()),
9437 cgroup_path: None,
9438 });
9439 }
9440 }
9441 let retry = state.approved_start_failure.is_some();
9442 state.approved_start_failure = None;
9443 retry
9444 };
9445 stage_supervisor_configuration(
9446 spec, runtime, snapshot, next.module_spec(), next.health,
9447 next.drain_timeout_ms,
9448 );
9449 runtime.restart_policy = next.restart;
9450 runtime.register_timeout = next.register_timeout;
9451 if let Some(active) = child.as_mut()
9452 .filter(|child| child.registration_deadline.is_some())
9453 {
9454 active.registration_deadline = Some(active.spawned_at + next.register_timeout);
9455 }
9456 let enabled = lock_snapshot(snapshot)?.enabled;
9457 let retry_gate_stopped = lock_snapshot(snapshot)?
9464 .needs_approved_gate_restart(&approved.identity());
9465 let retry_failed = enabled && child.is_none()
9466 && ((lock_snapshot(snapshot)?.state == ModuleState::Failed
9467 && (retry_approved_failure || approved.changes_entry(&spec.module_id)))
9468 || retry_gate_stopped);
9469 if retry_failed {
9470 update_snapshot(snapshot, Some(&spec.module_id), |state| {
9473 state.failed_retry.cancel();
9474 if retry_gate_stopped {
9475 state.stop_reason = None;
9476 state.config_exit_failed_approval = None;
9477 }
9478 })?;
9479 }
9480 if enabled != next.enabled || retry_failed {
9481 let failure = FailureContext {
9482 approval: next.enabled.then(|| approved.identity()),
9483 ..runtime.failure_context(false)
9484 };
9485 let held = next.enabled && runtime.config_delivery.as_ref()
9486 .is_some_and(|delivery| delivery.0.has_recorded_admission_hold(&spec.module_id)
9487 || delivery.0.check_module_hold(&spec.module_id).is_err_and(|error| error.code == "config_delivery_deferred"));
9488 if held {
9489 reset_restart_count(snapshot, &spec.module_id)?;
9492 update_snapshot(snapshot, Some(&spec.module_id), |state| {
9493 state.enabled = true;
9494 if child.is_none() && state.waiting_for_previous_pid.is_none() {
9495 state.state = ModuleState::Starting;
9496 }
9497 })?;
9498 if child.is_none() && lock_snapshot(snapshot)?.waiting_for_previous_pid.is_none() {
9499 hold_config_spawn(spec, runtime, snapshot, &failure)?;
9500 }
9501 } else {
9502 match set_child_enabled_with_budget_policy(
9503 spec, runtime, registry, process_liveness, snapshot, child,
9504 next.enabled, &failure, true,
9505 ).await {
9506 Ok(_) => {},
9507 Err(SuperviseError::ConfigHeld { .. }) if next.enabled => {},
9508 Err(error) => return Err(error),
9509 }
9510 if !next.enabled {
9511 containment::terminate(&runtime.child_roster.containment).await;
9512 }
9513 }
9514 }
9515 if (child.as_ref().is_some_and(|child| child.registration_spawn.is_some() && child.admission_base.is_none())
9516 || (child.is_none() && lock_snapshot(snapshot)?.state == ModuleState::Restarting
9517 && runtime.config_delivery.as_ref().is_some_and(|delivery| delivery.0.stamp_channel_cutover_pending())))
9518 && runtime.config_delivery.as_ref().is_some_and(|delivery| delivery.0.module_is_gated(&spec.module_id))
9519 {
9520 let failure = FailureContext {
9521 approval: Some(approved.identity()),
9522 ..runtime.failure_context(false)
9523 };
9524 if child.is_some() {
9528 restart_child_with_budget_policy(spec, runtime, registry,
9529 process_liveness, snapshot, child, runtime.drain_timeout, false).await?;
9530 }
9531 update_snapshot(snapshot, Some(&spec.module_id), |state| state.respawn_pending = false)?;
9534 match spawn_and_mark_running_async(spec, runtime, snapshot).await {
9535 Ok(replacement) => *child = Some(replacement),
9536 Err(SuperviseError::ConfigDeliveryDeferred { .. }) => {
9537 defer_config_spawn(spec, runtime, snapshot, RespawnKind::Spawn, &failure)?;
9538 }
9539 Err(SuperviseError::ConfigHeld { .. }) => hold_config_spawn(spec, runtime, snapshot, &failure)?,
9540 Err(error) => {
9541 fail_snapshot(snapshot, Some(&spec.module_id), None, &failure);
9542 return Err(error);
9543 }
9544 }
9545 }
9546 info!(module_id=%spec.module_id, config_generation=approved.generation(), "approved daemon entry reconciled");
9547 Ok(())
9548 }
9549 .await;
9550 if let Err(error) = &result {
9551 let _ = update_snapshot(snapshot, Some(&spec.module_id), |state| {
9552 if state.enabled
9553 && state.state == ModuleState::Failed
9554 && state
9555 .approved_start_failure
9556 .as_ref()
9557 .is_none_or(|(generation, _)| *generation != approved.identity())
9558 {
9559 state.approved_start_failure =
9560 Some((approved.identity(), error.to_string()));
9561 }
9562 });
9563 }
9564 let _ = reply.send(result);
9565 true
9566 }
9567 SupervisorCommand::ConfigPolicyProbe {
9568 connection,
9569 generation,
9570 attempt,
9571 } => {
9572 let Some(delivery) = &runtime.config_delivery else {
9573 return true;
9574 };
9575 let Ok(gate) = delivery.0.module_gate(&spec.module_id) else {
9576 return true;
9577 };
9578 let guard = gate.lock().await;
9579 let pending = delivery
9580 .0
9581 .on_blocking(move |owner| {
9582 owner.policy_probe_pending(connection, generation, attempt)
9583 })
9584 .await
9585 .unwrap_or(false)
9586 && child.is_some()
9587 && registry
9588 .get_module(&spec.module_id)
9589 .ok()
9590 .flatten()
9591 .is_some_and(|registration| registration.connection_id == connection);
9592 drop(guard);
9593 if pending {
9594 run_health_probe_cycle(spec, runtime, registry, process_liveness, snapshot, child)
9595 .await;
9596 }
9597 true
9598 }
9599 SupervisorCommand::ConfigApplyRestart {
9600 generation,
9601 supersedes_swap,
9602 } => {
9603 let Some(owner) = runtime
9604 .config_delivery
9605 .as_ref()
9606 .map(|delivery| delivery.0.clone())
9607 else {
9608 return true;
9609 };
9610 let Ok(gate) = owner.module_gate(&spec.module_id) else {
9611 return true;
9612 };
9613 let _guard = gate.lock().await;
9614 let spawn = child
9615 .as_ref()
9616 .and_then(|child| child.config_delivery.clone());
9617 let module = spec.module_id.clone();
9618 let current = owner
9619 .on_blocking(move |owner| {
9620 Ok((supersedes_swap
9621 || owner
9622 .pending_config_restarts()?
9623 .contains(&(module.clone(), generation)))
9624 && owner.requires_fresher_spawn(&module, spawn.as_ref())?
9625 == Some(generation))
9626 })
9627 .await
9628 .unwrap_or(false);
9629 if !current
9630 || !lock_snapshot(snapshot).is_ok_and(|state| {
9631 state.enabled && !state.respawn_pending && state.process_alive
9632 })
9633 {
9634 return true;
9635 }
9636 drop(_guard);
9637 info!(module_id=%spec.module_id, config_generation=generation, "approved configuration restart preserves the crash budget");
9638 if let Err(error) = restart_child_with_budget_policy(
9639 spec,
9640 runtime,
9641 registry,
9642 process_liveness,
9643 snapshot,
9644 child,
9645 runtime.drain_timeout,
9646 false,
9647 )
9648 .await
9649 {
9650 warn!(module_id=%spec.module_id, %error, "approved configuration restart could not be scheduled");
9651 }
9652 true
9653 }
9654 SupervisorCommand::ConfigExitHold { reply } => {
9655 let idle_start = child.is_none()
9659 && lock_snapshot(snapshot).is_ok_and(|state| {
9660 state.enabled
9661 && state.state == ModuleState::Starting
9662 && !state.process_alive
9663 && !state.respawn_pending
9664 && state.waiting_for_previous_pid.is_none()
9665 && state.waiting_for_previous_detail.is_none()
9666 && state.failed_retry.pending.is_none()
9667 })
9668 && runtime
9669 .scheduled_respawn
9670 .lock()
9671 .unwrap_or_else(|p| p.into_inner())
9672 .is_none()
9673 && runtime
9674 .deferred_reload_reply
9675 .lock()
9676 .unwrap_or_else(|p| p.into_inner())
9677 .is_none();
9678 let result = if idle_start {
9679 try_park_config_attempt(
9680 spec,
9681 runtime,
9682 snapshot,
9683 &runtime.failure_context(runtime.recovery_is_daemon),
9684 None,
9685 )
9686 .map(|_| ())
9687 } else {
9688 Ok(())
9689 };
9690 let _ = reply.send(result);
9691 true
9692 }
9693 SupervisorCommand::ConfigDeliveryFailed { connection, hold } => {
9694 if registry
9695 .get_module(&spec.module_id)
9696 .ok()
9697 .flatten()
9698 .is_none_or(|registration| registration.connection_id != connection)
9699 {
9700 return true;
9701 }
9702 runtime.recovery_is_daemon = true;
9703 if hold {
9707 let notice = begin_forwarding_drain_if_configured(
9708 spec,
9709 runtime,
9710 registry,
9711 snapshot,
9712 Some(true),
9713 RouteCloseReason::Disable,
9714 )
9715 .await;
9716 if let Ok(notice) = notice {
9717 let drained = drain_optional_child(
9718 &spec.module_id,
9719 spec.protocol,
9720 notice,
9721 registry,
9722 runtime.forwarding.as_deref(),
9723 snapshot,
9724 &runtime.terminal_ring,
9725 &runtime.spawn_events,
9726 child,
9727 runtime.drain_timeout,
9728 ModuleState::Failed,
9729 Some(true),
9730 &runtime.failure_context(true),
9731 )
9732 .await;
9733 if drained.is_ok() {
9734 match try_park_config_attempt(
9735 spec,
9736 runtime,
9737 snapshot,
9738 &runtime.failure_context(true),
9739 None,
9740 ) {
9741 Ok(true) => {}
9742 Ok(false) => {
9743 if let Err(error) = budgeted_restart_child(
9744 spec,
9745 runtime,
9746 registry,
9747 process_liveness,
9748 snapshot,
9749 child,
9750 SupervisorHealthStatus::Unknown,
9751 Some("configuration delivery failed after its hold ended"),
9752 unix_ms_now(),
9753 false,
9754 )
9755 .await
9756 {
9757 warn!(module_id=%spec.module_id, %error, "configuration failure restart could not be scheduled");
9758 }
9759 }
9760 Err(error) => {
9761 warn!(module_id=%spec.module_id, %error, "configuration failure could not be parked")
9762 }
9763 }
9764 }
9765 }
9766 } else if let Err(error) = budgeted_restart_child(
9767 spec,
9768 runtime,
9769 registry,
9770 process_liveness,
9771 snapshot,
9772 child,
9773 SupervisorHealthStatus::Unknown,
9774 Some("configuration delivery failed"),
9775 unix_ms_now(),
9776 false,
9777 )
9778 .await
9779 {
9780 warn!(module_id=%spec.module_id, %error, "configuration failure restart could not be scheduled");
9781 }
9782 true
9783 }
9784 SupervisorCommand::Drain { requester, reply } => {
9785 let result = drain_optional_child(
9788 &spec.module_id,
9789 spec.protocol,
9790 StopNotice::NotSent,
9791 registry,
9792 runtime.forwarding.as_deref(),
9793 snapshot,
9794 &runtime.terminal_ring,
9795 &runtime.spawn_events,
9796 child,
9797 runtime.drain_timeout,
9798 ModuleState::Stopped,
9799 None,
9800 &FailureContext::operator(),
9801 )
9802 .await;
9803 containment::terminate(&runtime.child_roster.containment).await;
9804 let registration_released = result.is_ok();
9805 if registration_released {
9806 runtime.record_activity_transition(
9807 "subc.module_stopped",
9808 &spec.module_id,
9809 requester,
9810 );
9811 }
9812 let _ = reply.send(result);
9813 if registration_released {
9814 process_liveness.untrack_if_current(&spec.module_id, snapshot);
9815 }
9816 false
9817 }
9818 SupervisorCommand::Retire { requester, reply } => {
9819 let result = async {
9820 let stop_notice = begin_forwarding_drain_if_configured(
9821 spec,
9822 runtime,
9823 registry,
9824 snapshot,
9825 None,
9826 RouteCloseReason::Disable,
9827 )
9828 .await?;
9829 drain_optional_child(
9830 &spec.module_id,
9831 spec.protocol,
9832 stop_notice,
9833 registry,
9834 runtime.forwarding.as_deref(),
9835 snapshot,
9836 &runtime.terminal_ring,
9837 &runtime.spawn_events,
9838 child,
9839 runtime.drain_timeout,
9840 ModuleState::Stopped,
9841 None,
9842 &FailureContext::operator(),
9843 )
9844 .await
9845 }
9846 .await;
9847 containment::terminate(&runtime.child_roster.containment).await;
9848 let registration_released = result.is_ok();
9849 if registration_released {
9850 runtime.record_activity_transition(
9851 "subc.module_stopped",
9852 &spec.module_id,
9853 requester,
9854 );
9855 }
9856 let _ = reply.send(result);
9857 if registration_released {
9858 process_liveness.untrack_if_current(&spec.module_id, snapshot);
9859 }
9860 false
9861 }
9862 SupervisorCommand::Restart {
9863 requester,
9864 drain_timeout_ms,
9865 received_at_generation,
9866 queued_at,
9867 reply,
9868 } => {
9869 info!(
9873 module_id = %spec.module_id,
9874 queued_ms = u64::try_from(queued_at.elapsed().as_millis()).unwrap_or(u64::MAX),
9875 "restart command dequeued"
9876 );
9877 let validation = match lock_snapshot(snapshot) {
9889 Ok(state) if !state.enabled => Err(SuperviseError::Disabled {
9890 module_id: spec.module_id.clone(),
9891 }),
9892 Ok(_) => Ok(()),
9893 Err(err) => Err(err),
9894 };
9895 let initiated = validation.is_ok();
9896 let _ = reply.send(validation);
9897 let satisfied_by_generation = if initiated
9911 && child
9912 .as_ref()
9913 .is_some_and(|child| child.spawned_at > queued_at)
9914 {
9915 lock_snapshot(snapshot).ok().and_then(|state| {
9916 (!state.configuration_updated_since_spawn).then_some(state.spawn_generation)
9917 })
9918 } else {
9919 None
9920 };
9921 let satisfied_by_pending = initiated
9922 && child.is_none()
9923 && lock_snapshot(snapshot).ok().is_some_and(|mut state| {
9924 let pending = state.respawn_pending && !state.configuration_updated_since_spawn;
9928 if pending {
9929 state.coalesced_restart_pending = true;
9930 }
9931 pending
9932 });
9933 if satisfied_by_pending {
9934 debug!(module_id = %spec.module_id, "restart coalesced into the pending replacement");
9935 } else if let Some(generation) = satisfied_by_generation {
9936 info!(
9937 module_id = %spec.module_id,
9938 received_at_generation,
9939 "restart already satisfied by generation {generation}; not restarting again"
9940 );
9941 } else if initiated {
9942 let drain_timeout = drain_timeout_ms
9945 .map(Duration::from_millis)
9946 .unwrap_or(runtime.drain_timeout);
9947 if let Err(err) = restart_child(
9948 spec,
9949 runtime,
9950 registry,
9951 process_liveness,
9952 snapshot,
9953 child,
9954 drain_timeout,
9955 )
9956 .await
9957 {
9958 warn!(
9959 module_id = %spec.module_id,
9960 error = %err,
9961 "operator restart failed after initiation ack; module state carries the outcome"
9962 );
9963 let _ = update_snapshot(snapshot, Some(&spec.module_id), |state| {
9964 state.state = ModuleState::Failed;
9965 clear_current_process_facts(state);
9966 });
9967 } else {
9968 runtime.start_requester = requester;
9969 runtime.pending_activity_restart = true;
9970 }
9971 }
9972 true
9973 }
9974 SupervisorCommand::Reload { requester, reply } => {
9975 let result =
9976 reload_child(spec, runtime, registry, process_liveness, snapshot, child).await;
9977 if result.is_ok() {
9978 runtime.start_requester = requester;
9979 runtime.pending_activity_restart = true;
9980 }
9981 if result.is_ok()
9982 && runtime
9983 .scheduled_respawn
9984 .lock()
9985 .unwrap_or_else(|p| p.into_inner())
9986 .as_ref()
9987 .is_some_and(|pending| matches!(pending.kind, RespawnKind::Reload))
9988 {
9989 *runtime
9990 .deferred_reload_reply
9991 .lock()
9992 .unwrap_or_else(|p| p.into_inner()) = Some(reply);
9993 } else {
9994 let _ = reply.send(result);
9995 }
9996 true
9997 }
9998 SupervisorCommand::SetEnabled {
9999 enabled,
10000 requester,
10001 reply,
10002 reset_budget,
10003 } => {
10004 let previous_requester = runtime.start_requester.clone();
10005 if enabled {
10006 runtime.start_requester = requester.clone();
10007 }
10008 let result = set_child_enabled_with_budget_policy(
10009 spec,
10010 runtime,
10011 registry,
10012 process_liveness,
10013 snapshot,
10014 child,
10015 enabled,
10016 &runtime.failure_context(false),
10017 reset_budget,
10018 )
10019 .await;
10020 if !enabled {
10021 containment::terminate(&runtime.child_roster.containment).await;
10022 }
10023 if matches!(result, Ok(true)) {
10024 runtime.pending_activity_restart = false;
10025 if enabled {
10026 runtime.start_requester = requester;
10027 } else {
10028 runtime.start_requester = None;
10029 runtime.record_activity_transition(
10030 "subc.module_disabled",
10031 &spec.module_id,
10032 requester,
10033 );
10034 }
10035 } else {
10036 runtime.start_requester = previous_requester;
10037 }
10038 let _ = reply.send(result);
10039 true
10040 }
10041 SupervisorCommand::UpdateConfiguration {
10042 spec: next_spec,
10043 health,
10044 drain_timeout_ms,
10045 reply,
10046 } => {
10047 if runtime
10048 .config_delivery
10049 .as_ref()
10050 .is_some_and(|delivery| delivery.0.file_configuration_is_gated(&spec.module_id))
10051 {
10052 let _ = reply.send(Err(SuperviseError::ConfigHeld {
10053 module_id: spec.module_id.clone(),
10054 code: "config_gated".into(),
10055 detail: "live file reconfiguration cannot replace stamped authority".into(),
10056 }));
10057 return true;
10058 }
10059 stage_supervisor_configuration(
10060 spec,
10061 runtime,
10062 snapshot,
10063 next_spec,
10064 health,
10065 drain_timeout_ms,
10066 );
10067 let _ = update_snapshot(snapshot, Some(&spec.module_id), |state| {
10068 state.configuration_updated_since_spawn = true;
10069 });
10070 let _ = reply.send(Ok(()));
10071 true
10072 }
10073 SupervisorCommand::Swap {
10074 requester,
10075 ready_timeout,
10076 reply,
10077 } => {
10078 if spec.carry_containment
10079 || containment::current(&runtime.child_roster.containment).is_some()
10080 {
10081 let _ = reply.send(Err(SuperviseError::InvalidSpec {
10082 reason: "a carried container cannot overlap a swap candidate".to_owned(),
10083 }));
10084 return true;
10085 }
10086 let mut swap_runtime = runtime.clone();
10087 swap_runtime.start_requester = requester;
10088 let before = lock_snapshot(snapshot)
10089 .ok()
10090 .map(|state| state.spawn_generation);
10091 let end = swap::run_swap(
10092 spec,
10093 &swap_runtime,
10094 registry,
10095 process_liveness,
10096 snapshot,
10097 child,
10098 commands,
10099 ready_timeout.unwrap_or(DEFAULT_SWAP_READY_TIMEOUT),
10100 reply,
10101 )
10102 .await;
10103 if before
10104 != lock_snapshot(snapshot)
10105 .ok()
10106 .map(|state| state.spawn_generation)
10107 {
10108 runtime.start_requester = swap_runtime.start_requester;
10109 }
10110 requeued.extend(end.requeue);
10111 true
10112 }
10113 }
10114}
10115
10116fn stage_supervisor_configuration(
10117 spec: &mut ModuleSpec,
10118 runtime: &mut SupervisorRuntimeConfig,
10119 snapshot: &SharedSnapshot,
10120 next_spec: ModuleSpec,
10121 health: HealthConfig,
10122 drain_timeout_ms: Option<u64>,
10123) {
10124 if let Some(handle) = &runtime.supervisor_handle {
10125 handle.apply_identity_configuration(&next_spec);
10126 }
10127 let launch_changed = *spec != next_spec;
10128 *spec = next_spec;
10129 if launch_changed {
10130 let _ = update_snapshot(snapshot, Some(&spec.module_id), |state| {
10131 state.configuration_updated_since_spawn = true
10132 });
10133 }
10134 let health_changed = runtime.health != health;
10135 runtime.health = health;
10136 if health_changed {
10139 let _ = update_snapshot(snapshot, Some(&spec.module_id), |state| {
10140 state.health = ModuleHealthStatus::default();
10141 state.last_probe_evidence = None;
10142 });
10143 }
10144 runtime.drain_timeout = drain_timeout_ms
10145 .map(Duration::from_millis)
10146 .unwrap_or(runtime.default_drain_timeout);
10147 *runtime
10148 .effective_drain_timeout
10149 .lock()
10150 .unwrap_or_else(|p| p.into_inner()) = runtime.drain_timeout;
10151}
10152
10153async fn restart_child(
10154 spec: &ModuleSpec,
10155 runtime: &SupervisorRuntimeConfig,
10156 registry: &Registry,
10157 process_liveness: &SupervisorProcessLiveness,
10158 snapshot: &SharedSnapshot,
10159 child: &mut Option<SupervisedChild>,
10160 drain_timeout: Duration,
10161) -> Result<(), SuperviseError> {
10162 restart_child_with_budget_policy(
10163 spec,
10164 runtime,
10165 registry,
10166 process_liveness,
10167 snapshot,
10168 child,
10169 drain_timeout,
10170 true,
10171 )
10172 .await
10173}
10174
10175#[allow(clippy::too_many_arguments)]
10176async fn restart_child_with_budget_policy(
10177 spec: &ModuleSpec,
10178 runtime: &SupervisorRuntimeConfig,
10179 registry: &Registry,
10180 process_liveness: &SupervisorProcessLiveness,
10181 snapshot: &SharedSnapshot,
10182 child: &mut Option<SupervisedChild>,
10183 drain_timeout: Duration,
10184 reset_budget: bool,
10185) -> Result<(), SuperviseError> {
10186 if !lock_snapshot(snapshot)?.enabled {
10188 return Err(SuperviseError::Disabled {
10189 module_id: spec.module_id.clone(),
10190 });
10191 }
10192 let stop_notice = begin_forwarding_drain_with_timeout(
10193 spec,
10194 runtime,
10195 registry,
10196 snapshot,
10197 None,
10198 RouteCloseReason::Restart,
10199 drain_timeout,
10200 )
10201 .await?;
10202
10203 if child.is_some() {
10204 drain_optional_child(
10205 &spec.module_id,
10206 spec.protocol,
10207 stop_notice,
10208 registry,
10209 runtime.forwarding.as_deref(),
10210 snapshot,
10211 &runtime.terminal_ring,
10212 &runtime.spawn_events,
10213 child,
10214 drain_timeout,
10215 ModuleState::Restarting,
10216 Some(true),
10217 &FailureContext::operator(),
10218 )
10219 .await?;
10220 } else {
10221 update_snapshot(snapshot, Some(&spec.module_id), |state| {
10222 state.enabled = true;
10223 state.state = ModuleState::Restarting;
10224 clear_current_process_facts(state);
10225 })?;
10226 release_dead_registration(
10227 registry,
10228 runtime.forwarding.as_deref(),
10229 snapshot,
10230 &spec.module_id,
10231 &runtime.failure_context(false),
10232 )
10233 .await?;
10234 }
10235
10236 if reset_budget {
10237 reset_restart_count(snapshot, &spec.module_id)?;
10238 }
10239 process_liveness.track(spec.module_id.clone(), Arc::clone(snapshot));
10240 schedule_respawn(
10241 runtime,
10242 snapshot,
10243 &spec.module_id,
10244 runtime.restart_policy.backoff,
10245 RespawnKind::Spawn,
10246 false,
10247 )
10248}
10249
10250async fn reload_child(
10251 spec: &ModuleSpec,
10252 runtime: &SupervisorRuntimeConfig,
10253 registry: &Registry,
10254 process_liveness: &SupervisorProcessLiveness,
10255 snapshot: &SharedSnapshot,
10256 child: &mut Option<SupervisedChild>,
10257) -> Result<(), SuperviseError> {
10258 if !lock_snapshot(snapshot)?.enabled {
10260 return Err(SuperviseError::Disabled {
10261 module_id: spec.module_id.clone(),
10262 });
10263 }
10264 let stop_notice = begin_forwarding_drain(
10265 spec,
10266 runtime,
10267 registry,
10268 snapshot,
10269 Some(true),
10270 RouteCloseReason::Reload,
10271 )
10272 .await?;
10273
10274 if child.is_some() {
10275 drain_optional_child(
10276 &spec.module_id,
10277 spec.protocol,
10278 stop_notice,
10279 registry,
10280 runtime.forwarding.as_deref(),
10281 snapshot,
10282 &runtime.terminal_ring,
10283 &runtime.spawn_events,
10284 child,
10285 runtime.drain_timeout,
10286 ModuleState::Restarting,
10287 Some(true),
10288 &FailureContext::operator(),
10289 )
10290 .await?;
10291 } else {
10292 update_snapshot(snapshot, Some(&spec.module_id), |state| {
10293 state.enabled = true;
10294 state.state = ModuleState::Restarting;
10295 clear_current_process_facts(state);
10296 })?;
10297 release_dead_registration(
10298 registry,
10299 runtime.forwarding.as_deref(),
10300 snapshot,
10301 &spec.module_id,
10302 &runtime.failure_context(false),
10303 )
10304 .await?;
10305 }
10306
10307 reset_restart_count(snapshot, &spec.module_id)?;
10308 process_liveness.track(spec.module_id.clone(), Arc::clone(snapshot));
10309 schedule_respawn(
10310 runtime,
10311 snapshot,
10312 &spec.module_id,
10313 runtime.restart_policy.backoff,
10314 RespawnKind::Reload,
10315 false,
10316 )
10317}
10318
10319async fn finish_reload_child(
10320 spec: &ModuleSpec,
10321 runtime: &SupervisorRuntimeConfig,
10322 registry: &Registry,
10323 process_liveness: &SupervisorProcessLiveness,
10324 snapshot: &SharedSnapshot,
10325 child: &mut Option<SupervisedChild>,
10326) -> Result<(), SuperviseError> {
10327 process_liveness.track(spec.module_id.clone(), Arc::clone(snapshot));
10328 let next_child = match spawn_and_mark_running_async(spec, runtime, snapshot).await {
10329 Ok(next_child) => next_child,
10330 Err(error @ SuperviseError::ConfigDeliveryDeferred { .. }) => {
10331 defer_config_spawn(
10332 spec,
10333 runtime,
10334 snapshot,
10335 RespawnKind::Reload,
10336 &runtime.failure_context(runtime.recovery_is_daemon),
10337 )?;
10338 return Err(error);
10339 }
10340 Err(error @ SuperviseError::ConfigHeld { .. }) => {
10341 let failure = runtime.failure_context(runtime.recovery_is_daemon);
10342 if try_park_config_attempt(spec, runtime, snapshot, &failure, None)? {
10343 return Err(error);
10344 }
10345 defer_config_spawn(spec, runtime, snapshot, RespawnKind::Reload, &failure)?;
10346 return Err(SuperviseError::ConfigDeliveryDeferred {
10347 module_id: spec.module_id.clone(),
10348 });
10349 }
10350 Err(err) => {
10351 return handle_reload_spawn_failure(
10352 spec,
10353 runtime,
10354 process_liveness,
10355 snapshot,
10356 child,
10357 format!("new child failed to spawn: {err}"),
10358 )
10359 .await;
10360 }
10361 };
10362 *child = Some(next_child);
10363
10364 let wait_outcome = {
10365 let active_child = child.as_mut().expect("new reload child was just stored");
10366 wait_for_registration_after_reload(
10367 registry,
10368 &spec.module_id,
10369 snapshot,
10370 active_child,
10371 REGISTRY_RELEASE_TIMEOUT,
10372 )
10373 .await?
10374 };
10375
10376 match wait_outcome {
10377 RegistrationWaitOutcome::Registered => {
10378 child
10379 .as_mut()
10380 .expect("registered reload child")
10381 .registration_deadline = None;
10382 debug!(module_id = %spec.module_id, "supervised module reloaded and registered");
10383 Ok(())
10384 }
10385 RegistrationWaitOutcome::Exited(exit_report) => {
10386 if let Some(active_child) = child.as_mut() {
10387 active_child.drain_stderr(&spec.module_id).await;
10388 }
10389 let mut exited_child = child.take().expect("exited reload child is still stored");
10392 #[cfg(test)]
10393 if let Some(gate) = &runtime.test_reload_exit_record_gate {
10394 gate.reached.notify_one();
10395 gate.resume.notified().await;
10396 }
10397 let result = handle_reload_child_registration_failure(
10398 spec,
10399 runtime,
10400 registry,
10401 process_liveness,
10402 snapshot,
10403 child,
10404 ReloadRegistrationFailure {
10405 exit_report: registration_failure_exit_report(exit_report),
10406 reason: exited_child
10407 .spawn_failure
10408 .clone()
10409 .unwrap_or_else(|| "new child exited before registering".to_string()),
10410 },
10411 )
10412 .await;
10413 exited_child
10414 .finish_exit(false)
10415 .await
10416 .map_err(|source| SuperviseError::Wait {
10417 module_id: spec.module_id.clone(),
10418 source,
10419 })?;
10420 result
10421 }
10422 RegistrationWaitOutcome::TimedOut => {
10423 let mut timed_out_child = child
10424 .take()
10425 .expect("timed-out reload child is still running");
10426 timed_out_child
10427 .start_kill()
10428 .map_err(|source| SuperviseError::Kill {
10429 module_id: spec.module_id.clone(),
10430 source,
10431 })?;
10432 let status = timed_out_child
10433 .wait()
10434 .await
10435 .map_err(|source| SuperviseError::Wait {
10436 module_id: spec.module_id.clone(),
10437 source,
10438 })?;
10439 timed_out_child.drain_stderr(&spec.module_id).await;
10440 let result = handle_reload_child_registration_failure(
10441 spec,
10442 runtime,
10443 registry,
10444 process_liveness,
10445 snapshot,
10446 child,
10447 ReloadRegistrationFailure {
10448 exit_report: registration_failure_exit_report(classify_reaped_child_exit(
10449 snapshot,
10450 &timed_out_child,
10451 &status,
10452 )),
10453 reason: format!(
10454 "new child did not register within {:?}",
10455 REGISTRY_RELEASE_TIMEOUT
10456 ),
10457 },
10458 )
10459 .await;
10460 timed_out_child
10461 .finish_exit(false)
10462 .await
10463 .map_err(|source| SuperviseError::Wait {
10464 module_id: spec.module_id.clone(),
10465 source,
10466 })?;
10467 result
10468 }
10469 }
10470}
10471
10472#[allow(clippy::too_many_arguments)]
10473async fn set_child_enabled(
10474 spec: &ModuleSpec,
10475 runtime: &SupervisorRuntimeConfig,
10476 registry: &Registry,
10477 process_liveness: &SupervisorProcessLiveness,
10478 snapshot: &SharedSnapshot,
10479 child: &mut Option<SupervisedChild>,
10480 enabled: bool,
10481 failure: &FailureContext,
10482) -> Result<bool, SuperviseError> {
10483 set_child_enabled_with_budget_policy(
10484 spec,
10485 runtime,
10486 registry,
10487 process_liveness,
10488 snapshot,
10489 child,
10490 enabled,
10491 failure,
10492 true,
10493 )
10494 .await
10495}
10496
10497#[allow(clippy::too_many_arguments)]
10498async fn set_child_enabled_with_budget_policy(
10499 spec: &ModuleSpec,
10500 runtime: &SupervisorRuntimeConfig,
10501 registry: &Registry,
10502 process_liveness: &SupervisorProcessLiveness,
10503 snapshot: &SharedSnapshot,
10504 child: &mut Option<SupervisedChild>,
10505 enabled: bool,
10506 failure: &FailureContext,
10507 reset_budget: bool,
10508) -> Result<bool, SuperviseError> {
10509 let (current_enabled, current_state, respawn_pending) = {
10510 let state = lock_snapshot(snapshot)?;
10511 (state.enabled, state.state, state.respawn_pending)
10512 };
10513 let revive_terminal = enabled
10521 && current_enabled
10522 && child.is_none()
10523 && (matches!(current_state, ModuleState::Failed | ModuleState::Stopped)
10524 || (current_state == ModuleState::Restarting && !respawn_pending));
10525 if current_enabled == enabled && !revive_terminal {
10526 return Ok(false);
10527 }
10528
10529 if enabled {
10530 update_snapshot(snapshot, Some(&spec.module_id), |state| {
10531 state.enabled = true;
10532 state.state = ModuleState::Starting;
10533 clear_current_process_facts(state);
10534 })?;
10535 #[cfg(test)]
10536 if runtime.test_seed_stale_facts_before_enable_spawn {
10537 update_snapshot(snapshot, Some(&spec.module_id), |state| {
10538 state.process_alive = true;
10539 state.pid = Some(41);
10540 state.spawned_at_ms = Some(42);
10541 state.spawned_from = Some(PathBuf::from("/spawned/module"));
10542 state.spawned_file_identity = Some(SpawnedFileIdentity {
10543 device: 43,
10544 inode: 44,
10545 #[cfg(windows)]
10546 windows_image: None,
10547 });
10548 })?;
10549 }
10550 release_dead_registration(
10551 registry,
10552 runtime.forwarding.as_deref(),
10553 snapshot,
10554 &spec.module_id,
10555 failure,
10556 )
10557 .await?;
10558 if reset_budget {
10559 reset_restart_count(snapshot, &spec.module_id)?;
10560 }
10561 process_liveness.track(spec.module_id.clone(), Arc::clone(snapshot));
10562 let next_child = match spawn_and_mark_running_async(spec, runtime, snapshot).await {
10563 Ok(next_child) => next_child,
10564 Err(SuperviseError::ConfigDeliveryDeferred { .. }) => {
10565 defer_config_spawn(spec, runtime, snapshot, RespawnKind::Spawn, failure)?;
10566 return Ok(true);
10567 }
10568 Err(error @ SuperviseError::ConfigHeld { .. }) => {
10569 if let Err(parking_error) = hold_config_spawn(spec, runtime, snapshot, failure) {
10570 process_liveness.untrack_if_current(&spec.module_id, snapshot);
10571 return Err(parking_error);
10572 }
10573 return Err(error);
10574 }
10575 Err(err) => {
10576 fail_snapshot(snapshot, Some(&spec.module_id), None, failure);
10577 process_liveness.untrack_if_current(&spec.module_id, snapshot);
10578 return Err(err);
10579 }
10580 };
10581 *child = Some(next_child);
10582 debug!(module_id = %spec.module_id, "supervised module enabled");
10583 Ok(true)
10584 } else {
10585 update_snapshot(snapshot, Some(&spec.module_id), |state| {
10586 state.config_hold_parked = false;
10587 state.config_hold_daemon_initiated = false;
10588 state.config_hold_approval = None;
10589 })?;
10590 let stop_notice = begin_forwarding_drain_if_configured(
10591 spec,
10592 runtime,
10593 registry,
10594 snapshot,
10595 Some(false),
10596 RouteCloseReason::Disable,
10597 )
10598 .await?;
10599 drain_optional_child(
10600 &spec.module_id,
10601 spec.protocol,
10602 stop_notice,
10603 registry,
10604 runtime.forwarding.as_deref(),
10605 snapshot,
10606 &runtime.terminal_ring,
10607 &runtime.spawn_events,
10608 child,
10609 runtime.drain_timeout,
10610 ModuleState::Disabled,
10611 Some(false),
10612 failure,
10613 )
10614 .await?;
10615 debug!(module_id = %spec.module_id, "supervised module disabled");
10616 Ok(true)
10617 }
10618}
10619
10620#[allow(clippy::too_many_arguments)]
10621async fn on_child_exit(
10622 spec: &ModuleSpec,
10623 policy: RestartPolicy,
10624 registry: &Registry,
10625 snapshot: &SharedSnapshot,
10626 terminal_ring: &Arc<Mutex<TerminalRing>>,
10627 spawn_events: &SpawnEventFeed,
10628 roster: &ChildRoster,
10629 exit_report: ExitReport,
10630 daemon_initiated: bool,
10631) -> NextAction {
10632 if roster.is_closed() {
10638 return on_child_exit_during_daemon_shutdown(
10639 spec,
10640 registry,
10641 snapshot,
10642 terminal_ring,
10643 spawn_events,
10644 exit_report,
10645 )
10646 .await;
10647 }
10648 let unrequested_clean_exit_of_protocol_none = exit_report.kind == ExitKind::Clean
10664 && running_protocol(spec, snapshot) == ModuleProtocol::None;
10665 match exit_report.kind {
10666 ExitKind::Clean if !unrequested_clean_exit_of_protocol_none => {
10667 info!(
10668 module_id = %spec.module_id,
10669 exit_code = ?exit_report.code,
10670 exit_signal = ?exit_report.signal,
10671 "supervised module exited cleanly"
10672 );
10673 if let Err(err) = publish_after_terminal_record(
10674 &spec.module_id,
10675 || {
10676 record_terminal(
10677 &spec.module_id,
10678 terminal_ring,
10679 spawn_events,
10680 &exit_report,
10681 TerminalDisposition::Stopped,
10682 )
10683 },
10684 || {
10685 update_snapshot(snapshot, Some(&spec.module_id), |state| {
10686 state.state = ModuleState::Stopped;
10687 clear_current_process_facts(state);
10688 state.last_exit = Some(exit_report.clone());
10689 })
10690 },
10691 ) {
10692 error!(module_id = %spec.module_id, error = %err, "failed to publish clean module exit");
10693 }
10694 let registration_released = match wait_for_registration_release(
10695 registry,
10696 &spec.module_id,
10697 REGISTRY_RELEASE_TIMEOUT,
10698 )
10699 .await
10700 {
10701 Ok(()) => true,
10702 Err(err) => {
10703 warn!(module_id = %spec.module_id, error = %err, "registration still active after clean exit");
10704 false
10705 }
10706 };
10707 NextAction::Stop {
10708 registration_released,
10709 }
10710 }
10711 ExitKind::Clean | ExitKind::Crash => {
10712 if unrequested_clean_exit_of_protocol_none {
10713 warn!(
10714 module_id = %spec.module_id,
10715 exit_code = ?exit_report.code,
10716 exit_signal = ?exit_report.signal,
10717 "protocol-none module exited cleanly without a stop request; handling it as a crash"
10718 );
10719 } else {
10720 warn!(
10721 module_id = %spec.module_id,
10722 exit_code = ?exit_report.code,
10723 exit_signal = ?exit_report.signal,
10724 "supervised module exited abnormally (crash)"
10725 );
10726 }
10727 let mut restart_schedule = None;
10728 let mut disposition = TerminalDisposition::Disabled;
10729 let mut final_state = ModuleState::Disabled;
10730 let mut disposition_detail = lock_snapshot(snapshot)
10734 .ok()
10735 .and_then(|mut state| state.spawn_failure.take());
10736 let now = Instant::now();
10737 if let Err(err) = update_snapshot(snapshot, Some(&spec.module_id), |state| {
10738 if state.enabled {
10739 if let Some(schedule) = state.next_crash_restart(&policy, now) {
10740 state.respawn_pending = true;
10746 final_state = ModuleState::Restarting;
10747 restart_schedule = Some(schedule);
10748 disposition = TerminalDisposition::Restarting;
10749 } else {
10750 final_state = ModuleState::Failed;
10751 disposition = TerminalDisposition::Failed;
10752 let budget = policy.budget_exhausted_detail();
10753 disposition_detail =
10754 Some(disposition_detail.take().map_or_else(
10755 || budget.clone(),
10756 |cause| format!("{cause}; {budget}"),
10757 ));
10758 }
10759 } else {
10760 disposition = TerminalDisposition::Disabled;
10761 }
10762 }) {
10763 error!(module_id = %spec.module_id, error = %err, "failed to record crashed module exit");
10764 return NextAction::Stop {
10765 registration_released: false,
10766 };
10767 }
10768 if disposition == TerminalDisposition::Failed {
10769 error!(
10774 module_id = %spec.module_id,
10775 max_restarts = policy.max_restarts,
10776 window_secs = policy.window.as_secs(),
10777 "module stopped: {}",
10778 policy.budget_exhausted_detail()
10779 );
10780 }
10781 if let Err(err) = publish_after_terminal_record(
10782 &spec.module_id,
10783 || {
10784 record_terminal_with_detail(
10785 &spec.module_id,
10786 terminal_ring,
10787 spawn_events,
10788 &exit_report,
10789 disposition,
10790 disposition_detail,
10791 )
10792 },
10793 || {
10794 update_snapshot(snapshot, Some(&spec.module_id), |state| {
10795 state.state = final_state;
10796 clear_current_process_facts(state);
10797 if final_state == ModuleState::Failed
10798 && daemon_initiated
10799 && !roster.is_closed()
10800 {
10801 state.failed_retry.schedule(&spec.module_id);
10802 }
10803 state.last_exit = Some(exit_report.clone());
10804 })
10805 },
10806 ) {
10807 error!(module_id = %spec.module_id, error = %err, "failed to publish crashed module exit");
10808 }
10809
10810 if let Some(schedule) = restart_schedule {
10811 NextAction::Restart {
10812 schedule: Some(schedule),
10813 }
10814 } else {
10815 let registration_released = match wait_for_registration_release(
10816 registry,
10817 &spec.module_id,
10818 REGISTRY_RELEASE_TIMEOUT,
10819 )
10820 .await
10821 {
10822 Ok(()) => true,
10823 Err(err) => {
10824 warn!(module_id = %spec.module_id, error = %err, "registration still active after failed module");
10825 false
10826 }
10827 };
10828 NextAction::Stop {
10829 registration_released,
10830 }
10831 }
10832 }
10833 ExitKind::DeliberateSeverance => {
10834 warn!(
10835 module_id = %spec.module_id,
10836 exit_code = ?exit_report.code,
10837 exit_signal = ?exit_report.signal,
10838 "supervised module exited after deliberate connection severance"
10839 );
10840 let should_restart = match lock_snapshot(snapshot) {
10841 Ok(state) => state.enabled,
10842 Err(err) => {
10843 error!(module_id = %spec.module_id, error = %err, "failed to read deliberately severed module state");
10844 return NextAction::Stop {
10845 registration_released: false,
10846 };
10847 }
10848 };
10849 let final_state = if should_restart {
10850 ModuleState::Restarting
10851 } else {
10852 ModuleState::Disabled
10853 };
10854 if let Err(err) = publish_after_terminal_record(
10855 &spec.module_id,
10856 || {
10857 record_terminal(
10858 &spec.module_id,
10859 terminal_ring,
10860 spawn_events,
10861 &exit_report,
10862 terminal_disposition(final_state),
10863 )
10864 },
10865 || {
10866 update_snapshot(snapshot, Some(&spec.module_id), |state| {
10867 state.state = final_state;
10868 clear_current_process_facts(state);
10869 state.last_exit = Some(exit_report.clone());
10870 state.lifetime_restarts += 1;
10871 })
10872 },
10873 ) {
10874 error!(module_id = %spec.module_id, error = %err, "failed to publish deliberately severed module exit");
10875 return NextAction::Stop {
10876 registration_released: false,
10877 };
10878 }
10879
10880 if should_restart {
10881 NextAction::Restart { schedule: None }
10882 } else {
10883 let registration_released = match wait_for_registration_release(
10884 registry,
10885 &spec.module_id,
10886 REGISTRY_RELEASE_TIMEOUT,
10887 )
10888 .await
10889 {
10890 Ok(()) => true,
10891 Err(err) => {
10892 warn!(module_id = %spec.module_id, error = %err, "registration still active after deliberately severed module exit");
10893 false
10894 }
10895 };
10896 NextAction::Stop {
10897 registration_released,
10898 }
10899 }
10900 }
10901 }
10902}
10903
10904async fn on_child_exit_during_daemon_shutdown(
10905 spec: &ModuleSpec,
10906 registry: &Registry,
10907 snapshot: &SharedSnapshot,
10908 terminal_ring: &Arc<Mutex<TerminalRing>>,
10909 spawn_events: &SpawnEventFeed,
10910 exit_report: ExitReport,
10911) -> NextAction {
10912 info!(
10913 module_id = %spec.module_id,
10914 exit_code = ?exit_report.code,
10915 exit_signal = ?exit_report.signal,
10916 exit_kind = ?exit_report.kind,
10917 "supervised module exited during daemon shutdown; not restarting it"
10918 );
10919 if let Err(err) = publish_after_terminal_record(
10920 &spec.module_id,
10921 || {
10922 record_terminal(
10923 &spec.module_id,
10924 terminal_ring,
10925 spawn_events,
10926 &exit_report,
10927 TerminalDisposition::DaemonShutdown,
10928 )
10929 },
10930 || {
10931 update_snapshot(snapshot, Some(&spec.module_id), |state| {
10932 state.state = ModuleState::Stopped;
10933 clear_current_process_facts(state);
10934 state.last_exit = Some(exit_report.clone());
10935 })
10936 },
10937 ) {
10938 error!(module_id = %spec.module_id, error = %err, "failed to publish module exit during daemon shutdown");
10939 }
10940 let registration_released =
10941 wait_for_registration_release(registry, &spec.module_id, REGISTRY_RELEASE_TIMEOUT)
10942 .await
10943 .is_ok();
10944 NextAction::Stop {
10945 registration_released,
10946 }
10947}
10948
10949fn publish_after_terminal_record<T>(
10954 module_id: &str,
10955 record: impl FnOnce(),
10956 publish: impl FnOnce() -> T,
10957) -> T {
10958 let recorded = std::panic::catch_unwind(std::panic::AssertUnwindSafe(record));
10959 let published = publish();
10960 if let Err(panic) = recorded {
10961 error!(
10962 module_id,
10963 "terminal recording panicked; published final state before resuming unwind"
10964 );
10965 std::panic::resume_unwind(panic);
10966 }
10967 published
10968}
10969
10970fn fail_after_wait_error(
10971 module_id: &str,
10972 snapshot: &SharedSnapshot,
10973 terminal_ring: &Arc<Mutex<TerminalRing>>,
10974 spawn_events: &SpawnEventFeed,
10975 detail: String,
10976 failure: &FailureContext,
10977) {
10978 let report = wait_error_exit_report();
10979 publish_after_terminal_record(
10980 module_id,
10981 || {
10982 record_terminal_with_detail(
10983 module_id,
10984 terminal_ring,
10985 spawn_events,
10986 &report,
10987 TerminalDisposition::Failed,
10988 Some(detail),
10989 )
10990 },
10991 || fail_snapshot(snapshot, Some(module_id), Some(report.clone()), failure),
10992 );
10993}
10994
10995#[cfg(test)]
10996fn record_wait_error_terminal(
10997 module_id: &str,
10998 terminal_ring: &Arc<Mutex<TerminalRing>>,
10999 spawn_events: &SpawnEventFeed,
11000) {
11001 record_terminal(
11002 module_id,
11003 terminal_ring,
11004 spawn_events,
11005 &wait_error_exit_report(),
11006 TerminalDisposition::Failed,
11007 );
11008}
11009
11010fn record_terminal(
11011 module_id: &str,
11012 terminal_ring: &Arc<Mutex<TerminalRing>>,
11013 spawn_events: &SpawnEventFeed,
11014 exit_report: &ExitReport,
11015 disposition: TerminalDisposition,
11016) {
11017 record_terminal_with_detail(
11018 module_id,
11019 terminal_ring,
11020 spawn_events,
11021 exit_report,
11022 disposition,
11023 None,
11024 );
11025}
11026
11027fn durable_terminal_history_of(
11031 terminal_ring: &Mutex<TerminalRing>,
11032 module_id: &str,
11033) -> subc_control::TerminalHistory {
11034 let read = terminal_ring
11035 .lock()
11036 .unwrap_or_else(|p| p.into_inner())
11037 .capture_durable_history();
11038 read.read(module_id)
11039}
11040
11041fn record_terminal_with_detail(
11042 module_id: &str,
11043 terminal_ring: &Arc<Mutex<TerminalRing>>,
11044 spawn_events: &SpawnEventFeed,
11045 exit_report: &ExitReport,
11046 disposition: TerminalDisposition,
11047 disposition_detail: Option<String>,
11048) {
11049 #[cfg(test)]
11050 terminal_history_tests::before_terminal_record(spawn_events);
11051 spawn_events.emit_exited(module_id, exit_report.code, exit_report.signal);
11052 let record = TerminalRecord {
11053 exit_code: exit_report.code,
11054 exit_signal: exit_report.signal,
11055 at_ms: exit_report.at_ms,
11056 disposition,
11057 exit_kind: exit_report.kind.into(),
11058 disposition_detail,
11059 };
11060 terminal_ring
11061 .lock()
11062 .unwrap_or_else(|poisoned| poisoned.into_inner())
11063 .record_exit(module_id, record);
11064}
11065
11066fn untrack_if_registration_released(
11067 process_liveness: &SupervisorProcessLiveness,
11068 registry: &Registry,
11069 module_id: &str,
11070 snapshot: &SharedSnapshot,
11071) {
11072 match registry.get_module(module_id) {
11073 Ok(None) => process_liveness.untrack_if_current(module_id, snapshot),
11074 Ok(Some(_)) => {}
11075 Err(err) => {
11076 warn!(module_id, error = %err, "could not determine whether supervisor liveness can be untracked");
11077 }
11078 }
11079}
11080
11081#[cfg(test)]
11095fn apply_wire_spawn_args(
11096 command: &mut Command,
11097 spec: &ModuleSpec,
11098 connection_file_path: Option<&std::path::Path>,
11099 handle: Option<&SupervisorHandle>,
11100) -> Result<Option<NonceHandoff>, SuperviseError> {
11101 apply_wire_spawn_args_for_role(
11102 command,
11103 spec,
11104 connection_file_path,
11105 handle,
11106 SpawnRole::Plain,
11107 )
11108}
11109
11110#[cfg(unix)]
11115type NonceHandoff = subc_os::LaunchNonceHandoff;
11116#[cfg(windows)]
11117type NonceHandoff = subc_os::LaunchNoncePipeHandoff;
11118#[cfg(not(any(unix, windows)))]
11119type NonceHandoff = std::convert::Infallible;
11120
11121#[cfg(test)]
11139fn apply_wire_spawn_args_for_role(
11140 command: &mut Command,
11141 spec: &ModuleSpec,
11142 connection_file_path: Option<&std::path::Path>,
11143 handle: Option<&SupervisorHandle>,
11144 role: SpawnRole,
11145) -> Result<Option<NonceHandoff>, SuperviseError> {
11146 apply_wire_spawn_args_for_admission(command, spec, connection_file_path, handle, role, None)
11147}
11148
11149fn apply_wire_spawn_args_for_admission(
11150 command: &mut Command,
11151 spec: &ModuleSpec,
11152 connection_file_path: Option<&std::path::Path>,
11153 handle: Option<&SupervisorHandle>,
11154 role: SpawnRole,
11155 admission: Option<&crate::config_gate::runtime::SpawnAdmission>,
11156) -> Result<Option<NonceHandoff>, SuperviseError> {
11157 command.env(SUBC_MODULE_ID_ENV, &spec.module_id);
11158 command.env_remove(subc_os::LAUNCH_NONCE_FD_ENV);
11163 #[cfg(windows)]
11164 {
11165 command.env_remove(subc_os::LAUNCH_NONCE_PIPE_ENV);
11166 command.env_remove(subc_os::LAUNCH_NONCE_PIPE_FALLBACK_ENV);
11167 }
11168 command.env_remove(SUBC_LAUNCH_NONCE_ENV);
11170 command.env_remove(subc_os::registration_channel::REGISTRATION_FD_ENV);
11171 command.env_remove(subc_os::registration_channel::REGISTRATION_PIPE_ENV);
11172 command.env_remove(subc_os::registration_channel::DAEMON_PID_ENV);
11173 if role == SpawnRole::Plain {
11174 if let Some(handle) = handle {
11175 #[cfg(all(test, unix))]
11176 {
11177 let pause = {
11178 let mut slot = handle.plain_identity_policy_pause.lock().unwrap();
11179 if slot
11180 .as_ref()
11181 .is_some_and(|(module, _)| module == &spec.module_id)
11182 {
11183 slot.take().map(|(_, pause)| pause)
11184 } else {
11185 None
11186 }
11187 };
11188 if let Some(pause) = pause {
11189 pause.reached.send(()).unwrap();
11190 pause.release.recv().unwrap();
11191 }
11192 }
11193 handle.apply_identity_policy(spec, spec.protocol);
11197 }
11198 }
11199 if spec.protocol == ModuleProtocol::None {
11200 return Ok(None);
11201 }
11202 if let Some(connection_file_path) = connection_file_path {
11203 command.arg(SUBC_ARG).arg(connection_file_path);
11204 }
11205
11206 if admission.is_some_and(|admission| admission.launch.is_some()) {
11207 if let Some(handle) = handle {
11208 match role {
11209 SpawnRole::Plain => {
11210 handle
11211 .spawn_nonces
11212 .lock()
11213 .unwrap_or_else(|p| p.into_inner())
11214 .remove(&spec.module_id);
11215 if spec.reserved {
11216 handle
11217 .reserved_nonces
11218 .lock()
11219 .unwrap_or_else(|p| p.into_inner())
11220 .insert(spec.module_id.clone(), None);
11221 }
11222 }
11223 SpawnRole::SwapCandidate => handle.open_swap(&spec.module_id, String::new()),
11226 }
11227 }
11228 return Ok(None);
11229 }
11230 let nonce = generate_launch_nonce()?;
11234 if let Some(handle) = handle {
11235 match role {
11236 SpawnRole::Plain => {
11237 handle.set_spawn_nonce(&spec.module_id, nonce.clone());
11238 if spec.reserved {
11239 handle.set_reserved_nonce(&spec.module_id, nonce.clone());
11240 }
11241 }
11242 SpawnRole::SwapCandidate => handle.open_swap(&spec.module_id, nonce.clone()),
11243 }
11244 }
11245 #[cfg(unix)]
11246 let handoff = {
11247 let handoff =
11248 subc_os::LaunchNonceHandoff::new(&nonce).map_err(|source| SuperviseError::Spawn {
11249 program: spec.program.clone(),
11250 source,
11251 cgroup_path: None,
11252 })?;
11253 command.env(subc_os::LAUNCH_NONCE_FD_ENV, handoff.fd_env_value());
11254 Some(handoff)
11255 };
11256 #[cfg(windows)]
11257 let handoff = prepare_windows_nonce_pipe(command, spec, &nonce);
11258 #[cfg(not(any(unix, windows)))]
11259 let handoff = None;
11260 #[cfg(not(unix))]
11264 command.env(SUBC_LAUNCH_NONCE_ENV, nonce);
11265 Ok(handoff)
11266}
11267
11268pub(crate) fn apply_child_env(command: &mut Command, spec: &ModuleSpec) {
11269 command.env_remove(CK_LOG_ENV);
11270 command.env_remove(SUBC_SPAWN_ROLE_ENV);
11277 for (key, value) in &spec.env {
11278 if matches!(
11282 key.as_str(),
11283 CAPTURE_MAX_FILE_MB_ENV | CAPTURE_KEEP_ENV | CAPTURE_MAX_AGE_DAYS_ENV
11284 ) || key == SUBC_SPAWN_ROLE_ENV
11285 {
11286 continue;
11287 }
11288 command.env(key, value);
11289 }
11290}
11291
11292#[cfg(windows)]
11293fn prepare_windows_registration_pipe(
11294 command: &mut Command,
11295 spec: &ModuleSpec,
11296) -> Result<Option<SpawnRegistrationChannel>, SuperviseError> {
11297 if spec.protocol == ModuleProtocol::None {
11298 return Ok(None);
11299 }
11300 let pipe = subc_os::registration_channel::RegistrationPipe::new().map_err(|source| {
11301 SuperviseError::Spawn {
11302 program: spec.program.clone(),
11303 source,
11304 cgroup_path: None,
11305 }
11306 })?;
11307 command
11308 .env(
11309 subc_os::registration_channel::REGISTRATION_PIPE_ENV,
11310 pipe.name(),
11311 )
11312 .env(
11313 subc_os::registration_channel::DAEMON_PID_ENV,
11314 std::process::id().to_string(),
11315 );
11316 Ok(Some(pipe))
11317}
11318
11319#[cfg(windows)]
11320fn prepare_windows_nonce_pipe(
11321 command: &mut Command,
11322 spec: &ModuleSpec,
11323 nonce: &str,
11324) -> Option<NonceHandoff> {
11325 match subc_os::LaunchNoncePipeHandoff::new(nonce) {
11326 Ok(handoff) => {
11327 command.env(subc_os::LAUNCH_NONCE_PIPE_ENV, handoff.name());
11328 command.env(subc_os::LAUNCH_NONCE_PIPE_FALLBACK_ENV, "env");
11333 Some(handoff)
11334 }
11335 Err(error) => {
11336 warn!(module_id = %spec.module_id, %error, "could not prepare launch nonce pipe; retaining the environment handoff");
11337 None
11338 }
11339 }
11340}
11341
11342#[cfg(windows)]
11343fn start_windows_nonce_pipe(
11344 handoff: Option<NonceHandoff>,
11345 pid: u32,
11346 spec: &ModuleSpec,
11347) -> Option<subc_os::LaunchNoncePipeDelivery> {
11348 handoff.and_then(|handoff| match handoff.serve(pid, REGISTRY_RELEASE_TIMEOUT) {
11352 Ok(delivery) => Some(delivery),
11353 Err(error) => {
11354 warn!(module_id = %spec.module_id, %error, "could not serve launch nonce pipe; retaining the environment handoff");
11355 None
11356 }
11357 })
11358}
11359
11360fn observed_launch_nonce_source(snapshot: &SupervisorSnapshot) -> Option<String> {
11361 #[cfg(windows)]
11362 if snapshot.process_alive && snapshot.spawned_protocol == Some(ModuleProtocol::Subc) {
11363 return Some(
11364 if snapshot
11365 .pipe_consumed
11366 .as_ref()
11367 .is_some_and(|consumed| consumed.get().is_some())
11368 {
11369 "pipe"
11370 } else {
11371 "env"
11372 }
11373 .to_owned(),
11374 );
11375 }
11376 let _ = snapshot;
11377 None
11378}
11379
11380#[derive(Debug, Clone, Copy, PartialEq, Eq)]
11383enum SpawnRole {
11384 Plain,
11385 SwapCandidate,
11386}
11387
11388fn apply_spawn_role(command: &mut Command, role: SpawnRole) {
11391 if role == SpawnRole::SwapCandidate {
11392 command.env(SUBC_SPAWN_ROLE_ENV, SPAWN_ROLE_SWAP_CANDIDATE);
11393 }
11394}
11395
11396fn previous_instance_error(
11397 spec: &ModuleSpec,
11398 pid: u32,
11399 reason: crate::live_children::RetainedReason,
11400) -> SuperviseError {
11401 SuperviseError::Spawn {
11402 program: spec.program.clone(),
11403 source: io::Error::new(io::ErrorKind::WouldBlock, reason.waiting_message(pid)),
11404 cgroup_path: None,
11405 }
11406}
11407
11408fn carries_containment(configured: bool, has_container: bool) -> bool {
11409 configured || has_container
11410}
11411
11412fn recovery_block_error(spec: &ModuleSpec, roster: &ChildRoster) -> Option<SuperviseError> {
11413 if !carries_containment(
11414 spec.carry_containment,
11415 containment::current(&roster.containment).is_some(),
11416 ) {
11417 return None;
11418 }
11419 Some(SuperviseError::Spawn {
11420 program: spec.program.clone(),
11421 source: io::Error::new(io::ErrorKind::WouldBlock, roster.recovery_block()?),
11422 cgroup_path: None,
11423 })
11424}
11425
11426#[allow(clippy::too_many_arguments)]
11427fn spawn_child(
11428 spec: &ModuleSpec,
11429 connection_file_path: Option<&std::path::Path>,
11430 handle: Option<&SupervisorHandle>,
11431 ring: &Arc<Mutex<StderrRing>>,
11432 capture_logs_dir: Option<&std::path::Path>,
11433 roster: &ChildRoster,
11434 config_delivery: Option<config_delivery::SpawnDelivery<'_>>,
11435 #[cfg(target_os = "linux")] cgroup_placement: Option<&subc_cgroup::Placement>,
11436 register_timeout: Duration,
11437) -> Result<SupervisedChild, SuperviseError> {
11438 #[cfg(all(test, unix))]
11439 if let Some(source) = containment_tests::join_failure(spec, roster) {
11440 containment::terminate_blocking(&roster.containment);
11441 return Err(SuperviseError::Spawn {
11442 program: spec.program.clone(),
11443 source,
11444 cgroup_path: None,
11445 });
11446 }
11447 let result = spawn_child_in_slot(
11448 spec,
11449 connection_file_path,
11450 handle,
11451 ring,
11452 capture_logs_dir,
11453 roster,
11454 config_delivery,
11455 #[cfg(target_os = "linux")]
11456 cgroup_placement,
11457 SpawnRole::Plain,
11458 false,
11459 register_timeout,
11460 );
11461 if result.is_err() {
11462 if roster.admission_pending(&spec.module_id) {
11463 containment::terminate_deferred(&roster.containment);
11464 } else {
11465 containment::terminate_blocking(&roster.containment);
11466 }
11467 }
11468 result
11469}
11470
11471#[allow(clippy::too_many_arguments)]
11484fn spawn_child_in_slot(
11485 spec: &ModuleSpec,
11486 connection_file_path: Option<&std::path::Path>,
11487 handle: Option<&SupervisorHandle>,
11488 ring: &Arc<Mutex<StderrRing>>,
11489 capture_logs_dir: Option<&std::path::Path>,
11490 roster: &ChildRoster,
11491 config_delivery: Option<config_delivery::SpawnDelivery<'_>>,
11492 #[cfg(target_os = "linux")] cgroup_placement: Option<&subc_cgroup::Placement>,
11493 role: SpawnRole,
11494 alternate_slot: bool,
11495 register_timeout: Duration,
11496) -> Result<SupervisedChild, SuperviseError> {
11497 let prepared = prepare_child_in_slot(
11498 spec,
11499 connection_file_path,
11500 handle,
11501 capture_logs_dir,
11502 roster,
11503 config_delivery,
11504 #[cfg(target_os = "linux")]
11505 cgroup_placement,
11506 role,
11507 alternate_slot,
11508 false,
11509 )?;
11510 let gate = config_delivery
11511 .map(|delivery| delivery.module_gate(spec))
11512 .transpose()?;
11513 let _gate = gate
11514 .as_ref()
11515 .map(|gate| {
11516 gate.try_lock()
11517 .map_err(|_| SuperviseError::ConfigDeliveryDeferred {
11518 module_id: spec.module_id.clone(),
11519 })
11520 })
11521 .transpose()?;
11522 if let (Some(delivery), Some(admission)) = (config_delivery, &prepared.spawn_admission) {
11523 delivery.check(spec, admission)?;
11524 }
11525 let checked_base = prepared
11526 .spawn_admission
11527 .as_ref()
11528 .and_then(|admission| admission.approved_base())
11529 .cloned();
11530 let mut result = launch_child_in_slot(spec, ring, roster, prepared, register_timeout);
11531 drop(_gate);
11532 if let (Some(delivery), Err(SuperviseError::ConfigHeld { code, detail, .. })) =
11533 (config_delivery, &result)
11534 {
11535 if let Some(base) = checked_base.as_ref().filter(|_| code == "program_refused") {
11536 if !delivery.record_refusal(spec, base, detail)? {
11537 result = Err(spawn_plan_stale(spec));
11538 }
11539 }
11540 }
11541 result
11542}
11543
11544struct AsyncInitialIdentity<'a> {
11545 supervisor: &'a Supervisor,
11546 module: String,
11547 committed: bool,
11548}
11549impl Drop for AsyncInitialIdentity<'_> {
11550 fn drop(&mut self) {
11551 if !self.committed {
11552 self.supervisor.abandon_unrostered(&self.module);
11553 }
11554 }
11555}
11556
11557struct PreparedChild {
11561 script_delivery: Option<script::Delivery>,
11562 registration_spawn: Option<Arc<RegistrationSpawn>>,
11563 registration_handle: Option<SupervisorHandle>,
11564 #[cfg(any(unix, windows))]
11565 registration_channel: Option<SpawnRegistrationChannel>,
11566 config_context: Option<config_delivery::ConfigDelivery>,
11567 binding_publication: Option<ProgramBindingPublication>,
11568 launch_spec: ModuleSpec,
11569 spawn_admission: Option<crate::config_gate::runtime::SpawnAdmission>,
11570 command: Command,
11571 carrying: bool,
11572 carried: Option<Arc<containment::CarriedContainment>>,
11573 #[cfg(unix)]
11574 launch_identity: Option<containment::LaunchIdentity>,
11575 #[cfg(unix)]
11576 group_target: containment::GroupTarget,
11577 #[cfg(target_os = "linux")]
11578 cgroup_placement: Option<subc_cgroup::Placement>,
11579 #[cfg(target_os = "linux")]
11580 cgroup_name: String,
11581 #[cfg(target_os = "linux")]
11582 cgroup_cleanup: PreparedCgroup,
11583 cgroup_path: Option<PathBuf>,
11584 #[cfg(unix)]
11585 admission_gate: Option<admission::Gate>,
11586 output_sink: ChildOutputSink,
11587 #[cfg(target_os = "macos")]
11588 privacy_exec: Option<PrivacyExec>,
11589 #[cfg(windows)]
11590 image_capture: subc_os::ExecutableCapture,
11591 #[cfg(windows)]
11592 nonce_handoff: Option<NonceHandoff>,
11593 spawned_at: Instant,
11594 config_delivery: Option<crate::config_gate::runtime::DeliverySpawn>,
11595 delivery_registration: Option<config_delivery::RegistrationGuard>,
11596 program_binding: Option<SpawnProgramBinding>,
11597}
11598
11599#[cfg(target_os = "linux")]
11600struct PreparedCgroup(Option<(subc_cgroup::Placement, String)>);
11601#[cfg(target_os = "linux")]
11602impl Drop for PreparedCgroup {
11603 fn drop(&mut self) {
11604 if let Some((placement, name)) = &self.0 {
11605 remove_module_cgroup(placement, name);
11606 }
11607 }
11608}
11609
11610pub(crate) struct SpawnProgramBinding {
11611 pub(crate) path: PathBuf,
11612 _file: std::fs::File,
11615 pub(crate) identity: subc_os::RetainedFileIdentity,
11616}
11617
11618impl SpawnProgramBinding {
11619 pub(crate) fn capture(spec: &ModuleSpec) -> Result<Self, SuperviseError> {
11620 Self::capture_after_resolution(spec, |_| {})
11621 }
11622
11623 fn capture_after_resolution(
11624 spec: &ModuleSpec,
11625 before_open: impl FnOnce(&std::path::Path),
11626 ) -> Result<Self, SuperviseError> {
11627 require_program_search_path(spec)?;
11628 #[cfg(windows)]
11629 let windows_capture =
11630 windows_spawn_capture(spec).map_err(|source| SuperviseError::Spawn {
11631 program: spec.program.clone(),
11632 source,
11633 cgroup_path: None,
11634 })?;
11635 #[cfg(windows)]
11636 let path = windows_capture.path().to_path_buf();
11637 #[cfg(not(windows))]
11638 let path = if spec.program.is_absolute() || spec.program.components().count() > 1 {
11639 spec.program.clone()
11640 } else {
11641 let path = spec
11642 .env
11643 .iter()
11644 .rev()
11645 .find(|(key, _)| key == "PATH")
11646 .map(|(_, path)| std::ffi::OsString::from(path))
11647 .or_else(|| std::env::var_os("PATH"))
11648 .unwrap_or_else(|| {
11649 if cfg!(target_os = "macos") {
11650 "/usr/bin:/bin".into()
11653 } else {
11654 Default::default()
11655 }
11656 });
11657 std::env::split_paths(&path)
11658 .map(|dir| dir.join(&spec.program))
11659 .find(|path| {
11660 if !path.is_file() {
11661 return false;
11662 }
11663 #[cfg(unix)]
11664 {
11665 rustix::fs::accessat(
11666 rustix::fs::CWD,
11667 path,
11668 rustix::fs::Access::EXEC_OK,
11669 rustix::fs::AtFlags::EACCESS,
11670 )
11671 .is_ok()
11672 }
11673 #[cfg(not(unix))]
11674 {
11675 true
11676 }
11677 })
11678 .ok_or_else(|| SuperviseError::Spawn {
11679 program: spec.program.clone(),
11680 source: io::Error::new(io::ErrorKind::NotFound, "executable not found in PATH"),
11681 cgroup_path: None,
11682 })?
11683 };
11684 let resolved = std::fs::canonicalize(&path).map_err(|source| SuperviseError::Spawn {
11685 program: spec.program.clone(),
11686 source,
11687 cgroup_path: None,
11688 })?;
11689 before_open(&resolved);
11690 let file =
11693 subc_os::open_regular_nofollow(&resolved).map_err(|source| SuperviseError::Spawn {
11694 program: spec.program.clone(),
11695 source: io::Error::other(source),
11696 cgroup_path: None,
11697 })?;
11698 let identity =
11699 subc_os::retained_file_identity(&file).map_err(|source| SuperviseError::Spawn {
11700 program: spec.program.clone(),
11701 source,
11702 cgroup_path: None,
11703 })?;
11704 Ok(Self {
11705 path,
11706 _file: file,
11707 identity,
11708 })
11709 }
11710 fn current(&self) -> bool {
11711 #[cfg(unix)]
11712 {
11713 subc_os::file_identity(&self.path) == Some(self.identity)
11714 }
11715 #[cfg(windows)]
11716 {
11717 std::fs::canonicalize(&self.path)
11718 .ok()
11719 .and_then(|path| subc_os::open_regular_nofollow(&path).ok())
11720 .and_then(|file| subc_os::retained_file_identity(&file).ok())
11721 == Some(self.identity)
11722 }
11723 }
11724}
11725
11726fn spawn_plan_stale(spec: &ModuleSpec) -> SuperviseError {
11727 SuperviseError::ConfigDeliveryDeferred {
11728 module_id: spec.module_id.clone(),
11729 }
11730}
11731
11732fn require_program_search_path(spec: &ModuleSpec) -> Result<(), SuperviseError> {
11733 if !cfg!(target_os = "macos")
11734 && !spec.program.is_absolute()
11735 && spec.program.components().count() == 1
11736 && !spec.env.iter().any(|(key, _)| {
11737 if cfg!(windows) {
11738 key.eq_ignore_ascii_case("PATH")
11739 } else {
11740 key == "PATH"
11741 }
11742 })
11743 && std::env::var_os("PATH").is_none()
11744 {
11745 return Err(SuperviseError::Spawn {
11748 program: spec.program.clone(),
11749 source: io::Error::new(io::ErrorKind::InvalidInput, "bare program requires PATH"),
11750 cgroup_path: None,
11751 });
11752 }
11753 Ok(())
11754}
11755
11756async fn spawn_child_async(
11757 spec: &ModuleSpec,
11758 runtime: &SupervisorRuntimeConfig,
11759) -> Result<SupervisedChild, SuperviseError> {
11760 #[cfg(all(test, unix))]
11761 if let Some(source) = containment_tests::join_failure(spec, &runtime.child_roster) {
11762 containment::terminate_blocking(&runtime.child_roster.containment);
11763 return Err(SuperviseError::Spawn {
11764 program: spec.program.clone(),
11765 source,
11766 cgroup_path: None,
11767 });
11768 }
11769 let result = spawn_child_in_slot_async(spec, runtime, SpawnRole::Plain, false).await;
11770 if result.is_err() {
11771 if runtime.child_roster.admission_pending(&spec.module_id) {
11772 containment::terminate_deferred(&runtime.child_roster.containment);
11773 } else {
11774 containment::terminate_blocking(&runtime.child_roster.containment);
11775 }
11776 }
11777 result
11778}
11779
11780async fn spawn_child_in_slot_async(
11781 spec: &ModuleSpec,
11782 runtime: &SupervisorRuntimeConfig,
11783 role: SpawnRole,
11784 alternate_slot: bool,
11785) -> Result<SupervisedChild, SuperviseError> {
11786 let gate = runtime
11787 .config_delivery
11788 .as_ref()
11789 .map(|delivery| delivery.0.module_gate(&spec.module_id))
11790 .transpose()
11791 .map_err(|error| SuperviseError::InvalidSpec {
11792 reason: error.to_string(),
11793 })?;
11794 if gate.as_ref().is_some_and(|gate| gate.try_lock().is_err()) {
11795 return Err(SuperviseError::ConfigDeliveryDeferred {
11796 module_id: spec.module_id.clone(),
11797 });
11798 }
11799 let prepared_spec = spec.clone();
11800 let prepared_runtime = runtime.clone();
11801 let prepared = tokio::task::spawn_blocking(move || {
11802 let prepared = prepare_child_in_slot(
11803 &prepared_spec,
11804 prepared_runtime.connection_file_path.as_deref(),
11805 prepared_runtime.supervisor_handle.as_ref(),
11806 prepared_runtime.capture_logs_dir.as_deref(),
11807 &prepared_runtime.child_roster,
11808 prepared_runtime.spawn_delivery(),
11809 #[cfg(target_os = "linux")]
11810 prepared_runtime.cgroup_placement.as_ref(),
11811 role,
11812 alternate_slot,
11813 true,
11814 );
11815 prepared
11816 })
11817 .await
11818 .map_err(|_| SuperviseError::InvalidSpec {
11819 reason: "configuration launch preparation stopped".into(),
11820 })??;
11821 if let Some(registration) = &prepared.delivery_registration {
11822 if !registration.launch_current()? {
11823 return Err(spawn_plan_stale(spec));
11824 }
11825 }
11826 let _gate = match gate {
11827 Some(gate) => {
11828 Some(
11829 gate.try_lock_owned()
11830 .map_err(|_| SuperviseError::ConfigDeliveryDeferred {
11831 module_id: spec.module_id.clone(),
11832 })?,
11833 )
11834 }
11835 None => None,
11836 };
11837 if let (Some(delivery), Some(admission)) = (runtime.spawn_delivery(), &prepared.spawn_admission)
11838 {
11839 delivery.check(spec, admission)?;
11840 }
11841 if prepared
11842 .spawn_admission
11843 .as_ref()
11844 .is_none_or(|admission| admission.launch.is_none())
11845 && prepared
11846 .program_binding
11847 .as_ref()
11848 .is_some_and(|binding| !binding.current())
11849 {
11850 return Err(spawn_plan_stale(spec));
11851 }
11852 let checked_base = prepared
11854 .spawn_admission
11855 .as_ref()
11856 .and_then(|admission| admission.approved_base())
11857 .cloned();
11858 let mut result = launch_child_in_slot(
11859 spec,
11860 &runtime.stderr_ring,
11861 &runtime.child_roster,
11862 prepared,
11863 runtime.register_timeout,
11864 );
11865 drop(_gate);
11866 if let (Some(delivery), Err(SuperviseError::ConfigHeld { code, detail, .. })) =
11867 (runtime.config_delivery.as_ref(), &result)
11868 {
11869 if let Some(base) = checked_base.filter(|_| code == "program_refused") {
11870 let module = spec.module_id.clone();
11871 let identity = subc_config::raw_digest(detail.as_bytes());
11872 let recorded = delivery
11873 .0
11874 .on_blocking(move |owner| owner.record_program_refusal(&module, &base, identity))
11875 .await;
11876 match recorded {
11877 Ok(true) => {}
11878 Ok(false) => result = Err(spawn_plan_stale(spec)),
11879 Err(error) => {
11880 result = Err(config_delivery::gate_hold_error(
11881 spec,
11882 error,
11883 Some(delivery),
11884 ))
11885 }
11886 }
11887 }
11888 }
11889 result
11890}
11891
11892fn validate_spawn_program(
11893 spec: &ModuleSpec,
11894 admission: Option<&crate::config_gate::runtime::SpawnAdmission>,
11895 delivery: Option<config_delivery::SpawnDelivery<'_>>,
11896) -> Result<(), SuperviseError> {
11897 if let Some(admission) = admission {
11898 if let Some(program) = &admission.launch {
11899 if let Err(error) = program.validate_for_spawn() {
11900 if let (Some(context), Some(base)) = (delivery, admission.approved_base()) {
11904 if error.code == "program_refused"
11905 && !context.record_refusal(spec, base, &error.detail)?
11906 {
11907 return Err(spawn_plan_stale(spec));
11908 }
11909 }
11910 return Err(config_delivery::gate_hold_error(
11911 spec,
11912 error,
11913 delivery.map(|context| context.runtime()).as_ref(),
11914 ));
11915 }
11916 }
11917 }
11918 Ok(())
11919}
11920
11921#[allow(clippy::too_many_arguments)]
11922fn prepare_child_in_slot(
11923 spec: &ModuleSpec,
11924 connection_file_path: Option<&std::path::Path>,
11925 handle: Option<&SupervisorHandle>,
11926 capture_logs_dir: Option<&std::path::Path>,
11927 roster: &ChildRoster,
11928 config_delivery: Option<config_delivery::SpawnDelivery<'_>>,
11929 #[cfg(target_os = "linux")] cgroup_placement: Option<&subc_cgroup::Placement>,
11930 role: SpawnRole,
11931 alternate_slot: bool,
11932 asynchronous: bool,
11933) -> Result<PreparedChild, SuperviseError> {
11934 require_program_search_path(spec)?;
11935 let spawn_admission = config_delivery
11936 .map(|delivery| delivery.prepare(spec))
11937 .transpose()?;
11938 #[cfg(target_os = "macos")]
11939 let gated = spawn_admission
11940 .as_ref()
11941 .is_some_and(|admission| admission.approved_base().is_some());
11942 let mut launch_spec = spawn_admission
11943 .as_ref()
11944 .and_then(|admission| admission.launch.as_ref())
11945 .map(|launch| launch.spec.clone())
11946 .unwrap_or_else(|| spec.clone());
11947 if launch_spec.script.is_some() {
11948 launch_spec.program = script::runner_path().map_err(|source| SuperviseError::Spawn {
11949 program: launch_spec.program.clone(),
11950 source,
11951 cgroup_path: None,
11952 })?;
11953 }
11954 let script_content = if let Some(path) = &launch_spec.script {
11955 Some(
11956 spawn_admission
11957 .as_ref()
11958 .and_then(|admission| admission.launch.as_ref())
11959 .and_then(|launch| launch.script.clone())
11960 .map_or_else(|| script::ScriptBytes::read(path), Ok)
11961 .map_err(|error| config_delivery::gate_hold_error(&launch_spec, error, None))?,
11962 )
11963 } else {
11964 None
11965 };
11966 let spec = &launch_spec;
11967 if let Some(error) = recovery_block_error(spec, roster) {
11968 return Err(error);
11969 }
11970 if let Some((pid, reason)) = roster.previous_instance(&spec.module_id) {
11971 return Err(previous_instance_error(spec, pid, reason));
11972 }
11973 if roster.is_closed() {
11974 return Err(SuperviseError::Spawn {
11975 program: spec.program.clone(),
11976 source: io::Error::other("the daemon is shutting down; not starting a new process"),
11977 cgroup_path: None,
11978 });
11979 }
11980 let carried = containment::current(&roster.containment);
11981 let carrying = carries_containment(spec.carry_containment, carried.is_some());
11982 if role != SpawnRole::Plain && carrying {
11983 return Err(SuperviseError::InvalidSpec {
11984 reason: "a carried container cannot overlap a swap candidate".to_owned(),
11985 });
11986 }
11987 #[cfg(unix)]
11988 let launch_identity = if carrying && carried.is_none() {
11989 Some(
11990 containment::LaunchIdentity::capture().map_err(|source| SuperviseError::Spawn {
11991 program: spec.program.clone(),
11992 source,
11993 cgroup_path: None,
11994 })?,
11995 )
11996 } else {
11997 None
11998 };
11999 #[cfg(target_os = "linux")]
12000 let cgroup_placement = match &carried {
12001 Some(container) => container.cgroup().map(|(placement, _)| placement),
12002 None => cgroup_placement,
12003 };
12004 #[cfg(unix)]
12005 let group_target = if let Some(container) = carried
12006 .as_ref()
12007 .filter(|container| container.group().is_some())
12008 {
12009 containment::GroupTarget::Pinned(Arc::clone(container))
12010 } else {
12011 #[cfg(target_os = "linux")]
12012 let cgroup_container = carrying && cgroup_placement.is_some();
12013 #[cfg(not(target_os = "linux"))]
12014 let cgroup_container = false;
12015 if cgroup_container {
12016 containment::GroupTarget::NoGroup
12017 } else {
12018 containment::GroupTarget::OwnLeader
12019 }
12020 };
12021 #[cfg(target_os = "linux")]
12022 let cgroup_name = {
12023 if let Some((_, name)) = carried.as_ref().and_then(|container| container.cgroup()) {
12028 name.to_owned()
12029 } else if cgroup_placement.is_none() {
12030 swap::cgroup_name(&spec.module_id, alternate_slot)
12031 } else {
12032 let nonce = generate_launch_nonce()?;
12033 let suffix = u128::from_str_radix(&nonce[..32], 16).expect("hex launch nonce");
12034 let mut end = spec.module_id.len().min(64);
12037 while !spec.module_id.is_char_boundary(end) {
12038 end -= 1;
12039 }
12040 format!(
12041 "{}_{suffix}",
12042 swap::cgroup_name(&spec.module_id[..end], alternate_slot)
12043 )
12044 }
12045 };
12046 #[cfg(not(target_os = "linux"))]
12047 let _ = alternate_slot;
12048 #[cfg(windows)]
12049 let image_capture = windows_spawn_capture(spec).map_err(|source| SuperviseError::Spawn {
12050 program: spec.program.clone(),
12051 source,
12052 cgroup_path: None,
12053 })?;
12054 #[cfg(windows)]
12055 let program_binding = if asynchronous {
12056 let mut resolved = spec.clone();
12057 resolved.program = image_capture.path().to_path_buf();
12058 Some(SpawnProgramBinding::capture(&resolved)?)
12059 } else {
12060 None
12061 };
12062 #[cfg(not(windows))]
12063 let program_binding = (asynchronous || cfg!(target_os = "macos"))
12064 .then(|| SpawnProgramBinding::capture(spec))
12065 .transpose()?;
12066 #[cfg(target_os = "macos")]
12067 validate_spawn_program(spec, spawn_admission.as_ref(), config_delivery)?;
12068 #[cfg(target_os = "macos")]
12069 let (mut command, privacy_exec, exec_ack) = privacy_command(
12070 spec,
12071 program_binding
12072 .as_ref()
12073 .expect("macOS captures every launch"),
12074 roster,
12075 gated,
12079 )?;
12080 #[cfg(windows)]
12081 let mut command = Command::new(image_capture.path());
12082 #[cfg(not(any(target_os = "macos", windows)))]
12083 let mut command = Command::new(
12084 program_binding
12085 .as_ref()
12086 .map_or(spec.program.as_path(), |binding| binding.path.as_path()),
12087 );
12088 command.args(&spec.args);
12089 apply_child_env(&mut command, spec);
12119 if spec.script.is_some() {
12120 script::remove_runtime_env(&mut command, spec);
12121 }
12122 let origin_witness = crate::program_origin::LaunchWitness::capture_command(spec, &command)
12123 .map_err(|source| SuperviseError::Spawn {
12124 program: spec.program.clone(),
12125 source,
12126 cgroup_path: None,
12127 })?;
12128 command.current_dir(&origin_witness.working_directory);
12129 apply_spawn_role(&mut command, role);
12130 let nonce_handoff = apply_wire_spawn_args_for_admission(
12131 &mut command,
12132 spec,
12133 connection_file_path,
12134 handle,
12135 role,
12136 spawn_admission.as_ref(),
12137 )?;
12138 #[cfg(unix)]
12139 let mut slots = subc_os::registration_channel::ChildDescriptorSlots::default();
12140 command.env_remove(subc_os::script_pipe::SCRIPT_FD_ENV);
12141 let script_delivery = script_content
12142 .map(|content| {
12143 script::Delivery::prepare(
12144 content,
12145 &mut command,
12146 #[cfg(unix)]
12147 &mut slots,
12148 )
12149 })
12150 .transpose()
12151 .map_err(|source| SuperviseError::Spawn {
12152 program: spec.program.clone(),
12153 source,
12154 cgroup_path: None,
12155 })?;
12156 #[cfg(unix)]
12157 let registration_channel = if spec.protocol == ModuleProtocol::Subc {
12158 let (parent, child, metadata) = subc_os::registration_channel::registration_socketpair()
12159 .map_err(|source| SuperviseError::Spawn {
12160 program: spec.program.clone(),
12161 source,
12162 cgroup_path: None,
12163 })?;
12164 slots
12165 .channel(child)
12166 .map_err(|source| SuperviseError::Spawn {
12167 program: spec.program.clone(),
12168 source,
12169 cgroup_path: None,
12170 })?;
12171 command
12172 .env(subc_os::registration_channel::REGISTRATION_FD_ENV, metadata)
12173 .env(
12174 subc_os::registration_channel::DAEMON_PID_ENV,
12175 std::process::id().to_string(),
12176 );
12177 Some(parent)
12178 } else {
12179 None
12180 };
12181 #[cfg(windows)]
12182 let registration_channel = prepare_windows_registration_pipe(&mut command, spec)?;
12183 let binding_publication = if spawn_admission
12184 .as_ref()
12185 .is_some_and(|admission| admission.launch.is_some())
12186 {
12187 handle.and_then(|handle| handle.begin_program_binding(&spec.module_id, role))
12188 } else {
12189 None
12190 };
12191 let registration_spawn = (spec.protocol == ModuleProtocol::Subc)
12192 .then(|| {
12193 handle.map(|handle| {
12194 let spawn = RegistrationSpawn::new_with_binding_requirement(
12195 handle,
12196 &spec.module_id,
12197 role,
12198 binding_publication.clone(),
12199 spawn_admission
12200 .as_ref()
12201 .is_some_and(|admission| admission.launch.is_some()),
12202 );
12203 if let Some(admission) = &spawn_admission {
12204 admission.initialize_origin(&spawn.authority, &origin_witness);
12205 }
12206 handle.record_nonce_launch(&spawn);
12207 spawn
12208 })
12209 })
12210 .flatten();
12211
12212 #[cfg(target_os = "linux")]
12213 let cgroup_path = cgroup_placement
12214 .map(|placement| {
12215 if let Some(container) = &carried {
12216 let path = container
12217 .cgroup_path()
12218 .ok_or_else(|| io::Error::other("carried container has no cgroup path"))?;
12219 placement.existing_module_path(path)?.ok_or_else(|| {
12220 io::Error::new(
12221 io::ErrorKind::NotFound,
12222 "carried cgroup disappeared before replacement",
12223 )
12224 })
12225 } else {
12226 if carrying {
12227 containment::prepare_cgroup_path(placement, &cgroup_name)
12228 } else {
12229 placement.module_path(&cgroup_name)
12230 }
12231 }
12232 })
12233 .transpose()
12234 .map_err(|source| SuperviseError::Cgroup {
12235 module_id: spec.module_id.clone(),
12236 source,
12237 })?;
12238 #[cfg(not(target_os = "linux"))]
12239 let cgroup_path: Option<PathBuf> = None;
12240 #[cfg(target_os = "linux")]
12243 let cgroup_cleanup = PreparedCgroup(
12244 cgroup_placement
12245 .filter(|_| carried.is_none())
12246 .map(|placement| (placement.clone(), cgroup_name.clone())),
12247 );
12248 #[cfg(unix)]
12249 let mut admission_gate = if carrying {
12250 Some(
12251 admission::Gate::install(&mut command, spec, roster).map_err(|source| {
12252 SuperviseError::Spawn {
12253 program: spec.program.clone(),
12254 source,
12255 cgroup_path: cgroup_path.clone(),
12256 }
12257 })?,
12258 )
12259 } else {
12260 None
12261 };
12262 #[cfg(target_os = "linux")]
12263 if let Some(path) = &cgroup_path {
12264 apply_cgroup_placement(&mut command, spec, path)?;
12265 }
12266
12267 let output_sink = if let Some(logs_dir) = capture_logs_dir {
12268 let path = logs_dir.join(format!("{}.stderr.log", spec.module_id));
12269 match ChildOutputSink::open(&path, capture_retention(spec)) {
12270 Ok(sink) => sink,
12271 Err(error) => {
12272 warn!(
12273 module_id = %spec.module_id,
12274 path = %path.display(),
12275 error = %error,
12276 "could not open child output capture file; forwarding to stderr"
12277 );
12278 ChildOutputSink::Stderr
12279 }
12280 }
12281 } else {
12282 ChildOutputSink::Stderr
12283 };
12284
12285 command.stdout(Stdio::piped());
12286 command.stderr(Stdio::piped());
12287 command.kill_on_drop(!carrying);
12288 #[cfg(unix)]
12305 command.process_group(
12306 carried
12307 .as_ref()
12308 .map(|container| container.verified_join_group())
12309 .transpose()
12310 .map_err(|source| SuperviseError::Spawn {
12311 program: spec.program.clone(),
12312 source,
12313 cgroup_path: cgroup_path.clone(),
12314 })?
12315 .flatten()
12316 .unwrap_or(0) as i32,
12317 );
12318 command.stdin(Stdio::null());
12319 #[cfg(unix)]
12320 if let Some(gate) = &mut admission_gate {
12321 gate.install_placement(&mut command);
12322 }
12323 #[cfg(unix)]
12326 {
12327 if let Some(handoff) = nonce_handoff {
12328 slots.nonce(handoff);
12329 }
12330 #[cfg(target_os = "macos")]
12331 slots
12332 .acknowledgement(exec_ack.into_descriptor())
12333 .map_err(|source| SuperviseError::Spawn {
12334 program: spec.program.clone(),
12335 source,
12336 cgroup_path: cgroup_path.clone(),
12337 })?;
12338 slots.install(command.as_std_mut());
12339 }
12340 #[cfg(not(any(unix, windows)))]
12341 let _ = nonce_handoff;
12342
12343 #[cfg(windows)]
12348 {
12349 subc_jobobject::ensure_root_job().map_err(|source| SuperviseError::Spawn {
12350 program: spec.program.clone(),
12351 source,
12352 cgroup_path: None,
12353 })?;
12354 subc_jobobject::suspend_on_create_async(&mut command);
12355 }
12356 let spawned_at = Instant::now();
12360 let delivery_context = config_delivery;
12361 let config_delivery = config_delivery
12362 .map(|delivery| {
12363 delivery.spawned(
12364 spec,
12365 spawn_admission
12366 .as_ref()
12367 .expect("delivery prepares spawn admission"),
12368 spawned_at,
12369 role,
12370 registration_spawn
12371 .as_ref()
12372 .map(|spawn| spawn.authority.clone()),
12373 )
12374 })
12375 .transpose()?
12376 .flatten();
12377 let mut delivery_registration =
12378 delivery_context.map(|context| context.registration_guard(config_delivery.as_ref(), role));
12379 if let Some(registration) = &mut delivery_registration {
12380 registration.defer_cleanup();
12381 }
12382 if asynchronous {
12383 #[cfg(test)]
12384 if let Some(context) = delivery_context {
12385 context.pause_preflight_for_test();
12386 }
12387 }
12388 validate_spawn_program(spec, spawn_admission.as_ref(), delivery_context)?;
12389 Ok(PreparedChild {
12390 script_delivery,
12391 registration_spawn,
12392 registration_handle: handle.cloned(),
12393 #[cfg(any(unix, windows))]
12394 registration_channel,
12395 config_context: delivery_context.map(|delivery| delivery.runtime()),
12396 binding_publication,
12397 launch_spec,
12398 spawn_admission,
12399 command,
12400 carrying,
12401 carried: carried.clone(),
12402 #[cfg(unix)]
12403 launch_identity,
12404 #[cfg(unix)]
12405 group_target,
12406 #[cfg(target_os = "linux")]
12407 cgroup_placement: cgroup_placement.cloned(),
12408 #[cfg(target_os = "linux")]
12409 cgroup_cleanup,
12410 #[cfg(target_os = "linux")]
12411 cgroup_name,
12412 cgroup_path,
12413 #[cfg(unix)]
12414 admission_gate,
12415 output_sink,
12416 #[cfg(target_os = "macos")]
12417 privacy_exec,
12418 #[cfg(windows)]
12419 image_capture,
12420 #[cfg(windows)]
12421 nonce_handoff,
12422 spawned_at,
12423 config_delivery,
12424 delivery_registration,
12425 program_binding,
12426 })
12427}
12428
12429fn launch_child_in_slot(
12430 _spec: &ModuleSpec,
12431 ring: &Arc<Mutex<StderrRing>>,
12432 roster: &ChildRoster,
12433 prepared: PreparedChild,
12434 register_timeout: Duration,
12435) -> Result<SupervisedChild, SuperviseError> {
12436 let launch_spec = prepared.launch_spec.clone();
12437 let registration_declaration = prepared
12438 .spawn_admission
12439 .as_ref()
12440 .and_then(|admission| admission.registration_declaration(&launch_spec.module_id));
12441 let admission_base = prepared
12442 .spawn_admission
12443 .as_ref()
12444 .and_then(|admission| admission.approved_base())
12445 .cloned();
12446 let spec = &launch_spec;
12447 let expected_image = prepared
12448 .spawn_admission
12449 .as_ref()
12450 .and_then(|admission| admission.launch.as_ref())
12451 .map(|launch| launch.image);
12452 let program_launch = prepared
12453 .spawn_admission
12454 .and_then(|admission| admission.launch);
12455 let config_owner = prepared
12456 .config_context
12457 .as_ref()
12458 .map(|context| context.0.clone());
12459 let binding_publication = prepared.binding_publication;
12460 #[cfg(test)]
12461 if let Some(context) = &prepared.config_context {
12462 context.0.before_spawn_exec_for_test();
12463 }
12464 #[cfg(feature = "test-support")]
12465 if let Some(context) = &prepared.config_context {
12466 context
12467 .0
12468 .pause_spawn_before_exec()
12469 .map_err(|error| config_delivery::gate_hold_error(spec, error, Some(context)))?;
12470 }
12471 let mut command = prepared.command;
12472 let carrying = prepared.carrying;
12473 let carried = prepared.carried;
12474 #[cfg(unix)]
12475 let launch_identity = prepared.launch_identity;
12476 #[cfg(unix)]
12477 let group_target = prepared.group_target;
12478 #[cfg(target_os = "linux")]
12479 let cgroup_placement = prepared.cgroup_placement.as_ref();
12480 #[cfg(target_os = "linux")]
12481 let cgroup_name = prepared.cgroup_name;
12482 #[cfg(target_os = "linux")]
12483 let mut cgroup_cleanup = prepared.cgroup_cleanup;
12484 #[cfg(target_os = "linux")]
12485 let _ = cgroup_cleanup.0.take();
12486 let cgroup_path = prepared.cgroup_path;
12487 #[cfg(unix)]
12488 let admission_gate = prepared.admission_gate;
12489 let output_sink = prepared.output_sink;
12490 #[cfg(target_os = "macos")]
12491 let mut privacy_exec = prepared.privacy_exec;
12492 #[cfg(target_os = "macos")]
12493 if let Some(pending) = &mut privacy_exec {
12494 pending.admission_binding = binding_publication.clone();
12495 if expected_image.is_some() {
12496 pending.expected = expected_image;
12497 }
12498 }
12499 #[cfg(any(unix, windows))]
12500 let registration_channel = prepared.registration_channel;
12501 let registration_spawn = prepared.registration_spawn;
12502 let registration_handle = prepared.registration_handle;
12503 #[cfg(windows)]
12504 let image_capture = prepared.image_capture;
12505 #[cfg(windows)]
12506 let nonce_handoff = prepared.nonce_handoff;
12507 let spawned_at = prepared.spawned_at;
12508 let config_delivery = prepared.config_delivery;
12509 let delivery_registration = prepared.delivery_registration;
12510 let _program_binding = prepared.program_binding;
12511 #[cfg(all(test, unix))]
12512 containment_tests::record_spawn_attempt(spec, &mut command);
12513 #[cfg(unix)]
12514 let spawned = match admission_gate {
12515 Some(gate) => gate
12516 .spawn(
12517 command,
12518 spec,
12519 roster,
12520 launch_identity,
12521 #[cfg(target_os = "linux")]
12522 cgroup_placement
12523 .zip(cgroup_path.as_ref())
12524 .map(|(placement, path)| {
12525 (placement.clone(), cgroup_name.clone(), path.clone())
12526 }),
12527 )
12528 .map(|(child, admitted)| (child, Some(admitted))),
12529 None => {
12530 let spawned =
12531 subc_os::child_spawn::tokio_spawn(&mut command).map(|child| (child, None));
12532 drop(command);
12535 spawned
12536 }
12537 };
12538 #[cfg(not(unix))]
12539 let spawned = subc_os::child_spawn::tokio_spawn(&mut command);
12540 let spawned = match spawned {
12541 Ok(child) => child,
12542 Err(source) => {
12543 if carried
12544 .as_ref()
12545 .and_then(|container| container.group())
12546 .is_some()
12547 {
12548 containment::terminate_blocking(&roster.containment);
12549 }
12550 #[cfg(target_os = "linux")]
12551 if let Some(placement) = cgroup_placement.filter(|_| carried.is_none()) {
12552 remove_module_cgroup(placement, &cgroup_name);
12553 }
12554 return Err(SuperviseError::Spawn {
12555 program: spec.program.clone(),
12556 source,
12557 cgroup_path,
12558 });
12559 }
12560 };
12561 #[cfg(unix)]
12562 let (mut child, mut admitted) = spawned;
12563 #[cfg(unix)]
12564 let group_target = admitted
12565 .as_ref()
12566 .map(|attempt| attempt.group_target())
12567 .unwrap_or(group_target);
12568 #[cfg(not(unix))]
12569 let mut child = spawned;
12570 if let Some(delivery) = prepared.script_delivery {
12571 delivery.start(child.id().expect("spawned child has a pid"));
12572 }
12573 #[cfg(test)]
12574 spawn_record_tests::after_create(&child, spec);
12575
12576 #[cfg(windows)]
12581 let windows_image = match image_capture.bind_tokio(&child) {
12582 Ok(image) => Arc::new(image),
12583 Err(source) => {
12584 kill_and_reap_rejected_child(&mut child);
12585 return Err(SuperviseError::Spawn {
12586 program: spec.program.clone(),
12587 source,
12588 cgroup_path: None,
12589 });
12590 }
12591 };
12592 #[cfg(windows)]
12595 let job = contain_spawned_child(&mut child, spec, carried.as_ref())?;
12596 let spawned_at_ms = unix_ms_now();
12597 #[cfg(not(windows))]
12598 let spawned_from = spec.program.clone();
12599 #[cfg(windows)]
12600 let spawned_from = windows_image.path().to_path_buf();
12601 #[cfg(not(windows))]
12602 let spawned_file_identity = spawned_file_identity(&spawned_from);
12603 #[cfg(windows)]
12604 let spawned_file_identity = Some(SpawnedFileIdentity {
12605 device: 0,
12606 inode: 0,
12607 windows_image: Some(Arc::clone(&windows_image)),
12608 });
12609 let pid = child.id().ok_or_else(|| SuperviseError::Spawn {
12610 program: spec.program.clone(),
12611 source: io::Error::other("spawned child exposed no live pid"),
12612 cgroup_path: cgroup_path.clone(),
12613 })?;
12614 #[cfg(windows)]
12615 let nonce_delivery = start_windows_nonce_pipe(nonce_handoff, pid, spec);
12616 #[cfg(not(windows))]
12617 let process_start_time = admitted
12618 .as_ref()
12619 .map(|admitted| admitted.birth)
12620 .or_else(|| crate::provenance::process_start_time(pid));
12621 #[cfg(windows)]
12622 let process_start_time = Some(windows_image.start_time());
12623 let process_identity = process_start_time.map(|start_time| ProcessIdentity { pid, start_time });
12624 #[cfg(all(test, target_os = "macos"))]
12625 privacy_exec_boundary_tests::before_image_sample(spec, pid);
12626 #[cfg(all(test, target_os = "linux"))]
12627 assert!(roster.admission_event_for_test(&spec.module_id, "before_image_binding", Some(pid)));
12628 #[cfg(not(windows))]
12633 let recorded_image = observe_spawned_image(pid);
12634 #[cfg(all(test, target_os = "linux"))]
12635 let recorded_image = if roster.admission_event_for_test(
12636 &spec.module_id,
12637 "after_spawn_image_sample",
12638 Some(pid),
12639 ) {
12640 recorded_image
12641 } else {
12642 None
12643 };
12644 #[cfg(target_os = "linux")]
12645 let exited_before_binding = expected_image.is_some()
12648 && recorded_image.and_then(|image| image.executable) != expected_image
12649 && child.try_wait().ok().flatten().is_some();
12650 #[cfg(target_os = "linux")]
12651 let unavailable_initial_image = expected_image.is_some()
12652 && recorded_image.and_then(|image| image.executable).is_none()
12653 && !exited_before_binding;
12654 #[cfg(not(target_os = "linux"))]
12655 let unavailable_initial_image = false;
12656 #[cfg(target_os = "linux")]
12657 if expected_image
12658 .is_some_and(|expected| recorded_image.and_then(|image| image.executable) != Some(expected))
12659 && !exited_before_binding
12660 && !unavailable_initial_image
12661 {
12662 kill_module_cgroup(cgroup_placement, &cgroup_name);
12663 kill_and_reap_rejected_child(&mut child, group_target.clone());
12664 return Err(SuperviseError::ConfigHeld {
12665 module_id: spec.module_id.clone(),
12666 code: "program_refused".into(),
12667 detail: format!(
12668 "spawned executable {:?} does not match hashed program {:?}",
12669 recorded_image.and_then(|image| image.executable),
12670 expected_image
12671 ),
12672 });
12673 }
12674 #[cfg(not(target_os = "linux"))]
12675 let _ = expected_image;
12676 #[cfg(target_os = "macos")]
12677 let binding_ready = privacy_exec.is_none();
12678 #[cfg(target_os = "linux")]
12679 let binding_ready = !exited_before_binding;
12680 #[cfg(not(any(target_os = "macos", target_os = "linux")))]
12681 let binding_ready = cfg!(windows);
12682 #[cfg(target_os = "macos")]
12686 let recorded_image = if privacy_exec.is_some() {
12687 None
12688 } else {
12689 recorded_image
12690 };
12691 #[cfg(target_os = "linux")]
12692 let recorded_cgroup_name = cgroup_path.as_ref().map(|_| cgroup_name.clone());
12693 #[cfg(not(target_os = "linux"))]
12694 let recorded_cgroup_name = None;
12695 #[cfg(unix)]
12696 let preadmitted = admitted
12697 .as_mut()
12698 .and_then(|admitted| admitted.leader.take());
12699 #[cfg(not(unix))]
12700 let preadmitted: Option<crate::child_roster::RosterGuard> = None;
12701 let roster_guard = if let Some(guard) = preadmitted {
12702 Ok(guard)
12703 } else {
12704 roster.admit(
12705 spec.module_id.clone(),
12706 pid,
12707 spec.protocol,
12708 process_start_time,
12709 crate::child_roster::RecordedIdentity {
12710 container: None,
12711 #[cfg(unix)]
12712 carried: None,
12713 #[cfg(not(windows))]
12714 start_time: recorded_image.map(|image| image.start_time),
12715 #[cfg(windows)]
12716 start_time: process_start_time,
12717 #[cfg(not(windows))]
12718 executable: recorded_image
12719 .and_then(|image| image.executable)
12720 .map(crate::live_children::ExecutableIdentity::from),
12721 #[cfg(windows)]
12722 executable: Some(crate::live_children::ExecutableIdentity::from(
12723 windows_image.identity(),
12724 )),
12725 cgroup_name: recorded_cgroup_name,
12726 #[cfg(target_os = "linux")]
12727 cgroup_placement: cgroup_placement.cloned(),
12728 },
12729 )
12730 };
12731 let roster_guard = match roster_guard {
12732 Ok(guard) => guard,
12733 Err(failure) => {
12734 return failure.with_cleanup(|source| {
12735 #[cfg(windows)]
12736 if let Some(job) = &job {
12737 let _ = job.lock().unwrap_or_else(|p| p.into_inner()).terminate();
12738 }
12739 #[cfg(target_os = "linux")]
12740 kill_module_cgroup(cgroup_placement, &cgroup_name);
12741 #[cfg(unix)]
12742 let rejected_group = group_target.clone();
12743 kill_and_reap_rejected_child(
12744 &mut child,
12745 #[cfg(unix)]
12746 rejected_group,
12747 );
12748 #[cfg(target_os = "linux")]
12749 if let Some(placement) = cgroup_placement.filter(|_| carried.is_none()) {
12750 remove_module_cgroup(placement, &cgroup_name);
12751 }
12752 Err(SuperviseError::Spawn {
12753 program: spec.program.clone(),
12754 source: io::Error::other(format!(
12755 "could not durably record module child: {source}"
12756 )),
12757 cgroup_path,
12758 })
12759 });
12760 }
12761 };
12762 #[cfg(unix)]
12763 if admitted.is_some() {
12764 if let Some(image) = recorded_image {
12765 if let Err(source) = roster_guard.confirm_executable(image) {
12766 kill_and_reap_rejected_child(&mut child, group_target.clone());
12767 return Err(SuperviseError::Spawn {
12768 program: spec.program.clone(),
12769 source,
12770 cgroup_path,
12771 });
12772 }
12773 }
12774 }
12775 if roster.is_closed() {
12780 if carrying {
12781 #[cfg(unix)]
12782 let rejected_group = admitted
12783 .as_ref()
12784 .map(|admitted| admitted.rejection_target())
12785 .unwrap_or_else(|| group_target.clone());
12786 #[cfg(windows)]
12787 if let Some(job) = &job {
12788 let _ = job.lock().unwrap_or_else(|p| p.into_inner()).terminate();
12789 }
12790 kill_and_reap_rejected_child(
12791 &mut child,
12792 #[cfg(unix)]
12793 rejected_group,
12794 );
12795 #[cfg(all(test, unix))]
12796 let _ =
12797 roster.admission_event_for_test(&spec.module_id, "after_rejected_child", Some(pid));
12798 drop(roster_guard);
12799 return Err(SuperviseError::Spawn {
12800 program: spec.program.clone(),
12801 source: io::Error::other(
12802 "the daemon began shutting down during carried admission; ended the container",
12803 ),
12804 cgroup_path,
12805 });
12806 }
12807 #[cfg(target_os = "linux")]
12809 kill_module_cgroup(cgroup_placement, &cgroup_name);
12810 if let Err(error) = child.start_kill() {
12811 debug!(module_id = %spec.module_id, pid, %error, "kill of a process spawned during daemon shutdown failed; it may already have exited");
12812 }
12813 #[cfg(target_os = "linux")]
12814 if let Some(placement) = cgroup_placement.filter(|_| carried.is_none()) {
12815 while matches!(child.try_wait(), Ok(None)) {
12819 std::thread::yield_now();
12820 }
12821 if matches!(
12822 subc_cgroup::kill_module(Some(placement), &cgroup_name),
12823 subc_cgroup::KillOutcome::Killed
12824 ) {
12825 if let Ok(path) = placement.module_path(&cgroup_name) {
12826 while std::fs::read_to_string(path.join("cgroup.events"))
12827 .ok()
12828 .is_some_and(|events| events.lines().any(|line| line == "populated 1"))
12829 {
12830 std::thread::yield_now();
12831 }
12832 }
12833 }
12834 remove_module_cgroup(placement, &cgroup_name);
12835 }
12836 drop(roster_guard);
12837 return Err(SuperviseError::Spawn {
12838 program: spec.program.clone(),
12839 source: io::Error::other(
12840 "the daemon began shutting down while this process was starting; ended it",
12841 ),
12842 cgroup_path,
12843 });
12844 }
12845
12846 let stdout_pump = match child.stdout.take() {
12847 Some(stdout) => Some(tokio::spawn(pump_stdout_to(stdout, output_sink.clone()))),
12848 None => {
12849 warn!(
12850 module_id = %spec.module_id,
12851 "spawned child exposed no stdout pipe; file capture will be incomplete"
12852 );
12853 None
12854 }
12855 };
12856 let stderr_pump = match child.stderr.take() {
12857 Some(stderr) => {
12858 let generation = ring
12859 .lock()
12860 .unwrap_or_else(|poisoned| poisoned.into_inner())
12861 .begin_process();
12862 Some(StderrPump {
12863 task: tokio::spawn(pump_stderr_to(
12864 stderr,
12865 Arc::clone(ring),
12866 generation,
12867 output_sink,
12868 )),
12869 generation,
12870 })
12871 }
12872 None => {
12873 ring.lock()
12877 .unwrap_or_else(|poisoned| poisoned.into_inner())
12878 .mark_not_captured("stderr pipe was not available on spawn");
12879 warn!(
12880 module_id = %spec.module_id,
12881 "spawned child exposed no stderr pipe; tail will be unavailable"
12882 );
12883 None
12884 }
12885 };
12886
12887 let exec = Arc::new(crate::activity::ExecOutcome::new());
12888 #[cfg(target_os = "macos")]
12889 let pending_exec = privacy_exec.is_some();
12890 #[cfg(not(target_os = "macos"))]
12891 let pending_exec = false;
12892 if !pending_exec {
12893 exec.finish(crate::activity::ExecState::CommittedExec);
12894 }
12895 #[cfg(windows)]
12896 let subtree_target = job
12897 .as_ref()
12898 .map(|job| crate::subtree_resources::Target::Job(Arc::downgrade(job)))
12899 .unwrap_or(crate::subtree_resources::Target::Unavailable("no_job"));
12900 #[cfg(target_os = "linux")]
12901 let subtree_target = cgroup_path
12902 .map(crate::subtree_resources::Target::Cgroup)
12903 .unwrap_or(crate::subtree_resources::Target::Unavailable("no_cgroup"));
12904 #[cfg(not(any(target_os = "linux", windows)))]
12905 let subtree_target = crate::subtree_resources::Target::Unavailable("unsupported_platform");
12906 let mut supervised = SupervisedChild {
12907 registration_spawn,
12908 #[cfg(any(unix, windows))]
12909 registration_channel,
12910 live_program_binding: None,
12911 registered_config_connection: None,
12912 admission_base,
12913 registration_declaration,
12914 _program_binding: binding_publication,
12915 subtree_target,
12916 child: containment::ProcessChild::plain(child, pid),
12917 exec: crate::activity::ExecOwner(exec),
12918 #[cfg(windows)]
12919 nonce_delivery,
12920 protocol: spec.protocol,
12921 #[cfg(target_os = "linux")]
12922 module_id: cgroup_name,
12923 #[cfg(target_os = "linux")]
12924 cgroup_placement: cgroup_placement.cloned(),
12925 #[cfg(windows)]
12926 job,
12927 stdout_pump,
12928 stderr_pump,
12929 stderr_ring: Arc::clone(ring),
12930 spawned_at,
12931 config_delivery,
12932 registration_deadline: None,
12933 registration_watch: None,
12934 spawned_at_ms,
12935 spawned_from,
12936 launch_spec_digest: LaunchSpecDigest::from_spec(spec),
12937 spawned_file_identity,
12938 process_start_time,
12939 process_identity,
12940 pid,
12941 roster_guard: Some(roster_guard),
12942 #[cfg(target_os = "macos")]
12943 privacy_exec,
12944 #[cfg(target_os = "macos")]
12945 report_ready: Arc::new(OnceLock::new()),
12946 spawn_failure: None,
12947 admission_kill_sent: false,
12948 };
12949 if let Err(error) = containment::adopt(&mut supervised, spec, roster) {
12950 supervised.child.reject(
12951 #[cfg(unix)]
12952 group_target,
12953 );
12954 #[cfg(all(test, unix))]
12955 let _ = roster.admission_event_for_test(&spec.module_id, "after_rejected_child", Some(pid));
12956 return Err(error);
12957 }
12958 if let (Some(launch), Some(owner), Some(base)) = (
12959 program_launch,
12960 config_owner,
12961 supervised.admission_base.clone(),
12962 ) {
12963 supervised.live_program_binding =
12964 content_binding::LiveProgramBinding::new(launch, &supervised, owner, base);
12965 if let (Some(publication), Some(live)) = (
12966 &supervised._program_binding,
12967 &supervised.live_program_binding,
12968 ) {
12969 publication.attach(live);
12970 }
12971 }
12972 #[cfg(unix)]
12973 if let Some(admitted) = admitted.take() {
12974 admitted.disarm();
12975 }
12976 #[cfg(windows)]
12977 if let Err(error) = resume_suspended_child(pid, spec) {
12978 containment::terminate_blocking(&roster.containment);
12979 let _ = supervised.child.start_kill();
12980 return Err(error);
12981 }
12982 if binding_ready {
12983 let checked = if unavailable_initial_image {
12987 Err(crate::config_gate::GateError::new(
12988 "config_delivery_deferred",
12989 "initial process image is unavailable",
12990 ))
12991 } else {
12992 match (
12993 &supervised._program_binding,
12994 &supervised.live_program_binding,
12995 ) {
12996 (Some(binding), _) => binding.verify(),
12997 (None, Some(live)) => live.check(),
12998 _ if supervised.admission_base.is_some() => {
12999 Err(crate::config_gate::GateError::new(
13000 "config_delivery_deferred",
13001 "live program witness is unavailable",
13002 ))
13003 }
13004 _ => Ok(()),
13005 }
13006 };
13007 #[cfg(all(test, target_os = "linux"))]
13008 let checked = if roster.admission_event_for_test(
13009 &spec.module_id,
13010 "after_content_witness_check",
13011 Some(pid),
13012 ) {
13013 checked
13014 } else {
13015 Err(crate::config_gate::GateError::new(
13016 "config_delivery_deferred",
13017 "current process image is unavailable",
13018 ))
13019 };
13020 if let Err(error) = checked {
13021 if let Some(binding) = &mut supervised._program_binding {
13022 binding.reject();
13023 }
13024 if error.code == "program_refused" || supervised.try_wait().ok().flatten().is_none() {
13027 supervised.spawn_failure = Some(format!("{}: {}", error.code, error.detail));
13028 #[cfg(all(test, target_os = "linux"))]
13029 assert!(roster.admission_event_for_test(
13030 &spec.module_id,
13031 if unavailable_initial_image {
13032 "before_initial_image_kill"
13033 } else {
13034 "before_content_witness_kill"
13035 },
13036 Some(pid),
13037 ));
13038 let _ = supervised.start_admission_kill();
13039 #[cfg(all(test, target_os = "linux"))]
13040 if supervised.admission_kill_sent {
13041 assert!(roster.admission_event_for_test(
13042 &spec.module_id,
13043 "after_admission_kill_sent",
13044 Some(pid),
13045 ));
13046 }
13047 }
13048 }
13049 }
13050 if let Some(registration) = delivery_registration {
13051 registration.commit();
13052 }
13053 if let Some(spawn) = &supervised.registration_spawn {
13054 if let Some(identity) = supervised.process_identity {
13055 let _ = spawn.identity.set(identity);
13056 }
13057 if supervised.spawn_failure.is_some() {
13058 spawn.refuse();
13059 } else if binding_ready {
13060 spawn.ready();
13061 }
13062 #[cfg(any(unix, windows))]
13063 if let (Some(handle), Some(channel)) =
13064 (registration_handle, supervised.registration_channel.take())
13065 {
13066 handle.serve_registration(channel, spawn.clone(), pid, spawned_at + register_timeout);
13067 }
13068 }
13069 Ok(supervised)
13070}
13071
13072#[cfg(test)]
13073mod spawn_record_tests {
13074 use super::*;
13075
13076 thread_local! {
13077 static LAST_PID: std::cell::Cell<Option<u32>> = const { std::cell::Cell::new(None) };
13078 }
13079
13080 pub(super) fn after_create(child: &Child, _spec: &ModuleSpec) {
13081 LAST_PID.set(child.id());
13082 #[cfg(windows)]
13083 if let Some((_, address)) = _spec
13084 .env
13085 .iter()
13086 .find(|(key, _)| key == "SUBC_TEST_AFTER_CREATE")
13087 {
13088 use std::io::{Read, Write};
13089 let mut stream = std::net::TcpStream::connect(address).unwrap();
13090 stream
13091 .write_all(&child.id().unwrap().to_ne_bytes())
13092 .unwrap();
13093 let mut release = [0];
13094 stream.read_exact(&mut release).unwrap();
13095 }
13096 }
13097
13098 #[cfg(windows)]
13099 pub(super) fn assignment_failpoint(spec: &ModuleSpec) -> io::Result<()> {
13100 if spec
13101 .env
13102 .iter()
13103 .any(|(key, _)| key == "SUBC_TEST_ASSIGNMENT_FAILURE")
13104 {
13105 Err(io::Error::other("injected module job assignment failure"))
13106 } else {
13107 Ok(())
13108 }
13109 }
13110
13111 fn spec() -> ModuleSpec {
13112 ModuleSpec {
13113 script: None,
13114 module_id: "spawn-record-worker".into(),
13115 #[cfg(unix)]
13116 program: "/bin/sleep".into(),
13117 #[cfg(windows)]
13118 program: std::env::current_exe().unwrap(),
13119 #[cfg(unix)]
13120 args: vec!["60".into()],
13121 #[cfg(windows)]
13122 args: vec![
13123 "--exact".into(),
13124 "supervise::spawn_record_tests::windows_store_touch_fixture".into(),
13125 "--nocapture".into(),
13126 ],
13127 env: vec![],
13128 reserved: false,
13129 reserved_prefixes: vec![],
13130 protocol: ModuleProtocol::None,
13131 overlap: Default::default(),
13132 carry_containment: false,
13133 }
13134 }
13135
13136 #[tokio::test]
13137 async fn record_sync_failure_fails_spawn_and_reaps_child() {
13138 let home = cortexkit_test_support::ScratchDir::new("record-sync-failure");
13139 let supervisor =
13140 Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default())
13141 .with_live_children_record(home.join("live-children.json"));
13142 crate::live_children::fail_next_file_sync();
13143 let result = supervisor.spawn(spec());
13144 let failed = result.is_err();
13145 if let Ok(module) = result {
13146 module.stop().await.unwrap();
13147 }
13148 assert!(failed, "record failure must fail spawn");
13149 let pid = LAST_PID.get().expect("the real OS child was created");
13150 assert!(
13151 subc_os::Process::open(pid).unwrap().is_none(),
13152 "child {pid} survived rejected admission or was not reaped"
13153 );
13154 }
13155
13156 #[cfg(target_os = "macos")]
13157 #[test]
13158 #[ignore = "manual on-device latency measurement"]
13159 fn durable_record_write_latency_50() {
13160 let home = cortexkit_test_support::ScratchDir::new("record-latency");
13161 let path = home.join("live-children.json");
13162 let children = vec![crate::live_children::LiveChild {
13163 module_id: "latency-worker".into(),
13164 pid: std::process::id(),
13165 protocol: ModuleProtocol::None,
13166 start_time: Some(1),
13167 executable: None,
13168 cgroup_name: None,
13169 container: None,
13170 }];
13171 let mut samples = Vec::new();
13172 for _ in 0..50 {
13173 let start = std::time::Instant::now();
13174 crate::live_children::write_record(&path, &children).unwrap();
13175 samples.push(start.elapsed());
13176 }
13177 samples.sort();
13178 eprintln!(
13179 "durable_record_write_latency_50: n=50 median={:?} p95={:?}",
13180 (samples[24] + samples[25]) / 2,
13181 samples[47]
13182 );
13183 }
13184
13185 #[cfg(windows)]
13186 #[test]
13187 fn windows_store_touch_fixture() {
13188 if let Some(path) = std::env::var_os("SUBC_TEST_STORE_TOUCH") {
13189 std::fs::write(path, b"module ran").unwrap();
13190 }
13191 }
13192
13193 #[cfg(windows)]
13194 #[tokio::test]
13195 async fn windows_daemon_fixture() {
13196 let Ok(address) = std::env::var("SUBC_TEST_DAEMON_BARRIER") else {
13197 return;
13198 };
13199 let mut spec = spec();
13200 spec.env.push(("SUBC_TEST_AFTER_CREATE".into(), address));
13201 let supervisor =
13202 Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default());
13203 let _module = supervisor.spawn(spec).unwrap();
13204 }
13205
13206 #[cfg(windows)]
13207 #[test]
13208 fn windows_daemon_death_before_module_job_assignment_leaves_no_child() {
13209 use std::io::Read;
13210 let listener = std::net::TcpListener::bind("127.0.0.1:0").unwrap();
13211 let mut command = std::process::Command::new(std::env::current_exe().unwrap());
13214 command
13215 .args([
13216 "--exact",
13217 "supervise::spawn_record_tests::windows_daemon_fixture",
13218 "--nocapture",
13219 ])
13220 .env(
13221 "SUBC_TEST_DAEMON_BARRIER",
13222 listener.local_addr().unwrap().to_string(),
13223 );
13224 let mut daemon = subc_jobobject::spawn_contained(&mut command).unwrap();
13225 listener.set_nonblocking(true).unwrap();
13226 let deadline = std::time::Instant::now() + Duration::from_secs(30);
13227 let mut stream = loop {
13228 if let Ok((stream, _)) = listener.accept() {
13229 break stream;
13230 }
13231 assert!(
13232 daemon.child.try_wait().unwrap().is_none(),
13233 "daemon failed before rendezvous"
13234 );
13235 assert!(
13236 std::time::Instant::now() < deadline,
13237 "daemon did not reach CreateProcess failpoint"
13238 );
13239 std::thread::yield_now();
13240 };
13241 stream
13242 .set_read_timeout(Some(Duration::from_secs(30)))
13243 .unwrap();
13244 let mut pid = [0; 4];
13245 stream.read_exact(&mut pid).unwrap();
13246 let pid = u32::from_ne_bytes(pid);
13247 assert!(subc_jobobject::process_exists(pid));
13248 daemon.child.kill().unwrap();
13251 daemon.child.wait().unwrap();
13252 let exited = subc_jobobject::wait_for_process_exit(pid, Duration::from_secs(10));
13253 if !exited {
13254 daemon.job.terminate().unwrap();
13255 }
13256 assert!(exited, "suspended child {pid} outlived hard-killed daemon");
13257 }
13258
13259 #[cfg(windows)]
13260 #[tokio::test]
13261 async fn windows_containment_failure_never_resumes() {
13262 let home = cortexkit_test_support::ScratchDir::new("assignment-refusal");
13263 let sentinel = home.join("store-touched");
13264 let mut spec = spec();
13265 spec.env
13266 .push(("SUBC_TEST_ASSIGNMENT_FAILURE".into(), "1".into()));
13267 spec.env.push((
13268 "SUBC_TEST_STORE_TOUCH".into(),
13269 sentinel.display().to_string(),
13270 ));
13271 let supervisor =
13272 Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default());
13273 let result = supervisor.spawn(spec);
13274 if let Ok(module) = result {
13277 let _ = tokio::time::timeout(Duration::from_secs(10), async {
13278 while !sentinel.exists() && module.state().unwrap() == ModuleState::Running {
13279 tokio::task::yield_now().await;
13280 }
13281 })
13282 .await;
13283 let _ = module.stop().await;
13284 panic!(
13285 "assignment failure resumed a module; store touched={}",
13286 sentinel.exists()
13287 );
13288 }
13289 assert!(
13290 !sentinel.exists(),
13291 "refused module executed store-touch sentinel"
13292 );
13293 assert!(!subc_jobobject::process_exists(LAST_PID.get().unwrap()));
13294 }
13295
13296 #[cfg(windows)]
13297 #[tokio::test]
13298 async fn windows_nested_daemon_fixture() {
13299 let Some(sentinel) = std::env::var_os("SUBC_TEST_NESTED_STORE") else {
13300 return;
13301 };
13302 let mut spec = spec();
13303 spec.env.push((
13304 "SUBC_TEST_STORE_TOUCH".into(),
13305 sentinel.to_str().unwrap().into(),
13306 ));
13307 let supervisor =
13308 Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default());
13309 let module = supervisor
13310 .spawn(spec)
13311 .expect("per-module job must nest below daemon and host jobs");
13312 tokio::time::timeout(Duration::from_secs(10), async {
13313 while !PathBuf::from(&sentinel).exists() {
13314 tokio::task::yield_now().await;
13315 }
13316 })
13317 .await
13318 .unwrap();
13319 module.stop().await.unwrap();
13320 }
13321
13322 #[cfg(windows)]
13323 #[test]
13324 fn windows_module_job_nests_under_host_job() {
13325 let home = cortexkit_test_support::ScratchDir::new("nested-jobs");
13326 let sentinel = home.join("nested-store-touched");
13327 let mut command = std::process::Command::new(std::env::current_exe().unwrap());
13328 command
13329 .args([
13330 "--exact",
13331 "supervise::spawn_record_tests::windows_nested_daemon_fixture",
13332 "--nocapture",
13333 ])
13334 .env("SUBC_TEST_NESTED_STORE", &sentinel);
13335 let mut daemon = subc_jobobject::spawn_contained(&mut command).unwrap();
13336 assert!(daemon.child.wait().unwrap().success());
13337 assert!(sentinel.exists(), "nested protocol:none module never ran");
13338 }
13339}
13340
13341#[cfg(target_os = "linux")]
13342pub(crate) fn kill_module_cgroup(placement: Option<&subc_cgroup::Placement>, module_id: &str) {
13343 use subc_cgroup::KillOutcome;
13344 match subc_cgroup::kill_module(placement, module_id) {
13345 KillOutcome::Killed => {}
13346 KillOutcome::NotPlaced | KillOutcome::Unsupported => {
13347 debug!(
13348 module_id,
13349 "cgroup tree kill unavailable; using direct-child kill"
13350 );
13351 }
13352 KillOutcome::IoError { path, error } => {
13353 warn!(module_id, path = %path.display(), %error, "cgroup tree kill failed; using direct-child kill");
13354 }
13355 }
13356}
13357
13358#[cfg(windows)]
13366fn contain_spawned_child(
13367 child: &mut Child,
13368 spec: &ModuleSpec,
13369 carried: Option<&Arc<containment::CarriedContainment>>,
13370) -> Result<Option<Arc<Mutex<subc_jobobject::JobObject>>>, SuperviseError> {
13371 let assignment = || {
13372 let job = match carried {
13373 Some(container) => container.job(),
13374 None => Arc::new(Mutex::new(subc_jobobject::JobObject::new()?)),
13375 };
13376 #[cfg(test)]
13377 spawn_record_tests::assignment_failpoint(spec)?;
13378 job.lock()
13379 .unwrap_or_else(|p| p.into_inner())
13380 .assign(child)?;
13381 Ok::<_, io::Error>(job)
13382 };
13383 match assignment() {
13384 Ok(job) => Ok(Some(job)),
13385 Err(source) => {
13386 kill_and_reap_rejected_child(child);
13387 Err(SuperviseError::Spawn {
13388 program: spec.program.clone(),
13389 source: io::Error::other(format!(
13390 "Windows module job containment failed: {source}"
13391 )),
13392 cgroup_path: None,
13393 })
13394 }
13395 }
13396}
13397
13398fn kill_and_reap_rejected_child(child: &mut Child, #[cfg(unix)] group: containment::GroupTarget) {
13401 #[cfg(unix)]
13402 group.signal_rejected(child);
13403 let _ = child.start_kill();
13404 while matches!(child.try_wait(), Ok(None)) {
13405 std::thread::yield_now();
13406 }
13407}
13408
13409#[cfg(windows)]
13414fn resume_suspended_child(pid: u32, spec: &ModuleSpec) -> Result<(), SuperviseError> {
13415 if let Err(source) = subc_jobobject::resume_main_thread(pid) {
13416 let _ = subc_os::child_spawn::status(
13420 std::process::Command::new("taskkill.exe")
13421 .args(["/PID", &pid.to_string(), "/T", "/F"])
13422 .stdin(Stdio::null())
13423 .stdout(Stdio::null())
13424 .stderr(Stdio::null()),
13425 );
13426 return Err(SuperviseError::Spawn {
13427 program: spec.program.clone(),
13428 source,
13429 cgroup_path: None,
13430 });
13431 }
13432 Ok(())
13433}
13434
13435#[cfg(target_os = "linux")]
13436fn remove_module_cgroup(placement: &subc_cgroup::Placement, module_id: &str) {
13437 match placement.remove_module(module_id) {
13438 Ok(()) => debug!(module_id, "removed module cgroup after process exit"),
13439 Err(error) => warn!(
13440 module_id,
13441 error = %error,
13442 "could not remove module cgroup after process exit; continuing teardown"
13443 ),
13444 }
13445}
13446
13447#[cfg(target_os = "linux")]
13448const REAPED_CGROUP_TERM_GRACE: Duration = Duration::from_secs(2);
13449#[cfg(target_os = "linux")]
13450const REAPED_CGROUP_FREEZE_TIMEOUT: Duration = Duration::from_millis(100);
13451
13452#[cfg(target_os = "linux")]
13453async fn cleanup_reaped_cgroup(placement: &subc_cgroup::Placement, module_id: &str) {
13454 let path = placement.module_path(module_id).ok();
13455 if path.as_ref().is_some_and(|path| {
13456 std::fs::read_to_string(path.join("cgroup.events"))
13457 .ok()
13458 .is_some_and(|events| events.lines().any(|line| line == "populated 0"))
13459 }) {
13460 remove_module_cgroup(placement, module_id);
13461 return;
13462 }
13463
13464 if terminate_reaped_cgroup(placement, module_id).await {
13467 if let Some(path) = &path {
13468 let deadline = Instant::now() + REAPED_CGROUP_TERM_GRACE;
13469 while cgroup_is_populated(path) && Instant::now() < deadline {
13470 sleep_until((Instant::now() + Duration::from_millis(1)).min(deadline)).await;
13471 }
13472 }
13473 }
13474
13475 if matches!(
13479 subc_cgroup::kill_module(Some(placement), module_id),
13480 subc_cgroup::KillOutcome::Killed
13481 ) {
13482 if let Some(path) = &path {
13483 while cgroup_is_populated(path) {
13484 sleep(Duration::from_millis(1)).await;
13485 }
13486 }
13487 }
13488 remove_module_cgroup(placement, module_id);
13489}
13490
13491#[cfg(target_os = "linux")]
13492fn cgroup_is_populated(path: &std::path::Path) -> bool {
13493 std::fs::read_to_string(path.join("cgroup.events"))
13494 .ok()
13495 .is_some_and(|events| events.lines().any(|line| line == "populated 1"))
13496}
13497
13498#[cfg(target_os = "linux")]
13499async fn terminate_reaped_cgroup(placement: &subc_cgroup::Placement, module_id: &str) -> bool {
13500 let frozen =
13501 match subc_cgroup::freeze_module(Some(placement), module_id, REAPED_CGROUP_FREEZE_TIMEOUT)
13502 .await
13503 {
13504 subc_cgroup::FreezeOutcome::Frozen(frozen) => frozen,
13505 outcome => {
13506 tracing::debug!(
13507 module_id,
13508 ?outcome,
13509 "cgroup freeze failed; skipping SIGTERM grace"
13510 );
13511 return false;
13512 }
13513 };
13514 let listed = match frozen.processes() {
13515 subc_cgroup::ProcsOutcome::Listed(pids) => {
13516 for pid in pids.into_iter().filter(|pid| *pid > 1) {
13517 if let Some(pid) = rustix::process::Pid::from_raw(pid) {
13518 if let Err(error) =
13519 rustix::process::kill_process(pid, rustix::process::Signal::TERM)
13520 {
13521 tracing::debug!(module_id, %pid, %error, "leftover cgroup SIGTERM failed");
13522 }
13523 }
13524 }
13525 true
13526 }
13527 outcome => {
13528 tracing::debug!(
13529 module_id,
13530 ?outcome,
13531 "cgroup process listing failed; skipping SIGTERM grace"
13532 );
13533 false
13534 }
13535 };
13536 match frozen.thaw() {
13537 subc_cgroup::ThawOutcome::Thawed => listed,
13538 outcome => {
13539 tracing::debug!(
13540 module_id,
13541 ?outcome,
13542 "cgroup thaw failed; proceeding to cgroup.kill"
13543 );
13544 false
13545 }
13546 }
13547}
13548
13549#[cfg(target_os = "linux")]
13550fn apply_cgroup_placement(
13551 command: &mut Command,
13552 spec: &ModuleSpec,
13553 path: &std::path::Path,
13554) -> Result<(), SuperviseError> {
13555 subc_cgroup::apply(command, path).map_err(|source| SuperviseError::Cgroup {
13556 module_id: spec.module_id.clone(),
13557 source,
13558 })
13559}
13560
13561fn capture_retention(spec: &ModuleSpec) -> Retention {
13562 let defaults = Retention::default();
13563 let value = |name: &str| {
13564 spec.env
13565 .iter()
13566 .rev()
13567 .find_map(|(key, value)| (key == name).then_some(value.as_str()))
13568 };
13569 Retention {
13570 max_file_mb: value(CAPTURE_MAX_FILE_MB_ENV)
13571 .and_then(|value| value.parse().ok())
13572 .unwrap_or(defaults.max_file_mb),
13573 keep: value(CAPTURE_KEEP_ENV)
13574 .and_then(|value| value.parse().ok())
13575 .unwrap_or(defaults.keep),
13576 max_age_days: value(CAPTURE_MAX_AGE_DAYS_ENV)
13577 .and_then(|value| value.parse().ok())
13578 .unwrap_or(defaults.max_age_days),
13579 }
13580}
13581
13582fn generate_launch_nonce() -> Result<String, SuperviseError> {
13585 let mut bytes = [0u8; 32];
13586 getrandom::getrandom(&mut bytes).map_err(|source| SuperviseError::LaunchNonce {
13587 reason: source.to_string(),
13588 })?;
13589 let mut hex = String::with_capacity(64);
13590 for b in bytes {
13591 use std::fmt::Write;
13592 let _ = write!(hex, "{b:02x}");
13593 }
13594 Ok(hex)
13595}
13596
13597fn constant_time_eq(a: &[u8], b: &[u8]) -> bool {
13600 if a.len() != b.len() {
13601 return false;
13602 }
13603 let mut diff = 0u8;
13604 for (x, y) in a.iter().zip(b.iter()) {
13605 diff |= x ^ y;
13606 }
13607 diff == 0
13608}
13609
13610#[cfg(not(windows))]
13614fn observe_spawned_image(pid: u32) -> Option<subc_os::Observation> {
13615 let process = subc_os::Process::open(pid).ok().flatten()?;
13616 #[cfg(target_os = "linux")]
13617 return sample_after_linux_exec(
13618 || {
13619 let stat = std::fs::read_to_string(format!("/proc/{pid}/stat")).ok()?;
13620 let committed = linux_exec_committed(&stat);
13621 #[cfg(test)]
13622 spawn_image_tests::report_exec_state(committed);
13623 committed
13624 },
13625 || process.observe(),
13626 );
13627 #[cfg(not(target_os = "linux"))]
13628 process.observe()
13629}
13630
13631#[cfg(target_os = "linux")]
13635fn linux_exec_committed(stat: &str) -> Option<bool> {
13636 const PF_FORKNOEXEC: u64 = 0x40;
13637 let flags = stat.rsplit_once(')')?.1.split_whitespace().nth(6)?;
13638 Some(flags.parse::<u64>().ok()? & PF_FORKNOEXEC == 0)
13639}
13640
13641#[cfg(target_os = "linux")]
13642fn sample_after_linux_exec(
13643 mut exec_committed: impl FnMut() -> Option<bool>,
13644 observe: impl FnOnce() -> Option<subc_os::Observation>,
13645) -> Option<subc_os::Observation> {
13646 let deadline = std::time::Instant::now() + Duration::from_secs(1);
13647 while !exec_committed()? {
13648 if std::time::Instant::now() >= deadline {
13652 return None;
13653 }
13654 std::thread::sleep(Duration::from_micros(50));
13655 }
13656 observe()
13657}
13658
13659#[cfg(all(test, target_os = "linux"))]
13660mod spawn_image_tests {
13661 use super::*;
13662 use std::cell::{Cell, RefCell};
13663
13664 enum ExecEvent {
13665 Uncommitted,
13666 Image(Option<subc_os::Observation>),
13667 }
13668
13669 thread_local! {
13670 static EXEC_EVENTS: RefCell<Option<std::sync::mpsc::Sender<ExecEvent>>> = const { RefCell::new(None) };
13671 }
13672
13673 pub(super) fn report_exec_state(committed: Option<bool>) {
13674 if committed == Some(false) {
13675 EXEC_EVENTS.with(|events| {
13676 if let Some(events) = events.borrow_mut().take() {
13677 let _ = events.send(ExecEvent::Uncommitted);
13678 }
13679 });
13680 }
13681 }
13682
13683 #[test]
13684 fn linux_spawn_observation_waits_for_a_paused_fork_to_exec() {
13685 use std::io::{Read, Write};
13686 use std::os::fd::AsRawFd;
13687
13688 let descriptor = std::fs::File::open("/dev/null").unwrap();
13689 let mut command = std::process::Command::new("/bin/sleep");
13690 command.arg("30");
13691 let (mut ready, mut release) =
13692 subc_os::fork_exec_test::pause_before_exec(&mut command, descriptor.as_raw_fd())
13693 .unwrap();
13694 let spawn = std::thread::spawn(move || command.spawn());
13695 let mut message = [0u8; 8];
13696 ready.read_exact(&mut message).unwrap();
13697 let pid = i32::from_ne_bytes(message[..4].try_into().unwrap()) as u32;
13698 let before = std::fs::read_to_string(format!("/proc/{pid}/stat"));
13699 let inherited = subc_os::Process::open(pid).unwrap().unwrap().observe();
13700 let (events, received) = std::sync::mpsc::channel();
13701 let observer = std::thread::spawn(move || {
13702 EXEC_EVENTS.with(|slot| *slot.borrow_mut() = Some(events.clone()));
13703 let image = observe_spawned_image(pid);
13704 let _ = events.send(ExecEvent::Image(image));
13705 });
13706 let first = received.recv_timeout(Duration::from_secs(5));
13710 release.write_all(&[1]).unwrap();
13711 let mut child = spawn.join().unwrap().unwrap();
13712 let after = match first {
13713 Ok(ExecEvent::Uncommitted) => match received.recv_timeout(Duration::from_secs(5)) {
13714 Ok(ExecEvent::Image(image)) => image,
13715 _ => None,
13716 },
13717 Ok(ExecEvent::Image(image)) => image,
13718 Err(_) => None,
13719 };
13720 observer.join().unwrap();
13721 child.kill().unwrap();
13722 child.wait().unwrap();
13723
13724 assert_eq!(linux_exec_committed(&before.unwrap()), Some(false));
13725 assert_eq!(
13726 inherited.unwrap().executable,
13727 subc_os::file_identity(&std::env::current_exe().unwrap())
13728 );
13729 assert_eq!(
13730 after.unwrap().executable,
13731 subc_os::file_identity(std::path::Path::new("/bin/sleep"))
13732 );
13733 }
13734
13735 #[test]
13736 fn spawned_image_is_not_sampled_until_linux_exec_commits() {
13737 let committed = Cell::new(false);
13738 let inherited = subc_os::Observation {
13739 start_time: 23,
13740 executable: Some(subc_os::FileIdentity {
13741 device: 29,
13742 inode: 31,
13743 }),
13744 };
13745 let module = subc_os::Observation {
13746 start_time: 23,
13747 executable: Some(subc_os::FileIdentity {
13748 device: 29,
13749 inode: 37,
13750 }),
13751 };
13752 let polls = Cell::new(0);
13753 let image = sample_after_linux_exec(
13754 || {
13755 polls.set(polls.get() + 1);
13756 committed.set(polls.get() == 2);
13759 Some(committed.get())
13760 },
13761 || Some(if committed.get() { module } else { inherited }),
13762 );
13763 assert_eq!(image, Some(module));
13764 assert_eq!(polls.get(), 2);
13765 }
13766
13767 #[test]
13768 fn linux_exec_flag_is_read_from_flags_after_the_last_parenthesis() {
13769 assert_eq!(
13770 linux_exec_committed("17 (a ) b) R 1 2 3 4 5 4194368"),
13771 Some(false)
13772 );
13773 assert_eq!(
13774 linux_exec_committed("17 (a ) b) R 1 2 3 4 5 4194304"),
13775 Some(true)
13776 );
13777 assert_eq!(linux_exec_committed("17 (a) R 1 2 3 4 5 invalid"), None);
13778 assert_eq!(linux_exec_committed("17 (a) R"), None);
13779 }
13780
13781 #[test]
13782 fn unreadable_exec_state_never_samples_an_inherited_image() {
13783 assert_eq!(
13784 sample_after_linux_exec(
13785 || None,
13786 || panic!("must not sample before exec confirmation")
13787 ),
13788 None
13789 );
13790 }
13791
13792 #[test]
13793 fn uncommitted_exec_times_out_without_sampling_an_inherited_image() {
13794 assert_eq!(
13795 sample_after_linux_exec(
13796 || Some(false),
13797 || panic!("must not sample after an exec timeout")
13798 ),
13799 None
13800 );
13801 }
13802}
13803
13804#[cfg(windows)]
13807pub(crate) fn windows_spawn_capture(spec: &ModuleSpec) -> io::Result<subc_os::ExecutableCapture> {
13808 if let Err(error) = require_program_search_path(spec) {
13809 return Err(match error {
13810 SuperviseError::Spawn { source, .. } => source,
13811 error => io::Error::other(error),
13812 });
13813 }
13814 let mut program = spec.program.clone();
13815 if program.is_absolute() || program.components().count() > 1 {
13816 match subc_os::ExecutableCapture::open(&program) {
13819 Ok(capture) => return Ok(capture),
13820 Err(error)
13821 if error.kind() == io::ErrorKind::NotFound && program.extension().is_none() => {}
13822 Err(error) => return Err(error),
13823 }
13824 program.set_extension("exe");
13825 return subc_os::ExecutableCapture::open(&program);
13826 }
13827 if program.extension().is_none() {
13828 program.set_extension("exe");
13829 }
13830 let custom_path = spec
13831 .env
13832 .iter()
13833 .rev()
13834 .find(|(key, _)| key.eq_ignore_ascii_case("PATH"));
13835 let mut directories = Vec::new();
13836 if let Some(parent) = std::env::current_exe()?.parent() {
13839 directories.push(parent.to_path_buf());
13840 }
13841 directories.push(std::env::current_dir()?);
13842 if let Some(windows) = std::env::var_os("SystemRoot") {
13843 let windows = PathBuf::from(windows);
13844 directories.push(windows.join("System32"));
13845 directories.push(windows.clone());
13846 }
13847 let path = custom_path
13848 .map(|(_, value)| std::ffi::OsString::from(value))
13849 .or_else(|| std::env::var_os("PATH"));
13850 if let Some(path) = path {
13851 directories.extend(std::env::split_paths(&path));
13852 }
13853 for directory in directories {
13854 match subc_os::ExecutableCapture::open(&directory.join(&program)) {
13855 Ok(capture) => return Ok(capture),
13856 Err(error) if error.kind() == io::ErrorKind::NotFound => continue,
13857 Err(error) => return Err(error),
13858 }
13859 }
13860 Err(io::Error::new(
13861 io::ErrorKind::NotFound,
13862 "resolved executable not found",
13863 ))
13864}
13865
13866async fn spawn_and_mark_running_async(
13867 spec: &ModuleSpec,
13868 runtime: &SupervisorRuntimeConfig,
13869 snapshot: &SharedSnapshot,
13870) -> Result<SupervisedChild, SuperviseError> {
13871 let activity_spawn = runtime.begin_activity_spawn(&spec.module_id);
13872 let registration_watch = watch_registration(&runtime.registry, spec);
13873 #[cfg(unix)]
13874 let joining_group = containment::current(&runtime.child_roster.containment)
13875 .is_some_and(|container| container.group().is_some());
13876 let result = spawn_child_async(spec, runtime).await;
13877 #[cfg(unix)]
13878 let result = if result.is_err()
13879 && joining_group
13880 && !runtime.child_roster.is_closed()
13881 && lock_snapshot(snapshot).is_ok_and(|state| state.enabled)
13882 && runtime
13883 .child_roster
13884 .previous_instance(&spec.module_id)
13885 .is_none()
13886 {
13887 warn!(module_id = %spec.module_id, "carried process-group spawn failed; trying one fresh container");
13890 spawn_child_async(spec, runtime).await
13891 } else {
13892 result
13893 };
13894 let mut child = result?;
13895 arm_registration_deadline(&mut child, runtime.register_timeout, registration_watch);
13896 if let Err(error) = set_running(snapshot, &child, &spec.module_id, &runtime.spawn_events) {
13897 containment::terminate_blocking(&runtime.child_roster.containment);
13898 return Err(error);
13899 }
13900 if let Some(spawn) = activity_spawn {
13901 spawn.complete(snapshot);
13902 }
13903 Ok(child)
13904}
13905
13906#[cfg(test)]
13907fn spawn_and_mark_running(
13908 spec: &ModuleSpec,
13909 runtime: &SupervisorRuntimeConfig,
13910 snapshot: &SharedSnapshot,
13911) -> Result<SupervisedChild, SuperviseError> {
13912 let activity_spawn = runtime.begin_activity_spawn(&spec.module_id);
13913 let registration_watch = watch_registration(&runtime.registry, spec);
13914 #[cfg(unix)]
13915 let joining_group = containment::current(&runtime.child_roster.containment)
13916 .is_some_and(|container| container.group().is_some());
13917 let spawn = || {
13918 spawn_child(
13919 spec,
13920 runtime.connection_file_path.as_deref(),
13921 runtime.supervisor_handle.as_ref(),
13922 &runtime.stderr_ring,
13923 runtime.capture_logs_dir.as_deref(),
13924 &runtime.child_roster,
13925 runtime.spawn_delivery(),
13926 #[cfg(target_os = "linux")]
13927 runtime.cgroup_placement.as_ref(),
13928 runtime.register_timeout,
13929 )
13930 };
13931 let result = spawn();
13932 #[cfg(unix)]
13933 let result = if result.is_err()
13934 && joining_group
13935 && !runtime.child_roster.is_closed()
13936 && lock_snapshot(snapshot).is_ok_and(|state| state.enabled)
13937 && runtime
13938 .child_roster
13939 .previous_instance(&spec.module_id)
13940 .is_none()
13941 {
13942 warn!(module_id = %spec.module_id, "carried process-group spawn failed; trying one fresh container");
13946 spawn()
13947 } else {
13948 result
13949 };
13950 let child = result?;
13951 let mut child = child;
13952 arm_registration_deadline(&mut child, runtime.register_timeout, registration_watch);
13953 if let Err(error) = set_running(snapshot, &child, &spec.module_id, &runtime.spawn_events) {
13954 containment::terminate_blocking(&runtime.child_roster.containment);
13955 return Err(error);
13956 }
13957 if let Some(spawn) = activity_spawn {
13958 spawn.complete(snapshot);
13959 }
13960 Ok(child)
13961}
13962
13963struct RegistrationWatch {
13964 changes: watch::Receiver<u64>,
13965 baseline: u64,
13966}
13967
13968fn watch_registration(registry: &Registry, spec: &ModuleSpec) -> Option<RegistrationWatch> {
13972 if spec.protocol != ModuleProtocol::Subc {
13973 return None;
13974 }
13975 let changes = registry.subscribe_module_changes(&spec.module_id).ok()?;
13976 let baseline = *changes.borrow();
13977 Some(RegistrationWatch { changes, baseline })
13978}
13979
13980fn arm_registration_deadline(
13981 child: &mut SupervisedChild,
13982 budget: Duration,
13983 registration_watch: Option<RegistrationWatch>,
13984) {
13985 child.registration_watch = registration_watch;
13986 child.registration_deadline = (child.protocol == ModuleProtocol::Subc
13987 && child.awaiting_registration())
13988 .then_some(child.spawned_at + budget);
13989}
13990
13991async fn stop_unregistered_child(child: &mut SupervisedChild) -> io::Result<()> {
13992 child.retire_registration();
13993 if let Some(container) = child.child.container() {
13994 container.stop_members().await;
13995 child.kill_leader()?;
13996 child.wait().await?;
13997 return Ok(());
13998 }
13999 #[cfg(unix)]
14000 {
14001 let group = unreaped_registration_group(child)?;
14004 let _ = rustix::process::kill_process_group(group, rustix::process::Signal::TERM);
14005 sleep(crate::child_roster::TERM_TO_KILL).await;
14010 let group = unreaped_registration_group(child)?;
14011 let _ = rustix::process::kill_process_group(group, rustix::process::Signal::KILL);
14012 }
14013 child.start_kill()?;
14016 child.wait().await?;
14017 Ok(())
14018}
14019
14020#[cfg(unix)]
14021fn unreaped_registration_group(child: &SupervisedChild) -> io::Result<rustix::process::Pid> {
14022 if let Some(container) = child.child.container() {
14023 return container
14024 .group()
14025 .and_then(|pid| rustix::process::Pid::from_raw(pid as i32))
14026 .ok_or_else(|| io::Error::other("a carried cgroup has no signalable process group"));
14027 }
14028 let group = i32::try_from(child.pid)
14031 .ok()
14032 .filter(|pid| *pid > 1)
14033 .and_then(rustix::process::Pid::from_raw)
14034 .ok_or_else(|| {
14035 io::Error::new(
14036 io::ErrorKind::InvalidInput,
14037 "invalid supervised process group",
14038 )
14039 })?;
14040 if child.child.id() != Some(child.pid) {
14041 return Err(io::Error::new(
14042 io::ErrorKind::InvalidInput,
14043 "refusing process-group signal after supervised child was reaped",
14044 ));
14045 }
14046 Ok(group)
14047}
14048
14049enum RegistrationWaitOutcome {
14050 Registered,
14051 Exited(ExitReport),
14052 TimedOut,
14053}
14054
14055struct ReloadRegistrationFailure {
14056 exit_report: ExitReport,
14057 reason: String,
14058}
14059
14060#[derive(Debug, Clone, Copy, PartialEq, Eq)]
14061enum BusyGaugeObservation {
14062 Quiescent,
14063 Busy,
14064 Omitted,
14065}
14066
14067#[derive(Debug, Clone, PartialEq, Eq)]
14069enum GaugeVerdict {
14070 Zero,
14072 NonZero(u64),
14074 NotACount(String),
14078 Missing,
14080 ProbeFailed(&'static str),
14083}
14084
14085impl GaugeVerdict {
14086 fn reads_quiet(&self) -> bool {
14087 matches!(self, Self::Zero)
14088 }
14089}
14090
14091impl fmt::Display for GaugeVerdict {
14092 fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
14093 match self {
14094 Self::Zero => f.write_str("zero"),
14095 Self::NonZero(value) => write!(f, "non_zero({value})"),
14096 Self::NotACount(value) => write!(f, "not_a_count({value})"),
14097 Self::Missing => f.write_str("missing_from_health_body"),
14098 Self::ProbeFailed(kind) => write!(f, "probe_failed({kind})"),
14099 }
14100 }
14101}
14102
14103const GAUGE_VALUE_TEXT_LIMIT: usize = 32;
14106
14107fn gauge_verdicts(metrics: Option<&Value>, gauges: &[String]) -> Vec<GaugeVerdict> {
14109 let metrics = metrics.and_then(Value::as_object);
14110 gauges
14111 .iter()
14112 .map(
14113 |gauge| match metrics.and_then(|metrics| metrics.get(gauge)) {
14114 None => GaugeVerdict::Missing,
14115 Some(value) => match value.as_u64() {
14116 Some(0) => GaugeVerdict::Zero,
14117 Some(count) => GaugeVerdict::NonZero(count),
14118 None => {
14119 let mut text = value.to_string();
14120 if text.len() > GAUGE_VALUE_TEXT_LIMIT {
14121 let mut end = GAUGE_VALUE_TEXT_LIMIT;
14122 while !text.is_char_boundary(end) {
14123 end -= 1;
14124 }
14125 text.truncate(end);
14126 text.push_str("...");
14127 }
14128 GaugeVerdict::NotACount(text)
14129 }
14130 },
14131 },
14132 )
14133 .collect()
14134}
14135
14136fn busy_gauge_observation(verdicts: &[GaugeVerdict]) -> BusyGaugeObservation {
14139 if verdicts
14140 .iter()
14141 .any(|verdict| matches!(verdict, GaugeVerdict::Missing))
14142 {
14143 BusyGaugeObservation::Omitted
14144 } else if verdicts.iter().all(GaugeVerdict::reads_quiet) {
14145 BusyGaugeObservation::Quiescent
14146 } else {
14147 BusyGaugeObservation::Busy
14148 }
14149}
14150
14151#[derive(Debug, Clone, Default, PartialEq, Eq)]
14156struct GaugeTrace {
14157 latest: Option<(GaugeVerdict, Duration)>,
14158 last_value: Option<(GaugeVerdict, Duration)>,
14159}
14160
14161#[derive(Debug, Clone, Default, PartialEq, Eq)]
14167struct DrainGaugeTrace {
14168 probes: u32,
14170 gauges: Vec<(String, GaugeTrace)>,
14172}
14173
14174impl DrainGaugeTrace {
14175 fn new(gauges: &[String]) -> Self {
14176 Self {
14177 probes: 0,
14178 gauges: gauges
14179 .iter()
14180 .map(|gauge| (gauge.clone(), GaugeTrace::default()))
14181 .collect(),
14182 }
14183 }
14184
14185 fn record(&mut self, verdicts: &[GaugeVerdict], at: Duration) {
14187 self.probes = self.probes.saturating_add(1);
14188 for ((_, trace), verdict) in self.gauges.iter_mut().zip(verdicts) {
14189 trace.latest = Some((verdict.clone(), at));
14190 if matches!(
14191 verdict,
14192 GaugeVerdict::Zero | GaugeVerdict::NonZero(_) | GaugeVerdict::NotACount(_)
14193 ) {
14194 trace.last_value = Some((verdict.clone(), at));
14195 }
14196 }
14197 }
14198}
14199
14200impl fmt::Display for DrainGaugeTrace {
14205 fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
14206 if self.gauges.is_empty() {
14207 return f.write_str("none declared");
14208 }
14209 for (index, (name, trace)) in self.gauges.iter().enumerate() {
14210 if index > 0 {
14211 f.write_str(" | ")?;
14212 }
14213 write!(f, "{name}: ")?;
14214 match &trace.latest {
14215 Some((verdict, at)) => write!(f, "{verdict} @{}ms", at.as_millis())?,
14216 None => f.write_str("never probed")?,
14217 }
14218 match &trace.last_value {
14219 Some((value, at)) => write!(f, "; last value {value} @{}ms", at.as_millis())?,
14220 None => f.write_str("; no value ever read")?,
14221 }
14222 }
14223 Ok(())
14224 }
14225}
14226
14227pub(crate) fn declared_busy_gauges(
14228 registry: &Registry,
14229 module_id: &str,
14230) -> Result<Vec<String>, SuperviseError> {
14231 busy_gauges_of(
14232 registry
14233 .get_module(module_id)
14234 .map_err(SuperviseError::Registry)?,
14235 )
14236}
14237
14238fn declared_busy_gauges_for_connection(
14242 registry: &Registry,
14243 connection_id: ConnectionId,
14244) -> Result<Vec<String>, SuperviseError> {
14245 busy_gauges_of(
14246 registry
14247 .get_module_by_connection(connection_id)
14248 .map_err(SuperviseError::Registry)?,
14249 )
14250}
14251
14252fn busy_gauges_of(
14253 registration: Option<crate::registry::ModuleRegistration>,
14254) -> Result<Vec<String>, SuperviseError> {
14255 let Some(registration) = registration else {
14256 return Ok(Vec::new());
14257 };
14258 let Some(self_signals) = registration.manifest.self_signals else {
14259 return Ok(Vec::new());
14260 };
14261
14262 let mut gauges = Vec::new();
14263 for declaration in self_signals {
14264 if declaration.kind != SelfSignalKind::Busy {
14265 continue;
14266 }
14267 match declaration.anchored_to {
14268 SignalAnchor::HealthGauges { gauges: declared } if !declared.is_empty() => {
14269 gauges.extend(declared)
14270 }
14271 _ => {
14272 gauges.push(String::new());
14275 }
14276 }
14277 }
14278 Ok(gauges)
14279}
14280
14281#[allow(clippy::too_many_arguments)]
14290async fn wait_for_forwarding_quiescence(
14291 forwarding: &ForwardingTable,
14292 module_id: &str,
14293 runtime: &SupervisorRuntimeConfig,
14294 endpoint: crate::ModuleEndpointId,
14295 deadline: Instant,
14296 busy_gauges: &[String],
14297 scope: DrainScope,
14298 trace: &mut DrainGaugeTrace,
14299) -> Result<bool, SuperviseError> {
14300 let wait_started = Instant::now();
14301 let mut gauges_quiescent = busy_gauges.is_empty();
14302 let mut next_probe_at = wait_started;
14303 let mut omission_reported = false;
14304
14305 loop {
14306 let now = Instant::now();
14307 if !busy_gauges.is_empty() && now >= next_probe_at && now < deadline {
14308 let report = match scope {
14309 DrainScope::Active => probe_module_health(module_id, runtime, Some(deadline)).await,
14310 DrainScope::Endpoint(endpoint) => {
14311 probe_endpoint_health(endpoint, runtime, Some(deadline)).await
14312 }
14313 };
14314 let verdicts = match &report {
14315 Ok(report) => gauge_verdicts(report.metrics.as_ref(), busy_gauges),
14316 Err(err) => vec![GaugeVerdict::ProbeFailed(err.label()); busy_gauges.len()],
14317 };
14318 trace.record(&verdicts, wait_started.elapsed());
14319 gauges_quiescent = match report {
14320 Ok(_) => match busy_gauge_observation(&verdicts) {
14321 BusyGaugeObservation::Quiescent => true,
14322 BusyGaugeObservation::Busy => false,
14323 BusyGaugeObservation::Omitted => {
14324 if !omission_reported {
14327 forwarding
14328 .counters()
14329 .increment_drains_with_undeclared_gauge();
14330 let missing = busy_gauges
14331 .iter()
14332 .zip(&verdicts)
14333 .filter(|(_, verdict)| matches!(verdict, GaugeVerdict::Missing))
14334 .map(|(gauge, _)| gauge.as_str())
14335 .collect::<Vec<_>>()
14336 .join(",");
14337 warn!(
14338 module_id,
14339 missing_gauges = %missing,
14340 "drain health.check answer lacks a declared busy gauge; the drain treats the module as busy until it reports it"
14341 );
14342 omission_reported = true;
14343 }
14344 false
14345 }
14346 },
14347 Err(err) => {
14348 warn!(
14349 module_id,
14350 error = %err,
14351 "drain health.check did not produce declared busy gauges; treating module as busy"
14352 );
14353 false
14354 }
14355 };
14356 next_probe_at = Instant::now() + drain_busy_reprobe_interval();
14359 }
14360
14361 let in_flight = forwarding
14362 .endpoint_in_flight_count(endpoint)
14363 .map_err(SuperviseError::Forwarding)?;
14364 if in_flight == 0 && gauges_quiescent {
14365 return Ok(true);
14366 }
14367
14368 let now = Instant::now();
14369 if now >= deadline {
14370 return Ok(false);
14371 }
14372 let mut wait = deadline
14373 .saturating_duration_since(now)
14374 .min(REGISTRY_RELEASE_POLL);
14375 if !busy_gauges.is_empty() {
14376 wait = wait.min(next_probe_at.saturating_duration_since(now));
14377 }
14378 sleep(wait).await;
14379 }
14380}
14381
14382fn drain_busy_reprobe_interval() -> Duration {
14386 DRAIN_BUSY_REPROBE_INTERVAL.max(REGISTRY_RELEASE_POLL)
14387}
14388
14389fn warn_drain_timed_out(
14404 module_id: &str,
14405 drain_timeout: Duration,
14406 reason: RouteCloseReason,
14407 holdouts: &crate::forwarding::DrainHoldouts,
14408 gauges: &DrainGaugeTrace,
14409) {
14410 warn!(
14411 module_id = %module_id,
14412 waited = ?drain_timeout,
14413 ?reason,
14414 held_requests = holdouts.requests,
14415 held_routes = holdouts.routes,
14416 total_routes = holdouts.total_routes,
14417 top_connections = ?holdouts.top_connections,
14418 held = %holdouts
14421 .held
14422 .iter()
14423 .map(|(channel, corr)| format!("{channel}:{corr}"))
14424 .collect::<Vec<_>>()
14425 .join(","),
14426 gauge_probes = gauges.probes,
14427 busy_gauges = %gauges,
14428 "route drain timed out before request quiescence; forcing teardown"
14429 );
14430}
14431
14432fn drained_after_quiescence_wait(wait_result: &Result<bool, SuperviseError>) -> bool {
14440 match wait_result {
14441 Ok(drained) => *drained,
14442 Err(_) => false,
14443 }
14444}
14445
14446fn send_route_goodbyes(forwarding: &ForwardingTable, released_routes: Vec<GoodbyeTarget>) {
14447 for released in released_routes {
14448 let frame = match Frame::build_with_version(
14449 released.negotiated_ver,
14450 FrameType::Goodbye,
14451 control_flags(),
14452 released.channel,
14453 released.epoch,
14454 0,
14455 Vec::new(),
14456 ) {
14457 Ok(frame) => frame,
14458 Err(err) => {
14459 warn!(
14460 route_channel = released.channel,
14461 error = %err,
14462 "failed to build supervisor drain route GOODBYE frame"
14463 );
14464 continue;
14465 }
14466 };
14467 if !released.close_on_delivery_failure() {
14468 crate::forwarding::send_module_route_goodbye(
14469 &forwarding.counters(),
14470 &released.sink,
14471 frame,
14472 released.module_id.as_deref(),
14473 "supervisor drain",
14474 );
14475 continue;
14476 }
14477 if let Err(err) = released.sink.try_send(frame) {
14478 warn!(
14479 target_connection_id = released.connection_id.get(),
14480 route_channel = released.channel,
14481 error = %err,
14482 "supervisor drain route GOODBYE was not delivered to client; closing target connection"
14483 );
14484 let _ = forwarding.escalate_client_delivery_failure(
14485 released.connection_id,
14486 released.channel,
14487 released.epoch,
14488 CloseReason::new(
14489 "route_goodbye_delivery_failed",
14490 format!(
14491 "failed to enqueue supervisor drain route GOODBYE for channel {}: {err}",
14492 released.channel
14493 ),
14494 ),
14495 crate::forwarding::UndeliveredFrame {
14496 module_id: released.module_id.as_deref(),
14497 sink: &released.sink,
14498 },
14499 );
14500 }
14501 }
14502}
14503
14504async fn send_module_draining(
14505 module_id: &str,
14506 reason: RouteCloseReason,
14507 deadline_ms: u64,
14508 target: &ModuleDrainTarget,
14509 _wait_for_flush: bool,
14510) -> bool {
14511 let body = match serde_json::to_vec(&ModuleControlCommand::Draining {
14512 reason,
14513 deadline_ms,
14514 }) {
14515 Ok(body) => body,
14516 Err(err) => {
14517 warn!(
14518 module_id,
14519 error = %err,
14520 "failed to encode module draining command"
14521 );
14522 return false;
14523 }
14524 };
14525 let frame = match Frame::build_with_version(
14526 target.negotiated_ver,
14527 FrameType::Push,
14528 control_flags(),
14529 0,
14530 0,
14531 0,
14532 body,
14533 ) {
14534 Ok(frame) => frame,
14535 Err(err) => {
14536 warn!(
14537 module_id,
14538 error = %err,
14539 "failed to build module draining command frame"
14540 );
14541 return false;
14542 }
14543 };
14544 #[cfg(unix)]
14545 if _wait_for_flush {
14546 return target.sink.send_flushed(frame).await.is_ok();
14547 }
14548 if let Err(err) = target.sink.try_send(frame) {
14549 warn!(
14550 module_id,
14551 target_connection_id = target.endpoint.connection_id.get(),
14552 error = %err,
14553 "module draining command was not delivered to peer"
14554 );
14555 return false;
14556 }
14557 true
14558}
14559
14560fn module_goodbye_frame(module_id: &str, negotiated_ver: u8) -> Option<Frame> {
14562 match Frame::build_with_version(
14563 negotiated_ver,
14564 FrameType::Goodbye,
14565 control_flags(),
14566 0,
14567 0,
14568 0,
14569 Vec::new(),
14570 ) {
14571 Ok(frame) => Some(frame),
14572 Err(err) => {
14573 warn!(
14574 module_id,
14575 error = %err,
14576 "failed to build module GOODBYE frame"
14577 );
14578 None
14579 }
14580 }
14581}
14582
14583#[cfg(unix)]
14597async fn send_module_goodbyes_for_daemon_shutdown(
14598 forwarding: &Arc<ForwardingTable>,
14599 reason: &CloseReason,
14600 wait_for_flush: bool,
14601) {
14602 const GOODBYE_BUDGET: Duration = Duration::from_millis(500);
14603 let targets = match forwarding.module_connections() {
14604 Ok(targets) => targets,
14605 Err(err) => {
14606 warn!(error = %err, "could not list module connections for shutdown GOODBYE");
14607 return;
14608 }
14609 };
14610 let deadline = Instant::now() + GOODBYE_BUDGET;
14611 let mut sends = tokio::task::JoinSet::new();
14612 for target in targets {
14613 let Some(frame) = module_goodbye_frame(&target.module_id, target.negotiated_ver) else {
14614 continue;
14615 };
14616 if !wait_for_flush {
14617 if let Err(err) = target.sink.try_send(frame) {
14618 debug!(
14619 module_id = %target.module_id,
14620 error = %err,
14621 "shutdown module GOODBYE was not queued"
14622 );
14623 }
14624 continue;
14625 }
14626 let forwarding = Arc::clone(forwarding);
14627 let reason = reason.clone();
14628 sends.spawn(async move {
14629 match timeout_at(deadline, target.sink.send_flushed(frame)).await {
14630 Ok(Ok(())) => {}
14631 Ok(Err(err)) => debug!(
14632 module_id = %target.module_id,
14633 error = %err,
14634 "module connection closed before its shutdown GOODBYE was written"
14635 ),
14636 Err(_) => warn!(
14637 module_id = %target.module_id,
14638 budget = ?GOODBYE_BUDGET,
14639 "shutdown module GOODBYE was not written within its budget; closing anyway"
14640 ),
14641 }
14642 forwarding.request_connection_close(target.endpoint.connection_id, reason);
14643 });
14644 }
14645 while sends.join_next().await.is_some() {}
14647}
14648
14649async fn send_module_goodbye(
14650 module_id: &str,
14651 forwarding: &ForwardingTable,
14652 target: &ModuleDrainTarget,
14653 wait_for_flush: bool,
14654) {
14655 let Some(frame) = module_goodbye_frame(module_id, target.negotiated_ver) else {
14656 return;
14657 };
14658 let sent = if wait_for_flush {
14661 timeout(Duration::from_millis(500), target.sink.send_flushed(frame))
14662 .await
14663 .map_err(|_| "module GOODBYE flush exceeded 500ms".to_owned())
14664 .and_then(|result| result.map_err(|err| err.to_string()))
14665 } else {
14666 target.sink.try_send(frame).map_err(|err| err.to_string())
14667 };
14668 if let Err(err) = sent {
14669 warn!(
14670 module_id,
14671 target_connection_id = target.endpoint.connection_id.get(),
14672 error = %err,
14673 "supervisor drain module GOODBYE was not delivered to peer; closing module connection"
14674 );
14675 forwarding.request_connection_close(
14676 target.endpoint.connection_id,
14677 CloseReason::new(
14678 "module_goodbye_delivery_failed",
14679 format!("failed to enqueue supervisor drain module GOODBYE for module '{module_id}': {err}"),
14680 ),
14681 );
14682 }
14683}
14684
14685#[derive(Clone, Copy)]
14686struct ForwardingDrainContext<'a> {
14687 spec: &'a ModuleSpec,
14688 runtime: &'a SupervisorRuntimeConfig,
14689 registry: &'a Registry,
14690 scope: DrainScope,
14691}
14692
14693#[derive(Debug, Clone, Copy, PartialEq, Eq)]
14695enum DrainScope {
14696 Active,
14699 Endpoint(crate::ModuleEndpointId),
14704}
14705
14706#[derive(Debug, Clone, Copy, PartialEq, Eq)]
14714enum StopNotice {
14715 SentOverConnection,
14718 NoConnection,
14722 NotSent,
14726}
14727
14728async fn begin_forwarding_drain(
14729 spec: &ModuleSpec,
14730 runtime: &SupervisorRuntimeConfig,
14731 registry: &Registry,
14732 snapshot: &SharedSnapshot,
14733 enabled: Option<bool>,
14734 reason: RouteCloseReason,
14735) -> Result<StopNotice, SuperviseError> {
14736 let Some(forwarding) = runtime.forwarding.as_ref() else {
14737 return Err(SuperviseError::ReloadUnavailable {
14738 module_id: spec.module_id.clone(),
14739 reason: "supervisor was not configured with a forwarding table".to_string(),
14740 });
14741 };
14742
14743 begin_forwarding_drain_with(
14744 forwarding,
14745 ForwardingDrainContext {
14746 spec,
14747 runtime,
14748 registry,
14749 scope: DrainScope::Active,
14750 },
14751 snapshot,
14752 enabled,
14753 reason,
14754 runtime.drain_timeout,
14755 )
14756 .await
14757}
14758
14759async fn begin_forwarding_drain_if_configured(
14760 spec: &ModuleSpec,
14761 runtime: &SupervisorRuntimeConfig,
14762 registry: &Registry,
14763 snapshot: &SharedSnapshot,
14764 enabled: Option<bool>,
14765 reason: RouteCloseReason,
14766) -> Result<StopNotice, SuperviseError> {
14767 begin_forwarding_drain_with_timeout(
14768 spec,
14769 runtime,
14770 registry,
14771 snapshot,
14772 enabled,
14773 reason,
14774 runtime.drain_timeout,
14775 )
14776 .await
14777}
14778
14779async fn begin_forwarding_drain_with_timeout(
14783 spec: &ModuleSpec,
14784 runtime: &SupervisorRuntimeConfig,
14785 registry: &Registry,
14786 snapshot: &SharedSnapshot,
14787 enabled: Option<bool>,
14788 reason: RouteCloseReason,
14789 drain_timeout: Duration,
14790) -> Result<StopNotice, SuperviseError> {
14791 let Some(forwarding) = runtime.forwarding.as_ref() else {
14792 return Ok(StopNotice::NotSent);
14793 };
14794
14795 begin_forwarding_drain_with(
14796 forwarding,
14797 ForwardingDrainContext {
14798 spec,
14799 runtime,
14800 registry,
14801 scope: DrainScope::Active,
14802 },
14803 snapshot,
14804 enabled,
14805 reason,
14806 drain_timeout,
14807 )
14808 .await
14809}
14810
14811async fn begin_forwarding_drain_with(
14812 forwarding: &ForwardingTable,
14813 context: ForwardingDrainContext<'_>,
14814 snapshot: &SharedSnapshot,
14815 enabled: Option<bool>,
14816 reason: RouteCloseReason,
14817 drain_timeout: Duration,
14818) -> Result<StopNotice, SuperviseError> {
14819 let ForwardingDrainContext {
14820 spec,
14821 runtime,
14822 registry,
14823 scope,
14824 } = context;
14825 debug_assert_ne!(reason, RouteCloseReason::Crash);
14826 let terminal = matches!(reason, RouteCloseReason::Disable);
14827 let drain_started_at = Instant::now();
14828 let drain_deadline = drain_started_at + drain_timeout;
14829 let deadline_ms =
14830 unix_ms_now().saturating_add(u64::try_from(drain_timeout.as_millis()).unwrap_or(u64::MAX));
14831 let busy_gauges = match scope {
14832 DrainScope::Active => declared_busy_gauges(registry, &spec.module_id)?,
14833 DrainScope::Endpoint(endpoint) => {
14834 declared_busy_gauges_for_connection(registry, endpoint.connection_id)?
14835 }
14836 };
14837
14838 let gate_started = Instant::now();
14841 let drain_target = match scope {
14842 DrainScope::Active => forwarding.begin_module_drain(&spec.module_id, reason),
14843 DrainScope::Endpoint(endpoint) => forwarding.begin_endpoint_drain(endpoint, reason),
14844 }
14845 .map_err(SuperviseError::Forwarding)?;
14846 info!(
14851 module_id = %spec.module_id,
14852 ?reason,
14853 gate_ms = u64::try_from(gate_started.elapsed().as_millis()).unwrap_or(u64::MAX),
14854 connected = drain_target.is_some(),
14855 "module drain began; route admission closed"
14856 );
14857 if scope == DrainScope::Active {
14858 update_snapshot(snapshot, Some(&spec.module_id), |state| {
14859 state.state = ModuleState::Draining;
14860 state.draining_to_replace =
14861 matches!(reason, RouteCloseReason::Restart | RouteCloseReason::Reload);
14862 if let Some(enabled) = enabled {
14863 state.enabled = enabled;
14864 }
14865 })?;
14866 }
14867
14868 let Some(target) = drain_target.as_ref() else {
14869 return Ok(StopNotice::NoConnection);
14873 };
14874 let flush_goodbye = runtime.supervisor_handle.as_ref().is_some_and(|handle| {
14875 handle
14876 .registration_spawn(target.endpoint.connection_id)
14877 .is_some()
14878 });
14879 {
14880 send_module_draining(&spec.module_id, reason, deadline_ms, target, false).await;
14881 #[cfg(feature = "test-support")]
14882 forwarding.wait_before_route_closing_for_test().await;
14883 let routes = forwarding
14884 .endpoint_routes(target.endpoint)
14885 .map_err(SuperviseError::Forwarding)?;
14886 let routes_notified = routes.len();
14887 crate::control::send_route_control_pushes(
14888 forwarding,
14889 routes.clone(),
14890 ClientControlPush::RouteClosing {
14891 module_id: spec.module_id.clone(),
14892 channels: Vec::new(),
14893 reason,
14894 },
14895 );
14896 send_route_goodbyes(forwarding, target.abandoned_bindings.clone());
14897
14898 let mut gauge_trace = DrainGaugeTrace::new(&busy_gauges);
14904 let wait_result = wait_for_forwarding_quiescence(
14905 forwarding,
14906 &spec.module_id,
14907 runtime,
14908 target.endpoint,
14909 drain_deadline,
14910 &busy_gauges,
14911 scope,
14912 &mut gauge_trace,
14913 )
14914 .await;
14915 let drained = drained_after_quiescence_wait(&wait_result);
14916 if let Err(err) = &wait_result {
14917 error!(
14918 module_id = %spec.module_id,
14919 ?reason,
14920 error = %err,
14921 "forwarding quiescence wait failed after route.closing; forcing route.closed(drained: false) so the client is not left waiting on an unfulfilled promise"
14922 );
14923 } else if !drained {
14924 let holdouts = forwarding
14926 .endpoint_drain_holdouts(target.endpoint)
14927 .unwrap_or_default();
14928 warn_drain_timed_out(
14929 &spec.module_id,
14930 drain_timeout,
14931 reason,
14932 &holdouts,
14933 &gauge_trace,
14934 );
14935 }
14936 crate::control::send_route_control_pushes(
14937 forwarding,
14938 routes,
14939 ClientControlPush::RouteClosed {
14940 module_id: spec.module_id.clone(),
14941 channels: Vec::new(),
14942 reason,
14943 drained,
14944 abandoned: target.abandoned_bindings.len() as u32,
14945 excluded_subscriptions: target.excluded_subscriptions,
14946 terminal: Some(terminal),
14947 },
14948 );
14949 wait_result?;
14950
14951 let released_routes = match forwarding.release_module_endpoint_routes(target.endpoint) {
14957 Ok(routes) => routes,
14958 Err(err) => {
14959 warn!(
14960 module_id = %spec.module_id,
14961 ?reason,
14962 error = %err,
14963 "failed to release module endpoint routes after route.closed; module GOODBYE will still be sent"
14964 );
14965 send_module_goodbye(&spec.module_id, forwarding, target, flush_goodbye).await;
14966 return Err(SuperviseError::Forwarding(err));
14967 }
14968 };
14969 let route_goodbye_count = released_routes.len();
14970 send_route_goodbyes(forwarding, released_routes);
14971 send_module_goodbye(&spec.module_id, forwarding, target, flush_goodbye).await;
14972
14973 info!(
14979 module_id = %spec.module_id,
14980 ?reason,
14981 routes_notified,
14982 route_goodbyes = route_goodbye_count,
14983 abandoned_reservations = target.abandoned_bindings.len(),
14984 excluded_subscriptions = target.excluded_subscriptions,
14985 drained,
14986 "module drain complete; consumers notified via route.closing/route.closed pushes and per-route GOODBYE frames"
14987 );
14988 }
14989
14990 Ok(StopNotice::SentOverConnection)
14991}
14992
14993async fn wait_for_registration_after_reload(
14996 registry: &Registry,
14997 module_id: &str,
14998 snapshot: &SharedSnapshot,
14999 child: &mut SupervisedChild,
15000 wait: Duration,
15001) -> Result<RegistrationWaitOutcome, SuperviseError> {
15002 wait_for_slot_registration(
15003 registry,
15004 crate::registry::RegistrationSlot::Active(module_id),
15005 module_id,
15006 snapshot,
15007 child,
15008 wait,
15009 )
15010 .await
15011}
15012
15013async fn wait_for_slot_registration(
15021 registry: &Registry,
15022 slot: crate::registry::RegistrationSlot<'_>,
15023 module_id: &str,
15024 snapshot: &SharedSnapshot,
15025 child: &mut SupervisedChild,
15026 wait: Duration,
15027) -> Result<RegistrationWaitOutcome, SuperviseError> {
15028 let deadline = Instant::now() + wait;
15029 loop {
15030 if registry
15031 .registration(slot)
15032 .map_err(SuperviseError::Registry)?
15033 .is_some()
15034 {
15035 return Ok(RegistrationWaitOutcome::Registered);
15036 }
15037
15038 let now = Instant::now();
15039 if now >= deadline {
15040 return Ok(RegistrationWaitOutcome::TimedOut);
15041 }
15042 let remaining = deadline.saturating_duration_since(now);
15043 let poll = remaining.min(REGISTRY_RELEASE_POLL);
15044
15045 tokio::select! {
15046 wait_result = child.wait() => {
15047 let status = wait_result.map_err(|source| SuperviseError::Wait {
15048 module_id: module_id.to_string(),
15049 source,
15050 })?;
15051 return Ok(RegistrationWaitOutcome::Exited(classify_reaped_child_exit(
15052 snapshot,
15053 child,
15054 &status,
15055 )));
15056 }
15057 _ = sleep(poll) => {}
15058 }
15059 }
15060}
15061
15062fn registration_failure_exit_report(mut exit_report: ExitReport) -> ExitReport {
15063 if exit_report.kind != ExitKind::DeliberateSeverance {
15067 exit_report.kind = ExitKind::Crash;
15068 }
15069 exit_report
15070}
15071
15072async fn handle_reload_child_registration_failure(
15073 spec: &ModuleSpec,
15074 runtime: &SupervisorRuntimeConfig,
15075 registry: &Registry,
15076 process_liveness: &SupervisorProcessLiveness,
15077 snapshot: &SharedSnapshot,
15078 _child: &mut Option<SupervisedChild>,
15079 failure: ReloadRegistrationFailure,
15080) -> Result<(), SuperviseError> {
15081 let ReloadRegistrationFailure {
15082 exit_report,
15083 reason,
15084 } = failure;
15085 match on_child_exit(
15086 spec,
15087 runtime.restart_policy,
15088 registry,
15089 snapshot,
15090 &runtime.terminal_ring,
15091 &runtime.spawn_events,
15092 &runtime.child_roster,
15093 exit_report,
15094 runtime.recovery_is_daemon,
15095 )
15096 .await
15097 {
15098 NextAction::Stop {
15099 registration_released,
15100 } => {
15101 if registration_released {
15102 process_liveness.untrack_if_current(&spec.module_id, snapshot);
15103 }
15104 }
15105 NextAction::Restart { schedule } => {
15106 let delay = schedule.map_or(runtime.restart_policy.delay_for_restart(0), |schedule| {
15107 schedule.delay
15108 });
15109 if let Some(schedule) = schedule {
15110 log_crash_respawn(&spec.module_id, schedule);
15111 }
15112 schedule_respawn(
15113 runtime,
15114 snapshot,
15115 &spec.module_id,
15116 delay,
15117 RespawnKind::Spawn,
15118 true,
15119 )?;
15120 }
15121 }
15122 Err(SuperviseError::ReloadFailed {
15123 module_id: spec.module_id.clone(),
15124 reason,
15125 })
15126}
15127
15128async fn handle_reload_spawn_failure(
15129 spec: &ModuleSpec,
15130 runtime: &SupervisorRuntimeConfig,
15131 process_liveness: &SupervisorProcessLiveness,
15132 snapshot: &SharedSnapshot,
15133 _child: &mut Option<SupervisedChild>,
15134 reason: String,
15135) -> Result<(), SuperviseError> {
15136 let now = Instant::now();
15137 let mut schedule = None;
15138 update_snapshot(snapshot, Some(&spec.module_id), |state| {
15139 clear_current_process_facts(state);
15140 if state.enabled {
15141 schedule = state.next_crash_restart(&runtime.restart_policy, now);
15142 state.state = if schedule.is_some() {
15143 ModuleState::Restarting
15144 } else {
15145 if runtime
15146 .failure_context(runtime.recovery_is_daemon)
15147 .may_retry()
15148 {
15149 state.failed_retry.schedule(&spec.module_id);
15150 }
15151 ModuleState::Failed
15152 };
15153 } else {
15154 state.state = ModuleState::Disabled;
15155 }
15156 })?;
15157 if let Some(schedule) = schedule {
15158 schedule_respawn(
15159 runtime,
15160 snapshot,
15161 &spec.module_id,
15162 schedule.delay,
15163 RespawnKind::Spawn,
15164 true,
15165 )?;
15166 } else {
15167 process_liveness.untrack_if_current(&spec.module_id, snapshot);
15168 }
15169 Err(SuperviseError::ReloadFailed {
15170 module_id: spec.module_id.clone(),
15171 reason,
15172 })
15173}
15174
15175fn control_flags() -> Flags {
15176 Flags::new(false, Priority::Passive, false)
15177}
15178
15179#[allow(clippy::too_many_arguments)]
15180async fn drain_optional_child(
15181 module_id: &str,
15182 protocol: ModuleProtocol,
15183 stop_notice: StopNotice,
15184 registry: &Registry,
15185 forwarding: Option<&ForwardingTable>,
15186 snapshot: &SharedSnapshot,
15187 terminal_ring: &Arc<Mutex<TerminalRing>>,
15188 spawn_events: &SpawnEventFeed,
15189 child: &mut Option<SupervisedChild>,
15190 drain_timeout: Duration,
15191 final_state: ModuleState,
15192 enabled: Option<bool>,
15193 failure: &FailureContext,
15194) -> Result<(), SuperviseError> {
15195 if let Some(child) = child.take() {
15196 drain_child_to_state(
15197 module_id,
15198 protocol,
15199 stop_notice,
15200 registry,
15201 forwarding,
15202 snapshot,
15203 terminal_ring,
15204 spawn_events,
15205 child,
15206 drain_timeout,
15207 final_state,
15208 enabled,
15209 failure,
15210 )
15211 .await
15212 } else {
15213 update_snapshot(snapshot, Some(module_id), |state| {
15214 state.state = final_state;
15215 if final_state == ModuleState::Failed {
15216 state.failed_retry.failed_at = Some(Instant::now());
15217 }
15218 if let Some(enabled) = enabled {
15219 state.enabled = enabled;
15220 }
15221 clear_current_process_facts(state);
15222 })?;
15223 release_dead_registration(registry, forwarding, snapshot, module_id, failure).await
15224 }
15225}
15226
15227#[allow(clippy::too_many_arguments)]
15228async fn drain_child_to_state(
15229 module_id: &str,
15230 _protocol: ModuleProtocol,
15231 stop_notice: StopNotice,
15232 registry: &Registry,
15233 forwarding: Option<&ForwardingTable>,
15234 snapshot: &SharedSnapshot,
15235 terminal_ring: &Arc<Mutex<TerminalRing>>,
15236 spawn_events: &SpawnEventFeed,
15237 mut child: SupervisedChild,
15238 drain_timeout: Duration,
15239 final_state: ModuleState,
15240 enabled: Option<bool>,
15241 failure: &FailureContext,
15242) -> Result<(), SuperviseError> {
15243 let protocol = child.protocol;
15244 child.retire_registration();
15245 update_snapshot(snapshot, Some(module_id), |state| {
15246 state.state = ModuleState::Draining;
15247 state.draining_to_replace = final_state == ModuleState::Restarting;
15248 if let Some(enabled) = enabled {
15249 state.enabled = enabled;
15250 }
15251 })?;
15252
15253 if stop_notice != StopNotice::SentOverConnection {
15264 if protocol == ModuleProtocol::Subc && stop_notice == StopNotice::NoConnection {
15265 info!(
15266 module_id,
15267 pid = child.pid,
15268 budget_ms = u64::try_from(drain_timeout.as_millis()).unwrap_or(u64::MAX),
15269 "module has no connection yet; requesting stop by signal"
15270 );
15271 }
15272 request_graceful_stop(module_id, &child);
15273 }
15274
15275 let exit_report = match timeout(drain_timeout, child.wait()).await {
15276 Ok(Ok(status)) => classify_reaped_child_exit(snapshot, &child, &status),
15277 Ok(Err(source)) => {
15278 fail_after_wait_error(
15279 module_id,
15280 snapshot,
15281 terminal_ring,
15282 spawn_events,
15283 format!("failed to wait during drain: {source}"),
15284 failure,
15285 );
15286 return Err(SuperviseError::Wait {
15287 module_id: module_id.to_string(),
15288 source,
15289 });
15290 }
15291 Err(_) => {
15292 warn!(
15305 module_id,
15306 pid = child.pid,
15307 budget_ms = u64::try_from(drain_timeout.as_millis()).unwrap_or(u64::MAX),
15308 reason = ?final_state,
15309 ?stop_notice,
15310 "drain budget expired before the module exited; killing it"
15311 );
15312 let kill =
15313 if final_state == ModuleState::Restarting && child.child.container().is_some() {
15314 child.kill_leader()
15315 } else {
15316 child.start_kill()
15317 };
15318 kill.map_err(|source| {
15319 fail_after_wait_error(
15320 module_id,
15321 snapshot,
15322 terminal_ring,
15323 spawn_events,
15324 format!("failed to kill during drain: {source}"),
15325 failure,
15326 );
15327 SuperviseError::Kill {
15328 module_id: module_id.to_string(),
15329 source,
15330 }
15331 })?;
15332 let status = child.wait().await.map_err(|source| {
15333 fail_after_wait_error(
15334 module_id,
15335 snapshot,
15336 terminal_ring,
15337 spawn_events,
15338 format!("failed to wait after drain kill: {source}"),
15339 failure,
15340 );
15341 SuperviseError::Wait {
15342 module_id: module_id.to_string(),
15343 source,
15344 }
15345 })?;
15346 classify_reaped_child_exit(snapshot, &child, &status)
15347 }
15348 };
15349
15350 let detail = lock_snapshot(snapshot)?.drain_disposition_detail.take();
15351 publish_after_terminal_record(
15352 module_id,
15353 || {
15354 record_terminal_with_detail(
15355 module_id,
15356 terminal_ring,
15357 spawn_events,
15358 &exit_report,
15359 terminal_disposition(final_state),
15360 detail,
15361 )
15362 },
15363 || {
15364 update_snapshot(snapshot, Some(module_id), |state| {
15365 state.state = final_state;
15366 if final_state == ModuleState::Failed {
15367 state.failed_retry.failed_at = Some(Instant::now());
15368 }
15369 if let Some(enabled) = enabled {
15370 state.enabled = enabled;
15371 }
15372 clear_current_process_facts(state);
15373 state.last_exit = Some(exit_report.clone());
15374 if exit_report.kind == ExitKind::DeliberateSeverance {
15375 state.lifetime_restarts += 1;
15376 }
15377 })
15378 },
15379 )?;
15380 child.drain_stderr(module_id).await;
15381
15382 let released =
15383 release_dead_registration(registry, forwarding, snapshot, module_id, failure).await;
15384 child
15385 .finish_exit(
15386 released.is_ok()
15387 && final_state == ModuleState::Restarting
15388 && child.spawn_failure.is_none(),
15389 )
15390 .await
15391 .map_err(|source| SuperviseError::Wait {
15392 module_id: module_id.to_owned(),
15393 source,
15394 })?;
15395 released
15396}
15397
15398#[cfg(unix)]
15418fn request_graceful_stop(module_id: &str, child: &SupervisedChild) {
15419 let Some(pid) = child
15420 .id()
15421 .and_then(|pid| i32::try_from(pid).ok())
15422 .and_then(rustix::process::Pid::from_raw)
15423 else {
15424 debug!(
15425 module_id,
15426 "no pid to signal for teardown; falling through to the drain wait"
15427 );
15428 return;
15429 };
15430 match rustix::process::kill_process(pid, rustix::process::Signal::TERM) {
15431 Ok(()) => debug!(
15432 module_id,
15433 "sent SIGTERM to a module nothing else asked to stop"
15434 ),
15435 Err(err) => debug!(
15436 module_id,
15437 error = %err,
15438 "SIGTERM to module failed; the drain wait and kill still apply"
15439 ),
15440 }
15441}
15442
15443#[cfg(not(unix))]
15451fn request_graceful_stop(module_id: &str, _child: &SupervisedChild) {
15452 debug!(
15453 module_id,
15454 "no graceful stop signal exists on this platform; teardown of a module nothing asked to stop waits, then kills"
15455 );
15456}
15457
15458fn terminal_disposition(final_state: ModuleState) -> TerminalDisposition {
15459 match final_state {
15460 ModuleState::Stopped => TerminalDisposition::Stopped,
15461 ModuleState::Disabled => TerminalDisposition::Disabled,
15462 ModuleState::Restarting => TerminalDisposition::Restarting,
15463 ModuleState::Failed => TerminalDisposition::Failed,
15464 ModuleState::Starting
15465 | ModuleState::Running
15466 | ModuleState::Unresponsive
15467 | ModuleState::Draining => {
15468 unreachable!("terminal exits only finish in terminal or restarting states")
15469 }
15470 }
15471}
15472
15473async fn release_dead_registration(
15481 registry: &Registry,
15482 forwarding: Option<&ForwardingTable>,
15483 snapshot: &SharedSnapshot,
15484 module_id: &str,
15485 failure: &FailureContext,
15486) -> Result<(), SuperviseError> {
15487 let result = async {
15488 let registration = registry
15489 .get_module(module_id)
15490 .map_err(SuperviseError::Registry)?;
15491 match wait_for_registration_release(registry, module_id, REGISTRY_RELEASE_TIMEOUT).await {
15492 Ok(()) => return Ok(()),
15493 Err(SuperviseError::RegistrationStillActive { .. }) => {}
15494 Err(err) => return Err(err),
15495 }
15496 let pid = lock_snapshot(snapshot)?.reaped_pid;
15497 if let (Some(registration), Some(forwarding), Some(pid)) = (registration, forwarding, pid) {
15498 warn!(
15499 module_id,
15500 pid,
15501 connection_id = registration.connection_id.get(),
15502 "reaped module registration outlived release grace; closing dead connection"
15503 );
15504 forwarding.request_connection_close(
15505 registration.connection_id,
15506 CloseReason::new(
15507 "supervised_process_reaped",
15508 format!("module '{module_id}' pid {pid} exited"),
15509 ),
15510 );
15511 wait_for_slot_registration_release(
15512 registry,
15513 crate::registry::RegistrationSlot::Connection(registration.connection_id),
15514 REGISTRY_RELEASE_TIMEOUT,
15515 )
15516 .await?;
15517 }
15518 wait_for_registration_release(registry, module_id, Duration::ZERO).await
15519 }
15520 .await;
15521 if let Err(err) = &result {
15522 fail_snapshot(snapshot, Some(module_id), None, failure);
15523 error!(module_id, error = %err, "registration release failed after child exit; module is failed and start can retry");
15524 }
15525 result
15526}
15527
15528async fn wait_for_registration_release(
15531 registry: &Registry,
15532 module_id: &str,
15533 wait: Duration,
15534) -> Result<(), SuperviseError> {
15535 wait_for_slot_registration_release(
15536 registry,
15537 crate::registry::RegistrationSlot::Active(module_id),
15538 wait,
15539 )
15540 .await
15541}
15542
15543async fn wait_for_slot_registration_release(
15551 registry: &Registry,
15552 slot: crate::registry::RegistrationSlot<'_>,
15553 wait: Duration,
15554) -> Result<(), SuperviseError> {
15555 let deadline = Instant::now() + wait;
15556 let mut release_events = registration_release_events().subscribe();
15557 let still_active = |registration: &crate::registry::ModuleRegistration| {
15558 SuperviseError::RegistrationStillActive {
15559 module_id: registration.manifest.module_id.clone(),
15560 waited: wait,
15561 }
15562 };
15563 loop {
15564 let _observed_generation = *release_events.borrow_and_update();
15565 let Some(registration) = registry
15566 .registration(slot)
15567 .map_err(SuperviseError::Registry)?
15568 else {
15569 return Ok(());
15570 };
15571
15572 let now = Instant::now();
15573 if now >= deadline {
15574 return Err(still_active(®istration));
15575 }
15576
15577 let remaining = deadline.saturating_duration_since(now);
15578 match timeout(remaining, release_events.changed()).await {
15579 Ok(Ok(())) | Ok(Err(_)) => {}
15580 Err(_) => return Err(still_active(®istration)),
15581 }
15582 }
15583}
15584
15585#[cfg(test)]
15586mod slot_registration_wait_tests {
15587 use super::*;
15588 use crate::registry::{ConnectionId, RegistrationSlot};
15589 use subc_protocol::manifest::ModuleManifest;
15590
15591 #[tokio::test]
15592 async fn enable_release_failure_is_failed_and_a_second_enable_retries() {
15593 let registry = Arc::new(Registry::default());
15594 let supervisor = Supervisor::new_for_test(Arc::clone(®istry), RestartPolicy::default());
15595 let runtime = supervisor.runtime_config();
15596 let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::disabled()));
15597 let spec = ModuleSpec {
15598 script: None,
15599 module_id: "enable-stale-registration".to_string(),
15600 program: PathBuf::from("/missing/enable-retry-test"),
15601 args: Vec::new(),
15602 env: Vec::new(),
15603 reserved: false,
15604 reserved_prefixes: Vec::new(),
15605 protocol: ModuleProtocol::Subc,
15606 overlap: Default::default(),
15607 carry_containment: false,
15608 };
15609 let connection = ConnectionId::new(90);
15610 registry
15611 .register_with_control_ops(
15612 ModuleManifest::builder(&spec.module_id, "0.1.0").build(),
15613 1,
15614 connection,
15615 Vec::new(),
15616 )
15617 .unwrap();
15618 let mut child = None;
15619 let err = set_child_enabled(
15620 &spec,
15621 &runtime,
15622 ®istry,
15623 &supervisor.process_liveness,
15624 &snapshot,
15625 &mut child,
15626 true,
15627 &runtime.failure_context(false),
15628 )
15629 .await
15630 .unwrap_err();
15631 assert!(matches!(
15632 err,
15633 SuperviseError::RegistrationStillActive { .. }
15634 ));
15635 assert_eq!(lock_snapshot(&snapshot).unwrap().state, ModuleState::Failed);
15636 assert!(child.is_none());
15637 registry.deregister_connection(connection).unwrap();
15638 let err = set_child_enabled(
15639 &spec,
15640 &runtime,
15641 ®istry,
15642 &supervisor.process_liveness,
15643 &snapshot,
15644 &mut child,
15645 true,
15646 &runtime.failure_context(false),
15647 )
15648 .await
15649 .unwrap_err();
15650 assert!(
15651 matches!(err, SuperviseError::Spawn { .. }),
15652 "second enable must attempt a spawn: {err}"
15653 );
15654 assert_eq!(lock_snapshot(&snapshot).unwrap().state, ModuleState::Failed);
15655 }
15656
15657 const INCUMBENT: u64 = 1;
15658 const CANDIDATE: u64 = 2;
15659
15660 fn swapped_registry() -> Arc<Registry> {
15661 let registry = Arc::new(Registry::default());
15662 let manifest = ModuleManifest::builder("m", "0.1.0").build();
15663 registry
15664 .register_with_control_ops(
15665 manifest.clone(),
15666 1,
15667 ConnectionId::new(INCUMBENT),
15668 Vec::new(),
15669 )
15670 .unwrap();
15671 registry
15672 .register_candidate_with_control_ops(
15673 manifest,
15674 1,
15675 ConnectionId::new(CANDIDATE),
15676 Vec::new(),
15677 )
15678 .unwrap();
15679 registry
15680 }
15681
15682 #[tokio::test]
15686 async fn incumbent_release_is_awaited_by_connection_not_by_module_id() {
15687 let registry = swapped_registry();
15688 registry.promote_candidate("m").unwrap().unwrap();
15689
15690 assert!(matches!(
15691 wait_for_registration_release(®istry, "m", Duration::from_millis(50)).await,
15692 Err(SuperviseError::RegistrationStillActive { .. })
15693 ));
15694
15695 assert!(matches!(
15697 wait_for_slot_registration_release(
15698 ®istry,
15699 RegistrationSlot::Connection(ConnectionId::new(INCUMBENT)),
15700 Duration::from_millis(50),
15701 )
15702 .await,
15703 Err(SuperviseError::RegistrationStillActive { .. })
15704 ));
15705
15706 let releaser = Arc::clone(®istry);
15707 let release = tokio::spawn(async move {
15708 sleep(Duration::from_millis(20)).await;
15709 releaser
15710 .deregister_connection(ConnectionId::new(INCUMBENT))
15711 .unwrap();
15712 notify_registration_release();
15713 });
15714 wait_for_slot_registration_release(
15715 ®istry,
15716 RegistrationSlot::Connection(ConnectionId::new(INCUMBENT)),
15717 Duration::from_secs(5),
15718 )
15719 .await
15720 .expect("the incumbent's own registration is released");
15721 release.await.unwrap();
15722 assert!(registry.get_module("m").unwrap().is_some());
15723 }
15724
15725 #[tokio::test]
15728 async fn candidate_slot_wait_ignores_the_incumbents_registration() {
15729 let registry = swapped_registry();
15730 assert!(matches!(
15731 wait_for_slot_registration_release(
15732 ®istry,
15733 RegistrationSlot::Candidate("m"),
15734 Duration::from_millis(50),
15735 )
15736 .await,
15737 Err(SuperviseError::RegistrationStillActive { .. })
15738 ));
15739 registry
15740 .deregister_connection(ConnectionId::new(CANDIDATE))
15741 .unwrap();
15742 wait_for_slot_registration_release(
15743 ®istry,
15744 RegistrationSlot::Candidate("m"),
15745 Duration::from_millis(50),
15746 )
15747 .await
15748 .expect("a candidate slot with no candidate is released");
15749 assert!(registry
15750 .registration(RegistrationSlot::Active("m"))
15751 .unwrap()
15752 .is_some());
15753 }
15754}
15755
15756fn classify_exit(status: &ExitStatus) -> ExitReport {
15757 ExitReport {
15758 kind: if status.success() {
15759 ExitKind::Clean
15760 } else {
15761 ExitKind::Crash
15762 },
15763 code: status.code(),
15764 signal: exit_signal(status),
15765 at_ms: unix_ms_now(),
15766 }
15767}
15768
15769fn wait_error_exit_report() -> ExitReport {
15775 ExitReport {
15776 kind: ExitKind::Crash,
15777 code: None,
15778 signal: None,
15779 at_ms: unix_ms_now(),
15780 }
15781}
15782
15783#[cfg(unix)]
15784fn exit_signal(status: &ExitStatus) -> Option<i32> {
15785 use std::os::unix::process::ExitStatusExt;
15786
15787 status.signal()
15788}
15789
15790#[cfg(not(unix))]
15791fn exit_signal(_status: &ExitStatus) -> Option<i32> {
15792 None
15793}
15794
15795fn reset_restart_count(snapshot: &SharedSnapshot, module_id: &str) -> Result<(), SuperviseError> {
15801 update_snapshot(snapshot, Some(module_id), |state| {
15802 state.clear_crash_restarts();
15803 })
15804}
15805
15806fn set_running(
15807 snapshot: &SharedSnapshot,
15808 child: &SupervisedChild,
15809 module_id: &str,
15810 spawn_events: &SpawnEventFeed,
15811) -> Result<(), SuperviseError> {
15812 let mut state = snapshot.lock().map_err(|_| SuperviseError::StatePoisoned {
15813 module_id: Some(module_id.to_string()),
15814 })?;
15815 state.spawn_generation = spawn_events.emit_spawned(
15816 module_id,
15817 child
15818 .registration_spawn
15819 .as_ref()
15820 .map(|spawn| spawn.authority.spawn_number()),
15821 child.pid,
15822 child.spawned_at_ms,
15823 );
15824 if std::mem::take(&mut state.coalesced_restart_pending) {
15825 let generation = state.spawn_generation;
15826 info!(module_id, "restart already satisfied by generation {generation}; coalesced pending request completed");
15827 }
15828 state.drain_disposition_detail = None;
15829 state.spawn_failure = None;
15830 state.admission_refused = None;
15831 state.config_hold_parked = false;
15832 state.config_hold_daemon_initiated = false;
15833 state.config_hold_approval = None;
15834 state.legacy_registration_spawn =
15835 child.registration_spawn.is_some() && child.admission_base.is_none();
15836 state.registration_declaration = child.registration_declaration.clone();
15837 state.stop_reason = None;
15838 state.config_exit_failed_approval = None;
15839 state.waiting_for_previous_pid = None;
15840 state.waiting_for_previous_detail = None;
15841 state.in_alternate_slot = false;
15844 state.configuration_updated_since_spawn = false;
15845 state.spawned_protocol = Some(child.protocol);
15846 state.state = ModuleState::Running;
15847 state.approved_start_failure = None;
15848 state.enabled = true;
15849 state.process_alive = true;
15850 state.pid = child.id();
15851 state.subtree_target = child.subtree_target.clone();
15852 state.health.status = SupervisorHealthStatus::Unknown;
15855 state.last_probe_evidence = None;
15856 state.health.consecutive_failures = 0;
15857 state.health.detail = None;
15858 state.health.metrics = None;
15859 state.health.last_probe_ms = None;
15860 state.failed_retry.healthy_since = None;
15861 state.failed_retry.failed_at = None;
15862 state.failed_retry.running_since = Some(child.spawned_at);
15863 #[cfg(windows)]
15864 {
15865 state.pipe_consumed = child
15866 .nonce_delivery
15867 .as_ref()
15868 .map(|delivery| delivery.consumed());
15869 }
15870 #[cfg(target_os = "macos")]
15871 {
15872 state.report_ready = Some(Arc::clone(&child.report_ready));
15873 }
15874 state.spawned_at_ms = Some(child.spawned_at_ms);
15875 state.exec_outcome = Some(Arc::clone(&child.exec.0));
15876 state.spawned_from = Some(child.spawned_from.clone());
15877 state.launch_spec_digest = Some(child.launch_spec_digest.clone());
15878 state.spawned_file_identity = child.spawned_file_identity.clone();
15879 state.process_start_time = child.process_start_time;
15880 Ok(())
15881}
15882
15883fn clear_current_process_facts(state: &mut SupervisorSnapshot) {
15884 state.legacy_registration_spawn = false;
15885 state.registration_declaration = None;
15886 state.subtree_target = crate::subtree_resources::Target::Unavailable("not_running");
15887 state.failed_retry.healthy_since = None;
15888 state.failed_retry.running_since = None;
15889 state.waiting_for_previous_pid = None;
15890 state.waiting_for_previous_detail = None;
15891 state.process_alive = false;
15892 state.spawned_protocol = None;
15893 state.pid = None;
15894 #[cfg(windows)]
15895 {
15896 state.pipe_consumed = None;
15897 }
15898 #[cfg(target_os = "macos")]
15899 {
15900 state.report_ready = None;
15901 }
15902 state.spawned_at_ms = None;
15903 state.exec_outcome = None;
15904 state.spawned_from = None;
15905 state.launch_spec_digest = None;
15906 state.spawned_file_identity = None;
15907 state.process_start_time = None;
15908 state.deliberate_severance = None;
15909}
15910
15911#[cfg(test)]
15912fn record_deliberate_severance(
15913 snapshot: &SharedSnapshot,
15914 identity: ProcessIdentity,
15915) -> Result<(), SuperviseError> {
15916 update_snapshot(snapshot, None, |state| {
15917 state.deliberate_severance = Some(identity);
15918 })
15919}
15920
15921fn apply_deliberate_severance_marker(
15922 snapshot: &SharedSnapshot,
15923 exited_identity: Option<ProcessIdentity>,
15924 mut exit_report: ExitReport,
15925) -> ExitReport {
15926 let marker = lock_snapshot(snapshot)
15927 .ok()
15928 .and_then(|mut state| state.deliberate_severance.take());
15929 if marker.is_some() && marker == exited_identity {
15930 exit_report.kind = ExitKind::DeliberateSeverance;
15931 }
15932 exit_report
15933}
15934
15935fn classify_reaped_child_exit(
15936 snapshot: &SharedSnapshot,
15937 child: &SupervisedChild,
15938 status: &ExitStatus,
15939) -> ExitReport {
15940 let _ = update_snapshot(snapshot, None, |state| {
15941 state.reaped_pid = Some(child.pid);
15942 state.spawn_failure = child.spawn_failure.clone();
15943 });
15944 if matches!(
15945 child.spawn_failure.as_deref(),
15946 Some("registration_timeout" | "registration_channel_closed")
15947 ) {
15948 registration_failure_exit_report(classify_exit(status))
15949 } else {
15950 apply_deliberate_severance_marker(snapshot, child.process_identity(), classify_exit(status))
15951 }
15952}
15953
15954fn fail_snapshot(
15955 snapshot: &SharedSnapshot,
15956 module_id: Option<&str>,
15957 last_exit: Option<ExitReport>,
15958 failure: &FailureContext,
15959) {
15960 if let Err(err) = update_snapshot(snapshot, module_id, |state| {
15961 state.state = ModuleState::Failed;
15962 state.config_hold_parked = false;
15963 state.config_hold_daemon_initiated = false;
15964 state.config_hold_approval = None;
15965 clear_current_process_facts(state);
15966 if failure.may_retry() && state.enabled {
15967 state.failed_retry.schedule(module_id.unwrap_or("unknown"));
15968 } else if failure.shutdown.is_set() {
15969 state.failed_retry.cancel();
15970 }
15971 if let Some(last_exit) = last_exit {
15972 state.last_exit = Some(last_exit);
15973 }
15974 }) {
15975 error!(error = %err, "failed to mark supervisor state failed");
15976 }
15977}
15978
15979fn update_snapshot(
15980 snapshot: &SharedSnapshot,
15981 module_id: Option<&str>,
15982 update: impl FnOnce(&mut SupervisorSnapshot),
15983) -> Result<(), SuperviseError> {
15984 let mut state = snapshot.lock().map_err(|_| SuperviseError::StatePoisoned {
15985 module_id: module_id.map(ToOwned::to_owned),
15986 })?;
15987 update(&mut state);
15988 Ok(())
15989}
15990
15991const SLOW_SNAPSHOT_LOCK_THRESHOLD: Duration = Duration::from_millis(250);
15992
15993fn lock_snapshot_for_control<'a>(
15994 snapshot: &'a SharedSnapshot,
15995 module_id: &str,
15996 caller: &'static str,
15997) -> Result<std::sync::MutexGuard<'a, SupervisorSnapshot>, SuperviseError> {
15998 let started_at = Instant::now();
15999 let guard = lock_snapshot(snapshot)?;
16000 let waited = started_at.elapsed();
16001 if waited >= SLOW_SNAPSHOT_LOCK_THRESHOLD {
16002 warn!(
16003 module_id = %module_id,
16004 waited_ms = waited.as_millis() as u64,
16005 caller = %caller,
16006 "slow snapshot lock"
16007 );
16008 }
16009 Ok(guard)
16010}
16011
16012fn lock_snapshot(
16013 snapshot: &SharedSnapshot,
16014) -> Result<std::sync::MutexGuard<'_, SupervisorSnapshot>, SuperviseError> {
16015 snapshot
16016 .lock()
16017 .map_err(|_| SuperviseError::StatePoisoned { module_id: None })
16018}
16019
16020#[cfg(test)]
16021mod terminal_history_tests {
16022 include!("supervise_route_open_recovery_tests.rs");
16023 use std::{
16024 path::PathBuf,
16025 sync::Arc,
16026 time::{Duration, Instant},
16027 };
16028
16029 use tokio::{sync::mpsc, time::sleep};
16030
16031 use super::{
16032 apply_deliberate_severance_marker, daemon_will_restart, drain_child_to_state,
16033 drained_after_quiescence_wait, handle_reload_spawn_failure, health_restart_child,
16034 lock_snapshot, on_child_exit, record_deliberate_severance, record_wait_error_terminal,
16035 reset_restart_count, spawn_and_mark_running, update_snapshot, wait_error_exit_report,
16036 ExitKind, ExitReport, ModuleProtocol, ModuleSpec, ModuleState, NextAction, ProcessIdentity,
16037 RestartPolicy, SpawnEventKind, StopNotice, SuperviseError, SupervisedConfiguration,
16038 SupervisedModule, SupervisedModuleInner, Supervisor, SupervisorHandle,
16039 SupervisorHealthStatus, SupervisorSnapshot,
16040 };
16041 use super::Instant as ClockInstant;
16046 use crate::{
16047 registry::Registry,
16048 terminal_ring::{TerminalRing, TerminalRingConfig},
16049 };
16050 use std::sync::Mutex;
16051 use subc_control::TerminalDisposition;
16052
16053 pub(super) fn fake_aft_stub_path() -> PathBuf {
16058 let path = cortexkit_test_support::cargo_profile_bin("fake-aft-stub");
16059 assert!(
16060 path.exists(),
16061 "fake-aft-stub not built at {}: run `cargo test -p subc-core` (which builds \
16062 [[bin]] targets) rather than `cargo test -p subc-core --lib` (which does not)",
16063 path.display()
16064 );
16065 path
16066 }
16067
16068 #[test]
16069 fn reserved_never_spawned_refuses_every_hello() {
16070 let supervisor = SupervisorHandle::default();
16075 supervisor.apply_identity_configuration(&ModuleSpec {
16076 script: None,
16077 module_id: "never-spawned".to_string(),
16078 program: PathBuf::from("/usr/bin/false"),
16079 args: Vec::new(),
16080 env: Vec::new(),
16081 reserved: true,
16082 reserved_prefixes: Vec::new(),
16083 protocol: ModuleProtocol::Subc,
16084 overlap: Default::default(),
16085 carry_containment: false,
16086 });
16087 assert!(
16088 supervisor
16089 .reserved_hello_verification("never-spawned", Some("any-forged-nonce"))
16090 .is_err(),
16091 "forged nonce must refuse on a reserved never-spawned id"
16092 );
16093 assert!(
16094 supervisor
16095 .reserved_hello_verification("never-spawned", None)
16096 .is_err(),
16097 "absent nonce must refuse on a reserved never-spawned id"
16098 );
16099 supervisor.set_spawn_nonce("never-spawned", "minted".to_string());
16101 supervisor.apply_identity_configuration(&ModuleSpec {
16102 script: None,
16103 module_id: "never-spawned".to_string(),
16104 program: PathBuf::from("/usr/bin/false"),
16105 args: Vec::new(),
16106 env: Vec::new(),
16107 reserved: true,
16108 reserved_prefixes: Vec::new(),
16109 protocol: ModuleProtocol::Subc,
16110 overlap: Default::default(),
16111 carry_containment: false,
16112 });
16113 assert!(supervisor
16114 .reserved_hello_verification("never-spawned", Some("minted"))
16115 .is_ok());
16116 assert!(supervisor
16117 .reserved_hello_verification("never-spawned", Some("forged"))
16118 .is_err());
16119 }
16120
16121 fn seed_crash_restarts(state: &mut SupervisorSnapshot, count: u32) {
16124 let now = ClockInstant::now();
16125 for _ in 0..count {
16126 state.crash_restarts.push_back(now);
16127 }
16128 }
16129
16130 fn age_oldest_crash_restart_out_of_window(state: &mut SupervisorSnapshot, window: Duration) {
16134 let aged = state
16135 .crash_restarts
16136 .front()
16137 .expect("a crash restart must be recorded before it can be aged")
16138 .checked_sub(window + Duration::from_secs(1))
16139 .expect("the test clock is far enough from its origin to age an instant");
16140 state.crash_restarts[0] = aged;
16141 }
16142
16143 fn snapshot_with_restarts(enabled: bool, count: u32) -> SupervisorSnapshot {
16144 let mut state = SupervisorSnapshot::new(ModuleState::Running, enabled);
16145 seed_crash_restarts(&mut state, count);
16146 state
16147 }
16148
16149 #[test]
16150 fn daemon_owned_recovery_predicate_uses_the_pre_increment_budget() {
16151 let policy = RestartPolicy::new(3, Duration::ZERO);
16152 let now = ClockInstant::now();
16153 assert!(daemon_will_restart(
16154 &mut snapshot_with_restarts(true, 2),
16155 &policy,
16156 now
16157 ));
16158 assert!(!daemon_will_restart(
16159 &mut snapshot_with_restarts(true, 3),
16160 &policy,
16161 now
16162 ));
16163 assert!(!daemon_will_restart(
16164 &mut snapshot_with_restarts(false, 0),
16165 &policy,
16166 now
16167 ));
16168 }
16169
16170 #[test]
16171 fn crash_restart_backoff_escalates_with_in_window_count() {
16172 let policy = RestartPolicy::new(4, Duration::from_millis(100))
16173 .with_max_backoff(Duration::from_secs(30));
16174 let now = ClockInstant::now();
16175 let mut state = SupervisorSnapshot::new(ModuleState::Running, true);
16176 let schedules = (0..4)
16177 .map(|_| {
16178 state
16179 .next_crash_restart(&policy, now)
16180 .expect("the test policy allows four crash restarts")
16181 })
16182 .collect::<Vec<_>>();
16183
16184 assert_eq!(
16185 schedules
16186 .iter()
16187 .map(|schedule| schedule.restart_in_window)
16188 .collect::<Vec<_>>(),
16189 vec![0, 1, 2, 3]
16190 );
16191 assert_eq!(
16192 schedules
16193 .iter()
16194 .map(|schedule| schedule.delay)
16195 .collect::<Vec<_>>(),
16196 vec![
16197 Duration::from_millis(100),
16198 Duration::from_secs(1),
16199 Duration::from_secs(10),
16200 Duration::from_secs(30),
16201 ]
16202 );
16203 }
16204
16205 #[test]
16206 fn crash_restart_backoff_resets_after_ring_clear() {
16207 let policy = RestartPolicy::new(3, Duration::from_millis(100));
16208 let now = ClockInstant::now();
16209 let mut state = SupervisorSnapshot::new(ModuleState::Running, true);
16210 assert_eq!(
16211 state.next_crash_restart(&policy, now).unwrap().delay,
16212 Duration::from_millis(100)
16213 );
16214 assert_eq!(
16215 state.next_crash_restart(&policy, now).unwrap().delay,
16216 Duration::from_secs(1)
16217 );
16218
16219 state.clear_crash_restarts();
16220 let schedule = state
16221 .next_crash_restart(&policy, now)
16222 .expect("a cleared ring must allow another restart");
16223 assert_eq!(schedule.restart_in_window, 0);
16224 assert_eq!(schedule.delay, Duration::from_millis(100));
16225 }
16226
16227 #[test]
16228 fn crash_restart_backoff_ignores_aged_restarts() {
16229 let policy = RestartPolicy::new(3, Duration::from_millis(100));
16230 let now = ClockInstant::now();
16231 let mut state = SupervisorSnapshot::new(ModuleState::Running, true);
16232 state
16233 .next_crash_restart(&policy, now)
16234 .expect("the first restart is allowed");
16235 state
16236 .next_crash_restart(&policy, now)
16237 .expect("the second restart is allowed");
16238 state.crash_restarts[0] = now
16239 .checked_sub(policy.window + Duration::from_secs(1))
16240 .expect("the fake clock can age a restart past the window");
16241
16242 let schedule = state
16243 .next_crash_restart(&policy, now)
16244 .expect("an aged restart must release its slot");
16245 assert_eq!(schedule.restart_in_window, 1);
16246 assert_eq!(schedule.delay, Duration::from_secs(1));
16247 assert_eq!(state.crash_restarts.len(), 2);
16248 }
16249
16250 #[test]
16254 fn a_budget_spent_before_the_window_no_longer_refuses() {
16255 let policy = RestartPolicy::new(3, Duration::ZERO);
16256 let mut state = snapshot_with_restarts(true, 3);
16257 let now = ClockInstant::now();
16258 assert!(!daemon_will_restart(&mut state, &policy, now));
16259
16260 assert!(daemon_will_restart(
16261 &mut state,
16262 &policy,
16263 now + policy.window + Duration::from_secs(1)
16264 ));
16265 assert!(
16266 state.crash_restarts.is_empty(),
16267 "reading the budget must drop the instants that left the window"
16268 );
16269 }
16270
16271 fn module_with_recovery_snapshot(
16272 state: ModuleState,
16273 enabled: bool,
16274 restart_count: u32,
16275 ) -> SupervisedModule {
16276 let registry = Arc::new(Registry::default());
16277 let supervisor =
16278 Supervisor::new_for_test(Arc::clone(®istry), RestartPolicy::new(3, Duration::ZERO));
16279 let runtime = supervisor.runtime_config();
16280 let spec = ModuleSpec {
16281 script: None,
16282 module_id: "recovery-snapshot".to_string(),
16283 program: fake_aft_stub_path(),
16284 args: Vec::new(),
16285 env: Vec::new(),
16286 reserved: false,
16287 reserved_prefixes: Vec::new(),
16288 protocol: ModuleProtocol::Subc,
16289 overlap: Default::default(),
16290 carry_containment: false,
16291 };
16292 let mut snapshot = SupervisorSnapshot::new(state, enabled);
16293 seed_crash_restarts(&mut snapshot, restart_count);
16294 let (commands, _rx) = mpsc::channel(4);
16297 let configured_enabled = snapshot.enabled;
16298 SupervisedModule {
16299 inner: Arc::new(SupervisedModuleInner {
16300 module_id: spec.module_id.clone(),
16301 registry,
16302 snapshot: Arc::new(Mutex::new(snapshot)),
16303 configuration: Arc::new(Mutex::new(SupervisedConfiguration {
16304 spec,
16305 enabled: configured_enabled,
16306 health: runtime.health,
16307 register_timeout: runtime.register_timeout,
16308 restart_policy: runtime.restart_policy,
16309 })),
16310 stderr_ring: runtime.stderr_ring,
16311 terminal_ring: runtime.terminal_ring,
16312 commands,
16313 monitor: Mutex::new(None),
16314 effective_drain_timeout: runtime.effective_drain_timeout,
16315 provenance_probe: supervisor.provenance_probe.clone(),
16316 retained: runtime.child_roster.retained_view(),
16317 }),
16318 }
16319 }
16320
16321 #[cfg(target_os = "linux")]
16322 #[tokio::test]
16323 async fn no_cgroup_placement_does_not_block_fake_aft_stub_spawn() {
16324 let supervisor =
16325 Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default())
16326 .with_cgroup_placement(None);
16327 let result = supervisor.spawn(ModuleSpec {
16328 script: None,
16329 module_id: "no-cgroup-placement".to_string(),
16330 program: fake_aft_stub_path(),
16331 args: Vec::new(),
16332 env: Vec::new(),
16333 reserved: false,
16334 reserved_prefixes: Vec::new(),
16335 protocol: ModuleProtocol::Subc,
16336 overlap: Default::default(),
16337 carry_containment: false,
16338 });
16339
16340 assert!(
16341 result.is_ok(),
16342 "no delegation must not turn an otherwise valid spawn into a failure: {result:?}"
16343 );
16344 }
16345
16346 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
16347 async fn undecided_snapshot_uses_shared_restart_predicate() {
16348 assert!(module_with_recovery_snapshot(ModuleState::Running, true, 2)
16349 .will_recover_after_connection_loss()
16350 .unwrap());
16351 assert!(
16352 !module_with_recovery_snapshot(ModuleState::Running, true, 3)
16353 .will_recover_after_connection_loss()
16354 .unwrap()
16355 );
16356 }
16357
16358 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
16359 async fn restarting_snapshot_at_exhausted_budget_is_non_terminal() {
16360 assert!(
16361 module_with_recovery_snapshot(ModuleState::Restarting, true, 3)
16362 .will_recover_after_connection_loss()
16363 .unwrap()
16364 );
16365 }
16366
16367 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
16368 async fn terminal_phase_snapshots_are_terminal_before_budget_exhaustion() {
16369 assert!(!module_with_recovery_snapshot(ModuleState::Failed, true, 0)
16370 .will_recover_after_connection_loss()
16371 .unwrap());
16372 assert!(
16373 !module_with_recovery_snapshot(ModuleState::Disabled, true, 0)
16374 .will_recover_after_connection_loss()
16375 .unwrap()
16376 );
16377 }
16378
16379 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
16380 async fn warming_snapshot_is_limited_to_startup_phases() {
16381 for state in [
16382 ModuleState::Starting,
16383 ModuleState::Running,
16384 ModuleState::Restarting,
16385 ] {
16386 assert!(
16387 module_with_recovery_snapshot(state, true, 0)
16388 .is_warming()
16389 .unwrap(),
16390 "{state:?} should be warming"
16391 );
16392 }
16393 for state in [
16394 ModuleState::Unresponsive,
16395 ModuleState::Draining,
16396 ModuleState::Stopped,
16397 ModuleState::Failed,
16398 ModuleState::Disabled,
16399 ] {
16400 assert!(
16401 !module_with_recovery_snapshot(state, true, 0)
16402 .is_warming()
16403 .unwrap(),
16404 "{state:?} should not be warming"
16405 );
16406 }
16407 }
16408
16409 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
16410 async fn terminal_history_survives_respawn_and_keeps_both_crashes_in_order() {
16411 let registry = Arc::new(Registry::default());
16412 let supervisor =
16413 Supervisor::new_for_test(Arc::clone(®istry), RestartPolicy::new(1, Duration::ZERO));
16414 let module = supervisor
16415 .spawn(ModuleSpec {
16416 script: None,
16417 module_id: "terminal-history".to_string(),
16418 program: fake_aft_stub_path(),
16419 args: Vec::new(),
16420 env: vec![("FAKE_AFT_EXIT_CODE".to_string(), "23".to_string())],
16421 reserved: false,
16422 reserved_prefixes: Vec::new(),
16423 protocol: ModuleProtocol::Subc,
16424 overlap: Default::default(),
16425 carry_containment: false,
16426 })
16427 .unwrap();
16428
16429 let deadline = Instant::now() + Duration::from_secs(5);
16430 loop {
16431 let history = module.terminal_history();
16432 if history.entries.len() == 2 {
16433 assert_eq!(module.status().unwrap().state, ModuleState::Failed);
16434 assert_eq!(history.dropped, 0);
16435 assert_eq!(
16436 history
16437 .entries
16438 .iter()
16439 .map(|entry| entry.exit_code)
16440 .collect::<Vec<_>>(),
16441 vec![Some(23), Some(23)]
16442 );
16443 assert!(history.entries[0].at_ms <= history.entries[1].at_ms);
16444 return;
16445 }
16446 assert!(
16447 Instant::now() < deadline,
16448 "module did not retain two terminal exits: {history:?}"
16449 );
16450 sleep(Duration::from_millis(10)).await;
16451 }
16452 }
16453
16454 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
16458 async fn disable_during_crash_backoff_cancels_pending_respawn() {
16459 let backoff = Duration::from_secs(2);
16460 let supervisor = Supervisor::new_for_test(
16461 Arc::new(Registry::default()),
16462 RestartPolicy::new(10, backoff),
16463 );
16464 let module = supervisor
16465 .spawn(ModuleSpec {
16466 script: None,
16467 module_id: "disable-during-backoff".to_string(),
16468 program: fake_aft_stub_path(),
16469 args: Vec::new(),
16470 env: vec![("FAKE_AFT_EXIT_CODE".to_string(), "23".to_string())],
16471 reserved: false,
16472 reserved_prefixes: Vec::new(),
16473 protocol: ModuleProtocol::Subc,
16474 overlap: Default::default(),
16475 carry_containment: false,
16476 })
16477 .unwrap();
16478
16479 let deadline = Instant::now() + Duration::from_secs(5);
16481 loop {
16482 if module.status().unwrap().state == ModuleState::Restarting {
16483 break;
16484 }
16485 assert!(
16486 Instant::now() < deadline,
16487 "module never entered the crash backoff"
16488 );
16489 sleep(Duration::from_millis(10)).await;
16490 }
16491
16492 let started = Instant::now();
16493 module.set_enabled(false).await.unwrap();
16494 let waited = started.elapsed();
16495
16496 assert!(
16497 waited < backoff / 2,
16498 "disable waited {waited:?} behind the {backoff:?} crash backoff; the operator command must preempt the pending respawn"
16499 );
16500 assert_eq!(module.status().unwrap().state, ModuleState::Disabled);
16501
16502 sleep(backoff + Duration::from_millis(500)).await;
16504 let status = module.status().unwrap();
16505 assert_eq!(status.state, ModuleState::Disabled);
16506 assert_eq!(
16507 status.spawn_generation, 1,
16508 "module respawned after the operator disabled it"
16509 );
16510 }
16511
16512 #[cfg(unix)]
16515 fn protocol_none_sigterm_exits_clean_spec(
16516 module_id: &str,
16517 dir: &std::path::Path,
16518 ) -> (ModuleSpec, PathBuf, PathBuf) {
16519 let ready = dir.join("ready");
16520 let marker = dir.join("sigterm");
16521 let spec = ModuleSpec {
16522 script: None,
16523 module_id: module_id.to_string(),
16524 program: fake_aft_stub_path(),
16525 args: Vec::new(),
16526 env: vec![
16527 ("FAKE_AFT_NEVER_CONNECT".to_string(), "1".to_string()),
16528 (
16529 "FAKE_AFT_SIGTERM_MARKER_PATH".to_string(),
16530 marker.display().to_string(),
16531 ),
16532 (
16533 "FAKE_AFT_NEVER_CONNECT_READY_PATH".to_string(),
16534 ready.display().to_string(),
16535 ),
16536 ],
16537 reserved: false,
16538 reserved_prefixes: Vec::new(),
16539 protocol: ModuleProtocol::None,
16540 overlap: Default::default(),
16541 carry_containment: false,
16542 };
16543 (spec, ready, marker)
16544 }
16545
16546 #[cfg(unix)]
16550 async fn wait_for_file(path: &std::path::Path) {
16551 let deadline = Instant::now() + Duration::from_secs(10);
16552 while !path.exists() {
16553 assert!(
16554 Instant::now() < deadline,
16555 "{} never appeared",
16556 path.display()
16557 );
16558 sleep(Duration::from_millis(10)).await;
16559 }
16560 }
16561
16562 #[cfg(unix)]
16566 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
16567 async fn protocol_none_unrequested_clean_exit_restarts_as_a_crash() {
16568 let dir = cortexkit_test_support::ScratchDir::new("none-unrequested-clean-exit");
16569 let (spec, ready, marker) =
16570 protocol_none_sigterm_exits_clean_spec("none-unrequested-clean-exit", dir.path());
16571 let supervisor = Supervisor::new_for_test(
16572 Arc::new(Registry::default()),
16573 RestartPolicy::new(3, Duration::ZERO),
16574 );
16575 let module = supervisor.spawn(spec).unwrap();
16576 wait_for_file(&ready).await;
16577 let deadline = Instant::now() + Duration::from_secs(10);
16581 let first_pid = loop {
16582 let status = module.status().unwrap();
16583 if status.state == ModuleState::Running {
16584 if let Some(pid) = status.pid {
16585 break pid;
16586 }
16587 }
16588 assert!(
16589 Instant::now() < deadline,
16590 "a running module must report its pid after exec confirmation: {status:?}"
16591 );
16592 sleep(Duration::from_millis(10)).await;
16593 };
16594
16595 rustix::process::kill_process(
16596 rustix::process::Pid::from_raw(i32::try_from(first_pid).unwrap()).unwrap(),
16597 rustix::process::Signal::TERM,
16598 )
16599 .unwrap();
16600
16601 let deadline = Instant::now() + Duration::from_secs(10);
16602 let respawned = loop {
16603 let status = module.status().unwrap();
16604 if status.state == ModuleState::Running
16605 && status.pid.is_some_and(|pid| pid != first_pid)
16606 {
16607 break status;
16608 }
16609 assert!(
16610 Instant::now() < deadline,
16611 "protocol-none module was not respawned after an unrequested clean exit: {status:?}"
16612 );
16613 sleep(Duration::from_millis(10)).await;
16614 };
16615 assert_eq!(respawned.spawn_generation, 2);
16616 assert!(
16617 marker.exists(),
16618 "the child must have exited through its SIGTERM handler (exit 0), or this proves nothing about clean exits"
16619 );
16620
16621 let history = module.terminal_history();
16622 assert_eq!(history.entries.len(), 1, "{history:?}");
16623 let entry = &history.entries[0];
16624 assert_eq!(entry.exit_code, Some(0));
16625 assert_eq!(entry.exit_kind, subc_control::TerminalExitKind::Clean);
16626 assert_eq!(entry.disposition, TerminalDisposition::Restarting);
16627
16628 module.stop().await.unwrap();
16629 }
16630
16631 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
16635 async fn protocol_none_repeated_clean_exits_exhaust_the_restart_budget() {
16636 let supervisor = Supervisor::new_for_test(
16637 Arc::new(Registry::default()),
16638 RestartPolicy::new(1, Duration::ZERO),
16639 );
16640 let module = supervisor
16641 .spawn(ModuleSpec {
16642 script: None,
16643 module_id: "none-clean-exit-budget".to_string(),
16644 program: fake_aft_stub_path(),
16645 args: Vec::new(),
16646 env: vec![("FAKE_AFT_EXIT_CODE".to_string(), "0".to_string())],
16647 reserved: false,
16648 reserved_prefixes: Vec::new(),
16649 protocol: ModuleProtocol::None,
16650 overlap: Default::default(),
16651 carry_containment: false,
16652 })
16653 .unwrap();
16654
16655 let deadline = Instant::now() + Duration::from_secs(10);
16658 loop {
16659 let status = module.status().unwrap();
16660 if status.state == ModuleState::Failed {
16661 break;
16662 }
16663 assert!(
16664 Instant::now() < deadline,
16665 "module never exhausted its budget: {status:?} {:?}",
16666 module.terminal_history()
16667 );
16668 sleep(Duration::from_millis(10)).await;
16669 }
16670 let history = module.terminal_history();
16671 assert_eq!(
16672 history
16673 .entries
16674 .iter()
16675 .map(|entry| (entry.exit_code, entry.disposition.clone()))
16676 .collect::<Vec<_>>(),
16677 vec![
16678 (Some(0), TerminalDisposition::Restarting),
16679 (Some(0), TerminalDisposition::Failed),
16680 ]
16681 );
16682 let detail = history.entries[1]
16683 .disposition_detail
16684 .as_deref()
16685 .expect("a budget failure names the budget");
16686 assert!(detail.contains("max_restarts=1"), "{detail}");
16687 assert_eq!(module.status().unwrap().spawn_generation, 2);
16688 }
16689
16690 #[test]
16691 fn restart_budget_failure_is_published_after_its_terminal_record() {
16692 use std::io::{Read, Write};
16693 let supervisor = Supervisor::new_for_test(
16694 Arc::new(Registry::default()),
16695 RestartPolicy::new(0, Duration::ZERO),
16696 );
16697 let runtime = supervisor.runtime_config();
16698 let spec = ModuleSpec {
16699 script: None,
16700 protocol: ModuleProtocol::None,
16701 ..windowed_crash_spec("budget-publication-order")
16702 };
16703 let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::new(
16704 ModuleState::Running,
16705 true,
16706 )));
16707 lock_snapshot(&snapshot).unwrap().process_alive = true;
16708 let reaped_snapshot = snapshot.clone();
16709 let mut barrier = terminal_record_barrier(&runtime.spawn_events);
16710 let history = runtime.terminal_ring.clone();
16711 let reap = std::thread::spawn(move || {
16712 tokio::runtime::Builder::new_current_thread()
16713 .enable_all()
16714 .build()
16715 .unwrap()
16716 .block_on(on_child_exit(
16717 &spec,
16718 runtime.restart_policy,
16719 &supervisor.registry,
16720 &reaped_snapshot,
16721 &runtime.terminal_ring,
16722 &runtime.spawn_events,
16723 &runtime.child_roster,
16724 ExitReport {
16725 kind: ExitKind::Clean,
16726 code: Some(0),
16727 signal: None,
16728 at_ms: 1,
16729 },
16730 true,
16731 ))
16732 });
16733 let mut reached = [0];
16734 barrier.read_exact(&mut reached).unwrap();
16735 assert_eq!(&reached, b"R");
16736 let (before_state, before_alive) = {
16737 let state = lock_snapshot(&snapshot).unwrap();
16738 (state.state, state.process_alive)
16739 };
16740 let before_history = history.lock().unwrap().snapshot();
16741 barrier.write_all(b"X").unwrap();
16742 assert!(matches!(reap.join().unwrap(), NextAction::Stop { .. }));
16743 assert!(before_history.entries.is_empty());
16744 assert_ne!(
16745 before_state,
16746 ModuleState::Failed,
16747 "Failed was visible before its terminal record could be written"
16748 );
16749 assert!(before_alive);
16750 assert_eq!(lock_snapshot(&snapshot).unwrap().state, ModuleState::Failed);
16751 let history = history.lock().unwrap().snapshot();
16752 assert_eq!(history.entries.len(), 1);
16753 assert_eq!(history.entries[0].exit_code, Some(0));
16754 assert_eq!(history.entries[0].disposition, TerminalDisposition::Failed);
16755 }
16756
16757 #[test]
16758 fn crash_restart_at_budget_limit_remains_recoverable_while_recording() {
16759 use std::io::{Read, Write};
16760
16761 let supervisor = Supervisor::new_for_test(
16762 Arc::new(Registry::default()),
16763 RestartPolicy::new(3, Duration::ZERO),
16764 );
16765 let mut runtime = supervisor.runtime_config();
16766 let module = module_with_recovery_snapshot(ModuleState::Running, true, 2);
16767 let snapshot = module.inner.snapshot.clone();
16768 lock_snapshot(&snapshot).unwrap().process_alive = true;
16769 runtime.terminal_ring = module.inner.terminal_ring.clone();
16770 let spec = windowed_crash_spec(module.module_id());
16771 let mut barrier = terminal_record_barrier(&runtime.spawn_events);
16772 let reaped_snapshot = snapshot.clone();
16773 let reap = std::thread::spawn(move || {
16774 tokio::runtime::Builder::new_current_thread()
16775 .enable_all()
16776 .build()
16777 .unwrap()
16778 .block_on(on_child_exit(
16779 &spec,
16780 runtime.restart_policy,
16781 &supervisor.registry,
16782 &reaped_snapshot,
16783 &runtime.terminal_ring,
16784 &runtime.spawn_events,
16785 &runtime.child_roster,
16786 ExitReport {
16787 kind: ExitKind::Crash,
16788 code: Some(2),
16789 signal: None,
16790 at_ms: 1,
16791 },
16792 true,
16793 ))
16794 });
16795
16796 let mut reached = [0];
16800 barrier.read_exact(&mut reached).unwrap();
16801 assert_eq!(&reached, b"R");
16802 let before_status = module.status().unwrap();
16803 let before_history = module.terminal_history();
16804 let before_recovers = module.will_recover_after_connection_loss().unwrap();
16805 barrier.write_all(b"X").unwrap();
16806 assert!(matches!(reap.join().unwrap(), NextAction::Restart { .. }));
16807
16808 assert_eq!(before_status.state, ModuleState::Running);
16809 assert!(before_status.process_alive);
16810 assert_eq!(before_status.restart_count, 3);
16811 assert!(before_history.entries.is_empty());
16812 assert!(
16813 before_recovers,
16814 "connection cleanup would report a terminal close for a scheduled crash restart"
16815 );
16816 let restarting = module.status().unwrap();
16817 assert_eq!(restarting.state, ModuleState::Restarting);
16818 assert_eq!(restarting.restart_count, 3);
16819 assert!(module.will_recover_after_connection_loss().unwrap());
16820 let history = module.terminal_history();
16821 assert_eq!(history.entries.len(), 1);
16822 assert_eq!(
16823 history.entries[0].disposition,
16824 TerminalDisposition::Restarting
16825 );
16826
16827 update_snapshot(&snapshot, Some(module.module_id()), |state| {
16830 state.state = ModuleState::Running;
16831 state.respawn_pending = false;
16832 })
16833 .unwrap();
16834 assert!(!module.will_recover_after_connection_loss().unwrap());
16835 update_snapshot(&snapshot, Some(module.module_id()), |state| {
16836 state.enabled = false;
16837 state.respawn_pending = true;
16838 })
16839 .unwrap();
16840 assert!(!module.will_recover_after_connection_loss().unwrap());
16841 }
16842
16843 fn terminal_record_barrier(events: &super::SpawnEventFeed) -> std::net::TcpStream {
16844 use std::net::{TcpListener, TcpStream};
16845 let listener = TcpListener::bind("127.0.0.1:0").unwrap();
16846 let writer = TcpStream::connect(listener.local_addr().unwrap()).unwrap();
16847 let (barrier, _) = listener.accept().unwrap();
16848 for stream in [&writer, &barrier] {
16849 stream
16850 .set_read_timeout(Some(Duration::from_secs(10)))
16851 .unwrap();
16852 stream
16853 .set_write_timeout(Some(Duration::from_secs(10)))
16854 .unwrap();
16855 }
16856 *events.1.lock().unwrap() = Some(writer);
16857 barrier
16858 }
16859
16860 pub(super) fn before_terminal_record(events: &super::SpawnEventFeed) {
16861 use std::io::{Read, Write};
16862 let barrier = events.1.lock().unwrap().take();
16863 if let Some(mut barrier) = barrier {
16864 barrier.write_all(b"R").unwrap();
16865 let mut release = [0];
16866 barrier.read_exact(&mut release).unwrap();
16867 assert_eq!(&release, b"X");
16868 }
16869 }
16870
16871 #[test]
16872 fn health_restart_budget_failure_is_published_after_its_terminal_record() {
16873 use std::io::{Read, Write};
16874
16875 let supervisor = Supervisor::new_for_test(
16876 Arc::new(Registry::default()),
16877 RestartPolicy::new(0, Duration::ZERO),
16878 );
16879 let mut runtime = supervisor.runtime_config();
16880 let module = module_with_recovery_snapshot(ModuleState::Running, true, 0);
16881 let snapshot = module.inner.snapshot.clone();
16882 runtime.terminal_ring = module.inner.terminal_ring.clone();
16883 let spec = ModuleSpec {
16884 script: None,
16885 module_id: module.module_id().into(),
16886 program: fake_aft_stub_path(),
16887 protocol: ModuleProtocol::None,
16888 env: vec![("FAKE_AFT_EXIT_CODE".into(), "17".into())],
16889 ..windowed_crash_spec(module.module_id())
16890 };
16891 let mut barrier = terminal_record_barrier(&runtime.spawn_events);
16892 let reap = std::thread::spawn(move || {
16893 tokio::runtime::Builder::new_current_thread()
16894 .enable_all()
16895 .build()
16896 .unwrap()
16897 .block_on(async {
16898 let mut child =
16899 Some(spawn_and_mark_running(&spec, &runtime, &snapshot).unwrap());
16900 health_restart_child(
16901 &spec,
16902 &runtime,
16903 &supervisor.registry,
16904 &supervisor.process_liveness,
16905 &snapshot,
16906 &mut child,
16907 SupervisorHealthStatus::Failing,
16908 Some("health probe did not answer"),
16909 1,
16910 )
16911 .await
16912 .unwrap();
16913 assert!(child.is_none());
16914 });
16915 });
16916 let mut reached = [0];
16921 barrier.read_exact(&mut reached).unwrap();
16922 assert_eq!(&reached, b"R");
16923 let before_status = module.status().unwrap();
16924 let before_history = module.terminal_history();
16925 barrier.write_all(b"X").unwrap();
16926 reap.join().unwrap();
16927 assert!(before_history.entries.is_empty());
16928 assert_ne!(
16929 before_status.state,
16930 ModuleState::Failed,
16931 "Failed was visible before its terminal record could be written"
16932 );
16933 assert!(
16934 before_status.process_alive,
16935 "process death was visible before its terminal record could be written"
16936 );
16937 let failed = module.status().unwrap();
16938 assert_eq!(failed.state, ModuleState::Failed);
16939 assert!(!failed.process_alive);
16940 let history = module.terminal_history();
16941 assert_eq!(history.entries.len(), 1);
16942 assert_eq!(history.entries[0].disposition, TerminalDisposition::Failed);
16943 assert_eq!(
16944 history.entries[0].disposition_detail.as_deref(),
16945 Some("crash budget exhausted: max_restarts=0 within window_secs=600")
16946 );
16947 }
16948
16949 #[test]
16950 fn observed_exits_publish_process_death_after_terminal_history() {
16951 use std::io::{Read, Write};
16952 for (kind, enabled, shutdown, wait_error, expected_state) in [
16953 (ExitKind::Clean, true, false, false, ModuleState::Stopped),
16954 (ExitKind::Crash, true, false, false, ModuleState::Restarting),
16955 (ExitKind::Crash, false, false, false, ModuleState::Disabled),
16956 (
16957 ExitKind::DeliberateSeverance,
16958 true,
16959 false,
16960 false,
16961 ModuleState::Restarting,
16962 ),
16963 #[cfg(unix)]
16964 (ExitKind::Crash, true, true, false, ModuleState::Stopped),
16965 (ExitKind::Crash, true, false, true, ModuleState::Failed),
16966 ] {
16967 let supervisor = Supervisor::new_for_test(
16968 Arc::new(Registry::default()),
16969 RestartPolicy::new(1, Duration::ZERO),
16970 );
16971 let runtime = supervisor.runtime_config();
16972 #[cfg(unix)]
16973 if shutdown {
16974 runtime.child_roster.close();
16975 }
16976 #[cfg(not(unix))]
16977 let _ = shutdown;
16978 let spec = windowed_crash_spec("exit-publication-order");
16979 let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::new(
16980 ModuleState::Running,
16981 enabled,
16982 )));
16983 lock_snapshot(&snapshot).unwrap().process_alive = true;
16984 let reaped_snapshot = snapshot.clone();
16985 let history = runtime.terminal_ring.clone();
16986 let mut barrier = terminal_record_barrier(&runtime.spawn_events);
16987 let reap = std::thread::spawn(move || {
16988 if wait_error {
16989 super::fail_after_wait_error(
16990 &spec.module_id,
16991 &reaped_snapshot,
16992 &runtime.terminal_ring,
16993 &runtime.spawn_events,
16994 "wait failed".into(),
16995 &runtime.failure_context(true),
16996 );
16997 } else {
16998 tokio::runtime::Builder::new_current_thread()
16999 .enable_all()
17000 .build()
17001 .unwrap()
17002 .block_on(on_child_exit(
17003 &spec,
17004 runtime.restart_policy,
17005 &supervisor.registry,
17006 &reaped_snapshot,
17007 &runtime.terminal_ring,
17008 &runtime.spawn_events,
17009 &runtime.child_roster,
17010 ExitReport {
17011 kind,
17012 code: Some(0),
17013 signal: None,
17014 at_ms: 1,
17015 },
17016 true,
17017 ));
17018 }
17019 });
17020 let mut reached = [0];
17021 barrier.read_exact(&mut reached).unwrap();
17022 assert_eq!(&reached, b"R");
17023 let before_alive = lock_snapshot(&snapshot).unwrap().process_alive;
17024 let before_history = history.lock().unwrap().snapshot();
17025 barrier.write_all(b"X").unwrap();
17026 reap.join().unwrap();
17027 assert!(
17028 before_alive,
17029 "{kind:?}: death was published before recording"
17030 );
17031 assert!(before_history.entries.is_empty());
17032 assert_eq!(lock_snapshot(&snapshot).unwrap().state, expected_state);
17033 assert!(!lock_snapshot(&snapshot).unwrap().process_alive);
17034 assert_eq!(history.lock().unwrap().snapshot().entries.len(), 1);
17035 }
17036 }
17037
17038 #[tokio::test]
17039 async fn health_restart_budget_failure_remains_failed_when_journal_append_fails() {
17040 let dir = cortexkit_test_support::ScratchDir::new("health-budget-journal-failure");
17041 let path = dir.join("terminals.jsonl");
17042 std::fs::create_dir(&path).unwrap();
17043 let supervisor = Supervisor::new_for_test(
17044 Arc::new(Registry::default()),
17045 RestartPolicy::new(0, Duration::ZERO),
17046 )
17047 .with_terminal_journal(path, "health-budget-journal-failure".into());
17048 let runtime = supervisor.runtime_config();
17049 let spec = ModuleSpec {
17050 script: None,
17051 program: fake_aft_stub_path(),
17052 protocol: ModuleProtocol::None,
17053 env: vec![("FAKE_AFT_EXIT_CODE".into(), "17".into())],
17054 ..windowed_crash_spec("health-budget-journal-failure")
17055 };
17056 let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::starting()));
17057 let mut child = Some(spawn_and_mark_running(&spec, &runtime, &snapshot).unwrap());
17058 health_restart_child(
17059 &spec,
17060 &runtime,
17061 &supervisor.registry,
17062 &supervisor.process_liveness,
17063 &snapshot,
17064 &mut child,
17065 SupervisorHealthStatus::Failing,
17066 None,
17067 1,
17068 )
17069 .await
17070 .unwrap();
17071 assert_eq!(lock_snapshot(&snapshot).unwrap().state, ModuleState::Failed);
17072 assert!(!lock_snapshot(&snapshot).unwrap().process_alive);
17073 let history = runtime
17074 .terminal_ring
17075 .lock()
17076 .unwrap()
17077 .durable_history(&spec.module_id);
17078 assert!(history.journal_write_failures > 0);
17079 assert_eq!(history.entries.len(), 1);
17080 assert_eq!(history.entries[0].disposition, TerminalDisposition::Failed);
17081 }
17082
17083 #[test]
17084 fn health_restart_budget_failure_remains_failed_when_exit_recording_panics() {
17085 let supervisor = Supervisor::new_for_test(
17086 Arc::new(Registry::default()),
17087 RestartPolicy::new(0, Duration::ZERO),
17088 );
17089 let runtime = supervisor.runtime_config();
17090 let spec = ModuleSpec {
17091 script: None,
17092 program: fake_aft_stub_path(),
17093 protocol: ModuleProtocol::None,
17094 env: vec![("FAKE_AFT_EXIT_CODE".into(), "17".into())],
17095 ..windowed_crash_spec("health-budget-recording-panic")
17096 };
17097 let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::starting()));
17098 let result = std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| {
17099 tokio::runtime::Builder::new_current_thread()
17100 .enable_all()
17101 .build()
17102 .unwrap()
17103 .block_on(async {
17104 let mut child =
17105 Some(spawn_and_mark_running(&spec, &runtime, &snapshot).unwrap());
17106 runtime.spawn_events.0.lock().unwrap().seq = u64::MAX;
17107 health_restart_child(
17108 &spec,
17109 &runtime,
17110 &supervisor.registry,
17111 &supervisor.process_liveness,
17112 &snapshot,
17113 &mut child,
17114 SupervisorHealthStatus::Failing,
17115 None,
17116 1,
17117 )
17118 .await
17119 .unwrap();
17120 });
17121 }));
17122 let panic = result.expect_err("recording must still unwind");
17123 assert_eq!(
17124 panic.downcast_ref::<String>().map(String::as_str),
17125 Some("spawn event sequence exhausted")
17126 );
17127 let state = lock_snapshot(&snapshot).unwrap();
17128 assert_eq!(state.state, ModuleState::Failed);
17129 assert!(!state.process_alive);
17130 assert!(state.last_exit.is_some());
17131 }
17132
17133 #[tokio::test]
17134 async fn restart_budget_failure_remains_failed_when_journal_append_fails() {
17135 let dir = cortexkit_test_support::ScratchDir::new("budget-journal-failure");
17136 let path = dir.join("terminals.jsonl");
17137 std::fs::create_dir(&path).unwrap();
17138 let supervisor = Supervisor::new_for_test(
17139 Arc::new(Registry::default()),
17140 RestartPolicy::new(0, Duration::ZERO),
17141 )
17142 .with_terminal_journal(path, "budget-journal-failure".into());
17143 let runtime = supervisor.runtime_config();
17144 let spec = windowed_crash_spec("budget-journal-failure");
17145 let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::starting()));
17146 assert!(matches!(
17147 on_child_exit(
17148 &spec,
17149 runtime.restart_policy,
17150 &supervisor.registry,
17151 &snapshot,
17152 &runtime.terminal_ring,
17153 &runtime.spawn_events,
17154 &runtime.child_roster,
17155 crash_exit_report(1),
17156 true,
17157 )
17158 .await,
17159 NextAction::Stop { .. }
17160 ));
17161 assert_eq!(lock_snapshot(&snapshot).unwrap().state, ModuleState::Failed);
17162 let history = runtime
17163 .terminal_ring
17164 .lock()
17165 .unwrap()
17166 .durable_history(&spec.module_id);
17167 assert!(history.journal_write_failures > 0);
17168 assert_eq!(history.entries.len(), 1);
17169 assert_eq!(history.entries[0].disposition, TerminalDisposition::Failed);
17170 }
17171
17172 #[test]
17173 fn restart_budget_failure_remains_failed_when_exit_recording_panics() {
17174 let supervisor = Supervisor::new_for_test(
17175 Arc::new(Registry::default()),
17176 RestartPolicy::new(0, Duration::ZERO),
17177 );
17178 let runtime = supervisor.runtime_config();
17179 let spec = windowed_crash_spec("budget-recording-panic");
17180 let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::starting()));
17181 runtime
17182 .spawn_events
17183 .emit_spawned(&spec.module_id, None, 1, 1);
17184 runtime.spawn_events.0.lock().unwrap().seq = u64::MAX;
17187 let result = std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| {
17188 tokio::runtime::Builder::new_current_thread()
17189 .enable_all()
17190 .build()
17191 .unwrap()
17192 .block_on(on_child_exit(
17193 &spec,
17194 runtime.restart_policy,
17195 &supervisor.registry,
17196 &snapshot,
17197 &runtime.terminal_ring,
17198 &runtime.spawn_events,
17199 &runtime.child_roster,
17200 crash_exit_report(1),
17201 true,
17202 ))
17203 }));
17204 let panic = result.err().expect("recording must still unwind");
17205 assert_eq!(
17206 panic.downcast_ref::<String>().map(String::as_str),
17207 Some("spawn event sequence exhausted")
17208 );
17209 assert_eq!(lock_snapshot(&snapshot).unwrap().state, ModuleState::Failed);
17210 }
17211
17212 #[cfg(unix)]
17215 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
17216 async fn protocol_none_requested_stop_and_disable_do_not_respawn() {
17217 for disable in [false, true] {
17218 let label = if disable {
17219 "none-requested-disable"
17220 } else {
17221 "none-requested-stop"
17222 };
17223 let dir = cortexkit_test_support::ScratchDir::new(label);
17224 let (spec, ready, marker) = protocol_none_sigterm_exits_clean_spec(label, dir.path());
17225 let supervisor = Supervisor::new_for_test(
17226 Arc::new(Registry::default()),
17227 RestartPolicy::new(3, Duration::ZERO),
17228 );
17229 let module = supervisor.spawn(spec).unwrap();
17230 wait_for_file(&ready).await;
17231
17232 if disable {
17233 module.set_enabled(false).await.unwrap();
17234 } else {
17235 module.stop().await.unwrap();
17236 }
17237 assert!(
17238 marker.exists(),
17239 "{label}: the child must have left through its SIGTERM handler with exit 0"
17240 );
17241
17242 sleep(Duration::from_millis(500)).await;
17245 let status = module.status().unwrap();
17246 let expected = if disable {
17247 ModuleState::Disabled
17248 } else {
17249 ModuleState::Stopped
17250 };
17251 assert_eq!(status.state, expected, "{label}");
17252 assert_eq!(
17253 status.spawn_generation, 1,
17254 "{label}: respawned after a requested stop"
17255 );
17256 let history = module.terminal_history();
17257 assert_eq!(history.entries.len(), 1, "{label}: {history:?}");
17258 assert_eq!(history.entries[0].exit_code, Some(0), "{label}");
17259 assert_ne!(
17260 history.entries[0].disposition,
17261 TerminalDisposition::Restarting,
17262 "{label}"
17263 );
17264 }
17265 }
17266
17267 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
17270 async fn subc_wire_clean_exit_is_still_a_stop() {
17271 let supervisor = Supervisor::new_for_test(
17272 Arc::new(Registry::default()),
17273 RestartPolicy::new(3, Duration::ZERO),
17274 );
17275 let module = supervisor
17276 .spawn(ModuleSpec {
17277 script: None,
17278 module_id: "wire-clean-exit".to_string(),
17279 program: fake_aft_stub_path(),
17280 args: Vec::new(),
17281 env: vec![("FAKE_AFT_EXIT_CODE".to_string(), "0".to_string())],
17282 reserved: false,
17283 reserved_prefixes: Vec::new(),
17284 protocol: ModuleProtocol::Subc,
17285 overlap: Default::default(),
17286 carry_containment: false,
17287 })
17288 .unwrap();
17289
17290 let deadline = Instant::now() + Duration::from_secs(10);
17291 while module.terminal_history().entries.is_empty() {
17292 assert!(Instant::now() < deadline, "module never exited");
17293 sleep(Duration::from_millis(10)).await;
17294 }
17295 sleep(Duration::from_millis(500)).await;
17297 let status = module.status().unwrap();
17298 assert_eq!(status.state, ModuleState::Stopped);
17299 assert_eq!(status.spawn_generation, 1);
17300 let history = module.terminal_history();
17301 assert_eq!(history.entries.len(), 1, "{history:?}");
17302 assert_eq!(history.entries[0].exit_code, Some(0));
17303 assert_eq!(history.entries[0].disposition, TerminalDisposition::Stopped);
17304 }
17305
17306 #[cfg(unix)]
17307 #[tokio::test]
17308 async fn reload_exit_keeps_roster_until_terminal_is_recorded_during_shutdown() {
17309 let dir = cortexkit_test_support::ScratchDir::new("reload-roster-terminal-order");
17310 let record = dir.join("live-children.json");
17311 let supervisor = Supervisor::new_for_test(
17312 Arc::new(Registry::default()),
17313 RestartPolicy::new(0, Duration::ZERO),
17314 );
17315 let mut runtime = supervisor.runtime_config();
17316 runtime.child_roster.record_to(record.clone());
17317 let gate = Arc::new(super::ReloadExitRecordGate::default());
17318 runtime.test_reload_exit_record_gate = Some(Arc::clone(&gate));
17319 let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::starting()));
17320 let spec = ModuleSpec {
17321 script: None,
17322 module_id: "reload-exit-roster".into(),
17323 program: fake_aft_stub_path(),
17324 args: Vec::new(),
17325 env: vec![("FAKE_AFT_EXIT_CODE".into(), "23".into())],
17326 reserved: false,
17327 reserved_prefixes: Vec::new(),
17328 protocol: ModuleProtocol::Subc,
17329 overlap: Default::default(),
17330 carry_containment: false,
17331 };
17332 let mut child = None;
17333 let reload = super::finish_reload_child(
17334 &spec,
17335 &runtime,
17336 &supervisor.registry,
17337 &supervisor.process_liveness,
17338 &snapshot,
17339 &mut child,
17340 );
17341 tokio::pin!(reload);
17342 tokio::select! {
17343 result = &mut reload => panic!("reload missed the before-record gate: {result:?}"),
17344 _ = gate.reached.notified() => {}
17345 }
17346 assert!(runtime
17347 .terminal_ring
17348 .lock()
17349 .unwrap()
17350 .snapshot()
17351 .entries
17352 .is_empty());
17353 assert_eq!(
17354 crate::live_children::read_record(&record).unwrap().len(),
17355 1,
17356 "shutdown must still wait for the reaped child until its terminal record exists"
17357 );
17358 runtime.child_roster.close();
17359 gate.resume.notify_one();
17360 assert!(reload.await.is_err());
17361 assert!(crate::live_children::read_record(&record)
17362 .unwrap()
17363 .is_empty());
17364 let history = runtime.terminal_ring.lock().unwrap().snapshot();
17365 assert_eq!(history.entries.len(), 1);
17366 assert_eq!(
17367 history.entries[0].disposition,
17368 TerminalDisposition::DaemonShutdown
17369 );
17370 }
17371
17372 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
17376 async fn every_restart_increment_path_advances_lifetime_count() {
17377 let supervisor = Supervisor::new_for_test(
17378 Arc::new(Registry::default()),
17379 RestartPolicy::new(1, Duration::ZERO),
17380 );
17381 let runtime = supervisor.runtime_config();
17382 let spec = ModuleSpec {
17383 script: None,
17384 module_id: "lifetime-increment-path".to_string(),
17385 program: PathBuf::from("/unused/lifetime-increment-path"),
17386 args: Vec::new(),
17387 env: Vec::new(),
17388 reserved: false,
17389 reserved_prefixes: Vec::new(),
17390 protocol: ModuleProtocol::Subc,
17391 overlap: Default::default(),
17392 carry_containment: false,
17393 };
17394
17395 let crash_snapshot = Arc::new(Mutex::new(SupervisorSnapshot::starting()));
17396 assert!(matches!(
17397 on_child_exit(
17398 &spec,
17399 runtime.restart_policy,
17400 &supervisor.registry,
17401 &crash_snapshot,
17402 &runtime.terminal_ring,
17403 &runtime.spawn_events,
17404 &runtime.child_roster,
17405 ExitReport {
17406 kind: ExitKind::Crash,
17407 code: Some(1),
17408 signal: None,
17409 at_ms: 1,
17410 },
17411 true,
17412 )
17413 .await,
17414 NextAction::Restart { schedule: _ }
17415 ));
17416 let (crash_restarts, crash_lifetime) = {
17417 let state = lock_snapshot(&crash_snapshot).unwrap();
17418 (state.crash_restarts.len(), state.lifetime_restarts)
17419 };
17420 assert_eq!(crash_restarts, 1);
17421 assert_eq!(crash_lifetime, 1);
17422
17423 let health_snapshot = Arc::new(Mutex::new(SupervisorSnapshot::starting()));
17424 let mut health_child = None;
17425 assert!(matches!(
17426 health_restart_child(
17427 &spec,
17428 &runtime,
17429 &supervisor.registry,
17430 &supervisor.process_liveness,
17431 &health_snapshot,
17432 &mut health_child,
17433 SupervisorHealthStatus::Failing,
17434 None,
17435 2,
17436 )
17437 .await,
17438 Ok(())
17439 ));
17440 assert!(health_child.is_none());
17441 assert!(lock_snapshot(&health_snapshot).unwrap().respawn_pending);
17442 assert!(runtime.scheduled_respawn.lock().unwrap().take().is_some());
17443 let (health_restarts, health_lifetime) = {
17444 let state = lock_snapshot(&health_snapshot).unwrap();
17445 (state.crash_restarts.len(), state.lifetime_restarts)
17446 };
17447 assert_eq!(health_restarts, 1);
17448 assert_eq!(health_lifetime, 1);
17449
17450 let reload_snapshot = Arc::new(Mutex::new(SupervisorSnapshot::starting()));
17451 let mut reload_child = None;
17452 assert!(matches!(
17453 handle_reload_spawn_failure(
17454 &spec,
17455 &runtime,
17456 &supervisor.process_liveness,
17457 &reload_snapshot,
17458 &mut reload_child,
17459 "forced reload spawn failure".to_string(),
17460 )
17461 .await,
17462 Err(SuperviseError::ReloadFailed { .. })
17463 ));
17464 let (reload_restarts, reload_lifetime) = {
17465 let state = lock_snapshot(&reload_snapshot).unwrap();
17466 (state.crash_restarts.len(), state.lifetime_restarts)
17467 };
17468 assert_eq!(reload_restarts, 1);
17469 assert_eq!(reload_lifetime, 1);
17470 }
17471
17472 #[tokio::test]
17473 async fn deliberately_severed_live_child_records_lifetime_without_spending_restart_budget() {
17474 let supervisor = Supervisor::new_for_test(
17475 Arc::new(Registry::default()),
17476 RestartPolicy::new(3, Duration::ZERO),
17477 );
17478 let runtime = supervisor.runtime_config();
17479 let spec = ModuleSpec {
17480 script: None,
17481 module_id: "deliberately-severed".to_string(),
17482 program: PathBuf::from("/unused/deliberately-severed"),
17483 args: Vec::new(),
17484 env: Vec::new(),
17485 reserved: false,
17486 reserved_prefixes: Vec::new(),
17487 protocol: ModuleProtocol::Subc,
17488 overlap: Default::default(),
17489 carry_containment: false,
17490 };
17491 let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::starting()));
17492 let process = ProcessIdentity {
17493 pid: 41,
17494 start_time: 101,
17495 };
17496 record_deliberate_severance(&snapshot, process).unwrap();
17497 let exit_report = apply_deliberate_severance_marker(
17498 &snapshot,
17499 Some(process),
17500 ExitReport {
17501 kind: ExitKind::Crash,
17502 code: Some(1),
17503 signal: None,
17504 at_ms: 1,
17505 },
17506 );
17507 assert_eq!(exit_report.kind, ExitKind::DeliberateSeverance);
17508
17509 assert!(matches!(
17510 on_child_exit(
17511 &spec,
17512 runtime.restart_policy,
17513 &supervisor.registry,
17514 &snapshot,
17515 &runtime.terminal_ring,
17516 &runtime.spawn_events,
17517 &runtime.child_roster,
17518 exit_report,
17519 true,
17520 )
17521 .await,
17522 NextAction::Restart { schedule: _ }
17523 ));
17524 let state = lock_snapshot(&snapshot).unwrap();
17525 assert_eq!(state.lifetime_restarts, 1);
17526 assert_eq!(state.crash_restarts.len(), 0);
17527 }
17528
17529 #[tokio::test]
17530 async fn genuine_crash_spends_restart_budget_and_records_lifetime() {
17531 let supervisor = Supervisor::new_for_test(
17532 Arc::new(Registry::default()),
17533 RestartPolicy::new(3, Duration::ZERO),
17534 );
17535 let runtime = supervisor.runtime_config();
17536 let spec = ModuleSpec {
17537 script: None,
17538 module_id: "genuine-crash".to_string(),
17539 program: PathBuf::from("/unused/genuine-crash"),
17540 args: Vec::new(),
17541 env: Vec::new(),
17542 reserved: false,
17543 reserved_prefixes: Vec::new(),
17544 protocol: ModuleProtocol::Subc,
17545 overlap: Default::default(),
17546 carry_containment: false,
17547 };
17548 let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::starting()));
17549
17550 assert!(matches!(
17551 on_child_exit(
17552 &spec,
17553 runtime.restart_policy,
17554 &supervisor.registry,
17555 &snapshot,
17556 &runtime.terminal_ring,
17557 &runtime.spawn_events,
17558 &runtime.child_roster,
17559 ExitReport {
17560 kind: ExitKind::Crash,
17561 code: Some(1),
17562 signal: None,
17563 at_ms: 1,
17564 },
17565 true,
17566 )
17567 .await,
17568 NextAction::Restart { schedule: _ }
17569 ));
17570 let state = lock_snapshot(&snapshot).unwrap();
17571 assert_eq!(state.lifetime_restarts, 1);
17572 assert_eq!(state.crash_restarts.len(), 1);
17573 }
17574
17575 fn crash_exit_report(at_ms: u64) -> ExitReport {
17576 ExitReport {
17577 kind: ExitKind::Crash,
17578 code: Some(1),
17579 signal: None,
17580 at_ms,
17581 }
17582 }
17583
17584 fn windowed_crash_spec(module_id: &str) -> ModuleSpec {
17585 ModuleSpec {
17586 script: None,
17587 module_id: module_id.to_string(),
17588 program: PathBuf::from("/unused").join(module_id),
17589 args: Vec::new(),
17590 env: Vec::new(),
17591 reserved: false,
17592 reserved_prefixes: Vec::new(),
17593 protocol: ModuleProtocol::Subc,
17594 overlap: Default::default(),
17595 carry_containment: false,
17596 }
17597 }
17598
17599 #[tokio::test]
17605 async fn three_crashes_inside_the_window_stop_the_module_and_name_the_window() {
17606 let (logs, _guard) = crate::router::test_log::log_capture(tracing::Level::ERROR);
17607 let supervisor = Supervisor::new_for_test(
17608 Arc::new(Registry::default()),
17609 RestartPolicy::new(2, Duration::ZERO),
17610 );
17611 let runtime = supervisor.runtime_config();
17612 let spec = windowed_crash_spec("crash-loop-in-window");
17613 let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::starting()));
17614
17615 for attempt in 1..=2 {
17616 assert!(
17617 matches!(
17618 on_child_exit(
17619 &spec,
17620 runtime.restart_policy,
17621 &supervisor.registry,
17622 &snapshot,
17623 &runtime.terminal_ring,
17624 &runtime.spawn_events,
17625 &runtime.child_roster,
17626 crash_exit_report(attempt),
17627 true,
17628 )
17629 .await,
17630 NextAction::Restart { schedule: _ }
17631 ),
17632 "crash {attempt} is inside the budget and must respawn"
17633 );
17634 }
17635
17636 assert!(matches!(
17637 on_child_exit(
17638 &spec,
17639 runtime.restart_policy,
17640 &supervisor.registry,
17641 &snapshot,
17642 &runtime.terminal_ring,
17643 &runtime.spawn_events,
17644 &runtime.child_roster,
17645 crash_exit_report(3),
17646 true,
17647 )
17648 .await,
17649 NextAction::Stop { .. }
17650 ));
17651
17652 {
17653 let state = lock_snapshot(&snapshot).unwrap();
17654 assert_eq!(state.state, ModuleState::Failed);
17655 assert_eq!(state.crash_restarts.len(), 2);
17656 assert_eq!(state.lifetime_restarts, 2);
17657 }
17658
17659 let history = runtime
17660 .terminal_ring
17661 .lock()
17662 .expect("terminal ring is not poisoned")
17663 .snapshot();
17664 let last = history
17665 .entries
17666 .last()
17667 .expect("the refused crash is retained");
17668 assert_eq!(last.disposition, TerminalDisposition::Failed);
17669 assert_eq!(
17670 last.disposition_detail.as_deref(),
17671 Some("crash budget exhausted: max_restarts=2 within window_secs=600")
17672 );
17673
17674 let captured = crate::router::test_log::captured_logs(&logs);
17675 assert!(
17676 captured.contains("crash budget exhausted: max_restarts=2 within window_secs=600"),
17677 "the stop must be logged with its window: {captured}"
17678 );
17679 }
17680
17681 #[tokio::test]
17689 async fn a_crash_older_than_the_window_frees_its_slot_for_a_later_crash() {
17690 let supervisor = Supervisor::new_for_test(
17691 Arc::new(Registry::default()),
17692 RestartPolicy::new(2, Duration::ZERO),
17693 );
17694 let runtime = supervisor.runtime_config();
17695 let spec = windowed_crash_spec("crash-across-windows");
17696 let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::starting()));
17697
17698 for attempt in 1..=2 {
17699 assert!(matches!(
17700 on_child_exit(
17701 &spec,
17702 runtime.restart_policy,
17703 &supervisor.registry,
17704 &snapshot,
17705 &runtime.terminal_ring,
17706 &runtime.spawn_events,
17707 &runtime.child_roster,
17708 crash_exit_report(attempt),
17709 true,
17710 )
17711 .await,
17712 NextAction::Restart { schedule: _ }
17713 ));
17714 }
17715
17716 update_snapshot(&snapshot, Some(&spec.module_id), |state| {
17719 age_oldest_crash_restart_out_of_window(state, runtime.restart_policy.window);
17720 })
17721 .unwrap();
17722
17723 assert!(
17724 matches!(
17725 on_child_exit(
17726 &spec,
17727 runtime.restart_policy,
17728 &supervisor.registry,
17729 &snapshot,
17730 &runtime.terminal_ring,
17731 &runtime.spawn_events,
17732 &runtime.child_roster,
17733 crash_exit_report(3),
17734 true,
17735 )
17736 .await,
17737 NextAction::Restart { schedule: _ }
17738 ),
17739 "a crash older than the window must not hold a budget slot"
17740 );
17741
17742 let state = lock_snapshot(&snapshot).unwrap();
17743 assert_eq!(state.state, ModuleState::Restarting);
17744 assert_eq!(
17745 state.crash_restarts.len(),
17746 2,
17747 "the aged instant is dropped and the new one takes its place"
17748 );
17749 assert_eq!(
17750 state.lifetime_restarts, 3,
17751 "the ledger counts every restart, including the ones the window forgot"
17752 );
17753 }
17754
17755 #[tokio::test]
17760 async fn an_operator_restart_clears_the_ring_and_leaves_the_ledger_alone() {
17761 let supervisor = Supervisor::new_for_test(
17762 Arc::new(Registry::default()),
17763 RestartPolicy::new(2, Duration::ZERO),
17764 );
17765 let runtime = supervisor.runtime_config();
17766 let spec = windowed_crash_spec("operator-cleared-budget");
17767 let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::starting()));
17768
17769 for attempt in 1..=2 {
17770 assert!(matches!(
17771 on_child_exit(
17772 &spec,
17773 runtime.restart_policy,
17774 &supervisor.registry,
17775 &snapshot,
17776 &runtime.terminal_ring,
17777 &runtime.spawn_events,
17778 &runtime.child_roster,
17779 crash_exit_report(attempt),
17780 true,
17781 )
17782 .await,
17783 NextAction::Restart { schedule: _ }
17784 ));
17785 }
17786
17787 reset_restart_count(&snapshot, &spec.module_id).unwrap();
17788 {
17789 let state = lock_snapshot(&snapshot).unwrap();
17790 assert!(
17791 state.crash_restarts.is_empty(),
17792 "an operator restart returns the full budget"
17793 );
17794 assert_eq!(
17795 state.lifetime_restarts, 2,
17796 "clearing the budget must not unmake the crashes"
17797 );
17798 }
17799
17800 assert!(
17801 matches!(
17802 on_child_exit(
17803 &spec,
17804 runtime.restart_policy,
17805 &supervisor.registry,
17806 &snapshot,
17807 &runtime.terminal_ring,
17808 &runtime.spawn_events,
17809 &runtime.child_roster,
17810 crash_exit_report(3),
17811 true,
17812 )
17813 .await,
17814 NextAction::Restart { schedule: _ }
17815 ),
17816 "the cleared budget must be spendable again"
17817 );
17818 let state = lock_snapshot(&snapshot).unwrap();
17819 assert_eq!(state.crash_restarts.len(), 1);
17820 assert_eq!(state.lifetime_restarts, 3);
17821 }
17822
17823 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
17824 async fn severance_marker_for_a_dead_child_does_not_label_its_successor() {
17825 let severed = ProcessIdentity {
17826 pid: 41,
17827 start_time: 101,
17828 };
17829 let successor = ProcessIdentity {
17830 pid: 41,
17831 start_time: 202,
17832 };
17833 let module = module_with_recovery_snapshot(ModuleState::Running, true, 0);
17834 update_snapshot(&module.inner.snapshot, Some("recovery-snapshot"), |state| {
17835 state.pid = Some(successor.pid);
17836 state.process_start_time = Some(successor.start_time);
17837 })
17838 .unwrap();
17839 assert!(!module.record_deliberate_severance(severed).unwrap());
17840
17841 let exit_report = apply_deliberate_severance_marker(
17842 &module.inner.snapshot,
17843 Some(successor),
17844 ExitReport {
17845 kind: ExitKind::Crash,
17846 code: Some(1),
17847 signal: None,
17848 at_ms: 1,
17849 },
17850 );
17851
17852 assert_eq!(exit_report.kind, ExitKind::Crash);
17853 }
17854
17855 #[tokio::test]
17856 async fn drain_reap_marks_deliberate_severance_and_records_lifetime_without_budget() {
17857 let registry = Registry::default();
17858 let supervisor = Supervisor::new_for_test(
17859 Arc::new(Registry::default()),
17860 RestartPolicy::new(3, Duration::ZERO),
17861 );
17862 let runtime = supervisor.runtime_config();
17863 let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::starting()));
17864 let spec = ModuleSpec {
17865 script: None,
17866 module_id: "drain-deliberate-severance".to_string(),
17867 program: fake_aft_stub_path(),
17868 args: Vec::new(),
17869 env: vec![("FAKE_AFT_EXIT_CODE".to_string(), "23".to_string())],
17870 reserved: false,
17871 reserved_prefixes: Vec::new(),
17872 protocol: ModuleProtocol::Subc,
17873 overlap: Default::default(),
17874 carry_containment: false,
17875 };
17876 let mut child = spawn_and_mark_running(&spec, &runtime, &snapshot).unwrap();
17877 let process = ProcessIdentity {
17878 pid: 41,
17879 start_time: 101,
17880 };
17881 child.process_identity = Some(process);
17882 update_snapshot(&snapshot, Some(&spec.module_id), |state| {
17883 state.pid = Some(process.pid);
17884 state.process_start_time = Some(process.start_time);
17885 })
17886 .unwrap();
17887 record_deliberate_severance(&snapshot, process).unwrap();
17888
17889 drain_child_to_state(
17890 &spec.module_id,
17891 spec.protocol,
17892 StopNotice::SentOverConnection,
17895 ®istry,
17896 None,
17897 &snapshot,
17898 &runtime.terminal_ring,
17899 &runtime.spawn_events,
17900 child,
17901 Duration::from_secs(1),
17902 ModuleState::Stopped,
17903 Some(false),
17904 &super::FailureContext::operator(),
17905 )
17906 .await
17907 .unwrap();
17908
17909 let state = lock_snapshot(&snapshot).unwrap();
17910 assert_eq!(
17911 state.last_exit.as_ref().map(|exit| exit.kind),
17912 Some(ExitKind::DeliberateSeverance)
17913 );
17914 assert_eq!(state.lifetime_restarts, 1);
17915 assert_eq!(state.crash_restarts.len(), 0);
17916 drop(state);
17917 let history = runtime.terminal_ring.lock().unwrap().snapshot();
17918 assert_eq!(
17919 history.entries[0].exit_kind,
17920 subc_control::TerminalExitKind::DeliberateSeverance
17921 );
17922 }
17923
17924 #[tokio::test]
17925 async fn ordinary_drain_reap_does_not_record_a_lifetime_restart() {
17926 let registry = Registry::default();
17927 let supervisor = Supervisor::new_for_test(
17928 Arc::new(Registry::default()),
17929 RestartPolicy::new(3, Duration::ZERO),
17930 );
17931 let runtime = supervisor.runtime_config();
17932 let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::starting()));
17933 let spec = ModuleSpec {
17934 script: None,
17935 module_id: "ordinary-drain".to_string(),
17936 program: fake_aft_stub_path(),
17937 args: Vec::new(),
17938 env: vec![("FAKE_AFT_EXIT_CODE".to_string(), "23".to_string())],
17939 reserved: false,
17940 reserved_prefixes: Vec::new(),
17941 protocol: ModuleProtocol::Subc,
17942 overlap: Default::default(),
17943 carry_containment: false,
17944 };
17945 let child = spawn_and_mark_running(&spec, &runtime, &snapshot).unwrap();
17946
17947 drain_child_to_state(
17948 &spec.module_id,
17949 spec.protocol,
17950 StopNotice::SentOverConnection,
17953 ®istry,
17954 None,
17955 &snapshot,
17956 &runtime.terminal_ring,
17957 &runtime.spawn_events,
17958 child,
17959 Duration::from_secs(1),
17960 ModuleState::Stopped,
17961 Some(false),
17962 &super::FailureContext::operator(),
17963 )
17964 .await
17965 .unwrap();
17966
17967 let state = lock_snapshot(&snapshot).unwrap();
17968 assert_eq!(
17969 state.last_exit.as_ref().map(|exit| exit.kind),
17970 Some(ExitKind::Crash)
17971 );
17972 assert_eq!(state.lifetime_restarts, 0);
17973 assert_eq!(state.crash_restarts.len(), 0);
17974 }
17975
17976 #[test]
17977 fn fatal_connection_teardown_cannot_arm_a_marker_for_a_surviving_process() {
17978 assert!(!include_str!("server.rs")
17984 .contains("router.record_deliberate_connection_severance(ctx.connection_id)"));
17985 }
17986
17987 #[test]
17994 fn drained_after_quiescence_wait_passes_ok_through_and_forces_false_on_err() {
17995 assert!(drained_after_quiescence_wait(&Ok(true)));
17996 assert!(!drained_after_quiescence_wait(&Ok(false)));
17997 assert!(!drained_after_quiescence_wait(&Err(
17998 SuperviseError::StatePoisoned { module_id: None }
17999 )));
18000 }
18001
18002 #[test]
18011 fn wait_error_exit_report_records_a_failed_terminal_with_no_code_or_signal() {
18012 let ring = Arc::new(Mutex::new(TerminalRing::new(
18013 TerminalRingConfig::default(),
18014 0,
18015 )));
18016 record_wait_error_terminal("wait-error", &ring, &super::SpawnEventFeed::default());
18017
18018 let snapshot = ring.lock().unwrap().snapshot();
18019 assert_eq!(snapshot.entries.len(), 1);
18020 let entry = &snapshot.entries[0];
18021 assert_eq!(entry.exit_code, None);
18022 assert_eq!(entry.exit_signal, None);
18023 assert_eq!(entry.disposition, TerminalDisposition::Failed);
18024 }
18025
18026 #[test]
18029 fn spawn_generation_of_a_failed_launch_is_never_reused() {
18030 let feed = super::SpawnEventFeed::default();
18031 feed.configure_incarnation("launch-numbering".to_string());
18032 let (incarnation, failed) = feed.allocate_generation("relaunched");
18033 assert_eq!(incarnation, "launch-numbering");
18034 let (_, next) = feed.allocate_generation("relaunched");
18035 assert!(next > failed);
18036 assert_eq!(feed.emit_spawned("relaunched", Some(next), 7, 1), next);
18037 assert!(feed.emit_spawned("relaunched", None, 8, 2) > next);
18039 assert_eq!(
18040 feed.allocate_generation("other").1,
18041 1,
18042 "numbering is per module"
18043 );
18044 }
18045
18046 #[test]
18047 fn wait_error_exit_path_preserves_spawn_event_density() {
18048 let feed = super::SpawnEventFeed::default();
18049 feed.configure_incarnation("wait-error-density".to_string());
18050 feed.emit_spawned("wait-error", None, 41, 1);
18051 let ring = Arc::new(Mutex::new(TerminalRing::new(
18052 TerminalRingConfig::default(),
18053 0,
18054 )));
18055
18056 record_wait_error_terminal("wait-error", &ring, &feed);
18057 feed.emit_spawned("after-wait-error", None, 42, 2);
18058
18059 let state = feed.0.lock().unwrap();
18060 let sequences = state
18061 .events
18062 .iter()
18063 .map(|event| event.cursor.seq)
18064 .collect::<Vec<_>>();
18065 assert_eq!(sequences, vec![1, 2, 3]);
18066 assert_eq!(state.events[1].kind, SpawnEventKind::Exited);
18067 assert_eq!(state.events[1].exit_code, None);
18068 assert_eq!(state.events[1].exit_signal, None);
18069 }
18070
18071 #[test]
18075 fn wait_error_exit_report_is_classified_as_a_crash() {
18076 assert_eq!(wait_error_exit_report().kind, ExitKind::Crash);
18077 }
18078}
18079
18080#[cfg(all(test, unix))]
18081mod restart_ordering_tests {
18082 use super::*;
18083 use std::io::Write;
18084 use tokio::io::AsyncReadExt;
18085
18086 #[test]
18087 #[ignore = "child process fixture"]
18088 fn serving_child() {
18089 let Ok(address_file) = std::env::var("RESTART_TEST_ADDRESS_FILE") else {
18090 return;
18091 };
18092 let listener = std::net::TcpListener::bind("127.0.0.1:0").unwrap();
18093 std::fs::write(address_file, listener.local_addr().unwrap().to_string()).unwrap();
18094 for stream in listener.incoming() {
18095 writeln!(stream.unwrap(), "{}", std::process::id()).unwrap();
18096 }
18097 }
18098
18099 async fn restart_before_publication(requests: usize) {
18100 let home = cortexkit_test_support::ScratchDir::new("restart-before-publication");
18101 let address_file = home.path().join("address");
18102 let spec = ModuleSpec {
18103 script: None,
18104 module_id: "restart-serving-child".into(),
18105 program: std::env::current_exe().unwrap(),
18106 args: vec![
18107 "--exact".into(),
18108 "supervise::restart_ordering_tests::serving_child".into(),
18109 "--ignored".into(),
18110 "--nocapture".into(),
18111 ],
18112 env: vec![(
18113 "RESTART_TEST_ADDRESS_FILE".into(),
18114 address_file.display().to_string(),
18115 )],
18116 reserved: false,
18117 reserved_prefixes: vec![],
18118 protocol: ModuleProtocol::None,
18119 overlap: Default::default(),
18120 carry_containment: false,
18121 };
18122 let mut supervisor =
18123 Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default());
18124 let (spawned_tx, spawned_rx) = oneshot::channel();
18125 let spawned_tx = Mutex::new(Some(spawned_tx));
18126 let (release_tx, release_rx) = std::sync::mpsc::channel();
18127 let release_rx = Mutex::new(release_rx);
18128 supervisor.test_after_first_spawn = AfterFirstSpawnHook(Some(Arc::new(move |_| {
18131 spawned_tx.lock().unwrap().take().unwrap().send(()).unwrap();
18132 release_rx
18133 .lock()
18134 .unwrap()
18135 .recv_timeout(Duration::from_secs(20))
18136 .unwrap();
18137 })));
18138 let registered = supervisor
18139 .register_configured_with_health(
18140 spec.clone(),
18141 true,
18142 HealthConfig::default(),
18143 None,
18144 RestartPolicy::new(3, Duration::ZERO),
18145 DEFAULT_REGISTER_TIMEOUT,
18146 )
18147 .unwrap();
18148 let module = registered.module.clone();
18149 let startup = tokio::task::spawn_blocking(move || {
18150 supervisor.start_registered_configured(registered, None, true)
18151 });
18152 spawned_rx.await.unwrap();
18153 let old_pid = tokio::time::timeout(Duration::from_secs(10), async {
18154 let address = loop {
18155 if let Ok(address) = std::fs::read_to_string(&address_file) {
18156 if !address.is_empty() {
18157 break address;
18158 }
18159 }
18160 tokio::time::sleep(Duration::from_millis(1)).await;
18161 };
18162 let mut stream = tokio::net::TcpStream::connect(address).await.unwrap();
18163 let mut reply = String::new();
18164 stream.read_to_string(&mut reply).await.unwrap();
18165 reply.trim().parse::<u32>().unwrap()
18166 })
18167 .await
18168 .expect("child must serve before generation publication");
18169 assert_eq!(
18170 lock_snapshot(&module.inner.snapshot)
18171 .unwrap()
18172 .spawn_generation,
18173 0
18174 );
18175 let mut replies = Vec::new();
18176 for _ in 0..requests {
18177 let mut restart = Box::pin(module.restart(Some(0)));
18178 std::future::poll_fn(|cx| {
18181 assert!(std::future::Future::poll(restart.as_mut(), cx).is_pending());
18182 std::task::Poll::Ready(())
18183 })
18184 .await;
18185 replies.push(restart);
18186 }
18187 release_tx.send(()).unwrap();
18188 startup.await.unwrap().unwrap();
18189 for reply in replies {
18190 reply.await.unwrap();
18191 }
18192 let replacement = tokio::time::timeout(Duration::from_secs(10), async {
18195 loop {
18196 if lock_snapshot(&module.inner.snapshot)
18197 .unwrap()
18198 .spawn_generation
18199 > 1
18200 {
18201 break;
18202 }
18203 tokio::time::sleep(Duration::from_millis(1)).await;
18204 }
18205 })
18206 .await;
18207 module.update_spec_for_test(spec).await.unwrap();
18211 let (generation, pid) = {
18212 let state = lock_snapshot(&module.inner.snapshot).unwrap();
18213 (state.spawn_generation, state.pid)
18214 };
18215 module.stop().await.unwrap();
18216 replacement.expect("restart must publish a replacement for the already-serving child");
18217 assert_ne!(
18218 pid,
18219 Some(old_pid),
18220 "a serving child predating the restart must be replaced"
18221 );
18222 assert_eq!(generation, 2, "queued restarts must share one replacement");
18223 }
18224
18225 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
18226 async fn restart_received_after_serving_before_generation_publication_replaces_child() {
18227 restart_before_publication(1).await;
18228 }
18229
18230 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
18231 async fn restart_queued_behind_restart_does_not_kill_its_replacement() {
18232 restart_before_publication(2).await;
18233 }
18234
18235 #[tokio::test]
18236 async fn restart_queued_before_replacement_spawn_keeps_the_new_child() {
18237 let supervisor =
18238 Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default());
18239 let registered = supervisor
18240 .register_configured_with_health(
18241 ModuleSpec {
18242 script: None,
18243 module_id: "queued-before-replacement".into(),
18244 program: "/bin/sleep".into(),
18249 args: vec!["600".into()],
18250 env: vec![],
18251 reserved: false,
18252 reserved_prefixes: vec![],
18253 protocol: ModuleProtocol::None,
18254 overlap: Default::default(),
18255 carry_containment: false,
18256 },
18257 true,
18258 HealthConfig::default(),
18259 None,
18260 RestartPolicy::default(),
18261 DEFAULT_REGISTER_TIMEOUT,
18262 )
18263 .unwrap();
18264 let RegisteredConfiguredModule {
18265 mut spec,
18266 mut runtime,
18267 module,
18268 mut commands,
18269 } = registered;
18270 let snapshot = &module.inner.snapshot;
18271 let mut restart = Box::pin(module.restart(Some(0)));
18272 std::future::poll_fn(|cx| {
18273 assert!(std::future::Future::poll(restart.as_mut(), cx).is_pending());
18274 std::task::Poll::Ready(())
18275 })
18276 .await;
18277 let mut child = Some(spawn_and_mark_running(&spec, &runtime, snapshot).unwrap());
18280 let replacement_pid = child.as_ref().unwrap().id();
18281 let command = commands.recv().await.unwrap();
18282 assert!(
18283 handle_supervisor_command(
18284 command,
18285 &mut spec,
18286 &mut runtime,
18287 &supervisor.registry,
18288 &supervisor.process_liveness,
18289 snapshot,
18290 &mut child,
18291 &mut commands,
18292 &mut VecDeque::new(),
18293 )
18294 .await
18295 );
18296 restart.await.unwrap();
18297 let kept_pid = child.as_ref().and_then(SupervisedChild::id);
18298 let pending = lock_snapshot(snapshot).unwrap().respawn_pending;
18299 if let Some(mut child) = child {
18300 child.start_kill().unwrap();
18301 child.wait().await.unwrap();
18302 }
18303 assert_eq!(
18304 kept_pid, replacement_pid,
18305 "a newer replacement must survive"
18306 );
18307 assert!(
18308 !pending,
18309 "a satisfied restart must not schedule another spawn"
18310 );
18311 }
18312}
18313
18314#[cfg(all(test, unix))]
18315mod health_event_tests {
18316 use super::*;
18317 use cortexkit_test_support::ScratchDir;
18318 use subc_protocol::{manifest::Concurrency, session::ModuleControlResponse};
18319
18320 struct Actor {
18321 module: SupervisedModule,
18322 registry: Arc<Registry>,
18323 forwarding: Arc<ForwardingTable>,
18324 spec: ModuleSpec,
18325 health: HealthConfig,
18326 rx: mpsc::Receiver<crate::router::OutboundFrame>,
18327 roster: ChildRoster,
18328 _home: ScratchDir,
18329 }
18330
18331 async fn wait_for<T>(reason: &str, mut observe: impl FnMut() -> Option<T>) -> T {
18332 let deadline = std::time::Instant::now() + Duration::from_secs(10);
18336 let tick = Instant::now();
18337 loop {
18338 if let Some(value) = observe() {
18339 assert_eq!(Instant::now(), tick, "{reason} must not wait for a timer");
18340 return value;
18341 }
18342 assert!(
18343 std::time::Instant::now() < deadline,
18344 "timed out waiting for {reason}"
18345 );
18346 tokio::task::yield_now().await;
18347 }
18348 }
18349
18350 impl Actor {
18351 async fn start(protocol: ModuleProtocol, cadence: Duration) -> Self {
18352 Self::start_with_policy(
18353 protocol,
18354 cadence,
18355 DEFAULT_REGISTER_TIMEOUT,
18356 RestartPolicy::new(3, Duration::ZERO),
18357 )
18358 .await
18359 }
18360
18361 async fn start_with_policy(
18362 protocol: ModuleProtocol,
18363 cadence: Duration,
18364 register_timeout: Duration,
18365 policy: RestartPolicy,
18366 ) -> Self {
18367 let home = ScratchDir::new("health-events");
18368 let registry = Arc::new(Registry::default());
18369 let forwarding = Arc::new(ForwardingTable::default());
18370 let health = HealthConfig {
18371 cadence,
18372 deadline: Duration::from_secs(3600),
18373 ..HealthConfig::default()
18374 };
18375 let supervisor = Supervisor::new_for_test(registry.clone(), policy)
18376 .with_forwarding(forwarding.clone())
18377 .with_health_config(health.clone());
18378 let spec = ModuleSpec {
18379 script: None,
18380 module_id: "event-health-child".into(),
18381 program: PathBuf::from("/bin/sleep"),
18389 args: vec!["600".into()],
18392 env: ["XDG_DATA_HOME", "XDG_RUNTIME_DIR", "XDG_CONFIG_HOME"]
18393 .into_iter()
18394 .map(|key| (key.into(), home.path().display().to_string()))
18395 .collect(),
18396 reserved: false,
18397 reserved_prefixes: vec![],
18398 protocol,
18399 overlap: Default::default(),
18400 carry_containment: false,
18401 };
18402 let module = supervisor
18403 .supervise_configured_requested(
18404 spec.clone(),
18405 true,
18406 health.clone(),
18407 Some(0),
18408 policy,
18409 register_timeout,
18410 None,
18411 )
18412 .unwrap();
18413 let (_, rx) = mpsc::channel(8);
18414 let actor = Self {
18415 module,
18416 registry,
18417 forwarding,
18418 spec,
18419 health,
18420 rx,
18421 roster: supervisor.child_roster.clone(),
18422 _home: home,
18423 };
18424 actor
18425 .wait_for_select(|checkpoint| checkpoint.generation > 0)
18426 .await;
18427 actor
18428 }
18429
18430 async fn expire_registration_wait(&self, remaining: Duration) {
18431 tokio::time::advance(remaining).await;
18432 wait_for("registration timeout TERM grace", || {
18433 lock_snapshot(&self.module.inner.snapshot)
18434 .unwrap()
18435 .actor_select
18436 .is_none()
18437 .then_some(())
18438 })
18439 .await;
18440 tokio::time::advance(crate::child_roster::TERM_TO_KILL).await;
18441 }
18442
18443 fn turns(&self) -> u64 {
18444 lock_snapshot(&self.module.inner.snapshot)
18445 .unwrap()
18446 .actor_turns
18447 }
18448
18449 async fn wait_for_select(
18450 &self,
18451 expected: impl Fn(&ActorSelectCheckpoint) -> bool,
18452 ) -> ActorSelectCheckpoint {
18453 wait_for("actor select checkpoint", || {
18454 let state = lock_snapshot(&self.module.inner.snapshot).unwrap();
18455 state.actor_select.filter(|checkpoint| {
18456 checkpoint.generation == state.spawn_generation
18457 && state.state == ModuleState::Running
18458 && state.process_alive
18459 && state.reported_pid().is_some()
18460 && expected(checkpoint)
18461 })
18462 })
18463 .await
18464 }
18465
18466 fn hello(&mut self, connection: u64, health: bool) {
18467 let manifest =
18468 subc_protocol::manifest::ModuleManifest::builder(&self.spec.module_id, "1.0.0")
18469 .protocol_ver(subc_protocol::PROTOCOL_VERSION)
18470 .build();
18471 let connection = ConnectionId::new(connection);
18472 let (tx, rx) = mpsc::channel(8);
18473 self.rx = rx;
18474 self.forwarding
18475 .register_module_connection(
18476 connection,
18477 self.spec.module_id.clone(),
18478 subc_protocol::PROTOCOL_VERSION,
18479 Concurrency::ModuleManaged,
18480 FrameSink::new(tx),
18481 )
18482 .unwrap();
18483 self.registry
18484 .register_with_control_ops(
18485 manifest,
18486 subc_protocol::PROTOCOL_VERSION,
18487 connection,
18488 if health {
18489 vec![MODULE_CONTROL_OP_HEALTH_CHECK.into()]
18490 } else {
18491 vec![]
18492 },
18493 )
18494 .unwrap();
18495 }
18496
18497 async fn probe(&mut self) -> crate::router::OutboundFrame {
18498 let frame = wait_for("outbound health probe", || match self.rx.try_recv() {
18499 Ok(frame) => Some(frame),
18500 Err(mpsc::error::TryRecvError::Empty) => None,
18501 Err(mpsc::error::TryRecvError::Disconnected) => panic!("health peer disconnected"),
18502 })
18503 .await;
18504 let body: Value = serde_json::from_slice(&frame.body).unwrap();
18505 assert_eq!(body["op"], MODULE_CONTROL_OP_HEALTH_CHECK);
18506 frame
18507 }
18508
18509 fn no_probe(&mut self) {
18510 assert!(matches!(
18511 self.rx.try_recv(),
18512 Err(mpsc::error::TryRecvError::Empty)
18513 ));
18514 }
18515
18516 fn answer(&self, connection: u64, frame: crate::router::OutboundFrame) {
18517 self.forwarding
18518 .complete_module_control_rpc(
18519 ConnectionId::new(connection),
18520 frame.header.corr,
18521 Some(MODULE_CONTROL_OP_HEALTH_CHECK),
18522 ModuleControlRpcOutcome::Response(ModuleControlResponse::HealthCheck {
18523 status: HealthStatus::Ok,
18524 detail: None,
18525 metrics: None,
18526 }),
18527 )
18528 .unwrap();
18529 }
18530 }
18531
18532 #[tokio::test(start_paused = true)]
18533 async fn registration_timeout_kills_at_120_seconds_and_respawns() {
18534 let actor = Actor::start(ModuleProtocol::Subc, Duration::from_secs(30)).await;
18535 let initial = actor.module.status().unwrap();
18536 tokio::time::advance(Duration::from_secs(119)).await;
18537 tokio::task::yield_now().await;
18538 assert_eq!(actor.module.status().unwrap().pid, initial.pid);
18539 assert!(actor.module.terminal_history().entries.is_empty());
18540 actor.expire_registration_wait(Duration::from_secs(1)).await;
18541 actor
18542 .wait_for_select(|at| at.generation > initial.spawn_generation)
18543 .await;
18544 let status = actor.module.status().unwrap();
18545 assert_ne!(status.pid, initial.pid);
18546 assert!(!cortexkit_test_support::process_alive(
18547 initial.pid.unwrap() as i32
18548 ));
18549 assert_eq!(status.restart_count, 1);
18550 assert_eq!(status.last_exit.unwrap().kind, ExitKind::Crash);
18551 let entries = actor.module.terminal_history().entries;
18552 assert_eq!(entries.len(), 1);
18553 assert_eq!(
18554 entries[0].disposition_detail.as_deref(),
18555 Some("registration_timeout")
18556 );
18557 actor.module.stop().await.unwrap();
18558 }
18559
18560 #[tokio::test(start_paused = true)]
18561 async fn registration_timeout_is_cancelled_by_hello_at_119_seconds() {
18562 let mut actor = Actor::start(ModuleProtocol::Subc, Duration::from_secs(30)).await;
18563 let pid = actor.module.status().unwrap().pid;
18564 tokio::time::advance(Duration::from_secs(119)).await;
18565 actor.hello(1, false);
18566 let registered = actor
18567 .wait_for_select(|at| at.registered_connection.is_some())
18568 .await;
18569 assert_eq!(
18570 registered.wake_after, None,
18571 "HELLO must cancel the deadline"
18572 );
18573 tokio::time::advance(Duration::from_secs(300)).await;
18574 tokio::task::yield_now().await;
18575 assert_eq!(actor.module.status().unwrap().pid, pid);
18576 assert!(actor.module.terminal_history().entries.is_empty());
18577 actor
18579 .registry
18580 .deregister_connection(ConnectionId::new(1))
18581 .unwrap();
18582 actor
18583 .wait_for_select(|at| at.registered_connection.is_none())
18584 .await;
18585 tokio::time::advance(Duration::from_secs(300)).await;
18586 tokio::task::yield_now().await;
18587 assert_eq!(actor.module.status().unwrap().pid, pid);
18588 assert!(actor.module.terminal_history().entries.is_empty());
18589 actor.module.stop().await.unwrap();
18590
18591 let mut transient = Actor::start(ModuleProtocol::Subc, Duration::from_secs(30)).await;
18592 let pid = transient.module.status().unwrap().pid;
18593 tokio::time::advance(Duration::from_secs(119)).await;
18594 let turns = transient.turns();
18595 transient.hello(1, false);
18598 transient
18599 .registry
18600 .deregister_connection(ConnectionId::new(1))
18601 .unwrap();
18602 let parked = transient.wait_for_select(|at| at.turn > turns).await;
18603 assert_eq!(
18604 parked.wake_after, None,
18605 "transient HELLO must cancel the deadline"
18606 );
18607 tokio::time::advance(Duration::from_secs(300)).await;
18608 tokio::task::yield_now().await;
18609 assert_eq!(transient.module.status().unwrap().pid, pid);
18610 assert!(transient.module.terminal_history().entries.is_empty());
18611 transient.module.stop().await.unwrap();
18612 }
18613
18614 #[tokio::test(start_paused = true)]
18615 async fn registration_timeouts_exhaust_budget_and_schedule_slow_retry() {
18616 let actor = Actor::start_with_policy(
18617 ModuleProtocol::Subc,
18618 Duration::from_secs(30),
18619 Duration::from_secs(10),
18620 RestartPolicy::new(1, Duration::ZERO),
18621 )
18622 .await;
18623 let generation = actor.module.status().unwrap().spawn_generation;
18624 actor
18625 .expire_registration_wait(Duration::from_secs(10))
18626 .await;
18627 actor.wait_for_select(|at| at.generation > generation).await;
18628 actor
18629 .expire_registration_wait(Duration::from_secs(10))
18630 .await;
18631 let retry = wait_for("failed retry schedule", || {
18632 let state = lock_snapshot(&actor.module.inner.snapshot).unwrap();
18633 (state.state == ModuleState::Failed)
18634 .then_some(state.failed_retry.pending)
18635 .flatten()
18636 })
18637 .await;
18638 assert_eq!(retry.delay, Duration::from_secs(600));
18639 assert_eq!(retry.attempt, 1);
18640 let entries = actor.module.terminal_history().entries;
18641 assert_eq!(entries.len(), 2);
18642 assert_eq!(entries[1].disposition, TerminalDisposition::Failed);
18643 assert!(entries[1]
18644 .disposition_detail
18645 .as_deref()
18646 .unwrap()
18647 .contains("registration_timeout"));
18648 tokio::time::advance(Duration::from_secs(599)).await;
18649 tokio::task::yield_now().await;
18650 assert_eq!(actor.module.state().unwrap(), ModuleState::Failed);
18651 tokio::time::advance(Duration::from_secs(1)).await;
18652 actor
18653 .wait_for_select(|at| at.generation > generation + 1)
18654 .await;
18655 actor
18657 .expire_registration_wait(Duration::from_secs(10))
18658 .await;
18659 wait_for("slow retry registration timeout", || {
18660 (actor.module.terminal_history().entries.len() == 3).then_some(())
18661 })
18662 .await;
18663 actor.module.stop().await.unwrap();
18664 }
18665
18666 #[tokio::test(start_paused = true)]
18667 async fn registration_timeout_honours_ten_second_override() {
18668 let actor = Actor::start_with_policy(
18669 ModuleProtocol::Subc,
18670 Duration::from_secs(30),
18671 Duration::from_secs(10),
18672 RestartPolicy::new(3, Duration::ZERO),
18673 )
18674 .await;
18675 let generation = actor.module.status().unwrap().spawn_generation;
18676 tokio::time::advance(Duration::from_secs(9)).await;
18677 tokio::task::yield_now().await;
18678 assert!(actor.module.terminal_history().entries.is_empty());
18679 actor.expire_registration_wait(Duration::from_secs(1)).await;
18680 actor.wait_for_select(|at| at.generation > generation).await;
18681 assert_eq!(actor.module.terminal_history().entries.len(), 1);
18682 actor.module.stop().await.unwrap();
18683 }
18684
18685 #[tokio::test]
18686 async fn registration_timeout_kills_term_ignoring_helper_before_reaping_leader() {
18687 use std::os::unix::process::{CommandExt, ExitStatusExt};
18688
18689 struct Helper(std::process::Child);
18690 impl Drop for Helper {
18691 fn drop(&mut self) {
18692 if matches!(self.0.try_wait(), Ok(None)) {
18695 let _ = self.0.kill();
18696 let _ = self.0.wait();
18697 }
18698 }
18699 }
18700
18701 let home = ScratchDir::new("registration-timeout-helper");
18702 let supervisor = Supervisor::new_for_test(
18703 Arc::new(Registry::default()),
18704 RestartPolicy::new(0, Duration::ZERO),
18705 );
18706 let spec = ModuleSpec {
18707 script: None,
18708 module_id: "registration-timeout-group".into(),
18709 program: "/bin/sleep".into(),
18710 args: vec!["120".into()],
18711 env: ["XDG_DATA_HOME", "XDG_RUNTIME_DIR", "XDG_CONFIG_HOME"]
18712 .into_iter()
18713 .map(|key| (key.into(), home.path().display().to_string()))
18714 .collect(),
18715 reserved: false,
18716 reserved_prefixes: vec![],
18717 protocol: ModuleProtocol::Subc,
18718 overlap: Default::default(),
18719 carry_containment: false,
18720 };
18721 let module = supervisor
18722 .supervise_configured_requested(
18723 spec,
18724 true,
18725 HealthConfig::default(),
18726 None,
18727 RestartPolicy::new(0, Duration::ZERO),
18728 Duration::from_secs(10),
18729 None,
18730 )
18731 .unwrap();
18732 let leader = timeout(Duration::from_secs(5), async {
18733 loop {
18734 if let Some(pid) = module.status().unwrap().pid {
18735 break pid;
18736 }
18737 sleep(Duration::from_millis(5)).await;
18738 }
18739 })
18740 .await
18741 .expect("supervised group leader must be reported before the deadline");
18742 assert!(leader > 1);
18743 let ready = home.join("helper.ready");
18744 let mut helper = Helper(
18747 std::process::Command::new("/bin/sh")
18748 .args([
18749 "-c",
18750 "trap '' TERM; : > \"$1\"; exec /bin/sleep 120",
18751 "helper",
18752 ])
18753 .arg(&ready)
18754 .process_group(i32::try_from(leader).unwrap())
18755 .stdout(std::process::Stdio::null())
18756 .stderr(std::process::Stdio::null())
18757 .spawn()
18758 .unwrap(),
18759 );
18760 timeout(Duration::from_secs(5), async {
18761 while !ready.exists() {
18762 assert!(helper.0.try_wait().unwrap().is_none());
18763 sleep(Duration::from_millis(5)).await;
18764 }
18765 })
18766 .await
18767 .expect("helper must install TERM ignore before the registration deadline");
18768 let helper_pid =
18769 rustix::process::Pid::from_raw(i32::try_from(helper.0.id()).unwrap()).unwrap();
18770 rustix::process::kill_process(helper_pid, rustix::process::Signal::TERM).unwrap();
18771 sleep(Duration::from_millis(20)).await;
18772 assert!(
18773 helper.0.try_wait().unwrap().is_none(),
18774 "helper must ignore TERM"
18775 );
18776
18777 timeout(Duration::from_secs(20), async {
18778 while module.state().unwrap() != ModuleState::Failed {
18779 sleep(Duration::from_millis(5)).await;
18780 }
18781 })
18782 .await
18783 .expect("registration timeout must stop and reap the group leader");
18784 let helper_exit = timeout(Duration::from_secs(3), async {
18785 loop {
18786 if let Some(status) = helper.0.try_wait().unwrap() {
18787 break status;
18788 }
18789 sleep(Duration::from_millis(5)).await;
18790 }
18791 })
18792 .await
18793 .expect("TERM-ignoring helper survived the registration timeout's group KILL");
18794 assert_eq!(helper_exit.signal(), Some(9));
18795 assert!(!cortexkit_test_support::process_alive(
18796 i32::try_from(leader).unwrap()
18797 ));
18798 assert_eq!(
18799 module.status().unwrap().last_exit.unwrap().kind,
18800 ExitKind::Crash
18801 );
18802 assert!(module.terminal_history().entries[0]
18803 .disposition_detail
18804 .as_deref()
18805 .unwrap()
18806 .contains("registration_timeout"));
18807 module.stop().await.unwrap();
18808 }
18809
18810 #[tokio::test(start_paused = true)]
18811 async fn registration_timeout_does_not_count_an_earlier_exit() {
18812 let actor = Actor::start_with_policy(
18813 ModuleProtocol::Subc,
18814 Duration::from_secs(30),
18815 DEFAULT_REGISTER_TIMEOUT,
18816 RestartPolicy::new(0, Duration::ZERO),
18817 )
18818 .await;
18819 let pid = actor.module.status().unwrap().pid.unwrap();
18820 rustix::process::kill_process(
18821 rustix::process::Pid::from_raw(pid as i32).unwrap(),
18822 rustix::process::Signal::TERM,
18823 )
18824 .unwrap();
18825 wait_for("early exit", || {
18826 (actor.module.state().unwrap() == ModuleState::Failed).then_some(())
18827 })
18828 .await;
18829 tokio::time::advance(Duration::from_secs(121)).await;
18830 tokio::task::yield_now().await;
18831 let entries = actor.module.terminal_history().entries;
18832 assert_eq!(entries.len(), 1);
18833 assert!(!entries[0]
18834 .disposition_detail
18835 .as_deref()
18836 .unwrap()
18837 .contains("registration_timeout"));
18838 assert_eq!(actor.module.status().unwrap().lifetime_restarts, 0);
18839 actor.module.stop().await.unwrap();
18840 }
18841
18842 #[tokio::test(start_paused = true)]
18843 async fn registration_timeout_is_cancelled_by_daemon_shutdown() {
18844 let actor = Actor::start(ModuleProtocol::Subc, Duration::from_secs(30)).await;
18845 let pid = actor.module.status().unwrap().pid;
18846 tokio::time::advance(Duration::from_secs(119)).await;
18847 actor.roster.close();
18848 let turns = actor.turns();
18849 tokio::time::advance(Duration::from_secs(300)).await;
18850 let parked = actor.wait_for_select(|at| at.turn > turns).await;
18851 assert_eq!(parked.wake_after, None);
18852 assert_eq!(actor.module.status().unwrap().pid, pid);
18853 assert!(actor.module.terminal_history().entries.is_empty());
18854 actor.module.stop().await.unwrap();
18855 let status = actor.module.status().unwrap();
18856 assert_eq!(status.restart_count, 0);
18857 assert_eq!(
18858 actor.module.terminal_history().entries[0].disposition,
18859 TerminalDisposition::DaemonShutdown
18860 );
18861 }
18862
18863 #[tokio::test(start_paused = true)]
18864 async fn no_health_children_park_for_sixty_seconds() {
18865 let none = Actor::start(ModuleProtocol::None, Duration::from_secs(30)).await;
18866 let unregistered = Actor::start(ModuleProtocol::Subc, Duration::from_secs(30)).await;
18867 let mut unadvertised = Actor::start(ModuleProtocol::Subc, Duration::from_secs(30)).await;
18868 unadvertised.hello(1, false);
18869 for _ in 0..6000 {
18874 tokio::time::advance(Duration::from_millis(10)).await;
18875 tokio::task::yield_now().await;
18876 }
18877 for actor in [&none, &unregistered, &unadvertised] {
18878 assert!(
18879 actor.turns() <= 5,
18880 "no-health actor woke {} times",
18881 actor.turns()
18882 );
18883 assert_eq!(actor.module.state().unwrap(), ModuleState::Running);
18884 }
18885 }
18886
18887 #[tokio::test(start_paused = true)]
18888 async fn late_hello_starts_a_due_probe_in_the_notification_tick() {
18889 let mut actor = Actor::start(ModuleProtocol::Subc, Duration::ZERO).await;
18893 let tick = Instant::now();
18894 actor.hello(1, true);
18895 actor.probe().await;
18896 assert_eq!(Instant::now(), tick, "HELLO must not wait for a poll");
18897 }
18898
18899 #[tokio::test(start_paused = true)]
18900 async fn hello_keeps_the_jittered_cadence_and_catalog_updates_do_not_reset_it() {
18901 let mut actor = Actor::start(ModuleProtocol::Subc, Duration::from_secs(30)).await;
18902 let hello_tick = Instant::now();
18903 actor.hello(1, true);
18904 let armed = actor
18905 .wait_for_select(|checkpoint| {
18906 checkpoint.registered_connection == Some(ConnectionId::new(1))
18907 && checkpoint.next_probe_at.is_some()
18908 })
18909 .await;
18910 let delay = jittered_health_delay(&actor.spec.module_id, 0, actor.health.cadence);
18911 assert!((Duration::from_secs(30)..Duration::from_secs(33)).contains(&delay));
18912 let first_deadline = hello_tick + delay;
18913 assert_eq!(armed.next_probe_at, Some(first_deadline));
18914 tokio::time::advance(Duration::from_secs(29)).await;
18915 let turns = actor.turns();
18916 actor
18917 .registry
18918 .replace_catalog_for_connection(ConnectionId::new(1), vec![], None, Some(true))
18919 .unwrap();
18920 let updated = actor
18921 .wait_for_select(|checkpoint| checkpoint.turn > turns)
18922 .await;
18923 assert_eq!(
18924 updated.next_probe_at,
18925 Some(first_deadline),
18926 "catalog update must not reset cadence"
18927 );
18928 actor.no_probe();
18929 tokio::time::advance(first_deadline - Instant::now()).await;
18930 let frame = actor.probe().await;
18931 actor.answer(1, frame);
18932 let next = jittered_health_delay(&actor.spec.module_id, 1, actor.health.cadence);
18933 let next_deadline = Instant::now() + next;
18934 let rearmed = actor
18935 .wait_for_select(|checkpoint| {
18936 checkpoint
18937 .next_probe_at
18938 .is_some_and(|deadline| deadline > first_deadline)
18939 })
18940 .await;
18941 assert_eq!(rearmed.next_probe_at, Some(next_deadline));
18942 tokio::time::advance(next - Duration::from_nanos(1)).await;
18943 actor.no_probe();
18944 tokio::time::advance(Duration::from_nanos(1)).await;
18945 actor.probe().await;
18946 }
18947
18948 #[tokio::test(start_paused = true)]
18949 async fn disconnect_disarms_and_reconnect_rearms_health() {
18950 let mut actor = Actor::start(ModuleProtocol::Subc, Duration::from_secs(30)).await;
18951 actor.hello(1, true);
18952 actor
18953 .wait_for_select(|checkpoint| {
18954 checkpoint.registered_connection == Some(ConnectionId::new(1))
18955 && checkpoint.next_probe_at.is_some()
18956 })
18957 .await;
18958 let turns = actor.turns();
18959 let tick = Instant::now();
18960 actor
18961 .registry
18962 .deregister_connection(ConnectionId::new(1))
18963 .unwrap();
18964 let disarmed = actor
18965 .wait_for_select(|checkpoint| {
18966 checkpoint.turn > turns && checkpoint.registered_connection.is_none()
18967 })
18968 .await;
18969 assert_eq!(disarmed.next_probe_at, None);
18970 assert_eq!(disarmed.wake_after, None);
18971 assert_eq!(
18972 Instant::now(),
18973 tick,
18974 "disconnect must disarm in the notification tick"
18975 );
18976 tokio::time::advance(Duration::from_secs(60)).await;
18977 actor.no_probe();
18978 let reconnect_tick = Instant::now();
18979 actor.hello(2, true);
18980 let rearmed = actor
18981 .wait_for_select(|checkpoint| {
18982 checkpoint.registered_connection == Some(ConnectionId::new(2))
18983 && checkpoint.next_probe_at.is_some()
18984 })
18985 .await;
18986 let delay = jittered_health_delay(&actor.spec.module_id, 0, actor.health.cadence);
18987 assert_eq!(rearmed.next_probe_at, Some(reconnect_tick + delay));
18988 tokio::time::advance(delay).await;
18989 actor.probe().await;
18990 }
18991
18992 #[tokio::test(start_paused = true)]
18993 async fn reregistration_without_health_disarms_the_old_schedule() {
18994 let mut actor = Actor::start(ModuleProtocol::Subc, Duration::from_secs(30)).await;
18995 actor.hello(1, true);
18996 actor
18997 .wait_for_select(|checkpoint| {
18998 checkpoint.registered_connection == Some(ConnectionId::new(1))
18999 && checkpoint.next_probe_at.is_some()
19000 })
19001 .await;
19002 actor
19003 .registry
19004 .deregister_connection(ConnectionId::new(1))
19005 .unwrap();
19006 actor.hello(2, false);
19007 let disarmed = actor
19008 .wait_for_select(|checkpoint| {
19009 checkpoint.registered_connection == Some(ConnectionId::new(2))
19010 })
19011 .await;
19012 assert_eq!(disarmed.next_probe_at, None);
19013 assert_eq!(disarmed.wake_after, None);
19014 tokio::time::advance(Duration::from_secs(60)).await;
19015 actor.no_probe();
19016 }
19017
19018 #[tokio::test(start_paused = true)]
19019 async fn restart_waits_for_its_own_hello_before_rearming_health() {
19020 let mut actor = Actor::start(ModuleProtocol::Subc, Duration::from_secs(30)).await;
19021 actor.hello(1, true);
19022 let armed = actor
19023 .wait_for_select(|checkpoint| {
19024 checkpoint.registered_connection == Some(ConnectionId::new(1))
19025 && checkpoint.next_probe_at.is_some()
19026 })
19027 .await;
19028 actor
19032 .registry
19033 .deregister_connection(ConnectionId::new(1))
19034 .unwrap();
19035 actor.module.restart(Some(0)).await.unwrap();
19036 let parked = actor
19037 .wait_for_select(|checkpoint| {
19038 checkpoint.generation > armed.generation
19039 && checkpoint.registered_connection.is_none()
19040 && checkpoint.next_probe_at.is_none()
19041 })
19042 .await;
19043 assert_eq!(parked.wake_after, Some(DEFAULT_REGISTER_TIMEOUT));
19044 actor.rx.close();
19045 while actor.rx.try_recv().is_ok() {}
19048 let turns = actor.turns();
19049 tokio::time::advance(Duration::from_secs(60)).await;
19050 let after = actor.turns();
19051 eprintln!(
19052 "restart park turns: before={turns} after={after} delta={}",
19053 after - turns
19054 );
19055 assert_eq!(after, turns, "replacement must park before HELLO");
19056 let hello_tick = Instant::now();
19057 actor.hello(2, true);
19058 let rearmed = actor
19059 .wait_for_select(|checkpoint| {
19060 checkpoint.registered_connection == Some(ConnectionId::new(2))
19061 && checkpoint.next_probe_at.is_some()
19062 })
19063 .await;
19064 let delay = jittered_health_delay(&actor.spec.module_id, 0, actor.health.cadence);
19065 assert_eq!(rearmed.next_probe_at, Some(hello_tick + delay));
19066 tokio::time::advance(delay).await;
19067 actor.probe().await;
19068 }
19069
19070 #[tokio::test(start_paused = true)]
19071 async fn http_probe_waits_a_full_cadence_for_each_spawn() {
19072 use tokio::{
19073 io::{AsyncReadExt, AsyncWriteExt},
19074 net::TcpListener,
19075 };
19076 let mut actor = Actor::start(ModuleProtocol::None, Duration::from_secs(30)).await;
19077 let listener = TcpListener::bind("127.0.0.1:0").await.unwrap();
19078 actor.health.http = Some(format!("http://{}/healthz", listener.local_addr().unwrap()));
19079 let (started_tx, mut started_rx) = mpsc::channel(8);
19080 let peer = tokio::spawn(async move {
19081 loop {
19082 let (mut socket, _) = listener.accept().await.unwrap();
19083 let mut request = [0; 1024];
19084 assert!(socket.read(&mut request).await.unwrap() > 0);
19085 started_tx.send(Instant::now()).await.unwrap();
19086 socket
19087 .write_all(b"HTTP/1.1 200 OK\r\nContent-Length: 0\r\n\r\n")
19088 .await
19089 .unwrap();
19090 }
19091 });
19092 actor
19093 .module
19094 .update_configuration(actor.spec.clone(), actor.health.clone(), None)
19095 .await
19096 .unwrap();
19097 let first_tick = Instant::now();
19098 let first = actor
19099 .wait_for_select(|checkpoint| checkpoint.next_probe_at.is_some())
19100 .await;
19101 assert!(first.next_probe_at.unwrap() >= first_tick + actor.health.cadence);
19102 assert!(started_rx.try_recv().is_err());
19103
19104 tokio::time::advance(
19107 first.next_probe_at.unwrap() - Instant::now() - Duration::from_secs(1),
19108 )
19109 .await;
19110 actor.module.restart(Some(0)).await.unwrap();
19111 let spawn_tick = Instant::now();
19112 let replacement = actor
19113 .wait_for_select(|checkpoint| checkpoint.generation > first.generation)
19114 .await;
19115 let deadline = replacement.next_probe_at.unwrap();
19116 assert!(
19117 deadline >= spawn_tick + actor.health.cadence,
19118 "a replacement must not inherit its predecessor's probe deadline"
19119 );
19120 tokio::time::advance(actor.health.cadence - Duration::from_nanos(1)).await;
19121 assert!(
19122 started_rx.try_recv().is_err(),
19123 "no HTTP probe during startup grace"
19124 );
19125 tokio::time::advance(deadline - Instant::now()).await;
19126 let landed = wait_for("first replacement HTTP probe", || {
19127 started_rx.try_recv().ok()
19128 })
19129 .await;
19130 assert!(landed >= spawn_tick + actor.health.cadence);
19131 wait_for("healthy replacement", || {
19132 (actor.module.status().unwrap().health.status == SupervisorHealthStatus::Ok)
19133 .then_some(())
19134 })
19135 .await;
19136 peer.abort();
19137 }
19138
19139 #[tokio::test(start_paused = true)]
19140 async fn rescan_adds_and_removes_http_health_without_polling() {
19141 use tokio::{
19142 io::{AsyncReadExt, AsyncWriteExt},
19143 net::TcpListener,
19144 };
19145 let mut actor = Actor::start(ModuleProtocol::None, Duration::from_secs(30)).await;
19146 let listener = TcpListener::bind("127.0.0.1:0").await.unwrap();
19147 actor.health.http = Some(format!("http://{}/health", listener.local_addr().unwrap()));
19148 actor.health.cadence = Duration::from_millis(1);
19150 let (started_tx, mut started_rx) = mpsc::channel(8);
19151 let peer = tokio::spawn(async move {
19152 loop {
19153 let (mut socket, _) = listener.accept().await.unwrap();
19154 let mut request = [0; 1024];
19155 assert!(socket.read(&mut request).await.unwrap() > 0);
19156 started_tx.send(()).await.unwrap();
19157 socket
19158 .write_all(b"HTTP/1.1 200 OK\r\nContent-Length: 0\r\n\r\n")
19159 .await
19160 .unwrap();
19161 }
19162 });
19163 let tick = Instant::now();
19164 let turns = actor.turns();
19165 actor
19166 .module
19167 .update_configuration(actor.spec.clone(), actor.health.clone(), None)
19168 .await
19169 .unwrap();
19170 let armed = actor
19171 .wait_for_select(|checkpoint| {
19172 checkpoint.turn > turns && checkpoint.next_probe_at.is_some()
19173 })
19174 .await;
19175 assert_eq!(armed.next_probe_at, Some(tick + Duration::from_millis(1)));
19176 assert_eq!(Instant::now(), tick);
19177 tokio::time::advance(Duration::from_millis(1)).await;
19178 let mut started = false;
19179 wait_for("successful HTTP probe", || {
19180 started |= started_rx.try_recv().is_ok();
19181 (started && actor.module.status().unwrap().health.status == SupervisorHealthStatus::Ok)
19182 .then_some(())
19183 })
19184 .await;
19185 assert!(started, "added HTTP check must start at its first deadline");
19186 assert_eq!(
19187 actor.module.status().unwrap().health.status,
19188 SupervisorHealthStatus::Ok
19189 );
19190 assert_eq!(Instant::now(), tick + Duration::from_millis(1));
19191 actor.health.http = None;
19192 let turns = actor.turns();
19193 actor
19194 .module
19195 .update_configuration(actor.spec.clone(), actor.health.clone(), None)
19196 .await
19197 .unwrap();
19198 let parked = actor
19199 .wait_for_select(|checkpoint| {
19200 checkpoint.turn > turns && checkpoint.next_probe_at.is_none()
19201 })
19202 .await;
19203 assert_eq!(parked.wake_after, None);
19204 let turns = actor.turns();
19205 tokio::time::advance(Duration::from_secs(60)).await;
19206 assert_eq!(
19207 actor.turns(),
19208 turns,
19209 "removed HTTP check must park the actor"
19210 );
19211 assert!(
19212 started_rx.try_recv().is_err(),
19213 "removed HTTP check must stay disarmed"
19214 );
19215 assert_eq!(
19216 actor.module.status().unwrap().health,
19217 ModuleHealthStatus::default()
19218 );
19219 peer.abort();
19220 }
19221}
19222
19223#[cfg(test)]
19224mod drain_busy_reprobe_tests {
19225 use super::*;
19229 use std::collections::BTreeMap;
19230 use std::sync::Mutex as StdMutex;
19231 use subc_protocol::{manifest::Concurrency, session::ModuleControlResponse};
19232
19233 const MODULE_ID: &str = "drain-reprobe-module";
19234 const CONNECTION: u64 = 7;
19235 const GAUGE: &str = "runs_in_flight";
19236 const CADENCE: Duration = Duration::from_secs(30);
19239 const CEILING: Duration = Duration::from_secs(30);
19240
19241 #[derive(Clone, Copy)]
19243 enum Answer {
19244 BusyUntil { until: Instant, latency: Duration },
19247 OmitsGauge,
19249 Silent,
19251 }
19252
19253 struct Drain {
19254 forwarding: Arc<ForwardingTable>,
19255 runtime: SupervisorRuntimeConfig,
19256 endpoint: crate::ModuleEndpointId,
19257 probes: Arc<StdMutex<Vec<Instant>>>,
19259 responder: tokio::task::JoinHandle<()>,
19260 }
19261
19262 impl Drain {
19263 fn start(answer: Answer) -> Self {
19264 let registry = Arc::new(Registry::default());
19265 let forwarding = Arc::new(ForwardingTable::default());
19266 let health = HealthConfig {
19267 cadence: CADENCE,
19268 ..HealthConfig::default()
19269 };
19270 let runtime = Supervisor::new_for_test(registry, RestartPolicy::new(3, Duration::ZERO))
19271 .with_forwarding(forwarding.clone())
19272 .with_health_config(health)
19273 .runtime_config();
19274 let (tx, mut rx) = mpsc::channel::<crate::router::OutboundFrame>(8);
19275 let endpoint = forwarding
19276 .register_module_connection(
19277 ConnectionId::new(CONNECTION),
19278 MODULE_ID.to_string(),
19279 subc_protocol::PROTOCOL_VERSION,
19280 Concurrency::ModuleManaged,
19281 FrameSink::new(tx),
19282 )
19283 .unwrap();
19284 let probes = Arc::new(StdMutex::new(Vec::new()));
19285 let responder = tokio::spawn({
19286 let forwarding = forwarding.clone();
19287 let probes = probes.clone();
19288 async move {
19289 while let Some(frame) = rx.recv().await {
19290 let body: Value = serde_json::from_slice(&frame.body).unwrap();
19291 assert_eq!(body["op"], MODULE_CONTROL_OP_HEALTH_CHECK);
19292 probes.lock().unwrap().push(Instant::now());
19293 let metrics = match answer {
19294 Answer::BusyUntil { until, latency } => {
19295 sleep(latency).await;
19296 let busy = u64::from(Instant::now() < until);
19297 serde_json::json!({ GAUGE: busy })
19298 }
19299 Answer::OmitsGauge => serde_json::json!({ "queue_depth": 0 }),
19300 Answer::Silent => continue,
19301 };
19302 let _ = forwarding.complete_module_control_rpc(
19305 ConnectionId::new(CONNECTION),
19306 frame.header.corr,
19307 Some(MODULE_CONTROL_OP_HEALTH_CHECK),
19308 ModuleControlRpcOutcome::Response(ModuleControlResponse::HealthCheck {
19309 status: HealthStatus::Ok,
19310 detail: None,
19311 metrics: Some(metrics),
19312 }),
19313 );
19314 }
19315 }
19316 });
19317 Self {
19318 forwarding,
19319 runtime,
19320 endpoint,
19321 probes,
19322 responder,
19323 }
19324 }
19325
19326 async fn run(&self, ceiling: Duration) -> (bool, Duration) {
19329 let (drained, elapsed, _) = self.run_traced(ceiling).await;
19330 (drained, elapsed)
19331 }
19332
19333 async fn run_traced(&self, ceiling: Duration) -> (bool, Duration, DrainGaugeTrace) {
19335 let started = Instant::now();
19336 let mut trace = DrainGaugeTrace::new(&[GAUGE.to_string()]);
19337 let drained = wait_for_forwarding_quiescence(
19338 &self.forwarding,
19339 MODULE_ID,
19340 &self.runtime,
19341 self.endpoint,
19342 started + ceiling,
19343 &[GAUGE.to_string()],
19344 DrainScope::Active,
19345 &mut trace,
19346 )
19347 .await
19348 .unwrap();
19349 (drained, started.elapsed(), trace)
19350 }
19351
19352 fn probes(&self) -> Vec<Instant> {
19353 self.probes.lock().unwrap().clone()
19354 }
19355 }
19356
19357 impl Drop for Drain {
19358 fn drop(&mut self) {
19359 self.responder.abort();
19360 }
19361 }
19362
19363 #[tokio::test(start_paused = true)]
19364 async fn busy_at_first_probe_then_quiet_drains_long_before_the_ceiling() {
19365 let drain = Drain::start(Answer::BusyUntil {
19366 until: Instant::now() + Duration::from_secs(1),
19367 latency: Duration::ZERO,
19368 });
19369 let (drained, elapsed) = drain.run(CEILING).await;
19370 assert!(
19371 drained,
19372 "gauges that read zero during the drain must let it finish drained, \
19373 not wait out the {CADENCE:?} health cadence (elapsed {elapsed:?})"
19374 );
19375 assert!(
19376 elapsed < Duration::from_secs(3),
19377 "drain took {elapsed:?} after the gauges went quiet at 1s"
19378 );
19379 let probes = drain.probes();
19380 assert!(
19381 probes.len() >= 2,
19382 "the module was busy at the first probe, so a quiet verdict needs a second: {probes:?}"
19383 );
19384 }
19385
19386 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
19391 async fn an_instant_busy_first_answer_does_not_stop_later_probes_in_real_time() {
19392 let drain = Drain::start(Answer::BusyUntil {
19393 until: Instant::now() + Duration::from_millis(600),
19394 latency: Duration::ZERO,
19395 });
19396 let (drained, elapsed, trace) = drain.run_traced(CEILING).await;
19397 assert!(drained, "the gauge fell at 600ms: {trace}");
19398 assert!(
19399 elapsed < Duration::from_secs(3),
19400 "drain took {elapsed:?} after the gauge fell at 600ms: {trace}"
19401 );
19402 let probes = drain.probes();
19403 assert!(
19404 probes.len() >= 2 && trace.probes as usize == probes.len(),
19405 "the drain must re-probe after the first busy answer: {probes:?}, {trace}"
19406 );
19407 }
19408
19409 #[tokio::test(start_paused = true)]
19410 async fn busy_throughout_ends_undrained_at_the_ceiling_with_bounded_probes() {
19411 let started = Instant::now();
19412 let drain = Drain::start(Answer::BusyUntil {
19413 until: started + Duration::from_secs(3600),
19414 latency: Duration::ZERO,
19415 });
19416 let (drained, elapsed) = drain.run(CEILING).await;
19417 assert!(!drained, "a module busy throughout must not drain");
19418 assert!(
19419 elapsed >= CEILING && elapsed < CEILING + drain_busy_reprobe_interval(),
19420 "a busy module is held until the ceiling and no longer: {elapsed:?}"
19421 );
19422 let probes = drain.probes();
19425 let max_probes = (CEILING.as_millis() / drain_busy_reprobe_interval().as_millis()) + 1;
19426 assert!(
19427 probes.len() as u128 <= max_probes,
19428 "{} probes in a {CEILING:?} drain exceeds the {max_probes} the interval allows",
19429 probes.len()
19430 );
19431 for pair in probes.windows(2) {
19432 assert!(
19433 pair[1] - pair[0] >= drain_busy_reprobe_interval(),
19434 "consecutive probes {:?} apart",
19435 pair[1] - pair[0]
19436 );
19437 }
19438 }
19439
19440 #[tokio::test(start_paused = true)]
19441 async fn a_slow_probe_is_never_overlapped_by_the_next() {
19442 let latency = Duration::from_secs(1);
19445 let drain = Drain::start(Answer::BusyUntil {
19446 until: Instant::now() + Duration::from_secs(3600),
19447 latency,
19448 });
19449 let (drained, _) = drain.run(Duration::from_secs(5)).await;
19450 assert!(!drained);
19451 let probes = drain.probes();
19452 assert!(probes.len() >= 2, "the drain must re-probe: {probes:?}");
19453 for pair in probes.windows(2) {
19454 assert!(
19455 pair[1] - pair[0] >= latency + drain_busy_reprobe_interval(),
19456 "a probe started {:?} after the previous one, before it could finish",
19457 pair[1] - pair[0]
19458 );
19459 }
19460 }
19461
19462 #[tokio::test(start_paused = true)]
19463 async fn an_unanswered_probe_cannot_hold_the_drain_past_its_ceiling() {
19464 let drain = Drain::start(Answer::Silent);
19465 let ceiling = Duration::from_secs(2);
19466 let (drained, elapsed) = drain.run(ceiling).await;
19467 assert!(!drained, "an unanswered probe counts as busy");
19468 assert!(
19469 elapsed >= ceiling && elapsed < ceiling + drain_busy_reprobe_interval(),
19470 "the probe deadline is capped at the drain ceiling: {elapsed:?}"
19471 );
19472 assert_eq!(
19473 drain.probes().len(),
19474 1,
19475 "the unanswered probe holds the only slot until the ceiling"
19476 );
19477 }
19478
19479 #[derive(Clone, Default)]
19481 struct LogCapture {
19482 events: Arc<StdMutex<Vec<BTreeMap<String, String>>>>,
19483 }
19484
19485 impl<S: tracing::Subscriber> tracing_subscriber::Layer<S> for LogCapture {
19486 fn on_event(
19487 &self,
19488 event: &tracing::Event<'_>,
19489 _context: tracing_subscriber::layer::Context<'_, S>,
19490 ) {
19491 struct Fields(BTreeMap<String, String>);
19492 impl tracing::field::Visit for Fields {
19493 fn record_debug(&mut self, field: &tracing::field::Field, value: &dyn fmt::Debug) {
19494 self.0
19495 .insert(field.name().to_string(), format!("{value:?}"));
19496 }
19497 }
19498 let mut fields = Fields(BTreeMap::new());
19499 event.record(&mut fields);
19500 self.events.lock().unwrap().push(fields.0);
19501 }
19502 }
19503
19504 impl LogCapture {
19505 fn install(&self) -> tracing::subscriber::DefaultGuard {
19506 use tracing_subscriber::layer::SubscriberExt;
19507 tracing::subscriber::set_default(tracing_subscriber::registry().with(self.clone()))
19508 }
19509
19510 fn with_message(&self, needle: &str) -> Vec<BTreeMap<String, String>> {
19511 self.events
19512 .lock()
19513 .unwrap()
19514 .iter()
19515 .filter(|fields| fields.get("message").is_some_and(|m| m.contains(needle)))
19516 .cloned()
19517 .collect()
19518 }
19519 }
19520
19521 #[tokio::test(start_paused = true)]
19526 async fn a_missing_declared_gauge_is_named_on_the_drain_timeout_line() {
19527 let capture = LogCapture::default();
19528 let _guard = capture.install();
19529 let drain = Drain::start(Answer::OmitsGauge);
19530 let (drained, _, trace) = drain.run_traced(Duration::from_secs(2)).await;
19531 assert!(!drained, "a gauge never reported cannot read quiet");
19532 assert!(trace.probes >= 2, "the drain must re-probe: {trace:?}");
19533
19534 let missing = capture.with_message("lacks a declared busy gauge");
19535 assert_eq!(missing.len(), 1, "one warning per drain: {missing:?}");
19536 assert_eq!(
19537 missing[0].get("missing_gauges").map(String::as_str),
19538 Some(GAUGE)
19539 );
19540
19541 warn_drain_timed_out(
19542 MODULE_ID,
19543 Duration::from_secs(2),
19544 RouteCloseReason::Restart,
19545 &Default::default(),
19546 &trace,
19547 );
19548 let timeouts = capture.with_message("route drain timed out");
19549 assert_eq!(timeouts.len(), 1, "{timeouts:?}");
19550 let gauges = &timeouts[0]["busy_gauges"];
19551 assert!(
19552 gauges.starts_with(&format!("{GAUGE}: missing_from_health_body @")),
19553 "the timeout line must name the gauge and why it read busy: {gauges}"
19554 );
19555 assert!(
19556 gauges.ends_with("; no value ever read"),
19557 "no value was ever read for the gauge: {gauges}"
19558 );
19559 assert_eq!(timeouts[0]["gauge_probes"], trace.probes.to_string());
19560 }
19561
19562 #[tokio::test(start_paused = true)]
19566 async fn the_drain_trace_names_a_busy_value_and_an_unanswered_probe() {
19567 let busy = Drain::start(Answer::BusyUntil {
19568 until: Instant::now() + Duration::from_secs(3600),
19569 latency: Duration::ZERO,
19570 });
19571 let (_, _, trace) = busy.run_traced(Duration::from_secs(1)).await;
19572 let rendered = trace.to_string();
19573 assert!(
19574 rendered.starts_with(&format!("{GAUGE}: non_zero(1) @"))
19575 && rendered.contains("; last value non_zero(1) @"),
19576 "{rendered}"
19577 );
19578
19579 let silent = Drain::start(Answer::Silent);
19580 let (_, _, trace) = silent.run_traced(Duration::from_secs(1)).await;
19581 assert_eq!(
19584 trace.to_string(),
19585 format!("{GAUGE}: probe_failed(no-answer) @1000ms; no value ever read")
19586 );
19587 assert_eq!(trace.probes, 1);
19588 }
19589
19590 #[test]
19592 fn a_gauge_that_is_not_a_count_is_busy_and_shown_as_sent() {
19593 let gauges = [GAUGE.to_string()];
19594 let verdicts = gauge_verdicts(Some(&serde_json::json!({ GAUGE: 0.0 })), &gauges);
19595 assert_eq!(verdicts, vec![GaugeVerdict::NotACount("0.0".to_string())]);
19596 assert_eq!(
19597 busy_gauge_observation(&verdicts),
19598 BusyGaugeObservation::Busy
19599 );
19600 let zero = gauge_verdicts(Some(&serde_json::json!({ GAUGE: 0 })), &gauges);
19601 assert_eq!(
19602 busy_gauge_observation(&zero),
19603 BusyGaugeObservation::Quiescent
19604 );
19605 let none = gauge_verdicts(None, &gauges);
19606 assert_eq!(busy_gauge_observation(&none), BusyGaugeObservation::Omitted);
19607 }
19608}
19609
19610#[cfg(test)]
19611mod health_evidence_tests {
19612 use super::{HealthProbeError, HealthProbeEvidence};
19613 use std::collections::HashSet;
19614
19615 #[test]
19623 fn only_a_dead_lane_is_proof_of_death() {
19624 assert!(HealthProbeError::lane_dead("gone").is_proof_of_death());
19625 assert!(!HealthProbeError::no_answer("timed out").is_proof_of_death());
19629 assert!(!HealthProbeError::bad_answer("garbage").is_proof_of_death());
19630 assert!(!HealthProbeError::misconfigured("no table").is_proof_of_death());
19631 }
19632
19633 #[test]
19639 fn every_evidence_class_has_a_distinct_label() {
19640 let labels = [
19641 HealthProbeError::lane_dead("").label(),
19642 HealthProbeError::no_answer("").label(),
19643 HealthProbeError::bad_answer("").label(),
19644 HealthProbeError::misconfigured("").label(),
19645 ];
19646 let unique: HashSet<_> = labels.iter().collect();
19647 assert_eq!(unique.len(), labels.len(), "labels collided: {labels:?}");
19648 }
19649
19650 #[test]
19656 fn classification_preserves_the_original_message() {
19657 let err = HealthProbeError::no_answer("module did not answer within 5s");
19658 assert_eq!(err.to_string(), "module did not answer within 5s");
19659 assert!(matches!(err.evidence, HealthProbeEvidence::NoAnswer));
19660 }
19661}
19662
19663#[cfg(test)]
19664mod health_tombstone_tests {
19665 use std::{path::PathBuf, sync::Arc, time::Duration};
19666
19667 use subc_protocol::{
19668 manifest::Concurrency,
19669 session::{HealthStatus, ModuleControlResponse},
19670 };
19671 use tokio::sync::mpsc;
19672
19673 use super::{
19674 probe_module_health, HealthAction, HealthConfig, HealthProbeEvidence, ModuleProtocol,
19675 ModuleSpec, RestartPolicy, Supervisor, SupervisorRuntimeConfig,
19676 };
19677 use crate::{
19678 control::ControlHandler,
19679 forwarding::{ForwardingTable, ModuleControlRpcCompletion, ModuleControlRpcOutcome},
19680 registry::{ConnectionId, Registry},
19681 router::FrameSink,
19682 };
19683
19684 struct ProbeHarness {
19685 spec: ModuleSpec,
19686 runtime: SupervisorRuntimeConfig,
19687 forwarding: Arc<ForwardingTable>,
19688 module_connection: ConnectionId,
19689 module_rx: mpsc::Receiver<crate::router::OutboundFrame>,
19690 handler: ControlHandler,
19691 module: super::SupervisedModule,
19692 }
19693
19694 fn probe_harness() -> ProbeHarness {
19695 let registry = Arc::new(Registry::default());
19696 let forwarding = Arc::new(ForwardingTable::default());
19697 let supervisor_handle = super::SupervisorHandle::new();
19698 let health = HealthConfig {
19699 http: None,
19700 cadence: Duration::from_secs(30),
19701 deadline: Duration::from_secs(5),
19702 failure_threshold: 3,
19703 on_degraded: HealthAction::Report,
19704 on_failing: HealthAction::Report,
19705 critical: false,
19706 };
19707 let supervisor = Supervisor::new_for_test(Arc::clone(®istry), RestartPolicy::default())
19708 .with_forwarding(Arc::clone(&forwarding))
19709 .with_handle(supervisor_handle.clone())
19710 .with_health_config(health);
19711 let spec = ModuleSpec {
19712 script: None,
19713 module_id: "late-health-module".to_string(),
19714 program: PathBuf::from("disabled-module"),
19715 args: Vec::new(),
19716 env: Vec::new(),
19717 reserved: false,
19718 reserved_prefixes: Vec::new(),
19719 protocol: ModuleProtocol::Subc,
19720 overlap: Default::default(),
19721 carry_containment: false,
19722 };
19723 let module = supervisor
19724 .supervise_configured(spec.clone(), false)
19725 .unwrap();
19726 let runtime = supervisor.runtime_config();
19727 let handler = ControlHandler::with_forwarding(registry, Arc::clone(&forwarding))
19728 .with_supervisor(supervisor_handle);
19729 let module_connection = ConnectionId::new(700);
19730 let (module_tx, module_rx) = mpsc::channel(8);
19731 forwarding
19732 .register_module_connection(
19733 module_connection,
19734 spec.module_id.clone(),
19735 subc_protocol::PROTOCOL_VERSION,
19736 Concurrency::ModuleManaged,
19737 FrameSink::new(module_tx),
19738 )
19739 .unwrap();
19740
19741 ProbeHarness {
19742 spec,
19743 runtime,
19744 forwarding,
19745 module_connection,
19746 module_rx,
19747 handler,
19748 module,
19749 }
19750 }
19751
19752 async fn finish_after(
19753 harness: &mut ProbeHarness,
19754 stall: Duration,
19755 ) -> ModuleControlRpcCompletion {
19756 assert!(stall > harness.runtime.health.deadline);
19757 let deadline = harness.runtime.health.deadline;
19758 let probe = probe_module_health(&harness.spec.module_id, &harness.runtime, None);
19759 let answer = async {
19760 let frame = harness.module_rx.recv().await.expect("health.check frame");
19761 tokio::time::advance(deadline).await;
19762 tokio::task::yield_now().await;
19763 tokio::time::advance(stall - deadline).await;
19764 harness
19765 .forwarding
19766 .complete_module_control_rpc(
19767 harness.module_connection,
19768 frame.header.corr,
19769 Some("health.check"),
19770 ModuleControlRpcOutcome::Response(ModuleControlResponse::HealthCheck {
19771 status: HealthStatus::Ok,
19772 detail: None,
19773 metrics: None,
19774 }),
19775 )
19776 .unwrap()
19777 };
19778 let (probe_result, completion) = tokio::join!(probe, answer);
19779 let err = probe_result.expect_err("probe must miss its deadline");
19780 assert!(matches!(err.evidence, HealthProbeEvidence::NoAnswer));
19781 completion
19782 }
19783
19784 async fn time_out_without_answer(harness: &mut ProbeHarness) {
19785 let deadline = harness.runtime.health.deadline;
19786 let probe = probe_module_health(&harness.spec.module_id, &harness.runtime, None);
19787 let exhaust_deadline = async {
19788 let _frame = harness.module_rx.recv().await.expect("health.check frame");
19789 tokio::time::advance(deadline).await;
19790 tokio::task::yield_now().await;
19791 };
19792 let (probe_result, ()) = tokio::join!(probe, exhaust_deadline);
19793 let err = probe_result.expect_err("probe must miss its deadline");
19794 assert!(matches!(err.evidence, HealthProbeEvidence::NoAnswer));
19795 }
19796
19797 async fn run_probe_cycle(harness: &mut ProbeHarness, answer: bool) {
19798 let registry = Arc::clone(&harness.module.inner.registry);
19799 let snapshot = Arc::clone(&harness.module.inner.snapshot);
19800 let process_liveness = super::SupervisorProcessLiveness::default();
19801 let mut child = None;
19802 let cycle = super::run_health_probe_cycle(
19803 &harness.spec,
19804 &harness.runtime,
19805 ®istry,
19806 &process_liveness,
19807 &snapshot,
19808 &mut child,
19809 );
19810 let peer = async {
19811 let frame = harness.module_rx.recv().await.expect("health.check frame");
19812 if answer {
19813 harness
19814 .forwarding
19815 .complete_module_control_rpc(
19816 harness.module_connection,
19817 frame.header.corr,
19818 Some("health.check"),
19819 ModuleControlRpcOutcome::Response(ModuleControlResponse::HealthCheck {
19820 status: HealthStatus::Ok,
19821 detail: None,
19822 metrics: Some(serde_json::json!({"ready": true})),
19823 }),
19824 )
19825 .unwrap();
19826 } else {
19827 tokio::time::advance(harness.runtime.health.deadline).await;
19828 tokio::task::yield_now().await;
19829 }
19830 };
19831 tokio::join!(cycle, peer);
19832 }
19833
19834 #[tokio::test(start_paused = true)]
19835 async fn unanswered_probe_is_unknown_until_threshold_and_ok_report_recovers() {
19836 let mut harness = probe_harness();
19837 let monitor = harness.module.inner.monitor.lock().unwrap().take().unwrap();
19841 monitor.abort();
19842 let _ = monitor.await;
19843 super::update_snapshot(&harness.module.inner.snapshot, None, |state| {
19844 state.enabled = true;
19845 state.state = super::ModuleState::Running;
19846 state.process_alive = true;
19847 })
19848 .unwrap();
19849
19850 run_probe_cycle(&mut harness, true).await;
19851 assert_eq!(
19852 harness.module.status().unwrap().health.status,
19853 super::SupervisorHealthStatus::Ok
19854 );
19855
19856 for failures in 1..harness.runtime.health.failure_threshold {
19857 run_probe_cycle(&mut harness, false).await;
19858 let status = harness.module.status().unwrap();
19859 assert_eq!(status.health.status, super::SupervisorHealthStatus::Unknown);
19860 assert_eq!(status.health.consecutive_failures, failures);
19861 assert!(status.health.last_probe_ms.is_some());
19862 assert!(status.health.detail.unwrap().starts_with("[no-answer]"));
19863 assert!(status.health.metrics.is_none());
19864 assert_eq!(status.state, super::ModuleState::Running);
19865 assert!(status.process_alive);
19866 assert_eq!(status.restart_count, 0);
19867 assert_eq!(status.lifetime_restarts, 0);
19868 assert!(status.health.last_action.is_none());
19869 assert!(harness.runtime.scheduled_respawn.lock().unwrap().is_none());
19870 }
19871
19872 run_probe_cycle(&mut harness, true).await;
19873 let recovered = harness.module.status().unwrap();
19874 assert_eq!(recovered.health.status, super::SupervisorHealthStatus::Ok);
19875 assert_eq!(recovered.health.consecutive_failures, 0);
19876 assert!(recovered.health.detail.is_none());
19877 assert_eq!(
19878 recovered.health.metrics,
19879 Some(serde_json::json!({"ready": true}))
19880 );
19881 assert_eq!(recovered.lifetime_restarts, 0);
19882
19883 for failures in 1..=harness.runtime.health.failure_threshold {
19884 run_probe_cycle(&mut harness, false).await;
19885 let status = harness.module.status().unwrap();
19886 assert_eq!(status.health.consecutive_failures, failures);
19887 if failures < harness.runtime.health.failure_threshold {
19888 assert_eq!(status.health.status, super::SupervisorHealthStatus::Unknown);
19889 assert_eq!(status.state, super::ModuleState::Running);
19890 assert_eq!(status.lifetime_restarts, 0);
19891 assert!(harness.runtime.scheduled_respawn.lock().unwrap().is_none());
19892 } else {
19893 assert_eq!(
19894 status.health.status,
19895 super::SupervisorHealthStatus::Unresponsive
19896 );
19897 assert_eq!(status.state, super::ModuleState::Restarting);
19898 assert_eq!(status.restart_count, 1);
19899 assert_eq!(status.lifetime_restarts, 1);
19900 assert_eq!(status.health.last_action.as_deref(), Some("restart"));
19901 assert!(harness.runtime.scheduled_respawn.lock().unwrap().is_some());
19902 }
19903 }
19904 }
19905
19906 #[tokio::test(start_paused = true)]
19907 async fn late_health_answers_record_start_anchored_latency_for_two_stalls() {
19908 let mut harness = probe_harness();
19909
19910 let first = finish_after(&mut harness, Duration::from_secs(8)).await;
19911 let first_latency = match &first {
19912 ModuleControlRpcCompletion::LateHealthAnswer { latency, .. } => *latency,
19913 other => panic!("late answer was not retained: {other:?}"),
19914 };
19915 assert!(harness.handler.observe_module_control_completion(first));
19916
19917 let second = finish_after(&mut harness, Duration::from_secs(11)).await;
19918 let second_latency = match &second {
19919 ModuleControlRpcCompletion::LateHealthAnswer { latency, .. } => *latency,
19920 other => panic!("late answer was not retained: {other:?}"),
19921 };
19922 assert!(harness.handler.observe_module_control_completion(second));
19923
19924 assert_eq!(first_latency, Duration::from_secs(8));
19925 assert_eq!(
19926 second_latency - first_latency,
19927 Duration::from_secs(3),
19928 "latency must grow linearly with the additional stall"
19929 );
19930 let health = harness.module.status().unwrap().health;
19931 assert_eq!(health.late_answer_count, 2);
19932 assert_eq!(health.last_late_answer_latency_ms, Some(11_000));
19933 }
19934
19935 #[tokio::test(start_paused = true)]
19943 async fn late_answer_clears_the_consecutive_failure_streak() {
19944 let mut harness = probe_harness();
19945
19946 time_out_without_answer(&mut harness).await;
19948 harness
19949 .module
19950 .record_health_probe_failure_for_test("[no-answer] test miss")
19951 .unwrap();
19952 assert_eq!(
19953 harness.module.status().unwrap().health.consecutive_failures,
19954 1,
19955 "precondition: the miss must be on the streak before the late answer"
19956 );
19957
19958 let late = finish_after(&mut harness, Duration::from_secs(9)).await;
19960 assert!(matches!(
19961 late,
19962 ModuleControlRpcCompletion::LateHealthAnswer { .. }
19963 ));
19964 assert!(harness.handler.observe_module_control_completion(late));
19965
19966 let health = harness.module.status().unwrap().health;
19967 assert_eq!(
19968 health.consecutive_failures, 0,
19969 "a late answer is an answer: the streak must reset"
19970 );
19971 assert_eq!(health.late_answer_count, 1);
19972 }
19973
19974 #[tokio::test(start_paused = true)]
19975 async fn repeated_serial_probe_cycles_keep_one_tombstone_per_endpoint() {
19976 let mut harness = probe_harness();
19977
19978 for _ in 0..20 {
19979 time_out_without_answer(&mut harness).await;
19980 assert_eq!(
19981 harness.forwarding.health_probe_tombstone_count().unwrap(),
19982 1
19983 );
19984 }
19985 }
19986}
19987
19988#[cfg(test)]
19989mod child_env_tests {
19990 #[cfg(windows)]
19991 use super::{
19992 clear_current_process_facts, observed_launch_nonce_source, ModuleState, SupervisorSnapshot,
19993 };
19994 #[cfg(windows)]
19995 use std::sync::{Arc, OnceLock};
19996 #[cfg(windows)]
19997 #[test]
19998 fn windows_wire_spawn_prepares_pipe_and_keeps_phase_one_env() {
19999 let spec = spec(Vec::new());
20000 let mut command = Command::new("unused");
20001 let handoff = apply_wire_spawn_args(&mut command, &spec, None, None)
20002 .unwrap()
20003 .unwrap();
20004 let name = command
20005 .as_std()
20006 .get_envs()
20007 .find(|(key, _)| *key == OsStr::new(subc_os::launch_nonce::LAUNCH_NONCE_PIPE_ENV))
20008 .and_then(|(_, value)| value)
20009 .unwrap();
20010 assert_eq!(name, OsStr::new(handoff.name()));
20011 assert!(handoff.name().starts_with(r"\\.\pipe\subc-launch-"));
20012 assert!(command.as_std().get_envs().any(|(key, value)| key
20013 == OsStr::new(subc_os::LAUNCH_NONCE_PIPE_FALLBACK_ENV)
20014 && value == Some(OsStr::new("env"))));
20015 assert!(command
20016 .as_std()
20017 .get_envs()
20018 .any(|(key, value)| key == OsStr::new(SUBC_LAUNCH_NONCE_ENV) && value.is_some()));
20019 }
20020
20021 #[cfg(windows)]
20022 #[test]
20023 fn windows_source_reporting_tracks_consumption_and_resets_per_spawn() {
20024 let mut snapshot = SupervisorSnapshot::new(ModuleState::Running, true);
20025 snapshot.process_alive = true;
20026 snapshot.spawned_protocol = Some(ModuleProtocol::Subc);
20027 assert_eq!(
20028 observed_launch_nonce_source(&snapshot).as_deref(),
20029 Some("env")
20030 );
20031 let consumed = Arc::new(OnceLock::new());
20032 snapshot.pipe_consumed = Some(consumed.clone());
20033 assert_eq!(
20034 observed_launch_nonce_source(&snapshot).as_deref(),
20035 Some("env")
20036 );
20037 consumed.set(()).unwrap();
20038 assert_eq!(
20039 observed_launch_nonce_source(&snapshot).as_deref(),
20040 Some("pipe")
20041 );
20042 snapshot.pipe_consumed = Some(Arc::new(OnceLock::new()));
20043 assert_eq!(
20044 observed_launch_nonce_source(&snapshot).as_deref(),
20045 Some("env")
20046 );
20047 clear_current_process_facts(&mut snapshot);
20048 assert_eq!(observed_launch_nonce_source(&snapshot), None);
20049 }
20050 use super::{
20051 apply_child_env, apply_spawn_role, apply_wire_spawn_args, LaunchSpecDigest, ModuleProtocol,
20052 ModuleSpec, SpawnRole, SupervisorHandle, SPAWN_ROLE_SWAP_CANDIDATE, SUBC_ARG,
20053 SUBC_LAUNCH_NONCE_ENV, SUBC_MODULE_ID_ENV, SUBC_SPAWN_ROLE_ENV,
20054 };
20055 use std::{ffi::OsStr, path::PathBuf};
20056 use tokio::process::Command;
20057
20058 fn spec(env: Vec<(String, String)>) -> ModuleSpec {
20059 ModuleSpec {
20060 script: None,
20061 module_id: "env-plan".to_string(),
20062 program: PathBuf::from("/nonexistent"),
20063 args: Vec::new(),
20064 env,
20065 reserved: false,
20066 reserved_prefixes: Vec::new(),
20067 protocol: ModuleProtocol::Subc,
20068 overlap: Default::default(),
20069 carry_containment: false,
20070 }
20071 }
20072
20073 #[test]
20074 fn configured_launch_spec_digest_is_canonical_and_excludes_injected_inputs() {
20075 let base = spec(vec![("A".into(), "B=C".into()), ("Z".into(), "v".into())]);
20076 let digest = LaunchSpecDigest::from_spec(&base);
20077 let mut reordered = base.clone();
20078 reordered.env.reverse();
20079 reordered.module_id = "different-id".into();
20080 reordered.reserved = true;
20081 reordered.overlap = Default::default();
20082 assert_eq!(digest, LaunchSpecDigest::from_spec(&reordered));
20083 let mut delimiter_collision = base.clone();
20084 delimiter_collision.env[0] = ("A=B".into(), "C".into());
20085 assert_ne!(digest, LaunchSpecDigest::from_spec(&delimiter_collision));
20086 let mut changed = base.clone();
20087 changed.protocol = ModuleProtocol::None;
20088 assert_ne!(digest, LaunchSpecDigest::from_spec(&changed));
20089 changed = base.clone();
20090 changed.args.push("--flag".into());
20091 assert_ne!(digest, LaunchSpecDigest::from_spec(&changed));
20092 changed = base.clone();
20093 changed.program = PathBuf::from("/other");
20094 assert_ne!(digest, LaunchSpecDigest::from_spec(&changed));
20095 changed = base.clone();
20096 changed.env[1].1 = "other".into();
20097 assert_ne!(digest, LaunchSpecDigest::from_spec(&changed));
20098 assert!(!format!("{digest:?}").contains("B=C"));
20099 assert!(!format!("{digest:?}").contains("[u8"));
20100 }
20101
20102 #[cfg(unix)]
20103 #[test]
20104 fn native_program_bytes_do_not_collapse_through_lossy_utf8() {
20105 use std::os::unix::ffi::OsStringExt;
20106 let mut left = spec(Vec::new());
20107 left.program = std::ffi::OsString::from_vec(vec![b'/', 0xff]).into();
20108 let mut right = left.clone();
20109 right.program = std::ffi::OsString::from_vec(vec![b'/', 0xfe]).into();
20110 assert_eq!(
20111 left.program.to_string_lossy(),
20112 right.program.to_string_lossy()
20113 );
20114 assert_ne!(
20115 LaunchSpecDigest::from_spec(&left),
20116 LaunchSpecDigest::from_spec(&right)
20117 );
20118 }
20119
20120 #[test]
20134 fn ambient_ck_log_is_removed_and_a_configured_one_survives() {
20135 let mut command = Command::new("/nonexistent");
20136 apply_child_env(&mut command, &spec(Vec::new()));
20137 let removed = command
20138 .as_std()
20139 .get_envs()
20140 .any(|(key, value)| key == OsStr::new("CK_LOG") && value.is_none());
20141 assert!(
20142 removed,
20143 "ambient CK_LOG must be explicitly removed for an unconfigured module"
20144 );
20145
20146 let mut configured = Command::new("/nonexistent");
20147 apply_child_env(
20148 &mut configured,
20149 &spec(vec![("CK_LOG".to_string(), "debug".to_string())]),
20150 );
20151 let effective = configured
20152 .as_std()
20153 .get_envs()
20154 .filter(|(key, _)| *key == OsStr::new("CK_LOG"))
20155 .last()
20156 .map(|(_, value)| value.map(|v| v.to_string_lossy().into_owned()));
20157 assert_eq!(
20158 effective,
20159 Some(Some("debug".to_string())),
20160 "a module's configured CK_LOG must survive the ambient removal"
20161 );
20162 }
20163
20164 #[test]
20173 fn protocol_none_spawn_carries_no_subc_argument_and_no_nonce() {
20174 let connection_file = std::path::Path::new("/run/subc-connection.json");
20175 let handle = SupervisorHandle::new();
20176
20177 let mut none_spec = spec(Vec::new());
20178 none_spec.protocol = ModuleProtocol::None;
20179 let mut none = Command::new("/nonexistent");
20180 let none_handoff =
20181 apply_wire_spawn_args(&mut none, &none_spec, Some(connection_file), Some(&handle))
20182 .expect("protocol-none spawn args apply");
20183 assert!(
20184 none_handoff.is_none(),
20185 "protocol:none spawn must not receive a nonce descriptor"
20186 );
20187 #[cfg(windows)]
20188 assert!(!none.as_std().get_envs().any(|(key, value)| key
20189 == OsStr::new(subc_os::LAUNCH_NONCE_PIPE_FALLBACK_ENV)
20190 && value.is_some()));
20191 assert!(
20192 !none.as_std().get_envs().any(|(key, value)| key
20193 == OsStr::new(subc_os::LAUNCH_NONCE_FD_ENV)
20194 && value.is_some()),
20195 "protocol:none spawn must not name a nonce descriptor"
20196 );
20197 let none_args: Vec<String> = none
20198 .as_std()
20199 .get_args()
20200 .map(|a| a.to_string_lossy().into_owned())
20201 .collect();
20202 assert!(
20203 !none_args.iter().any(|a| a == SUBC_ARG),
20204 "protocol:none argv must not carry --subc; got {none_args:?}"
20205 );
20206 let none_has_nonce = none
20207 .as_std()
20208 .get_envs()
20209 .any(|(key, value)| key == OsStr::new(SUBC_LAUNCH_NONCE_ENV) && value.is_some());
20210 assert!(
20211 !none_has_nonce,
20212 "protocol:none spawn must not receive a launch nonce"
20213 );
20214 let none_has_module_id = none
20215 .as_std()
20216 .get_envs()
20217 .any(|(key, value)| key == OsStr::new(SUBC_MODULE_ID_ENV) && value.is_some());
20218 assert!(
20219 none_has_module_id,
20220 "SUBC_MODULE_ID is inert and stays on every path"
20221 );
20222 assert!(
20223 handle.spawn_nonce(&none_spec.module_id).is_none(),
20224 "no nonce record for a process that will never present one"
20225 );
20226
20227 let wire_spec = spec(Vec::new());
20229 let mut wire = Command::new("/nonexistent");
20230 let wire_handoff =
20231 apply_wire_spawn_args(&mut wire, &wire_spec, Some(connection_file), Some(&handle))
20232 .expect("subc-wire spawn args apply");
20233 let wire_fd_env = wire
20234 .as_std()
20235 .get_envs()
20236 .find(|(key, _)| *key == OsStr::new(subc_os::LAUNCH_NONCE_FD_ENV))
20237 .and_then(|(_, value)| value.map(|v| v.to_string_lossy().into_owned()));
20238 #[cfg(unix)]
20239 assert_eq!(
20240 wire_fd_env,
20241 Some(wire_handoff.expect("a descriptor handoff").fd_env_value()),
20242 "a subc-wire spawn names the pipe it will receive at descriptor 3"
20243 );
20244 #[cfg(windows)]
20245 assert!(wire_handoff.is_some() && wire_fd_env.is_none());
20246 #[cfg(not(any(unix, windows)))]
20247 assert!(wire_handoff.is_none() && wire_fd_env.is_none());
20248 let wire_args: Vec<String> = wire
20249 .as_std()
20250 .get_args()
20251 .map(|a| a.to_string_lossy().into_owned())
20252 .collect();
20253 assert_eq!(
20254 wire_args,
20255 vec![
20256 SUBC_ARG.to_string(),
20257 connection_file.to_string_lossy().into_owned()
20258 ],
20259 "a subc-wire spawn still carries --subc <path>"
20260 );
20261 assert_eq!(
20262 wire.as_std()
20263 .get_envs()
20264 .any(|(key, value)| key == OsStr::new(SUBC_LAUNCH_NONCE_ENV) && value.is_some()),
20265 !cfg!(unix),
20266 "only Windows supplies the environment nonce"
20267 );
20268 assert!(handle.spawn_nonce(&wire_spec.module_id).is_some());
20269 }
20270
20271 #[test]
20281 fn plain_spawn_removes_the_spawn_role_even_when_the_spec_sets_it() {
20282 let role = |command: &Command| {
20283 command
20284 .as_std()
20285 .get_envs()
20286 .filter(|(key, _)| *key == OsStr::new(SUBC_SPAWN_ROLE_ENV))
20287 .last()
20288 .map(|(_, value)| value.map(|v| v.to_string_lossy().into_owned()))
20289 };
20290 let forged = spec(vec![(
20291 SUBC_SPAWN_ROLE_ENV.to_string(),
20292 SPAWN_ROLE_SWAP_CANDIDATE.to_string(),
20293 )]);
20294
20295 let mut plain = Command::new("/nonexistent");
20296 apply_child_env(&mut plain, &forged);
20297 apply_spawn_role(&mut plain, SpawnRole::Plain);
20298 assert_eq!(
20299 role(&plain),
20300 Some(None),
20301 "a plain spawn must remove SUBC_SPAWN_ROLE, whatever the spec says"
20302 );
20303
20304 let mut candidate = Command::new("/nonexistent");
20305 apply_child_env(&mut candidate, &spec(Vec::new()));
20306 apply_spawn_role(&mut candidate, SpawnRole::SwapCandidate);
20307 assert_eq!(
20308 role(&candidate),
20309 Some(Some(SPAWN_ROLE_SWAP_CANDIDATE.to_string()))
20310 );
20311 }
20312
20313 #[test]
20319 fn daemon_private_capture_keys_are_not_passed_to_the_child() {
20320 let mut command = Command::new("/nonexistent");
20321 apply_child_env(
20322 &mut command,
20323 &spec(vec![
20324 (super::CAPTURE_KEEP_ENV.to_string(), "5".to_string()),
20325 ("KEPT".to_string(), "yes".to_string()),
20326 ]),
20327 );
20328 let keys: Vec<String> = command
20329 .as_std()
20330 .get_envs()
20331 .filter(|(_, value)| value.is_some())
20332 .map(|(key, _)| key.to_string_lossy().into_owned())
20333 .collect();
20334 assert!(keys.contains(&"KEPT".to_string()), "got {keys:?}");
20335 assert!(
20336 !keys.contains(&super::CAPTURE_KEEP_ENV.to_string()),
20337 "daemon-private capture key leaked to the child: {keys:?}"
20338 );
20339 }
20340}
20341
20342#[cfg(test)]
20343mod jitter_tests {
20344 use super::jittered_health_delay;
20345 use std::{collections::HashSet, time::Duration};
20346
20347 const FLEET: [&str; 14] = [
20356 "aft",
20357 "alfonso-core",
20358 "magic-context",
20359 "broca",
20360 "thalamus",
20361 "quota",
20362 "engram",
20363 "plexus",
20364 "cerebellum",
20365 "astrocyte",
20366 "synapse",
20367 "subc-mcp",
20368 "cortexkit-credentials",
20369 "subc-federation",
20370 ];
20371
20372 #[test]
20380 fn probe_delays_disperse_across_the_fleet() {
20381 let cadence = Duration::from_secs(30);
20382 let delays: HashSet<Duration> = FLEET
20383 .iter()
20384 .map(|id| jittered_health_delay(id, 0, cadence))
20385 .collect();
20386 assert_eq!(
20387 delays.len(),
20388 FLEET.len(),
20389 "every supervised module must land on its own probe offset"
20390 );
20391 }
20392
20393 #[test]
20399 fn jitter_only_delays_and_stays_within_one_tenth_of_cadence() {
20400 let cadence = Duration::from_secs(30);
20401 let span = cadence / 10;
20402 for id in FLEET {
20403 for probe_index in 0..8 {
20404 let delay = jittered_health_delay(id, probe_index, cadence);
20405 assert!(
20406 delay >= cadence,
20407 "{id}#{probe_index}: jitter must not shorten the cadence"
20408 );
20409 assert!(
20410 delay < cadence + span,
20411 "{id}#{probe_index}: jitter must stay inside one tenth of the cadence"
20412 );
20413 }
20414 }
20415 }
20416
20417 #[test]
20423 fn a_module_offset_is_stable_across_restarts() {
20424 let cadence = Duration::from_secs(30);
20425 for id in FLEET {
20426 assert_eq!(
20427 jittered_health_delay(id, 0, cadence),
20428 jittered_health_delay(id, 0, cadence),
20429 "{id}: the same module and probe index must produce the same offset"
20430 );
20431 }
20432 }
20433
20434 #[test]
20436 fn zero_cadence_yields_zero_delay() {
20437 assert_eq!(
20438 jittered_health_delay("aft", 0, Duration::ZERO),
20439 Duration::ZERO
20440 );
20441 }
20442}
20443
20444#[cfg(all(test, target_os = "linux"))]
20445mod cgroup_placement_tests {
20446 use super::{
20447 apply_cgroup_placement, remove_module_cgroup, LaunchSpecDigest, ModuleProtocol, ModuleSpec,
20448 SuperviseError, SupervisedChild,
20449 };
20450 use crate::stderr_tail::{StderrRing, StderrTailConfig};
20451 use cortexkit_test_support::ScratchDir;
20452 use std::{
20453 fs, io,
20454 path::{Path, PathBuf},
20455 sync::{Arc, Mutex},
20456 };
20457 use tokio::process::Command;
20458
20459 #[tokio::test]
20460 async fn unique_spawn_cgroups_clean_up_across_restarts_and_shutdown() {
20461 use super::*;
20462 let dir = ScratchDir::new("unique-spawn-cgroups");
20463 let root = PathBuf::from(format!(
20464 "/sys/fs/cgroup/subc-unique-test-{}-{}",
20465 std::process::id(),
20466 unix_ms_now()
20467 ));
20468 if let Err(error) = fs::create_dir(&root) {
20469 assert!(
20470 std::env::var_os("SUBC_REQUIRE_CGROUP_TEST").is_none(),
20471 "required cgroup test cannot execute: {error}"
20472 );
20473 eprintln!("SKIP unique_spawn_cgroups_clean_up_across_restarts_and_shutdown: {error}");
20474 return;
20475 }
20476 let placement = subc_cgroup::prepare_at(&root).unwrap().unwrap();
20477 let group_count = || {
20478 fs::read_dir(root.join("subc-modules"))
20479 .unwrap()
20480 .map(|entry| entry.unwrap().file_type().unwrap())
20481 .filter(|kind| kind.is_dir())
20482 .count()
20483 };
20484 let supervisor =
20485 Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default())
20486 .with_cgroup_placement(Some(placement.clone()));
20487 let runtime = supervisor.runtime_config();
20488 let mut spec = ModuleSpec {
20489 script: None,
20490 module_id: "unique-spawn".into(),
20491 program: PathBuf::from("/bin/sleep"),
20492 args: vec!["60".into()],
20493 env: ["XDG_DATA_HOME", "XDG_RUNTIME_DIR", "XDG_CONFIG_HOME"]
20494 .into_iter()
20495 .map(|key| (key.into(), dir.to_string_lossy().into_owned()))
20496 .collect(),
20497 reserved: false,
20498 reserved_prefixes: vec![],
20499 protocol: ModuleProtocol::None,
20500 overlap: Default::default(),
20501 carry_containment: false,
20502 };
20503 let spawn = |spec: &ModuleSpec| {
20504 spawn_child(
20505 spec,
20506 None,
20507 None,
20508 &runtime.stderr_ring,
20509 None,
20510 &runtime.child_roster,
20511 runtime.spawn_delivery(),
20512 Some(&placement),
20513 runtime.register_timeout,
20514 )
20515 .unwrap()
20516 };
20517 let mut live = spawn(&spec);
20518 for _ in 0..3 {
20519 let next = spawn(&spec);
20521 assert_ne!(live.module_id, next.module_id);
20522 live.start_kill().unwrap();
20523 live.wait().await.unwrap();
20524 live = next;
20525 assert!(
20526 live.child.try_wait().unwrap().is_none(),
20527 "retiring the old slot must not kill the replacement"
20528 );
20529 assert_eq!(
20530 group_count(),
20531 1,
20532 "only the live spawn's cgroup should remain"
20533 );
20534 }
20535 supervisor.begin_daemon_shutdown();
20536 let reap = tokio::spawn(async move {
20537 live.wait().await.unwrap();
20538 });
20539 supervisor
20540 .end_children_for_daemon_shutdown(false, std::future::pending())
20541 .await;
20542 reap.await.unwrap();
20543 assert_eq!(group_count(), 0);
20544 spec.program = PathBuf::from("/bin/true");
20546 spec.args.clear();
20547 let fresh_roster = ChildRoster::default();
20548 let mut short = spawn_child(
20549 &spec,
20550 None,
20551 None,
20552 &runtime.stderr_ring,
20553 None,
20554 &fresh_roster,
20555 None,
20556 Some(&placement),
20557 runtime.register_timeout,
20558 )
20559 .unwrap();
20560 short.wait().await.unwrap();
20561 assert_eq!(group_count(), 0);
20562 spec.module_id = "_".repeat(255);
20563 let mut long_id = spawn_child(
20564 &spec,
20565 None,
20566 None,
20567 &runtime.stderr_ring,
20568 None,
20569 &fresh_roster,
20570 None,
20571 Some(&placement),
20572 runtime.register_timeout,
20573 )
20574 .unwrap();
20575 long_id.wait().await.unwrap();
20576 assert_eq!(
20577 group_count(),
20578 0,
20579 "valid long module IDs must not exceed cgroup NAME_MAX"
20580 );
20581 fs::remove_dir(root.join("subc-modules")).unwrap();
20582 fs::remove_dir(root).unwrap();
20583 eprintln!("EXECUTED unique_spawn_cgroups_clean_up_across_restarts_and_shutdown");
20584 }
20585
20586 #[test]
20587 fn failed_parent_cgroup_open_is_a_cgroup_supervision_error() {
20588 let path = Path::new("/definitely-missing-subc-cgroup");
20589 let mut command = Command::new("true");
20590 let error = apply_cgroup_placement(
20591 &mut command,
20592 &ModuleSpec {
20593 script: None,
20594 module_id: "broken-cgroup".to_string(),
20595 program: PathBuf::from("true"),
20596 args: Vec::new(),
20597 env: Vec::new(),
20598 reserved: false,
20599 reserved_prefixes: Vec::new(),
20600 protocol: ModuleProtocol::Subc,
20601 overlap: Default::default(),
20602 carry_containment: false,
20603 },
20604 path,
20605 )
20606 .expect_err("a parent cgroup open failure must reject the supervised spawn");
20607 let reason = error.to_string();
20608
20609 assert!(
20610 matches!(error, SuperviseError::Cgroup { .. }),
20611 "parent cgroup open must be reported as a cgroup supervision error: {reason}"
20612 );
20613 assert!(
20614 reason.contains("/definitely-missing-subc-cgroup/cgroup.procs"),
20615 "parent cgroup open failure must name cgroup.procs: {reason}"
20616 );
20617 }
20618
20619 #[tokio::test]
20620 async fn reaping_a_child_removes_its_empty_module_cgroup() {
20621 let root = ScratchDir::new("supervisor-reap-cgroup");
20622 fs::write(root.join("cgroup.procs"), b"").expect("write scratch cgroup marker");
20623 let placement = subc_cgroup::prepare_at(&root)
20624 .expect("prepare scratch cgroup root")
20625 .expect("scratch root has a cgroup.procs marker");
20626 let module_id = "reaped-module";
20627 let module = placement
20628 .module_path(module_id)
20629 .expect("create scratch module cgroup");
20630 let child = Command::new("true")
20631 .env("XDG_DATA_HOME", root.path())
20632 .env("XDG_RUNTIME_DIR", root.path())
20633 .env("XDG_CONFIG_HOME", root.path())
20634 .spawn()
20635 .expect("spawn short-lived child");
20636 let pid = child.id().expect("spawned child has pid");
20637 let mut child = SupervisedChild {
20638 registration_channel: None,
20639 registration_spawn: None,
20640 live_program_binding: None,
20641 registered_config_connection: None,
20642 admission_base: None,
20643 registration_declaration: None,
20644 _program_binding: None,
20645 subtree_target: crate::subtree_resources::Target::Unavailable("not_running"),
20646 child: crate::supervise::containment::ProcessChild::plain(child, pid),
20647 exec: crate::activity::ExecOwner(Arc::new(crate::activity::ExecOutcome::new())),
20648 protocol: ModuleProtocol::Subc,
20649 module_id: module_id.to_string(),
20650 cgroup_placement: Some(placement),
20651 stdout_pump: None,
20652 stderr_pump: None,
20653 stderr_ring: Arc::new(Mutex::new(StderrRing::new(StderrTailConfig::default()))),
20654 spawned_at: tokio::time::Instant::now(),
20655 config_delivery: None,
20656 registration_deadline: None,
20657 registration_watch: None,
20658 spawned_at_ms: 0,
20659 spawned_from: PathBuf::from("true"),
20660 launch_spec_digest: LaunchSpecDigest([0; 32]),
20661 spawned_file_identity: None,
20662 process_start_time: None,
20663 process_identity: None,
20664 pid,
20665 roster_guard: None,
20666 #[cfg(target_os = "macos")]
20667 privacy_exec: None,
20668 spawn_failure: None,
20669 admission_kill_sent: false,
20670 };
20671
20672 child.wait().await.expect("reap short-lived child");
20673
20674 assert!(
20675 !module.exists(),
20676 "waiting for a non-carried child must remove its empty cgroup before returning"
20677 );
20678 child.finish_exit(false).await.unwrap();
20679 }
20680
20681 #[test]
20682 fn non_empty_cgroup_removal_is_reported_without_blocking_teardown() {
20683 let root = ScratchDir::new("supervisor-non-empty-cgroup");
20684 fs::write(root.join("cgroup.procs"), b"").expect("write scratch cgroup marker");
20685 let placement = subc_cgroup::prepare_at(&root)
20686 .expect("prepare scratch cgroup root")
20687 .expect("scratch root has a cgroup.procs marker");
20688 let module = placement
20689 .module_path("surviving-module")
20690 .expect("create scratch module cgroup");
20691 fs::write(module.join("surviving-process"), b"still present")
20692 .expect("make scratch cgroup non-empty");
20693 let (logs, _guard) = crate::router::test_log::log_capture(tracing::Level::WARN);
20694
20695 remove_module_cgroup(&placement, "surviving-module");
20696
20697 let logs = crate::router::test_log::captured_logs(&logs);
20698 assert!(
20699 module.exists(),
20700 "failed removal must leave the cgroup intact"
20701 );
20702 assert!(
20703 logs.contains("could not remove module cgroup after process exit; continuing teardown")
20704 && logs.contains("surviving-module"),
20705 "best-effort removal must report the failure without returning it: {logs}"
20706 );
20707 }
20708
20709 #[test]
20710 fn cgroup_pre_exec_spawn_failure_names_the_cgroup_path() {
20711 let cgroup_path = PathBuf::from("/sys/fs/cgroup/subc-modules/broken-module");
20712 let reason = SuperviseError::Spawn {
20713 program: PathBuf::from("/bin/true"),
20714 source: io::Error::from_raw_os_error(13),
20715 cgroup_path: Some(cgroup_path.clone()),
20716 }
20717 .to_string();
20718
20719 assert!(
20720 reason.contains(&cgroup_path.display().to_string()),
20721 "a pre_exec spawn failure must name the cgroup path: {reason}"
20722 );
20723 }
20724}
20725
20726#[cfg(test)]
20727mod spawn_subscriber_lag_tests {
20728 use super::*;
20729
20730 #[tokio::test]
20735 async fn lagged_spawn_subscriber_receives_a_terminal_lagged_error_after_its_queued_frames() {
20736 let feed = SpawnEventFeed::default();
20737 feed.configure_incarnation("lag-incarnation".to_string());
20738 let (tx, mut rx) = mpsc::channel(1);
20741 feed.subscribe(ConnectionId::new(1), 7, 1, None, FrameSink::new(tx))
20742 .expect("subscribe");
20743 let emitted = SPAWN_SUBSCRIBER_BUFFER + 16;
20744 for index in 0..emitted {
20745 feed.emit_spawned(&format!("lag-module-{index}"), None, 1000, 0);
20746 tokio::task::yield_now().await;
20749 }
20750 assert_eq!(
20751 feed.subscriber_count(),
20752 0,
20753 "the lagged subscriber must be removed"
20754 );
20755
20756 let mut data = Vec::new();
20757 let mut last = None;
20758 loop {
20759 let next = tokio::time::timeout(Duration::from_secs(5), rx.recv())
20760 .await
20761 .expect("the forwarder must finish once the subscriber is dropped");
20762 let Some(outbound) = next else { break };
20763 let frame = outbound.frame;
20764 if frame.header.ty == FrameType::StreamData {
20765 assert!(last.is_none(), "no data may follow the terminal frame");
20766 let event: SpawnEvent = serde_json::from_slice(&frame.body).unwrap();
20767 data.push(event.cursor.seq);
20768 } else {
20769 assert!(last.is_none(), "exactly one terminal frame");
20770 last = Some(frame);
20771 }
20772 }
20773 assert!(!data.is_empty(), "queued frames drain before the terminal");
20774 for pair in data.windows(2) {
20775 assert_eq!(
20776 pair[1],
20777 pair[0] + 1,
20778 "queued frames arrive dense and in order"
20779 );
20780 }
20781 let terminal = last.expect("a lagged subscriber must receive a terminal frame");
20782 assert_eq!(terminal.header.ty, FrameType::Error);
20783 assert_eq!(terminal.header.corr, 7);
20784 let body: subc_protocol::ErrorBody = serde_json::from_slice(&terminal.body).unwrap();
20785 assert_eq!(body.code, SPAWN_SUBSCRIBER_LAGGED_CODE);
20786 let detail = body.detail.expect("lagged error carries detail");
20787 assert_eq!(
20788 detail["first_undelivered_cursor"]["seq"],
20789 data.last().unwrap() + 1,
20790 "the named cursor is the first event the subscriber did not receive"
20791 );
20792 assert_eq!(
20793 detail["first_undelivered_cursor"]["daemon_incarnation"],
20794 "lag-incarnation"
20795 );
20796 }
20797}
20798
20799#[cfg(test)]
20800mod terminal_history_read_concurrency_tests {
20801 use super::*;
20802 use crate::terminal_journal::read_pause;
20803 use cortexkit_test_support::ScratchDir;
20804 use std::sync::mpsc as std_mpsc;
20805
20806 fn journaled_ring(
20807 journal: &Arc<crate::terminal_journal::TerminalJournal>,
20808 ) -> Arc<Mutex<TerminalRing>> {
20809 Arc::new(Mutex::new(
20810 TerminalRing::new(TerminalRingConfig::default(), 1)
20811 .with_journal(Some(Arc::clone(journal))),
20812 ))
20813 }
20814
20815 fn crash(at_ms: u64) -> ExitReport {
20816 ExitReport {
20817 kind: ExitKind::Crash,
20818 code: Some(1),
20819 signal: None,
20820 at_ms,
20821 }
20822 }
20823
20824 fn record_within(
20827 module_id: &'static str,
20828 ring: &Arc<Mutex<TerminalRing>>,
20829 at_ms: u64,
20830 bound: Duration,
20831 ) -> bool {
20832 let ring = Arc::clone(ring);
20833 let (done, done_rx) = std_mpsc::channel();
20834 std::thread::spawn(move || {
20835 record_terminal(
20836 module_id,
20837 &ring,
20838 &SpawnEventFeed::default(),
20839 &crash(at_ms),
20840 TerminalDisposition::Restarting,
20841 );
20842 let _ = done.send(());
20843 });
20844 done_rx.recv_timeout(bound).is_ok()
20845 }
20846
20847 #[test]
20852 fn exits_recorded_during_a_paused_history_read_are_not_blocked_or_half_merged() {
20853 let dir = ScratchDir::new("terminal-history-concurrent-read");
20854 let path = dir.join("terminals.jsonl");
20855 let journal = Arc::new(crate::terminal_journal::TerminalJournal::open(
20856 path.clone(),
20857 "daemon".into(),
20858 ));
20859 let reader_ring = journaled_ring(&journal);
20860 let other_ring = journaled_ring(&journal);
20861 assert!(record_within(
20862 "reader-module",
20863 &reader_ring,
20864 10,
20865 Duration::from_secs(5)
20866 ));
20867
20868 let (started, release) = read_pause::install(&path);
20869 let reading = {
20870 let ring = Arc::clone(&reader_ring);
20871 std::thread::spawn(move || durable_terminal_history_of(&ring, "reader-module"))
20872 };
20873 started
20874 .recv_timeout(Duration::from_secs(5))
20875 .expect("the history read reached its pause");
20876
20877 let bound = Duration::from_secs(1);
20878 assert!(
20879 record_within("other-module", &other_ring, 20, bound),
20880 "another module's exit waited on a history read (journal writer held)"
20881 );
20882 assert!(
20883 record_within("reader-module", &reader_ring, 30, bound),
20884 "the read module's own exit waited on its history read (ring held)"
20885 );
20886
20887 drop(release);
20888 let paused = reading.join().unwrap();
20889 assert_eq!(
20890 paused.entries.iter().map(|e| e.at_ms).collect::<Vec<_>>(),
20891 vec![10],
20892 "an exit recorded after the read began lands in neither half of it"
20893 );
20894 assert_eq!(paused.journal_skipped_lines, 0);
20895 assert_eq!(paused.journal_read_errors, 0);
20896
20897 let after = durable_terminal_history_of(&reader_ring, "reader-module");
20898 assert_eq!(
20899 after.entries.iter().map(|e| e.at_ms).collect::<Vec<_>>(),
20900 vec![10, 30],
20901 "the next read merges ring and journal with no duplicate"
20902 );
20903 assert_eq!(after.journal_skipped_lines, 0);
20904 }
20905}
20906
20907#[cfg(test)]
20912mod stderr_settle_tests {
20913 use std::{
20914 future::Future,
20915 io,
20916 pin::Pin,
20917 sync::{Arc, Mutex},
20918 task::{Context, Poll},
20919 time::Duration,
20920 };
20921
20922 use tokio::{
20923 io::{AsyncRead, ReadBuf},
20924 sync::oneshot,
20925 time::Instant,
20926 };
20927
20928 use super::{settle_stderr_pump, settle_stdout_pump, StderrPump};
20929 use crate::stderr_tail::{
20930 pump_stderr_to, pump_stdout_to, split_capture_stamp, untimed, CaptureState,
20931 ChildOutputSink, OutputSink, StderrRing, StderrTailConfig, TailEntry,
20932 };
20933
20934 const BOUND: Duration = Duration::from_millis(250);
20935
20936 struct HeldReader {
20940 before: Option<Vec<u8>>,
20941 gate: Option<oneshot::Receiver<()>>,
20942 after: io::Cursor<Vec<u8>>,
20943 }
20944
20945 impl AsyncRead for HeldReader {
20946 fn poll_read(
20947 mut self: Pin<&mut Self>,
20948 cx: &mut Context<'_>,
20949 buf: &mut ReadBuf<'_>,
20950 ) -> Poll<io::Result<()>> {
20951 if let Some(bytes) = self.before.take() {
20952 buf.put_slice(&bytes);
20953 return Poll::Ready(Ok(()));
20954 }
20955 if let Some(gate) = self.gate.as_mut() {
20956 match Pin::new(gate).poll(cx) {
20957 Poll::Pending => return Poll::Pending,
20958 Poll::Ready(_) => self.gate = None,
20959 }
20960 }
20961 Pin::new(&mut self.after).poll_read(cx, buf)
20962 }
20963 }
20964
20965 struct DiscardSink;
20966
20967 impl OutputSink for DiscardSink {
20968 fn write_line(&mut self, _line: &[u8]) {}
20969 }
20970
20971 fn line(text: &str) -> TailEntry {
20972 TailEntry::Line {
20973 text: text.to_string(),
20974 truncated: false,
20975 at_ms: None,
20976 }
20977 }
20978
20979 fn lock(ring: &Arc<Mutex<StderrRing>>) -> std::sync::MutexGuard<'_, StderrRing> {
20980 ring.lock().unwrap()
20981 }
20982
20983 fn held_pump(
20987 ring: &Arc<Mutex<StderrRing>>,
20988 before: &str,
20989 after: &str,
20990 ) -> (StderrPump, oneshot::Sender<()>) {
20991 let generation = lock(ring).begin_process();
20992 let (release, gate) = oneshot::channel();
20993 let reader = HeldReader {
20994 before: Some(before.as_bytes().to_vec()),
20995 gate: Some(gate),
20996 after: io::Cursor::new(after.as_bytes().to_vec()),
20997 };
20998 let task = tokio::spawn(pump_stderr_to(
20999 reader,
21000 Arc::clone(ring),
21001 generation,
21002 DiscardSink,
21003 ));
21004 (StderrPump { task, generation }, release)
21005 }
21006
21007 async fn wait_until(ring: &Arc<Mutex<StderrRing>>, done: impl Fn(&StderrRing) -> bool) {
21008 for _ in 0..1000 {
21009 if done(&lock(ring)) {
21010 return;
21011 }
21012 tokio::time::sleep(Duration::from_millis(1)).await;
21013 }
21014 panic!(
21015 "ring never reached the expected state: {:?}",
21016 lock(ring).snapshot(None, None)
21017 );
21018 }
21019
21020 #[tokio::test(start_paused = true)]
21021 async fn a_crash_line_the_reader_had_not_reached_by_the_bound_is_kept_before_the_restart() {
21022 let ring = Arc::new(Mutex::new(StderrRing::new(StderrTailConfig::default())));
21023 let (pump, release) = held_pump(&ring, "booting\n", "config error: missing storage\n");
21024
21025 settle_stderr_pump("crasher", &ring, pump, BOUND).await;
21026 let before_release = lock(&ring).snapshot(None, None);
21027 assert!(
21028 matches!(before_release.capture, CaptureState::Incomplete { .. }),
21029 "a reader that has not reached EOF cannot claim a whole tail: {before_release:?}"
21030 );
21031
21032 let next = lock(&ring).begin_process();
21035 lock(&ring).push_line_from(next, "next process booting");
21036 release.send(()).unwrap();
21037 wait_until(&ring, |ring| {
21038 ring.snapshot(None, None).capture == CaptureState::Captured
21039 })
21040 .await;
21041
21042 assert_eq!(
21043 untimed(lock(&ring).snapshot(None, None).entries),
21044 vec![
21045 line("booting"),
21046 line("config error: missing storage"),
21047 TailEntry::ProcessStart,
21048 line("next process booting"),
21049 ],
21050 "the crash's last line must survive a slow reader and stay in the crashed process's section"
21051 );
21052 }
21053
21054 #[tokio::test(start_paused = true)]
21055 async fn a_pipe_held_open_by_a_descendant_reads_incomplete_without_delaying_the_restart_past_the_bound(
21056 ) {
21057 let ring = Arc::new(Mutex::new(StderrRing::new(StderrTailConfig::default())));
21058 let (pump, _held) = held_pump(&ring, "parent exiting\n", "");
21061
21062 let started = Instant::now();
21063 settle_stderr_pump("orphaning", &ring, pump, BOUND).await;
21064 assert_eq!(
21065 started.elapsed(),
21066 BOUND,
21067 "the restart must wait exactly the bound for a pipe that stays open, no longer"
21068 );
21069
21070 let next = lock(&ring).begin_process();
21071 lock(&ring).push_line_from(next, "next process booting");
21072 tokio::time::sleep(Duration::from_secs(60)).await;
21073
21074 let snapshot = lock(&ring).snapshot(None, None);
21075 match &snapshot.capture {
21076 CaptureState::Incomplete { reason } => assert!(
21077 reason.contains("had not reached EOF") && reason.contains("250ms"),
21078 "the reason must say what is missing and after how long: {reason}"
21079 ),
21080 other => panic!("expected Incomplete while the pipe is held open, got {other:?}"),
21081 }
21082 assert_eq!(
21083 untimed(snapshot.entries),
21084 vec![
21085 line("parent exiting"),
21086 TailEntry::ProcessStart,
21087 line("next process booting"),
21088 ]
21089 );
21090 }
21091
21092 #[tokio::test(start_paused = true)]
21093 async fn a_reader_that_reaches_eof_within_the_bound_leaves_the_tail_captured() {
21094 let ring = Arc::new(Mutex::new(StderrRing::new(StderrTailConfig::default())));
21095 let (pump, release) = held_pump(&ring, "one\n", "two\n");
21096 release.send(()).unwrap();
21097
21098 settle_stderr_pump("clean", &ring, pump, BOUND).await;
21099
21100 let snapshot = lock(&ring).snapshot(None, None);
21101 assert_eq!(snapshot.capture, CaptureState::Captured);
21102 assert_eq!(untimed(snapshot.entries), vec![line("one"), line("two")]);
21103 }
21104
21105 #[tokio::test(start_paused = true)]
21106 async fn stdout_buffered_past_the_exit_drain_bound_reaches_the_capture_file() {
21107 const BURST: usize = 150;
21108 let capture = cortexkit_test_support::ScratchDir::new("stdout-exit-drain");
21109 let path = capture.join("two-pipe.stderr.log");
21110 let sink = ChildOutputSink::open(&path, cortexkit_log::Retention::default()).unwrap();
21111 let ring = Arc::new(Mutex::new(StderrRing::new(StderrTailConfig::default())));
21112 let generation = lock(&ring).begin_process();
21113 let stdout = (0..BURST)
21114 .map(|i| format!("out-{i:04}-{}\n", "o".repeat(64)))
21115 .collect::<String>();
21116 let stderr = (0..BURST)
21117 .map(|i| format!("err-{i:04}-{}\n", "e".repeat(64)))
21118 .collect::<String>();
21119 let first_chunk = stdout.find('\n').unwrap() + 1 + "out-0001-".len();
21124 let (release, gate) = oneshot::channel();
21125 let reader = HeldReader {
21126 before: Some(stdout.as_bytes()[..first_chunk].to_vec()),
21127 gate: Some(gate),
21128 after: io::Cursor::new(stdout.as_bytes()[first_chunk..stdout.len() - 1].to_vec()),
21129 };
21130 let (finished, completion) = oneshot::channel();
21131 let stdout_sink = sink.clone();
21132 let pump = tokio::spawn(async move {
21133 pump_stdout_to(reader, stdout_sink).await;
21134 finished.send(()).unwrap();
21135 });
21136 pump_stderr_to(io::Cursor::new(stderr.into_bytes()), ring, generation, sink).await;
21137
21138 let started = Instant::now();
21139 settle_stdout_pump("two-pipe", pump, BOUND).await;
21140 assert_eq!(
21141 started.elapsed(),
21142 BOUND,
21143 "a late reader must not hold up restart"
21144 );
21145 assert_eq!(
21146 std::fs::read_to_string(&path).unwrap().lines().count(),
21147 BURST + 1
21148 );
21149
21150 release
21151 .send(())
21152 .expect("stdout reader must survive the exit drain bound");
21153 tokio::time::timeout(Duration::from_secs(1), completion)
21154 .await
21155 .expect("released stdout reader must reach EOF")
21156 .expect("stdout capture must finish at EOF, not be cancelled");
21157 let contents = std::fs::read_to_string(&path).unwrap();
21158 let mut lines = contents
21159 .lines()
21160 .map(|line| split_capture_stamp(line).expect("capture stamp").1)
21161 .collect::<Vec<_>>();
21162 let mut expected = (0..BURST)
21163 .flat_map(|i| {
21164 [
21165 format!("out-{i:04}-{}", "o".repeat(64)),
21166 format!("err-{i:04}-{}", "e".repeat(64)),
21167 ]
21168 })
21169 .collect::<Vec<_>>();
21170 lines.sort_unstable();
21171 expected.sort_unstable();
21172 assert_eq!(
21173 lines, expected,
21174 "both pipes must retain every complete line exactly once"
21175 );
21176 }
21177}
21178
21179#[cfg(all(test, windows))]
21193mod job_containment_tests {
21194 use super::*;
21195 use cortexkit_test_support::ScratchDir;
21196 use std::{
21197 path::{Path, PathBuf},
21198 sync::{Arc, Mutex},
21199 time::{Duration, Instant},
21200 };
21201
21202 fn stub_path() -> PathBuf {
21208 let path = cortexkit_test_support::cargo_profile_bin("fake-aft-stub");
21209 assert!(
21210 path.exists(),
21211 "fake-aft-stub not built at {}: run `cargo test -p subc-core` (which builds \
21212 [[bin]] targets) rather than `cargo test -p subc-core --lib` (which does not)",
21213 path.display()
21214 );
21215 path
21216 }
21217
21218 fn read_grandchild_pid(path: &Path) -> u32 {
21220 let deadline = Instant::now() + Duration::from_secs(10);
21221 loop {
21222 if let Ok(contents) = std::fs::read_to_string(path) {
21223 if let Ok(pid) = contents.trim().parse() {
21224 return pid;
21225 }
21226 }
21227 assert!(
21228 Instant::now() < deadline,
21229 "the stub never recorded a grandchild pid at {}",
21230 path.display()
21231 );
21232 std::thread::sleep(Duration::from_millis(10));
21233 }
21234 }
21235
21236 struct Fixture {
21239 _dir: ScratchDir,
21240 _roster: ChildRoster,
21241 module_id: String,
21242 grandchild: u32,
21243 child: Option<SupervisedChild>,
21244 registry: Arc<Registry>,
21245 snapshot: Arc<Mutex<SupervisorSnapshot>>,
21246 terminal_ring: Arc<Mutex<TerminalRing>>,
21247 spawn_events: SpawnEventFeed,
21248 }
21249
21250 fn fixture(label: &str, module_id: &str) -> Fixture {
21251 fixture_with_carry(label, module_id, false)
21252 }
21253
21254 fn fixture_with_carry(label: &str, module_id: &str, carry_containment: bool) -> Fixture {
21255 let dir = ScratchDir::new(label);
21256 let pid_file = dir.join("grandchild.pid");
21257 let supervisor = Supervisor::new_for_test(
21258 Arc::new(Registry::default()),
21259 RestartPolicy::new(3, Duration::ZERO),
21260 );
21261 let runtime = supervisor.runtime_config();
21262 let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::starting()));
21263 let spec = ModuleSpec {
21264 script: None,
21265 module_id: module_id.to_string(),
21266 program: stub_path(),
21267 args: Vec::new(),
21271 env: vec![
21272 ("FAKE_AFT_NEVER_CONNECT".to_string(), "1".to_string()),
21273 (
21274 "FAKE_AFT_GRANDCHILD_PID_FILE".to_string(),
21275 pid_file.display().to_string(),
21276 ),
21277 ],
21278 reserved: false,
21279 reserved_prefixes: Vec::new(),
21280 protocol: ModuleProtocol::Subc,
21281 overlap: Default::default(),
21282 carry_containment,
21283 };
21284 let child = spawn_and_mark_running(&spec, &runtime, &snapshot)
21285 .expect("spawn the supervised fixture");
21286 let grandchild = read_grandchild_pid(&pid_file);
21287 Fixture {
21288 _dir: dir,
21289 _roster: runtime.child_roster.clone(),
21290 module_id: module_id.to_string(),
21291 grandchild,
21292 child: Some(child),
21293 registry: Arc::new(Registry::default()),
21294 snapshot,
21295 terminal_ring: Arc::clone(&runtime.terminal_ring),
21296 spawn_events: SpawnEventFeed::default(),
21297 }
21298 }
21299
21300 async fn assert_admission_kill_reaps_grandchild(carry: bool) {
21301 let label = if carry {
21302 "admission-kill-carried-tree"
21303 } else {
21304 "admission-kill-plain-tree"
21305 };
21306 let mut fixture = fixture_with_carry(label, label, carry);
21307 let descendant = subc_os::Process::open(fixture.grandchild)
21308 .unwrap()
21309 .expect("the stub descendant must be alive before admission kill");
21310 assert!(descendant.observe().is_some());
21311 let child = fixture.child.as_mut().unwrap();
21312 assert_eq!(child.child.container().is_some(), carry);
21313 assert_eq!(child.job.is_some(), !carry);
21314
21315 child.start_admission_kill().unwrap();
21316 assert!(child.admission_kill_sent);
21317 let status = timeout(Duration::from_secs(10), child.wait())
21318 .await
21319 .unwrap()
21320 .unwrap();
21321 assert_eq!(
21322 status.code(),
21323 Some(admission_exit::WINDOWS_ADMISSION_KILL_EXIT_CODE),
21324 "containment teardown must preserve the leader's distinctive status"
21325 );
21326 assert!(
21329 descendant.wait_for_exit(Duration::from_secs(10)).unwrap(),
21330 "admission kill left a descendant running before exit finalization"
21331 );
21332 assert_eq!(child.child.container().is_some(), carry);
21333 assert_eq!(child.job.is_some(), !carry);
21334 child.finish_exit(false).await.unwrap();
21335 }
21336
21337 #[tokio::test]
21338 async fn admission_kill_reaps_grandchild_with_plain_job() {
21339 assert_admission_kill_reaps_grandchild(false).await;
21340 }
21341
21342 #[tokio::test]
21343 async fn admission_kill_reaps_grandchild_with_carried_job() {
21344 assert_admission_kill_reaps_grandchild(true).await;
21345 }
21346
21347 #[tokio::test]
21348 async fn admission_kill_retires_registration_before_wait() {
21349 let mut fixture = fixture("admission-registration", "admission-registration");
21350 let handle = SupervisorHandle::new();
21351 let spawn =
21352 RegistrationSpawn::new(&handle, "admission-registration", SpawnRole::Plain, None);
21353 spawn.ready();
21354 let child = fixture.child.as_mut().unwrap();
21355 child.registration_spawn = Some(spawn.clone());
21356 child.start_admission_kill().unwrap();
21357 assert!(
21358 !spawn.launch_ready(),
21359 "admission kill must retire before its caller can await the child"
21360 );
21361 timeout(Duration::from_secs(10), child.wait())
21362 .await
21363 .unwrap()
21364 .unwrap();
21365 child.finish_exit(false).await.unwrap();
21366 }
21367
21368 impl Fixture {
21369 async fn drain(&mut self) {
21371 let child = self
21372 .child
21373 .take()
21374 .expect("the fixture child is still present");
21375 drain_child_to_state(
21376 &self.module_id,
21377 ModuleProtocol::Subc,
21378 StopNotice::NotSent,
21381 &self.registry,
21382 None,
21383 &self.snapshot,
21384 &self.terminal_ring,
21385 &self.spawn_events,
21386 child,
21387 Duration::from_millis(500),
21388 ModuleState::Stopped,
21389 Some(false),
21390 &FailureContext::operator(),
21391 )
21392 .await
21393 .expect("drain the supervised fixture");
21394 }
21395 }
21396
21397 #[tokio::test]
21403 async fn teardown_reaps_the_grandchild() {
21404 let mut fixture = fixture("teardown-grandchild", "tree-teardown");
21405 let grandchild = fixture.grandchild;
21406
21407 assert!(
21408 subc_jobobject::process_exists(grandchild),
21409 "grandchild {grandchild} must be alive before teardown, or this proves nothing"
21410 );
21411
21412 fixture.drain().await;
21413
21414 assert!(
21415 subc_jobobject::wait_for_process_exit(grandchild, Duration::from_secs(10)),
21416 "grandchild {grandchild} outlived module teardown: the tree was not contained"
21417 );
21418 }
21419
21420 #[test]
21433 fn an_uncontained_grandchild_survives_a_direct_child_kill() {
21434 let dir = ScratchDir::new("teardown-uncontained");
21435 let pid_file = dir.join("grandchild.pid");
21436 let mut child = std::process::Command::new(stub_path())
21437 .env("FAKE_AFT_NEVER_CONNECT", "1")
21438 .env(
21439 "FAKE_AFT_GRANDCHILD_PID_FILE",
21440 pid_file.display().to_string(),
21441 )
21442 .stdin(std::process::Stdio::null())
21443 .stdout(std::process::Stdio::null())
21444 .stderr(std::process::Stdio::null())
21445 .spawn()
21446 .expect("spawn the uncontained fixture");
21447 let grandchild = read_grandchild_pid(&pid_file);
21448
21449 child.kill().expect("kill the direct child");
21451 let _ = child.wait();
21452
21453 assert!(
21454 subc_jobobject::process_exists(grandchild),
21455 "grandchild {grandchild} died with the direct child, so this control no longer \
21456 distinguishes contained from uncontained teardown and the regression test is \
21457 passing vacuously"
21458 );
21459
21460 kill_tree(grandchild);
21463 }
21464
21465 #[tokio::test]
21474 async fn dropping_containment_reaps_the_grandchild() {
21475 let mut fixture = fixture("drop-containment", "tree-drop");
21476 let grandchild = fixture.grandchild;
21477
21478 assert!(subc_jobobject::process_exists(grandchild));
21479
21480 fixture.child.as_mut().expect("child present").job = None;
21482
21483 assert!(
21484 subc_jobobject::wait_for_process_exit(grandchild, Duration::from_secs(10)),
21485 "grandchild {grandchild} survived the containment handle closing, so a daemon \
21486 crash would leave the tree behind"
21487 );
21488 }
21489
21490 fn kill_tree(pid: u32) {
21492 let _ = std::process::Command::new("taskkill.exe")
21493 .args(["/PID", &pid.to_string(), "/T", "/F"])
21494 .stdin(std::process::Stdio::null())
21495 .stdout(std::process::Stdio::null())
21496 .stderr(std::process::Stdio::null())
21497 .status();
21498 assert!(
21499 subc_jobobject::wait_for_process_exit(pid, Duration::from_secs(10)),
21500 "could not clean up grandchild {pid}"
21501 );
21502 }
21503}
21504
21505#[cfg(test)]
21506mod privacy_trampoline_configuration_tests {
21507 #[tokio::test]
21508 #[cfg_attr(not(target_os = "macos"), ignore = "requires macOS privacy trampoline")]
21509 async fn macos_spawn_without_a_configured_trampoline_refuses_by_name() {
21510 #[cfg(target_os = "macos")]
21511 {
21512 let supervisor = super::Supervisor::new(
21513 std::sync::Arc::new(crate::Registry::default()),
21514 super::RestartPolicy::default(),
21515 );
21516 let error = supervisor.spawn(spec()).unwrap_err();
21517 assert!(
21518 error
21519 .to_string()
21520 .contains("no privacy trampoline configured"),
21521 "{error}"
21522 );
21523 }
21524 }
21525
21526 #[tokio::test]
21527 #[cfg_attr(not(target_os = "macos"), ignore = "requires macOS privacy trampoline")]
21528 async fn macos_wrong_trampoline_is_rejected_by_the_startup_probe() {
21529 #[cfg(target_os = "macos")]
21530 {
21531 let supervisor = super::Supervisor::new(
21532 std::sync::Arc::new(crate::Registry::default()),
21533 super::RestartPolicy::default(),
21534 )
21535 .with_privacy_trampoline(std::env::current_exe().unwrap());
21536 let error = supervisor.spawn(spec()).unwrap_err();
21537 assert!(
21538 error
21539 .to_string()
21540 .contains("binary does not implement the privacy trampoline protocol"),
21541 "{error}"
21542 );
21543 }
21544 }
21545
21546 #[cfg(target_os = "macos")]
21547 fn spec() -> super::ModuleSpec {
21548 super::ModuleSpec {
21549 module_id: "privacy-configuration".into(),
21550 program: "/bin/sleep".into(),
21551 args: vec!["30".into()],
21552 env: vec![],
21553 reserved: false,
21554 reserved_prefixes: vec![],
21555 protocol: subc_control::ModuleProtocol::None,
21556 overlap: super::ModuleOverlap::Exclusive,
21557 carry_containment: false,
21558 script: None,
21559 }
21560 }
21561}
21562
21563#[cfg(test)]
21564mod privacy_exec_boundary_tests {
21565 #[cfg(target_os = "macos")]
21566 use super::*;
21567 #[cfg(target_os = "macos")]
21568 use std::{
21569 io::{Read, Write},
21570 net::{TcpListener, TcpStream},
21571 };
21572
21573 #[cfg(target_os = "macos")]
21576 pub(super) fn before_image_sample(spec: &ModuleSpec, pid: u32) {
21577 if let Some((_, path)) = spec
21578 .env
21579 .iter()
21580 .find(|(key, _)| key == "SUBC_TEST_PRIVACY_SAMPLE_BARRIER")
21581 {
21582 let mut barrier = TcpStream::connect(path).unwrap();
21583 barrier
21584 .set_read_timeout(Some(Duration::from_secs(30)))
21585 .unwrap();
21586 barrier.write_all(&pid.to_ne_bytes()).unwrap();
21587 let mut release = [0];
21588 barrier.read_exact(&mut release).unwrap();
21589 assert_eq!(&release, b"X");
21590 }
21591 }
21592
21593 #[cfg(target_os = "macos")]
21594 fn spec(program: &str, args: &[&str]) -> ModuleSpec {
21595 ModuleSpec {
21596 script: None,
21597 module_id: "privacy-boundary".into(),
21598 program: program.into(),
21599 args: args.iter().map(|arg| (*arg).into()).collect(),
21600 env: vec![],
21601 reserved: false,
21602 reserved_prefixes: vec![],
21603 protocol: ModuleProtocol::None,
21604 overlap: ModuleOverlap::Exclusive,
21605 carry_containment: false,
21606 }
21607 }
21608
21609 #[cfg(target_os = "macos")]
21610 async fn accept(listener: TcpListener) -> TcpStream {
21611 let listener = tokio::net::TcpListener::from_std({
21613 listener.set_nonblocking(true).unwrap();
21614 listener
21615 })
21616 .unwrap();
21617 let (stream, _) = tokio::time::timeout(Duration::from_secs(30), listener.accept())
21618 .await
21619 .unwrap()
21620 .unwrap();
21621 let stream = stream.into_std().unwrap();
21622 stream.set_nonblocking(false).unwrap();
21623 stream
21624 .set_read_timeout(Some(Duration::from_secs(30)))
21625 .unwrap();
21626 stream
21627 }
21628
21629 #[tokio::test]
21630 #[cfg_attr(not(target_os = "macos"), ignore = "requires macOS privacy trampoline")]
21631 async fn macos_roster_withholds_a_nonnull_trampoline_image_until_exec_confirmation() {
21632 #[cfg(target_os = "macos")]
21633 {
21634 let root = cortexkit_test_support::ScratchDir::new("privacy-roster-barrier");
21635 let exec_listener = TcpListener::bind("127.0.0.1:0").unwrap();
21638 let sample_listener = TcpListener::bind("127.0.0.1:0").unwrap();
21639 let record = root.join("live-children.json");
21640 let supervisor =
21641 Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default())
21642 .with_live_children_record(&record);
21643 let runtime = supervisor.runtime_config();
21644 let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::starting()));
21645 let mut spec = spec("/bin/sleep", &["30"]);
21646 spec.env = vec![
21647 (
21648 "SUBC_TEST_PRIVACY_EXEC_BARRIER".into(),
21649 exec_listener.local_addr().unwrap().to_string(),
21650 ),
21651 (
21652 "SUBC_TEST_PRIVACY_SAMPLE_BARRIER".into(),
21653 sample_listener.local_addr().unwrap().to_string(),
21654 ),
21655 ];
21656 let spawn = tokio::task::spawn_blocking(move || {
21657 spawn_and_mark_running(&spec, &runtime, &snapshot).unwrap()
21658 });
21659 let mut sample = accept(sample_listener).await;
21660 let mut pid = [0; 4];
21661 sample.read_exact(&mut pid).unwrap();
21662 let pid = u32::from_ne_bytes(pid);
21663 let mut exec = accept(exec_listener).await;
21664 let mut ready = [0];
21665 exec.read_exact(&mut ready).unwrap();
21666 assert_eq!(&ready, b"R");
21667 let trampoline = subc_os::file_identity(&test_privacy_trampoline()).unwrap();
21670 assert_eq!(
21671 observe_spawned_image(pid).unwrap().executable,
21672 Some(trampoline)
21673 );
21674 sample.write_all(b"X").unwrap();
21675 let mut child = spawn.await.unwrap();
21676 let early = crate::live_children::read_record(&record).unwrap();
21677 assert_eq!(early.len(), 1);
21678 assert_eq!(early[0].pid, pid);
21679 assert_eq!(
21680 early[0].executable, None,
21681 "unconfirmed trampoline image entered the roster"
21682 );
21683 assert!(child.report_ready.get().is_none());
21684 child.privacy_exec.as_mut().unwrap().deadline =
21687 tokio::time::Instant::now() + Duration::from_secs(30);
21688 exec.write_all(b"X").unwrap();
21689 child.confirm_privacy_exec().await;
21690 assert_eq!(child.spawn_failure, None);
21691 assert!(child.report_ready.get().is_some());
21692 let confirmed = crate::live_children::read_record(&record).unwrap();
21693 let module = subc_os::file_identity(std::path::Path::new("/bin/sleep")).unwrap();
21694 assert_ne!(module, trampoline);
21695 assert_eq!(confirmed[0].executable, Some(module.into()));
21696 child.start_kill().unwrap();
21697 child.wait().await.unwrap();
21698 child.release_roster();
21699 }
21700 }
21701
21702 #[tokio::test]
21703 #[cfg_attr(not(target_os = "macos"), ignore = "requires macOS privacy trampoline")]
21704 async fn macos_unstamped_reexec_records_the_observed_native_image() {
21705 #[cfg(target_os = "macos")]
21706 {
21707 let root = cortexkit_test_support::ScratchDir::new("privacy-reexec");
21708 let record = root.join("live-children.json");
21709 let supervisor =
21710 Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default())
21711 .with_live_children_record(&record);
21712 let runtime = supervisor.runtime_config();
21713 let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::starting()));
21714 let fixture = crate::config_gate::program::native_fixture_for_test();
21715 let mut spec = spec(
21716 "/bin/sh",
21717 &["-c", "exec \"$1\" 600", "reexec", fixture.to_str().unwrap()],
21718 );
21719 spec.env
21720 .push(("SUBC_TEST_PRIVACY_IMAGE_DELAY_MS".into(), "5".into()));
21721 let mut child = spawn_and_mark_running(&spec, &runtime, &snapshot).unwrap();
21722 let native_image = subc_os::file_identity(&fixture).unwrap();
21723 let reexec = tokio::time::timeout(Duration::from_secs(5), async {
21726 while observe_spawned_image(child.pid).and_then(|image| image.executable)
21727 != Some(native_image)
21728 {
21729 tokio::time::sleep(Duration::from_millis(1)).await;
21730 }
21731 })
21732 .await;
21733 child.confirm_privacy_exec().await;
21734 let failure = child.spawn_failure.clone();
21735 let ready = child.report_ready.get().is_some();
21736 let confirmed = crate::live_children::read_record(&record).unwrap();
21737 child.start_kill().unwrap();
21738 tokio::time::timeout(Duration::from_secs(5), child.wait())
21739 .await
21740 .unwrap()
21741 .unwrap();
21742 child.release_roster();
21743
21744 reexec.expect("the shell must re-exec the native fixture");
21745 assert_eq!(
21746 failure, None,
21747 "an unstamped program may re-exec before observation"
21748 );
21749 assert!(ready);
21750 assert_eq!(confirmed.len(), 1);
21751 assert_eq!(confirmed[0].executable, Some(native_image.into()));
21752 }
21753 }
21754
21755 #[tokio::test]
21756 #[cfg_attr(not(target_os = "macos"), ignore = "requires macOS privacy trampoline")]
21757 async fn macos_unstamped_ack_without_exec_still_refuses_the_trampoline() {
21758 #[cfg(target_os = "macos")]
21759 {
21760 let listener = TcpListener::bind("127.0.0.1:0").unwrap();
21761 let supervisor =
21762 Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default());
21763 let runtime = supervisor.runtime_config();
21764 let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::starting()));
21765 let fixture = crate::config_gate::program::native_fixture_for_test();
21766 let mut spec = spec(fixture.to_str().unwrap(), &["600"]);
21767 spec.env.push((
21768 "SUBC_TEST_PRIVACY_EXEC_BARRIER".into(),
21769 listener.local_addr().unwrap().to_string(),
21770 ));
21771 let mut child = spawn_and_mark_running(&spec, &runtime, &snapshot).unwrap();
21772 let mut exec = accept(listener).await;
21773 let mut ready = [0];
21774 exec.read_exact(&mut ready).unwrap();
21775 assert_eq!(&ready, b"R");
21776 assert_eq!(
21777 observe_spawned_image(child.pid).unwrap().executable,
21778 subc_os::file_identity(&test_privacy_trampoline())
21779 );
21780 let (reader, writer) = std::io::pipe().unwrap();
21783 drop(writer);
21784 child.privacy_exec.as_mut().unwrap().reader =
21785 tokio::io::unix::AsyncFd::new(reader).unwrap();
21786 child.confirm_privacy_exec().await;
21787 let failure = child.spawn_failure.clone();
21788 let ready = child.report_ready.get().is_some();
21789 child.start_kill().unwrap();
21790 tokio::time::timeout(Duration::from_secs(5), child.wait())
21791 .await
21792 .unwrap()
21793 .unwrap();
21794 child.release_roster();
21795 assert_eq!(
21796 failure.as_deref(),
21797 Some("privacy identity trampoline did not exec the module")
21798 );
21799 assert!(!ready);
21800 }
21801 }
21802
21803 #[tokio::test]
21804 #[cfg_attr(not(target_os = "macos"), ignore = "requires macOS privacy trampoline")]
21805 async fn macos_resumed_121_is_a_module_exit_not_a_trampoline_refusal() {
21806 #[cfg(target_os = "macos")]
21807 {
21808 let registry = Arc::new(Registry::default());
21809 let policy = RestartPolicy::new(0, Duration::ZERO);
21810 let fixture = crate::config_gate::program::native_fixture_for_test();
21811 let spec = spec(fixture.to_str().unwrap(), &["exit121"]);
21812 let scratch = cortexkit_test_support::ScratchDir::new("resumed-exit");
21813 let supervisor = Supervisor::new_for_test(Arc::clone(®istry), policy)
21814 .with_config_runtime(suspended_spawn_tests::gated_owner(&scratch, &spec, true));
21815 let runtime = supervisor.runtime_config();
21816 let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::starting()));
21817 let mut child = spawn_and_mark_running(&spec, &runtime, &snapshot).unwrap();
21820 child.confirm_privacy_exec().await;
21821 assert_eq!(child.spawn_failure, None);
21822 assert!(child.report_ready.get().is_some());
21823 let pid = child.pid;
21824 tokio::task::spawn_blocking(move || {
21825 subc_os::fork_exec_test::wait_for_child_exit_without_reaping(pid)
21826 })
21827 .await
21828 .unwrap()
21829 .unwrap();
21830 let status = child.wait().await.unwrap();
21831 assert_eq!(status.code(), Some(121));
21832 assert!(child.privacy_exec.is_none());
21833 let report = classify_reaped_child_exit(&snapshot, &child, &status);
21834 on_child_exit(
21835 &spec,
21836 policy,
21837 ®istry,
21838 &snapshot,
21839 &runtime.terminal_ring,
21840 &runtime.spawn_events,
21841 &runtime.child_roster,
21842 report,
21843 true,
21844 )
21845 .await;
21846 let state = lock_snapshot(&snapshot).unwrap();
21847 assert_eq!(state.state, ModuleState::Failed);
21848 assert_eq!(state.last_exit.as_ref().unwrap().code, Some(121));
21849 assert_eq!(state.reported_pid(), None);
21850 drop(state);
21851 let history = runtime.terminal_ring.lock().unwrap().snapshot();
21852 assert_eq!(history.entries.len(), 1);
21853 let terminal = &history.entries[0];
21854 assert_eq!(terminal.exit_code, Some(121));
21855 assert_eq!(terminal.exit_kind, subc_control::TerminalExitKind::Crash);
21856 assert_eq!(terminal.disposition, TerminalDisposition::Failed);
21857 assert_eq!(
21858 terminal.disposition_detail.as_deref(),
21859 Some(policy.budget_exhausted_detail().as_str()),
21860 "module exit 121 was classified as a trampoline refusal: {terminal:?}"
21861 );
21862 assert_eq!(child.spawn_failure, None);
21863 child.release_roster();
21864 }
21865 }
21866}
21867
21868#[cfg(all(test, target_os = "macos"))]
21869#[path = "supervise/suspended_spawn_tests.rs"]
21870mod suspended_spawn_tests;
21871
21872#[cfg(all(test, unix))]
21876mod launch_nonce_descriptor_tests {
21877 use super::{ChildRoster, ModuleProtocol, ModuleSpec, SupervisorHandle};
21878 use crate::stderr_tail::{StderrRing, StderrTailConfig};
21879 use cortexkit_test_support::ScratchDir;
21880 use std::{
21881 path::PathBuf,
21882 sync::{Arc, Mutex},
21883 time::{Duration, Instant},
21884 };
21885
21886 async fn probe(role: super::SpawnRole) {
21887 let scratch = ScratchDir::new("launch-nonce-descriptor");
21888 let fd_copy = scratch.join("from-descriptor");
21889 let env_copy = scratch.join("environment");
21890 let script = format!(
21891 "env > '{env}.tmp' && mv '{env}.tmp' '{env}'; cat <&3 > '{fd}.tmp' && mv '{fd}.tmp' '{fd}'; sleep 30",
21892 fd = fd_copy.display(), env = env_copy.display(),
21893 );
21894 let xdg = |name: &str| (name.to_string(), scratch.join(name).display().to_string());
21895 let spec = ModuleSpec {
21896 script: None,
21897 module_id: "nonce-descriptor-probe".to_string(),
21898 program: PathBuf::from("/bin/sh"),
21899 args: vec!["-c".to_string(), script],
21900 env: vec![
21901 xdg("XDG_DATA_HOME"),
21902 xdg("XDG_RUNTIME_DIR"),
21903 xdg("XDG_CONFIG_HOME"),
21904 ("SUBC_LAUNCH_NONCE".to_string(), "must-not-leak".to_string()),
21905 ],
21906 reserved: true,
21907 reserved_prefixes: Vec::new(),
21908 protocol: ModuleProtocol::Subc,
21909 overlap: Default::default(),
21910 carry_containment: false,
21911 };
21912 let handle = SupervisorHandle::new();
21913 let ring = Arc::new(Mutex::new(StderrRing::new(StderrTailConfig::default())));
21914 let roster = ChildRoster::default();
21915 #[cfg(target_os = "macos")]
21916 {
21917 let path = super::test_privacy_trampoline();
21918 roster.set_privacy_trampoline(super::probe_privacy_trampoline(&path).map(|()| path));
21919 }
21920 let child = super::spawn_child_in_slot(
21921 &spec,
21922 None,
21923 Some(&handle),
21924 &ring,
21925 None,
21926 &roster,
21927 None,
21928 #[cfg(target_os = "linux")]
21929 None,
21930 role,
21931 matches!(role, super::SpawnRole::SwapCandidate),
21932 super::DEFAULT_REGISTER_TIMEOUT,
21933 )
21934 .expect("spawn probe");
21935 let deadline = Instant::now() + Duration::from_secs(10);
21936 while !(fd_copy.exists() && env_copy.exists()) {
21937 assert!(Instant::now() < deadline, "probe never wrote its copies");
21938 tokio::time::sleep(Duration::from_millis(20)).await;
21939 }
21940 let nonce = std::fs::read_to_string(fd_copy).unwrap();
21941 assert!(!nonce.is_empty());
21942 let environment = std::fs::read_to_string(env_copy).unwrap();
21943 assert!(environment
21944 .lines()
21945 .any(|line| line.starts_with("SUBC_LAUNCH_NONCE_FD=3:")));
21946 let copy = environment
21947 .lines()
21948 .find_map(|line| line.strip_prefix("SUBC_LAUNCH_NONCE="));
21949 assert_eq!(
21950 copy, None,
21951 "Unix children must never receive the environment nonce"
21952 );
21953 if matches!(role, super::SpawnRole::Plain) {
21954 assert_eq!(
21955 handle.spawn_nonce(&spec.module_id).as_deref(),
21956 Some(nonce.as_str())
21957 );
21958 }
21959 drop(child);
21960 }
21961
21962 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
21963 async fn a_spawned_module_receives_its_nonce_only_on_descriptor_3() {
21964 probe(super::SpawnRole::Plain).await;
21965 }
21966
21967 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
21968 async fn swap_candidate_receives_its_nonce_only_on_descriptor_3() {
21969 probe(super::SpawnRole::SwapCandidate).await;
21970 }
21971}
21972
21973#[cfg(all(test, target_os = "linux"))]
21974#[path = "supervise/cgroup_reap_tests.rs"]
21975mod cgroup_reap_tests;
21976
21977#[cfg(all(test, target_os = "linux"))]
21978mod cgroup_containment_tests {
21979 use super::*;
21980 use cortexkit_test_support::ScratchDir;
21981
21982 #[tokio::test]
21983 async fn linux_subtree_memory_covers_module_children() {
21984 let dir = ScratchDir::new("subtree-memory");
21985 let root = PathBuf::from(format!(
21986 "/sys/fs/cgroup/subc-memory-test-{}-{}",
21987 std::process::id(),
21988 unix_ms_now()
21989 ));
21990 if let Err(error) = std::fs::create_dir(&root) {
21991 assert!(
21992 std::env::var_os("SUBC_REQUIRE_CGROUP_TEST").is_none(),
21993 "required cgroup test cannot execute: {error}"
21994 );
21995 eprintln!("SKIP linux_subtree_memory_covers_module_children: {error}");
21996 return;
21997 }
21998 let placement = subc_cgroup::prepare_at(&root)
21999 .unwrap()
22000 .expect("delegated cgroup");
22001 let supervisor = Supervisor::new_for_test(
22002 Arc::new(Registry::default()),
22003 RestartPolicy::new(3, Duration::ZERO),
22004 )
22005 .with_cgroup_placement(Some(placement));
22006 let ready = dir.join("ready");
22007 let module = supervisor
22008 .spawn(ModuleSpec {
22009 script: None,
22010 module_id: "memory-tree".into(),
22011 program: PathBuf::from("/bin/sh"),
22012 args: vec![
22013 "-c".into(),
22014 "sleep 600 & sleep 600 & sleep 600 & echo ready > \"$1\"; wait".into(),
22015 "fixture".into(),
22016 ready.display().to_string(),
22017 ],
22018 env: ["XDG_DATA_HOME", "XDG_RUNTIME_DIR", "XDG_CONFIG_HOME"]
22019 .into_iter()
22020 .map(|key| (key.into(), dir.display().to_string()))
22021 .collect(),
22022 reserved: false,
22023 reserved_prefixes: Vec::new(),
22024 protocol: ModuleProtocol::None,
22025 overlap: Default::default(),
22026 carry_containment: false,
22027 })
22028 .unwrap();
22029 let deadline = tokio::time::Instant::now() + Duration::from_secs(10);
22030 while !ready.exists() {
22031 assert!(
22032 tokio::time::Instant::now() < deadline,
22033 "children never started"
22034 );
22035 tokio::time::sleep(Duration::from_millis(10)).await;
22036 }
22037 let readings = crate::subtree_resources::read(vec![module.subtree_target()]).await;
22038 let reading = &readings[0];
22039 assert!(
22040 reading.process_count.is_some_and(|count| count >= 4),
22041 "{reading:?}"
22042 );
22043 assert!(
22044 reading.memory_bytes.is_some_and(|bytes| bytes > 0),
22045 "{reading:?}"
22046 );
22047 module.stop().await.unwrap();
22048 drop(module);
22049 std::fs::remove_dir(root.join("subc-modules")).unwrap();
22050 std::fs::remove_dir(&root).unwrap();
22051 eprintln!("EXECUTED linux_subtree_memory_covers_module_children: {reading:?}");
22052 }
22053
22054 fn running(pid: u32) -> bool {
22055 std::fs::read_to_string(format!("/proc/{pid}/stat"))
22057 .ok()
22058 .and_then(|stat| {
22059 stat.rsplit_once(") ")
22060 .map(|(_, rest)| rest.starts_with('Z'))
22061 })
22062 .is_some_and(|zombie| !zombie)
22063 }
22064
22065 #[tokio::test]
22066 async fn linux_teardown_reaps_the_grandchild() {
22067 teardown_tree("linux_teardown_reaps_the_grandchild", false).await;
22068 }
22069
22070 #[tokio::test]
22071 async fn linux_shutdown_straggler_reaps_the_grandchild() {
22072 teardown_tree("linux_shutdown_straggler_reaps_the_grandchild", true).await;
22073 }
22074
22075 async fn teardown_tree(test_name: &str, shutdown: bool) {
22076 let dir = ScratchDir::new(test_name);
22077 let root = PathBuf::from(format!(
22078 "/sys/fs/cgroup/subc-tree-test-{test_name}-{}-{}",
22079 std::process::id(),
22080 unix_ms_now()
22081 ));
22082 if let Err(error) = std::fs::create_dir(&root) {
22083 assert!(
22084 std::env::var_os("SUBC_REQUIRE_CGROUP_TEST").is_none(),
22085 "required cgroup test cannot execute: {error}"
22086 );
22087 eprintln!(
22088 "SKIP {test_name}: no writable delegated cgroup at {}: {error}",
22089 root.display()
22090 );
22091 return;
22092 }
22093 let placement = subc_cgroup::prepare_at(&root)
22094 .expect("prepare isolated kernel cgroup")
22095 .expect("isolated cgroup is delegated");
22096 let module_id = "tree-teardown";
22097 let module = placement
22098 .module_path(module_id)
22099 .expect("create isolated module cgroup");
22100 if !module.join("cgroup.kill").exists() {
22101 std::fs::remove_dir(&module).unwrap();
22102 std::fs::remove_dir(root.join("subc-modules")).unwrap();
22103 std::fs::remove_dir(&root).unwrap();
22104 assert!(
22105 std::env::var_os("SUBC_REQUIRE_CGROUP_TEST").is_none(),
22106 "required cgroup.kill interface unavailable"
22107 );
22108 eprintln!("SKIP {test_name}: cgroup.kill unavailable (kernel < 5.14)");
22109 return;
22110 }
22111 let supervisor = Supervisor::new_for_test(
22112 Arc::new(Registry::default()),
22113 RestartPolicy::new(3, Duration::ZERO),
22114 )
22115 .with_cgroup_placement(Some(placement));
22116 let mut runtime = supervisor.runtime_config();
22117 runtime.child_roster = runtime
22118 .child_roster
22119 .for_module(Arc::new(Mutex::new(Duration::from_millis(100))));
22120 let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::starting()));
22121 let pid_file = dir.join("grandchild.pid");
22122 let spec = ModuleSpec {
22123 script: None,
22124 module_id: module_id.to_string(),
22125 program: PathBuf::from("/bin/sh"),
22126 args: vec![
22127 "-c".into(),
22128 "trap '' TERM; sleep 600 & echo $! > \"$1\"; wait".into(),
22129 "fixture".into(),
22130 pid_file.display().to_string(),
22131 ],
22132 env: ["XDG_DATA_HOME", "XDG_RUNTIME_DIR", "XDG_CONFIG_HOME"]
22133 .into_iter()
22134 .map(|key| (key.to_string(), dir.display().to_string()))
22135 .collect(),
22136 reserved: false,
22137 reserved_prefixes: Vec::new(),
22138 protocol: ModuleProtocol::None,
22139 overlap: Default::default(),
22140 carry_containment: false,
22141 };
22142 let child =
22143 spawn_and_mark_running(&spec, &runtime, &snapshot).expect("spawn supervised tree");
22144 let deadline = tokio::time::Instant::now() + Duration::from_secs(10);
22145 let grandchild: u32 = loop {
22146 if let Ok(contents) = std::fs::read_to_string(&pid_file) {
22147 if let Ok(pid) = contents.trim().parse() {
22148 break pid;
22149 }
22150 }
22151 assert!(
22152 tokio::time::Instant::now() < deadline,
22153 "grandchild pid was not recorded"
22154 );
22155 tokio::time::sleep(Duration::from_millis(10)).await;
22156 };
22157 assert!(
22158 running(grandchild),
22159 "grandchild must be alive before teardown"
22160 );
22161 if shutdown {
22162 let mut child = child;
22163 crate::child_roster::end_children_for_daemon_shutdown(
22164 &runtime.child_roster,
22165 false,
22166 std::future::pending(),
22167 )
22168 .await;
22169 child.wait().await.expect("reap shutdown straggler");
22170 } else {
22171 drain_child_to_state(
22172 module_id,
22173 ModuleProtocol::None,
22174 StopNotice::NotSent,
22175 &Registry::default(),
22176 None,
22177 &snapshot,
22178 &runtime.terminal_ring,
22179 &SpawnEventFeed::default(),
22180 child,
22181 Duration::from_millis(100),
22182 ModuleState::Stopped,
22183 Some(false),
22184 &FailureContext::operator(),
22185 )
22186 .await
22187 .expect("real supervisor teardown");
22188 }
22189 let deadline = tokio::time::Instant::now() + Duration::from_secs(5);
22190 while running(grandchild) && tokio::time::Instant::now() < deadline {
22191 tokio::time::sleep(Duration::from_millis(10)).await;
22192 }
22193 let survived = running(grandchild);
22194 if survived {
22196 let pid = rustix::process::Pid::from_raw(grandchild as i32).unwrap();
22197 let _ = rustix::process::kill_process(pid, rustix::process::Signal::KILL);
22198 tokio::time::sleep(Duration::from_millis(100)).await;
22199 }
22200 if module.exists() {
22201 std::fs::remove_dir(&module).expect("remove empty module cgroup");
22202 }
22203 std::fs::remove_dir(root.join("subc-modules")).unwrap();
22204 std::fs::remove_dir(&root).unwrap();
22205 assert!(
22206 !survived,
22207 "grandchild {grandchild} outlived module teardown"
22208 );
22209 eprintln!("EXECUTED {test_name}: grandchild {grandchild} killed in isolated cgroup");
22210 }
22211}
22212
22213#[cfg(all(test, unix))]
22216mod record_write_stall_tests {
22217 use super::*;
22218 use std::sync::atomic::{AtomicBool, Ordering};
22219 use subc_transport::{authenticate_client, ConnectionInfo, Endpoint, SCHEMA_VERSION};
22220
22221 const MODULE_AUTH_DEADLINE: Duration = Duration::from_secs(2);
22224 const HELD_RECORD_WRITE: Duration = Duration::from_secs(3);
22228
22229 fn sleeping_spec() -> ModuleSpec {
22241 ModuleSpec {
22242 script: None,
22243 module_id: "record-write-stall".into(),
22244 program: "/bin/sleep".into(),
22245 args: vec!["600".into()],
22246 env: vec![],
22247 reserved: false,
22248 reserved_prefixes: vec![],
22249 protocol: ModuleProtocol::None,
22250 overlap: Default::default(),
22251 carry_containment: false,
22252 }
22253 }
22254
22255 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
22256 async fn module_auth_completes_while_the_restart_waits_on_the_record_write() {
22257 let home = cortexkit_test_support::ScratchDir::new("record-write-stall");
22258 let supervisor =
22259 Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default())
22260 .with_live_children_record(home.join("live-children.json"));
22261 let module = supervisor.spawn(sleeping_spec()).unwrap();
22262 let old_pid = tokio::time::timeout(Duration::from_secs(10), async {
22267 loop {
22268 if let Some(pid) = lock_snapshot(&module.inner.snapshot)
22269 .unwrap()
22270 .reported_pid()
22271 {
22272 return pid;
22273 }
22274 tokio::time::sleep(Duration::from_millis(10)).await;
22275 }
22276 })
22277 .await
22278 .expect("the first process was admitted as the module");
22279
22280 let key = vec![0x42; 32];
22283 let daemon_id = [0x24; 16];
22284 let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
22285 let port = listener.local_addr().unwrap().port();
22286 let server = tokio::spawn(crate::server::serve_listener(
22287 listener,
22288 Arc::new(crate::router::Router::with_default_self_handler()),
22289 crate::server::ServerAuth::with_limits(
22290 key.clone(),
22291 daemon_id,
22292 "test",
22293 crate::server::DEFAULT_AUTH_DEADLINE,
22294 4,
22295 ),
22296 ));
22297
22298 let (held_tx, held_rx) = std::sync::mpsc::channel();
22301 let armed = AtomicBool::new(true);
22302 supervisor
22303 .child_roster
22304 .set_record_write_hook_for_test(move |live| {
22305 let admitting_replacement = live.values().any(|entry| entry.pid != old_pid);
22306 if admitting_replacement && armed.swap(false, Ordering::SeqCst) {
22307 let _ = held_tx.send(());
22308 std::thread::sleep(HELD_RECORD_WRITE);
22309 }
22310 });
22311
22312 let module_process = std::thread::spawn(move || {
22315 held_rx
22316 .recv_timeout(Duration::from_secs(30))
22317 .expect("the restart reached the replacement's record write");
22318 let conn = ConnectionInfo {
22319 schema: SCHEMA_VERSION,
22320 wire_version: None,
22321 endpoints: vec![Endpoint {
22322 host: "127.0.0.1".to_owned(),
22323 port,
22324 }],
22325 key,
22326 daemon_id,
22327 pid: std::process::id(),
22328 daemon_ver: "test".to_owned(),
22329 };
22330 tokio::runtime::Builder::new_current_thread()
22331 .enable_all()
22332 .build()
22333 .unwrap()
22334 .block_on(async {
22335 let started = Instant::now();
22336 let mut stream = tokio::net::TcpStream::connect(("127.0.0.1", port))
22337 .await
22338 .unwrap();
22339 let result =
22340 authenticate_client(&mut stream, &conn, MODULE_AUTH_DEADLINE).await;
22341 (result, started.elapsed())
22342 })
22343 });
22344
22345 module.restart(Some(0)).await.unwrap();
22346 let (auth, elapsed) = tokio::task::spawn_blocking(move || module_process.join().unwrap())
22347 .await
22348 .unwrap();
22349 assert!(
22350 auth.is_ok(),
22351 "the daemon must answer a module's handshake while a restart waits on the \
22352 live-children record write; got {auth:?} after {elapsed:?}"
22353 );
22354
22355 tokio::time::timeout(Duration::from_secs(20), async {
22356 while lock_snapshot(&module.inner.snapshot)
22357 .unwrap()
22358 .reported_pid()
22359 .is_none_or(|pid| pid == old_pid)
22360 {
22361 tokio::time::sleep(Duration::from_millis(10)).await;
22362 }
22363 })
22364 .await
22365 .expect("the restart published its replacement");
22366 module.stop().await.unwrap();
22367 server.abort();
22368 }
22369}
22370
22371#[cfg(all(test, unix))]
22372mod retained_status_tests {
22373 use super::*;
22374
22375 #[tokio::test]
22376 async fn cached_container_block_is_visible_in_status_and_health_without_rechecking() {
22377 use crate::live_children::{ContainerIdentity, RetainedReason};
22378 for (reason, carried, expected) in [
22379 (
22380 RetainedReason::ContainerSurvivors(vec![4242]),
22381 true,
22382 Some("waiting: container survivors (pids [4242])".to_owned()),
22383 ),
22384 (
22385 RetainedReason::ContainerUnverifiable("member inspection denied".into()),
22386 true,
22387 Some(
22388 "waiting: container survivors could not be verified: member inspection denied"
22389 .to_owned(),
22390 ),
22391 ),
22392 (RetainedReason::Survived, false, None),
22393 ] {
22394 let supervisor =
22395 Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default());
22396 let spec = ModuleSpec {
22397 script: None,
22398 module_id: "retained-status-fixture".into(),
22399 program: PathBuf::from("/not-spawned"),
22400 args: Vec::new(),
22401 env: Vec::new(),
22402 reserved: false,
22403 reserved_prefixes: Vec::new(),
22404 protocol: ModuleProtocol::None,
22405 overlap: ModuleOverlap::Exclusive,
22406 carry_containment: carried,
22407 };
22408 let module = supervisor
22409 .supervise_configured(spec.clone(), false)
22410 .unwrap();
22411 let guard = supervisor
22415 .child_roster
22416 .admit(
22417 spec.module_id,
22418 0,
22419 ModuleProtocol::None,
22420 None,
22421 crate::child_roster::RecordedIdentity {
22422 container: carried.then_some(ContainerIdentity {
22423 pgid: Some(0),
22424 created_start_time: Some(0),
22425 cgroup_path: None,
22426 ..ContainerIdentity::test_metadata()
22427 }),
22428 ..crate::child_roster::RecordedIdentity::default()
22429 },
22430 )
22431 .unwrap();
22432 guard.retain(reason);
22433 let status = module.status().unwrap();
22434 assert_eq!(status.waiting_for_previous_detail, expected.clone());
22435 assert_eq!(status.health.detail, expected);
22436 assert!(!status.process_alive);
22437 }
22438 }
22439}
22440
22441fn registration_child_is_running(child: &mut SupervisedChild) -> bool {
22442 child.try_wait().is_ok_and(|status| status.is_none())
22443}