1#[cfg(all(test, target_os = "linux"))]
6#[path = "supervise/audit_identity_tests.rs"]
7mod audit_identity_tests;
8mod resource_launch;
9pub(crate) use resource_launch::{ResourceCommitError, ResourceLaunch};
10
11use std::{
12 collections::{HashMap, HashSet, VecDeque},
13 error::Error,
14 fmt, io,
15 path::PathBuf,
16 process::{ExitStatus, Stdio},
17 sync::{Arc, Mutex, OnceLock},
18 time::{Duration, SystemTime, UNIX_EPOCH},
19};
20
21use cortexkit_log::Retention;
22use serde_json::Value;
23use sha2::{Digest, Sha256};
24use subc_control::{
25 ClientControlPush, LiveSpawn, ModuleProtocol, RouteCloseReason, SpawnCursor, SpawnEvent,
26 SpawnEventKind, SpawnSnapshot, SupervisorHealthStatus, TerminalDisposition, TerminalExitKind,
27};
28use subc_protocol::{
29 manifest::{SelfSignalKind, SignalAnchor},
30 session::{
31 HealthReport, HealthStatus, ModuleControlCommand, ModuleControlRequest,
32 MODULE_CONTROL_OP_HEALTH_CHECK,
33 },
34 Flags, FrameType, Principal, Priority, SUBC_LAUNCH_NONCE_ENV, SUBC_MODULE_ID_ENV,
35};
36use tokio::{
37 process::{Child, Command},
38 sync::{mpsc, oneshot, watch, Mutex as AsyncMutex},
39 task::JoinHandle,
40 time::{sleep, sleep_until, timeout, timeout_at, Instant},
41};
42use tracing::{debug, error, info, warn};
43
44use crate::{
45 child_roster::ChildRoster,
46 daemon_config::{
47 CAPTURE_KEEP_ENV, CAPTURE_MAX_AGE_DAYS_ENV, CAPTURE_MAX_FILE_MB_ENV, CK_LOG_ENV,
48 },
49 forwarding::{
50 CloseReason, ForwardingError, ForwardingTable, GoodbyeTarget, ModuleControlRpcOutcome,
51 ModuleDrainTarget, PendingModuleControlRpc,
52 },
53 provenance::{ExecutableIdentityProbe, SpawnedFileIdentity},
54 registry::{ConnectionId, RegistrationEndReason, RegistryError},
55 stderr_tail::{
56 pump_stderr_to, pump_stdout_to, ChildOutputSink, StderrRing, StderrTailConfig,
57 StderrTailSnapshot,
58 },
59 terminal_ring::{TerminalHistorySnapshot, TerminalRecord, TerminalRing, TerminalRingConfig},
60 Frame, FrameSink, Registry,
61};
62
63#[cfg(not(windows))]
64use crate::provenance::spawned_file_identity;
65
66#[path = "supervise_swap.rs"]
67mod swap;
68
69#[path = "supervise_activity.rs"]
70mod activity;
71
72#[cfg(unix)]
73#[path = "supervise/admission.rs"]
74mod admission;
75mod admission_exit;
76#[cfg(all(test, target_os = "linux"))]
77mod async_spawn_tests;
78#[cfg(all(test, windows))]
79mod carried_job_tests;
80mod config_delivery;
81mod content_binding;
82mod program_binding;
83mod registration;
84pub(crate) mod script;
85type NonceLaunches = HashMap<String, (String, std::sync::Weak<RegistrationSpawn>)>;
87pub(crate) use registration::NonceAttestation;
88#[cfg(all(test, unix))]
89pub(crate) use registration::RegistrationProof;
90pub(crate) use registration::RegistrationSpawn;
91pub(crate) use registration::REGISTRATION_CHANNEL_CLOSED;
92#[cfg(all(test, unix))]
93pub(crate) mod unix_channel_placement_tests;
94use program_binding::ProgramBindingPublication;
95mod config_exit_failure;
96#[cfg(all(test, any(target_os = "linux", windows)))]
97mod missing_path_tests;
98#[cfg(all(test, unix))]
99mod program_binding_tests;
100#[cfg(all(test, windows))]
101mod program_binding_windows_tests;
102#[cfg(all(test, windows))]
103mod windows_channel_tests;
104pub use config_exit_failure::ModuleStopReason;
105#[cfg(all(test, unix))]
106mod config_exit_hold_tests;
107pub(crate) mod containment;
108#[cfg(all(test, unix))]
109mod containment_tests;
110mod slow_retry;
111use slow_retry::{FailedRetry, FailureContext};
112mod host_health;
113use host_health::HostHealth;
114
115#[cfg(all(test, unix))]
116mod mark_readiness_tests;
117#[cfg(all(test, unix))]
118mod slow_retry_tests;
119
120pub const SUBC_ARG: &str = "--subc";
126
127const DEFAULT_MAX_RESTARTS: u32 = 3;
128const DEFAULT_BACKOFF: Duration = Duration::from_millis(100);
129const DEFAULT_MAX_BACKOFF: Duration = Duration::from_secs(30);
130const DEFAULT_RESTART_WINDOW: Duration = Duration::from_secs(600);
134pub const DEFAULT_DRAIN_TIMEOUT: Duration = Duration::from_secs(30);
145pub(crate) const DEFAULT_REGISTER_TIMEOUT: Duration = Duration::from_secs(120);
146const REGISTRY_RELEASE_TIMEOUT: Duration = Duration::from_secs(1);
147const REGISTRY_RELEASE_POLL: Duration = Duration::from_millis(10);
148const DRAIN_BUSY_REPROBE_INTERVAL: Duration = Duration::from_millis(250);
160const STDERR_PUMP_DRAIN_TIMEOUT: Duration = Duration::from_millis(250);
183pub const SPAWN_EVENT_RING_CAPACITY: usize = 4096;
185const SPAWN_SUBSCRIBER_BUFFER: usize = SPAWN_EVENT_RING_CAPACITY + 1;
186pub(crate) const SPAWN_SUBSCRIBER_LAGGED_CODE: &str = "spawn_subscriber_lagged";
191
192#[cfg(target_os = "macos")]
193struct PrivacyExec {
194 content_check: Option<tokio::task::JoinHandle<Result<(), crate::config_gate::GateError>>>,
195 gated: bool,
196 admission_binding: Option<ProgramBindingPublication>,
197 reader: tokio::io::unix::AsyncFd<std::io::PipeReader>,
198 deadline: tokio::time::Instant,
199 expected: Option<subc_os::FileIdentity>,
200 trampoline: Option<subc_os::FileIdentity>,
201 signature: Option<Arc<subc_os::privacy_identity::SuspendedSignature>>,
202 module_id: String,
203 #[cfg(test)]
204 confirmation_pause: Option<(String, Option<tokio::net::TcpStream>)>,
205 #[cfg(test)]
206 image_probe: Option<fn(u32) -> Option<subc_os::Observation>>,
207 #[cfg(test)]
208 image_read_delay: Option<Duration>,
209}
210
211#[cfg(all(test, target_os = "macos"))]
212pub(crate) fn test_privacy_trampoline() -> PathBuf {
213 let path = cortexkit_test_support::cargo_profile_bin("privacy-trampoline-fixture");
216 assert!(
220 path.exists(),
221 "privacy-trampoline-fixture not built at {}: run `cargo build -p subc-daemon \
222 --bins --features test-support` or `cargo test -p subc-daemon` first",
223 path.display()
224 );
225 path
226}
227
228#[cfg(target_os = "macos")]
229fn probe_privacy_trampoline(path: &std::path::Path) -> Result<(), String> {
230 use std::io::Read;
231 let mut probe = subc_os::child_spawn::spawn(
232 std::process::Command::new(path)
233 .args(["__disclaim-exec", "--probe-suspended"])
234 .stdin(Stdio::null())
235 .stdout(Stdio::piped())
236 .stderr(Stdio::piped()),
237 )
238 .map_err(|error| {
239 format!(
240 "privacy trampoline probe failed for {}: {error}",
241 path.display()
242 )
243 })?;
244 let deadline = std::time::Instant::now() + Duration::from_secs(5);
245 let status = loop {
246 match probe.try_wait() {
247 Ok(Some(status)) => break status,
248 Ok(None) if std::time::Instant::now() < deadline => {
249 std::thread::sleep(Duration::from_millis(5))
250 }
251 result => {
252 let _ = probe.kill();
253 let _ = probe.wait();
254 return Err(format!(
255 "privacy trampoline probe failed or timed out for {}: {result:?}",
256 path.display()
257 ));
258 }
259 }
260 };
261 let mut answer = String::new();
262 if let Some(stdout) = probe.stdout.take() {
263 let _ = stdout.take(256).read_to_string(&mut answer);
264 }
265 if status.success() && answer.trim() == subc_os::privacy_identity::SUSPENDED_TRAMPOLINE_PROBE {
266 return Ok(());
267 }
268 let mut diagnostic = String::new();
269 if let Some(stderr) = probe.stderr.take() {
270 let _ = stderr.take(1024).read_to_string(&mut diagnostic);
271 }
272 let cause = diagnostic
273 .trim()
274 .strip_prefix("ck-subc: own privacy identity refused: ")
275 .unwrap_or("binary does not implement the privacy trampoline protocol");
276 Err(format!(
277 "{cause}: probe of {} exited {status}",
278 path.display()
279 ))
280}
281
282#[cfg(target_os = "macos")]
283fn privacy_command(
284 spec: &ModuleSpec,
285 binding: &SpawnProgramBinding,
286 roster: &ChildRoster,
287 gated: bool,
288) -> Result<
289 (
290 Command,
291 Option<PrivacyExec>,
292 subc_os::privacy_identity::ExecAcknowledgement,
293 ),
294 SuperviseError,
295> {
296 let failure = |cause: String| {
297 warn!(module_id = %spec.module_id, %cause, "privacy identity trampoline refused module spawn");
298 SuperviseError::Spawn {
299 program: spec.program.clone(),
300 source: io::Error::other(cause),
301 cgroup_path: None,
302 }
303 };
304 let trampoline = roster.privacy_trampoline().map_err(failure)?;
305 let expected = Some(binding.identity);
306 let trampoline_image = subc_os::file_identity(&trampoline);
307 let signature = gated
308 .then(|| {
309 subc_os::privacy_identity::SuspendedSignature::capture(&binding._file, &binding.path)
310 .map(Arc::new)
311 })
312 .transpose()
313 .map_err(|error| failure(error.to_string()))?;
314 if expected == trampoline_image {
315 return Err(failure(
316 "privacy identity module executable is missing or is the trampoline itself".to_string(),
317 ));
318 }
319 let (reader, ack) = subc_os::privacy_identity::ExecAcknowledgement::pipe()
320 .map_err(|error| failure(error.to_string()))?;
321 let reader =
322 tokio::io::unix::AsyncFd::new(reader).map_err(|error| failure(error.to_string()))?;
323 let mut command = Command::new(&trampoline);
324 command
325 .arg(if gated {
326 "__disclaim-exec-suspended"
327 } else {
328 "__disclaim-exec"
329 })
330 .arg(ack.fd().to_string())
331 .arg(&binding.path);
332 Ok((
333 command,
334 Some(PrivacyExec {
335 gated,
336 admission_binding: None,
337 content_check: None,
338 reader,
339 deadline: tokio::time::Instant::now() + Duration::from_secs(5),
340 expected,
341 trampoline: trampoline_image,
342 signature,
343 module_id: spec.module_id.clone(),
344 #[cfg(test)]
345 image_probe: None,
346 #[cfg(test)]
347 image_read_delay: spec
348 .env
349 .iter()
350 .find(|(key, _)| key == "SUBC_TEST_PRIVACY_IMAGE_DELAY_MS")
351 .map(|(_, millis)| Duration::from_millis(millis.parse().unwrap())),
352 #[cfg(test)]
353 confirmation_pause: spec
354 .env
355 .iter()
356 .find(|(key, _)| key == "SUBC_TEST_BEFORE_EXEC_CONFIRM")
357 .map(|(_, address)| (address.clone(), None)),
358 }),
359 ack,
360 ))
361}
362
363#[cfg(unix)]
364type SpawnRegistrationChannel = std::os::unix::net::UnixStream;
365#[cfg(windows)]
366type SpawnRegistrationChannel = subc_os::registration_channel::RegistrationPipe;
367#[cfg(test)]
368type CapturedRegistrationChannels =
369 Arc<Mutex<Option<Vec<(Arc<RegistrationSpawn>, CapturedRegistrationEndpoint)>>>>;
370#[cfg(test)]
371struct CapturedRegistrationEndpoint {
372 _channel: SpawnRegistrationChannel,
373}
374#[cfg(test)]
375impl fmt::Debug for CapturedRegistrationEndpoint {
376 fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
377 f.write_str("CapturedRegistrationEndpoint")
378 }
379}
380
381struct SupervisedChild {
382 registration_spawn: Option<Arc<RegistrationSpawn>>,
383 #[cfg(any(unix, windows))]
385 registration_channel: Option<SpawnRegistrationChannel>,
386 live_program_binding: Option<Arc<content_binding::LiveProgramBinding>>,
387 registered_config_connection: Option<ConnectionId>,
388 admission_base: Option<subc_config::Digest>,
389 registration_declaration: Option<String>,
390 _program_binding: Option<ProgramBindingPublication>,
391 subtree_target: crate::subtree_resources::Target,
392 child: containment::ProcessChild,
393 exec: crate::activity::ExecOwner,
394 #[cfg(windows)]
395 nonce_delivery: Option<subc_os::LaunchNoncePipeDelivery>,
396 #[cfg(target_os = "macos")]
397 privacy_exec: Option<PrivacyExec>,
398 #[cfg(target_os = "macos")]
405 report_ready: Arc<OnceLock<()>>,
406 spawn_failure: Option<String>,
409 admission_kill_sent: bool,
412 protocol: ModuleProtocol,
417 #[cfg(target_os = "linux")]
424 module_id: String,
425 #[cfg(target_os = "linux")]
426 cgroup_placement: Option<subc_cgroup::Placement>,
427 #[cfg(windows)]
450 job: Option<Arc<Mutex<subc_jobobject::JobObject>>>,
451 stdout_pump: Option<JoinHandle<()>>,
452 stderr_pump: Option<StderrPump>,
453 stderr_ring: Arc<Mutex<StderrRing>>,
454 spawned_at: Instant,
457 config_delivery: Option<crate::config_gate::runtime::DeliverySpawn>,
458 registration_deadline: Option<Instant>,
461 registration_watch: Option<RegistrationWatch>,
462 spawned_at_ms: u64,
463 spawned_from: PathBuf,
464 launch_spec_digest: LaunchSpecDigest,
465 spawned_file_identity: Option<SpawnedFileIdentity>,
466 process_start_time: Option<u64>,
467 process_identity: Option<ProcessIdentity>,
468 pid: u32,
469 roster_guard: Option<crate::child_roster::RosterGuard>,
472}
473
474impl SupervisedChild {
475 fn retire_registration(&self) {
476 if let Some(spawn) = &self.registration_spawn {
481 spawn.retire();
482 } else if let Some(binding) = &self._program_binding {
483 binding.retire();
484 }
485 #[cfg(unix)]
486 if let Some(channel) = &self.registration_channel {
487 let _ = channel.shutdown(std::net::Shutdown::Both);
488 }
489 #[cfg(windows)]
490 if let Some(channel) = &self.registration_channel {
491 if let Ok(shutdown) = channel.shutdown_handle() {
492 shutdown.shutdown();
493 }
494 }
495 }
496 fn registered_since_spawn(&self) -> bool {
497 self.registration_watch
498 .as_ref()
499 .is_some_and(|watch| *watch.changes.borrow() != watch.baseline)
500 }
501
502 fn awaiting_registration(&self) -> bool {
503 let admission_failed = self.admission_kill_sent && self.spawn_failure.is_some();
507 !admission_failed && !self.registered_since_spawn()
508 }
509
510 fn id(&self) -> Option<u32> {
511 Some(self.pid)
512 }
513
514 fn process_identity(&self) -> Option<ProcessIdentity> {
515 self.process_identity
516 }
517
518 fn try_wait(&mut self) -> io::Result<Option<ExitStatus>> {
519 let result = self.child.try_wait();
520 if !matches!(result, Ok(None)) {
521 self.retire_registration();
522 #[cfg(any(unix, windows))]
523 self.registration_channel.take();
524 if let Some(binding) = &self._program_binding {
525 binding.retire();
526 }
527 }
528 result
529 }
530
531 async fn wait(&mut self) -> io::Result<ExitStatus> {
532 #[cfg(target_os = "macos")]
533 self.confirm_privacy_exec().await;
534 let result = self.child.wait().await;
542 self.retire_registration();
543 #[cfg(any(unix, windows))]
544 self.registration_channel.take();
545 if let Some(binding) = &self._program_binding {
548 binding.retire();
549 }
550 #[cfg(all(test, target_os = "linux"))]
551 if let Some(binding) = &self._program_binding {
552 binding.pause_after_wait_for_test().await;
553 }
554 #[cfg(target_os = "linux")]
555 if result.is_ok() && self.child.container().is_none() {
556 if let Some(placement) = self.cgroup_placement.as_ref() {
557 cleanup_reaped_cgroup(placement, &self.module_id).await;
561 self.cgroup_placement = None;
562 }
563 }
564 result
565 }
566
567 async fn finish_exit(&mut self, carry: bool) -> io::Result<()> {
568 self.retire_registration();
569 if let Some(binding) = self._program_binding.take() {
571 binding.retire();
572 }
573 if self.child.container().is_some() {
574 if !carry {
575 self.child.terminate_container().await;
576 }
577 } else {
578 #[cfg(target_os = "linux")]
579 if let Some(placement) = self.cgroup_placement.as_ref() {
580 cleanup_reaped_cgroup(placement, &self.module_id).await;
581 self.cgroup_placement = None;
582 }
583 }
584 self.child.reap().await?;
585 self.release_roster();
586 Ok(())
587 }
588
589 #[cfg(target_os = "macos")]
590 async fn confirm_privacy_exec(&mut self) {
591 if self
592 .privacy_exec
593 .as_ref()
594 .is_some_and(|pending| pending.content_check.is_some())
595 {
596 self.finish_privacy_content_check().await;
597 return;
598 }
599 let Some(pending) = &mut self.privacy_exec else {
600 return;
601 };
602 let result = tokio::time::timeout_at(pending.deadline, async {
603 let mut record = Vec::new();
604 loop {
605 let mut ready = pending.reader.readable().await?;
606 let read = ready.try_io(|reader| {
607 use std::io::Read;
608 let mut reader = reader.get_ref();
609 let mut buffer = [0u8; 256];
610 reader.read(&mut buffer).map(|count| (count, buffer))
611 });
612 match read {
613 Ok(Ok((0, _))) => return Ok::<_, io::Error>(record),
614 Ok(Ok((count, buffer))) => {
615 if record.len() + count > 1024 {
616 return Err(io::Error::other(
617 "privacy exec refusal record is too long",
618 ));
619 }
620 record.extend_from_slice(&buffer[..count]);
621 }
622 Ok(Err(error)) => return Err(error),
623 Err(_) => continue,
624 }
625 }
626 })
627 .await;
628 #[cfg(test)]
629 let result = if let Some((address, stream)) = &mut pending.confirmation_pause {
630 use tokio::io::{AsyncReadExt, AsyncWriteExt};
631 if stream.is_none() {
632 *stream = Some(
633 tokio::net::TcpStream::connect(address.as_str())
634 .await
635 .unwrap(),
636 );
637 stream.as_mut().unwrap().write_all(b"R").await.unwrap();
638 stream
639 .as_mut()
640 .unwrap()
641 .write_all(&self.pid.to_ne_bytes())
642 .await
643 .unwrap();
644 }
645 let mut reply = [0];
646 stream
647 .as_mut()
648 .unwrap()
649 .read_exact(&mut reply)
650 .await
651 .unwrap();
652 pending.confirmation_pause = None;
653 if &reply == b"F" {
654 Ok(Err(io::Error::other("injected late exec refusal")))
655 } else {
656 assert_eq!(&reply, b"X");
657 result
658 }
659 } else {
660 result
661 };
662 let waited = if matches!(&result, Ok(Ok(record)) if record.is_empty()) {
667 Some(self.try_wait())
668 } else {
669 None
670 };
671 let pending = self.privacy_exec.as_ref().expect("pending exec");
672 let cause = match result {
673 Err(_) => Some("privacy identity trampoline did not exec within 5s".to_string()),
674 Ok(Err(error)) => Some(format!(
675 "privacy identity exec acknowledgement failed: {error}"
676 )),
677 Ok(Ok(record)) if !record.is_empty() => Some(
678 std::str::from_utf8(&record)
679 .ok()
680 .and_then(|record| {
681 record
682 .trim()
683 .strip_prefix(subc_os::privacy_identity::EXEC_REFUSAL_TAG)
684 })
685 .filter(|cause| !cause.is_empty())
686 .unwrap_or("invalid privacy exec refusal record")
687 .to_string(),
688 ),
689 Ok(Ok(_)) if !pending.gated => match waited.expect("exec EOF checked child") {
690 Ok(Some(_)) => None,
691 Err(error) => Some(format!("privacy identity trampoline wait failed: {error}")),
692 Ok(None) => {
693 #[cfg(test)]
694 if let Some(delay) = pending.image_read_delay {
695 tokio::time::sleep(delay).await;
696 }
697 let image = observe_spawned_image(self.pid);
698 #[cfg(test)]
699 let image = pending.image_probe.map_or(image, |probe| probe(self.pid));
700 if let Some(image) = image.filter(|image| {
704 image.executable.is_some() && image.executable != pending.trampoline
705 }) {
706 match self
707 .roster_guard
708 .as_ref()
709 .map(|guard| guard.confirm_executable(image))
710 .transpose()
711 {
712 Err(error) => Some(format!(
713 "could not durably record confirmed module image: {error}"
714 )),
715 Ok(_) => {
716 let _ = self.report_ready.set(());
717 None
718 }
719 }
720 } else if image.is_none()
721 || image.is_some_and(|image| image.executable.is_none())
722 {
723 match tokio::time::timeout_at(pending.deadline, self.child.wait()).await {
726 Ok(result) => {
727 self.retire_registration();
728 self.registration_channel.take();
729 if let Some(binding) = &self._program_binding {
730 binding.retire();
731 }
732 result.err().map(|error| format!("privacy identity trampoline wait failed: {error}"))
733 }
734 Err(_) => Some("privacy identity module executable remained unreadable until the 5s exec deadline".to_string()),
735 }
736 } else {
737 Some("privacy identity trampoline did not exec the module".to_string())
738 }
739 }
740 },
741 Ok(Ok(_)) => match waited.expect("exec EOF checked child") {
742 Ok(Some(_)) => {
743 Some("privacy identity target exited before suspended admission".to_string())
744 }
745 Err(error) => Some(format!("privacy identity trampoline wait failed: {error}")),
746 Ok(None) => {
747 let stopped = tokio::time::timeout_at(pending.deadline, async {
750 loop {
751 if subc_os::privacy_identity::SuspendedSignature::is_suspended(
752 self.pid,
753 )? {
754 return Ok::<_, io::Error>(());
755 }
756 tokio::time::sleep(Duration::from_millis(1)).await;
757 }
758 })
759 .await;
760 match stopped {
761 Err(_) => {
762 Some("privacy identity target was not suspended within 5s".to_string())
763 }
764 Ok(Err(error)) => Some(format!(
765 "privacy identity suspended target unavailable: {error}"
766 )),
767 Ok(Ok(())) => {
768 let image = observe_spawned_image(self.pid);
769 if image.is_some_and(|image| {
770 image.executable.is_some()
771 && image.executable != pending.trampoline
772 && image.executable == pending.expected
773 }) {
774 None
775 } else {
776 Some("privacy identity trampoline executable mismatch: expected the suspended module program".to_string())
777 }
778 }
779 }
780 }
781 },
782 };
783 #[cfg(test)]
784 if let Some(binding) = &self._program_binding {
785 binding.pause_after_wait_for_test().await;
786 }
787 if cause.is_none() && pending.gated {
788 let publication = pending.admission_binding.clone();
789 let live = self.live_program_binding.clone();
790 let signature = Arc::clone(pending.signature.as_ref().expect("gated signature"));
791 let pid = self.pid;
792 let gated = pending.gated;
793 self.privacy_exec
794 .as_mut()
795 .expect("pending exec")
796 .content_check = Some(tokio::task::spawn_blocking(move || {
797 signature.verify(pid).map_err(|error| {
798 crate::config_gate::GateError::new("program_refused", error)
799 })?;
800 if !gated {
801 return Ok(());
802 }
803 match (publication, live) {
804 (Some(publication), _) => publication.verify(),
805 (None, Some(live)) => live.check(),
806 _ => Err(crate::config_gate::GateError::new(
807 "config_delivery_deferred",
808 "live program witness is unavailable",
809 )),
810 }
811 }));
812 self.finish_privacy_content_check().await;
813 } else {
814 self.finish_privacy_exec(cause);
815 }
816 }
817
818 #[cfg(target_os = "macos")]
819 async fn finish_privacy_content_check(&mut self) {
820 let result = self
821 .privacy_exec
822 .as_mut()
823 .expect("pending exec")
824 .content_check
825 .as_mut()
826 .expect("pending content check")
827 .await;
828 let cause = match result {
829 Ok(Ok(())) => None,
830 Ok(Err(error)) => Some(format!("{}: {}", error.code, error.detail)),
831 Err(error) => Some(format!(
832 "config_delivery_deferred: content check stopped: {error}"
833 )),
834 };
835 self.finish_privacy_exec(cause);
836 }
837
838 #[cfg(target_os = "macos")]
839 fn finish_privacy_exec(&mut self, mut cause: Option<String>) {
840 let mut pending = self.privacy_exec.take().expect("pending exec");
841 if cause.is_none() && pending.gated {
842 let admission = (|| -> io::Result<()> {
843 let image = observe_spawned_image(self.pid)
844 .filter(|image| image.executable == pending.expected)
845 .ok_or_else(|| io::Error::other("suspended executable mismatch"))?;
846 if let Some(guard) = &self.roster_guard {
847 guard.confirm_executable(image)?;
848 }
849 pending
850 .signature
851 .as_ref()
852 .expect("gated signature")
853 .resume(self.pid)?;
854 let _ = self.report_ready.set(());
855 info!(module_id = %pending.module_id, pid = self.pid,
856 "verified suspended module resumed with own privacy identity");
857 Ok(())
858 })();
859 if let Err(error) = admission {
860 cause = Some(format!("program_refused: {error}"));
861 }
862 }
863 if cause.is_some() {
864 if let Some(spawn) = &self.registration_spawn {
865 spawn.refuse();
866 }
867 if let Some(binding) = &mut pending.admission_binding {
868 binding.reject();
869 }
870 } else if let Some(spawn) = &self.registration_spawn {
871 spawn.ready();
872 }
873 self.exec.0.finish(if cause.is_some() {
876 crate::activity::ExecState::Refused
877 } else {
878 crate::activity::ExecState::CommittedExec
879 });
880 if let Some(cause) = cause {
881 warn!(module_id = %pending.module_id, pid = self.pid, %cause, "privacy identity trampoline refused module spawn");
882 self.spawn_failure = Some(
883 if pending.gated
884 && (cause.contains("executable mismatch")
885 || cause.starts_with("program_refused:"))
886 {
887 format!("program_refused: {cause}")
888 } else if pending.gated {
889 format!("config_delivery_deferred: {cause}")
890 } else {
891 cause
892 },
893 );
894 let _ = self.start_admission_kill();
897 }
898 }
899
900 fn release_roster(&mut self) {
904 self.roster_guard = None;
905 }
906
907 fn start_kill(&mut self) -> io::Result<()> {
920 self.retire_registration();
921 if let Some(container) = self.child.container() {
922 container.hard_kill();
923 return self.kill_leader();
924 }
925 #[cfg(windows)]
926 if let Some(job) = &self.job {
927 if let Err(error) = job.lock().unwrap_or_else(|p| p.into_inner()).terminate() {
928 debug!(
929 error = %error,
930 "job termination failed; the direct-child kill still owns the outcome"
931 );
932 }
933 }
934 #[cfg(target_os = "linux")]
935 kill_module_cgroup(self.cgroup_placement.as_ref(), &self.module_id);
936 self.child.start_kill()
937 }
938
939 fn kill_leader(&mut self) -> io::Result<()> {
940 self.retire_registration();
941 self.child.start_kill()
942 }
943
944 fn start_admission_kill(&mut self) -> io::Result<()> {
945 self.retire_registration();
946 #[cfg(windows)]
947 let result = {
948 let result = self
951 .spawned_file_identity
952 .as_ref()
953 .and_then(|identity| identity.windows_image.as_ref())
954 .ok_or_else(|| io::Error::other("admission child handle unavailable"))
955 .and_then(|image| {
956 image.process().terminate_with_exit_code(
957 admission_exit::WINDOWS_ADMISSION_KILL_EXIT_CODE as u32,
958 )
959 });
960 if let Some(container) = self.child.container() {
961 container.hard_kill();
962 } else if let Some(job) = &self.job {
963 if let Err(error) = job.lock().unwrap_or_else(|p| p.into_inner()).terminate() {
964 debug!(%error, "job termination failed after admission leader termination");
965 }
966 }
967 result
968 };
969 #[cfg(not(windows))]
970 let result = self.start_kill().map(|()| true);
971 #[cfg(unix)]
972 let result = {
973 let group_sent = self.child.container().is_none()
976 && rustix::process::Pid::from_raw(self.pid as i32).is_some_and(|group| {
977 rustix::process::kill_process_group(group, rustix::process::Signal::KILL)
978 .is_ok()
979 });
980 if group_sent {
981 Ok(true)
982 } else {
983 result
984 }
985 };
986 self.admission_kill_sent |= result?;
987 Ok(())
988 }
989
990 async fn drain_stderr(&mut self, module_id: &str) {
991 if let Some(pump) = self.stdout_pump.take() {
992 settle_stdout_pump(module_id, pump, STDERR_PUMP_DRAIN_TIMEOUT).await;
993 }
994
995 let Some(pump) = self.stderr_pump.take() else {
996 return;
997 };
998 settle_stderr_pump(
999 module_id,
1000 &self.stderr_ring,
1001 pump,
1002 STDERR_PUMP_DRAIN_TIMEOUT,
1003 )
1004 .await;
1005 }
1006}
1007
1008async fn settle_stdout_pump(module_id: &str, mut pump: JoinHandle<()>, bound: Duration) {
1012 match timeout(bound, &mut pump).await {
1013 Ok(Ok(())) => {}
1014 Ok(Err(error)) => {
1015 warn!(module_id, error = %error, "stdout pump ended unexpectedly");
1016 }
1017 Err(_) => {
1018 drop(pump);
1019 warn!(
1020 module_id,
1021 waited = ?bound,
1022 "stdout pipe of the exited process has not drained; its reader keeps running without delaying the restart"
1023 );
1024 }
1025 }
1026}
1027
1028struct StderrPump {
1031 task: JoinHandle<()>,
1032 generation: u64,
1033}
1034
1035async fn settle_stderr_pump(
1041 module_id: &str,
1042 ring: &Arc<Mutex<StderrRing>>,
1043 pump: StderrPump,
1044 bound: Duration,
1045) {
1046 let lock = || ring.lock().unwrap_or_else(|poisoned| poisoned.into_inner());
1047 let StderrPump {
1048 mut task,
1049 generation,
1050 } = pump;
1051 lock().retire_pump(generation);
1052 match timeout(bound, &mut task).await {
1053 Ok(Ok(())) => {}
1054 Ok(Err(err)) => {
1055 let mut ring = lock();
1056 ring.mark_incomplete_from(generation, format!("stderr pump ended unexpectedly: {err}"));
1057 ring.finish_pump(generation);
1058 warn!(module_id, error = %err, "stderr pump ended before clean EOF");
1059 }
1060 Err(_) => {
1061 drop(task);
1063 lock().mark_pump_late(
1064 generation,
1065 format!(
1066 "stderr of the exited process had not reached EOF {bound:?} after it was \
1067 retired (a descendant may still hold the pipe open); lines it still \
1068 writes are kept in that process's section"
1069 ),
1070 );
1071 warn!(
1072 module_id,
1073 waited = ?bound,
1074 "stderr pipe of the exited process is still open; its reader keeps running without delaying the restart"
1075 );
1076 }
1077 }
1078}
1079
1080fn registration_release_events() -> &'static watch::Sender<u64> {
1081 static EVENTS: OnceLock<watch::Sender<u64>> = OnceLock::new();
1082 EVENTS.get_or_init(|| {
1083 let (sender, _receiver) = watch::channel(0);
1084 sender
1085 })
1086}
1087
1088pub(crate) fn notify_registration_release() {
1089 let events = registration_release_events();
1090 let next_generation = (*events.borrow()).wrapping_add(1);
1091 events.send_replace(next_generation);
1092}
1093
1094#[derive(Debug, Clone, Copy, PartialEq, Eq)]
1096#[non_exhaustive]
1097pub enum ScriptRuntime {
1098 Bun,
1099 Python,
1100}
1101
1102impl ScriptRuntime {
1103 pub const fn as_str(self) -> &'static str {
1104 match self {
1105 Self::Bun => "bun",
1106 Self::Python => "python",
1107 }
1108 }
1109}
1110
1111#[derive(Debug, Clone, PartialEq, Eq)]
1113#[non_exhaustive]
1114pub struct ScriptEntry {
1115 pub runtime: ScriptRuntime,
1116 pub path: PathBuf,
1117}
1118
1119impl ScriptEntry {
1120 pub fn new(runtime: ScriptRuntime, path: PathBuf) -> Self {
1121 Self { runtime, path }
1122 }
1123}
1124
1125#[derive(Debug, Clone, PartialEq, Eq)]
1127pub struct ModuleSpec {
1128 pub module_id: String,
1129 pub program: PathBuf,
1130 pub script: Option<ScriptEntry>,
1132 pub args: Vec<String>,
1133 pub env: Vec<(String, String)>,
1134 pub reserved: bool,
1139 pub reserved_prefixes: Vec<String>,
1144 pub protocol: ModuleProtocol,
1163 pub overlap: ModuleOverlap,
1168 pub carry_containment: bool,
1172}
1173
1174#[derive(Clone, PartialEq, Eq)]
1177pub(crate) struct LaunchSpecDigest([u8; 32]);
1178
1179impl fmt::Debug for LaunchSpecDigest {
1180 fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
1181 f.write_str("LaunchSpecDigest([redacted])")
1182 }
1183}
1184
1185impl LaunchSpecDigest {
1186 pub(crate) fn from_spec(spec: &ModuleSpec) -> Self {
1187 fn field(hash: &mut Sha256, bytes: &[u8]) {
1188 hash.update((bytes.len() as u64).to_le_bytes());
1189 hash.update(bytes);
1190 }
1191 let mut hash = Sha256::new();
1192 if let Some(script) = &spec.script {
1193 field(
1194 &mut hash,
1195 match script.runtime {
1196 ScriptRuntime::Bun => b"bun-script",
1197 ScriptRuntime::Python => b"python-script",
1198 },
1199 );
1200 field(&mut hash, script.path.as_os_str().as_encoded_bytes());
1201 }
1202 if let Some(program) = spec.program.to_str() {
1203 field(&mut hash, b"utf8-program");
1204 field(&mut hash, program.as_bytes());
1205 } else {
1206 #[cfg(unix)]
1209 {
1210 use std::os::unix::ffi::OsStrExt;
1211 field(&mut hash, b"unix-program");
1212 field(&mut hash, spec.program.as_os_str().as_bytes());
1213 }
1214 #[cfg(windows)]
1215 {
1216 use std::os::windows::ffi::OsStrExt;
1217 field(&mut hash, b"windows-program");
1218 let bytes = spec
1219 .program
1220 .as_os_str()
1221 .encode_wide()
1222 .flat_map(u16::to_le_bytes)
1223 .collect::<Vec<_>>();
1224 field(&mut hash, &bytes);
1225 }
1226 #[cfg(not(any(unix, windows)))]
1227 field(&mut hash, spec.program.to_string_lossy().as_bytes());
1228 }
1229 hash.update((spec.args.len() as u64).to_le_bytes());
1230 for arg in &spec.args {
1231 field(&mut hash, arg.as_bytes());
1232 }
1233 let mut env = spec.env.iter().collect::<Vec<_>>();
1234 env.sort_unstable();
1235 hash.update((env.len() as u64).to_le_bytes());
1236 for (key, value) in env {
1237 field(&mut hash, key.as_bytes());
1238 field(&mut hash, value.as_bytes());
1239 }
1240 field(
1241 &mut hash,
1242 match spec.protocol {
1243 ModuleProtocol::Subc => b"subc",
1244 ModuleProtocol::None => b"none",
1245 },
1246 );
1247 Self(hash.finalize().into())
1248 }
1249}
1250
1251#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)]
1258pub enum ModuleOverlap {
1259 #[default]
1261 Exclusive,
1262 Safe,
1274}
1275
1276impl ModuleOverlap {
1277 pub fn as_str(self) -> &'static str {
1278 match self {
1279 Self::Exclusive => "exclusive",
1280 Self::Safe => "safe",
1281 }
1282 }
1283}
1284
1285pub const SUBC_SPAWN_ROLE_ENV: &str = "SUBC_SPAWN_ROLE";
1295pub const SPAWN_ROLE_SWAP_CANDIDATE: &str = "swap_candidate";
1297pub const DEFAULT_SWAP_READY_TIMEOUT: Duration = Duration::from_secs(100);
1302
1303#[derive(Debug, Clone, Copy, PartialEq, Eq)]
1321pub struct RestartPolicy {
1322 pub max_restarts: u32,
1323 pub backoff: Duration,
1326 pub max_backoff: Duration,
1328 pub window: Duration,
1332}
1333
1334impl RestartPolicy {
1335 pub fn new(max_restarts: u32, backoff: Duration) -> Self {
1339 Self {
1340 max_restarts,
1341 backoff,
1342 max_backoff: DEFAULT_MAX_BACKOFF,
1343 window: DEFAULT_RESTART_WINDOW,
1344 }
1345 }
1346
1347 pub fn with_max_backoff(mut self, max_backoff: Duration) -> Self {
1348 self.max_backoff = max_backoff;
1349 self
1350 }
1351
1352 pub fn with_window(mut self, window: Duration) -> Self {
1353 self.window = window;
1354 self
1355 }
1356
1357 fn delay_for_restart(&self, restart_in_window: u32) -> Duration {
1362 if self.backoff.is_zero() || self.max_backoff.is_zero() {
1363 return Duration::ZERO;
1364 }
1365
1366 let mut delay = self.backoff;
1367 for _ in 0..restart_in_window {
1368 if delay >= self.max_backoff {
1369 return self.max_backoff;
1370 }
1371 delay = delay
1372 .checked_mul(10)
1373 .unwrap_or(self.max_backoff)
1374 .min(self.max_backoff);
1375 }
1376 delay.min(self.max_backoff)
1377 }
1378
1379 fn budget_exhausted_detail(&self) -> String {
1384 format!(
1385 "crash budget exhausted: max_restarts={} within window_secs={}",
1386 self.max_restarts,
1387 self.window.as_secs()
1388 )
1389 }
1390}
1391
1392impl Default for RestartPolicy {
1393 fn default() -> Self {
1394 Self {
1395 max_restarts: DEFAULT_MAX_RESTARTS,
1396 backoff: DEFAULT_BACKOFF,
1397 max_backoff: DEFAULT_MAX_BACKOFF,
1398 window: DEFAULT_RESTART_WINDOW,
1399 }
1400 }
1401}
1402
1403#[derive(Debug, Clone, Copy, PartialEq, Eq)]
1404struct CrashRestartSchedule {
1405 restart_in_window: u32,
1406 delay: Duration,
1407}
1408
1409fn daemon_will_restart(
1416 state: &mut SupervisorSnapshot,
1417 policy: &RestartPolicy,
1418 now: Instant,
1419) -> bool {
1420 state.enabled && state.crash_restarts_in_window(policy.window, now) < policy.max_restarts
1421}
1422
1423const DEFAULT_HEALTH_CADENCE: Duration = Duration::from_secs(30);
1424const DEFAULT_HEALTH_DEADLINE: Duration = Duration::from_secs(5);
1425const DEFAULT_HEALTH_FAILURE_THRESHOLD: u32 = 3;
1426const MAX_HEALTH_METRICS_BYTES: usize = 16 * 1024;
1427
1428#[derive(Debug, Clone, Copy, PartialEq, Eq)]
1429pub enum HealthAction {
1430 Report,
1431 Restart,
1432 Alert,
1433}
1434
1435impl fmt::Display for HealthAction {
1436 fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
1437 f.write_str(match self {
1438 Self::Report => "report",
1439 Self::Restart => "restart",
1440 Self::Alert => "alert",
1441 })
1442 }
1443}
1444
1445#[derive(Debug, Clone, PartialEq, Eq)]
1446pub struct HealthConfig {
1447 pub http: Option<String>,
1451 pub cadence: Duration,
1452 pub deadline: Duration,
1453 pub failure_threshold: u32,
1454 pub on_degraded: HealthAction,
1455 pub on_failing: HealthAction,
1456 pub critical: bool,
1457}
1458
1459impl Default for HealthConfig {
1460 fn default() -> Self {
1461 Self {
1462 http: None,
1463 cadence: DEFAULT_HEALTH_CADENCE,
1464 deadline: DEFAULT_HEALTH_DEADLINE,
1465 failure_threshold: DEFAULT_HEALTH_FAILURE_THRESHOLD,
1466 on_degraded: HealthAction::Report,
1467 on_failing: HealthAction::Report,
1468 critical: false,
1469 }
1470 }
1471}
1472
1473#[derive(Debug, Clone, PartialEq)]
1491pub struct ModuleHealthStatus {
1492 pub status: SupervisorHealthStatus,
1493 pub last_probe_ms: Option<u64>,
1494 pub detail: Option<String>,
1495 pub metrics: Option<Value>,
1496 pub consecutive_failures: u32,
1497 pub late_answer_count: u64,
1500 pub last_late_answer_latency_ms: Option<u64>,
1502 pub last_action: Option<String>,
1503 pub last_action_ms: Option<u64>,
1507}
1508
1509impl Default for ModuleHealthStatus {
1510 fn default() -> Self {
1511 Self {
1512 status: SupervisorHealthStatus::Unknown,
1513 last_probe_ms: None,
1514 detail: None,
1515 metrics: None,
1516 consecutive_failures: 0,
1517 late_answer_count: 0,
1518 last_late_answer_latency_ms: None,
1519 last_action: None,
1520 last_action_ms: None,
1521 }
1522 }
1523}
1524
1525#[derive(Debug, Clone, Copy, PartialEq, Eq)]
1527pub enum ModuleState {
1528 Starting,
1529 Running,
1530 Unresponsive,
1531 Restarting,
1532 Draining,
1533 Stopped,
1534 Failed,
1535 Disabled,
1536}
1537
1538impl fmt::Display for ModuleState {
1539 fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
1540 f.write_str(match self {
1541 Self::Starting => "starting",
1542 Self::Running => "running",
1543 Self::Unresponsive => "unresponsive",
1544 Self::Restarting => "restarting",
1545 Self::Draining => "draining",
1546 Self::Stopped => "stopped",
1547 Self::Failed => "failed",
1548 Self::Disabled => "disabled",
1549 })
1550 }
1551}
1552
1553#[derive(Debug, Clone, Copy, PartialEq, Eq)]
1555pub enum ExitKind {
1556 Clean,
1557 Crash,
1558 DeliberateSeverance,
1559}
1560
1561impl From<ExitKind> for TerminalExitKind {
1562 fn from(kind: ExitKind) -> Self {
1563 match kind {
1564 ExitKind::Clean => Self::Clean,
1565 ExitKind::Crash => Self::Crash,
1566 ExitKind::DeliberateSeverance => Self::DeliberateSeverance,
1567 }
1568 }
1569}
1570
1571#[derive(Debug, Clone, Copy, PartialEq, Eq)]
1574pub(crate) struct ProcessIdentity {
1575 pub(crate) pid: u32,
1576 pub(crate) start_time: u64,
1577}
1578
1579#[derive(Debug, Clone, PartialEq, Eq)]
1581pub struct ExitReport {
1582 pub kind: ExitKind,
1583 pub code: Option<i32>,
1584 pub signal: Option<i32>,
1585 pub at_ms: u64,
1586}
1587
1588#[derive(Debug, Clone, PartialEq)]
1591pub struct ModuleStatus {
1592 pub(crate) admission_refused: Option<String>,
1593 pub(crate) legacy_registration_spawn: bool,
1594 pub(crate) registration_declaration: Option<String>,
1595 pub module_id: String,
1596 pub stop_reason: Option<ModuleStopReason>,
1599 pub waiting_for_previous_pid: Option<u32>,
1601 pub waiting_for_previous_detail: Option<String>,
1603 pub launch_nonce_source: Option<String>,
1608 pub state: ModuleState,
1609 pub enabled: bool,
1610 pub process_alive: bool,
1611 pub registration_active: bool,
1612 pub protocol: ModuleProtocol,
1617 pub live: bool,
1628 pub restart_count: u32,
1632 pub lifetime_restarts: u32,
1636 pub spawn_generation: u64,
1637 pub swap_outcome: Option<SwapOutcome>,
1639 pub max_restarts: u32,
1644 pub restart_window: Duration,
1648 pub drain_timeout: Duration,
1652 pub restart_backoff: Duration,
1653 pub restart_max_backoff: Duration,
1654 pub pid: Option<u32>,
1659 pub spawned_at_ms: Option<u64>,
1660 pub spawned_from: Option<PathBuf>,
1661 pub(crate) launch_spec_digest: Option<LaunchSpecDigest>,
1662 pub process_start_time: Option<u64>,
1663 pub last_exit: Option<ExitReport>,
1664 pub health: ModuleHealthStatus,
1665}
1666
1667#[cfg(test)]
1668#[derive(Debug, Clone, Copy, PartialEq)]
1669struct ActorSelectCheckpoint {
1670 generation: u64,
1671 turn: u64,
1672 registered_connection: Option<ConnectionId>,
1673 next_probe_at: Option<Instant>,
1674 wake_after: Option<Duration>,
1675}
1676
1677#[derive(Debug, Clone, PartialEq)]
1678struct SupervisorSnapshot {
1679 admission_refused: Option<String>,
1680 config_hold_parked: bool,
1682 config_hold_daemon_initiated: bool,
1685 config_hold_approval: Option<crate::config_gate::runtime::ApprovalIdentity>,
1687 approved_start_failure: Option<(crate::config_gate::runtime::ApprovalIdentity, String)>,
1689 legacy_registration_spawn: bool,
1690 registration_declaration: Option<String>,
1691 stop_reason: Option<ModuleStopReason>,
1692 config_exit_failed_approval: Option<crate::config_gate::runtime::ApprovalIdentity>,
1693 subtree_target: crate::subtree_resources::Target,
1694 failed_retry: FailedRetry,
1695 exec_outcome: Option<Arc<crate::activity::ExecOutcome>>,
1696 #[cfg(windows)]
1697 pipe_consumed: Option<Arc<OnceLock<()>>>,
1698 #[cfg(test)]
1700 actor_turns: u64,
1701 #[cfg(test)]
1704 actor_select: Option<ActorSelectCheckpoint>,
1705 state: ModuleState,
1706 enabled: bool,
1707 process_alive: bool,
1708 spawned_protocol: Option<ModuleProtocol>,
1709 spawn_failure: Option<String>,
1710 waiting_for_previous_pid: Option<u32>,
1711 waiting_for_previous_detail: Option<String>,
1712 crash_restarts: VecDeque<Instant>,
1718 lifetime_restarts: u32,
1719 spawn_generation: u64,
1728 swap_outcome: Option<SwapOutcome>,
1729 pid: Option<u32>,
1730 #[cfg(target_os = "macos")]
1731 report_ready: Option<Arc<OnceLock<()>>>,
1732 reaped_pid: Option<u32>,
1734 respawn_pending: bool,
1736 coalesced_restart_pending: bool,
1738 spawned_at_ms: Option<u64>,
1739 spawned_from: Option<PathBuf>,
1740 launch_spec_digest: Option<LaunchSpecDigest>,
1741 spawned_file_identity: Option<SpawnedFileIdentity>,
1742 process_start_time: Option<u64>,
1743 deliberate_severance: Option<ProcessIdentity>,
1744 last_exit: Option<ExitReport>,
1745 drain_disposition_detail: Option<String>,
1747 health: ModuleHealthStatus,
1748 last_probe_evidence: Option<HealthProbeEvidence>,
1749 in_alternate_slot: bool,
1754 draining_to_replace: bool,
1761 configuration_updated_since_spawn: bool,
1767}
1768
1769impl SupervisorSnapshot {
1770 fn reported_pid(&self) -> Option<u32> {
1775 #[cfg(target_os = "macos")]
1776 if self
1777 .report_ready
1778 .as_ref()
1779 .is_some_and(|ready| ready.get().is_none())
1780 {
1781 return None;
1782 }
1783 self.pid
1784 }
1785
1786 fn starting() -> Self {
1787 Self::new(ModuleState::Starting, true)
1788 }
1789
1790 fn disabled() -> Self {
1791 Self::new(ModuleState::Disabled, false)
1792 }
1793
1794 fn crash_restarts_in_window(&mut self, window: Duration, now: Instant) -> u32 {
1798 while let Some(oldest) = self.crash_restarts.front() {
1799 if now.duration_since(*oldest) > window {
1800 self.crash_restarts.pop_front();
1801 } else {
1802 break;
1803 }
1804 }
1805 u32::try_from(self.crash_restarts.len()).unwrap_or(u32::MAX)
1806 }
1807
1808 fn record_crash_restart(&mut self, policy: &RestartPolicy, now: Instant) {
1814 self.crash_restarts.push_back(now);
1815 while self.crash_restarts.len() > policy.max_restarts as usize {
1816 self.crash_restarts.pop_front();
1817 }
1818 self.lifetime_restarts += 1;
1819 }
1820
1821 fn next_crash_restart(
1825 &mut self,
1826 policy: &RestartPolicy,
1827 now: Instant,
1828 ) -> Option<CrashRestartSchedule> {
1829 let restart_in_window = self.crash_restarts_in_window(policy.window, now);
1830 if restart_in_window >= policy.max_restarts {
1831 return None;
1832 }
1833 self.record_crash_restart(policy, now);
1834 Some(CrashRestartSchedule {
1835 restart_in_window,
1836 delay: policy.delay_for_restart(restart_in_window),
1837 })
1838 }
1839
1840 fn clear_crash_restarts(&mut self) {
1845 self.crash_restarts.clear();
1846 }
1847
1848 fn new(state: ModuleState, enabled: bool) -> Self {
1849 Self {
1850 admission_refused: None,
1851 config_hold_parked: false,
1852 config_hold_daemon_initiated: false,
1853 config_hold_approval: None,
1854 approved_start_failure: None,
1855 legacy_registration_spawn: false,
1856 registration_declaration: None,
1857 stop_reason: None,
1858 config_exit_failed_approval: None,
1859 subtree_target: crate::subtree_resources::Target::Unavailable("not_running"),
1860 failed_retry: FailedRetry::default(),
1861 exec_outcome: None,
1862 #[cfg(test)]
1863 actor_turns: 0,
1864 #[cfg(test)]
1865 actor_select: None,
1866 state,
1867 enabled,
1868 process_alive: false,
1869 #[cfg(windows)]
1870 pipe_consumed: None,
1871 spawned_protocol: None,
1872 spawn_failure: None,
1873 waiting_for_previous_pid: None,
1874 waiting_for_previous_detail: None,
1875 crash_restarts: VecDeque::new(),
1876 lifetime_restarts: 0,
1877 spawn_generation: 0,
1878 swap_outcome: None,
1879 pid: None,
1880 #[cfg(target_os = "macos")]
1881 report_ready: None,
1882 reaped_pid: None,
1883 respawn_pending: false,
1884 coalesced_restart_pending: false,
1885 spawned_at_ms: None,
1886 spawned_from: None,
1887 launch_spec_digest: None,
1888 spawned_file_identity: None,
1889 process_start_time: None,
1890 deliberate_severance: None,
1891 last_exit: None,
1892 drain_disposition_detail: None,
1893 health: ModuleHealthStatus::default(),
1894 last_probe_evidence: None,
1895 in_alternate_slot: false,
1896 draining_to_replace: false,
1897 configuration_updated_since_spawn: false,
1898 }
1899 }
1900}
1901
1902type SharedSnapshot = Arc<Mutex<SupervisorSnapshot>>;
1903
1904type SpawnSubscriberKey = (ConnectionId, u64);
1905
1906#[derive(Debug)]
1907struct SpawnSubscriber {
1908 version: u8,
1909 frames: mpsc::Sender<Frame>,
1910 lagged: Option<oneshot::Sender<SpawnCursor>>,
1914}
1915
1916#[derive(Debug)]
1917struct SpawnEventState {
1918 daemon_incarnation: String,
1919 seq: u64,
1920 capacity: usize,
1921 live: HashMap<String, LiveSpawn>,
1922 generations: HashMap<String, u64>,
1923 events: VecDeque<SpawnEvent>,
1924 subscribers: HashMap<SpawnSubscriberKey, SpawnSubscriber>,
1925 process_changes: HashMap<String, tokio::sync::watch::Sender<u64>>,
1926}
1927
1928impl Default for SpawnEventState {
1929 fn default() -> Self {
1930 Self {
1931 daemon_incarnation: "unconfigured".to_string(),
1932 seq: 0,
1933 capacity: SPAWN_EVENT_RING_CAPACITY,
1934 live: HashMap::new(),
1935 generations: HashMap::new(),
1936 events: VecDeque::new(),
1937 subscribers: HashMap::new(),
1938 process_changes: HashMap::new(),
1939 }
1940 }
1941}
1942
1943#[derive(Debug, Clone, Default)]
1944struct SpawnEventFeed(
1945 Arc<Mutex<SpawnEventState>>,
1946 #[cfg(test)] Arc<Mutex<Option<std::net::TcpStream>>>,
1947);
1948
1949#[derive(Debug, Clone, PartialEq, Eq)]
1950pub(crate) enum SpawnSubscribeRefusal {
1951 ForeignIncarnation { current: String },
1952 TooOld { oldest: SpawnCursor },
1953 Frame(String),
1954}
1955
1956impl SpawnEventFeed {
1957 fn configure_incarnation(&self, daemon_incarnation: String) {
1958 let mut state = self.0.lock().unwrap_or_else(|p| p.into_inner());
1959 state.daemon_incarnation = daemon_incarnation;
1960 state.seq = 0;
1961 state.live.clear();
1962 state.generations.clear();
1963 state.events.clear();
1964 state.subscribers.clear();
1965 for changed in state.process_changes.values() {
1966 changed.send_replace(0);
1967 }
1968 }
1969
1970 fn cursor(state: &SpawnEventState) -> SpawnCursor {
1971 SpawnCursor {
1972 daemon_incarnation: state.daemon_incarnation.clone(),
1973 seq: state.seq,
1974 }
1975 }
1976
1977 fn snapshot(&self) -> SpawnSnapshot {
1978 let state = self.0.lock().unwrap_or_else(|p| p.into_inner());
1979 let mut live = state.live.values().cloned().collect::<Vec<_>>();
1980 live.sort_by(|left, right| left.module_id.cmp(&right.module_id));
1981 SpawnSnapshot {
1982 cursor: Self::cursor(&state),
1983 ring_bound: state.capacity as u64,
1984 live,
1985 }
1986 }
1987
1988 fn allocate_generation_locked(state: &mut SpawnEventState, module_id: &str) -> u64 {
1989 let generation = state
1990 .generations
1991 .get(module_id)
1992 .copied()
1993 .unwrap_or(0)
1994 .checked_add(1)
1995 .expect("spawn generation exhausted");
1996 state.generations.insert(module_id.to_string(), generation);
1997 generation
1998 }
1999
2000 fn allocate_generation(&self, module_id: &str) -> (String, u64) {
2005 let mut state = self.0.lock().unwrap_or_else(|p| p.into_inner());
2006 let generation = Self::allocate_generation_locked(&mut state, module_id);
2007 (state.daemon_incarnation.clone(), generation)
2008 }
2009
2010 fn emit_spawned(
2014 &self,
2015 module_id: &str,
2016 allocated: Option<u64>,
2017 pid: u32,
2018 spawned_at_ms: u64,
2019 ) -> u64 {
2020 let mut state = self.0.lock().unwrap_or_else(|p| p.into_inner());
2021 let generation = match allocated {
2022 Some(generation) => generation,
2023 None => Self::allocate_generation_locked(&mut state, module_id),
2024 };
2025 let live = LiveSpawn {
2026 module_id: module_id.to_string(),
2027 spawn_generation: generation,
2028 pid,
2029 spawned_at_ms,
2030 };
2031 state.live.insert(module_id.to_string(), live);
2032 Self::emit_locked(
2033 &mut state,
2034 SpawnEventKind::Spawned,
2035 module_id.to_string(),
2036 generation,
2037 pid,
2038 None,
2039 None,
2040 );
2041 generation
2042 }
2043
2044 fn emit_exited(&self, module_id: &str, exit_code: Option<i32>, exit_signal: Option<i32>) {
2045 let mut state = self.0.lock().unwrap_or_else(|p| p.into_inner());
2046 let Some(live) = state.live.remove(module_id) else {
2047 warn!(
2048 module_id,
2049 "terminal record had no live spawn event identity"
2050 );
2051 return;
2052 };
2053 Self::emit_locked(
2054 &mut state,
2055 SpawnEventKind::Exited,
2056 module_id.to_string(),
2057 live.spawn_generation,
2058 live.pid,
2059 exit_code,
2060 exit_signal,
2061 );
2062 }
2063
2064 fn emit_superseded_exited(
2071 &self,
2072 module_id: &str,
2073 spawn_generation: u64,
2074 pid: u32,
2075 exit_code: Option<i32>,
2076 exit_signal: Option<i32>,
2077 ) {
2078 let mut state = self.0.lock().unwrap_or_else(|p| p.into_inner());
2079 if state
2080 .live
2081 .get(module_id)
2082 .is_some_and(|live| live.spawn_generation == spawn_generation)
2083 {
2084 state.live.remove(module_id);
2085 }
2086 Self::emit_locked(
2087 &mut state,
2088 SpawnEventKind::Exited,
2089 module_id.to_string(),
2090 spawn_generation,
2091 pid,
2092 exit_code,
2093 exit_signal,
2094 );
2095 }
2096
2097 #[allow(clippy::too_many_arguments)]
2098 fn emit_locked(
2099 state: &mut SpawnEventState,
2100 kind: SpawnEventKind,
2101 module_id: String,
2102 spawn_generation: u64,
2103 pid: u32,
2104 exit_code: Option<i32>,
2105 exit_signal: Option<i32>,
2106 ) {
2107 state.seq = state
2108 .seq
2109 .checked_add(1)
2110 .expect("spawn event sequence exhausted");
2111 let event = SpawnEvent {
2112 cursor: Self::cursor(state),
2113 kind,
2114 module_id,
2115 spawn_generation,
2116 pid,
2117 exit_code,
2118 exit_signal,
2119 };
2120 state
2121 .process_changes
2122 .retain(|_, changed| changed.receiver_count() > 0);
2123 if let Some(changed) = state.process_changes.get(&event.module_id) {
2124 changed.send_replace(state.seq);
2125 }
2126 state.events.push_back(event.clone());
2127 while state.events.len() > state.capacity {
2128 state.events.pop_front();
2129 }
2130 let body = match serde_json::to_vec(&event) {
2131 Ok(body) => body,
2132 Err(error) => {
2133 error!(%error, "failed to serialize supervisor spawn event");
2134 return;
2135 }
2136 };
2137 state.subscribers.retain(|(connection_id, corr), subscriber| {
2138 let frame = Frame::build_with_version(
2139 subscriber.version,
2140 FrameType::StreamData,
2141 control_flags(),
2142 0,
2143 0,
2144 *corr,
2145 body.clone(),
2146 );
2147 match frame {
2148 Ok(frame) => {
2149 if subscriber.frames.try_send(frame).is_ok() {
2150 true
2151 } else {
2152 warn!(connection_id = connection_id.get(), corr, "dropping lagged supervisor spawn subscriber");
2153 if let Some(lagged) = subscriber.lagged.take() {
2154 let _ = lagged.send(event.cursor.clone());
2155 }
2156 false
2157 }
2158 }
2159 Err(error) => {
2160 warn!(connection_id = connection_id.get(), corr, %error, "dropping supervisor spawn subscriber after frame build failure");
2161 false
2162 }
2163 }
2164 });
2165 }
2166
2167 fn subscribe(
2168 &self,
2169 connection_id: ConnectionId,
2170 corr: u64,
2171 version: u8,
2172 since: Option<SpawnCursor>,
2173 sink: FrameSink,
2174 ) -> Result<(), SpawnSubscribeRefusal> {
2175 let (frames, mut receiver) = mpsc::channel(SPAWN_SUBSCRIBER_BUFFER);
2176 let (lagged, mut lagged_rx) = oneshot::channel::<SpawnCursor>();
2177 {
2178 let mut state = self.0.lock().unwrap_or_else(|p| p.into_inner());
2179 let replay = if let Some(since) = since {
2180 if since.daemon_incarnation != state.daemon_incarnation {
2181 return Err(SpawnSubscribeRefusal::ForeignIncarnation {
2182 current: state.daemon_incarnation.clone(),
2183 });
2184 }
2185 if let Some(oldest) = state.events.front().map(|event| event.cursor.clone()) {
2186 if since.seq < oldest.seq.saturating_sub(1) {
2187 return Err(SpawnSubscribeRefusal::TooOld { oldest });
2188 }
2189 }
2190 state
2191 .events
2192 .iter()
2193 .filter(|event| event.cursor.seq > since.seq)
2194 .cloned()
2195 .collect::<Vec<_>>()
2196 } else {
2197 Vec::new()
2198 };
2199 for event in replay {
2200 let body = serde_json::to_vec(&event)
2201 .map_err(|error| SpawnSubscribeRefusal::Frame(error.to_string()))?;
2202 let frame = Frame::build_with_version(
2203 version,
2204 FrameType::StreamData,
2205 control_flags(),
2206 0,
2207 0,
2208 corr,
2209 body,
2210 )
2211 .map_err(|error| SpawnSubscribeRefusal::Frame(error.to_string()))?;
2212 frames
2213 .try_send(frame)
2214 .map_err(|error| SpawnSubscribeRefusal::Frame(error.to_string()))?;
2215 }
2216 state.subscribers.insert(
2217 (connection_id, corr),
2218 SpawnSubscriber {
2219 version,
2220 frames,
2221 lagged: Some(lagged),
2222 },
2223 );
2224 }
2225 tokio::spawn(async move {
2236 while let Some(frame) = receiver.recv().await {
2237 if sink.send(frame).await.is_err() {
2238 return;
2239 }
2240 }
2241 let Ok(first_undelivered) = lagged_rx.try_recv() else {
2242 return;
2243 };
2244 match spawn_subscriber_lagged_frame(version, corr, first_undelivered) {
2245 Ok(frame) => {
2246 let _ = sink.send(frame).await;
2247 }
2248 Err(error) => {
2249 error!(%error, corr, "failed to build lagged spawn subscriber terminal frame");
2250 }
2251 }
2252 });
2253 Ok(())
2254 }
2255
2256 fn cancel(&self, connection_id: ConnectionId, corr: u64) -> bool {
2257 let Some(subscriber) = self
2258 .0
2259 .lock()
2260 .unwrap_or_else(|p| p.into_inner())
2261 .subscribers
2262 .remove(&(connection_id, corr))
2263 else {
2264 return false;
2265 };
2266 if let Ok(frame) = Frame::build_with_version(
2267 subscriber.version,
2268 FrameType::StreamEnd,
2269 control_flags(),
2270 0,
2271 0,
2272 corr,
2273 Vec::new(),
2274 ) {
2275 tokio::spawn(async move {
2276 let _ = subscriber.frames.send(frame).await;
2277 });
2278 }
2279 true
2280 }
2281
2282 fn remove_connection(&self, connection_id: ConnectionId) {
2283 self.0
2284 .lock()
2285 .unwrap_or_else(|p| p.into_inner())
2286 .subscribers
2287 .retain(|(subscriber_connection, _), _| *subscriber_connection != connection_id);
2288 }
2289
2290 #[cfg(any(test, feature = "test-support"))]
2291 fn set_capacity(&self, capacity: usize) {
2292 self.0.lock().unwrap_or_else(|p| p.into_inner()).capacity = capacity;
2293 }
2294
2295 #[cfg(any(test, feature = "test-support"))]
2296 fn subscriber_count(&self) -> usize {
2297 self.0
2298 .lock()
2299 .unwrap_or_else(|p| p.into_inner())
2300 .subscribers
2301 .len()
2302 }
2303}
2304
2305fn spawn_subscriber_lagged_frame(
2308 version: u8,
2309 corr: u64,
2310 first_undelivered: SpawnCursor,
2311) -> Result<Frame, String> {
2312 let body = serde_json::to_vec(&subc_protocol::ErrorBody {
2314 code: SPAWN_SUBSCRIBER_LAGGED_CODE.to_string(),
2315 message: "spawn subscriber fell behind and was dropped; resubscribe from the last cursor received"
2316 .to_string(),
2317 detail: Some(serde_json::json!({
2318 "first_undelivered_cursor": first_undelivered
2319 })),
2320 })
2321 .map_err(|error| error.to_string())?;
2322 Frame::build_with_version(
2323 version,
2324 FrameType::Error,
2325 crate::router::daemon_error_flags(),
2326 0,
2327 0,
2328 corr,
2329 body,
2330 )
2331 .map_err(|error| error.to_string())
2332}
2333
2334pub trait ModuleProcessLiveness: Send + Sync {
2335 fn process_live(&self, module_id: &str) -> Option<bool>;
2336
2337 fn process_replacing(&self, _module_id: &str) -> bool {
2343 false
2344 }
2345}
2346
2347#[derive(Debug, Clone, Default)]
2349pub struct SupervisorProcessLiveness {
2350 snapshots: Arc<Mutex<HashMap<String, SharedSnapshot>>>,
2351}
2352
2353impl SupervisorProcessLiveness {
2354 pub fn new() -> Self {
2355 Self::default()
2356 }
2357
2358 fn track(&self, module_id: String, snapshot: SharedSnapshot) {
2359 let mut snapshots = self
2360 .snapshots
2361 .lock()
2362 .unwrap_or_else(|poisoned| poisoned.into_inner());
2363 snapshots.insert(module_id, snapshot);
2364 }
2365
2366 fn untrack_if_current(&self, module_id: &str, snapshot: &SharedSnapshot) {
2367 let mut snapshots = self
2368 .snapshots
2369 .lock()
2370 .unwrap_or_else(|poisoned| poisoned.into_inner());
2371 let is_current = snapshots
2372 .get(module_id)
2373 .map(|tracked| Arc::ptr_eq(tracked, snapshot))
2374 .unwrap_or(false);
2375 if is_current {
2376 snapshots.remove(module_id);
2377 }
2378 }
2379}
2380
2381impl ModuleProcessLiveness for SupervisorProcessLiveness {
2382 fn process_live(&self, module_id: &str) -> Option<bool> {
2383 let snapshot = {
2384 let snapshots = self
2385 .snapshots
2386 .lock()
2387 .unwrap_or_else(|poisoned| poisoned.into_inner());
2388 snapshots.get(module_id).cloned()
2389 }?;
2390 let snapshot = snapshot
2391 .lock()
2392 .unwrap_or_else(|poisoned| poisoned.into_inner());
2393 Some(snapshot.state == ModuleState::Running && snapshot.process_alive)
2394 }
2395
2396 fn process_replacing(&self, module_id: &str) -> bool {
2397 let Some(snapshot) = self
2398 .snapshots
2399 .lock()
2400 .unwrap_or_else(|poisoned| poisoned.into_inner())
2401 .get(module_id)
2402 .cloned()
2403 else {
2404 return false;
2405 };
2406 let snapshot = snapshot
2407 .lock()
2408 .unwrap_or_else(|poisoned| poisoned.into_inner());
2409 snapshot.enabled
2410 && match snapshot.state {
2411 ModuleState::Restarting => true,
2412 ModuleState::Draining => snapshot.draining_to_replace,
2413 ModuleState::Starting
2414 | ModuleState::Running
2415 | ModuleState::Unresponsive
2416 | ModuleState::Stopped
2417 | ModuleState::Failed
2418 | ModuleState::Disabled => false,
2419 }
2420 }
2421}
2422
2423#[cfg(test)]
2424#[derive(Debug, Default)]
2425struct ReloadExitRecordGate {
2426 reached: tokio::sync::Notify,
2427 resume: tokio::sync::Notify,
2428}
2429
2430#[derive(Debug, Clone, Copy)]
2431enum RespawnKind {
2432 Spawn,
2433 Reload,
2434}
2435
2436#[derive(Debug, Clone)]
2437struct PendingRespawn {
2438 deadline: Instant,
2439 kind: RespawnKind,
2440 daemon_initiated: bool,
2441 approval: Option<crate::config_gate::runtime::ApprovalIdentity>,
2442}
2443
2444impl PendingRespawn {
2445 fn failure_context(&self, runtime: &SupervisorRuntimeConfig) -> FailureContext {
2446 FailureContext {
2447 approval: self.approval.clone(),
2448 ..runtime.failure_context(self.daemon_initiated)
2449 }
2450 }
2451}
2452
2453type ReloadReply = oneshot::Sender<Result<(), SuperviseError>>;
2454
2455#[derive(Debug, Clone)]
2456struct SupervisorRuntimeConfig {
2457 registry: Arc<Registry>,
2458 config_delivery: Option<config_delivery::ConfigDelivery>,
2459 recovery_is_daemon: bool,
2461 start_requester: Option<Principal>,
2462 pending_activity_restart: bool,
2463 provenance_probe: ExecutableIdentityProbe,
2464 scheduled_respawn: Arc<Mutex<Option<PendingRespawn>>>,
2466 deferred_reload_reply: Arc<Mutex<Option<ReloadReply>>>,
2468 restart_policy: RestartPolicy,
2469 register_timeout: Duration,
2470 drain_timeout: Duration,
2473 effective_drain_timeout: Arc<Mutex<Duration>>,
2476 default_drain_timeout: Duration,
2479 health: HealthConfig,
2480 host_health: HostHealth,
2481 connection_file_path: Option<PathBuf>,
2482 capture_logs_dir: Option<PathBuf>,
2483 forwarding: Option<Arc<ForwardingTable>>,
2484 supervisor_handle: Option<SupervisorHandle>,
2487 stderr_ring: Arc<Mutex<StderrRing>>,
2494 terminal_ring: Arc<Mutex<TerminalRing>>,
2495 spawn_events: SpawnEventFeed,
2496 child_roster: ChildRoster,
2497 #[cfg(target_os = "linux")]
2498 cgroup_placement: Option<subc_cgroup::Placement>,
2499 #[cfg(test)]
2500 test_seed_stale_facts_before_enable_spawn: bool,
2501 #[cfg(test)]
2502 test_reload_exit_record_gate: Option<Arc<ReloadExitRecordGate>>,
2503}
2504
2505impl SupervisorRuntimeConfig {
2506 fn failure_context(&self, daemon_initiated: bool) -> FailureContext {
2507 FailureContext {
2508 daemon_initiated,
2509 approval: None,
2510 shutdown: self.child_roster.shutdown_flag(),
2511 }
2512 }
2513}
2514
2515#[derive(Debug, Clone, PartialEq, Eq)]
2516struct SupervisedConfiguration {
2517 spec: ModuleSpec,
2518 enabled: bool,
2519 health: HealthConfig,
2520 register_timeout: Duration,
2521 restart_policy: RestartPolicy,
2522}
2523
2524#[derive(Debug, Clone, Default)]
2530pub struct SupervisorHandle {
2531 #[cfg(test)]
2532 resource_commit_pause: Arc<Mutex<Option<resource_launch::ResourceCommitPause>>>,
2533 #[cfg(test)]
2534 registration_test_channels: CapturedRegistrationChannels,
2535 registration_service: Arc<Mutex<Option<registration::RegistrationService>>>,
2536 registration_connections: Arc<Mutex<HashMap<ConnectionId, Arc<RegistrationSpawn>>>>,
2537 registration_launches: Arc<Mutex<Vec<std::sync::Weak<RegistrationSpawn>>>>,
2538 gate_runtime: Arc<Mutex<Option<std::sync::Weak<crate::config_gate::runtime::RuntimeOwner>>>>,
2539 program_bindings: program_binding::ProgramBindings,
2540 #[cfg(test)]
2541 identity_sample_pause: Arc<Mutex<Option<IdentitySamplePause>>>,
2542 #[cfg(all(test, unix))]
2543 plain_identity_policy_pause: Arc<Mutex<Option<(String, IdentitySamplePause)>>>,
2544 host_health: HostHealth,
2545 restart_intent: Arc<Mutex<DaemonRestartIntent>>,
2546 activity: Arc<Mutex<Option<crate::activity::ActivityLedger>>>,
2547 activity_images: Arc<Mutex<HashMap<String, activity::ActivityImage>>>,
2548 modules: Arc<Mutex<HashMap<String, SupervisedModule>>>,
2549 configured_ids: Arc<Mutex<HashSet<String>>>,
2561 spawn_events: SpawnEventFeed,
2562 reserved_nonces: Arc<Mutex<HashMap<String, Option<String>>>>,
2573 implicit_protocol_none: Arc<Mutex<HashSet<String>>>,
2577 removal_tombstones: Arc<Mutex<HashMap<String, u64>>>,
2583 spawn_nonces: Arc<Mutex<HashMap<String, String>>>,
2587 nonce_launches: Arc<Mutex<NonceLaunches>>,
2593 reserved_prefix_owners: Arc<Mutex<HashMap<String, String>>>,
2601 swaps: Arc<Mutex<HashMap<String, OpenSwap>>>,
2607 promotion_observer: PromotionObserverSlot,
2609 operation_lock: Arc<AsyncMutex<()>>,
2613}
2614
2615#[cfg(test)]
2616#[derive(Debug)]
2617struct IdentitySamplePause {
2618 reached: std::sync::mpsc::Sender<()>,
2619 release: std::sync::mpsc::Receiver<()>,
2620}
2621
2622pub(crate) trait SwapPromotionObserver: Send + Sync {
2631 fn swap_promoted(&self, registration: &crate::registry::ModuleRegistration);
2632}
2633
2634#[derive(Clone, Default)]
2638struct PromotionObserverSlot(Arc<Mutex<Option<std::sync::Weak<dyn SwapPromotionObserver>>>>);
2639
2640impl fmt::Debug for PromotionObserverSlot {
2641 fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
2642 f.write_str("PromotionObserverSlot")
2643 }
2644}
2645
2646#[derive(Debug, Clone)]
2648struct OpenSwap {
2649 candidate_nonce: String,
2652 incumbent_nonce: Option<String>,
2655 candidate_admitted: bool,
2659}
2660
2661#[derive(Debug, Clone, Copy, PartialEq, Eq)]
2664pub(crate) enum SwapHelloAdmission {
2665 NotSwapping,
2668 Candidate,
2670 Refused,
2673 ProtocolNone,
2676 ChannelRequired,
2679}
2680
2681#[derive(Debug, Clone, PartialEq, Eq)]
2682pub(crate) enum ReservedHelloRejection {
2683 Exact {
2684 module_id: String,
2685 },
2686 Prefix {
2687 prefix: String,
2688 owner_module_id: String,
2689 },
2690}
2691
2692#[derive(Debug, Default)]
2694pub(crate) struct ReservedHelloVerification {
2695 pub(crate) nonce_verified: bool,
2696 pub(crate) prefix_owner: Option<String>,
2697}
2698
2699#[derive(Debug)]
2700struct DaemonRestartIntent {
2701 expires: Option<Duration>,
2702 window: Duration,
2703}
2704
2705impl Default for DaemonRestartIntent {
2706 fn default() -> Self {
2707 Self {
2708 expires: None,
2709 window: Duration::from_secs(120),
2710 }
2711 }
2712}
2713
2714impl SupervisorHandle {
2715 pub(crate) fn module_is_gated(&self, module: &str) -> bool {
2716 let runtime = {
2717 let Ok(runtime) = self.gate_runtime.lock() else {
2718 return true;
2719 };
2720 runtime.clone()
2721 };
2722 runtime.is_some_and(|runtime| {
2723 runtime
2724 .upgrade()
2725 .is_none_or(|owner| owner.module_is_gated(module))
2726 })
2727 }
2728 pub(crate) fn announce_daemon_restart(&self) -> u64 {
2729 let mut intent = self
2730 .restart_intent
2731 .lock()
2732 .unwrap_or_else(|p| p.into_inner());
2733 intent.expires = Some(subc_uptime::suspend_inclusive_now() + intent.window);
2734 unix_ms_now().saturating_add(intent.window.as_millis() as u64)
2735 }
2736
2737 #[cfg(feature = "test-support")]
2738 pub(crate) fn set_shutdown_intent_window(&self, window: Duration) {
2739 self.restart_intent
2740 .lock()
2741 .unwrap_or_else(|p| p.into_inner())
2742 .window = window;
2743 }
2744
2745 #[cfg(unix)]
2746 fn take_daemon_shutdown_reason(&self) -> RouteCloseReason {
2747 let expiry = self
2748 .restart_intent
2749 .lock()
2750 .unwrap_or_else(|p| p.into_inner())
2751 .expires
2752 .take();
2753 if expiry.is_some_and(|expiry| subc_uptime::suspend_inclusive_now() < expiry) {
2754 RouteCloseReason::Restart
2755 } else {
2756 RouteCloseReason::Disable
2757 }
2758 }
2759
2760 pub fn new() -> Self {
2761 Self::default()
2762 }
2763
2764 pub(crate) fn spawn_snapshot(&self) -> SpawnSnapshot {
2765 self.spawn_events.snapshot()
2766 }
2767
2768 pub(crate) fn subscribe_process_changes(
2771 &self,
2772 module: &str,
2773 ) -> tokio::sync::watch::Receiver<u64> {
2774 let mut state = self
2775 .spawn_events
2776 .0
2777 .lock()
2778 .unwrap_or_else(|p| p.into_inner());
2779 state
2780 .process_changes
2781 .retain(|_, changed| changed.receiver_count() > 0);
2782 let seq = state.seq;
2783 state
2784 .process_changes
2785 .entry(module.into())
2786 .or_insert_with(|| tokio::sync::watch::channel(seq).0)
2787 .subscribe()
2788 }
2789
2790 pub(crate) fn current_spawn(&self, module: &str) -> Option<LiveSpawn> {
2791 self.spawn_events
2792 .0
2793 .lock()
2794 .unwrap_or_else(|p| p.into_inner())
2795 .live
2796 .get(module)
2797 .cloned()
2798 }
2799
2800 pub(crate) fn subscribe_spawns(
2801 &self,
2802 connection_id: ConnectionId,
2803 corr: u64,
2804 version: u8,
2805 since: Option<SpawnCursor>,
2806 sink: FrameSink,
2807 ) -> Result<(), SpawnSubscribeRefusal> {
2808 self.spawn_events
2809 .subscribe(connection_id, corr, version, since, sink)
2810 }
2811
2812 pub(crate) fn cancel_spawn_subscription(&self, connection_id: ConnectionId, corr: u64) -> bool {
2813 self.spawn_events.cancel(connection_id, corr)
2814 }
2815
2816 pub(crate) fn remove_spawn_subscribers(&self, connection_id: ConnectionId) {
2817 self.spawn_events.remove_connection(connection_id);
2818 }
2819
2820 #[cfg(any(test, feature = "test-support"))]
2821 pub fn set_spawn_event_capacity_for_test(&self, capacity: usize) {
2822 assert!(capacity > 0, "spawn event capacity must be non-zero");
2823 self.spawn_events.set_capacity(capacity);
2824 }
2825
2826 #[cfg(any(test, feature = "test-support"))]
2827 pub fn spawn_subscriber_count_for_test(&self) -> usize {
2828 self.spawn_events.subscriber_count()
2829 }
2830
2831 pub fn set_spawn_nonce(&self, module_id: &str, nonce: String) {
2834 self.spawn_nonces
2835 .lock()
2836 .unwrap_or_else(|poisoned| poisoned.into_inner())
2837 .insert(module_id.to_string(), nonce);
2838 self.refresh_program_binding_nonces(module_id);
2839 }
2840
2841 pub fn set_reserved_nonce(&self, module_id: &str, nonce: String) {
2844 {
2845 let mut reserved = self
2846 .reserved_nonces
2847 .lock()
2848 .unwrap_or_else(|p| p.into_inner());
2849 self.implicit_protocol_none
2850 .lock()
2851 .unwrap_or_else(|p| p.into_inner())
2852 .remove(module_id);
2853 reserved.insert(module_id.to_string(), Some(nonce));
2854 }
2855 self.refresh_program_binding_nonces(module_id);
2856 }
2857
2858 pub fn set_reserved_prefixes(&self, owner_module_id: &str, prefixes: &[String]) {
2860 let mut owners = self
2861 .reserved_prefix_owners
2862 .lock()
2863 .unwrap_or_else(|poisoned| poisoned.into_inner());
2864 owners.retain(|_, owner| owner != owner_module_id);
2865 for prefix in prefixes {
2866 owners.insert(prefix.clone(), owner_module_id.to_string());
2867 }
2868 }
2869
2870 pub(crate) fn spawn_nonce(&self, module_id: &str) -> Option<String> {
2872 if self.module_is_gated(module_id) {
2873 return None;
2874 }
2875 self.spawn_nonces
2876 .lock()
2877 .unwrap_or_else(|poisoned| poisoned.into_inner())
2878 .get(module_id)
2879 .cloned()
2880 }
2881
2882 #[cfg(all(test, unix))]
2883 pub(crate) fn legacy_nonce_present_for_test(&self, module: &str) -> bool {
2884 self.spawn_nonces.lock().unwrap().contains_key(module)
2885 }
2886
2887 pub(crate) fn apply_identity_configuration(&self, spec: &ModuleSpec) {
2888 self.set_reserved_prefixes(&spec.module_id, &spec.reserved_prefixes);
2891 let protocol = self
2892 .get(&spec.module_id)
2893 .and_then(|module| {
2894 let snapshot = module.inner.snapshot.lock().ok()?;
2895 snapshot
2896 .process_alive
2897 .then_some(snapshot.spawned_protocol.unwrap_or(spec.protocol))
2898 })
2899 .unwrap_or(spec.protocol);
2900 self.apply_identity_policy(spec, protocol);
2901 }
2902
2903 #[cfg(all(test, unix))]
2904 pub(crate) fn pause_plain_identity_policy_for_test(
2905 &self,
2906 module: &str,
2907 ) -> (std::sync::mpsc::Receiver<()>, std::sync::mpsc::Sender<()>) {
2908 let (reached, waiting) = std::sync::mpsc::channel();
2909 let (release, paused) = std::sync::mpsc::channel();
2910 *self.plain_identity_policy_pause.lock().unwrap() = Some((
2911 module.to_owned(),
2912 IdentitySamplePause {
2913 reached,
2914 release: paused,
2915 },
2916 ));
2917 (waiting, release)
2918 }
2919
2920 fn apply_identity_policy(&self, spec: &ModuleSpec, protocol: ModuleProtocol) {
2921 let channel_only = self.module_is_gated(&spec.module_id);
2922 let mut current_nonces = self
2923 .spawn_nonces
2924 .lock()
2925 .unwrap_or_else(|poisoned| poisoned.into_inner());
2926 if protocol == ModuleProtocol::None || channel_only {
2927 current_nonces.remove(&spec.module_id);
2928 }
2929 let spawn_nonce = current_nonces.get(&spec.module_id).cloned();
2930 #[cfg(test)]
2931 if let Some(pause) = self.identity_sample_pause.lock().unwrap().take() {
2932 pause.reached.send(()).unwrap();
2933 pause.release.recv().unwrap();
2934 }
2935 let mut reserved_nonces = self
2936 .reserved_nonces
2937 .lock()
2938 .unwrap_or_else(|poisoned| poisoned.into_inner());
2939 let mut implicit = self
2940 .implicit_protocol_none
2941 .lock()
2942 .unwrap_or_else(|p| p.into_inner());
2943 if spec.reserved {
2944 implicit.remove(&spec.module_id);
2945 } else if protocol == ModuleProtocol::None {
2946 if !reserved_nonces.contains_key(&spec.module_id) {
2947 implicit.insert(spec.module_id.clone());
2948 }
2949 } else if implicit.remove(&spec.module_id) {
2950 reserved_nonces.remove(&spec.module_id);
2951 }
2952 if spec.reserved || protocol == ModuleProtocol::None {
2953 reserved_nonces.insert(spec.module_id.clone(), spawn_nonce);
2958 }
2959 drop(implicit);
2960 drop(reserved_nonces);
2961 drop(current_nonces);
2962 self.refresh_program_binding_nonces(&spec.module_id);
2963 self.removal_tombstones
2967 .lock()
2968 .unwrap_or_else(|poisoned| poisoned.into_inner())
2969 .remove(&spec.module_id);
2970 }
2971
2972 pub fn reserved_hello_authorized(&self, module_id: &str, presented: Option<&str>) -> bool {
2977 self.reserved_hello_verification(module_id, presented)
2978 .is_ok()
2979 }
2980
2981 pub(crate) fn reserved_hello_verification(
2984 &self,
2985 module_id: &str,
2986 presented: Option<&str>,
2987 ) -> Result<ReservedHelloVerification, ReservedHelloRejection> {
2988 if self.module_is_gated(module_id) {
2989 return Err(ReservedHelloRejection::Exact {
2990 module_id: module_id.into(),
2991 });
2992 }
2993 self.hello_verification(module_id, presented, false)
2994 }
2995
2996 pub(crate) fn channel_hello_verification(
2997 &self,
2998 module_id: &str,
2999 ) -> Result<ReservedHelloVerification, ReservedHelloRejection> {
3000 if self
3001 .get(module_id)
3002 .is_some_and(|module| module.declared_protocol().ok() == Some(ModuleProtocol::None))
3003 {
3004 return Err(ReservedHelloRejection::Exact {
3005 module_id: module_id.into(),
3006 });
3007 }
3008 let mut verification = self.hello_verification(module_id, None, true)?;
3009 verification.nonce_verified = true;
3010 Ok(verification)
3011 }
3012
3013 pub(crate) fn prefix_claim_authorized(&self, owner: &str, module_id: &str) -> bool {
3014 self.prefix_claim_publication(owner, module_id).is_some()
3015 }
3016
3017 #[cfg(all(test, unix))]
3018 pub(crate) fn prefix_publication_locked_for_test(&self) -> bool {
3019 matches!(
3020 self.reserved_prefix_owners.try_lock(),
3021 Err(std::sync::TryLockError::WouldBlock)
3022 )
3023 }
3024
3025 pub(crate) fn prefix_claim_publication(
3029 &self,
3030 owner: &str,
3031 module_id: &str,
3032 ) -> Option<std::sync::MutexGuard<'_, HashMap<String, String>>> {
3033 if subc_protocol::manifest::validate_module_id(module_id).is_err()
3034 || self.module_is_gated(module_id)
3035 || self
3036 .reserved_nonces
3037 .lock()
3038 .unwrap_or_else(|p| p.into_inner())
3039 .contains_key(module_id)
3040 {
3041 return None;
3042 }
3043 let owners = self
3044 .reserved_prefix_owners
3045 .lock()
3046 .unwrap_or_else(|p| p.into_inner());
3047 let authorized = owners
3048 .iter()
3049 .filter(|(prefix, _)| module_id.starts_with(prefix.as_str()))
3050 .max_by_key(|(prefix, _)| prefix.len())
3051 .is_some_and(|(prefix, holder)| holder == owner && module_id.len() > prefix.len());
3052 authorized.then_some(owners)
3053 }
3054
3055 fn hello_verification(
3056 &self,
3057 module_id: &str,
3058 presented: Option<&str>,
3059 channel: bool,
3060 ) -> Result<ReservedHelloVerification, ReservedHelloRejection> {
3061 let nonces = self
3062 .reserved_nonces
3063 .lock()
3064 .unwrap_or_else(|poisoned| poisoned.into_inner());
3065 if let Some(expected) = nonces.get(module_id) {
3066 let authorized = channel
3070 || match expected {
3071 Some(expected) => presented.is_some_and(|p| {
3072 !p.is_empty() && constant_time_eq(expected.as_bytes(), p.as_bytes())
3073 }),
3074 None => false,
3075 };
3076 if authorized {
3077 return Ok(ReservedHelloVerification {
3078 nonce_verified: true,
3079 prefix_owner: None,
3080 });
3081 }
3082 return Err(ReservedHelloRejection::Exact {
3083 module_id: module_id.to_string(),
3084 });
3085 }
3086 drop(nonces);
3087
3088 let matched_prefix = self
3089 .reserved_prefix_owners
3090 .lock()
3091 .unwrap_or_else(|poisoned| poisoned.into_inner())
3092 .iter()
3093 .filter(|(prefix, _)| module_id.starts_with(prefix.as_str()))
3094 .max_by_key(|(prefix, _)| prefix.len())
3095 .map(|(prefix, owner)| (prefix.clone(), owner.clone()));
3096 let Some((prefix, owner_module_id)) = matched_prefix else {
3097 return Ok(ReservedHelloVerification::default());
3098 };
3099
3100 if module_id.len() == prefix.len()
3101 || (!channel
3102 && (self.module_is_gated(&owner_module_id) || self.protocol_none(&owner_module_id)))
3103 {
3104 return Err(ReservedHelloRejection::Prefix {
3105 prefix,
3106 owner_module_id,
3107 });
3108 }
3109
3110 let authorized = if channel {
3111 owner_module_id == module_id
3112 } else {
3113 presented.is_some_and(|presented| {
3114 self.spawn_nonces
3115 .lock()
3116 .unwrap_or_else(|poisoned| poisoned.into_inner())
3117 .get(&owner_module_id)
3118 .is_some_and(|expected| !presented.is_empty() && constant_time_eq(expected.as_bytes(), presented.as_bytes()))
3119 || self.swap_nonce_matches(&owner_module_id, presented)
3122 })
3123 };
3124 if authorized {
3125 Ok(ReservedHelloVerification {
3126 nonce_verified: true,
3127 prefix_owner: (owner_module_id != module_id).then_some(owner_module_id),
3128 })
3129 } else {
3130 Err(ReservedHelloRejection::Prefix {
3131 prefix,
3132 owner_module_id,
3133 })
3134 }
3135 }
3136
3137 pub fn spawned_consumer_authorized(&self, module_id: &str, presented: &str) -> bool {
3142 if self.module_is_gated(module_id) || self.protocol_none(module_id) {
3143 return false;
3144 }
3145 let nonces = self
3146 .spawn_nonces
3147 .lock()
3148 .unwrap_or_else(|poisoned| poisoned.into_inner());
3149 let current = nonces.get(module_id).is_some_and(|expected| {
3150 !presented.is_empty() && constant_time_eq(expected.as_bytes(), presented.as_bytes())
3151 });
3152 drop(nonces);
3153 current || self.swap_nonce_matches(module_id, presented)
3156 }
3157
3158 fn swap_nonce_matches(&self, module_id: &str, presented: &str) -> bool {
3160 if self.module_is_gated(module_id) || self.protocol_none(module_id) || presented.is_empty()
3161 {
3162 return false;
3163 }
3164 let swaps = self
3165 .swaps
3166 .lock()
3167 .unwrap_or_else(|poisoned| poisoned.into_inner());
3168 swaps.get(module_id).is_some_and(|swap| {
3169 constant_time_eq(swap.candidate_nonce.as_bytes(), presented.as_bytes())
3170 || swap.incumbent_nonce.as_deref().is_some_and(|incumbent| {
3171 constant_time_eq(incumbent.as_bytes(), presented.as_bytes())
3172 })
3173 })
3174 }
3175
3176 pub(crate) fn open_swap(&self, module_id: &str, candidate_nonce: String) {
3179 let incumbent_nonce = self
3180 .spawn_nonces
3181 .lock()
3182 .unwrap_or_else(|poisoned| poisoned.into_inner())
3183 .get(module_id)
3184 .cloned();
3185 self.swaps
3186 .lock()
3187 .unwrap_or_else(|poisoned| poisoned.into_inner())
3188 .insert(
3189 module_id.to_string(),
3190 OpenSwap {
3191 candidate_nonce,
3192 incumbent_nonce,
3193 candidate_admitted: false,
3194 },
3195 );
3196 self.refresh_program_binding_nonces(module_id);
3197 }
3198
3199 pub(crate) fn close_swap(&self, module_id: &str) {
3202 self.swaps
3203 .lock()
3204 .unwrap_or_else(|poisoned| poisoned.into_inner())
3205 .remove(module_id);
3206 self.refresh_program_binding_nonces(module_id);
3207 }
3208
3209 pub(crate) fn set_swap_promotion_observer(
3212 &self,
3213 observer: std::sync::Weak<dyn SwapPromotionObserver>,
3214 ) {
3215 *self
3216 .promotion_observer
3217 .0
3218 .lock()
3219 .unwrap_or_else(|poisoned| poisoned.into_inner()) = Some(observer);
3220 }
3221
3222 fn notify_swap_promoted(&self, registration: &crate::registry::ModuleRegistration) {
3225 let observer = self
3226 .promotion_observer
3227 .0
3228 .lock()
3229 .unwrap_or_else(|poisoned| poisoned.into_inner())
3230 .as_ref()
3231 .and_then(std::sync::Weak::upgrade);
3232 if let Some(observer) = observer {
3233 observer.swap_promoted(registration);
3234 }
3235 }
3236
3237 pub(crate) fn swap_open(&self, module_id: &str) -> bool {
3239 self.swaps
3240 .lock()
3241 .unwrap_or_else(|poisoned| poisoned.into_inner())
3242 .contains_key(module_id)
3243 }
3244
3245 fn promote_swap_nonce(&self, module_id: &str, reserved: bool) {
3250 let candidate_nonce = self
3251 .swaps
3252 .lock()
3253 .unwrap_or_else(|poisoned| poisoned.into_inner())
3254 .get_mut(module_id)
3255 .map(|swap| {
3256 swap.incumbent_nonce = None;
3257 swap.candidate_nonce.clone()
3258 });
3259 self.spawn_nonces
3260 .lock()
3261 .unwrap_or_else(|p| p.into_inner())
3262 .remove(module_id);
3263 {
3264 let mut nonces = self
3265 .reserved_nonces
3266 .lock()
3267 .unwrap_or_else(|p| p.into_inner());
3268 if reserved || nonces.contains_key(module_id) {
3269 nonces.insert(module_id.into(), None);
3270 }
3271 }
3272 if self.module_is_gated(module_id) {
3273 return;
3274 }
3275 let Some(nonce) = candidate_nonce.filter(|nonce| !nonce.is_empty()) else {
3276 return;
3277 };
3278 self.set_spawn_nonce(module_id, nonce.clone());
3279 if reserved {
3280 self.set_reserved_nonce(module_id, nonce);
3281 }
3282 }
3283
3284 pub(crate) fn swap_hello_admission(
3299 &self,
3300 module_id: &str,
3301 presented: Option<&str>,
3302 ) -> SwapHelloAdmission {
3303 if self.protocol_none(module_id) {
3304 return SwapHelloAdmission::ProtocolNone;
3305 }
3306 if self.module_is_gated(module_id) {
3307 return SwapHelloAdmission::ChannelRequired;
3308 }
3309 let swaps = self
3310 .swaps
3311 .lock()
3312 .unwrap_or_else(|poisoned| poisoned.into_inner());
3313 let Some(swap) = swaps.get(module_id) else {
3314 return SwapHelloAdmission::NotSwapping;
3315 };
3316 let Some(presented) = presented.filter(|nonce| !nonce.is_empty()) else {
3317 return SwapHelloAdmission::Refused;
3318 };
3319 if constant_time_eq(swap.candidate_nonce.as_bytes(), presented.as_bytes()) {
3320 return if swap.candidate_admitted {
3321 SwapHelloAdmission::Refused
3322 } else {
3323 SwapHelloAdmission::Candidate
3324 };
3325 }
3326 if swap
3327 .incumbent_nonce
3328 .as_deref()
3329 .is_some_and(|incumbent| constant_time_eq(incumbent.as_bytes(), presented.as_bytes()))
3330 {
3331 return SwapHelloAdmission::NotSwapping;
3332 }
3333 SwapHelloAdmission::Refused
3334 }
3335
3336 fn protocol_none(&self, module: &str) -> bool {
3337 self.get(module)
3338 .is_some_and(|module| module.declared_protocol().ok() == Some(ModuleProtocol::None))
3339 }
3340
3341 pub(crate) fn with_swap_candidate_admission<T>(
3343 &self,
3344 module_id: &str,
3345 publish: impl FnOnce() -> (T, bool),
3346 ) -> Option<T> {
3347 let mut swaps = self.swaps.lock().unwrap_or_else(|p| p.into_inner());
3348 let swap = swaps
3349 .get_mut(module_id)
3350 .filter(|swap| !swap.candidate_admitted)?;
3351 let (result, admitted) = publish();
3352 swap.candidate_admitted = admitted;
3353 Some(result)
3354 }
3355
3356 pub fn spawn_launch_nonce_for(&self, module_id: &str) -> Option<String> {
3358 self.spawn_nonce(module_id)
3359 }
3360
3361 pub fn reserved_launch_nonce_for(&self, module_id: &str) -> Option<String> {
3363 if self.module_is_gated(module_id) {
3364 return None;
3365 }
3366 self.reserved_nonces
3367 .lock()
3368 .unwrap_or_else(|poisoned| poisoned.into_inner())
3369 .get(module_id)
3370 .cloned()
3371 .flatten()
3372 }
3373
3374 pub fn insert(&self, module: SupervisedModule) -> Option<SupervisedModule> {
3375 self.mark_configured(module.module_id());
3379 let mut modules = self
3380 .modules
3381 .lock()
3382 .unwrap_or_else(|poisoned| poisoned.into_inner());
3383 modules.insert(module.module_id().to_string(), module)
3384 }
3385
3386 fn mark_configured(&self, module_id: &str) {
3390 self.update_configured_ids(|ids| {
3391 ids.insert(module_id.to_string());
3392 });
3393 }
3394
3395 fn publish_configured_ids(&self, ids: Option<&HashSet<String>>) {
3396 let owner = self
3397 .gate_runtime
3398 .lock()
3399 .unwrap_or_else(|p| p.into_inner())
3400 .clone()
3401 .and_then(|owner| owner.upgrade());
3402 if let Some(owner) = owner {
3403 owner.observe_supervisor_configured_ids(ids, Arc::downgrade(&self.configured_ids));
3404 }
3405 }
3406
3407 fn update_configured_ids(&self, update: impl FnOnce(&mut HashSet<String>)) {
3408 match self.configured_ids.lock() {
3409 Ok(mut ids) => {
3410 update(&mut ids);
3411 self.publish_configured_ids(Some(&ids));
3414 }
3415 Err(poisoned) => {
3416 update(&mut poisoned.into_inner());
3417 self.publish_configured_ids(None);
3418 }
3419 }
3420 }
3421
3422 fn unmark_configured_unless_rostered(&self, module_id: &str) {
3426 let modules = self
3427 .modules
3428 .lock()
3429 .unwrap_or_else(|poisoned| poisoned.into_inner());
3430 if !modules.contains_key(module_id) {
3431 self.update_configured_ids(|ids| {
3432 ids.remove(module_id);
3433 });
3434 }
3435 }
3436
3437 pub(crate) fn is_configured(&self, module_id: &str) -> bool {
3445 self.configured_ids
3446 .lock()
3447 .unwrap_or_else(|poisoned| poisoned.into_inner())
3448 .contains(module_id)
3449 }
3450
3451 pub fn get(&self, module_id: &str) -> Option<SupervisedModule> {
3452 let modules = self
3453 .modules
3454 .lock()
3455 .unwrap_or_else(|poisoned| poisoned.into_inner());
3456 modules.get(module_id).cloned()
3457 }
3458
3459 pub(crate) fn record_late_health_answer(
3460 &self,
3461 module_id: &str,
3462 latency_ms: u64,
3463 ) -> Result<bool, SuperviseError> {
3464 let Some(module) = self.get(module_id) else {
3465 return Ok(false);
3466 };
3467 self.host_health.record_late_module(module_id);
3468 update_snapshot(&module.inner.snapshot, Some(module_id), |state| {
3469 state.health.late_answer_count = state.health.late_answer_count.saturating_add(1);
3470 state.health.last_late_answer_latency_ms = Some(latency_ms);
3471 state.health.consecutive_failures = 0;
3479 })?;
3480 Ok(true)
3481 }
3482
3483 pub(crate) fn spawn_host_health_monitor(&self) -> JoinHandle<()> {
3484 self.host_health.spawn_monitor()
3485 }
3486
3487 pub fn record_deliberate_severance(&self, module_id: &str) -> Result<bool, SuperviseError> {
3493 let Some(module) = self.get(module_id) else {
3494 return Ok(false);
3495 };
3496 let snapshot = lock_snapshot(&module.inner.snapshot)?;
3497 let Some((pid, start_time)) = snapshot.pid.zip(snapshot.process_start_time) else {
3498 return Ok(false);
3499 };
3500 drop(snapshot);
3501 module.record_deliberate_severance(ProcessIdentity { pid, start_time })
3502 }
3503
3504 pub fn list(&self) -> Vec<SupervisedModule> {
3505 let modules = self
3506 .modules
3507 .lock()
3508 .unwrap_or_else(|poisoned| poisoned.into_inner());
3509 let mut modules = modules.values().cloned().collect::<Vec<_>>();
3510 modules.sort_by(|left, right| left.module_id().cmp(right.module_id()));
3511 modules
3512 }
3513
3514 pub(crate) fn retire(&self, module_id: &str) -> Option<SupervisedModule> {
3515 self.activity_images
3516 .lock()
3517 .unwrap_or_else(|p| p.into_inner())
3518 .remove(module_id);
3519 self.spawn_nonces
3520 .lock()
3521 .unwrap_or_else(|poisoned| poisoned.into_inner())
3522 .remove(module_id);
3523 self.close_swap(module_id);
3524 let mut reserved_nonces = self
3525 .reserved_nonces
3526 .lock()
3527 .unwrap_or_else(|poisoned| poisoned.into_inner());
3528 if reserved_nonces.contains_key(module_id) {
3529 reserved_nonces.insert(module_id.to_string(), None);
3532 }
3533 drop(reserved_nonces);
3534 self.reserved_prefix_owners
3535 .lock()
3536 .unwrap_or_else(|poisoned| poisoned.into_inner())
3537 .retain(|_, owner| owner != module_id);
3538 let removed = self
3539 .modules
3540 .lock()
3541 .unwrap_or_else(|poisoned| poisoned.into_inner())
3542 .remove(module_id);
3543 self.update_configured_ids(|ids| {
3544 ids.remove(module_id);
3545 });
3546 self.remove_program_binding_records(module_id);
3547 removed
3548 }
3549
3550 pub(crate) fn record_rescan_removal(&self, module_id: &str) {
3553 self.removal_tombstones
3554 .lock()
3555 .unwrap_or_else(|poisoned| poisoned.into_inner())
3556 .insert(module_id.to_string(), unix_ms_now());
3557 }
3558
3559 pub(crate) fn removal_tombstone_age_ms(&self, module_id: &str) -> Option<u64> {
3561 self.removal_tombstones
3562 .lock()
3563 .unwrap_or_else(|poisoned| poisoned.into_inner())
3564 .get(module_id)
3565 .copied()
3566 .map(|removed_at_ms| unix_ms_now().saturating_sub(removed_at_ms))
3567 }
3568
3569 pub(crate) fn release_retained_reserved_gate(&self, module_id: &str) -> bool {
3574 if self.get(module_id).is_some() {
3575 return false;
3576 }
3577 let mut reserved_nonces = self
3578 .reserved_nonces
3579 .lock()
3580 .unwrap_or_else(|poisoned| poisoned.into_inner());
3581 if !matches!(reserved_nonces.get(module_id), Some(None)) {
3582 return false;
3583 }
3584 reserved_nonces.remove(module_id);
3585 self.implicit_protocol_none
3586 .lock()
3587 .unwrap_or_else(|p| p.into_inner())
3588 .remove(module_id);
3589 true
3590 }
3591
3592 pub(crate) fn operation_lock(&self) -> Arc<AsyncMutex<()>> {
3593 Arc::clone(&self.operation_lock)
3594 }
3595}
3596
3597#[derive(Debug, Clone)]
3599pub struct Supervisor {
3600 registry: Arc<Registry>,
3601 config_delivery: Option<config_delivery::ConfigDelivery>,
3602 restart_policy: RestartPolicy,
3603 drain_timeout: Duration,
3604 connection_file_path: Option<PathBuf>,
3605 capture_logs_dir: Option<PathBuf>,
3606 forwarding: Option<Arc<ForwardingTable>>,
3607 process_liveness: Arc<SupervisorProcessLiveness>,
3608 supervisor_handle: Option<SupervisorHandle>,
3609 health: HealthConfig,
3610 host_health: HostHealth,
3611 daemon_start_clock: crate::clock::StartClock,
3612 terminal_journal: Option<Arc<crate::terminal_journal::TerminalJournal>>,
3613 spawn_events: SpawnEventFeed,
3614 provenance_probe: ExecutableIdentityProbe,
3615 child_roster: ChildRoster,
3618 #[cfg(target_os = "linux")]
3619 cgroup_placement: Option<subc_cgroup::Placement>,
3620 #[cfg(test)]
3621 test_after_first_spawn: AfterFirstSpawnHook,
3622}
3623
3624pub(crate) struct RegisteredConfiguredModule {
3628 spec: ModuleSpec,
3629 runtime: SupervisorRuntimeConfig,
3630 module: SupervisedModule,
3631 commands: mpsc::Receiver<SupervisorCommand>,
3632}
3633
3634#[cfg(test)]
3640#[derive(Clone, Default)]
3641struct AfterFirstSpawnHook(Option<AfterFirstSpawnFn>);
3642
3643#[cfg(test)]
3644type AfterFirstSpawnFn = Arc<dyn Fn(&str) + Send + Sync>;
3645
3646#[cfg(test)]
3647impl fmt::Debug for AfterFirstSpawnHook {
3648 fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
3649 f.write_str("AfterFirstSpawnHook")
3650 }
3651}
3652
3653#[cfg(test)]
3654impl AfterFirstSpawnHook {
3655 fn run(&self, module_id: &str) {
3656 if let Some(hook) = &self.0 {
3657 hook(module_id);
3658 }
3659 }
3660}
3661
3662impl Supervisor {
3663 #[cfg(all(test, target_os = "linux"))]
3664 pub(crate) fn set_admission_hook_for_test(
3665 &self,
3666 hook: impl Fn(&str, &'static str, Option<u32>) -> bool + Send + Sync + 'static,
3667 ) {
3668 self.child_roster.set_admission_hook_for_test(hook);
3669 }
3670
3671 #[cfg(test)]
3672 pub(crate) fn new_for_test(registry: Arc<Registry>, policy: RestartPolicy) -> Self {
3673 let supervisor = Self::new(registry, policy);
3674 #[cfg(target_os = "macos")]
3675 let supervisor = supervisor.with_privacy_trampoline(test_privacy_trampoline());
3676 supervisor
3677 }
3678 pub fn with_privacy_trampoline(self, path: impl Into<PathBuf>) -> Self {
3683 let path = path.into();
3684 #[cfg(target_os = "macos")]
3685 {
3686 let result = probe_privacy_trampoline(&path).map(|()| path);
3687 if let Err(cause) = &result {
3688 error!(%cause, "macOS privacy identity unavailable; supervised launches will refuse");
3689 }
3690 self.child_roster.set_privacy_trampoline(result);
3691 }
3692 #[cfg(not(target_os = "macos"))]
3693 let _ = path;
3694 self
3695 }
3696 pub(crate) fn daemon_shutdown_flag(&self) -> crate::child_roster::DaemonShutdownFlag {
3699 self.child_roster.shutdown_flag()
3700 }
3701
3702 #[cfg(unix)]
3713 pub(crate) fn begin_daemon_shutdown(&self) {
3714 self.child_roster.close();
3715 if let Some(journal) = &self.terminal_journal {
3716 journal.stamp_shutdown();
3717 }
3718 }
3719
3720 #[cfg(unix)]
3725 pub(crate) async fn drain_for_daemon_shutdown(&self) -> Result<(), SuperviseError> {
3726 const NOTICE_BUDGET: Duration = Duration::from_millis(500);
3727 const DRAIN_BUDGET: Duration = Duration::from_secs(2);
3728 let Some(forwarding) = &self.forwarding else {
3729 return Ok(());
3730 };
3731 let notice_started = Instant::now();
3732 let modules = forwarding
3733 .begin_daemon_drain()
3734 .map_err(SuperviseError::Forwarding)?;
3735 let deadline_ms =
3736 unix_ms_now().saturating_add((NOTICE_BUDGET + DRAIN_BUDGET).as_millis() as u64);
3737 let shutdown_reason = self
3738 .supervisor_handle
3739 .as_ref()
3740 .map(SupervisorHandle::take_daemon_shutdown_reason)
3741 .unwrap_or(RouteCloseReason::Disable);
3742 let mut modules_told = 0;
3743 let mut notices = tokio::task::JoinSet::new();
3744 let mut drains = Vec::new();
3745 for connection in forwarding
3746 .module_connections()
3747 .map_err(SuperviseError::Forwarding)?
3748 {
3749 let target = ModuleDrainTarget {
3750 endpoint: connection.endpoint,
3751 sink: connection.sink,
3752 negotiated_ver: connection.negotiated_ver,
3753 abandoned_bindings: Vec::new(),
3754 excluded_subscriptions: 0,
3755 };
3756 notices.spawn(async move {
3757 (
3758 true,
3759 send_module_draining(
3760 &connection.module_id,
3761 shutdown_reason,
3762 deadline_ms,
3763 &target,
3764 true,
3765 )
3766 .await,
3767 )
3768 });
3769 }
3770 for (module_id, endpoint) in modules {
3771 let Some(target) = forwarding
3772 .begin_endpoint_drain(endpoint, RouteCloseReason::Restart)
3773 .map_err(SuperviseError::Forwarding)?
3774 else {
3775 continue;
3776 };
3777 let routes = forwarding
3778 .endpoint_routes(target.endpoint)
3779 .map_err(SuperviseError::Forwarding)?;
3780 let mut recipients = Vec::new();
3783 let mut clients: Vec<(crate::forwarding::GoodbyeTarget, Vec<u16>)> = Vec::new();
3784 for route in routes {
3785 let client = route.goodbye_target;
3786 if let Some((_, channels)) = clients
3787 .iter_mut()
3788 .find(|(existing, _)| existing.connection_id == client.connection_id)
3789 {
3790 channels.push(client.channel);
3791 } else {
3792 let channel = client.channel;
3793 clients.push((client, vec![channel]));
3794 }
3795 }
3796 for (client, mut channels) in clients {
3797 channels.sort_unstable();
3798 channels.dedup();
3799 let closing = serde_json::to_vec(&ClientControlPush::RouteClosing {
3800 module_id: module_id.clone(),
3801 channels,
3802 reason: RouteCloseReason::Restart,
3803 })
3804 .expect("route closing serializes");
3805 recipients.push((client.sink, client.negotiated_ver, closing));
3806 }
3807 for (sink, version, body) in recipients {
3808 notices.spawn(async move {
3809 let frame = Frame::build_with_version(
3810 version,
3811 FrameType::Push,
3812 control_flags(),
3813 0,
3814 0,
3815 0,
3816 body,
3817 )
3818 .expect("bounded lifecycle notice frame builds");
3819 (false, sink.send_flushed(frame).await.is_ok())
3820 });
3821 }
3822 let gauges = declared_busy_gauges(&self.registry, &module_id)?;
3823 drains.push((module_id, target.endpoint, gauges));
3824 }
3825 let notice_deadline = Instant::now() + NOTICE_BUDGET;
3828 while let Ok(Some(result)) = timeout_at(notice_deadline, notices.join_next()).await {
3829 match result {
3830 Ok((true, true)) => modules_told += 1,
3831 Ok((false, true)) => {}
3832 result => warn!(?result, "daemon shutdown notice delivery failed"),
3833 }
3834 }
3835 notices.abort_all();
3836 info!(
3837 ?shutdown_reason,
3838 modules_told, "daemon shutdown module notices delivered"
3839 );
3840 let notice_elapsed = notice_started.elapsed();
3841 let drain_started = Instant::now();
3842 let deadline = drain_started + DRAIN_BUDGET;
3843 let mut waits = tokio::task::JoinSet::new();
3844 for (module_id, endpoint, gauges) in drains {
3845 let forwarding = Arc::clone(forwarding);
3846 let mut runtime = self.runtime_config();
3847 runtime.health.cadence = Duration::from_millis(100);
3848 waits.spawn(async move {
3849 let wait_result = wait_for_forwarding_quiescence(
3850 &forwarding,
3851 &module_id,
3852 &runtime,
3853 endpoint,
3854 deadline,
3855 &gauges,
3856 DrainScope::Endpoint(endpoint),
3857 &mut DrainGaugeTrace::new(&gauges),
3858 )
3859 .await;
3860 forwarding
3864 .record_daemon_drain_result(
3865 endpoint,
3866 drained_after_quiescence_wait(&wait_result),
3867 )
3868 .map_err(SuperviseError::Forwarding)?;
3869 wait_result
3870 });
3871 }
3872 while let Ok(Some(result)) = timeout_at(deadline, waits.join_next()).await {
3873 if !matches!(result, Ok(Ok(true))) {
3874 warn!(?result, "daemon shutdown drain did not reach quiescence");
3875 }
3876 }
3877 info!(
3880 notice_elapsed_ms = notice_elapsed.as_millis() as u64,
3881 drain_elapsed_ms = drain_started.elapsed().as_millis() as u64,
3882 "daemon shutdown forwarding drain complete"
3883 );
3884 Ok(())
3885 }
3886
3887 #[cfg(unix)]
3899 pub(crate) async fn end_children_for_daemon_shutdown(
3900 &self,
3901 already_escalated: bool,
3902 escalate: impl std::future::Future<Output = ()>,
3903 ) {
3904 tokio::pin!(escalate);
3905 let mut escalated = already_escalated;
3906 if let Some(forwarding) = &self.forwarding {
3907 let reason = CloseReason::new(
3908 "daemon_shutdown",
3909 "the daemon is exiting after its shutdown notice and drain",
3910 );
3911 if escalated {
3912 send_module_goodbyes_for_daemon_shutdown(forwarding, &reason, false).await;
3915 } else {
3916 tokio::select! {
3917 biased;
3918 _ = escalate.as_mut() => {
3919 info!("second SIGTERM: abandoning module GOODBYE delivery");
3920 escalated = true;
3921 }
3922 _ = send_module_goodbyes_for_daemon_shutdown(forwarding, &reason, true) => {}
3923 }
3924 }
3925 let closed = forwarding.close_all_connections(&reason);
3926 debug!(closed, "closed established connections for daemon shutdown");
3927 }
3928 let escalated_here = escalated && !already_escalated;
3932 let remaining_escalate = async move {
3933 if escalated_here {
3934 std::future::pending::<()>().await;
3935 } else {
3936 escalate.await;
3937 }
3938 };
3939 crate::child_roster::end_children_for_daemon_shutdown(
3940 &self.child_roster,
3941 escalated,
3942 remaining_escalate,
3943 )
3944 .await;
3945 }
3946
3947 pub fn new(registry: Arc<Registry>, restart_policy: RestartPolicy) -> Self {
3948 Self {
3949 registry,
3950 config_delivery: None,
3951 restart_policy,
3952 drain_timeout: DEFAULT_DRAIN_TIMEOUT,
3953 connection_file_path: None,
3954 capture_logs_dir: None,
3955 forwarding: None,
3956 process_liveness: Arc::new(SupervisorProcessLiveness::default()),
3957 supervisor_handle: None,
3958 health: HealthConfig::default(),
3959 host_health: HostHealth::default(),
3960 daemon_start_clock: crate::clock::StartClock::capture(),
3961 terminal_journal: None,
3962 spawn_events: SpawnEventFeed::default(),
3963 provenance_probe: ExecutableIdentityProbe::default(),
3964 child_roster: ChildRoster::default(),
3965 #[cfg(target_os = "linux")]
3966 cgroup_placement: None,
3967 #[cfg(test)]
3968 test_after_first_spawn: AfterFirstSpawnHook::default(),
3969 }
3970 }
3971
3972 pub fn with_drain_timeout(mut self, drain_timeout: Duration) -> Self {
3973 self.drain_timeout = drain_timeout;
3974 self
3975 }
3976
3977 pub(crate) fn default_drain_timeout(&self) -> Duration {
3979 self.drain_timeout
3980 }
3981
3982 pub fn with_process_liveness(
3983 mut self,
3984 process_liveness: Arc<SupervisorProcessLiveness>,
3985 ) -> Self {
3986 self.process_liveness = process_liveness;
3987 self
3988 }
3989
3990 pub fn with_config_runtime(
3992 mut self,
3993 owner: Arc<crate::config_gate::runtime::RuntimeOwner>,
3994 ) -> Self {
3995 if let Some(handle) = &self.supervisor_handle {
3996 handle.install_binding_retirement(&owner);
3997 }
3998 self.config_delivery = Some(config_delivery::ConfigDelivery(owner));
3999 self
4000 }
4001
4002 pub fn with_connection_file_path(mut self, connection_file_path: impl Into<PathBuf>) -> Self {
4003 self.connection_file_path = Some(connection_file_path.into());
4004 self
4005 }
4006
4007 pub fn with_capture_logs_dir(mut self, logs_dir: impl Into<PathBuf>) -> Self {
4009 self.capture_logs_dir = Some(logs_dir.into());
4010 self
4011 }
4012
4013 pub fn with_daemon_incarnation(self, daemon_incarnation: String) -> Self {
4016 self.spawn_events.configure_incarnation(daemon_incarnation);
4020 self
4021 }
4022
4023 pub fn with_terminal_journal(self, path: PathBuf, daemon_incarnation: String) -> Self {
4026 let mut this = self.with_daemon_incarnation(daemon_incarnation.clone());
4027 this.terminal_journal = Some(Arc::new(crate::terminal_journal::TerminalJournal::open(
4028 path,
4029 daemon_incarnation,
4030 )));
4031 this
4032 }
4033
4034 pub fn with_forwarding(mut self, forwarding: Arc<ForwardingTable>) -> Self {
4035 self.child_roster.counters = forwarding.counters();
4036 self.forwarding = Some(forwarding);
4037 self
4038 }
4039
4040 pub fn with_handle(mut self, supervisor_handle: SupervisorHandle) -> Self {
4041 if let Some(delivery) = &self.config_delivery {
4042 supervisor_handle.install_binding_retirement(&delivery.0);
4043 }
4044 self.host_health = supervisor_handle.host_health.clone();
4045 self.spawn_events = supervisor_handle.spawn_events.clone();
4046 self.supervisor_handle = Some(supervisor_handle);
4047 self
4048 }
4049
4050 pub fn with_health_config(mut self, health: HealthConfig) -> Self {
4051 self.health = health;
4052 self
4053 }
4054
4055 pub fn with_live_children_record(self, path: impl Into<PathBuf>) -> Self {
4059 self.child_roster.record_to(path.into());
4060 self
4061 }
4062
4063 #[cfg(target_os = "linux")]
4064 pub fn with_cgroup_placement(
4065 mut self,
4066 cgroup_placement: Option<subc_cgroup::Placement>,
4067 ) -> Self {
4068 self.child_roster
4069 .set_cgroup_placement(cgroup_placement.clone());
4070 self.cgroup_placement = cgroup_placement;
4071 self
4072 }
4073
4074 pub fn spawn(&self, spec: ModuleSpec) -> Result<SupervisedModule, SuperviseError> {
4083 validate_spec(&spec)?;
4084 self.establish_identity(&spec);
4085
4086 let runtime = self.runtime_config();
4087 let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::starting()));
4088 let activity_spawn = runtime.begin_activity_spawn(&spec.module_id);
4089 let registration_watch = watch_registration(&self.registry, &spec);
4090 let spawned = spawn_child(
4091 &spec,
4092 runtime.connection_file_path.as_deref(),
4093 self.supervisor_handle.as_ref(),
4094 &runtime.stderr_ring,
4095 runtime.capture_logs_dir.as_deref(),
4096 &runtime.child_roster,
4097 runtime.spawn_delivery(),
4098 #[cfg(target_os = "linux")]
4099 runtime.cgroup_placement.as_ref(),
4100 runtime.register_timeout,
4101 );
4102 #[cfg(test)]
4103 self.test_after_first_spawn.run(&spec.module_id);
4104 let mut child = match spawned {
4105 Ok(child) => child,
4106 Err(SuperviseError::ConfigDeliveryDeferred { .. }) => {
4107 defer_config_spawn(
4108 &spec,
4109 &runtime,
4110 &snapshot,
4111 RespawnKind::Spawn,
4112 &runtime.failure_context(runtime.recovery_is_daemon),
4113 )?;
4114 return Ok(self.supervised_module(spec, runtime, snapshot, None));
4115 }
4116 Err(SuperviseError::ConfigHeld { .. }) => {
4117 hold_config_spawn(
4118 &spec,
4119 &runtime,
4120 &snapshot,
4121 &runtime.failure_context(runtime.recovery_is_daemon),
4122 )?;
4123 return Ok(self.supervised_module(spec, runtime, snapshot, None));
4124 }
4125 Err(err) => {
4126 self.abandon_unrostered(&spec.module_id);
4129 return Err(err);
4130 }
4131 };
4132 arm_registration_deadline(&mut child, runtime.register_timeout, registration_watch);
4133 self.mark_first_process_running(&spec, &runtime, &snapshot, &mut child)?;
4134 if let Some(spawn) = activity_spawn {
4135 spawn.complete(&snapshot);
4136 }
4137
4138 Ok(self.supervised_module(spec, runtime, snapshot, Some(child)))
4139 }
4140
4141 pub async fn spawn_async(&self, spec: ModuleSpec) -> Result<SupervisedModule, SuperviseError> {
4144 validate_spec(&spec)?;
4145 self.establish_identity(&spec);
4146 let mut identity = AsyncInitialIdentity {
4147 supervisor: self,
4148 module: spec.module_id.clone(),
4149 committed: false,
4150 };
4151 let runtime = self.runtime_config();
4152 let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::starting()));
4153 let activity_spawn = runtime.begin_activity_spawn(&spec.module_id);
4154 let registration_watch = watch_registration(&self.registry, &spec);
4155 let spawned = spawn_child_async(&spec, &runtime).await;
4156 #[cfg(test)]
4157 self.test_after_first_spawn.run(&spec.module_id);
4158 let mut child = match spawned {
4159 Ok(child) => child,
4160 Err(SuperviseError::ConfigDeliveryDeferred { .. }) => {
4161 defer_config_spawn(
4162 &spec,
4163 &runtime,
4164 &snapshot,
4165 RespawnKind::Spawn,
4166 &runtime.failure_context(runtime.recovery_is_daemon),
4167 )?;
4168 identity.committed = true;
4169 return Ok(self.supervised_module(spec, runtime, snapshot, None));
4170 }
4171 Err(SuperviseError::ConfigHeld { .. }) => {
4172 hold_config_spawn(
4173 &spec,
4174 &runtime,
4175 &snapshot,
4176 &runtime.failure_context(runtime.recovery_is_daemon),
4177 )?;
4178 identity.committed = true;
4179 return Ok(self.supervised_module(spec, runtime, snapshot, None));
4180 }
4181 Err(err) => {
4182 self.abandon_unrostered(&spec.module_id);
4183 return Err(err);
4184 }
4185 };
4186 arm_registration_deadline(&mut child, runtime.register_timeout, registration_watch);
4187 self.mark_first_process_running(&spec, &runtime, &snapshot, &mut child)?;
4188 if let Some(spawn) = activity_spawn {
4189 spawn.complete(&snapshot);
4190 }
4191 identity.committed = true;
4192 Ok(self.supervised_module(spec, runtime, snapshot, Some(child)))
4193 }
4194
4195 fn establish_identity(&self, spec: &ModuleSpec) {
4208 if let Some(supervisor_handle) = &self.supervisor_handle {
4209 supervisor_handle.apply_identity_configuration(spec);
4210 supervisor_handle.mark_configured(&spec.module_id);
4211 }
4212 }
4213
4214 fn abandon_unrostered(&self, module_id: &str) {
4217 if let Some(supervisor_handle) = &self.supervisor_handle {
4218 supervisor_handle.unmark_configured_unless_rostered(module_id);
4219 }
4220 }
4221
4222 fn mark_first_process_running(
4225 &self,
4226 spec: &ModuleSpec,
4227 runtime: &SupervisorRuntimeConfig,
4228 snapshot: &SharedSnapshot,
4229 child: &mut SupervisedChild,
4230 ) -> Result<(), SuperviseError> {
4231 if let Err(err) = set_running(snapshot, child, &spec.module_id, &runtime.spawn_events) {
4232 containment::terminate_blocking(&runtime.child_roster.containment);
4233 self.abandon_unrostered(&spec.module_id);
4234 return Err(err);
4235 }
4236 self.process_liveness
4237 .track(spec.module_id.clone(), Arc::clone(snapshot));
4238 Ok(())
4239 }
4240
4241 pub fn supervise_configured(
4247 &self,
4248 spec: ModuleSpec,
4249 enabled: bool,
4250 ) -> Result<SupervisedModule, SuperviseError> {
4251 self.supervise_configured_requested(
4252 spec,
4253 enabled,
4254 self.health.clone(),
4255 None,
4256 self.restart_policy,
4257 DEFAULT_REGISTER_TIMEOUT,
4258 None,
4259 )
4260 }
4261
4262 pub fn supervise_configured_with_health(
4268 &self,
4269 spec: ModuleSpec,
4270 enabled: bool,
4271 health: HealthConfig,
4272 drain_timeout_ms: Option<u64>,
4273 restart_policy: RestartPolicy,
4274 ) -> Result<SupervisedModule, SuperviseError> {
4275 self.supervise_configured_requested(
4276 spec,
4277 enabled,
4278 health,
4279 drain_timeout_ms,
4280 restart_policy,
4281 DEFAULT_REGISTER_TIMEOUT,
4282 None,
4283 )
4284 }
4285
4286 #[allow(clippy::too_many_arguments)]
4287 pub(crate) fn supervise_configured_requested(
4288 &self,
4289 spec: ModuleSpec,
4290 enabled: bool,
4291 health: HealthConfig,
4292 drain_timeout_ms: Option<u64>,
4293 restart_policy: RestartPolicy,
4294 register_timeout: Duration,
4295 requester: Option<Principal>,
4296 ) -> Result<SupervisedModule, SuperviseError> {
4297 let registered = self.register_configured_with_health(
4298 spec,
4299 enabled,
4300 health,
4301 drain_timeout_ms,
4302 restart_policy,
4303 register_timeout,
4304 )?;
4305 let daemon_initiated = requester.is_none();
4306 self.start_registered_configured(registered, requester, daemon_initiated)
4307 }
4308
4309 #[allow(clippy::too_many_arguments)]
4310 pub(crate) async fn supervise_configured_requested_async(
4311 &self,
4312 spec: ModuleSpec,
4313 enabled: bool,
4314 health: HealthConfig,
4315 drain_timeout_ms: Option<u64>,
4316 restart_policy: RestartPolicy,
4317 register_timeout: Duration,
4318 requester: Option<Principal>,
4319 ) -> Result<SupervisedModule, SuperviseError> {
4320 let registered = self.register_configured_with_health(
4321 spec,
4322 enabled,
4323 health,
4324 drain_timeout_ms,
4325 restart_policy,
4326 register_timeout,
4327 )?;
4328 let daemon_initiated = requester.is_none();
4329 self.start_registered_configured_async(registered, requester, daemon_initiated)
4330 .await
4331 }
4332
4333 pub(crate) fn register_configured_with_health(
4338 &self,
4339 spec: ModuleSpec,
4340 enabled: bool,
4341 health: HealthConfig,
4342 drain_timeout_ms: Option<u64>,
4343 restart_policy: RestartPolicy,
4344 register_timeout: Duration,
4345 ) -> Result<RegisteredConfiguredModule, SuperviseError> {
4346 validate_spec(&spec)?;
4347 self.establish_identity(&spec);
4348
4349 let mut runtime = self.runtime_config();
4350 runtime.health = health;
4351 runtime.restart_policy = restart_policy;
4352 runtime.register_timeout = register_timeout;
4353 if let Some(ms) = drain_timeout_ms {
4354 runtime.drain_timeout = Duration::from_millis(ms);
4355 *runtime
4356 .effective_drain_timeout
4357 .lock()
4358 .unwrap_or_else(|poisoned| poisoned.into_inner()) = runtime.drain_timeout;
4359 }
4360 let snapshot = if enabled {
4361 SupervisorSnapshot::starting()
4362 } else {
4363 SupervisorSnapshot::disabled()
4364 };
4365 let snapshot = {
4366 let mut snapshot = snapshot;
4367 if enabled {
4368 if let Some((pid, reason)) = runtime.child_roster.previous_instance(&spec.module_id)
4369 {
4370 snapshot.waiting_for_previous_pid = Some(pid);
4371 snapshot.waiting_for_previous_detail = Some(reason.waiting_message(pid));
4372 }
4373 }
4374 snapshot
4375 };
4376 let (module, commands) =
4377 self.register_module(&spec, &runtime, Arc::new(Mutex::new(snapshot)));
4378 Ok(RegisteredConfiguredModule {
4379 spec,
4380 runtime,
4381 module,
4382 commands,
4383 })
4384 }
4385
4386 pub(crate) fn start_registered_configured(
4389 &self,
4390 registered: RegisteredConfiguredModule,
4391 requester: Option<Principal>,
4392 daemon_initiated: bool,
4393 ) -> Result<SupervisedModule, SuperviseError> {
4394 let RegisteredConfiguredModule {
4395 spec,
4396 mut runtime,
4397 module,
4398 commands,
4399 } = registered;
4400 runtime.start_requester = requester;
4401 runtime.recovery_is_daemon = daemon_initiated;
4402 let snapshot = &module.inner.snapshot;
4403 let should_spawn = {
4404 let state = lock_snapshot(snapshot)?;
4405 state.enabled && state.waiting_for_previous_pid.is_none()
4406 };
4407 let child = if should_spawn {
4408 let activity_spawn = runtime.begin_activity_spawn(&spec.module_id);
4409 let registration_watch = watch_registration(&self.registry, &spec);
4410 let spawned = spawn_child(
4411 &spec,
4412 runtime.connection_file_path.as_deref(),
4413 self.supervisor_handle.as_ref(),
4414 &runtime.stderr_ring,
4415 runtime.capture_logs_dir.as_deref(),
4416 &runtime.child_roster,
4417 runtime.spawn_delivery(),
4418 #[cfg(target_os = "linux")]
4419 runtime.cgroup_placement.as_ref(),
4420 runtime.register_timeout,
4421 );
4422 #[cfg(test)]
4423 self.test_after_first_spawn.run(&spec.module_id);
4424 match spawned {
4425 Ok(mut child) => {
4426 arm_registration_deadline(
4427 &mut child,
4428 runtime.register_timeout,
4429 registration_watch,
4430 );
4431 self.mark_first_process_running(&spec, &runtime, snapshot, &mut child)?;
4432 if let Some(spawn) = activity_spawn {
4433 spawn.complete(snapshot);
4434 }
4435 Some(child)
4436 }
4437 Err(SuperviseError::ConfigDeliveryDeferred { .. }) => {
4438 defer_config_spawn(
4439 &spec,
4440 &runtime,
4441 snapshot,
4442 RespawnKind::Spawn,
4443 &runtime.failure_context(daemon_initiated),
4444 )?;
4445 None
4446 }
4447 Err(mut err) => {
4448 if matches!(err, SuperviseError::ConfigHeld { .. }) {
4449 match hold_config_spawn(
4450 &spec,
4451 &runtime,
4452 snapshot,
4453 &runtime.failure_context(daemon_initiated),
4454 ) {
4455 Ok(()) => {
4456 self.start_monitor(spec, runtime, &module, None, commands);
4457 return Ok(module);
4458 }
4459 Err(parking_error) => err = parking_error,
4460 }
4461 }
4462 if runtime.health.critical {
4463 error!(
4464 module_id = %spec.module_id,
4465 program = %spec.program.display(),
4466 error = %err,
4467 "critical configured module failed to spawn; marking failed and alerting"
4468 );
4469 } else {
4470 error!(
4471 module_id = %spec.module_id,
4472 program = %spec.program.display(),
4473 error = %err,
4474 "configured module failed to spawn; marking failed and continuing"
4475 );
4476 }
4477 fail_snapshot(
4478 snapshot,
4479 Some(&spec.module_id),
4480 None,
4481 &runtime.failure_context(daemon_initiated),
4482 );
4483 None
4484 }
4485 }
4486 } else {
4487 None
4488 };
4489 self.start_monitor(spec, runtime, &module, child, commands);
4490 Ok(module)
4491 }
4492
4493 pub(crate) async fn start_registered_configured_async(
4494 &self,
4495 registered: RegisteredConfiguredModule,
4496 requester: Option<Principal>,
4497 daemon_initiated: bool,
4498 ) -> Result<SupervisedModule, SuperviseError> {
4499 self.start_registered_with_failure_policy(registered, requester, daemon_initiated, None)
4500 .await
4501 }
4502
4503 pub(crate) async fn start_registered_approved_async(
4504 &self,
4505 registered: RegisteredConfiguredModule,
4506 requester: Principal,
4507 generation: crate::config_gate::runtime::ApprovalIdentity,
4508 ) -> Result<SupervisedModule, SuperviseError> {
4509 self.start_registered_with_failure_policy(
4510 registered,
4511 Some(requester),
4512 false,
4513 Some(generation),
4514 )
4515 .await
4516 }
4517
4518 async fn start_registered_with_failure_policy(
4519 &self,
4520 registered: RegisteredConfiguredModule,
4521 requester: Option<Principal>,
4522 daemon_initiated: bool,
4523 approved_generation: Option<crate::config_gate::runtime::ApprovalIdentity>,
4524 ) -> Result<SupervisedModule, SuperviseError> {
4525 let RegisteredConfiguredModule {
4526 spec,
4527 mut runtime,
4528 module,
4529 commands,
4530 } = registered;
4531 runtime.start_requester = requester;
4532 runtime.recovery_is_daemon = daemon_initiated;
4533 let snapshot = &module.inner.snapshot;
4534 let mut startup_error = None;
4535 let failure = FailureContext {
4536 approval: approved_generation.clone(),
4537 ..runtime.failure_context(daemon_initiated)
4538 };
4539 let should_spawn = {
4540 let state = lock_snapshot(snapshot)?;
4541 state.enabled && state.waiting_for_previous_pid.is_none()
4542 };
4543 let child = if should_spawn {
4544 let activity_spawn = runtime.begin_activity_spawn(&spec.module_id);
4545 let registration_watch = watch_registration(&self.registry, &spec);
4546 let spawned = spawn_child_async(&spec, &runtime).await;
4547 #[cfg(test)]
4548 self.test_after_first_spawn.run(&spec.module_id);
4549 match spawned {
4550 Ok(mut child) => {
4551 arm_registration_deadline(
4552 &mut child,
4553 runtime.register_timeout,
4554 registration_watch,
4555 );
4556 self.mark_first_process_running(&spec, &runtime, snapshot, &mut child)?;
4557 if let Some(spawn) = activity_spawn {
4558 spawn.complete(snapshot);
4559 }
4560 Some(child)
4561 }
4562 Err(SuperviseError::ConfigDeliveryDeferred { .. }) => {
4563 defer_config_spawn(&spec, &runtime, snapshot, RespawnKind::Spawn, &failure)?;
4564 None
4565 }
4566 Err(mut err) => {
4567 if matches!(err, SuperviseError::ConfigHeld { .. }) {
4568 match hold_config_spawn(&spec, &runtime, snapshot, &failure) {
4569 Ok(()) => {
4570 self.start_monitor(spec, runtime, &module, None, commands);
4571 return Ok(module);
4572 }
4573 Err(parking_error) => err = parking_error,
4574 }
4575 }
4576 if runtime.health.critical {
4577 error!(module_id = %spec.module_id, program = %spec.program.display(), error = %err,
4578 "critical configured module failed to spawn; marking failed and alerting");
4579 } else {
4580 error!(module_id = %spec.module_id, program = %spec.program.display(), error = %err,
4581 "configured module failed to spawn; marking failed and continuing");
4582 }
4583 fail_snapshot(snapshot, Some(&spec.module_id), None, &failure);
4584 if let Some(generation) = approved_generation {
4585 let _ = update_snapshot(snapshot, Some(&spec.module_id), |state| {
4586 state.approved_start_failure = Some((generation, err.to_string()));
4587 });
4588 startup_error = Some(err);
4589 }
4590 None
4591 }
4592 }
4593 } else {
4594 None
4595 };
4596 self.start_monitor(spec, runtime, &module, child, commands);
4597 startup_error.map_or(Ok(module), Err)
4601 }
4602
4603 fn runtime_config(&self) -> SupervisorRuntimeConfig {
4604 let effective_drain_timeout = Arc::new(Mutex::new(self.drain_timeout));
4605 SupervisorRuntimeConfig {
4606 registry: self.registry.clone(),
4607 config_delivery: self.config_delivery.clone(),
4608 recovery_is_daemon: true,
4609 start_requester: None,
4610 pending_activity_restart: false,
4611 provenance_probe: self.provenance_probe.clone(),
4612 scheduled_respawn: Arc::default(),
4613 deferred_reload_reply: Arc::default(),
4614 restart_policy: self.restart_policy,
4615 register_timeout: DEFAULT_REGISTER_TIMEOUT,
4616 drain_timeout: self.drain_timeout,
4617 child_roster: self
4620 .child_roster
4621 .for_module(Arc::clone(&effective_drain_timeout)),
4622 effective_drain_timeout,
4623 default_drain_timeout: self.drain_timeout,
4624 health: self.health.clone(),
4625 host_health: self.host_health.clone(),
4626 connection_file_path: self.connection_file_path.clone(),
4627 capture_logs_dir: self.capture_logs_dir.clone(),
4628 forwarding: self.forwarding.clone(),
4629 supervisor_handle: self.supervisor_handle.clone(),
4630 stderr_ring: Arc::new(Mutex::new(StderrRing::new(StderrTailConfig::default()))),
4631 terminal_ring: Arc::new(Mutex::new(
4632 TerminalRing::new(
4633 TerminalRingConfig::default(),
4634 self.daemon_start_clock.started_at_ms(),
4635 )
4636 .with_start_clock(self.daemon_start_clock)
4637 .with_journal(self.terminal_journal.clone())
4638 .with_daemon_shutdown(self.child_roster.shutdown_flag()),
4639 )),
4640 spawn_events: self.spawn_events.clone(),
4641 #[cfg(target_os = "linux")]
4642 cgroup_placement: self.cgroup_placement.clone(),
4643 #[cfg(test)]
4644 test_seed_stale_facts_before_enable_spawn: false,
4645 #[cfg(test)]
4646 test_reload_exit_record_gate: None,
4647 }
4648 }
4649
4650 fn supervised_module(
4651 &self,
4652 spec: ModuleSpec,
4653 runtime: SupervisorRuntimeConfig,
4654 snapshot: SharedSnapshot,
4655 child: Option<SupervisedChild>,
4656 ) -> SupervisedModule {
4657 let (module, commands) = self.register_module(&spec, &runtime, snapshot);
4658 self.start_monitor(spec, runtime, &module, child, commands);
4659 module
4660 }
4661
4662 fn register_module(
4663 &self,
4664 spec: &ModuleSpec,
4665 runtime: &SupervisorRuntimeConfig,
4666 snapshot: SharedSnapshot,
4667 ) -> (SupervisedModule, mpsc::Receiver<SupervisorCommand>) {
4668 runtime.host_health.track(spec, &snapshot);
4669 let configuration = Arc::new(Mutex::new(SupervisedConfiguration {
4670 spec: spec.clone(),
4671 enabled: snapshot.lock().unwrap_or_else(|p| p.into_inner()).enabled,
4672 health: runtime.health.clone(),
4673 register_timeout: runtime.register_timeout,
4674 restart_policy: runtime.restart_policy,
4675 }));
4676 let stderr_ring = Arc::clone(&runtime.stderr_ring);
4677 let terminal_ring = Arc::clone(&runtime.terminal_ring);
4678 let effective_drain_timeout = Arc::clone(&runtime.effective_drain_timeout);
4679 let (tx, rx) = mpsc::channel(4);
4680 let module_id = spec.module_id.clone();
4681 let module = SupervisedModule {
4682 inner: Arc::new(SupervisedModuleInner {
4683 module_id: module_id.clone(),
4684 registry: Arc::clone(&self.registry),
4685 snapshot,
4686 configuration,
4687 stderr_ring,
4688 terminal_ring,
4689 commands: tx,
4690 monitor: Mutex::new(None),
4691 effective_drain_timeout,
4692 provenance_probe: self.provenance_probe.clone(),
4693 retained: runtime.child_roster.retained_view(),
4694 }),
4695 };
4696 if let Some(supervisor_handle) = &self.supervisor_handle {
4699 supervisor_handle.insert(module.clone());
4700 }
4701 (module, rx)
4702 }
4703
4704 fn start_monitor(
4705 &self,
4706 spec: ModuleSpec,
4707 runtime: SupervisorRuntimeConfig,
4708 module: &SupervisedModule,
4709 child: Option<SupervisedChild>,
4710 commands: mpsc::Receiver<SupervisorCommand>,
4711 ) {
4712 let monitor = tokio::spawn(supervise_loop(
4713 spec,
4714 runtime,
4715 Arc::clone(&self.registry),
4716 Arc::clone(&self.process_liveness),
4717 Arc::clone(&module.inner.snapshot),
4718 child,
4719 commands,
4720 ));
4721 *module
4722 .inner
4723 .monitor
4724 .lock()
4725 .unwrap_or_else(|p| p.into_inner()) = Some(monitor);
4726 }
4727}
4728
4729impl Default for Supervisor {
4730 fn default() -> Self {
4731 Self::new(Arc::new(Registry::default()), RestartPolicy::default())
4732 }
4733}
4734
4735#[derive(Clone)]
4737pub struct SupervisedModule {
4738 inner: Arc<SupervisedModuleInner>,
4739}
4740
4741struct SupervisedModuleInner {
4742 retained: crate::child_roster::RetainedView,
4743 module_id: String,
4744 registry: Arc<Registry>,
4745 snapshot: SharedSnapshot,
4746 configuration: Arc<Mutex<SupervisedConfiguration>>,
4747 stderr_ring: Arc<Mutex<StderrRing>>,
4748 terminal_ring: Arc<Mutex<TerminalRing>>,
4749 commands: mpsc::Sender<SupervisorCommand>,
4750 monitor: Mutex<Option<JoinHandle<()>>>,
4751 effective_drain_timeout: Arc<Mutex<Duration>>,
4752 provenance_probe: ExecutableIdentityProbe,
4753}
4754
4755impl fmt::Debug for SupervisedModule {
4756 fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
4757 f.debug_struct("SupervisedModule")
4758 .field("module_id", &self.inner.module_id)
4759 .field("status", &self.status())
4760 .finish_non_exhaustive()
4761 }
4762}
4763
4764impl SupervisedModule {
4765 pub(crate) fn check_approved_start_failure(
4766 &self,
4767 generation: &crate::config_gate::runtime::ApprovalIdentity,
4768 ) -> Result<bool, SuperviseError> {
4769 let state = lock_snapshot(&self.inner.snapshot)?;
4770 if let Some((failed_generation, detail)) = &state.approved_start_failure {
4771 if failed_generation == generation {
4772 return Err(SuperviseError::ReloadFailed {
4773 module_id: self.inner.module_id.clone(),
4774 reason: detail.clone(),
4775 });
4776 }
4777 }
4778 Ok(state.approved_start_failure.is_some())
4779 }
4780
4781 pub(crate) fn has_approved_start_failure(
4782 &self,
4783 identity: &crate::config_gate::runtime::ApprovalIdentity,
4784 ) -> bool {
4785 lock_snapshot(&self.inner.snapshot).is_ok_and(|state| {
4786 state
4787 .approved_start_failure
4788 .as_ref()
4789 .is_some_and(|(failed, _)| failed == identity)
4790 })
4791 }
4792
4793 pub(crate) fn needs_approved_gate_restart(
4794 &self,
4795 approval: &crate::config_gate::runtime::ApprovalIdentity,
4796 ) -> bool {
4797 lock_snapshot(&self.inner.snapshot)
4798 .is_ok_and(|state| state.needs_approved_gate_restart(approval))
4799 }
4800
4801 #[cfg(all(test, unix))]
4802 pub(crate) fn config_hold_origin_for_test(
4803 &self,
4804 ) -> (bool, Option<crate::config_gate::runtime::ApprovalIdentity>) {
4805 let state = lock_snapshot(&self.inner.snapshot).unwrap();
4806 (
4807 state.config_hold_daemon_initiated,
4808 state.config_hold_approval.clone(),
4809 )
4810 }
4811
4812 pub(crate) async fn config_policy_probe(
4813 &self,
4814 connection: crate::ConnectionId,
4815 generation: u64,
4816 attempt: u64,
4817 ) -> Result<(), SuperviseError> {
4818 self.inner
4819 .commands
4820 .send(SupervisorCommand::ConfigPolicyProbe {
4821 connection,
4822 generation,
4823 attempt,
4824 })
4825 .await
4826 .map_err(|_| SuperviseError::CommandClosed {
4827 module_id: self.inner.module_id.clone(),
4828 })
4829 }
4830 pub(crate) async fn config_exit_hold(&self) -> Result<(), SuperviseError> {
4831 let (reply, result) = oneshot::channel();
4832 self.inner
4833 .commands
4834 .send(SupervisorCommand::ConfigExitHold { reply })
4835 .await
4836 .map_err(|_| SuperviseError::CommandClosed {
4837 module_id: self.module_id().to_string(),
4838 })?;
4839 result.await.map_err(|_| SuperviseError::CommandClosed {
4840 module_id: self.module_id().to_string(),
4841 })?
4842 }
4843 pub(crate) async fn config_delivery_failed(
4844 &self,
4845 connection: crate::ConnectionId,
4846 hold: bool,
4847 ) -> Result<(), SuperviseError> {
4848 self.inner
4849 .commands
4850 .send(SupervisorCommand::ConfigDeliveryFailed { connection, hold })
4851 .await
4852 .map_err(|_| SuperviseError::CommandClosed {
4853 module_id: self.module_id().to_string(),
4854 })
4855 }
4856
4857 pub(crate) async fn config_apply_restart(&self, generation: u64) -> Result<(), SuperviseError> {
4858 self.inner
4859 .commands
4860 .send(SupervisorCommand::ConfigApplyRestart {
4861 generation,
4862 supersedes_swap: false,
4863 })
4864 .await
4865 .map_err(|_| SuperviseError::CommandClosed {
4866 module_id: self.inner.module_id.clone(),
4867 })
4868 }
4869 pub fn module_id(&self) -> &str {
4870 &self.inner.module_id
4871 }
4872
4873 #[cfg(test)]
4877 pub(crate) fn record_health_probe_failure_for_test(
4878 &self,
4879 detail: &str,
4880 ) -> Result<(), SuperviseError> {
4881 update_snapshot(&self.inner.snapshot, Some(&self.inner.module_id), |state| {
4882 state.health.consecutive_failures = state.health.consecutive_failures.saturating_add(1);
4883 state.health.detail = Some(detail.to_string());
4884 })
4885 }
4886
4887 pub fn state(&self) -> Result<ModuleState, SuperviseError> {
4888 Ok(lock_snapshot(&self.inner.snapshot)?.state)
4889 }
4890
4891 pub(crate) fn registration_end_reason(
4892 &self,
4893 ) -> Result<Option<RegistrationEndReason>, SuperviseError> {
4894 let snapshot = lock_snapshot(&self.inner.snapshot)?;
4895 Ok(match snapshot.state {
4896 ModuleState::Draining if snapshot.draining_to_replace => {
4897 Some(RegistrationEndReason::SupervisorRestart)
4898 }
4899 ModuleState::Draining => Some(RegistrationEndReason::SupervisorStop),
4900 ModuleState::Restarting if snapshot.draining_to_replace => {
4901 Some(RegistrationEndReason::SupervisorRestart)
4902 }
4903 _ => None,
4904 })
4905 }
4906
4907 pub fn stderr_tail(
4914 &self,
4915 max_lines: Option<usize>,
4916 max_bytes: Option<usize>,
4917 ) -> StderrTailSnapshot {
4918 self.inner
4919 .stderr_ring
4920 .lock()
4921 .unwrap_or_else(|poisoned| poisoned.into_inner())
4922 .snapshot(max_lines, max_bytes)
4923 }
4924
4925 pub fn terminal_history(&self) -> TerminalHistorySnapshot {
4930 self.inner
4931 .terminal_ring
4932 .lock()
4933 .unwrap_or_else(|poisoned| poisoned.into_inner())
4934 .snapshot()
4935 }
4936
4937 pub fn durable_terminal_history(&self) -> subc_control::TerminalHistory {
4942 durable_terminal_history_of(&self.inner.terminal_ring, &self.inner.module_id)
4943 }
4944
4945 pub(crate) async fn read_durable_terminal_history(
4950 &self,
4951 ) -> Result<subc_control::TerminalHistory, tokio::task::JoinError> {
4952 let terminal_ring = Arc::clone(&self.inner.terminal_ring);
4953 let module_id = self.inner.module_id.clone();
4954 tokio::task::spawn_blocking(move || durable_terminal_history_of(&terminal_ring, &module_id))
4955 .await
4956 }
4957
4958 pub fn status(&self) -> Result<ModuleStatus, SuperviseError> {
4959 self.status_with_snapshot_lock(&self.inner.snapshot, None)
4960 .map(|(status, _)| status)
4961 }
4962
4963 pub(crate) fn next_retry_at_ms(&self) -> Result<Option<u64>, SuperviseError> {
4964 let state = lock_snapshot(&self.inner.snapshot)?;
4965 Ok(state
4966 .failed_retry
4967 .pending
4968 .filter(|_| state.state == ModuleState::Failed && state.enabled)
4969 .map(|retry| {
4970 unix_ms_now().saturating_add(
4971 u64::try_from(
4972 retry
4973 .deadline
4974 .saturating_duration_since(Instant::now())
4975 .as_millis(),
4976 )
4977 .unwrap_or(u64::MAX),
4978 )
4979 }))
4980 }
4981
4982 #[cfg(all(test, unix))]
4983 pub(crate) fn retry_deadline_for_test(&self) -> Option<Instant> {
4984 lock_snapshot(&self.inner.snapshot)
4985 .unwrap()
4986 .failed_retry
4987 .pending
4988 .map(|retry| retry.deadline)
4989 }
4990
4991 pub(crate) fn record_deliberate_severance(
4992 &self,
4993 identity: ProcessIdentity,
4994 ) -> Result<bool, SuperviseError> {
4995 let mut snapshot = lock_snapshot(&self.inner.snapshot)?;
4996 if snapshot.pid != Some(identity.pid)
4997 || snapshot.process_start_time != Some(identity.start_time)
4998 {
4999 return Ok(false);
5000 }
5001 snapshot.deliberate_severance = Some(identity);
5002 Ok(true)
5003 }
5004
5005 pub(crate) fn status_for_control(
5010 &self,
5011 caller: &'static str,
5012 ) -> Result<ModuleStatus, SuperviseError> {
5013 self.status_with_snapshot_lock(&self.inner.snapshot, Some(caller))
5014 .map(|(status, _)| status)
5015 }
5016
5017 fn status_with_snapshot_lock(
5018 &self,
5019 snapshot: &SharedSnapshot,
5020 caller: Option<&'static str>,
5021 ) -> Result<(ModuleStatus, Option<SpawnedFileIdentity>), SuperviseError> {
5022 let restart_policy = self
5023 .inner
5024 .configuration
5025 .lock()
5026 .map_err(|_| SuperviseError::StatePoisoned {
5027 module_id: Some(self.inner.module_id.clone()),
5028 })?
5029 .restart_policy;
5030 let mut guard = match caller {
5031 Some(caller) => lock_snapshot_for_control(snapshot, &self.inner.module_id, caller)?,
5032 None => lock_snapshot(snapshot)?,
5033 };
5034 let restart_count = guard.crash_restarts_in_window(restart_policy.window, Instant::now());
5037 let snapshot = guard.clone();
5038 drop(guard);
5039 let drain_timeout = *self.inner.effective_drain_timeout.lock().map_err(|_| {
5040 SuperviseError::StatePoisoned {
5041 module_id: Some(self.inner.module_id.clone()),
5042 }
5043 })?;
5044 let registration_active = self
5045 .inner
5046 .registry
5047 .get_module(&self.inner.module_id)
5048 .map_err(SuperviseError::Registry)?
5049 .is_some();
5050 let protocol = snapshot
5051 .spawned_protocol
5052 .unwrap_or(self.declared_protocol()?);
5053 let running_process =
5054 snapshot.enabled && snapshot.state == ModuleState::Running && snapshot.process_alive;
5055 let live = match protocol {
5061 ModuleProtocol::Subc => running_process && registration_active,
5062 ModuleProtocol::None => running_process,
5063 };
5064 let retained = self.inner.retained.waiting_detail(&self.inner.module_id);
5067 let carry_configured = self
5068 .inner
5069 .configuration
5070 .lock()
5071 .map_err(|_| SuperviseError::StatePoisoned {
5072 module_id: Some(self.inner.module_id.clone()),
5073 })?
5074 .spec
5075 .carry_containment;
5076 let recovery_block = carries_containment(
5077 carry_configured,
5078 self.inner
5079 .retained
5080 .has_carried_container(&self.inner.module_id),
5081 )
5082 .then(|| self.inner.retained.recovery_block())
5083 .flatten();
5084 let launch_nonce_source = observed_launch_nonce_source(&snapshot);
5085 let pid = snapshot.reported_pid();
5086 let mut health = snapshot.health;
5087 if let Some(detail) = recovery_block
5088 .as_ref()
5089 .or_else(|| retained.as_ref().map(|(_, detail)| detail))
5090 {
5091 health.detail = Some(match health.detail {
5092 Some(previous) => format!("{previous}; {detail}"),
5093 None => detail.clone(),
5094 });
5095 }
5096
5097 Ok((
5098 ModuleStatus {
5099 admission_refused: snapshot.admission_refused,
5100 legacy_registration_spawn: snapshot.legacy_registration_spawn,
5101 registration_declaration: snapshot.registration_declaration.clone(),
5102 stop_reason: snapshot.stop_reason,
5103 waiting_for_previous_pid: retained
5104 .as_ref()
5105 .map(|(pid, _)| *pid)
5106 .filter(|pid| *pid > 0)
5107 .or(snapshot.waiting_for_previous_pid),
5108 waiting_for_previous_detail: recovery_block
5109 .or_else(|| retained.map(|(_, detail)| detail))
5110 .or(snapshot.waiting_for_previous_detail),
5111 module_id: self.inner.module_id.clone(),
5112 launch_nonce_source,
5113 state: snapshot.state,
5114 enabled: snapshot.enabled,
5115 process_alive: snapshot.process_alive,
5116 registration_active,
5117 protocol,
5118 live,
5119 restart_count,
5120 lifetime_restarts: snapshot.lifetime_restarts,
5121 spawn_generation: snapshot.spawn_generation,
5122 swap_outcome: snapshot.swap_outcome,
5123 max_restarts: restart_policy.max_restarts,
5124 restart_window: restart_policy.window,
5125 drain_timeout,
5126 restart_backoff: restart_policy.backoff,
5127 restart_max_backoff: restart_policy.max_backoff,
5128 pid,
5129 spawned_at_ms: snapshot.spawned_at_ms,
5130 spawned_from: snapshot.spawned_from,
5131 launch_spec_digest: snapshot.launch_spec_digest,
5132 process_start_time: snapshot.process_start_time,
5133 last_exit: snapshot.last_exit,
5134 health,
5135 },
5136 snapshot.spawned_file_identity.clone(),
5137 ))
5138 }
5139
5140 #[cfg(test)]
5141 pub(crate) fn hold_snapshot_for_test(
5142 &self,
5143 acquired: std::sync::mpsc::Sender<()>,
5144 hold: Duration,
5145 ) -> std::thread::JoinHandle<()> {
5146 let snapshot = Arc::clone(&self.inner.snapshot);
5147 std::thread::spawn(move || {
5148 let _guard = snapshot.lock().expect("test snapshot lock is not poisoned");
5149 acquired
5150 .send(())
5151 .expect("test receiver waits for snapshot lock");
5152 std::thread::sleep(hold);
5153 })
5154 }
5155
5156 pub(crate) async fn status_and_running_image_agreement(
5162 &self,
5163 ) -> Result<(ModuleStatus, subc_control::RunningImageAgreement), SuperviseError> {
5164 let (status, identity) = self.status_with_snapshot_lock(&self.inner.snapshot, None)?;
5165 let image = self
5166 .inner
5167 .provenance_probe
5168 .observe(
5169 status.pid,
5170 status.spawned_from.as_deref(),
5171 identity,
5172 status.process_start_time,
5173 )
5174 .await;
5175 Ok((status, image))
5176 }
5177
5178 pub(crate) async fn running_image_agreement(&self) -> subc_control::RunningImageAgreement {
5179 let snapshot = match lock_snapshot(&self.inner.snapshot) {
5180 Ok(snapshot) => snapshot.clone(),
5181 Err(_) => {
5182 return subc_control::RunningImageAgreement::Unavailable {
5183 reason: subc_control::RunningImageUnavailableReason::NotRunning,
5184 };
5185 }
5186 };
5187 self.inner
5188 .provenance_probe
5189 .observe(
5190 snapshot.reported_pid(),
5191 snapshot.spawned_from.as_deref(),
5192 snapshot.spawned_file_identity.clone(),
5193 snapshot.process_start_time,
5194 )
5195 .await
5196 }
5197
5198 pub(crate) fn child_resource_usage(&self) -> subc_control::ChildResourceUsage {
5201 #[cfg(windows)]
5202 {
5203 let image = lock_snapshot(&self.inner.snapshot)
5204 .ok()
5205 .and_then(|snapshot| {
5206 snapshot
5207 .spawned_file_identity
5208 .as_ref()
5209 .and_then(|identity| identity.windows_image.clone())
5210 });
5211 match image {
5212 Some(image) => {
5213 crate::child_resources::read_windows_handle(image.process(), image.start_time())
5214 }
5215 None => subc_control::ChildResourceUsage::Unavailable {
5216 reason: subc_control::ChildResourceUnavailableReason::NotRunning,
5217 },
5218 }
5219 }
5220 #[cfg(not(windows))]
5221 {
5222 let (pid, start_time) = match lock_snapshot(&self.inner.snapshot) {
5223 Ok(snapshot) => (snapshot.reported_pid(), snapshot.process_start_time),
5224 Err(_) => {
5225 return subc_control::ChildResourceUsage::Unavailable {
5226 reason: subc_control::ChildResourceUnavailableReason::Unreadable,
5227 }
5228 }
5229 };
5230 crate::child_resources::read(pid, start_time)
5231 }
5232 }
5233
5234 #[cfg(all(test, any(target_os = "linux", windows)))]
5235 pub(crate) fn subtree_target(&self) -> crate::subtree_resources::Target {
5236 self.inner
5237 .snapshot
5238 .try_lock()
5239 .map(|snapshot| snapshot.subtree_target.clone())
5240 .unwrap_or(crate::subtree_resources::Target::Unavailable("read_failed"))
5241 }
5242
5243 #[cfg(test)]
5244 pub(crate) fn set_subtree_target_for_test(&self, target: crate::subtree_resources::Target) {
5245 self.inner.snapshot.lock().unwrap().subtree_target = target;
5246 }
5247
5248 pub(crate) fn will_recover_after_connection_loss(&self) -> Result<bool, SuperviseError> {
5249 let mut snapshot = lock_snapshot(&self.inner.snapshot)?;
5250 Ok(match snapshot.state {
5251 ModuleState::Restarting => true,
5252 ModuleState::Failed | ModuleState::Disabled => false,
5253 _ if snapshot.enabled && snapshot.respawn_pending => true,
5254 _ => daemon_will_restart(
5255 &mut snapshot,
5256 &self
5257 .inner
5258 .configuration
5259 .lock()
5260 .map_err(|_| SuperviseError::StatePoisoned {
5261 module_id: Some(self.inner.module_id.clone()),
5262 })?
5263 .restart_policy,
5264 Instant::now(),
5265 ),
5266 })
5267 }
5268
5269 #[cfg(test)]
5270 pub(crate) fn is_warming(&self) -> Result<bool, SuperviseError> {
5271 self.is_warming_with_snapshot_lock(None)
5272 }
5273
5274 pub(crate) fn is_warming_for_control(
5275 &self,
5276 caller: &'static str,
5277 ) -> Result<bool, SuperviseError> {
5278 self.is_warming_with_snapshot_lock(Some(caller))
5279 }
5280
5281 fn is_warming_with_snapshot_lock(
5282 &self,
5283 caller: Option<&'static str>,
5284 ) -> Result<bool, SuperviseError> {
5285 let snapshot = match caller {
5286 Some(caller) => {
5287 lock_snapshot_for_control(&self.inner.snapshot, &self.inner.module_id, caller)?
5288 }
5289 None => lock_snapshot(&self.inner.snapshot)?,
5290 }
5291 .clone();
5292 Ok(matches!(
5293 snapshot.state,
5294 ModuleState::Starting | ModuleState::Running | ModuleState::Restarting
5295 ))
5296 }
5297
5298 pub async fn drain(&self) -> Result<(), SuperviseError> {
5300 self.stop().await
5301 }
5302
5303 pub(crate) async fn retire_requested(
5304 &self,
5305 requester: Option<Principal>,
5306 ) -> Result<(), SuperviseError> {
5307 match self.state()? {
5308 ModuleState::Stopped => return Ok(()),
5309 ModuleState::Failed
5310 if lock_snapshot(&self.inner.snapshot)?
5311 .failed_retry
5312 .pending
5313 .is_none() =>
5314 {
5315 return Ok(())
5316 }
5317 ModuleState::Failed
5318 | ModuleState::Starting
5319 | ModuleState::Running
5320 | ModuleState::Unresponsive
5321 | ModuleState::Restarting
5322 | ModuleState::Draining
5323 | ModuleState::Disabled => {}
5324 }
5325
5326 let (reply_tx, reply_rx) = oneshot::channel();
5327 self.inner
5328 .commands
5329 .send(SupervisorCommand::Retire {
5330 requester,
5331 reply: reply_tx,
5332 })
5333 .await
5334 .map_err(|_| SuperviseError::CommandClosed {
5335 module_id: self.inner.module_id.clone(),
5336 })?;
5337 reply_rx.await.map_err(|_| SuperviseError::CommandClosed {
5338 module_id: self.inner.module_id.clone(),
5339 })?
5340 }
5341
5342 pub async fn stop(&self) -> Result<(), SuperviseError> {
5343 self.stop_requested(None).await
5344 }
5345
5346 pub(crate) async fn stop_requested(
5347 &self,
5348 requester: Option<Principal>,
5349 ) -> Result<(), SuperviseError> {
5350 match self.state()? {
5351 ModuleState::Stopped => return Ok(()),
5352 ModuleState::Failed
5353 if lock_snapshot(&self.inner.snapshot)?
5354 .failed_retry
5355 .pending
5356 .is_none() =>
5357 {
5358 return Ok(())
5359 }
5360 ModuleState::Failed
5361 | ModuleState::Starting
5362 | ModuleState::Running
5363 | ModuleState::Unresponsive
5364 | ModuleState::Restarting
5365 | ModuleState::Draining
5366 | ModuleState::Disabled => {}
5367 }
5368
5369 let (reply_tx, reply_rx) = oneshot::channel();
5370 self.inner
5371 .commands
5372 .send(SupervisorCommand::Drain {
5373 requester,
5374 reply: reply_tx,
5375 })
5376 .await
5377 .map_err(|_| SuperviseError::CommandClosed {
5378 module_id: self.inner.module_id.clone(),
5379 })?;
5380 reply_rx.await.map_err(|_| SuperviseError::CommandClosed {
5381 module_id: self.inner.module_id.clone(),
5382 })?
5383 }
5384
5385 pub async fn restart(&self, drain_timeout_ms: Option<u64>) -> Result<(), SuperviseError> {
5386 self.restart_requested(drain_timeout_ms, None).await
5387 }
5388
5389 pub(crate) async fn restart_requested(
5390 &self,
5391 drain_timeout_ms: Option<u64>,
5392 requester: Option<Principal>,
5393 ) -> Result<(), SuperviseError> {
5394 let queued_at = Instant::now();
5395 let received_at_generation = lock_snapshot(&self.inner.snapshot)?.spawn_generation;
5396 let (reply_tx, reply_rx) = oneshot::channel();
5397 self.inner
5398 .commands
5399 .send(SupervisorCommand::Restart {
5400 requester,
5401 drain_timeout_ms,
5402 received_at_generation,
5403 queued_at,
5404 reply: reply_tx,
5405 })
5406 .await
5407 .map_err(|_| SuperviseError::CommandClosed {
5408 module_id: self.inner.module_id.clone(),
5409 })?;
5410 reply_rx.await.map_err(|_| SuperviseError::CommandClosed {
5411 module_id: self.inner.module_id.clone(),
5412 })?
5413 }
5414
5415 pub async fn swap(
5421 &self,
5422 ready_timeout: Option<Duration>,
5423 ) -> Result<SwapOutcome, SuperviseError> {
5424 self.swap_requested(ready_timeout, None).await
5425 }
5426
5427 pub(crate) async fn swap_requested(
5428 &self,
5429 ready_timeout: Option<Duration>,
5430 requester: Option<Principal>,
5431 ) -> Result<SwapOutcome, SuperviseError> {
5432 let (reply_tx, reply_rx) = oneshot::channel();
5433 self.inner
5434 .commands
5435 .send(SupervisorCommand::Swap {
5436 requester,
5437 ready_timeout,
5438 reply: reply_tx,
5439 })
5440 .await
5441 .map_err(|_| SuperviseError::CommandClosed {
5442 module_id: self.inner.module_id.clone(),
5443 })?;
5444 reply_rx.await.map_err(|_| SuperviseError::CommandClosed {
5445 module_id: self.inner.module_id.clone(),
5446 })?
5447 }
5448
5449 pub async fn reload(&self) -> Result<(), SuperviseError> {
5450 self.reload_requested(None).await
5451 }
5452
5453 pub(crate) async fn reload_requested(
5454 &self,
5455 requester: Option<Principal>,
5456 ) -> Result<(), SuperviseError> {
5457 let (reply_tx, reply_rx) = oneshot::channel();
5458 self.inner
5459 .commands
5460 .send(SupervisorCommand::Reload {
5461 requester,
5462 reply: reply_tx,
5463 })
5464 .await
5465 .map_err(|_| SuperviseError::CommandClosed {
5466 module_id: self.inner.module_id.clone(),
5467 })?;
5468 reply_rx.await.map_err(|_| SuperviseError::CommandClosed {
5469 module_id: self.inner.module_id.clone(),
5470 })?
5471 }
5472
5473 pub async fn set_enabled(&self, enabled: bool) -> Result<bool, SuperviseError> {
5474 self.set_enabled_requested(enabled, None).await
5475 }
5476
5477 pub(crate) async fn set_enabled_requested(
5478 &self,
5479 enabled: bool,
5480 requester: Option<Principal>,
5481 ) -> Result<bool, SuperviseError> {
5482 self.set_enabled_with_budget_policy(enabled, requester, true)
5483 .await
5484 }
5485
5486 pub(crate) async fn release_config_hold(&self) -> Result<bool, SuperviseError> {
5487 self.set_enabled_with_budget_policy(true, None, false).await
5488 }
5489
5490 #[cfg(all(test, unix))]
5491 pub(crate) fn seed_crash_budget_for_test(&self, count: u32) {
5492 let policy = self.inner.configuration.lock().unwrap().restart_policy;
5493 let mut state = self.inner.snapshot.lock().unwrap();
5494 for _ in 0..count {
5495 state.record_crash_restart(&policy, Instant::now());
5496 }
5497 }
5498
5499 async fn set_enabled_with_budget_policy(
5500 &self,
5501 enabled: bool,
5502 requester: Option<Principal>,
5503 reset_budget: bool,
5504 ) -> Result<bool, SuperviseError> {
5505 let (reply_tx, reply_rx) = oneshot::channel();
5506 self.inner
5507 .commands
5508 .send(SupervisorCommand::SetEnabled {
5509 requester,
5510 enabled,
5511 reset_budget,
5512 reply: reply_tx,
5513 })
5514 .await
5515 .map_err(|_| SuperviseError::CommandClosed {
5516 module_id: self.inner.module_id.clone(),
5517 })?;
5518 reply_rx.await.map_err(|_| SuperviseError::CommandClosed {
5519 module_id: self.inner.module_id.clone(),
5520 })?
5521 }
5522
5523 pub(crate) fn declared_protocol(&self) -> Result<ModuleProtocol, SuperviseError> {
5527 let configured = self
5528 .inner
5529 .configuration
5530 .lock()
5531 .map_err(|_| SuperviseError::StatePoisoned {
5532 module_id: Some(self.inner.module_id.clone()),
5533 })?
5534 .spec
5535 .protocol;
5536 let state = lock_snapshot(&self.inner.snapshot)?;
5537 Ok(state.spawned_protocol.unwrap_or(configured))
5538 }
5539
5540 pub(crate) fn configuration(&self) -> Result<(ModuleSpec, HealthConfig), SuperviseError> {
5541 let configuration =
5542 self.inner
5543 .configuration
5544 .lock()
5545 .map_err(|_| SuperviseError::StatePoisoned {
5546 module_id: Some(self.inner.module_id.clone()),
5547 })?;
5548 Ok((configuration.spec.clone(), configuration.health.clone()))
5549 }
5550
5551 pub(crate) fn configured_register_timeout(&self) -> Result<Duration, SuperviseError> {
5552 self.inner
5553 .configuration
5554 .lock()
5555 .map(|configuration| configuration.register_timeout)
5556 .map_err(|_| SuperviseError::StatePoisoned {
5557 module_id: Some(self.inner.module_id.clone()),
5558 })
5559 }
5560
5561 pub(crate) fn configured_enabled(&self) -> Result<bool, SuperviseError> {
5562 self.inner
5563 .configuration
5564 .lock()
5565 .map(|configuration| configuration.enabled)
5566 .map_err(|_| SuperviseError::StatePoisoned {
5567 module_id: Some(self.inner.module_id.clone()),
5568 })
5569 }
5570
5571 #[cfg(any(test, feature = "test-support"))]
5575 pub async fn update_spec_for_test(&self, spec: ModuleSpec) -> Result<(), SuperviseError> {
5576 let (_, health) = self.configuration()?;
5577 let drain_timeout_ms = u64::try_from(
5578 self.inner
5579 .effective_drain_timeout
5580 .lock()
5581 .unwrap_or_else(|poisoned| poisoned.into_inner())
5582 .as_millis(),
5583 )
5584 .ok();
5585 self.update_configuration(spec, health, drain_timeout_ms)
5586 .await
5587 }
5588
5589 pub(crate) async fn reconcile_approved_configuration(
5590 &self,
5591 approved: crate::config_gate::runtime::ApprovedDaemonConfiguration,
5592 ) -> Result<(), SuperviseError> {
5593 let entry = approved
5594 .module_entry(&self.inner.module_id)
5595 .map_err(|error| SuperviseError::InvalidSpec {
5596 reason: error.to_string(),
5597 })?;
5598 validate_spec(&entry.module_spec())?;
5599 let (reply, received) = oneshot::channel();
5600 self.inner
5601 .commands
5602 .send(SupervisorCommand::ApprovedConfiguration { approved, reply })
5603 .await
5604 .map_err(|_| SuperviseError::CommandClosed {
5605 module_id: self.inner.module_id.clone(),
5606 })?;
5607 received.await.map_err(|_| SuperviseError::CommandClosed {
5608 module_id: self.inner.module_id.clone(),
5609 })??;
5610 let mut configuration =
5611 self.inner
5612 .configuration
5613 .lock()
5614 .map_err(|_| SuperviseError::StatePoisoned {
5615 module_id: Some(self.inner.module_id.clone()),
5616 })?;
5617 configuration.spec = entry.module_spec();
5618 configuration.enabled = entry.enabled;
5619 configuration.health = entry.health;
5620 configuration.register_timeout = entry.register_timeout;
5621 configuration.restart_policy = entry.restart;
5622 Ok(())
5623 }
5624
5625 pub(crate) async fn update_configuration(
5626 &self,
5627 spec: ModuleSpec,
5628 health: HealthConfig,
5629 drain_timeout_ms: Option<u64>,
5630 ) -> Result<(), SuperviseError> {
5631 if spec.module_id != self.inner.module_id {
5632 return Err(SuperviseError::InvalidSpec {
5633 reason: "a supervised module's module_id cannot be changed".to_string(),
5634 });
5635 }
5636 validate_spec(&spec)?;
5637 let (reply_tx, reply_rx) = oneshot::channel();
5638 self.inner
5639 .commands
5640 .send(SupervisorCommand::UpdateConfiguration {
5641 spec: spec.clone(),
5642 health: health.clone(),
5643 drain_timeout_ms,
5644 reply: reply_tx,
5645 })
5646 .await
5647 .map_err(|_| SuperviseError::CommandClosed {
5648 module_id: self.inner.module_id.clone(),
5649 })?;
5650 reply_rx.await.map_err(|_| SuperviseError::CommandClosed {
5651 module_id: self.inner.module_id.clone(),
5652 })??;
5653 let mut configuration =
5654 self.inner
5655 .configuration
5656 .lock()
5657 .map_err(|_| SuperviseError::StatePoisoned {
5658 module_id: Some(self.inner.module_id.clone()),
5659 })?;
5660 configuration.spec = spec;
5661 configuration.health = health;
5662 Ok(())
5663 }
5664}
5665
5666impl Drop for SupervisedModuleInner {
5667 fn drop(&mut self) {
5668 let Ok(mut monitor) = self.monitor.lock() else {
5669 return;
5670 };
5671 if let Some(monitor) = monitor.as_ref().filter(|monitor| !monitor.is_finished()) {
5672 let _ = update_snapshot(&self.snapshot, Some(&self.module_id), |state| {
5673 state.state = ModuleState::Stopped;
5674 clear_current_process_facts(state);
5675 });
5676 monitor.abort();
5677 }
5678 let _ = monitor.take();
5679 }
5680}
5681
5682#[derive(Debug)]
5683enum SupervisorCommand {
5684 ApprovedConfiguration {
5685 approved: crate::config_gate::runtime::ApprovedDaemonConfiguration,
5686 reply: oneshot::Sender<Result<(), SuperviseError>>,
5687 },
5688 ConfigPolicyProbe {
5689 connection: crate::ConnectionId,
5690 generation: u64,
5691 attempt: u64,
5692 },
5693 ConfigApplyRestart {
5694 generation: u64,
5695 supersedes_swap: bool,
5696 },
5697 ConfigExitHold {
5698 reply: oneshot::Sender<Result<(), SuperviseError>>,
5699 },
5700 ConfigDeliveryFailed {
5701 connection: crate::ConnectionId,
5702 hold: bool,
5703 },
5704 Drain {
5705 requester: Option<Principal>,
5706 reply: oneshot::Sender<Result<(), SuperviseError>>,
5707 },
5708 Retire {
5709 requester: Option<Principal>,
5710 reply: oneshot::Sender<Result<(), SuperviseError>>,
5711 },
5712 Restart {
5713 requester: Option<Principal>,
5714 drain_timeout_ms: Option<u64>,
5719 received_at_generation: u64,
5723 queued_at: Instant,
5726 reply: oneshot::Sender<Result<(), SuperviseError>>,
5727 },
5728 Reload {
5729 requester: Option<Principal>,
5730 reply: oneshot::Sender<Result<(), SuperviseError>>,
5731 },
5732 SetEnabled {
5733 reset_budget: bool,
5734 requester: Option<Principal>,
5735 enabled: bool,
5736 reply: oneshot::Sender<Result<bool, SuperviseError>>,
5737 },
5738 UpdateConfiguration {
5739 spec: ModuleSpec,
5740 health: HealthConfig,
5741 drain_timeout_ms: Option<u64>,
5744 reply: oneshot::Sender<Result<(), SuperviseError>>,
5745 },
5746 Swap {
5747 requester: Option<Principal>,
5748 ready_timeout: Option<Duration>,
5751 reply: oneshot::Sender<Result<SwapOutcome, SuperviseError>>,
5753 },
5754}
5755
5756#[derive(Clone, Copy, Debug, PartialEq, Eq)]
5760pub enum SwapOutcome {
5761 Cutover,
5762 SupersededByConfigRestart,
5763}
5764
5765impl SwapOutcome {
5766 pub fn as_str(self) -> &'static str {
5767 match self {
5768 Self::Cutover => "cutover",
5769 Self::SupersededByConfigRestart => "superseded_by_config_restart",
5770 }
5771 }
5772}
5773
5774#[derive(Debug)]
5775pub enum SuperviseError {
5776 ConfigHeld {
5777 module_id: String,
5778 code: String,
5779 detail: String,
5780 },
5781 ConfigDeliveryDeferred {
5782 module_id: String,
5783 },
5784 InvalidSpec {
5785 reason: String,
5786 },
5787 Spawn {
5788 program: PathBuf,
5789 source: io::Error,
5790 cgroup_path: Option<PathBuf>,
5791 },
5792 Cgroup {
5793 module_id: String,
5794 source: io::Error,
5795 },
5796 LaunchNonce {
5799 reason: String,
5800 },
5801 Wait {
5802 module_id: String,
5803 source: io::Error,
5804 },
5805 Kill {
5806 module_id: String,
5807 source: io::Error,
5808 },
5809 Forwarding(ForwardingError),
5810 Registry(RegistryError),
5811 ReloadUnavailable {
5812 module_id: String,
5813 reason: String,
5814 },
5815 Disabled {
5820 module_id: String,
5821 },
5822 ReloadFailed {
5823 module_id: String,
5824 reason: String,
5825 },
5826 RegistrationStillActive {
5827 module_id: String,
5828 waited: Duration,
5829 },
5830 StatePoisoned {
5831 module_id: Option<String>,
5832 },
5833 CommandClosed {
5834 module_id: String,
5835 },
5836 SwapInProgress {
5840 module_id: String,
5841 },
5842 SwapRefused {
5844 module_id: String,
5845 reason: SwapRefusal,
5846 },
5847 SwapFailed {
5851 module_id: String,
5852 arm: SwapFailureArm,
5853 detail: String,
5854 candidate_exit: Option<ExitReport>,
5857 },
5858}
5859
5860#[derive(Debug, Clone, Copy, PartialEq, Eq)]
5862pub enum SwapRefusal {
5863 OverlapExclusive,
5865 NotRegistered,
5868 ProtocolNone,
5871 NotConfigured,
5874 AlreadySwapping,
5876}
5877
5878impl SwapRefusal {
5879 pub fn as_str(self) -> &'static str {
5880 match self {
5881 Self::OverlapExclusive => "overlap_exclusive",
5882 Self::NotRegistered => "not_registered",
5883 Self::ProtocolNone => "protocol_none",
5884 Self::NotConfigured => "not_configured",
5885 Self::AlreadySwapping => "already_swapping",
5886 }
5887 }
5888}
5889
5890#[derive(Debug, Clone, Copy, PartialEq, Eq)]
5893pub enum SwapFailureArm {
5894 SpawnFailed,
5896 NeverRegistered,
5898 NeverReady,
5900 CandidateExited,
5902 CandidateUnhealthy,
5904 Interrupted,
5908 CutoverLost,
5913}
5914
5915impl SwapFailureArm {
5916 pub fn as_str(self) -> &'static str {
5917 match self {
5918 Self::SpawnFailed => "spawn_failed",
5919 Self::NeverRegistered => "never_registered",
5920 Self::NeverReady => "never_ready",
5921 Self::CandidateExited => "candidate_exited",
5922 Self::CandidateUnhealthy => "candidate_unhealthy",
5923 Self::Interrupted => "interrupted",
5924 Self::CutoverLost => "cutover_lost",
5925 }
5926 }
5927}
5928
5929impl SuperviseError {
5930 pub(crate) fn is_config_gate_error(&self) -> bool {
5931 match self {
5932 Self::ConfigHeld { .. } => true,
5933 Self::Spawn { source, .. } => source
5934 .get_ref()
5935 .is_some_and(|cause| cause.is::<crate::config_gate::GateError>()),
5936 _ => false,
5937 }
5938 }
5939}
5940
5941impl fmt::Display for SuperviseError {
5942 fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
5943 match self {
5944 Self::ConfigHeld { module_id, code, detail } => write!(f, "{code}: module '{module_id}' held: {detail}"),
5945 Self::ConfigDeliveryDeferred { module_id } => write!(f, "config_gated: module '{module_id}' is waiting for its exited delivery target's failure record"),
5946 Self::InvalidSpec { reason } => write!(f, "invalid module spec: {reason}"),
5947 Self::Spawn {
5948 program,
5949 source,
5950 cgroup_path: Some(cgroup_path),
5951 } => write!(
5952 f,
5953 "failed to place module in cgroup '{}' while spawning '{}': {source}",
5954 cgroup_path.display(),
5955 program.display()
5956 ),
5957 Self::Spawn {
5958 program,
5959 source,
5960 cgroup_path: None,
5961 } => write!(
5962 f,
5963 "failed to spawn module '{}': {source}",
5964 program.display()
5965 ),
5966 Self::Cgroup { module_id, source } => {
5967 write!(
5968 f,
5969 "failed to prepare cgroup for module '{module_id}': {source}"
5970 )
5971 }
5972 Self::LaunchNonce { reason } => {
5973 write!(
5974 f,
5975 "failed to generate reserved-module launch nonce: {reason}"
5976 )
5977 }
5978 Self::Wait { module_id, source } => {
5979 write!(f, "failed to wait for module '{module_id}': {source}")
5980 }
5981 Self::Kill { module_id, source } => {
5982 write!(f, "failed to kill module '{module_id}': {source}")
5983 }
5984 Self::Forwarding(err) => write!(f, "forwarding error: {err}"),
5985 Self::Registry(err) => write!(f, "registry error: {err}"),
5986 Self::ReloadUnavailable { module_id, reason } => {
5987 write!(f, "reload unavailable for module '{module_id}': {reason}")
5988 }
5989 Self::Disabled { module_id } => {
5990 write!(
5991 f,
5992 "module '{module_id}' is disabled; enable it before restart or reload"
5993 )
5994 }
5995 Self::ReloadFailed { module_id, reason } => {
5996 write!(f, "reload failed for module '{module_id}': {reason}")
5997 }
5998 Self::RegistrationStillActive { module_id, waited } => write!(
5999 f,
6000 "module '{module_id}' registration remained active after waiting {waited:?}"
6001 ),
6002 Self::StatePoisoned { module_id } => match module_id {
6003 Some(module_id) => {
6004 write!(f, "supervisor state for module '{module_id}' was poisoned")
6005 }
6006 None => write!(f, "supervisor state was poisoned"),
6007 },
6008 Self::CommandClosed { module_id } => {
6009 write!(
6010 f,
6011 "supervisor command channel for module '{module_id}' is closed"
6012 )
6013 }
6014 Self::SwapInProgress { module_id } => write!(
6015 f,
6016 "module '{module_id}' is being swapped; retry once the swap has cut over or failed, or stop the module to abort the swap"
6017 ),
6018 Self::SwapRefused { module_id, reason } => match reason {
6019 SwapRefusal::OverlapExclusive => write!(
6020 f,
6021 "module '{module_id}' is declared overlap: \"exclusive\" (the default): two processes of it must not run at once, so it cannot be swapped; use a plain restart, or declare overlap: \"safe\" in its config if it really tolerates a second process"
6022 ),
6023 SwapRefusal::NotRegistered => write!(
6024 f,
6025 "module '{module_id}' is not registered, so there is no serving process to keep while a replacement warms; use a plain restart"
6026 ),
6027 SwapRefusal::ProtocolNone => write!(
6028 f,
6029 "module '{module_id}' is protocol: \"none\" and never registers, so a swap could never see its replacement become ready; use a plain restart"
6030 ),
6031 SwapRefusal::NotConfigured => write!(
6032 f,
6033 "module '{module_id}' cannot be swapped: the supervisor was built without the forwarding table or shared handle a swap needs"
6034 ),
6035 SwapRefusal::AlreadySwapping => {
6036 write!(f, "module '{module_id}' is already being swapped")
6037 }
6038 },
6039 Self::SwapFailed {
6040 module_id,
6041 arm,
6042 detail,
6043 ..
6044 } => write!(
6045 f,
6046 "swap of module '{module_id}' failed ({}): {detail}; the running process was left serving",
6047 arm.as_str()
6048 ),
6049 }
6050 }
6051}
6052
6053impl Error for SuperviseError {
6054 fn source(&self) -> Option<&(dyn Error + 'static)> {
6055 match self {
6056 Self::Spawn { source, .. }
6057 | Self::Cgroup { source, .. }
6058 | Self::Wait { source, .. }
6059 | Self::Kill { source, .. } => Some(source),
6060 Self::Forwarding(err) => Some(err),
6061 Self::Registry(err) => Some(err),
6062 Self::LaunchNonce { .. }
6063 | Self::ConfigDeliveryDeferred { .. }
6064 | Self::ConfigHeld { .. }
6065 | Self::InvalidSpec { .. }
6066 | Self::ReloadUnavailable { .. }
6067 | Self::Disabled { .. }
6068 | Self::ReloadFailed { .. }
6069 | Self::RegistrationStillActive { .. }
6070 | Self::StatePoisoned { .. }
6071 | Self::CommandClosed { .. }
6072 | Self::SwapInProgress { .. }
6073 | Self::SwapRefused { .. }
6074 | Self::SwapFailed { .. } => None,
6075 }
6076 }
6077}
6078
6079pub(crate) fn validate_spec(spec: &ModuleSpec) -> Result<(), SuperviseError> {
6080 if spec.carry_containment && spec.overlap == ModuleOverlap::Safe {
6081 return Err(SuperviseError::InvalidSpec {
6082 reason: "carry_containment cannot be combined with overlap: safe".to_owned(),
6083 });
6084 }
6085 if spec.module_id.trim().is_empty() {
6086 return Err(SuperviseError::InvalidSpec {
6087 reason: "module_id must not be empty".to_string(),
6088 });
6089 }
6090
6091 Ok(())
6092}
6093
6094#[derive(Debug, Default)]
6095struct HealthProbeRuntime {
6096 spawn_generation: Option<u64>,
6097 configured_health: Option<HealthConfig>,
6098 registered_connection: Option<crate::ConnectionId>,
6099 advertised: bool,
6100 next_probe_at: Option<Instant>,
6101 probe_index: u64,
6102}
6103
6104fn running_protocol(spec: &ModuleSpec, snapshot: &SharedSnapshot) -> ModuleProtocol {
6105 lock_snapshot(snapshot)
6106 .ok()
6107 .and_then(|state| state.spawned_protocol)
6108 .unwrap_or(spec.protocol)
6109}
6110
6111impl HealthProbeRuntime {
6112 fn refresh_registration(
6113 &mut self,
6114 spec: &ModuleSpec,
6115 runtime: &SupervisorRuntimeConfig,
6116 registry: &Registry,
6117 snapshot: &SharedSnapshot,
6118 ) {
6119 let generation = lock_snapshot(snapshot)
6120 .ok()
6121 .map(|state| state.spawn_generation);
6122 if self.spawn_generation != generation {
6123 *self = Self {
6127 spawn_generation: generation,
6128 ..Self::default()
6129 };
6130 }
6131 if self.configured_health.as_ref() != Some(&runtime.health) {
6132 self.configured_health = Some(runtime.health.clone());
6133 self.next_probe_at = None;
6134 self.registered_connection = None;
6135 self.probe_index = 0;
6136 }
6137 if running_protocol(spec, snapshot) == ModuleProtocol::None {
6141 self.registered_connection = None;
6142 self.advertised = runtime.health.http.is_some();
6143 if !self.advertised {
6144 self.next_probe_at = None;
6145 let _ = update_snapshot(snapshot, Some(&spec.module_id), |state| {
6146 let unknown = ModuleHealthStatus::default();
6147 if state.health != unknown {
6148 state.health = unknown;
6149 state.last_probe_evidence = None;
6150 }
6151 });
6152 } else if self.next_probe_at.is_none() {
6153 self.next_probe_at = Some(
6154 Instant::now()
6155 + jittered_health_delay(&spec.module_id, 0, runtime.health.cadence),
6156 );
6157 }
6158 return;
6159 }
6160
6161 let registration = match registry.get_module(&spec.module_id) {
6162 Ok(registration) => registration,
6163 Err(err) => {
6164 warn!(module_id = %spec.module_id, error = %err, "health prober could not read registry");
6165 self.advertised = false;
6166 self.next_probe_at = None;
6167 return;
6168 }
6169 };
6170
6171 let Some(registration) = registration else {
6172 self.registered_connection = None;
6173 self.advertised = false;
6174 self.next_probe_at = None;
6175 return;
6176 };
6177
6178 let advertised = registration
6179 .control_ops
6180 .iter()
6181 .any(|op| op == MODULE_CONTROL_OP_HEALTH_CHECK);
6182 if !advertised {
6183 self.registered_connection = Some(registration.connection_id);
6184 self.advertised = false;
6185 self.next_probe_at = None;
6186 let _ = update_snapshot(snapshot, Some(&spec.module_id), |state| {
6187 if state.health.status != SupervisorHealthStatus::Unknown
6188 || state.health.consecutive_failures != 0
6189 || state.health.last_probe_ms.is_some()
6190 || state.health.detail.is_some()
6191 || state.health.metrics.is_some()
6192 {
6193 state.health.status = SupervisorHealthStatus::Unknown;
6194 state.last_probe_evidence = None;
6195 state.health.consecutive_failures = 0;
6196 state.health.last_probe_ms = None;
6197 state.health.detail = None;
6198 state.health.metrics = None;
6199 }
6200 });
6201 return;
6202 }
6203
6204 let reregistered = self.registered_connection != Some(registration.connection_id);
6205 self.registered_connection = Some(registration.connection_id);
6206 self.advertised = true;
6207 if reregistered || self.next_probe_at.is_none() {
6208 self.probe_index = 0;
6209 self.next_probe_at = Some(
6210 Instant::now() + jittered_health_delay(&spec.module_id, 0, runtime.health.cadence),
6211 );
6212 let _ = update_snapshot(snapshot, Some(&spec.module_id), |state| {
6213 state.health.status = SupervisorHealthStatus::Unknown;
6214 state.last_probe_evidence = None;
6215 state.health.consecutive_failures = 0;
6216 state.health.detail = None;
6217 state.health.metrics = None;
6218 });
6219 }
6220 }
6221
6222 fn wake_at(&self) -> Option<Instant> {
6223 if !self.advertised {
6224 return None;
6225 }
6226 self.next_probe_at
6227 }
6228
6229 fn due(&self) -> bool {
6230 self.advertised
6231 && self
6232 .next_probe_at
6233 .is_some_and(|next| Instant::now() >= next)
6234 }
6235
6236 fn schedule_next(&mut self, spec: &ModuleSpec, cadence: Duration) {
6237 self.probe_index = self.probe_index.wrapping_add(1);
6238 self.next_probe_at = Some(
6239 Instant::now() + jittered_health_delay(&spec.module_id, self.probe_index, cadence),
6240 );
6241 }
6242}
6243
6244#[derive(Debug, Clone, Copy, PartialEq, Eq)]
6248enum HealthProbeEvidence {
6249 LaneDead,
6251 NoAnswer,
6253 BadAnswer,
6255 Misconfigured,
6257}
6258
6259#[derive(Debug)]
6260struct HealthProbeError {
6261 evidence: HealthProbeEvidence,
6262 message: String,
6263}
6264
6265impl HealthProbeError {
6266 fn lane_dead(message: impl Into<String>) -> Self {
6267 Self::with(HealthProbeEvidence::LaneDead, message)
6268 }
6269
6270 fn no_answer(message: impl Into<String>) -> Self {
6271 Self::with(HealthProbeEvidence::NoAnswer, message)
6272 }
6273
6274 fn bad_answer(message: impl Into<String>) -> Self {
6275 Self::with(HealthProbeEvidence::BadAnswer, message)
6276 }
6277
6278 fn misconfigured(message: impl Into<String>) -> Self {
6279 Self::with(HealthProbeEvidence::Misconfigured, message)
6280 }
6281
6282 fn with(evidence: HealthProbeEvidence, message: impl Into<String>) -> Self {
6283 Self {
6284 evidence,
6285 message: message.into(),
6286 }
6287 }
6288
6289 #[allow(dead_code)]
6298 fn is_proof_of_death(&self) -> bool {
6299 matches!(self.evidence, HealthProbeEvidence::LaneDead)
6300 }
6301
6302 fn label(&self) -> &'static str {
6310 match self.evidence {
6311 HealthProbeEvidence::LaneDead => "lane-dead",
6312 HealthProbeEvidence::NoAnswer => "no-answer",
6313 HealthProbeEvidence::BadAnswer => "bad-answer",
6314 HealthProbeEvidence::Misconfigured => "daemon-misconfigured",
6315 }
6316 }
6317}
6318
6319impl fmt::Display for HealthProbeError {
6320 fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
6321 f.write_str(&self.message)
6322 }
6323}
6324
6325async fn run_health_probe_cycle(
6326 spec: &ModuleSpec,
6327 runtime: &SupervisorRuntimeConfig,
6328 registry: &Registry,
6329 process_liveness: &SupervisorProcessLiveness,
6330 snapshot: &SharedSnapshot,
6331 child: &mut Option<SupervisedChild>,
6332) {
6333 let now_ms = unix_ms_now();
6334 let delivery_probe = match &runtime.config_delivery {
6335 Some(delivery) => {
6336 delivery
6337 .health_probe(&spec.module_id, runtime.forwarding.as_deref(), registry)
6338 .await
6339 }
6340 None => None,
6341 };
6342 let http = (running_protocol(spec, snapshot) == ModuleProtocol::None)
6343 .then_some(runtime.health.http.as_deref())
6344 .flatten();
6345 let result = match http {
6346 Some(url) => probe_http_health(url, runtime.health.deadline)
6347 .await
6348 .map(|report| (report, None)),
6349 None => probe_module_health_from(&spec.module_id, runtime, None)
6350 .await
6351 .map(|(report, probed)| (report, Some(probed))),
6352 };
6353 match result {
6354 Ok((report, probed)) => {
6355 handle_health_report(
6356 spec,
6357 runtime,
6358 registry,
6359 process_liveness,
6360 snapshot,
6361 child,
6362 report,
6363 now_ms,
6364 delivery_probe,
6365 probed,
6366 )
6367 .await;
6368 }
6369 Err(err) => {
6370 if http.is_some() {
6371 let _ = update_snapshot(snapshot, Some(&spec.module_id), |state| {
6372 state.health.status = SupervisorHealthStatus::Failing;
6373 });
6374 }
6375 handle_health_probe_failure(
6376 spec,
6377 runtime,
6378 registry,
6379 process_liveness,
6380 snapshot,
6381 child,
6382 err,
6383 now_ms,
6384 delivery_probe,
6385 )
6386 .await;
6387 }
6388 }
6389}
6390
6391pub(crate) struct HttpProbeTarget<'a> {
6392 address: std::net::SocketAddr,
6393 localhost: bool,
6394 authority: &'a str,
6395 path: String,
6396}
6397
6398pub(crate) fn parse_http_probe_url(url: &str) -> Result<HttpProbeTarget<'_>, String> {
6401 if url.bytes().any(|b| b <= b' ' || b == 127) || url.contains('#') {
6402 return Err("must not contain whitespace, controls, or a fragment".into());
6403 }
6404 let rest = url
6405 .strip_prefix("http://")
6406 .ok_or("must use plain http://")?;
6407 let split = rest.find(['/', '?']).unwrap_or(rest.len());
6408 let (authority, suffix) = rest.split_at(split);
6409 let (host, port) = if let Some(rest) = authority.strip_prefix("[::1]") {
6410 ("::1", rest)
6411 } else {
6412 let split = authority.find(':').unwrap_or(authority.len());
6413 authority.split_at(split)
6414 };
6415 let ip: std::net::IpAddr = match host {
6416 "127.0.0.1" | "localhost" => std::net::Ipv4Addr::LOCALHOST.into(),
6417 "::1" => std::net::Ipv6Addr::LOCALHOST.into(),
6418 _ => return Err("host must be 127.0.0.1, [::1], or localhost".into()),
6419 };
6420 let port = if port.is_empty() {
6421 80
6422 } else {
6423 port.strip_prefix(':')
6424 .and_then(|p| p.parse::<u16>().ok())
6425 .filter(|p| *p > 0)
6426 .ok_or("must have a valid nonzero TCP port")?
6427 };
6428 let path = if suffix.is_empty() {
6429 "/".into()
6430 } else if suffix.starts_with('?') {
6431 format!("/{suffix}")
6432 } else {
6433 suffix.into()
6434 };
6435 Ok(HttpProbeTarget {
6436 address: std::net::SocketAddr::new(ip, port),
6437 localhost: host == "localhost",
6438 authority,
6439 path,
6440 })
6441}
6442
6443async fn probe_http_health(
6444 url: &str,
6445 deadline: Duration,
6446) -> Result<HealthReport, HealthProbeError> {
6447 use tokio::io::{AsyncReadExt, AsyncWriteExt, BufReader};
6448 let target = parse_http_probe_url(url).map_err(HealthProbeError::misconfigured)?;
6449 let mut response_status = String::new();
6452 let mut body = Vec::new();
6453 let probe = async {
6454 let connection = match tokio::net::TcpStream::connect(target.address).await {
6457 Err(_) if target.localhost => {
6458 tokio::net::TcpStream::connect((
6459 std::net::Ipv6Addr::LOCALHOST,
6460 target.address.port(),
6461 ))
6462 .await
6463 }
6464 result => result,
6465 };
6466 let mut stream = connection.map_err(|error| {
6467 HealthProbeError::no_answer(format!("HTTP connect failed: {error}"))
6468 })?;
6469 stream.write_all(format!("GET {} HTTP/1.1\r\nHost: {}\r\nAccept-Encoding: identity\r\nConnection: close\r\n\r\n", target.path, target.authority).as_bytes()).await
6470 .map_err(|error| HealthProbeError::no_answer(format!("HTTP write failed: {error}")))?;
6471 let mut reader = BufReader::new(stream);
6472 let mut budget = 16 * 1024;
6473 let status = http_line(&mut reader, &mut budget).await?;
6474 let mut words = status.split_ascii_whitespace();
6475 let version = words.next();
6476 let code = words
6477 .next()
6478 .filter(|word| word.len() == 3)
6479 .and_then(|word| word.parse::<u16>().ok());
6480 if !matches!(version, Some("HTTP/1.1" | "HTTP/1.0"))
6481 || !code.is_some_and(|code| (100..600).contains(&code))
6482 {
6483 return Err(HealthProbeError::bad_answer(format!(
6484 "invalid HTTP status: {status}"
6485 )));
6486 }
6487 let code = code.expect("validated status code");
6488 response_status = status.clone();
6489 let mut length = None;
6490 let mut chunked = false;
6491 loop {
6492 let line = http_line(&mut reader, &mut budget).await?;
6493 if line.is_empty() {
6494 break;
6495 }
6496 if let Some((name, value)) = line.split_once(':') {
6497 if name.eq_ignore_ascii_case("content-length") {
6498 length = Some(value.trim().parse::<u64>().map_err(|_| {
6499 HealthProbeError::bad_answer("invalid HTTP Content-Length")
6500 })?);
6501 } else if name.eq_ignore_ascii_case("transfer-encoding") {
6502 chunked = value.trim().eq_ignore_ascii_case("chunked");
6503 }
6504 }
6505 }
6506 if chunked {
6507 while body.len() < 200 {
6508 let line = http_line(&mut reader, &mut budget).await?;
6509 let size = u64::from_str_radix(line.split(';').next().unwrap_or("").trim(), 16)
6510 .map_err(|_| HealthProbeError::bad_answer("invalid HTTP chunk size"))?;
6511 if size == 0 {
6512 break;
6513 }
6514 let count = size.min((200 - body.len()) as u64) as usize;
6515 let start = body.len();
6516 (&mut reader)
6517 .take(count as u64)
6518 .read_to_end(&mut body)
6519 .await
6520 .map_err(|error| {
6521 HealthProbeError::no_answer(format!("HTTP body read failed: {error}"))
6522 })?;
6523 if body.len() - start != count {
6524 return Err(HealthProbeError::bad_answer("truncated HTTP chunk"));
6525 }
6526 if size > count as u64 || body.len() == 200 {
6527 break;
6528 }
6529 if !http_line(&mut reader, &mut budget).await?.is_empty() {
6530 return Err(HealthProbeError::bad_answer("invalid HTTP chunk delimiter"));
6531 }
6532 }
6533 } else {
6534 reader
6535 .take(length.unwrap_or(200).min(200))
6536 .read_to_end(&mut body)
6537 .await
6538 .map_err(|error| {
6539 HealthProbeError::no_answer(format!("HTTP body read failed: {error}"))
6540 })?;
6541 }
6542 if (200..300).contains(&code) {
6543 Ok(HealthReport::ok())
6544 } else {
6545 Err(HealthProbeError::bad_answer(
6546 "HTTP health endpoint returned non-2xx",
6547 ))
6548 }
6549 };
6550 let mut result = timeout(deadline, probe).await.unwrap_or_else(|_| {
6551 Err(HealthProbeError::no_answer(format!(
6552 "HTTP probe timed out after {deadline:?}"
6553 )))
6554 });
6555 if let Err(error) = &mut result {
6556 if !response_status.is_empty() {
6557 error.message = format!(
6558 "{}; {response_status}: {}",
6559 error.message,
6560 String::from_utf8_lossy(&body)
6561 );
6562 }
6563 }
6564 result
6565}
6566
6567async fn http_line(
6568 reader: &mut tokio::io::BufReader<tokio::net::TcpStream>,
6569 remaining: &mut usize,
6570) -> Result<String, HealthProbeError> {
6571 use tokio::io::{AsyncBufReadExt, AsyncReadExt};
6572 let mut line = Vec::new();
6573 (&mut *reader)
6574 .take(*remaining as u64)
6575 .read_until(b'\n', &mut line)
6576 .await
6577 .map_err(|error| {
6578 HealthProbeError::no_answer(format!("HTTP header read failed: {error}"))
6579 })?;
6580 *remaining -= line.len();
6581 if !line.ends_with(b"\r\n") {
6582 return Err(HealthProbeError::bad_answer(
6583 "HTTP headers are incomplete or exceed 16 KiB",
6584 ));
6585 }
6586 line.truncate(line.len() - 2);
6587 String::from_utf8(line).map_err(|_| HealthProbeError::bad_answer("HTTP header is not UTF-8"))
6588}
6589
6590async fn probe_module_health(
6591 module_id: &str,
6592 runtime: &SupervisorRuntimeConfig,
6593 drain_deadline: Option<Instant>,
6594) -> Result<HealthReport, HealthProbeError> {
6595 probe_module_health_from(module_id, runtime, drain_deadline)
6596 .await
6597 .map(|(report, _)| report)
6598}
6599
6600async fn probe_module_health_from(
6603 module_id: &str,
6604 runtime: &SupervisorRuntimeConfig,
6605 drain_deadline: Option<Instant>,
6606) -> Result<(HealthReport, ConnectionId), HealthProbeError> {
6607 let Some(forwarding) = runtime.forwarding.as_ref() else {
6608 return Err(HealthProbeError::misconfigured(
6609 "supervisor was not configured with a forwarding table",
6610 ));
6611 };
6612 let probe_started_at = Instant::now();
6613 let mut deadline = probe_started_at + runtime.health.deadline;
6614 if let Some(drain_deadline) = drain_deadline {
6615 deadline = deadline.min(drain_deadline);
6616 }
6617 let pending = if drain_deadline.is_some() {
6618 forwarding.begin_drain_health_probe_rpc_for(
6619 module_id,
6620 MODULE_CONTROL_OP_HEALTH_CHECK,
6621 probe_started_at,
6622 deadline,
6623 )
6624 } else {
6625 forwarding.begin_health_probe_rpc_for(
6626 module_id,
6627 MODULE_CONTROL_OP_HEALTH_CHECK,
6628 probe_started_at,
6629 deadline,
6630 )
6631 }
6632 .map_err(|err| {
6633 HealthProbeError::lane_dead(format!("failed to begin health.check RPC: {err}"))
6636 })?;
6637 let probed = pending.endpoint.connection_id;
6638 await_health_probe(forwarding, pending, deadline, runtime.health.deadline)
6639 .await
6640 .map(|report| (report, probed))
6641}
6642
6643async fn probe_endpoint_health(
6650 endpoint: crate::ModuleEndpointId,
6651 runtime: &SupervisorRuntimeConfig,
6652 deadline_cap: Option<Instant>,
6653) -> Result<HealthReport, HealthProbeError> {
6654 let Some(forwarding) = runtime.forwarding.as_ref() else {
6655 return Err(HealthProbeError::misconfigured(
6656 "supervisor was not configured with a forwarding table",
6657 ));
6658 };
6659 let probe_started_at = Instant::now();
6660 let mut deadline = probe_started_at + runtime.health.deadline;
6661 if let Some(cap) = deadline_cap {
6662 deadline = deadline.min(cap);
6663 }
6664 let pending = forwarding
6665 .begin_endpoint_health_probe_rpc_for(
6666 endpoint,
6667 MODULE_CONTROL_OP_HEALTH_CHECK,
6668 probe_started_at,
6669 deadline,
6670 )
6671 .map_err(|err| {
6672 HealthProbeError::lane_dead(format!("failed to begin health.check RPC: {err}"))
6673 })?;
6674 await_health_probe(forwarding, pending, deadline, runtime.health.deadline).await
6675}
6676
6677async fn await_health_probe(
6679 forwarding: &ForwardingTable,
6680 pending: PendingModuleControlRpc,
6681 deadline: Instant,
6682 probe_budget: Duration,
6683) -> Result<HealthReport, HealthProbeError> {
6684 let PendingModuleControlRpc {
6685 endpoint,
6686 module_sink,
6687 negotiated_ver,
6688 corr,
6689 receiver,
6690 } = pending;
6691 let body = serde_json::to_vec(&ModuleControlRequest::HealthCheck {}).map_err(|err| {
6692 HealthProbeError::misconfigured(format!("failed to encode health.check: {err}"))
6693 })?;
6694 let frame = Frame::build_with_version(
6695 negotiated_ver,
6696 FrameType::Request,
6697 control_flags(),
6698 0,
6699 0,
6700 corr,
6701 body,
6702 )
6703 .map_err(|err| {
6704 HealthProbeError::misconfigured(format!("failed to build health.check frame: {err}"))
6705 })?;
6706
6707 match timeout_at(deadline, module_sink.send(frame)).await {
6713 Ok(Ok(())) => {}
6714 Ok(Err(err)) => {
6715 let _ = forwarding.cancel_module_control_rpc(endpoint, corr);
6716 return Err(HealthProbeError::lane_dead(format!(
6719 "failed to send health.check: {err}"
6720 )));
6721 }
6722 Err(_elapsed) => {
6723 let _ = forwarding.cancel_module_control_rpc(endpoint, corr);
6724 return Err(HealthProbeError::no_answer(
6728 "health.check send timed out before enqueue (module egress full)",
6729 ));
6730 }
6731 }
6732
6733 match timeout_at(deadline, receiver).await {
6734 Ok(Ok(ModuleControlRpcOutcome::RawResponse(_))) => Err(HealthProbeError::bad_answer(
6735 "health.check returned an unexpected config response",
6736 )),
6737 Ok(Ok(ModuleControlRpcOutcome::Response(response))) => {
6741 response.health_report().ok_or_else(|| {
6742 HealthProbeError::bad_answer("health.check RPC returned a non-health response")
6743 })
6744 }
6745 Ok(Ok(ModuleControlRpcOutcome::Rejected(body))) => Err(HealthProbeError::bad_answer(
6746 format!("health.check rejected: {}", body.message),
6747 )),
6748 Ok(Ok(ModuleControlRpcOutcome::ModuleGone(message))) => {
6749 Err(HealthProbeError::lane_dead(message))
6750 }
6751 Ok(Ok(ModuleControlRpcOutcome::MalformedResponse(message))) => {
6752 Err(HealthProbeError::bad_answer(message))
6753 }
6754 Ok(Ok(ModuleControlRpcOutcome::UnexpectedOp { expected, actual })) => {
6755 Err(HealthProbeError::bad_answer(format!(
6756 "expected module-control op '{expected}', got '{actual}'"
6757 )))
6758 }
6759 Ok(Ok(ModuleControlRpcOutcome::DeadlineElapsed)) => Err(HealthProbeError::bad_answer(
6763 "module answered health.check after its daemon deadline",
6764 )),
6765 Ok(Err(_)) => Err(HealthProbeError::misconfigured(
6766 "health.check waiter was canceled before the module responded",
6767 )),
6768 Err(_) => {
6769 let _ = forwarding.tombstone_health_probe_rpc(endpoint, corr);
6770 Err(HealthProbeError::no_answer(format!(
6771 "module did not answer health.check within {probe_budget:?}"
6772 )))
6773 }
6774 }
6775}
6776
6777#[allow(clippy::too_many_arguments)]
6781async fn handle_health_report(
6782 spec: &ModuleSpec,
6783 runtime: &SupervisorRuntimeConfig,
6784 registry: &Registry,
6785 process_liveness: &SupervisorProcessLiveness,
6786 snapshot: &SharedSnapshot,
6787 child: &mut Option<SupervisedChild>,
6788 report: HealthReport,
6789 now_ms: u64,
6790 delivery_probe: Option<crate::config_gate::runtime::DeliveryProbe>,
6791 probed: Option<ConnectionId>,
6792) {
6793 let delivery_failed = match (&runtime.config_delivery, delivery_probe) {
6794 (Some(delivery), Some(probe)) => {
6795 delivery
6796 .health_result(
6797 &spec.module_id,
6798 runtime.forwarding.as_deref(),
6799 registry,
6800 probe,
6801 report.status == HealthStatus::Ok,
6802 )
6803 .await
6804 }
6805 _ => false,
6806 };
6807 if let (Some(delivery), Some(spawn)) = (
6808 &runtime.config_delivery,
6809 child
6810 .as_ref()
6811 .and_then(|child| child.config_delivery.as_ref()),
6812 ) {
6813 delivery
6814 .mark_readiness(
6815 &spec.module_id,
6816 runtime.forwarding.as_deref(),
6817 probed,
6818 spawn,
6819 report.status == HealthStatus::Ok,
6820 )
6821 .await;
6822 }
6823 let status = supervisor_health_status(report.status);
6824 let detail = report.detail.clone();
6825 let metrics = truncate_health_metrics(report.metrics);
6826 let _ = update_snapshot(snapshot, Some(&spec.module_id), |state| {
6827 state.health.status = status;
6828 state.last_probe_evidence = None;
6829 state.health.last_probe_ms = Some(now_ms);
6830 state.health.detail = detail.clone();
6831 state.health.metrics = metrics.clone();
6832 state.health.consecutive_failures = 0;
6833 state.failed_retry.observe_health(
6834 report.status == HealthStatus::Ok,
6835 runtime.restart_policy.window,
6836 );
6837 });
6838
6839 if delivery_failed {
6840 return;
6841 }
6842 let action = match report.status {
6843 HealthStatus::Ok => return,
6844 HealthStatus::Degraded => runtime.health.on_degraded,
6845 HealthStatus::Failing => runtime.health.on_failing,
6846 };
6847 apply_l3_health_action(
6848 spec,
6849 runtime,
6850 registry,
6851 process_liveness,
6852 snapshot,
6853 child,
6854 status,
6855 detail.as_deref(),
6856 action,
6857 now_ms,
6858 )
6859 .await;
6860}
6861
6862#[allow(clippy::too_many_arguments)]
6863async fn handle_health_probe_failure(
6864 spec: &ModuleSpec,
6865 runtime: &SupervisorRuntimeConfig,
6866 registry: &Registry,
6867 process_liveness: &SupervisorProcessLiveness,
6868 snapshot: &SharedSnapshot,
6869 child: &mut Option<SupervisedChild>,
6870 err: HealthProbeError,
6871 now_ms: u64,
6872 delivery_probe: Option<crate::config_gate::runtime::DeliveryProbe>,
6873) {
6874 let delivery_failed = match (&runtime.config_delivery, delivery_probe) {
6875 (Some(delivery), Some(probe)) => {
6876 delivery
6877 .health_result(
6878 &spec.module_id,
6879 runtime.forwarding.as_deref(),
6880 registry,
6881 probe,
6882 false,
6883 )
6884 .await
6885 }
6886 _ => false,
6887 };
6888 let threshold = runtime.health.failure_threshold.max(1);
6889 let held = if err.evidence == HealthProbeEvidence::NoAnswer {
6890 runtime.host_health.record_late_module(&spec.module_id)
6891 } else {
6892 false
6893 };
6894 let mut failures = 0;
6895 let detail = if held {
6900 format!("[{}] {err}; host overloaded", err.label())
6901 } else {
6902 format!("[{}] {err}", err.label())
6903 };
6904 let _ = update_snapshot(snapshot, Some(&spec.module_id), |state| {
6905 state.last_probe_evidence = Some(err.evidence);
6906 state
6907 .failed_retry
6908 .observe_health(false, runtime.restart_policy.window);
6909 if held {
6912 state.health.status = SupervisorHealthStatus::Degraded;
6913 } else if state.spawned_protocol.unwrap_or(spec.protocol) == ModuleProtocol::Subc {
6914 state.health.status = SupervisorHealthStatus::Unknown;
6915 }
6916 state.health.last_probe_ms = Some(now_ms);
6917 state.health.consecutive_failures = if held {
6920 0
6921 } else {
6922 state.health.consecutive_failures.saturating_add(1)
6923 };
6924 state.health.detail = Some(detail.clone());
6925 state.health.metrics = None;
6926 failures = state.health.consecutive_failures;
6927 });
6928
6929 if delivery_failed {
6930 return;
6931 }
6932 if held || failures < threshold {
6933 warn!(
6934 module_id = %spec.module_id,
6935 consecutive_failures = failures,
6936 threshold,
6937 evidence = err.label(),
6938 detail = %detail,
6939 "health.check probe failed"
6940 );
6941 return;
6942 }
6943
6944 let _ = update_snapshot(snapshot, Some(&spec.module_id), |state| {
6945 state.state = ModuleState::Unresponsive;
6946 state.health.status = SupervisorHealthStatus::Unresponsive;
6947 });
6948 if runtime.health.critical {
6952 error!(
6953 module_id = %spec.module_id,
6954 status = "unresponsive",
6955 evidence = err.label(),
6956 detail = %detail,
6957 "critical module health alert"
6958 );
6959 } else {
6960 warn!(
6961 module_id = %spec.module_id,
6962 status = "unresponsive",
6963 evidence = err.label(),
6964 detail = %detail,
6965 "module health threshold breached"
6966 );
6967 }
6968 if let Err(err) = health_restart_child(
6969 spec,
6970 runtime,
6971 registry,
6972 process_liveness,
6973 snapshot,
6974 child,
6975 SupervisorHealthStatus::Unresponsive,
6976 Some(&detail),
6977 now_ms,
6978 )
6979 .await
6980 {
6981 error!(module_id = %spec.module_id, error = %err, "health-triggered restart failed");
6982 }
6983}
6984
6985#[allow(clippy::too_many_arguments)]
6986async fn apply_l3_health_action(
6987 spec: &ModuleSpec,
6988 runtime: &SupervisorRuntimeConfig,
6989 registry: &Registry,
6990 process_liveness: &SupervisorProcessLiveness,
6991 snapshot: &SharedSnapshot,
6992 child: &mut Option<SupervisedChild>,
6993 status: SupervisorHealthStatus,
6994 detail: Option<&str>,
6995 action: HealthAction,
6996 now_ms: u64,
6997) {
6998 record_health_action(snapshot, &spec.module_id, action.to_string(), now_ms);
6999 match action {
7000 HealthAction::Report => {
7001 info!(
7002 module_id = %spec.module_id,
7003 status = ?status,
7004 detail,
7005 "module reported non-ok health"
7006 );
7007 }
7008 HealthAction::Alert => {
7009 error!(
7010 module_id = %spec.module_id,
7011 status = ?status,
7012 detail,
7013 "module health alert"
7014 );
7015 }
7016 HealthAction::Restart => {
7017 if let Err(err) = health_restart_child(
7018 spec,
7019 runtime,
7020 registry,
7021 process_liveness,
7022 snapshot,
7023 child,
7024 status,
7025 detail,
7026 now_ms,
7027 )
7028 .await
7029 {
7030 error!(module_id = %spec.module_id, error = %err, "health-triggered restart failed");
7031 }
7032 }
7033 }
7034}
7035
7036#[allow(clippy::too_many_arguments)]
7037async fn health_restart_child(
7038 spec: &ModuleSpec,
7039 runtime: &SupervisorRuntimeConfig,
7040 registry: &Registry,
7041 process_liveness: &SupervisorProcessLiveness,
7042 snapshot: &SharedSnapshot,
7043 child: &mut Option<SupervisedChild>,
7044 status: SupervisorHealthStatus,
7045 detail: Option<&str>,
7046 now_ms: u64,
7047) -> Result<(), SuperviseError> {
7048 budgeted_restart_child(
7049 spec,
7050 runtime,
7051 registry,
7052 process_liveness,
7053 snapshot,
7054 child,
7055 status,
7056 detail,
7057 now_ms,
7058 true,
7059 )
7060 .await
7061}
7062
7063#[allow(clippy::too_many_arguments)]
7064async fn budgeted_restart_child(
7065 spec: &ModuleSpec,
7066 runtime: &SupervisorRuntimeConfig,
7067 registry: &Registry,
7068 process_liveness: &SupervisorProcessLiveness,
7069 snapshot: &SharedSnapshot,
7070 child: &mut Option<SupervisedChild>,
7071 status: SupervisorHealthStatus,
7072 detail: Option<&str>,
7073 now_ms: u64,
7074 health_failure: bool,
7075) -> Result<(), SuperviseError> {
7076 if let Some(error) = runtime
7077 .config_delivery
7078 .as_ref()
7079 .and_then(|delivery| delivery.0.check_module_hold(&spec.module_id).err())
7080 .filter(|error| error.code != "config_delivery_deferred")
7081 {
7082 return Err(config_delivery::gate_hold_error(
7083 spec,
7084 error,
7085 runtime.config_delivery.as_ref(),
7086 ));
7087 }
7088 let recovery_reason = if health_failure {
7089 "health_check"
7090 } else {
7091 "configuration_delivery"
7092 };
7093 let (enabled, schedule) = {
7094 let mut state = lock_snapshot(snapshot)?;
7095 let enabled = state.enabled;
7096 let schedule = if enabled {
7097 state.next_crash_restart(&runtime.restart_policy, Instant::now())
7098 } else {
7099 None
7100 };
7101 (enabled, schedule)
7102 };
7103
7104 if !enabled {
7105 return Err(SuperviseError::Disabled {
7106 module_id: spec.module_id.clone(),
7107 });
7108 }
7109
7110 if schedule.is_none() {
7111 if health_failure {
7112 record_health_action(snapshot, &spec.module_id, "failed".to_string(), now_ms);
7113 }
7114 error!(
7115 module_id = %spec.module_id,
7116 status = ?status,
7117 detail,
7118 max_restarts = runtime.restart_policy.max_restarts,
7119 window_secs = runtime.restart_policy.window.as_secs(),
7120 reason = %runtime.restart_policy.budget_exhausted_detail(),
7121 recovery_reason,
7122 "automatic restart budget exhausted; marking module failed"
7123 );
7124 let stop_notice = begin_forwarding_drain_if_configured(
7125 spec,
7126 runtime,
7127 registry,
7128 snapshot,
7129 Some(true),
7130 RouteCloseReason::Disable,
7131 )
7132 .await?;
7133 update_snapshot(snapshot, Some(&spec.module_id), |state| {
7134 state.drain_disposition_detail = Some(runtime.restart_policy.budget_exhausted_detail());
7135 })?;
7136 let result = drain_optional_child(
7137 &spec.module_id,
7138 spec.protocol,
7139 stop_notice,
7140 registry,
7141 runtime.forwarding.as_deref(),
7142 snapshot,
7143 &runtime.terminal_ring,
7144 &runtime.spawn_events,
7145 child,
7146 runtime.drain_timeout,
7147 ModuleState::Failed,
7148 Some(true),
7149 &runtime.failure_context(true),
7150 )
7151 .await;
7152 update_snapshot(snapshot, Some(&spec.module_id), |state| {
7153 if state.state == ModuleState::Failed && !runtime.child_roster.is_closed() {
7154 state.failed_retry.schedule(&spec.module_id);
7155 }
7156 })?;
7157 result?;
7158 process_liveness.untrack_if_current(&spec.module_id, snapshot);
7159 return Ok(());
7160 }
7161
7162 let schedule = schedule.expect("a health restart must have a crash-restart schedule");
7163 let mut restart_count = 0;
7164 update_snapshot(snapshot, Some(&spec.module_id), |state| {
7165 restart_count = state.crash_restarts.len();
7166 state.state = ModuleState::Unresponsive;
7167 if health_failure {
7168 state.health.status = status;
7169 state.health.last_action = Some(HealthAction::Restart.to_string());
7170 state.health.last_action_ms = Some(now_ms);
7171 }
7172 })?;
7173 warn!(
7174 module_id = %spec.module_id,
7175 status = ?status,
7176 detail,
7177 restart_count,
7178 restart_in_window = schedule.restart_in_window,
7179 delay_ms = schedule.delay.as_millis() as u64,
7180 recovery_reason,
7181 "automatic module restart"
7182 );
7183
7184 let stop_notice = begin_forwarding_drain_if_configured(
7185 spec,
7186 runtime,
7187 registry,
7188 snapshot,
7189 Some(true),
7190 RouteCloseReason::Restart,
7191 )
7192 .await?;
7193 drain_optional_child(
7194 &spec.module_id,
7195 spec.protocol,
7196 stop_notice,
7197 registry,
7198 runtime.forwarding.as_deref(),
7199 snapshot,
7200 &runtime.terminal_ring,
7201 &runtime.spawn_events,
7202 child,
7203 runtime.drain_timeout,
7204 ModuleState::Restarting,
7205 Some(true),
7206 &runtime.failure_context(true),
7207 )
7208 .await?;
7209 schedule_respawn(
7210 runtime,
7211 snapshot,
7212 &spec.module_id,
7213 schedule.delay,
7214 RespawnKind::Spawn,
7215 true,
7216 )
7217}
7218
7219fn cancel_deferred_reload(runtime: &SupervisorRuntimeConfig, module_id: &str, reason: &str) {
7220 if let Some(reply) = runtime
7221 .deferred_reload_reply
7222 .lock()
7223 .unwrap_or_else(|p| p.into_inner())
7224 .take()
7225 {
7226 let _ = reply.send(Err(SuperviseError::ReloadFailed {
7227 module_id: module_id.to_string(),
7228 reason: reason.to_string(),
7229 }));
7230 }
7231}
7232
7233fn park_config_spawn(
7234 spec: &ModuleSpec,
7235 runtime: &SupervisorRuntimeConfig,
7236 snapshot: &SharedSnapshot,
7237 failure: &FailureContext,
7238 admission_refused: Option<String>,
7239) -> Result<(), SuperviseError> {
7240 *runtime
7241 .scheduled_respawn
7242 .lock()
7243 .unwrap_or_else(|poisoned| poisoned.into_inner()) = None;
7244 if let Some(reply) = runtime
7245 .deferred_reload_reply
7246 .lock()
7247 .unwrap_or_else(|poisoned| poisoned.into_inner())
7248 .take()
7249 {
7250 let _ = reply.send(Err(SuperviseError::ConfigHeld {
7251 module_id: spec.module_id.clone(),
7252 code: "config_gated".into(),
7253 detail: "module is held by configuration admission".into(),
7254 }));
7255 }
7256 update_snapshot(snapshot, Some(&spec.module_id), |state| {
7257 state.admission_refused = admission_refused;
7258 state.config_hold_daemon_initiated = failure.daemon_initiated;
7259 state.config_hold_approval = failure.approval.clone();
7260 state.config_hold_parked = true;
7261 state.state = ModuleState::Stopped;
7262 state.respawn_pending = false;
7263 state.failed_retry.pending = None;
7264 state.failed_retry.failed_at = None;
7267 clear_current_process_facts(state);
7268 })
7269}
7270
7271fn try_park_config_attempt(
7272 spec: &ModuleSpec,
7273 runtime: &SupervisorRuntimeConfig,
7274 snapshot: &SharedSnapshot,
7275 failure: &FailureContext,
7276 program_base: Option<&subc_config::Digest>,
7277) -> Result<bool, SuperviseError> {
7278 let result = (|| {
7279 if !lock_snapshot(snapshot)?.enabled {
7280 return Ok(false);
7281 }
7282 #[cfg(test)]
7283 if let Some(delivery) = &runtime.config_delivery {
7284 delivery.0.pause_before_config_park_for_test();
7285 }
7286 let Some(delivery) = &runtime.config_delivery else {
7287 return Ok(false);
7288 };
7289 let parked = delivery
7290 .0
7291 .with_recorded_admission_hold(&spec.module_id, program_base, |refusal| {
7292 park_config_spawn(spec, runtime, snapshot, failure, refusal)
7293 })
7294 .map_err(|error| config_delivery::gate_hold_error(spec, error, Some(delivery)))?;
7295 if let Some(result) = parked {
7296 result?;
7297 Ok(true)
7298 } else {
7299 Ok(false)
7300 }
7301 })();
7302 if let Err(error) = &result {
7303 fail_snapshot(snapshot, Some(&spec.module_id), None, failure);
7307 record_approved_attempt_failure(spec, snapshot, failure, error);
7308 }
7309 result
7310}
7311
7312fn record_approved_attempt_failure(
7313 spec: &ModuleSpec,
7314 snapshot: &SharedSnapshot,
7315 failure: &FailureContext,
7316 error: &SuperviseError,
7317) {
7318 let Some(approval) = &failure.approval else {
7319 return;
7320 };
7321 let _ = update_snapshot(snapshot, Some(&spec.module_id), |state| {
7322 if state.enabled
7323 && state.state == ModuleState::Failed
7324 && state
7325 .approved_start_failure
7326 .as_ref()
7327 .is_none_or(|(identity, _)| identity != approval)
7328 {
7329 state.approved_start_failure = Some((approval.clone(), error.to_string()));
7330 }
7331 });
7332}
7333
7334fn hold_config_spawn(
7335 spec: &ModuleSpec,
7336 runtime: &SupervisorRuntimeConfig,
7337 snapshot: &SharedSnapshot,
7338 failure: &FailureContext,
7339) -> Result<(), SuperviseError> {
7340 if !lock_snapshot(snapshot)?.enabled {
7341 return Ok(());
7342 }
7343 if !try_park_config_attempt(spec, runtime, snapshot, failure, None)? {
7344 defer_config_spawn(spec, runtime, snapshot, RespawnKind::Spawn, failure)?;
7347 }
7348 Ok(())
7349}
7350
7351fn schedule_respawn(
7352 runtime: &SupervisorRuntimeConfig,
7353 snapshot: &SharedSnapshot,
7354 module_id: &str,
7355 delay: Duration,
7356 kind: RespawnKind,
7357 daemon_initiated: bool,
7358) -> Result<(), SuperviseError> {
7359 cancel_deferred_reload(runtime, module_id, "respawn superseded by another restart");
7360 update_snapshot(snapshot, Some(module_id), |state| {
7361 state.respawn_pending = true
7362 })?;
7363 *runtime
7364 .scheduled_respawn
7365 .lock()
7366 .unwrap_or_else(|p| p.into_inner()) = Some(PendingRespawn {
7367 deadline: Instant::now() + delay,
7368 kind,
7369 daemon_initiated,
7370 approval: None,
7371 });
7372 Ok(())
7373}
7374
7375const CONFIG_ADMISSION_RETRY_DELAY: Duration = Duration::from_millis(100);
7376
7377fn defer_config_spawn(
7378 spec: &ModuleSpec,
7379 runtime: &SupervisorRuntimeConfig,
7380 snapshot: &SharedSnapshot,
7381 kind: RespawnKind,
7382 failure: &FailureContext,
7383) -> Result<(), SuperviseError> {
7384 update_snapshot(snapshot, Some(&spec.module_id), |state| {
7385 state.state = ModuleState::Restarting;
7386 state.respawn_pending = true;
7387 clear_current_process_facts(state);
7388 })?;
7389 *runtime
7390 .scheduled_respawn
7391 .lock()
7392 .unwrap_or_else(|p| p.into_inner()) = Some(PendingRespawn {
7393 deadline: Instant::now() + CONFIG_ADMISSION_RETRY_DELAY,
7394 kind,
7395 daemon_initiated: failure.daemon_initiated,
7396 approval: failure.approval.clone(),
7397 });
7398 Ok(())
7399}
7400
7401fn record_health_action(snapshot: &SharedSnapshot, module_id: &str, action: String, now_ms: u64) {
7402 let _ = update_snapshot(snapshot, Some(module_id), |state| {
7403 state.health.last_action = Some(action);
7404 state.health.last_action_ms = Some(now_ms);
7405 });
7406}
7407
7408fn supervisor_health_status(status: HealthStatus) -> SupervisorHealthStatus {
7409 match status {
7410 HealthStatus::Ok => SupervisorHealthStatus::Ok,
7411 HealthStatus::Degraded => SupervisorHealthStatus::Degraded,
7412 HealthStatus::Failing => SupervisorHealthStatus::Failing,
7413 }
7414}
7415
7416fn truncate_health_metrics(metrics: Option<Value>) -> Option<Value> {
7428 let metrics = metrics?;
7429 match serde_json::to_vec(&metrics) {
7430 Ok(encoded) if encoded.len() > MAX_HEALTH_METRICS_BYTES => Some(serde_json::json!({
7431 "truncated": true,
7432 "original_bytes": encoded.len(),
7433 })),
7434 Ok(_) | Err(_) => Some(metrics),
7435 }
7436}
7437
7438fn jittered_health_delay(module_id: &str, probe_index: u64, cadence: Duration) -> Duration {
7444 if cadence.is_zero() {
7445 return Duration::ZERO;
7446 }
7447 let cadence_ms = cadence.as_millis() as u64;
7448 if cadence_ms == 0 {
7464 return cadence;
7465 }
7466 let jitter_span = (cadence_ms / 10).max(1);
7481 let hash = module_id.as_bytes().iter().fold(
7482 probe_index.wrapping_mul(0x9E37_79B9_7F4A_7C15),
7483 |acc, byte| {
7484 acc.wrapping_mul(1099511628211)
7485 .wrapping_add(u64::from(*byte))
7486 },
7487 );
7488 cadence + Duration::from_millis(hash % jitter_span)
7489}
7490
7491#[cfg(test)]
7492mod tests {
7493 use super::*;
7494
7495 #[test]
7496 fn registration_retirement_self_prefixed_owner_is_not_a_prefix_submodule() {
7497 let handle = SupervisorHandle::new();
7498 handle.set_spawn_nonce("prefix:owner", "owner-nonce".into());
7499 handle.set_reserved_prefixes("prefix:owner", &["prefix:".into()]);
7500 let owner = handle
7501 .reserved_hello_verification("prefix:owner", Some("owner-nonce"))
7502 .unwrap();
7503 assert!(owner.nonce_verified);
7504 assert!(
7505 owner.prefix_owner.is_none(),
7506 "an unreserved programmatic owner is not its own prefix child"
7507 );
7508 let child = handle
7509 .reserved_hello_verification("prefix:child", Some("owner-nonce"))
7510 .unwrap();
7511 assert!(child.nonce_verified);
7512 assert_eq!(child.prefix_owner.as_deref(), Some("prefix:owner"));
7513 }
7514
7515 #[test]
7516 fn readding_a_module_clears_its_rescan_removal_tombstone() {
7517 let handle = SupervisorHandle::new();
7518 let module_id = "readded-tombstone";
7519 handle.record_rescan_removal(module_id);
7520 assert!(handle.removal_tombstone_age_ms(module_id).is_some());
7521
7522 handle.apply_identity_configuration(&ModuleSpec {
7523 script: None,
7524 module_id: module_id.to_string(),
7525 program: PathBuf::from("/test/module"),
7526 args: Vec::new(),
7527 env: Vec::new(),
7528 reserved: false,
7529 reserved_prefixes: Vec::new(),
7530 protocol: ModuleProtocol::Subc,
7531 overlap: Default::default(),
7532 carry_containment: false,
7533 });
7534
7535 assert!(
7536 handle.removal_tombstone_age_ms(module_id).is_none(),
7537 "a re-added module must not retain a stale removal tombstone"
7538 );
7539 }
7540
7541 #[derive(Debug, PartialEq, Eq)]
7544 struct OwnerInSpawnWindow {
7545 module_id: String,
7546 configured: bool,
7547 on_roster: bool,
7548 admission_refusal: Option<&'static str>,
7549 }
7550
7551 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
7562 async fn a_new_module_is_configured_before_its_first_process_can_register() {
7563 use crate::scopes::ScopeTable;
7564 use subc_protocol::{error_codes, scope::ScopeSelector, Principal};
7565
7566 let dir = cortexkit_test_support::ScratchDir::new("configured-before-spawn");
7567 let stub = |module_id: &str, program: PathBuf| ModuleSpec {
7568 script: None,
7569 module_id: module_id.to_string(),
7570 program,
7571 args: Vec::new(),
7572 env: ["XDG_DATA_HOME", "XDG_RUNTIME_DIR", "XDG_CONFIG_HOME"]
7573 .into_iter()
7574 .map(|key| (key.to_string(), dir.path().display().to_string()))
7575 .chain([("FAKE_AFT_NEVER_CONNECT".to_string(), "1".to_string())])
7576 .collect(),
7577 reserved: false,
7578 reserved_prefixes: Vec::new(),
7579 protocol: ModuleProtocol::Subc,
7580 overlap: Default::default(),
7581 carry_containment: false,
7582 };
7583 let live = super::terminal_history_tests::fake_aft_stub_path();
7584 let missing = dir.path().join("definitely-missing-module");
7585
7586 let handle = SupervisorHandle::new();
7587 let mut supervisor =
7588 Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default())
7589 .with_handle(handle.clone());
7590 let observed = Arc::new(Mutex::new(Vec::<OwnerInSpawnWindow>::new()));
7591 let hook_handle = handle.clone();
7592 let hook_observed = Arc::clone(&observed);
7593 supervisor.test_after_first_spawn = AfterFirstSpawnHook(Some(Arc::new(move |module_id| {
7594 let configured = hook_handle.is_configured(module_id);
7597 let selector = ScopeSelector::new(
7598 Principal::Reserved {
7599 module_id: module_id.to_string(),
7600 },
7601 "s".to_string(),
7602 Some(1),
7603 );
7604 let carrier = Principal::Reserved {
7605 module_id: "carrier".to_string(),
7606 };
7607 let admission_refusal = match ScopeTable::new(Vec::<String>::new())
7608 .admit(&carrier, module_id, &selector, configured)
7609 {
7610 Ok(_) => None,
7611 Err(refusal) => Some(refusal.code),
7612 };
7613 hook_observed.lock().unwrap().push(OwnerInSpawnWindow {
7614 module_id: module_id.to_string(),
7615 configured,
7616 on_roster: hook_handle.get(module_id).is_some(),
7617 admission_refusal,
7618 });
7619 })));
7620
7621 let plain = supervisor.spawn(stub("plain", live.clone())).unwrap();
7622 let configured = supervisor
7623 .supervise_configured(stub("configured", live.clone()), true)
7624 .unwrap();
7625 let with_health = supervisor
7626 .supervise_configured_with_health(
7627 stub("with-health", live.clone()),
7628 true,
7629 HealthConfig::default(),
7630 None,
7631 RestartPolicy::default(),
7632 )
7633 .unwrap();
7634 let failed = supervisor
7637 .supervise_configured_with_health(
7638 stub("failed-spawn", missing.clone()),
7639 true,
7640 HealthConfig::default(),
7641 None,
7642 RestartPolicy::default(),
7643 )
7644 .unwrap();
7645 assert!(supervisor.spawn(stub("spawn-error", missing)).is_err());
7648
7649 let expected = [
7650 "plain",
7651 "configured",
7652 "with-health",
7653 "failed-spawn",
7654 "spawn-error",
7655 ]
7656 .into_iter()
7657 .map(|module_id| OwnerInSpawnWindow {
7658 module_id: module_id.to_string(),
7659 configured: true,
7660 on_roster: !matches!(module_id, "plain" | "spawn-error"),
7661 admission_refusal: Some(error_codes::SCOPE_NOT_SYNCED),
7662 })
7663 .collect::<Vec<_>>();
7664 assert_eq!(*observed.lock().unwrap(), expected);
7665
7666 for module_id in ["plain", "configured", "with-health", "failed-spawn"] {
7667 assert!(
7668 handle.get(module_id).is_some(),
7669 "{module_id} is on the roster"
7670 );
7671 assert!(
7672 handle.is_configured(module_id),
7673 "{module_id} stays configured"
7674 );
7675 }
7676 assert!(handle.get("spawn-error").is_none());
7677 assert!(
7678 !handle.is_configured("spawn-error"),
7679 "a plain spawn that failed must not leave its module marked configured"
7680 );
7681
7682 handle.retire("failed-spawn");
7684 assert!(!handle.is_configured("failed-spawn"));
7685
7686 for module in [plain, configured, with_health] {
7687 module.stop().await.unwrap();
7688 }
7689 drop(failed);
7690 }
7691
7692 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
7699 async fn a_configured_module_reads_starting_while_its_first_process_already_runs() {
7700 let dir = cortexkit_test_support::ScratchDir::new("starting-while-spawned");
7701 let spec = ModuleSpec {
7702 script: None,
7703 module_id: "spawned-not-admitted".to_string(),
7704 program: super::terminal_history_tests::fake_aft_stub_path(),
7705 args: Vec::new(),
7706 env: ["XDG_DATA_HOME", "XDG_RUNTIME_DIR", "XDG_CONFIG_HOME"]
7707 .into_iter()
7708 .map(|key| (key.to_string(), dir.path().display().to_string()))
7709 .chain([("FAKE_AFT_NEVER_CONNECT".to_string(), "1".to_string())])
7710 .collect(),
7711 reserved: false,
7712 reserved_prefixes: Vec::new(),
7713 protocol: ModuleProtocol::Subc,
7714 overlap: Default::default(),
7715 carry_containment: false,
7716 };
7717 let handle = SupervisorHandle::new();
7718 let mut supervisor =
7719 Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default())
7720 .with_handle(handle.clone());
7721 let observed = Arc::new(Mutex::new(None));
7722 let hook_handle = handle.clone();
7723 let hook_observed = Arc::clone(&observed);
7724 supervisor.test_after_first_spawn = AfterFirstSpawnHook(Some(Arc::new(move |module_id| {
7725 let state = hook_handle
7726 .get(module_id)
7727 .expect("a configured module is on the roster before it is spawned")
7728 .state()
7729 .unwrap();
7730 *hook_observed.lock().unwrap() = Some(state);
7731 })));
7732
7733 let module = supervisor.supervise_configured(spec, true).unwrap();
7734 assert_eq!(
7735 *observed.lock().unwrap(),
7736 Some(ModuleState::Starting),
7737 "between the spawn and its admission the supervisor still reports starting"
7738 );
7739 assert_eq!(module.state().unwrap(), ModuleState::Running);
7740 module.stop().await.unwrap();
7741 }
7742
7743 fn stale_process_snapshot(state: ModuleState, enabled: bool) -> SharedSnapshot {
7744 let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::new(state, enabled)));
7745 update_snapshot(&snapshot, Some("stale-process-facts"), |snapshot| {
7746 snapshot.process_alive = true;
7747 snapshot.pid = Some(41);
7748 snapshot.spawned_at_ms = Some(42);
7749 snapshot.spawned_from = Some(PathBuf::from("/spawned/module"));
7750 snapshot.launch_spec_digest = Some(LaunchSpecDigest([0xA5; 32]));
7751 snapshot.spawned_file_identity = Some(SpawnedFileIdentity {
7752 device: 43,
7753 inode: 44,
7754 #[cfg(windows)]
7755 windows_image: None,
7756 });
7757 })
7758 .unwrap();
7759 snapshot
7760 }
7761
7762 fn assert_snapshot_process_facts_cleared(snapshot: &SharedSnapshot) {
7763 let snapshot = lock_snapshot(snapshot).unwrap();
7764 assert!(!snapshot.process_alive);
7765 assert_eq!(snapshot.pid, None);
7766 assert_eq!(snapshot.spawned_at_ms, None);
7767 assert_eq!(snapshot.spawned_from, None);
7768 assert_eq!(snapshot.launch_spec_digest, None);
7769 assert_eq!(snapshot.spawned_file_identity, None);
7770 }
7771
7772 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
7773 async fn failed_enable_spawn_clears_preexisting_current_process_facts() {
7774 let supervisor =
7775 Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default());
7776 let mut runtime = supervisor.runtime_config();
7777 runtime.test_seed_stale_facts_before_enable_spawn = true;
7778 let snapshot = stale_process_snapshot(ModuleState::Disabled, false);
7779 let mut child = None;
7780 let spec = ModuleSpec {
7781 script: None,
7782 module_id: "failed-enable-clears-facts".to_string(),
7783 program: PathBuf::from("/definitely/missing/failed-enable-module"),
7784 args: Vec::new(),
7785 env: Vec::new(),
7786 reserved: false,
7787 reserved_prefixes: Vec::new(),
7788 protocol: ModuleProtocol::Subc,
7789 overlap: Default::default(),
7790 carry_containment: false,
7791 };
7792
7793 let result = set_child_enabled(
7794 &spec,
7795 &runtime,
7796 &supervisor.registry,
7797 &supervisor.process_liveness,
7798 &snapshot,
7799 &mut child,
7800 true,
7801 &runtime.failure_context(false),
7802 )
7803 .await;
7804
7805 assert!(matches!(result, Err(SuperviseError::Spawn { .. })));
7806 assert_eq!(lock_snapshot(&snapshot).unwrap().state, ModuleState::Failed);
7807 assert_snapshot_process_facts_cleared(&snapshot);
7808 }
7809
7810 #[tokio::test]
7811 async fn start_revives_stranded_restarting_but_not_pending_backoff() {
7812 let supervisor =
7813 Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default());
7814 let runtime = supervisor.runtime_config();
7815 let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::new(
7816 ModuleState::Restarting,
7817 true,
7818 )));
7819 let spec = ModuleSpec {
7820 script: None,
7821 module_id: "start-stranded-restarting".to_string(),
7822 program: super::terminal_history_tests::fake_aft_stub_path(),
7823 args: Vec::new(),
7824 env: vec![("FAKE_AFT_NEVER_CONNECT".to_string(), "1".to_string())],
7825 reserved: false,
7826 reserved_prefixes: Vec::new(),
7827 protocol: ModuleProtocol::None,
7828 overlap: Default::default(),
7829 carry_containment: false,
7830 };
7831 let mut child = None;
7832 lock_snapshot(&snapshot).unwrap().respawn_pending = true;
7833 assert!(!super::set_child_enabled(
7834 &spec,
7835 &runtime,
7836 &Registry::default(),
7837 &supervisor.process_liveness,
7838 &snapshot,
7839 &mut child,
7840 true,
7841 &runtime.failure_context(false),
7842 )
7843 .await
7844 .unwrap());
7845 assert!(child.is_none());
7846 lock_snapshot(&snapshot).unwrap().respawn_pending = false;
7847 assert!(super::set_child_enabled(
7848 &spec,
7849 &runtime,
7850 &Registry::default(),
7851 &supervisor.process_liveness,
7852 &snapshot,
7853 &mut child,
7854 true,
7855 &runtime.failure_context(false),
7856 )
7857 .await
7858 .unwrap());
7859 assert_eq!(
7860 lock_snapshot(&snapshot).unwrap().state,
7861 ModuleState::Running
7862 );
7863 let mut child = child.unwrap();
7864 child.start_kill().unwrap();
7865 child.wait().await.unwrap();
7866 }
7867
7868 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
7869 async fn failed_reload_spawn_clears_current_process_facts() {
7870 let supervisor =
7871 Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default());
7872 let mut runtime = supervisor.runtime_config();
7873 runtime.restart_policy = RestartPolicy::new(0, Duration::ZERO);
7874 let snapshot = stale_process_snapshot(ModuleState::Running, true);
7875 let mut child = None;
7876 let spec = ModuleSpec {
7877 script: None,
7878 module_id: "failed-reload-clears-facts".to_string(),
7879 program: PathBuf::from("/unused/failed-reload-module"),
7880 args: Vec::new(),
7881 env: Vec::new(),
7882 reserved: false,
7883 reserved_prefixes: Vec::new(),
7884 protocol: ModuleProtocol::Subc,
7885 overlap: Default::default(),
7886 carry_containment: false,
7887 };
7888
7889 let result = handle_reload_spawn_failure(
7890 &spec,
7891 &runtime,
7892 &supervisor.process_liveness,
7893 &snapshot,
7894 &mut child,
7895 "forced reload spawn failure".to_string(),
7896 )
7897 .await;
7898
7899 assert!(matches!(result, Err(SuperviseError::ReloadFailed { .. })));
7900 assert_eq!(lock_snapshot(&snapshot).unwrap().state, ModuleState::Failed);
7901 assert_snapshot_process_facts_cleared(&snapshot);
7902 }
7903
7904 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
7905 async fn dropping_a_module_with_an_active_monitor_clears_current_process_facts() {
7906 let supervisor =
7907 Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default());
7908 let snapshot = stale_process_snapshot(ModuleState::Running, true);
7909 let module = supervisor.supervised_module(
7910 ModuleSpec {
7911 script: None,
7912 module_id: "drop-clears-facts".to_string(),
7913 program: PathBuf::from("/unused/drop-module"),
7914 args: Vec::new(),
7915 env: Vec::new(),
7916 reserved: false,
7917 reserved_prefixes: Vec::new(),
7918 protocol: ModuleProtocol::Subc,
7919 overlap: Default::default(),
7920 carry_containment: false,
7921 },
7922 supervisor.runtime_config(),
7923 Arc::clone(&snapshot),
7924 None,
7925 );
7926 assert!(!module
7927 .inner
7928 .monitor
7929 .lock()
7930 .unwrap()
7931 .as_ref()
7932 .unwrap()
7933 .is_finished());
7934
7935 drop(module);
7936
7937 assert_eq!(
7938 lock_snapshot(&snapshot).unwrap().state,
7939 ModuleState::Stopped
7940 );
7941 assert_snapshot_process_facts_cleared(&snapshot);
7942 }
7943
7944 #[cfg(unix)]
7945 #[tokio::test]
7946 async fn rescan_preserves_running_protocol_until_respawn() {
7947 let dir = cortexkit_test_support::ScratchDir::new("rescan-protocol");
7948 let initial = ModuleSpec {
7949 script: None,
7950 module_id: "rescan-protocol".into(),
7951 program: PathBuf::from("/bin/sleep"),
7952 args: vec!["60".into()],
7953 env: ["XDG_DATA_HOME", "XDG_RUNTIME_DIR", "XDG_CONFIG_HOME"]
7954 .into_iter()
7955 .map(|key| (key.into(), dir.path().to_string_lossy().into_owned()))
7956 .collect(),
7957 reserved: false,
7958 reserved_prefixes: vec![],
7959 protocol: ModuleProtocol::None,
7960 overlap: Default::default(),
7961 carry_containment: false,
7962 };
7963 let supervisor =
7964 Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default());
7965 let module = supervisor.spawn(initial.clone()).unwrap();
7966 assert!(module.status().unwrap().live);
7967 let mut next = initial;
7968 next.protocol = ModuleProtocol::Subc;
7969 module
7970 .update_configuration(next.clone(), HealthConfig::default(), None)
7971 .await
7972 .unwrap();
7973 assert!(
7974 module.status().unwrap().live,
7975 "rescan must not require HELLO from the old non-wire process"
7976 );
7977 let runtime = supervisor.runtime_config();
7978 let action = on_child_exit(
7979 &next,
7980 RestartPolicy::default(),
7981 &supervisor.registry,
7982 &module.inner.snapshot,
7983 &runtime.terminal_ring,
7984 &runtime.spawn_events,
7985 &runtime.child_roster,
7986 ExitReport {
7987 kind: ExitKind::Clean,
7988 code: Some(0),
7989 signal: None,
7990 at_ms: unix_ms_now(),
7991 },
7992 true,
7993 )
7994 .await;
7995 assert!(
7996 matches!(action, NextAction::Restart { .. }),
7997 "the old non-wire process's clean exit must restart"
7998 );
7999 module.drain().await.unwrap();
8000 }
8001
8002 #[cfg(unix)]
8003 #[tokio::test]
8004 async fn orphan_identity_matches_path_names_and_shebang_interpreters() {
8005 use std::os::unix::fs::PermissionsExt;
8006 let dir = cortexkit_test_support::ScratchDir::new("spawn-image-identity");
8007 let script = dir.join("module.sh");
8008 std::fs::write(&script, "#!/bin/sh\nwhile :; do sleep 0.1; done\n").unwrap();
8009 std::fs::set_permissions(&script, std::fs::Permissions::from_mode(0o700)).unwrap();
8010 let record_path = dir.join("live-children.json");
8011 let supervisor =
8012 Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default())
8013 .with_live_children_record(&record_path);
8014 for (program, args) in [
8015 (PathBuf::from("sleep"), vec!["60".into()]),
8016 (script, vec![]),
8017 ] {
8018 let spec = ModuleSpec {
8019 script: None,
8020 module_id: "image-identity".into(),
8021 program: program.clone(),
8022 args,
8023 env: ["XDG_DATA_HOME", "XDG_RUNTIME_DIR", "XDG_CONFIG_HOME"]
8024 .into_iter()
8025 .map(|key| (key.into(), dir.to_string_lossy().into_owned()))
8026 .collect(),
8027 reserved: false,
8028 reserved_prefixes: vec![],
8029 protocol: ModuleProtocol::None,
8030 overlap: Default::default(),
8031 carry_containment: false,
8032 };
8033 let module = supervisor.spawn(spec).unwrap();
8034 #[cfg(target_os = "macos")]
8035 {
8036 let deadline = Instant::now() + Duration::from_secs(5);
8039 while crate::live_children::read_record(&record_path)
8040 .unwrap()
8041 .iter()
8042 .all(|entry| entry.executable.is_none())
8043 {
8044 assert!(Instant::now() < deadline, "module image was not confirmed");
8045 tokio::time::sleep(Duration::from_millis(5)).await;
8046 }
8047 }
8048 let entry = crate::live_children::read_record(&record_path)
8049 .unwrap()
8050 .pop()
8051 .unwrap();
8052 let observed = subc_os::Process::open(entry.pid)
8053 .unwrap()
8054 .unwrap()
8055 .observe()
8056 .unwrap();
8057 let verdict = crate::live_children::identity_verdict(&entry, entry.pid, &observed);
8058 module.drain().await.unwrap();
8059 assert_eq!(
8060 verdict,
8061 crate::live_children::IdentityVerdict::Matches,
8062 "program {program:?}: recorded {entry:?}, observed {observed:?}"
8063 );
8064 }
8065 }
8066
8067 #[cfg(unix)]
8068 fn http_fixture(
8069 dir: &std::path::Path,
8070 url: &str,
8071 threshold: u32,
8072 ) -> crate::daemon_config::ConfiguredModule {
8073 let path = dir.join("subc.jsonc");
8074 std::fs::write(&path, serde_json::json!({"version": 1, "modules": {"http-process": {
8075 "program": "/bin/sleep", "args": ["60"], "protocol": "none",
8076 "env": {"XDG_DATA_HOME":dir,"XDG_RUNTIME_DIR":dir,"XDG_CONFIG_HOME":dir},
8077 "health": {"http":url,"cadence_ms":20,"deadline_ms":200,"failure_threshold":threshold},
8078 "restart": {"backoff_ms":1,"max_backoff_ms":1}, "drain_timeout_ms":10
8079 }}}).to_string()).unwrap();
8080 crate::daemon_config::load(&path)
8081 .unwrap()
8082 .unwrap()
8083 .modules
8084 .pop()
8085 .unwrap()
8086 }
8087
8088 #[cfg(unix)]
8089 async fn wait_http_health(module: &SupervisedModule, status: SupervisorHealthStatus) {
8090 timeout(Duration::from_secs(5), async {
8091 loop {
8092 if module.status().unwrap().health.status == status {
8093 break;
8094 }
8095 sleep(Duration::from_millis(5)).await;
8096 }
8097 })
8098 .await
8099 .unwrap_or_else(|_| {
8100 panic!(
8101 "expected {status:?}, got {:?}",
8102 module.status().unwrap().health
8103 )
8104 });
8105 }
8106
8107 #[cfg(unix)]
8108 #[tokio::test]
8109 async fn http_health_status_flips_ok_failing_ok() {
8110 use tokio::io::{AsyncReadExt, AsyncWriteExt};
8111 let dir = cortexkit_test_support::ScratchDir::new("http-status-flips");
8112 let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
8113 let status = Arc::new(std::sync::atomic::AtomicU16::new(200));
8114 let serving_status = status.clone();
8115 let url = format!("http://{}/healthz", listener.local_addr().unwrap());
8116 let server = tokio::spawn(async move {
8117 loop {
8118 let (mut stream, _) = listener.accept().await.unwrap();
8119 let mut request = [0u8; 2048];
8120 let count = stream.read(&mut request).await.unwrap();
8121 assert!(count > 0, "a probe must send an HTTP request");
8122 let code = serving_status.load(std::sync::atomic::Ordering::SeqCst);
8123 let body = if code == 200 {
8124 "ready"
8125 } else {
8126 "{\"status\":\"unavailable\",\"error\":\"scratch failure\"}"
8127 };
8128 let response = format!("HTTP/1.1 {code} scratch\r\nContent-Length: {}\r\nConnection: close\r\n\r\n{body}", body.len());
8129 let _ = stream.write_all(response.as_bytes()).await;
8130 }
8131 });
8132 let configured = http_fixture(&dir, &url, 1000);
8133 let module =
8134 Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default())
8135 .supervise_configured_with_health(
8136 configured.module_spec(),
8137 true,
8138 configured.health,
8139 configured.drain_timeout_ms,
8140 configured.restart,
8141 )
8142 .unwrap();
8143 wait_http_health(&module, SupervisorHealthStatus::Ok).await;
8144 status.store(503, std::sync::atomic::Ordering::SeqCst);
8145 wait_http_health(&module, SupervisorHealthStatus::Failing).await;
8146 assert!(module
8147 .status()
8148 .unwrap()
8149 .health
8150 .detail
8151 .unwrap()
8152 .contains("scratch failure"));
8153 status.store(200, std::sync::atomic::Ordering::SeqCst);
8154 wait_http_health(&module, SupervisorHealthStatus::Ok).await;
8155 assert_eq!(module.status().unwrap().health.consecutive_failures, 0);
8156 let before = module.status().unwrap();
8157 let (spec, mut health) = module.configuration().unwrap();
8158 health.http = None;
8159 module
8160 .update_configuration(spec.clone(), health.clone(), Some(10))
8161 .await
8162 .unwrap();
8163 wait_http_health(&module, SupervisorHealthStatus::Unknown).await;
8164 health.http = Some(url);
8165 module
8166 .update_configuration(spec, health, Some(10))
8167 .await
8168 .unwrap();
8169 wait_http_health(&module, SupervisorHealthStatus::Ok).await;
8170 assert_eq!(
8171 module.status().unwrap().pid,
8172 before.pid,
8173 "changing a probe must apply live, not restart its process"
8174 );
8175 let (spec, mut health) = module.configuration().unwrap();
8176 health.failure_threshold = 2;
8177 module
8178 .update_configuration(spec, health, Some(10))
8179 .await
8180 .unwrap();
8181 status.store(503, std::sync::atomic::Ordering::SeqCst);
8182 timeout(Duration::from_secs(5), async {
8183 while module.status().unwrap().spawn_generation == before.spawn_generation {
8184 sleep(Duration::from_millis(5)).await;
8185 }
8186 })
8187 .await
8188 .expect("sustained HTTP 503 responses must trigger the health restart policy");
8189 module.drain().await.unwrap();
8190 server.abort();
8191 }
8192
8193 #[cfg(unix)]
8194 #[tokio::test]
8195 async fn http_health_replacement_recovers_after_one_restart() {
8196 use tokio::io::{AsyncReadExt, AsyncWriteExt};
8197 let dir = cortexkit_test_support::ScratchDir::new("http-replacement-recovers");
8198 let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
8199 let url = format!("http://{}/healthz", listener.local_addr().unwrap());
8200 let mut configured = http_fixture(&dir, &url, 2);
8201 configured.restart.max_restarts = 2;
8202 let module =
8203 Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default())
8204 .supervise_configured_with_health(
8205 configured.module_spec(),
8206 true,
8207 configured.health,
8208 configured.drain_timeout_ms,
8209 configured.restart,
8210 )
8211 .unwrap();
8212 let first_generation = module.status().unwrap().spawn_generation;
8213 let observed = module.clone();
8214 let (healthy_tx, mut healthy_rx) = mpsc::channel(8);
8215 let server = tokio::spawn(async move {
8216 let mut generation = first_generation;
8217 let mut probes = 0;
8218 loop {
8219 let (mut stream, _) = listener.accept().await.unwrap();
8220 let mut request = [0; 2048];
8221 assert!(stream.read(&mut request).await.unwrap() > 0);
8222 let current = observed.status().unwrap().spawn_generation;
8223 if current != generation {
8224 generation = current;
8225 probes = 0;
8226 }
8227 probes += 1;
8228 let healthy = generation != first_generation && probes > 1;
8231 let code = if healthy { 200 } else { 503 };
8232 let response = format!(
8233 "HTTP/1.1 {code} scratch\r\nContent-Length: 0\r\nConnection: close\r\n\r\n"
8234 );
8235 let _ = stream.write_all(response.as_bytes()).await;
8236 if healthy {
8237 healthy_tx.send(()).await.unwrap();
8238 }
8239 }
8240 });
8241 timeout(Duration::from_secs(5), async {
8242 let mut healthy_probes = 0;
8243 loop {
8244 if module.state().unwrap() == ModuleState::Failed {
8245 break;
8246 }
8247 while healthy_rx.try_recv().is_ok() {
8248 healthy_probes += 1;
8249 }
8250 if healthy_probes >= 3
8251 && module.status().unwrap().health.status == SupervisorHealthStatus::Ok
8252 {
8253 break;
8254 }
8255 sleep(Duration::from_millis(5)).await;
8256 }
8257 })
8258 .await
8259 .expect("the replacement must settle healthy without exhausting its restart budget");
8260 let status = module.status().unwrap();
8261 module.drain().await.unwrap();
8262 server.abort();
8263 assert_eq!(status.state, ModuleState::Running);
8264 assert_eq!(status.health.status, SupervisorHealthStatus::Ok);
8265 assert_eq!(status.health.consecutive_failures, 0);
8266 assert_eq!(status.lifetime_restarts, 1);
8267 assert_eq!(status.restart_count, 1);
8268 }
8269
8270 #[cfg(unix)]
8271 #[tokio::test]
8272 async fn http_health_sustained_failure_exhausts_restart_budget() {
8273 use std::collections::BTreeMap;
8274 use tokio::io::{AsyncReadExt, AsyncWriteExt};
8275 let dir = cortexkit_test_support::ScratchDir::new("http-health-budget");
8276 let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
8277 let url = format!("http://{}/healthz", listener.local_addr().unwrap());
8278 let mut configured = http_fixture(&dir, &url, 2);
8279 configured.restart.max_restarts = 2;
8280 let module =
8281 Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default())
8282 .supervise_configured_with_health(
8283 configured.module_spec(),
8284 true,
8285 configured.health,
8286 configured.drain_timeout_ms,
8287 configured.restart,
8288 )
8289 .unwrap();
8290 let observed = module.clone();
8291 let probes = Arc::new(Mutex::new(BTreeMap::<u64, u32>::new()));
8292 let served = probes.clone();
8293 let server = tokio::spawn(async move {
8294 loop {
8295 let (mut stream, _) = listener.accept().await.unwrap();
8296 let mut request = [0; 2048];
8297 assert!(stream.read(&mut request).await.unwrap() > 0);
8298 *served
8299 .lock()
8300 .unwrap()
8301 .entry(observed.status().unwrap().spawn_generation)
8302 .or_default() += 1;
8303 let _ = stream
8304 .write_all(b"HTTP/1.1 503 Unavailable\r\nContent-Length: 0\r\nConnection: close\r\n\r\n")
8305 .await;
8306 }
8307 });
8308 timeout(Duration::from_secs(5), async {
8309 while module.state().unwrap() != ModuleState::Failed {
8310 sleep(Duration::from_millis(5)).await;
8311 }
8312 })
8313 .await
8314 .expect("sustained failures must still exhaust the health restart budget");
8315 let status = module.status().unwrap();
8316 server.abort();
8317 assert_eq!(status.restart_count, 2);
8318 assert_eq!(status.lifetime_restarts, 2);
8319 assert!(!status.process_alive);
8320 assert_eq!(
8321 probes.lock().unwrap().values().copied().collect::<Vec<_>>(),
8322 vec![2, 2, 2]
8323 );
8324 }
8325
8326 #[cfg(unix)]
8327 #[tokio::test]
8328 async fn spawn_clears_the_previous_process_health() {
8329 let dir = cortexkit_test_support::ScratchDir::new("spawn-fresh-health");
8330 let mut configured = http_fixture(&dir, "http://127.0.0.1:1/healthz", 2);
8331 configured.health.cadence = Duration::from_secs(3600);
8332 let module =
8333 Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default())
8334 .supervise_configured_with_health(
8335 configured.module_spec(),
8336 false,
8337 configured.health,
8338 configured.drain_timeout_ms,
8339 configured.restart,
8340 )
8341 .unwrap();
8342 update_snapshot(&module.inner.snapshot, Some(module.module_id()), |state| {
8343 state.health.status = SupervisorHealthStatus::Unresponsive;
8344 state.health.consecutive_failures = 3;
8345 state.health.detail = Some("old process refused connections".into());
8346 state.health.metrics = Some(serde_json::json!({"old": true}));
8347 state.health.last_probe_ms = Some(123);
8348 state.health.last_action = Some("restart".into());
8349 state.health.last_action_ms = Some(124);
8350 })
8351 .unwrap();
8352 module.set_enabled(true).await.unwrap();
8353 let health = module.status().unwrap().health;
8354 module.drain().await.unwrap();
8355 assert_eq!(health.consecutive_failures, 0);
8356 assert_eq!(health.status, SupervisorHealthStatus::Unknown);
8357 assert_eq!(health.detail, None);
8358 assert_eq!(health.metrics, None);
8359 assert_eq!(health.last_probe_ms, None);
8360 assert_eq!(health.last_action.as_deref(), Some("restart"));
8361 assert_eq!(health.last_action_ms, Some(124));
8362 }
8363
8364 #[cfg(unix)]
8365 #[tokio::test]
8366 async fn http_health_no_listener_restarts_after_consecutive_failures() {
8367 let dir = cortexkit_test_support::ScratchDir::new("http-refused");
8368 let unused = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
8369 let url = format!("http://{}/healthz", unused.local_addr().unwrap());
8370 drop(unused);
8371 let configured = http_fixture(&dir, &url, 2);
8372 let module =
8373 Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default())
8374 .supervise_configured_with_health(
8375 configured.module_spec(),
8376 true,
8377 configured.health,
8378 configured.drain_timeout_ms,
8379 configured.restart,
8380 )
8381 .unwrap();
8382 let before = module.status().unwrap().spawn_generation;
8383 timeout(Duration::from_secs(5), async {
8384 loop {
8385 let status = module.status().unwrap();
8386 if status.spawn_generation > before {
8387 assert!(status.lifetime_restarts > 0);
8388 break;
8389 }
8390 sleep(Duration::from_millis(5)).await;
8391 }
8392 })
8393 .await
8394 .expect("sustained HTTP refusal must trigger the health restart policy");
8395 module.drain().await.unwrap();
8396 }
8397
8398 #[cfg(unix)]
8399 #[tokio::test]
8400 async fn http_health_timeout_honours_deadline() {
8401 let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
8402 let url = format!("http://{}/healthz", listener.local_addr().unwrap());
8403 let server = tokio::spawn(async move {
8404 let _held = listener.accept().await.unwrap();
8405 std::future::pending::<()>().await;
8406 });
8407 let error = timeout(
8408 Duration::from_secs(1),
8409 probe_http_health(&url, Duration::from_millis(10)),
8410 )
8411 .await
8412 .expect("the probe must enforce its own deadline")
8413 .unwrap_err();
8414 server.abort();
8415 assert!(error.to_string().contains("timed out"));
8416 }
8417
8418 #[cfg(unix)]
8419 #[tokio::test]
8420 async fn http_health_localhost_reaches_an_ipv6_only_listener() {
8421 use tokio::io::{AsyncReadExt, AsyncWriteExt};
8422 let listener = tokio::net::TcpListener::bind("[::1]:0").await.unwrap();
8423 let url = format!(
8424 "http://localhost:{}/healthz",
8425 listener.local_addr().unwrap().port()
8426 );
8427 let server = tokio::spawn(async move {
8428 let (mut stream, _) = listener.accept().await.unwrap();
8429 let mut request = [0u8; 2048];
8430 assert!(stream.read(&mut request).await.unwrap() > 0);
8431 stream
8432 .write_all(b"HTTP/1.1 200 OK\r\nContent-Length: 2\r\n\r\nok")
8433 .await
8434 .unwrap();
8435 });
8436 assert_eq!(
8441 probe_http_health(&url, Duration::from_secs(10))
8442 .await
8443 .unwrap()
8444 .status,
8445 HealthStatus::Ok
8446 );
8447 server.await.unwrap();
8448 }
8449
8450 #[cfg(unix)]
8451 #[tokio::test]
8452 async fn http_health_timeout_keeps_partial_status_and_body() {
8453 use tokio::io::{AsyncReadExt, AsyncWriteExt};
8454 let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
8455 let url = format!("http://{}/healthz", listener.local_addr().unwrap());
8456 let server = tokio::spawn(async move {
8457 let (mut stream, _) = listener.accept().await.unwrap();
8458 let mut request = [0u8; 2048];
8459 assert!(stream.read(&mut request).await.unwrap() > 0);
8460 stream
8461 .write_all(
8462 b"HTTP/1.1 503 Unavailable\r\nContent-Length: 100\r\n\r\npartial diagnostic",
8463 )
8464 .await
8465 .unwrap();
8466 std::future::pending::<()>().await;
8467 });
8468 let error = probe_http_health(&url, Duration::from_secs(1))
8469 .await
8470 .unwrap_err()
8471 .to_string();
8472 server.abort();
8473 assert!(
8474 error.contains("timed out")
8475 && error.contains("503 Unavailable")
8476 && error.contains("partial diagnostic"),
8477 "{error}"
8478 );
8479 }
8480
8481 #[cfg(unix)]
8482 #[tokio::test]
8483 async fn http_health_diagnostic_body_is_bounded_to_200_bytes() {
8484 use tokio::io::{AsyncReadExt, AsyncWriteExt};
8485 let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
8486 let url = format!("http://{}/healthz", listener.local_addr().unwrap());
8487 let server = tokio::spawn(async move {
8488 let (mut stream, _) = listener.accept().await.unwrap();
8489 let mut request = [0u8; 2048];
8490 assert!(stream.read(&mut request).await.unwrap() > 0);
8491 let body = format!("{}not-in-diagnostic", "x".repeat(400));
8492 let response = format!(
8493 "HTTP/1.1 503 Unavailable\r\nContent-Length: {}\r\n\r\n{body}",
8494 body.len()
8495 );
8496 stream.write_all(response.as_bytes()).await.unwrap();
8497 });
8498 let error = probe_http_health(&url, Duration::from_secs(1))
8499 .await
8500 .unwrap_err()
8501 .to_string();
8502 server.await.unwrap();
8503 assert!(error.ends_with(&"x".repeat(200)), "{error}");
8504 assert_eq!(error.split(": ").last().unwrap().len(), 200);
8505 assert!(!error.contains("not-in-diagnostic"));
8506 }
8507
8508 #[cfg(unix)]
8509 #[tokio::test]
8510 async fn http_health_real_nats_server_monitoring() {
8511 if std::process::Command::new("nats-server")
8512 .arg("--version")
8513 .env("XDG_DATA_HOME", std::env::temp_dir())
8514 .env("XDG_RUNTIME_DIR", std::env::temp_dir())
8515 .env("XDG_CONFIG_HOME", std::env::temp_dir())
8516 .output()
8517 .is_err()
8518 {
8519 eprintln!(
8520 "skipped: http_health_real_nats_server_monitoring (nats-server is not on PATH)"
8521 );
8522 return;
8523 }
8524 let dir = cortexkit_test_support::ScratchDir::new("nats-http-monitoring");
8525 let monitor = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
8526 let port = monitor.local_addr().unwrap().port();
8527 let client = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
8528 let client_port = client.local_addr().unwrap().port();
8529 let config = dir.join("server.conf");
8530 std::fs::write(&config, format!("listen: 127.0.0.1:{client_port}\nhttp: 127.0.0.1:{port}\njetstream {{ store_dir: \"{}\" }}\n", dir.join("jetstream").display())).unwrap();
8531 let mut configured = http_fixture(&dir, &format!("http://127.0.0.1:{port}/healthz"), 1000);
8532 configured.program = PathBuf::from("nats-server");
8533 configured.args = vec!["-c".into(), config.to_string_lossy().into_owned()];
8534 drop(monitor);
8535 drop(client);
8536 let module =
8537 Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default())
8538 .supervise_configured_with_health(
8539 configured.module_spec(),
8540 true,
8541 configured.health,
8542 configured.drain_timeout_ms,
8543 configured.restart,
8544 )
8545 .unwrap();
8546 wait_http_health(&module, SupervisorHealthStatus::Ok).await;
8547 module.drain().await.unwrap();
8548 }
8549
8550 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
8551 async fn configuration_update_does_not_replace_captured_running_process_facts() {
8552 let supervisor =
8553 Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default());
8554 let snapshot = stale_process_snapshot(ModuleState::Running, true);
8555 let initial = ModuleSpec {
8556 script: None,
8557 module_id: "rescan-preserves-spawn-facts".to_string(),
8558 program: PathBuf::from("/spawned/module"),
8559 args: Vec::new(),
8560 env: Vec::new(),
8561 reserved: false,
8562 reserved_prefixes: Vec::new(),
8563 protocol: ModuleProtocol::Subc,
8564 overlap: Default::default(),
8565 carry_containment: false,
8566 };
8567 let module = supervisor.supervised_module(
8568 initial.clone(),
8569 supervisor.runtime_config(),
8570 snapshot,
8571 None,
8572 );
8573 let before = module.status().unwrap();
8574 let mut replacement = initial;
8575 replacement.program = PathBuf::from("/rescanned/replacement-module");
8576
8577 module
8578 .update_configuration(replacement, HealthConfig::default(), None)
8579 .await
8580 .unwrap();
8581
8582 let after = module.status().unwrap();
8583 assert_eq!(after.pid, before.pid);
8584 assert_eq!(after.spawned_at_ms, before.spawned_at_ms);
8585 assert_eq!(after.spawned_from, before.spawned_from);
8586 drop(module);
8587 }
8588}
8589
8590fn unix_ms_now() -> u64 {
8591 SystemTime::now()
8592 .duration_since(UNIX_EPOCH)
8593 .map(|duration| duration.as_millis().min(u128::from(u64::MAX)) as u64)
8594 .unwrap_or(0)
8595}
8596
8597async fn supervise_loop(
8598 mut spec: ModuleSpec,
8599 mut runtime: SupervisorRuntimeConfig,
8600 registry: Arc<Registry>,
8601 process_liveness: Arc<SupervisorProcessLiveness>,
8602 snapshot: SharedSnapshot,
8603 mut child: Option<SupervisedChild>,
8604 mut commands: mpsc::Receiver<SupervisorCommand>,
8605) {
8606 let mut health_probe = HealthProbeRuntime::default();
8607 let mut registration_changes = match registry.subscribe_module_changes(&spec.module_id) {
8611 Ok(changes) => Some(changes),
8612 Err(err) => {
8613 warn!(module_id = %spec.module_id, error = %err, "health prober could not subscribe to registry");
8616 None
8617 }
8618 };
8619 let mut pending_respawn: Option<PendingRespawn> = None;
8623 let mut requeued: VecDeque<SupervisorCommand> = VecDeque::new();
8626 let mut previous_instance_recheck = tokio::time::interval_at(
8627 Instant::now() + Duration::from_secs(5),
8628 Duration::from_secs(5),
8629 );
8630 previous_instance_recheck.set_missed_tick_behavior(tokio::time::MissedTickBehavior::Skip);
8631 'supervision: loop {
8632 #[cfg(test)]
8633 {
8634 lock_snapshot(&snapshot).unwrap().actor_select = None;
8635 }
8636 #[cfg(target_os = "macos")]
8637 if let Some(active) = child.as_mut() {
8638 active.confirm_privacy_exec().await;
8639 }
8640 if let Some(scheduled) = runtime
8641 .scheduled_respawn
8642 .lock()
8643 .unwrap_or_else(|p| p.into_inner())
8644 .take()
8645 {
8646 runtime.recovery_is_daemon = scheduled.daemon_initiated;
8647 pending_respawn = Some(scheduled);
8648 }
8649 if pending_respawn.is_some() && (child.is_some() || !respawn_still_pending(&snapshot)) {
8650 pending_respawn = None;
8651 cancel_deferred_reload(
8652 &runtime,
8653 &spec.module_id,
8654 "respawn cancelled by a supervisor command",
8655 );
8656 }
8657 if child.is_none()
8658 && pending_respawn.is_none()
8659 && lock_snapshot(&snapshot).is_ok_and(|state| state.waiting_for_previous_pid.is_none())
8660 {
8661 cancel_deferred_reload(
8662 &runtime,
8663 &spec.module_id,
8664 "respawn cancelled before a replacement was spawned",
8665 );
8666 let mut stranded = false;
8667 let _ = update_snapshot(&snapshot, Some(&spec.module_id), |state| {
8668 state.respawn_pending = false;
8669 state.coalesced_restart_pending = false;
8670 if matches!(
8671 state.state,
8672 ModuleState::Restarting
8673 | ModuleState::Starting
8674 | ModuleState::Draining
8675 | ModuleState::Unresponsive
8676 ) {
8677 error!(module_id = %spec.module_id, state = ?state.state,
8678 "supervision operation ended without a child or pending respawn; marking failed so start can retry");
8679 stranded = true;
8680 }
8681 });
8682 if stranded {
8683 fail_snapshot(
8684 &snapshot,
8685 Some(&spec.module_id),
8686 None,
8687 &runtime.failure_context(runtime.recovery_is_daemon),
8688 );
8689 }
8690 }
8691 if child.is_none()
8692 && (runtime.child_roster.is_closed()
8693 || pending_respawn.is_none()
8694 || !lock_snapshot(&snapshot)
8695 .is_ok_and(|state| state.enabled && state.state == ModuleState::Restarting))
8696 {
8697 containment::terminate(&runtime.child_roster.containment).await;
8698 }
8699 if let Some(command) = requeued.pop_front() {
8700 if !handle_supervisor_command(
8701 command,
8702 &mut spec,
8703 &mut runtime,
8704 ®istry,
8705 &process_liveness,
8706 &snapshot,
8707 &mut child,
8708 &mut commands,
8709 &mut requeued,
8710 )
8711 .await
8712 {
8713 containment::terminate(&runtime.child_roster.containment).await;
8714 return;
8715 }
8716 if child.is_some() || !respawn_still_pending(&snapshot) {
8717 pending_respawn = None;
8718 let _ = update_snapshot(&snapshot, Some(&spec.module_id), |state| {
8719 state.respawn_pending = false
8720 });
8721 }
8722 continue;
8723 }
8724 if child.is_some() {
8725 runtime.recovery_is_daemon = true;
8726 #[cfg(test)]
8727 {
8728 lock_snapshot(&snapshot).unwrap().actor_turns += 1;
8729 }
8730 health_probe.refresh_registration(&spec, &runtime, ®istry, &snapshot);
8731 let active = child.as_mut().expect("child checked above");
8732 if active.registered_since_spawn() {
8733 if let Some(delivery) = &runtime.config_delivery {
8734 delivery.queue_registered(
8735 &spec.module_id,
8736 runtime.forwarding.as_ref(),
8737 ®istry,
8738 active,
8739 );
8740 }
8741 }
8742 if runtime.child_roster.is_closed() || !active.awaiting_registration() {
8743 active.registration_deadline = None;
8744 }
8745 let registration_deadline = active.registration_deadline;
8746 let no_checks = runtime.health.http.is_none()
8747 && (running_protocol(&spec, &snapshot) == ModuleProtocol::None
8748 || (health_probe.registered_connection.is_some() && !health_probe.advertised));
8749 let uptime_reset = lock_snapshot(&snapshot).ok().and_then(|mut state| {
8750 state
8751 .failed_retry
8752 .uptime_reset_at(no_checks, runtime.restart_policy.window)
8753 });
8754 let wake_at = match (health_probe.wake_at(), uptime_reset) {
8755 (probe, Some(reset)) => Some(probe.map_or(reset, |probe| probe.min(reset))),
8756 (probe, None) => probe,
8757 };
8758 let wake_at = match (wake_at, registration_deadline) {
8759 (Some(wake), Some(register)) => Some(wake.min(register)),
8760 (wake, register) => wake.or(register),
8761 };
8762 #[cfg(test)]
8763 let wake_after = wake_at.map(|at| at.saturating_duration_since(Instant::now()));
8764 let probe_sleep = supervisor_sleep(wake_at);
8765 tokio::pin!(probe_sleep);
8766 let wire_child = running_protocol(&spec, &snapshot) == ModuleProtocol::Subc;
8771 #[cfg(test)]
8772 {
8773 let registration_pending = wire_child
8774 && registration_changes
8775 .as_ref()
8776 .is_some_and(|changes| changes.has_changed().unwrap());
8777 let mut state = lock_snapshot(&snapshot).unwrap();
8778 state.actor_select = (!registration_pending).then_some(ActorSelectCheckpoint {
8779 generation: state.spawn_generation,
8780 turn: state.actor_turns,
8781 registered_connection: health_probe.registered_connection,
8782 next_probe_at: health_probe.next_probe_at,
8783 wake_after,
8784 });
8785 }
8786 let registration = active
8787 .awaiting_registration()
8788 .then(|| active.registration_spawn.clone())
8789 .flatten();
8790 #[cfg(all(test, target_os = "linux"))]
8791 let registration_before_exit = !runtime.child_roster.admission_event_for_test(
8792 &spec.module_id,
8793 "observe_registration_before_exit",
8794 Some(active.pid),
8795 ) && registration.is_some();
8796 #[cfg(not(all(test, target_os = "linux")))]
8797 let registration_before_exit = false;
8798 #[cfg(all(test, target_os = "linux"))]
8799 let registration_child_is_running = |child: &mut SupervisedChild| {
8800 registration_before_exit || registration_child_is_running(child)
8803 };
8804 let registration_failed = async {
8805 match ®istration {
8806 Some(spawn) => spawn.wait_registration_failure().await,
8807 None => std::future::pending().await,
8808 }
8809 };
8810 let active_child = child.as_mut().expect("child checked above");
8811 tokio::select! {
8812 _ = registration_failed => {
8813 if !active_child.registered_since_spawn() && registration_child_is_running(active_child) {
8814 active_child.registration_deadline = None;
8815 active_child.spawn_failure = Some("registration_channel_closed".into());
8816 if let Err(error) = stop_unregistered_child(active_child).await {
8817 fail_after_wait_error(&spec.module_id, &snapshot, &runtime.terminal_ring,
8818 &runtime.spawn_events, format!("registration_channel_closed: {error}"),
8819 &runtime.failure_context(true));
8820 child = None;
8821 }
8822 }
8823 continue;
8824 }
8825 wait_result = active_child.wait(), if !registration_before_exit => {
8826 if let Ok(mut state) = lock_snapshot(&snapshot) {
8827 state.failed_retry.uptime_reset_at(no_checks, runtime.restart_policy.window);
8828 }
8829 #[cfg(test)]
8830 { lock_snapshot(&snapshot).unwrap().actor_select = None; }
8831 let exit_report = match wait_result {
8840 Ok(status) => classify_reaped_child_exit(&snapshot, active_child, &status),
8841 Err(err) => {
8842 active_child.drain_stderr(&spec.module_id).await;
8843 fail_after_wait_error(
8844 &spec.module_id,
8845 &snapshot,
8846 &runtime.terminal_ring,
8847 &runtime.spawn_events,
8848 format!("failed to wait for supervised module: {err}"),
8849 &runtime.failure_context(true),
8850 );
8851 untrack_if_registration_released(
8852 &process_liveness,
8853 ®istry,
8854 &spec.module_id,
8855 &snapshot,
8856 );
8857 error!(module_id = %spec.module_id, error = %err, "failed to wait for supervised module");
8858 child = None;
8859 continue;
8860 }
8861 };
8862 active_child.drain_stderr(&spec.module_id).await;
8863
8864 let delivery=runtime.config_delivery.clone();
8865 let exited_spawn=active_child.spawned_at;
8866 let spawn=active_child.config_delivery.clone();
8867 let refusal=active_child.spawn_failure.as_deref().filter(|detail|detail.starts_with("program_refused:") && !active_child.live_program_binding.as_ref().is_some_and(|binding|binding.is_refused())).zip(active_child.admission_base.clone()).map(|(detail,base)|(base,subc_config::raw_digest(detail.as_bytes())));
8868 let module_id=spec.module_id.clone();
8869 let observed=async move {
8873 if let Some(delivery)=delivery {
8874 delivery.0.on_blocking(move |owner| {
8875 if let Some(spawn)=spawn { owner.delivery_child_exited(&spawn)?; }
8876 if let Some((base,identity))=refusal {owner.record_program_refusal(&module_id,&base,identity)?;}
8877 Ok(())
8878 }).await
8879 } else {Ok(())}
8880 };
8881 tokio::pin!(observed);
8882 loop {
8883 tokio::select! {
8884 result=&mut observed=>{
8885 if let Err(error)=result {warn!(module_id=%spec.module_id,%error,"configuration exit observation failed");}
8886 break;
8887 }
8888 command=commands.recv()=>{
8889 let Some(command)=command else {containment::terminate(&runtime.child_roster.containment).await;return;};
8890 let deferred=||SuperviseError::ConfigDeliveryDeferred {module_id:spec.module_id.clone()};
8891 let command=match command {
8892 SupervisorCommand::Restart {reply,..}|SupervisorCommand::Reload {reply,..}=>{let _=reply.send(Err(deferred()));continue;}
8893 SupervisorCommand::Swap {reply,..}=>{let _=reply.send(Err(deferred()));continue;}
8894 command @ (SupervisorCommand::ConfigApplyRestart {..}|SupervisorCommand::ConfigPolicyProbe {..}|SupervisorCommand::ConfigDeliveryFailed {..})=>{requeued.push_back(command);continue;}
8895 command=>command,
8896 };
8897 if !handle_supervisor_command(command,&mut spec,&mut runtime,®istry,&process_liveness,&snapshot,&mut child,&mut commands,&mut requeued).await {
8898 containment::terminate(&runtime.child_roster.containment).await;return;
8899 }
8900 if child.as_ref().is_none_or(|current|current.spawned_at!=exited_spawn) {continue 'supervision;}
8901 }
8902 }
8903 }
8904 let active_child=child.as_mut().expect("exit observation retained its child");
8905 let admission_kill = admission_exit::is_admission_kill(active_child.admission_kill_sent, &exit_report);
8909 let deferred_spawn=admission_kill && active_child.spawn_failure.as_deref().is_some_and(|detail|detail.starts_with("config_delivery_deferred:"));
8910 let refused_program = active_child.live_program_binding.as_ref().is_some_and(|binding| binding.is_refused());
8911 let exit_held = !deferred_spawn && (refused_program || runtime.config_delivery.as_ref().is_some_and(|delivery| delivery.0.exit_is_held(&spec.module_id)));
8912 let parked = exit_held && (!refused_program || active_child.admission_base.is_some())
8913 && try_park_config_attempt(&spec, &runtime, &snapshot, &runtime.failure_context(true),
8914 if refused_program { active_child.admission_base.as_ref() } else { None })
8915 .unwrap_or_else(|error| { warn!(module_id=%spec.module_id, %error, "exit hold could not be recorded; using ordinary exit accounting"); false });
8916 let next = if deferred_spawn {
8917 record_terminal_with_detail(&spec.module_id,&runtime.terminal_ring,&runtime.spawn_events,&exit_report,TerminalDisposition::Restarting,active_child.spawn_failure.clone());
8918 let _=update_snapshot(&snapshot,Some(&spec.module_id),|state| {
8919 state.state=ModuleState::Restarting;
8920 clear_current_process_facts(state);
8921 state.last_exit=Some(exit_report.clone());
8922 });
8923 NextAction::Restart {schedule:None}
8924 } else if parked {
8925 record_terminal_with_detail(&spec.module_id, &runtime.terminal_ring, &runtime.spawn_events, &exit_report, TerminalDisposition::Stopped, active_child.spawn_failure.clone().or_else(|| Some("config_gated".into())));
8926 NextAction::Stop { registration_released: registry.get_module(&spec.module_id).ok().flatten().is_none() }
8927 } else { on_child_exit(
8928 &spec,
8929 runtime.restart_policy,
8930 ®istry,
8931 &snapshot,
8932 &runtime.terminal_ring,
8933 &runtime.spawn_events,
8934 &runtime.child_roster,
8935 exit_report,
8936 true,
8937 ).await };
8938 runtime.start_requester = None;
8939 runtime.pending_activity_restart = false;
8940 let carry = matches!(next, NextAction::Restart { .. })
8943 && active_child.spawn_failure.is_none()
8944 && !runtime.child_roster.is_closed();
8945 if let Err(error) = active_child.finish_exit(carry).await {
8946 containment::terminate(&runtime.child_roster.containment).await;
8947 fail_after_wait_error(&spec.module_id, &snapshot, &runtime.terminal_ring,
8948 &runtime.spawn_events, format!("failed to reap classified exit: {error}"),
8949 &runtime.failure_context(true));
8950 child = None;
8951 continue;
8952 }
8953 match next {
8954 NextAction::Stop { registration_released } => {
8955 if registration_released {
8956 process_liveness.untrack_if_current(&spec.module_id, &snapshot);
8957 }
8958 child = None;
8959 }
8960 NextAction::Restart { schedule } => {
8961 let delay = if deferred_spawn {CONFIG_ADMISSION_RETRY_DELAY} else {schedule.map_or(
8962 runtime.restart_policy.delay_for_restart(0),
8963 |schedule| schedule.delay,
8964 )};
8965 if let Some(schedule) = schedule {
8966 log_crash_respawn(&spec.module_id, schedule);
8967 }
8968 child = None;
8976 pending_respawn = Some(PendingRespawn { deadline: Instant::now() + delay, kind: RespawnKind::Spawn, daemon_initiated: true, approval: None });
8977 let _ = update_snapshot(&snapshot, Some(&spec.module_id), |state| state.respawn_pending = true);
8978 }
8979 }
8980 }
8981 command = commands.recv() => {
8982 #[cfg(test)]
8983 { lock_snapshot(&snapshot).unwrap().actor_select = None; }
8984 let Some(command) = command else {
8985 containment::terminate(&runtime.child_roster.containment).await;
8986 return;
8987 };
8988 if !handle_supervisor_command(
8989 command,
8990 &mut spec,
8991 &mut runtime,
8992 ®istry,
8993 &process_liveness,
8994 &snapshot,
8995 &mut child,
8996 &mut commands,
8997 &mut requeued,
8998 ).await {
8999 containment::terminate(&runtime.child_roster.containment).await;
9000 return;
9001 }
9002 }
9003 _ = async {
9004 match registration_changes.as_mut() {
9005 Some(changes) => { let _ = changes.changed().await; }
9006 None => std::future::pending().await,
9007 }
9008 }, if wire_child => {
9009 #[cfg(test)]
9010 { lock_snapshot(&snapshot).unwrap().actor_select = None; }
9011 }
9012 _ = &mut probe_sleep => {
9013 #[cfg(test)]
9014 { lock_snapshot(&snapshot).unwrap().actor_select = None; }
9015 if registration_deadline.is_some_and(|deadline| Instant::now() >= deadline) {
9016 active_child.registration_deadline = None;
9019 if !runtime.child_roster.is_closed()
9020 && !active_child.registered_since_spawn()
9021 && registration_child_is_running(active_child)
9022 {
9023 warn!(module_id = %spec.module_id, pid = active_child.pid,
9024 wait_secs = runtime.register_timeout.as_secs(),
9025 "supervised module registration_timeout: first HELLO deadline expired; stopping child");
9026 active_child.spawn_failure = Some("registration_timeout".into());
9027 if let Err(error) = stop_unregistered_child(active_child).await {
9028 error!(module_id = %spec.module_id, %error, "failed to stop unregistered child");
9029 fail_after_wait_error(&spec.module_id, &snapshot, &runtime.terminal_ring,
9030 &runtime.spawn_events, format!("registration_timeout: {error}"),
9031 &runtime.failure_context(true));
9032 child = None;
9033 }
9034 continue;
9037 }
9038 }
9039 if health_probe.due() {
9040 run_health_probe_cycle(
9041 &spec,
9042 &runtime,
9043 ®istry,
9044 &process_liveness,
9045 &snapshot,
9046 &mut child,
9047 ).await;
9048 if child.is_none() {
9049 runtime.start_requester = None;
9050 runtime.pending_activity_restart = false;
9051 }
9052 if child.is_some() {
9053 health_probe.schedule_next(&spec, runtime.health.cadence);
9054 }
9055 }
9056 }
9057 }
9058 } else if let Some(pending) = pending_respawn.clone() {
9059 tokio::select! {
9060 _ = runtime.child_roster.closed() => {
9061 pending_respawn = None;
9062 let _ = update_snapshot(&snapshot, Some(&spec.module_id), |state| {
9063 state.state = ModuleState::Stopped;
9064 state.respawn_pending = false;
9065 });
9066 containment::terminate(&runtime.child_roster.containment).await;
9067 }
9068 _ = async {
9069 sleep_until(pending.deadline).await;
9070 if let Some(delivery) = &runtime.config_delivery {
9071 delivery.unfenced(&spec.module_id).await;
9072 }
9073 } => {
9074 pending_respawn = None;
9075 let failure = pending.failure_context(&runtime);
9076 let _ = update_snapshot(&snapshot, Some(&spec.module_id), |state| state.respawn_pending = false);
9077 if !respawn_still_pending(&snapshot) {
9081 continue;
9082 }
9083 if runtime.child_roster.is_closed() {
9088 let _ = update_snapshot(&snapshot, Some(&spec.module_id), |state| {
9089 state.state = ModuleState::Stopped;
9090 });
9091 debug!(module_id = %spec.module_id, "crash respawn cancelled by daemon shutdown");
9092 continue;
9093 }
9094 if let Err(err) = release_dead_registration(
9095 ®istry,
9096 runtime.forwarding.as_deref(),
9097 &snapshot,
9098 &spec.module_id,
9099 &failure,
9100 ).await {
9101 record_approved_attempt_failure(&spec, &snapshot, &failure, &err);
9102 error!(module_id = %spec.module_id, error = %err, "registration did not release before restart");
9103 continue;
9104 }
9105
9106 if matches!(pending.kind, RespawnKind::Reload) {
9107 let reply = runtime.deferred_reload_reply.lock().unwrap_or_else(|p| p.into_inner()).take();
9108 let result = finish_reload_child(&spec, &runtime, ®istry, &process_liveness, &snapshot, &mut child).await;
9109 if matches!(result, Err(SuperviseError::ConfigDeliveryDeferred { .. })) {
9110 *runtime.deferred_reload_reply.lock().unwrap_or_else(|p| p.into_inner()) = reply;
9111 continue;
9112 }
9113 if result.is_ok() && runtime.pending_activity_restart {
9114 runtime.record_activity_transition("subc.module_restarted", &spec.module_id, runtime.start_requester.clone());
9115 }
9116 runtime.pending_activity_restart = false;
9117 if let Some(reply) = reply { let _ = reply.send(result); }
9118 continue;
9119 }
9120 process_liveness.track(spec.module_id.clone(), Arc::clone(&snapshot));
9121 match spawn_and_mark_running_async(&spec, &runtime, &snapshot).await {
9122 Ok(next_child) => {
9123 child = Some(next_child);
9124 if runtime.pending_activity_restart {
9125 runtime.record_activity_transition("subc.module_restarted", &spec.module_id, runtime.start_requester.clone());
9126 runtime.pending_activity_restart = false;
9127 }
9128 debug!(module_id = %spec.module_id, "supervised module restarted after crash");
9129 }
9130 Err(SuperviseError::ConfigDeliveryDeferred { .. }) => {
9131 pending_respawn = Some(PendingRespawn { deadline:Instant::now()+CONFIG_ADMISSION_RETRY_DELAY, ..pending });
9132 let _ = update_snapshot(&snapshot, Some(&spec.module_id), |state| state.respawn_pending = true);
9133 }
9134 Err(SuperviseError::ConfigHeld { .. }) => {
9135 let _ = hold_config_spawn(&spec, &runtime, &snapshot, &failure);
9136 runtime.pending_activity_restart = false;
9137 process_liveness.untrack_if_current(&spec.module_id, &snapshot);
9138 }
9139 Err(err) => {
9140 fail_snapshot(&snapshot, Some(&spec.module_id), None, &failure);
9141 record_approved_attempt_failure(&spec, &snapshot, &failure, &err);
9142 process_liveness.untrack_if_current(&spec.module_id, &snapshot);
9143 error!(module_id = %spec.module_id, error = %err, "failed to restart supervised module");
9144 }
9145 }
9146 }
9147 command = commands.recv() => {
9148 let Some(command) = command else {
9149 containment::terminate(&runtime.child_roster.containment).await;
9150 return;
9151 };
9152 if !handle_supervisor_command(
9153 command,
9154 &mut spec,
9155 &mut runtime,
9156 ®istry,
9157 &process_liveness,
9158 &snapshot,
9159 &mut child,
9160 &mut commands,
9161 &mut requeued,
9162 ).await {
9163 containment::terminate(&runtime.child_roster.containment).await;
9164 return;
9165 }
9166 if child.is_some() || !respawn_still_pending(&snapshot) {
9171 pending_respawn = None;
9172 let _ = update_snapshot(&snapshot, Some(&spec.module_id), |state| state.respawn_pending = false);
9173 }
9174 }
9175 }
9176 } else {
9177 let slow_retry = lock_snapshot(&snapshot).ok().and_then(|state| {
9178 (state.enabled && state.state == ModuleState::Failed)
9179 .then_some(state.failed_retry.pending)
9180 .flatten()
9181 });
9182 let command = if lock_snapshot(&snapshot)
9183 .is_ok_and(|state| state.waiting_for_previous_pid.is_some())
9184 {
9185 tokio::select! {
9186 command = commands.recv() => command,
9187 _ = previous_instance_recheck.tick() => {
9188 if runtime.child_roster.previous_instance_pid(&spec.module_id).is_none() {
9189 let enabled = lock_snapshot(&snapshot).is_ok_and(|state| state.enabled);
9190 if enabled && !runtime.child_roster.is_closed() {
9191 process_liveness.track(spec.module_id.clone(), Arc::clone(&snapshot));
9192 match spawn_and_mark_running_async(&spec, &runtime, &snapshot).await {
9193 Ok(next_child) => child = Some(next_child),
9194 Err(SuperviseError::ConfigDeliveryDeferred { .. }) => {
9195 let _ = defer_config_spawn(&spec, &runtime, &snapshot, RespawnKind::Spawn, &runtime.failure_context(true));
9196 }
9197 Err(SuperviseError::ConfigHeld { .. }) => {
9198 let _ = hold_config_spawn(&spec, &runtime, &snapshot, &runtime.failure_context(true));
9199 }
9200 Err(error) => {
9201 fail_snapshot(&snapshot, Some(&spec.module_id), None, &runtime.failure_context(true));
9202 error!(module_id = %spec.module_id, %error, "failed to start module after its previous instance exited");
9203 }
9204 }
9205 } else {
9206 let _ = update_snapshot(&snapshot, Some(&spec.module_id), |state| {
9207 state.waiting_for_previous_pid = None;
9208 state.waiting_for_previous_detail = None;
9209 });
9210 }
9211 }
9212 continue;
9213 }
9214 }
9215 } else {
9216 tokio::select! {
9217 biased;
9218 command = commands.recv() => command,
9219 _ = async {
9220 match slow_retry {
9221 Some(retry) => sleep_until(retry.deadline).await,
9222 None => std::future::pending().await,
9223 }
9224 } => {
9225 let retry = slow_retry.expect("only a scheduled slow retry can wake");
9226 let _ = update_snapshot(&snapshot, Some(&spec.module_id), |state| {
9227 state.failed_retry.pending = None;
9228 });
9229 if runtime.child_roster.is_closed() {
9230 continue;
9231 }
9232 info!(module_id = %spec.module_id, attempt = retry.attempt,
9233 delay_secs = retry.delay.as_secs(), "starting failed module retry");
9234 if let Err(error) = set_child_enabled(
9235 &spec, &runtime, ®istry, &process_liveness, &snapshot, &mut child, true,
9236 &runtime.failure_context(true),
9237 ).await {
9238 error!(module_id = %spec.module_id, %error, "failed module retry could not start");
9239 }
9240 continue;
9241 }
9242 }
9243 };
9244 let Some(command) = command else {
9245 containment::terminate(&runtime.child_roster.containment).await;
9246 return;
9247 };
9248 if !handle_supervisor_command(
9249 command,
9250 &mut spec,
9251 &mut runtime,
9252 ®istry,
9253 &process_liveness,
9254 &snapshot,
9255 &mut child,
9256 &mut commands,
9257 &mut requeued,
9258 )
9259 .await
9260 {
9261 containment::terminate(&runtime.child_roster.containment).await;
9262 return;
9263 }
9264 }
9265 }
9266}
9267
9268async fn supervisor_sleep(wake_at: Option<Instant>) {
9271 match wake_at {
9272 Some(deadline) => sleep_until(deadline).await,
9273 None => std::future::pending().await,
9274 }
9275}
9276
9277fn log_crash_respawn(module_id: &str, schedule: CrashRestartSchedule) {
9278 info!(
9279 module_id,
9280 restart_in_window = schedule.restart_in_window,
9281 delay_ms = schedule.delay.as_millis() as u64,
9282 "respawning after crash"
9283 );
9284}
9285
9286fn respawn_still_pending(snapshot: &SharedSnapshot) -> bool {
9292 matches!(
9293 lock_snapshot(snapshot),
9294 Ok(state) if state.enabled && state.state == ModuleState::Restarting
9295 )
9296}
9297
9298enum NextAction {
9299 Stop {
9300 registration_released: bool,
9301 },
9302 Restart {
9303 schedule: Option<CrashRestartSchedule>,
9304 },
9305}
9306
9307#[allow(clippy::too_many_arguments)]
9308async fn handle_supervisor_command(
9309 command: SupervisorCommand,
9310 spec: &mut ModuleSpec,
9311 runtime: &mut SupervisorRuntimeConfig,
9312 registry: &Arc<Registry>,
9313 process_liveness: &SupervisorProcessLiveness,
9314 snapshot: &SharedSnapshot,
9315 child: &mut Option<SupervisedChild>,
9316 commands: &mut mpsc::Receiver<SupervisorCommand>,
9317 requeued: &mut VecDeque<SupervisorCommand>,
9318) -> bool {
9319 if let SupervisorCommand::SetEnabled {
9320 enabled: true,
9321 reset_budget: false,
9322 reply,
9323 ..
9324 } = command
9325 {
9326 let parked = lock_snapshot(snapshot).ok().and_then(|state| {
9327 (state.config_hold_parked
9328 && state.enabled
9329 && !state.process_alive
9330 && child.is_none()
9331 && state.state == ModuleState::Stopped)
9332 .then(|| {
9333 (
9334 state.config_hold_daemon_initiated,
9335 state.config_hold_approval.clone(),
9336 )
9337 })
9338 });
9339 let result = if let Some((daemon_initiated, approval)) = parked {
9343 runtime.recovery_is_daemon = daemon_initiated;
9344 let failure = FailureContext {
9345 approval,
9346 ..runtime.failure_context(daemon_initiated)
9347 };
9348 let result = set_child_enabled_with_budget_policy(
9349 spec,
9350 runtime,
9351 registry,
9352 process_liveness,
9353 snapshot,
9354 child,
9355 true,
9356 &failure,
9357 false,
9358 )
9359 .await;
9360 if let Err(error) = &result {
9361 record_approved_attempt_failure(spec, snapshot, &failure, error);
9362 }
9363 result
9364 } else {
9365 Ok(false)
9366 };
9367 let _ = reply.send(result);
9368 return true;
9369 }
9370 let blocked = runtime
9371 .config_delivery
9372 .as_ref()
9373 .and_then(|delivery| delivery.0.check_module_hold(&spec.module_id).err())
9374 .filter(|error| error.code != "config_delivery_deferred")
9375 .map(|error| {
9376 config_delivery::gate_hold_error(spec, error, runtime.config_delivery.as_ref())
9377 })
9378 .or_else(|| recovery_block_error(spec, &runtime.child_roster))
9379 .or_else(|| {
9380 runtime
9381 .child_roster
9382 .previous_instance(&spec.module_id)
9383 .map(|(pid, reason)| previous_instance_error(spec, pid, reason))
9384 });
9385 let command = if let Some(error) = blocked {
9386 match command {
9387 SupervisorCommand::Restart { reply, .. } | SupervisorCommand::Reload { reply, .. } => {
9388 let _ = reply.send(Err(error));
9389 return true;
9390 }
9391 SupervisorCommand::Swap { reply, .. } => {
9392 let _ = reply.send(Err(error));
9393 return true;
9394 }
9395 SupervisorCommand::ConfigApplyRestart { .. } => return true,
9396 SupervisorCommand::SetEnabled {
9397 enabled: true,
9398 requester,
9399 reply,
9400 ..
9401 } => {
9402 if matches!(error, SuperviseError::ConfigHeld { .. })
9403 && lock_snapshot(snapshot).is_ok_and(|state| !state.enabled)
9404 && runtime
9405 .config_delivery
9406 .as_ref()
9407 .is_some_and(|delivery| delivery.0.can_stage_unstamp_enable())
9408 {
9409 let mut changed = false;
9413 let result = update_snapshot(snapshot, Some(&spec.module_id), |state| {
9414 changed = !state.enabled;
9415 state.enabled = true;
9416 if child.is_none() && state.waiting_for_previous_pid.is_none() {
9417 state.state = ModuleState::Starting;
9418 state.failed_retry.cancel();
9419 }
9420 })
9421 .and_then(|_| {
9422 if child.is_none()
9423 && lock_snapshot(snapshot)?.waiting_for_previous_pid.is_none()
9424 {
9425 hold_config_spawn(
9426 spec,
9427 runtime,
9428 snapshot,
9429 &runtime.failure_context(false),
9430 )?;
9431 }
9432 Ok(changed)
9433 });
9434 if result.is_ok() && changed {
9435 runtime.start_requester = requester;
9436 }
9437 let _ = reply.send(result);
9438 return true;
9439 }
9440 let _ = reply.send(Err(error));
9441 return true;
9442 }
9443 other => other,
9444 }
9445 } else {
9446 command
9447 };
9448 if matches!(
9451 command,
9452 SupervisorCommand::Drain { .. }
9453 | SupervisorCommand::Retire { .. }
9454 | SupervisorCommand::SetEnabled { .. }
9455 | SupervisorCommand::Restart { .. }
9456 ) {
9457 let _ = update_snapshot(snapshot, Some(&spec.module_id), |state| {
9458 state.failed_retry.cancel();
9459 });
9460 }
9461 match command {
9462 SupervisorCommand::ApprovedConfiguration { approved, reply } => {
9463 let result = async {
9464 let next = approved.module_entry(&spec.module_id).map_err(|error| {
9465 SuperviseError::InvalidSpec { reason: error.to_string() }
9466 })?;
9467 let retry_approved_failure = {
9468 let mut state = lock_snapshot(snapshot)?;
9469 if let Some((generation, detail)) = &state.approved_start_failure {
9470 if *generation == approved.identity() && next.enabled {
9471 return Err(SuperviseError::Spawn {
9472 program: spec.program.clone(),
9473 source: std::io::Error::other(detail.clone()),
9474 cgroup_path: None,
9475 });
9476 }
9477 }
9478 let retry = state.approved_start_failure.is_some();
9479 state.approved_start_failure = None;
9480 retry
9481 };
9482 stage_supervisor_configuration(
9483 spec, runtime, snapshot, next.module_spec(), next.health,
9484 next.drain_timeout_ms,
9485 );
9486 runtime.restart_policy = next.restart;
9487 runtime.register_timeout = next.register_timeout;
9488 if let Some(active) = child.as_mut()
9489 .filter(|child| child.registration_deadline.is_some())
9490 {
9491 active.registration_deadline = Some(active.spawned_at + next.register_timeout);
9492 }
9493 let enabled = lock_snapshot(snapshot)?.enabled;
9494 let retry_gate_stopped = lock_snapshot(snapshot)?
9501 .needs_approved_gate_restart(&approved.identity());
9502 let retry_failed = enabled && child.is_none()
9503 && ((lock_snapshot(snapshot)?.state == ModuleState::Failed
9504 && (retry_approved_failure || approved.changes_entry(&spec.module_id)))
9505 || retry_gate_stopped);
9506 if retry_failed {
9507 update_snapshot(snapshot, Some(&spec.module_id), |state| {
9510 state.failed_retry.cancel();
9511 if retry_gate_stopped {
9512 state.stop_reason = None;
9513 state.config_exit_failed_approval = None;
9514 }
9515 })?;
9516 }
9517 if enabled != next.enabled || retry_failed {
9518 let failure = FailureContext {
9519 approval: next.enabled.then(|| approved.identity()),
9520 ..runtime.failure_context(false)
9521 };
9522 let held = next.enabled && runtime.config_delivery.as_ref()
9523 .is_some_and(|delivery| delivery.0.has_recorded_admission_hold(&spec.module_id)
9524 || delivery.0.check_module_hold(&spec.module_id).is_err_and(|error| error.code == "config_delivery_deferred"));
9525 if held {
9526 reset_restart_count(snapshot, &spec.module_id)?;
9529 update_snapshot(snapshot, Some(&spec.module_id), |state| {
9530 state.enabled = true;
9531 if child.is_none() && state.waiting_for_previous_pid.is_none() {
9532 state.state = ModuleState::Starting;
9533 }
9534 })?;
9535 if child.is_none() && lock_snapshot(snapshot)?.waiting_for_previous_pid.is_none() {
9536 hold_config_spawn(spec, runtime, snapshot, &failure)?;
9537 }
9538 } else {
9539 match set_child_enabled_with_budget_policy(
9540 spec, runtime, registry, process_liveness, snapshot, child,
9541 next.enabled, &failure, true,
9542 ).await {
9543 Ok(_) => {},
9544 Err(SuperviseError::ConfigHeld { .. }) if next.enabled => {},
9545 Err(error) => return Err(error),
9546 }
9547 if !next.enabled {
9548 containment::terminate(&runtime.child_roster.containment).await;
9549 }
9550 }
9551 }
9552 if (child.as_ref().is_some_and(|child| child.registration_spawn.is_some() && child.admission_base.is_none())
9553 || (child.is_none() && lock_snapshot(snapshot)?.state == ModuleState::Restarting
9554 && runtime.config_delivery.as_ref().is_some_and(|delivery| delivery.0.stamp_channel_cutover_pending())))
9555 && runtime.config_delivery.as_ref().is_some_and(|delivery| delivery.0.module_is_gated(&spec.module_id))
9556 {
9557 let failure = FailureContext {
9558 approval: Some(approved.identity()),
9559 ..runtime.failure_context(false)
9560 };
9561 if child.is_some() {
9565 restart_child_with_budget_policy(spec, runtime, registry,
9566 process_liveness, snapshot, child, runtime.drain_timeout, false).await?;
9567 }
9568 update_snapshot(snapshot, Some(&spec.module_id), |state| state.respawn_pending = false)?;
9571 match spawn_and_mark_running_async(spec, runtime, snapshot).await {
9572 Ok(replacement) => *child = Some(replacement),
9573 Err(SuperviseError::ConfigDeliveryDeferred { .. }) => {
9574 defer_config_spawn(spec, runtime, snapshot, RespawnKind::Spawn, &failure)?;
9575 }
9576 Err(SuperviseError::ConfigHeld { .. }) => hold_config_spawn(spec, runtime, snapshot, &failure)?,
9577 Err(error) => {
9578 fail_snapshot(snapshot, Some(&spec.module_id), None, &failure);
9579 return Err(error);
9580 }
9581 }
9582 }
9583 info!(module_id=%spec.module_id, config_generation=approved.generation(), "approved daemon entry reconciled");
9584 Ok(())
9585 }
9586 .await;
9587 if let Err(error) = &result {
9588 let _ = update_snapshot(snapshot, Some(&spec.module_id), |state| {
9589 if state.enabled
9590 && state.state == ModuleState::Failed
9591 && state
9592 .approved_start_failure
9593 .as_ref()
9594 .is_none_or(|(generation, _)| *generation != approved.identity())
9595 {
9596 state.approved_start_failure =
9597 Some((approved.identity(), error.to_string()));
9598 }
9599 });
9600 }
9601 let _ = reply.send(result);
9602 true
9603 }
9604 SupervisorCommand::ConfigPolicyProbe {
9605 connection,
9606 generation,
9607 attempt,
9608 } => {
9609 let Some(delivery) = &runtime.config_delivery else {
9610 return true;
9611 };
9612 let Ok(gate) = delivery.0.module_gate(&spec.module_id) else {
9613 return true;
9614 };
9615 let guard = gate.lock().await;
9616 let pending = delivery
9617 .0
9618 .on_blocking(move |owner| {
9619 owner.policy_probe_pending(connection, generation, attempt)
9620 })
9621 .await
9622 .unwrap_or(false)
9623 && child.is_some()
9624 && registry
9625 .get_module(&spec.module_id)
9626 .ok()
9627 .flatten()
9628 .is_some_and(|registration| registration.connection_id == connection);
9629 drop(guard);
9630 if pending {
9631 run_health_probe_cycle(spec, runtime, registry, process_liveness, snapshot, child)
9632 .await;
9633 }
9634 true
9635 }
9636 SupervisorCommand::ConfigApplyRestart {
9637 generation,
9638 supersedes_swap,
9639 } => {
9640 let Some(owner) = runtime
9641 .config_delivery
9642 .as_ref()
9643 .map(|delivery| delivery.0.clone())
9644 else {
9645 return true;
9646 };
9647 let Ok(gate) = owner.module_gate(&spec.module_id) else {
9648 return true;
9649 };
9650 let _guard = gate.lock().await;
9651 let spawn = child
9652 .as_ref()
9653 .and_then(|child| child.config_delivery.clone());
9654 let module = spec.module_id.clone();
9655 let current = owner
9656 .on_blocking(move |owner| {
9657 Ok((supersedes_swap
9658 || owner
9659 .pending_config_restarts()?
9660 .contains(&(module.clone(), generation)))
9661 && owner.requires_fresher_spawn(&module, spawn.as_ref())?
9662 == Some(generation))
9663 })
9664 .await
9665 .unwrap_or(false);
9666 if !current
9667 || !lock_snapshot(snapshot).is_ok_and(|state| {
9668 state.enabled && !state.respawn_pending && state.process_alive
9669 })
9670 {
9671 return true;
9672 }
9673 drop(_guard);
9674 info!(module_id=%spec.module_id, config_generation=generation, "approved configuration restart preserves the crash budget");
9675 if let Err(error) = restart_child_with_budget_policy(
9676 spec,
9677 runtime,
9678 registry,
9679 process_liveness,
9680 snapshot,
9681 child,
9682 runtime.drain_timeout,
9683 false,
9684 )
9685 .await
9686 {
9687 warn!(module_id=%spec.module_id, %error, "approved configuration restart could not be scheduled");
9688 }
9689 true
9690 }
9691 SupervisorCommand::ConfigExitHold { reply } => {
9692 let idle_start = child.is_none()
9696 && lock_snapshot(snapshot).is_ok_and(|state| {
9697 state.enabled
9698 && state.state == ModuleState::Starting
9699 && !state.process_alive
9700 && !state.respawn_pending
9701 && state.waiting_for_previous_pid.is_none()
9702 && state.waiting_for_previous_detail.is_none()
9703 && state.failed_retry.pending.is_none()
9704 })
9705 && runtime
9706 .scheduled_respawn
9707 .lock()
9708 .unwrap_or_else(|p| p.into_inner())
9709 .is_none()
9710 && runtime
9711 .deferred_reload_reply
9712 .lock()
9713 .unwrap_or_else(|p| p.into_inner())
9714 .is_none();
9715 let result = if idle_start {
9716 try_park_config_attempt(
9717 spec,
9718 runtime,
9719 snapshot,
9720 &runtime.failure_context(runtime.recovery_is_daemon),
9721 None,
9722 )
9723 .map(|_| ())
9724 } else {
9725 Ok(())
9726 };
9727 let _ = reply.send(result);
9728 true
9729 }
9730 SupervisorCommand::ConfigDeliveryFailed { connection, hold } => {
9731 if registry
9732 .get_module(&spec.module_id)
9733 .ok()
9734 .flatten()
9735 .is_none_or(|registration| registration.connection_id != connection)
9736 {
9737 return true;
9738 }
9739 runtime.recovery_is_daemon = true;
9740 if hold {
9744 let notice = begin_forwarding_drain_if_configured(
9745 spec,
9746 runtime,
9747 registry,
9748 snapshot,
9749 Some(true),
9750 RouteCloseReason::Disable,
9751 )
9752 .await;
9753 if let Ok(notice) = notice {
9754 let drained = drain_optional_child(
9755 &spec.module_id,
9756 spec.protocol,
9757 notice,
9758 registry,
9759 runtime.forwarding.as_deref(),
9760 snapshot,
9761 &runtime.terminal_ring,
9762 &runtime.spawn_events,
9763 child,
9764 runtime.drain_timeout,
9765 ModuleState::Failed,
9766 Some(true),
9767 &runtime.failure_context(true),
9768 )
9769 .await;
9770 if drained.is_ok() {
9771 match try_park_config_attempt(
9772 spec,
9773 runtime,
9774 snapshot,
9775 &runtime.failure_context(true),
9776 None,
9777 ) {
9778 Ok(true) => {}
9779 Ok(false) => {
9780 if let Err(error) = budgeted_restart_child(
9781 spec,
9782 runtime,
9783 registry,
9784 process_liveness,
9785 snapshot,
9786 child,
9787 SupervisorHealthStatus::Unknown,
9788 Some("configuration delivery failed after its hold ended"),
9789 unix_ms_now(),
9790 false,
9791 )
9792 .await
9793 {
9794 warn!(module_id=%spec.module_id, %error, "configuration failure restart could not be scheduled");
9795 }
9796 }
9797 Err(error) => {
9798 warn!(module_id=%spec.module_id, %error, "configuration failure could not be parked")
9799 }
9800 }
9801 }
9802 }
9803 } else if let Err(error) = budgeted_restart_child(
9804 spec,
9805 runtime,
9806 registry,
9807 process_liveness,
9808 snapshot,
9809 child,
9810 SupervisorHealthStatus::Unknown,
9811 Some("configuration delivery failed"),
9812 unix_ms_now(),
9813 false,
9814 )
9815 .await
9816 {
9817 warn!(module_id=%spec.module_id, %error, "configuration failure restart could not be scheduled");
9818 }
9819 true
9820 }
9821 SupervisorCommand::Drain { requester, reply } => {
9822 let result = drain_optional_child(
9825 &spec.module_id,
9826 spec.protocol,
9827 StopNotice::NotSent,
9828 registry,
9829 runtime.forwarding.as_deref(),
9830 snapshot,
9831 &runtime.terminal_ring,
9832 &runtime.spawn_events,
9833 child,
9834 runtime.drain_timeout,
9835 ModuleState::Stopped,
9836 None,
9837 &FailureContext::operator(),
9838 )
9839 .await;
9840 containment::terminate(&runtime.child_roster.containment).await;
9841 let registration_released = result.is_ok();
9842 if registration_released {
9843 runtime.record_activity_transition(
9844 "subc.module_stopped",
9845 &spec.module_id,
9846 requester,
9847 );
9848 }
9849 let _ = reply.send(result);
9850 if registration_released {
9851 process_liveness.untrack_if_current(&spec.module_id, snapshot);
9852 }
9853 false
9854 }
9855 SupervisorCommand::Retire { requester, reply } => {
9856 let result = async {
9857 let stop_notice = begin_forwarding_drain_if_configured(
9858 spec,
9859 runtime,
9860 registry,
9861 snapshot,
9862 None,
9863 RouteCloseReason::Disable,
9864 )
9865 .await?;
9866 drain_optional_child(
9867 &spec.module_id,
9868 spec.protocol,
9869 stop_notice,
9870 registry,
9871 runtime.forwarding.as_deref(),
9872 snapshot,
9873 &runtime.terminal_ring,
9874 &runtime.spawn_events,
9875 child,
9876 runtime.drain_timeout,
9877 ModuleState::Stopped,
9878 None,
9879 &FailureContext::operator(),
9880 )
9881 .await
9882 }
9883 .await;
9884 containment::terminate(&runtime.child_roster.containment).await;
9885 let registration_released = result.is_ok();
9886 if registration_released {
9887 runtime.record_activity_transition(
9888 "subc.module_stopped",
9889 &spec.module_id,
9890 requester,
9891 );
9892 }
9893 let _ = reply.send(result);
9894 if registration_released {
9895 process_liveness.untrack_if_current(&spec.module_id, snapshot);
9896 }
9897 false
9898 }
9899 SupervisorCommand::Restart {
9900 requester,
9901 drain_timeout_ms,
9902 received_at_generation,
9903 queued_at,
9904 reply,
9905 } => {
9906 info!(
9910 module_id = %spec.module_id,
9911 queued_ms = u64::try_from(queued_at.elapsed().as_millis()).unwrap_or(u64::MAX),
9912 "restart command dequeued"
9913 );
9914 let validation = match lock_snapshot(snapshot) {
9926 Ok(state) if !state.enabled => Err(SuperviseError::Disabled {
9927 module_id: spec.module_id.clone(),
9928 }),
9929 Ok(_) => Ok(()),
9930 Err(err) => Err(err),
9931 };
9932 let initiated = validation.is_ok();
9933 let _ = reply.send(validation);
9934 let satisfied_by_generation = if initiated
9948 && child
9949 .as_ref()
9950 .is_some_and(|child| child.spawned_at > queued_at)
9951 {
9952 lock_snapshot(snapshot).ok().and_then(|state| {
9953 (!state.configuration_updated_since_spawn).then_some(state.spawn_generation)
9954 })
9955 } else {
9956 None
9957 };
9958 let satisfied_by_pending = initiated
9959 && child.is_none()
9960 && lock_snapshot(snapshot).ok().is_some_and(|mut state| {
9961 let pending = state.respawn_pending && !state.configuration_updated_since_spawn;
9965 if pending {
9966 state.coalesced_restart_pending = true;
9967 }
9968 pending
9969 });
9970 if satisfied_by_pending {
9971 debug!(module_id = %spec.module_id, "restart coalesced into the pending replacement");
9972 } else if let Some(generation) = satisfied_by_generation {
9973 info!(
9974 module_id = %spec.module_id,
9975 received_at_generation,
9976 "restart already satisfied by generation {generation}; not restarting again"
9977 );
9978 } else if initiated {
9979 let drain_timeout = drain_timeout_ms
9982 .map(Duration::from_millis)
9983 .unwrap_or(runtime.drain_timeout);
9984 if let Err(err) = restart_child(
9985 spec,
9986 runtime,
9987 registry,
9988 process_liveness,
9989 snapshot,
9990 child,
9991 drain_timeout,
9992 )
9993 .await
9994 {
9995 warn!(
9996 module_id = %spec.module_id,
9997 error = %err,
9998 "operator restart failed after initiation ack; module state carries the outcome"
9999 );
10000 let _ = update_snapshot(snapshot, Some(&spec.module_id), |state| {
10001 state.state = ModuleState::Failed;
10002 clear_current_process_facts(state);
10003 });
10004 } else {
10005 runtime.start_requester = requester;
10006 runtime.pending_activity_restart = true;
10007 }
10008 }
10009 true
10010 }
10011 SupervisorCommand::Reload { requester, reply } => {
10012 let result =
10013 reload_child(spec, runtime, registry, process_liveness, snapshot, child).await;
10014 if result.is_ok() {
10015 runtime.start_requester = requester;
10016 runtime.pending_activity_restart = true;
10017 }
10018 if result.is_ok()
10019 && runtime
10020 .scheduled_respawn
10021 .lock()
10022 .unwrap_or_else(|p| p.into_inner())
10023 .as_ref()
10024 .is_some_and(|pending| matches!(pending.kind, RespawnKind::Reload))
10025 {
10026 *runtime
10027 .deferred_reload_reply
10028 .lock()
10029 .unwrap_or_else(|p| p.into_inner()) = Some(reply);
10030 } else {
10031 let _ = reply.send(result);
10032 }
10033 true
10034 }
10035 SupervisorCommand::SetEnabled {
10036 enabled,
10037 requester,
10038 reply,
10039 reset_budget,
10040 } => {
10041 let previous_requester = runtime.start_requester.clone();
10042 if enabled {
10043 runtime.start_requester = requester.clone();
10044 }
10045 let result = set_child_enabled_with_budget_policy(
10046 spec,
10047 runtime,
10048 registry,
10049 process_liveness,
10050 snapshot,
10051 child,
10052 enabled,
10053 &runtime.failure_context(false),
10054 reset_budget,
10055 )
10056 .await;
10057 if !enabled {
10058 containment::terminate(&runtime.child_roster.containment).await;
10059 }
10060 if matches!(result, Ok(true)) {
10061 runtime.pending_activity_restart = false;
10062 if enabled {
10063 runtime.start_requester = requester;
10064 } else {
10065 runtime.start_requester = None;
10066 runtime.record_activity_transition(
10067 "subc.module_disabled",
10068 &spec.module_id,
10069 requester,
10070 );
10071 }
10072 } else {
10073 runtime.start_requester = previous_requester;
10074 }
10075 let _ = reply.send(result);
10076 true
10077 }
10078 SupervisorCommand::UpdateConfiguration {
10079 spec: next_spec,
10080 health,
10081 drain_timeout_ms,
10082 reply,
10083 } => {
10084 if runtime
10085 .config_delivery
10086 .as_ref()
10087 .is_some_and(|delivery| delivery.0.file_configuration_is_gated(&spec.module_id))
10088 {
10089 let _ = reply.send(Err(SuperviseError::ConfigHeld {
10090 module_id: spec.module_id.clone(),
10091 code: "config_gated".into(),
10092 detail: "live file reconfiguration cannot replace stamped authority".into(),
10093 }));
10094 return true;
10095 }
10096 stage_supervisor_configuration(
10097 spec,
10098 runtime,
10099 snapshot,
10100 next_spec,
10101 health,
10102 drain_timeout_ms,
10103 );
10104 let _ = update_snapshot(snapshot, Some(&spec.module_id), |state| {
10105 state.configuration_updated_since_spawn = true;
10106 });
10107 let _ = reply.send(Ok(()));
10108 true
10109 }
10110 SupervisorCommand::Swap {
10111 requester,
10112 ready_timeout,
10113 reply,
10114 } => {
10115 if spec.carry_containment
10116 || containment::current(&runtime.child_roster.containment).is_some()
10117 {
10118 let _ = reply.send(Err(SuperviseError::InvalidSpec {
10119 reason: "a carried container cannot overlap a swap candidate".to_owned(),
10120 }));
10121 return true;
10122 }
10123 let mut swap_runtime = runtime.clone();
10124 swap_runtime.start_requester = requester;
10125 let before = lock_snapshot(snapshot)
10126 .ok()
10127 .map(|state| state.spawn_generation);
10128 let end = swap::run_swap(
10129 spec,
10130 &swap_runtime,
10131 registry,
10132 process_liveness,
10133 snapshot,
10134 child,
10135 commands,
10136 ready_timeout.unwrap_or(DEFAULT_SWAP_READY_TIMEOUT),
10137 reply,
10138 )
10139 .await;
10140 if before
10141 != lock_snapshot(snapshot)
10142 .ok()
10143 .map(|state| state.spawn_generation)
10144 {
10145 runtime.start_requester = swap_runtime.start_requester;
10146 }
10147 requeued.extend(end.requeue);
10148 true
10149 }
10150 }
10151}
10152
10153fn stage_supervisor_configuration(
10154 spec: &mut ModuleSpec,
10155 runtime: &mut SupervisorRuntimeConfig,
10156 snapshot: &SharedSnapshot,
10157 next_spec: ModuleSpec,
10158 health: HealthConfig,
10159 drain_timeout_ms: Option<u64>,
10160) {
10161 if let Some(handle) = &runtime.supervisor_handle {
10162 handle.apply_identity_configuration(&next_spec);
10163 }
10164 let launch_changed = *spec != next_spec;
10165 *spec = next_spec;
10166 if launch_changed {
10167 let _ = update_snapshot(snapshot, Some(&spec.module_id), |state| {
10168 state.configuration_updated_since_spawn = true
10169 });
10170 }
10171 let health_changed = runtime.health != health;
10172 runtime.health = health;
10173 if health_changed {
10176 let _ = update_snapshot(snapshot, Some(&spec.module_id), |state| {
10177 state.health = ModuleHealthStatus::default();
10178 state.last_probe_evidence = None;
10179 });
10180 }
10181 runtime.drain_timeout = drain_timeout_ms
10182 .map(Duration::from_millis)
10183 .unwrap_or(runtime.default_drain_timeout);
10184 *runtime
10185 .effective_drain_timeout
10186 .lock()
10187 .unwrap_or_else(|p| p.into_inner()) = runtime.drain_timeout;
10188}
10189
10190async fn restart_child(
10191 spec: &ModuleSpec,
10192 runtime: &SupervisorRuntimeConfig,
10193 registry: &Registry,
10194 process_liveness: &SupervisorProcessLiveness,
10195 snapshot: &SharedSnapshot,
10196 child: &mut Option<SupervisedChild>,
10197 drain_timeout: Duration,
10198) -> Result<(), SuperviseError> {
10199 restart_child_with_budget_policy(
10200 spec,
10201 runtime,
10202 registry,
10203 process_liveness,
10204 snapshot,
10205 child,
10206 drain_timeout,
10207 true,
10208 )
10209 .await
10210}
10211
10212#[allow(clippy::too_many_arguments)]
10213async fn restart_child_with_budget_policy(
10214 spec: &ModuleSpec,
10215 runtime: &SupervisorRuntimeConfig,
10216 registry: &Registry,
10217 process_liveness: &SupervisorProcessLiveness,
10218 snapshot: &SharedSnapshot,
10219 child: &mut Option<SupervisedChild>,
10220 drain_timeout: Duration,
10221 reset_budget: bool,
10222) -> Result<(), SuperviseError> {
10223 if !lock_snapshot(snapshot)?.enabled {
10225 return Err(SuperviseError::Disabled {
10226 module_id: spec.module_id.clone(),
10227 });
10228 }
10229 let stop_notice = begin_forwarding_drain_with_timeout(
10230 spec,
10231 runtime,
10232 registry,
10233 snapshot,
10234 None,
10235 RouteCloseReason::Restart,
10236 drain_timeout,
10237 )
10238 .await?;
10239
10240 if child.is_some() {
10241 drain_optional_child(
10242 &spec.module_id,
10243 spec.protocol,
10244 stop_notice,
10245 registry,
10246 runtime.forwarding.as_deref(),
10247 snapshot,
10248 &runtime.terminal_ring,
10249 &runtime.spawn_events,
10250 child,
10251 drain_timeout,
10252 ModuleState::Restarting,
10253 Some(true),
10254 &FailureContext::operator(),
10255 )
10256 .await?;
10257 } else {
10258 update_snapshot(snapshot, Some(&spec.module_id), |state| {
10259 state.enabled = true;
10260 state.state = ModuleState::Restarting;
10261 clear_current_process_facts(state);
10262 })?;
10263 release_dead_registration(
10264 registry,
10265 runtime.forwarding.as_deref(),
10266 snapshot,
10267 &spec.module_id,
10268 &runtime.failure_context(false),
10269 )
10270 .await?;
10271 }
10272
10273 if reset_budget {
10274 reset_restart_count(snapshot, &spec.module_id)?;
10275 }
10276 process_liveness.track(spec.module_id.clone(), Arc::clone(snapshot));
10277 schedule_respawn(
10278 runtime,
10279 snapshot,
10280 &spec.module_id,
10281 runtime.restart_policy.backoff,
10282 RespawnKind::Spawn,
10283 false,
10284 )
10285}
10286
10287async fn reload_child(
10288 spec: &ModuleSpec,
10289 runtime: &SupervisorRuntimeConfig,
10290 registry: &Registry,
10291 process_liveness: &SupervisorProcessLiveness,
10292 snapshot: &SharedSnapshot,
10293 child: &mut Option<SupervisedChild>,
10294) -> Result<(), SuperviseError> {
10295 if !lock_snapshot(snapshot)?.enabled {
10297 return Err(SuperviseError::Disabled {
10298 module_id: spec.module_id.clone(),
10299 });
10300 }
10301 let stop_notice = begin_forwarding_drain(
10302 spec,
10303 runtime,
10304 registry,
10305 snapshot,
10306 Some(true),
10307 RouteCloseReason::Reload,
10308 )
10309 .await?;
10310
10311 if child.is_some() {
10312 drain_optional_child(
10313 &spec.module_id,
10314 spec.protocol,
10315 stop_notice,
10316 registry,
10317 runtime.forwarding.as_deref(),
10318 snapshot,
10319 &runtime.terminal_ring,
10320 &runtime.spawn_events,
10321 child,
10322 runtime.drain_timeout,
10323 ModuleState::Restarting,
10324 Some(true),
10325 &FailureContext::operator(),
10326 )
10327 .await?;
10328 } else {
10329 update_snapshot(snapshot, Some(&spec.module_id), |state| {
10330 state.enabled = true;
10331 state.state = ModuleState::Restarting;
10332 clear_current_process_facts(state);
10333 })?;
10334 release_dead_registration(
10335 registry,
10336 runtime.forwarding.as_deref(),
10337 snapshot,
10338 &spec.module_id,
10339 &runtime.failure_context(false),
10340 )
10341 .await?;
10342 }
10343
10344 reset_restart_count(snapshot, &spec.module_id)?;
10345 process_liveness.track(spec.module_id.clone(), Arc::clone(snapshot));
10346 schedule_respawn(
10347 runtime,
10348 snapshot,
10349 &spec.module_id,
10350 runtime.restart_policy.backoff,
10351 RespawnKind::Reload,
10352 false,
10353 )
10354}
10355
10356async fn finish_reload_child(
10357 spec: &ModuleSpec,
10358 runtime: &SupervisorRuntimeConfig,
10359 registry: &Registry,
10360 process_liveness: &SupervisorProcessLiveness,
10361 snapshot: &SharedSnapshot,
10362 child: &mut Option<SupervisedChild>,
10363) -> Result<(), SuperviseError> {
10364 process_liveness.track(spec.module_id.clone(), Arc::clone(snapshot));
10365 let next_child = match spawn_and_mark_running_async(spec, runtime, snapshot).await {
10366 Ok(next_child) => next_child,
10367 Err(error @ SuperviseError::ConfigDeliveryDeferred { .. }) => {
10368 defer_config_spawn(
10369 spec,
10370 runtime,
10371 snapshot,
10372 RespawnKind::Reload,
10373 &runtime.failure_context(runtime.recovery_is_daemon),
10374 )?;
10375 return Err(error);
10376 }
10377 Err(error @ SuperviseError::ConfigHeld { .. }) => {
10378 let failure = runtime.failure_context(runtime.recovery_is_daemon);
10379 if try_park_config_attempt(spec, runtime, snapshot, &failure, None)? {
10380 return Err(error);
10381 }
10382 defer_config_spawn(spec, runtime, snapshot, RespawnKind::Reload, &failure)?;
10383 return Err(SuperviseError::ConfigDeliveryDeferred {
10384 module_id: spec.module_id.clone(),
10385 });
10386 }
10387 Err(err) => {
10388 return handle_reload_spawn_failure(
10389 spec,
10390 runtime,
10391 process_liveness,
10392 snapshot,
10393 child,
10394 format!("new child failed to spawn: {err}"),
10395 )
10396 .await;
10397 }
10398 };
10399 *child = Some(next_child);
10400
10401 let wait_outcome = {
10402 let active_child = child.as_mut().expect("new reload child was just stored");
10403 wait_for_registration_after_reload(
10404 registry,
10405 &spec.module_id,
10406 snapshot,
10407 active_child,
10408 REGISTRY_RELEASE_TIMEOUT,
10409 )
10410 .await?
10411 };
10412
10413 match wait_outcome {
10414 RegistrationWaitOutcome::Registered => {
10415 child
10416 .as_mut()
10417 .expect("registered reload child")
10418 .registration_deadline = None;
10419 debug!(module_id = %spec.module_id, "supervised module reloaded and registered");
10420 Ok(())
10421 }
10422 RegistrationWaitOutcome::Exited(exit_report) => {
10423 if let Some(active_child) = child.as_mut() {
10424 active_child.drain_stderr(&spec.module_id).await;
10425 }
10426 let mut exited_child = child.take().expect("exited reload child is still stored");
10429 #[cfg(test)]
10430 if let Some(gate) = &runtime.test_reload_exit_record_gate {
10431 gate.reached.notify_one();
10432 gate.resume.notified().await;
10433 }
10434 let result = handle_reload_child_registration_failure(
10435 spec,
10436 runtime,
10437 registry,
10438 process_liveness,
10439 snapshot,
10440 child,
10441 ReloadRegistrationFailure {
10442 exit_report: registration_failure_exit_report(exit_report),
10443 reason: exited_child
10444 .spawn_failure
10445 .clone()
10446 .unwrap_or_else(|| "new child exited before registering".to_string()),
10447 },
10448 )
10449 .await;
10450 exited_child
10451 .finish_exit(false)
10452 .await
10453 .map_err(|source| SuperviseError::Wait {
10454 module_id: spec.module_id.clone(),
10455 source,
10456 })?;
10457 result
10458 }
10459 RegistrationWaitOutcome::TimedOut => {
10460 let mut timed_out_child = child
10461 .take()
10462 .expect("timed-out reload child is still running");
10463 timed_out_child
10464 .start_kill()
10465 .map_err(|source| SuperviseError::Kill {
10466 module_id: spec.module_id.clone(),
10467 source,
10468 })?;
10469 let status = timed_out_child
10470 .wait()
10471 .await
10472 .map_err(|source| SuperviseError::Wait {
10473 module_id: spec.module_id.clone(),
10474 source,
10475 })?;
10476 timed_out_child.drain_stderr(&spec.module_id).await;
10477 let result = handle_reload_child_registration_failure(
10478 spec,
10479 runtime,
10480 registry,
10481 process_liveness,
10482 snapshot,
10483 child,
10484 ReloadRegistrationFailure {
10485 exit_report: registration_failure_exit_report(classify_reaped_child_exit(
10486 snapshot,
10487 &timed_out_child,
10488 &status,
10489 )),
10490 reason: format!(
10491 "new child did not register within {:?}",
10492 REGISTRY_RELEASE_TIMEOUT
10493 ),
10494 },
10495 )
10496 .await;
10497 timed_out_child
10498 .finish_exit(false)
10499 .await
10500 .map_err(|source| SuperviseError::Wait {
10501 module_id: spec.module_id.clone(),
10502 source,
10503 })?;
10504 result
10505 }
10506 }
10507}
10508
10509#[allow(clippy::too_many_arguments)]
10510async fn set_child_enabled(
10511 spec: &ModuleSpec,
10512 runtime: &SupervisorRuntimeConfig,
10513 registry: &Registry,
10514 process_liveness: &SupervisorProcessLiveness,
10515 snapshot: &SharedSnapshot,
10516 child: &mut Option<SupervisedChild>,
10517 enabled: bool,
10518 failure: &FailureContext,
10519) -> Result<bool, SuperviseError> {
10520 set_child_enabled_with_budget_policy(
10521 spec,
10522 runtime,
10523 registry,
10524 process_liveness,
10525 snapshot,
10526 child,
10527 enabled,
10528 failure,
10529 true,
10530 )
10531 .await
10532}
10533
10534#[allow(clippy::too_many_arguments)]
10535async fn set_child_enabled_with_budget_policy(
10536 spec: &ModuleSpec,
10537 runtime: &SupervisorRuntimeConfig,
10538 registry: &Registry,
10539 process_liveness: &SupervisorProcessLiveness,
10540 snapshot: &SharedSnapshot,
10541 child: &mut Option<SupervisedChild>,
10542 enabled: bool,
10543 failure: &FailureContext,
10544 reset_budget: bool,
10545) -> Result<bool, SuperviseError> {
10546 let (current_enabled, current_state, respawn_pending) = {
10547 let state = lock_snapshot(snapshot)?;
10548 (state.enabled, state.state, state.respawn_pending)
10549 };
10550 let revive_terminal = enabled
10558 && current_enabled
10559 && child.is_none()
10560 && (matches!(current_state, ModuleState::Failed | ModuleState::Stopped)
10561 || (current_state == ModuleState::Restarting && !respawn_pending));
10562 if current_enabled == enabled && !revive_terminal {
10563 return Ok(false);
10564 }
10565
10566 if enabled {
10567 update_snapshot(snapshot, Some(&spec.module_id), |state| {
10568 state.enabled = true;
10569 state.state = ModuleState::Starting;
10570 clear_current_process_facts(state);
10571 })?;
10572 #[cfg(test)]
10573 if runtime.test_seed_stale_facts_before_enable_spawn {
10574 update_snapshot(snapshot, Some(&spec.module_id), |state| {
10575 state.process_alive = true;
10576 state.pid = Some(41);
10577 state.spawned_at_ms = Some(42);
10578 state.spawned_from = Some(PathBuf::from("/spawned/module"));
10579 state.spawned_file_identity = Some(SpawnedFileIdentity {
10580 device: 43,
10581 inode: 44,
10582 #[cfg(windows)]
10583 windows_image: None,
10584 });
10585 })?;
10586 }
10587 release_dead_registration(
10588 registry,
10589 runtime.forwarding.as_deref(),
10590 snapshot,
10591 &spec.module_id,
10592 failure,
10593 )
10594 .await?;
10595 if reset_budget {
10596 reset_restart_count(snapshot, &spec.module_id)?;
10597 }
10598 process_liveness.track(spec.module_id.clone(), Arc::clone(snapshot));
10599 let next_child = match spawn_and_mark_running_async(spec, runtime, snapshot).await {
10600 Ok(next_child) => next_child,
10601 Err(SuperviseError::ConfigDeliveryDeferred { .. }) => {
10602 defer_config_spawn(spec, runtime, snapshot, RespawnKind::Spawn, failure)?;
10603 return Ok(true);
10604 }
10605 Err(error @ SuperviseError::ConfigHeld { .. }) => {
10606 if let Err(parking_error) = hold_config_spawn(spec, runtime, snapshot, failure) {
10607 process_liveness.untrack_if_current(&spec.module_id, snapshot);
10608 return Err(parking_error);
10609 }
10610 return Err(error);
10611 }
10612 Err(err) => {
10613 fail_snapshot(snapshot, Some(&spec.module_id), None, failure);
10614 process_liveness.untrack_if_current(&spec.module_id, snapshot);
10615 return Err(err);
10616 }
10617 };
10618 *child = Some(next_child);
10619 debug!(module_id = %spec.module_id, "supervised module enabled");
10620 Ok(true)
10621 } else {
10622 update_snapshot(snapshot, Some(&spec.module_id), |state| {
10623 state.config_hold_parked = false;
10624 state.config_hold_daemon_initiated = false;
10625 state.config_hold_approval = None;
10626 })?;
10627 let stop_notice = begin_forwarding_drain_if_configured(
10628 spec,
10629 runtime,
10630 registry,
10631 snapshot,
10632 Some(false),
10633 RouteCloseReason::Disable,
10634 )
10635 .await?;
10636 drain_optional_child(
10637 &spec.module_id,
10638 spec.protocol,
10639 stop_notice,
10640 registry,
10641 runtime.forwarding.as_deref(),
10642 snapshot,
10643 &runtime.terminal_ring,
10644 &runtime.spawn_events,
10645 child,
10646 runtime.drain_timeout,
10647 ModuleState::Disabled,
10648 Some(false),
10649 failure,
10650 )
10651 .await?;
10652 debug!(module_id = %spec.module_id, "supervised module disabled");
10653 Ok(true)
10654 }
10655}
10656
10657#[allow(clippy::too_many_arguments)]
10658async fn on_child_exit(
10659 spec: &ModuleSpec,
10660 policy: RestartPolicy,
10661 registry: &Registry,
10662 snapshot: &SharedSnapshot,
10663 terminal_ring: &Arc<Mutex<TerminalRing>>,
10664 spawn_events: &SpawnEventFeed,
10665 roster: &ChildRoster,
10666 exit_report: ExitReport,
10667 daemon_initiated: bool,
10668) -> NextAction {
10669 if roster.is_closed() {
10675 return on_child_exit_during_daemon_shutdown(
10676 spec,
10677 registry,
10678 snapshot,
10679 terminal_ring,
10680 spawn_events,
10681 exit_report,
10682 )
10683 .await;
10684 }
10685 let unrequested_clean_exit_of_protocol_none = exit_report.kind == ExitKind::Clean
10701 && running_protocol(spec, snapshot) == ModuleProtocol::None;
10702 match exit_report.kind {
10703 ExitKind::Clean if !unrequested_clean_exit_of_protocol_none => {
10704 info!(
10705 module_id = %spec.module_id,
10706 exit_code = ?exit_report.code,
10707 exit_signal = ?exit_report.signal,
10708 "supervised module exited cleanly"
10709 );
10710 if let Err(err) = publish_after_terminal_record(
10711 &spec.module_id,
10712 || {
10713 record_terminal(
10714 &spec.module_id,
10715 terminal_ring,
10716 spawn_events,
10717 &exit_report,
10718 TerminalDisposition::Stopped,
10719 )
10720 },
10721 || {
10722 update_snapshot(snapshot, Some(&spec.module_id), |state| {
10723 state.state = ModuleState::Stopped;
10724 clear_current_process_facts(state);
10725 state.last_exit = Some(exit_report.clone());
10726 })
10727 },
10728 ) {
10729 error!(module_id = %spec.module_id, error = %err, "failed to publish clean module exit");
10730 }
10731 let registration_released = match wait_for_registration_release(
10732 registry,
10733 &spec.module_id,
10734 REGISTRY_RELEASE_TIMEOUT,
10735 )
10736 .await
10737 {
10738 Ok(()) => true,
10739 Err(err) => {
10740 warn!(module_id = %spec.module_id, error = %err, "registration still active after clean exit");
10741 false
10742 }
10743 };
10744 NextAction::Stop {
10745 registration_released,
10746 }
10747 }
10748 ExitKind::Clean | ExitKind::Crash => {
10749 if unrequested_clean_exit_of_protocol_none {
10750 warn!(
10751 module_id = %spec.module_id,
10752 exit_code = ?exit_report.code,
10753 exit_signal = ?exit_report.signal,
10754 "protocol-none module exited cleanly without a stop request; handling it as a crash"
10755 );
10756 } else {
10757 warn!(
10758 module_id = %spec.module_id,
10759 exit_code = ?exit_report.code,
10760 exit_signal = ?exit_report.signal,
10761 "supervised module exited abnormally (crash)"
10762 );
10763 }
10764 let mut restart_schedule = None;
10765 let mut disposition = TerminalDisposition::Disabled;
10766 let mut final_state = ModuleState::Disabled;
10767 let mut disposition_detail = lock_snapshot(snapshot)
10771 .ok()
10772 .and_then(|mut state| state.spawn_failure.take());
10773 let now = Instant::now();
10774 if let Err(err) = update_snapshot(snapshot, Some(&spec.module_id), |state| {
10775 if state.enabled {
10776 if let Some(schedule) = state.next_crash_restart(&policy, now) {
10777 state.respawn_pending = true;
10783 final_state = ModuleState::Restarting;
10784 restart_schedule = Some(schedule);
10785 disposition = TerminalDisposition::Restarting;
10786 } else {
10787 final_state = ModuleState::Failed;
10788 disposition = TerminalDisposition::Failed;
10789 let budget = policy.budget_exhausted_detail();
10790 disposition_detail =
10791 Some(disposition_detail.take().map_or_else(
10792 || budget.clone(),
10793 |cause| format!("{cause}; {budget}"),
10794 ));
10795 }
10796 } else {
10797 disposition = TerminalDisposition::Disabled;
10798 }
10799 }) {
10800 error!(module_id = %spec.module_id, error = %err, "failed to record crashed module exit");
10801 return NextAction::Stop {
10802 registration_released: false,
10803 };
10804 }
10805 if disposition == TerminalDisposition::Failed {
10806 error!(
10811 module_id = %spec.module_id,
10812 max_restarts = policy.max_restarts,
10813 window_secs = policy.window.as_secs(),
10814 "module stopped: {}",
10815 policy.budget_exhausted_detail()
10816 );
10817 }
10818 if let Err(err) = publish_after_terminal_record(
10819 &spec.module_id,
10820 || {
10821 record_terminal_with_detail(
10822 &spec.module_id,
10823 terminal_ring,
10824 spawn_events,
10825 &exit_report,
10826 disposition,
10827 disposition_detail,
10828 )
10829 },
10830 || {
10831 update_snapshot(snapshot, Some(&spec.module_id), |state| {
10832 state.state = final_state;
10833 clear_current_process_facts(state);
10834 if final_state == ModuleState::Failed
10835 && daemon_initiated
10836 && !roster.is_closed()
10837 {
10838 state.failed_retry.schedule(&spec.module_id);
10839 }
10840 state.last_exit = Some(exit_report.clone());
10841 })
10842 },
10843 ) {
10844 error!(module_id = %spec.module_id, error = %err, "failed to publish crashed module exit");
10845 }
10846
10847 if let Some(schedule) = restart_schedule {
10848 NextAction::Restart {
10849 schedule: Some(schedule),
10850 }
10851 } else {
10852 let registration_released = match wait_for_registration_release(
10853 registry,
10854 &spec.module_id,
10855 REGISTRY_RELEASE_TIMEOUT,
10856 )
10857 .await
10858 {
10859 Ok(()) => true,
10860 Err(err) => {
10861 warn!(module_id = %spec.module_id, error = %err, "registration still active after failed module");
10862 false
10863 }
10864 };
10865 NextAction::Stop {
10866 registration_released,
10867 }
10868 }
10869 }
10870 ExitKind::DeliberateSeverance => {
10871 warn!(
10872 module_id = %spec.module_id,
10873 exit_code = ?exit_report.code,
10874 exit_signal = ?exit_report.signal,
10875 "supervised module exited after deliberate connection severance"
10876 );
10877 let should_restart = match lock_snapshot(snapshot) {
10878 Ok(state) => state.enabled,
10879 Err(err) => {
10880 error!(module_id = %spec.module_id, error = %err, "failed to read deliberately severed module state");
10881 return NextAction::Stop {
10882 registration_released: false,
10883 };
10884 }
10885 };
10886 let final_state = if should_restart {
10887 ModuleState::Restarting
10888 } else {
10889 ModuleState::Disabled
10890 };
10891 if let Err(err) = publish_after_terminal_record(
10892 &spec.module_id,
10893 || {
10894 record_terminal(
10895 &spec.module_id,
10896 terminal_ring,
10897 spawn_events,
10898 &exit_report,
10899 terminal_disposition(final_state),
10900 )
10901 },
10902 || {
10903 update_snapshot(snapshot, Some(&spec.module_id), |state| {
10904 state.state = final_state;
10905 clear_current_process_facts(state);
10906 state.last_exit = Some(exit_report.clone());
10907 state.lifetime_restarts += 1;
10908 })
10909 },
10910 ) {
10911 error!(module_id = %spec.module_id, error = %err, "failed to publish deliberately severed module exit");
10912 return NextAction::Stop {
10913 registration_released: false,
10914 };
10915 }
10916
10917 if should_restart {
10918 NextAction::Restart { schedule: None }
10919 } else {
10920 let registration_released = match wait_for_registration_release(
10921 registry,
10922 &spec.module_id,
10923 REGISTRY_RELEASE_TIMEOUT,
10924 )
10925 .await
10926 {
10927 Ok(()) => true,
10928 Err(err) => {
10929 warn!(module_id = %spec.module_id, error = %err, "registration still active after deliberately severed module exit");
10930 false
10931 }
10932 };
10933 NextAction::Stop {
10934 registration_released,
10935 }
10936 }
10937 }
10938 }
10939}
10940
10941async fn on_child_exit_during_daemon_shutdown(
10942 spec: &ModuleSpec,
10943 registry: &Registry,
10944 snapshot: &SharedSnapshot,
10945 terminal_ring: &Arc<Mutex<TerminalRing>>,
10946 spawn_events: &SpawnEventFeed,
10947 exit_report: ExitReport,
10948) -> NextAction {
10949 info!(
10950 module_id = %spec.module_id,
10951 exit_code = ?exit_report.code,
10952 exit_signal = ?exit_report.signal,
10953 exit_kind = ?exit_report.kind,
10954 "supervised module exited during daemon shutdown; not restarting it"
10955 );
10956 if let Err(err) = publish_after_terminal_record(
10957 &spec.module_id,
10958 || {
10959 record_terminal(
10960 &spec.module_id,
10961 terminal_ring,
10962 spawn_events,
10963 &exit_report,
10964 TerminalDisposition::DaemonShutdown,
10965 )
10966 },
10967 || {
10968 update_snapshot(snapshot, Some(&spec.module_id), |state| {
10969 state.state = ModuleState::Stopped;
10970 clear_current_process_facts(state);
10971 state.last_exit = Some(exit_report.clone());
10972 })
10973 },
10974 ) {
10975 error!(module_id = %spec.module_id, error = %err, "failed to publish module exit during daemon shutdown");
10976 }
10977 let registration_released =
10978 wait_for_registration_release(registry, &spec.module_id, REGISTRY_RELEASE_TIMEOUT)
10979 .await
10980 .is_ok();
10981 NextAction::Stop {
10982 registration_released,
10983 }
10984}
10985
10986fn publish_after_terminal_record<T>(
10991 module_id: &str,
10992 record: impl FnOnce(),
10993 publish: impl FnOnce() -> T,
10994) -> T {
10995 let recorded = std::panic::catch_unwind(std::panic::AssertUnwindSafe(record));
10996 let published = publish();
10997 if let Err(panic) = recorded {
10998 error!(
10999 module_id,
11000 "terminal recording panicked; published final state before resuming unwind"
11001 );
11002 std::panic::resume_unwind(panic);
11003 }
11004 published
11005}
11006
11007fn fail_after_wait_error(
11008 module_id: &str,
11009 snapshot: &SharedSnapshot,
11010 terminal_ring: &Arc<Mutex<TerminalRing>>,
11011 spawn_events: &SpawnEventFeed,
11012 detail: String,
11013 failure: &FailureContext,
11014) {
11015 let report = wait_error_exit_report();
11016 publish_after_terminal_record(
11017 module_id,
11018 || {
11019 record_terminal_with_detail(
11020 module_id,
11021 terminal_ring,
11022 spawn_events,
11023 &report,
11024 TerminalDisposition::Failed,
11025 Some(detail),
11026 )
11027 },
11028 || fail_snapshot(snapshot, Some(module_id), Some(report.clone()), failure),
11029 );
11030}
11031
11032#[cfg(test)]
11033fn record_wait_error_terminal(
11034 module_id: &str,
11035 terminal_ring: &Arc<Mutex<TerminalRing>>,
11036 spawn_events: &SpawnEventFeed,
11037) {
11038 record_terminal(
11039 module_id,
11040 terminal_ring,
11041 spawn_events,
11042 &wait_error_exit_report(),
11043 TerminalDisposition::Failed,
11044 );
11045}
11046
11047fn record_terminal(
11048 module_id: &str,
11049 terminal_ring: &Arc<Mutex<TerminalRing>>,
11050 spawn_events: &SpawnEventFeed,
11051 exit_report: &ExitReport,
11052 disposition: TerminalDisposition,
11053) {
11054 record_terminal_with_detail(
11055 module_id,
11056 terminal_ring,
11057 spawn_events,
11058 exit_report,
11059 disposition,
11060 None,
11061 );
11062}
11063
11064fn durable_terminal_history_of(
11068 terminal_ring: &Mutex<TerminalRing>,
11069 module_id: &str,
11070) -> subc_control::TerminalHistory {
11071 let read = terminal_ring
11072 .lock()
11073 .unwrap_or_else(|p| p.into_inner())
11074 .capture_durable_history();
11075 read.read(module_id)
11076}
11077
11078fn record_terminal_with_detail(
11079 module_id: &str,
11080 terminal_ring: &Arc<Mutex<TerminalRing>>,
11081 spawn_events: &SpawnEventFeed,
11082 exit_report: &ExitReport,
11083 disposition: TerminalDisposition,
11084 disposition_detail: Option<String>,
11085) {
11086 #[cfg(test)]
11087 terminal_history_tests::before_terminal_record(spawn_events);
11088 spawn_events.emit_exited(module_id, exit_report.code, exit_report.signal);
11089 let record = TerminalRecord {
11090 exit_code: exit_report.code,
11091 exit_signal: exit_report.signal,
11092 at_ms: exit_report.at_ms,
11093 disposition,
11094 exit_kind: exit_report.kind.into(),
11095 disposition_detail,
11096 };
11097 terminal_ring
11098 .lock()
11099 .unwrap_or_else(|poisoned| poisoned.into_inner())
11100 .record_exit(module_id, record);
11101}
11102
11103fn untrack_if_registration_released(
11104 process_liveness: &SupervisorProcessLiveness,
11105 registry: &Registry,
11106 module_id: &str,
11107 snapshot: &SharedSnapshot,
11108) {
11109 match registry.get_module(module_id) {
11110 Ok(None) => process_liveness.untrack_if_current(module_id, snapshot),
11111 Ok(Some(_)) => {}
11112 Err(err) => {
11113 warn!(module_id, error = %err, "could not determine whether supervisor liveness can be untracked");
11114 }
11115 }
11116}
11117
11118#[cfg(test)]
11132fn apply_wire_spawn_args(
11133 command: &mut Command,
11134 spec: &ModuleSpec,
11135 connection_file_path: Option<&std::path::Path>,
11136 handle: Option<&SupervisorHandle>,
11137) -> Result<Option<NonceHandoff>, SuperviseError> {
11138 apply_wire_spawn_args_for_role(
11139 command,
11140 spec,
11141 connection_file_path,
11142 handle,
11143 SpawnRole::Plain,
11144 )
11145}
11146
11147#[cfg(unix)]
11152type NonceHandoff = subc_os::LaunchNonceHandoff;
11153#[cfg(windows)]
11154type NonceHandoff = subc_os::LaunchNoncePipeHandoff;
11155#[cfg(not(any(unix, windows)))]
11156type NonceHandoff = std::convert::Infallible;
11157
11158#[cfg(test)]
11176fn apply_wire_spawn_args_for_role(
11177 command: &mut Command,
11178 spec: &ModuleSpec,
11179 connection_file_path: Option<&std::path::Path>,
11180 handle: Option<&SupervisorHandle>,
11181 role: SpawnRole,
11182) -> Result<Option<NonceHandoff>, SuperviseError> {
11183 apply_wire_spawn_args_for_admission(command, spec, connection_file_path, handle, role, None)
11184}
11185
11186fn apply_wire_spawn_args_for_admission(
11187 command: &mut Command,
11188 spec: &ModuleSpec,
11189 connection_file_path: Option<&std::path::Path>,
11190 handle: Option<&SupervisorHandle>,
11191 role: SpawnRole,
11192 admission: Option<&crate::config_gate::runtime::SpawnAdmission>,
11193) -> Result<Option<NonceHandoff>, SuperviseError> {
11194 command.env(SUBC_MODULE_ID_ENV, &spec.module_id);
11195 command.env_remove(subc_os::LAUNCH_NONCE_FD_ENV);
11200 #[cfg(windows)]
11201 {
11202 command.env_remove(subc_os::LAUNCH_NONCE_PIPE_ENV);
11203 command.env_remove(subc_os::LAUNCH_NONCE_PIPE_FALLBACK_ENV);
11204 }
11205 command.env_remove(SUBC_LAUNCH_NONCE_ENV);
11207 command.env_remove(subc_os::registration_channel::REGISTRATION_FD_ENV);
11208 command.env_remove(subc_os::registration_channel::REGISTRATION_PIPE_ENV);
11209 command.env_remove(subc_os::registration_channel::DAEMON_PID_ENV);
11210 if role == SpawnRole::Plain {
11211 if let Some(handle) = handle {
11212 #[cfg(all(test, unix))]
11213 {
11214 let pause = {
11215 let mut slot = handle.plain_identity_policy_pause.lock().unwrap();
11216 if slot
11217 .as_ref()
11218 .is_some_and(|(module, _)| module == &spec.module_id)
11219 {
11220 slot.take().map(|(_, pause)| pause)
11221 } else {
11222 None
11223 }
11224 };
11225 if let Some(pause) = pause {
11226 pause.reached.send(()).unwrap();
11227 pause.release.recv().unwrap();
11228 }
11229 }
11230 handle.apply_identity_policy(spec, spec.protocol);
11234 }
11235 }
11236 if spec.protocol == ModuleProtocol::None {
11237 return Ok(None);
11238 }
11239 if let Some(connection_file_path) = connection_file_path {
11240 command.arg(SUBC_ARG).arg(connection_file_path);
11241 }
11242
11243 if admission.is_some_and(|admission| admission.launch.is_some()) {
11244 if let Some(handle) = handle {
11245 match role {
11246 SpawnRole::Plain => {
11247 handle
11248 .spawn_nonces
11249 .lock()
11250 .unwrap_or_else(|p| p.into_inner())
11251 .remove(&spec.module_id);
11252 if spec.reserved {
11253 handle
11254 .reserved_nonces
11255 .lock()
11256 .unwrap_or_else(|p| p.into_inner())
11257 .insert(spec.module_id.clone(), None);
11258 }
11259 }
11260 SpawnRole::SwapCandidate => handle.open_swap(&spec.module_id, String::new()),
11263 }
11264 }
11265 return Ok(None);
11266 }
11267 let nonce = generate_launch_nonce()?;
11271 if let Some(handle) = handle {
11272 match role {
11273 SpawnRole::Plain => {
11274 handle.set_spawn_nonce(&spec.module_id, nonce.clone());
11275 if spec.reserved {
11276 handle.set_reserved_nonce(&spec.module_id, nonce.clone());
11277 }
11278 }
11279 SpawnRole::SwapCandidate => handle.open_swap(&spec.module_id, nonce.clone()),
11280 }
11281 }
11282 #[cfg(unix)]
11283 let handoff = {
11284 let handoff =
11285 subc_os::LaunchNonceHandoff::new(&nonce).map_err(|source| SuperviseError::Spawn {
11286 program: spec.program.clone(),
11287 source,
11288 cgroup_path: None,
11289 })?;
11290 command.env(subc_os::LAUNCH_NONCE_FD_ENV, handoff.fd_env_value());
11291 Some(handoff)
11292 };
11293 #[cfg(windows)]
11294 let handoff = prepare_windows_nonce_pipe(command, spec, &nonce);
11295 #[cfg(not(any(unix, windows)))]
11296 let handoff = None;
11297 #[cfg(not(unix))]
11301 command.env(SUBC_LAUNCH_NONCE_ENV, nonce);
11302 Ok(handoff)
11303}
11304
11305pub(crate) fn apply_child_env(command: &mut Command, spec: &ModuleSpec) {
11306 command.env_remove(CK_LOG_ENV);
11307 command.env_remove(SUBC_SPAWN_ROLE_ENV);
11314 for (key, value) in &spec.env {
11315 if matches!(
11319 key.as_str(),
11320 CAPTURE_MAX_FILE_MB_ENV | CAPTURE_KEEP_ENV | CAPTURE_MAX_AGE_DAYS_ENV
11321 ) || key == SUBC_SPAWN_ROLE_ENV
11322 {
11323 continue;
11324 }
11325 command.env(key, value);
11326 }
11327}
11328
11329#[cfg(windows)]
11330fn prepare_windows_registration_pipe(
11331 command: &mut Command,
11332 spec: &ModuleSpec,
11333) -> Result<Option<SpawnRegistrationChannel>, SuperviseError> {
11334 if spec.protocol == ModuleProtocol::None {
11335 return Ok(None);
11336 }
11337 let pipe = subc_os::registration_channel::RegistrationPipe::new().map_err(|source| {
11338 SuperviseError::Spawn {
11339 program: spec.program.clone(),
11340 source,
11341 cgroup_path: None,
11342 }
11343 })?;
11344 command
11345 .env(
11346 subc_os::registration_channel::REGISTRATION_PIPE_ENV,
11347 pipe.name(),
11348 )
11349 .env(
11350 subc_os::registration_channel::DAEMON_PID_ENV,
11351 std::process::id().to_string(),
11352 );
11353 Ok(Some(pipe))
11354}
11355
11356#[cfg(windows)]
11357fn prepare_windows_nonce_pipe(
11358 command: &mut Command,
11359 spec: &ModuleSpec,
11360 nonce: &str,
11361) -> Option<NonceHandoff> {
11362 match subc_os::LaunchNoncePipeHandoff::new(nonce) {
11363 Ok(handoff) => {
11364 command.env(subc_os::LAUNCH_NONCE_PIPE_ENV, handoff.name());
11365 command.env(subc_os::LAUNCH_NONCE_PIPE_FALLBACK_ENV, "env");
11370 Some(handoff)
11371 }
11372 Err(error) => {
11373 warn!(module_id = %spec.module_id, %error, "could not prepare launch nonce pipe; retaining the environment handoff");
11374 None
11375 }
11376 }
11377}
11378
11379#[cfg(windows)]
11380fn start_windows_nonce_pipe(
11381 handoff: Option<NonceHandoff>,
11382 pid: u32,
11383 spec: &ModuleSpec,
11384) -> Option<subc_os::LaunchNoncePipeDelivery> {
11385 handoff.and_then(|handoff| match handoff.serve(pid, REGISTRY_RELEASE_TIMEOUT) {
11389 Ok(delivery) => Some(delivery),
11390 Err(error) => {
11391 warn!(module_id = %spec.module_id, %error, "could not serve launch nonce pipe; retaining the environment handoff");
11392 None
11393 }
11394 })
11395}
11396
11397fn observed_launch_nonce_source(snapshot: &SupervisorSnapshot) -> Option<String> {
11398 #[cfg(windows)]
11399 if snapshot.process_alive && snapshot.spawned_protocol == Some(ModuleProtocol::Subc) {
11400 return Some(
11401 if snapshot
11402 .pipe_consumed
11403 .as_ref()
11404 .is_some_and(|consumed| consumed.get().is_some())
11405 {
11406 "pipe"
11407 } else {
11408 "env"
11409 }
11410 .to_owned(),
11411 );
11412 }
11413 let _ = snapshot;
11414 None
11415}
11416
11417#[derive(Debug, Clone, Copy, PartialEq, Eq)]
11420enum SpawnRole {
11421 Plain,
11422 SwapCandidate,
11423}
11424
11425fn apply_spawn_role(command: &mut Command, role: SpawnRole) {
11428 if role == SpawnRole::SwapCandidate {
11429 command.env(SUBC_SPAWN_ROLE_ENV, SPAWN_ROLE_SWAP_CANDIDATE);
11430 }
11431}
11432
11433fn previous_instance_error(
11434 spec: &ModuleSpec,
11435 pid: u32,
11436 reason: crate::live_children::RetainedReason,
11437) -> SuperviseError {
11438 SuperviseError::Spawn {
11439 program: spec.program.clone(),
11440 source: io::Error::new(io::ErrorKind::WouldBlock, reason.waiting_message(pid)),
11441 cgroup_path: None,
11442 }
11443}
11444
11445fn carries_containment(configured: bool, has_container: bool) -> bool {
11446 configured || has_container
11447}
11448
11449fn recovery_block_error(spec: &ModuleSpec, roster: &ChildRoster) -> Option<SuperviseError> {
11450 if !carries_containment(
11451 spec.carry_containment,
11452 containment::current(&roster.containment).is_some(),
11453 ) {
11454 return None;
11455 }
11456 Some(SuperviseError::Spawn {
11457 program: spec.program.clone(),
11458 source: io::Error::new(io::ErrorKind::WouldBlock, roster.recovery_block()?),
11459 cgroup_path: None,
11460 })
11461}
11462
11463#[allow(clippy::too_many_arguments)]
11464fn spawn_child(
11465 spec: &ModuleSpec,
11466 connection_file_path: Option<&std::path::Path>,
11467 handle: Option<&SupervisorHandle>,
11468 ring: &Arc<Mutex<StderrRing>>,
11469 capture_logs_dir: Option<&std::path::Path>,
11470 roster: &ChildRoster,
11471 config_delivery: Option<config_delivery::SpawnDelivery<'_>>,
11472 #[cfg(target_os = "linux")] cgroup_placement: Option<&subc_cgroup::Placement>,
11473 register_timeout: Duration,
11474) -> Result<SupervisedChild, SuperviseError> {
11475 #[cfg(all(test, unix))]
11476 if let Some(source) = containment_tests::join_failure(spec, roster) {
11477 containment::terminate_blocking(&roster.containment);
11478 return Err(SuperviseError::Spawn {
11479 program: spec.program.clone(),
11480 source,
11481 cgroup_path: None,
11482 });
11483 }
11484 let result = spawn_child_in_slot(
11485 spec,
11486 connection_file_path,
11487 handle,
11488 ring,
11489 capture_logs_dir,
11490 roster,
11491 config_delivery,
11492 #[cfg(target_os = "linux")]
11493 cgroup_placement,
11494 SpawnRole::Plain,
11495 false,
11496 register_timeout,
11497 );
11498 if result.is_err() {
11499 if roster.admission_pending(&spec.module_id) {
11500 containment::terminate_deferred(&roster.containment);
11501 } else {
11502 containment::terminate_blocking(&roster.containment);
11503 }
11504 }
11505 result
11506}
11507
11508#[allow(clippy::too_many_arguments)]
11521fn spawn_child_in_slot(
11522 spec: &ModuleSpec,
11523 connection_file_path: Option<&std::path::Path>,
11524 handle: Option<&SupervisorHandle>,
11525 ring: &Arc<Mutex<StderrRing>>,
11526 capture_logs_dir: Option<&std::path::Path>,
11527 roster: &ChildRoster,
11528 config_delivery: Option<config_delivery::SpawnDelivery<'_>>,
11529 #[cfg(target_os = "linux")] cgroup_placement: Option<&subc_cgroup::Placement>,
11530 role: SpawnRole,
11531 alternate_slot: bool,
11532 register_timeout: Duration,
11533) -> Result<SupervisedChild, SuperviseError> {
11534 let prepared = prepare_child_in_slot(
11535 spec,
11536 connection_file_path,
11537 handle,
11538 capture_logs_dir,
11539 roster,
11540 config_delivery,
11541 #[cfg(target_os = "linux")]
11542 cgroup_placement,
11543 role,
11544 alternate_slot,
11545 false,
11546 )?;
11547 let gate = config_delivery
11548 .map(|delivery| delivery.module_gate(spec))
11549 .transpose()?;
11550 let _gate = gate
11551 .as_ref()
11552 .map(|gate| {
11553 gate.try_lock()
11554 .map_err(|_| SuperviseError::ConfigDeliveryDeferred {
11555 module_id: spec.module_id.clone(),
11556 })
11557 })
11558 .transpose()?;
11559 if let (Some(delivery), Some(admission)) = (config_delivery, &prepared.spawn_admission) {
11560 delivery.check(spec, admission)?;
11561 }
11562 let checked_base = prepared
11563 .spawn_admission
11564 .as_ref()
11565 .and_then(|admission| admission.approved_base())
11566 .cloned();
11567 let mut result = launch_child_in_slot(spec, ring, roster, prepared, register_timeout);
11568 drop(_gate);
11569 if let (Some(delivery), Err(SuperviseError::ConfigHeld { code, detail, .. })) =
11570 (config_delivery, &result)
11571 {
11572 if let Some(base) = checked_base.as_ref().filter(|_| code == "program_refused") {
11573 if !delivery.record_refusal(spec, base, detail)? {
11574 result = Err(spawn_plan_stale(spec));
11575 }
11576 }
11577 }
11578 result
11579}
11580
11581struct AsyncInitialIdentity<'a> {
11582 supervisor: &'a Supervisor,
11583 module: String,
11584 committed: bool,
11585}
11586impl Drop for AsyncInitialIdentity<'_> {
11587 fn drop(&mut self) {
11588 if !self.committed {
11589 self.supervisor.abandon_unrostered(&self.module);
11590 }
11591 }
11592}
11593
11594struct PreparedChild {
11598 script_delivery: Option<script::Delivery>,
11599 registration_spawn: Option<Arc<RegistrationSpawn>>,
11600 registration_handle: Option<SupervisorHandle>,
11601 #[cfg(any(unix, windows))]
11602 registration_channel: Option<SpawnRegistrationChannel>,
11603 config_context: Option<config_delivery::ConfigDelivery>,
11604 binding_publication: Option<ProgramBindingPublication>,
11605 launch_spec: ModuleSpec,
11606 spawn_admission: Option<crate::config_gate::runtime::SpawnAdmission>,
11607 command: Command,
11608 carrying: bool,
11609 carried: Option<Arc<containment::CarriedContainment>>,
11610 #[cfg(unix)]
11611 launch_identity: Option<containment::LaunchIdentity>,
11612 #[cfg(unix)]
11613 group_target: containment::GroupTarget,
11614 #[cfg(target_os = "linux")]
11615 cgroup_placement: Option<subc_cgroup::Placement>,
11616 #[cfg(target_os = "linux")]
11617 cgroup_name: String,
11618 #[cfg(target_os = "linux")]
11619 cgroup_cleanup: PreparedCgroup,
11620 cgroup_path: Option<PathBuf>,
11621 #[cfg(unix)]
11622 admission_gate: Option<admission::Gate>,
11623 output_sink: ChildOutputSink,
11624 #[cfg(target_os = "macos")]
11625 privacy_exec: Option<PrivacyExec>,
11626 #[cfg(windows)]
11627 image_capture: subc_os::ExecutableCapture,
11628 #[cfg(windows)]
11629 nonce_handoff: Option<NonceHandoff>,
11630 spawned_at: Instant,
11631 config_delivery: Option<crate::config_gate::runtime::DeliverySpawn>,
11632 delivery_registration: Option<config_delivery::RegistrationGuard>,
11633 program_binding: Option<SpawnProgramBinding>,
11634}
11635
11636#[cfg(target_os = "linux")]
11637struct PreparedCgroup(Option<(subc_cgroup::Placement, String)>);
11638#[cfg(target_os = "linux")]
11639impl Drop for PreparedCgroup {
11640 fn drop(&mut self) {
11641 if let Some((placement, name)) = &self.0 {
11642 remove_module_cgroup(placement, name);
11643 }
11644 }
11645}
11646
11647pub(crate) struct SpawnProgramBinding {
11648 pub(crate) path: PathBuf,
11649 _file: std::fs::File,
11652 pub(crate) identity: subc_os::RetainedFileIdentity,
11653}
11654
11655impl SpawnProgramBinding {
11656 pub(crate) fn capture(spec: &ModuleSpec) -> Result<Self, SuperviseError> {
11657 Self::capture_after_resolution(spec, |_| {})
11658 }
11659
11660 fn capture_after_resolution(
11661 spec: &ModuleSpec,
11662 before_open: impl FnOnce(&std::path::Path),
11663 ) -> Result<Self, SuperviseError> {
11664 require_program_search_path(spec)?;
11665 #[cfg(windows)]
11666 let windows_capture =
11667 windows_spawn_capture(spec).map_err(|source| SuperviseError::Spawn {
11668 program: spec.program.clone(),
11669 source,
11670 cgroup_path: None,
11671 })?;
11672 #[cfg(windows)]
11673 let path = windows_capture.path().to_path_buf();
11674 #[cfg(not(windows))]
11675 let path = if spec.program.is_absolute() || spec.program.components().count() > 1 {
11676 spec.program.clone()
11677 } else {
11678 let path = spec
11679 .env
11680 .iter()
11681 .rev()
11682 .find(|(key, _)| key == "PATH")
11683 .map(|(_, path)| std::ffi::OsString::from(path))
11684 .or_else(|| std::env::var_os("PATH"))
11685 .unwrap_or_else(|| {
11686 if cfg!(target_os = "macos") {
11687 "/usr/bin:/bin".into()
11690 } else {
11691 Default::default()
11692 }
11693 });
11694 std::env::split_paths(&path)
11695 .map(|dir| dir.join(&spec.program))
11696 .find(|path| {
11697 if !path.is_file() {
11698 return false;
11699 }
11700 #[cfg(unix)]
11701 {
11702 rustix::fs::accessat(
11703 rustix::fs::CWD,
11704 path,
11705 rustix::fs::Access::EXEC_OK,
11706 rustix::fs::AtFlags::EACCESS,
11707 )
11708 .is_ok()
11709 }
11710 #[cfg(not(unix))]
11711 {
11712 true
11713 }
11714 })
11715 .ok_or_else(|| SuperviseError::Spawn {
11716 program: spec.program.clone(),
11717 source: io::Error::new(io::ErrorKind::NotFound, "executable not found in PATH"),
11718 cgroup_path: None,
11719 })?
11720 };
11721 let resolved = std::fs::canonicalize(&path).map_err(|source| SuperviseError::Spawn {
11722 program: spec.program.clone(),
11723 source,
11724 cgroup_path: None,
11725 })?;
11726 before_open(&resolved);
11727 let file =
11730 subc_os::open_regular_nofollow(&resolved).map_err(|source| SuperviseError::Spawn {
11731 program: spec.program.clone(),
11732 source: io::Error::other(source),
11733 cgroup_path: None,
11734 })?;
11735 let identity =
11736 subc_os::retained_file_identity(&file).map_err(|source| SuperviseError::Spawn {
11737 program: spec.program.clone(),
11738 source,
11739 cgroup_path: None,
11740 })?;
11741 Ok(Self {
11742 path,
11743 _file: file,
11744 identity,
11745 })
11746 }
11747 fn current(&self) -> bool {
11748 #[cfg(unix)]
11749 {
11750 subc_os::file_identity(&self.path) == Some(self.identity)
11751 }
11752 #[cfg(windows)]
11753 {
11754 std::fs::canonicalize(&self.path)
11755 .ok()
11756 .and_then(|path| subc_os::open_regular_nofollow(&path).ok())
11757 .and_then(|file| subc_os::retained_file_identity(&file).ok())
11758 == Some(self.identity)
11759 }
11760 }
11761}
11762
11763fn spawn_plan_stale(spec: &ModuleSpec) -> SuperviseError {
11764 SuperviseError::ConfigDeliveryDeferred {
11765 module_id: spec.module_id.clone(),
11766 }
11767}
11768
11769fn require_program_search_path(spec: &ModuleSpec) -> Result<(), SuperviseError> {
11770 if !cfg!(target_os = "macos")
11771 && !spec.program.is_absolute()
11772 && spec.program.components().count() == 1
11773 && !spec.env.iter().any(|(key, _)| {
11774 if cfg!(windows) {
11775 key.eq_ignore_ascii_case("PATH")
11776 } else {
11777 key == "PATH"
11778 }
11779 })
11780 && std::env::var_os("PATH").is_none()
11781 {
11782 return Err(SuperviseError::Spawn {
11785 program: spec.program.clone(),
11786 source: io::Error::new(io::ErrorKind::InvalidInput, "bare program requires PATH"),
11787 cgroup_path: None,
11788 });
11789 }
11790 Ok(())
11791}
11792
11793async fn spawn_child_async(
11794 spec: &ModuleSpec,
11795 runtime: &SupervisorRuntimeConfig,
11796) -> Result<SupervisedChild, SuperviseError> {
11797 #[cfg(all(test, unix))]
11798 if let Some(source) = containment_tests::join_failure(spec, &runtime.child_roster) {
11799 containment::terminate_blocking(&runtime.child_roster.containment);
11800 return Err(SuperviseError::Spawn {
11801 program: spec.program.clone(),
11802 source,
11803 cgroup_path: None,
11804 });
11805 }
11806 let result = spawn_child_in_slot_async(spec, runtime, SpawnRole::Plain, false).await;
11807 if result.is_err() {
11808 if runtime.child_roster.admission_pending(&spec.module_id) {
11809 containment::terminate_deferred(&runtime.child_roster.containment);
11810 } else {
11811 containment::terminate_blocking(&runtime.child_roster.containment);
11812 }
11813 }
11814 result
11815}
11816
11817async fn spawn_child_in_slot_async(
11818 spec: &ModuleSpec,
11819 runtime: &SupervisorRuntimeConfig,
11820 role: SpawnRole,
11821 alternate_slot: bool,
11822) -> Result<SupervisedChild, SuperviseError> {
11823 let gate = runtime
11824 .config_delivery
11825 .as_ref()
11826 .map(|delivery| delivery.0.module_gate(&spec.module_id))
11827 .transpose()
11828 .map_err(|error| SuperviseError::InvalidSpec {
11829 reason: error.to_string(),
11830 })?;
11831 if gate.as_ref().is_some_and(|gate| gate.try_lock().is_err()) {
11832 return Err(SuperviseError::ConfigDeliveryDeferred {
11833 module_id: spec.module_id.clone(),
11834 });
11835 }
11836 let prepared_spec = spec.clone();
11837 let prepared_runtime = runtime.clone();
11838 let prepared = tokio::task::spawn_blocking(move || {
11839 let prepared = prepare_child_in_slot(
11840 &prepared_spec,
11841 prepared_runtime.connection_file_path.as_deref(),
11842 prepared_runtime.supervisor_handle.as_ref(),
11843 prepared_runtime.capture_logs_dir.as_deref(),
11844 &prepared_runtime.child_roster,
11845 prepared_runtime.spawn_delivery(),
11846 #[cfg(target_os = "linux")]
11847 prepared_runtime.cgroup_placement.as_ref(),
11848 role,
11849 alternate_slot,
11850 true,
11851 );
11852 prepared
11853 })
11854 .await
11855 .map_err(|_| SuperviseError::InvalidSpec {
11856 reason: "configuration launch preparation stopped".into(),
11857 })??;
11858 if let Some(registration) = &prepared.delivery_registration {
11859 if !registration.launch_current()? {
11860 return Err(spawn_plan_stale(spec));
11861 }
11862 }
11863 let _gate = match gate {
11864 Some(gate) => {
11865 Some(
11866 gate.try_lock_owned()
11867 .map_err(|_| SuperviseError::ConfigDeliveryDeferred {
11868 module_id: spec.module_id.clone(),
11869 })?,
11870 )
11871 }
11872 None => None,
11873 };
11874 if let (Some(delivery), Some(admission)) = (runtime.spawn_delivery(), &prepared.spawn_admission)
11875 {
11876 delivery.check(spec, admission)?;
11877 }
11878 if prepared
11879 .spawn_admission
11880 .as_ref()
11881 .is_none_or(|admission| admission.launch.is_none())
11882 && prepared
11883 .program_binding
11884 .as_ref()
11885 .is_some_and(|binding| !binding.current())
11886 {
11887 return Err(spawn_plan_stale(spec));
11888 }
11889 let checked_base = prepared
11891 .spawn_admission
11892 .as_ref()
11893 .and_then(|admission| admission.approved_base())
11894 .cloned();
11895 let mut result = launch_child_in_slot(
11896 spec,
11897 &runtime.stderr_ring,
11898 &runtime.child_roster,
11899 prepared,
11900 runtime.register_timeout,
11901 );
11902 drop(_gate);
11903 if let (Some(delivery), Err(SuperviseError::ConfigHeld { code, detail, .. })) =
11904 (runtime.config_delivery.as_ref(), &result)
11905 {
11906 if let Some(base) = checked_base.filter(|_| code == "program_refused") {
11907 let module = spec.module_id.clone();
11908 let identity = subc_config::raw_digest(detail.as_bytes());
11909 let recorded = delivery
11910 .0
11911 .on_blocking(move |owner| owner.record_program_refusal(&module, &base, identity))
11912 .await;
11913 match recorded {
11914 Ok(true) => {}
11915 Ok(false) => result = Err(spawn_plan_stale(spec)),
11916 Err(error) => {
11917 result = Err(config_delivery::gate_hold_error(
11918 spec,
11919 error,
11920 Some(delivery),
11921 ))
11922 }
11923 }
11924 }
11925 }
11926 result
11927}
11928
11929fn validate_spawn_program(
11930 spec: &ModuleSpec,
11931 admission: Option<&crate::config_gate::runtime::SpawnAdmission>,
11932 delivery: Option<config_delivery::SpawnDelivery<'_>>,
11933) -> Result<(), SuperviseError> {
11934 if let Some(admission) = admission {
11935 if let Some(program) = &admission.launch {
11936 if let Err(error) = program.validate_for_spawn() {
11937 if let (Some(context), Some(base)) = (delivery, admission.approved_base()) {
11941 if error.code == "program_refused"
11942 && !context.record_refusal(spec, base, &error.detail)?
11943 {
11944 return Err(spawn_plan_stale(spec));
11945 }
11946 }
11947 return Err(config_delivery::gate_hold_error(
11948 spec,
11949 error,
11950 delivery.map(|context| context.runtime()).as_ref(),
11951 ));
11952 }
11953 }
11954 }
11955 Ok(())
11956}
11957
11958#[allow(clippy::too_many_arguments)]
11959fn prepare_child_in_slot(
11960 spec: &ModuleSpec,
11961 connection_file_path: Option<&std::path::Path>,
11962 handle: Option<&SupervisorHandle>,
11963 capture_logs_dir: Option<&std::path::Path>,
11964 roster: &ChildRoster,
11965 config_delivery: Option<config_delivery::SpawnDelivery<'_>>,
11966 #[cfg(target_os = "linux")] cgroup_placement: Option<&subc_cgroup::Placement>,
11967 role: SpawnRole,
11968 alternate_slot: bool,
11969 asynchronous: bool,
11970) -> Result<PreparedChild, SuperviseError> {
11971 require_program_search_path(spec)?;
11972 let spawn_admission = config_delivery
11973 .map(|delivery| delivery.prepare(spec))
11974 .transpose()?;
11975 #[cfg(target_os = "macos")]
11976 let gated = spawn_admission
11977 .as_ref()
11978 .is_some_and(|admission| admission.approved_base().is_some());
11979 let mut launch_spec = spawn_admission
11980 .as_ref()
11981 .and_then(|admission| admission.launch.as_ref())
11982 .map(|launch| launch.spec.clone())
11983 .unwrap_or_else(|| spec.clone());
11984 if let Some(script) = &launch_spec.script {
11985 launch_spec.program =
11986 script::runner_path(script.runtime).map_err(|source| SuperviseError::Spawn {
11987 program: launch_spec.program.clone(),
11988 source,
11989 cgroup_path: None,
11990 })?;
11991 }
11992 let script_content = if let Some(entry) = &launch_spec.script {
11993 Some(
11994 spawn_admission
11995 .as_ref()
11996 .and_then(|admission| admission.launch.as_ref())
11997 .and_then(|launch| launch.script.clone())
11998 .map_or_else(|| script::ScriptBytes::read(&entry.path), Ok)
11999 .map_err(|error| config_delivery::gate_hold_error(&launch_spec, error, None))?,
12000 )
12001 } else {
12002 None
12003 };
12004 let spec = &launch_spec;
12005 if let Some(error) = recovery_block_error(spec, roster) {
12006 return Err(error);
12007 }
12008 if let Some((pid, reason)) = roster.previous_instance(&spec.module_id) {
12009 return Err(previous_instance_error(spec, pid, reason));
12010 }
12011 if roster.is_closed() {
12012 return Err(SuperviseError::Spawn {
12013 program: spec.program.clone(),
12014 source: io::Error::other("the daemon is shutting down; not starting a new process"),
12015 cgroup_path: None,
12016 });
12017 }
12018 let carried = containment::current(&roster.containment);
12019 let carrying = carries_containment(spec.carry_containment, carried.is_some());
12020 if role != SpawnRole::Plain && carrying {
12021 return Err(SuperviseError::InvalidSpec {
12022 reason: "a carried container cannot overlap a swap candidate".to_owned(),
12023 });
12024 }
12025 #[cfg(unix)]
12026 let launch_identity = if carrying && carried.is_none() {
12027 Some(
12028 containment::LaunchIdentity::capture().map_err(|source| SuperviseError::Spawn {
12029 program: spec.program.clone(),
12030 source,
12031 cgroup_path: None,
12032 })?,
12033 )
12034 } else {
12035 None
12036 };
12037 #[cfg(target_os = "linux")]
12038 let cgroup_placement = match &carried {
12039 Some(container) => container.cgroup().map(|(placement, _)| placement),
12040 None => cgroup_placement,
12041 };
12042 #[cfg(unix)]
12043 let group_target = if let Some(container) = carried
12044 .as_ref()
12045 .filter(|container| container.group().is_some())
12046 {
12047 containment::GroupTarget::Pinned(Arc::clone(container))
12048 } else {
12049 #[cfg(target_os = "linux")]
12050 let cgroup_container = carrying && cgroup_placement.is_some();
12051 #[cfg(not(target_os = "linux"))]
12052 let cgroup_container = false;
12053 if cgroup_container {
12054 containment::GroupTarget::NoGroup
12055 } else {
12056 containment::GroupTarget::OwnLeader
12057 }
12058 };
12059 #[cfg(target_os = "linux")]
12060 let cgroup_name = {
12061 if let Some((_, name)) = carried.as_ref().and_then(|container| container.cgroup()) {
12066 name.to_owned()
12067 } else if cgroup_placement.is_none() {
12068 swap::cgroup_name(&spec.module_id, alternate_slot)
12069 } else {
12070 let nonce = generate_launch_nonce()?;
12071 let suffix = u128::from_str_radix(&nonce[..32], 16).expect("hex launch nonce");
12072 let mut end = spec.module_id.len().min(64);
12075 while !spec.module_id.is_char_boundary(end) {
12076 end -= 1;
12077 }
12078 format!(
12079 "{}_{suffix}",
12080 swap::cgroup_name(&spec.module_id[..end], alternate_slot)
12081 )
12082 }
12083 };
12084 #[cfg(not(target_os = "linux"))]
12085 let _ = alternate_slot;
12086 #[cfg(windows)]
12087 let image_capture = windows_spawn_capture(spec).map_err(|source| SuperviseError::Spawn {
12088 program: spec.program.clone(),
12089 source,
12090 cgroup_path: None,
12091 })?;
12092 #[cfg(windows)]
12093 let program_binding = if asynchronous {
12094 let mut resolved = spec.clone();
12095 resolved.program = image_capture.path().to_path_buf();
12096 Some(SpawnProgramBinding::capture(&resolved)?)
12097 } else {
12098 None
12099 };
12100 #[cfg(not(windows))]
12101 let program_binding = (asynchronous || cfg!(target_os = "macos"))
12102 .then(|| SpawnProgramBinding::capture(spec))
12103 .transpose()?;
12104 #[cfg(target_os = "macos")]
12105 validate_spawn_program(spec, spawn_admission.as_ref(), config_delivery)?;
12106 #[cfg(target_os = "macos")]
12107 let (mut command, privacy_exec, exec_ack) = privacy_command(
12108 spec,
12109 program_binding
12110 .as_ref()
12111 .expect("macOS captures every launch"),
12112 roster,
12113 gated,
12117 )?;
12118 #[cfg(windows)]
12119 let mut command = Command::new(image_capture.path());
12120 #[cfg(not(any(target_os = "macos", windows)))]
12121 let mut command = Command::new(
12122 program_binding
12123 .as_ref()
12124 .map_or(spec.program.as_path(), |binding| binding.path.as_path()),
12125 );
12126 command.args(&spec.args);
12127 apply_child_env(&mut command, spec);
12157 if spec.script.is_some() {
12158 script::remove_runtime_env(&mut command, spec);
12159 }
12160 let origin_witness = crate::program_origin::LaunchWitness::capture_command(spec, &command)
12161 .map_err(|source| SuperviseError::Spawn {
12162 program: spec.program.clone(),
12163 source,
12164 cgroup_path: None,
12165 })?;
12166 command.current_dir(&origin_witness.working_directory);
12167 apply_spawn_role(&mut command, role);
12168 let nonce_handoff = apply_wire_spawn_args_for_admission(
12169 &mut command,
12170 spec,
12171 connection_file_path,
12172 handle,
12173 role,
12174 spawn_admission.as_ref(),
12175 )?;
12176 #[cfg(unix)]
12177 let mut slots = subc_os::registration_channel::ChildDescriptorSlots::default();
12178 command.env_remove(subc_os::script_pipe::SCRIPT_FD_ENV);
12179 let script_delivery = script_content
12180 .map(|content| {
12181 script::Delivery::prepare(
12182 content,
12183 &mut command,
12184 #[cfg(unix)]
12185 &mut slots,
12186 )
12187 })
12188 .transpose()
12189 .map_err(|source| SuperviseError::Spawn {
12190 program: spec.program.clone(),
12191 source,
12192 cgroup_path: None,
12193 })?;
12194 #[cfg(unix)]
12195 let registration_channel = if spec.protocol == ModuleProtocol::Subc {
12196 let (parent, child, metadata) = subc_os::registration_channel::registration_socketpair()
12197 .map_err(|source| SuperviseError::Spawn {
12198 program: spec.program.clone(),
12199 source,
12200 cgroup_path: None,
12201 })?;
12202 slots
12203 .channel(child)
12204 .map_err(|source| SuperviseError::Spawn {
12205 program: spec.program.clone(),
12206 source,
12207 cgroup_path: None,
12208 })?;
12209 command
12210 .env(subc_os::registration_channel::REGISTRATION_FD_ENV, metadata)
12211 .env(
12212 subc_os::registration_channel::DAEMON_PID_ENV,
12213 std::process::id().to_string(),
12214 );
12215 Some(parent)
12216 } else {
12217 None
12218 };
12219 #[cfg(windows)]
12220 let registration_channel = prepare_windows_registration_pipe(&mut command, spec)?;
12221 let binding_publication = if spawn_admission
12222 .as_ref()
12223 .is_some_and(|admission| admission.launch.is_some())
12224 {
12225 handle.and_then(|handle| handle.begin_program_binding(&spec.module_id, role))
12226 } else {
12227 None
12228 };
12229 let registration_spawn = (spec.protocol == ModuleProtocol::Subc)
12230 .then(|| {
12231 handle.map(|handle| {
12232 let spawn = RegistrationSpawn::new_with_binding_requirement(
12233 handle,
12234 &spec.module_id,
12235 role,
12236 binding_publication.clone(),
12237 spawn_admission
12238 .as_ref()
12239 .is_some_and(|admission| admission.launch.is_some()),
12240 );
12241 if let Some(admission) = &spawn_admission {
12242 admission.initialize_origin(&spawn.authority, &origin_witness);
12243 }
12244 handle.record_nonce_launch(&spawn);
12245 spawn
12246 })
12247 })
12248 .flatten();
12249
12250 #[cfg(target_os = "linux")]
12251 let cgroup_path = cgroup_placement
12252 .map(|placement| {
12253 if let Some(container) = &carried {
12254 let path = container
12255 .cgroup_path()
12256 .ok_or_else(|| io::Error::other("carried container has no cgroup path"))?;
12257 placement.existing_module_path(path)?.ok_or_else(|| {
12258 io::Error::new(
12259 io::ErrorKind::NotFound,
12260 "carried cgroup disappeared before replacement",
12261 )
12262 })
12263 } else {
12264 if carrying {
12265 containment::prepare_cgroup_path(placement, &cgroup_name)
12266 } else {
12267 placement.module_path(&cgroup_name)
12268 }
12269 }
12270 })
12271 .transpose()
12272 .map_err(|source| SuperviseError::Cgroup {
12273 module_id: spec.module_id.clone(),
12274 source,
12275 })?;
12276 #[cfg(not(target_os = "linux"))]
12277 let cgroup_path: Option<PathBuf> = None;
12278 #[cfg(target_os = "linux")]
12281 let cgroup_cleanup = PreparedCgroup(
12282 cgroup_placement
12283 .filter(|_| carried.is_none())
12284 .map(|placement| (placement.clone(), cgroup_name.clone())),
12285 );
12286 #[cfg(unix)]
12287 let mut admission_gate = if carrying {
12288 Some(
12289 admission::Gate::install(&mut command, spec, roster).map_err(|source| {
12290 SuperviseError::Spawn {
12291 program: spec.program.clone(),
12292 source,
12293 cgroup_path: cgroup_path.clone(),
12294 }
12295 })?,
12296 )
12297 } else {
12298 None
12299 };
12300 #[cfg(target_os = "linux")]
12301 if let Some(path) = &cgroup_path {
12302 apply_cgroup_placement(&mut command, spec, path)?;
12303 }
12304
12305 let output_sink = if let Some(logs_dir) = capture_logs_dir {
12306 let path = logs_dir.join(format!("{}.stderr.log", spec.module_id));
12307 match ChildOutputSink::open(&path, capture_retention(spec)) {
12308 Ok(sink) => sink,
12309 Err(error) => {
12310 warn!(
12311 module_id = %spec.module_id,
12312 path = %path.display(),
12313 error = %error,
12314 "could not open child output capture file; forwarding to stderr"
12315 );
12316 ChildOutputSink::Stderr
12317 }
12318 }
12319 } else {
12320 ChildOutputSink::Stderr
12321 };
12322
12323 command.stdout(Stdio::piped());
12324 command.stderr(Stdio::piped());
12325 command.kill_on_drop(!carrying);
12326 #[cfg(unix)]
12343 command.process_group(
12344 carried
12345 .as_ref()
12346 .map(|container| container.verified_join_group())
12347 .transpose()
12348 .map_err(|source| SuperviseError::Spawn {
12349 program: spec.program.clone(),
12350 source,
12351 cgroup_path: cgroup_path.clone(),
12352 })?
12353 .flatten()
12354 .unwrap_or(0) as i32,
12355 );
12356 command.stdin(Stdio::null());
12357 #[cfg(unix)]
12358 if let Some(gate) = &mut admission_gate {
12359 gate.install_placement(&mut command);
12360 }
12361 #[cfg(unix)]
12364 {
12365 if let Some(handoff) = nonce_handoff {
12366 slots.nonce(handoff);
12367 }
12368 #[cfg(target_os = "macos")]
12369 slots
12370 .acknowledgement(exec_ack.into_descriptor())
12371 .map_err(|source| SuperviseError::Spawn {
12372 program: spec.program.clone(),
12373 source,
12374 cgroup_path: cgroup_path.clone(),
12375 })?;
12376 slots.install(command.as_std_mut());
12377 }
12378 #[cfg(not(any(unix, windows)))]
12379 let _ = nonce_handoff;
12380
12381 #[cfg(windows)]
12386 {
12387 subc_jobobject::ensure_root_job().map_err(|source| SuperviseError::Spawn {
12388 program: spec.program.clone(),
12389 source,
12390 cgroup_path: None,
12391 })?;
12392 subc_jobobject::suspend_on_create_async(&mut command);
12393 }
12394 let spawned_at = Instant::now();
12398 let delivery_context = config_delivery;
12399 let config_delivery = config_delivery
12400 .map(|delivery| {
12401 delivery.spawned(
12402 spec,
12403 spawn_admission
12404 .as_ref()
12405 .expect("delivery prepares spawn admission"),
12406 spawned_at,
12407 role,
12408 registration_spawn
12409 .as_ref()
12410 .map(|spawn| spawn.authority.clone()),
12411 )
12412 })
12413 .transpose()?
12414 .flatten();
12415 let mut delivery_registration =
12416 delivery_context.map(|context| context.registration_guard(config_delivery.as_ref(), role));
12417 if let Some(registration) = &mut delivery_registration {
12418 registration.defer_cleanup();
12419 }
12420 if asynchronous {
12421 #[cfg(test)]
12422 if let Some(context) = delivery_context {
12423 context.pause_preflight_for_test();
12424 }
12425 }
12426 validate_spawn_program(spec, spawn_admission.as_ref(), delivery_context)?;
12427 Ok(PreparedChild {
12428 script_delivery,
12429 registration_spawn,
12430 registration_handle: handle.cloned(),
12431 #[cfg(any(unix, windows))]
12432 registration_channel,
12433 config_context: delivery_context.map(|delivery| delivery.runtime()),
12434 binding_publication,
12435 launch_spec,
12436 spawn_admission,
12437 command,
12438 carrying,
12439 carried: carried.clone(),
12440 #[cfg(unix)]
12441 launch_identity,
12442 #[cfg(unix)]
12443 group_target,
12444 #[cfg(target_os = "linux")]
12445 cgroup_placement: cgroup_placement.cloned(),
12446 #[cfg(target_os = "linux")]
12447 cgroup_cleanup,
12448 #[cfg(target_os = "linux")]
12449 cgroup_name,
12450 cgroup_path,
12451 #[cfg(unix)]
12452 admission_gate,
12453 output_sink,
12454 #[cfg(target_os = "macos")]
12455 privacy_exec,
12456 #[cfg(windows)]
12457 image_capture,
12458 #[cfg(windows)]
12459 nonce_handoff,
12460 spawned_at,
12461 config_delivery,
12462 delivery_registration,
12463 program_binding,
12464 })
12465}
12466
12467fn launch_child_in_slot(
12468 _spec: &ModuleSpec,
12469 ring: &Arc<Mutex<StderrRing>>,
12470 roster: &ChildRoster,
12471 prepared: PreparedChild,
12472 register_timeout: Duration,
12473) -> Result<SupervisedChild, SuperviseError> {
12474 let launch_spec = prepared.launch_spec.clone();
12475 let registration_declaration = prepared
12476 .spawn_admission
12477 .as_ref()
12478 .and_then(|admission| admission.registration_declaration(&launch_spec.module_id));
12479 let admission_base = prepared
12480 .spawn_admission
12481 .as_ref()
12482 .and_then(|admission| admission.approved_base())
12483 .cloned();
12484 let spec = &launch_spec;
12485 let expected_image = prepared
12486 .spawn_admission
12487 .as_ref()
12488 .and_then(|admission| admission.launch.as_ref())
12489 .map(|launch| launch.image);
12490 let program_launch = prepared
12491 .spawn_admission
12492 .and_then(|admission| admission.launch);
12493 let config_owner = prepared
12494 .config_context
12495 .as_ref()
12496 .map(|context| context.0.clone());
12497 let binding_publication = prepared.binding_publication;
12498 #[cfg(test)]
12499 if let Some(context) = &prepared.config_context {
12500 context.0.before_spawn_exec_for_test();
12501 }
12502 #[cfg(feature = "test-support")]
12503 if let Some(context) = &prepared.config_context {
12504 context
12505 .0
12506 .pause_spawn_before_exec()
12507 .map_err(|error| config_delivery::gate_hold_error(spec, error, Some(context)))?;
12508 }
12509 let mut command = prepared.command;
12510 let carrying = prepared.carrying;
12511 let carried = prepared.carried;
12512 #[cfg(unix)]
12513 let launch_identity = prepared.launch_identity;
12514 #[cfg(unix)]
12515 let group_target = prepared.group_target;
12516 #[cfg(target_os = "linux")]
12517 let cgroup_placement = prepared.cgroup_placement.as_ref();
12518 #[cfg(target_os = "linux")]
12519 let cgroup_name = prepared.cgroup_name;
12520 #[cfg(target_os = "linux")]
12521 let mut cgroup_cleanup = prepared.cgroup_cleanup;
12522 #[cfg(target_os = "linux")]
12523 let _ = cgroup_cleanup.0.take();
12524 let cgroup_path = prepared.cgroup_path;
12525 #[cfg(unix)]
12526 let admission_gate = prepared.admission_gate;
12527 let output_sink = prepared.output_sink;
12528 #[cfg(target_os = "macos")]
12529 let mut privacy_exec = prepared.privacy_exec;
12530 #[cfg(target_os = "macos")]
12531 if let Some(pending) = &mut privacy_exec {
12532 pending.admission_binding = binding_publication.clone();
12533 if expected_image.is_some() {
12534 pending.expected = expected_image;
12535 }
12536 }
12537 #[cfg(any(unix, windows))]
12538 let registration_channel = prepared.registration_channel;
12539 let registration_spawn = prepared.registration_spawn;
12540 let registration_handle = prepared.registration_handle;
12541 #[cfg(windows)]
12542 let image_capture = prepared.image_capture;
12543 #[cfg(windows)]
12544 let nonce_handoff = prepared.nonce_handoff;
12545 let spawned_at = prepared.spawned_at;
12546 let config_delivery = prepared.config_delivery;
12547 let delivery_registration = prepared.delivery_registration;
12548 let _program_binding = prepared.program_binding;
12549 #[cfg(all(test, unix))]
12550 containment_tests::record_spawn_attempt(spec, &mut command);
12551 #[cfg(unix)]
12552 let spawned = match admission_gate {
12553 Some(gate) => gate
12554 .spawn(
12555 command,
12556 spec,
12557 roster,
12558 launch_identity,
12559 #[cfg(target_os = "linux")]
12560 cgroup_placement
12561 .zip(cgroup_path.as_ref())
12562 .map(|(placement, path)| {
12563 (placement.clone(), cgroup_name.clone(), path.clone())
12564 }),
12565 )
12566 .map(|(child, admitted)| (child, Some(admitted))),
12567 None => {
12568 let spawned =
12569 subc_os::child_spawn::tokio_spawn(&mut command).map(|child| (child, None));
12570 drop(command);
12573 spawned
12574 }
12575 };
12576 #[cfg(not(unix))]
12577 let spawned = subc_os::child_spawn::tokio_spawn(&mut command);
12578 let spawned = match spawned {
12579 Ok(child) => child,
12580 Err(source) => {
12581 if carried
12582 .as_ref()
12583 .and_then(|container| container.group())
12584 .is_some()
12585 {
12586 containment::terminate_blocking(&roster.containment);
12587 }
12588 #[cfg(target_os = "linux")]
12589 if let Some(placement) = cgroup_placement.filter(|_| carried.is_none()) {
12590 remove_module_cgroup(placement, &cgroup_name);
12591 }
12592 return Err(SuperviseError::Spawn {
12593 program: spec.program.clone(),
12594 source,
12595 cgroup_path,
12596 });
12597 }
12598 };
12599 #[cfg(unix)]
12600 let (mut child, mut admitted) = spawned;
12601 #[cfg(unix)]
12602 let group_target = admitted
12603 .as_ref()
12604 .map(|attempt| attempt.group_target())
12605 .unwrap_or(group_target);
12606 #[cfg(not(unix))]
12607 let mut child = spawned;
12608 if let Some(delivery) = prepared.script_delivery {
12609 delivery.start(child.id().expect("spawned child has a pid"));
12610 }
12611 #[cfg(test)]
12612 spawn_record_tests::after_create(&child, spec);
12613
12614 #[cfg(windows)]
12619 let windows_image = match image_capture.bind_tokio(&child) {
12620 Ok(image) => Arc::new(image),
12621 Err(source) => {
12622 kill_and_reap_rejected_child(&mut child);
12623 return Err(SuperviseError::Spawn {
12624 program: spec.program.clone(),
12625 source,
12626 cgroup_path: None,
12627 });
12628 }
12629 };
12630 #[cfg(windows)]
12633 let job = contain_spawned_child(&mut child, spec, carried.as_ref())?;
12634 let spawned_at_ms = unix_ms_now();
12635 #[cfg(not(windows))]
12636 let spawned_from = spec.program.clone();
12637 #[cfg(windows)]
12638 let spawned_from = windows_image.path().to_path_buf();
12639 #[cfg(not(windows))]
12640 let spawned_file_identity = spawned_file_identity(&spawned_from);
12641 #[cfg(windows)]
12642 let spawned_file_identity = Some(SpawnedFileIdentity {
12643 device: 0,
12644 inode: 0,
12645 windows_image: Some(Arc::clone(&windows_image)),
12646 });
12647 let pid = child.id().ok_or_else(|| SuperviseError::Spawn {
12648 program: spec.program.clone(),
12649 source: io::Error::other("spawned child exposed no live pid"),
12650 cgroup_path: cgroup_path.clone(),
12651 })?;
12652 #[cfg(windows)]
12653 let nonce_delivery = start_windows_nonce_pipe(nonce_handoff, pid, spec);
12654 #[cfg(not(windows))]
12655 let process_start_time = admitted
12656 .as_ref()
12657 .map(|admitted| admitted.birth)
12658 .or_else(|| crate::provenance::process_start_time(pid));
12659 #[cfg(windows)]
12660 let process_start_time = Some(windows_image.start_time());
12661 let process_identity = process_start_time.map(|start_time| ProcessIdentity { pid, start_time });
12662 #[cfg(all(test, target_os = "macos"))]
12663 privacy_exec_boundary_tests::before_image_sample(spec, pid);
12664 #[cfg(all(test, target_os = "linux"))]
12665 assert!(roster.admission_event_for_test(&spec.module_id, "before_image_binding", Some(pid)));
12666 #[cfg(not(windows))]
12671 let recorded_image = observe_spawned_image(pid);
12672 #[cfg(all(test, target_os = "linux"))]
12673 let recorded_image = if roster.admission_event_for_test(
12674 &spec.module_id,
12675 "after_spawn_image_sample",
12676 Some(pid),
12677 ) {
12678 recorded_image
12679 } else {
12680 None
12681 };
12682 #[cfg(target_os = "linux")]
12683 let exited_before_binding = expected_image.is_some()
12686 && recorded_image.and_then(|image| image.executable) != expected_image
12687 && child.try_wait().ok().flatten().is_some();
12688 #[cfg(target_os = "linux")]
12689 let unavailable_initial_image = expected_image.is_some()
12690 && recorded_image.and_then(|image| image.executable).is_none()
12691 && !exited_before_binding;
12692 #[cfg(not(target_os = "linux"))]
12693 let unavailable_initial_image = false;
12694 #[cfg(target_os = "linux")]
12695 if expected_image
12696 .is_some_and(|expected| recorded_image.and_then(|image| image.executable) != Some(expected))
12697 && !exited_before_binding
12698 && !unavailable_initial_image
12699 {
12700 kill_module_cgroup(cgroup_placement, &cgroup_name);
12701 kill_and_reap_rejected_child(&mut child, group_target.clone());
12702 return Err(SuperviseError::ConfigHeld {
12703 module_id: spec.module_id.clone(),
12704 code: "program_refused".into(),
12705 detail: format!(
12706 "spawned executable {:?} does not match hashed program {:?}",
12707 recorded_image.and_then(|image| image.executable),
12708 expected_image
12709 ),
12710 });
12711 }
12712 #[cfg(not(target_os = "linux"))]
12713 let _ = expected_image;
12714 #[cfg(target_os = "macos")]
12715 let binding_ready = privacy_exec.is_none();
12716 #[cfg(target_os = "linux")]
12717 let binding_ready = !exited_before_binding;
12718 #[cfg(not(any(target_os = "macos", target_os = "linux")))]
12719 let binding_ready = cfg!(windows);
12720 #[cfg(target_os = "macos")]
12724 let recorded_image = if privacy_exec.is_some() {
12725 None
12726 } else {
12727 recorded_image
12728 };
12729 #[cfg(target_os = "linux")]
12730 let recorded_cgroup_name = cgroup_path.as_ref().map(|_| cgroup_name.clone());
12731 #[cfg(not(target_os = "linux"))]
12732 let recorded_cgroup_name = None;
12733 #[cfg(unix)]
12734 let preadmitted = admitted
12735 .as_mut()
12736 .and_then(|admitted| admitted.leader.take());
12737 #[cfg(not(unix))]
12738 let preadmitted: Option<crate::child_roster::RosterGuard> = None;
12739 let roster_guard = if let Some(guard) = preadmitted {
12740 Ok(guard)
12741 } else {
12742 roster.admit(
12743 spec.module_id.clone(),
12744 pid,
12745 spec.protocol,
12746 process_start_time,
12747 crate::child_roster::RecordedIdentity {
12748 container: None,
12749 #[cfg(unix)]
12750 carried: None,
12751 #[cfg(not(windows))]
12752 start_time: recorded_image.map(|image| image.start_time),
12753 #[cfg(windows)]
12754 start_time: process_start_time,
12755 #[cfg(not(windows))]
12756 executable: recorded_image
12757 .and_then(|image| image.executable)
12758 .map(crate::live_children::ExecutableIdentity::from),
12759 #[cfg(windows)]
12760 executable: Some(crate::live_children::ExecutableIdentity::from(
12761 windows_image.identity(),
12762 )),
12763 cgroup_name: recorded_cgroup_name,
12764 #[cfg(target_os = "linux")]
12765 cgroup_placement: cgroup_placement.cloned(),
12766 },
12767 )
12768 };
12769 let roster_guard = match roster_guard {
12770 Ok(guard) => guard,
12771 Err(failure) => {
12772 return failure.with_cleanup(|source| {
12773 #[cfg(windows)]
12774 if let Some(job) = &job {
12775 let _ = job.lock().unwrap_or_else(|p| p.into_inner()).terminate();
12776 }
12777 #[cfg(target_os = "linux")]
12778 kill_module_cgroup(cgroup_placement, &cgroup_name);
12779 #[cfg(unix)]
12780 let rejected_group = group_target.clone();
12781 kill_and_reap_rejected_child(
12782 &mut child,
12783 #[cfg(unix)]
12784 rejected_group,
12785 );
12786 #[cfg(target_os = "linux")]
12787 if let Some(placement) = cgroup_placement.filter(|_| carried.is_none()) {
12788 remove_module_cgroup(placement, &cgroup_name);
12789 }
12790 Err(SuperviseError::Spawn {
12791 program: spec.program.clone(),
12792 source: io::Error::other(format!(
12793 "could not durably record module child: {source}"
12794 )),
12795 cgroup_path,
12796 })
12797 });
12798 }
12799 };
12800 #[cfg(unix)]
12801 if admitted.is_some() {
12802 if let Some(image) = recorded_image {
12803 if let Err(source) = roster_guard.confirm_executable(image) {
12804 kill_and_reap_rejected_child(&mut child, group_target.clone());
12805 return Err(SuperviseError::Spawn {
12806 program: spec.program.clone(),
12807 source,
12808 cgroup_path,
12809 });
12810 }
12811 }
12812 }
12813 if roster.is_closed() {
12818 if carrying {
12819 #[cfg(unix)]
12820 let rejected_group = admitted
12821 .as_ref()
12822 .map(|admitted| admitted.rejection_target())
12823 .unwrap_or_else(|| group_target.clone());
12824 #[cfg(windows)]
12825 if let Some(job) = &job {
12826 let _ = job.lock().unwrap_or_else(|p| p.into_inner()).terminate();
12827 }
12828 kill_and_reap_rejected_child(
12829 &mut child,
12830 #[cfg(unix)]
12831 rejected_group,
12832 );
12833 #[cfg(all(test, unix))]
12834 let _ =
12835 roster.admission_event_for_test(&spec.module_id, "after_rejected_child", Some(pid));
12836 drop(roster_guard);
12837 return Err(SuperviseError::Spawn {
12838 program: spec.program.clone(),
12839 source: io::Error::other(
12840 "the daemon began shutting down during carried admission; ended the container",
12841 ),
12842 cgroup_path,
12843 });
12844 }
12845 #[cfg(target_os = "linux")]
12847 kill_module_cgroup(cgroup_placement, &cgroup_name);
12848 if let Err(error) = child.start_kill() {
12849 debug!(module_id = %spec.module_id, pid, %error, "kill of a process spawned during daemon shutdown failed; it may already have exited");
12850 }
12851 #[cfg(target_os = "linux")]
12852 if let Some(placement) = cgroup_placement.filter(|_| carried.is_none()) {
12853 while matches!(child.try_wait(), Ok(None)) {
12857 std::thread::yield_now();
12858 }
12859 if matches!(
12860 subc_cgroup::kill_module(Some(placement), &cgroup_name),
12861 subc_cgroup::KillOutcome::Killed
12862 ) {
12863 if let Ok(path) = placement.module_path(&cgroup_name) {
12864 while std::fs::read_to_string(path.join("cgroup.events"))
12865 .ok()
12866 .is_some_and(|events| events.lines().any(|line| line == "populated 1"))
12867 {
12868 std::thread::yield_now();
12869 }
12870 }
12871 }
12872 remove_module_cgroup(placement, &cgroup_name);
12873 }
12874 drop(roster_guard);
12875 return Err(SuperviseError::Spawn {
12876 program: spec.program.clone(),
12877 source: io::Error::other(
12878 "the daemon began shutting down while this process was starting; ended it",
12879 ),
12880 cgroup_path,
12881 });
12882 }
12883
12884 let stdout_pump = match child.stdout.take() {
12885 Some(stdout) => Some(tokio::spawn(pump_stdout_to(stdout, output_sink.clone()))),
12886 None => {
12887 warn!(
12888 module_id = %spec.module_id,
12889 "spawned child exposed no stdout pipe; file capture will be incomplete"
12890 );
12891 None
12892 }
12893 };
12894 let stderr_pump = match child.stderr.take() {
12895 Some(stderr) => {
12896 let generation = ring
12897 .lock()
12898 .unwrap_or_else(|poisoned| poisoned.into_inner())
12899 .begin_process();
12900 Some(StderrPump {
12901 task: tokio::spawn(pump_stderr_to(
12902 stderr,
12903 Arc::clone(ring),
12904 generation,
12905 output_sink,
12906 )),
12907 generation,
12908 })
12909 }
12910 None => {
12911 ring.lock()
12915 .unwrap_or_else(|poisoned| poisoned.into_inner())
12916 .mark_not_captured("stderr pipe was not available on spawn");
12917 warn!(
12918 module_id = %spec.module_id,
12919 "spawned child exposed no stderr pipe; tail will be unavailable"
12920 );
12921 None
12922 }
12923 };
12924
12925 let exec = Arc::new(crate::activity::ExecOutcome::new());
12926 #[cfg(target_os = "macos")]
12927 let pending_exec = privacy_exec.is_some();
12928 #[cfg(not(target_os = "macos"))]
12929 let pending_exec = false;
12930 if !pending_exec {
12931 exec.finish(crate::activity::ExecState::CommittedExec);
12932 }
12933 #[cfg(windows)]
12934 let subtree_target = job
12935 .as_ref()
12936 .map(|job| crate::subtree_resources::Target::Job(Arc::downgrade(job)))
12937 .unwrap_or(crate::subtree_resources::Target::Unavailable("no_job"));
12938 #[cfg(target_os = "linux")]
12939 let subtree_target = cgroup_path
12940 .map(crate::subtree_resources::Target::Cgroup)
12941 .unwrap_or(crate::subtree_resources::Target::Unavailable("no_cgroup"));
12942 #[cfg(not(any(target_os = "linux", windows)))]
12943 let subtree_target = crate::subtree_resources::Target::Unavailable("unsupported_platform");
12944 let mut supervised = SupervisedChild {
12945 registration_spawn,
12946 #[cfg(any(unix, windows))]
12947 registration_channel,
12948 live_program_binding: None,
12949 registered_config_connection: None,
12950 admission_base,
12951 registration_declaration,
12952 _program_binding: binding_publication,
12953 subtree_target,
12954 child: containment::ProcessChild::plain(child, pid),
12955 exec: crate::activity::ExecOwner(exec),
12956 #[cfg(windows)]
12957 nonce_delivery,
12958 protocol: spec.protocol,
12959 #[cfg(target_os = "linux")]
12960 module_id: cgroup_name,
12961 #[cfg(target_os = "linux")]
12962 cgroup_placement: cgroup_placement.cloned(),
12963 #[cfg(windows)]
12964 job,
12965 stdout_pump,
12966 stderr_pump,
12967 stderr_ring: Arc::clone(ring),
12968 spawned_at,
12969 config_delivery,
12970 registration_deadline: None,
12971 registration_watch: None,
12972 spawned_at_ms,
12973 spawned_from,
12974 launch_spec_digest: LaunchSpecDigest::from_spec(spec),
12975 spawned_file_identity,
12976 process_start_time,
12977 process_identity,
12978 pid,
12979 roster_guard: Some(roster_guard),
12980 #[cfg(target_os = "macos")]
12981 privacy_exec,
12982 #[cfg(target_os = "macos")]
12983 report_ready: Arc::new(OnceLock::new()),
12984 spawn_failure: None,
12985 admission_kill_sent: false,
12986 };
12987 if let Err(error) = containment::adopt(&mut supervised, spec, roster) {
12988 supervised.child.reject(
12989 #[cfg(unix)]
12990 group_target,
12991 );
12992 #[cfg(all(test, unix))]
12993 let _ = roster.admission_event_for_test(&spec.module_id, "after_rejected_child", Some(pid));
12994 return Err(error);
12995 }
12996 if let (Some(launch), Some(owner), Some(base)) = (
12997 program_launch,
12998 config_owner,
12999 supervised.admission_base.clone(),
13000 ) {
13001 supervised.live_program_binding =
13002 content_binding::LiveProgramBinding::new(launch, &supervised, owner, base);
13003 if let (Some(publication), Some(live)) = (
13004 &supervised._program_binding,
13005 &supervised.live_program_binding,
13006 ) {
13007 publication.attach(live);
13008 }
13009 }
13010 #[cfg(unix)]
13011 if let Some(admitted) = admitted.take() {
13012 admitted.disarm();
13013 }
13014 #[cfg(windows)]
13015 if let Err(error) = resume_suspended_child(pid, spec) {
13016 containment::terminate_blocking(&roster.containment);
13017 let _ = supervised.child.start_kill();
13018 return Err(error);
13019 }
13020 if binding_ready {
13021 let checked = if unavailable_initial_image {
13025 Err(crate::config_gate::GateError::new(
13026 "config_delivery_deferred",
13027 "initial process image is unavailable",
13028 ))
13029 } else {
13030 match (
13031 &supervised._program_binding,
13032 &supervised.live_program_binding,
13033 ) {
13034 (Some(binding), _) => binding.verify(),
13035 (None, Some(live)) => live.check(),
13036 _ if supervised.admission_base.is_some() => {
13037 Err(crate::config_gate::GateError::new(
13038 "config_delivery_deferred",
13039 "live program witness is unavailable",
13040 ))
13041 }
13042 _ => Ok(()),
13043 }
13044 };
13045 #[cfg(all(test, target_os = "linux"))]
13046 let checked = if roster.admission_event_for_test(
13047 &spec.module_id,
13048 "after_content_witness_check",
13049 Some(pid),
13050 ) {
13051 checked
13052 } else {
13053 Err(crate::config_gate::GateError::new(
13054 "config_delivery_deferred",
13055 "current process image is unavailable",
13056 ))
13057 };
13058 if let Err(error) = checked {
13059 if let Some(binding) = &mut supervised._program_binding {
13060 binding.reject();
13061 }
13062 if error.code == "program_refused" || supervised.try_wait().ok().flatten().is_none() {
13065 supervised.spawn_failure = Some(format!("{}: {}", error.code, error.detail));
13066 #[cfg(all(test, target_os = "linux"))]
13067 assert!(roster.admission_event_for_test(
13068 &spec.module_id,
13069 if unavailable_initial_image {
13070 "before_initial_image_kill"
13071 } else {
13072 "before_content_witness_kill"
13073 },
13074 Some(pid),
13075 ));
13076 let _ = supervised.start_admission_kill();
13077 #[cfg(all(test, target_os = "linux"))]
13078 if supervised.admission_kill_sent {
13079 assert!(roster.admission_event_for_test(
13080 &spec.module_id,
13081 "after_admission_kill_sent",
13082 Some(pid),
13083 ));
13084 }
13085 }
13086 }
13087 }
13088 if let Some(registration) = delivery_registration {
13089 registration.commit();
13090 }
13091 if let Some(spawn) = &supervised.registration_spawn {
13092 if let Some(identity) = supervised.process_identity {
13093 let _ = spawn.identity.set(identity);
13094 }
13095 if supervised.spawn_failure.is_some() {
13096 spawn.refuse();
13097 } else if binding_ready {
13098 spawn.ready();
13099 }
13100 #[cfg(any(unix, windows))]
13101 if let (Some(handle), Some(channel)) =
13102 (registration_handle, supervised.registration_channel.take())
13103 {
13104 handle.serve_registration(channel, spawn.clone(), pid, spawned_at + register_timeout);
13105 }
13106 }
13107 Ok(supervised)
13108}
13109
13110#[cfg(test)]
13111mod spawn_record_tests {
13112 use super::*;
13113
13114 thread_local! {
13115 static LAST_PID: std::cell::Cell<Option<u32>> = const { std::cell::Cell::new(None) };
13116 }
13117
13118 pub(super) fn after_create(child: &Child, _spec: &ModuleSpec) {
13119 LAST_PID.set(child.id());
13120 #[cfg(windows)]
13121 if let Some((_, address)) = _spec
13122 .env
13123 .iter()
13124 .find(|(key, _)| key == "SUBC_TEST_AFTER_CREATE")
13125 {
13126 use std::io::{Read, Write};
13127 let mut stream = std::net::TcpStream::connect(address).unwrap();
13128 stream
13129 .write_all(&child.id().unwrap().to_ne_bytes())
13130 .unwrap();
13131 let mut release = [0];
13132 stream.read_exact(&mut release).unwrap();
13133 }
13134 }
13135
13136 #[cfg(windows)]
13137 pub(super) fn assignment_failpoint(spec: &ModuleSpec) -> io::Result<()> {
13138 if spec
13139 .env
13140 .iter()
13141 .any(|(key, _)| key == "SUBC_TEST_ASSIGNMENT_FAILURE")
13142 {
13143 Err(io::Error::other("injected module job assignment failure"))
13144 } else {
13145 Ok(())
13146 }
13147 }
13148
13149 fn spec() -> ModuleSpec {
13150 ModuleSpec {
13151 script: None,
13152 module_id: "spawn-record-worker".into(),
13153 #[cfg(unix)]
13154 program: "/bin/sleep".into(),
13155 #[cfg(windows)]
13156 program: std::env::current_exe().unwrap(),
13157 #[cfg(unix)]
13158 args: vec!["60".into()],
13159 #[cfg(windows)]
13160 args: vec![
13161 "--exact".into(),
13162 "supervise::spawn_record_tests::windows_store_touch_fixture".into(),
13163 "--nocapture".into(),
13164 ],
13165 env: vec![],
13166 reserved: false,
13167 reserved_prefixes: vec![],
13168 protocol: ModuleProtocol::None,
13169 overlap: Default::default(),
13170 carry_containment: false,
13171 }
13172 }
13173
13174 #[tokio::test]
13175 async fn record_sync_failure_fails_spawn_and_reaps_child() {
13176 let home = cortexkit_test_support::ScratchDir::new("record-sync-failure");
13177 let supervisor =
13178 Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default())
13179 .with_live_children_record(home.join("live-children.json"));
13180 crate::live_children::fail_next_file_sync();
13181 let result = supervisor.spawn(spec());
13182 let failed = result.is_err();
13183 if let Ok(module) = result {
13184 module.stop().await.unwrap();
13185 }
13186 assert!(failed, "record failure must fail spawn");
13187 let pid = LAST_PID.get().expect("the real OS child was created");
13188 assert!(
13189 subc_os::Process::open(pid).unwrap().is_none(),
13190 "child {pid} survived rejected admission or was not reaped"
13191 );
13192 }
13193
13194 #[cfg(target_os = "macos")]
13195 #[test]
13196 #[ignore = "manual on-device latency measurement"]
13197 fn durable_record_write_latency_50() {
13198 let home = cortexkit_test_support::ScratchDir::new("record-latency");
13199 let path = home.join("live-children.json");
13200 let children = vec![crate::live_children::LiveChild {
13201 module_id: "latency-worker".into(),
13202 pid: std::process::id(),
13203 protocol: ModuleProtocol::None,
13204 start_time: Some(1),
13205 executable: None,
13206 cgroup_name: None,
13207 container: None,
13208 }];
13209 let mut samples = Vec::new();
13210 for _ in 0..50 {
13211 let start = std::time::Instant::now();
13212 crate::live_children::write_record(&path, &children).unwrap();
13213 samples.push(start.elapsed());
13214 }
13215 samples.sort();
13216 eprintln!(
13217 "durable_record_write_latency_50: n=50 median={:?} p95={:?}",
13218 (samples[24] + samples[25]) / 2,
13219 samples[47]
13220 );
13221 }
13222
13223 #[cfg(windows)]
13224 #[test]
13225 fn windows_store_touch_fixture() {
13226 if let Some(path) = std::env::var_os("SUBC_TEST_STORE_TOUCH") {
13227 std::fs::write(path, b"module ran").unwrap();
13228 }
13229 }
13230
13231 #[cfg(windows)]
13232 #[tokio::test]
13233 async fn windows_daemon_fixture() {
13234 let Ok(address) = std::env::var("SUBC_TEST_DAEMON_BARRIER") else {
13235 return;
13236 };
13237 let mut spec = spec();
13238 spec.env.push(("SUBC_TEST_AFTER_CREATE".into(), address));
13239 let supervisor =
13240 Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default());
13241 let _module = supervisor.spawn(spec).unwrap();
13242 }
13243
13244 #[cfg(windows)]
13245 #[test]
13246 fn windows_daemon_death_before_module_job_assignment_leaves_no_child() {
13247 use std::io::Read;
13248 let listener = std::net::TcpListener::bind("127.0.0.1:0").unwrap();
13249 let mut command = std::process::Command::new(std::env::current_exe().unwrap());
13252 command
13253 .args([
13254 "--exact",
13255 "supervise::spawn_record_tests::windows_daemon_fixture",
13256 "--nocapture",
13257 ])
13258 .env(
13259 "SUBC_TEST_DAEMON_BARRIER",
13260 listener.local_addr().unwrap().to_string(),
13261 );
13262 let mut daemon = subc_jobobject::spawn_contained(&mut command).unwrap();
13263 listener.set_nonblocking(true).unwrap();
13264 let deadline = std::time::Instant::now() + Duration::from_secs(30);
13265 let mut stream = loop {
13266 if let Ok((stream, _)) = listener.accept() {
13267 break stream;
13268 }
13269 assert!(
13270 daemon.child.try_wait().unwrap().is_none(),
13271 "daemon failed before rendezvous"
13272 );
13273 assert!(
13274 std::time::Instant::now() < deadline,
13275 "daemon did not reach CreateProcess failpoint"
13276 );
13277 std::thread::yield_now();
13278 };
13279 stream
13280 .set_read_timeout(Some(Duration::from_secs(30)))
13281 .unwrap();
13282 let mut pid = [0; 4];
13283 stream.read_exact(&mut pid).unwrap();
13284 let pid = u32::from_ne_bytes(pid);
13285 assert!(subc_jobobject::process_exists(pid));
13286 daemon.child.kill().unwrap();
13289 daemon.child.wait().unwrap();
13290 let exited = subc_jobobject::wait_for_process_exit(pid, Duration::from_secs(10));
13291 if !exited {
13292 daemon.job.terminate().unwrap();
13293 }
13294 assert!(exited, "suspended child {pid} outlived hard-killed daemon");
13295 }
13296
13297 #[cfg(windows)]
13298 #[tokio::test]
13299 async fn windows_containment_failure_never_resumes() {
13300 let home = cortexkit_test_support::ScratchDir::new("assignment-refusal");
13301 let sentinel = home.join("store-touched");
13302 let mut spec = spec();
13303 spec.env
13304 .push(("SUBC_TEST_ASSIGNMENT_FAILURE".into(), "1".into()));
13305 spec.env.push((
13306 "SUBC_TEST_STORE_TOUCH".into(),
13307 sentinel.display().to_string(),
13308 ));
13309 let supervisor =
13310 Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default());
13311 let result = supervisor.spawn(spec);
13312 if let Ok(module) = result {
13315 let _ = tokio::time::timeout(Duration::from_secs(10), async {
13316 while !sentinel.exists() && module.state().unwrap() == ModuleState::Running {
13317 tokio::task::yield_now().await;
13318 }
13319 })
13320 .await;
13321 let _ = module.stop().await;
13322 panic!(
13323 "assignment failure resumed a module; store touched={}",
13324 sentinel.exists()
13325 );
13326 }
13327 assert!(
13328 !sentinel.exists(),
13329 "refused module executed store-touch sentinel"
13330 );
13331 assert!(!subc_jobobject::process_exists(LAST_PID.get().unwrap()));
13332 }
13333
13334 #[cfg(windows)]
13335 #[tokio::test]
13336 async fn windows_nested_daemon_fixture() {
13337 let Some(sentinel) = std::env::var_os("SUBC_TEST_NESTED_STORE") else {
13338 return;
13339 };
13340 let mut spec = spec();
13341 spec.env.push((
13342 "SUBC_TEST_STORE_TOUCH".into(),
13343 sentinel.to_str().unwrap().into(),
13344 ));
13345 let supervisor =
13346 Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default());
13347 let module = supervisor
13348 .spawn(spec)
13349 .expect("per-module job must nest below daemon and host jobs");
13350 tokio::time::timeout(Duration::from_secs(10), async {
13351 while !PathBuf::from(&sentinel).exists() {
13352 tokio::task::yield_now().await;
13353 }
13354 })
13355 .await
13356 .unwrap();
13357 module.stop().await.unwrap();
13358 }
13359
13360 #[cfg(windows)]
13361 #[test]
13362 fn windows_module_job_nests_under_host_job() {
13363 let home = cortexkit_test_support::ScratchDir::new("nested-jobs");
13364 let sentinel = home.join("nested-store-touched");
13365 let mut command = std::process::Command::new(std::env::current_exe().unwrap());
13366 command
13367 .args([
13368 "--exact",
13369 "supervise::spawn_record_tests::windows_nested_daemon_fixture",
13370 "--nocapture",
13371 ])
13372 .env("SUBC_TEST_NESTED_STORE", &sentinel);
13373 let mut daemon = subc_jobobject::spawn_contained(&mut command).unwrap();
13374 assert!(daemon.child.wait().unwrap().success());
13375 assert!(sentinel.exists(), "nested protocol:none module never ran");
13376 }
13377}
13378
13379#[cfg(target_os = "linux")]
13380pub(crate) fn kill_module_cgroup(placement: Option<&subc_cgroup::Placement>, module_id: &str) {
13381 use subc_cgroup::KillOutcome;
13382 match subc_cgroup::kill_module(placement, module_id) {
13383 KillOutcome::Killed => {}
13384 KillOutcome::NotPlaced | KillOutcome::Unsupported => {
13385 debug!(
13386 module_id,
13387 "cgroup tree kill unavailable; using direct-child kill"
13388 );
13389 }
13390 KillOutcome::IoError { path, error } => {
13391 warn!(module_id, path = %path.display(), %error, "cgroup tree kill failed; using direct-child kill");
13392 }
13393 }
13394}
13395
13396#[cfg(windows)]
13404fn contain_spawned_child(
13405 child: &mut Child,
13406 spec: &ModuleSpec,
13407 carried: Option<&Arc<containment::CarriedContainment>>,
13408) -> Result<Option<Arc<Mutex<subc_jobobject::JobObject>>>, SuperviseError> {
13409 let assignment = || {
13410 let job = match carried {
13411 Some(container) => container.job(),
13412 None => Arc::new(Mutex::new(subc_jobobject::JobObject::new()?)),
13413 };
13414 #[cfg(test)]
13415 spawn_record_tests::assignment_failpoint(spec)?;
13416 job.lock()
13417 .unwrap_or_else(|p| p.into_inner())
13418 .assign(child)?;
13419 Ok::<_, io::Error>(job)
13420 };
13421 match assignment() {
13422 Ok(job) => Ok(Some(job)),
13423 Err(source) => {
13424 kill_and_reap_rejected_child(child);
13425 Err(SuperviseError::Spawn {
13426 program: spec.program.clone(),
13427 source: io::Error::other(format!(
13428 "Windows module job containment failed: {source}"
13429 )),
13430 cgroup_path: None,
13431 })
13432 }
13433 }
13434}
13435
13436fn kill_and_reap_rejected_child(child: &mut Child, #[cfg(unix)] group: containment::GroupTarget) {
13439 #[cfg(unix)]
13440 group.signal_rejected(child);
13441 let _ = child.start_kill();
13442 while matches!(child.try_wait(), Ok(None)) {
13443 std::thread::yield_now();
13444 }
13445}
13446
13447#[cfg(windows)]
13452fn resume_suspended_child(pid: u32, spec: &ModuleSpec) -> Result<(), SuperviseError> {
13453 if let Err(source) = subc_jobobject::resume_main_thread(pid) {
13454 let _ = subc_os::child_spawn::status(
13458 std::process::Command::new("taskkill.exe")
13459 .args(["/PID", &pid.to_string(), "/T", "/F"])
13460 .stdin(Stdio::null())
13461 .stdout(Stdio::null())
13462 .stderr(Stdio::null()),
13463 );
13464 return Err(SuperviseError::Spawn {
13465 program: spec.program.clone(),
13466 source,
13467 cgroup_path: None,
13468 });
13469 }
13470 Ok(())
13471}
13472
13473#[cfg(target_os = "linux")]
13474fn remove_module_cgroup(placement: &subc_cgroup::Placement, module_id: &str) {
13475 match placement.remove_module(module_id) {
13476 Ok(()) => debug!(module_id, "removed module cgroup after process exit"),
13477 Err(error) => warn!(
13478 module_id,
13479 error = %error,
13480 "could not remove module cgroup after process exit; continuing teardown"
13481 ),
13482 }
13483}
13484
13485#[cfg(target_os = "linux")]
13486const REAPED_CGROUP_TERM_GRACE: Duration = Duration::from_secs(2);
13487#[cfg(target_os = "linux")]
13488const REAPED_CGROUP_FREEZE_TIMEOUT: Duration = Duration::from_millis(100);
13489
13490#[cfg(target_os = "linux")]
13491async fn cleanup_reaped_cgroup(placement: &subc_cgroup::Placement, module_id: &str) {
13492 let path = placement.module_path(module_id).ok();
13493 if path.as_ref().is_some_and(|path| {
13494 std::fs::read_to_string(path.join("cgroup.events"))
13495 .ok()
13496 .is_some_and(|events| events.lines().any(|line| line == "populated 0"))
13497 }) {
13498 remove_module_cgroup(placement, module_id);
13499 return;
13500 }
13501
13502 if terminate_reaped_cgroup(placement, module_id).await {
13505 if let Some(path) = &path {
13506 let deadline = Instant::now() + REAPED_CGROUP_TERM_GRACE;
13507 while cgroup_is_populated(path) && Instant::now() < deadline {
13508 sleep_until((Instant::now() + Duration::from_millis(1)).min(deadline)).await;
13509 }
13510 }
13511 }
13512
13513 if matches!(
13517 subc_cgroup::kill_module(Some(placement), module_id),
13518 subc_cgroup::KillOutcome::Killed
13519 ) {
13520 if let Some(path) = &path {
13521 while cgroup_is_populated(path) {
13522 sleep(Duration::from_millis(1)).await;
13523 }
13524 }
13525 }
13526 remove_module_cgroup(placement, module_id);
13527}
13528
13529#[cfg(target_os = "linux")]
13530fn cgroup_is_populated(path: &std::path::Path) -> bool {
13531 std::fs::read_to_string(path.join("cgroup.events"))
13532 .ok()
13533 .is_some_and(|events| events.lines().any(|line| line == "populated 1"))
13534}
13535
13536#[cfg(target_os = "linux")]
13537async fn terminate_reaped_cgroup(placement: &subc_cgroup::Placement, module_id: &str) -> bool {
13538 let frozen =
13539 match subc_cgroup::freeze_module(Some(placement), module_id, REAPED_CGROUP_FREEZE_TIMEOUT)
13540 .await
13541 {
13542 subc_cgroup::FreezeOutcome::Frozen(frozen) => frozen,
13543 outcome => {
13544 tracing::debug!(
13545 module_id,
13546 ?outcome,
13547 "cgroup freeze failed; skipping SIGTERM grace"
13548 );
13549 return false;
13550 }
13551 };
13552 let listed = match frozen.processes() {
13553 subc_cgroup::ProcsOutcome::Listed(pids) => {
13554 for pid in pids.into_iter().filter(|pid| *pid > 1) {
13555 if let Some(pid) = rustix::process::Pid::from_raw(pid) {
13556 if let Err(error) =
13557 rustix::process::kill_process(pid, rustix::process::Signal::TERM)
13558 {
13559 tracing::debug!(module_id, %pid, %error, "leftover cgroup SIGTERM failed");
13560 }
13561 }
13562 }
13563 true
13564 }
13565 outcome => {
13566 tracing::debug!(
13567 module_id,
13568 ?outcome,
13569 "cgroup process listing failed; skipping SIGTERM grace"
13570 );
13571 false
13572 }
13573 };
13574 match frozen.thaw() {
13575 subc_cgroup::ThawOutcome::Thawed => listed,
13576 outcome => {
13577 tracing::debug!(
13578 module_id,
13579 ?outcome,
13580 "cgroup thaw failed; proceeding to cgroup.kill"
13581 );
13582 false
13583 }
13584 }
13585}
13586
13587#[cfg(target_os = "linux")]
13588fn apply_cgroup_placement(
13589 command: &mut Command,
13590 spec: &ModuleSpec,
13591 path: &std::path::Path,
13592) -> Result<(), SuperviseError> {
13593 subc_cgroup::apply(command, path).map_err(|source| SuperviseError::Cgroup {
13594 module_id: spec.module_id.clone(),
13595 source,
13596 })
13597}
13598
13599fn capture_retention(spec: &ModuleSpec) -> Retention {
13600 let defaults = Retention::default();
13601 let value = |name: &str| {
13602 spec.env
13603 .iter()
13604 .rev()
13605 .find_map(|(key, value)| (key == name).then_some(value.as_str()))
13606 };
13607 Retention {
13608 max_file_mb: value(CAPTURE_MAX_FILE_MB_ENV)
13609 .and_then(|value| value.parse().ok())
13610 .unwrap_or(defaults.max_file_mb),
13611 keep: value(CAPTURE_KEEP_ENV)
13612 .and_then(|value| value.parse().ok())
13613 .unwrap_or(defaults.keep),
13614 max_age_days: value(CAPTURE_MAX_AGE_DAYS_ENV)
13615 .and_then(|value| value.parse().ok())
13616 .unwrap_or(defaults.max_age_days),
13617 }
13618}
13619
13620fn generate_launch_nonce() -> Result<String, SuperviseError> {
13623 let mut bytes = [0u8; 32];
13624 getrandom::getrandom(&mut bytes).map_err(|source| SuperviseError::LaunchNonce {
13625 reason: source.to_string(),
13626 })?;
13627 let mut hex = String::with_capacity(64);
13628 for b in bytes {
13629 use std::fmt::Write;
13630 let _ = write!(hex, "{b:02x}");
13631 }
13632 Ok(hex)
13633}
13634
13635fn constant_time_eq(a: &[u8], b: &[u8]) -> bool {
13638 if a.len() != b.len() {
13639 return false;
13640 }
13641 let mut diff = 0u8;
13642 for (x, y) in a.iter().zip(b.iter()) {
13643 diff |= x ^ y;
13644 }
13645 diff == 0
13646}
13647
13648#[cfg(not(windows))]
13652fn observe_spawned_image(pid: u32) -> Option<subc_os::Observation> {
13653 let process = subc_os::Process::open(pid).ok().flatten()?;
13654 #[cfg(target_os = "linux")]
13655 return sample_after_linux_exec(
13656 || {
13657 let stat = std::fs::read_to_string(format!("/proc/{pid}/stat")).ok()?;
13658 let committed = linux_exec_committed(&stat);
13659 #[cfg(test)]
13660 spawn_image_tests::report_exec_state(committed);
13661 committed
13662 },
13663 || process.observe(),
13664 );
13665 #[cfg(not(target_os = "linux"))]
13666 process.observe()
13667}
13668
13669#[cfg(target_os = "linux")]
13673fn linux_exec_committed(stat: &str) -> Option<bool> {
13674 const PF_FORKNOEXEC: u64 = 0x40;
13675 let flags = stat.rsplit_once(')')?.1.split_whitespace().nth(6)?;
13676 Some(flags.parse::<u64>().ok()? & PF_FORKNOEXEC == 0)
13677}
13678
13679#[cfg(target_os = "linux")]
13680fn sample_after_linux_exec(
13681 mut exec_committed: impl FnMut() -> Option<bool>,
13682 observe: impl FnOnce() -> Option<subc_os::Observation>,
13683) -> Option<subc_os::Observation> {
13684 let deadline = std::time::Instant::now() + Duration::from_secs(1);
13685 while !exec_committed()? {
13686 if std::time::Instant::now() >= deadline {
13690 return None;
13691 }
13692 std::thread::sleep(Duration::from_micros(50));
13693 }
13694 observe()
13695}
13696
13697#[cfg(all(test, target_os = "linux"))]
13698mod spawn_image_tests {
13699 use super::*;
13700 use std::cell::{Cell, RefCell};
13701
13702 enum ExecEvent {
13703 Uncommitted,
13704 Image(Option<subc_os::Observation>),
13705 }
13706
13707 thread_local! {
13708 static EXEC_EVENTS: RefCell<Option<std::sync::mpsc::Sender<ExecEvent>>> = const { RefCell::new(None) };
13709 }
13710
13711 pub(super) fn report_exec_state(committed: Option<bool>) {
13712 if committed == Some(false) {
13713 EXEC_EVENTS.with(|events| {
13714 if let Some(events) = events.borrow_mut().take() {
13715 let _ = events.send(ExecEvent::Uncommitted);
13716 }
13717 });
13718 }
13719 }
13720
13721 #[test]
13722 fn linux_spawn_observation_waits_for_a_paused_fork_to_exec() {
13723 use std::io::{Read, Write};
13724 use std::os::fd::AsRawFd;
13725
13726 let descriptor = std::fs::File::open("/dev/null").unwrap();
13727 let mut command = std::process::Command::new("/bin/sleep");
13728 command.arg("30");
13729 let (mut ready, mut release) =
13730 subc_os::fork_exec_test::pause_before_exec(&mut command, descriptor.as_raw_fd())
13731 .unwrap();
13732 let spawn = std::thread::spawn(move || command.spawn());
13733 let mut message = [0u8; 8];
13734 ready.read_exact(&mut message).unwrap();
13735 let pid = i32::from_ne_bytes(message[..4].try_into().unwrap()) as u32;
13736 let before = std::fs::read_to_string(format!("/proc/{pid}/stat"));
13737 let inherited = subc_os::Process::open(pid).unwrap().unwrap().observe();
13738 let (events, received) = std::sync::mpsc::channel();
13739 let observer = std::thread::spawn(move || {
13740 EXEC_EVENTS.with(|slot| *slot.borrow_mut() = Some(events.clone()));
13741 let image = observe_spawned_image(pid);
13742 let _ = events.send(ExecEvent::Image(image));
13743 });
13744 let first = received.recv_timeout(Duration::from_secs(5));
13748 release.write_all(&[1]).unwrap();
13749 let mut child = spawn.join().unwrap().unwrap();
13750 let after = match first {
13751 Ok(ExecEvent::Uncommitted) => match received.recv_timeout(Duration::from_secs(5)) {
13752 Ok(ExecEvent::Image(image)) => image,
13753 _ => None,
13754 },
13755 Ok(ExecEvent::Image(image)) => image,
13756 Err(_) => None,
13757 };
13758 observer.join().unwrap();
13759 child.kill().unwrap();
13760 child.wait().unwrap();
13761
13762 assert_eq!(linux_exec_committed(&before.unwrap()), Some(false));
13763 assert_eq!(
13764 inherited.unwrap().executable,
13765 subc_os::file_identity(&std::env::current_exe().unwrap())
13766 );
13767 assert_eq!(
13768 after.unwrap().executable,
13769 subc_os::file_identity(std::path::Path::new("/bin/sleep"))
13770 );
13771 }
13772
13773 #[test]
13774 fn spawned_image_is_not_sampled_until_linux_exec_commits() {
13775 let committed = Cell::new(false);
13776 let inherited = subc_os::Observation {
13777 start_time: 23,
13778 executable: Some(subc_os::FileIdentity {
13779 device: 29,
13780 inode: 31,
13781 }),
13782 };
13783 let module = subc_os::Observation {
13784 start_time: 23,
13785 executable: Some(subc_os::FileIdentity {
13786 device: 29,
13787 inode: 37,
13788 }),
13789 };
13790 let polls = Cell::new(0);
13791 let image = sample_after_linux_exec(
13792 || {
13793 polls.set(polls.get() + 1);
13794 committed.set(polls.get() == 2);
13797 Some(committed.get())
13798 },
13799 || Some(if committed.get() { module } else { inherited }),
13800 );
13801 assert_eq!(image, Some(module));
13802 assert_eq!(polls.get(), 2);
13803 }
13804
13805 #[test]
13806 fn linux_exec_flag_is_read_from_flags_after_the_last_parenthesis() {
13807 assert_eq!(
13808 linux_exec_committed("17 (a ) b) R 1 2 3 4 5 4194368"),
13809 Some(false)
13810 );
13811 assert_eq!(
13812 linux_exec_committed("17 (a ) b) R 1 2 3 4 5 4194304"),
13813 Some(true)
13814 );
13815 assert_eq!(linux_exec_committed("17 (a) R 1 2 3 4 5 invalid"), None);
13816 assert_eq!(linux_exec_committed("17 (a) R"), None);
13817 }
13818
13819 #[test]
13820 fn unreadable_exec_state_never_samples_an_inherited_image() {
13821 assert_eq!(
13822 sample_after_linux_exec(
13823 || None,
13824 || panic!("must not sample before exec confirmation")
13825 ),
13826 None
13827 );
13828 }
13829
13830 #[test]
13831 fn uncommitted_exec_times_out_without_sampling_an_inherited_image() {
13832 assert_eq!(
13833 sample_after_linux_exec(
13834 || Some(false),
13835 || panic!("must not sample after an exec timeout")
13836 ),
13837 None
13838 );
13839 }
13840}
13841
13842#[cfg(windows)]
13845pub(crate) fn windows_spawn_capture(spec: &ModuleSpec) -> io::Result<subc_os::ExecutableCapture> {
13846 if let Err(error) = require_program_search_path(spec) {
13847 return Err(match error {
13848 SuperviseError::Spawn { source, .. } => source,
13849 error => io::Error::other(error),
13850 });
13851 }
13852 let mut program = spec.program.clone();
13853 if program.is_absolute() || program.components().count() > 1 {
13854 match subc_os::ExecutableCapture::open(&program) {
13857 Ok(capture) => return Ok(capture),
13858 Err(error)
13859 if error.kind() == io::ErrorKind::NotFound && program.extension().is_none() => {}
13860 Err(error) => return Err(error),
13861 }
13862 program.set_extension("exe");
13863 return subc_os::ExecutableCapture::open(&program);
13864 }
13865 if program.extension().is_none() {
13866 program.set_extension("exe");
13867 }
13868 let custom_path = spec
13869 .env
13870 .iter()
13871 .rev()
13872 .find(|(key, _)| key.eq_ignore_ascii_case("PATH"));
13873 let mut directories = Vec::new();
13874 if let Some(parent) = std::env::current_exe()?.parent() {
13877 directories.push(parent.to_path_buf());
13878 }
13879 directories.push(std::env::current_dir()?);
13880 if let Some(windows) = std::env::var_os("SystemRoot") {
13881 let windows = PathBuf::from(windows);
13882 directories.push(windows.join("System32"));
13883 directories.push(windows.clone());
13884 }
13885 let path = custom_path
13886 .map(|(_, value)| std::ffi::OsString::from(value))
13887 .or_else(|| std::env::var_os("PATH"));
13888 if let Some(path) = path {
13889 directories.extend(std::env::split_paths(&path));
13890 }
13891 for directory in directories {
13892 match subc_os::ExecutableCapture::open(&directory.join(&program)) {
13893 Ok(capture) => return Ok(capture),
13894 Err(error) if error.kind() == io::ErrorKind::NotFound => continue,
13895 Err(error) => return Err(error),
13896 }
13897 }
13898 Err(io::Error::new(
13899 io::ErrorKind::NotFound,
13900 "resolved executable not found",
13901 ))
13902}
13903
13904async fn spawn_and_mark_running_async(
13905 spec: &ModuleSpec,
13906 runtime: &SupervisorRuntimeConfig,
13907 snapshot: &SharedSnapshot,
13908) -> Result<SupervisedChild, SuperviseError> {
13909 let activity_spawn = runtime.begin_activity_spawn(&spec.module_id);
13910 let registration_watch = watch_registration(&runtime.registry, spec);
13911 #[cfg(unix)]
13912 let joining_group = containment::current(&runtime.child_roster.containment)
13913 .is_some_and(|container| container.group().is_some());
13914 let result = spawn_child_async(spec, runtime).await;
13915 #[cfg(unix)]
13916 let result = if result.is_err()
13917 && joining_group
13918 && !runtime.child_roster.is_closed()
13919 && lock_snapshot(snapshot).is_ok_and(|state| state.enabled)
13920 && runtime
13921 .child_roster
13922 .previous_instance(&spec.module_id)
13923 .is_none()
13924 {
13925 warn!(module_id = %spec.module_id, "carried process-group spawn failed; trying one fresh container");
13928 spawn_child_async(spec, runtime).await
13929 } else {
13930 result
13931 };
13932 let mut child = result?;
13933 arm_registration_deadline(&mut child, runtime.register_timeout, registration_watch);
13934 if let Err(error) = set_running(snapshot, &child, &spec.module_id, &runtime.spawn_events) {
13935 containment::terminate_blocking(&runtime.child_roster.containment);
13936 return Err(error);
13937 }
13938 if let Some(spawn) = activity_spawn {
13939 spawn.complete(snapshot);
13940 }
13941 Ok(child)
13942}
13943
13944#[cfg(test)]
13945fn spawn_and_mark_running(
13946 spec: &ModuleSpec,
13947 runtime: &SupervisorRuntimeConfig,
13948 snapshot: &SharedSnapshot,
13949) -> Result<SupervisedChild, SuperviseError> {
13950 let activity_spawn = runtime.begin_activity_spawn(&spec.module_id);
13951 let registration_watch = watch_registration(&runtime.registry, spec);
13952 #[cfg(unix)]
13953 let joining_group = containment::current(&runtime.child_roster.containment)
13954 .is_some_and(|container| container.group().is_some());
13955 let spawn = || {
13956 spawn_child(
13957 spec,
13958 runtime.connection_file_path.as_deref(),
13959 runtime.supervisor_handle.as_ref(),
13960 &runtime.stderr_ring,
13961 runtime.capture_logs_dir.as_deref(),
13962 &runtime.child_roster,
13963 runtime.spawn_delivery(),
13964 #[cfg(target_os = "linux")]
13965 runtime.cgroup_placement.as_ref(),
13966 runtime.register_timeout,
13967 )
13968 };
13969 let result = spawn();
13970 #[cfg(unix)]
13971 let result = if result.is_err()
13972 && joining_group
13973 && !runtime.child_roster.is_closed()
13974 && lock_snapshot(snapshot).is_ok_and(|state| state.enabled)
13975 && runtime
13976 .child_roster
13977 .previous_instance(&spec.module_id)
13978 .is_none()
13979 {
13980 warn!(module_id = %spec.module_id, "carried process-group spawn failed; trying one fresh container");
13984 spawn()
13985 } else {
13986 result
13987 };
13988 let child = result?;
13989 let mut child = child;
13990 arm_registration_deadline(&mut child, runtime.register_timeout, registration_watch);
13991 if let Err(error) = set_running(snapshot, &child, &spec.module_id, &runtime.spawn_events) {
13992 containment::terminate_blocking(&runtime.child_roster.containment);
13993 return Err(error);
13994 }
13995 if let Some(spawn) = activity_spawn {
13996 spawn.complete(snapshot);
13997 }
13998 Ok(child)
13999}
14000
14001struct RegistrationWatch {
14002 changes: watch::Receiver<u64>,
14003 baseline: u64,
14004}
14005
14006fn watch_registration(registry: &Registry, spec: &ModuleSpec) -> Option<RegistrationWatch> {
14010 if spec.protocol != ModuleProtocol::Subc {
14011 return None;
14012 }
14013 let changes = registry.subscribe_module_changes(&spec.module_id).ok()?;
14014 let baseline = *changes.borrow();
14015 Some(RegistrationWatch { changes, baseline })
14016}
14017
14018fn arm_registration_deadline(
14019 child: &mut SupervisedChild,
14020 budget: Duration,
14021 registration_watch: Option<RegistrationWatch>,
14022) {
14023 child.registration_watch = registration_watch;
14024 child.registration_deadline = (child.protocol == ModuleProtocol::Subc
14025 && child.awaiting_registration())
14026 .then_some(child.spawned_at + budget);
14027}
14028
14029async fn stop_unregistered_child(child: &mut SupervisedChild) -> io::Result<()> {
14030 child.retire_registration();
14031 if let Some(container) = child.child.container() {
14032 container.stop_members().await;
14033 child.kill_leader()?;
14034 child.wait().await?;
14035 return Ok(());
14036 }
14037 #[cfg(unix)]
14038 {
14039 let group = unreaped_registration_group(child)?;
14042 let _ = rustix::process::kill_process_group(group, rustix::process::Signal::TERM);
14043 sleep(crate::child_roster::TERM_TO_KILL).await;
14048 let group = unreaped_registration_group(child)?;
14049 let _ = rustix::process::kill_process_group(group, rustix::process::Signal::KILL);
14050 }
14051 child.start_kill()?;
14054 child.wait().await?;
14055 Ok(())
14056}
14057
14058#[cfg(unix)]
14059fn unreaped_registration_group(child: &SupervisedChild) -> io::Result<rustix::process::Pid> {
14060 if let Some(container) = child.child.container() {
14061 return container
14062 .group()
14063 .and_then(|pid| rustix::process::Pid::from_raw(pid as i32))
14064 .ok_or_else(|| io::Error::other("a carried cgroup has no signalable process group"));
14065 }
14066 let group = i32::try_from(child.pid)
14069 .ok()
14070 .filter(|pid| *pid > 1)
14071 .and_then(rustix::process::Pid::from_raw)
14072 .ok_or_else(|| {
14073 io::Error::new(
14074 io::ErrorKind::InvalidInput,
14075 "invalid supervised process group",
14076 )
14077 })?;
14078 if child.child.id() != Some(child.pid) {
14079 return Err(io::Error::new(
14080 io::ErrorKind::InvalidInput,
14081 "refusing process-group signal after supervised child was reaped",
14082 ));
14083 }
14084 Ok(group)
14085}
14086
14087enum RegistrationWaitOutcome {
14088 Registered,
14089 Exited(ExitReport),
14090 TimedOut,
14091}
14092
14093struct ReloadRegistrationFailure {
14094 exit_report: ExitReport,
14095 reason: String,
14096}
14097
14098#[derive(Debug, Clone, Copy, PartialEq, Eq)]
14099enum BusyGaugeObservation {
14100 Quiescent,
14101 Busy,
14102 Omitted,
14103}
14104
14105#[derive(Debug, Clone, PartialEq, Eq)]
14107enum GaugeVerdict {
14108 Zero,
14110 NonZero(u64),
14112 NotACount(String),
14116 Missing,
14118 ProbeFailed(&'static str),
14121}
14122
14123impl GaugeVerdict {
14124 fn reads_quiet(&self) -> bool {
14125 matches!(self, Self::Zero)
14126 }
14127}
14128
14129impl fmt::Display for GaugeVerdict {
14130 fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
14131 match self {
14132 Self::Zero => f.write_str("zero"),
14133 Self::NonZero(value) => write!(f, "non_zero({value})"),
14134 Self::NotACount(value) => write!(f, "not_a_count({value})"),
14135 Self::Missing => f.write_str("missing_from_health_body"),
14136 Self::ProbeFailed(kind) => write!(f, "probe_failed({kind})"),
14137 }
14138 }
14139}
14140
14141const GAUGE_VALUE_TEXT_LIMIT: usize = 32;
14144
14145fn gauge_verdicts(metrics: Option<&Value>, gauges: &[String]) -> Vec<GaugeVerdict> {
14147 let metrics = metrics.and_then(Value::as_object);
14148 gauges
14149 .iter()
14150 .map(
14151 |gauge| match metrics.and_then(|metrics| metrics.get(gauge)) {
14152 None => GaugeVerdict::Missing,
14153 Some(value) => match value.as_u64() {
14154 Some(0) => GaugeVerdict::Zero,
14155 Some(count) => GaugeVerdict::NonZero(count),
14156 None => {
14157 let mut text = value.to_string();
14158 if text.len() > GAUGE_VALUE_TEXT_LIMIT {
14159 let mut end = GAUGE_VALUE_TEXT_LIMIT;
14160 while !text.is_char_boundary(end) {
14161 end -= 1;
14162 }
14163 text.truncate(end);
14164 text.push_str("...");
14165 }
14166 GaugeVerdict::NotACount(text)
14167 }
14168 },
14169 },
14170 )
14171 .collect()
14172}
14173
14174fn busy_gauge_observation(verdicts: &[GaugeVerdict]) -> BusyGaugeObservation {
14177 if verdicts
14178 .iter()
14179 .any(|verdict| matches!(verdict, GaugeVerdict::Missing))
14180 {
14181 BusyGaugeObservation::Omitted
14182 } else if verdicts.iter().all(GaugeVerdict::reads_quiet) {
14183 BusyGaugeObservation::Quiescent
14184 } else {
14185 BusyGaugeObservation::Busy
14186 }
14187}
14188
14189#[derive(Debug, Clone, Default, PartialEq, Eq)]
14194struct GaugeTrace {
14195 latest: Option<(GaugeVerdict, Duration)>,
14196 last_value: Option<(GaugeVerdict, Duration)>,
14197}
14198
14199#[derive(Debug, Clone, Default, PartialEq, Eq)]
14205struct DrainGaugeTrace {
14206 probes: u32,
14208 gauges: Vec<(String, GaugeTrace)>,
14210}
14211
14212impl DrainGaugeTrace {
14213 fn new(gauges: &[String]) -> Self {
14214 Self {
14215 probes: 0,
14216 gauges: gauges
14217 .iter()
14218 .map(|gauge| (gauge.clone(), GaugeTrace::default()))
14219 .collect(),
14220 }
14221 }
14222
14223 fn record(&mut self, verdicts: &[GaugeVerdict], at: Duration) {
14225 self.probes = self.probes.saturating_add(1);
14226 for ((_, trace), verdict) in self.gauges.iter_mut().zip(verdicts) {
14227 trace.latest = Some((verdict.clone(), at));
14228 if matches!(
14229 verdict,
14230 GaugeVerdict::Zero | GaugeVerdict::NonZero(_) | GaugeVerdict::NotACount(_)
14231 ) {
14232 trace.last_value = Some((verdict.clone(), at));
14233 }
14234 }
14235 }
14236}
14237
14238impl fmt::Display for DrainGaugeTrace {
14243 fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
14244 if self.gauges.is_empty() {
14245 return f.write_str("none declared");
14246 }
14247 for (index, (name, trace)) in self.gauges.iter().enumerate() {
14248 if index > 0 {
14249 f.write_str(" | ")?;
14250 }
14251 write!(f, "{name}: ")?;
14252 match &trace.latest {
14253 Some((verdict, at)) => write!(f, "{verdict} @{}ms", at.as_millis())?,
14254 None => f.write_str("never probed")?,
14255 }
14256 match &trace.last_value {
14257 Some((value, at)) => write!(f, "; last value {value} @{}ms", at.as_millis())?,
14258 None => f.write_str("; no value ever read")?,
14259 }
14260 }
14261 Ok(())
14262 }
14263}
14264
14265pub(crate) fn declared_busy_gauges(
14266 registry: &Registry,
14267 module_id: &str,
14268) -> Result<Vec<String>, SuperviseError> {
14269 busy_gauges_of(
14270 registry
14271 .get_module(module_id)
14272 .map_err(SuperviseError::Registry)?,
14273 )
14274}
14275
14276fn declared_busy_gauges_for_connection(
14280 registry: &Registry,
14281 connection_id: ConnectionId,
14282) -> Result<Vec<String>, SuperviseError> {
14283 busy_gauges_of(
14284 registry
14285 .get_module_by_connection(connection_id)
14286 .map_err(SuperviseError::Registry)?,
14287 )
14288}
14289
14290fn busy_gauges_of(
14291 registration: Option<crate::registry::ModuleRegistration>,
14292) -> Result<Vec<String>, SuperviseError> {
14293 let Some(registration) = registration else {
14294 return Ok(Vec::new());
14295 };
14296 let Some(self_signals) = registration.manifest.self_signals else {
14297 return Ok(Vec::new());
14298 };
14299
14300 let mut gauges = Vec::new();
14301 for declaration in self_signals {
14302 if declaration.kind != SelfSignalKind::Busy {
14303 continue;
14304 }
14305 match declaration.anchored_to {
14306 SignalAnchor::HealthGauges { gauges: declared } if !declared.is_empty() => {
14307 gauges.extend(declared)
14308 }
14309 _ => {
14310 gauges.push(String::new());
14313 }
14314 }
14315 }
14316 Ok(gauges)
14317}
14318
14319#[allow(clippy::too_many_arguments)]
14328async fn wait_for_forwarding_quiescence(
14329 forwarding: &ForwardingTable,
14330 module_id: &str,
14331 runtime: &SupervisorRuntimeConfig,
14332 endpoint: crate::ModuleEndpointId,
14333 deadline: Instant,
14334 busy_gauges: &[String],
14335 scope: DrainScope,
14336 trace: &mut DrainGaugeTrace,
14337) -> Result<bool, SuperviseError> {
14338 let wait_started = Instant::now();
14339 let mut gauges_quiescent = busy_gauges.is_empty();
14340 let mut next_probe_at = wait_started;
14341 let mut omission_reported = false;
14342
14343 loop {
14344 let now = Instant::now();
14345 if !busy_gauges.is_empty() && now >= next_probe_at && now < deadline {
14346 let report = match scope {
14347 DrainScope::Active => probe_module_health(module_id, runtime, Some(deadline)).await,
14348 DrainScope::Endpoint(endpoint) => {
14349 probe_endpoint_health(endpoint, runtime, Some(deadline)).await
14350 }
14351 };
14352 let verdicts = match &report {
14353 Ok(report) => gauge_verdicts(report.metrics.as_ref(), busy_gauges),
14354 Err(err) => vec![GaugeVerdict::ProbeFailed(err.label()); busy_gauges.len()],
14355 };
14356 trace.record(&verdicts, wait_started.elapsed());
14357 gauges_quiescent = match report {
14358 Ok(_) => match busy_gauge_observation(&verdicts) {
14359 BusyGaugeObservation::Quiescent => true,
14360 BusyGaugeObservation::Busy => false,
14361 BusyGaugeObservation::Omitted => {
14362 if !omission_reported {
14365 forwarding
14366 .counters()
14367 .increment_drains_with_undeclared_gauge();
14368 let missing = busy_gauges
14369 .iter()
14370 .zip(&verdicts)
14371 .filter(|(_, verdict)| matches!(verdict, GaugeVerdict::Missing))
14372 .map(|(gauge, _)| gauge.as_str())
14373 .collect::<Vec<_>>()
14374 .join(",");
14375 warn!(
14376 module_id,
14377 missing_gauges = %missing,
14378 "drain health.check answer lacks a declared busy gauge; the drain treats the module as busy until it reports it"
14379 );
14380 omission_reported = true;
14381 }
14382 false
14383 }
14384 },
14385 Err(err) => {
14386 warn!(
14387 module_id,
14388 error = %err,
14389 "drain health.check did not produce declared busy gauges; treating module as busy"
14390 );
14391 false
14392 }
14393 };
14394 next_probe_at = Instant::now() + drain_busy_reprobe_interval();
14397 }
14398
14399 let in_flight = forwarding
14400 .endpoint_in_flight_count(endpoint)
14401 .map_err(SuperviseError::Forwarding)?;
14402 if in_flight == 0 && gauges_quiescent {
14403 return Ok(true);
14404 }
14405
14406 let now = Instant::now();
14407 if now >= deadline {
14408 return Ok(false);
14409 }
14410 let mut wait = deadline
14411 .saturating_duration_since(now)
14412 .min(REGISTRY_RELEASE_POLL);
14413 if !busy_gauges.is_empty() {
14414 wait = wait.min(next_probe_at.saturating_duration_since(now));
14415 }
14416 sleep(wait).await;
14417 }
14418}
14419
14420fn drain_busy_reprobe_interval() -> Duration {
14424 DRAIN_BUSY_REPROBE_INTERVAL.max(REGISTRY_RELEASE_POLL)
14425}
14426
14427fn warn_drain_timed_out(
14442 module_id: &str,
14443 drain_timeout: Duration,
14444 reason: RouteCloseReason,
14445 holdouts: &crate::forwarding::DrainHoldouts,
14446 gauges: &DrainGaugeTrace,
14447) {
14448 warn!(
14449 module_id = %module_id,
14450 waited = ?drain_timeout,
14451 ?reason,
14452 held_requests = holdouts.requests,
14453 held_routes = holdouts.routes,
14454 total_routes = holdouts.total_routes,
14455 top_connections = ?holdouts.top_connections,
14456 held = %holdouts
14459 .held
14460 .iter()
14461 .map(|(channel, corr)| format!("{channel}:{corr}"))
14462 .collect::<Vec<_>>()
14463 .join(","),
14464 gauge_probes = gauges.probes,
14465 busy_gauges = %gauges,
14466 "route drain timed out before request quiescence; forcing teardown"
14467 );
14468}
14469
14470fn drained_after_quiescence_wait(wait_result: &Result<bool, SuperviseError>) -> bool {
14478 match wait_result {
14479 Ok(drained) => *drained,
14480 Err(_) => false,
14481 }
14482}
14483
14484fn send_route_goodbyes(forwarding: &ForwardingTable, released_routes: Vec<GoodbyeTarget>) {
14485 for released in released_routes {
14486 let frame = match Frame::build_with_version(
14487 released.negotiated_ver,
14488 FrameType::Goodbye,
14489 control_flags(),
14490 released.channel,
14491 released.epoch,
14492 0,
14493 Vec::new(),
14494 ) {
14495 Ok(frame) => frame,
14496 Err(err) => {
14497 warn!(
14498 route_channel = released.channel,
14499 error = %err,
14500 "failed to build supervisor drain route GOODBYE frame"
14501 );
14502 continue;
14503 }
14504 };
14505 if !released.close_on_delivery_failure() {
14506 crate::forwarding::send_module_route_goodbye(
14507 &forwarding.counters(),
14508 &released.sink,
14509 frame,
14510 released.module_id.as_deref(),
14511 "supervisor drain",
14512 );
14513 continue;
14514 }
14515 if let Err(err) = released.sink.try_send(frame) {
14516 warn!(
14517 target_connection_id = released.connection_id.get(),
14518 route_channel = released.channel,
14519 error = %err,
14520 "supervisor drain route GOODBYE was not delivered to client; closing target connection"
14521 );
14522 let _ = forwarding.escalate_client_delivery_failure(
14523 released.connection_id,
14524 released.channel,
14525 released.epoch,
14526 CloseReason::new(
14527 "route_goodbye_delivery_failed",
14528 format!(
14529 "failed to enqueue supervisor drain route GOODBYE for channel {}: {err}",
14530 released.channel
14531 ),
14532 ),
14533 crate::forwarding::UndeliveredFrame {
14534 module_id: released.module_id.as_deref(),
14535 sink: &released.sink,
14536 },
14537 );
14538 }
14539 }
14540}
14541
14542async fn send_module_draining(
14543 module_id: &str,
14544 reason: RouteCloseReason,
14545 deadline_ms: u64,
14546 target: &ModuleDrainTarget,
14547 _wait_for_flush: bool,
14548) -> bool {
14549 let body = match serde_json::to_vec(&ModuleControlCommand::Draining {
14550 reason,
14551 deadline_ms,
14552 }) {
14553 Ok(body) => body,
14554 Err(err) => {
14555 warn!(
14556 module_id,
14557 error = %err,
14558 "failed to encode module draining command"
14559 );
14560 return false;
14561 }
14562 };
14563 let frame = match Frame::build_with_version(
14564 target.negotiated_ver,
14565 FrameType::Push,
14566 control_flags(),
14567 0,
14568 0,
14569 0,
14570 body,
14571 ) {
14572 Ok(frame) => frame,
14573 Err(err) => {
14574 warn!(
14575 module_id,
14576 error = %err,
14577 "failed to build module draining command frame"
14578 );
14579 return false;
14580 }
14581 };
14582 #[cfg(unix)]
14583 if _wait_for_flush {
14584 return target.sink.send_flushed(frame).await.is_ok();
14585 }
14586 if let Err(err) = target.sink.try_send(frame) {
14587 warn!(
14588 module_id,
14589 target_connection_id = target.endpoint.connection_id.get(),
14590 error = %err,
14591 "module draining command was not delivered to peer"
14592 );
14593 return false;
14594 }
14595 true
14596}
14597
14598fn module_goodbye_frame(module_id: &str, negotiated_ver: u8) -> Option<Frame> {
14600 match Frame::build_with_version(
14601 negotiated_ver,
14602 FrameType::Goodbye,
14603 control_flags(),
14604 0,
14605 0,
14606 0,
14607 Vec::new(),
14608 ) {
14609 Ok(frame) => Some(frame),
14610 Err(err) => {
14611 warn!(
14612 module_id,
14613 error = %err,
14614 "failed to build module GOODBYE frame"
14615 );
14616 None
14617 }
14618 }
14619}
14620
14621#[cfg(unix)]
14635async fn send_module_goodbyes_for_daemon_shutdown(
14636 forwarding: &Arc<ForwardingTable>,
14637 reason: &CloseReason,
14638 wait_for_flush: bool,
14639) {
14640 const GOODBYE_BUDGET: Duration = Duration::from_millis(500);
14641 let targets = match forwarding.module_connections() {
14642 Ok(targets) => targets,
14643 Err(err) => {
14644 warn!(error = %err, "could not list module connections for shutdown GOODBYE");
14645 return;
14646 }
14647 };
14648 let deadline = Instant::now() + GOODBYE_BUDGET;
14649 let mut sends = tokio::task::JoinSet::new();
14650 for target in targets {
14651 let Some(frame) = module_goodbye_frame(&target.module_id, target.negotiated_ver) else {
14652 continue;
14653 };
14654 if !wait_for_flush {
14655 if let Err(err) = target.sink.try_send(frame) {
14656 debug!(
14657 module_id = %target.module_id,
14658 error = %err,
14659 "shutdown module GOODBYE was not queued"
14660 );
14661 }
14662 continue;
14663 }
14664 let forwarding = Arc::clone(forwarding);
14665 let reason = reason.clone();
14666 sends.spawn(async move {
14667 match timeout_at(deadline, target.sink.send_flushed(frame)).await {
14668 Ok(Ok(())) => {}
14669 Ok(Err(err)) => debug!(
14670 module_id = %target.module_id,
14671 error = %err,
14672 "module connection closed before its shutdown GOODBYE was written"
14673 ),
14674 Err(_) => warn!(
14675 module_id = %target.module_id,
14676 budget = ?GOODBYE_BUDGET,
14677 "shutdown module GOODBYE was not written within its budget; closing anyway"
14678 ),
14679 }
14680 forwarding.request_connection_close(target.endpoint.connection_id, reason);
14681 });
14682 }
14683 while sends.join_next().await.is_some() {}
14685}
14686
14687async fn send_module_goodbye(
14688 module_id: &str,
14689 forwarding: &ForwardingTable,
14690 target: &ModuleDrainTarget,
14691 wait_for_flush: bool,
14692) {
14693 let Some(frame) = module_goodbye_frame(module_id, target.negotiated_ver) else {
14694 return;
14695 };
14696 let sent = if wait_for_flush {
14699 timeout(Duration::from_millis(500), target.sink.send_flushed(frame))
14700 .await
14701 .map_err(|_| "module GOODBYE flush exceeded 500ms".to_owned())
14702 .and_then(|result| result.map_err(|err| err.to_string()))
14703 } else {
14704 target.sink.try_send(frame).map_err(|err| err.to_string())
14705 };
14706 if let Err(err) = sent {
14707 warn!(
14708 module_id,
14709 target_connection_id = target.endpoint.connection_id.get(),
14710 error = %err,
14711 "supervisor drain module GOODBYE was not delivered to peer; closing module connection"
14712 );
14713 forwarding.request_connection_close(
14714 target.endpoint.connection_id,
14715 CloseReason::new(
14716 "module_goodbye_delivery_failed",
14717 format!("failed to enqueue supervisor drain module GOODBYE for module '{module_id}': {err}"),
14718 ),
14719 );
14720 }
14721}
14722
14723#[derive(Clone, Copy)]
14724struct ForwardingDrainContext<'a> {
14725 spec: &'a ModuleSpec,
14726 runtime: &'a SupervisorRuntimeConfig,
14727 registry: &'a Registry,
14728 scope: DrainScope,
14729}
14730
14731#[derive(Debug, Clone, Copy, PartialEq, Eq)]
14733enum DrainScope {
14734 Active,
14737 Endpoint(crate::ModuleEndpointId),
14742}
14743
14744#[derive(Debug, Clone, Copy, PartialEq, Eq)]
14752enum StopNotice {
14753 SentOverConnection,
14756 NoConnection,
14760 NotSent,
14764}
14765
14766async fn begin_forwarding_drain(
14767 spec: &ModuleSpec,
14768 runtime: &SupervisorRuntimeConfig,
14769 registry: &Registry,
14770 snapshot: &SharedSnapshot,
14771 enabled: Option<bool>,
14772 reason: RouteCloseReason,
14773) -> Result<StopNotice, SuperviseError> {
14774 let Some(forwarding) = runtime.forwarding.as_ref() else {
14775 return Err(SuperviseError::ReloadUnavailable {
14776 module_id: spec.module_id.clone(),
14777 reason: "supervisor was not configured with a forwarding table".to_string(),
14778 });
14779 };
14780
14781 begin_forwarding_drain_with(
14782 forwarding,
14783 ForwardingDrainContext {
14784 spec,
14785 runtime,
14786 registry,
14787 scope: DrainScope::Active,
14788 },
14789 snapshot,
14790 enabled,
14791 reason,
14792 runtime.drain_timeout,
14793 )
14794 .await
14795}
14796
14797async fn begin_forwarding_drain_if_configured(
14798 spec: &ModuleSpec,
14799 runtime: &SupervisorRuntimeConfig,
14800 registry: &Registry,
14801 snapshot: &SharedSnapshot,
14802 enabled: Option<bool>,
14803 reason: RouteCloseReason,
14804) -> Result<StopNotice, SuperviseError> {
14805 begin_forwarding_drain_with_timeout(
14806 spec,
14807 runtime,
14808 registry,
14809 snapshot,
14810 enabled,
14811 reason,
14812 runtime.drain_timeout,
14813 )
14814 .await
14815}
14816
14817async fn begin_forwarding_drain_with_timeout(
14821 spec: &ModuleSpec,
14822 runtime: &SupervisorRuntimeConfig,
14823 registry: &Registry,
14824 snapshot: &SharedSnapshot,
14825 enabled: Option<bool>,
14826 reason: RouteCloseReason,
14827 drain_timeout: Duration,
14828) -> Result<StopNotice, SuperviseError> {
14829 let Some(forwarding) = runtime.forwarding.as_ref() else {
14830 return Ok(StopNotice::NotSent);
14831 };
14832
14833 begin_forwarding_drain_with(
14834 forwarding,
14835 ForwardingDrainContext {
14836 spec,
14837 runtime,
14838 registry,
14839 scope: DrainScope::Active,
14840 },
14841 snapshot,
14842 enabled,
14843 reason,
14844 drain_timeout,
14845 )
14846 .await
14847}
14848
14849async fn begin_forwarding_drain_with(
14850 forwarding: &ForwardingTable,
14851 context: ForwardingDrainContext<'_>,
14852 snapshot: &SharedSnapshot,
14853 enabled: Option<bool>,
14854 reason: RouteCloseReason,
14855 drain_timeout: Duration,
14856) -> Result<StopNotice, SuperviseError> {
14857 let ForwardingDrainContext {
14858 spec,
14859 runtime,
14860 registry,
14861 scope,
14862 } = context;
14863 debug_assert_ne!(reason, RouteCloseReason::Crash);
14864 let terminal = matches!(reason, RouteCloseReason::Disable);
14865 let drain_started_at = Instant::now();
14866 let drain_deadline = drain_started_at + drain_timeout;
14867 let deadline_ms =
14868 unix_ms_now().saturating_add(u64::try_from(drain_timeout.as_millis()).unwrap_or(u64::MAX));
14869 let busy_gauges = match scope {
14870 DrainScope::Active => declared_busy_gauges(registry, &spec.module_id)?,
14871 DrainScope::Endpoint(endpoint) => {
14872 declared_busy_gauges_for_connection(registry, endpoint.connection_id)?
14873 }
14874 };
14875
14876 let gate_started = Instant::now();
14879 let drain_target = match scope {
14880 DrainScope::Active => forwarding.begin_module_drain(&spec.module_id, reason),
14881 DrainScope::Endpoint(endpoint) => forwarding.begin_endpoint_drain(endpoint, reason),
14882 }
14883 .map_err(SuperviseError::Forwarding)?;
14884 info!(
14889 module_id = %spec.module_id,
14890 ?reason,
14891 gate_ms = u64::try_from(gate_started.elapsed().as_millis()).unwrap_or(u64::MAX),
14892 connected = drain_target.is_some(),
14893 "module drain began; route admission closed"
14894 );
14895 if scope == DrainScope::Active {
14896 update_snapshot(snapshot, Some(&spec.module_id), |state| {
14897 state.state = ModuleState::Draining;
14898 state.draining_to_replace =
14899 matches!(reason, RouteCloseReason::Restart | RouteCloseReason::Reload);
14900 if let Some(enabled) = enabled {
14901 state.enabled = enabled;
14902 }
14903 })?;
14904 }
14905
14906 let Some(target) = drain_target.as_ref() else {
14907 return Ok(StopNotice::NoConnection);
14911 };
14912 let flush_goodbye = runtime.supervisor_handle.as_ref().is_some_and(|handle| {
14913 handle
14914 .registration_spawn(target.endpoint.connection_id)
14915 .is_some()
14916 });
14917 {
14918 send_module_draining(&spec.module_id, reason, deadline_ms, target, false).await;
14919 #[cfg(feature = "test-support")]
14920 forwarding.wait_before_route_closing_for_test().await;
14921 let routes = forwarding
14922 .endpoint_routes(target.endpoint)
14923 .map_err(SuperviseError::Forwarding)?;
14924 let routes_notified = routes.len();
14925 crate::control::send_route_control_pushes(
14926 forwarding,
14927 routes.clone(),
14928 ClientControlPush::RouteClosing {
14929 module_id: spec.module_id.clone(),
14930 channels: Vec::new(),
14931 reason,
14932 },
14933 );
14934 send_route_goodbyes(forwarding, target.abandoned_bindings.clone());
14935
14936 let mut gauge_trace = DrainGaugeTrace::new(&busy_gauges);
14942 let wait_result = wait_for_forwarding_quiescence(
14943 forwarding,
14944 &spec.module_id,
14945 runtime,
14946 target.endpoint,
14947 drain_deadline,
14948 &busy_gauges,
14949 scope,
14950 &mut gauge_trace,
14951 )
14952 .await;
14953 let drained = drained_after_quiescence_wait(&wait_result);
14954 if let Err(err) = &wait_result {
14955 error!(
14956 module_id = %spec.module_id,
14957 ?reason,
14958 error = %err,
14959 "forwarding quiescence wait failed after route.closing; forcing route.closed(drained: false) so the client is not left waiting on an unfulfilled promise"
14960 );
14961 } else if !drained {
14962 let holdouts = forwarding
14964 .endpoint_drain_holdouts(target.endpoint)
14965 .unwrap_or_default();
14966 warn_drain_timed_out(
14967 &spec.module_id,
14968 drain_timeout,
14969 reason,
14970 &holdouts,
14971 &gauge_trace,
14972 );
14973 }
14974 crate::control::send_route_control_pushes(
14975 forwarding,
14976 routes,
14977 ClientControlPush::RouteClosed {
14978 module_id: spec.module_id.clone(),
14979 channels: Vec::new(),
14980 reason,
14981 drained,
14982 abandoned: target.abandoned_bindings.len() as u32,
14983 excluded_subscriptions: target.excluded_subscriptions,
14984 terminal: Some(terminal),
14985 },
14986 );
14987 wait_result?;
14988
14989 let released_routes = match forwarding.release_module_endpoint_routes(target.endpoint) {
14995 Ok(routes) => routes,
14996 Err(err) => {
14997 warn!(
14998 module_id = %spec.module_id,
14999 ?reason,
15000 error = %err,
15001 "failed to release module endpoint routes after route.closed; module GOODBYE will still be sent"
15002 );
15003 send_module_goodbye(&spec.module_id, forwarding, target, flush_goodbye).await;
15004 return Err(SuperviseError::Forwarding(err));
15005 }
15006 };
15007 let route_goodbye_count = released_routes.len();
15008 send_route_goodbyes(forwarding, released_routes);
15009 send_module_goodbye(&spec.module_id, forwarding, target, flush_goodbye).await;
15010
15011 info!(
15017 module_id = %spec.module_id,
15018 ?reason,
15019 routes_notified,
15020 route_goodbyes = route_goodbye_count,
15021 abandoned_reservations = target.abandoned_bindings.len(),
15022 excluded_subscriptions = target.excluded_subscriptions,
15023 drained,
15024 "module drain complete; consumers notified via route.closing/route.closed pushes and per-route GOODBYE frames"
15025 );
15026 }
15027
15028 Ok(StopNotice::SentOverConnection)
15029}
15030
15031async fn wait_for_registration_after_reload(
15034 registry: &Registry,
15035 module_id: &str,
15036 snapshot: &SharedSnapshot,
15037 child: &mut SupervisedChild,
15038 wait: Duration,
15039) -> Result<RegistrationWaitOutcome, SuperviseError> {
15040 wait_for_slot_registration(
15041 registry,
15042 crate::registry::RegistrationSlot::Active(module_id),
15043 module_id,
15044 snapshot,
15045 child,
15046 wait,
15047 )
15048 .await
15049}
15050
15051async fn wait_for_slot_registration(
15059 registry: &Registry,
15060 slot: crate::registry::RegistrationSlot<'_>,
15061 module_id: &str,
15062 snapshot: &SharedSnapshot,
15063 child: &mut SupervisedChild,
15064 wait: Duration,
15065) -> Result<RegistrationWaitOutcome, SuperviseError> {
15066 let deadline = Instant::now() + wait;
15067 loop {
15068 if registry
15069 .registration(slot)
15070 .map_err(SuperviseError::Registry)?
15071 .is_some()
15072 {
15073 return Ok(RegistrationWaitOutcome::Registered);
15074 }
15075
15076 let now = Instant::now();
15077 if now >= deadline {
15078 return Ok(RegistrationWaitOutcome::TimedOut);
15079 }
15080 let remaining = deadline.saturating_duration_since(now);
15081 let poll = remaining.min(REGISTRY_RELEASE_POLL);
15082
15083 tokio::select! {
15084 wait_result = child.wait() => {
15085 let status = wait_result.map_err(|source| SuperviseError::Wait {
15086 module_id: module_id.to_string(),
15087 source,
15088 })?;
15089 return Ok(RegistrationWaitOutcome::Exited(classify_reaped_child_exit(
15090 snapshot,
15091 child,
15092 &status,
15093 )));
15094 }
15095 _ = sleep(poll) => {}
15096 }
15097 }
15098}
15099
15100fn registration_failure_exit_report(mut exit_report: ExitReport) -> ExitReport {
15101 if exit_report.kind != ExitKind::DeliberateSeverance {
15105 exit_report.kind = ExitKind::Crash;
15106 }
15107 exit_report
15108}
15109
15110async fn handle_reload_child_registration_failure(
15111 spec: &ModuleSpec,
15112 runtime: &SupervisorRuntimeConfig,
15113 registry: &Registry,
15114 process_liveness: &SupervisorProcessLiveness,
15115 snapshot: &SharedSnapshot,
15116 _child: &mut Option<SupervisedChild>,
15117 failure: ReloadRegistrationFailure,
15118) -> Result<(), SuperviseError> {
15119 let ReloadRegistrationFailure {
15120 exit_report,
15121 reason,
15122 } = failure;
15123 match on_child_exit(
15124 spec,
15125 runtime.restart_policy,
15126 registry,
15127 snapshot,
15128 &runtime.terminal_ring,
15129 &runtime.spawn_events,
15130 &runtime.child_roster,
15131 exit_report,
15132 runtime.recovery_is_daemon,
15133 )
15134 .await
15135 {
15136 NextAction::Stop {
15137 registration_released,
15138 } => {
15139 if registration_released {
15140 process_liveness.untrack_if_current(&spec.module_id, snapshot);
15141 }
15142 }
15143 NextAction::Restart { schedule } => {
15144 let delay = schedule.map_or(runtime.restart_policy.delay_for_restart(0), |schedule| {
15145 schedule.delay
15146 });
15147 if let Some(schedule) = schedule {
15148 log_crash_respawn(&spec.module_id, schedule);
15149 }
15150 schedule_respawn(
15151 runtime,
15152 snapshot,
15153 &spec.module_id,
15154 delay,
15155 RespawnKind::Spawn,
15156 true,
15157 )?;
15158 }
15159 }
15160 Err(SuperviseError::ReloadFailed {
15161 module_id: spec.module_id.clone(),
15162 reason,
15163 })
15164}
15165
15166async fn handle_reload_spawn_failure(
15167 spec: &ModuleSpec,
15168 runtime: &SupervisorRuntimeConfig,
15169 process_liveness: &SupervisorProcessLiveness,
15170 snapshot: &SharedSnapshot,
15171 _child: &mut Option<SupervisedChild>,
15172 reason: String,
15173) -> Result<(), SuperviseError> {
15174 let now = Instant::now();
15175 let mut schedule = None;
15176 update_snapshot(snapshot, Some(&spec.module_id), |state| {
15177 clear_current_process_facts(state);
15178 if state.enabled {
15179 schedule = state.next_crash_restart(&runtime.restart_policy, now);
15180 state.state = if schedule.is_some() {
15181 ModuleState::Restarting
15182 } else {
15183 if runtime
15184 .failure_context(runtime.recovery_is_daemon)
15185 .may_retry()
15186 {
15187 state.failed_retry.schedule(&spec.module_id);
15188 }
15189 ModuleState::Failed
15190 };
15191 } else {
15192 state.state = ModuleState::Disabled;
15193 }
15194 })?;
15195 if let Some(schedule) = schedule {
15196 schedule_respawn(
15197 runtime,
15198 snapshot,
15199 &spec.module_id,
15200 schedule.delay,
15201 RespawnKind::Spawn,
15202 true,
15203 )?;
15204 } else {
15205 process_liveness.untrack_if_current(&spec.module_id, snapshot);
15206 }
15207 Err(SuperviseError::ReloadFailed {
15208 module_id: spec.module_id.clone(),
15209 reason,
15210 })
15211}
15212
15213fn control_flags() -> Flags {
15214 Flags::new(false, Priority::Passive, false)
15215}
15216
15217#[allow(clippy::too_many_arguments)]
15218async fn drain_optional_child(
15219 module_id: &str,
15220 protocol: ModuleProtocol,
15221 stop_notice: StopNotice,
15222 registry: &Registry,
15223 forwarding: Option<&ForwardingTable>,
15224 snapshot: &SharedSnapshot,
15225 terminal_ring: &Arc<Mutex<TerminalRing>>,
15226 spawn_events: &SpawnEventFeed,
15227 child: &mut Option<SupervisedChild>,
15228 drain_timeout: Duration,
15229 final_state: ModuleState,
15230 enabled: Option<bool>,
15231 failure: &FailureContext,
15232) -> Result<(), SuperviseError> {
15233 if let Some(child) = child.take() {
15234 drain_child_to_state(
15235 module_id,
15236 protocol,
15237 stop_notice,
15238 registry,
15239 forwarding,
15240 snapshot,
15241 terminal_ring,
15242 spawn_events,
15243 child,
15244 drain_timeout,
15245 final_state,
15246 enabled,
15247 failure,
15248 )
15249 .await
15250 } else {
15251 update_snapshot(snapshot, Some(module_id), |state| {
15252 state.state = final_state;
15253 if final_state == ModuleState::Failed {
15254 state.failed_retry.failed_at = Some(Instant::now());
15255 }
15256 if let Some(enabled) = enabled {
15257 state.enabled = enabled;
15258 }
15259 clear_current_process_facts(state);
15260 })?;
15261 release_dead_registration(registry, forwarding, snapshot, module_id, failure).await
15262 }
15263}
15264
15265#[allow(clippy::too_many_arguments)]
15266async fn drain_child_to_state(
15267 module_id: &str,
15268 _protocol: ModuleProtocol,
15269 stop_notice: StopNotice,
15270 registry: &Registry,
15271 forwarding: Option<&ForwardingTable>,
15272 snapshot: &SharedSnapshot,
15273 terminal_ring: &Arc<Mutex<TerminalRing>>,
15274 spawn_events: &SpawnEventFeed,
15275 mut child: SupervisedChild,
15276 drain_timeout: Duration,
15277 final_state: ModuleState,
15278 enabled: Option<bool>,
15279 failure: &FailureContext,
15280) -> Result<(), SuperviseError> {
15281 let protocol = child.protocol;
15282 child.retire_registration();
15283 update_snapshot(snapshot, Some(module_id), |state| {
15284 state.state = ModuleState::Draining;
15285 state.draining_to_replace = final_state == ModuleState::Restarting;
15286 if let Some(enabled) = enabled {
15287 state.enabled = enabled;
15288 }
15289 })?;
15290
15291 if stop_notice != StopNotice::SentOverConnection {
15302 if protocol == ModuleProtocol::Subc && stop_notice == StopNotice::NoConnection {
15303 info!(
15304 module_id,
15305 pid = child.pid,
15306 budget_ms = u64::try_from(drain_timeout.as_millis()).unwrap_or(u64::MAX),
15307 "module has no connection yet; requesting stop by signal"
15308 );
15309 }
15310 request_graceful_stop(module_id, &child);
15311 }
15312
15313 let exit_report = match timeout(drain_timeout, child.wait()).await {
15314 Ok(Ok(status)) => classify_reaped_child_exit(snapshot, &child, &status),
15315 Ok(Err(source)) => {
15316 fail_after_wait_error(
15317 module_id,
15318 snapshot,
15319 terminal_ring,
15320 spawn_events,
15321 format!("failed to wait during drain: {source}"),
15322 failure,
15323 );
15324 return Err(SuperviseError::Wait {
15325 module_id: module_id.to_string(),
15326 source,
15327 });
15328 }
15329 Err(_) => {
15330 warn!(
15343 module_id,
15344 pid = child.pid,
15345 budget_ms = u64::try_from(drain_timeout.as_millis()).unwrap_or(u64::MAX),
15346 reason = ?final_state,
15347 ?stop_notice,
15348 "drain budget expired before the module exited; killing it"
15349 );
15350 let kill =
15351 if final_state == ModuleState::Restarting && child.child.container().is_some() {
15352 child.kill_leader()
15353 } else {
15354 child.start_kill()
15355 };
15356 kill.map_err(|source| {
15357 fail_after_wait_error(
15358 module_id,
15359 snapshot,
15360 terminal_ring,
15361 spawn_events,
15362 format!("failed to kill during drain: {source}"),
15363 failure,
15364 );
15365 SuperviseError::Kill {
15366 module_id: module_id.to_string(),
15367 source,
15368 }
15369 })?;
15370 let status = child.wait().await.map_err(|source| {
15371 fail_after_wait_error(
15372 module_id,
15373 snapshot,
15374 terminal_ring,
15375 spawn_events,
15376 format!("failed to wait after drain kill: {source}"),
15377 failure,
15378 );
15379 SuperviseError::Wait {
15380 module_id: module_id.to_string(),
15381 source,
15382 }
15383 })?;
15384 classify_reaped_child_exit(snapshot, &child, &status)
15385 }
15386 };
15387
15388 let detail = lock_snapshot(snapshot)?.drain_disposition_detail.take();
15389 publish_after_terminal_record(
15390 module_id,
15391 || {
15392 record_terminal_with_detail(
15393 module_id,
15394 terminal_ring,
15395 spawn_events,
15396 &exit_report,
15397 terminal_disposition(final_state),
15398 detail,
15399 )
15400 },
15401 || {
15402 update_snapshot(snapshot, Some(module_id), |state| {
15403 state.state = final_state;
15404 if final_state == ModuleState::Failed {
15405 state.failed_retry.failed_at = Some(Instant::now());
15406 }
15407 if let Some(enabled) = enabled {
15408 state.enabled = enabled;
15409 }
15410 clear_current_process_facts(state);
15411 state.last_exit = Some(exit_report.clone());
15412 if exit_report.kind == ExitKind::DeliberateSeverance {
15413 state.lifetime_restarts += 1;
15414 }
15415 })
15416 },
15417 )?;
15418 child.drain_stderr(module_id).await;
15419
15420 let released =
15421 release_dead_registration(registry, forwarding, snapshot, module_id, failure).await;
15422 child
15423 .finish_exit(
15424 released.is_ok()
15425 && final_state == ModuleState::Restarting
15426 && child.spawn_failure.is_none(),
15427 )
15428 .await
15429 .map_err(|source| SuperviseError::Wait {
15430 module_id: module_id.to_owned(),
15431 source,
15432 })?;
15433 released
15434}
15435
15436#[cfg(unix)]
15456fn request_graceful_stop(module_id: &str, child: &SupervisedChild) {
15457 let Some(pid) = child
15458 .id()
15459 .and_then(|pid| i32::try_from(pid).ok())
15460 .and_then(rustix::process::Pid::from_raw)
15461 else {
15462 debug!(
15463 module_id,
15464 "no pid to signal for teardown; falling through to the drain wait"
15465 );
15466 return;
15467 };
15468 match rustix::process::kill_process(pid, rustix::process::Signal::TERM) {
15469 Ok(()) => debug!(
15470 module_id,
15471 "sent SIGTERM to a module nothing else asked to stop"
15472 ),
15473 Err(err) => debug!(
15474 module_id,
15475 error = %err,
15476 "SIGTERM to module failed; the drain wait and kill still apply"
15477 ),
15478 }
15479}
15480
15481#[cfg(not(unix))]
15489fn request_graceful_stop(module_id: &str, _child: &SupervisedChild) {
15490 debug!(
15491 module_id,
15492 "no graceful stop signal exists on this platform; teardown of a module nothing asked to stop waits, then kills"
15493 );
15494}
15495
15496fn terminal_disposition(final_state: ModuleState) -> TerminalDisposition {
15497 match final_state {
15498 ModuleState::Stopped => TerminalDisposition::Stopped,
15499 ModuleState::Disabled => TerminalDisposition::Disabled,
15500 ModuleState::Restarting => TerminalDisposition::Restarting,
15501 ModuleState::Failed => TerminalDisposition::Failed,
15502 ModuleState::Starting
15503 | ModuleState::Running
15504 | ModuleState::Unresponsive
15505 | ModuleState::Draining => {
15506 unreachable!("terminal exits only finish in terminal or restarting states")
15507 }
15508 }
15509}
15510
15511async fn release_dead_registration(
15519 registry: &Registry,
15520 forwarding: Option<&ForwardingTable>,
15521 snapshot: &SharedSnapshot,
15522 module_id: &str,
15523 failure: &FailureContext,
15524) -> Result<(), SuperviseError> {
15525 let result = async {
15526 let registration = registry
15527 .get_module(module_id)
15528 .map_err(SuperviseError::Registry)?;
15529 match wait_for_registration_release(registry, module_id, REGISTRY_RELEASE_TIMEOUT).await {
15530 Ok(()) => return Ok(()),
15531 Err(SuperviseError::RegistrationStillActive { .. }) => {}
15532 Err(err) => return Err(err),
15533 }
15534 let pid = lock_snapshot(snapshot)?.reaped_pid;
15535 if let (Some(registration), Some(forwarding), Some(pid)) = (registration, forwarding, pid) {
15536 warn!(
15537 module_id,
15538 pid,
15539 connection_id = registration.connection_id.get(),
15540 "reaped module registration outlived release grace; closing dead connection"
15541 );
15542 forwarding.request_connection_close(
15543 registration.connection_id,
15544 CloseReason::new(
15545 "supervised_process_reaped",
15546 format!("module '{module_id}' pid {pid} exited"),
15547 ),
15548 );
15549 wait_for_slot_registration_release(
15550 registry,
15551 crate::registry::RegistrationSlot::Connection(registration.connection_id),
15552 REGISTRY_RELEASE_TIMEOUT,
15553 )
15554 .await?;
15555 }
15556 wait_for_registration_release(registry, module_id, Duration::ZERO).await
15557 }
15558 .await;
15559 if let Err(err) = &result {
15560 fail_snapshot(snapshot, Some(module_id), None, failure);
15561 error!(module_id, error = %err, "registration release failed after child exit; module is failed and start can retry");
15562 }
15563 result
15564}
15565
15566async fn wait_for_registration_release(
15569 registry: &Registry,
15570 module_id: &str,
15571 wait: Duration,
15572) -> Result<(), SuperviseError> {
15573 wait_for_slot_registration_release(
15574 registry,
15575 crate::registry::RegistrationSlot::Active(module_id),
15576 wait,
15577 )
15578 .await
15579}
15580
15581async fn wait_for_slot_registration_release(
15589 registry: &Registry,
15590 slot: crate::registry::RegistrationSlot<'_>,
15591 wait: Duration,
15592) -> Result<(), SuperviseError> {
15593 let deadline = Instant::now() + wait;
15594 let mut release_events = registration_release_events().subscribe();
15595 let still_active = |registration: &crate::registry::ModuleRegistration| {
15596 SuperviseError::RegistrationStillActive {
15597 module_id: registration.manifest.module_id.clone(),
15598 waited: wait,
15599 }
15600 };
15601 loop {
15602 let _observed_generation = *release_events.borrow_and_update();
15603 let Some(registration) = registry
15604 .registration(slot)
15605 .map_err(SuperviseError::Registry)?
15606 else {
15607 return Ok(());
15608 };
15609
15610 let now = Instant::now();
15611 if now >= deadline {
15612 return Err(still_active(®istration));
15613 }
15614
15615 let remaining = deadline.saturating_duration_since(now);
15616 match timeout(remaining, release_events.changed()).await {
15617 Ok(Ok(())) | Ok(Err(_)) => {}
15618 Err(_) => return Err(still_active(®istration)),
15619 }
15620 }
15621}
15622
15623#[cfg(test)]
15624mod slot_registration_wait_tests {
15625 use super::*;
15626 use crate::registry::{ConnectionId, RegistrationSlot};
15627 use subc_protocol::manifest::ModuleManifest;
15628
15629 #[tokio::test]
15630 async fn enable_release_failure_is_failed_and_a_second_enable_retries() {
15631 let registry = Arc::new(Registry::default());
15632 let supervisor = Supervisor::new_for_test(Arc::clone(®istry), RestartPolicy::default());
15633 let runtime = supervisor.runtime_config();
15634 let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::disabled()));
15635 let spec = ModuleSpec {
15636 script: None,
15637 module_id: "enable-stale-registration".to_string(),
15638 program: PathBuf::from("/missing/enable-retry-test"),
15639 args: Vec::new(),
15640 env: Vec::new(),
15641 reserved: false,
15642 reserved_prefixes: Vec::new(),
15643 protocol: ModuleProtocol::Subc,
15644 overlap: Default::default(),
15645 carry_containment: false,
15646 };
15647 let connection = ConnectionId::new(90);
15648 registry
15649 .register_with_control_ops(
15650 ModuleManifest::builder(&spec.module_id, "0.1.0").build(),
15651 1,
15652 connection,
15653 Vec::new(),
15654 )
15655 .unwrap();
15656 let mut child = None;
15657 let err = set_child_enabled(
15658 &spec,
15659 &runtime,
15660 ®istry,
15661 &supervisor.process_liveness,
15662 &snapshot,
15663 &mut child,
15664 true,
15665 &runtime.failure_context(false),
15666 )
15667 .await
15668 .unwrap_err();
15669 assert!(matches!(
15670 err,
15671 SuperviseError::RegistrationStillActive { .. }
15672 ));
15673 assert_eq!(lock_snapshot(&snapshot).unwrap().state, ModuleState::Failed);
15674 assert!(child.is_none());
15675 registry.deregister_connection(connection).unwrap();
15676 let err = set_child_enabled(
15677 &spec,
15678 &runtime,
15679 ®istry,
15680 &supervisor.process_liveness,
15681 &snapshot,
15682 &mut child,
15683 true,
15684 &runtime.failure_context(false),
15685 )
15686 .await
15687 .unwrap_err();
15688 assert!(
15689 matches!(err, SuperviseError::Spawn { .. }),
15690 "second enable must attempt a spawn: {err}"
15691 );
15692 assert_eq!(lock_snapshot(&snapshot).unwrap().state, ModuleState::Failed);
15693 }
15694
15695 const INCUMBENT: u64 = 1;
15696 const CANDIDATE: u64 = 2;
15697
15698 fn swapped_registry() -> Arc<Registry> {
15699 let registry = Arc::new(Registry::default());
15700 let manifest = ModuleManifest::builder("m", "0.1.0").build();
15701 registry
15702 .register_with_control_ops(
15703 manifest.clone(),
15704 1,
15705 ConnectionId::new(INCUMBENT),
15706 Vec::new(),
15707 )
15708 .unwrap();
15709 registry
15710 .register_candidate_with_control_ops(
15711 manifest,
15712 1,
15713 ConnectionId::new(CANDIDATE),
15714 Vec::new(),
15715 )
15716 .unwrap();
15717 registry
15718 }
15719
15720 #[tokio::test]
15724 async fn incumbent_release_is_awaited_by_connection_not_by_module_id() {
15725 let registry = swapped_registry();
15726 registry.promote_candidate("m").unwrap().unwrap();
15727
15728 assert!(matches!(
15729 wait_for_registration_release(®istry, "m", Duration::from_millis(50)).await,
15730 Err(SuperviseError::RegistrationStillActive { .. })
15731 ));
15732
15733 assert!(matches!(
15735 wait_for_slot_registration_release(
15736 ®istry,
15737 RegistrationSlot::Connection(ConnectionId::new(INCUMBENT)),
15738 Duration::from_millis(50),
15739 )
15740 .await,
15741 Err(SuperviseError::RegistrationStillActive { .. })
15742 ));
15743
15744 let releaser = Arc::clone(®istry);
15745 let release = tokio::spawn(async move {
15746 sleep(Duration::from_millis(20)).await;
15747 releaser
15748 .deregister_connection(ConnectionId::new(INCUMBENT))
15749 .unwrap();
15750 notify_registration_release();
15751 });
15752 wait_for_slot_registration_release(
15753 ®istry,
15754 RegistrationSlot::Connection(ConnectionId::new(INCUMBENT)),
15755 Duration::from_secs(5),
15756 )
15757 .await
15758 .expect("the incumbent's own registration is released");
15759 release.await.unwrap();
15760 assert!(registry.get_module("m").unwrap().is_some());
15761 }
15762
15763 #[tokio::test]
15766 async fn candidate_slot_wait_ignores_the_incumbents_registration() {
15767 let registry = swapped_registry();
15768 assert!(matches!(
15769 wait_for_slot_registration_release(
15770 ®istry,
15771 RegistrationSlot::Candidate("m"),
15772 Duration::from_millis(50),
15773 )
15774 .await,
15775 Err(SuperviseError::RegistrationStillActive { .. })
15776 ));
15777 registry
15778 .deregister_connection(ConnectionId::new(CANDIDATE))
15779 .unwrap();
15780 wait_for_slot_registration_release(
15781 ®istry,
15782 RegistrationSlot::Candidate("m"),
15783 Duration::from_millis(50),
15784 )
15785 .await
15786 .expect("a candidate slot with no candidate is released");
15787 assert!(registry
15788 .registration(RegistrationSlot::Active("m"))
15789 .unwrap()
15790 .is_some());
15791 }
15792}
15793
15794fn classify_exit(status: &ExitStatus) -> ExitReport {
15795 ExitReport {
15796 kind: if status.success() {
15797 ExitKind::Clean
15798 } else {
15799 ExitKind::Crash
15800 },
15801 code: status.code(),
15802 signal: exit_signal(status),
15803 at_ms: unix_ms_now(),
15804 }
15805}
15806
15807fn wait_error_exit_report() -> ExitReport {
15813 ExitReport {
15814 kind: ExitKind::Crash,
15815 code: None,
15816 signal: None,
15817 at_ms: unix_ms_now(),
15818 }
15819}
15820
15821#[cfg(unix)]
15822fn exit_signal(status: &ExitStatus) -> Option<i32> {
15823 use std::os::unix::process::ExitStatusExt;
15824
15825 status.signal()
15826}
15827
15828#[cfg(not(unix))]
15829fn exit_signal(_status: &ExitStatus) -> Option<i32> {
15830 None
15831}
15832
15833fn reset_restart_count(snapshot: &SharedSnapshot, module_id: &str) -> Result<(), SuperviseError> {
15839 update_snapshot(snapshot, Some(module_id), |state| {
15840 state.clear_crash_restarts();
15841 })
15842}
15843
15844fn set_running(
15845 snapshot: &SharedSnapshot,
15846 child: &SupervisedChild,
15847 module_id: &str,
15848 spawn_events: &SpawnEventFeed,
15849) -> Result<(), SuperviseError> {
15850 let mut state = snapshot.lock().map_err(|_| SuperviseError::StatePoisoned {
15851 module_id: Some(module_id.to_string()),
15852 })?;
15853 state.spawn_generation = spawn_events.emit_spawned(
15854 module_id,
15855 child
15856 .registration_spawn
15857 .as_ref()
15858 .map(|spawn| spawn.authority.spawn_number()),
15859 child.pid,
15860 child.spawned_at_ms,
15861 );
15862 if std::mem::take(&mut state.coalesced_restart_pending) {
15863 let generation = state.spawn_generation;
15864 info!(module_id, "restart already satisfied by generation {generation}; coalesced pending request completed");
15865 }
15866 state.drain_disposition_detail = None;
15867 state.spawn_failure = None;
15868 state.admission_refused = None;
15869 state.config_hold_parked = false;
15870 state.config_hold_daemon_initiated = false;
15871 state.config_hold_approval = None;
15872 state.legacy_registration_spawn =
15873 child.registration_spawn.is_some() && child.admission_base.is_none();
15874 state.registration_declaration = child.registration_declaration.clone();
15875 state.stop_reason = None;
15876 state.config_exit_failed_approval = None;
15877 state.waiting_for_previous_pid = None;
15878 state.waiting_for_previous_detail = None;
15879 state.in_alternate_slot = false;
15882 state.configuration_updated_since_spawn = false;
15883 state.spawned_protocol = Some(child.protocol);
15884 state.state = ModuleState::Running;
15885 state.approved_start_failure = None;
15886 state.enabled = true;
15887 state.process_alive = true;
15888 state.pid = child.id();
15889 state.subtree_target = child.subtree_target.clone();
15890 state.health.status = SupervisorHealthStatus::Unknown;
15893 state.last_probe_evidence = None;
15894 state.health.consecutive_failures = 0;
15895 state.health.detail = None;
15896 state.health.metrics = None;
15897 state.health.last_probe_ms = None;
15898 state.failed_retry.healthy_since = None;
15899 state.failed_retry.failed_at = None;
15900 state.failed_retry.running_since = Some(child.spawned_at);
15901 #[cfg(windows)]
15902 {
15903 state.pipe_consumed = child
15904 .nonce_delivery
15905 .as_ref()
15906 .map(|delivery| delivery.consumed());
15907 }
15908 #[cfg(target_os = "macos")]
15909 {
15910 state.report_ready = Some(Arc::clone(&child.report_ready));
15911 }
15912 state.spawned_at_ms = Some(child.spawned_at_ms);
15913 state.exec_outcome = Some(Arc::clone(&child.exec.0));
15914 state.spawned_from = Some(child.spawned_from.clone());
15915 state.launch_spec_digest = Some(child.launch_spec_digest.clone());
15916 state.spawned_file_identity = child.spawned_file_identity.clone();
15917 state.process_start_time = child.process_start_time;
15918 Ok(())
15919}
15920
15921fn clear_current_process_facts(state: &mut SupervisorSnapshot) {
15922 state.legacy_registration_spawn = false;
15923 state.registration_declaration = None;
15924 state.subtree_target = crate::subtree_resources::Target::Unavailable("not_running");
15925 state.failed_retry.healthy_since = None;
15926 state.failed_retry.running_since = None;
15927 state.waiting_for_previous_pid = None;
15928 state.waiting_for_previous_detail = None;
15929 state.process_alive = false;
15930 state.spawned_protocol = None;
15931 state.pid = None;
15932 #[cfg(windows)]
15933 {
15934 state.pipe_consumed = None;
15935 }
15936 #[cfg(target_os = "macos")]
15937 {
15938 state.report_ready = None;
15939 }
15940 state.spawned_at_ms = None;
15941 state.exec_outcome = None;
15942 state.spawned_from = None;
15943 state.launch_spec_digest = None;
15944 state.spawned_file_identity = None;
15945 state.process_start_time = None;
15946 state.deliberate_severance = None;
15947}
15948
15949#[cfg(test)]
15950fn record_deliberate_severance(
15951 snapshot: &SharedSnapshot,
15952 identity: ProcessIdentity,
15953) -> Result<(), SuperviseError> {
15954 update_snapshot(snapshot, None, |state| {
15955 state.deliberate_severance = Some(identity);
15956 })
15957}
15958
15959fn apply_deliberate_severance_marker(
15960 snapshot: &SharedSnapshot,
15961 exited_identity: Option<ProcessIdentity>,
15962 mut exit_report: ExitReport,
15963) -> ExitReport {
15964 let marker = lock_snapshot(snapshot)
15965 .ok()
15966 .and_then(|mut state| state.deliberate_severance.take());
15967 if marker.is_some() && marker == exited_identity {
15968 exit_report.kind = ExitKind::DeliberateSeverance;
15969 }
15970 exit_report
15971}
15972
15973fn classify_reaped_child_exit(
15974 snapshot: &SharedSnapshot,
15975 child: &SupervisedChild,
15976 status: &ExitStatus,
15977) -> ExitReport {
15978 let _ = update_snapshot(snapshot, None, |state| {
15979 state.reaped_pid = Some(child.pid);
15980 state.spawn_failure = child.spawn_failure.clone();
15981 });
15982 if matches!(
15983 child.spawn_failure.as_deref(),
15984 Some("registration_timeout" | "registration_channel_closed")
15985 ) {
15986 registration_failure_exit_report(classify_exit(status))
15987 } else {
15988 apply_deliberate_severance_marker(snapshot, child.process_identity(), classify_exit(status))
15989 }
15990}
15991
15992fn fail_snapshot(
15993 snapshot: &SharedSnapshot,
15994 module_id: Option<&str>,
15995 last_exit: Option<ExitReport>,
15996 failure: &FailureContext,
15997) {
15998 if let Err(err) = update_snapshot(snapshot, module_id, |state| {
15999 state.state = ModuleState::Failed;
16000 state.config_hold_parked = false;
16001 state.config_hold_daemon_initiated = false;
16002 state.config_hold_approval = None;
16003 clear_current_process_facts(state);
16004 if failure.may_retry() && state.enabled {
16005 state.failed_retry.schedule(module_id.unwrap_or("unknown"));
16006 } else if failure.shutdown.is_set() {
16007 state.failed_retry.cancel();
16008 }
16009 if let Some(last_exit) = last_exit {
16010 state.last_exit = Some(last_exit);
16011 }
16012 }) {
16013 error!(error = %err, "failed to mark supervisor state failed");
16014 }
16015}
16016
16017fn update_snapshot(
16018 snapshot: &SharedSnapshot,
16019 module_id: Option<&str>,
16020 update: impl FnOnce(&mut SupervisorSnapshot),
16021) -> Result<(), SuperviseError> {
16022 let mut state = snapshot.lock().map_err(|_| SuperviseError::StatePoisoned {
16023 module_id: module_id.map(ToOwned::to_owned),
16024 })?;
16025 update(&mut state);
16026 Ok(())
16027}
16028
16029const SLOW_SNAPSHOT_LOCK_THRESHOLD: Duration = Duration::from_millis(250);
16030
16031fn lock_snapshot_for_control<'a>(
16032 snapshot: &'a SharedSnapshot,
16033 module_id: &str,
16034 caller: &'static str,
16035) -> Result<std::sync::MutexGuard<'a, SupervisorSnapshot>, SuperviseError> {
16036 let started_at = Instant::now();
16037 let guard = lock_snapshot(snapshot)?;
16038 let waited = started_at.elapsed();
16039 if waited >= SLOW_SNAPSHOT_LOCK_THRESHOLD {
16040 warn!(
16041 module_id = %module_id,
16042 waited_ms = waited.as_millis() as u64,
16043 caller = %caller,
16044 "slow snapshot lock"
16045 );
16046 }
16047 Ok(guard)
16048}
16049
16050fn lock_snapshot(
16051 snapshot: &SharedSnapshot,
16052) -> Result<std::sync::MutexGuard<'_, SupervisorSnapshot>, SuperviseError> {
16053 snapshot
16054 .lock()
16055 .map_err(|_| SuperviseError::StatePoisoned { module_id: None })
16056}
16057
16058#[cfg(test)]
16059mod terminal_history_tests {
16060 include!("supervise_route_open_recovery_tests.rs");
16061 use std::{
16062 path::PathBuf,
16063 sync::Arc,
16064 time::{Duration, Instant},
16065 };
16066
16067 use tokio::{sync::mpsc, time::sleep};
16068
16069 use super::{
16070 apply_deliberate_severance_marker, daemon_will_restart, drain_child_to_state,
16071 drained_after_quiescence_wait, handle_reload_spawn_failure, health_restart_child,
16072 lock_snapshot, on_child_exit, record_deliberate_severance, record_wait_error_terminal,
16073 reset_restart_count, spawn_and_mark_running, update_snapshot, wait_error_exit_report,
16074 ExitKind, ExitReport, ModuleProtocol, ModuleSpec, ModuleState, NextAction, ProcessIdentity,
16075 RestartPolicy, SpawnEventKind, StopNotice, SuperviseError, SupervisedConfiguration,
16076 SupervisedModule, SupervisedModuleInner, Supervisor, SupervisorHandle,
16077 SupervisorHealthStatus, SupervisorSnapshot,
16078 };
16079 use super::Instant as ClockInstant;
16084 use crate::{
16085 registry::Registry,
16086 terminal_ring::{TerminalRing, TerminalRingConfig},
16087 };
16088 use std::sync::Mutex;
16089 use subc_control::TerminalDisposition;
16090
16091 pub(super) fn fake_aft_stub_path() -> PathBuf {
16096 let path = cortexkit_test_support::cargo_profile_bin("fake-aft-stub");
16097 assert!(
16098 path.exists(),
16099 "fake-aft-stub not built at {}: run `cargo test -p subc-core` (which builds \
16100 [[bin]] targets) rather than `cargo test -p subc-core --lib` (which does not)",
16101 path.display()
16102 );
16103 path
16104 }
16105
16106 #[test]
16107 fn reserved_never_spawned_refuses_every_hello() {
16108 let supervisor = SupervisorHandle::default();
16113 supervisor.apply_identity_configuration(&ModuleSpec {
16114 script: None,
16115 module_id: "never-spawned".to_string(),
16116 program: PathBuf::from("/usr/bin/false"),
16117 args: Vec::new(),
16118 env: Vec::new(),
16119 reserved: true,
16120 reserved_prefixes: Vec::new(),
16121 protocol: ModuleProtocol::Subc,
16122 overlap: Default::default(),
16123 carry_containment: false,
16124 });
16125 assert!(
16126 supervisor
16127 .reserved_hello_verification("never-spawned", Some("any-forged-nonce"))
16128 .is_err(),
16129 "forged nonce must refuse on a reserved never-spawned id"
16130 );
16131 assert!(
16132 supervisor
16133 .reserved_hello_verification("never-spawned", None)
16134 .is_err(),
16135 "absent nonce must refuse on a reserved never-spawned id"
16136 );
16137 supervisor.set_spawn_nonce("never-spawned", "minted".to_string());
16139 supervisor.apply_identity_configuration(&ModuleSpec {
16140 script: None,
16141 module_id: "never-spawned".to_string(),
16142 program: PathBuf::from("/usr/bin/false"),
16143 args: Vec::new(),
16144 env: Vec::new(),
16145 reserved: true,
16146 reserved_prefixes: Vec::new(),
16147 protocol: ModuleProtocol::Subc,
16148 overlap: Default::default(),
16149 carry_containment: false,
16150 });
16151 assert!(supervisor
16152 .reserved_hello_verification("never-spawned", Some("minted"))
16153 .is_ok());
16154 assert!(supervisor
16155 .reserved_hello_verification("never-spawned", Some("forged"))
16156 .is_err());
16157 }
16158
16159 fn seed_crash_restarts(state: &mut SupervisorSnapshot, count: u32) {
16162 let now = ClockInstant::now();
16163 for _ in 0..count {
16164 state.crash_restarts.push_back(now);
16165 }
16166 }
16167
16168 fn age_oldest_crash_restart_out_of_window(state: &mut SupervisorSnapshot, window: Duration) {
16172 let aged = state
16173 .crash_restarts
16174 .front()
16175 .expect("a crash restart must be recorded before it can be aged")
16176 .checked_sub(window + Duration::from_secs(1))
16177 .expect("the test clock is far enough from its origin to age an instant");
16178 state.crash_restarts[0] = aged;
16179 }
16180
16181 fn snapshot_with_restarts(enabled: bool, count: u32) -> SupervisorSnapshot {
16182 let mut state = SupervisorSnapshot::new(ModuleState::Running, enabled);
16183 seed_crash_restarts(&mut state, count);
16184 state
16185 }
16186
16187 #[test]
16188 fn daemon_owned_recovery_predicate_uses_the_pre_increment_budget() {
16189 let policy = RestartPolicy::new(3, Duration::ZERO);
16190 let now = ClockInstant::now();
16191 assert!(daemon_will_restart(
16192 &mut snapshot_with_restarts(true, 2),
16193 &policy,
16194 now
16195 ));
16196 assert!(!daemon_will_restart(
16197 &mut snapshot_with_restarts(true, 3),
16198 &policy,
16199 now
16200 ));
16201 assert!(!daemon_will_restart(
16202 &mut snapshot_with_restarts(false, 0),
16203 &policy,
16204 now
16205 ));
16206 }
16207
16208 #[test]
16209 fn crash_restart_backoff_escalates_with_in_window_count() {
16210 let policy = RestartPolicy::new(4, Duration::from_millis(100))
16211 .with_max_backoff(Duration::from_secs(30));
16212 let now = ClockInstant::now();
16213 let mut state = SupervisorSnapshot::new(ModuleState::Running, true);
16214 let schedules = (0..4)
16215 .map(|_| {
16216 state
16217 .next_crash_restart(&policy, now)
16218 .expect("the test policy allows four crash restarts")
16219 })
16220 .collect::<Vec<_>>();
16221
16222 assert_eq!(
16223 schedules
16224 .iter()
16225 .map(|schedule| schedule.restart_in_window)
16226 .collect::<Vec<_>>(),
16227 vec![0, 1, 2, 3]
16228 );
16229 assert_eq!(
16230 schedules
16231 .iter()
16232 .map(|schedule| schedule.delay)
16233 .collect::<Vec<_>>(),
16234 vec![
16235 Duration::from_millis(100),
16236 Duration::from_secs(1),
16237 Duration::from_secs(10),
16238 Duration::from_secs(30),
16239 ]
16240 );
16241 }
16242
16243 #[test]
16244 fn crash_restart_backoff_resets_after_ring_clear() {
16245 let policy = RestartPolicy::new(3, Duration::from_millis(100));
16246 let now = ClockInstant::now();
16247 let mut state = SupervisorSnapshot::new(ModuleState::Running, true);
16248 assert_eq!(
16249 state.next_crash_restart(&policy, now).unwrap().delay,
16250 Duration::from_millis(100)
16251 );
16252 assert_eq!(
16253 state.next_crash_restart(&policy, now).unwrap().delay,
16254 Duration::from_secs(1)
16255 );
16256
16257 state.clear_crash_restarts();
16258 let schedule = state
16259 .next_crash_restart(&policy, now)
16260 .expect("a cleared ring must allow another restart");
16261 assert_eq!(schedule.restart_in_window, 0);
16262 assert_eq!(schedule.delay, Duration::from_millis(100));
16263 }
16264
16265 #[test]
16266 fn crash_restart_backoff_ignores_aged_restarts() {
16267 let policy = RestartPolicy::new(3, Duration::from_millis(100));
16268 let now = ClockInstant::now();
16269 let mut state = SupervisorSnapshot::new(ModuleState::Running, true);
16270 state
16271 .next_crash_restart(&policy, now)
16272 .expect("the first restart is allowed");
16273 state
16274 .next_crash_restart(&policy, now)
16275 .expect("the second restart is allowed");
16276 state.crash_restarts[0] = now
16277 .checked_sub(policy.window + Duration::from_secs(1))
16278 .expect("the fake clock can age a restart past the window");
16279
16280 let schedule = state
16281 .next_crash_restart(&policy, now)
16282 .expect("an aged restart must release its slot");
16283 assert_eq!(schedule.restart_in_window, 1);
16284 assert_eq!(schedule.delay, Duration::from_secs(1));
16285 assert_eq!(state.crash_restarts.len(), 2);
16286 }
16287
16288 #[test]
16292 fn a_budget_spent_before_the_window_no_longer_refuses() {
16293 let policy = RestartPolicy::new(3, Duration::ZERO);
16294 let mut state = snapshot_with_restarts(true, 3);
16295 let now = ClockInstant::now();
16296 assert!(!daemon_will_restart(&mut state, &policy, now));
16297
16298 assert!(daemon_will_restart(
16299 &mut state,
16300 &policy,
16301 now + policy.window + Duration::from_secs(1)
16302 ));
16303 assert!(
16304 state.crash_restarts.is_empty(),
16305 "reading the budget must drop the instants that left the window"
16306 );
16307 }
16308
16309 fn module_with_recovery_snapshot(
16310 state: ModuleState,
16311 enabled: bool,
16312 restart_count: u32,
16313 ) -> SupervisedModule {
16314 let registry = Arc::new(Registry::default());
16315 let supervisor =
16316 Supervisor::new_for_test(Arc::clone(®istry), RestartPolicy::new(3, Duration::ZERO));
16317 let runtime = supervisor.runtime_config();
16318 let spec = ModuleSpec {
16319 script: None,
16320 module_id: "recovery-snapshot".to_string(),
16321 program: fake_aft_stub_path(),
16322 args: Vec::new(),
16323 env: Vec::new(),
16324 reserved: false,
16325 reserved_prefixes: Vec::new(),
16326 protocol: ModuleProtocol::Subc,
16327 overlap: Default::default(),
16328 carry_containment: false,
16329 };
16330 let mut snapshot = SupervisorSnapshot::new(state, enabled);
16331 seed_crash_restarts(&mut snapshot, restart_count);
16332 let (commands, _rx) = mpsc::channel(4);
16335 let configured_enabled = snapshot.enabled;
16336 SupervisedModule {
16337 inner: Arc::new(SupervisedModuleInner {
16338 module_id: spec.module_id.clone(),
16339 registry,
16340 snapshot: Arc::new(Mutex::new(snapshot)),
16341 configuration: Arc::new(Mutex::new(SupervisedConfiguration {
16342 spec,
16343 enabled: configured_enabled,
16344 health: runtime.health,
16345 register_timeout: runtime.register_timeout,
16346 restart_policy: runtime.restart_policy,
16347 })),
16348 stderr_ring: runtime.stderr_ring,
16349 terminal_ring: runtime.terminal_ring,
16350 commands,
16351 monitor: Mutex::new(None),
16352 effective_drain_timeout: runtime.effective_drain_timeout,
16353 provenance_probe: supervisor.provenance_probe.clone(),
16354 retained: runtime.child_roster.retained_view(),
16355 }),
16356 }
16357 }
16358
16359 #[cfg(target_os = "linux")]
16360 #[tokio::test]
16361 async fn no_cgroup_placement_does_not_block_fake_aft_stub_spawn() {
16362 let supervisor =
16363 Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default())
16364 .with_cgroup_placement(None);
16365 let result = supervisor.spawn(ModuleSpec {
16366 script: None,
16367 module_id: "no-cgroup-placement".to_string(),
16368 program: fake_aft_stub_path(),
16369 args: Vec::new(),
16370 env: Vec::new(),
16371 reserved: false,
16372 reserved_prefixes: Vec::new(),
16373 protocol: ModuleProtocol::Subc,
16374 overlap: Default::default(),
16375 carry_containment: false,
16376 });
16377
16378 assert!(
16379 result.is_ok(),
16380 "no delegation must not turn an otherwise valid spawn into a failure: {result:?}"
16381 );
16382 }
16383
16384 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
16385 async fn undecided_snapshot_uses_shared_restart_predicate() {
16386 assert!(module_with_recovery_snapshot(ModuleState::Running, true, 2)
16387 .will_recover_after_connection_loss()
16388 .unwrap());
16389 assert!(
16390 !module_with_recovery_snapshot(ModuleState::Running, true, 3)
16391 .will_recover_after_connection_loss()
16392 .unwrap()
16393 );
16394 }
16395
16396 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
16397 async fn restarting_snapshot_at_exhausted_budget_is_non_terminal() {
16398 assert!(
16399 module_with_recovery_snapshot(ModuleState::Restarting, true, 3)
16400 .will_recover_after_connection_loss()
16401 .unwrap()
16402 );
16403 }
16404
16405 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
16406 async fn terminal_phase_snapshots_are_terminal_before_budget_exhaustion() {
16407 assert!(!module_with_recovery_snapshot(ModuleState::Failed, true, 0)
16408 .will_recover_after_connection_loss()
16409 .unwrap());
16410 assert!(
16411 !module_with_recovery_snapshot(ModuleState::Disabled, true, 0)
16412 .will_recover_after_connection_loss()
16413 .unwrap()
16414 );
16415 }
16416
16417 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
16418 async fn warming_snapshot_is_limited_to_startup_phases() {
16419 for state in [
16420 ModuleState::Starting,
16421 ModuleState::Running,
16422 ModuleState::Restarting,
16423 ] {
16424 assert!(
16425 module_with_recovery_snapshot(state, true, 0)
16426 .is_warming()
16427 .unwrap(),
16428 "{state:?} should be warming"
16429 );
16430 }
16431 for state in [
16432 ModuleState::Unresponsive,
16433 ModuleState::Draining,
16434 ModuleState::Stopped,
16435 ModuleState::Failed,
16436 ModuleState::Disabled,
16437 ] {
16438 assert!(
16439 !module_with_recovery_snapshot(state, true, 0)
16440 .is_warming()
16441 .unwrap(),
16442 "{state:?} should not be warming"
16443 );
16444 }
16445 }
16446
16447 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
16448 async fn terminal_history_survives_respawn_and_keeps_both_crashes_in_order() {
16449 let registry = Arc::new(Registry::default());
16450 let supervisor =
16451 Supervisor::new_for_test(Arc::clone(®istry), RestartPolicy::new(1, Duration::ZERO));
16452 let module = supervisor
16453 .spawn(ModuleSpec {
16454 script: None,
16455 module_id: "terminal-history".to_string(),
16456 program: fake_aft_stub_path(),
16457 args: Vec::new(),
16458 env: vec![("FAKE_AFT_EXIT_CODE".to_string(), "23".to_string())],
16459 reserved: false,
16460 reserved_prefixes: Vec::new(),
16461 protocol: ModuleProtocol::Subc,
16462 overlap: Default::default(),
16463 carry_containment: false,
16464 })
16465 .unwrap();
16466
16467 let deadline = Instant::now() + Duration::from_secs(5);
16468 loop {
16469 let history = module.terminal_history();
16470 if history.entries.len() == 2 {
16471 assert_eq!(module.status().unwrap().state, ModuleState::Failed);
16472 assert_eq!(history.dropped, 0);
16473 assert_eq!(
16474 history
16475 .entries
16476 .iter()
16477 .map(|entry| entry.exit_code)
16478 .collect::<Vec<_>>(),
16479 vec![Some(23), Some(23)]
16480 );
16481 assert!(history.entries[0].at_ms <= history.entries[1].at_ms);
16482 return;
16483 }
16484 assert!(
16485 Instant::now() < deadline,
16486 "module did not retain two terminal exits: {history:?}"
16487 );
16488 sleep(Duration::from_millis(10)).await;
16489 }
16490 }
16491
16492 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
16496 async fn disable_during_crash_backoff_cancels_pending_respawn() {
16497 let backoff = Duration::from_secs(2);
16498 let supervisor = Supervisor::new_for_test(
16499 Arc::new(Registry::default()),
16500 RestartPolicy::new(10, backoff),
16501 );
16502 let module = supervisor
16503 .spawn(ModuleSpec {
16504 script: None,
16505 module_id: "disable-during-backoff".to_string(),
16506 program: fake_aft_stub_path(),
16507 args: Vec::new(),
16508 env: vec![("FAKE_AFT_EXIT_CODE".to_string(), "23".to_string())],
16509 reserved: false,
16510 reserved_prefixes: Vec::new(),
16511 protocol: ModuleProtocol::Subc,
16512 overlap: Default::default(),
16513 carry_containment: false,
16514 })
16515 .unwrap();
16516
16517 let deadline = Instant::now() + Duration::from_secs(5);
16519 loop {
16520 if module.status().unwrap().state == ModuleState::Restarting {
16521 break;
16522 }
16523 assert!(
16524 Instant::now() < deadline,
16525 "module never entered the crash backoff"
16526 );
16527 sleep(Duration::from_millis(10)).await;
16528 }
16529
16530 let started = Instant::now();
16531 module.set_enabled(false).await.unwrap();
16532 let waited = started.elapsed();
16533
16534 assert!(
16535 waited < backoff / 2,
16536 "disable waited {waited:?} behind the {backoff:?} crash backoff; the operator command must preempt the pending respawn"
16537 );
16538 assert_eq!(module.status().unwrap().state, ModuleState::Disabled);
16539
16540 sleep(backoff + Duration::from_millis(500)).await;
16542 let status = module.status().unwrap();
16543 assert_eq!(status.state, ModuleState::Disabled);
16544 assert_eq!(
16545 status.spawn_generation, 1,
16546 "module respawned after the operator disabled it"
16547 );
16548 }
16549
16550 #[cfg(unix)]
16553 fn protocol_none_sigterm_exits_clean_spec(
16554 module_id: &str,
16555 dir: &std::path::Path,
16556 ) -> (ModuleSpec, PathBuf, PathBuf) {
16557 let ready = dir.join("ready");
16558 let marker = dir.join("sigterm");
16559 let spec = ModuleSpec {
16560 script: None,
16561 module_id: module_id.to_string(),
16562 program: fake_aft_stub_path(),
16563 args: Vec::new(),
16564 env: vec![
16565 ("FAKE_AFT_NEVER_CONNECT".to_string(), "1".to_string()),
16566 (
16567 "FAKE_AFT_SIGTERM_MARKER_PATH".to_string(),
16568 marker.display().to_string(),
16569 ),
16570 (
16571 "FAKE_AFT_NEVER_CONNECT_READY_PATH".to_string(),
16572 ready.display().to_string(),
16573 ),
16574 ],
16575 reserved: false,
16576 reserved_prefixes: Vec::new(),
16577 protocol: ModuleProtocol::None,
16578 overlap: Default::default(),
16579 carry_containment: false,
16580 };
16581 (spec, ready, marker)
16582 }
16583
16584 #[cfg(unix)]
16588 async fn wait_for_file(path: &std::path::Path) {
16589 let deadline = Instant::now() + Duration::from_secs(10);
16590 while !path.exists() {
16591 assert!(
16592 Instant::now() < deadline,
16593 "{} never appeared",
16594 path.display()
16595 );
16596 sleep(Duration::from_millis(10)).await;
16597 }
16598 }
16599
16600 #[cfg(unix)]
16604 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
16605 async fn protocol_none_unrequested_clean_exit_restarts_as_a_crash() {
16606 let dir = cortexkit_test_support::ScratchDir::new("none-unrequested-clean-exit");
16607 let (spec, ready, marker) =
16608 protocol_none_sigterm_exits_clean_spec("none-unrequested-clean-exit", dir.path());
16609 let supervisor = Supervisor::new_for_test(
16610 Arc::new(Registry::default()),
16611 RestartPolicy::new(3, Duration::ZERO),
16612 );
16613 let module = supervisor.spawn(spec).unwrap();
16614 wait_for_file(&ready).await;
16615 let deadline = Instant::now() + Duration::from_secs(10);
16619 let first_pid = loop {
16620 let status = module.status().unwrap();
16621 if status.state == ModuleState::Running {
16622 if let Some(pid) = status.pid {
16623 break pid;
16624 }
16625 }
16626 assert!(
16627 Instant::now() < deadline,
16628 "a running module must report its pid after exec confirmation: {status:?}"
16629 );
16630 sleep(Duration::from_millis(10)).await;
16631 };
16632
16633 rustix::process::kill_process(
16634 rustix::process::Pid::from_raw(i32::try_from(first_pid).unwrap()).unwrap(),
16635 rustix::process::Signal::TERM,
16636 )
16637 .unwrap();
16638
16639 let deadline = Instant::now() + Duration::from_secs(10);
16640 let respawned = loop {
16641 let status = module.status().unwrap();
16642 if status.state == ModuleState::Running
16643 && status.pid.is_some_and(|pid| pid != first_pid)
16644 {
16645 break status;
16646 }
16647 assert!(
16648 Instant::now() < deadline,
16649 "protocol-none module was not respawned after an unrequested clean exit: {status:?}"
16650 );
16651 sleep(Duration::from_millis(10)).await;
16652 };
16653 assert_eq!(respawned.spawn_generation, 2);
16654 assert!(
16655 marker.exists(),
16656 "the child must have exited through its SIGTERM handler (exit 0), or this proves nothing about clean exits"
16657 );
16658
16659 let history = module.terminal_history();
16660 assert_eq!(history.entries.len(), 1, "{history:?}");
16661 let entry = &history.entries[0];
16662 assert_eq!(entry.exit_code, Some(0));
16663 assert_eq!(entry.exit_kind, subc_control::TerminalExitKind::Clean);
16664 assert_eq!(entry.disposition, TerminalDisposition::Restarting);
16665
16666 module.stop().await.unwrap();
16667 }
16668
16669 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
16673 async fn protocol_none_repeated_clean_exits_exhaust_the_restart_budget() {
16674 let supervisor = Supervisor::new_for_test(
16675 Arc::new(Registry::default()),
16676 RestartPolicy::new(1, Duration::ZERO),
16677 );
16678 let module = supervisor
16679 .spawn(ModuleSpec {
16680 script: None,
16681 module_id: "none-clean-exit-budget".to_string(),
16682 program: fake_aft_stub_path(),
16683 args: Vec::new(),
16684 env: vec![("FAKE_AFT_EXIT_CODE".to_string(), "0".to_string())],
16685 reserved: false,
16686 reserved_prefixes: Vec::new(),
16687 protocol: ModuleProtocol::None,
16688 overlap: Default::default(),
16689 carry_containment: false,
16690 })
16691 .unwrap();
16692
16693 let deadline = Instant::now() + Duration::from_secs(10);
16696 loop {
16697 let status = module.status().unwrap();
16698 if status.state == ModuleState::Failed {
16699 break;
16700 }
16701 assert!(
16702 Instant::now() < deadline,
16703 "module never exhausted its budget: {status:?} {:?}",
16704 module.terminal_history()
16705 );
16706 sleep(Duration::from_millis(10)).await;
16707 }
16708 let history = module.terminal_history();
16709 assert_eq!(
16710 history
16711 .entries
16712 .iter()
16713 .map(|entry| (entry.exit_code, entry.disposition.clone()))
16714 .collect::<Vec<_>>(),
16715 vec![
16716 (Some(0), TerminalDisposition::Restarting),
16717 (Some(0), TerminalDisposition::Failed),
16718 ]
16719 );
16720 let detail = history.entries[1]
16721 .disposition_detail
16722 .as_deref()
16723 .expect("a budget failure names the budget");
16724 assert!(detail.contains("max_restarts=1"), "{detail}");
16725 assert_eq!(module.status().unwrap().spawn_generation, 2);
16726 }
16727
16728 #[test]
16729 fn restart_budget_failure_is_published_after_its_terminal_record() {
16730 use std::io::{Read, Write};
16731 let supervisor = Supervisor::new_for_test(
16732 Arc::new(Registry::default()),
16733 RestartPolicy::new(0, Duration::ZERO),
16734 );
16735 let runtime = supervisor.runtime_config();
16736 let spec = ModuleSpec {
16737 script: None,
16738 protocol: ModuleProtocol::None,
16739 ..windowed_crash_spec("budget-publication-order")
16740 };
16741 let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::new(
16742 ModuleState::Running,
16743 true,
16744 )));
16745 lock_snapshot(&snapshot).unwrap().process_alive = true;
16746 let reaped_snapshot = snapshot.clone();
16747 let mut barrier = terminal_record_barrier(&runtime.spawn_events);
16748 let history = runtime.terminal_ring.clone();
16749 let reap = std::thread::spawn(move || {
16750 tokio::runtime::Builder::new_current_thread()
16751 .enable_all()
16752 .build()
16753 .unwrap()
16754 .block_on(on_child_exit(
16755 &spec,
16756 runtime.restart_policy,
16757 &supervisor.registry,
16758 &reaped_snapshot,
16759 &runtime.terminal_ring,
16760 &runtime.spawn_events,
16761 &runtime.child_roster,
16762 ExitReport {
16763 kind: ExitKind::Clean,
16764 code: Some(0),
16765 signal: None,
16766 at_ms: 1,
16767 },
16768 true,
16769 ))
16770 });
16771 let mut reached = [0];
16772 barrier.read_exact(&mut reached).unwrap();
16773 assert_eq!(&reached, b"R");
16774 let (before_state, before_alive) = {
16775 let state = lock_snapshot(&snapshot).unwrap();
16776 (state.state, state.process_alive)
16777 };
16778 let before_history = history.lock().unwrap().snapshot();
16779 barrier.write_all(b"X").unwrap();
16780 assert!(matches!(reap.join().unwrap(), NextAction::Stop { .. }));
16781 assert!(before_history.entries.is_empty());
16782 assert_ne!(
16783 before_state,
16784 ModuleState::Failed,
16785 "Failed was visible before its terminal record could be written"
16786 );
16787 assert!(before_alive);
16788 assert_eq!(lock_snapshot(&snapshot).unwrap().state, ModuleState::Failed);
16789 let history = history.lock().unwrap().snapshot();
16790 assert_eq!(history.entries.len(), 1);
16791 assert_eq!(history.entries[0].exit_code, Some(0));
16792 assert_eq!(history.entries[0].disposition, TerminalDisposition::Failed);
16793 }
16794
16795 #[test]
16796 fn crash_restart_at_budget_limit_remains_recoverable_while_recording() {
16797 use std::io::{Read, Write};
16798
16799 let supervisor = Supervisor::new_for_test(
16800 Arc::new(Registry::default()),
16801 RestartPolicy::new(3, Duration::ZERO),
16802 );
16803 let mut runtime = supervisor.runtime_config();
16804 let module = module_with_recovery_snapshot(ModuleState::Running, true, 2);
16805 let snapshot = module.inner.snapshot.clone();
16806 lock_snapshot(&snapshot).unwrap().process_alive = true;
16807 runtime.terminal_ring = module.inner.terminal_ring.clone();
16808 let spec = windowed_crash_spec(module.module_id());
16809 let mut barrier = terminal_record_barrier(&runtime.spawn_events);
16810 let reaped_snapshot = snapshot.clone();
16811 let reap = std::thread::spawn(move || {
16812 tokio::runtime::Builder::new_current_thread()
16813 .enable_all()
16814 .build()
16815 .unwrap()
16816 .block_on(on_child_exit(
16817 &spec,
16818 runtime.restart_policy,
16819 &supervisor.registry,
16820 &reaped_snapshot,
16821 &runtime.terminal_ring,
16822 &runtime.spawn_events,
16823 &runtime.child_roster,
16824 ExitReport {
16825 kind: ExitKind::Crash,
16826 code: Some(2),
16827 signal: None,
16828 at_ms: 1,
16829 },
16830 true,
16831 ))
16832 });
16833
16834 let mut reached = [0];
16838 barrier.read_exact(&mut reached).unwrap();
16839 assert_eq!(&reached, b"R");
16840 let before_status = module.status().unwrap();
16841 let before_history = module.terminal_history();
16842 let before_recovers = module.will_recover_after_connection_loss().unwrap();
16843 barrier.write_all(b"X").unwrap();
16844 assert!(matches!(reap.join().unwrap(), NextAction::Restart { .. }));
16845
16846 assert_eq!(before_status.state, ModuleState::Running);
16847 assert!(before_status.process_alive);
16848 assert_eq!(before_status.restart_count, 3);
16849 assert!(before_history.entries.is_empty());
16850 assert!(
16851 before_recovers,
16852 "connection cleanup would report a terminal close for a scheduled crash restart"
16853 );
16854 let restarting = module.status().unwrap();
16855 assert_eq!(restarting.state, ModuleState::Restarting);
16856 assert_eq!(restarting.restart_count, 3);
16857 assert!(module.will_recover_after_connection_loss().unwrap());
16858 let history = module.terminal_history();
16859 assert_eq!(history.entries.len(), 1);
16860 assert_eq!(
16861 history.entries[0].disposition,
16862 TerminalDisposition::Restarting
16863 );
16864
16865 update_snapshot(&snapshot, Some(module.module_id()), |state| {
16868 state.state = ModuleState::Running;
16869 state.respawn_pending = false;
16870 })
16871 .unwrap();
16872 assert!(!module.will_recover_after_connection_loss().unwrap());
16873 update_snapshot(&snapshot, Some(module.module_id()), |state| {
16874 state.enabled = false;
16875 state.respawn_pending = true;
16876 })
16877 .unwrap();
16878 assert!(!module.will_recover_after_connection_loss().unwrap());
16879 }
16880
16881 fn terminal_record_barrier(events: &super::SpawnEventFeed) -> std::net::TcpStream {
16882 use std::net::{TcpListener, TcpStream};
16883 let listener = TcpListener::bind("127.0.0.1:0").unwrap();
16884 let writer = TcpStream::connect(listener.local_addr().unwrap()).unwrap();
16885 let (barrier, _) = listener.accept().unwrap();
16886 for stream in [&writer, &barrier] {
16887 stream
16888 .set_read_timeout(Some(Duration::from_secs(10)))
16889 .unwrap();
16890 stream
16891 .set_write_timeout(Some(Duration::from_secs(10)))
16892 .unwrap();
16893 }
16894 *events.1.lock().unwrap() = Some(writer);
16895 barrier
16896 }
16897
16898 pub(super) fn before_terminal_record(events: &super::SpawnEventFeed) {
16899 use std::io::{Read, Write};
16900 let barrier = events.1.lock().unwrap().take();
16901 if let Some(mut barrier) = barrier {
16902 barrier.write_all(b"R").unwrap();
16903 let mut release = [0];
16904 barrier.read_exact(&mut release).unwrap();
16905 assert_eq!(&release, b"X");
16906 }
16907 }
16908
16909 #[test]
16910 fn health_restart_budget_failure_is_published_after_its_terminal_record() {
16911 use std::io::{Read, Write};
16912
16913 let supervisor = Supervisor::new_for_test(
16914 Arc::new(Registry::default()),
16915 RestartPolicy::new(0, Duration::ZERO),
16916 );
16917 let mut runtime = supervisor.runtime_config();
16918 let module = module_with_recovery_snapshot(ModuleState::Running, true, 0);
16919 let snapshot = module.inner.snapshot.clone();
16920 runtime.terminal_ring = module.inner.terminal_ring.clone();
16921 let spec = ModuleSpec {
16922 script: None,
16923 module_id: module.module_id().into(),
16924 program: fake_aft_stub_path(),
16925 protocol: ModuleProtocol::None,
16926 env: vec![("FAKE_AFT_EXIT_CODE".into(), "17".into())],
16927 ..windowed_crash_spec(module.module_id())
16928 };
16929 let mut barrier = terminal_record_barrier(&runtime.spawn_events);
16930 let reap = std::thread::spawn(move || {
16931 tokio::runtime::Builder::new_current_thread()
16932 .enable_all()
16933 .build()
16934 .unwrap()
16935 .block_on(async {
16936 let mut child =
16937 Some(spawn_and_mark_running(&spec, &runtime, &snapshot).unwrap());
16938 health_restart_child(
16939 &spec,
16940 &runtime,
16941 &supervisor.registry,
16942 &supervisor.process_liveness,
16943 &snapshot,
16944 &mut child,
16945 SupervisorHealthStatus::Failing,
16946 Some("health probe did not answer"),
16947 1,
16948 )
16949 .await
16950 .unwrap();
16951 assert!(child.is_none());
16952 });
16953 });
16954 let mut reached = [0];
16959 barrier.read_exact(&mut reached).unwrap();
16960 assert_eq!(&reached, b"R");
16961 let before_status = module.status().unwrap();
16962 let before_history = module.terminal_history();
16963 barrier.write_all(b"X").unwrap();
16964 reap.join().unwrap();
16965 assert!(before_history.entries.is_empty());
16966 assert_ne!(
16967 before_status.state,
16968 ModuleState::Failed,
16969 "Failed was visible before its terminal record could be written"
16970 );
16971 assert!(
16972 before_status.process_alive,
16973 "process death was visible before its terminal record could be written"
16974 );
16975 let failed = module.status().unwrap();
16976 assert_eq!(failed.state, ModuleState::Failed);
16977 assert!(!failed.process_alive);
16978 let history = module.terminal_history();
16979 assert_eq!(history.entries.len(), 1);
16980 assert_eq!(history.entries[0].disposition, TerminalDisposition::Failed);
16981 assert_eq!(
16982 history.entries[0].disposition_detail.as_deref(),
16983 Some("crash budget exhausted: max_restarts=0 within window_secs=600")
16984 );
16985 }
16986
16987 #[test]
16988 fn observed_exits_publish_process_death_after_terminal_history() {
16989 use std::io::{Read, Write};
16990 for (kind, enabled, shutdown, wait_error, expected_state) in [
16991 (ExitKind::Clean, true, false, false, ModuleState::Stopped),
16992 (ExitKind::Crash, true, false, false, ModuleState::Restarting),
16993 (ExitKind::Crash, false, false, false, ModuleState::Disabled),
16994 (
16995 ExitKind::DeliberateSeverance,
16996 true,
16997 false,
16998 false,
16999 ModuleState::Restarting,
17000 ),
17001 #[cfg(unix)]
17002 (ExitKind::Crash, true, true, false, ModuleState::Stopped),
17003 (ExitKind::Crash, true, false, true, ModuleState::Failed),
17004 ] {
17005 let supervisor = Supervisor::new_for_test(
17006 Arc::new(Registry::default()),
17007 RestartPolicy::new(1, Duration::ZERO),
17008 );
17009 let runtime = supervisor.runtime_config();
17010 #[cfg(unix)]
17011 if shutdown {
17012 runtime.child_roster.close();
17013 }
17014 #[cfg(not(unix))]
17015 let _ = shutdown;
17016 let spec = windowed_crash_spec("exit-publication-order");
17017 let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::new(
17018 ModuleState::Running,
17019 enabled,
17020 )));
17021 lock_snapshot(&snapshot).unwrap().process_alive = true;
17022 let reaped_snapshot = snapshot.clone();
17023 let history = runtime.terminal_ring.clone();
17024 let mut barrier = terminal_record_barrier(&runtime.spawn_events);
17025 let reap = std::thread::spawn(move || {
17026 if wait_error {
17027 super::fail_after_wait_error(
17028 &spec.module_id,
17029 &reaped_snapshot,
17030 &runtime.terminal_ring,
17031 &runtime.spawn_events,
17032 "wait failed".into(),
17033 &runtime.failure_context(true),
17034 );
17035 } else {
17036 tokio::runtime::Builder::new_current_thread()
17037 .enable_all()
17038 .build()
17039 .unwrap()
17040 .block_on(on_child_exit(
17041 &spec,
17042 runtime.restart_policy,
17043 &supervisor.registry,
17044 &reaped_snapshot,
17045 &runtime.terminal_ring,
17046 &runtime.spawn_events,
17047 &runtime.child_roster,
17048 ExitReport {
17049 kind,
17050 code: Some(0),
17051 signal: None,
17052 at_ms: 1,
17053 },
17054 true,
17055 ));
17056 }
17057 });
17058 let mut reached = [0];
17059 barrier.read_exact(&mut reached).unwrap();
17060 assert_eq!(&reached, b"R");
17061 let before_alive = lock_snapshot(&snapshot).unwrap().process_alive;
17062 let before_history = history.lock().unwrap().snapshot();
17063 barrier.write_all(b"X").unwrap();
17064 reap.join().unwrap();
17065 assert!(
17066 before_alive,
17067 "{kind:?}: death was published before recording"
17068 );
17069 assert!(before_history.entries.is_empty());
17070 assert_eq!(lock_snapshot(&snapshot).unwrap().state, expected_state);
17071 assert!(!lock_snapshot(&snapshot).unwrap().process_alive);
17072 assert_eq!(history.lock().unwrap().snapshot().entries.len(), 1);
17073 }
17074 }
17075
17076 #[tokio::test]
17077 async fn health_restart_budget_failure_remains_failed_when_journal_append_fails() {
17078 let dir = cortexkit_test_support::ScratchDir::new("health-budget-journal-failure");
17079 let path = dir.join("terminals.jsonl");
17080 std::fs::create_dir(&path).unwrap();
17081 let supervisor = Supervisor::new_for_test(
17082 Arc::new(Registry::default()),
17083 RestartPolicy::new(0, Duration::ZERO),
17084 )
17085 .with_terminal_journal(path, "health-budget-journal-failure".into());
17086 let runtime = supervisor.runtime_config();
17087 let spec = ModuleSpec {
17088 script: None,
17089 program: fake_aft_stub_path(),
17090 protocol: ModuleProtocol::None,
17091 env: vec![("FAKE_AFT_EXIT_CODE".into(), "17".into())],
17092 ..windowed_crash_spec("health-budget-journal-failure")
17093 };
17094 let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::starting()));
17095 let mut child = Some(spawn_and_mark_running(&spec, &runtime, &snapshot).unwrap());
17096 health_restart_child(
17097 &spec,
17098 &runtime,
17099 &supervisor.registry,
17100 &supervisor.process_liveness,
17101 &snapshot,
17102 &mut child,
17103 SupervisorHealthStatus::Failing,
17104 None,
17105 1,
17106 )
17107 .await
17108 .unwrap();
17109 assert_eq!(lock_snapshot(&snapshot).unwrap().state, ModuleState::Failed);
17110 assert!(!lock_snapshot(&snapshot).unwrap().process_alive);
17111 let history = runtime
17112 .terminal_ring
17113 .lock()
17114 .unwrap()
17115 .durable_history(&spec.module_id);
17116 assert!(history.journal_write_failures > 0);
17117 assert_eq!(history.entries.len(), 1);
17118 assert_eq!(history.entries[0].disposition, TerminalDisposition::Failed);
17119 }
17120
17121 #[test]
17122 fn health_restart_budget_failure_remains_failed_when_exit_recording_panics() {
17123 let supervisor = Supervisor::new_for_test(
17124 Arc::new(Registry::default()),
17125 RestartPolicy::new(0, Duration::ZERO),
17126 );
17127 let runtime = supervisor.runtime_config();
17128 let spec = ModuleSpec {
17129 script: None,
17130 program: fake_aft_stub_path(),
17131 protocol: ModuleProtocol::None,
17132 env: vec![("FAKE_AFT_EXIT_CODE".into(), "17".into())],
17133 ..windowed_crash_spec("health-budget-recording-panic")
17134 };
17135 let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::starting()));
17136 let result = std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| {
17137 tokio::runtime::Builder::new_current_thread()
17138 .enable_all()
17139 .build()
17140 .unwrap()
17141 .block_on(async {
17142 let mut child =
17143 Some(spawn_and_mark_running(&spec, &runtime, &snapshot).unwrap());
17144 runtime.spawn_events.0.lock().unwrap().seq = u64::MAX;
17145 health_restart_child(
17146 &spec,
17147 &runtime,
17148 &supervisor.registry,
17149 &supervisor.process_liveness,
17150 &snapshot,
17151 &mut child,
17152 SupervisorHealthStatus::Failing,
17153 None,
17154 1,
17155 )
17156 .await
17157 .unwrap();
17158 });
17159 }));
17160 let panic = result.expect_err("recording must still unwind");
17161 assert_eq!(
17162 panic.downcast_ref::<String>().map(String::as_str),
17163 Some("spawn event sequence exhausted")
17164 );
17165 let state = lock_snapshot(&snapshot).unwrap();
17166 assert_eq!(state.state, ModuleState::Failed);
17167 assert!(!state.process_alive);
17168 assert!(state.last_exit.is_some());
17169 }
17170
17171 #[tokio::test]
17172 async fn restart_budget_failure_remains_failed_when_journal_append_fails() {
17173 let dir = cortexkit_test_support::ScratchDir::new("budget-journal-failure");
17174 let path = dir.join("terminals.jsonl");
17175 std::fs::create_dir(&path).unwrap();
17176 let supervisor = Supervisor::new_for_test(
17177 Arc::new(Registry::default()),
17178 RestartPolicy::new(0, Duration::ZERO),
17179 )
17180 .with_terminal_journal(path, "budget-journal-failure".into());
17181 let runtime = supervisor.runtime_config();
17182 let spec = windowed_crash_spec("budget-journal-failure");
17183 let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::starting()));
17184 assert!(matches!(
17185 on_child_exit(
17186 &spec,
17187 runtime.restart_policy,
17188 &supervisor.registry,
17189 &snapshot,
17190 &runtime.terminal_ring,
17191 &runtime.spawn_events,
17192 &runtime.child_roster,
17193 crash_exit_report(1),
17194 true,
17195 )
17196 .await,
17197 NextAction::Stop { .. }
17198 ));
17199 assert_eq!(lock_snapshot(&snapshot).unwrap().state, ModuleState::Failed);
17200 let history = runtime
17201 .terminal_ring
17202 .lock()
17203 .unwrap()
17204 .durable_history(&spec.module_id);
17205 assert!(history.journal_write_failures > 0);
17206 assert_eq!(history.entries.len(), 1);
17207 assert_eq!(history.entries[0].disposition, TerminalDisposition::Failed);
17208 }
17209
17210 #[test]
17211 fn restart_budget_failure_remains_failed_when_exit_recording_panics() {
17212 let supervisor = Supervisor::new_for_test(
17213 Arc::new(Registry::default()),
17214 RestartPolicy::new(0, Duration::ZERO),
17215 );
17216 let runtime = supervisor.runtime_config();
17217 let spec = windowed_crash_spec("budget-recording-panic");
17218 let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::starting()));
17219 runtime
17220 .spawn_events
17221 .emit_spawned(&spec.module_id, None, 1, 1);
17222 runtime.spawn_events.0.lock().unwrap().seq = u64::MAX;
17225 let result = std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| {
17226 tokio::runtime::Builder::new_current_thread()
17227 .enable_all()
17228 .build()
17229 .unwrap()
17230 .block_on(on_child_exit(
17231 &spec,
17232 runtime.restart_policy,
17233 &supervisor.registry,
17234 &snapshot,
17235 &runtime.terminal_ring,
17236 &runtime.spawn_events,
17237 &runtime.child_roster,
17238 crash_exit_report(1),
17239 true,
17240 ))
17241 }));
17242 let panic = result.err().expect("recording must still unwind");
17243 assert_eq!(
17244 panic.downcast_ref::<String>().map(String::as_str),
17245 Some("spawn event sequence exhausted")
17246 );
17247 assert_eq!(lock_snapshot(&snapshot).unwrap().state, ModuleState::Failed);
17248 }
17249
17250 #[cfg(unix)]
17253 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
17254 async fn protocol_none_requested_stop_and_disable_do_not_respawn() {
17255 for disable in [false, true] {
17256 let label = if disable {
17257 "none-requested-disable"
17258 } else {
17259 "none-requested-stop"
17260 };
17261 let dir = cortexkit_test_support::ScratchDir::new(label);
17262 let (spec, ready, marker) = protocol_none_sigterm_exits_clean_spec(label, dir.path());
17263 let supervisor = Supervisor::new_for_test(
17264 Arc::new(Registry::default()),
17265 RestartPolicy::new(3, Duration::ZERO),
17266 );
17267 let module = supervisor.spawn(spec).unwrap();
17268 wait_for_file(&ready).await;
17269
17270 if disable {
17271 module.set_enabled(false).await.unwrap();
17272 } else {
17273 module.stop().await.unwrap();
17274 }
17275 assert!(
17276 marker.exists(),
17277 "{label}: the child must have left through its SIGTERM handler with exit 0"
17278 );
17279
17280 sleep(Duration::from_millis(500)).await;
17283 let status = module.status().unwrap();
17284 let expected = if disable {
17285 ModuleState::Disabled
17286 } else {
17287 ModuleState::Stopped
17288 };
17289 assert_eq!(status.state, expected, "{label}");
17290 assert_eq!(
17291 status.spawn_generation, 1,
17292 "{label}: respawned after a requested stop"
17293 );
17294 let history = module.terminal_history();
17295 assert_eq!(history.entries.len(), 1, "{label}: {history:?}");
17296 assert_eq!(history.entries[0].exit_code, Some(0), "{label}");
17297 assert_ne!(
17298 history.entries[0].disposition,
17299 TerminalDisposition::Restarting,
17300 "{label}"
17301 );
17302 }
17303 }
17304
17305 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
17308 async fn subc_wire_clean_exit_is_still_a_stop() {
17309 let supervisor = Supervisor::new_for_test(
17310 Arc::new(Registry::default()),
17311 RestartPolicy::new(3, Duration::ZERO),
17312 );
17313 let module = supervisor
17314 .spawn(ModuleSpec {
17315 script: None,
17316 module_id: "wire-clean-exit".to_string(),
17317 program: fake_aft_stub_path(),
17318 args: Vec::new(),
17319 env: vec![("FAKE_AFT_EXIT_CODE".to_string(), "0".to_string())],
17320 reserved: false,
17321 reserved_prefixes: Vec::new(),
17322 protocol: ModuleProtocol::Subc,
17323 overlap: Default::default(),
17324 carry_containment: false,
17325 })
17326 .unwrap();
17327
17328 let deadline = Instant::now() + Duration::from_secs(10);
17329 while module.terminal_history().entries.is_empty() {
17330 assert!(Instant::now() < deadline, "module never exited");
17331 sleep(Duration::from_millis(10)).await;
17332 }
17333 sleep(Duration::from_millis(500)).await;
17335 let status = module.status().unwrap();
17336 assert_eq!(status.state, ModuleState::Stopped);
17337 assert_eq!(status.spawn_generation, 1);
17338 let history = module.terminal_history();
17339 assert_eq!(history.entries.len(), 1, "{history:?}");
17340 assert_eq!(history.entries[0].exit_code, Some(0));
17341 assert_eq!(history.entries[0].disposition, TerminalDisposition::Stopped);
17342 }
17343
17344 #[cfg(unix)]
17345 #[tokio::test]
17346 async fn reload_exit_keeps_roster_until_terminal_is_recorded_during_shutdown() {
17347 let dir = cortexkit_test_support::ScratchDir::new("reload-roster-terminal-order");
17348 let record = dir.join("live-children.json");
17349 let supervisor = Supervisor::new_for_test(
17350 Arc::new(Registry::default()),
17351 RestartPolicy::new(0, Duration::ZERO),
17352 );
17353 let mut runtime = supervisor.runtime_config();
17354 runtime.child_roster.record_to(record.clone());
17355 let gate = Arc::new(super::ReloadExitRecordGate::default());
17356 runtime.test_reload_exit_record_gate = Some(Arc::clone(&gate));
17357 let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::starting()));
17358 let spec = ModuleSpec {
17359 script: None,
17360 module_id: "reload-exit-roster".into(),
17361 program: fake_aft_stub_path(),
17362 args: Vec::new(),
17363 env: vec![("FAKE_AFT_EXIT_CODE".into(), "23".into())],
17364 reserved: false,
17365 reserved_prefixes: Vec::new(),
17366 protocol: ModuleProtocol::Subc,
17367 overlap: Default::default(),
17368 carry_containment: false,
17369 };
17370 let mut child = None;
17371 let reload = super::finish_reload_child(
17372 &spec,
17373 &runtime,
17374 &supervisor.registry,
17375 &supervisor.process_liveness,
17376 &snapshot,
17377 &mut child,
17378 );
17379 tokio::pin!(reload);
17380 tokio::select! {
17381 result = &mut reload => panic!("reload missed the before-record gate: {result:?}"),
17382 _ = gate.reached.notified() => {}
17383 }
17384 assert!(runtime
17385 .terminal_ring
17386 .lock()
17387 .unwrap()
17388 .snapshot()
17389 .entries
17390 .is_empty());
17391 assert_eq!(
17392 crate::live_children::read_record(&record).unwrap().len(),
17393 1,
17394 "shutdown must still wait for the reaped child until its terminal record exists"
17395 );
17396 runtime.child_roster.close();
17397 gate.resume.notify_one();
17398 assert!(reload.await.is_err());
17399 assert!(crate::live_children::read_record(&record)
17400 .unwrap()
17401 .is_empty());
17402 let history = runtime.terminal_ring.lock().unwrap().snapshot();
17403 assert_eq!(history.entries.len(), 1);
17404 assert_eq!(
17405 history.entries[0].disposition,
17406 TerminalDisposition::DaemonShutdown
17407 );
17408 }
17409
17410 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
17414 async fn every_restart_increment_path_advances_lifetime_count() {
17415 let supervisor = Supervisor::new_for_test(
17416 Arc::new(Registry::default()),
17417 RestartPolicy::new(1, Duration::ZERO),
17418 );
17419 let runtime = supervisor.runtime_config();
17420 let spec = ModuleSpec {
17421 script: None,
17422 module_id: "lifetime-increment-path".to_string(),
17423 program: PathBuf::from("/unused/lifetime-increment-path"),
17424 args: Vec::new(),
17425 env: Vec::new(),
17426 reserved: false,
17427 reserved_prefixes: Vec::new(),
17428 protocol: ModuleProtocol::Subc,
17429 overlap: Default::default(),
17430 carry_containment: false,
17431 };
17432
17433 let crash_snapshot = Arc::new(Mutex::new(SupervisorSnapshot::starting()));
17434 assert!(matches!(
17435 on_child_exit(
17436 &spec,
17437 runtime.restart_policy,
17438 &supervisor.registry,
17439 &crash_snapshot,
17440 &runtime.terminal_ring,
17441 &runtime.spawn_events,
17442 &runtime.child_roster,
17443 ExitReport {
17444 kind: ExitKind::Crash,
17445 code: Some(1),
17446 signal: None,
17447 at_ms: 1,
17448 },
17449 true,
17450 )
17451 .await,
17452 NextAction::Restart { schedule: _ }
17453 ));
17454 let (crash_restarts, crash_lifetime) = {
17455 let state = lock_snapshot(&crash_snapshot).unwrap();
17456 (state.crash_restarts.len(), state.lifetime_restarts)
17457 };
17458 assert_eq!(crash_restarts, 1);
17459 assert_eq!(crash_lifetime, 1);
17460
17461 let health_snapshot = Arc::new(Mutex::new(SupervisorSnapshot::starting()));
17462 let mut health_child = None;
17463 assert!(matches!(
17464 health_restart_child(
17465 &spec,
17466 &runtime,
17467 &supervisor.registry,
17468 &supervisor.process_liveness,
17469 &health_snapshot,
17470 &mut health_child,
17471 SupervisorHealthStatus::Failing,
17472 None,
17473 2,
17474 )
17475 .await,
17476 Ok(())
17477 ));
17478 assert!(health_child.is_none());
17479 assert!(lock_snapshot(&health_snapshot).unwrap().respawn_pending);
17480 assert!(runtime.scheduled_respawn.lock().unwrap().take().is_some());
17481 let (health_restarts, health_lifetime) = {
17482 let state = lock_snapshot(&health_snapshot).unwrap();
17483 (state.crash_restarts.len(), state.lifetime_restarts)
17484 };
17485 assert_eq!(health_restarts, 1);
17486 assert_eq!(health_lifetime, 1);
17487
17488 let reload_snapshot = Arc::new(Mutex::new(SupervisorSnapshot::starting()));
17489 let mut reload_child = None;
17490 assert!(matches!(
17491 handle_reload_spawn_failure(
17492 &spec,
17493 &runtime,
17494 &supervisor.process_liveness,
17495 &reload_snapshot,
17496 &mut reload_child,
17497 "forced reload spawn failure".to_string(),
17498 )
17499 .await,
17500 Err(SuperviseError::ReloadFailed { .. })
17501 ));
17502 let (reload_restarts, reload_lifetime) = {
17503 let state = lock_snapshot(&reload_snapshot).unwrap();
17504 (state.crash_restarts.len(), state.lifetime_restarts)
17505 };
17506 assert_eq!(reload_restarts, 1);
17507 assert_eq!(reload_lifetime, 1);
17508 }
17509
17510 #[tokio::test]
17511 async fn deliberately_severed_live_child_records_lifetime_without_spending_restart_budget() {
17512 let supervisor = Supervisor::new_for_test(
17513 Arc::new(Registry::default()),
17514 RestartPolicy::new(3, Duration::ZERO),
17515 );
17516 let runtime = supervisor.runtime_config();
17517 let spec = ModuleSpec {
17518 script: None,
17519 module_id: "deliberately-severed".to_string(),
17520 program: PathBuf::from("/unused/deliberately-severed"),
17521 args: Vec::new(),
17522 env: Vec::new(),
17523 reserved: false,
17524 reserved_prefixes: Vec::new(),
17525 protocol: ModuleProtocol::Subc,
17526 overlap: Default::default(),
17527 carry_containment: false,
17528 };
17529 let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::starting()));
17530 let process = ProcessIdentity {
17531 pid: 41,
17532 start_time: 101,
17533 };
17534 record_deliberate_severance(&snapshot, process).unwrap();
17535 let exit_report = apply_deliberate_severance_marker(
17536 &snapshot,
17537 Some(process),
17538 ExitReport {
17539 kind: ExitKind::Crash,
17540 code: Some(1),
17541 signal: None,
17542 at_ms: 1,
17543 },
17544 );
17545 assert_eq!(exit_report.kind, ExitKind::DeliberateSeverance);
17546
17547 assert!(matches!(
17548 on_child_exit(
17549 &spec,
17550 runtime.restart_policy,
17551 &supervisor.registry,
17552 &snapshot,
17553 &runtime.terminal_ring,
17554 &runtime.spawn_events,
17555 &runtime.child_roster,
17556 exit_report,
17557 true,
17558 )
17559 .await,
17560 NextAction::Restart { schedule: _ }
17561 ));
17562 let state = lock_snapshot(&snapshot).unwrap();
17563 assert_eq!(state.lifetime_restarts, 1);
17564 assert_eq!(state.crash_restarts.len(), 0);
17565 }
17566
17567 #[tokio::test]
17568 async fn genuine_crash_spends_restart_budget_and_records_lifetime() {
17569 let supervisor = Supervisor::new_for_test(
17570 Arc::new(Registry::default()),
17571 RestartPolicy::new(3, Duration::ZERO),
17572 );
17573 let runtime = supervisor.runtime_config();
17574 let spec = ModuleSpec {
17575 script: None,
17576 module_id: "genuine-crash".to_string(),
17577 program: PathBuf::from("/unused/genuine-crash"),
17578 args: Vec::new(),
17579 env: Vec::new(),
17580 reserved: false,
17581 reserved_prefixes: Vec::new(),
17582 protocol: ModuleProtocol::Subc,
17583 overlap: Default::default(),
17584 carry_containment: false,
17585 };
17586 let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::starting()));
17587
17588 assert!(matches!(
17589 on_child_exit(
17590 &spec,
17591 runtime.restart_policy,
17592 &supervisor.registry,
17593 &snapshot,
17594 &runtime.terminal_ring,
17595 &runtime.spawn_events,
17596 &runtime.child_roster,
17597 ExitReport {
17598 kind: ExitKind::Crash,
17599 code: Some(1),
17600 signal: None,
17601 at_ms: 1,
17602 },
17603 true,
17604 )
17605 .await,
17606 NextAction::Restart { schedule: _ }
17607 ));
17608 let state = lock_snapshot(&snapshot).unwrap();
17609 assert_eq!(state.lifetime_restarts, 1);
17610 assert_eq!(state.crash_restarts.len(), 1);
17611 }
17612
17613 fn crash_exit_report(at_ms: u64) -> ExitReport {
17614 ExitReport {
17615 kind: ExitKind::Crash,
17616 code: Some(1),
17617 signal: None,
17618 at_ms,
17619 }
17620 }
17621
17622 fn windowed_crash_spec(module_id: &str) -> ModuleSpec {
17623 ModuleSpec {
17624 script: None,
17625 module_id: module_id.to_string(),
17626 program: PathBuf::from("/unused").join(module_id),
17627 args: Vec::new(),
17628 env: Vec::new(),
17629 reserved: false,
17630 reserved_prefixes: Vec::new(),
17631 protocol: ModuleProtocol::Subc,
17632 overlap: Default::default(),
17633 carry_containment: false,
17634 }
17635 }
17636
17637 #[tokio::test]
17643 async fn three_crashes_inside_the_window_stop_the_module_and_name_the_window() {
17644 let (logs, _guard) = crate::router::test_log::log_capture(tracing::Level::ERROR);
17645 let supervisor = Supervisor::new_for_test(
17646 Arc::new(Registry::default()),
17647 RestartPolicy::new(2, Duration::ZERO),
17648 );
17649 let runtime = supervisor.runtime_config();
17650 let spec = windowed_crash_spec("crash-loop-in-window");
17651 let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::starting()));
17652
17653 for attempt in 1..=2 {
17654 assert!(
17655 matches!(
17656 on_child_exit(
17657 &spec,
17658 runtime.restart_policy,
17659 &supervisor.registry,
17660 &snapshot,
17661 &runtime.terminal_ring,
17662 &runtime.spawn_events,
17663 &runtime.child_roster,
17664 crash_exit_report(attempt),
17665 true,
17666 )
17667 .await,
17668 NextAction::Restart { schedule: _ }
17669 ),
17670 "crash {attempt} is inside the budget and must respawn"
17671 );
17672 }
17673
17674 assert!(matches!(
17675 on_child_exit(
17676 &spec,
17677 runtime.restart_policy,
17678 &supervisor.registry,
17679 &snapshot,
17680 &runtime.terminal_ring,
17681 &runtime.spawn_events,
17682 &runtime.child_roster,
17683 crash_exit_report(3),
17684 true,
17685 )
17686 .await,
17687 NextAction::Stop { .. }
17688 ));
17689
17690 {
17691 let state = lock_snapshot(&snapshot).unwrap();
17692 assert_eq!(state.state, ModuleState::Failed);
17693 assert_eq!(state.crash_restarts.len(), 2);
17694 assert_eq!(state.lifetime_restarts, 2);
17695 }
17696
17697 let history = runtime
17698 .terminal_ring
17699 .lock()
17700 .expect("terminal ring is not poisoned")
17701 .snapshot();
17702 let last = history
17703 .entries
17704 .last()
17705 .expect("the refused crash is retained");
17706 assert_eq!(last.disposition, TerminalDisposition::Failed);
17707 assert_eq!(
17708 last.disposition_detail.as_deref(),
17709 Some("crash budget exhausted: max_restarts=2 within window_secs=600")
17710 );
17711
17712 let captured = crate::router::test_log::captured_logs(&logs);
17713 assert!(
17714 captured.contains("crash budget exhausted: max_restarts=2 within window_secs=600"),
17715 "the stop must be logged with its window: {captured}"
17716 );
17717 }
17718
17719 #[tokio::test]
17727 async fn a_crash_older_than_the_window_frees_its_slot_for_a_later_crash() {
17728 let supervisor = Supervisor::new_for_test(
17729 Arc::new(Registry::default()),
17730 RestartPolicy::new(2, Duration::ZERO),
17731 );
17732 let runtime = supervisor.runtime_config();
17733 let spec = windowed_crash_spec("crash-across-windows");
17734 let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::starting()));
17735
17736 for attempt in 1..=2 {
17737 assert!(matches!(
17738 on_child_exit(
17739 &spec,
17740 runtime.restart_policy,
17741 &supervisor.registry,
17742 &snapshot,
17743 &runtime.terminal_ring,
17744 &runtime.spawn_events,
17745 &runtime.child_roster,
17746 crash_exit_report(attempt),
17747 true,
17748 )
17749 .await,
17750 NextAction::Restart { schedule: _ }
17751 ));
17752 }
17753
17754 update_snapshot(&snapshot, Some(&spec.module_id), |state| {
17757 age_oldest_crash_restart_out_of_window(state, runtime.restart_policy.window);
17758 })
17759 .unwrap();
17760
17761 assert!(
17762 matches!(
17763 on_child_exit(
17764 &spec,
17765 runtime.restart_policy,
17766 &supervisor.registry,
17767 &snapshot,
17768 &runtime.terminal_ring,
17769 &runtime.spawn_events,
17770 &runtime.child_roster,
17771 crash_exit_report(3),
17772 true,
17773 )
17774 .await,
17775 NextAction::Restart { schedule: _ }
17776 ),
17777 "a crash older than the window must not hold a budget slot"
17778 );
17779
17780 let state = lock_snapshot(&snapshot).unwrap();
17781 assert_eq!(state.state, ModuleState::Restarting);
17782 assert_eq!(
17783 state.crash_restarts.len(),
17784 2,
17785 "the aged instant is dropped and the new one takes its place"
17786 );
17787 assert_eq!(
17788 state.lifetime_restarts, 3,
17789 "the ledger counts every restart, including the ones the window forgot"
17790 );
17791 }
17792
17793 #[tokio::test]
17798 async fn an_operator_restart_clears_the_ring_and_leaves_the_ledger_alone() {
17799 let supervisor = Supervisor::new_for_test(
17800 Arc::new(Registry::default()),
17801 RestartPolicy::new(2, Duration::ZERO),
17802 );
17803 let runtime = supervisor.runtime_config();
17804 let spec = windowed_crash_spec("operator-cleared-budget");
17805 let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::starting()));
17806
17807 for attempt in 1..=2 {
17808 assert!(matches!(
17809 on_child_exit(
17810 &spec,
17811 runtime.restart_policy,
17812 &supervisor.registry,
17813 &snapshot,
17814 &runtime.terminal_ring,
17815 &runtime.spawn_events,
17816 &runtime.child_roster,
17817 crash_exit_report(attempt),
17818 true,
17819 )
17820 .await,
17821 NextAction::Restart { schedule: _ }
17822 ));
17823 }
17824
17825 reset_restart_count(&snapshot, &spec.module_id).unwrap();
17826 {
17827 let state = lock_snapshot(&snapshot).unwrap();
17828 assert!(
17829 state.crash_restarts.is_empty(),
17830 "an operator restart returns the full budget"
17831 );
17832 assert_eq!(
17833 state.lifetime_restarts, 2,
17834 "clearing the budget must not unmake the crashes"
17835 );
17836 }
17837
17838 assert!(
17839 matches!(
17840 on_child_exit(
17841 &spec,
17842 runtime.restart_policy,
17843 &supervisor.registry,
17844 &snapshot,
17845 &runtime.terminal_ring,
17846 &runtime.spawn_events,
17847 &runtime.child_roster,
17848 crash_exit_report(3),
17849 true,
17850 )
17851 .await,
17852 NextAction::Restart { schedule: _ }
17853 ),
17854 "the cleared budget must be spendable again"
17855 );
17856 let state = lock_snapshot(&snapshot).unwrap();
17857 assert_eq!(state.crash_restarts.len(), 1);
17858 assert_eq!(state.lifetime_restarts, 3);
17859 }
17860
17861 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
17862 async fn severance_marker_for_a_dead_child_does_not_label_its_successor() {
17863 let severed = ProcessIdentity {
17864 pid: 41,
17865 start_time: 101,
17866 };
17867 let successor = ProcessIdentity {
17868 pid: 41,
17869 start_time: 202,
17870 };
17871 let module = module_with_recovery_snapshot(ModuleState::Running, true, 0);
17872 update_snapshot(&module.inner.snapshot, Some("recovery-snapshot"), |state| {
17873 state.pid = Some(successor.pid);
17874 state.process_start_time = Some(successor.start_time);
17875 })
17876 .unwrap();
17877 assert!(!module.record_deliberate_severance(severed).unwrap());
17878
17879 let exit_report = apply_deliberate_severance_marker(
17880 &module.inner.snapshot,
17881 Some(successor),
17882 ExitReport {
17883 kind: ExitKind::Crash,
17884 code: Some(1),
17885 signal: None,
17886 at_ms: 1,
17887 },
17888 );
17889
17890 assert_eq!(exit_report.kind, ExitKind::Crash);
17891 }
17892
17893 #[tokio::test]
17894 async fn drain_reap_marks_deliberate_severance_and_records_lifetime_without_budget() {
17895 let registry = Registry::default();
17896 let supervisor = Supervisor::new_for_test(
17897 Arc::new(Registry::default()),
17898 RestartPolicy::new(3, Duration::ZERO),
17899 );
17900 let runtime = supervisor.runtime_config();
17901 let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::starting()));
17902 let spec = ModuleSpec {
17903 script: None,
17904 module_id: "drain-deliberate-severance".to_string(),
17905 program: fake_aft_stub_path(),
17906 args: Vec::new(),
17907 env: vec![("FAKE_AFT_EXIT_CODE".to_string(), "23".to_string())],
17908 reserved: false,
17909 reserved_prefixes: Vec::new(),
17910 protocol: ModuleProtocol::Subc,
17911 overlap: Default::default(),
17912 carry_containment: false,
17913 };
17914 let mut child = spawn_and_mark_running(&spec, &runtime, &snapshot).unwrap();
17915 let process = ProcessIdentity {
17916 pid: 41,
17917 start_time: 101,
17918 };
17919 child.process_identity = Some(process);
17920 update_snapshot(&snapshot, Some(&spec.module_id), |state| {
17921 state.pid = Some(process.pid);
17922 state.process_start_time = Some(process.start_time);
17923 })
17924 .unwrap();
17925 record_deliberate_severance(&snapshot, process).unwrap();
17926
17927 drain_child_to_state(
17928 &spec.module_id,
17929 spec.protocol,
17930 StopNotice::SentOverConnection,
17933 ®istry,
17934 None,
17935 &snapshot,
17936 &runtime.terminal_ring,
17937 &runtime.spawn_events,
17938 child,
17939 Duration::from_secs(1),
17940 ModuleState::Stopped,
17941 Some(false),
17942 &super::FailureContext::operator(),
17943 )
17944 .await
17945 .unwrap();
17946
17947 let state = lock_snapshot(&snapshot).unwrap();
17948 assert_eq!(
17949 state.last_exit.as_ref().map(|exit| exit.kind),
17950 Some(ExitKind::DeliberateSeverance)
17951 );
17952 assert_eq!(state.lifetime_restarts, 1);
17953 assert_eq!(state.crash_restarts.len(), 0);
17954 drop(state);
17955 let history = runtime.terminal_ring.lock().unwrap().snapshot();
17956 assert_eq!(
17957 history.entries[0].exit_kind,
17958 subc_control::TerminalExitKind::DeliberateSeverance
17959 );
17960 }
17961
17962 #[tokio::test]
17963 async fn ordinary_drain_reap_does_not_record_a_lifetime_restart() {
17964 let registry = Registry::default();
17965 let supervisor = Supervisor::new_for_test(
17966 Arc::new(Registry::default()),
17967 RestartPolicy::new(3, Duration::ZERO),
17968 );
17969 let runtime = supervisor.runtime_config();
17970 let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::starting()));
17971 let spec = ModuleSpec {
17972 script: None,
17973 module_id: "ordinary-drain".to_string(),
17974 program: fake_aft_stub_path(),
17975 args: Vec::new(),
17976 env: vec![("FAKE_AFT_EXIT_CODE".to_string(), "23".to_string())],
17977 reserved: false,
17978 reserved_prefixes: Vec::new(),
17979 protocol: ModuleProtocol::Subc,
17980 overlap: Default::default(),
17981 carry_containment: false,
17982 };
17983 let child = spawn_and_mark_running(&spec, &runtime, &snapshot).unwrap();
17984
17985 drain_child_to_state(
17986 &spec.module_id,
17987 spec.protocol,
17988 StopNotice::SentOverConnection,
17991 ®istry,
17992 None,
17993 &snapshot,
17994 &runtime.terminal_ring,
17995 &runtime.spawn_events,
17996 child,
17997 Duration::from_secs(1),
17998 ModuleState::Stopped,
17999 Some(false),
18000 &super::FailureContext::operator(),
18001 )
18002 .await
18003 .unwrap();
18004
18005 let state = lock_snapshot(&snapshot).unwrap();
18006 assert_eq!(
18007 state.last_exit.as_ref().map(|exit| exit.kind),
18008 Some(ExitKind::Crash)
18009 );
18010 assert_eq!(state.lifetime_restarts, 0);
18011 assert_eq!(state.crash_restarts.len(), 0);
18012 }
18013
18014 #[test]
18015 fn fatal_connection_teardown_cannot_arm_a_marker_for_a_surviving_process() {
18016 assert!(!include_str!("server.rs")
18022 .contains("router.record_deliberate_connection_severance(ctx.connection_id)"));
18023 }
18024
18025 #[test]
18032 fn drained_after_quiescence_wait_passes_ok_through_and_forces_false_on_err() {
18033 assert!(drained_after_quiescence_wait(&Ok(true)));
18034 assert!(!drained_after_quiescence_wait(&Ok(false)));
18035 assert!(!drained_after_quiescence_wait(&Err(
18036 SuperviseError::StatePoisoned { module_id: None }
18037 )));
18038 }
18039
18040 #[test]
18049 fn wait_error_exit_report_records_a_failed_terminal_with_no_code_or_signal() {
18050 let ring = Arc::new(Mutex::new(TerminalRing::new(
18051 TerminalRingConfig::default(),
18052 0,
18053 )));
18054 record_wait_error_terminal("wait-error", &ring, &super::SpawnEventFeed::default());
18055
18056 let snapshot = ring.lock().unwrap().snapshot();
18057 assert_eq!(snapshot.entries.len(), 1);
18058 let entry = &snapshot.entries[0];
18059 assert_eq!(entry.exit_code, None);
18060 assert_eq!(entry.exit_signal, None);
18061 assert_eq!(entry.disposition, TerminalDisposition::Failed);
18062 }
18063
18064 #[test]
18067 fn spawn_generation_of_a_failed_launch_is_never_reused() {
18068 let feed = super::SpawnEventFeed::default();
18069 feed.configure_incarnation("launch-numbering".to_string());
18070 let (incarnation, failed) = feed.allocate_generation("relaunched");
18071 assert_eq!(incarnation, "launch-numbering");
18072 let (_, next) = feed.allocate_generation("relaunched");
18073 assert!(next > failed);
18074 assert_eq!(feed.emit_spawned("relaunched", Some(next), 7, 1), next);
18075 assert!(feed.emit_spawned("relaunched", None, 8, 2) > next);
18077 assert_eq!(
18078 feed.allocate_generation("other").1,
18079 1,
18080 "numbering is per module"
18081 );
18082 }
18083
18084 #[test]
18085 fn wait_error_exit_path_preserves_spawn_event_density() {
18086 let feed = super::SpawnEventFeed::default();
18087 feed.configure_incarnation("wait-error-density".to_string());
18088 feed.emit_spawned("wait-error", None, 41, 1);
18089 let ring = Arc::new(Mutex::new(TerminalRing::new(
18090 TerminalRingConfig::default(),
18091 0,
18092 )));
18093
18094 record_wait_error_terminal("wait-error", &ring, &feed);
18095 feed.emit_spawned("after-wait-error", None, 42, 2);
18096
18097 let state = feed.0.lock().unwrap();
18098 let sequences = state
18099 .events
18100 .iter()
18101 .map(|event| event.cursor.seq)
18102 .collect::<Vec<_>>();
18103 assert_eq!(sequences, vec![1, 2, 3]);
18104 assert_eq!(state.events[1].kind, SpawnEventKind::Exited);
18105 assert_eq!(state.events[1].exit_code, None);
18106 assert_eq!(state.events[1].exit_signal, None);
18107 }
18108
18109 #[test]
18113 fn wait_error_exit_report_is_classified_as_a_crash() {
18114 assert_eq!(wait_error_exit_report().kind, ExitKind::Crash);
18115 }
18116}
18117
18118#[cfg(all(test, unix))]
18119mod restart_ordering_tests {
18120 use super::*;
18121 use std::io::Write;
18122 use tokio::io::AsyncReadExt;
18123
18124 #[test]
18125 #[ignore = "child process fixture"]
18126 fn serving_child() {
18127 let Ok(address_file) = std::env::var("RESTART_TEST_ADDRESS_FILE") else {
18128 return;
18129 };
18130 let listener = std::net::TcpListener::bind("127.0.0.1:0").unwrap();
18131 std::fs::write(address_file, listener.local_addr().unwrap().to_string()).unwrap();
18132 for stream in listener.incoming() {
18133 writeln!(stream.unwrap(), "{}", std::process::id()).unwrap();
18134 }
18135 }
18136
18137 async fn restart_before_publication(requests: usize) {
18138 let home = cortexkit_test_support::ScratchDir::new("restart-before-publication");
18139 let address_file = home.path().join("address");
18140 let spec = ModuleSpec {
18141 script: None,
18142 module_id: "restart-serving-child".into(),
18143 program: std::env::current_exe().unwrap(),
18144 args: vec![
18145 "--exact".into(),
18146 "supervise::restart_ordering_tests::serving_child".into(),
18147 "--ignored".into(),
18148 "--nocapture".into(),
18149 ],
18150 env: vec![(
18151 "RESTART_TEST_ADDRESS_FILE".into(),
18152 address_file.display().to_string(),
18153 )],
18154 reserved: false,
18155 reserved_prefixes: vec![],
18156 protocol: ModuleProtocol::None,
18157 overlap: Default::default(),
18158 carry_containment: false,
18159 };
18160 let mut supervisor =
18161 Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default());
18162 let (spawned_tx, spawned_rx) = oneshot::channel();
18163 let spawned_tx = Mutex::new(Some(spawned_tx));
18164 let (release_tx, release_rx) = std::sync::mpsc::channel();
18165 let release_rx = Mutex::new(release_rx);
18166 supervisor.test_after_first_spawn = AfterFirstSpawnHook(Some(Arc::new(move |_| {
18169 spawned_tx.lock().unwrap().take().unwrap().send(()).unwrap();
18170 release_rx
18171 .lock()
18172 .unwrap()
18173 .recv_timeout(Duration::from_secs(20))
18174 .unwrap();
18175 })));
18176 let registered = supervisor
18177 .register_configured_with_health(
18178 spec.clone(),
18179 true,
18180 HealthConfig::default(),
18181 None,
18182 RestartPolicy::new(3, Duration::ZERO),
18183 DEFAULT_REGISTER_TIMEOUT,
18184 )
18185 .unwrap();
18186 let module = registered.module.clone();
18187 let startup = tokio::task::spawn_blocking(move || {
18188 supervisor.start_registered_configured(registered, None, true)
18189 });
18190 spawned_rx.await.unwrap();
18191 let old_pid = tokio::time::timeout(Duration::from_secs(10), async {
18192 let address = loop {
18193 if let Ok(address) = std::fs::read_to_string(&address_file) {
18194 if !address.is_empty() {
18195 break address;
18196 }
18197 }
18198 tokio::time::sleep(Duration::from_millis(1)).await;
18199 };
18200 let mut stream = tokio::net::TcpStream::connect(address).await.unwrap();
18201 let mut reply = String::new();
18202 stream.read_to_string(&mut reply).await.unwrap();
18203 reply.trim().parse::<u32>().unwrap()
18204 })
18205 .await
18206 .expect("child must serve before generation publication");
18207 assert_eq!(
18208 lock_snapshot(&module.inner.snapshot)
18209 .unwrap()
18210 .spawn_generation,
18211 0
18212 );
18213 let mut replies = Vec::new();
18214 for _ in 0..requests {
18215 let mut restart = Box::pin(module.restart(Some(0)));
18216 std::future::poll_fn(|cx| {
18219 assert!(std::future::Future::poll(restart.as_mut(), cx).is_pending());
18220 std::task::Poll::Ready(())
18221 })
18222 .await;
18223 replies.push(restart);
18224 }
18225 release_tx.send(()).unwrap();
18226 startup.await.unwrap().unwrap();
18227 for reply in replies {
18228 reply.await.unwrap();
18229 }
18230 let replacement = tokio::time::timeout(Duration::from_secs(10), async {
18233 loop {
18234 if lock_snapshot(&module.inner.snapshot)
18235 .unwrap()
18236 .spawn_generation
18237 > 1
18238 {
18239 break;
18240 }
18241 tokio::time::sleep(Duration::from_millis(1)).await;
18242 }
18243 })
18244 .await;
18245 module.update_spec_for_test(spec).await.unwrap();
18249 let (generation, pid) = {
18250 let state = lock_snapshot(&module.inner.snapshot).unwrap();
18251 (state.spawn_generation, state.pid)
18252 };
18253 module.stop().await.unwrap();
18254 replacement.expect("restart must publish a replacement for the already-serving child");
18255 assert_ne!(
18256 pid,
18257 Some(old_pid),
18258 "a serving child predating the restart must be replaced"
18259 );
18260 assert_eq!(generation, 2, "queued restarts must share one replacement");
18261 }
18262
18263 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
18264 async fn restart_received_after_serving_before_generation_publication_replaces_child() {
18265 restart_before_publication(1).await;
18266 }
18267
18268 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
18269 async fn restart_queued_behind_restart_does_not_kill_its_replacement() {
18270 restart_before_publication(2).await;
18271 }
18272
18273 #[tokio::test]
18274 async fn restart_queued_before_replacement_spawn_keeps_the_new_child() {
18275 let supervisor =
18276 Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default());
18277 let registered = supervisor
18278 .register_configured_with_health(
18279 ModuleSpec {
18280 script: None,
18281 module_id: "queued-before-replacement".into(),
18282 program: "/bin/sleep".into(),
18287 args: vec!["600".into()],
18288 env: vec![],
18289 reserved: false,
18290 reserved_prefixes: vec![],
18291 protocol: ModuleProtocol::None,
18292 overlap: Default::default(),
18293 carry_containment: false,
18294 },
18295 true,
18296 HealthConfig::default(),
18297 None,
18298 RestartPolicy::default(),
18299 DEFAULT_REGISTER_TIMEOUT,
18300 )
18301 .unwrap();
18302 let RegisteredConfiguredModule {
18303 mut spec,
18304 mut runtime,
18305 module,
18306 mut commands,
18307 } = registered;
18308 let snapshot = &module.inner.snapshot;
18309 let mut restart = Box::pin(module.restart(Some(0)));
18310 std::future::poll_fn(|cx| {
18311 assert!(std::future::Future::poll(restart.as_mut(), cx).is_pending());
18312 std::task::Poll::Ready(())
18313 })
18314 .await;
18315 let mut child = Some(spawn_and_mark_running(&spec, &runtime, snapshot).unwrap());
18318 let replacement_pid = child.as_ref().unwrap().id();
18319 let command = commands.recv().await.unwrap();
18320 assert!(
18321 handle_supervisor_command(
18322 command,
18323 &mut spec,
18324 &mut runtime,
18325 &supervisor.registry,
18326 &supervisor.process_liveness,
18327 snapshot,
18328 &mut child,
18329 &mut commands,
18330 &mut VecDeque::new(),
18331 )
18332 .await
18333 );
18334 restart.await.unwrap();
18335 let kept_pid = child.as_ref().and_then(SupervisedChild::id);
18336 let pending = lock_snapshot(snapshot).unwrap().respawn_pending;
18337 if let Some(mut child) = child {
18338 child.start_kill().unwrap();
18339 child.wait().await.unwrap();
18340 }
18341 assert_eq!(
18342 kept_pid, replacement_pid,
18343 "a newer replacement must survive"
18344 );
18345 assert!(
18346 !pending,
18347 "a satisfied restart must not schedule another spawn"
18348 );
18349 }
18350}
18351
18352#[cfg(all(test, unix))]
18353mod health_event_tests {
18354 use super::*;
18355 use cortexkit_test_support::ScratchDir;
18356 use subc_protocol::{manifest::Concurrency, session::ModuleControlResponse};
18357
18358 struct Actor {
18359 module: SupervisedModule,
18360 registry: Arc<Registry>,
18361 forwarding: Arc<ForwardingTable>,
18362 spec: ModuleSpec,
18363 health: HealthConfig,
18364 rx: mpsc::Receiver<crate::router::OutboundFrame>,
18365 roster: ChildRoster,
18366 _home: ScratchDir,
18367 }
18368
18369 async fn wait_for<T>(reason: &str, mut observe: impl FnMut() -> Option<T>) -> T {
18370 let deadline = std::time::Instant::now() + Duration::from_secs(10);
18374 let tick = Instant::now();
18375 loop {
18376 if let Some(value) = observe() {
18377 assert_eq!(Instant::now(), tick, "{reason} must not wait for a timer");
18378 return value;
18379 }
18380 assert!(
18381 std::time::Instant::now() < deadline,
18382 "timed out waiting for {reason}"
18383 );
18384 tokio::task::yield_now().await;
18385 }
18386 }
18387
18388 impl Actor {
18389 async fn start(protocol: ModuleProtocol, cadence: Duration) -> Self {
18390 Self::start_with_policy(
18391 protocol,
18392 cadence,
18393 DEFAULT_REGISTER_TIMEOUT,
18394 RestartPolicy::new(3, Duration::ZERO),
18395 )
18396 .await
18397 }
18398
18399 async fn start_with_policy(
18400 protocol: ModuleProtocol,
18401 cadence: Duration,
18402 register_timeout: Duration,
18403 policy: RestartPolicy,
18404 ) -> Self {
18405 let home = ScratchDir::new("health-events");
18406 let registry = Arc::new(Registry::default());
18407 let forwarding = Arc::new(ForwardingTable::default());
18408 let health = HealthConfig {
18409 cadence,
18410 deadline: Duration::from_secs(3600),
18411 ..HealthConfig::default()
18412 };
18413 let supervisor = Supervisor::new_for_test(registry.clone(), policy)
18414 .with_forwarding(forwarding.clone())
18415 .with_health_config(health.clone());
18416 let spec = ModuleSpec {
18417 script: None,
18418 module_id: "event-health-child".into(),
18419 program: PathBuf::from("/bin/sleep"),
18427 args: vec!["600".into()],
18430 env: ["XDG_DATA_HOME", "XDG_RUNTIME_DIR", "XDG_CONFIG_HOME"]
18431 .into_iter()
18432 .map(|key| (key.into(), home.path().display().to_string()))
18433 .collect(),
18434 reserved: false,
18435 reserved_prefixes: vec![],
18436 protocol,
18437 overlap: Default::default(),
18438 carry_containment: false,
18439 };
18440 let module = supervisor
18441 .supervise_configured_requested(
18442 spec.clone(),
18443 true,
18444 health.clone(),
18445 Some(0),
18446 policy,
18447 register_timeout,
18448 None,
18449 )
18450 .unwrap();
18451 let (_, rx) = mpsc::channel(8);
18452 let actor = Self {
18453 module,
18454 registry,
18455 forwarding,
18456 spec,
18457 health,
18458 rx,
18459 roster: supervisor.child_roster.clone(),
18460 _home: home,
18461 };
18462 actor
18463 .wait_for_select(|checkpoint| checkpoint.generation > 0)
18464 .await;
18465 actor
18466 }
18467
18468 async fn expire_registration_wait(&self, remaining: Duration) {
18469 tokio::time::advance(remaining).await;
18470 wait_for("registration timeout TERM grace", || {
18471 lock_snapshot(&self.module.inner.snapshot)
18472 .unwrap()
18473 .actor_select
18474 .is_none()
18475 .then_some(())
18476 })
18477 .await;
18478 tokio::time::advance(crate::child_roster::TERM_TO_KILL).await;
18479 }
18480
18481 fn turns(&self) -> u64 {
18482 lock_snapshot(&self.module.inner.snapshot)
18483 .unwrap()
18484 .actor_turns
18485 }
18486
18487 async fn wait_for_select(
18488 &self,
18489 expected: impl Fn(&ActorSelectCheckpoint) -> bool,
18490 ) -> ActorSelectCheckpoint {
18491 wait_for("actor select checkpoint", || {
18492 let state = lock_snapshot(&self.module.inner.snapshot).unwrap();
18493 state.actor_select.filter(|checkpoint| {
18494 checkpoint.generation == state.spawn_generation
18495 && state.state == ModuleState::Running
18496 && state.process_alive
18497 && state.reported_pid().is_some()
18498 && expected(checkpoint)
18499 })
18500 })
18501 .await
18502 }
18503
18504 fn hello(&mut self, connection: u64, health: bool) {
18505 let manifest =
18506 subc_protocol::manifest::ModuleManifest::builder(&self.spec.module_id, "1.0.0")
18507 .protocol_ver(subc_protocol::PROTOCOL_VERSION)
18508 .build();
18509 let connection = ConnectionId::new(connection);
18510 let (tx, rx) = mpsc::channel(8);
18511 self.rx = rx;
18512 self.forwarding
18513 .register_module_connection(
18514 connection,
18515 self.spec.module_id.clone(),
18516 subc_protocol::PROTOCOL_VERSION,
18517 Concurrency::ModuleManaged,
18518 FrameSink::new(tx),
18519 )
18520 .unwrap();
18521 self.registry
18522 .register_with_control_ops(
18523 manifest,
18524 subc_protocol::PROTOCOL_VERSION,
18525 connection,
18526 if health {
18527 vec![MODULE_CONTROL_OP_HEALTH_CHECK.into()]
18528 } else {
18529 vec![]
18530 },
18531 )
18532 .unwrap();
18533 }
18534
18535 async fn probe(&mut self) -> crate::router::OutboundFrame {
18536 let frame = wait_for("outbound health probe", || match self.rx.try_recv() {
18537 Ok(frame) => Some(frame),
18538 Err(mpsc::error::TryRecvError::Empty) => None,
18539 Err(mpsc::error::TryRecvError::Disconnected) => panic!("health peer disconnected"),
18540 })
18541 .await;
18542 let body: Value = serde_json::from_slice(&frame.body).unwrap();
18543 assert_eq!(body["op"], MODULE_CONTROL_OP_HEALTH_CHECK);
18544 frame
18545 }
18546
18547 fn no_probe(&mut self) {
18548 assert!(matches!(
18549 self.rx.try_recv(),
18550 Err(mpsc::error::TryRecvError::Empty)
18551 ));
18552 }
18553
18554 fn answer(&self, connection: u64, frame: crate::router::OutboundFrame) {
18555 self.forwarding
18556 .complete_module_control_rpc(
18557 ConnectionId::new(connection),
18558 frame.header.corr,
18559 Some(MODULE_CONTROL_OP_HEALTH_CHECK),
18560 ModuleControlRpcOutcome::Response(ModuleControlResponse::HealthCheck {
18561 status: HealthStatus::Ok,
18562 detail: None,
18563 metrics: None,
18564 }),
18565 )
18566 .unwrap();
18567 }
18568 }
18569
18570 #[tokio::test(start_paused = true)]
18571 async fn registration_timeout_kills_at_120_seconds_and_respawns() {
18572 let actor = Actor::start(ModuleProtocol::Subc, Duration::from_secs(30)).await;
18573 let initial = actor.module.status().unwrap();
18574 tokio::time::advance(Duration::from_secs(119)).await;
18575 tokio::task::yield_now().await;
18576 assert_eq!(actor.module.status().unwrap().pid, initial.pid);
18577 assert!(actor.module.terminal_history().entries.is_empty());
18578 actor.expire_registration_wait(Duration::from_secs(1)).await;
18579 actor
18580 .wait_for_select(|at| at.generation > initial.spawn_generation)
18581 .await;
18582 let status = actor.module.status().unwrap();
18583 assert_ne!(status.pid, initial.pid);
18584 assert!(!cortexkit_test_support::process_alive(
18585 initial.pid.unwrap() as i32
18586 ));
18587 assert_eq!(status.restart_count, 1);
18588 assert_eq!(status.last_exit.unwrap().kind, ExitKind::Crash);
18589 let entries = actor.module.terminal_history().entries;
18590 assert_eq!(entries.len(), 1);
18591 assert_eq!(
18592 entries[0].disposition_detail.as_deref(),
18593 Some("registration_timeout")
18594 );
18595 actor.module.stop().await.unwrap();
18596 }
18597
18598 #[tokio::test(start_paused = true)]
18599 async fn registration_timeout_is_cancelled_by_hello_at_119_seconds() {
18600 let mut actor = Actor::start(ModuleProtocol::Subc, Duration::from_secs(30)).await;
18601 let pid = actor.module.status().unwrap().pid;
18602 tokio::time::advance(Duration::from_secs(119)).await;
18603 actor.hello(1, false);
18604 let registered = actor
18605 .wait_for_select(|at| at.registered_connection.is_some())
18606 .await;
18607 assert_eq!(
18608 registered.wake_after, None,
18609 "HELLO must cancel the deadline"
18610 );
18611 tokio::time::advance(Duration::from_secs(300)).await;
18612 tokio::task::yield_now().await;
18613 assert_eq!(actor.module.status().unwrap().pid, pid);
18614 assert!(actor.module.terminal_history().entries.is_empty());
18615 actor
18617 .registry
18618 .deregister_connection(ConnectionId::new(1))
18619 .unwrap();
18620 actor
18621 .wait_for_select(|at| at.registered_connection.is_none())
18622 .await;
18623 tokio::time::advance(Duration::from_secs(300)).await;
18624 tokio::task::yield_now().await;
18625 assert_eq!(actor.module.status().unwrap().pid, pid);
18626 assert!(actor.module.terminal_history().entries.is_empty());
18627 actor.module.stop().await.unwrap();
18628
18629 let mut transient = Actor::start(ModuleProtocol::Subc, Duration::from_secs(30)).await;
18630 let pid = transient.module.status().unwrap().pid;
18631 tokio::time::advance(Duration::from_secs(119)).await;
18632 let turns = transient.turns();
18633 transient.hello(1, false);
18636 transient
18637 .registry
18638 .deregister_connection(ConnectionId::new(1))
18639 .unwrap();
18640 let parked = transient.wait_for_select(|at| at.turn > turns).await;
18641 assert_eq!(
18642 parked.wake_after, None,
18643 "transient HELLO must cancel the deadline"
18644 );
18645 tokio::time::advance(Duration::from_secs(300)).await;
18646 tokio::task::yield_now().await;
18647 assert_eq!(transient.module.status().unwrap().pid, pid);
18648 assert!(transient.module.terminal_history().entries.is_empty());
18649 transient.module.stop().await.unwrap();
18650 }
18651
18652 #[tokio::test(start_paused = true)]
18653 async fn registration_timeouts_exhaust_budget_and_schedule_slow_retry() {
18654 let actor = Actor::start_with_policy(
18655 ModuleProtocol::Subc,
18656 Duration::from_secs(30),
18657 Duration::from_secs(10),
18658 RestartPolicy::new(1, Duration::ZERO),
18659 )
18660 .await;
18661 let generation = actor.module.status().unwrap().spawn_generation;
18662 actor
18663 .expire_registration_wait(Duration::from_secs(10))
18664 .await;
18665 actor.wait_for_select(|at| at.generation > generation).await;
18666 actor
18667 .expire_registration_wait(Duration::from_secs(10))
18668 .await;
18669 let retry = wait_for("failed retry schedule", || {
18670 let state = lock_snapshot(&actor.module.inner.snapshot).unwrap();
18671 (state.state == ModuleState::Failed)
18672 .then_some(state.failed_retry.pending)
18673 .flatten()
18674 })
18675 .await;
18676 assert_eq!(retry.delay, Duration::from_secs(600));
18677 assert_eq!(retry.attempt, 1);
18678 let entries = actor.module.terminal_history().entries;
18679 assert_eq!(entries.len(), 2);
18680 assert_eq!(entries[1].disposition, TerminalDisposition::Failed);
18681 assert!(entries[1]
18682 .disposition_detail
18683 .as_deref()
18684 .unwrap()
18685 .contains("registration_timeout"));
18686 tokio::time::advance(Duration::from_secs(599)).await;
18687 tokio::task::yield_now().await;
18688 assert_eq!(actor.module.state().unwrap(), ModuleState::Failed);
18689 tokio::time::advance(Duration::from_secs(1)).await;
18690 actor
18691 .wait_for_select(|at| at.generation > generation + 1)
18692 .await;
18693 actor
18695 .expire_registration_wait(Duration::from_secs(10))
18696 .await;
18697 wait_for("slow retry registration timeout", || {
18698 (actor.module.terminal_history().entries.len() == 3).then_some(())
18699 })
18700 .await;
18701 actor.module.stop().await.unwrap();
18702 }
18703
18704 #[tokio::test(start_paused = true)]
18705 async fn registration_timeout_honours_ten_second_override() {
18706 let actor = Actor::start_with_policy(
18707 ModuleProtocol::Subc,
18708 Duration::from_secs(30),
18709 Duration::from_secs(10),
18710 RestartPolicy::new(3, Duration::ZERO),
18711 )
18712 .await;
18713 let generation = actor.module.status().unwrap().spawn_generation;
18714 tokio::time::advance(Duration::from_secs(9)).await;
18715 tokio::task::yield_now().await;
18716 assert!(actor.module.terminal_history().entries.is_empty());
18717 actor.expire_registration_wait(Duration::from_secs(1)).await;
18718 actor.wait_for_select(|at| at.generation > generation).await;
18719 assert_eq!(actor.module.terminal_history().entries.len(), 1);
18720 actor.module.stop().await.unwrap();
18721 }
18722
18723 #[tokio::test]
18724 async fn registration_timeout_kills_term_ignoring_helper_before_reaping_leader() {
18725 use std::os::unix::process::{CommandExt, ExitStatusExt};
18726
18727 struct Helper(std::process::Child);
18728 impl Drop for Helper {
18729 fn drop(&mut self) {
18730 if matches!(self.0.try_wait(), Ok(None)) {
18733 let _ = self.0.kill();
18734 let _ = self.0.wait();
18735 }
18736 }
18737 }
18738
18739 let home = ScratchDir::new("registration-timeout-helper");
18740 let supervisor = Supervisor::new_for_test(
18741 Arc::new(Registry::default()),
18742 RestartPolicy::new(0, Duration::ZERO),
18743 );
18744 let spec = ModuleSpec {
18745 script: None,
18746 module_id: "registration-timeout-group".into(),
18747 program: "/bin/sleep".into(),
18748 args: vec!["120".into()],
18749 env: ["XDG_DATA_HOME", "XDG_RUNTIME_DIR", "XDG_CONFIG_HOME"]
18750 .into_iter()
18751 .map(|key| (key.into(), home.path().display().to_string()))
18752 .collect(),
18753 reserved: false,
18754 reserved_prefixes: vec![],
18755 protocol: ModuleProtocol::Subc,
18756 overlap: Default::default(),
18757 carry_containment: false,
18758 };
18759 let module = supervisor
18760 .supervise_configured_requested(
18761 spec,
18762 true,
18763 HealthConfig::default(),
18764 None,
18765 RestartPolicy::new(0, Duration::ZERO),
18766 Duration::from_secs(10),
18767 None,
18768 )
18769 .unwrap();
18770 let leader = timeout(Duration::from_secs(5), async {
18771 loop {
18772 if let Some(pid) = module.status().unwrap().pid {
18773 break pid;
18774 }
18775 sleep(Duration::from_millis(5)).await;
18776 }
18777 })
18778 .await
18779 .expect("supervised group leader must be reported before the deadline");
18780 assert!(leader > 1);
18781 let ready = home.join("helper.ready");
18782 let mut helper = Helper(
18785 std::process::Command::new("/bin/sh")
18786 .args([
18787 "-c",
18788 "trap '' TERM; : > \"$1\"; exec /bin/sleep 120",
18789 "helper",
18790 ])
18791 .arg(&ready)
18792 .process_group(i32::try_from(leader).unwrap())
18793 .stdout(std::process::Stdio::null())
18794 .stderr(std::process::Stdio::null())
18795 .spawn()
18796 .unwrap(),
18797 );
18798 timeout(Duration::from_secs(5), async {
18799 while !ready.exists() {
18800 assert!(helper.0.try_wait().unwrap().is_none());
18801 sleep(Duration::from_millis(5)).await;
18802 }
18803 })
18804 .await
18805 .expect("helper must install TERM ignore before the registration deadline");
18806 let helper_pid =
18807 rustix::process::Pid::from_raw(i32::try_from(helper.0.id()).unwrap()).unwrap();
18808 rustix::process::kill_process(helper_pid, rustix::process::Signal::TERM).unwrap();
18809 sleep(Duration::from_millis(20)).await;
18810 assert!(
18811 helper.0.try_wait().unwrap().is_none(),
18812 "helper must ignore TERM"
18813 );
18814
18815 timeout(Duration::from_secs(20), async {
18816 while module.state().unwrap() != ModuleState::Failed {
18817 sleep(Duration::from_millis(5)).await;
18818 }
18819 })
18820 .await
18821 .expect("registration timeout must stop and reap the group leader");
18822 let helper_exit = timeout(Duration::from_secs(3), async {
18823 loop {
18824 if let Some(status) = helper.0.try_wait().unwrap() {
18825 break status;
18826 }
18827 sleep(Duration::from_millis(5)).await;
18828 }
18829 })
18830 .await
18831 .expect("TERM-ignoring helper survived the registration timeout's group KILL");
18832 assert_eq!(helper_exit.signal(), Some(9));
18833 assert!(!cortexkit_test_support::process_alive(
18834 i32::try_from(leader).unwrap()
18835 ));
18836 assert_eq!(
18837 module.status().unwrap().last_exit.unwrap().kind,
18838 ExitKind::Crash
18839 );
18840 assert!(module.terminal_history().entries[0]
18841 .disposition_detail
18842 .as_deref()
18843 .unwrap()
18844 .contains("registration_timeout"));
18845 module.stop().await.unwrap();
18846 }
18847
18848 #[tokio::test(start_paused = true)]
18849 async fn registration_timeout_does_not_count_an_earlier_exit() {
18850 let actor = Actor::start_with_policy(
18851 ModuleProtocol::Subc,
18852 Duration::from_secs(30),
18853 DEFAULT_REGISTER_TIMEOUT,
18854 RestartPolicy::new(0, Duration::ZERO),
18855 )
18856 .await;
18857 let pid = actor.module.status().unwrap().pid.unwrap();
18858 rustix::process::kill_process(
18859 rustix::process::Pid::from_raw(pid as i32).unwrap(),
18860 rustix::process::Signal::TERM,
18861 )
18862 .unwrap();
18863 wait_for("early exit", || {
18864 (actor.module.state().unwrap() == ModuleState::Failed).then_some(())
18865 })
18866 .await;
18867 tokio::time::advance(Duration::from_secs(121)).await;
18868 tokio::task::yield_now().await;
18869 let entries = actor.module.terminal_history().entries;
18870 assert_eq!(entries.len(), 1);
18871 assert!(!entries[0]
18872 .disposition_detail
18873 .as_deref()
18874 .unwrap()
18875 .contains("registration_timeout"));
18876 assert_eq!(actor.module.status().unwrap().lifetime_restarts, 0);
18877 actor.module.stop().await.unwrap();
18878 }
18879
18880 #[tokio::test(start_paused = true)]
18881 async fn registration_timeout_is_cancelled_by_daemon_shutdown() {
18882 let actor = Actor::start(ModuleProtocol::Subc, Duration::from_secs(30)).await;
18883 let pid = actor.module.status().unwrap().pid;
18884 tokio::time::advance(Duration::from_secs(119)).await;
18885 actor.roster.close();
18886 let turns = actor.turns();
18887 tokio::time::advance(Duration::from_secs(300)).await;
18888 let parked = actor.wait_for_select(|at| at.turn > turns).await;
18889 assert_eq!(parked.wake_after, None);
18890 assert_eq!(actor.module.status().unwrap().pid, pid);
18891 assert!(actor.module.terminal_history().entries.is_empty());
18892 actor.module.stop().await.unwrap();
18893 let status = actor.module.status().unwrap();
18894 assert_eq!(status.restart_count, 0);
18895 assert_eq!(
18896 actor.module.terminal_history().entries[0].disposition,
18897 TerminalDisposition::DaemonShutdown
18898 );
18899 }
18900
18901 #[tokio::test(start_paused = true)]
18902 async fn no_health_children_park_for_sixty_seconds() {
18903 let none = Actor::start(ModuleProtocol::None, Duration::from_secs(30)).await;
18904 let unregistered = Actor::start(ModuleProtocol::Subc, Duration::from_secs(30)).await;
18905 let mut unadvertised = Actor::start(ModuleProtocol::Subc, Duration::from_secs(30)).await;
18906 unadvertised.hello(1, false);
18907 for _ in 0..6000 {
18912 tokio::time::advance(Duration::from_millis(10)).await;
18913 tokio::task::yield_now().await;
18914 }
18915 for actor in [&none, &unregistered, &unadvertised] {
18916 assert!(
18917 actor.turns() <= 5,
18918 "no-health actor woke {} times",
18919 actor.turns()
18920 );
18921 assert_eq!(actor.module.state().unwrap(), ModuleState::Running);
18922 }
18923 }
18924
18925 #[tokio::test(start_paused = true)]
18926 async fn late_hello_starts_a_due_probe_in_the_notification_tick() {
18927 let mut actor = Actor::start(ModuleProtocol::Subc, Duration::ZERO).await;
18931 let tick = Instant::now();
18932 actor.hello(1, true);
18933 actor.probe().await;
18934 assert_eq!(Instant::now(), tick, "HELLO must not wait for a poll");
18935 }
18936
18937 #[tokio::test(start_paused = true)]
18938 async fn hello_keeps_the_jittered_cadence_and_catalog_updates_do_not_reset_it() {
18939 let mut actor = Actor::start(ModuleProtocol::Subc, Duration::from_secs(30)).await;
18940 let hello_tick = Instant::now();
18941 actor.hello(1, true);
18942 let armed = actor
18943 .wait_for_select(|checkpoint| {
18944 checkpoint.registered_connection == Some(ConnectionId::new(1))
18945 && checkpoint.next_probe_at.is_some()
18946 })
18947 .await;
18948 let delay = jittered_health_delay(&actor.spec.module_id, 0, actor.health.cadence);
18949 assert!((Duration::from_secs(30)..Duration::from_secs(33)).contains(&delay));
18950 let first_deadline = hello_tick + delay;
18951 assert_eq!(armed.next_probe_at, Some(first_deadline));
18952 tokio::time::advance(Duration::from_secs(29)).await;
18953 let turns = actor.turns();
18954 actor
18955 .registry
18956 .replace_catalog_for_connection(ConnectionId::new(1), vec![], None, Some(true))
18957 .unwrap();
18958 let updated = actor
18959 .wait_for_select(|checkpoint| checkpoint.turn > turns)
18960 .await;
18961 assert_eq!(
18962 updated.next_probe_at,
18963 Some(first_deadline),
18964 "catalog update must not reset cadence"
18965 );
18966 actor.no_probe();
18967 tokio::time::advance(first_deadline - Instant::now()).await;
18968 let frame = actor.probe().await;
18969 actor.answer(1, frame);
18970 let next = jittered_health_delay(&actor.spec.module_id, 1, actor.health.cadence);
18971 let next_deadline = Instant::now() + next;
18972 let rearmed = actor
18973 .wait_for_select(|checkpoint| {
18974 checkpoint
18975 .next_probe_at
18976 .is_some_and(|deadline| deadline > first_deadline)
18977 })
18978 .await;
18979 assert_eq!(rearmed.next_probe_at, Some(next_deadline));
18980 tokio::time::advance(next - Duration::from_nanos(1)).await;
18981 actor.no_probe();
18982 tokio::time::advance(Duration::from_nanos(1)).await;
18983 actor.probe().await;
18984 }
18985
18986 #[tokio::test(start_paused = true)]
18987 async fn disconnect_disarms_and_reconnect_rearms_health() {
18988 let mut actor = Actor::start(ModuleProtocol::Subc, Duration::from_secs(30)).await;
18989 actor.hello(1, true);
18990 actor
18991 .wait_for_select(|checkpoint| {
18992 checkpoint.registered_connection == Some(ConnectionId::new(1))
18993 && checkpoint.next_probe_at.is_some()
18994 })
18995 .await;
18996 let turns = actor.turns();
18997 let tick = Instant::now();
18998 actor
18999 .registry
19000 .deregister_connection(ConnectionId::new(1))
19001 .unwrap();
19002 let disarmed = actor
19003 .wait_for_select(|checkpoint| {
19004 checkpoint.turn > turns && checkpoint.registered_connection.is_none()
19005 })
19006 .await;
19007 assert_eq!(disarmed.next_probe_at, None);
19008 assert_eq!(disarmed.wake_after, None);
19009 assert_eq!(
19010 Instant::now(),
19011 tick,
19012 "disconnect must disarm in the notification tick"
19013 );
19014 tokio::time::advance(Duration::from_secs(60)).await;
19015 actor.no_probe();
19016 let reconnect_tick = Instant::now();
19017 actor.hello(2, true);
19018 let rearmed = actor
19019 .wait_for_select(|checkpoint| {
19020 checkpoint.registered_connection == Some(ConnectionId::new(2))
19021 && checkpoint.next_probe_at.is_some()
19022 })
19023 .await;
19024 let delay = jittered_health_delay(&actor.spec.module_id, 0, actor.health.cadence);
19025 assert_eq!(rearmed.next_probe_at, Some(reconnect_tick + delay));
19026 tokio::time::advance(delay).await;
19027 actor.probe().await;
19028 }
19029
19030 #[tokio::test(start_paused = true)]
19031 async fn reregistration_without_health_disarms_the_old_schedule() {
19032 let mut actor = Actor::start(ModuleProtocol::Subc, Duration::from_secs(30)).await;
19033 actor.hello(1, true);
19034 actor
19035 .wait_for_select(|checkpoint| {
19036 checkpoint.registered_connection == Some(ConnectionId::new(1))
19037 && checkpoint.next_probe_at.is_some()
19038 })
19039 .await;
19040 actor
19041 .registry
19042 .deregister_connection(ConnectionId::new(1))
19043 .unwrap();
19044 actor.hello(2, false);
19045 let disarmed = actor
19046 .wait_for_select(|checkpoint| {
19047 checkpoint.registered_connection == Some(ConnectionId::new(2))
19048 })
19049 .await;
19050 assert_eq!(disarmed.next_probe_at, None);
19051 assert_eq!(disarmed.wake_after, None);
19052 tokio::time::advance(Duration::from_secs(60)).await;
19053 actor.no_probe();
19054 }
19055
19056 #[tokio::test(start_paused = true)]
19057 async fn restart_waits_for_its_own_hello_before_rearming_health() {
19058 let mut actor = Actor::start(ModuleProtocol::Subc, Duration::from_secs(30)).await;
19059 actor.hello(1, true);
19060 let armed = actor
19061 .wait_for_select(|checkpoint| {
19062 checkpoint.registered_connection == Some(ConnectionId::new(1))
19063 && checkpoint.next_probe_at.is_some()
19064 })
19065 .await;
19066 actor
19070 .registry
19071 .deregister_connection(ConnectionId::new(1))
19072 .unwrap();
19073 actor.module.restart(Some(0)).await.unwrap();
19074 let parked = actor
19075 .wait_for_select(|checkpoint| {
19076 checkpoint.generation > armed.generation
19077 && checkpoint.registered_connection.is_none()
19078 && checkpoint.next_probe_at.is_none()
19079 })
19080 .await;
19081 assert_eq!(parked.wake_after, Some(DEFAULT_REGISTER_TIMEOUT));
19082 actor.rx.close();
19083 while actor.rx.try_recv().is_ok() {}
19086 let turns = actor.turns();
19087 tokio::time::advance(Duration::from_secs(60)).await;
19088 let after = actor.turns();
19089 eprintln!(
19090 "restart park turns: before={turns} after={after} delta={}",
19091 after - turns
19092 );
19093 assert_eq!(after, turns, "replacement must park before HELLO");
19094 let hello_tick = Instant::now();
19095 actor.hello(2, true);
19096 let rearmed = actor
19097 .wait_for_select(|checkpoint| {
19098 checkpoint.registered_connection == Some(ConnectionId::new(2))
19099 && checkpoint.next_probe_at.is_some()
19100 })
19101 .await;
19102 let delay = jittered_health_delay(&actor.spec.module_id, 0, actor.health.cadence);
19103 assert_eq!(rearmed.next_probe_at, Some(hello_tick + delay));
19104 tokio::time::advance(delay).await;
19105 actor.probe().await;
19106 }
19107
19108 #[tokio::test(start_paused = true)]
19109 async fn http_probe_waits_a_full_cadence_for_each_spawn() {
19110 use tokio::{
19111 io::{AsyncReadExt, AsyncWriteExt},
19112 net::TcpListener,
19113 };
19114 let mut actor = Actor::start(ModuleProtocol::None, Duration::from_secs(30)).await;
19115 let listener = TcpListener::bind("127.0.0.1:0").await.unwrap();
19116 actor.health.http = Some(format!("http://{}/healthz", listener.local_addr().unwrap()));
19117 let (started_tx, mut started_rx) = mpsc::channel(8);
19118 let peer = tokio::spawn(async move {
19119 loop {
19120 let (mut socket, _) = listener.accept().await.unwrap();
19121 let mut request = [0; 1024];
19122 assert!(socket.read(&mut request).await.unwrap() > 0);
19123 started_tx.send(Instant::now()).await.unwrap();
19124 socket
19125 .write_all(b"HTTP/1.1 200 OK\r\nContent-Length: 0\r\n\r\n")
19126 .await
19127 .unwrap();
19128 }
19129 });
19130 actor
19131 .module
19132 .update_configuration(actor.spec.clone(), actor.health.clone(), None)
19133 .await
19134 .unwrap();
19135 let first_tick = Instant::now();
19136 let first = actor
19137 .wait_for_select(|checkpoint| checkpoint.next_probe_at.is_some())
19138 .await;
19139 assert!(first.next_probe_at.unwrap() >= first_tick + actor.health.cadence);
19140 assert!(started_rx.try_recv().is_err());
19141
19142 tokio::time::advance(
19145 first.next_probe_at.unwrap() - Instant::now() - Duration::from_secs(1),
19146 )
19147 .await;
19148 actor.module.restart(Some(0)).await.unwrap();
19149 let spawn_tick = Instant::now();
19150 let replacement = actor
19151 .wait_for_select(|checkpoint| checkpoint.generation > first.generation)
19152 .await;
19153 let deadline = replacement.next_probe_at.unwrap();
19154 assert!(
19155 deadline >= spawn_tick + actor.health.cadence,
19156 "a replacement must not inherit its predecessor's probe deadline"
19157 );
19158 tokio::time::advance(actor.health.cadence - Duration::from_nanos(1)).await;
19159 assert!(
19160 started_rx.try_recv().is_err(),
19161 "no HTTP probe during startup grace"
19162 );
19163 tokio::time::advance(deadline - Instant::now()).await;
19164 let landed = wait_for("first replacement HTTP probe", || {
19165 started_rx.try_recv().ok()
19166 })
19167 .await;
19168 assert!(landed >= spawn_tick + actor.health.cadence);
19169 wait_for("healthy replacement", || {
19170 (actor.module.status().unwrap().health.status == SupervisorHealthStatus::Ok)
19171 .then_some(())
19172 })
19173 .await;
19174 peer.abort();
19175 }
19176
19177 #[tokio::test(start_paused = true)]
19178 async fn rescan_adds_and_removes_http_health_without_polling() {
19179 use tokio::{
19180 io::{AsyncReadExt, AsyncWriteExt},
19181 net::TcpListener,
19182 };
19183 let mut actor = Actor::start(ModuleProtocol::None, Duration::from_secs(30)).await;
19184 let listener = TcpListener::bind("127.0.0.1:0").await.unwrap();
19185 actor.health.http = Some(format!("http://{}/health", listener.local_addr().unwrap()));
19186 actor.health.cadence = Duration::from_millis(1);
19188 let (started_tx, mut started_rx) = mpsc::channel(8);
19189 let peer = tokio::spawn(async move {
19190 loop {
19191 let (mut socket, _) = listener.accept().await.unwrap();
19192 let mut request = [0; 1024];
19193 assert!(socket.read(&mut request).await.unwrap() > 0);
19194 started_tx.send(()).await.unwrap();
19195 socket
19196 .write_all(b"HTTP/1.1 200 OK\r\nContent-Length: 0\r\n\r\n")
19197 .await
19198 .unwrap();
19199 }
19200 });
19201 let tick = Instant::now();
19202 let turns = actor.turns();
19203 actor
19204 .module
19205 .update_configuration(actor.spec.clone(), actor.health.clone(), None)
19206 .await
19207 .unwrap();
19208 let armed = actor
19209 .wait_for_select(|checkpoint| {
19210 checkpoint.turn > turns && checkpoint.next_probe_at.is_some()
19211 })
19212 .await;
19213 assert_eq!(armed.next_probe_at, Some(tick + Duration::from_millis(1)));
19214 assert_eq!(Instant::now(), tick);
19215 tokio::time::advance(Duration::from_millis(1)).await;
19216 let mut started = false;
19217 wait_for("successful HTTP probe", || {
19218 started |= started_rx.try_recv().is_ok();
19219 (started && actor.module.status().unwrap().health.status == SupervisorHealthStatus::Ok)
19220 .then_some(())
19221 })
19222 .await;
19223 assert!(started, "added HTTP check must start at its first deadline");
19224 assert_eq!(
19225 actor.module.status().unwrap().health.status,
19226 SupervisorHealthStatus::Ok
19227 );
19228 assert_eq!(Instant::now(), tick + Duration::from_millis(1));
19229 actor.health.http = None;
19230 let turns = actor.turns();
19231 actor
19232 .module
19233 .update_configuration(actor.spec.clone(), actor.health.clone(), None)
19234 .await
19235 .unwrap();
19236 let parked = actor
19237 .wait_for_select(|checkpoint| {
19238 checkpoint.turn > turns && checkpoint.next_probe_at.is_none()
19239 })
19240 .await;
19241 assert_eq!(parked.wake_after, None);
19242 let turns = actor.turns();
19243 tokio::time::advance(Duration::from_secs(60)).await;
19244 assert_eq!(
19245 actor.turns(),
19246 turns,
19247 "removed HTTP check must park the actor"
19248 );
19249 assert!(
19250 started_rx.try_recv().is_err(),
19251 "removed HTTP check must stay disarmed"
19252 );
19253 assert_eq!(
19254 actor.module.status().unwrap().health,
19255 ModuleHealthStatus::default()
19256 );
19257 peer.abort();
19258 }
19259}
19260
19261#[cfg(test)]
19262mod drain_busy_reprobe_tests {
19263 use super::*;
19267 use std::collections::BTreeMap;
19268 use std::sync::Mutex as StdMutex;
19269 use subc_protocol::{manifest::Concurrency, session::ModuleControlResponse};
19270
19271 const MODULE_ID: &str = "drain-reprobe-module";
19272 const CONNECTION: u64 = 7;
19273 const GAUGE: &str = "runs_in_flight";
19274 const CADENCE: Duration = Duration::from_secs(30);
19277 const CEILING: Duration = Duration::from_secs(30);
19278
19279 #[derive(Clone, Copy)]
19281 enum Answer {
19282 BusyUntil { until: Instant, latency: Duration },
19285 OmitsGauge,
19287 Silent,
19289 }
19290
19291 struct Drain {
19292 forwarding: Arc<ForwardingTable>,
19293 runtime: SupervisorRuntimeConfig,
19294 endpoint: crate::ModuleEndpointId,
19295 probes: Arc<StdMutex<Vec<Instant>>>,
19297 responder: tokio::task::JoinHandle<()>,
19298 }
19299
19300 impl Drain {
19301 fn start(answer: Answer) -> Self {
19302 let registry = Arc::new(Registry::default());
19303 let forwarding = Arc::new(ForwardingTable::default());
19304 let health = HealthConfig {
19305 cadence: CADENCE,
19306 ..HealthConfig::default()
19307 };
19308 let runtime = Supervisor::new_for_test(registry, RestartPolicy::new(3, Duration::ZERO))
19309 .with_forwarding(forwarding.clone())
19310 .with_health_config(health)
19311 .runtime_config();
19312 let (tx, mut rx) = mpsc::channel::<crate::router::OutboundFrame>(8);
19313 let endpoint = forwarding
19314 .register_module_connection(
19315 ConnectionId::new(CONNECTION),
19316 MODULE_ID.to_string(),
19317 subc_protocol::PROTOCOL_VERSION,
19318 Concurrency::ModuleManaged,
19319 FrameSink::new(tx),
19320 )
19321 .unwrap();
19322 let probes = Arc::new(StdMutex::new(Vec::new()));
19323 let responder = tokio::spawn({
19324 let forwarding = forwarding.clone();
19325 let probes = probes.clone();
19326 async move {
19327 while let Some(frame) = rx.recv().await {
19328 let body: Value = serde_json::from_slice(&frame.body).unwrap();
19329 assert_eq!(body["op"], MODULE_CONTROL_OP_HEALTH_CHECK);
19330 probes.lock().unwrap().push(Instant::now());
19331 let metrics = match answer {
19332 Answer::BusyUntil { until, latency } => {
19333 sleep(latency).await;
19334 let busy = u64::from(Instant::now() < until);
19335 serde_json::json!({ GAUGE: busy })
19336 }
19337 Answer::OmitsGauge => serde_json::json!({ "queue_depth": 0 }),
19338 Answer::Silent => continue,
19339 };
19340 let _ = forwarding.complete_module_control_rpc(
19343 ConnectionId::new(CONNECTION),
19344 frame.header.corr,
19345 Some(MODULE_CONTROL_OP_HEALTH_CHECK),
19346 ModuleControlRpcOutcome::Response(ModuleControlResponse::HealthCheck {
19347 status: HealthStatus::Ok,
19348 detail: None,
19349 metrics: Some(metrics),
19350 }),
19351 );
19352 }
19353 }
19354 });
19355 Self {
19356 forwarding,
19357 runtime,
19358 endpoint,
19359 probes,
19360 responder,
19361 }
19362 }
19363
19364 async fn run(&self, ceiling: Duration) -> (bool, Duration) {
19367 let (drained, elapsed, _) = self.run_traced(ceiling).await;
19368 (drained, elapsed)
19369 }
19370
19371 async fn run_traced(&self, ceiling: Duration) -> (bool, Duration, DrainGaugeTrace) {
19373 let started = Instant::now();
19374 let mut trace = DrainGaugeTrace::new(&[GAUGE.to_string()]);
19375 let drained = wait_for_forwarding_quiescence(
19376 &self.forwarding,
19377 MODULE_ID,
19378 &self.runtime,
19379 self.endpoint,
19380 started + ceiling,
19381 &[GAUGE.to_string()],
19382 DrainScope::Active,
19383 &mut trace,
19384 )
19385 .await
19386 .unwrap();
19387 (drained, started.elapsed(), trace)
19388 }
19389
19390 fn probes(&self) -> Vec<Instant> {
19391 self.probes.lock().unwrap().clone()
19392 }
19393 }
19394
19395 impl Drop for Drain {
19396 fn drop(&mut self) {
19397 self.responder.abort();
19398 }
19399 }
19400
19401 #[tokio::test(start_paused = true)]
19402 async fn busy_at_first_probe_then_quiet_drains_long_before_the_ceiling() {
19403 let drain = Drain::start(Answer::BusyUntil {
19404 until: Instant::now() + Duration::from_secs(1),
19405 latency: Duration::ZERO,
19406 });
19407 let (drained, elapsed) = drain.run(CEILING).await;
19408 assert!(
19409 drained,
19410 "gauges that read zero during the drain must let it finish drained, \
19411 not wait out the {CADENCE:?} health cadence (elapsed {elapsed:?})"
19412 );
19413 assert!(
19414 elapsed < Duration::from_secs(3),
19415 "drain took {elapsed:?} after the gauges went quiet at 1s"
19416 );
19417 let probes = drain.probes();
19418 assert!(
19419 probes.len() >= 2,
19420 "the module was busy at the first probe, so a quiet verdict needs a second: {probes:?}"
19421 );
19422 }
19423
19424 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
19429 async fn an_instant_busy_first_answer_does_not_stop_later_probes_in_real_time() {
19430 let drain = Drain::start(Answer::BusyUntil {
19431 until: Instant::now() + Duration::from_millis(600),
19432 latency: Duration::ZERO,
19433 });
19434 let (drained, elapsed, trace) = drain.run_traced(CEILING).await;
19435 assert!(drained, "the gauge fell at 600ms: {trace}");
19436 assert!(
19437 elapsed < Duration::from_secs(3),
19438 "drain took {elapsed:?} after the gauge fell at 600ms: {trace}"
19439 );
19440 let probes = drain.probes();
19441 assert!(
19442 probes.len() >= 2 && trace.probes as usize == probes.len(),
19443 "the drain must re-probe after the first busy answer: {probes:?}, {trace}"
19444 );
19445 }
19446
19447 #[tokio::test(start_paused = true)]
19448 async fn busy_throughout_ends_undrained_at_the_ceiling_with_bounded_probes() {
19449 let started = Instant::now();
19450 let drain = Drain::start(Answer::BusyUntil {
19451 until: started + Duration::from_secs(3600),
19452 latency: Duration::ZERO,
19453 });
19454 let (drained, elapsed) = drain.run(CEILING).await;
19455 assert!(!drained, "a module busy throughout must not drain");
19456 assert!(
19457 elapsed >= CEILING && elapsed < CEILING + drain_busy_reprobe_interval(),
19458 "a busy module is held until the ceiling and no longer: {elapsed:?}"
19459 );
19460 let probes = drain.probes();
19463 let max_probes = (CEILING.as_millis() / drain_busy_reprobe_interval().as_millis()) + 1;
19464 assert!(
19465 probes.len() as u128 <= max_probes,
19466 "{} probes in a {CEILING:?} drain exceeds the {max_probes} the interval allows",
19467 probes.len()
19468 );
19469 for pair in probes.windows(2) {
19470 assert!(
19471 pair[1] - pair[0] >= drain_busy_reprobe_interval(),
19472 "consecutive probes {:?} apart",
19473 pair[1] - pair[0]
19474 );
19475 }
19476 }
19477
19478 #[tokio::test(start_paused = true)]
19479 async fn a_slow_probe_is_never_overlapped_by_the_next() {
19480 let latency = Duration::from_secs(1);
19483 let drain = Drain::start(Answer::BusyUntil {
19484 until: Instant::now() + Duration::from_secs(3600),
19485 latency,
19486 });
19487 let (drained, _) = drain.run(Duration::from_secs(5)).await;
19488 assert!(!drained);
19489 let probes = drain.probes();
19490 assert!(probes.len() >= 2, "the drain must re-probe: {probes:?}");
19491 for pair in probes.windows(2) {
19492 assert!(
19493 pair[1] - pair[0] >= latency + drain_busy_reprobe_interval(),
19494 "a probe started {:?} after the previous one, before it could finish",
19495 pair[1] - pair[0]
19496 );
19497 }
19498 }
19499
19500 #[tokio::test(start_paused = true)]
19501 async fn an_unanswered_probe_cannot_hold_the_drain_past_its_ceiling() {
19502 let drain = Drain::start(Answer::Silent);
19503 let ceiling = Duration::from_secs(2);
19504 let (drained, elapsed) = drain.run(ceiling).await;
19505 assert!(!drained, "an unanswered probe counts as busy");
19506 assert!(
19507 elapsed >= ceiling && elapsed < ceiling + drain_busy_reprobe_interval(),
19508 "the probe deadline is capped at the drain ceiling: {elapsed:?}"
19509 );
19510 assert_eq!(
19511 drain.probes().len(),
19512 1,
19513 "the unanswered probe holds the only slot until the ceiling"
19514 );
19515 }
19516
19517 #[derive(Clone, Default)]
19519 struct LogCapture {
19520 events: Arc<StdMutex<Vec<BTreeMap<String, String>>>>,
19521 }
19522
19523 impl<S: tracing::Subscriber> tracing_subscriber::Layer<S> for LogCapture {
19524 fn on_event(
19525 &self,
19526 event: &tracing::Event<'_>,
19527 _context: tracing_subscriber::layer::Context<'_, S>,
19528 ) {
19529 struct Fields(BTreeMap<String, String>);
19530 impl tracing::field::Visit for Fields {
19531 fn record_debug(&mut self, field: &tracing::field::Field, value: &dyn fmt::Debug) {
19532 self.0
19533 .insert(field.name().to_string(), format!("{value:?}"));
19534 }
19535 }
19536 let mut fields = Fields(BTreeMap::new());
19537 event.record(&mut fields);
19538 self.events.lock().unwrap().push(fields.0);
19539 }
19540 }
19541
19542 impl LogCapture {
19543 fn install(&self) -> tracing::subscriber::DefaultGuard {
19544 use tracing_subscriber::layer::SubscriberExt;
19545 tracing::subscriber::set_default(tracing_subscriber::registry().with(self.clone()))
19546 }
19547
19548 fn with_message(&self, needle: &str) -> Vec<BTreeMap<String, String>> {
19549 self.events
19550 .lock()
19551 .unwrap()
19552 .iter()
19553 .filter(|fields| fields.get("message").is_some_and(|m| m.contains(needle)))
19554 .cloned()
19555 .collect()
19556 }
19557 }
19558
19559 #[tokio::test(start_paused = true)]
19564 async fn a_missing_declared_gauge_is_named_on_the_drain_timeout_line() {
19565 let capture = LogCapture::default();
19566 let _guard = capture.install();
19567 let drain = Drain::start(Answer::OmitsGauge);
19568 let (drained, _, trace) = drain.run_traced(Duration::from_secs(2)).await;
19569 assert!(!drained, "a gauge never reported cannot read quiet");
19570 assert!(trace.probes >= 2, "the drain must re-probe: {trace:?}");
19571
19572 let missing = capture.with_message("lacks a declared busy gauge");
19573 assert_eq!(missing.len(), 1, "one warning per drain: {missing:?}");
19574 assert_eq!(
19575 missing[0].get("missing_gauges").map(String::as_str),
19576 Some(GAUGE)
19577 );
19578
19579 warn_drain_timed_out(
19580 MODULE_ID,
19581 Duration::from_secs(2),
19582 RouteCloseReason::Restart,
19583 &Default::default(),
19584 &trace,
19585 );
19586 let timeouts = capture.with_message("route drain timed out");
19587 assert_eq!(timeouts.len(), 1, "{timeouts:?}");
19588 let gauges = &timeouts[0]["busy_gauges"];
19589 assert!(
19590 gauges.starts_with(&format!("{GAUGE}: missing_from_health_body @")),
19591 "the timeout line must name the gauge and why it read busy: {gauges}"
19592 );
19593 assert!(
19594 gauges.ends_with("; no value ever read"),
19595 "no value was ever read for the gauge: {gauges}"
19596 );
19597 assert_eq!(timeouts[0]["gauge_probes"], trace.probes.to_string());
19598 }
19599
19600 #[tokio::test(start_paused = true)]
19604 async fn the_drain_trace_names_a_busy_value_and_an_unanswered_probe() {
19605 let busy = Drain::start(Answer::BusyUntil {
19606 until: Instant::now() + Duration::from_secs(3600),
19607 latency: Duration::ZERO,
19608 });
19609 let (_, _, trace) = busy.run_traced(Duration::from_secs(1)).await;
19610 let rendered = trace.to_string();
19611 assert!(
19612 rendered.starts_with(&format!("{GAUGE}: non_zero(1) @"))
19613 && rendered.contains("; last value non_zero(1) @"),
19614 "{rendered}"
19615 );
19616
19617 let silent = Drain::start(Answer::Silent);
19618 let (_, _, trace) = silent.run_traced(Duration::from_secs(1)).await;
19619 assert_eq!(
19622 trace.to_string(),
19623 format!("{GAUGE}: probe_failed(no-answer) @1000ms; no value ever read")
19624 );
19625 assert_eq!(trace.probes, 1);
19626 }
19627
19628 #[test]
19630 fn a_gauge_that_is_not_a_count_is_busy_and_shown_as_sent() {
19631 let gauges = [GAUGE.to_string()];
19632 let verdicts = gauge_verdicts(Some(&serde_json::json!({ GAUGE: 0.0 })), &gauges);
19633 assert_eq!(verdicts, vec![GaugeVerdict::NotACount("0.0".to_string())]);
19634 assert_eq!(
19635 busy_gauge_observation(&verdicts),
19636 BusyGaugeObservation::Busy
19637 );
19638 let zero = gauge_verdicts(Some(&serde_json::json!({ GAUGE: 0 })), &gauges);
19639 assert_eq!(
19640 busy_gauge_observation(&zero),
19641 BusyGaugeObservation::Quiescent
19642 );
19643 let none = gauge_verdicts(None, &gauges);
19644 assert_eq!(busy_gauge_observation(&none), BusyGaugeObservation::Omitted);
19645 }
19646}
19647
19648#[cfg(test)]
19649mod health_evidence_tests {
19650 use super::{HealthProbeError, HealthProbeEvidence};
19651 use std::collections::HashSet;
19652
19653 #[test]
19661 fn only_a_dead_lane_is_proof_of_death() {
19662 assert!(HealthProbeError::lane_dead("gone").is_proof_of_death());
19663 assert!(!HealthProbeError::no_answer("timed out").is_proof_of_death());
19667 assert!(!HealthProbeError::bad_answer("garbage").is_proof_of_death());
19668 assert!(!HealthProbeError::misconfigured("no table").is_proof_of_death());
19669 }
19670
19671 #[test]
19677 fn every_evidence_class_has_a_distinct_label() {
19678 let labels = [
19679 HealthProbeError::lane_dead("").label(),
19680 HealthProbeError::no_answer("").label(),
19681 HealthProbeError::bad_answer("").label(),
19682 HealthProbeError::misconfigured("").label(),
19683 ];
19684 let unique: HashSet<_> = labels.iter().collect();
19685 assert_eq!(unique.len(), labels.len(), "labels collided: {labels:?}");
19686 }
19687
19688 #[test]
19694 fn classification_preserves_the_original_message() {
19695 let err = HealthProbeError::no_answer("module did not answer within 5s");
19696 assert_eq!(err.to_string(), "module did not answer within 5s");
19697 assert!(matches!(err.evidence, HealthProbeEvidence::NoAnswer));
19698 }
19699}
19700
19701#[cfg(test)]
19702mod health_tombstone_tests {
19703 use std::{path::PathBuf, sync::Arc, time::Duration};
19704
19705 use subc_protocol::{
19706 manifest::Concurrency,
19707 session::{HealthStatus, ModuleControlResponse},
19708 };
19709 use tokio::sync::mpsc;
19710
19711 use super::{
19712 probe_module_health, HealthAction, HealthConfig, HealthProbeEvidence, ModuleProtocol,
19713 ModuleSpec, RestartPolicy, Supervisor, SupervisorRuntimeConfig,
19714 };
19715 use crate::{
19716 control::ControlHandler,
19717 forwarding::{ForwardingTable, ModuleControlRpcCompletion, ModuleControlRpcOutcome},
19718 registry::{ConnectionId, Registry},
19719 router::FrameSink,
19720 };
19721
19722 struct ProbeHarness {
19723 spec: ModuleSpec,
19724 runtime: SupervisorRuntimeConfig,
19725 forwarding: Arc<ForwardingTable>,
19726 module_connection: ConnectionId,
19727 module_rx: mpsc::Receiver<crate::router::OutboundFrame>,
19728 handler: ControlHandler,
19729 module: super::SupervisedModule,
19730 }
19731
19732 fn probe_harness() -> ProbeHarness {
19733 let registry = Arc::new(Registry::default());
19734 let forwarding = Arc::new(ForwardingTable::default());
19735 let supervisor_handle = super::SupervisorHandle::new();
19736 let health = HealthConfig {
19737 http: None,
19738 cadence: Duration::from_secs(30),
19739 deadline: Duration::from_secs(5),
19740 failure_threshold: 3,
19741 on_degraded: HealthAction::Report,
19742 on_failing: HealthAction::Report,
19743 critical: false,
19744 };
19745 let supervisor = Supervisor::new_for_test(Arc::clone(®istry), RestartPolicy::default())
19746 .with_forwarding(Arc::clone(&forwarding))
19747 .with_handle(supervisor_handle.clone())
19748 .with_health_config(health);
19749 let spec = ModuleSpec {
19750 script: None,
19751 module_id: "late-health-module".to_string(),
19752 program: PathBuf::from("disabled-module"),
19753 args: Vec::new(),
19754 env: Vec::new(),
19755 reserved: false,
19756 reserved_prefixes: Vec::new(),
19757 protocol: ModuleProtocol::Subc,
19758 overlap: Default::default(),
19759 carry_containment: false,
19760 };
19761 let module = supervisor
19762 .supervise_configured(spec.clone(), false)
19763 .unwrap();
19764 let runtime = supervisor.runtime_config();
19765 let handler = ControlHandler::with_forwarding(registry, Arc::clone(&forwarding))
19766 .with_supervisor(supervisor_handle);
19767 let module_connection = ConnectionId::new(700);
19768 let (module_tx, module_rx) = mpsc::channel(8);
19769 forwarding
19770 .register_module_connection(
19771 module_connection,
19772 spec.module_id.clone(),
19773 subc_protocol::PROTOCOL_VERSION,
19774 Concurrency::ModuleManaged,
19775 FrameSink::new(module_tx),
19776 )
19777 .unwrap();
19778
19779 ProbeHarness {
19780 spec,
19781 runtime,
19782 forwarding,
19783 module_connection,
19784 module_rx,
19785 handler,
19786 module,
19787 }
19788 }
19789
19790 async fn finish_after(
19791 harness: &mut ProbeHarness,
19792 stall: Duration,
19793 ) -> ModuleControlRpcCompletion {
19794 assert!(stall > harness.runtime.health.deadline);
19795 let deadline = harness.runtime.health.deadline;
19796 let probe = probe_module_health(&harness.spec.module_id, &harness.runtime, None);
19797 let answer = async {
19798 let frame = harness.module_rx.recv().await.expect("health.check frame");
19799 tokio::time::advance(deadline).await;
19800 tokio::task::yield_now().await;
19801 tokio::time::advance(stall - deadline).await;
19802 harness
19803 .forwarding
19804 .complete_module_control_rpc(
19805 harness.module_connection,
19806 frame.header.corr,
19807 Some("health.check"),
19808 ModuleControlRpcOutcome::Response(ModuleControlResponse::HealthCheck {
19809 status: HealthStatus::Ok,
19810 detail: None,
19811 metrics: None,
19812 }),
19813 )
19814 .unwrap()
19815 };
19816 let (probe_result, completion) = tokio::join!(probe, answer);
19817 let err = probe_result.expect_err("probe must miss its deadline");
19818 assert!(matches!(err.evidence, HealthProbeEvidence::NoAnswer));
19819 completion
19820 }
19821
19822 async fn time_out_without_answer(harness: &mut ProbeHarness) {
19823 let deadline = harness.runtime.health.deadline;
19824 let probe = probe_module_health(&harness.spec.module_id, &harness.runtime, None);
19825 let exhaust_deadline = async {
19826 let _frame = harness.module_rx.recv().await.expect("health.check frame");
19827 tokio::time::advance(deadline).await;
19828 tokio::task::yield_now().await;
19829 };
19830 let (probe_result, ()) = tokio::join!(probe, exhaust_deadline);
19831 let err = probe_result.expect_err("probe must miss its deadline");
19832 assert!(matches!(err.evidence, HealthProbeEvidence::NoAnswer));
19833 }
19834
19835 async fn run_probe_cycle(harness: &mut ProbeHarness, answer: bool) {
19836 let registry = Arc::clone(&harness.module.inner.registry);
19837 let snapshot = Arc::clone(&harness.module.inner.snapshot);
19838 let process_liveness = super::SupervisorProcessLiveness::default();
19839 let mut child = None;
19840 let cycle = super::run_health_probe_cycle(
19841 &harness.spec,
19842 &harness.runtime,
19843 ®istry,
19844 &process_liveness,
19845 &snapshot,
19846 &mut child,
19847 );
19848 let peer = async {
19849 let frame = harness.module_rx.recv().await.expect("health.check frame");
19850 if answer {
19851 harness
19852 .forwarding
19853 .complete_module_control_rpc(
19854 harness.module_connection,
19855 frame.header.corr,
19856 Some("health.check"),
19857 ModuleControlRpcOutcome::Response(ModuleControlResponse::HealthCheck {
19858 status: HealthStatus::Ok,
19859 detail: None,
19860 metrics: Some(serde_json::json!({"ready": true})),
19861 }),
19862 )
19863 .unwrap();
19864 } else {
19865 tokio::time::advance(harness.runtime.health.deadline).await;
19866 tokio::task::yield_now().await;
19867 }
19868 };
19869 tokio::join!(cycle, peer);
19870 }
19871
19872 #[tokio::test(start_paused = true)]
19873 async fn unanswered_probe_is_unknown_until_threshold_and_ok_report_recovers() {
19874 let mut harness = probe_harness();
19875 let monitor = harness.module.inner.monitor.lock().unwrap().take().unwrap();
19879 monitor.abort();
19880 let _ = monitor.await;
19881 super::update_snapshot(&harness.module.inner.snapshot, None, |state| {
19882 state.enabled = true;
19883 state.state = super::ModuleState::Running;
19884 state.process_alive = true;
19885 })
19886 .unwrap();
19887
19888 run_probe_cycle(&mut harness, true).await;
19889 assert_eq!(
19890 harness.module.status().unwrap().health.status,
19891 super::SupervisorHealthStatus::Ok
19892 );
19893
19894 for failures in 1..harness.runtime.health.failure_threshold {
19895 run_probe_cycle(&mut harness, false).await;
19896 let status = harness.module.status().unwrap();
19897 assert_eq!(status.health.status, super::SupervisorHealthStatus::Unknown);
19898 assert_eq!(status.health.consecutive_failures, failures);
19899 assert!(status.health.last_probe_ms.is_some());
19900 assert!(status.health.detail.unwrap().starts_with("[no-answer]"));
19901 assert!(status.health.metrics.is_none());
19902 assert_eq!(status.state, super::ModuleState::Running);
19903 assert!(status.process_alive);
19904 assert_eq!(status.restart_count, 0);
19905 assert_eq!(status.lifetime_restarts, 0);
19906 assert!(status.health.last_action.is_none());
19907 assert!(harness.runtime.scheduled_respawn.lock().unwrap().is_none());
19908 }
19909
19910 run_probe_cycle(&mut harness, true).await;
19911 let recovered = harness.module.status().unwrap();
19912 assert_eq!(recovered.health.status, super::SupervisorHealthStatus::Ok);
19913 assert_eq!(recovered.health.consecutive_failures, 0);
19914 assert!(recovered.health.detail.is_none());
19915 assert_eq!(
19916 recovered.health.metrics,
19917 Some(serde_json::json!({"ready": true}))
19918 );
19919 assert_eq!(recovered.lifetime_restarts, 0);
19920
19921 for failures in 1..=harness.runtime.health.failure_threshold {
19922 run_probe_cycle(&mut harness, false).await;
19923 let status = harness.module.status().unwrap();
19924 assert_eq!(status.health.consecutive_failures, failures);
19925 if failures < harness.runtime.health.failure_threshold {
19926 assert_eq!(status.health.status, super::SupervisorHealthStatus::Unknown);
19927 assert_eq!(status.state, super::ModuleState::Running);
19928 assert_eq!(status.lifetime_restarts, 0);
19929 assert!(harness.runtime.scheduled_respawn.lock().unwrap().is_none());
19930 } else {
19931 assert_eq!(
19932 status.health.status,
19933 super::SupervisorHealthStatus::Unresponsive
19934 );
19935 assert_eq!(status.state, super::ModuleState::Restarting);
19936 assert_eq!(status.restart_count, 1);
19937 assert_eq!(status.lifetime_restarts, 1);
19938 assert_eq!(status.health.last_action.as_deref(), Some("restart"));
19939 assert!(harness.runtime.scheduled_respawn.lock().unwrap().is_some());
19940 }
19941 }
19942 }
19943
19944 #[tokio::test(start_paused = true)]
19945 async fn late_health_answers_record_start_anchored_latency_for_two_stalls() {
19946 let mut harness = probe_harness();
19947
19948 let first = finish_after(&mut harness, Duration::from_secs(8)).await;
19949 let first_latency = match &first {
19950 ModuleControlRpcCompletion::LateHealthAnswer { latency, .. } => *latency,
19951 other => panic!("late answer was not retained: {other:?}"),
19952 };
19953 assert!(harness.handler.observe_module_control_completion(first));
19954
19955 let second = finish_after(&mut harness, Duration::from_secs(11)).await;
19956 let second_latency = match &second {
19957 ModuleControlRpcCompletion::LateHealthAnswer { latency, .. } => *latency,
19958 other => panic!("late answer was not retained: {other:?}"),
19959 };
19960 assert!(harness.handler.observe_module_control_completion(second));
19961
19962 assert_eq!(first_latency, Duration::from_secs(8));
19963 assert_eq!(
19964 second_latency - first_latency,
19965 Duration::from_secs(3),
19966 "latency must grow linearly with the additional stall"
19967 );
19968 let health = harness.module.status().unwrap().health;
19969 assert_eq!(health.late_answer_count, 2);
19970 assert_eq!(health.last_late_answer_latency_ms, Some(11_000));
19971 }
19972
19973 #[tokio::test(start_paused = true)]
19981 async fn late_answer_clears_the_consecutive_failure_streak() {
19982 let mut harness = probe_harness();
19983
19984 time_out_without_answer(&mut harness).await;
19986 harness
19987 .module
19988 .record_health_probe_failure_for_test("[no-answer] test miss")
19989 .unwrap();
19990 assert_eq!(
19991 harness.module.status().unwrap().health.consecutive_failures,
19992 1,
19993 "precondition: the miss must be on the streak before the late answer"
19994 );
19995
19996 let late = finish_after(&mut harness, Duration::from_secs(9)).await;
19998 assert!(matches!(
19999 late,
20000 ModuleControlRpcCompletion::LateHealthAnswer { .. }
20001 ));
20002 assert!(harness.handler.observe_module_control_completion(late));
20003
20004 let health = harness.module.status().unwrap().health;
20005 assert_eq!(
20006 health.consecutive_failures, 0,
20007 "a late answer is an answer: the streak must reset"
20008 );
20009 assert_eq!(health.late_answer_count, 1);
20010 }
20011
20012 #[tokio::test(start_paused = true)]
20013 async fn repeated_serial_probe_cycles_keep_one_tombstone_per_endpoint() {
20014 let mut harness = probe_harness();
20015
20016 for _ in 0..20 {
20017 time_out_without_answer(&mut harness).await;
20018 assert_eq!(
20019 harness.forwarding.health_probe_tombstone_count().unwrap(),
20020 1
20021 );
20022 }
20023 }
20024}
20025
20026#[cfg(test)]
20027mod child_env_tests {
20028 #[cfg(windows)]
20029 use super::{
20030 clear_current_process_facts, observed_launch_nonce_source, ModuleState, SupervisorSnapshot,
20031 };
20032 #[cfg(windows)]
20033 use std::sync::{Arc, OnceLock};
20034 #[cfg(windows)]
20035 #[test]
20036 fn windows_wire_spawn_prepares_pipe_and_keeps_phase_one_env() {
20037 let spec = spec(Vec::new());
20038 let mut command = Command::new("unused");
20039 let handoff = apply_wire_spawn_args(&mut command, &spec, None, None)
20040 .unwrap()
20041 .unwrap();
20042 let name = command
20043 .as_std()
20044 .get_envs()
20045 .find(|(key, _)| *key == OsStr::new(subc_os::launch_nonce::LAUNCH_NONCE_PIPE_ENV))
20046 .and_then(|(_, value)| value)
20047 .unwrap();
20048 assert_eq!(name, OsStr::new(handoff.name()));
20049 assert!(handoff.name().starts_with(r"\\.\pipe\subc-launch-"));
20050 assert!(command.as_std().get_envs().any(|(key, value)| key
20051 == OsStr::new(subc_os::LAUNCH_NONCE_PIPE_FALLBACK_ENV)
20052 && value == Some(OsStr::new("env"))));
20053 assert!(command
20054 .as_std()
20055 .get_envs()
20056 .any(|(key, value)| key == OsStr::new(SUBC_LAUNCH_NONCE_ENV) && value.is_some()));
20057 }
20058
20059 #[cfg(windows)]
20060 #[test]
20061 fn windows_source_reporting_tracks_consumption_and_resets_per_spawn() {
20062 let mut snapshot = SupervisorSnapshot::new(ModuleState::Running, true);
20063 snapshot.process_alive = true;
20064 snapshot.spawned_protocol = Some(ModuleProtocol::Subc);
20065 assert_eq!(
20066 observed_launch_nonce_source(&snapshot).as_deref(),
20067 Some("env")
20068 );
20069 let consumed = Arc::new(OnceLock::new());
20070 snapshot.pipe_consumed = Some(consumed.clone());
20071 assert_eq!(
20072 observed_launch_nonce_source(&snapshot).as_deref(),
20073 Some("env")
20074 );
20075 consumed.set(()).unwrap();
20076 assert_eq!(
20077 observed_launch_nonce_source(&snapshot).as_deref(),
20078 Some("pipe")
20079 );
20080 snapshot.pipe_consumed = Some(Arc::new(OnceLock::new()));
20081 assert_eq!(
20082 observed_launch_nonce_source(&snapshot).as_deref(),
20083 Some("env")
20084 );
20085 clear_current_process_facts(&mut snapshot);
20086 assert_eq!(observed_launch_nonce_source(&snapshot), None);
20087 }
20088 use super::{
20089 apply_child_env, apply_spawn_role, apply_wire_spawn_args, LaunchSpecDigest, ModuleProtocol,
20090 ModuleSpec, SpawnRole, SupervisorHandle, SPAWN_ROLE_SWAP_CANDIDATE, SUBC_ARG,
20091 SUBC_LAUNCH_NONCE_ENV, SUBC_MODULE_ID_ENV, SUBC_SPAWN_ROLE_ENV,
20092 };
20093 use std::{ffi::OsStr, path::PathBuf};
20094 use tokio::process::Command;
20095
20096 fn spec(env: Vec<(String, String)>) -> ModuleSpec {
20097 ModuleSpec {
20098 script: None,
20099 module_id: "env-plan".to_string(),
20100 program: PathBuf::from("/nonexistent"),
20101 args: Vec::new(),
20102 env,
20103 reserved: false,
20104 reserved_prefixes: Vec::new(),
20105 protocol: ModuleProtocol::Subc,
20106 overlap: Default::default(),
20107 carry_containment: false,
20108 }
20109 }
20110
20111 #[test]
20112 fn configured_launch_spec_digest_is_canonical_and_excludes_injected_inputs() {
20113 let base = spec(vec![("A".into(), "B=C".into()), ("Z".into(), "v".into())]);
20114 let digest = LaunchSpecDigest::from_spec(&base);
20115 let mut reordered = base.clone();
20116 reordered.env.reverse();
20117 reordered.module_id = "different-id".into();
20118 reordered.reserved = true;
20119 reordered.overlap = Default::default();
20120 assert_eq!(digest, LaunchSpecDigest::from_spec(&reordered));
20121 let mut delimiter_collision = base.clone();
20122 delimiter_collision.env[0] = ("A=B".into(), "C".into());
20123 assert_ne!(digest, LaunchSpecDigest::from_spec(&delimiter_collision));
20124 let mut changed = base.clone();
20125 changed.protocol = ModuleProtocol::None;
20126 assert_ne!(digest, LaunchSpecDigest::from_spec(&changed));
20127 changed = base.clone();
20128 changed.args.push("--flag".into());
20129 assert_ne!(digest, LaunchSpecDigest::from_spec(&changed));
20130 changed = base.clone();
20131 changed.program = PathBuf::from("/other");
20132 assert_ne!(digest, LaunchSpecDigest::from_spec(&changed));
20133 changed = base.clone();
20134 changed.env[1].1 = "other".into();
20135 assert_ne!(digest, LaunchSpecDigest::from_spec(&changed));
20136 assert!(!format!("{digest:?}").contains("B=C"));
20137 assert!(!format!("{digest:?}").contains("[u8"));
20138 }
20139
20140 #[cfg(unix)]
20141 #[test]
20142 fn native_program_bytes_do_not_collapse_through_lossy_utf8() {
20143 use std::os::unix::ffi::OsStringExt;
20144 let mut left = spec(Vec::new());
20145 left.program = std::ffi::OsString::from_vec(vec![b'/', 0xff]).into();
20146 let mut right = left.clone();
20147 right.program = std::ffi::OsString::from_vec(vec![b'/', 0xfe]).into();
20148 assert_eq!(
20149 left.program.to_string_lossy(),
20150 right.program.to_string_lossy()
20151 );
20152 assert_ne!(
20153 LaunchSpecDigest::from_spec(&left),
20154 LaunchSpecDigest::from_spec(&right)
20155 );
20156 }
20157
20158 #[test]
20172 fn ambient_ck_log_is_removed_and_a_configured_one_survives() {
20173 let mut command = Command::new("/nonexistent");
20174 apply_child_env(&mut command, &spec(Vec::new()));
20175 let removed = command
20176 .as_std()
20177 .get_envs()
20178 .any(|(key, value)| key == OsStr::new("CK_LOG") && value.is_none());
20179 assert!(
20180 removed,
20181 "ambient CK_LOG must be explicitly removed for an unconfigured module"
20182 );
20183
20184 let mut configured = Command::new("/nonexistent");
20185 apply_child_env(
20186 &mut configured,
20187 &spec(vec![("CK_LOG".to_string(), "debug".to_string())]),
20188 );
20189 let effective = configured
20190 .as_std()
20191 .get_envs()
20192 .filter(|(key, _)| *key == OsStr::new("CK_LOG"))
20193 .last()
20194 .map(|(_, value)| value.map(|v| v.to_string_lossy().into_owned()));
20195 assert_eq!(
20196 effective,
20197 Some(Some("debug".to_string())),
20198 "a module's configured CK_LOG must survive the ambient removal"
20199 );
20200 }
20201
20202 #[test]
20211 fn protocol_none_spawn_carries_no_subc_argument_and_no_nonce() {
20212 let connection_file = std::path::Path::new("/run/subc-connection.json");
20213 let handle = SupervisorHandle::new();
20214
20215 let mut none_spec = spec(Vec::new());
20216 none_spec.protocol = ModuleProtocol::None;
20217 let mut none = Command::new("/nonexistent");
20218 let none_handoff =
20219 apply_wire_spawn_args(&mut none, &none_spec, Some(connection_file), Some(&handle))
20220 .expect("protocol-none spawn args apply");
20221 assert!(
20222 none_handoff.is_none(),
20223 "protocol:none spawn must not receive a nonce descriptor"
20224 );
20225 #[cfg(windows)]
20226 assert!(!none.as_std().get_envs().any(|(key, value)| key
20227 == OsStr::new(subc_os::LAUNCH_NONCE_PIPE_FALLBACK_ENV)
20228 && value.is_some()));
20229 assert!(
20230 !none.as_std().get_envs().any(|(key, value)| key
20231 == OsStr::new(subc_os::LAUNCH_NONCE_FD_ENV)
20232 && value.is_some()),
20233 "protocol:none spawn must not name a nonce descriptor"
20234 );
20235 let none_args: Vec<String> = none
20236 .as_std()
20237 .get_args()
20238 .map(|a| a.to_string_lossy().into_owned())
20239 .collect();
20240 assert!(
20241 !none_args.iter().any(|a| a == SUBC_ARG),
20242 "protocol:none argv must not carry --subc; got {none_args:?}"
20243 );
20244 let none_has_nonce = none
20245 .as_std()
20246 .get_envs()
20247 .any(|(key, value)| key == OsStr::new(SUBC_LAUNCH_NONCE_ENV) && value.is_some());
20248 assert!(
20249 !none_has_nonce,
20250 "protocol:none spawn must not receive a launch nonce"
20251 );
20252 let none_has_module_id = none
20253 .as_std()
20254 .get_envs()
20255 .any(|(key, value)| key == OsStr::new(SUBC_MODULE_ID_ENV) && value.is_some());
20256 assert!(
20257 none_has_module_id,
20258 "SUBC_MODULE_ID is inert and stays on every path"
20259 );
20260 assert!(
20261 handle.spawn_nonce(&none_spec.module_id).is_none(),
20262 "no nonce record for a process that will never present one"
20263 );
20264
20265 let wire_spec = spec(Vec::new());
20267 let mut wire = Command::new("/nonexistent");
20268 let wire_handoff =
20269 apply_wire_spawn_args(&mut wire, &wire_spec, Some(connection_file), Some(&handle))
20270 .expect("subc-wire spawn args apply");
20271 let wire_fd_env = wire
20272 .as_std()
20273 .get_envs()
20274 .find(|(key, _)| *key == OsStr::new(subc_os::LAUNCH_NONCE_FD_ENV))
20275 .and_then(|(_, value)| value.map(|v| v.to_string_lossy().into_owned()));
20276 #[cfg(unix)]
20277 assert_eq!(
20278 wire_fd_env,
20279 Some(wire_handoff.expect("a descriptor handoff").fd_env_value()),
20280 "a subc-wire spawn names the pipe it will receive at descriptor 3"
20281 );
20282 #[cfg(windows)]
20283 assert!(wire_handoff.is_some() && wire_fd_env.is_none());
20284 #[cfg(not(any(unix, windows)))]
20285 assert!(wire_handoff.is_none() && wire_fd_env.is_none());
20286 let wire_args: Vec<String> = wire
20287 .as_std()
20288 .get_args()
20289 .map(|a| a.to_string_lossy().into_owned())
20290 .collect();
20291 assert_eq!(
20292 wire_args,
20293 vec![
20294 SUBC_ARG.to_string(),
20295 connection_file.to_string_lossy().into_owned()
20296 ],
20297 "a subc-wire spawn still carries --subc <path>"
20298 );
20299 assert_eq!(
20300 wire.as_std()
20301 .get_envs()
20302 .any(|(key, value)| key == OsStr::new(SUBC_LAUNCH_NONCE_ENV) && value.is_some()),
20303 !cfg!(unix),
20304 "only Windows supplies the environment nonce"
20305 );
20306 assert!(handle.spawn_nonce(&wire_spec.module_id).is_some());
20307 }
20308
20309 #[test]
20319 fn plain_spawn_removes_the_spawn_role_even_when_the_spec_sets_it() {
20320 let role = |command: &Command| {
20321 command
20322 .as_std()
20323 .get_envs()
20324 .filter(|(key, _)| *key == OsStr::new(SUBC_SPAWN_ROLE_ENV))
20325 .last()
20326 .map(|(_, value)| value.map(|v| v.to_string_lossy().into_owned()))
20327 };
20328 let forged = spec(vec![(
20329 SUBC_SPAWN_ROLE_ENV.to_string(),
20330 SPAWN_ROLE_SWAP_CANDIDATE.to_string(),
20331 )]);
20332
20333 let mut plain = Command::new("/nonexistent");
20334 apply_child_env(&mut plain, &forged);
20335 apply_spawn_role(&mut plain, SpawnRole::Plain);
20336 assert_eq!(
20337 role(&plain),
20338 Some(None),
20339 "a plain spawn must remove SUBC_SPAWN_ROLE, whatever the spec says"
20340 );
20341
20342 let mut candidate = Command::new("/nonexistent");
20343 apply_child_env(&mut candidate, &spec(Vec::new()));
20344 apply_spawn_role(&mut candidate, SpawnRole::SwapCandidate);
20345 assert_eq!(
20346 role(&candidate),
20347 Some(Some(SPAWN_ROLE_SWAP_CANDIDATE.to_string()))
20348 );
20349 }
20350
20351 #[test]
20357 fn daemon_private_capture_keys_are_not_passed_to_the_child() {
20358 let mut command = Command::new("/nonexistent");
20359 apply_child_env(
20360 &mut command,
20361 &spec(vec![
20362 (super::CAPTURE_KEEP_ENV.to_string(), "5".to_string()),
20363 ("KEPT".to_string(), "yes".to_string()),
20364 ]),
20365 );
20366 let keys: Vec<String> = command
20367 .as_std()
20368 .get_envs()
20369 .filter(|(_, value)| value.is_some())
20370 .map(|(key, _)| key.to_string_lossy().into_owned())
20371 .collect();
20372 assert!(keys.contains(&"KEPT".to_string()), "got {keys:?}");
20373 assert!(
20374 !keys.contains(&super::CAPTURE_KEEP_ENV.to_string()),
20375 "daemon-private capture key leaked to the child: {keys:?}"
20376 );
20377 }
20378}
20379
20380#[cfg(test)]
20381mod jitter_tests {
20382 use super::jittered_health_delay;
20383 use std::{collections::HashSet, time::Duration};
20384
20385 const FLEET: [&str; 14] = [
20394 "aft",
20395 "alfonso-core",
20396 "magic-context",
20397 "broca",
20398 "thalamus",
20399 "quota",
20400 "engram",
20401 "plexus",
20402 "cerebellum",
20403 "astrocyte",
20404 "synapse",
20405 "subc-mcp",
20406 "cortexkit-credentials",
20407 "subc-federation",
20408 ];
20409
20410 #[test]
20418 fn probe_delays_disperse_across_the_fleet() {
20419 let cadence = Duration::from_secs(30);
20420 let delays: HashSet<Duration> = FLEET
20421 .iter()
20422 .map(|id| jittered_health_delay(id, 0, cadence))
20423 .collect();
20424 assert_eq!(
20425 delays.len(),
20426 FLEET.len(),
20427 "every supervised module must land on its own probe offset"
20428 );
20429 }
20430
20431 #[test]
20437 fn jitter_only_delays_and_stays_within_one_tenth_of_cadence() {
20438 let cadence = Duration::from_secs(30);
20439 let span = cadence / 10;
20440 for id in FLEET {
20441 for probe_index in 0..8 {
20442 let delay = jittered_health_delay(id, probe_index, cadence);
20443 assert!(
20444 delay >= cadence,
20445 "{id}#{probe_index}: jitter must not shorten the cadence"
20446 );
20447 assert!(
20448 delay < cadence + span,
20449 "{id}#{probe_index}: jitter must stay inside one tenth of the cadence"
20450 );
20451 }
20452 }
20453 }
20454
20455 #[test]
20461 fn a_module_offset_is_stable_across_restarts() {
20462 let cadence = Duration::from_secs(30);
20463 for id in FLEET {
20464 assert_eq!(
20465 jittered_health_delay(id, 0, cadence),
20466 jittered_health_delay(id, 0, cadence),
20467 "{id}: the same module and probe index must produce the same offset"
20468 );
20469 }
20470 }
20471
20472 #[test]
20474 fn zero_cadence_yields_zero_delay() {
20475 assert_eq!(
20476 jittered_health_delay("aft", 0, Duration::ZERO),
20477 Duration::ZERO
20478 );
20479 }
20480}
20481
20482#[cfg(all(test, target_os = "linux"))]
20483mod cgroup_placement_tests {
20484 use super::{
20485 apply_cgroup_placement, remove_module_cgroup, LaunchSpecDigest, ModuleProtocol, ModuleSpec,
20486 SuperviseError, SupervisedChild,
20487 };
20488 use crate::stderr_tail::{StderrRing, StderrTailConfig};
20489 use cortexkit_test_support::ScratchDir;
20490 use std::{
20491 fs, io,
20492 path::{Path, PathBuf},
20493 sync::{Arc, Mutex},
20494 };
20495 use tokio::process::Command;
20496
20497 #[tokio::test]
20498 async fn unique_spawn_cgroups_clean_up_across_restarts_and_shutdown() {
20499 use super::*;
20500 let dir = ScratchDir::new("unique-spawn-cgroups");
20501 let root = PathBuf::from(format!(
20502 "/sys/fs/cgroup/subc-unique-test-{}-{}",
20503 std::process::id(),
20504 unix_ms_now()
20505 ));
20506 if let Err(error) = fs::create_dir(&root) {
20507 assert!(
20508 std::env::var_os("SUBC_REQUIRE_CGROUP_TEST").is_none(),
20509 "required cgroup test cannot execute: {error}"
20510 );
20511 eprintln!("SKIP unique_spawn_cgroups_clean_up_across_restarts_and_shutdown: {error}");
20512 return;
20513 }
20514 let placement = subc_cgroup::prepare_at(&root).unwrap().unwrap();
20515 let group_count = || {
20516 fs::read_dir(root.join("subc-modules"))
20517 .unwrap()
20518 .map(|entry| entry.unwrap().file_type().unwrap())
20519 .filter(|kind| kind.is_dir())
20520 .count()
20521 };
20522 let supervisor =
20523 Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default())
20524 .with_cgroup_placement(Some(placement.clone()));
20525 let runtime = supervisor.runtime_config();
20526 let mut spec = ModuleSpec {
20527 script: None,
20528 module_id: "unique-spawn".into(),
20529 program: PathBuf::from("/bin/sleep"),
20530 args: vec!["60".into()],
20531 env: ["XDG_DATA_HOME", "XDG_RUNTIME_DIR", "XDG_CONFIG_HOME"]
20532 .into_iter()
20533 .map(|key| (key.into(), dir.to_string_lossy().into_owned()))
20534 .collect(),
20535 reserved: false,
20536 reserved_prefixes: vec![],
20537 protocol: ModuleProtocol::None,
20538 overlap: Default::default(),
20539 carry_containment: false,
20540 };
20541 let spawn = |spec: &ModuleSpec| {
20542 spawn_child(
20543 spec,
20544 None,
20545 None,
20546 &runtime.stderr_ring,
20547 None,
20548 &runtime.child_roster,
20549 runtime.spawn_delivery(),
20550 Some(&placement),
20551 runtime.register_timeout,
20552 )
20553 .unwrap()
20554 };
20555 let mut live = spawn(&spec);
20556 for _ in 0..3 {
20557 let next = spawn(&spec);
20559 assert_ne!(live.module_id, next.module_id);
20560 live.start_kill().unwrap();
20561 live.wait().await.unwrap();
20562 live = next;
20563 assert!(
20564 live.child.try_wait().unwrap().is_none(),
20565 "retiring the old slot must not kill the replacement"
20566 );
20567 assert_eq!(
20568 group_count(),
20569 1,
20570 "only the live spawn's cgroup should remain"
20571 );
20572 }
20573 supervisor.begin_daemon_shutdown();
20574 let reap = tokio::spawn(async move {
20575 live.wait().await.unwrap();
20576 });
20577 supervisor
20578 .end_children_for_daemon_shutdown(false, std::future::pending())
20579 .await;
20580 reap.await.unwrap();
20581 assert_eq!(group_count(), 0);
20582 spec.program = PathBuf::from("/bin/true");
20584 spec.args.clear();
20585 let fresh_roster = ChildRoster::default();
20586 let mut short = spawn_child(
20587 &spec,
20588 None,
20589 None,
20590 &runtime.stderr_ring,
20591 None,
20592 &fresh_roster,
20593 None,
20594 Some(&placement),
20595 runtime.register_timeout,
20596 )
20597 .unwrap();
20598 short.wait().await.unwrap();
20599 assert_eq!(group_count(), 0);
20600 spec.module_id = "_".repeat(255);
20601 let mut long_id = spawn_child(
20602 &spec,
20603 None,
20604 None,
20605 &runtime.stderr_ring,
20606 None,
20607 &fresh_roster,
20608 None,
20609 Some(&placement),
20610 runtime.register_timeout,
20611 )
20612 .unwrap();
20613 long_id.wait().await.unwrap();
20614 assert_eq!(
20615 group_count(),
20616 0,
20617 "valid long module IDs must not exceed cgroup NAME_MAX"
20618 );
20619 fs::remove_dir(root.join("subc-modules")).unwrap();
20620 fs::remove_dir(root).unwrap();
20621 eprintln!("EXECUTED unique_spawn_cgroups_clean_up_across_restarts_and_shutdown");
20622 }
20623
20624 #[test]
20625 fn failed_parent_cgroup_open_is_a_cgroup_supervision_error() {
20626 let path = Path::new("/definitely-missing-subc-cgroup");
20627 let mut command = Command::new("true");
20628 let error = apply_cgroup_placement(
20629 &mut command,
20630 &ModuleSpec {
20631 script: None,
20632 module_id: "broken-cgroup".to_string(),
20633 program: PathBuf::from("true"),
20634 args: Vec::new(),
20635 env: Vec::new(),
20636 reserved: false,
20637 reserved_prefixes: Vec::new(),
20638 protocol: ModuleProtocol::Subc,
20639 overlap: Default::default(),
20640 carry_containment: false,
20641 },
20642 path,
20643 )
20644 .expect_err("a parent cgroup open failure must reject the supervised spawn");
20645 let reason = error.to_string();
20646
20647 assert!(
20648 matches!(error, SuperviseError::Cgroup { .. }),
20649 "parent cgroup open must be reported as a cgroup supervision error: {reason}"
20650 );
20651 assert!(
20652 reason.contains("/definitely-missing-subc-cgroup/cgroup.procs"),
20653 "parent cgroup open failure must name cgroup.procs: {reason}"
20654 );
20655 }
20656
20657 #[tokio::test]
20658 async fn reaping_a_child_removes_its_empty_module_cgroup() {
20659 let root = ScratchDir::new("supervisor-reap-cgroup");
20660 fs::write(root.join("cgroup.procs"), b"").expect("write scratch cgroup marker");
20661 let placement = subc_cgroup::prepare_at(&root)
20662 .expect("prepare scratch cgroup root")
20663 .expect("scratch root has a cgroup.procs marker");
20664 let module_id = "reaped-module";
20665 let module = placement
20666 .module_path(module_id)
20667 .expect("create scratch module cgroup");
20668 let child = Command::new("true")
20669 .env("XDG_DATA_HOME", root.path())
20670 .env("XDG_RUNTIME_DIR", root.path())
20671 .env("XDG_CONFIG_HOME", root.path())
20672 .spawn()
20673 .expect("spawn short-lived child");
20674 let pid = child.id().expect("spawned child has pid");
20675 let mut child = SupervisedChild {
20676 registration_channel: None,
20677 registration_spawn: None,
20678 live_program_binding: None,
20679 registered_config_connection: None,
20680 admission_base: None,
20681 registration_declaration: None,
20682 _program_binding: None,
20683 subtree_target: crate::subtree_resources::Target::Unavailable("not_running"),
20684 child: crate::supervise::containment::ProcessChild::plain(child, pid),
20685 exec: crate::activity::ExecOwner(Arc::new(crate::activity::ExecOutcome::new())),
20686 protocol: ModuleProtocol::Subc,
20687 module_id: module_id.to_string(),
20688 cgroup_placement: Some(placement),
20689 stdout_pump: None,
20690 stderr_pump: None,
20691 stderr_ring: Arc::new(Mutex::new(StderrRing::new(StderrTailConfig::default()))),
20692 spawned_at: tokio::time::Instant::now(),
20693 config_delivery: None,
20694 registration_deadline: None,
20695 registration_watch: None,
20696 spawned_at_ms: 0,
20697 spawned_from: PathBuf::from("true"),
20698 launch_spec_digest: LaunchSpecDigest([0; 32]),
20699 spawned_file_identity: None,
20700 process_start_time: None,
20701 process_identity: None,
20702 pid,
20703 roster_guard: None,
20704 #[cfg(target_os = "macos")]
20705 privacy_exec: None,
20706 spawn_failure: None,
20707 admission_kill_sent: false,
20708 };
20709
20710 child.wait().await.expect("reap short-lived child");
20711
20712 assert!(
20713 !module.exists(),
20714 "waiting for a non-carried child must remove its empty cgroup before returning"
20715 );
20716 child.finish_exit(false).await.unwrap();
20717 }
20718
20719 #[test]
20720 fn non_empty_cgroup_removal_is_reported_without_blocking_teardown() {
20721 let root = ScratchDir::new("supervisor-non-empty-cgroup");
20722 fs::write(root.join("cgroup.procs"), b"").expect("write scratch cgroup marker");
20723 let placement = subc_cgroup::prepare_at(&root)
20724 .expect("prepare scratch cgroup root")
20725 .expect("scratch root has a cgroup.procs marker");
20726 let module = placement
20727 .module_path("surviving-module")
20728 .expect("create scratch module cgroup");
20729 fs::write(module.join("surviving-process"), b"still present")
20730 .expect("make scratch cgroup non-empty");
20731 let (logs, _guard) = crate::router::test_log::log_capture(tracing::Level::WARN);
20732
20733 remove_module_cgroup(&placement, "surviving-module");
20734
20735 let logs = crate::router::test_log::captured_logs(&logs);
20736 assert!(
20737 module.exists(),
20738 "failed removal must leave the cgroup intact"
20739 );
20740 assert!(
20741 logs.contains("could not remove module cgroup after process exit; continuing teardown")
20742 && logs.contains("surviving-module"),
20743 "best-effort removal must report the failure without returning it: {logs}"
20744 );
20745 }
20746
20747 #[test]
20748 fn cgroup_pre_exec_spawn_failure_names_the_cgroup_path() {
20749 let cgroup_path = PathBuf::from("/sys/fs/cgroup/subc-modules/broken-module");
20750 let reason = SuperviseError::Spawn {
20751 program: PathBuf::from("/bin/true"),
20752 source: io::Error::from_raw_os_error(13),
20753 cgroup_path: Some(cgroup_path.clone()),
20754 }
20755 .to_string();
20756
20757 assert!(
20758 reason.contains(&cgroup_path.display().to_string()),
20759 "a pre_exec spawn failure must name the cgroup path: {reason}"
20760 );
20761 }
20762}
20763
20764#[cfg(test)]
20765mod spawn_subscriber_lag_tests {
20766 use super::*;
20767
20768 #[tokio::test]
20773 async fn lagged_spawn_subscriber_receives_a_terminal_lagged_error_after_its_queued_frames() {
20774 let feed = SpawnEventFeed::default();
20775 feed.configure_incarnation("lag-incarnation".to_string());
20776 let (tx, mut rx) = mpsc::channel(1);
20779 feed.subscribe(ConnectionId::new(1), 7, 1, None, FrameSink::new(tx))
20780 .expect("subscribe");
20781 let emitted = SPAWN_SUBSCRIBER_BUFFER + 16;
20782 for index in 0..emitted {
20783 feed.emit_spawned(&format!("lag-module-{index}"), None, 1000, 0);
20784 tokio::task::yield_now().await;
20787 }
20788 assert_eq!(
20789 feed.subscriber_count(),
20790 0,
20791 "the lagged subscriber must be removed"
20792 );
20793
20794 let mut data = Vec::new();
20795 let mut last = None;
20796 loop {
20797 let next = tokio::time::timeout(Duration::from_secs(5), rx.recv())
20798 .await
20799 .expect("the forwarder must finish once the subscriber is dropped");
20800 let Some(outbound) = next else { break };
20801 let frame = outbound.frame;
20802 if frame.header.ty == FrameType::StreamData {
20803 assert!(last.is_none(), "no data may follow the terminal frame");
20804 let event: SpawnEvent = serde_json::from_slice(&frame.body).unwrap();
20805 data.push(event.cursor.seq);
20806 } else {
20807 assert!(last.is_none(), "exactly one terminal frame");
20808 last = Some(frame);
20809 }
20810 }
20811 assert!(!data.is_empty(), "queued frames drain before the terminal");
20812 for pair in data.windows(2) {
20813 assert_eq!(
20814 pair[1],
20815 pair[0] + 1,
20816 "queued frames arrive dense and in order"
20817 );
20818 }
20819 let terminal = last.expect("a lagged subscriber must receive a terminal frame");
20820 assert_eq!(terminal.header.ty, FrameType::Error);
20821 assert_eq!(terminal.header.corr, 7);
20822 let body: subc_protocol::ErrorBody = serde_json::from_slice(&terminal.body).unwrap();
20823 assert_eq!(body.code, SPAWN_SUBSCRIBER_LAGGED_CODE);
20824 let detail = body.detail.expect("lagged error carries detail");
20825 assert_eq!(
20826 detail["first_undelivered_cursor"]["seq"],
20827 data.last().unwrap() + 1,
20828 "the named cursor is the first event the subscriber did not receive"
20829 );
20830 assert_eq!(
20831 detail["first_undelivered_cursor"]["daemon_incarnation"],
20832 "lag-incarnation"
20833 );
20834 }
20835}
20836
20837#[cfg(test)]
20838mod terminal_history_read_concurrency_tests {
20839 use super::*;
20840 use crate::terminal_journal::read_pause;
20841 use cortexkit_test_support::ScratchDir;
20842 use std::sync::mpsc as std_mpsc;
20843
20844 fn journaled_ring(
20845 journal: &Arc<crate::terminal_journal::TerminalJournal>,
20846 ) -> Arc<Mutex<TerminalRing>> {
20847 Arc::new(Mutex::new(
20848 TerminalRing::new(TerminalRingConfig::default(), 1)
20849 .with_journal(Some(Arc::clone(journal))),
20850 ))
20851 }
20852
20853 fn crash(at_ms: u64) -> ExitReport {
20854 ExitReport {
20855 kind: ExitKind::Crash,
20856 code: Some(1),
20857 signal: None,
20858 at_ms,
20859 }
20860 }
20861
20862 fn record_within(
20865 module_id: &'static str,
20866 ring: &Arc<Mutex<TerminalRing>>,
20867 at_ms: u64,
20868 bound: Duration,
20869 ) -> bool {
20870 let ring = Arc::clone(ring);
20871 let (done, done_rx) = std_mpsc::channel();
20872 std::thread::spawn(move || {
20873 record_terminal(
20874 module_id,
20875 &ring,
20876 &SpawnEventFeed::default(),
20877 &crash(at_ms),
20878 TerminalDisposition::Restarting,
20879 );
20880 let _ = done.send(());
20881 });
20882 done_rx.recv_timeout(bound).is_ok()
20883 }
20884
20885 #[test]
20890 fn exits_recorded_during_a_paused_history_read_are_not_blocked_or_half_merged() {
20891 let dir = ScratchDir::new("terminal-history-concurrent-read");
20892 let path = dir.join("terminals.jsonl");
20893 let journal = Arc::new(crate::terminal_journal::TerminalJournal::open(
20894 path.clone(),
20895 "daemon".into(),
20896 ));
20897 let reader_ring = journaled_ring(&journal);
20898 let other_ring = journaled_ring(&journal);
20899 assert!(record_within(
20900 "reader-module",
20901 &reader_ring,
20902 10,
20903 Duration::from_secs(5)
20904 ));
20905
20906 let (started, release) = read_pause::install(&path);
20907 let reading = {
20908 let ring = Arc::clone(&reader_ring);
20909 std::thread::spawn(move || durable_terminal_history_of(&ring, "reader-module"))
20910 };
20911 started
20912 .recv_timeout(Duration::from_secs(5))
20913 .expect("the history read reached its pause");
20914
20915 let bound = Duration::from_secs(1);
20916 assert!(
20917 record_within("other-module", &other_ring, 20, bound),
20918 "another module's exit waited on a history read (journal writer held)"
20919 );
20920 assert!(
20921 record_within("reader-module", &reader_ring, 30, bound),
20922 "the read module's own exit waited on its history read (ring held)"
20923 );
20924
20925 drop(release);
20926 let paused = reading.join().unwrap();
20927 assert_eq!(
20928 paused.entries.iter().map(|e| e.at_ms).collect::<Vec<_>>(),
20929 vec![10],
20930 "an exit recorded after the read began lands in neither half of it"
20931 );
20932 assert_eq!(paused.journal_skipped_lines, 0);
20933 assert_eq!(paused.journal_read_errors, 0);
20934
20935 let after = durable_terminal_history_of(&reader_ring, "reader-module");
20936 assert_eq!(
20937 after.entries.iter().map(|e| e.at_ms).collect::<Vec<_>>(),
20938 vec![10, 30],
20939 "the next read merges ring and journal with no duplicate"
20940 );
20941 assert_eq!(after.journal_skipped_lines, 0);
20942 }
20943}
20944
20945#[cfg(test)]
20950mod stderr_settle_tests {
20951 use std::{
20952 future::Future,
20953 io,
20954 pin::Pin,
20955 sync::{Arc, Mutex},
20956 task::{Context, Poll},
20957 time::Duration,
20958 };
20959
20960 use tokio::{
20961 io::{AsyncRead, ReadBuf},
20962 sync::oneshot,
20963 time::Instant,
20964 };
20965
20966 use super::{settle_stderr_pump, settle_stdout_pump, StderrPump};
20967 use crate::stderr_tail::{
20968 pump_stderr_to, pump_stdout_to, split_capture_stamp, untimed, CaptureState,
20969 ChildOutputSink, OutputSink, StderrRing, StderrTailConfig, TailEntry,
20970 };
20971
20972 const BOUND: Duration = Duration::from_millis(250);
20973
20974 struct HeldReader {
20978 before: Option<Vec<u8>>,
20979 gate: Option<oneshot::Receiver<()>>,
20980 after: io::Cursor<Vec<u8>>,
20981 }
20982
20983 impl AsyncRead for HeldReader {
20984 fn poll_read(
20985 mut self: Pin<&mut Self>,
20986 cx: &mut Context<'_>,
20987 buf: &mut ReadBuf<'_>,
20988 ) -> Poll<io::Result<()>> {
20989 if let Some(bytes) = self.before.take() {
20990 buf.put_slice(&bytes);
20991 return Poll::Ready(Ok(()));
20992 }
20993 if let Some(gate) = self.gate.as_mut() {
20994 match Pin::new(gate).poll(cx) {
20995 Poll::Pending => return Poll::Pending,
20996 Poll::Ready(_) => self.gate = None,
20997 }
20998 }
20999 Pin::new(&mut self.after).poll_read(cx, buf)
21000 }
21001 }
21002
21003 struct DiscardSink;
21004
21005 impl OutputSink for DiscardSink {
21006 fn write_line(&mut self, _line: &[u8]) {}
21007 }
21008
21009 fn line(text: &str) -> TailEntry {
21010 TailEntry::Line {
21011 text: text.to_string(),
21012 truncated: false,
21013 at_ms: None,
21014 }
21015 }
21016
21017 fn lock(ring: &Arc<Mutex<StderrRing>>) -> std::sync::MutexGuard<'_, StderrRing> {
21018 ring.lock().unwrap()
21019 }
21020
21021 fn held_pump(
21025 ring: &Arc<Mutex<StderrRing>>,
21026 before: &str,
21027 after: &str,
21028 ) -> (StderrPump, oneshot::Sender<()>) {
21029 let generation = lock(ring).begin_process();
21030 let (release, gate) = oneshot::channel();
21031 let reader = HeldReader {
21032 before: Some(before.as_bytes().to_vec()),
21033 gate: Some(gate),
21034 after: io::Cursor::new(after.as_bytes().to_vec()),
21035 };
21036 let task = tokio::spawn(pump_stderr_to(
21037 reader,
21038 Arc::clone(ring),
21039 generation,
21040 DiscardSink,
21041 ));
21042 (StderrPump { task, generation }, release)
21043 }
21044
21045 async fn wait_until(ring: &Arc<Mutex<StderrRing>>, done: impl Fn(&StderrRing) -> bool) {
21046 for _ in 0..1000 {
21047 if done(&lock(ring)) {
21048 return;
21049 }
21050 tokio::time::sleep(Duration::from_millis(1)).await;
21051 }
21052 panic!(
21053 "ring never reached the expected state: {:?}",
21054 lock(ring).snapshot(None, None)
21055 );
21056 }
21057
21058 #[tokio::test(start_paused = true)]
21059 async fn a_crash_line_the_reader_had_not_reached_by_the_bound_is_kept_before_the_restart() {
21060 let ring = Arc::new(Mutex::new(StderrRing::new(StderrTailConfig::default())));
21061 let (pump, release) = held_pump(&ring, "booting\n", "config error: missing storage\n");
21062
21063 settle_stderr_pump("crasher", &ring, pump, BOUND).await;
21064 let before_release = lock(&ring).snapshot(None, None);
21065 assert!(
21066 matches!(before_release.capture, CaptureState::Incomplete { .. }),
21067 "a reader that has not reached EOF cannot claim a whole tail: {before_release:?}"
21068 );
21069
21070 let next = lock(&ring).begin_process();
21073 lock(&ring).push_line_from(next, "next process booting");
21074 release.send(()).unwrap();
21075 wait_until(&ring, |ring| {
21076 ring.snapshot(None, None).capture == CaptureState::Captured
21077 })
21078 .await;
21079
21080 assert_eq!(
21081 untimed(lock(&ring).snapshot(None, None).entries),
21082 vec![
21083 line("booting"),
21084 line("config error: missing storage"),
21085 TailEntry::ProcessStart,
21086 line("next process booting"),
21087 ],
21088 "the crash's last line must survive a slow reader and stay in the crashed process's section"
21089 );
21090 }
21091
21092 #[tokio::test(start_paused = true)]
21093 async fn a_pipe_held_open_by_a_descendant_reads_incomplete_without_delaying_the_restart_past_the_bound(
21094 ) {
21095 let ring = Arc::new(Mutex::new(StderrRing::new(StderrTailConfig::default())));
21096 let (pump, _held) = held_pump(&ring, "parent exiting\n", "");
21099
21100 let started = Instant::now();
21101 settle_stderr_pump("orphaning", &ring, pump, BOUND).await;
21102 assert_eq!(
21103 started.elapsed(),
21104 BOUND,
21105 "the restart must wait exactly the bound for a pipe that stays open, no longer"
21106 );
21107
21108 let next = lock(&ring).begin_process();
21109 lock(&ring).push_line_from(next, "next process booting");
21110 tokio::time::sleep(Duration::from_secs(60)).await;
21111
21112 let snapshot = lock(&ring).snapshot(None, None);
21113 match &snapshot.capture {
21114 CaptureState::Incomplete { reason } => assert!(
21115 reason.contains("had not reached EOF") && reason.contains("250ms"),
21116 "the reason must say what is missing and after how long: {reason}"
21117 ),
21118 other => panic!("expected Incomplete while the pipe is held open, got {other:?}"),
21119 }
21120 assert_eq!(
21121 untimed(snapshot.entries),
21122 vec![
21123 line("parent exiting"),
21124 TailEntry::ProcessStart,
21125 line("next process booting"),
21126 ]
21127 );
21128 }
21129
21130 #[tokio::test(start_paused = true)]
21131 async fn a_reader_that_reaches_eof_within_the_bound_leaves_the_tail_captured() {
21132 let ring = Arc::new(Mutex::new(StderrRing::new(StderrTailConfig::default())));
21133 let (pump, release) = held_pump(&ring, "one\n", "two\n");
21134 release.send(()).unwrap();
21135
21136 settle_stderr_pump("clean", &ring, pump, BOUND).await;
21137
21138 let snapshot = lock(&ring).snapshot(None, None);
21139 assert_eq!(snapshot.capture, CaptureState::Captured);
21140 assert_eq!(untimed(snapshot.entries), vec![line("one"), line("two")]);
21141 }
21142
21143 #[tokio::test(start_paused = true)]
21144 async fn stdout_buffered_past_the_exit_drain_bound_reaches_the_capture_file() {
21145 const BURST: usize = 150;
21146 let capture = cortexkit_test_support::ScratchDir::new("stdout-exit-drain");
21147 let path = capture.join("two-pipe.stderr.log");
21148 let sink = ChildOutputSink::open(&path, cortexkit_log::Retention::default()).unwrap();
21149 let ring = Arc::new(Mutex::new(StderrRing::new(StderrTailConfig::default())));
21150 let generation = lock(&ring).begin_process();
21151 let stdout = (0..BURST)
21152 .map(|i| format!("out-{i:04}-{}\n", "o".repeat(64)))
21153 .collect::<String>();
21154 let stderr = (0..BURST)
21155 .map(|i| format!("err-{i:04}-{}\n", "e".repeat(64)))
21156 .collect::<String>();
21157 let first_chunk = stdout.find('\n').unwrap() + 1 + "out-0001-".len();
21162 let (release, gate) = oneshot::channel();
21163 let reader = HeldReader {
21164 before: Some(stdout.as_bytes()[..first_chunk].to_vec()),
21165 gate: Some(gate),
21166 after: io::Cursor::new(stdout.as_bytes()[first_chunk..stdout.len() - 1].to_vec()),
21167 };
21168 let (finished, completion) = oneshot::channel();
21169 let stdout_sink = sink.clone();
21170 let pump = tokio::spawn(async move {
21171 pump_stdout_to(reader, stdout_sink).await;
21172 finished.send(()).unwrap();
21173 });
21174 pump_stderr_to(io::Cursor::new(stderr.into_bytes()), ring, generation, sink).await;
21175
21176 let started = Instant::now();
21177 settle_stdout_pump("two-pipe", pump, BOUND).await;
21178 assert_eq!(
21179 started.elapsed(),
21180 BOUND,
21181 "a late reader must not hold up restart"
21182 );
21183 assert_eq!(
21184 std::fs::read_to_string(&path).unwrap().lines().count(),
21185 BURST + 1
21186 );
21187
21188 release
21189 .send(())
21190 .expect("stdout reader must survive the exit drain bound");
21191 tokio::time::timeout(Duration::from_secs(1), completion)
21192 .await
21193 .expect("released stdout reader must reach EOF")
21194 .expect("stdout capture must finish at EOF, not be cancelled");
21195 let contents = std::fs::read_to_string(&path).unwrap();
21196 let mut lines = contents
21197 .lines()
21198 .map(|line| split_capture_stamp(line).expect("capture stamp").1)
21199 .collect::<Vec<_>>();
21200 let mut expected = (0..BURST)
21201 .flat_map(|i| {
21202 [
21203 format!("out-{i:04}-{}", "o".repeat(64)),
21204 format!("err-{i:04}-{}", "e".repeat(64)),
21205 ]
21206 })
21207 .collect::<Vec<_>>();
21208 lines.sort_unstable();
21209 expected.sort_unstable();
21210 assert_eq!(
21211 lines, expected,
21212 "both pipes must retain every complete line exactly once"
21213 );
21214 }
21215}
21216
21217#[cfg(all(test, windows))]
21231mod job_containment_tests {
21232 use super::*;
21233 use cortexkit_test_support::ScratchDir;
21234 use std::{
21235 path::{Path, PathBuf},
21236 sync::{Arc, Mutex},
21237 time::{Duration, Instant},
21238 };
21239
21240 fn stub_path() -> PathBuf {
21246 let path = cortexkit_test_support::cargo_profile_bin("fake-aft-stub");
21247 assert!(
21248 path.exists(),
21249 "fake-aft-stub not built at {}: run `cargo test -p subc-core` (which builds \
21250 [[bin]] targets) rather than `cargo test -p subc-core --lib` (which does not)",
21251 path.display()
21252 );
21253 path
21254 }
21255
21256 fn read_grandchild_pid(path: &Path) -> u32 {
21258 let deadline = Instant::now() + Duration::from_secs(10);
21259 loop {
21260 if let Ok(contents) = std::fs::read_to_string(path) {
21261 if let Ok(pid) = contents.trim().parse() {
21262 return pid;
21263 }
21264 }
21265 assert!(
21266 Instant::now() < deadline,
21267 "the stub never recorded a grandchild pid at {}",
21268 path.display()
21269 );
21270 std::thread::sleep(Duration::from_millis(10));
21271 }
21272 }
21273
21274 struct Fixture {
21277 _dir: ScratchDir,
21278 _roster: ChildRoster,
21279 module_id: String,
21280 grandchild: u32,
21281 child: Option<SupervisedChild>,
21282 registry: Arc<Registry>,
21283 snapshot: Arc<Mutex<SupervisorSnapshot>>,
21284 terminal_ring: Arc<Mutex<TerminalRing>>,
21285 spawn_events: SpawnEventFeed,
21286 }
21287
21288 fn fixture(label: &str, module_id: &str) -> Fixture {
21289 fixture_with_carry(label, module_id, false)
21290 }
21291
21292 fn fixture_with_carry(label: &str, module_id: &str, carry_containment: bool) -> Fixture {
21293 let dir = ScratchDir::new(label);
21294 let pid_file = dir.join("grandchild.pid");
21295 let supervisor = Supervisor::new_for_test(
21296 Arc::new(Registry::default()),
21297 RestartPolicy::new(3, Duration::ZERO),
21298 );
21299 let runtime = supervisor.runtime_config();
21300 let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::starting()));
21301 let spec = ModuleSpec {
21302 script: None,
21303 module_id: module_id.to_string(),
21304 program: stub_path(),
21305 args: Vec::new(),
21309 env: vec![
21310 ("FAKE_AFT_NEVER_CONNECT".to_string(), "1".to_string()),
21311 (
21312 "FAKE_AFT_GRANDCHILD_PID_FILE".to_string(),
21313 pid_file.display().to_string(),
21314 ),
21315 ],
21316 reserved: false,
21317 reserved_prefixes: Vec::new(),
21318 protocol: ModuleProtocol::Subc,
21319 overlap: Default::default(),
21320 carry_containment,
21321 };
21322 let child = spawn_and_mark_running(&spec, &runtime, &snapshot)
21323 .expect("spawn the supervised fixture");
21324 let grandchild = read_grandchild_pid(&pid_file);
21325 Fixture {
21326 _dir: dir,
21327 _roster: runtime.child_roster.clone(),
21328 module_id: module_id.to_string(),
21329 grandchild,
21330 child: Some(child),
21331 registry: Arc::new(Registry::default()),
21332 snapshot,
21333 terminal_ring: Arc::clone(&runtime.terminal_ring),
21334 spawn_events: SpawnEventFeed::default(),
21335 }
21336 }
21337
21338 async fn assert_admission_kill_reaps_grandchild(carry: bool) {
21339 let label = if carry {
21340 "admission-kill-carried-tree"
21341 } else {
21342 "admission-kill-plain-tree"
21343 };
21344 let mut fixture = fixture_with_carry(label, label, carry);
21345 let descendant = subc_os::Process::open(fixture.grandchild)
21346 .unwrap()
21347 .expect("the stub descendant must be alive before admission kill");
21348 assert!(descendant.observe().is_some());
21349 let child = fixture.child.as_mut().unwrap();
21350 assert_eq!(child.child.container().is_some(), carry);
21351 assert_eq!(child.job.is_some(), !carry);
21352
21353 child.start_admission_kill().unwrap();
21354 assert!(child.admission_kill_sent);
21355 let status = timeout(Duration::from_secs(10), child.wait())
21356 .await
21357 .unwrap()
21358 .unwrap();
21359 assert_eq!(
21360 status.code(),
21361 Some(admission_exit::WINDOWS_ADMISSION_KILL_EXIT_CODE),
21362 "containment teardown must preserve the leader's distinctive status"
21363 );
21364 assert!(
21367 descendant.wait_for_exit(Duration::from_secs(10)).unwrap(),
21368 "admission kill left a descendant running before exit finalization"
21369 );
21370 assert_eq!(child.child.container().is_some(), carry);
21371 assert_eq!(child.job.is_some(), !carry);
21372 child.finish_exit(false).await.unwrap();
21373 }
21374
21375 #[tokio::test]
21376 async fn admission_kill_reaps_grandchild_with_plain_job() {
21377 assert_admission_kill_reaps_grandchild(false).await;
21378 }
21379
21380 #[tokio::test]
21381 async fn admission_kill_reaps_grandchild_with_carried_job() {
21382 assert_admission_kill_reaps_grandchild(true).await;
21383 }
21384
21385 #[tokio::test]
21386 async fn admission_kill_retires_registration_before_wait() {
21387 let mut fixture = fixture("admission-registration", "admission-registration");
21388 let handle = SupervisorHandle::new();
21389 let spawn =
21390 RegistrationSpawn::new(&handle, "admission-registration", SpawnRole::Plain, None);
21391 spawn.ready();
21392 let child = fixture.child.as_mut().unwrap();
21393 child.registration_spawn = Some(spawn.clone());
21394 child.start_admission_kill().unwrap();
21395 assert!(
21396 !spawn.launch_ready(),
21397 "admission kill must retire before its caller can await the child"
21398 );
21399 timeout(Duration::from_secs(10), child.wait())
21400 .await
21401 .unwrap()
21402 .unwrap();
21403 child.finish_exit(false).await.unwrap();
21404 }
21405
21406 impl Fixture {
21407 async fn drain(&mut self) {
21409 let child = self
21410 .child
21411 .take()
21412 .expect("the fixture child is still present");
21413 drain_child_to_state(
21414 &self.module_id,
21415 ModuleProtocol::Subc,
21416 StopNotice::NotSent,
21419 &self.registry,
21420 None,
21421 &self.snapshot,
21422 &self.terminal_ring,
21423 &self.spawn_events,
21424 child,
21425 Duration::from_millis(500),
21426 ModuleState::Stopped,
21427 Some(false),
21428 &FailureContext::operator(),
21429 )
21430 .await
21431 .expect("drain the supervised fixture");
21432 }
21433 }
21434
21435 #[tokio::test]
21441 async fn teardown_reaps_the_grandchild() {
21442 let mut fixture = fixture("teardown-grandchild", "tree-teardown");
21443 let grandchild = fixture.grandchild;
21444
21445 assert!(
21446 subc_jobobject::process_exists(grandchild),
21447 "grandchild {grandchild} must be alive before teardown, or this proves nothing"
21448 );
21449
21450 fixture.drain().await;
21451
21452 assert!(
21453 subc_jobobject::wait_for_process_exit(grandchild, Duration::from_secs(10)),
21454 "grandchild {grandchild} outlived module teardown: the tree was not contained"
21455 );
21456 }
21457
21458 #[test]
21471 fn an_uncontained_grandchild_survives_a_direct_child_kill() {
21472 let dir = ScratchDir::new("teardown-uncontained");
21473 let pid_file = dir.join("grandchild.pid");
21474 let mut child = std::process::Command::new(stub_path())
21475 .env("FAKE_AFT_NEVER_CONNECT", "1")
21476 .env(
21477 "FAKE_AFT_GRANDCHILD_PID_FILE",
21478 pid_file.display().to_string(),
21479 )
21480 .stdin(std::process::Stdio::null())
21481 .stdout(std::process::Stdio::null())
21482 .stderr(std::process::Stdio::null())
21483 .spawn()
21484 .expect("spawn the uncontained fixture");
21485 let grandchild = read_grandchild_pid(&pid_file);
21486
21487 child.kill().expect("kill the direct child");
21489 let _ = child.wait();
21490
21491 assert!(
21492 subc_jobobject::process_exists(grandchild),
21493 "grandchild {grandchild} died with the direct child, so this control no longer \
21494 distinguishes contained from uncontained teardown and the regression test is \
21495 passing vacuously"
21496 );
21497
21498 kill_tree(grandchild);
21501 }
21502
21503 #[tokio::test]
21512 async fn dropping_containment_reaps_the_grandchild() {
21513 let mut fixture = fixture("drop-containment", "tree-drop");
21514 let grandchild = fixture.grandchild;
21515
21516 assert!(subc_jobobject::process_exists(grandchild));
21517
21518 fixture.child.as_mut().expect("child present").job = None;
21520
21521 assert!(
21522 subc_jobobject::wait_for_process_exit(grandchild, Duration::from_secs(10)),
21523 "grandchild {grandchild} survived the containment handle closing, so a daemon \
21524 crash would leave the tree behind"
21525 );
21526 }
21527
21528 fn kill_tree(pid: u32) {
21530 let _ = std::process::Command::new("taskkill.exe")
21531 .args(["/PID", &pid.to_string(), "/T", "/F"])
21532 .stdin(std::process::Stdio::null())
21533 .stdout(std::process::Stdio::null())
21534 .stderr(std::process::Stdio::null())
21535 .status();
21536 assert!(
21537 subc_jobobject::wait_for_process_exit(pid, Duration::from_secs(10)),
21538 "could not clean up grandchild {pid}"
21539 );
21540 }
21541}
21542
21543#[cfg(test)]
21544mod privacy_trampoline_configuration_tests {
21545 #[tokio::test]
21546 #[cfg_attr(not(target_os = "macos"), ignore = "requires macOS privacy trampoline")]
21547 async fn macos_spawn_without_a_configured_trampoline_refuses_by_name() {
21548 #[cfg(target_os = "macos")]
21549 {
21550 let supervisor = super::Supervisor::new(
21551 std::sync::Arc::new(crate::Registry::default()),
21552 super::RestartPolicy::default(),
21553 );
21554 let error = supervisor.spawn(spec()).unwrap_err();
21555 assert!(
21556 error
21557 .to_string()
21558 .contains("no privacy trampoline configured"),
21559 "{error}"
21560 );
21561 }
21562 }
21563
21564 #[tokio::test]
21565 #[cfg_attr(not(target_os = "macos"), ignore = "requires macOS privacy trampoline")]
21566 async fn macos_wrong_trampoline_is_rejected_by_the_startup_probe() {
21567 #[cfg(target_os = "macos")]
21568 {
21569 let supervisor = super::Supervisor::new(
21570 std::sync::Arc::new(crate::Registry::default()),
21571 super::RestartPolicy::default(),
21572 )
21573 .with_privacy_trampoline(std::env::current_exe().unwrap());
21574 let error = supervisor.spawn(spec()).unwrap_err();
21575 assert!(
21576 error
21577 .to_string()
21578 .contains("binary does not implement the privacy trampoline protocol"),
21579 "{error}"
21580 );
21581 }
21582 }
21583
21584 #[cfg(target_os = "macos")]
21585 fn spec() -> super::ModuleSpec {
21586 super::ModuleSpec {
21587 module_id: "privacy-configuration".into(),
21588 program: "/bin/sleep".into(),
21589 args: vec!["30".into()],
21590 env: vec![],
21591 reserved: false,
21592 reserved_prefixes: vec![],
21593 protocol: subc_control::ModuleProtocol::None,
21594 overlap: super::ModuleOverlap::Exclusive,
21595 carry_containment: false,
21596 script: None,
21597 }
21598 }
21599}
21600
21601#[cfg(test)]
21602mod privacy_exec_boundary_tests {
21603 #[cfg(target_os = "macos")]
21604 use super::*;
21605 #[cfg(target_os = "macos")]
21606 use std::{
21607 io::{Read, Write},
21608 net::{TcpListener, TcpStream},
21609 };
21610
21611 #[cfg(target_os = "macos")]
21614 pub(super) fn before_image_sample(spec: &ModuleSpec, pid: u32) {
21615 if let Some((_, path)) = spec
21616 .env
21617 .iter()
21618 .find(|(key, _)| key == "SUBC_TEST_PRIVACY_SAMPLE_BARRIER")
21619 {
21620 let mut barrier = TcpStream::connect(path).unwrap();
21621 barrier
21622 .set_read_timeout(Some(Duration::from_secs(30)))
21623 .unwrap();
21624 barrier.write_all(&pid.to_ne_bytes()).unwrap();
21625 let mut release = [0];
21626 barrier.read_exact(&mut release).unwrap();
21627 assert_eq!(&release, b"X");
21628 }
21629 }
21630
21631 #[cfg(target_os = "macos")]
21632 fn spec(program: &str, args: &[&str]) -> ModuleSpec {
21633 ModuleSpec {
21634 script: None,
21635 module_id: "privacy-boundary".into(),
21636 program: program.into(),
21637 args: args.iter().map(|arg| (*arg).into()).collect(),
21638 env: vec![],
21639 reserved: false,
21640 reserved_prefixes: vec![],
21641 protocol: ModuleProtocol::None,
21642 overlap: ModuleOverlap::Exclusive,
21643 carry_containment: false,
21644 }
21645 }
21646
21647 #[cfg(target_os = "macos")]
21648 async fn accept(listener: TcpListener) -> TcpStream {
21649 let listener = tokio::net::TcpListener::from_std({
21651 listener.set_nonblocking(true).unwrap();
21652 listener
21653 })
21654 .unwrap();
21655 let (stream, _) = tokio::time::timeout(Duration::from_secs(30), listener.accept())
21656 .await
21657 .unwrap()
21658 .unwrap();
21659 let stream = stream.into_std().unwrap();
21660 stream.set_nonblocking(false).unwrap();
21661 stream
21662 .set_read_timeout(Some(Duration::from_secs(30)))
21663 .unwrap();
21664 stream
21665 }
21666
21667 #[tokio::test]
21668 #[cfg_attr(not(target_os = "macos"), ignore = "requires macOS privacy trampoline")]
21669 async fn macos_roster_withholds_a_nonnull_trampoline_image_until_exec_confirmation() {
21670 #[cfg(target_os = "macos")]
21671 {
21672 let root = cortexkit_test_support::ScratchDir::new("privacy-roster-barrier");
21673 let exec_listener = TcpListener::bind("127.0.0.1:0").unwrap();
21676 let sample_listener = TcpListener::bind("127.0.0.1:0").unwrap();
21677 let record = root.join("live-children.json");
21678 let supervisor =
21679 Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default())
21680 .with_live_children_record(&record);
21681 let runtime = supervisor.runtime_config();
21682 let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::starting()));
21683 let mut spec = spec("/bin/sleep", &["30"]);
21684 spec.env = vec![
21685 (
21686 "SUBC_TEST_PRIVACY_EXEC_BARRIER".into(),
21687 exec_listener.local_addr().unwrap().to_string(),
21688 ),
21689 (
21690 "SUBC_TEST_PRIVACY_SAMPLE_BARRIER".into(),
21691 sample_listener.local_addr().unwrap().to_string(),
21692 ),
21693 ];
21694 let spawn = tokio::task::spawn_blocking(move || {
21695 spawn_and_mark_running(&spec, &runtime, &snapshot).unwrap()
21696 });
21697 let mut sample = accept(sample_listener).await;
21698 let mut pid = [0; 4];
21699 sample.read_exact(&mut pid).unwrap();
21700 let pid = u32::from_ne_bytes(pid);
21701 let mut exec = accept(exec_listener).await;
21702 let mut ready = [0];
21703 exec.read_exact(&mut ready).unwrap();
21704 assert_eq!(&ready, b"R");
21705 let trampoline = subc_os::file_identity(&test_privacy_trampoline()).unwrap();
21708 assert_eq!(
21709 observe_spawned_image(pid).unwrap().executable,
21710 Some(trampoline)
21711 );
21712 sample.write_all(b"X").unwrap();
21713 let mut child = spawn.await.unwrap();
21714 let early = crate::live_children::read_record(&record).unwrap();
21715 assert_eq!(early.len(), 1);
21716 assert_eq!(early[0].pid, pid);
21717 assert_eq!(
21718 early[0].executable, None,
21719 "unconfirmed trampoline image entered the roster"
21720 );
21721 assert!(child.report_ready.get().is_none());
21722 child.privacy_exec.as_mut().unwrap().deadline =
21725 tokio::time::Instant::now() + Duration::from_secs(30);
21726 exec.write_all(b"X").unwrap();
21727 child.confirm_privacy_exec().await;
21728 assert_eq!(child.spawn_failure, None);
21729 assert!(child.report_ready.get().is_some());
21730 let confirmed = crate::live_children::read_record(&record).unwrap();
21731 let module = subc_os::file_identity(std::path::Path::new("/bin/sleep")).unwrap();
21732 assert_ne!(module, trampoline);
21733 assert_eq!(confirmed[0].executable, Some(module.into()));
21734 child.start_kill().unwrap();
21735 child.wait().await.unwrap();
21736 child.release_roster();
21737 }
21738 }
21739
21740 #[tokio::test]
21741 #[cfg_attr(not(target_os = "macos"), ignore = "requires macOS privacy trampoline")]
21742 async fn macos_unstamped_reexec_records_the_observed_native_image() {
21743 #[cfg(target_os = "macos")]
21744 {
21745 let root = cortexkit_test_support::ScratchDir::new("privacy-reexec");
21746 let record = root.join("live-children.json");
21747 let supervisor =
21748 Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default())
21749 .with_live_children_record(&record);
21750 let runtime = supervisor.runtime_config();
21751 let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::starting()));
21752 let fixture = crate::config_gate::program::native_fixture_for_test();
21753 let mut spec = spec(
21754 "/bin/sh",
21755 &["-c", "exec \"$1\" 600", "reexec", fixture.to_str().unwrap()],
21756 );
21757 spec.env
21758 .push(("SUBC_TEST_PRIVACY_IMAGE_DELAY_MS".into(), "5".into()));
21759 let mut child = spawn_and_mark_running(&spec, &runtime, &snapshot).unwrap();
21760 let native_image = subc_os::file_identity(&fixture).unwrap();
21761 let reexec = tokio::time::timeout(Duration::from_secs(5), async {
21764 while observe_spawned_image(child.pid).and_then(|image| image.executable)
21765 != Some(native_image)
21766 {
21767 tokio::time::sleep(Duration::from_millis(1)).await;
21768 }
21769 })
21770 .await;
21771 child.confirm_privacy_exec().await;
21772 let failure = child.spawn_failure.clone();
21773 let ready = child.report_ready.get().is_some();
21774 let confirmed = crate::live_children::read_record(&record).unwrap();
21775 child.start_kill().unwrap();
21776 tokio::time::timeout(Duration::from_secs(5), child.wait())
21777 .await
21778 .unwrap()
21779 .unwrap();
21780 child.release_roster();
21781
21782 reexec.expect("the shell must re-exec the native fixture");
21783 assert_eq!(
21784 failure, None,
21785 "an unstamped program may re-exec before observation"
21786 );
21787 assert!(ready);
21788 assert_eq!(confirmed.len(), 1);
21789 assert_eq!(confirmed[0].executable, Some(native_image.into()));
21790 }
21791 }
21792
21793 #[tokio::test]
21794 #[cfg_attr(not(target_os = "macos"), ignore = "requires macOS privacy trampoline")]
21795 async fn macos_unstamped_ack_without_exec_still_refuses_the_trampoline() {
21796 #[cfg(target_os = "macos")]
21797 {
21798 let listener = TcpListener::bind("127.0.0.1:0").unwrap();
21799 let supervisor =
21800 Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default());
21801 let runtime = supervisor.runtime_config();
21802 let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::starting()));
21803 let fixture = crate::config_gate::program::native_fixture_for_test();
21804 let mut spec = spec(fixture.to_str().unwrap(), &["600"]);
21805 spec.env.push((
21806 "SUBC_TEST_PRIVACY_EXEC_BARRIER".into(),
21807 listener.local_addr().unwrap().to_string(),
21808 ));
21809 let mut child = spawn_and_mark_running(&spec, &runtime, &snapshot).unwrap();
21810 let mut exec = accept(listener).await;
21811 let mut ready = [0];
21812 exec.read_exact(&mut ready).unwrap();
21813 assert_eq!(&ready, b"R");
21814 assert_eq!(
21815 observe_spawned_image(child.pid).unwrap().executable,
21816 subc_os::file_identity(&test_privacy_trampoline())
21817 );
21818 let (reader, writer) = std::io::pipe().unwrap();
21821 drop(writer);
21822 child.privacy_exec.as_mut().unwrap().reader =
21823 tokio::io::unix::AsyncFd::new(reader).unwrap();
21824 child.confirm_privacy_exec().await;
21825 let failure = child.spawn_failure.clone();
21826 let ready = child.report_ready.get().is_some();
21827 child.start_kill().unwrap();
21828 tokio::time::timeout(Duration::from_secs(5), child.wait())
21829 .await
21830 .unwrap()
21831 .unwrap();
21832 child.release_roster();
21833 assert_eq!(
21834 failure.as_deref(),
21835 Some("privacy identity trampoline did not exec the module")
21836 );
21837 assert!(!ready);
21838 }
21839 }
21840
21841 #[tokio::test]
21842 #[cfg_attr(not(target_os = "macos"), ignore = "requires macOS privacy trampoline")]
21843 async fn macos_resumed_121_is_a_module_exit_not_a_trampoline_refusal() {
21844 #[cfg(target_os = "macos")]
21845 {
21846 let registry = Arc::new(Registry::default());
21847 let policy = RestartPolicy::new(0, Duration::ZERO);
21848 let fixture = crate::config_gate::program::native_fixture_for_test();
21849 let spec = spec(fixture.to_str().unwrap(), &["exit121"]);
21850 let scratch = cortexkit_test_support::ScratchDir::new("resumed-exit");
21851 let supervisor = Supervisor::new_for_test(Arc::clone(®istry), policy)
21852 .with_config_runtime(suspended_spawn_tests::gated_owner(&scratch, &spec, true));
21853 let runtime = supervisor.runtime_config();
21854 let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::starting()));
21855 let mut child = spawn_and_mark_running(&spec, &runtime, &snapshot).unwrap();
21858 child.confirm_privacy_exec().await;
21859 assert_eq!(child.spawn_failure, None);
21860 assert!(child.report_ready.get().is_some());
21861 let pid = child.pid;
21862 tokio::task::spawn_blocking(move || {
21863 subc_os::fork_exec_test::wait_for_child_exit_without_reaping(pid)
21864 })
21865 .await
21866 .unwrap()
21867 .unwrap();
21868 let status = child.wait().await.unwrap();
21869 assert_eq!(status.code(), Some(121));
21870 assert!(child.privacy_exec.is_none());
21871 let report = classify_reaped_child_exit(&snapshot, &child, &status);
21872 on_child_exit(
21873 &spec,
21874 policy,
21875 ®istry,
21876 &snapshot,
21877 &runtime.terminal_ring,
21878 &runtime.spawn_events,
21879 &runtime.child_roster,
21880 report,
21881 true,
21882 )
21883 .await;
21884 let state = lock_snapshot(&snapshot).unwrap();
21885 assert_eq!(state.state, ModuleState::Failed);
21886 assert_eq!(state.last_exit.as_ref().unwrap().code, Some(121));
21887 assert_eq!(state.reported_pid(), None);
21888 drop(state);
21889 let history = runtime.terminal_ring.lock().unwrap().snapshot();
21890 assert_eq!(history.entries.len(), 1);
21891 let terminal = &history.entries[0];
21892 assert_eq!(terminal.exit_code, Some(121));
21893 assert_eq!(terminal.exit_kind, subc_control::TerminalExitKind::Crash);
21894 assert_eq!(terminal.disposition, TerminalDisposition::Failed);
21895 assert_eq!(
21896 terminal.disposition_detail.as_deref(),
21897 Some(policy.budget_exhausted_detail().as_str()),
21898 "module exit 121 was classified as a trampoline refusal: {terminal:?}"
21899 );
21900 assert_eq!(child.spawn_failure, None);
21901 child.release_roster();
21902 }
21903 }
21904}
21905
21906#[cfg(all(test, target_os = "macos"))]
21907#[path = "supervise/suspended_spawn_tests.rs"]
21908mod suspended_spawn_tests;
21909
21910#[cfg(all(test, unix))]
21914mod launch_nonce_descriptor_tests {
21915 use super::{ChildRoster, ModuleProtocol, ModuleSpec, SupervisorHandle};
21916 use crate::stderr_tail::{StderrRing, StderrTailConfig};
21917 use cortexkit_test_support::ScratchDir;
21918 use std::{
21919 path::PathBuf,
21920 sync::{Arc, Mutex},
21921 time::{Duration, Instant},
21922 };
21923
21924 async fn probe(role: super::SpawnRole) {
21925 let scratch = ScratchDir::new("launch-nonce-descriptor");
21926 let fd_copy = scratch.join("from-descriptor");
21927 let env_copy = scratch.join("environment");
21928 let script = format!(
21929 "env > '{env}.tmp' && mv '{env}.tmp' '{env}'; cat <&3 > '{fd}.tmp' && mv '{fd}.tmp' '{fd}'; sleep 30",
21930 fd = fd_copy.display(), env = env_copy.display(),
21931 );
21932 let xdg = |name: &str| (name.to_string(), scratch.join(name).display().to_string());
21933 let spec = ModuleSpec {
21934 script: None,
21935 module_id: "nonce-descriptor-probe".to_string(),
21936 program: PathBuf::from("/bin/sh"),
21937 args: vec!["-c".to_string(), script],
21938 env: vec![
21939 xdg("XDG_DATA_HOME"),
21940 xdg("XDG_RUNTIME_DIR"),
21941 xdg("XDG_CONFIG_HOME"),
21942 ("SUBC_LAUNCH_NONCE".to_string(), "must-not-leak".to_string()),
21943 ],
21944 reserved: true,
21945 reserved_prefixes: Vec::new(),
21946 protocol: ModuleProtocol::Subc,
21947 overlap: Default::default(),
21948 carry_containment: false,
21949 };
21950 let handle = SupervisorHandle::new();
21951 let ring = Arc::new(Mutex::new(StderrRing::new(StderrTailConfig::default())));
21952 let roster = ChildRoster::default();
21953 #[cfg(target_os = "macos")]
21954 {
21955 let path = super::test_privacy_trampoline();
21956 roster.set_privacy_trampoline(super::probe_privacy_trampoline(&path).map(|()| path));
21957 }
21958 let child = super::spawn_child_in_slot(
21959 &spec,
21960 None,
21961 Some(&handle),
21962 &ring,
21963 None,
21964 &roster,
21965 None,
21966 #[cfg(target_os = "linux")]
21967 None,
21968 role,
21969 matches!(role, super::SpawnRole::SwapCandidate),
21970 super::DEFAULT_REGISTER_TIMEOUT,
21971 )
21972 .expect("spawn probe");
21973 let deadline = Instant::now() + Duration::from_secs(10);
21974 while !(fd_copy.exists() && env_copy.exists()) {
21975 assert!(Instant::now() < deadline, "probe never wrote its copies");
21976 tokio::time::sleep(Duration::from_millis(20)).await;
21977 }
21978 let nonce = std::fs::read_to_string(fd_copy).unwrap();
21979 assert!(!nonce.is_empty());
21980 let environment = std::fs::read_to_string(env_copy).unwrap();
21981 assert!(environment
21982 .lines()
21983 .any(|line| line.starts_with("SUBC_LAUNCH_NONCE_FD=3:")));
21984 let copy = environment
21985 .lines()
21986 .find_map(|line| line.strip_prefix("SUBC_LAUNCH_NONCE="));
21987 assert_eq!(
21988 copy, None,
21989 "Unix children must never receive the environment nonce"
21990 );
21991 if matches!(role, super::SpawnRole::Plain) {
21992 assert_eq!(
21993 handle.spawn_nonce(&spec.module_id).as_deref(),
21994 Some(nonce.as_str())
21995 );
21996 }
21997 drop(child);
21998 }
21999
22000 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
22001 async fn a_spawned_module_receives_its_nonce_only_on_descriptor_3() {
22002 probe(super::SpawnRole::Plain).await;
22003 }
22004
22005 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
22006 async fn swap_candidate_receives_its_nonce_only_on_descriptor_3() {
22007 probe(super::SpawnRole::SwapCandidate).await;
22008 }
22009}
22010
22011#[cfg(all(test, target_os = "linux"))]
22012#[path = "supervise/cgroup_reap_tests.rs"]
22013mod cgroup_reap_tests;
22014
22015#[cfg(all(test, target_os = "linux"))]
22016mod cgroup_containment_tests {
22017 use super::*;
22018 use cortexkit_test_support::ScratchDir;
22019
22020 #[tokio::test]
22021 async fn linux_subtree_memory_covers_module_children() {
22022 let dir = ScratchDir::new("subtree-memory");
22023 let root = PathBuf::from(format!(
22024 "/sys/fs/cgroup/subc-memory-test-{}-{}",
22025 std::process::id(),
22026 unix_ms_now()
22027 ));
22028 if let Err(error) = std::fs::create_dir(&root) {
22029 assert!(
22030 std::env::var_os("SUBC_REQUIRE_CGROUP_TEST").is_none(),
22031 "required cgroup test cannot execute: {error}"
22032 );
22033 eprintln!("SKIP linux_subtree_memory_covers_module_children: {error}");
22034 return;
22035 }
22036 let placement = subc_cgroup::prepare_at(&root)
22037 .unwrap()
22038 .expect("delegated cgroup");
22039 let supervisor = Supervisor::new_for_test(
22040 Arc::new(Registry::default()),
22041 RestartPolicy::new(3, Duration::ZERO),
22042 )
22043 .with_cgroup_placement(Some(placement));
22044 let ready = dir.join("ready");
22045 let module = supervisor
22046 .spawn(ModuleSpec {
22047 script: None,
22048 module_id: "memory-tree".into(),
22049 program: PathBuf::from("/bin/sh"),
22050 args: vec![
22051 "-c".into(),
22052 "sleep 600 & sleep 600 & sleep 600 & echo ready > \"$1\"; wait".into(),
22053 "fixture".into(),
22054 ready.display().to_string(),
22055 ],
22056 env: ["XDG_DATA_HOME", "XDG_RUNTIME_DIR", "XDG_CONFIG_HOME"]
22057 .into_iter()
22058 .map(|key| (key.into(), dir.display().to_string()))
22059 .collect(),
22060 reserved: false,
22061 reserved_prefixes: Vec::new(),
22062 protocol: ModuleProtocol::None,
22063 overlap: Default::default(),
22064 carry_containment: false,
22065 })
22066 .unwrap();
22067 let deadline = tokio::time::Instant::now() + Duration::from_secs(10);
22068 while !ready.exists() {
22069 assert!(
22070 tokio::time::Instant::now() < deadline,
22071 "children never started"
22072 );
22073 tokio::time::sleep(Duration::from_millis(10)).await;
22074 }
22075 let readings = crate::subtree_resources::read(vec![module.subtree_target()]).await;
22076 let reading = &readings[0];
22077 assert!(
22078 reading.process_count.is_some_and(|count| count >= 4),
22079 "{reading:?}"
22080 );
22081 assert!(
22082 reading.memory_bytes.is_some_and(|bytes| bytes > 0),
22083 "{reading:?}"
22084 );
22085 module.stop().await.unwrap();
22086 drop(module);
22087 std::fs::remove_dir(root.join("subc-modules")).unwrap();
22088 std::fs::remove_dir(&root).unwrap();
22089 eprintln!("EXECUTED linux_subtree_memory_covers_module_children: {reading:?}");
22090 }
22091
22092 fn running(pid: u32) -> bool {
22093 std::fs::read_to_string(format!("/proc/{pid}/stat"))
22095 .ok()
22096 .and_then(|stat| {
22097 stat.rsplit_once(") ")
22098 .map(|(_, rest)| rest.starts_with('Z'))
22099 })
22100 .is_some_and(|zombie| !zombie)
22101 }
22102
22103 #[tokio::test]
22104 async fn linux_teardown_reaps_the_grandchild() {
22105 teardown_tree("linux_teardown_reaps_the_grandchild", false).await;
22106 }
22107
22108 #[tokio::test]
22109 async fn linux_shutdown_straggler_reaps_the_grandchild() {
22110 teardown_tree("linux_shutdown_straggler_reaps_the_grandchild", true).await;
22111 }
22112
22113 async fn teardown_tree(test_name: &str, shutdown: bool) {
22114 let dir = ScratchDir::new(test_name);
22115 let root = PathBuf::from(format!(
22116 "/sys/fs/cgroup/subc-tree-test-{test_name}-{}-{}",
22117 std::process::id(),
22118 unix_ms_now()
22119 ));
22120 if let Err(error) = std::fs::create_dir(&root) {
22121 assert!(
22122 std::env::var_os("SUBC_REQUIRE_CGROUP_TEST").is_none(),
22123 "required cgroup test cannot execute: {error}"
22124 );
22125 eprintln!(
22126 "SKIP {test_name}: no writable delegated cgroup at {}: {error}",
22127 root.display()
22128 );
22129 return;
22130 }
22131 let placement = subc_cgroup::prepare_at(&root)
22132 .expect("prepare isolated kernel cgroup")
22133 .expect("isolated cgroup is delegated");
22134 let module_id = "tree-teardown";
22135 let module = placement
22136 .module_path(module_id)
22137 .expect("create isolated module cgroup");
22138 if !module.join("cgroup.kill").exists() {
22139 std::fs::remove_dir(&module).unwrap();
22140 std::fs::remove_dir(root.join("subc-modules")).unwrap();
22141 std::fs::remove_dir(&root).unwrap();
22142 assert!(
22143 std::env::var_os("SUBC_REQUIRE_CGROUP_TEST").is_none(),
22144 "required cgroup.kill interface unavailable"
22145 );
22146 eprintln!("SKIP {test_name}: cgroup.kill unavailable (kernel < 5.14)");
22147 return;
22148 }
22149 let supervisor = Supervisor::new_for_test(
22150 Arc::new(Registry::default()),
22151 RestartPolicy::new(3, Duration::ZERO),
22152 )
22153 .with_cgroup_placement(Some(placement));
22154 let mut runtime = supervisor.runtime_config();
22155 runtime.child_roster = runtime
22156 .child_roster
22157 .for_module(Arc::new(Mutex::new(Duration::from_millis(100))));
22158 let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::starting()));
22159 let pid_file = dir.join("grandchild.pid");
22160 let spec = ModuleSpec {
22161 script: None,
22162 module_id: module_id.to_string(),
22163 program: PathBuf::from("/bin/sh"),
22164 args: vec![
22165 "-c".into(),
22166 "trap '' TERM; sleep 600 & echo $! > \"$1\"; wait".into(),
22167 "fixture".into(),
22168 pid_file.display().to_string(),
22169 ],
22170 env: ["XDG_DATA_HOME", "XDG_RUNTIME_DIR", "XDG_CONFIG_HOME"]
22171 .into_iter()
22172 .map(|key| (key.to_string(), dir.display().to_string()))
22173 .collect(),
22174 reserved: false,
22175 reserved_prefixes: Vec::new(),
22176 protocol: ModuleProtocol::None,
22177 overlap: Default::default(),
22178 carry_containment: false,
22179 };
22180 let child =
22181 spawn_and_mark_running(&spec, &runtime, &snapshot).expect("spawn supervised tree");
22182 let deadline = tokio::time::Instant::now() + Duration::from_secs(10);
22183 let grandchild: u32 = loop {
22184 if let Ok(contents) = std::fs::read_to_string(&pid_file) {
22185 if let Ok(pid) = contents.trim().parse() {
22186 break pid;
22187 }
22188 }
22189 assert!(
22190 tokio::time::Instant::now() < deadline,
22191 "grandchild pid was not recorded"
22192 );
22193 tokio::time::sleep(Duration::from_millis(10)).await;
22194 };
22195 assert!(
22196 running(grandchild),
22197 "grandchild must be alive before teardown"
22198 );
22199 if shutdown {
22200 let mut child = child;
22201 crate::child_roster::end_children_for_daemon_shutdown(
22202 &runtime.child_roster,
22203 false,
22204 std::future::pending(),
22205 )
22206 .await;
22207 child.wait().await.expect("reap shutdown straggler");
22208 } else {
22209 drain_child_to_state(
22210 module_id,
22211 ModuleProtocol::None,
22212 StopNotice::NotSent,
22213 &Registry::default(),
22214 None,
22215 &snapshot,
22216 &runtime.terminal_ring,
22217 &SpawnEventFeed::default(),
22218 child,
22219 Duration::from_millis(100),
22220 ModuleState::Stopped,
22221 Some(false),
22222 &FailureContext::operator(),
22223 )
22224 .await
22225 .expect("real supervisor teardown");
22226 }
22227 let deadline = tokio::time::Instant::now() + Duration::from_secs(5);
22228 while running(grandchild) && tokio::time::Instant::now() < deadline {
22229 tokio::time::sleep(Duration::from_millis(10)).await;
22230 }
22231 let survived = running(grandchild);
22232 if survived {
22234 let pid = rustix::process::Pid::from_raw(grandchild as i32).unwrap();
22235 let _ = rustix::process::kill_process(pid, rustix::process::Signal::KILL);
22236 tokio::time::sleep(Duration::from_millis(100)).await;
22237 }
22238 if module.exists() {
22239 std::fs::remove_dir(&module).expect("remove empty module cgroup");
22240 }
22241 std::fs::remove_dir(root.join("subc-modules")).unwrap();
22242 std::fs::remove_dir(&root).unwrap();
22243 assert!(
22244 !survived,
22245 "grandchild {grandchild} outlived module teardown"
22246 );
22247 eprintln!("EXECUTED {test_name}: grandchild {grandchild} killed in isolated cgroup");
22248 }
22249}
22250
22251#[cfg(all(test, unix))]
22254mod record_write_stall_tests {
22255 use super::*;
22256 use std::sync::atomic::{AtomicBool, Ordering};
22257 use subc_transport::{authenticate_client, ConnectionInfo, Endpoint, SCHEMA_VERSION};
22258
22259 const MODULE_AUTH_DEADLINE: Duration = Duration::from_secs(2);
22262 const HELD_RECORD_WRITE: Duration = Duration::from_secs(3);
22266
22267 fn sleeping_spec() -> ModuleSpec {
22279 ModuleSpec {
22280 script: None,
22281 module_id: "record-write-stall".into(),
22282 program: "/bin/sleep".into(),
22283 args: vec!["600".into()],
22284 env: vec![],
22285 reserved: false,
22286 reserved_prefixes: vec![],
22287 protocol: ModuleProtocol::None,
22288 overlap: Default::default(),
22289 carry_containment: false,
22290 }
22291 }
22292
22293 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
22294 async fn module_auth_completes_while_the_restart_waits_on_the_record_write() {
22295 let home = cortexkit_test_support::ScratchDir::new("record-write-stall");
22296 let supervisor =
22297 Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default())
22298 .with_live_children_record(home.join("live-children.json"));
22299 let module = supervisor.spawn(sleeping_spec()).unwrap();
22300 let old_pid = tokio::time::timeout(Duration::from_secs(10), async {
22305 loop {
22306 if let Some(pid) = lock_snapshot(&module.inner.snapshot)
22307 .unwrap()
22308 .reported_pid()
22309 {
22310 return pid;
22311 }
22312 tokio::time::sleep(Duration::from_millis(10)).await;
22313 }
22314 })
22315 .await
22316 .expect("the first process was admitted as the module");
22317
22318 let key = vec![0x42; 32];
22321 let daemon_id = [0x24; 16];
22322 let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
22323 let port = listener.local_addr().unwrap().port();
22324 let server = tokio::spawn(crate::server::serve_listener(
22325 listener,
22326 Arc::new(crate::router::Router::with_default_self_handler()),
22327 crate::server::ServerAuth::with_limits(
22328 key.clone(),
22329 daemon_id,
22330 "test",
22331 crate::server::DEFAULT_AUTH_DEADLINE,
22332 4,
22333 ),
22334 ));
22335
22336 let (held_tx, held_rx) = std::sync::mpsc::channel();
22339 let armed = AtomicBool::new(true);
22340 supervisor
22341 .child_roster
22342 .set_record_write_hook_for_test(move |live| {
22343 let admitting_replacement = live.values().any(|entry| entry.pid != old_pid);
22344 if admitting_replacement && armed.swap(false, Ordering::SeqCst) {
22345 let _ = held_tx.send(());
22346 std::thread::sleep(HELD_RECORD_WRITE);
22347 }
22348 });
22349
22350 let module_process = std::thread::spawn(move || {
22353 held_rx
22354 .recv_timeout(Duration::from_secs(30))
22355 .expect("the restart reached the replacement's record write");
22356 let conn = ConnectionInfo {
22357 schema: SCHEMA_VERSION,
22358 wire_version: None,
22359 endpoints: vec![Endpoint {
22360 host: "127.0.0.1".to_owned(),
22361 port,
22362 }],
22363 key,
22364 daemon_id,
22365 pid: std::process::id(),
22366 daemon_ver: "test".to_owned(),
22367 };
22368 tokio::runtime::Builder::new_current_thread()
22369 .enable_all()
22370 .build()
22371 .unwrap()
22372 .block_on(async {
22373 let started = Instant::now();
22374 let mut stream = tokio::net::TcpStream::connect(("127.0.0.1", port))
22375 .await
22376 .unwrap();
22377 let result =
22378 authenticate_client(&mut stream, &conn, MODULE_AUTH_DEADLINE).await;
22379 (result, started.elapsed())
22380 })
22381 });
22382
22383 module.restart(Some(0)).await.unwrap();
22384 let (auth, elapsed) = tokio::task::spawn_blocking(move || module_process.join().unwrap())
22385 .await
22386 .unwrap();
22387 assert!(
22388 auth.is_ok(),
22389 "the daemon must answer a module's handshake while a restart waits on the \
22390 live-children record write; got {auth:?} after {elapsed:?}"
22391 );
22392
22393 tokio::time::timeout(Duration::from_secs(20), async {
22394 while lock_snapshot(&module.inner.snapshot)
22395 .unwrap()
22396 .reported_pid()
22397 .is_none_or(|pid| pid == old_pid)
22398 {
22399 tokio::time::sleep(Duration::from_millis(10)).await;
22400 }
22401 })
22402 .await
22403 .expect("the restart published its replacement");
22404 module.stop().await.unwrap();
22405 server.abort();
22406 }
22407}
22408
22409#[cfg(all(test, unix))]
22410mod retained_status_tests {
22411 use super::*;
22412
22413 #[tokio::test]
22414 async fn cached_container_block_is_visible_in_status_and_health_without_rechecking() {
22415 use crate::live_children::{ContainerIdentity, RetainedReason};
22416 for (reason, carried, expected) in [
22417 (
22418 RetainedReason::ContainerSurvivors(vec![4242]),
22419 true,
22420 Some("waiting: container survivors (pids [4242])".to_owned()),
22421 ),
22422 (
22423 RetainedReason::ContainerUnverifiable("member inspection denied".into()),
22424 true,
22425 Some(
22426 "waiting: container survivors could not be verified: member inspection denied"
22427 .to_owned(),
22428 ),
22429 ),
22430 (RetainedReason::Survived, false, None),
22431 ] {
22432 let supervisor =
22433 Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default());
22434 let spec = ModuleSpec {
22435 script: None,
22436 module_id: "retained-status-fixture".into(),
22437 program: PathBuf::from("/not-spawned"),
22438 args: Vec::new(),
22439 env: Vec::new(),
22440 reserved: false,
22441 reserved_prefixes: Vec::new(),
22442 protocol: ModuleProtocol::None,
22443 overlap: ModuleOverlap::Exclusive,
22444 carry_containment: carried,
22445 };
22446 let module = supervisor
22447 .supervise_configured(spec.clone(), false)
22448 .unwrap();
22449 let guard = supervisor
22453 .child_roster
22454 .admit(
22455 spec.module_id,
22456 0,
22457 ModuleProtocol::None,
22458 None,
22459 crate::child_roster::RecordedIdentity {
22460 container: carried.then_some(ContainerIdentity {
22461 pgid: Some(0),
22462 created_start_time: Some(0),
22463 cgroup_path: None,
22464 ..ContainerIdentity::test_metadata()
22465 }),
22466 ..crate::child_roster::RecordedIdentity::default()
22467 },
22468 )
22469 .unwrap();
22470 guard.retain(reason);
22471 let status = module.status().unwrap();
22472 assert_eq!(status.waiting_for_previous_detail, expected.clone());
22473 assert_eq!(status.health.detail, expected);
22474 assert!(!status.process_alive);
22475 }
22476 }
22477}
22478
22479fn registration_child_is_running(child: &mut SupervisedChild) -> bool {
22480 child.try_wait().is_ok_and(|status| status.is_none())
22481}