1#[cfg(all(test, target_os = "linux"))]
6#[path = "supervise/audit_identity_tests.rs"]
7mod audit_identity_tests;
8mod resource_launch;
9pub(crate) use resource_launch::{ResourceCommitError, ResourceLaunch};
10
11use std::{
12 collections::{HashMap, HashSet, VecDeque},
13 error::Error,
14 fmt, io,
15 path::PathBuf,
16 process::{ExitStatus, Stdio},
17 sync::{Arc, Mutex, OnceLock},
18 time::{Duration, SystemTime, UNIX_EPOCH},
19};
20
21use cortexkit_log::Retention;
22use serde_json::Value;
23use sha2::{Digest, Sha256};
24use subc_control::{
25 ClientControlPush, LiveSpawn, ModuleProtocol, RouteCloseReason, SpawnCursor, SpawnEvent,
26 SpawnEventKind, SpawnSnapshot, SupervisorHealthStatus, TerminalDisposition, TerminalExitKind,
27};
28use subc_protocol::{
29 manifest::{SelfSignalKind, SignalAnchor},
30 session::{
31 HealthReport, HealthStatus, ModuleControlCommand, ModuleControlRequest,
32 MODULE_CONTROL_OP_HEALTH_CHECK,
33 },
34 Flags, FrameType, Principal, Priority, SUBC_LAUNCH_NONCE_ENV, SUBC_MODULE_ID_ENV,
35};
36use tokio::{
37 process::{Child, Command},
38 sync::{mpsc, oneshot, watch, Mutex as AsyncMutex},
39 task::JoinHandle,
40 time::{sleep, sleep_until, timeout, timeout_at, Instant},
41};
42use tracing::{debug, error, info, warn};
43
44use crate::{
45 child_roster::ChildRoster,
46 daemon_config::{
47 CAPTURE_KEEP_ENV, CAPTURE_MAX_AGE_DAYS_ENV, CAPTURE_MAX_FILE_MB_ENV, CK_LOG_ENV,
48 },
49 forwarding::{
50 CloseReason, ForwardingError, ForwardingTable, GoodbyeTarget, ModuleControlRpcOutcome,
51 ModuleDrainTarget, PendingModuleControlRpc,
52 },
53 provenance::{ExecutableIdentityProbe, SpawnedFileIdentity},
54 registry::{ConnectionId, RegistrationEndReason, RegistryError},
55 stderr_tail::{
56 pump_stderr_to, pump_stdout_to, ChildOutputSink, StderrRing, StderrTailConfig,
57 StderrTailSnapshot,
58 },
59 terminal_ring::{TerminalHistorySnapshot, TerminalRecord, TerminalRing, TerminalRingConfig},
60 Frame, FrameSink, Registry,
61};
62
63#[cfg(not(windows))]
64use crate::provenance::spawned_file_identity;
65
66#[path = "supervise_swap.rs"]
67mod swap;
68
69#[path = "supervise_activity.rs"]
70mod activity;
71
72#[cfg(unix)]
73#[path = "supervise/admission.rs"]
74mod admission;
75mod admission_exit;
76#[cfg(all(test, target_os = "linux"))]
77mod async_spawn_tests;
78#[cfg(all(test, windows))]
79mod carried_job_tests;
80mod config_delivery;
81mod content_binding;
82mod program_binding;
83mod registration;
84#[cfg(all(test, unix))]
85pub(crate) use registration::RegistrationProof;
86pub(crate) use registration::RegistrationSpawn;
87pub(crate) use registration::REGISTRATION_CHANNEL_CLOSED;
88#[cfg(all(test, unix))]
89pub(crate) mod unix_channel_placement_tests;
90use program_binding::ProgramBindingPublication;
91mod config_exit_failure;
92#[cfg(all(test, any(target_os = "linux", windows)))]
93mod missing_path_tests;
94#[cfg(all(test, unix))]
95mod program_binding_tests;
96#[cfg(all(test, windows))]
97mod program_binding_windows_tests;
98#[cfg(all(test, windows))]
99mod windows_channel_tests;
100pub use config_exit_failure::ModuleStopReason;
101#[cfg(all(test, unix))]
102mod config_exit_hold_tests;
103pub(crate) mod containment;
104#[cfg(all(test, unix))]
105mod containment_tests;
106mod slow_retry;
107use slow_retry::{FailedRetry, FailureContext};
108mod host_health;
109use host_health::HostHealth;
110
111#[cfg(all(test, unix))]
112mod slow_retry_tests;
113
114pub const SUBC_ARG: &str = "--subc";
120
121const DEFAULT_MAX_RESTARTS: u32 = 3;
122const DEFAULT_BACKOFF: Duration = Duration::from_millis(100);
123const DEFAULT_MAX_BACKOFF: Duration = Duration::from_secs(30);
124const DEFAULT_RESTART_WINDOW: Duration = Duration::from_secs(600);
128pub const DEFAULT_DRAIN_TIMEOUT: Duration = Duration::from_secs(30);
139pub(crate) const DEFAULT_REGISTER_TIMEOUT: Duration = Duration::from_secs(120);
140const REGISTRY_RELEASE_TIMEOUT: Duration = Duration::from_secs(1);
141const REGISTRY_RELEASE_POLL: Duration = Duration::from_millis(10);
142const DRAIN_BUSY_REPROBE_INTERVAL: Duration = Duration::from_millis(250);
154const STDERR_PUMP_DRAIN_TIMEOUT: Duration = Duration::from_millis(250);
177pub const SPAWN_EVENT_RING_CAPACITY: usize = 4096;
179const SPAWN_SUBSCRIBER_BUFFER: usize = SPAWN_EVENT_RING_CAPACITY + 1;
180pub(crate) const SPAWN_SUBSCRIBER_LAGGED_CODE: &str = "spawn_subscriber_lagged";
185
186#[cfg(target_os = "macos")]
187struct PrivacyExec {
188 content_check: Option<tokio::task::JoinHandle<Result<(), crate::config_gate::GateError>>>,
189 gated: bool,
190 admission_binding: Option<ProgramBindingPublication>,
191 reader: tokio::io::unix::AsyncFd<std::io::PipeReader>,
192 deadline: tokio::time::Instant,
193 expected: Option<subc_os::FileIdentity>,
194 trampoline: Option<subc_os::FileIdentity>,
195 signature: Option<Arc<subc_os::privacy_identity::SuspendedSignature>>,
196 module_id: String,
197 #[cfg(test)]
198 confirmation_pause: Option<(String, Option<tokio::net::TcpStream>)>,
199 #[cfg(test)]
200 image_probe: Option<fn(u32) -> Option<subc_os::Observation>>,
201 #[cfg(test)]
202 image_read_delay: Option<Duration>,
203}
204
205#[cfg(all(test, target_os = "macos"))]
206pub(crate) fn test_privacy_trampoline() -> PathBuf {
207 let path = cortexkit_test_support::cargo_profile_bin("privacy-trampoline-fixture");
210 assert!(
214 path.exists(),
215 "privacy-trampoline-fixture not built at {}: run `cargo build -p subc-daemon \
216 --bins --features test-support` or `cargo test -p subc-daemon` first",
217 path.display()
218 );
219 path
220}
221
222#[cfg(target_os = "macos")]
223fn probe_privacy_trampoline(path: &std::path::Path) -> Result<(), String> {
224 use std::io::Read;
225 let mut probe = subc_os::child_spawn::spawn(
226 std::process::Command::new(path)
227 .args(["__disclaim-exec", "--probe-suspended"])
228 .stdin(Stdio::null())
229 .stdout(Stdio::piped())
230 .stderr(Stdio::piped()),
231 )
232 .map_err(|error| {
233 format!(
234 "privacy trampoline probe failed for {}: {error}",
235 path.display()
236 )
237 })?;
238 let deadline = std::time::Instant::now() + Duration::from_secs(5);
239 let status = loop {
240 match probe.try_wait() {
241 Ok(Some(status)) => break status,
242 Ok(None) if std::time::Instant::now() < deadline => {
243 std::thread::sleep(Duration::from_millis(5))
244 }
245 result => {
246 let _ = probe.kill();
247 let _ = probe.wait();
248 return Err(format!(
249 "privacy trampoline probe failed or timed out for {}: {result:?}",
250 path.display()
251 ));
252 }
253 }
254 };
255 let mut answer = String::new();
256 if let Some(stdout) = probe.stdout.take() {
257 let _ = stdout.take(256).read_to_string(&mut answer);
258 }
259 if status.success() && answer.trim() == subc_os::privacy_identity::SUSPENDED_TRAMPOLINE_PROBE {
260 return Ok(());
261 }
262 let mut diagnostic = String::new();
263 if let Some(stderr) = probe.stderr.take() {
264 let _ = stderr.take(1024).read_to_string(&mut diagnostic);
265 }
266 let cause = diagnostic
267 .trim()
268 .strip_prefix("ck-subc: own privacy identity refused: ")
269 .unwrap_or("binary does not implement the privacy trampoline protocol");
270 Err(format!(
271 "{cause}: probe of {} exited {status}",
272 path.display()
273 ))
274}
275
276#[cfg(target_os = "macos")]
277fn privacy_command(
278 spec: &ModuleSpec,
279 binding: &SpawnProgramBinding,
280 roster: &ChildRoster,
281 gated: bool,
282) -> Result<
283 (
284 Command,
285 Option<PrivacyExec>,
286 subc_os::privacy_identity::ExecAcknowledgement,
287 ),
288 SuperviseError,
289> {
290 let failure = |cause: String| {
291 warn!(module_id = %spec.module_id, %cause, "privacy identity trampoline refused module spawn");
292 SuperviseError::Spawn {
293 program: spec.program.clone(),
294 source: io::Error::other(cause),
295 cgroup_path: None,
296 }
297 };
298 let trampoline = roster.privacy_trampoline().map_err(failure)?;
299 let expected = Some(binding.identity);
300 let trampoline_image = subc_os::file_identity(&trampoline);
301 let signature = gated
302 .then(|| {
303 subc_os::privacy_identity::SuspendedSignature::capture(&binding._file, &binding.path)
304 .map(Arc::new)
305 })
306 .transpose()
307 .map_err(|error| failure(error.to_string()))?;
308 if expected == trampoline_image {
309 return Err(failure(
310 "privacy identity module executable is missing or is the trampoline itself".to_string(),
311 ));
312 }
313 let (reader, ack) = subc_os::privacy_identity::ExecAcknowledgement::pipe()
314 .map_err(|error| failure(error.to_string()))?;
315 let reader =
316 tokio::io::unix::AsyncFd::new(reader).map_err(|error| failure(error.to_string()))?;
317 let mut command = Command::new(&trampoline);
318 command
319 .arg(if gated {
320 "__disclaim-exec-suspended"
321 } else {
322 "__disclaim-exec"
323 })
324 .arg(ack.fd().to_string())
325 .arg(&binding.path);
326 Ok((
327 command,
328 Some(PrivacyExec {
329 gated,
330 admission_binding: None,
331 content_check: None,
332 reader,
333 deadline: tokio::time::Instant::now() + Duration::from_secs(5),
334 expected,
335 trampoline: trampoline_image,
336 signature,
337 module_id: spec.module_id.clone(),
338 #[cfg(test)]
339 image_probe: None,
340 #[cfg(test)]
341 image_read_delay: spec
342 .env
343 .iter()
344 .find(|(key, _)| key == "SUBC_TEST_PRIVACY_IMAGE_DELAY_MS")
345 .map(|(_, millis)| Duration::from_millis(millis.parse().unwrap())),
346 #[cfg(test)]
347 confirmation_pause: spec
348 .env
349 .iter()
350 .find(|(key, _)| key == "SUBC_TEST_BEFORE_EXEC_CONFIRM")
351 .map(|(_, address)| (address.clone(), None)),
352 }),
353 ack,
354 ))
355}
356
357#[cfg(unix)]
358type SpawnRegistrationChannel = std::os::unix::net::UnixStream;
359#[cfg(windows)]
360type SpawnRegistrationChannel = subc_os::registration_channel::RegistrationPipe;
361#[cfg(test)]
362type CapturedRegistrationChannels =
363 Arc<Mutex<Option<Vec<(Arc<RegistrationSpawn>, CapturedRegistrationEndpoint)>>>>;
364#[cfg(test)]
365struct CapturedRegistrationEndpoint {
366 _channel: SpawnRegistrationChannel,
367}
368#[cfg(test)]
369impl fmt::Debug for CapturedRegistrationEndpoint {
370 fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
371 f.write_str("CapturedRegistrationEndpoint")
372 }
373}
374
375struct SupervisedChild {
376 registration_spawn: Option<Arc<RegistrationSpawn>>,
377 #[cfg(any(unix, windows))]
379 registration_channel: Option<SpawnRegistrationChannel>,
380 live_program_binding: Option<Arc<content_binding::LiveProgramBinding>>,
381 registered_config_connection: Option<ConnectionId>,
382 admission_base: Option<subc_config::Digest>,
383 registration_declaration: Option<String>,
384 _program_binding: Option<ProgramBindingPublication>,
385 subtree_target: crate::subtree_resources::Target,
386 child: containment::ProcessChild,
387 exec: crate::activity::ExecOwner,
388 #[cfg(windows)]
389 nonce_delivery: Option<subc_os::LaunchNoncePipeDelivery>,
390 #[cfg(target_os = "macos")]
391 privacy_exec: Option<PrivacyExec>,
392 #[cfg(target_os = "macos")]
399 report_ready: Arc<OnceLock<()>>,
400 spawn_failure: Option<String>,
403 admission_kill_sent: bool,
406 protocol: ModuleProtocol,
411 #[cfg(target_os = "linux")]
418 module_id: String,
419 #[cfg(target_os = "linux")]
420 cgroup_placement: Option<subc_cgroup::Placement>,
421 #[cfg(windows)]
444 job: Option<Arc<Mutex<subc_jobobject::JobObject>>>,
445 stdout_pump: Option<JoinHandle<()>>,
446 stderr_pump: Option<StderrPump>,
447 stderr_ring: Arc<Mutex<StderrRing>>,
448 spawned_at: Instant,
451 config_delivery: Option<crate::config_gate::runtime::DeliverySpawn>,
452 registration_deadline: Option<Instant>,
455 registration_watch: Option<RegistrationWatch>,
456 spawned_at_ms: u64,
457 spawned_from: PathBuf,
458 launch_spec_digest: LaunchSpecDigest,
459 spawned_file_identity: Option<SpawnedFileIdentity>,
460 process_start_time: Option<u64>,
461 process_identity: Option<ProcessIdentity>,
462 pid: u32,
463 roster_guard: Option<crate::child_roster::RosterGuard>,
466}
467
468impl SupervisedChild {
469 fn retire_registration(&self) {
470 if let Some(spawn) = &self.registration_spawn {
475 spawn.retire();
476 } else if let Some(binding) = &self._program_binding {
477 binding.retire();
478 }
479 #[cfg(unix)]
480 if let Some(channel) = &self.registration_channel {
481 let _ = channel.shutdown(std::net::Shutdown::Both);
482 }
483 #[cfg(windows)]
484 if let Some(channel) = &self.registration_channel {
485 if let Ok(shutdown) = channel.shutdown_handle() {
486 shutdown.shutdown();
487 }
488 }
489 }
490 fn registered_since_spawn(&self) -> bool {
491 self.registration_watch
492 .as_ref()
493 .is_some_and(|watch| *watch.changes.borrow() != watch.baseline)
494 }
495
496 fn awaiting_registration(&self) -> bool {
497 let admission_failed = self.admission_kill_sent && self.spawn_failure.is_some();
501 !admission_failed && !self.registered_since_spawn()
502 }
503
504 fn id(&self) -> Option<u32> {
505 Some(self.pid)
506 }
507
508 fn process_identity(&self) -> Option<ProcessIdentity> {
509 self.process_identity
510 }
511
512 fn try_wait(&mut self) -> io::Result<Option<ExitStatus>> {
513 let result = self.child.try_wait();
514 if !matches!(result, Ok(None)) {
515 self.retire_registration();
516 #[cfg(any(unix, windows))]
517 self.registration_channel.take();
518 if let Some(binding) = &self._program_binding {
519 binding.retire();
520 }
521 }
522 result
523 }
524
525 async fn wait(&mut self) -> io::Result<ExitStatus> {
526 #[cfg(target_os = "macos")]
527 self.confirm_privacy_exec().await;
528 let result = self.child.wait().await;
536 self.retire_registration();
537 #[cfg(any(unix, windows))]
538 self.registration_channel.take();
539 if let Some(binding) = &self._program_binding {
542 binding.retire();
543 }
544 #[cfg(all(test, target_os = "linux"))]
545 if let Some(binding) = &self._program_binding {
546 binding.pause_after_wait_for_test().await;
547 }
548 #[cfg(target_os = "linux")]
549 if result.is_ok() && self.child.container().is_none() {
550 if let Some(placement) = self.cgroup_placement.as_ref() {
551 cleanup_reaped_cgroup(placement, &self.module_id).await;
555 self.cgroup_placement = None;
556 }
557 }
558 result
559 }
560
561 async fn finish_exit(&mut self, carry: bool) -> io::Result<()> {
562 self.retire_registration();
563 if let Some(binding) = self._program_binding.take() {
565 binding.retire();
566 }
567 if self.child.container().is_some() {
568 if !carry {
569 self.child.terminate_container().await;
570 }
571 } else {
572 #[cfg(target_os = "linux")]
573 if let Some(placement) = self.cgroup_placement.as_ref() {
574 cleanup_reaped_cgroup(placement, &self.module_id).await;
575 self.cgroup_placement = None;
576 }
577 }
578 self.child.reap().await?;
579 self.release_roster();
580 Ok(())
581 }
582
583 #[cfg(target_os = "macos")]
584 async fn confirm_privacy_exec(&mut self) {
585 if self
586 .privacy_exec
587 .as_ref()
588 .is_some_and(|pending| pending.content_check.is_some())
589 {
590 self.finish_privacy_content_check().await;
591 return;
592 }
593 let Some(pending) = &mut self.privacy_exec else {
594 return;
595 };
596 let result = tokio::time::timeout_at(pending.deadline, async {
597 let mut record = Vec::new();
598 loop {
599 let mut ready = pending.reader.readable().await?;
600 let read = ready.try_io(|reader| {
601 use std::io::Read;
602 let mut reader = reader.get_ref();
603 let mut buffer = [0u8; 256];
604 reader.read(&mut buffer).map(|count| (count, buffer))
605 });
606 match read {
607 Ok(Ok((0, _))) => return Ok::<_, io::Error>(record),
608 Ok(Ok((count, buffer))) => {
609 if record.len() + count > 1024 {
610 return Err(io::Error::other(
611 "privacy exec refusal record is too long",
612 ));
613 }
614 record.extend_from_slice(&buffer[..count]);
615 }
616 Ok(Err(error)) => return Err(error),
617 Err(_) => continue,
618 }
619 }
620 })
621 .await;
622 #[cfg(test)]
623 let result = if let Some((address, stream)) = &mut pending.confirmation_pause {
624 use tokio::io::{AsyncReadExt, AsyncWriteExt};
625 if stream.is_none() {
626 *stream = Some(
627 tokio::net::TcpStream::connect(address.as_str())
628 .await
629 .unwrap(),
630 );
631 stream.as_mut().unwrap().write_all(b"R").await.unwrap();
632 stream
633 .as_mut()
634 .unwrap()
635 .write_all(&self.pid.to_ne_bytes())
636 .await
637 .unwrap();
638 }
639 let mut reply = [0];
640 stream
641 .as_mut()
642 .unwrap()
643 .read_exact(&mut reply)
644 .await
645 .unwrap();
646 pending.confirmation_pause = None;
647 if &reply == b"F" {
648 Ok(Err(io::Error::other("injected late exec refusal")))
649 } else {
650 assert_eq!(&reply, b"X");
651 result
652 }
653 } else {
654 result
655 };
656 let waited = if matches!(&result, Ok(Ok(record)) if record.is_empty()) {
661 Some(self.try_wait())
662 } else {
663 None
664 };
665 let pending = self.privacy_exec.as_ref().expect("pending exec");
666 let cause = match result {
667 Err(_) => Some("privacy identity trampoline did not exec within 5s".to_string()),
668 Ok(Err(error)) => Some(format!(
669 "privacy identity exec acknowledgement failed: {error}"
670 )),
671 Ok(Ok(record)) if !record.is_empty() => Some(
672 std::str::from_utf8(&record)
673 .ok()
674 .and_then(|record| {
675 record
676 .trim()
677 .strip_prefix(subc_os::privacy_identity::EXEC_REFUSAL_TAG)
678 })
679 .filter(|cause| !cause.is_empty())
680 .unwrap_or("invalid privacy exec refusal record")
681 .to_string(),
682 ),
683 Ok(Ok(_)) if !pending.gated => match waited.expect("exec EOF checked child") {
684 Ok(Some(_)) => None,
685 Err(error) => Some(format!("privacy identity trampoline wait failed: {error}")),
686 Ok(None) => {
687 #[cfg(test)]
688 if let Some(delay) = pending.image_read_delay {
689 tokio::time::sleep(delay).await;
690 }
691 let image = observe_spawned_image(self.pid);
692 #[cfg(test)]
693 let image = pending.image_probe.map_or(image, |probe| probe(self.pid));
694 if let Some(image) = image.filter(|image| {
698 image.executable.is_some() && image.executable != pending.trampoline
699 }) {
700 match self
701 .roster_guard
702 .as_ref()
703 .map(|guard| guard.confirm_executable(image))
704 .transpose()
705 {
706 Err(error) => Some(format!(
707 "could not durably record confirmed module image: {error}"
708 )),
709 Ok(_) => {
710 let _ = self.report_ready.set(());
711 None
712 }
713 }
714 } else if image.is_none()
715 || image.is_some_and(|image| image.executable.is_none())
716 {
717 match tokio::time::timeout_at(pending.deadline, self.child.wait()).await {
720 Ok(result) => {
721 self.retire_registration();
722 self.registration_channel.take();
723 if let Some(binding) = &self._program_binding {
724 binding.retire();
725 }
726 result.err().map(|error| format!("privacy identity trampoline wait failed: {error}"))
727 }
728 Err(_) => Some("privacy identity module executable remained unreadable until the 5s exec deadline".to_string()),
729 }
730 } else {
731 Some("privacy identity trampoline did not exec the module".to_string())
732 }
733 }
734 },
735 Ok(Ok(_)) => match waited.expect("exec EOF checked child") {
736 Ok(Some(_)) => {
737 Some("privacy identity target exited before suspended admission".to_string())
738 }
739 Err(error) => Some(format!("privacy identity trampoline wait failed: {error}")),
740 Ok(None) => {
741 let stopped = tokio::time::timeout_at(pending.deadline, async {
744 loop {
745 if subc_os::privacy_identity::SuspendedSignature::is_suspended(
746 self.pid,
747 )? {
748 return Ok::<_, io::Error>(());
749 }
750 tokio::time::sleep(Duration::from_millis(1)).await;
751 }
752 })
753 .await;
754 match stopped {
755 Err(_) => {
756 Some("privacy identity target was not suspended within 5s".to_string())
757 }
758 Ok(Err(error)) => Some(format!(
759 "privacy identity suspended target unavailable: {error}"
760 )),
761 Ok(Ok(())) => {
762 let image = observe_spawned_image(self.pid);
763 if image.is_some_and(|image| {
764 image.executable.is_some()
765 && image.executable != pending.trampoline
766 && image.executable == pending.expected
767 }) {
768 None
769 } else {
770 Some("privacy identity trampoline executable mismatch: expected the suspended module program".to_string())
771 }
772 }
773 }
774 }
775 },
776 };
777 #[cfg(test)]
778 if let Some(binding) = &self._program_binding {
779 binding.pause_after_wait_for_test().await;
780 }
781 if cause.is_none() && pending.gated {
782 let publication = pending.admission_binding.clone();
783 let live = self.live_program_binding.clone();
784 let signature = Arc::clone(pending.signature.as_ref().expect("gated signature"));
785 let pid = self.pid;
786 let gated = pending.gated;
787 self.privacy_exec
788 .as_mut()
789 .expect("pending exec")
790 .content_check = Some(tokio::task::spawn_blocking(move || {
791 signature.verify(pid).map_err(|error| {
792 crate::config_gate::GateError::new("program_refused", error)
793 })?;
794 if !gated {
795 return Ok(());
796 }
797 match (publication, live) {
798 (Some(publication), _) => publication.verify(),
799 (None, Some(live)) => live.check(),
800 _ => Err(crate::config_gate::GateError::new(
801 "config_delivery_deferred",
802 "live program witness is unavailable",
803 )),
804 }
805 }));
806 self.finish_privacy_content_check().await;
807 } else {
808 self.finish_privacy_exec(cause);
809 }
810 }
811
812 #[cfg(target_os = "macos")]
813 async fn finish_privacy_content_check(&mut self) {
814 let result = self
815 .privacy_exec
816 .as_mut()
817 .expect("pending exec")
818 .content_check
819 .as_mut()
820 .expect("pending content check")
821 .await;
822 let cause = match result {
823 Ok(Ok(())) => None,
824 Ok(Err(error)) => Some(format!("{}: {}", error.code, error.detail)),
825 Err(error) => Some(format!(
826 "config_delivery_deferred: content check stopped: {error}"
827 )),
828 };
829 self.finish_privacy_exec(cause);
830 }
831
832 #[cfg(target_os = "macos")]
833 fn finish_privacy_exec(&mut self, mut cause: Option<String>) {
834 let mut pending = self.privacy_exec.take().expect("pending exec");
835 if cause.is_none() && pending.gated {
836 let admission = (|| -> io::Result<()> {
837 let image = observe_spawned_image(self.pid)
838 .filter(|image| image.executable == pending.expected)
839 .ok_or_else(|| io::Error::other("suspended executable mismatch"))?;
840 if let Some(guard) = &self.roster_guard {
841 guard.confirm_executable(image)?;
842 }
843 pending
844 .signature
845 .as_ref()
846 .expect("gated signature")
847 .resume(self.pid)?;
848 let _ = self.report_ready.set(());
849 info!(module_id = %pending.module_id, pid = self.pid,
850 "verified suspended module resumed with own privacy identity");
851 Ok(())
852 })();
853 if let Err(error) = admission {
854 cause = Some(format!("program_refused: {error}"));
855 }
856 }
857 if cause.is_some() {
858 if let Some(spawn) = &self.registration_spawn {
859 spawn.refuse();
860 }
861 if let Some(binding) = &mut pending.admission_binding {
862 binding.reject();
863 }
864 } else if let Some(spawn) = &self.registration_spawn {
865 spawn.ready();
866 }
867 self.exec.0.finish(if cause.is_some() {
870 crate::activity::ExecState::Refused
871 } else {
872 crate::activity::ExecState::CommittedExec
873 });
874 if let Some(cause) = cause {
875 warn!(module_id = %pending.module_id, pid = self.pid, %cause, "privacy identity trampoline refused module spawn");
876 self.spawn_failure = Some(
877 if pending.gated
878 && (cause.contains("executable mismatch")
879 || cause.starts_with("program_refused:"))
880 {
881 format!("program_refused: {cause}")
882 } else if pending.gated {
883 format!("config_delivery_deferred: {cause}")
884 } else {
885 cause
886 },
887 );
888 let _ = self.start_admission_kill();
891 }
892 }
893
894 fn release_roster(&mut self) {
898 self.roster_guard = None;
899 }
900
901 fn start_kill(&mut self) -> io::Result<()> {
914 self.retire_registration();
915 if let Some(container) = self.child.container() {
916 container.hard_kill();
917 return self.kill_leader();
918 }
919 #[cfg(windows)]
920 if let Some(job) = &self.job {
921 if let Err(error) = job.lock().unwrap_or_else(|p| p.into_inner()).terminate() {
922 debug!(
923 error = %error,
924 "job termination failed; the direct-child kill still owns the outcome"
925 );
926 }
927 }
928 #[cfg(target_os = "linux")]
929 kill_module_cgroup(self.cgroup_placement.as_ref(), &self.module_id);
930 self.child.start_kill()
931 }
932
933 fn kill_leader(&mut self) -> io::Result<()> {
934 self.retire_registration();
935 self.child.start_kill()
936 }
937
938 fn start_admission_kill(&mut self) -> io::Result<()> {
939 self.retire_registration();
940 #[cfg(windows)]
941 let result = {
942 let result = self
945 .spawned_file_identity
946 .as_ref()
947 .and_then(|identity| identity.windows_image.as_ref())
948 .ok_or_else(|| io::Error::other("admission child handle unavailable"))
949 .and_then(|image| {
950 image.process().terminate_with_exit_code(
951 admission_exit::WINDOWS_ADMISSION_KILL_EXIT_CODE as u32,
952 )
953 });
954 if let Some(container) = self.child.container() {
955 container.hard_kill();
956 } else if let Some(job) = &self.job {
957 if let Err(error) = job.lock().unwrap_or_else(|p| p.into_inner()).terminate() {
958 debug!(%error, "job termination failed after admission leader termination");
959 }
960 }
961 result
962 };
963 #[cfg(not(windows))]
964 let result = self.start_kill().map(|()| true);
965 #[cfg(unix)]
966 let result = {
967 let group_sent = self.child.container().is_none()
970 && rustix::process::Pid::from_raw(self.pid as i32).is_some_and(|group| {
971 rustix::process::kill_process_group(group, rustix::process::Signal::KILL)
972 .is_ok()
973 });
974 if group_sent {
975 Ok(true)
976 } else {
977 result
978 }
979 };
980 self.admission_kill_sent |= result?;
981 Ok(())
982 }
983
984 async fn drain_stderr(&mut self, module_id: &str) {
985 if let Some(pump) = self.stdout_pump.take() {
986 settle_stdout_pump(module_id, pump, STDERR_PUMP_DRAIN_TIMEOUT).await;
987 }
988
989 let Some(pump) = self.stderr_pump.take() else {
990 return;
991 };
992 settle_stderr_pump(
993 module_id,
994 &self.stderr_ring,
995 pump,
996 STDERR_PUMP_DRAIN_TIMEOUT,
997 )
998 .await;
999 }
1000}
1001
1002async fn settle_stdout_pump(module_id: &str, mut pump: JoinHandle<()>, bound: Duration) {
1006 match timeout(bound, &mut pump).await {
1007 Ok(Ok(())) => {}
1008 Ok(Err(error)) => {
1009 warn!(module_id, error = %error, "stdout pump ended unexpectedly");
1010 }
1011 Err(_) => {
1012 drop(pump);
1013 warn!(
1014 module_id,
1015 waited = ?bound,
1016 "stdout pipe of the exited process has not drained; its reader keeps running without delaying the restart"
1017 );
1018 }
1019 }
1020}
1021
1022struct StderrPump {
1025 task: JoinHandle<()>,
1026 generation: u64,
1027}
1028
1029async fn settle_stderr_pump(
1035 module_id: &str,
1036 ring: &Arc<Mutex<StderrRing>>,
1037 pump: StderrPump,
1038 bound: Duration,
1039) {
1040 let lock = || ring.lock().unwrap_or_else(|poisoned| poisoned.into_inner());
1041 let StderrPump {
1042 mut task,
1043 generation,
1044 } = pump;
1045 lock().retire_pump(generation);
1046 match timeout(bound, &mut task).await {
1047 Ok(Ok(())) => {}
1048 Ok(Err(err)) => {
1049 let mut ring = lock();
1050 ring.mark_incomplete_from(generation, format!("stderr pump ended unexpectedly: {err}"));
1051 ring.finish_pump(generation);
1052 warn!(module_id, error = %err, "stderr pump ended before clean EOF");
1053 }
1054 Err(_) => {
1055 drop(task);
1057 lock().mark_pump_late(
1058 generation,
1059 format!(
1060 "stderr of the exited process had not reached EOF {bound:?} after it was \
1061 retired (a descendant may still hold the pipe open); lines it still \
1062 writes are kept in that process's section"
1063 ),
1064 );
1065 warn!(
1066 module_id,
1067 waited = ?bound,
1068 "stderr pipe of the exited process is still open; its reader keeps running without delaying the restart"
1069 );
1070 }
1071 }
1072}
1073
1074fn registration_release_events() -> &'static watch::Sender<u64> {
1075 static EVENTS: OnceLock<watch::Sender<u64>> = OnceLock::new();
1076 EVENTS.get_or_init(|| {
1077 let (sender, _receiver) = watch::channel(0);
1078 sender
1079 })
1080}
1081
1082pub(crate) fn notify_registration_release() {
1083 let events = registration_release_events();
1084 let next_generation = (*events.borrow()).wrapping_add(1);
1085 events.send_replace(next_generation);
1086}
1087
1088#[derive(Debug, Clone, PartialEq, Eq)]
1090pub struct ModuleSpec {
1091 pub module_id: String,
1092 pub program: PathBuf,
1093 pub args: Vec<String>,
1094 pub env: Vec<(String, String)>,
1095 pub reserved: bool,
1100 pub reserved_prefixes: Vec<String>,
1105 pub protocol: ModuleProtocol,
1124 pub overlap: ModuleOverlap,
1129 pub carry_containment: bool,
1133}
1134
1135#[derive(Clone, PartialEq, Eq)]
1138pub(crate) struct LaunchSpecDigest([u8; 32]);
1139
1140impl fmt::Debug for LaunchSpecDigest {
1141 fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
1142 f.write_str("LaunchSpecDigest([redacted])")
1143 }
1144}
1145
1146impl LaunchSpecDigest {
1147 pub(crate) fn from_spec(spec: &ModuleSpec) -> Self {
1148 fn field(hash: &mut Sha256, bytes: &[u8]) {
1149 hash.update((bytes.len() as u64).to_le_bytes());
1150 hash.update(bytes);
1151 }
1152 let mut hash = Sha256::new();
1153 if let Some(program) = spec.program.to_str() {
1154 field(&mut hash, b"utf8-program");
1155 field(&mut hash, program.as_bytes());
1156 } else {
1157 #[cfg(unix)]
1160 {
1161 use std::os::unix::ffi::OsStrExt;
1162 field(&mut hash, b"unix-program");
1163 field(&mut hash, spec.program.as_os_str().as_bytes());
1164 }
1165 #[cfg(windows)]
1166 {
1167 use std::os::windows::ffi::OsStrExt;
1168 field(&mut hash, b"windows-program");
1169 let bytes = spec
1170 .program
1171 .as_os_str()
1172 .encode_wide()
1173 .flat_map(u16::to_le_bytes)
1174 .collect::<Vec<_>>();
1175 field(&mut hash, &bytes);
1176 }
1177 #[cfg(not(any(unix, windows)))]
1178 field(&mut hash, spec.program.to_string_lossy().as_bytes());
1179 }
1180 hash.update((spec.args.len() as u64).to_le_bytes());
1181 for arg in &spec.args {
1182 field(&mut hash, arg.as_bytes());
1183 }
1184 let mut env = spec.env.iter().collect::<Vec<_>>();
1185 env.sort_unstable();
1186 hash.update((env.len() as u64).to_le_bytes());
1187 for (key, value) in env {
1188 field(&mut hash, key.as_bytes());
1189 field(&mut hash, value.as_bytes());
1190 }
1191 field(
1192 &mut hash,
1193 match spec.protocol {
1194 ModuleProtocol::Subc => b"subc",
1195 ModuleProtocol::None => b"none",
1196 },
1197 );
1198 Self(hash.finalize().into())
1199 }
1200}
1201
1202#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)]
1209pub enum ModuleOverlap {
1210 #[default]
1212 Exclusive,
1213 Safe,
1225}
1226
1227impl ModuleOverlap {
1228 pub fn as_str(self) -> &'static str {
1229 match self {
1230 Self::Exclusive => "exclusive",
1231 Self::Safe => "safe",
1232 }
1233 }
1234}
1235
1236pub const SUBC_SPAWN_ROLE_ENV: &str = "SUBC_SPAWN_ROLE";
1246pub const SPAWN_ROLE_SWAP_CANDIDATE: &str = "swap_candidate";
1248pub const DEFAULT_SWAP_READY_TIMEOUT: Duration = Duration::from_secs(100);
1253
1254#[derive(Debug, Clone, Copy, PartialEq, Eq)]
1272pub struct RestartPolicy {
1273 pub max_restarts: u32,
1274 pub backoff: Duration,
1277 pub max_backoff: Duration,
1279 pub window: Duration,
1283}
1284
1285impl RestartPolicy {
1286 pub fn new(max_restarts: u32, backoff: Duration) -> Self {
1290 Self {
1291 max_restarts,
1292 backoff,
1293 max_backoff: DEFAULT_MAX_BACKOFF,
1294 window: DEFAULT_RESTART_WINDOW,
1295 }
1296 }
1297
1298 pub fn with_max_backoff(mut self, max_backoff: Duration) -> Self {
1299 self.max_backoff = max_backoff;
1300 self
1301 }
1302
1303 pub fn with_window(mut self, window: Duration) -> Self {
1304 self.window = window;
1305 self
1306 }
1307
1308 fn delay_for_restart(&self, restart_in_window: u32) -> Duration {
1313 if self.backoff.is_zero() || self.max_backoff.is_zero() {
1314 return Duration::ZERO;
1315 }
1316
1317 let mut delay = self.backoff;
1318 for _ in 0..restart_in_window {
1319 if delay >= self.max_backoff {
1320 return self.max_backoff;
1321 }
1322 delay = delay
1323 .checked_mul(10)
1324 .unwrap_or(self.max_backoff)
1325 .min(self.max_backoff);
1326 }
1327 delay.min(self.max_backoff)
1328 }
1329
1330 fn budget_exhausted_detail(&self) -> String {
1335 format!(
1336 "crash budget exhausted: max_restarts={} within window_secs={}",
1337 self.max_restarts,
1338 self.window.as_secs()
1339 )
1340 }
1341}
1342
1343impl Default for RestartPolicy {
1344 fn default() -> Self {
1345 Self {
1346 max_restarts: DEFAULT_MAX_RESTARTS,
1347 backoff: DEFAULT_BACKOFF,
1348 max_backoff: DEFAULT_MAX_BACKOFF,
1349 window: DEFAULT_RESTART_WINDOW,
1350 }
1351 }
1352}
1353
1354#[derive(Debug, Clone, Copy, PartialEq, Eq)]
1355struct CrashRestartSchedule {
1356 restart_in_window: u32,
1357 delay: Duration,
1358}
1359
1360fn daemon_will_restart(
1367 state: &mut SupervisorSnapshot,
1368 policy: &RestartPolicy,
1369 now: Instant,
1370) -> bool {
1371 state.enabled && state.crash_restarts_in_window(policy.window, now) < policy.max_restarts
1372}
1373
1374const DEFAULT_HEALTH_CADENCE: Duration = Duration::from_secs(30);
1375const DEFAULT_HEALTH_DEADLINE: Duration = Duration::from_secs(5);
1376const DEFAULT_HEALTH_FAILURE_THRESHOLD: u32 = 3;
1377const MAX_HEALTH_METRICS_BYTES: usize = 16 * 1024;
1378
1379#[derive(Debug, Clone, Copy, PartialEq, Eq)]
1380pub enum HealthAction {
1381 Report,
1382 Restart,
1383 Alert,
1384}
1385
1386impl fmt::Display for HealthAction {
1387 fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
1388 f.write_str(match self {
1389 Self::Report => "report",
1390 Self::Restart => "restart",
1391 Self::Alert => "alert",
1392 })
1393 }
1394}
1395
1396#[derive(Debug, Clone, PartialEq, Eq)]
1397pub struct HealthConfig {
1398 pub http: Option<String>,
1402 pub cadence: Duration,
1403 pub deadline: Duration,
1404 pub failure_threshold: u32,
1405 pub on_degraded: HealthAction,
1406 pub on_failing: HealthAction,
1407 pub critical: bool,
1408}
1409
1410impl Default for HealthConfig {
1411 fn default() -> Self {
1412 Self {
1413 http: None,
1414 cadence: DEFAULT_HEALTH_CADENCE,
1415 deadline: DEFAULT_HEALTH_DEADLINE,
1416 failure_threshold: DEFAULT_HEALTH_FAILURE_THRESHOLD,
1417 on_degraded: HealthAction::Report,
1418 on_failing: HealthAction::Report,
1419 critical: false,
1420 }
1421 }
1422}
1423
1424#[derive(Debug, Clone, PartialEq)]
1442pub struct ModuleHealthStatus {
1443 pub status: SupervisorHealthStatus,
1444 pub last_probe_ms: Option<u64>,
1445 pub detail: Option<String>,
1446 pub metrics: Option<Value>,
1447 pub consecutive_failures: u32,
1448 pub late_answer_count: u64,
1451 pub last_late_answer_latency_ms: Option<u64>,
1453 pub last_action: Option<String>,
1454 pub last_action_ms: Option<u64>,
1458}
1459
1460impl Default for ModuleHealthStatus {
1461 fn default() -> Self {
1462 Self {
1463 status: SupervisorHealthStatus::Unknown,
1464 last_probe_ms: None,
1465 detail: None,
1466 metrics: None,
1467 consecutive_failures: 0,
1468 late_answer_count: 0,
1469 last_late_answer_latency_ms: None,
1470 last_action: None,
1471 last_action_ms: None,
1472 }
1473 }
1474}
1475
1476#[derive(Debug, Clone, Copy, PartialEq, Eq)]
1478pub enum ModuleState {
1479 Starting,
1480 Running,
1481 Unresponsive,
1482 Restarting,
1483 Draining,
1484 Stopped,
1485 Failed,
1486 Disabled,
1487}
1488
1489impl fmt::Display for ModuleState {
1490 fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
1491 f.write_str(match self {
1492 Self::Starting => "starting",
1493 Self::Running => "running",
1494 Self::Unresponsive => "unresponsive",
1495 Self::Restarting => "restarting",
1496 Self::Draining => "draining",
1497 Self::Stopped => "stopped",
1498 Self::Failed => "failed",
1499 Self::Disabled => "disabled",
1500 })
1501 }
1502}
1503
1504#[derive(Debug, Clone, Copy, PartialEq, Eq)]
1506pub enum ExitKind {
1507 Clean,
1508 Crash,
1509 DeliberateSeverance,
1510}
1511
1512impl From<ExitKind> for TerminalExitKind {
1513 fn from(kind: ExitKind) -> Self {
1514 match kind {
1515 ExitKind::Clean => Self::Clean,
1516 ExitKind::Crash => Self::Crash,
1517 ExitKind::DeliberateSeverance => Self::DeliberateSeverance,
1518 }
1519 }
1520}
1521
1522#[derive(Debug, Clone, Copy, PartialEq, Eq)]
1525pub(crate) struct ProcessIdentity {
1526 pub(crate) pid: u32,
1527 pub(crate) start_time: u64,
1528}
1529
1530#[derive(Debug, Clone, PartialEq, Eq)]
1532pub struct ExitReport {
1533 pub kind: ExitKind,
1534 pub code: Option<i32>,
1535 pub signal: Option<i32>,
1536 pub at_ms: u64,
1537}
1538
1539#[derive(Debug, Clone, PartialEq)]
1542pub struct ModuleStatus {
1543 pub(crate) admission_refused: Option<String>,
1544 pub(crate) legacy_registration_spawn: bool,
1545 pub(crate) registration_declaration: Option<String>,
1546 pub module_id: String,
1547 pub stop_reason: Option<ModuleStopReason>,
1550 pub waiting_for_previous_pid: Option<u32>,
1552 pub waiting_for_previous_detail: Option<String>,
1554 pub launch_nonce_source: Option<String>,
1559 pub state: ModuleState,
1560 pub enabled: bool,
1561 pub process_alive: bool,
1562 pub registration_active: bool,
1563 pub protocol: ModuleProtocol,
1568 pub live: bool,
1579 pub restart_count: u32,
1583 pub lifetime_restarts: u32,
1587 pub spawn_generation: u64,
1588 pub swap_outcome: Option<SwapOutcome>,
1590 pub max_restarts: u32,
1595 pub restart_window: Duration,
1599 pub drain_timeout: Duration,
1603 pub restart_backoff: Duration,
1604 pub restart_max_backoff: Duration,
1605 pub pid: Option<u32>,
1610 pub spawned_at_ms: Option<u64>,
1611 pub spawned_from: Option<PathBuf>,
1612 pub(crate) launch_spec_digest: Option<LaunchSpecDigest>,
1613 pub process_start_time: Option<u64>,
1614 pub last_exit: Option<ExitReport>,
1615 pub health: ModuleHealthStatus,
1616}
1617
1618#[cfg(test)]
1619#[derive(Debug, Clone, Copy, PartialEq)]
1620struct ActorSelectCheckpoint {
1621 generation: u64,
1622 turn: u64,
1623 registered_connection: Option<ConnectionId>,
1624 next_probe_at: Option<Instant>,
1625 wake_after: Option<Duration>,
1626}
1627
1628#[derive(Debug, Clone, PartialEq)]
1629struct SupervisorSnapshot {
1630 admission_refused: Option<String>,
1631 config_hold_parked: bool,
1633 config_hold_daemon_initiated: bool,
1636 config_hold_approval: Option<crate::config_gate::runtime::ApprovalIdentity>,
1638 approved_start_failure: Option<(crate::config_gate::runtime::ApprovalIdentity, String)>,
1640 legacy_registration_spawn: bool,
1641 registration_declaration: Option<String>,
1642 stop_reason: Option<ModuleStopReason>,
1643 config_exit_failed_approval: Option<crate::config_gate::runtime::ApprovalIdentity>,
1644 subtree_target: crate::subtree_resources::Target,
1645 failed_retry: FailedRetry,
1646 exec_outcome: Option<Arc<crate::activity::ExecOutcome>>,
1647 #[cfg(windows)]
1648 pipe_consumed: Option<Arc<OnceLock<()>>>,
1649 #[cfg(test)]
1651 actor_turns: u64,
1652 #[cfg(test)]
1655 actor_select: Option<ActorSelectCheckpoint>,
1656 state: ModuleState,
1657 enabled: bool,
1658 process_alive: bool,
1659 spawned_protocol: Option<ModuleProtocol>,
1660 spawn_failure: Option<String>,
1661 waiting_for_previous_pid: Option<u32>,
1662 waiting_for_previous_detail: Option<String>,
1663 crash_restarts: VecDeque<Instant>,
1669 lifetime_restarts: u32,
1670 spawn_generation: u64,
1679 swap_outcome: Option<SwapOutcome>,
1680 pid: Option<u32>,
1681 #[cfg(target_os = "macos")]
1682 report_ready: Option<Arc<OnceLock<()>>>,
1683 reaped_pid: Option<u32>,
1685 respawn_pending: bool,
1687 coalesced_restart_pending: bool,
1689 spawned_at_ms: Option<u64>,
1690 spawned_from: Option<PathBuf>,
1691 launch_spec_digest: Option<LaunchSpecDigest>,
1692 spawned_file_identity: Option<SpawnedFileIdentity>,
1693 process_start_time: Option<u64>,
1694 deliberate_severance: Option<ProcessIdentity>,
1695 last_exit: Option<ExitReport>,
1696 drain_disposition_detail: Option<String>,
1698 health: ModuleHealthStatus,
1699 last_probe_evidence: Option<HealthProbeEvidence>,
1700 in_alternate_slot: bool,
1705 draining_to_replace: bool,
1712 configuration_updated_since_spawn: bool,
1718}
1719
1720impl SupervisorSnapshot {
1721 fn reported_pid(&self) -> Option<u32> {
1726 #[cfg(target_os = "macos")]
1727 if self
1728 .report_ready
1729 .as_ref()
1730 .is_some_and(|ready| ready.get().is_none())
1731 {
1732 return None;
1733 }
1734 self.pid
1735 }
1736
1737 fn starting() -> Self {
1738 Self::new(ModuleState::Starting, true)
1739 }
1740
1741 fn disabled() -> Self {
1742 Self::new(ModuleState::Disabled, false)
1743 }
1744
1745 fn crash_restarts_in_window(&mut self, window: Duration, now: Instant) -> u32 {
1749 while let Some(oldest) = self.crash_restarts.front() {
1750 if now.duration_since(*oldest) > window {
1751 self.crash_restarts.pop_front();
1752 } else {
1753 break;
1754 }
1755 }
1756 u32::try_from(self.crash_restarts.len()).unwrap_or(u32::MAX)
1757 }
1758
1759 fn record_crash_restart(&mut self, policy: &RestartPolicy, now: Instant) {
1765 self.crash_restarts.push_back(now);
1766 while self.crash_restarts.len() > policy.max_restarts as usize {
1767 self.crash_restarts.pop_front();
1768 }
1769 self.lifetime_restarts += 1;
1770 }
1771
1772 fn next_crash_restart(
1776 &mut self,
1777 policy: &RestartPolicy,
1778 now: Instant,
1779 ) -> Option<CrashRestartSchedule> {
1780 let restart_in_window = self.crash_restarts_in_window(policy.window, now);
1781 if restart_in_window >= policy.max_restarts {
1782 return None;
1783 }
1784 self.record_crash_restart(policy, now);
1785 Some(CrashRestartSchedule {
1786 restart_in_window,
1787 delay: policy.delay_for_restart(restart_in_window),
1788 })
1789 }
1790
1791 fn clear_crash_restarts(&mut self) {
1796 self.crash_restarts.clear();
1797 }
1798
1799 fn new(state: ModuleState, enabled: bool) -> Self {
1800 Self {
1801 admission_refused: None,
1802 config_hold_parked: false,
1803 config_hold_daemon_initiated: false,
1804 config_hold_approval: None,
1805 approved_start_failure: None,
1806 legacy_registration_spawn: false,
1807 registration_declaration: None,
1808 stop_reason: None,
1809 config_exit_failed_approval: None,
1810 subtree_target: crate::subtree_resources::Target::Unavailable("not_running"),
1811 failed_retry: FailedRetry::default(),
1812 exec_outcome: None,
1813 #[cfg(test)]
1814 actor_turns: 0,
1815 #[cfg(test)]
1816 actor_select: None,
1817 state,
1818 enabled,
1819 process_alive: false,
1820 #[cfg(windows)]
1821 pipe_consumed: None,
1822 spawned_protocol: None,
1823 spawn_failure: None,
1824 waiting_for_previous_pid: None,
1825 waiting_for_previous_detail: None,
1826 crash_restarts: VecDeque::new(),
1827 lifetime_restarts: 0,
1828 spawn_generation: 0,
1829 swap_outcome: None,
1830 pid: None,
1831 #[cfg(target_os = "macos")]
1832 report_ready: None,
1833 reaped_pid: None,
1834 respawn_pending: false,
1835 coalesced_restart_pending: false,
1836 spawned_at_ms: None,
1837 spawned_from: None,
1838 launch_spec_digest: None,
1839 spawned_file_identity: None,
1840 process_start_time: None,
1841 deliberate_severance: None,
1842 last_exit: None,
1843 drain_disposition_detail: None,
1844 health: ModuleHealthStatus::default(),
1845 last_probe_evidence: None,
1846 in_alternate_slot: false,
1847 draining_to_replace: false,
1848 configuration_updated_since_spawn: false,
1849 }
1850 }
1851}
1852
1853type SharedSnapshot = Arc<Mutex<SupervisorSnapshot>>;
1854
1855type SpawnSubscriberKey = (ConnectionId, u64);
1856
1857#[derive(Debug)]
1858struct SpawnSubscriber {
1859 version: u8,
1860 frames: mpsc::Sender<Frame>,
1861 lagged: Option<oneshot::Sender<SpawnCursor>>,
1865}
1866
1867#[derive(Debug)]
1868struct SpawnEventState {
1869 daemon_incarnation: String,
1870 seq: u64,
1871 capacity: usize,
1872 live: HashMap<String, LiveSpawn>,
1873 generations: HashMap<String, u64>,
1874 events: VecDeque<SpawnEvent>,
1875 subscribers: HashMap<SpawnSubscriberKey, SpawnSubscriber>,
1876 process_changes: HashMap<String, tokio::sync::watch::Sender<u64>>,
1877}
1878
1879impl Default for SpawnEventState {
1880 fn default() -> Self {
1881 Self {
1882 daemon_incarnation: "unconfigured".to_string(),
1883 seq: 0,
1884 capacity: SPAWN_EVENT_RING_CAPACITY,
1885 live: HashMap::new(),
1886 generations: HashMap::new(),
1887 events: VecDeque::new(),
1888 subscribers: HashMap::new(),
1889 process_changes: HashMap::new(),
1890 }
1891 }
1892}
1893
1894#[derive(Debug, Clone, Default)]
1895struct SpawnEventFeed(
1896 Arc<Mutex<SpawnEventState>>,
1897 #[cfg(test)] Arc<Mutex<Option<std::net::TcpStream>>>,
1898);
1899
1900#[derive(Debug, Clone, PartialEq, Eq)]
1901pub(crate) enum SpawnSubscribeRefusal {
1902 ForeignIncarnation { current: String },
1903 TooOld { oldest: SpawnCursor },
1904 Frame(String),
1905}
1906
1907impl SpawnEventFeed {
1908 fn configure_incarnation(&self, daemon_incarnation: String) {
1909 let mut state = self.0.lock().unwrap_or_else(|p| p.into_inner());
1910 state.daemon_incarnation = daemon_incarnation;
1911 state.seq = 0;
1912 state.live.clear();
1913 state.generations.clear();
1914 state.events.clear();
1915 state.subscribers.clear();
1916 for changed in state.process_changes.values() {
1917 changed.send_replace(0);
1918 }
1919 }
1920
1921 fn cursor(state: &SpawnEventState) -> SpawnCursor {
1922 SpawnCursor {
1923 daemon_incarnation: state.daemon_incarnation.clone(),
1924 seq: state.seq,
1925 }
1926 }
1927
1928 fn snapshot(&self) -> SpawnSnapshot {
1929 let state = self.0.lock().unwrap_or_else(|p| p.into_inner());
1930 let mut live = state.live.values().cloned().collect::<Vec<_>>();
1931 live.sort_by(|left, right| left.module_id.cmp(&right.module_id));
1932 SpawnSnapshot {
1933 cursor: Self::cursor(&state),
1934 ring_bound: state.capacity as u64,
1935 live,
1936 }
1937 }
1938
1939 fn emit_spawned(&self, module_id: &str, pid: u32, spawned_at_ms: u64) -> u64 {
1940 let mut state = self.0.lock().unwrap_or_else(|p| p.into_inner());
1941 let generation = state
1942 .generations
1943 .get(module_id)
1944 .copied()
1945 .unwrap_or(0)
1946 .checked_add(1)
1947 .expect("spawn generation exhausted");
1948 state.generations.insert(module_id.to_string(), generation);
1949 let live = LiveSpawn {
1950 module_id: module_id.to_string(),
1951 spawn_generation: generation,
1952 pid,
1953 spawned_at_ms,
1954 };
1955 state.live.insert(module_id.to_string(), live);
1956 Self::emit_locked(
1957 &mut state,
1958 SpawnEventKind::Spawned,
1959 module_id.to_string(),
1960 generation,
1961 pid,
1962 None,
1963 None,
1964 );
1965 generation
1966 }
1967
1968 fn emit_exited(&self, module_id: &str, exit_code: Option<i32>, exit_signal: Option<i32>) {
1969 let mut state = self.0.lock().unwrap_or_else(|p| p.into_inner());
1970 let Some(live) = state.live.remove(module_id) else {
1971 warn!(
1972 module_id,
1973 "terminal record had no live spawn event identity"
1974 );
1975 return;
1976 };
1977 Self::emit_locked(
1978 &mut state,
1979 SpawnEventKind::Exited,
1980 module_id.to_string(),
1981 live.spawn_generation,
1982 live.pid,
1983 exit_code,
1984 exit_signal,
1985 );
1986 }
1987
1988 fn emit_superseded_exited(
1995 &self,
1996 module_id: &str,
1997 spawn_generation: u64,
1998 pid: u32,
1999 exit_code: Option<i32>,
2000 exit_signal: Option<i32>,
2001 ) {
2002 let mut state = self.0.lock().unwrap_or_else(|p| p.into_inner());
2003 if state
2004 .live
2005 .get(module_id)
2006 .is_some_and(|live| live.spawn_generation == spawn_generation)
2007 {
2008 state.live.remove(module_id);
2009 }
2010 Self::emit_locked(
2011 &mut state,
2012 SpawnEventKind::Exited,
2013 module_id.to_string(),
2014 spawn_generation,
2015 pid,
2016 exit_code,
2017 exit_signal,
2018 );
2019 }
2020
2021 #[allow(clippy::too_many_arguments)]
2022 fn emit_locked(
2023 state: &mut SpawnEventState,
2024 kind: SpawnEventKind,
2025 module_id: String,
2026 spawn_generation: u64,
2027 pid: u32,
2028 exit_code: Option<i32>,
2029 exit_signal: Option<i32>,
2030 ) {
2031 state.seq = state
2032 .seq
2033 .checked_add(1)
2034 .expect("spawn event sequence exhausted");
2035 let event = SpawnEvent {
2036 cursor: Self::cursor(state),
2037 kind,
2038 module_id,
2039 spawn_generation,
2040 pid,
2041 exit_code,
2042 exit_signal,
2043 };
2044 state
2045 .process_changes
2046 .retain(|_, changed| changed.receiver_count() > 0);
2047 if let Some(changed) = state.process_changes.get(&event.module_id) {
2048 changed.send_replace(state.seq);
2049 }
2050 state.events.push_back(event.clone());
2051 while state.events.len() > state.capacity {
2052 state.events.pop_front();
2053 }
2054 let body = match serde_json::to_vec(&event) {
2055 Ok(body) => body,
2056 Err(error) => {
2057 error!(%error, "failed to serialize supervisor spawn event");
2058 return;
2059 }
2060 };
2061 state.subscribers.retain(|(connection_id, corr), subscriber| {
2062 let frame = Frame::build_with_version(
2063 subscriber.version,
2064 FrameType::StreamData,
2065 control_flags(),
2066 0,
2067 0,
2068 *corr,
2069 body.clone(),
2070 );
2071 match frame {
2072 Ok(frame) => {
2073 if subscriber.frames.try_send(frame).is_ok() {
2074 true
2075 } else {
2076 warn!(connection_id = connection_id.get(), corr, "dropping lagged supervisor spawn subscriber");
2077 if let Some(lagged) = subscriber.lagged.take() {
2078 let _ = lagged.send(event.cursor.clone());
2079 }
2080 false
2081 }
2082 }
2083 Err(error) => {
2084 warn!(connection_id = connection_id.get(), corr, %error, "dropping supervisor spawn subscriber after frame build failure");
2085 false
2086 }
2087 }
2088 });
2089 }
2090
2091 fn subscribe(
2092 &self,
2093 connection_id: ConnectionId,
2094 corr: u64,
2095 version: u8,
2096 since: Option<SpawnCursor>,
2097 sink: FrameSink,
2098 ) -> Result<(), SpawnSubscribeRefusal> {
2099 let (frames, mut receiver) = mpsc::channel(SPAWN_SUBSCRIBER_BUFFER);
2100 let (lagged, mut lagged_rx) = oneshot::channel::<SpawnCursor>();
2101 {
2102 let mut state = self.0.lock().unwrap_or_else(|p| p.into_inner());
2103 let replay = if let Some(since) = since {
2104 if since.daemon_incarnation != state.daemon_incarnation {
2105 return Err(SpawnSubscribeRefusal::ForeignIncarnation {
2106 current: state.daemon_incarnation.clone(),
2107 });
2108 }
2109 if let Some(oldest) = state.events.front().map(|event| event.cursor.clone()) {
2110 if since.seq < oldest.seq.saturating_sub(1) {
2111 return Err(SpawnSubscribeRefusal::TooOld { oldest });
2112 }
2113 }
2114 state
2115 .events
2116 .iter()
2117 .filter(|event| event.cursor.seq > since.seq)
2118 .cloned()
2119 .collect::<Vec<_>>()
2120 } else {
2121 Vec::new()
2122 };
2123 for event in replay {
2124 let body = serde_json::to_vec(&event)
2125 .map_err(|error| SpawnSubscribeRefusal::Frame(error.to_string()))?;
2126 let frame = Frame::build_with_version(
2127 version,
2128 FrameType::StreamData,
2129 control_flags(),
2130 0,
2131 0,
2132 corr,
2133 body,
2134 )
2135 .map_err(|error| SpawnSubscribeRefusal::Frame(error.to_string()))?;
2136 frames
2137 .try_send(frame)
2138 .map_err(|error| SpawnSubscribeRefusal::Frame(error.to_string()))?;
2139 }
2140 state.subscribers.insert(
2141 (connection_id, corr),
2142 SpawnSubscriber {
2143 version,
2144 frames,
2145 lagged: Some(lagged),
2146 },
2147 );
2148 }
2149 tokio::spawn(async move {
2160 while let Some(frame) = receiver.recv().await {
2161 if sink.send(frame).await.is_err() {
2162 return;
2163 }
2164 }
2165 let Ok(first_undelivered) = lagged_rx.try_recv() else {
2166 return;
2167 };
2168 match spawn_subscriber_lagged_frame(version, corr, first_undelivered) {
2169 Ok(frame) => {
2170 let _ = sink.send(frame).await;
2171 }
2172 Err(error) => {
2173 error!(%error, corr, "failed to build lagged spawn subscriber terminal frame");
2174 }
2175 }
2176 });
2177 Ok(())
2178 }
2179
2180 fn cancel(&self, connection_id: ConnectionId, corr: u64) -> bool {
2181 let Some(subscriber) = self
2182 .0
2183 .lock()
2184 .unwrap_or_else(|p| p.into_inner())
2185 .subscribers
2186 .remove(&(connection_id, corr))
2187 else {
2188 return false;
2189 };
2190 if let Ok(frame) = Frame::build_with_version(
2191 subscriber.version,
2192 FrameType::StreamEnd,
2193 control_flags(),
2194 0,
2195 0,
2196 corr,
2197 Vec::new(),
2198 ) {
2199 tokio::spawn(async move {
2200 let _ = subscriber.frames.send(frame).await;
2201 });
2202 }
2203 true
2204 }
2205
2206 fn remove_connection(&self, connection_id: ConnectionId) {
2207 self.0
2208 .lock()
2209 .unwrap_or_else(|p| p.into_inner())
2210 .subscribers
2211 .retain(|(subscriber_connection, _), _| *subscriber_connection != connection_id);
2212 }
2213
2214 #[cfg(any(test, feature = "test-support"))]
2215 fn set_capacity(&self, capacity: usize) {
2216 self.0.lock().unwrap_or_else(|p| p.into_inner()).capacity = capacity;
2217 }
2218
2219 #[cfg(any(test, feature = "test-support"))]
2220 fn subscriber_count(&self) -> usize {
2221 self.0
2222 .lock()
2223 .unwrap_or_else(|p| p.into_inner())
2224 .subscribers
2225 .len()
2226 }
2227}
2228
2229fn spawn_subscriber_lagged_frame(
2232 version: u8,
2233 corr: u64,
2234 first_undelivered: SpawnCursor,
2235) -> Result<Frame, String> {
2236 let body = serde_json::to_vec(&subc_protocol::ErrorBody {
2238 code: SPAWN_SUBSCRIBER_LAGGED_CODE.to_string(),
2239 message: "spawn subscriber fell behind and was dropped; resubscribe from the last cursor received"
2240 .to_string(),
2241 detail: Some(serde_json::json!({
2242 "first_undelivered_cursor": first_undelivered
2243 })),
2244 })
2245 .map_err(|error| error.to_string())?;
2246 Frame::build_with_version(
2247 version,
2248 FrameType::Error,
2249 crate::router::daemon_error_flags(),
2250 0,
2251 0,
2252 corr,
2253 body,
2254 )
2255 .map_err(|error| error.to_string())
2256}
2257
2258pub trait ModuleProcessLiveness: Send + Sync {
2259 fn process_live(&self, module_id: &str) -> Option<bool>;
2260
2261 fn process_replacing(&self, _module_id: &str) -> bool {
2267 false
2268 }
2269}
2270
2271#[derive(Debug, Clone, Default)]
2273pub struct SupervisorProcessLiveness {
2274 snapshots: Arc<Mutex<HashMap<String, SharedSnapshot>>>,
2275}
2276
2277impl SupervisorProcessLiveness {
2278 pub fn new() -> Self {
2279 Self::default()
2280 }
2281
2282 fn track(&self, module_id: String, snapshot: SharedSnapshot) {
2283 let mut snapshots = self
2284 .snapshots
2285 .lock()
2286 .unwrap_or_else(|poisoned| poisoned.into_inner());
2287 snapshots.insert(module_id, snapshot);
2288 }
2289
2290 fn untrack_if_current(&self, module_id: &str, snapshot: &SharedSnapshot) {
2291 let mut snapshots = self
2292 .snapshots
2293 .lock()
2294 .unwrap_or_else(|poisoned| poisoned.into_inner());
2295 let is_current = snapshots
2296 .get(module_id)
2297 .map(|tracked| Arc::ptr_eq(tracked, snapshot))
2298 .unwrap_or(false);
2299 if is_current {
2300 snapshots.remove(module_id);
2301 }
2302 }
2303}
2304
2305impl ModuleProcessLiveness for SupervisorProcessLiveness {
2306 fn process_live(&self, module_id: &str) -> Option<bool> {
2307 let snapshot = {
2308 let snapshots = self
2309 .snapshots
2310 .lock()
2311 .unwrap_or_else(|poisoned| poisoned.into_inner());
2312 snapshots.get(module_id).cloned()
2313 }?;
2314 let snapshot = snapshot
2315 .lock()
2316 .unwrap_or_else(|poisoned| poisoned.into_inner());
2317 Some(snapshot.state == ModuleState::Running && snapshot.process_alive)
2318 }
2319
2320 fn process_replacing(&self, module_id: &str) -> bool {
2321 let Some(snapshot) = self
2322 .snapshots
2323 .lock()
2324 .unwrap_or_else(|poisoned| poisoned.into_inner())
2325 .get(module_id)
2326 .cloned()
2327 else {
2328 return false;
2329 };
2330 let snapshot = snapshot
2331 .lock()
2332 .unwrap_or_else(|poisoned| poisoned.into_inner());
2333 snapshot.enabled
2334 && match snapshot.state {
2335 ModuleState::Restarting => true,
2336 ModuleState::Draining => snapshot.draining_to_replace,
2337 ModuleState::Starting
2338 | ModuleState::Running
2339 | ModuleState::Unresponsive
2340 | ModuleState::Stopped
2341 | ModuleState::Failed
2342 | ModuleState::Disabled => false,
2343 }
2344 }
2345}
2346
2347#[cfg(test)]
2348#[derive(Debug, Default)]
2349struct ReloadExitRecordGate {
2350 reached: tokio::sync::Notify,
2351 resume: tokio::sync::Notify,
2352}
2353
2354#[derive(Debug, Clone, Copy)]
2355enum RespawnKind {
2356 Spawn,
2357 Reload,
2358}
2359
2360#[derive(Debug, Clone)]
2361struct PendingRespawn {
2362 deadline: Instant,
2363 kind: RespawnKind,
2364 daemon_initiated: bool,
2365 approval: Option<crate::config_gate::runtime::ApprovalIdentity>,
2366}
2367
2368impl PendingRespawn {
2369 fn failure_context(&self, runtime: &SupervisorRuntimeConfig) -> FailureContext {
2370 FailureContext {
2371 approval: self.approval.clone(),
2372 ..runtime.failure_context(self.daemon_initiated)
2373 }
2374 }
2375}
2376
2377type ReloadReply = oneshot::Sender<Result<(), SuperviseError>>;
2378
2379#[derive(Debug, Clone)]
2380struct SupervisorRuntimeConfig {
2381 registry: Arc<Registry>,
2382 config_delivery: Option<config_delivery::ConfigDelivery>,
2383 recovery_is_daemon: bool,
2385 start_requester: Option<Principal>,
2386 pending_activity_restart: bool,
2387 provenance_probe: ExecutableIdentityProbe,
2388 scheduled_respawn: Arc<Mutex<Option<PendingRespawn>>>,
2390 deferred_reload_reply: Arc<Mutex<Option<ReloadReply>>>,
2392 restart_policy: RestartPolicy,
2393 register_timeout: Duration,
2394 drain_timeout: Duration,
2397 effective_drain_timeout: Arc<Mutex<Duration>>,
2400 default_drain_timeout: Duration,
2403 health: HealthConfig,
2404 host_health: HostHealth,
2405 connection_file_path: Option<PathBuf>,
2406 capture_logs_dir: Option<PathBuf>,
2407 forwarding: Option<Arc<ForwardingTable>>,
2408 supervisor_handle: Option<SupervisorHandle>,
2411 stderr_ring: Arc<Mutex<StderrRing>>,
2418 terminal_ring: Arc<Mutex<TerminalRing>>,
2419 spawn_events: SpawnEventFeed,
2420 child_roster: ChildRoster,
2421 #[cfg(target_os = "linux")]
2422 cgroup_placement: Option<subc_cgroup::Placement>,
2423 #[cfg(test)]
2424 test_seed_stale_facts_before_enable_spawn: bool,
2425 #[cfg(test)]
2426 test_reload_exit_record_gate: Option<Arc<ReloadExitRecordGate>>,
2427}
2428
2429impl SupervisorRuntimeConfig {
2430 fn failure_context(&self, daemon_initiated: bool) -> FailureContext {
2431 FailureContext {
2432 daemon_initiated,
2433 approval: None,
2434 shutdown: self.child_roster.shutdown_flag(),
2435 }
2436 }
2437}
2438
2439#[derive(Debug, Clone, PartialEq, Eq)]
2440struct SupervisedConfiguration {
2441 spec: ModuleSpec,
2442 enabled: bool,
2443 health: HealthConfig,
2444 register_timeout: Duration,
2445 restart_policy: RestartPolicy,
2446}
2447
2448#[derive(Debug, Clone, Default)]
2454pub struct SupervisorHandle {
2455 #[cfg(test)]
2456 resource_commit_pause: Arc<Mutex<Option<resource_launch::ResourceCommitPause>>>,
2457 #[cfg(test)]
2458 registration_test_channels: CapturedRegistrationChannels,
2459 registration_service: Arc<Mutex<Option<registration::RegistrationService>>>,
2460 registration_connections: Arc<Mutex<HashMap<ConnectionId, Arc<RegistrationSpawn>>>>,
2461 registration_launches: Arc<Mutex<Vec<std::sync::Weak<RegistrationSpawn>>>>,
2462 gate_runtime: Arc<Mutex<Option<std::sync::Weak<crate::config_gate::runtime::RuntimeOwner>>>>,
2463 program_bindings: program_binding::ProgramBindings,
2464 #[cfg(test)]
2465 identity_sample_pause: Arc<Mutex<Option<IdentitySamplePause>>>,
2466 #[cfg(all(test, unix))]
2467 plain_identity_policy_pause: Arc<Mutex<Option<(String, IdentitySamplePause)>>>,
2468 host_health: HostHealth,
2469 restart_intent: Arc<Mutex<DaemonRestartIntent>>,
2470 activity: Arc<Mutex<Option<crate::activity::ActivityLedger>>>,
2471 activity_images: Arc<Mutex<HashMap<String, activity::ActivityImage>>>,
2472 modules: Arc<Mutex<HashMap<String, SupervisedModule>>>,
2473 configured_ids: Arc<Mutex<HashSet<String>>>,
2485 spawn_events: SpawnEventFeed,
2486 reserved_nonces: Arc<Mutex<HashMap<String, Option<String>>>>,
2497 implicit_protocol_none: Arc<Mutex<HashSet<String>>>,
2501 removal_tombstones: Arc<Mutex<HashMap<String, u64>>>,
2507 spawn_nonces: Arc<Mutex<HashMap<String, String>>>,
2511 reserved_prefix_owners: Arc<Mutex<HashMap<String, String>>>,
2519 swaps: Arc<Mutex<HashMap<String, OpenSwap>>>,
2525 promotion_observer: PromotionObserverSlot,
2527 operation_lock: Arc<AsyncMutex<()>>,
2531}
2532
2533#[cfg(test)]
2534#[derive(Debug)]
2535struct IdentitySamplePause {
2536 reached: std::sync::mpsc::Sender<()>,
2537 release: std::sync::mpsc::Receiver<()>,
2538}
2539
2540pub(crate) trait SwapPromotionObserver: Send + Sync {
2549 fn swap_promoted(&self, registration: &crate::registry::ModuleRegistration);
2550}
2551
2552#[derive(Clone, Default)]
2556struct PromotionObserverSlot(Arc<Mutex<Option<std::sync::Weak<dyn SwapPromotionObserver>>>>);
2557
2558impl fmt::Debug for PromotionObserverSlot {
2559 fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
2560 f.write_str("PromotionObserverSlot")
2561 }
2562}
2563
2564#[derive(Debug, Clone)]
2566struct OpenSwap {
2567 candidate_nonce: String,
2570 incumbent_nonce: Option<String>,
2573 candidate_admitted: bool,
2577}
2578
2579#[derive(Debug, Clone, Copy, PartialEq, Eq)]
2582pub(crate) enum SwapHelloAdmission {
2583 NotSwapping,
2586 Candidate,
2588 Refused,
2591 ProtocolNone,
2594 ChannelRequired,
2597}
2598
2599#[derive(Debug, Clone, PartialEq, Eq)]
2600pub(crate) enum ReservedHelloRejection {
2601 Exact {
2602 module_id: String,
2603 },
2604 Prefix {
2605 prefix: String,
2606 owner_module_id: String,
2607 },
2608}
2609
2610#[derive(Debug, Default)]
2612pub(crate) struct ReservedHelloVerification {
2613 pub(crate) nonce_verified: bool,
2614 pub(crate) prefix_owner: Option<String>,
2615}
2616
2617#[derive(Debug)]
2618struct DaemonRestartIntent {
2619 expires: Option<Duration>,
2620 window: Duration,
2621}
2622
2623impl Default for DaemonRestartIntent {
2624 fn default() -> Self {
2625 Self {
2626 expires: None,
2627 window: Duration::from_secs(120),
2628 }
2629 }
2630}
2631
2632impl SupervisorHandle {
2633 pub(crate) fn module_is_gated(&self, module: &str) -> bool {
2634 let runtime = {
2635 let Ok(runtime) = self.gate_runtime.lock() else {
2636 return true;
2637 };
2638 runtime.clone()
2639 };
2640 runtime.is_some_and(|runtime| {
2641 runtime
2642 .upgrade()
2643 .is_none_or(|owner| owner.module_is_gated(module))
2644 })
2645 }
2646 pub(crate) fn announce_daemon_restart(&self) -> u64 {
2647 let mut intent = self
2648 .restart_intent
2649 .lock()
2650 .unwrap_or_else(|p| p.into_inner());
2651 intent.expires = Some(subc_uptime::suspend_inclusive_now() + intent.window);
2652 unix_ms_now().saturating_add(intent.window.as_millis() as u64)
2653 }
2654
2655 #[cfg(feature = "test-support")]
2656 pub(crate) fn set_shutdown_intent_window(&self, window: Duration) {
2657 self.restart_intent
2658 .lock()
2659 .unwrap_or_else(|p| p.into_inner())
2660 .window = window;
2661 }
2662
2663 #[cfg(unix)]
2664 fn take_daemon_shutdown_reason(&self) -> RouteCloseReason {
2665 let expiry = self
2666 .restart_intent
2667 .lock()
2668 .unwrap_or_else(|p| p.into_inner())
2669 .expires
2670 .take();
2671 if expiry.is_some_and(|expiry| subc_uptime::suspend_inclusive_now() < expiry) {
2672 RouteCloseReason::Restart
2673 } else {
2674 RouteCloseReason::Disable
2675 }
2676 }
2677
2678 pub fn new() -> Self {
2679 Self::default()
2680 }
2681
2682 pub(crate) fn spawn_snapshot(&self) -> SpawnSnapshot {
2683 self.spawn_events.snapshot()
2684 }
2685
2686 pub(crate) fn subscribe_process_changes(
2689 &self,
2690 module: &str,
2691 ) -> tokio::sync::watch::Receiver<u64> {
2692 let mut state = self
2693 .spawn_events
2694 .0
2695 .lock()
2696 .unwrap_or_else(|p| p.into_inner());
2697 state
2698 .process_changes
2699 .retain(|_, changed| changed.receiver_count() > 0);
2700 let seq = state.seq;
2701 state
2702 .process_changes
2703 .entry(module.into())
2704 .or_insert_with(|| tokio::sync::watch::channel(seq).0)
2705 .subscribe()
2706 }
2707
2708 pub(crate) fn current_spawn(&self, module: &str) -> Option<LiveSpawn> {
2709 self.spawn_events
2710 .0
2711 .lock()
2712 .unwrap_or_else(|p| p.into_inner())
2713 .live
2714 .get(module)
2715 .cloned()
2716 }
2717
2718 pub(crate) fn subscribe_spawns(
2719 &self,
2720 connection_id: ConnectionId,
2721 corr: u64,
2722 version: u8,
2723 since: Option<SpawnCursor>,
2724 sink: FrameSink,
2725 ) -> Result<(), SpawnSubscribeRefusal> {
2726 self.spawn_events
2727 .subscribe(connection_id, corr, version, since, sink)
2728 }
2729
2730 pub(crate) fn cancel_spawn_subscription(&self, connection_id: ConnectionId, corr: u64) -> bool {
2731 self.spawn_events.cancel(connection_id, corr)
2732 }
2733
2734 pub(crate) fn remove_spawn_subscribers(&self, connection_id: ConnectionId) {
2735 self.spawn_events.remove_connection(connection_id);
2736 }
2737
2738 #[cfg(any(test, feature = "test-support"))]
2739 pub fn set_spawn_event_capacity_for_test(&self, capacity: usize) {
2740 assert!(capacity > 0, "spawn event capacity must be non-zero");
2741 self.spawn_events.set_capacity(capacity);
2742 }
2743
2744 #[cfg(any(test, feature = "test-support"))]
2745 pub fn spawn_subscriber_count_for_test(&self) -> usize {
2746 self.spawn_events.subscriber_count()
2747 }
2748
2749 pub fn set_spawn_nonce(&self, module_id: &str, nonce: String) {
2752 self.spawn_nonces
2753 .lock()
2754 .unwrap_or_else(|poisoned| poisoned.into_inner())
2755 .insert(module_id.to_string(), nonce);
2756 self.refresh_program_binding_nonces(module_id);
2757 }
2758
2759 pub fn set_reserved_nonce(&self, module_id: &str, nonce: String) {
2762 {
2763 let mut reserved = self
2764 .reserved_nonces
2765 .lock()
2766 .unwrap_or_else(|p| p.into_inner());
2767 self.implicit_protocol_none
2768 .lock()
2769 .unwrap_or_else(|p| p.into_inner())
2770 .remove(module_id);
2771 reserved.insert(module_id.to_string(), Some(nonce));
2772 }
2773 self.refresh_program_binding_nonces(module_id);
2774 }
2775
2776 pub fn set_reserved_prefixes(&self, owner_module_id: &str, prefixes: &[String]) {
2778 let mut owners = self
2779 .reserved_prefix_owners
2780 .lock()
2781 .unwrap_or_else(|poisoned| poisoned.into_inner());
2782 owners.retain(|_, owner| owner != owner_module_id);
2783 for prefix in prefixes {
2784 owners.insert(prefix.clone(), owner_module_id.to_string());
2785 }
2786 }
2787
2788 pub(crate) fn spawn_nonce(&self, module_id: &str) -> Option<String> {
2790 if self.module_is_gated(module_id) {
2791 return None;
2792 }
2793 self.spawn_nonces
2794 .lock()
2795 .unwrap_or_else(|poisoned| poisoned.into_inner())
2796 .get(module_id)
2797 .cloned()
2798 }
2799
2800 #[cfg(all(test, unix))]
2801 pub(crate) fn legacy_nonce_present_for_test(&self, module: &str) -> bool {
2802 self.spawn_nonces.lock().unwrap().contains_key(module)
2803 }
2804
2805 pub(crate) fn apply_identity_configuration(&self, spec: &ModuleSpec) {
2806 self.set_reserved_prefixes(&spec.module_id, &spec.reserved_prefixes);
2809 let protocol = self
2810 .get(&spec.module_id)
2811 .and_then(|module| {
2812 let snapshot = module.inner.snapshot.lock().ok()?;
2813 snapshot
2814 .process_alive
2815 .then_some(snapshot.spawned_protocol.unwrap_or(spec.protocol))
2816 })
2817 .unwrap_or(spec.protocol);
2818 self.apply_identity_policy(spec, protocol);
2819 }
2820
2821 #[cfg(all(test, unix))]
2822 pub(crate) fn pause_plain_identity_policy_for_test(
2823 &self,
2824 module: &str,
2825 ) -> (std::sync::mpsc::Receiver<()>, std::sync::mpsc::Sender<()>) {
2826 let (reached, waiting) = std::sync::mpsc::channel();
2827 let (release, paused) = std::sync::mpsc::channel();
2828 *self.plain_identity_policy_pause.lock().unwrap() = Some((
2829 module.to_owned(),
2830 IdentitySamplePause {
2831 reached,
2832 release: paused,
2833 },
2834 ));
2835 (waiting, release)
2836 }
2837
2838 fn apply_identity_policy(&self, spec: &ModuleSpec, protocol: ModuleProtocol) {
2839 let channel_only = self.module_is_gated(&spec.module_id);
2840 let mut current_nonces = self
2841 .spawn_nonces
2842 .lock()
2843 .unwrap_or_else(|poisoned| poisoned.into_inner());
2844 if protocol == ModuleProtocol::None || channel_only {
2845 current_nonces.remove(&spec.module_id);
2846 }
2847 let spawn_nonce = current_nonces.get(&spec.module_id).cloned();
2848 #[cfg(test)]
2849 if let Some(pause) = self.identity_sample_pause.lock().unwrap().take() {
2850 pause.reached.send(()).unwrap();
2851 pause.release.recv().unwrap();
2852 }
2853 let mut reserved_nonces = self
2854 .reserved_nonces
2855 .lock()
2856 .unwrap_or_else(|poisoned| poisoned.into_inner());
2857 let mut implicit = self
2858 .implicit_protocol_none
2859 .lock()
2860 .unwrap_or_else(|p| p.into_inner());
2861 if spec.reserved {
2862 implicit.remove(&spec.module_id);
2863 } else if protocol == ModuleProtocol::None {
2864 if !reserved_nonces.contains_key(&spec.module_id) {
2865 implicit.insert(spec.module_id.clone());
2866 }
2867 } else if implicit.remove(&spec.module_id) {
2868 reserved_nonces.remove(&spec.module_id);
2869 }
2870 if spec.reserved || protocol == ModuleProtocol::None {
2871 reserved_nonces.insert(spec.module_id.clone(), spawn_nonce);
2876 }
2877 drop(implicit);
2878 drop(reserved_nonces);
2879 drop(current_nonces);
2880 self.refresh_program_binding_nonces(&spec.module_id);
2881 self.removal_tombstones
2885 .lock()
2886 .unwrap_or_else(|poisoned| poisoned.into_inner())
2887 .remove(&spec.module_id);
2888 }
2889
2890 pub fn reserved_hello_authorized(&self, module_id: &str, presented: Option<&str>) -> bool {
2895 self.reserved_hello_verification(module_id, presented)
2896 .is_ok()
2897 }
2898
2899 pub(crate) fn reserved_hello_verification(
2902 &self,
2903 module_id: &str,
2904 presented: Option<&str>,
2905 ) -> Result<ReservedHelloVerification, ReservedHelloRejection> {
2906 if self.module_is_gated(module_id) {
2907 return Err(ReservedHelloRejection::Exact {
2908 module_id: module_id.into(),
2909 });
2910 }
2911 self.hello_verification(module_id, presented, false)
2912 }
2913
2914 pub(crate) fn channel_hello_verification(
2915 &self,
2916 module_id: &str,
2917 ) -> Result<ReservedHelloVerification, ReservedHelloRejection> {
2918 if self
2919 .get(module_id)
2920 .is_some_and(|module| module.declared_protocol().ok() == Some(ModuleProtocol::None))
2921 {
2922 return Err(ReservedHelloRejection::Exact {
2923 module_id: module_id.into(),
2924 });
2925 }
2926 let mut verification = self.hello_verification(module_id, None, true)?;
2927 verification.nonce_verified = true;
2928 Ok(verification)
2929 }
2930
2931 pub(crate) fn prefix_claim_authorized(&self, owner: &str, module_id: &str) -> bool {
2932 self.prefix_claim_publication(owner, module_id).is_some()
2933 }
2934
2935 #[cfg(all(test, unix))]
2936 pub(crate) fn prefix_publication_locked_for_test(&self) -> bool {
2937 matches!(
2938 self.reserved_prefix_owners.try_lock(),
2939 Err(std::sync::TryLockError::WouldBlock)
2940 )
2941 }
2942
2943 pub(crate) fn prefix_claim_publication(
2947 &self,
2948 owner: &str,
2949 module_id: &str,
2950 ) -> Option<std::sync::MutexGuard<'_, HashMap<String, String>>> {
2951 if subc_protocol::manifest::validate_module_id(module_id).is_err()
2952 || self.module_is_gated(module_id)
2953 || self
2954 .reserved_nonces
2955 .lock()
2956 .unwrap_or_else(|p| p.into_inner())
2957 .contains_key(module_id)
2958 {
2959 return None;
2960 }
2961 let owners = self
2962 .reserved_prefix_owners
2963 .lock()
2964 .unwrap_or_else(|p| p.into_inner());
2965 let authorized = owners
2966 .iter()
2967 .filter(|(prefix, _)| module_id.starts_with(prefix.as_str()))
2968 .max_by_key(|(prefix, _)| prefix.len())
2969 .is_some_and(|(prefix, holder)| holder == owner && module_id.len() > prefix.len());
2970 authorized.then_some(owners)
2971 }
2972
2973 fn hello_verification(
2974 &self,
2975 module_id: &str,
2976 presented: Option<&str>,
2977 channel: bool,
2978 ) -> Result<ReservedHelloVerification, ReservedHelloRejection> {
2979 let nonces = self
2980 .reserved_nonces
2981 .lock()
2982 .unwrap_or_else(|poisoned| poisoned.into_inner());
2983 if let Some(expected) = nonces.get(module_id) {
2984 let authorized = channel
2988 || match expected {
2989 Some(expected) => presented.is_some_and(|p| {
2990 !p.is_empty() && constant_time_eq(expected.as_bytes(), p.as_bytes())
2991 }),
2992 None => false,
2993 };
2994 if authorized {
2995 return Ok(ReservedHelloVerification {
2996 nonce_verified: true,
2997 prefix_owner: None,
2998 });
2999 }
3000 return Err(ReservedHelloRejection::Exact {
3001 module_id: module_id.to_string(),
3002 });
3003 }
3004 drop(nonces);
3005
3006 let matched_prefix = self
3007 .reserved_prefix_owners
3008 .lock()
3009 .unwrap_or_else(|poisoned| poisoned.into_inner())
3010 .iter()
3011 .filter(|(prefix, _)| module_id.starts_with(prefix.as_str()))
3012 .max_by_key(|(prefix, _)| prefix.len())
3013 .map(|(prefix, owner)| (prefix.clone(), owner.clone()));
3014 let Some((prefix, owner_module_id)) = matched_prefix else {
3015 return Ok(ReservedHelloVerification::default());
3016 };
3017
3018 if module_id.len() == prefix.len()
3019 || (!channel
3020 && (self.module_is_gated(&owner_module_id) || self.protocol_none(&owner_module_id)))
3021 {
3022 return Err(ReservedHelloRejection::Prefix {
3023 prefix,
3024 owner_module_id,
3025 });
3026 }
3027
3028 let authorized = if channel {
3029 owner_module_id == module_id
3030 } else {
3031 presented.is_some_and(|presented| {
3032 self.spawn_nonces
3033 .lock()
3034 .unwrap_or_else(|poisoned| poisoned.into_inner())
3035 .get(&owner_module_id)
3036 .is_some_and(|expected| !presented.is_empty() && constant_time_eq(expected.as_bytes(), presented.as_bytes()))
3037 || self.swap_nonce_matches(&owner_module_id, presented)
3040 })
3041 };
3042 if authorized {
3043 Ok(ReservedHelloVerification {
3044 nonce_verified: true,
3045 prefix_owner: (owner_module_id != module_id).then_some(owner_module_id),
3046 })
3047 } else {
3048 Err(ReservedHelloRejection::Prefix {
3049 prefix,
3050 owner_module_id,
3051 })
3052 }
3053 }
3054
3055 pub fn spawned_consumer_authorized(&self, module_id: &str, presented: &str) -> bool {
3060 if self.module_is_gated(module_id) || self.protocol_none(module_id) {
3061 return false;
3062 }
3063 let nonces = self
3064 .spawn_nonces
3065 .lock()
3066 .unwrap_or_else(|poisoned| poisoned.into_inner());
3067 let current = nonces.get(module_id).is_some_and(|expected| {
3068 !presented.is_empty() && constant_time_eq(expected.as_bytes(), presented.as_bytes())
3069 });
3070 drop(nonces);
3071 current || self.swap_nonce_matches(module_id, presented)
3074 }
3075
3076 fn swap_nonce_matches(&self, module_id: &str, presented: &str) -> bool {
3078 if self.module_is_gated(module_id) || self.protocol_none(module_id) || presented.is_empty()
3079 {
3080 return false;
3081 }
3082 let swaps = self
3083 .swaps
3084 .lock()
3085 .unwrap_or_else(|poisoned| poisoned.into_inner());
3086 swaps.get(module_id).is_some_and(|swap| {
3087 constant_time_eq(swap.candidate_nonce.as_bytes(), presented.as_bytes())
3088 || swap.incumbent_nonce.as_deref().is_some_and(|incumbent| {
3089 constant_time_eq(incumbent.as_bytes(), presented.as_bytes())
3090 })
3091 })
3092 }
3093
3094 pub(crate) fn open_swap(&self, module_id: &str, candidate_nonce: String) {
3097 let incumbent_nonce = self
3098 .spawn_nonces
3099 .lock()
3100 .unwrap_or_else(|poisoned| poisoned.into_inner())
3101 .get(module_id)
3102 .cloned();
3103 self.swaps
3104 .lock()
3105 .unwrap_or_else(|poisoned| poisoned.into_inner())
3106 .insert(
3107 module_id.to_string(),
3108 OpenSwap {
3109 candidate_nonce,
3110 incumbent_nonce,
3111 candidate_admitted: false,
3112 },
3113 );
3114 self.refresh_program_binding_nonces(module_id);
3115 }
3116
3117 pub(crate) fn close_swap(&self, module_id: &str) {
3120 self.swaps
3121 .lock()
3122 .unwrap_or_else(|poisoned| poisoned.into_inner())
3123 .remove(module_id);
3124 self.refresh_program_binding_nonces(module_id);
3125 }
3126
3127 pub(crate) fn set_swap_promotion_observer(
3130 &self,
3131 observer: std::sync::Weak<dyn SwapPromotionObserver>,
3132 ) {
3133 *self
3134 .promotion_observer
3135 .0
3136 .lock()
3137 .unwrap_or_else(|poisoned| poisoned.into_inner()) = Some(observer);
3138 }
3139
3140 fn notify_swap_promoted(&self, registration: &crate::registry::ModuleRegistration) {
3143 let observer = self
3144 .promotion_observer
3145 .0
3146 .lock()
3147 .unwrap_or_else(|poisoned| poisoned.into_inner())
3148 .as_ref()
3149 .and_then(std::sync::Weak::upgrade);
3150 if let Some(observer) = observer {
3151 observer.swap_promoted(registration);
3152 }
3153 }
3154
3155 pub(crate) fn swap_open(&self, module_id: &str) -> bool {
3157 self.swaps
3158 .lock()
3159 .unwrap_or_else(|poisoned| poisoned.into_inner())
3160 .contains_key(module_id)
3161 }
3162
3163 fn promote_swap_nonce(&self, module_id: &str, reserved: bool) {
3168 let candidate_nonce = self
3169 .swaps
3170 .lock()
3171 .unwrap_or_else(|poisoned| poisoned.into_inner())
3172 .get_mut(module_id)
3173 .map(|swap| {
3174 swap.incumbent_nonce = None;
3175 swap.candidate_nonce.clone()
3176 });
3177 self.spawn_nonces
3178 .lock()
3179 .unwrap_or_else(|p| p.into_inner())
3180 .remove(module_id);
3181 {
3182 let mut nonces = self
3183 .reserved_nonces
3184 .lock()
3185 .unwrap_or_else(|p| p.into_inner());
3186 if reserved || nonces.contains_key(module_id) {
3187 nonces.insert(module_id.into(), None);
3188 }
3189 }
3190 if self.module_is_gated(module_id) {
3191 return;
3192 }
3193 let Some(nonce) = candidate_nonce.filter(|nonce| !nonce.is_empty()) else {
3194 return;
3195 };
3196 self.set_spawn_nonce(module_id, nonce.clone());
3197 if reserved {
3198 self.set_reserved_nonce(module_id, nonce);
3199 }
3200 }
3201
3202 pub(crate) fn swap_hello_admission(
3217 &self,
3218 module_id: &str,
3219 presented: Option<&str>,
3220 ) -> SwapHelloAdmission {
3221 if self.protocol_none(module_id) {
3222 return SwapHelloAdmission::ProtocolNone;
3223 }
3224 if self.module_is_gated(module_id) {
3225 return SwapHelloAdmission::ChannelRequired;
3226 }
3227 let swaps = self
3228 .swaps
3229 .lock()
3230 .unwrap_or_else(|poisoned| poisoned.into_inner());
3231 let Some(swap) = swaps.get(module_id) else {
3232 return SwapHelloAdmission::NotSwapping;
3233 };
3234 let Some(presented) = presented.filter(|nonce| !nonce.is_empty()) else {
3235 return SwapHelloAdmission::Refused;
3236 };
3237 if constant_time_eq(swap.candidate_nonce.as_bytes(), presented.as_bytes()) {
3238 return if swap.candidate_admitted {
3239 SwapHelloAdmission::Refused
3240 } else {
3241 SwapHelloAdmission::Candidate
3242 };
3243 }
3244 if swap
3245 .incumbent_nonce
3246 .as_deref()
3247 .is_some_and(|incumbent| constant_time_eq(incumbent.as_bytes(), presented.as_bytes()))
3248 {
3249 return SwapHelloAdmission::NotSwapping;
3250 }
3251 SwapHelloAdmission::Refused
3252 }
3253
3254 fn protocol_none(&self, module: &str) -> bool {
3255 self.get(module)
3256 .is_some_and(|module| module.declared_protocol().ok() == Some(ModuleProtocol::None))
3257 }
3258
3259 pub(crate) fn with_swap_candidate_admission<T>(
3261 &self,
3262 module_id: &str,
3263 publish: impl FnOnce() -> (T, bool),
3264 ) -> Option<T> {
3265 let mut swaps = self.swaps.lock().unwrap_or_else(|p| p.into_inner());
3266 let swap = swaps
3267 .get_mut(module_id)
3268 .filter(|swap| !swap.candidate_admitted)?;
3269 let (result, admitted) = publish();
3270 swap.candidate_admitted = admitted;
3271 Some(result)
3272 }
3273
3274 pub fn spawn_launch_nonce_for(&self, module_id: &str) -> Option<String> {
3276 self.spawn_nonce(module_id)
3277 }
3278
3279 pub fn reserved_launch_nonce_for(&self, module_id: &str) -> Option<String> {
3281 if self.module_is_gated(module_id) {
3282 return None;
3283 }
3284 self.reserved_nonces
3285 .lock()
3286 .unwrap_or_else(|poisoned| poisoned.into_inner())
3287 .get(module_id)
3288 .cloned()
3289 .flatten()
3290 }
3291
3292 pub fn insert(&self, module: SupervisedModule) -> Option<SupervisedModule> {
3293 self.mark_configured(module.module_id());
3297 let mut modules = self
3298 .modules
3299 .lock()
3300 .unwrap_or_else(|poisoned| poisoned.into_inner());
3301 modules.insert(module.module_id().to_string(), module)
3302 }
3303
3304 fn mark_configured(&self, module_id: &str) {
3308 self.update_configured_ids(|ids| {
3309 ids.insert(module_id.to_string());
3310 });
3311 }
3312
3313 fn publish_configured_ids(&self, ids: Option<&HashSet<String>>) {
3314 let owner = self
3315 .gate_runtime
3316 .lock()
3317 .unwrap_or_else(|p| p.into_inner())
3318 .clone()
3319 .and_then(|owner| owner.upgrade());
3320 if let Some(owner) = owner {
3321 owner.observe_supervisor_configured_ids(ids, Arc::downgrade(&self.configured_ids));
3322 }
3323 }
3324
3325 fn update_configured_ids(&self, update: impl FnOnce(&mut HashSet<String>)) {
3326 match self.configured_ids.lock() {
3327 Ok(mut ids) => {
3328 update(&mut ids);
3329 self.publish_configured_ids(Some(&ids));
3332 }
3333 Err(poisoned) => {
3334 update(&mut poisoned.into_inner());
3335 self.publish_configured_ids(None);
3336 }
3337 }
3338 }
3339
3340 fn unmark_configured_unless_rostered(&self, module_id: &str) {
3344 let modules = self
3345 .modules
3346 .lock()
3347 .unwrap_or_else(|poisoned| poisoned.into_inner());
3348 if !modules.contains_key(module_id) {
3349 self.update_configured_ids(|ids| {
3350 ids.remove(module_id);
3351 });
3352 }
3353 }
3354
3355 pub(crate) fn is_configured(&self, module_id: &str) -> bool {
3363 self.configured_ids
3364 .lock()
3365 .unwrap_or_else(|poisoned| poisoned.into_inner())
3366 .contains(module_id)
3367 }
3368
3369 pub fn get(&self, module_id: &str) -> Option<SupervisedModule> {
3370 let modules = self
3371 .modules
3372 .lock()
3373 .unwrap_or_else(|poisoned| poisoned.into_inner());
3374 modules.get(module_id).cloned()
3375 }
3376
3377 pub(crate) fn record_late_health_answer(
3378 &self,
3379 module_id: &str,
3380 latency_ms: u64,
3381 ) -> Result<bool, SuperviseError> {
3382 let Some(module) = self.get(module_id) else {
3383 return Ok(false);
3384 };
3385 self.host_health.record_late_module(module_id);
3386 update_snapshot(&module.inner.snapshot, Some(module_id), |state| {
3387 state.health.late_answer_count = state.health.late_answer_count.saturating_add(1);
3388 state.health.last_late_answer_latency_ms = Some(latency_ms);
3389 state.health.consecutive_failures = 0;
3397 })?;
3398 Ok(true)
3399 }
3400
3401 pub(crate) fn spawn_host_health_monitor(&self) -> JoinHandle<()> {
3402 self.host_health.spawn_monitor()
3403 }
3404
3405 pub fn record_deliberate_severance(&self, module_id: &str) -> Result<bool, SuperviseError> {
3411 let Some(module) = self.get(module_id) else {
3412 return Ok(false);
3413 };
3414 let snapshot = lock_snapshot(&module.inner.snapshot)?;
3415 let Some((pid, start_time)) = snapshot.pid.zip(snapshot.process_start_time) else {
3416 return Ok(false);
3417 };
3418 drop(snapshot);
3419 module.record_deliberate_severance(ProcessIdentity { pid, start_time })
3420 }
3421
3422 pub fn list(&self) -> Vec<SupervisedModule> {
3423 let modules = self
3424 .modules
3425 .lock()
3426 .unwrap_or_else(|poisoned| poisoned.into_inner());
3427 let mut modules = modules.values().cloned().collect::<Vec<_>>();
3428 modules.sort_by(|left, right| left.module_id().cmp(right.module_id()));
3429 modules
3430 }
3431
3432 pub(crate) fn retire(&self, module_id: &str) -> Option<SupervisedModule> {
3433 self.activity_images
3434 .lock()
3435 .unwrap_or_else(|p| p.into_inner())
3436 .remove(module_id);
3437 self.spawn_nonces
3438 .lock()
3439 .unwrap_or_else(|poisoned| poisoned.into_inner())
3440 .remove(module_id);
3441 self.close_swap(module_id);
3442 let mut reserved_nonces = self
3443 .reserved_nonces
3444 .lock()
3445 .unwrap_or_else(|poisoned| poisoned.into_inner());
3446 if reserved_nonces.contains_key(module_id) {
3447 reserved_nonces.insert(module_id.to_string(), None);
3450 }
3451 drop(reserved_nonces);
3452 self.reserved_prefix_owners
3453 .lock()
3454 .unwrap_or_else(|poisoned| poisoned.into_inner())
3455 .retain(|_, owner| owner != module_id);
3456 let removed = self
3457 .modules
3458 .lock()
3459 .unwrap_or_else(|poisoned| poisoned.into_inner())
3460 .remove(module_id);
3461 self.update_configured_ids(|ids| {
3462 ids.remove(module_id);
3463 });
3464 self.remove_program_binding_records(module_id);
3465 removed
3466 }
3467
3468 pub(crate) fn record_rescan_removal(&self, module_id: &str) {
3471 self.removal_tombstones
3472 .lock()
3473 .unwrap_or_else(|poisoned| poisoned.into_inner())
3474 .insert(module_id.to_string(), unix_ms_now());
3475 }
3476
3477 pub(crate) fn removal_tombstone_age_ms(&self, module_id: &str) -> Option<u64> {
3479 self.removal_tombstones
3480 .lock()
3481 .unwrap_or_else(|poisoned| poisoned.into_inner())
3482 .get(module_id)
3483 .copied()
3484 .map(|removed_at_ms| unix_ms_now().saturating_sub(removed_at_ms))
3485 }
3486
3487 pub(crate) fn release_retained_reserved_gate(&self, module_id: &str) -> bool {
3492 if self.get(module_id).is_some() {
3493 return false;
3494 }
3495 let mut reserved_nonces = self
3496 .reserved_nonces
3497 .lock()
3498 .unwrap_or_else(|poisoned| poisoned.into_inner());
3499 if !matches!(reserved_nonces.get(module_id), Some(None)) {
3500 return false;
3501 }
3502 reserved_nonces.remove(module_id);
3503 self.implicit_protocol_none
3504 .lock()
3505 .unwrap_or_else(|p| p.into_inner())
3506 .remove(module_id);
3507 true
3508 }
3509
3510 pub(crate) fn operation_lock(&self) -> Arc<AsyncMutex<()>> {
3511 Arc::clone(&self.operation_lock)
3512 }
3513}
3514
3515#[derive(Debug, Clone)]
3517pub struct Supervisor {
3518 registry: Arc<Registry>,
3519 config_delivery: Option<config_delivery::ConfigDelivery>,
3520 restart_policy: RestartPolicy,
3521 drain_timeout: Duration,
3522 connection_file_path: Option<PathBuf>,
3523 capture_logs_dir: Option<PathBuf>,
3524 forwarding: Option<Arc<ForwardingTable>>,
3525 process_liveness: Arc<SupervisorProcessLiveness>,
3526 supervisor_handle: Option<SupervisorHandle>,
3527 health: HealthConfig,
3528 host_health: HostHealth,
3529 daemon_start_clock: crate::clock::StartClock,
3530 terminal_journal: Option<Arc<crate::terminal_journal::TerminalJournal>>,
3531 spawn_events: SpawnEventFeed,
3532 provenance_probe: ExecutableIdentityProbe,
3533 child_roster: ChildRoster,
3536 #[cfg(target_os = "linux")]
3537 cgroup_placement: Option<subc_cgroup::Placement>,
3538 #[cfg(test)]
3539 test_after_first_spawn: AfterFirstSpawnHook,
3540}
3541
3542pub(crate) struct RegisteredConfiguredModule {
3546 spec: ModuleSpec,
3547 runtime: SupervisorRuntimeConfig,
3548 module: SupervisedModule,
3549 commands: mpsc::Receiver<SupervisorCommand>,
3550}
3551
3552#[cfg(test)]
3558#[derive(Clone, Default)]
3559struct AfterFirstSpawnHook(Option<AfterFirstSpawnFn>);
3560
3561#[cfg(test)]
3562type AfterFirstSpawnFn = Arc<dyn Fn(&str) + Send + Sync>;
3563
3564#[cfg(test)]
3565impl fmt::Debug for AfterFirstSpawnHook {
3566 fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
3567 f.write_str("AfterFirstSpawnHook")
3568 }
3569}
3570
3571#[cfg(test)]
3572impl AfterFirstSpawnHook {
3573 fn run(&self, module_id: &str) {
3574 if let Some(hook) = &self.0 {
3575 hook(module_id);
3576 }
3577 }
3578}
3579
3580impl Supervisor {
3581 #[cfg(all(test, target_os = "linux"))]
3582 pub(crate) fn set_admission_hook_for_test(
3583 &self,
3584 hook: impl Fn(&str, &'static str, Option<u32>) -> bool + Send + Sync + 'static,
3585 ) {
3586 self.child_roster.set_admission_hook_for_test(hook);
3587 }
3588
3589 #[cfg(test)]
3590 pub(crate) fn new_for_test(registry: Arc<Registry>, policy: RestartPolicy) -> Self {
3591 let supervisor = Self::new(registry, policy);
3592 #[cfg(target_os = "macos")]
3593 let supervisor = supervisor.with_privacy_trampoline(test_privacy_trampoline());
3594 supervisor
3595 }
3596 pub fn with_privacy_trampoline(self, path: impl Into<PathBuf>) -> Self {
3601 let path = path.into();
3602 #[cfg(target_os = "macos")]
3603 {
3604 let result = probe_privacy_trampoline(&path).map(|()| path);
3605 if let Err(cause) = &result {
3606 error!(%cause, "macOS privacy identity unavailable; supervised launches will refuse");
3607 }
3608 self.child_roster.set_privacy_trampoline(result);
3609 }
3610 #[cfg(not(target_os = "macos"))]
3611 let _ = path;
3612 self
3613 }
3614 pub(crate) fn daemon_shutdown_flag(&self) -> crate::child_roster::DaemonShutdownFlag {
3617 self.child_roster.shutdown_flag()
3618 }
3619
3620 #[cfg(unix)]
3631 pub(crate) fn begin_daemon_shutdown(&self) {
3632 self.child_roster.close();
3633 if let Some(journal) = &self.terminal_journal {
3634 journal.stamp_shutdown();
3635 }
3636 }
3637
3638 #[cfg(unix)]
3643 pub(crate) async fn drain_for_daemon_shutdown(&self) -> Result<(), SuperviseError> {
3644 const NOTICE_BUDGET: Duration = Duration::from_millis(500);
3645 const DRAIN_BUDGET: Duration = Duration::from_secs(2);
3646 let Some(forwarding) = &self.forwarding else {
3647 return Ok(());
3648 };
3649 let notice_started = Instant::now();
3650 let modules = forwarding
3651 .begin_daemon_drain()
3652 .map_err(SuperviseError::Forwarding)?;
3653 let deadline_ms =
3654 unix_ms_now().saturating_add((NOTICE_BUDGET + DRAIN_BUDGET).as_millis() as u64);
3655 let shutdown_reason = self
3656 .supervisor_handle
3657 .as_ref()
3658 .map(SupervisorHandle::take_daemon_shutdown_reason)
3659 .unwrap_or(RouteCloseReason::Disable);
3660 let mut modules_told = 0;
3661 let mut notices = tokio::task::JoinSet::new();
3662 let mut drains = Vec::new();
3663 for connection in forwarding
3664 .module_connections()
3665 .map_err(SuperviseError::Forwarding)?
3666 {
3667 let target = ModuleDrainTarget {
3668 endpoint: connection.endpoint,
3669 sink: connection.sink,
3670 negotiated_ver: connection.negotiated_ver,
3671 abandoned_bindings: Vec::new(),
3672 excluded_subscriptions: 0,
3673 };
3674 notices.spawn(async move {
3675 (
3676 true,
3677 send_module_draining(
3678 &connection.module_id,
3679 shutdown_reason,
3680 deadline_ms,
3681 &target,
3682 true,
3683 )
3684 .await,
3685 )
3686 });
3687 }
3688 for (module_id, endpoint) in modules {
3689 let Some(target) = forwarding
3690 .begin_endpoint_drain(endpoint, RouteCloseReason::Restart)
3691 .map_err(SuperviseError::Forwarding)?
3692 else {
3693 continue;
3694 };
3695 let routes = forwarding
3696 .endpoint_routes(target.endpoint)
3697 .map_err(SuperviseError::Forwarding)?;
3698 let mut recipients = Vec::new();
3701 let mut clients: Vec<(crate::forwarding::GoodbyeTarget, Vec<u16>)> = Vec::new();
3702 for route in routes {
3703 let client = route.goodbye_target;
3704 if let Some((_, channels)) = clients
3705 .iter_mut()
3706 .find(|(existing, _)| existing.connection_id == client.connection_id)
3707 {
3708 channels.push(client.channel);
3709 } else {
3710 let channel = client.channel;
3711 clients.push((client, vec![channel]));
3712 }
3713 }
3714 for (client, mut channels) in clients {
3715 channels.sort_unstable();
3716 channels.dedup();
3717 let closing = serde_json::to_vec(&ClientControlPush::RouteClosing {
3718 module_id: module_id.clone(),
3719 channels,
3720 reason: RouteCloseReason::Restart,
3721 })
3722 .expect("route closing serializes");
3723 recipients.push((client.sink, client.negotiated_ver, closing));
3724 }
3725 for (sink, version, body) in recipients {
3726 notices.spawn(async move {
3727 let frame = Frame::build_with_version(
3728 version,
3729 FrameType::Push,
3730 control_flags(),
3731 0,
3732 0,
3733 0,
3734 body,
3735 )
3736 .expect("bounded lifecycle notice frame builds");
3737 (false, sink.send_flushed(frame).await.is_ok())
3738 });
3739 }
3740 let gauges = declared_busy_gauges(&self.registry, &module_id)?;
3741 drains.push((module_id, target.endpoint, gauges));
3742 }
3743 let notice_deadline = Instant::now() + NOTICE_BUDGET;
3746 while let Ok(Some(result)) = timeout_at(notice_deadline, notices.join_next()).await {
3747 match result {
3748 Ok((true, true)) => modules_told += 1,
3749 Ok((false, true)) => {}
3750 result => warn!(?result, "daemon shutdown notice delivery failed"),
3751 }
3752 }
3753 notices.abort_all();
3754 info!(
3755 ?shutdown_reason,
3756 modules_told, "daemon shutdown module notices delivered"
3757 );
3758 let notice_elapsed = notice_started.elapsed();
3759 let drain_started = Instant::now();
3760 let deadline = drain_started + DRAIN_BUDGET;
3761 let mut waits = tokio::task::JoinSet::new();
3762 for (module_id, endpoint, gauges) in drains {
3763 let forwarding = Arc::clone(forwarding);
3764 let mut runtime = self.runtime_config();
3765 runtime.health.cadence = Duration::from_millis(100);
3766 waits.spawn(async move {
3767 let wait_result = wait_for_forwarding_quiescence(
3768 &forwarding,
3769 &module_id,
3770 &runtime,
3771 endpoint,
3772 deadline,
3773 &gauges,
3774 DrainScope::Endpoint(endpoint),
3775 &mut DrainGaugeTrace::new(&gauges),
3776 )
3777 .await;
3778 forwarding
3782 .record_daemon_drain_result(
3783 endpoint,
3784 drained_after_quiescence_wait(&wait_result),
3785 )
3786 .map_err(SuperviseError::Forwarding)?;
3787 wait_result
3788 });
3789 }
3790 while let Ok(Some(result)) = timeout_at(deadline, waits.join_next()).await {
3791 if !matches!(result, Ok(Ok(true))) {
3792 warn!(?result, "daemon shutdown drain did not reach quiescence");
3793 }
3794 }
3795 info!(
3798 notice_elapsed_ms = notice_elapsed.as_millis() as u64,
3799 drain_elapsed_ms = drain_started.elapsed().as_millis() as u64,
3800 "daemon shutdown forwarding drain complete"
3801 );
3802 Ok(())
3803 }
3804
3805 #[cfg(unix)]
3817 pub(crate) async fn end_children_for_daemon_shutdown(
3818 &self,
3819 already_escalated: bool,
3820 escalate: impl std::future::Future<Output = ()>,
3821 ) {
3822 tokio::pin!(escalate);
3823 let mut escalated = already_escalated;
3824 if let Some(forwarding) = &self.forwarding {
3825 let reason = CloseReason::new(
3826 "daemon_shutdown",
3827 "the daemon is exiting after its shutdown notice and drain",
3828 );
3829 if escalated {
3830 send_module_goodbyes_for_daemon_shutdown(forwarding, &reason, false).await;
3833 } else {
3834 tokio::select! {
3835 biased;
3836 _ = escalate.as_mut() => {
3837 info!("second SIGTERM: abandoning module GOODBYE delivery");
3838 escalated = true;
3839 }
3840 _ = send_module_goodbyes_for_daemon_shutdown(forwarding, &reason, true) => {}
3841 }
3842 }
3843 let closed = forwarding.close_all_connections(&reason);
3844 debug!(closed, "closed established connections for daemon shutdown");
3845 }
3846 let escalated_here = escalated && !already_escalated;
3850 let remaining_escalate = async move {
3851 if escalated_here {
3852 std::future::pending::<()>().await;
3853 } else {
3854 escalate.await;
3855 }
3856 };
3857 crate::child_roster::end_children_for_daemon_shutdown(
3858 &self.child_roster,
3859 escalated,
3860 remaining_escalate,
3861 )
3862 .await;
3863 }
3864
3865 pub fn new(registry: Arc<Registry>, restart_policy: RestartPolicy) -> Self {
3866 Self {
3867 registry,
3868 config_delivery: None,
3869 restart_policy,
3870 drain_timeout: DEFAULT_DRAIN_TIMEOUT,
3871 connection_file_path: None,
3872 capture_logs_dir: None,
3873 forwarding: None,
3874 process_liveness: Arc::new(SupervisorProcessLiveness::default()),
3875 supervisor_handle: None,
3876 health: HealthConfig::default(),
3877 host_health: HostHealth::default(),
3878 daemon_start_clock: crate::clock::StartClock::capture(),
3879 terminal_journal: None,
3880 spawn_events: SpawnEventFeed::default(),
3881 provenance_probe: ExecutableIdentityProbe::default(),
3882 child_roster: ChildRoster::default(),
3883 #[cfg(target_os = "linux")]
3884 cgroup_placement: None,
3885 #[cfg(test)]
3886 test_after_first_spawn: AfterFirstSpawnHook::default(),
3887 }
3888 }
3889
3890 pub fn with_drain_timeout(mut self, drain_timeout: Duration) -> Self {
3891 self.drain_timeout = drain_timeout;
3892 self
3893 }
3894
3895 pub(crate) fn default_drain_timeout(&self) -> Duration {
3897 self.drain_timeout
3898 }
3899
3900 pub fn with_process_liveness(
3901 mut self,
3902 process_liveness: Arc<SupervisorProcessLiveness>,
3903 ) -> Self {
3904 self.process_liveness = process_liveness;
3905 self
3906 }
3907
3908 pub fn with_config_runtime(
3910 mut self,
3911 owner: Arc<crate::config_gate::runtime::RuntimeOwner>,
3912 ) -> Self {
3913 if let Some(handle) = &self.supervisor_handle {
3914 handle.install_binding_retirement(&owner);
3915 }
3916 self.config_delivery = Some(config_delivery::ConfigDelivery(owner));
3917 self
3918 }
3919
3920 pub fn with_connection_file_path(mut self, connection_file_path: impl Into<PathBuf>) -> Self {
3921 self.connection_file_path = Some(connection_file_path.into());
3922 self
3923 }
3924
3925 pub fn with_capture_logs_dir(mut self, logs_dir: impl Into<PathBuf>) -> Self {
3927 self.capture_logs_dir = Some(logs_dir.into());
3928 self
3929 }
3930
3931 pub fn with_daemon_incarnation(self, daemon_incarnation: String) -> Self {
3934 self.spawn_events.configure_incarnation(daemon_incarnation);
3938 self
3939 }
3940
3941 pub fn with_terminal_journal(self, path: PathBuf, daemon_incarnation: String) -> Self {
3944 let mut this = self.with_daemon_incarnation(daemon_incarnation.clone());
3945 this.terminal_journal = Some(Arc::new(crate::terminal_journal::TerminalJournal::open(
3946 path,
3947 daemon_incarnation,
3948 )));
3949 this
3950 }
3951
3952 pub fn with_forwarding(mut self, forwarding: Arc<ForwardingTable>) -> Self {
3953 self.child_roster.counters = forwarding.counters();
3954 self.forwarding = Some(forwarding);
3955 self
3956 }
3957
3958 pub fn with_handle(mut self, supervisor_handle: SupervisorHandle) -> Self {
3959 if let Some(delivery) = &self.config_delivery {
3960 supervisor_handle.install_binding_retirement(&delivery.0);
3961 }
3962 self.host_health = supervisor_handle.host_health.clone();
3963 self.spawn_events = supervisor_handle.spawn_events.clone();
3964 self.supervisor_handle = Some(supervisor_handle);
3965 self
3966 }
3967
3968 pub fn with_health_config(mut self, health: HealthConfig) -> Self {
3969 self.health = health;
3970 self
3971 }
3972
3973 pub fn with_live_children_record(self, path: impl Into<PathBuf>) -> Self {
3977 self.child_roster.record_to(path.into());
3978 self
3979 }
3980
3981 #[cfg(target_os = "linux")]
3982 pub fn with_cgroup_placement(
3983 mut self,
3984 cgroup_placement: Option<subc_cgroup::Placement>,
3985 ) -> Self {
3986 self.child_roster
3987 .set_cgroup_placement(cgroup_placement.clone());
3988 self.cgroup_placement = cgroup_placement;
3989 self
3990 }
3991
3992 pub fn spawn(&self, spec: ModuleSpec) -> Result<SupervisedModule, SuperviseError> {
4001 validate_spec(&spec)?;
4002 self.establish_identity(&spec);
4003
4004 let runtime = self.runtime_config();
4005 let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::starting()));
4006 let activity_spawn = runtime.begin_activity_spawn(&spec.module_id);
4007 let registration_watch = watch_registration(&self.registry, &spec);
4008 let spawned = spawn_child(
4009 &spec,
4010 runtime.connection_file_path.as_deref(),
4011 self.supervisor_handle.as_ref(),
4012 &runtime.stderr_ring,
4013 runtime.capture_logs_dir.as_deref(),
4014 &runtime.child_roster,
4015 runtime.spawn_delivery(),
4016 #[cfg(target_os = "linux")]
4017 runtime.cgroup_placement.as_ref(),
4018 runtime.register_timeout,
4019 );
4020 #[cfg(test)]
4021 self.test_after_first_spawn.run(&spec.module_id);
4022 let mut child = match spawned {
4023 Ok(child) => child,
4024 Err(SuperviseError::ConfigDeliveryDeferred { .. }) => {
4025 defer_config_spawn(
4026 &spec,
4027 &runtime,
4028 &snapshot,
4029 RespawnKind::Spawn,
4030 &runtime.failure_context(runtime.recovery_is_daemon),
4031 )?;
4032 return Ok(self.supervised_module(spec, runtime, snapshot, None));
4033 }
4034 Err(SuperviseError::ConfigHeld { .. }) => {
4035 hold_config_spawn(
4036 &spec,
4037 &runtime,
4038 &snapshot,
4039 &runtime.failure_context(runtime.recovery_is_daemon),
4040 )?;
4041 return Ok(self.supervised_module(spec, runtime, snapshot, None));
4042 }
4043 Err(err) => {
4044 self.abandon_unrostered(&spec.module_id);
4047 return Err(err);
4048 }
4049 };
4050 arm_registration_deadline(&mut child, runtime.register_timeout, registration_watch);
4051 self.mark_first_process_running(&spec, &runtime, &snapshot, &mut child)?;
4052 if let Some(spawn) = activity_spawn {
4053 spawn.complete(&snapshot);
4054 }
4055
4056 Ok(self.supervised_module(spec, runtime, snapshot, Some(child)))
4057 }
4058
4059 pub async fn spawn_async(&self, spec: ModuleSpec) -> Result<SupervisedModule, SuperviseError> {
4062 validate_spec(&spec)?;
4063 self.establish_identity(&spec);
4064 let mut identity = AsyncInitialIdentity {
4065 supervisor: self,
4066 module: spec.module_id.clone(),
4067 committed: false,
4068 };
4069 let runtime = self.runtime_config();
4070 let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::starting()));
4071 let activity_spawn = runtime.begin_activity_spawn(&spec.module_id);
4072 let registration_watch = watch_registration(&self.registry, &spec);
4073 let spawned = spawn_child_async(&spec, &runtime).await;
4074 #[cfg(test)]
4075 self.test_after_first_spawn.run(&spec.module_id);
4076 let mut child = match spawned {
4077 Ok(child) => child,
4078 Err(SuperviseError::ConfigDeliveryDeferred { .. }) => {
4079 defer_config_spawn(
4080 &spec,
4081 &runtime,
4082 &snapshot,
4083 RespawnKind::Spawn,
4084 &runtime.failure_context(runtime.recovery_is_daemon),
4085 )?;
4086 identity.committed = true;
4087 return Ok(self.supervised_module(spec, runtime, snapshot, None));
4088 }
4089 Err(SuperviseError::ConfigHeld { .. }) => {
4090 hold_config_spawn(
4091 &spec,
4092 &runtime,
4093 &snapshot,
4094 &runtime.failure_context(runtime.recovery_is_daemon),
4095 )?;
4096 identity.committed = true;
4097 return Ok(self.supervised_module(spec, runtime, snapshot, None));
4098 }
4099 Err(err) => {
4100 self.abandon_unrostered(&spec.module_id);
4101 return Err(err);
4102 }
4103 };
4104 arm_registration_deadline(&mut child, runtime.register_timeout, registration_watch);
4105 self.mark_first_process_running(&spec, &runtime, &snapshot, &mut child)?;
4106 if let Some(spawn) = activity_spawn {
4107 spawn.complete(&snapshot);
4108 }
4109 identity.committed = true;
4110 Ok(self.supervised_module(spec, runtime, snapshot, Some(child)))
4111 }
4112
4113 fn establish_identity(&self, spec: &ModuleSpec) {
4126 if let Some(supervisor_handle) = &self.supervisor_handle {
4127 supervisor_handle.apply_identity_configuration(spec);
4128 supervisor_handle.mark_configured(&spec.module_id);
4129 }
4130 }
4131
4132 fn abandon_unrostered(&self, module_id: &str) {
4135 if let Some(supervisor_handle) = &self.supervisor_handle {
4136 supervisor_handle.unmark_configured_unless_rostered(module_id);
4137 }
4138 }
4139
4140 fn mark_first_process_running(
4143 &self,
4144 spec: &ModuleSpec,
4145 runtime: &SupervisorRuntimeConfig,
4146 snapshot: &SharedSnapshot,
4147 child: &mut SupervisedChild,
4148 ) -> Result<(), SuperviseError> {
4149 if let Err(err) = set_running(snapshot, child, &spec.module_id, &runtime.spawn_events) {
4150 containment::terminate_blocking(&runtime.child_roster.containment);
4151 self.abandon_unrostered(&spec.module_id);
4152 return Err(err);
4153 }
4154 self.process_liveness
4155 .track(spec.module_id.clone(), Arc::clone(snapshot));
4156 Ok(())
4157 }
4158
4159 pub fn supervise_configured(
4165 &self,
4166 spec: ModuleSpec,
4167 enabled: bool,
4168 ) -> Result<SupervisedModule, SuperviseError> {
4169 self.supervise_configured_requested(
4170 spec,
4171 enabled,
4172 self.health.clone(),
4173 None,
4174 self.restart_policy,
4175 DEFAULT_REGISTER_TIMEOUT,
4176 None,
4177 )
4178 }
4179
4180 pub fn supervise_configured_with_health(
4186 &self,
4187 spec: ModuleSpec,
4188 enabled: bool,
4189 health: HealthConfig,
4190 drain_timeout_ms: Option<u64>,
4191 restart_policy: RestartPolicy,
4192 ) -> Result<SupervisedModule, SuperviseError> {
4193 self.supervise_configured_requested(
4194 spec,
4195 enabled,
4196 health,
4197 drain_timeout_ms,
4198 restart_policy,
4199 DEFAULT_REGISTER_TIMEOUT,
4200 None,
4201 )
4202 }
4203
4204 #[allow(clippy::too_many_arguments)]
4205 pub(crate) fn supervise_configured_requested(
4206 &self,
4207 spec: ModuleSpec,
4208 enabled: bool,
4209 health: HealthConfig,
4210 drain_timeout_ms: Option<u64>,
4211 restart_policy: RestartPolicy,
4212 register_timeout: Duration,
4213 requester: Option<Principal>,
4214 ) -> Result<SupervisedModule, SuperviseError> {
4215 let registered = self.register_configured_with_health(
4216 spec,
4217 enabled,
4218 health,
4219 drain_timeout_ms,
4220 restart_policy,
4221 register_timeout,
4222 )?;
4223 let daemon_initiated = requester.is_none();
4224 self.start_registered_configured(registered, requester, daemon_initiated)
4225 }
4226
4227 #[allow(clippy::too_many_arguments)]
4228 pub(crate) async fn supervise_configured_requested_async(
4229 &self,
4230 spec: ModuleSpec,
4231 enabled: bool,
4232 health: HealthConfig,
4233 drain_timeout_ms: Option<u64>,
4234 restart_policy: RestartPolicy,
4235 register_timeout: Duration,
4236 requester: Option<Principal>,
4237 ) -> Result<SupervisedModule, SuperviseError> {
4238 let registered = self.register_configured_with_health(
4239 spec,
4240 enabled,
4241 health,
4242 drain_timeout_ms,
4243 restart_policy,
4244 register_timeout,
4245 )?;
4246 let daemon_initiated = requester.is_none();
4247 self.start_registered_configured_async(registered, requester, daemon_initiated)
4248 .await
4249 }
4250
4251 pub(crate) fn register_configured_with_health(
4256 &self,
4257 spec: ModuleSpec,
4258 enabled: bool,
4259 health: HealthConfig,
4260 drain_timeout_ms: Option<u64>,
4261 restart_policy: RestartPolicy,
4262 register_timeout: Duration,
4263 ) -> Result<RegisteredConfiguredModule, SuperviseError> {
4264 validate_spec(&spec)?;
4265 self.establish_identity(&spec);
4266
4267 let mut runtime = self.runtime_config();
4268 runtime.health = health;
4269 runtime.restart_policy = restart_policy;
4270 runtime.register_timeout = register_timeout;
4271 if let Some(ms) = drain_timeout_ms {
4272 runtime.drain_timeout = Duration::from_millis(ms);
4273 *runtime
4274 .effective_drain_timeout
4275 .lock()
4276 .unwrap_or_else(|poisoned| poisoned.into_inner()) = runtime.drain_timeout;
4277 }
4278 let snapshot = if enabled {
4279 SupervisorSnapshot::starting()
4280 } else {
4281 SupervisorSnapshot::disabled()
4282 };
4283 let snapshot = {
4284 let mut snapshot = snapshot;
4285 if enabled {
4286 if let Some((pid, reason)) = runtime.child_roster.previous_instance(&spec.module_id)
4287 {
4288 snapshot.waiting_for_previous_pid = Some(pid);
4289 snapshot.waiting_for_previous_detail = Some(reason.waiting_message(pid));
4290 }
4291 }
4292 snapshot
4293 };
4294 let (module, commands) =
4295 self.register_module(&spec, &runtime, Arc::new(Mutex::new(snapshot)));
4296 Ok(RegisteredConfiguredModule {
4297 spec,
4298 runtime,
4299 module,
4300 commands,
4301 })
4302 }
4303
4304 pub(crate) fn start_registered_configured(
4307 &self,
4308 registered: RegisteredConfiguredModule,
4309 requester: Option<Principal>,
4310 daemon_initiated: bool,
4311 ) -> Result<SupervisedModule, SuperviseError> {
4312 let RegisteredConfiguredModule {
4313 spec,
4314 mut runtime,
4315 module,
4316 commands,
4317 } = registered;
4318 runtime.start_requester = requester;
4319 runtime.recovery_is_daemon = daemon_initiated;
4320 let snapshot = &module.inner.snapshot;
4321 let should_spawn = {
4322 let state = lock_snapshot(snapshot)?;
4323 state.enabled && state.waiting_for_previous_pid.is_none()
4324 };
4325 let child = if should_spawn {
4326 let activity_spawn = runtime.begin_activity_spawn(&spec.module_id);
4327 let registration_watch = watch_registration(&self.registry, &spec);
4328 let spawned = spawn_child(
4329 &spec,
4330 runtime.connection_file_path.as_deref(),
4331 self.supervisor_handle.as_ref(),
4332 &runtime.stderr_ring,
4333 runtime.capture_logs_dir.as_deref(),
4334 &runtime.child_roster,
4335 runtime.spawn_delivery(),
4336 #[cfg(target_os = "linux")]
4337 runtime.cgroup_placement.as_ref(),
4338 runtime.register_timeout,
4339 );
4340 #[cfg(test)]
4341 self.test_after_first_spawn.run(&spec.module_id);
4342 match spawned {
4343 Ok(mut child) => {
4344 arm_registration_deadline(
4345 &mut child,
4346 runtime.register_timeout,
4347 registration_watch,
4348 );
4349 self.mark_first_process_running(&spec, &runtime, snapshot, &mut child)?;
4350 if let Some(spawn) = activity_spawn {
4351 spawn.complete(snapshot);
4352 }
4353 Some(child)
4354 }
4355 Err(SuperviseError::ConfigDeliveryDeferred { .. }) => {
4356 defer_config_spawn(
4357 &spec,
4358 &runtime,
4359 snapshot,
4360 RespawnKind::Spawn,
4361 &runtime.failure_context(daemon_initiated),
4362 )?;
4363 None
4364 }
4365 Err(mut err) => {
4366 if matches!(err, SuperviseError::ConfigHeld { .. }) {
4367 match hold_config_spawn(
4368 &spec,
4369 &runtime,
4370 snapshot,
4371 &runtime.failure_context(daemon_initiated),
4372 ) {
4373 Ok(()) => {
4374 self.start_monitor(spec, runtime, &module, None, commands);
4375 return Ok(module);
4376 }
4377 Err(parking_error) => err = parking_error,
4378 }
4379 }
4380 if runtime.health.critical {
4381 error!(
4382 module_id = %spec.module_id,
4383 program = %spec.program.display(),
4384 error = %err,
4385 "critical configured module failed to spawn; marking failed and alerting"
4386 );
4387 } else {
4388 error!(
4389 module_id = %spec.module_id,
4390 program = %spec.program.display(),
4391 error = %err,
4392 "configured module failed to spawn; marking failed and continuing"
4393 );
4394 }
4395 fail_snapshot(
4396 snapshot,
4397 Some(&spec.module_id),
4398 None,
4399 &runtime.failure_context(daemon_initiated),
4400 );
4401 None
4402 }
4403 }
4404 } else {
4405 None
4406 };
4407 self.start_monitor(spec, runtime, &module, child, commands);
4408 Ok(module)
4409 }
4410
4411 pub(crate) async fn start_registered_configured_async(
4412 &self,
4413 registered: RegisteredConfiguredModule,
4414 requester: Option<Principal>,
4415 daemon_initiated: bool,
4416 ) -> Result<SupervisedModule, SuperviseError> {
4417 self.start_registered_with_failure_policy(registered, requester, daemon_initiated, None)
4418 .await
4419 }
4420
4421 pub(crate) async fn start_registered_approved_async(
4422 &self,
4423 registered: RegisteredConfiguredModule,
4424 requester: Principal,
4425 generation: crate::config_gate::runtime::ApprovalIdentity,
4426 ) -> Result<SupervisedModule, SuperviseError> {
4427 self.start_registered_with_failure_policy(
4428 registered,
4429 Some(requester),
4430 false,
4431 Some(generation),
4432 )
4433 .await
4434 }
4435
4436 async fn start_registered_with_failure_policy(
4437 &self,
4438 registered: RegisteredConfiguredModule,
4439 requester: Option<Principal>,
4440 daemon_initiated: bool,
4441 approved_generation: Option<crate::config_gate::runtime::ApprovalIdentity>,
4442 ) -> Result<SupervisedModule, SuperviseError> {
4443 let RegisteredConfiguredModule {
4444 spec,
4445 mut runtime,
4446 module,
4447 commands,
4448 } = registered;
4449 runtime.start_requester = requester;
4450 runtime.recovery_is_daemon = daemon_initiated;
4451 let snapshot = &module.inner.snapshot;
4452 let mut startup_error = None;
4453 let failure = FailureContext {
4454 approval: approved_generation.clone(),
4455 ..runtime.failure_context(daemon_initiated)
4456 };
4457 let should_spawn = {
4458 let state = lock_snapshot(snapshot)?;
4459 state.enabled && state.waiting_for_previous_pid.is_none()
4460 };
4461 let child = if should_spawn {
4462 let activity_spawn = runtime.begin_activity_spawn(&spec.module_id);
4463 let registration_watch = watch_registration(&self.registry, &spec);
4464 let spawned = spawn_child_async(&spec, &runtime).await;
4465 #[cfg(test)]
4466 self.test_after_first_spawn.run(&spec.module_id);
4467 match spawned {
4468 Ok(mut child) => {
4469 arm_registration_deadline(
4470 &mut child,
4471 runtime.register_timeout,
4472 registration_watch,
4473 );
4474 self.mark_first_process_running(&spec, &runtime, snapshot, &mut child)?;
4475 if let Some(spawn) = activity_spawn {
4476 spawn.complete(snapshot);
4477 }
4478 Some(child)
4479 }
4480 Err(SuperviseError::ConfigDeliveryDeferred { .. }) => {
4481 defer_config_spawn(&spec, &runtime, snapshot, RespawnKind::Spawn, &failure)?;
4482 None
4483 }
4484 Err(mut err) => {
4485 if matches!(err, SuperviseError::ConfigHeld { .. }) {
4486 match hold_config_spawn(&spec, &runtime, snapshot, &failure) {
4487 Ok(()) => {
4488 self.start_monitor(spec, runtime, &module, None, commands);
4489 return Ok(module);
4490 }
4491 Err(parking_error) => err = parking_error,
4492 }
4493 }
4494 if runtime.health.critical {
4495 error!(module_id = %spec.module_id, program = %spec.program.display(), error = %err,
4496 "critical configured module failed to spawn; marking failed and alerting");
4497 } else {
4498 error!(module_id = %spec.module_id, program = %spec.program.display(), error = %err,
4499 "configured module failed to spawn; marking failed and continuing");
4500 }
4501 fail_snapshot(snapshot, Some(&spec.module_id), None, &failure);
4502 if let Some(generation) = approved_generation {
4503 let _ = update_snapshot(snapshot, Some(&spec.module_id), |state| {
4504 state.approved_start_failure = Some((generation, err.to_string()));
4505 });
4506 startup_error = Some(err);
4507 }
4508 None
4509 }
4510 }
4511 } else {
4512 None
4513 };
4514 self.start_monitor(spec, runtime, &module, child, commands);
4515 startup_error.map_or(Ok(module), Err)
4519 }
4520
4521 fn runtime_config(&self) -> SupervisorRuntimeConfig {
4522 let effective_drain_timeout = Arc::new(Mutex::new(self.drain_timeout));
4523 SupervisorRuntimeConfig {
4524 registry: self.registry.clone(),
4525 config_delivery: self.config_delivery.clone(),
4526 recovery_is_daemon: true,
4527 start_requester: None,
4528 pending_activity_restart: false,
4529 provenance_probe: self.provenance_probe.clone(),
4530 scheduled_respawn: Arc::default(),
4531 deferred_reload_reply: Arc::default(),
4532 restart_policy: self.restart_policy,
4533 register_timeout: DEFAULT_REGISTER_TIMEOUT,
4534 drain_timeout: self.drain_timeout,
4535 child_roster: self
4538 .child_roster
4539 .for_module(Arc::clone(&effective_drain_timeout)),
4540 effective_drain_timeout,
4541 default_drain_timeout: self.drain_timeout,
4542 health: self.health.clone(),
4543 host_health: self.host_health.clone(),
4544 connection_file_path: self.connection_file_path.clone(),
4545 capture_logs_dir: self.capture_logs_dir.clone(),
4546 forwarding: self.forwarding.clone(),
4547 supervisor_handle: self.supervisor_handle.clone(),
4548 stderr_ring: Arc::new(Mutex::new(StderrRing::new(StderrTailConfig::default()))),
4549 terminal_ring: Arc::new(Mutex::new(
4550 TerminalRing::new(
4551 TerminalRingConfig::default(),
4552 self.daemon_start_clock.started_at_ms(),
4553 )
4554 .with_start_clock(self.daemon_start_clock)
4555 .with_journal(self.terminal_journal.clone())
4556 .with_daemon_shutdown(self.child_roster.shutdown_flag()),
4557 )),
4558 spawn_events: self.spawn_events.clone(),
4559 #[cfg(target_os = "linux")]
4560 cgroup_placement: self.cgroup_placement.clone(),
4561 #[cfg(test)]
4562 test_seed_stale_facts_before_enable_spawn: false,
4563 #[cfg(test)]
4564 test_reload_exit_record_gate: None,
4565 }
4566 }
4567
4568 fn supervised_module(
4569 &self,
4570 spec: ModuleSpec,
4571 runtime: SupervisorRuntimeConfig,
4572 snapshot: SharedSnapshot,
4573 child: Option<SupervisedChild>,
4574 ) -> SupervisedModule {
4575 let (module, commands) = self.register_module(&spec, &runtime, snapshot);
4576 self.start_monitor(spec, runtime, &module, child, commands);
4577 module
4578 }
4579
4580 fn register_module(
4581 &self,
4582 spec: &ModuleSpec,
4583 runtime: &SupervisorRuntimeConfig,
4584 snapshot: SharedSnapshot,
4585 ) -> (SupervisedModule, mpsc::Receiver<SupervisorCommand>) {
4586 runtime.host_health.track(spec, &snapshot);
4587 let configuration = Arc::new(Mutex::new(SupervisedConfiguration {
4588 spec: spec.clone(),
4589 enabled: snapshot.lock().unwrap_or_else(|p| p.into_inner()).enabled,
4590 health: runtime.health.clone(),
4591 register_timeout: runtime.register_timeout,
4592 restart_policy: runtime.restart_policy,
4593 }));
4594 let stderr_ring = Arc::clone(&runtime.stderr_ring);
4595 let terminal_ring = Arc::clone(&runtime.terminal_ring);
4596 let effective_drain_timeout = Arc::clone(&runtime.effective_drain_timeout);
4597 let (tx, rx) = mpsc::channel(4);
4598 let module_id = spec.module_id.clone();
4599 let module = SupervisedModule {
4600 inner: Arc::new(SupervisedModuleInner {
4601 module_id: module_id.clone(),
4602 registry: Arc::clone(&self.registry),
4603 snapshot,
4604 configuration,
4605 stderr_ring,
4606 terminal_ring,
4607 commands: tx,
4608 monitor: Mutex::new(None),
4609 effective_drain_timeout,
4610 provenance_probe: self.provenance_probe.clone(),
4611 retained: runtime.child_roster.retained_view(),
4612 }),
4613 };
4614 if let Some(supervisor_handle) = &self.supervisor_handle {
4617 supervisor_handle.insert(module.clone());
4618 }
4619 (module, rx)
4620 }
4621
4622 fn start_monitor(
4623 &self,
4624 spec: ModuleSpec,
4625 runtime: SupervisorRuntimeConfig,
4626 module: &SupervisedModule,
4627 child: Option<SupervisedChild>,
4628 commands: mpsc::Receiver<SupervisorCommand>,
4629 ) {
4630 let monitor = tokio::spawn(supervise_loop(
4631 spec,
4632 runtime,
4633 Arc::clone(&self.registry),
4634 Arc::clone(&self.process_liveness),
4635 Arc::clone(&module.inner.snapshot),
4636 child,
4637 commands,
4638 ));
4639 *module
4640 .inner
4641 .monitor
4642 .lock()
4643 .unwrap_or_else(|p| p.into_inner()) = Some(monitor);
4644 }
4645}
4646
4647impl Default for Supervisor {
4648 fn default() -> Self {
4649 Self::new(Arc::new(Registry::default()), RestartPolicy::default())
4650 }
4651}
4652
4653#[derive(Clone)]
4655pub struct SupervisedModule {
4656 inner: Arc<SupervisedModuleInner>,
4657}
4658
4659struct SupervisedModuleInner {
4660 retained: crate::child_roster::RetainedView,
4661 module_id: String,
4662 registry: Arc<Registry>,
4663 snapshot: SharedSnapshot,
4664 configuration: Arc<Mutex<SupervisedConfiguration>>,
4665 stderr_ring: Arc<Mutex<StderrRing>>,
4666 terminal_ring: Arc<Mutex<TerminalRing>>,
4667 commands: mpsc::Sender<SupervisorCommand>,
4668 monitor: Mutex<Option<JoinHandle<()>>>,
4669 effective_drain_timeout: Arc<Mutex<Duration>>,
4670 provenance_probe: ExecutableIdentityProbe,
4671}
4672
4673impl fmt::Debug for SupervisedModule {
4674 fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
4675 f.debug_struct("SupervisedModule")
4676 .field("module_id", &self.inner.module_id)
4677 .field("status", &self.status())
4678 .finish_non_exhaustive()
4679 }
4680}
4681
4682impl SupervisedModule {
4683 pub(crate) fn check_approved_start_failure(
4684 &self,
4685 generation: &crate::config_gate::runtime::ApprovalIdentity,
4686 ) -> Result<bool, SuperviseError> {
4687 let state = lock_snapshot(&self.inner.snapshot)?;
4688 if let Some((failed_generation, detail)) = &state.approved_start_failure {
4689 if failed_generation == generation {
4690 return Err(SuperviseError::ReloadFailed {
4691 module_id: self.inner.module_id.clone(),
4692 reason: detail.clone(),
4693 });
4694 }
4695 }
4696 Ok(state.approved_start_failure.is_some())
4697 }
4698
4699 pub(crate) fn has_approved_start_failure(
4700 &self,
4701 identity: &crate::config_gate::runtime::ApprovalIdentity,
4702 ) -> bool {
4703 lock_snapshot(&self.inner.snapshot).is_ok_and(|state| {
4704 state
4705 .approved_start_failure
4706 .as_ref()
4707 .is_some_and(|(failed, _)| failed == identity)
4708 })
4709 }
4710
4711 pub(crate) fn needs_approved_gate_restart(
4712 &self,
4713 approval: &crate::config_gate::runtime::ApprovalIdentity,
4714 ) -> bool {
4715 lock_snapshot(&self.inner.snapshot)
4716 .is_ok_and(|state| state.needs_approved_gate_restart(approval))
4717 }
4718
4719 #[cfg(all(test, unix))]
4720 pub(crate) fn config_hold_origin_for_test(
4721 &self,
4722 ) -> (bool, Option<crate::config_gate::runtime::ApprovalIdentity>) {
4723 let state = lock_snapshot(&self.inner.snapshot).unwrap();
4724 (
4725 state.config_hold_daemon_initiated,
4726 state.config_hold_approval.clone(),
4727 )
4728 }
4729
4730 pub(crate) async fn config_policy_probe(
4731 &self,
4732 connection: crate::ConnectionId,
4733 generation: u64,
4734 attempt: u64,
4735 ) -> Result<(), SuperviseError> {
4736 self.inner
4737 .commands
4738 .send(SupervisorCommand::ConfigPolicyProbe {
4739 connection,
4740 generation,
4741 attempt,
4742 })
4743 .await
4744 .map_err(|_| SuperviseError::CommandClosed {
4745 module_id: self.inner.module_id.clone(),
4746 })
4747 }
4748 pub(crate) async fn config_exit_hold(&self) -> Result<(), SuperviseError> {
4749 let (reply, result) = oneshot::channel();
4750 self.inner
4751 .commands
4752 .send(SupervisorCommand::ConfigExitHold { reply })
4753 .await
4754 .map_err(|_| SuperviseError::CommandClosed {
4755 module_id: self.module_id().to_string(),
4756 })?;
4757 result.await.map_err(|_| SuperviseError::CommandClosed {
4758 module_id: self.module_id().to_string(),
4759 })?
4760 }
4761 pub(crate) async fn config_delivery_failed(
4762 &self,
4763 connection: crate::ConnectionId,
4764 hold: bool,
4765 ) -> Result<(), SuperviseError> {
4766 self.inner
4767 .commands
4768 .send(SupervisorCommand::ConfigDeliveryFailed { connection, hold })
4769 .await
4770 .map_err(|_| SuperviseError::CommandClosed {
4771 module_id: self.module_id().to_string(),
4772 })
4773 }
4774
4775 pub(crate) async fn config_apply_restart(&self, generation: u64) -> Result<(), SuperviseError> {
4776 self.inner
4777 .commands
4778 .send(SupervisorCommand::ConfigApplyRestart {
4779 generation,
4780 supersedes_swap: false,
4781 })
4782 .await
4783 .map_err(|_| SuperviseError::CommandClosed {
4784 module_id: self.inner.module_id.clone(),
4785 })
4786 }
4787 pub fn module_id(&self) -> &str {
4788 &self.inner.module_id
4789 }
4790
4791 #[cfg(test)]
4795 pub(crate) fn record_health_probe_failure_for_test(
4796 &self,
4797 detail: &str,
4798 ) -> Result<(), SuperviseError> {
4799 update_snapshot(&self.inner.snapshot, Some(&self.inner.module_id), |state| {
4800 state.health.consecutive_failures = state.health.consecutive_failures.saturating_add(1);
4801 state.health.detail = Some(detail.to_string());
4802 })
4803 }
4804
4805 pub fn state(&self) -> Result<ModuleState, SuperviseError> {
4806 Ok(lock_snapshot(&self.inner.snapshot)?.state)
4807 }
4808
4809 pub(crate) fn registration_end_reason(
4810 &self,
4811 ) -> Result<Option<RegistrationEndReason>, SuperviseError> {
4812 let snapshot = lock_snapshot(&self.inner.snapshot)?;
4813 Ok(match snapshot.state {
4814 ModuleState::Draining if snapshot.draining_to_replace => {
4815 Some(RegistrationEndReason::SupervisorRestart)
4816 }
4817 ModuleState::Draining => Some(RegistrationEndReason::SupervisorStop),
4818 ModuleState::Restarting if snapshot.draining_to_replace => {
4819 Some(RegistrationEndReason::SupervisorRestart)
4820 }
4821 _ => None,
4822 })
4823 }
4824
4825 pub fn stderr_tail(
4832 &self,
4833 max_lines: Option<usize>,
4834 max_bytes: Option<usize>,
4835 ) -> StderrTailSnapshot {
4836 self.inner
4837 .stderr_ring
4838 .lock()
4839 .unwrap_or_else(|poisoned| poisoned.into_inner())
4840 .snapshot(max_lines, max_bytes)
4841 }
4842
4843 pub fn terminal_history(&self) -> TerminalHistorySnapshot {
4848 self.inner
4849 .terminal_ring
4850 .lock()
4851 .unwrap_or_else(|poisoned| poisoned.into_inner())
4852 .snapshot()
4853 }
4854
4855 pub fn durable_terminal_history(&self) -> subc_control::TerminalHistory {
4860 durable_terminal_history_of(&self.inner.terminal_ring, &self.inner.module_id)
4861 }
4862
4863 pub(crate) async fn read_durable_terminal_history(
4868 &self,
4869 ) -> Result<subc_control::TerminalHistory, tokio::task::JoinError> {
4870 let terminal_ring = Arc::clone(&self.inner.terminal_ring);
4871 let module_id = self.inner.module_id.clone();
4872 tokio::task::spawn_blocking(move || durable_terminal_history_of(&terminal_ring, &module_id))
4873 .await
4874 }
4875
4876 pub fn status(&self) -> Result<ModuleStatus, SuperviseError> {
4877 self.status_with_snapshot_lock(&self.inner.snapshot, None)
4878 .map(|(status, _)| status)
4879 }
4880
4881 pub(crate) fn next_retry_at_ms(&self) -> Result<Option<u64>, SuperviseError> {
4882 let state = lock_snapshot(&self.inner.snapshot)?;
4883 Ok(state
4884 .failed_retry
4885 .pending
4886 .filter(|_| state.state == ModuleState::Failed && state.enabled)
4887 .map(|retry| {
4888 unix_ms_now().saturating_add(
4889 u64::try_from(
4890 retry
4891 .deadline
4892 .saturating_duration_since(Instant::now())
4893 .as_millis(),
4894 )
4895 .unwrap_or(u64::MAX),
4896 )
4897 }))
4898 }
4899
4900 #[cfg(all(test, unix))]
4901 pub(crate) fn retry_deadline_for_test(&self) -> Option<Instant> {
4902 lock_snapshot(&self.inner.snapshot)
4903 .unwrap()
4904 .failed_retry
4905 .pending
4906 .map(|retry| retry.deadline)
4907 }
4908
4909 pub(crate) fn record_deliberate_severance(
4910 &self,
4911 identity: ProcessIdentity,
4912 ) -> Result<bool, SuperviseError> {
4913 let mut snapshot = lock_snapshot(&self.inner.snapshot)?;
4914 if snapshot.pid != Some(identity.pid)
4915 || snapshot.process_start_time != Some(identity.start_time)
4916 {
4917 return Ok(false);
4918 }
4919 snapshot.deliberate_severance = Some(identity);
4920 Ok(true)
4921 }
4922
4923 pub(crate) fn status_for_control(
4928 &self,
4929 caller: &'static str,
4930 ) -> Result<ModuleStatus, SuperviseError> {
4931 self.status_with_snapshot_lock(&self.inner.snapshot, Some(caller))
4932 .map(|(status, _)| status)
4933 }
4934
4935 fn status_with_snapshot_lock(
4936 &self,
4937 snapshot: &SharedSnapshot,
4938 caller: Option<&'static str>,
4939 ) -> Result<(ModuleStatus, Option<SpawnedFileIdentity>), SuperviseError> {
4940 let restart_policy = self
4941 .inner
4942 .configuration
4943 .lock()
4944 .map_err(|_| SuperviseError::StatePoisoned {
4945 module_id: Some(self.inner.module_id.clone()),
4946 })?
4947 .restart_policy;
4948 let mut guard = match caller {
4949 Some(caller) => lock_snapshot_for_control(snapshot, &self.inner.module_id, caller)?,
4950 None => lock_snapshot(snapshot)?,
4951 };
4952 let restart_count = guard.crash_restarts_in_window(restart_policy.window, Instant::now());
4955 let snapshot = guard.clone();
4956 drop(guard);
4957 let drain_timeout = *self.inner.effective_drain_timeout.lock().map_err(|_| {
4958 SuperviseError::StatePoisoned {
4959 module_id: Some(self.inner.module_id.clone()),
4960 }
4961 })?;
4962 let registration_active = self
4963 .inner
4964 .registry
4965 .get_module(&self.inner.module_id)
4966 .map_err(SuperviseError::Registry)?
4967 .is_some();
4968 let protocol = snapshot
4969 .spawned_protocol
4970 .unwrap_or(self.declared_protocol()?);
4971 let running_process =
4972 snapshot.enabled && snapshot.state == ModuleState::Running && snapshot.process_alive;
4973 let live = match protocol {
4979 ModuleProtocol::Subc => running_process && registration_active,
4980 ModuleProtocol::None => running_process,
4981 };
4982 let retained = self.inner.retained.waiting_detail(&self.inner.module_id);
4985 let carry_configured = self
4986 .inner
4987 .configuration
4988 .lock()
4989 .map_err(|_| SuperviseError::StatePoisoned {
4990 module_id: Some(self.inner.module_id.clone()),
4991 })?
4992 .spec
4993 .carry_containment;
4994 let recovery_block = carries_containment(
4995 carry_configured,
4996 self.inner
4997 .retained
4998 .has_carried_container(&self.inner.module_id),
4999 )
5000 .then(|| self.inner.retained.recovery_block())
5001 .flatten();
5002 let launch_nonce_source = observed_launch_nonce_source(&snapshot);
5003 let pid = snapshot.reported_pid();
5004 let mut health = snapshot.health;
5005 if let Some(detail) = recovery_block
5006 .as_ref()
5007 .or_else(|| retained.as_ref().map(|(_, detail)| detail))
5008 {
5009 health.detail = Some(match health.detail {
5010 Some(previous) => format!("{previous}; {detail}"),
5011 None => detail.clone(),
5012 });
5013 }
5014
5015 Ok((
5016 ModuleStatus {
5017 admission_refused: snapshot.admission_refused,
5018 legacy_registration_spawn: snapshot.legacy_registration_spawn,
5019 registration_declaration: snapshot.registration_declaration.clone(),
5020 stop_reason: snapshot.stop_reason,
5021 waiting_for_previous_pid: retained
5022 .as_ref()
5023 .map(|(pid, _)| *pid)
5024 .filter(|pid| *pid > 0)
5025 .or(snapshot.waiting_for_previous_pid),
5026 waiting_for_previous_detail: recovery_block
5027 .or_else(|| retained.map(|(_, detail)| detail))
5028 .or(snapshot.waiting_for_previous_detail),
5029 module_id: self.inner.module_id.clone(),
5030 launch_nonce_source,
5031 state: snapshot.state,
5032 enabled: snapshot.enabled,
5033 process_alive: snapshot.process_alive,
5034 registration_active,
5035 protocol,
5036 live,
5037 restart_count,
5038 lifetime_restarts: snapshot.lifetime_restarts,
5039 spawn_generation: snapshot.spawn_generation,
5040 swap_outcome: snapshot.swap_outcome,
5041 max_restarts: restart_policy.max_restarts,
5042 restart_window: restart_policy.window,
5043 drain_timeout,
5044 restart_backoff: restart_policy.backoff,
5045 restart_max_backoff: restart_policy.max_backoff,
5046 pid,
5047 spawned_at_ms: snapshot.spawned_at_ms,
5048 spawned_from: snapshot.spawned_from,
5049 launch_spec_digest: snapshot.launch_spec_digest,
5050 process_start_time: snapshot.process_start_time,
5051 last_exit: snapshot.last_exit,
5052 health,
5053 },
5054 snapshot.spawned_file_identity.clone(),
5055 ))
5056 }
5057
5058 #[cfg(test)]
5059 pub(crate) fn hold_snapshot_for_test(
5060 &self,
5061 acquired: std::sync::mpsc::Sender<()>,
5062 hold: Duration,
5063 ) -> std::thread::JoinHandle<()> {
5064 let snapshot = Arc::clone(&self.inner.snapshot);
5065 std::thread::spawn(move || {
5066 let _guard = snapshot.lock().expect("test snapshot lock is not poisoned");
5067 acquired
5068 .send(())
5069 .expect("test receiver waits for snapshot lock");
5070 std::thread::sleep(hold);
5071 })
5072 }
5073
5074 pub(crate) async fn status_and_running_image_agreement(
5080 &self,
5081 ) -> Result<(ModuleStatus, subc_control::RunningImageAgreement), SuperviseError> {
5082 let (status, identity) = self.status_with_snapshot_lock(&self.inner.snapshot, None)?;
5083 let image = self
5084 .inner
5085 .provenance_probe
5086 .observe(
5087 status.pid,
5088 status.spawned_from.as_deref(),
5089 identity,
5090 status.process_start_time,
5091 )
5092 .await;
5093 Ok((status, image))
5094 }
5095
5096 pub(crate) async fn running_image_agreement(&self) -> subc_control::RunningImageAgreement {
5097 let snapshot = match lock_snapshot(&self.inner.snapshot) {
5098 Ok(snapshot) => snapshot.clone(),
5099 Err(_) => {
5100 return subc_control::RunningImageAgreement::Unavailable {
5101 reason: subc_control::RunningImageUnavailableReason::NotRunning,
5102 };
5103 }
5104 };
5105 self.inner
5106 .provenance_probe
5107 .observe(
5108 snapshot.reported_pid(),
5109 snapshot.spawned_from.as_deref(),
5110 snapshot.spawned_file_identity.clone(),
5111 snapshot.process_start_time,
5112 )
5113 .await
5114 }
5115
5116 pub(crate) fn child_resource_usage(&self) -> subc_control::ChildResourceUsage {
5119 #[cfg(windows)]
5120 {
5121 let image = lock_snapshot(&self.inner.snapshot)
5122 .ok()
5123 .and_then(|snapshot| {
5124 snapshot
5125 .spawned_file_identity
5126 .as_ref()
5127 .and_then(|identity| identity.windows_image.clone())
5128 });
5129 match image {
5130 Some(image) => {
5131 crate::child_resources::read_windows_handle(image.process(), image.start_time())
5132 }
5133 None => subc_control::ChildResourceUsage::Unavailable {
5134 reason: subc_control::ChildResourceUnavailableReason::NotRunning,
5135 },
5136 }
5137 }
5138 #[cfg(not(windows))]
5139 {
5140 let (pid, start_time) = match lock_snapshot(&self.inner.snapshot) {
5141 Ok(snapshot) => (snapshot.reported_pid(), snapshot.process_start_time),
5142 Err(_) => {
5143 return subc_control::ChildResourceUsage::Unavailable {
5144 reason: subc_control::ChildResourceUnavailableReason::Unreadable,
5145 }
5146 }
5147 };
5148 crate::child_resources::read(pid, start_time)
5149 }
5150 }
5151
5152 #[cfg(all(test, any(target_os = "linux", windows)))]
5153 pub(crate) fn subtree_target(&self) -> crate::subtree_resources::Target {
5154 self.inner
5155 .snapshot
5156 .try_lock()
5157 .map(|snapshot| snapshot.subtree_target.clone())
5158 .unwrap_or(crate::subtree_resources::Target::Unavailable("read_failed"))
5159 }
5160
5161 #[cfg(test)]
5162 pub(crate) fn set_subtree_target_for_test(&self, target: crate::subtree_resources::Target) {
5163 self.inner.snapshot.lock().unwrap().subtree_target = target;
5164 }
5165
5166 pub(crate) fn will_recover_after_connection_loss(&self) -> Result<bool, SuperviseError> {
5167 let mut snapshot = lock_snapshot(&self.inner.snapshot)?;
5168 Ok(match snapshot.state {
5169 ModuleState::Restarting => true,
5170 ModuleState::Failed | ModuleState::Disabled => false,
5171 _ if snapshot.enabled && snapshot.respawn_pending => true,
5172 _ => daemon_will_restart(
5173 &mut snapshot,
5174 &self
5175 .inner
5176 .configuration
5177 .lock()
5178 .map_err(|_| SuperviseError::StatePoisoned {
5179 module_id: Some(self.inner.module_id.clone()),
5180 })?
5181 .restart_policy,
5182 Instant::now(),
5183 ),
5184 })
5185 }
5186
5187 #[cfg(test)]
5188 pub(crate) fn is_warming(&self) -> Result<bool, SuperviseError> {
5189 self.is_warming_with_snapshot_lock(None)
5190 }
5191
5192 pub(crate) fn is_warming_for_control(
5193 &self,
5194 caller: &'static str,
5195 ) -> Result<bool, SuperviseError> {
5196 self.is_warming_with_snapshot_lock(Some(caller))
5197 }
5198
5199 fn is_warming_with_snapshot_lock(
5200 &self,
5201 caller: Option<&'static str>,
5202 ) -> Result<bool, SuperviseError> {
5203 let snapshot = match caller {
5204 Some(caller) => {
5205 lock_snapshot_for_control(&self.inner.snapshot, &self.inner.module_id, caller)?
5206 }
5207 None => lock_snapshot(&self.inner.snapshot)?,
5208 }
5209 .clone();
5210 Ok(matches!(
5211 snapshot.state,
5212 ModuleState::Starting | ModuleState::Running | ModuleState::Restarting
5213 ))
5214 }
5215
5216 pub async fn drain(&self) -> Result<(), SuperviseError> {
5218 self.stop().await
5219 }
5220
5221 pub(crate) async fn retire_requested(
5222 &self,
5223 requester: Option<Principal>,
5224 ) -> Result<(), SuperviseError> {
5225 match self.state()? {
5226 ModuleState::Stopped => return Ok(()),
5227 ModuleState::Failed
5228 if lock_snapshot(&self.inner.snapshot)?
5229 .failed_retry
5230 .pending
5231 .is_none() =>
5232 {
5233 return Ok(())
5234 }
5235 ModuleState::Failed
5236 | ModuleState::Starting
5237 | ModuleState::Running
5238 | ModuleState::Unresponsive
5239 | ModuleState::Restarting
5240 | ModuleState::Draining
5241 | ModuleState::Disabled => {}
5242 }
5243
5244 let (reply_tx, reply_rx) = oneshot::channel();
5245 self.inner
5246 .commands
5247 .send(SupervisorCommand::Retire {
5248 requester,
5249 reply: reply_tx,
5250 })
5251 .await
5252 .map_err(|_| SuperviseError::CommandClosed {
5253 module_id: self.inner.module_id.clone(),
5254 })?;
5255 reply_rx.await.map_err(|_| SuperviseError::CommandClosed {
5256 module_id: self.inner.module_id.clone(),
5257 })?
5258 }
5259
5260 pub async fn stop(&self) -> Result<(), SuperviseError> {
5261 self.stop_requested(None).await
5262 }
5263
5264 pub(crate) async fn stop_requested(
5265 &self,
5266 requester: Option<Principal>,
5267 ) -> Result<(), SuperviseError> {
5268 match self.state()? {
5269 ModuleState::Stopped => return Ok(()),
5270 ModuleState::Failed
5271 if lock_snapshot(&self.inner.snapshot)?
5272 .failed_retry
5273 .pending
5274 .is_none() =>
5275 {
5276 return Ok(())
5277 }
5278 ModuleState::Failed
5279 | ModuleState::Starting
5280 | ModuleState::Running
5281 | ModuleState::Unresponsive
5282 | ModuleState::Restarting
5283 | ModuleState::Draining
5284 | ModuleState::Disabled => {}
5285 }
5286
5287 let (reply_tx, reply_rx) = oneshot::channel();
5288 self.inner
5289 .commands
5290 .send(SupervisorCommand::Drain {
5291 requester,
5292 reply: reply_tx,
5293 })
5294 .await
5295 .map_err(|_| SuperviseError::CommandClosed {
5296 module_id: self.inner.module_id.clone(),
5297 })?;
5298 reply_rx.await.map_err(|_| SuperviseError::CommandClosed {
5299 module_id: self.inner.module_id.clone(),
5300 })?
5301 }
5302
5303 pub async fn restart(&self, drain_timeout_ms: Option<u64>) -> Result<(), SuperviseError> {
5304 self.restart_requested(drain_timeout_ms, None).await
5305 }
5306
5307 pub(crate) async fn restart_requested(
5308 &self,
5309 drain_timeout_ms: Option<u64>,
5310 requester: Option<Principal>,
5311 ) -> Result<(), SuperviseError> {
5312 let queued_at = Instant::now();
5313 let received_at_generation = lock_snapshot(&self.inner.snapshot)?.spawn_generation;
5314 let (reply_tx, reply_rx) = oneshot::channel();
5315 self.inner
5316 .commands
5317 .send(SupervisorCommand::Restart {
5318 requester,
5319 drain_timeout_ms,
5320 received_at_generation,
5321 queued_at,
5322 reply: reply_tx,
5323 })
5324 .await
5325 .map_err(|_| SuperviseError::CommandClosed {
5326 module_id: self.inner.module_id.clone(),
5327 })?;
5328 reply_rx.await.map_err(|_| SuperviseError::CommandClosed {
5329 module_id: self.inner.module_id.clone(),
5330 })?
5331 }
5332
5333 pub async fn swap(
5339 &self,
5340 ready_timeout: Option<Duration>,
5341 ) -> Result<SwapOutcome, SuperviseError> {
5342 self.swap_requested(ready_timeout, None).await
5343 }
5344
5345 pub(crate) async fn swap_requested(
5346 &self,
5347 ready_timeout: Option<Duration>,
5348 requester: Option<Principal>,
5349 ) -> Result<SwapOutcome, SuperviseError> {
5350 let (reply_tx, reply_rx) = oneshot::channel();
5351 self.inner
5352 .commands
5353 .send(SupervisorCommand::Swap {
5354 requester,
5355 ready_timeout,
5356 reply: reply_tx,
5357 })
5358 .await
5359 .map_err(|_| SuperviseError::CommandClosed {
5360 module_id: self.inner.module_id.clone(),
5361 })?;
5362 reply_rx.await.map_err(|_| SuperviseError::CommandClosed {
5363 module_id: self.inner.module_id.clone(),
5364 })?
5365 }
5366
5367 pub async fn reload(&self) -> Result<(), SuperviseError> {
5368 self.reload_requested(None).await
5369 }
5370
5371 pub(crate) async fn reload_requested(
5372 &self,
5373 requester: Option<Principal>,
5374 ) -> Result<(), SuperviseError> {
5375 let (reply_tx, reply_rx) = oneshot::channel();
5376 self.inner
5377 .commands
5378 .send(SupervisorCommand::Reload {
5379 requester,
5380 reply: reply_tx,
5381 })
5382 .await
5383 .map_err(|_| SuperviseError::CommandClosed {
5384 module_id: self.inner.module_id.clone(),
5385 })?;
5386 reply_rx.await.map_err(|_| SuperviseError::CommandClosed {
5387 module_id: self.inner.module_id.clone(),
5388 })?
5389 }
5390
5391 pub async fn set_enabled(&self, enabled: bool) -> Result<bool, SuperviseError> {
5392 self.set_enabled_requested(enabled, None).await
5393 }
5394
5395 pub(crate) async fn set_enabled_requested(
5396 &self,
5397 enabled: bool,
5398 requester: Option<Principal>,
5399 ) -> Result<bool, SuperviseError> {
5400 self.set_enabled_with_budget_policy(enabled, requester, true)
5401 .await
5402 }
5403
5404 pub(crate) async fn release_config_hold(&self) -> Result<bool, SuperviseError> {
5405 self.set_enabled_with_budget_policy(true, None, false).await
5406 }
5407
5408 #[cfg(all(test, unix))]
5409 pub(crate) fn seed_crash_budget_for_test(&self, count: u32) {
5410 let policy = self.inner.configuration.lock().unwrap().restart_policy;
5411 let mut state = self.inner.snapshot.lock().unwrap();
5412 for _ in 0..count {
5413 state.record_crash_restart(&policy, Instant::now());
5414 }
5415 }
5416
5417 async fn set_enabled_with_budget_policy(
5418 &self,
5419 enabled: bool,
5420 requester: Option<Principal>,
5421 reset_budget: bool,
5422 ) -> Result<bool, SuperviseError> {
5423 let (reply_tx, reply_rx) = oneshot::channel();
5424 self.inner
5425 .commands
5426 .send(SupervisorCommand::SetEnabled {
5427 requester,
5428 enabled,
5429 reset_budget,
5430 reply: reply_tx,
5431 })
5432 .await
5433 .map_err(|_| SuperviseError::CommandClosed {
5434 module_id: self.inner.module_id.clone(),
5435 })?;
5436 reply_rx.await.map_err(|_| SuperviseError::CommandClosed {
5437 module_id: self.inner.module_id.clone(),
5438 })?
5439 }
5440
5441 pub(crate) fn declared_protocol(&self) -> Result<ModuleProtocol, SuperviseError> {
5445 let configured = self
5446 .inner
5447 .configuration
5448 .lock()
5449 .map_err(|_| SuperviseError::StatePoisoned {
5450 module_id: Some(self.inner.module_id.clone()),
5451 })?
5452 .spec
5453 .protocol;
5454 let state = lock_snapshot(&self.inner.snapshot)?;
5455 Ok(state.spawned_protocol.unwrap_or(configured))
5456 }
5457
5458 pub(crate) fn configuration(&self) -> Result<(ModuleSpec, HealthConfig), SuperviseError> {
5459 let configuration =
5460 self.inner
5461 .configuration
5462 .lock()
5463 .map_err(|_| SuperviseError::StatePoisoned {
5464 module_id: Some(self.inner.module_id.clone()),
5465 })?;
5466 Ok((configuration.spec.clone(), configuration.health.clone()))
5467 }
5468
5469 pub(crate) fn configured_register_timeout(&self) -> Result<Duration, SuperviseError> {
5470 self.inner
5471 .configuration
5472 .lock()
5473 .map(|configuration| configuration.register_timeout)
5474 .map_err(|_| SuperviseError::StatePoisoned {
5475 module_id: Some(self.inner.module_id.clone()),
5476 })
5477 }
5478
5479 pub(crate) fn configured_enabled(&self) -> Result<bool, SuperviseError> {
5480 self.inner
5481 .configuration
5482 .lock()
5483 .map(|configuration| configuration.enabled)
5484 .map_err(|_| SuperviseError::StatePoisoned {
5485 module_id: Some(self.inner.module_id.clone()),
5486 })
5487 }
5488
5489 #[cfg(any(test, feature = "test-support"))]
5493 pub async fn update_spec_for_test(&self, spec: ModuleSpec) -> Result<(), SuperviseError> {
5494 let (_, health) = self.configuration()?;
5495 let drain_timeout_ms = u64::try_from(
5496 self.inner
5497 .effective_drain_timeout
5498 .lock()
5499 .unwrap_or_else(|poisoned| poisoned.into_inner())
5500 .as_millis(),
5501 )
5502 .ok();
5503 self.update_configuration(spec, health, drain_timeout_ms)
5504 .await
5505 }
5506
5507 pub(crate) async fn reconcile_approved_configuration(
5508 &self,
5509 approved: crate::config_gate::runtime::ApprovedDaemonConfiguration,
5510 ) -> Result<(), SuperviseError> {
5511 let entry = approved
5512 .module_entry(&self.inner.module_id)
5513 .map_err(|error| SuperviseError::InvalidSpec {
5514 reason: error.to_string(),
5515 })?;
5516 validate_spec(&entry.module_spec())?;
5517 let (reply, received) = oneshot::channel();
5518 self.inner
5519 .commands
5520 .send(SupervisorCommand::ApprovedConfiguration { approved, reply })
5521 .await
5522 .map_err(|_| SuperviseError::CommandClosed {
5523 module_id: self.inner.module_id.clone(),
5524 })?;
5525 received.await.map_err(|_| SuperviseError::CommandClosed {
5526 module_id: self.inner.module_id.clone(),
5527 })??;
5528 let mut configuration =
5529 self.inner
5530 .configuration
5531 .lock()
5532 .map_err(|_| SuperviseError::StatePoisoned {
5533 module_id: Some(self.inner.module_id.clone()),
5534 })?;
5535 configuration.spec = entry.module_spec();
5536 configuration.enabled = entry.enabled;
5537 configuration.health = entry.health;
5538 configuration.register_timeout = entry.register_timeout;
5539 configuration.restart_policy = entry.restart;
5540 Ok(())
5541 }
5542
5543 pub(crate) async fn update_configuration(
5544 &self,
5545 spec: ModuleSpec,
5546 health: HealthConfig,
5547 drain_timeout_ms: Option<u64>,
5548 ) -> Result<(), SuperviseError> {
5549 if spec.module_id != self.inner.module_id {
5550 return Err(SuperviseError::InvalidSpec {
5551 reason: "a supervised module's module_id cannot be changed".to_string(),
5552 });
5553 }
5554 validate_spec(&spec)?;
5555 let (reply_tx, reply_rx) = oneshot::channel();
5556 self.inner
5557 .commands
5558 .send(SupervisorCommand::UpdateConfiguration {
5559 spec: spec.clone(),
5560 health: health.clone(),
5561 drain_timeout_ms,
5562 reply: reply_tx,
5563 })
5564 .await
5565 .map_err(|_| SuperviseError::CommandClosed {
5566 module_id: self.inner.module_id.clone(),
5567 })?;
5568 reply_rx.await.map_err(|_| SuperviseError::CommandClosed {
5569 module_id: self.inner.module_id.clone(),
5570 })??;
5571 let mut configuration =
5572 self.inner
5573 .configuration
5574 .lock()
5575 .map_err(|_| SuperviseError::StatePoisoned {
5576 module_id: Some(self.inner.module_id.clone()),
5577 })?;
5578 configuration.spec = spec;
5579 configuration.health = health;
5580 Ok(())
5581 }
5582}
5583
5584impl Drop for SupervisedModuleInner {
5585 fn drop(&mut self) {
5586 let Ok(mut monitor) = self.monitor.lock() else {
5587 return;
5588 };
5589 if let Some(monitor) = monitor.as_ref().filter(|monitor| !monitor.is_finished()) {
5590 let _ = update_snapshot(&self.snapshot, Some(&self.module_id), |state| {
5591 state.state = ModuleState::Stopped;
5592 clear_current_process_facts(state);
5593 });
5594 monitor.abort();
5595 }
5596 let _ = monitor.take();
5597 }
5598}
5599
5600#[derive(Debug)]
5601enum SupervisorCommand {
5602 ApprovedConfiguration {
5603 approved: crate::config_gate::runtime::ApprovedDaemonConfiguration,
5604 reply: oneshot::Sender<Result<(), SuperviseError>>,
5605 },
5606 ConfigPolicyProbe {
5607 connection: crate::ConnectionId,
5608 generation: u64,
5609 attempt: u64,
5610 },
5611 ConfigApplyRestart {
5612 generation: u64,
5613 supersedes_swap: bool,
5614 },
5615 ConfigExitHold {
5616 reply: oneshot::Sender<Result<(), SuperviseError>>,
5617 },
5618 ConfigDeliveryFailed {
5619 connection: crate::ConnectionId,
5620 hold: bool,
5621 },
5622 Drain {
5623 requester: Option<Principal>,
5624 reply: oneshot::Sender<Result<(), SuperviseError>>,
5625 },
5626 Retire {
5627 requester: Option<Principal>,
5628 reply: oneshot::Sender<Result<(), SuperviseError>>,
5629 },
5630 Restart {
5631 requester: Option<Principal>,
5632 drain_timeout_ms: Option<u64>,
5637 received_at_generation: u64,
5641 queued_at: Instant,
5644 reply: oneshot::Sender<Result<(), SuperviseError>>,
5645 },
5646 Reload {
5647 requester: Option<Principal>,
5648 reply: oneshot::Sender<Result<(), SuperviseError>>,
5649 },
5650 SetEnabled {
5651 reset_budget: bool,
5652 requester: Option<Principal>,
5653 enabled: bool,
5654 reply: oneshot::Sender<Result<bool, SuperviseError>>,
5655 },
5656 UpdateConfiguration {
5657 spec: ModuleSpec,
5658 health: HealthConfig,
5659 drain_timeout_ms: Option<u64>,
5662 reply: oneshot::Sender<Result<(), SuperviseError>>,
5663 },
5664 Swap {
5665 requester: Option<Principal>,
5666 ready_timeout: Option<Duration>,
5669 reply: oneshot::Sender<Result<SwapOutcome, SuperviseError>>,
5671 },
5672}
5673
5674#[derive(Clone, Copy, Debug, PartialEq, Eq)]
5678pub enum SwapOutcome {
5679 Cutover,
5680 SupersededByConfigRestart,
5681}
5682
5683impl SwapOutcome {
5684 pub fn as_str(self) -> &'static str {
5685 match self {
5686 Self::Cutover => "cutover",
5687 Self::SupersededByConfigRestart => "superseded_by_config_restart",
5688 }
5689 }
5690}
5691
5692#[derive(Debug)]
5693pub enum SuperviseError {
5694 ConfigHeld {
5695 module_id: String,
5696 code: String,
5697 detail: String,
5698 },
5699 ConfigDeliveryDeferred {
5700 module_id: String,
5701 },
5702 InvalidSpec {
5703 reason: String,
5704 },
5705 Spawn {
5706 program: PathBuf,
5707 source: io::Error,
5708 cgroup_path: Option<PathBuf>,
5709 },
5710 Cgroup {
5711 module_id: String,
5712 source: io::Error,
5713 },
5714 LaunchNonce {
5717 reason: String,
5718 },
5719 Wait {
5720 module_id: String,
5721 source: io::Error,
5722 },
5723 Kill {
5724 module_id: String,
5725 source: io::Error,
5726 },
5727 Forwarding(ForwardingError),
5728 Registry(RegistryError),
5729 ReloadUnavailable {
5730 module_id: String,
5731 reason: String,
5732 },
5733 Disabled {
5738 module_id: String,
5739 },
5740 ReloadFailed {
5741 module_id: String,
5742 reason: String,
5743 },
5744 RegistrationStillActive {
5745 module_id: String,
5746 waited: Duration,
5747 },
5748 StatePoisoned {
5749 module_id: Option<String>,
5750 },
5751 CommandClosed {
5752 module_id: String,
5753 },
5754 SwapInProgress {
5758 module_id: String,
5759 },
5760 SwapRefused {
5762 module_id: String,
5763 reason: SwapRefusal,
5764 },
5765 SwapFailed {
5769 module_id: String,
5770 arm: SwapFailureArm,
5771 detail: String,
5772 candidate_exit: Option<ExitReport>,
5775 },
5776}
5777
5778#[derive(Debug, Clone, Copy, PartialEq, Eq)]
5780pub enum SwapRefusal {
5781 OverlapExclusive,
5783 NotRegistered,
5786 ProtocolNone,
5789 NotConfigured,
5792 AlreadySwapping,
5794}
5795
5796impl SwapRefusal {
5797 pub fn as_str(self) -> &'static str {
5798 match self {
5799 Self::OverlapExclusive => "overlap_exclusive",
5800 Self::NotRegistered => "not_registered",
5801 Self::ProtocolNone => "protocol_none",
5802 Self::NotConfigured => "not_configured",
5803 Self::AlreadySwapping => "already_swapping",
5804 }
5805 }
5806}
5807
5808#[derive(Debug, Clone, Copy, PartialEq, Eq)]
5811pub enum SwapFailureArm {
5812 SpawnFailed,
5814 NeverRegistered,
5816 NeverReady,
5818 CandidateExited,
5820 CandidateUnhealthy,
5822 Interrupted,
5826 CutoverLost,
5831}
5832
5833impl SwapFailureArm {
5834 pub fn as_str(self) -> &'static str {
5835 match self {
5836 Self::SpawnFailed => "spawn_failed",
5837 Self::NeverRegistered => "never_registered",
5838 Self::NeverReady => "never_ready",
5839 Self::CandidateExited => "candidate_exited",
5840 Self::CandidateUnhealthy => "candidate_unhealthy",
5841 Self::Interrupted => "interrupted",
5842 Self::CutoverLost => "cutover_lost",
5843 }
5844 }
5845}
5846
5847impl SuperviseError {
5848 pub(crate) fn is_config_gate_error(&self) -> bool {
5849 match self {
5850 Self::ConfigHeld { .. } => true,
5851 Self::Spawn { source, .. } => source
5852 .get_ref()
5853 .is_some_and(|cause| cause.is::<crate::config_gate::GateError>()),
5854 _ => false,
5855 }
5856 }
5857}
5858
5859impl fmt::Display for SuperviseError {
5860 fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
5861 match self {
5862 Self::ConfigHeld { module_id, code, detail } => write!(f, "{code}: module '{module_id}' held: {detail}"),
5863 Self::ConfigDeliveryDeferred { module_id } => write!(f, "config_gated: module '{module_id}' is waiting for its exited delivery target's failure record"),
5864 Self::InvalidSpec { reason } => write!(f, "invalid module spec: {reason}"),
5865 Self::Spawn {
5866 program,
5867 source,
5868 cgroup_path: Some(cgroup_path),
5869 } => write!(
5870 f,
5871 "failed to place module in cgroup '{}' while spawning '{}': {source}",
5872 cgroup_path.display(),
5873 program.display()
5874 ),
5875 Self::Spawn {
5876 program,
5877 source,
5878 cgroup_path: None,
5879 } => write!(
5880 f,
5881 "failed to spawn module '{}': {source}",
5882 program.display()
5883 ),
5884 Self::Cgroup { module_id, source } => {
5885 write!(
5886 f,
5887 "failed to prepare cgroup for module '{module_id}': {source}"
5888 )
5889 }
5890 Self::LaunchNonce { reason } => {
5891 write!(
5892 f,
5893 "failed to generate reserved-module launch nonce: {reason}"
5894 )
5895 }
5896 Self::Wait { module_id, source } => {
5897 write!(f, "failed to wait for module '{module_id}': {source}")
5898 }
5899 Self::Kill { module_id, source } => {
5900 write!(f, "failed to kill module '{module_id}': {source}")
5901 }
5902 Self::Forwarding(err) => write!(f, "forwarding error: {err}"),
5903 Self::Registry(err) => write!(f, "registry error: {err}"),
5904 Self::ReloadUnavailable { module_id, reason } => {
5905 write!(f, "reload unavailable for module '{module_id}': {reason}")
5906 }
5907 Self::Disabled { module_id } => {
5908 write!(
5909 f,
5910 "module '{module_id}' is disabled; enable it before restart or reload"
5911 )
5912 }
5913 Self::ReloadFailed { module_id, reason } => {
5914 write!(f, "reload failed for module '{module_id}': {reason}")
5915 }
5916 Self::RegistrationStillActive { module_id, waited } => write!(
5917 f,
5918 "module '{module_id}' registration remained active after waiting {waited:?}"
5919 ),
5920 Self::StatePoisoned { module_id } => match module_id {
5921 Some(module_id) => {
5922 write!(f, "supervisor state for module '{module_id}' was poisoned")
5923 }
5924 None => write!(f, "supervisor state was poisoned"),
5925 },
5926 Self::CommandClosed { module_id } => {
5927 write!(
5928 f,
5929 "supervisor command channel for module '{module_id}' is closed"
5930 )
5931 }
5932 Self::SwapInProgress { module_id } => write!(
5933 f,
5934 "module '{module_id}' is being swapped; retry once the swap has cut over or failed, or stop the module to abort the swap"
5935 ),
5936 Self::SwapRefused { module_id, reason } => match reason {
5937 SwapRefusal::OverlapExclusive => write!(
5938 f,
5939 "module '{module_id}' is declared overlap: \"exclusive\" (the default): two processes of it must not run at once, so it cannot be swapped; use a plain restart, or declare overlap: \"safe\" in its config if it really tolerates a second process"
5940 ),
5941 SwapRefusal::NotRegistered => write!(
5942 f,
5943 "module '{module_id}' is not registered, so there is no serving process to keep while a replacement warms; use a plain restart"
5944 ),
5945 SwapRefusal::ProtocolNone => write!(
5946 f,
5947 "module '{module_id}' is protocol: \"none\" and never registers, so a swap could never see its replacement become ready; use a plain restart"
5948 ),
5949 SwapRefusal::NotConfigured => write!(
5950 f,
5951 "module '{module_id}' cannot be swapped: the supervisor was built without the forwarding table or shared handle a swap needs"
5952 ),
5953 SwapRefusal::AlreadySwapping => {
5954 write!(f, "module '{module_id}' is already being swapped")
5955 }
5956 },
5957 Self::SwapFailed {
5958 module_id,
5959 arm,
5960 detail,
5961 ..
5962 } => write!(
5963 f,
5964 "swap of module '{module_id}' failed ({}): {detail}; the running process was left serving",
5965 arm.as_str()
5966 ),
5967 }
5968 }
5969}
5970
5971impl Error for SuperviseError {
5972 fn source(&self) -> Option<&(dyn Error + 'static)> {
5973 match self {
5974 Self::Spawn { source, .. }
5975 | Self::Cgroup { source, .. }
5976 | Self::Wait { source, .. }
5977 | Self::Kill { source, .. } => Some(source),
5978 Self::Forwarding(err) => Some(err),
5979 Self::Registry(err) => Some(err),
5980 Self::LaunchNonce { .. }
5981 | Self::ConfigDeliveryDeferred { .. }
5982 | Self::ConfigHeld { .. }
5983 | Self::InvalidSpec { .. }
5984 | Self::ReloadUnavailable { .. }
5985 | Self::Disabled { .. }
5986 | Self::ReloadFailed { .. }
5987 | Self::RegistrationStillActive { .. }
5988 | Self::StatePoisoned { .. }
5989 | Self::CommandClosed { .. }
5990 | Self::SwapInProgress { .. }
5991 | Self::SwapRefused { .. }
5992 | Self::SwapFailed { .. } => None,
5993 }
5994 }
5995}
5996
5997pub(crate) fn validate_spec(spec: &ModuleSpec) -> Result<(), SuperviseError> {
5998 if spec.carry_containment && spec.overlap == ModuleOverlap::Safe {
5999 return Err(SuperviseError::InvalidSpec {
6000 reason: "carry_containment cannot be combined with overlap: safe".to_owned(),
6001 });
6002 }
6003 if spec.module_id.trim().is_empty() {
6004 return Err(SuperviseError::InvalidSpec {
6005 reason: "module_id must not be empty".to_string(),
6006 });
6007 }
6008
6009 Ok(())
6010}
6011
6012#[derive(Debug, Default)]
6013struct HealthProbeRuntime {
6014 spawn_generation: Option<u64>,
6015 configured_health: Option<HealthConfig>,
6016 registered_connection: Option<crate::ConnectionId>,
6017 advertised: bool,
6018 next_probe_at: Option<Instant>,
6019 probe_index: u64,
6020}
6021
6022fn running_protocol(spec: &ModuleSpec, snapshot: &SharedSnapshot) -> ModuleProtocol {
6023 lock_snapshot(snapshot)
6024 .ok()
6025 .and_then(|state| state.spawned_protocol)
6026 .unwrap_or(spec.protocol)
6027}
6028
6029impl HealthProbeRuntime {
6030 fn refresh_registration(
6031 &mut self,
6032 spec: &ModuleSpec,
6033 runtime: &SupervisorRuntimeConfig,
6034 registry: &Registry,
6035 snapshot: &SharedSnapshot,
6036 ) {
6037 let generation = lock_snapshot(snapshot)
6038 .ok()
6039 .map(|state| state.spawn_generation);
6040 if self.spawn_generation != generation {
6041 *self = Self {
6045 spawn_generation: generation,
6046 ..Self::default()
6047 };
6048 }
6049 if self.configured_health.as_ref() != Some(&runtime.health) {
6050 self.configured_health = Some(runtime.health.clone());
6051 self.next_probe_at = None;
6052 self.registered_connection = None;
6053 self.probe_index = 0;
6054 }
6055 if running_protocol(spec, snapshot) == ModuleProtocol::None {
6059 self.registered_connection = None;
6060 self.advertised = runtime.health.http.is_some();
6061 if !self.advertised {
6062 self.next_probe_at = None;
6063 let _ = update_snapshot(snapshot, Some(&spec.module_id), |state| {
6064 let unknown = ModuleHealthStatus::default();
6065 if state.health != unknown {
6066 state.health = unknown;
6067 state.last_probe_evidence = None;
6068 }
6069 });
6070 } else if self.next_probe_at.is_none() {
6071 self.next_probe_at = Some(
6072 Instant::now()
6073 + jittered_health_delay(&spec.module_id, 0, runtime.health.cadence),
6074 );
6075 }
6076 return;
6077 }
6078
6079 let registration = match registry.get_module(&spec.module_id) {
6080 Ok(registration) => registration,
6081 Err(err) => {
6082 warn!(module_id = %spec.module_id, error = %err, "health prober could not read registry");
6083 self.advertised = false;
6084 self.next_probe_at = None;
6085 return;
6086 }
6087 };
6088
6089 let Some(registration) = registration else {
6090 self.registered_connection = None;
6091 self.advertised = false;
6092 self.next_probe_at = None;
6093 return;
6094 };
6095
6096 let advertised = registration
6097 .control_ops
6098 .iter()
6099 .any(|op| op == MODULE_CONTROL_OP_HEALTH_CHECK);
6100 if !advertised {
6101 self.registered_connection = Some(registration.connection_id);
6102 self.advertised = false;
6103 self.next_probe_at = None;
6104 let _ = update_snapshot(snapshot, Some(&spec.module_id), |state| {
6105 if state.health.status != SupervisorHealthStatus::Unknown
6106 || state.health.consecutive_failures != 0
6107 || state.health.last_probe_ms.is_some()
6108 || state.health.detail.is_some()
6109 || state.health.metrics.is_some()
6110 {
6111 state.health.status = SupervisorHealthStatus::Unknown;
6112 state.last_probe_evidence = None;
6113 state.health.consecutive_failures = 0;
6114 state.health.last_probe_ms = None;
6115 state.health.detail = None;
6116 state.health.metrics = None;
6117 }
6118 });
6119 return;
6120 }
6121
6122 let reregistered = self.registered_connection != Some(registration.connection_id);
6123 self.registered_connection = Some(registration.connection_id);
6124 self.advertised = true;
6125 if reregistered || self.next_probe_at.is_none() {
6126 self.probe_index = 0;
6127 self.next_probe_at = Some(
6128 Instant::now() + jittered_health_delay(&spec.module_id, 0, runtime.health.cadence),
6129 );
6130 let _ = update_snapshot(snapshot, Some(&spec.module_id), |state| {
6131 state.health.status = SupervisorHealthStatus::Unknown;
6132 state.last_probe_evidence = None;
6133 state.health.consecutive_failures = 0;
6134 state.health.detail = None;
6135 state.health.metrics = None;
6136 });
6137 }
6138 }
6139
6140 fn wake_at(&self) -> Option<Instant> {
6141 if !self.advertised {
6142 return None;
6143 }
6144 self.next_probe_at
6145 }
6146
6147 fn due(&self) -> bool {
6148 self.advertised
6149 && self
6150 .next_probe_at
6151 .is_some_and(|next| Instant::now() >= next)
6152 }
6153
6154 fn schedule_next(&mut self, spec: &ModuleSpec, cadence: Duration) {
6155 self.probe_index = self.probe_index.wrapping_add(1);
6156 self.next_probe_at = Some(
6157 Instant::now() + jittered_health_delay(&spec.module_id, self.probe_index, cadence),
6158 );
6159 }
6160}
6161
6162#[derive(Debug, Clone, Copy, PartialEq, Eq)]
6166enum HealthProbeEvidence {
6167 LaneDead,
6169 NoAnswer,
6171 BadAnswer,
6173 Misconfigured,
6175}
6176
6177#[derive(Debug)]
6178struct HealthProbeError {
6179 evidence: HealthProbeEvidence,
6180 message: String,
6181}
6182
6183impl HealthProbeError {
6184 fn lane_dead(message: impl Into<String>) -> Self {
6185 Self::with(HealthProbeEvidence::LaneDead, message)
6186 }
6187
6188 fn no_answer(message: impl Into<String>) -> Self {
6189 Self::with(HealthProbeEvidence::NoAnswer, message)
6190 }
6191
6192 fn bad_answer(message: impl Into<String>) -> Self {
6193 Self::with(HealthProbeEvidence::BadAnswer, message)
6194 }
6195
6196 fn misconfigured(message: impl Into<String>) -> Self {
6197 Self::with(HealthProbeEvidence::Misconfigured, message)
6198 }
6199
6200 fn with(evidence: HealthProbeEvidence, message: impl Into<String>) -> Self {
6201 Self {
6202 evidence,
6203 message: message.into(),
6204 }
6205 }
6206
6207 #[allow(dead_code)]
6216 fn is_proof_of_death(&self) -> bool {
6217 matches!(self.evidence, HealthProbeEvidence::LaneDead)
6218 }
6219
6220 fn label(&self) -> &'static str {
6228 match self.evidence {
6229 HealthProbeEvidence::LaneDead => "lane-dead",
6230 HealthProbeEvidence::NoAnswer => "no-answer",
6231 HealthProbeEvidence::BadAnswer => "bad-answer",
6232 HealthProbeEvidence::Misconfigured => "daemon-misconfigured",
6233 }
6234 }
6235}
6236
6237impl fmt::Display for HealthProbeError {
6238 fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
6239 f.write_str(&self.message)
6240 }
6241}
6242
6243async fn run_health_probe_cycle(
6244 spec: &ModuleSpec,
6245 runtime: &SupervisorRuntimeConfig,
6246 registry: &Registry,
6247 process_liveness: &SupervisorProcessLiveness,
6248 snapshot: &SharedSnapshot,
6249 child: &mut Option<SupervisedChild>,
6250) {
6251 let now_ms = unix_ms_now();
6252 let delivery_probe = match &runtime.config_delivery {
6253 Some(delivery) => {
6254 delivery
6255 .health_probe(&spec.module_id, runtime.forwarding.as_deref(), registry)
6256 .await
6257 }
6258 None => None,
6259 };
6260 let http = (running_protocol(spec, snapshot) == ModuleProtocol::None)
6261 .then_some(runtime.health.http.as_deref())
6262 .flatten();
6263 let result = match http {
6264 Some(url) => probe_http_health(url, runtime.health.deadline).await,
6265 None => probe_module_health(&spec.module_id, runtime, None).await,
6266 };
6267 match result {
6268 Ok(report) => {
6269 handle_health_report(
6270 spec,
6271 runtime,
6272 registry,
6273 process_liveness,
6274 snapshot,
6275 child,
6276 report,
6277 now_ms,
6278 delivery_probe,
6279 )
6280 .await;
6281 }
6282 Err(err) => {
6283 if http.is_some() {
6284 let _ = update_snapshot(snapshot, Some(&spec.module_id), |state| {
6285 state.health.status = SupervisorHealthStatus::Failing;
6286 });
6287 }
6288 handle_health_probe_failure(
6289 spec,
6290 runtime,
6291 registry,
6292 process_liveness,
6293 snapshot,
6294 child,
6295 err,
6296 now_ms,
6297 delivery_probe,
6298 )
6299 .await;
6300 }
6301 }
6302}
6303
6304pub(crate) struct HttpProbeTarget<'a> {
6305 address: std::net::SocketAddr,
6306 localhost: bool,
6307 authority: &'a str,
6308 path: String,
6309}
6310
6311pub(crate) fn parse_http_probe_url(url: &str) -> Result<HttpProbeTarget<'_>, String> {
6314 if url.bytes().any(|b| b <= b' ' || b == 127) || url.contains('#') {
6315 return Err("must not contain whitespace, controls, or a fragment".into());
6316 }
6317 let rest = url
6318 .strip_prefix("http://")
6319 .ok_or("must use plain http://")?;
6320 let split = rest.find(['/', '?']).unwrap_or(rest.len());
6321 let (authority, suffix) = rest.split_at(split);
6322 let (host, port) = if let Some(rest) = authority.strip_prefix("[::1]") {
6323 ("::1", rest)
6324 } else {
6325 let split = authority.find(':').unwrap_or(authority.len());
6326 authority.split_at(split)
6327 };
6328 let ip: std::net::IpAddr = match host {
6329 "127.0.0.1" | "localhost" => std::net::Ipv4Addr::LOCALHOST.into(),
6330 "::1" => std::net::Ipv6Addr::LOCALHOST.into(),
6331 _ => return Err("host must be 127.0.0.1, [::1], or localhost".into()),
6332 };
6333 let port = if port.is_empty() {
6334 80
6335 } else {
6336 port.strip_prefix(':')
6337 .and_then(|p| p.parse::<u16>().ok())
6338 .filter(|p| *p > 0)
6339 .ok_or("must have a valid nonzero TCP port")?
6340 };
6341 let path = if suffix.is_empty() {
6342 "/".into()
6343 } else if suffix.starts_with('?') {
6344 format!("/{suffix}")
6345 } else {
6346 suffix.into()
6347 };
6348 Ok(HttpProbeTarget {
6349 address: std::net::SocketAddr::new(ip, port),
6350 localhost: host == "localhost",
6351 authority,
6352 path,
6353 })
6354}
6355
6356async fn probe_http_health(
6357 url: &str,
6358 deadline: Duration,
6359) -> Result<HealthReport, HealthProbeError> {
6360 use tokio::io::{AsyncReadExt, AsyncWriteExt, BufReader};
6361 let target = parse_http_probe_url(url).map_err(HealthProbeError::misconfigured)?;
6362 let mut response_status = String::new();
6365 let mut body = Vec::new();
6366 let probe = async {
6367 let connection = match tokio::net::TcpStream::connect(target.address).await {
6370 Err(_) if target.localhost => {
6371 tokio::net::TcpStream::connect((
6372 std::net::Ipv6Addr::LOCALHOST,
6373 target.address.port(),
6374 ))
6375 .await
6376 }
6377 result => result,
6378 };
6379 let mut stream = connection.map_err(|error| {
6380 HealthProbeError::no_answer(format!("HTTP connect failed: {error}"))
6381 })?;
6382 stream.write_all(format!("GET {} HTTP/1.1\r\nHost: {}\r\nAccept-Encoding: identity\r\nConnection: close\r\n\r\n", target.path, target.authority).as_bytes()).await
6383 .map_err(|error| HealthProbeError::no_answer(format!("HTTP write failed: {error}")))?;
6384 let mut reader = BufReader::new(stream);
6385 let mut budget = 16 * 1024;
6386 let status = http_line(&mut reader, &mut budget).await?;
6387 let mut words = status.split_ascii_whitespace();
6388 let version = words.next();
6389 let code = words
6390 .next()
6391 .filter(|word| word.len() == 3)
6392 .and_then(|word| word.parse::<u16>().ok());
6393 if !matches!(version, Some("HTTP/1.1" | "HTTP/1.0"))
6394 || !code.is_some_and(|code| (100..600).contains(&code))
6395 {
6396 return Err(HealthProbeError::bad_answer(format!(
6397 "invalid HTTP status: {status}"
6398 )));
6399 }
6400 let code = code.expect("validated status code");
6401 response_status = status.clone();
6402 let mut length = None;
6403 let mut chunked = false;
6404 loop {
6405 let line = http_line(&mut reader, &mut budget).await?;
6406 if line.is_empty() {
6407 break;
6408 }
6409 if let Some((name, value)) = line.split_once(':') {
6410 if name.eq_ignore_ascii_case("content-length") {
6411 length = Some(value.trim().parse::<u64>().map_err(|_| {
6412 HealthProbeError::bad_answer("invalid HTTP Content-Length")
6413 })?);
6414 } else if name.eq_ignore_ascii_case("transfer-encoding") {
6415 chunked = value.trim().eq_ignore_ascii_case("chunked");
6416 }
6417 }
6418 }
6419 if chunked {
6420 while body.len() < 200 {
6421 let line = http_line(&mut reader, &mut budget).await?;
6422 let size = u64::from_str_radix(line.split(';').next().unwrap_or("").trim(), 16)
6423 .map_err(|_| HealthProbeError::bad_answer("invalid HTTP chunk size"))?;
6424 if size == 0 {
6425 break;
6426 }
6427 let count = size.min((200 - body.len()) as u64) as usize;
6428 let start = body.len();
6429 (&mut reader)
6430 .take(count as u64)
6431 .read_to_end(&mut body)
6432 .await
6433 .map_err(|error| {
6434 HealthProbeError::no_answer(format!("HTTP body read failed: {error}"))
6435 })?;
6436 if body.len() - start != count {
6437 return Err(HealthProbeError::bad_answer("truncated HTTP chunk"));
6438 }
6439 if size > count as u64 || body.len() == 200 {
6440 break;
6441 }
6442 if !http_line(&mut reader, &mut budget).await?.is_empty() {
6443 return Err(HealthProbeError::bad_answer("invalid HTTP chunk delimiter"));
6444 }
6445 }
6446 } else {
6447 reader
6448 .take(length.unwrap_or(200).min(200))
6449 .read_to_end(&mut body)
6450 .await
6451 .map_err(|error| {
6452 HealthProbeError::no_answer(format!("HTTP body read failed: {error}"))
6453 })?;
6454 }
6455 if (200..300).contains(&code) {
6456 Ok(HealthReport::ok())
6457 } else {
6458 Err(HealthProbeError::bad_answer(
6459 "HTTP health endpoint returned non-2xx",
6460 ))
6461 }
6462 };
6463 let mut result = timeout(deadline, probe).await.unwrap_or_else(|_| {
6464 Err(HealthProbeError::no_answer(format!(
6465 "HTTP probe timed out after {deadline:?}"
6466 )))
6467 });
6468 if let Err(error) = &mut result {
6469 if !response_status.is_empty() {
6470 error.message = format!(
6471 "{}; {response_status}: {}",
6472 error.message,
6473 String::from_utf8_lossy(&body)
6474 );
6475 }
6476 }
6477 result
6478}
6479
6480async fn http_line(
6481 reader: &mut tokio::io::BufReader<tokio::net::TcpStream>,
6482 remaining: &mut usize,
6483) -> Result<String, HealthProbeError> {
6484 use tokio::io::{AsyncBufReadExt, AsyncReadExt};
6485 let mut line = Vec::new();
6486 (&mut *reader)
6487 .take(*remaining as u64)
6488 .read_until(b'\n', &mut line)
6489 .await
6490 .map_err(|error| {
6491 HealthProbeError::no_answer(format!("HTTP header read failed: {error}"))
6492 })?;
6493 *remaining -= line.len();
6494 if !line.ends_with(b"\r\n") {
6495 return Err(HealthProbeError::bad_answer(
6496 "HTTP headers are incomplete or exceed 16 KiB",
6497 ));
6498 }
6499 line.truncate(line.len() - 2);
6500 String::from_utf8(line).map_err(|_| HealthProbeError::bad_answer("HTTP header is not UTF-8"))
6501}
6502
6503async fn probe_module_health(
6504 module_id: &str,
6505 runtime: &SupervisorRuntimeConfig,
6506 drain_deadline: Option<Instant>,
6507) -> Result<HealthReport, HealthProbeError> {
6508 let Some(forwarding) = runtime.forwarding.as_ref() else {
6509 return Err(HealthProbeError::misconfigured(
6510 "supervisor was not configured with a forwarding table",
6511 ));
6512 };
6513 let probe_started_at = Instant::now();
6514 let mut deadline = probe_started_at + runtime.health.deadline;
6515 if let Some(drain_deadline) = drain_deadline {
6516 deadline = deadline.min(drain_deadline);
6517 }
6518 let pending = if drain_deadline.is_some() {
6519 forwarding.begin_drain_health_probe_rpc_for(
6520 module_id,
6521 MODULE_CONTROL_OP_HEALTH_CHECK,
6522 probe_started_at,
6523 deadline,
6524 )
6525 } else {
6526 forwarding.begin_health_probe_rpc_for(
6527 module_id,
6528 MODULE_CONTROL_OP_HEALTH_CHECK,
6529 probe_started_at,
6530 deadline,
6531 )
6532 }
6533 .map_err(|err| {
6534 HealthProbeError::lane_dead(format!("failed to begin health.check RPC: {err}"))
6537 })?;
6538 await_health_probe(forwarding, pending, deadline, runtime.health.deadline).await
6539}
6540
6541async fn probe_endpoint_health(
6548 endpoint: crate::ModuleEndpointId,
6549 runtime: &SupervisorRuntimeConfig,
6550 deadline_cap: Option<Instant>,
6551) -> Result<HealthReport, HealthProbeError> {
6552 let Some(forwarding) = runtime.forwarding.as_ref() else {
6553 return Err(HealthProbeError::misconfigured(
6554 "supervisor was not configured with a forwarding table",
6555 ));
6556 };
6557 let probe_started_at = Instant::now();
6558 let mut deadline = probe_started_at + runtime.health.deadline;
6559 if let Some(cap) = deadline_cap {
6560 deadline = deadline.min(cap);
6561 }
6562 let pending = forwarding
6563 .begin_endpoint_health_probe_rpc_for(
6564 endpoint,
6565 MODULE_CONTROL_OP_HEALTH_CHECK,
6566 probe_started_at,
6567 deadline,
6568 )
6569 .map_err(|err| {
6570 HealthProbeError::lane_dead(format!("failed to begin health.check RPC: {err}"))
6571 })?;
6572 await_health_probe(forwarding, pending, deadline, runtime.health.deadline).await
6573}
6574
6575async fn await_health_probe(
6577 forwarding: &ForwardingTable,
6578 pending: PendingModuleControlRpc,
6579 deadline: Instant,
6580 probe_budget: Duration,
6581) -> Result<HealthReport, HealthProbeError> {
6582 let PendingModuleControlRpc {
6583 endpoint,
6584 module_sink,
6585 negotiated_ver,
6586 corr,
6587 receiver,
6588 } = pending;
6589 let body = serde_json::to_vec(&ModuleControlRequest::HealthCheck {}).map_err(|err| {
6590 HealthProbeError::misconfigured(format!("failed to encode health.check: {err}"))
6591 })?;
6592 let frame = Frame::build_with_version(
6593 negotiated_ver,
6594 FrameType::Request,
6595 control_flags(),
6596 0,
6597 0,
6598 corr,
6599 body,
6600 )
6601 .map_err(|err| {
6602 HealthProbeError::misconfigured(format!("failed to build health.check frame: {err}"))
6603 })?;
6604
6605 match timeout_at(deadline, module_sink.send(frame)).await {
6611 Ok(Ok(())) => {}
6612 Ok(Err(err)) => {
6613 let _ = forwarding.cancel_module_control_rpc(endpoint, corr);
6614 return Err(HealthProbeError::lane_dead(format!(
6617 "failed to send health.check: {err}"
6618 )));
6619 }
6620 Err(_elapsed) => {
6621 let _ = forwarding.cancel_module_control_rpc(endpoint, corr);
6622 return Err(HealthProbeError::no_answer(
6626 "health.check send timed out before enqueue (module egress full)",
6627 ));
6628 }
6629 }
6630
6631 match timeout_at(deadline, receiver).await {
6632 Ok(Ok(ModuleControlRpcOutcome::RawResponse(_))) => Err(HealthProbeError::bad_answer(
6633 "health.check returned an unexpected config response",
6634 )),
6635 Ok(Ok(ModuleControlRpcOutcome::Response(response))) => {
6639 response.health_report().ok_or_else(|| {
6640 HealthProbeError::bad_answer("health.check RPC returned a non-health response")
6641 })
6642 }
6643 Ok(Ok(ModuleControlRpcOutcome::Rejected(body))) => Err(HealthProbeError::bad_answer(
6644 format!("health.check rejected: {}", body.message),
6645 )),
6646 Ok(Ok(ModuleControlRpcOutcome::ModuleGone(message))) => {
6647 Err(HealthProbeError::lane_dead(message))
6648 }
6649 Ok(Ok(ModuleControlRpcOutcome::MalformedResponse(message))) => {
6650 Err(HealthProbeError::bad_answer(message))
6651 }
6652 Ok(Ok(ModuleControlRpcOutcome::UnexpectedOp { expected, actual })) => {
6653 Err(HealthProbeError::bad_answer(format!(
6654 "expected module-control op '{expected}', got '{actual}'"
6655 )))
6656 }
6657 Ok(Ok(ModuleControlRpcOutcome::DeadlineElapsed)) => Err(HealthProbeError::bad_answer(
6661 "module answered health.check after its daemon deadline",
6662 )),
6663 Ok(Err(_)) => Err(HealthProbeError::misconfigured(
6664 "health.check waiter was canceled before the module responded",
6665 )),
6666 Err(_) => {
6667 let _ = forwarding.tombstone_health_probe_rpc(endpoint, corr);
6668 Err(HealthProbeError::no_answer(format!(
6669 "module did not answer health.check within {probe_budget:?}"
6670 )))
6671 }
6672 }
6673}
6674
6675#[allow(clippy::too_many_arguments)]
6676async fn handle_health_report(
6677 spec: &ModuleSpec,
6678 runtime: &SupervisorRuntimeConfig,
6679 registry: &Registry,
6680 process_liveness: &SupervisorProcessLiveness,
6681 snapshot: &SharedSnapshot,
6682 child: &mut Option<SupervisedChild>,
6683 report: HealthReport,
6684 now_ms: u64,
6685 delivery_probe: Option<crate::config_gate::runtime::DeliveryProbe>,
6686) {
6687 let delivery_failed = match (&runtime.config_delivery, delivery_probe) {
6688 (Some(delivery), Some(probe)) => {
6689 delivery
6690 .health_result(
6691 &spec.module_id,
6692 runtime.forwarding.as_deref(),
6693 registry,
6694 probe,
6695 report.status == HealthStatus::Ok,
6696 )
6697 .await
6698 }
6699 _ => false,
6700 };
6701 let status = supervisor_health_status(report.status);
6702 let detail = report.detail.clone();
6703 let metrics = truncate_health_metrics(report.metrics);
6704 let _ = update_snapshot(snapshot, Some(&spec.module_id), |state| {
6705 state.health.status = status;
6706 state.last_probe_evidence = None;
6707 state.health.last_probe_ms = Some(now_ms);
6708 state.health.detail = detail.clone();
6709 state.health.metrics = metrics.clone();
6710 state.health.consecutive_failures = 0;
6711 state.failed_retry.observe_health(
6712 report.status == HealthStatus::Ok,
6713 runtime.restart_policy.window,
6714 );
6715 });
6716
6717 if delivery_failed {
6718 return;
6719 }
6720 let action = match report.status {
6721 HealthStatus::Ok => return,
6722 HealthStatus::Degraded => runtime.health.on_degraded,
6723 HealthStatus::Failing => runtime.health.on_failing,
6724 };
6725 apply_l3_health_action(
6726 spec,
6727 runtime,
6728 registry,
6729 process_liveness,
6730 snapshot,
6731 child,
6732 status,
6733 detail.as_deref(),
6734 action,
6735 now_ms,
6736 )
6737 .await;
6738}
6739
6740#[allow(clippy::too_many_arguments)]
6741async fn handle_health_probe_failure(
6742 spec: &ModuleSpec,
6743 runtime: &SupervisorRuntimeConfig,
6744 registry: &Registry,
6745 process_liveness: &SupervisorProcessLiveness,
6746 snapshot: &SharedSnapshot,
6747 child: &mut Option<SupervisedChild>,
6748 err: HealthProbeError,
6749 now_ms: u64,
6750 delivery_probe: Option<crate::config_gate::runtime::DeliveryProbe>,
6751) {
6752 let delivery_failed = match (&runtime.config_delivery, delivery_probe) {
6753 (Some(delivery), Some(probe)) => {
6754 delivery
6755 .health_result(
6756 &spec.module_id,
6757 runtime.forwarding.as_deref(),
6758 registry,
6759 probe,
6760 false,
6761 )
6762 .await
6763 }
6764 _ => false,
6765 };
6766 let threshold = runtime.health.failure_threshold.max(1);
6767 let held = if err.evidence == HealthProbeEvidence::NoAnswer {
6768 runtime.host_health.record_late_module(&spec.module_id)
6769 } else {
6770 false
6771 };
6772 let mut failures = 0;
6773 let detail = if held {
6778 format!("[{}] {err}; host overloaded", err.label())
6779 } else {
6780 format!("[{}] {err}", err.label())
6781 };
6782 let _ = update_snapshot(snapshot, Some(&spec.module_id), |state| {
6783 state.last_probe_evidence = Some(err.evidence);
6784 state
6785 .failed_retry
6786 .observe_health(false, runtime.restart_policy.window);
6787 if held {
6790 state.health.status = SupervisorHealthStatus::Degraded;
6791 } else if state.spawned_protocol.unwrap_or(spec.protocol) == ModuleProtocol::Subc {
6792 state.health.status = SupervisorHealthStatus::Unknown;
6793 }
6794 state.health.last_probe_ms = Some(now_ms);
6795 state.health.consecutive_failures = if held {
6798 0
6799 } else {
6800 state.health.consecutive_failures.saturating_add(1)
6801 };
6802 state.health.detail = Some(detail.clone());
6803 state.health.metrics = None;
6804 failures = state.health.consecutive_failures;
6805 });
6806
6807 if delivery_failed {
6808 return;
6809 }
6810 if held || failures < threshold {
6811 warn!(
6812 module_id = %spec.module_id,
6813 consecutive_failures = failures,
6814 threshold,
6815 evidence = err.label(),
6816 detail = %detail,
6817 "health.check probe failed"
6818 );
6819 return;
6820 }
6821
6822 let _ = update_snapshot(snapshot, Some(&spec.module_id), |state| {
6823 state.state = ModuleState::Unresponsive;
6824 state.health.status = SupervisorHealthStatus::Unresponsive;
6825 });
6826 if runtime.health.critical {
6830 error!(
6831 module_id = %spec.module_id,
6832 status = "unresponsive",
6833 evidence = err.label(),
6834 detail = %detail,
6835 "critical module health alert"
6836 );
6837 } else {
6838 warn!(
6839 module_id = %spec.module_id,
6840 status = "unresponsive",
6841 evidence = err.label(),
6842 detail = %detail,
6843 "module health threshold breached"
6844 );
6845 }
6846 if let Err(err) = health_restart_child(
6847 spec,
6848 runtime,
6849 registry,
6850 process_liveness,
6851 snapshot,
6852 child,
6853 SupervisorHealthStatus::Unresponsive,
6854 Some(&detail),
6855 now_ms,
6856 )
6857 .await
6858 {
6859 error!(module_id = %spec.module_id, error = %err, "health-triggered restart failed");
6860 }
6861}
6862
6863#[allow(clippy::too_many_arguments)]
6864async fn apply_l3_health_action(
6865 spec: &ModuleSpec,
6866 runtime: &SupervisorRuntimeConfig,
6867 registry: &Registry,
6868 process_liveness: &SupervisorProcessLiveness,
6869 snapshot: &SharedSnapshot,
6870 child: &mut Option<SupervisedChild>,
6871 status: SupervisorHealthStatus,
6872 detail: Option<&str>,
6873 action: HealthAction,
6874 now_ms: u64,
6875) {
6876 record_health_action(snapshot, &spec.module_id, action.to_string(), now_ms);
6877 match action {
6878 HealthAction::Report => {
6879 info!(
6880 module_id = %spec.module_id,
6881 status = ?status,
6882 detail,
6883 "module reported non-ok health"
6884 );
6885 }
6886 HealthAction::Alert => {
6887 error!(
6888 module_id = %spec.module_id,
6889 status = ?status,
6890 detail,
6891 "module health alert"
6892 );
6893 }
6894 HealthAction::Restart => {
6895 if let Err(err) = health_restart_child(
6896 spec,
6897 runtime,
6898 registry,
6899 process_liveness,
6900 snapshot,
6901 child,
6902 status,
6903 detail,
6904 now_ms,
6905 )
6906 .await
6907 {
6908 error!(module_id = %spec.module_id, error = %err, "health-triggered restart failed");
6909 }
6910 }
6911 }
6912}
6913
6914#[allow(clippy::too_many_arguments)]
6915async fn health_restart_child(
6916 spec: &ModuleSpec,
6917 runtime: &SupervisorRuntimeConfig,
6918 registry: &Registry,
6919 process_liveness: &SupervisorProcessLiveness,
6920 snapshot: &SharedSnapshot,
6921 child: &mut Option<SupervisedChild>,
6922 status: SupervisorHealthStatus,
6923 detail: Option<&str>,
6924 now_ms: u64,
6925) -> Result<(), SuperviseError> {
6926 budgeted_restart_child(
6927 spec,
6928 runtime,
6929 registry,
6930 process_liveness,
6931 snapshot,
6932 child,
6933 status,
6934 detail,
6935 now_ms,
6936 true,
6937 )
6938 .await
6939}
6940
6941#[allow(clippy::too_many_arguments)]
6942async fn budgeted_restart_child(
6943 spec: &ModuleSpec,
6944 runtime: &SupervisorRuntimeConfig,
6945 registry: &Registry,
6946 process_liveness: &SupervisorProcessLiveness,
6947 snapshot: &SharedSnapshot,
6948 child: &mut Option<SupervisedChild>,
6949 status: SupervisorHealthStatus,
6950 detail: Option<&str>,
6951 now_ms: u64,
6952 health_failure: bool,
6953) -> Result<(), SuperviseError> {
6954 if let Some(error) = runtime
6955 .config_delivery
6956 .as_ref()
6957 .and_then(|delivery| delivery.0.check_module_hold(&spec.module_id).err())
6958 .filter(|error| error.code != "config_delivery_deferred")
6959 {
6960 return Err(config_delivery::gate_hold_error(
6961 spec,
6962 error,
6963 runtime.config_delivery.as_ref(),
6964 ));
6965 }
6966 let recovery_reason = if health_failure {
6967 "health_check"
6968 } else {
6969 "configuration_delivery"
6970 };
6971 let (enabled, schedule) = {
6972 let mut state = lock_snapshot(snapshot)?;
6973 let enabled = state.enabled;
6974 let schedule = if enabled {
6975 state.next_crash_restart(&runtime.restart_policy, Instant::now())
6976 } else {
6977 None
6978 };
6979 (enabled, schedule)
6980 };
6981
6982 if !enabled {
6983 return Err(SuperviseError::Disabled {
6984 module_id: spec.module_id.clone(),
6985 });
6986 }
6987
6988 if schedule.is_none() {
6989 if health_failure {
6990 record_health_action(snapshot, &spec.module_id, "failed".to_string(), now_ms);
6991 }
6992 error!(
6993 module_id = %spec.module_id,
6994 status = ?status,
6995 detail,
6996 max_restarts = runtime.restart_policy.max_restarts,
6997 window_secs = runtime.restart_policy.window.as_secs(),
6998 reason = %runtime.restart_policy.budget_exhausted_detail(),
6999 recovery_reason,
7000 "automatic restart budget exhausted; marking module failed"
7001 );
7002 let stop_notice = begin_forwarding_drain_if_configured(
7003 spec,
7004 runtime,
7005 registry,
7006 snapshot,
7007 Some(true),
7008 RouteCloseReason::Disable,
7009 )
7010 .await?;
7011 update_snapshot(snapshot, Some(&spec.module_id), |state| {
7012 state.drain_disposition_detail = Some(runtime.restart_policy.budget_exhausted_detail());
7013 })?;
7014 let result = drain_optional_child(
7015 &spec.module_id,
7016 spec.protocol,
7017 stop_notice,
7018 registry,
7019 runtime.forwarding.as_deref(),
7020 snapshot,
7021 &runtime.terminal_ring,
7022 &runtime.spawn_events,
7023 child,
7024 runtime.drain_timeout,
7025 ModuleState::Failed,
7026 Some(true),
7027 &runtime.failure_context(true),
7028 )
7029 .await;
7030 update_snapshot(snapshot, Some(&spec.module_id), |state| {
7031 if state.state == ModuleState::Failed && !runtime.child_roster.is_closed() {
7032 state.failed_retry.schedule(&spec.module_id);
7033 }
7034 })?;
7035 result?;
7036 process_liveness.untrack_if_current(&spec.module_id, snapshot);
7037 return Ok(());
7038 }
7039
7040 let schedule = schedule.expect("a health restart must have a crash-restart schedule");
7041 let mut restart_count = 0;
7042 update_snapshot(snapshot, Some(&spec.module_id), |state| {
7043 restart_count = state.crash_restarts.len();
7044 state.state = ModuleState::Unresponsive;
7045 if health_failure {
7046 state.health.status = status;
7047 state.health.last_action = Some(HealthAction::Restart.to_string());
7048 state.health.last_action_ms = Some(now_ms);
7049 }
7050 })?;
7051 warn!(
7052 module_id = %spec.module_id,
7053 status = ?status,
7054 detail,
7055 restart_count,
7056 restart_in_window = schedule.restart_in_window,
7057 delay_ms = schedule.delay.as_millis() as u64,
7058 recovery_reason,
7059 "automatic module restart"
7060 );
7061
7062 let stop_notice = begin_forwarding_drain_if_configured(
7063 spec,
7064 runtime,
7065 registry,
7066 snapshot,
7067 Some(true),
7068 RouteCloseReason::Restart,
7069 )
7070 .await?;
7071 drain_optional_child(
7072 &spec.module_id,
7073 spec.protocol,
7074 stop_notice,
7075 registry,
7076 runtime.forwarding.as_deref(),
7077 snapshot,
7078 &runtime.terminal_ring,
7079 &runtime.spawn_events,
7080 child,
7081 runtime.drain_timeout,
7082 ModuleState::Restarting,
7083 Some(true),
7084 &runtime.failure_context(true),
7085 )
7086 .await?;
7087 schedule_respawn(
7088 runtime,
7089 snapshot,
7090 &spec.module_id,
7091 schedule.delay,
7092 RespawnKind::Spawn,
7093 true,
7094 )
7095}
7096
7097fn cancel_deferred_reload(runtime: &SupervisorRuntimeConfig, module_id: &str, reason: &str) {
7098 if let Some(reply) = runtime
7099 .deferred_reload_reply
7100 .lock()
7101 .unwrap_or_else(|p| p.into_inner())
7102 .take()
7103 {
7104 let _ = reply.send(Err(SuperviseError::ReloadFailed {
7105 module_id: module_id.to_string(),
7106 reason: reason.to_string(),
7107 }));
7108 }
7109}
7110
7111fn park_config_spawn(
7112 spec: &ModuleSpec,
7113 runtime: &SupervisorRuntimeConfig,
7114 snapshot: &SharedSnapshot,
7115 failure: &FailureContext,
7116 admission_refused: Option<String>,
7117) -> Result<(), SuperviseError> {
7118 *runtime
7119 .scheduled_respawn
7120 .lock()
7121 .unwrap_or_else(|poisoned| poisoned.into_inner()) = None;
7122 if let Some(reply) = runtime
7123 .deferred_reload_reply
7124 .lock()
7125 .unwrap_or_else(|poisoned| poisoned.into_inner())
7126 .take()
7127 {
7128 let _ = reply.send(Err(SuperviseError::ConfigHeld {
7129 module_id: spec.module_id.clone(),
7130 code: "config_gated".into(),
7131 detail: "module is held by configuration admission".into(),
7132 }));
7133 }
7134 update_snapshot(snapshot, Some(&spec.module_id), |state| {
7135 state.admission_refused = admission_refused;
7136 state.config_hold_daemon_initiated = failure.daemon_initiated;
7137 state.config_hold_approval = failure.approval.clone();
7138 state.config_hold_parked = true;
7139 state.state = ModuleState::Stopped;
7140 state.respawn_pending = false;
7141 state.failed_retry.pending = None;
7142 state.failed_retry.failed_at = None;
7145 clear_current_process_facts(state);
7146 })
7147}
7148
7149fn try_park_config_attempt(
7150 spec: &ModuleSpec,
7151 runtime: &SupervisorRuntimeConfig,
7152 snapshot: &SharedSnapshot,
7153 failure: &FailureContext,
7154 program_base: Option<&subc_config::Digest>,
7155) -> Result<bool, SuperviseError> {
7156 let result = (|| {
7157 if !lock_snapshot(snapshot)?.enabled {
7158 return Ok(false);
7159 }
7160 #[cfg(test)]
7161 if let Some(delivery) = &runtime.config_delivery {
7162 delivery.0.pause_before_config_park_for_test();
7163 }
7164 let Some(delivery) = &runtime.config_delivery else {
7165 return Ok(false);
7166 };
7167 let parked = delivery
7168 .0
7169 .with_recorded_admission_hold(&spec.module_id, program_base, |refusal| {
7170 park_config_spawn(spec, runtime, snapshot, failure, refusal)
7171 })
7172 .map_err(|error| config_delivery::gate_hold_error(spec, error, Some(delivery)))?;
7173 if let Some(result) = parked {
7174 result?;
7175 Ok(true)
7176 } else {
7177 Ok(false)
7178 }
7179 })();
7180 if let Err(error) = &result {
7181 fail_snapshot(snapshot, Some(&spec.module_id), None, failure);
7185 record_approved_attempt_failure(spec, snapshot, failure, error);
7186 }
7187 result
7188}
7189
7190fn record_approved_attempt_failure(
7191 spec: &ModuleSpec,
7192 snapshot: &SharedSnapshot,
7193 failure: &FailureContext,
7194 error: &SuperviseError,
7195) {
7196 let Some(approval) = &failure.approval else {
7197 return;
7198 };
7199 let _ = update_snapshot(snapshot, Some(&spec.module_id), |state| {
7200 if state.enabled
7201 && state.state == ModuleState::Failed
7202 && state
7203 .approved_start_failure
7204 .as_ref()
7205 .is_none_or(|(identity, _)| identity != approval)
7206 {
7207 state.approved_start_failure = Some((approval.clone(), error.to_string()));
7208 }
7209 });
7210}
7211
7212fn hold_config_spawn(
7213 spec: &ModuleSpec,
7214 runtime: &SupervisorRuntimeConfig,
7215 snapshot: &SharedSnapshot,
7216 failure: &FailureContext,
7217) -> Result<(), SuperviseError> {
7218 if !lock_snapshot(snapshot)?.enabled {
7219 return Ok(());
7220 }
7221 if !try_park_config_attempt(spec, runtime, snapshot, failure, None)? {
7222 defer_config_spawn(spec, runtime, snapshot, RespawnKind::Spawn, failure)?;
7225 }
7226 Ok(())
7227}
7228
7229fn schedule_respawn(
7230 runtime: &SupervisorRuntimeConfig,
7231 snapshot: &SharedSnapshot,
7232 module_id: &str,
7233 delay: Duration,
7234 kind: RespawnKind,
7235 daemon_initiated: bool,
7236) -> Result<(), SuperviseError> {
7237 cancel_deferred_reload(runtime, module_id, "respawn superseded by another restart");
7238 update_snapshot(snapshot, Some(module_id), |state| {
7239 state.respawn_pending = true
7240 })?;
7241 *runtime
7242 .scheduled_respawn
7243 .lock()
7244 .unwrap_or_else(|p| p.into_inner()) = Some(PendingRespawn {
7245 deadline: Instant::now() + delay,
7246 kind,
7247 daemon_initiated,
7248 approval: None,
7249 });
7250 Ok(())
7251}
7252
7253const CONFIG_ADMISSION_RETRY_DELAY: Duration = Duration::from_millis(100);
7254
7255fn defer_config_spawn(
7256 spec: &ModuleSpec,
7257 runtime: &SupervisorRuntimeConfig,
7258 snapshot: &SharedSnapshot,
7259 kind: RespawnKind,
7260 failure: &FailureContext,
7261) -> Result<(), SuperviseError> {
7262 update_snapshot(snapshot, Some(&spec.module_id), |state| {
7263 state.state = ModuleState::Restarting;
7264 state.respawn_pending = true;
7265 clear_current_process_facts(state);
7266 })?;
7267 *runtime
7268 .scheduled_respawn
7269 .lock()
7270 .unwrap_or_else(|p| p.into_inner()) = Some(PendingRespawn {
7271 deadline: Instant::now() + CONFIG_ADMISSION_RETRY_DELAY,
7272 kind,
7273 daemon_initiated: failure.daemon_initiated,
7274 approval: failure.approval.clone(),
7275 });
7276 Ok(())
7277}
7278
7279fn record_health_action(snapshot: &SharedSnapshot, module_id: &str, action: String, now_ms: u64) {
7280 let _ = update_snapshot(snapshot, Some(module_id), |state| {
7281 state.health.last_action = Some(action);
7282 state.health.last_action_ms = Some(now_ms);
7283 });
7284}
7285
7286fn supervisor_health_status(status: HealthStatus) -> SupervisorHealthStatus {
7287 match status {
7288 HealthStatus::Ok => SupervisorHealthStatus::Ok,
7289 HealthStatus::Degraded => SupervisorHealthStatus::Degraded,
7290 HealthStatus::Failing => SupervisorHealthStatus::Failing,
7291 }
7292}
7293
7294fn truncate_health_metrics(metrics: Option<Value>) -> Option<Value> {
7306 let metrics = metrics?;
7307 match serde_json::to_vec(&metrics) {
7308 Ok(encoded) if encoded.len() > MAX_HEALTH_METRICS_BYTES => Some(serde_json::json!({
7309 "truncated": true,
7310 "original_bytes": encoded.len(),
7311 })),
7312 Ok(_) | Err(_) => Some(metrics),
7313 }
7314}
7315
7316fn jittered_health_delay(module_id: &str, probe_index: u64, cadence: Duration) -> Duration {
7322 if cadence.is_zero() {
7323 return Duration::ZERO;
7324 }
7325 let cadence_ms = cadence.as_millis() as u64;
7326 if cadence_ms == 0 {
7342 return cadence;
7343 }
7344 let jitter_span = (cadence_ms / 10).max(1);
7359 let hash = module_id.as_bytes().iter().fold(
7360 probe_index.wrapping_mul(0x9E37_79B9_7F4A_7C15),
7361 |acc, byte| {
7362 acc.wrapping_mul(1099511628211)
7363 .wrapping_add(u64::from(*byte))
7364 },
7365 );
7366 cadence + Duration::from_millis(hash % jitter_span)
7367}
7368
7369#[cfg(test)]
7370mod tests {
7371 use super::*;
7372
7373 #[test]
7374 fn registration_retirement_self_prefixed_owner_is_not_a_prefix_submodule() {
7375 let handle = SupervisorHandle::new();
7376 handle.set_spawn_nonce("prefix:owner", "owner-nonce".into());
7377 handle.set_reserved_prefixes("prefix:owner", &["prefix:".into()]);
7378 let owner = handle
7379 .reserved_hello_verification("prefix:owner", Some("owner-nonce"))
7380 .unwrap();
7381 assert!(owner.nonce_verified);
7382 assert!(
7383 owner.prefix_owner.is_none(),
7384 "an unreserved programmatic owner is not its own prefix child"
7385 );
7386 let child = handle
7387 .reserved_hello_verification("prefix:child", Some("owner-nonce"))
7388 .unwrap();
7389 assert!(child.nonce_verified);
7390 assert_eq!(child.prefix_owner.as_deref(), Some("prefix:owner"));
7391 }
7392
7393 #[test]
7394 fn readding_a_module_clears_its_rescan_removal_tombstone() {
7395 let handle = SupervisorHandle::new();
7396 let module_id = "readded-tombstone";
7397 handle.record_rescan_removal(module_id);
7398 assert!(handle.removal_tombstone_age_ms(module_id).is_some());
7399
7400 handle.apply_identity_configuration(&ModuleSpec {
7401 module_id: module_id.to_string(),
7402 program: PathBuf::from("/test/module"),
7403 args: Vec::new(),
7404 env: Vec::new(),
7405 reserved: false,
7406 reserved_prefixes: Vec::new(),
7407 protocol: ModuleProtocol::Subc,
7408 overlap: Default::default(),
7409 carry_containment: false,
7410 });
7411
7412 assert!(
7413 handle.removal_tombstone_age_ms(module_id).is_none(),
7414 "a re-added module must not retain a stale removal tombstone"
7415 );
7416 }
7417
7418 #[derive(Debug, PartialEq, Eq)]
7421 struct OwnerInSpawnWindow {
7422 module_id: String,
7423 configured: bool,
7424 on_roster: bool,
7425 admission_refusal: Option<&'static str>,
7426 }
7427
7428 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
7439 async fn a_new_module_is_configured_before_its_first_process_can_register() {
7440 use crate::scopes::ScopeTable;
7441 use subc_protocol::{error_codes, scope::ScopeSelector, Principal};
7442
7443 let dir = cortexkit_test_support::ScratchDir::new("configured-before-spawn");
7444 let stub = |module_id: &str, program: PathBuf| ModuleSpec {
7445 module_id: module_id.to_string(),
7446 program,
7447 args: Vec::new(),
7448 env: ["XDG_DATA_HOME", "XDG_RUNTIME_DIR", "XDG_CONFIG_HOME"]
7449 .into_iter()
7450 .map(|key| (key.to_string(), dir.path().display().to_string()))
7451 .chain([("FAKE_AFT_NEVER_CONNECT".to_string(), "1".to_string())])
7452 .collect(),
7453 reserved: false,
7454 reserved_prefixes: Vec::new(),
7455 protocol: ModuleProtocol::Subc,
7456 overlap: Default::default(),
7457 carry_containment: false,
7458 };
7459 let live = super::terminal_history_tests::fake_aft_stub_path();
7460 let missing = dir.path().join("definitely-missing-module");
7461
7462 let handle = SupervisorHandle::new();
7463 let mut supervisor =
7464 Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default())
7465 .with_handle(handle.clone());
7466 let observed = Arc::new(Mutex::new(Vec::<OwnerInSpawnWindow>::new()));
7467 let hook_handle = handle.clone();
7468 let hook_observed = Arc::clone(&observed);
7469 supervisor.test_after_first_spawn = AfterFirstSpawnHook(Some(Arc::new(move |module_id| {
7470 let configured = hook_handle.is_configured(module_id);
7473 let selector = ScopeSelector::new(
7474 Principal::Reserved {
7475 module_id: module_id.to_string(),
7476 },
7477 "s".to_string(),
7478 Some(1),
7479 );
7480 let carrier = Principal::Reserved {
7481 module_id: "carrier".to_string(),
7482 };
7483 let admission_refusal = match ScopeTable::new(Vec::<String>::new())
7484 .admit(&carrier, module_id, &selector, configured)
7485 {
7486 Ok(_) => None,
7487 Err(refusal) => Some(refusal.code),
7488 };
7489 hook_observed.lock().unwrap().push(OwnerInSpawnWindow {
7490 module_id: module_id.to_string(),
7491 configured,
7492 on_roster: hook_handle.get(module_id).is_some(),
7493 admission_refusal,
7494 });
7495 })));
7496
7497 let plain = supervisor.spawn(stub("plain", live.clone())).unwrap();
7498 let configured = supervisor
7499 .supervise_configured(stub("configured", live.clone()), true)
7500 .unwrap();
7501 let with_health = supervisor
7502 .supervise_configured_with_health(
7503 stub("with-health", live.clone()),
7504 true,
7505 HealthConfig::default(),
7506 None,
7507 RestartPolicy::default(),
7508 )
7509 .unwrap();
7510 let failed = supervisor
7513 .supervise_configured_with_health(
7514 stub("failed-spawn", missing.clone()),
7515 true,
7516 HealthConfig::default(),
7517 None,
7518 RestartPolicy::default(),
7519 )
7520 .unwrap();
7521 assert!(supervisor.spawn(stub("spawn-error", missing)).is_err());
7524
7525 let expected = [
7526 "plain",
7527 "configured",
7528 "with-health",
7529 "failed-spawn",
7530 "spawn-error",
7531 ]
7532 .into_iter()
7533 .map(|module_id| OwnerInSpawnWindow {
7534 module_id: module_id.to_string(),
7535 configured: true,
7536 on_roster: !matches!(module_id, "plain" | "spawn-error"),
7537 admission_refusal: Some(error_codes::SCOPE_NOT_SYNCED),
7538 })
7539 .collect::<Vec<_>>();
7540 assert_eq!(*observed.lock().unwrap(), expected);
7541
7542 for module_id in ["plain", "configured", "with-health", "failed-spawn"] {
7543 assert!(
7544 handle.get(module_id).is_some(),
7545 "{module_id} is on the roster"
7546 );
7547 assert!(
7548 handle.is_configured(module_id),
7549 "{module_id} stays configured"
7550 );
7551 }
7552 assert!(handle.get("spawn-error").is_none());
7553 assert!(
7554 !handle.is_configured("spawn-error"),
7555 "a plain spawn that failed must not leave its module marked configured"
7556 );
7557
7558 handle.retire("failed-spawn");
7560 assert!(!handle.is_configured("failed-spawn"));
7561
7562 for module in [plain, configured, with_health] {
7563 module.stop().await.unwrap();
7564 }
7565 drop(failed);
7566 }
7567
7568 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
7575 async fn a_configured_module_reads_starting_while_its_first_process_already_runs() {
7576 let dir = cortexkit_test_support::ScratchDir::new("starting-while-spawned");
7577 let spec = ModuleSpec {
7578 module_id: "spawned-not-admitted".to_string(),
7579 program: super::terminal_history_tests::fake_aft_stub_path(),
7580 args: Vec::new(),
7581 env: ["XDG_DATA_HOME", "XDG_RUNTIME_DIR", "XDG_CONFIG_HOME"]
7582 .into_iter()
7583 .map(|key| (key.to_string(), dir.path().display().to_string()))
7584 .chain([("FAKE_AFT_NEVER_CONNECT".to_string(), "1".to_string())])
7585 .collect(),
7586 reserved: false,
7587 reserved_prefixes: Vec::new(),
7588 protocol: ModuleProtocol::Subc,
7589 overlap: Default::default(),
7590 carry_containment: false,
7591 };
7592 let handle = SupervisorHandle::new();
7593 let mut supervisor =
7594 Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default())
7595 .with_handle(handle.clone());
7596 let observed = Arc::new(Mutex::new(None));
7597 let hook_handle = handle.clone();
7598 let hook_observed = Arc::clone(&observed);
7599 supervisor.test_after_first_spawn = AfterFirstSpawnHook(Some(Arc::new(move |module_id| {
7600 let state = hook_handle
7601 .get(module_id)
7602 .expect("a configured module is on the roster before it is spawned")
7603 .state()
7604 .unwrap();
7605 *hook_observed.lock().unwrap() = Some(state);
7606 })));
7607
7608 let module = supervisor.supervise_configured(spec, true).unwrap();
7609 assert_eq!(
7610 *observed.lock().unwrap(),
7611 Some(ModuleState::Starting),
7612 "between the spawn and its admission the supervisor still reports starting"
7613 );
7614 assert_eq!(module.state().unwrap(), ModuleState::Running);
7615 module.stop().await.unwrap();
7616 }
7617
7618 fn stale_process_snapshot(state: ModuleState, enabled: bool) -> SharedSnapshot {
7619 let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::new(state, enabled)));
7620 update_snapshot(&snapshot, Some("stale-process-facts"), |snapshot| {
7621 snapshot.process_alive = true;
7622 snapshot.pid = Some(41);
7623 snapshot.spawned_at_ms = Some(42);
7624 snapshot.spawned_from = Some(PathBuf::from("/spawned/module"));
7625 snapshot.launch_spec_digest = Some(LaunchSpecDigest([0xA5; 32]));
7626 snapshot.spawned_file_identity = Some(SpawnedFileIdentity {
7627 device: 43,
7628 inode: 44,
7629 #[cfg(windows)]
7630 windows_image: None,
7631 });
7632 })
7633 .unwrap();
7634 snapshot
7635 }
7636
7637 fn assert_snapshot_process_facts_cleared(snapshot: &SharedSnapshot) {
7638 let snapshot = lock_snapshot(snapshot).unwrap();
7639 assert!(!snapshot.process_alive);
7640 assert_eq!(snapshot.pid, None);
7641 assert_eq!(snapshot.spawned_at_ms, None);
7642 assert_eq!(snapshot.spawned_from, None);
7643 assert_eq!(snapshot.launch_spec_digest, None);
7644 assert_eq!(snapshot.spawned_file_identity, None);
7645 }
7646
7647 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
7648 async fn failed_enable_spawn_clears_preexisting_current_process_facts() {
7649 let supervisor =
7650 Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default());
7651 let mut runtime = supervisor.runtime_config();
7652 runtime.test_seed_stale_facts_before_enable_spawn = true;
7653 let snapshot = stale_process_snapshot(ModuleState::Disabled, false);
7654 let mut child = None;
7655 let spec = ModuleSpec {
7656 module_id: "failed-enable-clears-facts".to_string(),
7657 program: PathBuf::from("/definitely/missing/failed-enable-module"),
7658 args: Vec::new(),
7659 env: Vec::new(),
7660 reserved: false,
7661 reserved_prefixes: Vec::new(),
7662 protocol: ModuleProtocol::Subc,
7663 overlap: Default::default(),
7664 carry_containment: false,
7665 };
7666
7667 let result = set_child_enabled(
7668 &spec,
7669 &runtime,
7670 &supervisor.registry,
7671 &supervisor.process_liveness,
7672 &snapshot,
7673 &mut child,
7674 true,
7675 &runtime.failure_context(false),
7676 )
7677 .await;
7678
7679 assert!(matches!(result, Err(SuperviseError::Spawn { .. })));
7680 assert_eq!(lock_snapshot(&snapshot).unwrap().state, ModuleState::Failed);
7681 assert_snapshot_process_facts_cleared(&snapshot);
7682 }
7683
7684 #[tokio::test]
7685 async fn start_revives_stranded_restarting_but_not_pending_backoff() {
7686 let supervisor =
7687 Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default());
7688 let runtime = supervisor.runtime_config();
7689 let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::new(
7690 ModuleState::Restarting,
7691 true,
7692 )));
7693 let spec = ModuleSpec {
7694 module_id: "start-stranded-restarting".to_string(),
7695 program: super::terminal_history_tests::fake_aft_stub_path(),
7696 args: Vec::new(),
7697 env: vec![("FAKE_AFT_NEVER_CONNECT".to_string(), "1".to_string())],
7698 reserved: false,
7699 reserved_prefixes: Vec::new(),
7700 protocol: ModuleProtocol::None,
7701 overlap: Default::default(),
7702 carry_containment: false,
7703 };
7704 let mut child = None;
7705 lock_snapshot(&snapshot).unwrap().respawn_pending = true;
7706 assert!(!super::set_child_enabled(
7707 &spec,
7708 &runtime,
7709 &Registry::default(),
7710 &supervisor.process_liveness,
7711 &snapshot,
7712 &mut child,
7713 true,
7714 &runtime.failure_context(false),
7715 )
7716 .await
7717 .unwrap());
7718 assert!(child.is_none());
7719 lock_snapshot(&snapshot).unwrap().respawn_pending = false;
7720 assert!(super::set_child_enabled(
7721 &spec,
7722 &runtime,
7723 &Registry::default(),
7724 &supervisor.process_liveness,
7725 &snapshot,
7726 &mut child,
7727 true,
7728 &runtime.failure_context(false),
7729 )
7730 .await
7731 .unwrap());
7732 assert_eq!(
7733 lock_snapshot(&snapshot).unwrap().state,
7734 ModuleState::Running
7735 );
7736 let mut child = child.unwrap();
7737 child.start_kill().unwrap();
7738 child.wait().await.unwrap();
7739 }
7740
7741 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
7742 async fn failed_reload_spawn_clears_current_process_facts() {
7743 let supervisor =
7744 Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default());
7745 let mut runtime = supervisor.runtime_config();
7746 runtime.restart_policy = RestartPolicy::new(0, Duration::ZERO);
7747 let snapshot = stale_process_snapshot(ModuleState::Running, true);
7748 let mut child = None;
7749 let spec = ModuleSpec {
7750 module_id: "failed-reload-clears-facts".to_string(),
7751 program: PathBuf::from("/unused/failed-reload-module"),
7752 args: Vec::new(),
7753 env: Vec::new(),
7754 reserved: false,
7755 reserved_prefixes: Vec::new(),
7756 protocol: ModuleProtocol::Subc,
7757 overlap: Default::default(),
7758 carry_containment: false,
7759 };
7760
7761 let result = handle_reload_spawn_failure(
7762 &spec,
7763 &runtime,
7764 &supervisor.process_liveness,
7765 &snapshot,
7766 &mut child,
7767 "forced reload spawn failure".to_string(),
7768 )
7769 .await;
7770
7771 assert!(matches!(result, Err(SuperviseError::ReloadFailed { .. })));
7772 assert_eq!(lock_snapshot(&snapshot).unwrap().state, ModuleState::Failed);
7773 assert_snapshot_process_facts_cleared(&snapshot);
7774 }
7775
7776 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
7777 async fn dropping_a_module_with_an_active_monitor_clears_current_process_facts() {
7778 let supervisor =
7779 Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default());
7780 let snapshot = stale_process_snapshot(ModuleState::Running, true);
7781 let module = supervisor.supervised_module(
7782 ModuleSpec {
7783 module_id: "drop-clears-facts".to_string(),
7784 program: PathBuf::from("/unused/drop-module"),
7785 args: Vec::new(),
7786 env: Vec::new(),
7787 reserved: false,
7788 reserved_prefixes: Vec::new(),
7789 protocol: ModuleProtocol::Subc,
7790 overlap: Default::default(),
7791 carry_containment: false,
7792 },
7793 supervisor.runtime_config(),
7794 Arc::clone(&snapshot),
7795 None,
7796 );
7797 assert!(!module
7798 .inner
7799 .monitor
7800 .lock()
7801 .unwrap()
7802 .as_ref()
7803 .unwrap()
7804 .is_finished());
7805
7806 drop(module);
7807
7808 assert_eq!(
7809 lock_snapshot(&snapshot).unwrap().state,
7810 ModuleState::Stopped
7811 );
7812 assert_snapshot_process_facts_cleared(&snapshot);
7813 }
7814
7815 #[cfg(unix)]
7816 #[tokio::test]
7817 async fn rescan_preserves_running_protocol_until_respawn() {
7818 let dir = cortexkit_test_support::ScratchDir::new("rescan-protocol");
7819 let initial = ModuleSpec {
7820 module_id: "rescan-protocol".into(),
7821 program: PathBuf::from("/bin/sleep"),
7822 args: vec!["60".into()],
7823 env: ["XDG_DATA_HOME", "XDG_RUNTIME_DIR", "XDG_CONFIG_HOME"]
7824 .into_iter()
7825 .map(|key| (key.into(), dir.path().to_string_lossy().into_owned()))
7826 .collect(),
7827 reserved: false,
7828 reserved_prefixes: vec![],
7829 protocol: ModuleProtocol::None,
7830 overlap: Default::default(),
7831 carry_containment: false,
7832 };
7833 let supervisor =
7834 Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default());
7835 let module = supervisor.spawn(initial.clone()).unwrap();
7836 assert!(module.status().unwrap().live);
7837 let mut next = initial;
7838 next.protocol = ModuleProtocol::Subc;
7839 module
7840 .update_configuration(next.clone(), HealthConfig::default(), None)
7841 .await
7842 .unwrap();
7843 assert!(
7844 module.status().unwrap().live,
7845 "rescan must not require HELLO from the old non-wire process"
7846 );
7847 let runtime = supervisor.runtime_config();
7848 let action = on_child_exit(
7849 &next,
7850 RestartPolicy::default(),
7851 &supervisor.registry,
7852 &module.inner.snapshot,
7853 &runtime.terminal_ring,
7854 &runtime.spawn_events,
7855 &runtime.child_roster,
7856 ExitReport {
7857 kind: ExitKind::Clean,
7858 code: Some(0),
7859 signal: None,
7860 at_ms: unix_ms_now(),
7861 },
7862 true,
7863 )
7864 .await;
7865 assert!(
7866 matches!(action, NextAction::Restart { .. }),
7867 "the old non-wire process's clean exit must restart"
7868 );
7869 module.drain().await.unwrap();
7870 }
7871
7872 #[cfg(unix)]
7873 #[tokio::test]
7874 async fn orphan_identity_matches_path_names_and_shebang_interpreters() {
7875 use std::os::unix::fs::PermissionsExt;
7876 let dir = cortexkit_test_support::ScratchDir::new("spawn-image-identity");
7877 let script = dir.join("module.sh");
7878 std::fs::write(&script, "#!/bin/sh\nwhile :; do sleep 0.1; done\n").unwrap();
7879 std::fs::set_permissions(&script, std::fs::Permissions::from_mode(0o700)).unwrap();
7880 let record_path = dir.join("live-children.json");
7881 let supervisor =
7882 Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default())
7883 .with_live_children_record(&record_path);
7884 for (program, args) in [
7885 (PathBuf::from("sleep"), vec!["60".into()]),
7886 (script, vec![]),
7887 ] {
7888 let spec = ModuleSpec {
7889 module_id: "image-identity".into(),
7890 program: program.clone(),
7891 args,
7892 env: ["XDG_DATA_HOME", "XDG_RUNTIME_DIR", "XDG_CONFIG_HOME"]
7893 .into_iter()
7894 .map(|key| (key.into(), dir.to_string_lossy().into_owned()))
7895 .collect(),
7896 reserved: false,
7897 reserved_prefixes: vec![],
7898 protocol: ModuleProtocol::None,
7899 overlap: Default::default(),
7900 carry_containment: false,
7901 };
7902 let module = supervisor.spawn(spec).unwrap();
7903 #[cfg(target_os = "macos")]
7904 {
7905 let deadline = Instant::now() + Duration::from_secs(5);
7908 while crate::live_children::read_record(&record_path)
7909 .unwrap()
7910 .iter()
7911 .all(|entry| entry.executable.is_none())
7912 {
7913 assert!(Instant::now() < deadline, "module image was not confirmed");
7914 tokio::time::sleep(Duration::from_millis(5)).await;
7915 }
7916 }
7917 let entry = crate::live_children::read_record(&record_path)
7918 .unwrap()
7919 .pop()
7920 .unwrap();
7921 let observed = subc_os::Process::open(entry.pid)
7922 .unwrap()
7923 .unwrap()
7924 .observe()
7925 .unwrap();
7926 let verdict = crate::live_children::identity_verdict(&entry, entry.pid, &observed);
7927 module.drain().await.unwrap();
7928 assert_eq!(
7929 verdict,
7930 crate::live_children::IdentityVerdict::Matches,
7931 "program {program:?}: recorded {entry:?}, observed {observed:?}"
7932 );
7933 }
7934 }
7935
7936 #[cfg(unix)]
7937 fn http_fixture(
7938 dir: &std::path::Path,
7939 url: &str,
7940 threshold: u32,
7941 ) -> crate::daemon_config::ConfiguredModule {
7942 let path = dir.join("subc.jsonc");
7943 std::fs::write(&path, serde_json::json!({"version": 1, "modules": {"http-process": {
7944 "program": "/bin/sleep", "args": ["60"], "protocol": "none",
7945 "env": {"XDG_DATA_HOME":dir,"XDG_RUNTIME_DIR":dir,"XDG_CONFIG_HOME":dir},
7946 "health": {"http":url,"cadence_ms":20,"deadline_ms":200,"failure_threshold":threshold},
7947 "restart": {"backoff_ms":1,"max_backoff_ms":1}, "drain_timeout_ms":10
7948 }}}).to_string()).unwrap();
7949 crate::daemon_config::load(&path)
7950 .unwrap()
7951 .unwrap()
7952 .modules
7953 .pop()
7954 .unwrap()
7955 }
7956
7957 #[cfg(unix)]
7958 async fn wait_http_health(module: &SupervisedModule, status: SupervisorHealthStatus) {
7959 timeout(Duration::from_secs(5), async {
7960 loop {
7961 if module.status().unwrap().health.status == status {
7962 break;
7963 }
7964 sleep(Duration::from_millis(5)).await;
7965 }
7966 })
7967 .await
7968 .unwrap_or_else(|_| {
7969 panic!(
7970 "expected {status:?}, got {:?}",
7971 module.status().unwrap().health
7972 )
7973 });
7974 }
7975
7976 #[cfg(unix)]
7977 #[tokio::test]
7978 async fn http_health_status_flips_ok_failing_ok() {
7979 use tokio::io::{AsyncReadExt, AsyncWriteExt};
7980 let dir = cortexkit_test_support::ScratchDir::new("http-status-flips");
7981 let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
7982 let status = Arc::new(std::sync::atomic::AtomicU16::new(200));
7983 let serving_status = status.clone();
7984 let url = format!("http://{}/healthz", listener.local_addr().unwrap());
7985 let server = tokio::spawn(async move {
7986 loop {
7987 let (mut stream, _) = listener.accept().await.unwrap();
7988 let mut request = [0u8; 2048];
7989 let count = stream.read(&mut request).await.unwrap();
7990 assert!(count > 0, "a probe must send an HTTP request");
7991 let code = serving_status.load(std::sync::atomic::Ordering::SeqCst);
7992 let body = if code == 200 {
7993 "ready"
7994 } else {
7995 "{\"status\":\"unavailable\",\"error\":\"scratch failure\"}"
7996 };
7997 let response = format!("HTTP/1.1 {code} scratch\r\nContent-Length: {}\r\nConnection: close\r\n\r\n{body}", body.len());
7998 let _ = stream.write_all(response.as_bytes()).await;
7999 }
8000 });
8001 let configured = http_fixture(&dir, &url, 1000);
8002 let module =
8003 Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default())
8004 .supervise_configured_with_health(
8005 configured.module_spec(),
8006 true,
8007 configured.health,
8008 configured.drain_timeout_ms,
8009 configured.restart,
8010 )
8011 .unwrap();
8012 wait_http_health(&module, SupervisorHealthStatus::Ok).await;
8013 status.store(503, std::sync::atomic::Ordering::SeqCst);
8014 wait_http_health(&module, SupervisorHealthStatus::Failing).await;
8015 assert!(module
8016 .status()
8017 .unwrap()
8018 .health
8019 .detail
8020 .unwrap()
8021 .contains("scratch failure"));
8022 status.store(200, std::sync::atomic::Ordering::SeqCst);
8023 wait_http_health(&module, SupervisorHealthStatus::Ok).await;
8024 assert_eq!(module.status().unwrap().health.consecutive_failures, 0);
8025 let before = module.status().unwrap();
8026 let (spec, mut health) = module.configuration().unwrap();
8027 health.http = None;
8028 module
8029 .update_configuration(spec.clone(), health.clone(), Some(10))
8030 .await
8031 .unwrap();
8032 wait_http_health(&module, SupervisorHealthStatus::Unknown).await;
8033 health.http = Some(url);
8034 module
8035 .update_configuration(spec, health, Some(10))
8036 .await
8037 .unwrap();
8038 wait_http_health(&module, SupervisorHealthStatus::Ok).await;
8039 assert_eq!(
8040 module.status().unwrap().pid,
8041 before.pid,
8042 "changing a probe must apply live, not restart its process"
8043 );
8044 let (spec, mut health) = module.configuration().unwrap();
8045 health.failure_threshold = 2;
8046 module
8047 .update_configuration(spec, health, Some(10))
8048 .await
8049 .unwrap();
8050 status.store(503, std::sync::atomic::Ordering::SeqCst);
8051 timeout(Duration::from_secs(5), async {
8052 while module.status().unwrap().spawn_generation == before.spawn_generation {
8053 sleep(Duration::from_millis(5)).await;
8054 }
8055 })
8056 .await
8057 .expect("sustained HTTP 503 responses must trigger the health restart policy");
8058 module.drain().await.unwrap();
8059 server.abort();
8060 }
8061
8062 #[cfg(unix)]
8063 #[tokio::test]
8064 async fn http_health_replacement_recovers_after_one_restart() {
8065 use tokio::io::{AsyncReadExt, AsyncWriteExt};
8066 let dir = cortexkit_test_support::ScratchDir::new("http-replacement-recovers");
8067 let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
8068 let url = format!("http://{}/healthz", listener.local_addr().unwrap());
8069 let mut configured = http_fixture(&dir, &url, 2);
8070 configured.restart.max_restarts = 2;
8071 let module =
8072 Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default())
8073 .supervise_configured_with_health(
8074 configured.module_spec(),
8075 true,
8076 configured.health,
8077 configured.drain_timeout_ms,
8078 configured.restart,
8079 )
8080 .unwrap();
8081 let first_generation = module.status().unwrap().spawn_generation;
8082 let observed = module.clone();
8083 let (healthy_tx, mut healthy_rx) = mpsc::channel(8);
8084 let server = tokio::spawn(async move {
8085 let mut generation = first_generation;
8086 let mut probes = 0;
8087 loop {
8088 let (mut stream, _) = listener.accept().await.unwrap();
8089 let mut request = [0; 2048];
8090 assert!(stream.read(&mut request).await.unwrap() > 0);
8091 let current = observed.status().unwrap().spawn_generation;
8092 if current != generation {
8093 generation = current;
8094 probes = 0;
8095 }
8096 probes += 1;
8097 let healthy = generation != first_generation && probes > 1;
8100 let code = if healthy { 200 } else { 503 };
8101 let response = format!(
8102 "HTTP/1.1 {code} scratch\r\nContent-Length: 0\r\nConnection: close\r\n\r\n"
8103 );
8104 let _ = stream.write_all(response.as_bytes()).await;
8105 if healthy {
8106 healthy_tx.send(()).await.unwrap();
8107 }
8108 }
8109 });
8110 timeout(Duration::from_secs(5), async {
8111 let mut healthy_probes = 0;
8112 loop {
8113 if module.state().unwrap() == ModuleState::Failed {
8114 break;
8115 }
8116 while healthy_rx.try_recv().is_ok() {
8117 healthy_probes += 1;
8118 }
8119 if healthy_probes >= 3
8120 && module.status().unwrap().health.status == SupervisorHealthStatus::Ok
8121 {
8122 break;
8123 }
8124 sleep(Duration::from_millis(5)).await;
8125 }
8126 })
8127 .await
8128 .expect("the replacement must settle healthy without exhausting its restart budget");
8129 let status = module.status().unwrap();
8130 module.drain().await.unwrap();
8131 server.abort();
8132 assert_eq!(status.state, ModuleState::Running);
8133 assert_eq!(status.health.status, SupervisorHealthStatus::Ok);
8134 assert_eq!(status.health.consecutive_failures, 0);
8135 assert_eq!(status.lifetime_restarts, 1);
8136 assert_eq!(status.restart_count, 1);
8137 }
8138
8139 #[cfg(unix)]
8140 #[tokio::test]
8141 async fn http_health_sustained_failure_exhausts_restart_budget() {
8142 use std::collections::BTreeMap;
8143 use tokio::io::{AsyncReadExt, AsyncWriteExt};
8144 let dir = cortexkit_test_support::ScratchDir::new("http-health-budget");
8145 let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
8146 let url = format!("http://{}/healthz", listener.local_addr().unwrap());
8147 let mut configured = http_fixture(&dir, &url, 2);
8148 configured.restart.max_restarts = 2;
8149 let module =
8150 Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default())
8151 .supervise_configured_with_health(
8152 configured.module_spec(),
8153 true,
8154 configured.health,
8155 configured.drain_timeout_ms,
8156 configured.restart,
8157 )
8158 .unwrap();
8159 let observed = module.clone();
8160 let probes = Arc::new(Mutex::new(BTreeMap::<u64, u32>::new()));
8161 let served = probes.clone();
8162 let server = tokio::spawn(async move {
8163 loop {
8164 let (mut stream, _) = listener.accept().await.unwrap();
8165 let mut request = [0; 2048];
8166 assert!(stream.read(&mut request).await.unwrap() > 0);
8167 *served
8168 .lock()
8169 .unwrap()
8170 .entry(observed.status().unwrap().spawn_generation)
8171 .or_default() += 1;
8172 let _ = stream
8173 .write_all(b"HTTP/1.1 503 Unavailable\r\nContent-Length: 0\r\nConnection: close\r\n\r\n")
8174 .await;
8175 }
8176 });
8177 timeout(Duration::from_secs(5), async {
8178 while module.state().unwrap() != ModuleState::Failed {
8179 sleep(Duration::from_millis(5)).await;
8180 }
8181 })
8182 .await
8183 .expect("sustained failures must still exhaust the health restart budget");
8184 let status = module.status().unwrap();
8185 server.abort();
8186 assert_eq!(status.restart_count, 2);
8187 assert_eq!(status.lifetime_restarts, 2);
8188 assert!(!status.process_alive);
8189 assert_eq!(
8190 probes.lock().unwrap().values().copied().collect::<Vec<_>>(),
8191 vec![2, 2, 2]
8192 );
8193 }
8194
8195 #[cfg(unix)]
8196 #[tokio::test]
8197 async fn spawn_clears_the_previous_process_health() {
8198 let dir = cortexkit_test_support::ScratchDir::new("spawn-fresh-health");
8199 let mut configured = http_fixture(&dir, "http://127.0.0.1:1/healthz", 2);
8200 configured.health.cadence = Duration::from_secs(3600);
8201 let module =
8202 Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default())
8203 .supervise_configured_with_health(
8204 configured.module_spec(),
8205 false,
8206 configured.health,
8207 configured.drain_timeout_ms,
8208 configured.restart,
8209 )
8210 .unwrap();
8211 update_snapshot(&module.inner.snapshot, Some(module.module_id()), |state| {
8212 state.health.status = SupervisorHealthStatus::Unresponsive;
8213 state.health.consecutive_failures = 3;
8214 state.health.detail = Some("old process refused connections".into());
8215 state.health.metrics = Some(serde_json::json!({"old": true}));
8216 state.health.last_probe_ms = Some(123);
8217 state.health.last_action = Some("restart".into());
8218 state.health.last_action_ms = Some(124);
8219 })
8220 .unwrap();
8221 module.set_enabled(true).await.unwrap();
8222 let health = module.status().unwrap().health;
8223 module.drain().await.unwrap();
8224 assert_eq!(health.consecutive_failures, 0);
8225 assert_eq!(health.status, SupervisorHealthStatus::Unknown);
8226 assert_eq!(health.detail, None);
8227 assert_eq!(health.metrics, None);
8228 assert_eq!(health.last_probe_ms, None);
8229 assert_eq!(health.last_action.as_deref(), Some("restart"));
8230 assert_eq!(health.last_action_ms, Some(124));
8231 }
8232
8233 #[cfg(unix)]
8234 #[tokio::test]
8235 async fn http_health_no_listener_restarts_after_consecutive_failures() {
8236 let dir = cortexkit_test_support::ScratchDir::new("http-refused");
8237 let unused = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
8238 let url = format!("http://{}/healthz", unused.local_addr().unwrap());
8239 drop(unused);
8240 let configured = http_fixture(&dir, &url, 2);
8241 let module =
8242 Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default())
8243 .supervise_configured_with_health(
8244 configured.module_spec(),
8245 true,
8246 configured.health,
8247 configured.drain_timeout_ms,
8248 configured.restart,
8249 )
8250 .unwrap();
8251 let before = module.status().unwrap().spawn_generation;
8252 timeout(Duration::from_secs(5), async {
8253 loop {
8254 let status = module.status().unwrap();
8255 if status.spawn_generation > before {
8256 assert!(status.lifetime_restarts > 0);
8257 break;
8258 }
8259 sleep(Duration::from_millis(5)).await;
8260 }
8261 })
8262 .await
8263 .expect("sustained HTTP refusal must trigger the health restart policy");
8264 module.drain().await.unwrap();
8265 }
8266
8267 #[cfg(unix)]
8268 #[tokio::test]
8269 async fn http_health_timeout_honours_deadline() {
8270 let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
8271 let url = format!("http://{}/healthz", listener.local_addr().unwrap());
8272 let server = tokio::spawn(async move {
8273 let _held = listener.accept().await.unwrap();
8274 std::future::pending::<()>().await;
8275 });
8276 let error = timeout(
8277 Duration::from_secs(1),
8278 probe_http_health(&url, Duration::from_millis(10)),
8279 )
8280 .await
8281 .expect("the probe must enforce its own deadline")
8282 .unwrap_err();
8283 server.abort();
8284 assert!(error.to_string().contains("timed out"));
8285 }
8286
8287 #[cfg(unix)]
8288 #[tokio::test]
8289 async fn http_health_localhost_reaches_an_ipv6_only_listener() {
8290 use tokio::io::{AsyncReadExt, AsyncWriteExt};
8291 let listener = tokio::net::TcpListener::bind("[::1]:0").await.unwrap();
8292 let url = format!(
8293 "http://localhost:{}/healthz",
8294 listener.local_addr().unwrap().port()
8295 );
8296 let server = tokio::spawn(async move {
8297 let (mut stream, _) = listener.accept().await.unwrap();
8298 let mut request = [0u8; 2048];
8299 assert!(stream.read(&mut request).await.unwrap() > 0);
8300 stream
8301 .write_all(b"HTTP/1.1 200 OK\r\nContent-Length: 2\r\n\r\nok")
8302 .await
8303 .unwrap();
8304 });
8305 assert_eq!(
8310 probe_http_health(&url, Duration::from_secs(10))
8311 .await
8312 .unwrap()
8313 .status,
8314 HealthStatus::Ok
8315 );
8316 server.await.unwrap();
8317 }
8318
8319 #[cfg(unix)]
8320 #[tokio::test]
8321 async fn http_health_timeout_keeps_partial_status_and_body() {
8322 use tokio::io::{AsyncReadExt, AsyncWriteExt};
8323 let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
8324 let url = format!("http://{}/healthz", listener.local_addr().unwrap());
8325 let server = tokio::spawn(async move {
8326 let (mut stream, _) = listener.accept().await.unwrap();
8327 let mut request = [0u8; 2048];
8328 assert!(stream.read(&mut request).await.unwrap() > 0);
8329 stream
8330 .write_all(
8331 b"HTTP/1.1 503 Unavailable\r\nContent-Length: 100\r\n\r\npartial diagnostic",
8332 )
8333 .await
8334 .unwrap();
8335 std::future::pending::<()>().await;
8336 });
8337 let error = probe_http_health(&url, Duration::from_secs(1))
8338 .await
8339 .unwrap_err()
8340 .to_string();
8341 server.abort();
8342 assert!(
8343 error.contains("timed out")
8344 && error.contains("503 Unavailable")
8345 && error.contains("partial diagnostic"),
8346 "{error}"
8347 );
8348 }
8349
8350 #[cfg(unix)]
8351 #[tokio::test]
8352 async fn http_health_diagnostic_body_is_bounded_to_200_bytes() {
8353 use tokio::io::{AsyncReadExt, AsyncWriteExt};
8354 let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
8355 let url = format!("http://{}/healthz", listener.local_addr().unwrap());
8356 let server = tokio::spawn(async move {
8357 let (mut stream, _) = listener.accept().await.unwrap();
8358 let mut request = [0u8; 2048];
8359 assert!(stream.read(&mut request).await.unwrap() > 0);
8360 let body = format!("{}not-in-diagnostic", "x".repeat(400));
8361 let response = format!(
8362 "HTTP/1.1 503 Unavailable\r\nContent-Length: {}\r\n\r\n{body}",
8363 body.len()
8364 );
8365 stream.write_all(response.as_bytes()).await.unwrap();
8366 });
8367 let error = probe_http_health(&url, Duration::from_secs(1))
8368 .await
8369 .unwrap_err()
8370 .to_string();
8371 server.await.unwrap();
8372 assert!(error.ends_with(&"x".repeat(200)), "{error}");
8373 assert_eq!(error.split(": ").last().unwrap().len(), 200);
8374 assert!(!error.contains("not-in-diagnostic"));
8375 }
8376
8377 #[cfg(unix)]
8378 #[tokio::test]
8379 async fn http_health_real_nats_server_monitoring() {
8380 if std::process::Command::new("nats-server")
8381 .arg("--version")
8382 .env("XDG_DATA_HOME", std::env::temp_dir())
8383 .env("XDG_RUNTIME_DIR", std::env::temp_dir())
8384 .env("XDG_CONFIG_HOME", std::env::temp_dir())
8385 .output()
8386 .is_err()
8387 {
8388 eprintln!(
8389 "skipped: http_health_real_nats_server_monitoring (nats-server is not on PATH)"
8390 );
8391 return;
8392 }
8393 let dir = cortexkit_test_support::ScratchDir::new("nats-http-monitoring");
8394 let monitor = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
8395 let port = monitor.local_addr().unwrap().port();
8396 let client = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
8397 let client_port = client.local_addr().unwrap().port();
8398 let config = dir.join("server.conf");
8399 std::fs::write(&config, format!("listen: 127.0.0.1:{client_port}\nhttp: 127.0.0.1:{port}\njetstream {{ store_dir: \"{}\" }}\n", dir.join("jetstream").display())).unwrap();
8400 let mut configured = http_fixture(&dir, &format!("http://127.0.0.1:{port}/healthz"), 1000);
8401 configured.program = PathBuf::from("nats-server");
8402 configured.args = vec!["-c".into(), config.to_string_lossy().into_owned()];
8403 drop(monitor);
8404 drop(client);
8405 let module =
8406 Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default())
8407 .supervise_configured_with_health(
8408 configured.module_spec(),
8409 true,
8410 configured.health,
8411 configured.drain_timeout_ms,
8412 configured.restart,
8413 )
8414 .unwrap();
8415 wait_http_health(&module, SupervisorHealthStatus::Ok).await;
8416 module.drain().await.unwrap();
8417 }
8418
8419 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
8420 async fn configuration_update_does_not_replace_captured_running_process_facts() {
8421 let supervisor =
8422 Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default());
8423 let snapshot = stale_process_snapshot(ModuleState::Running, true);
8424 let initial = ModuleSpec {
8425 module_id: "rescan-preserves-spawn-facts".to_string(),
8426 program: PathBuf::from("/spawned/module"),
8427 args: Vec::new(),
8428 env: Vec::new(),
8429 reserved: false,
8430 reserved_prefixes: Vec::new(),
8431 protocol: ModuleProtocol::Subc,
8432 overlap: Default::default(),
8433 carry_containment: false,
8434 };
8435 let module = supervisor.supervised_module(
8436 initial.clone(),
8437 supervisor.runtime_config(),
8438 snapshot,
8439 None,
8440 );
8441 let before = module.status().unwrap();
8442 let mut replacement = initial;
8443 replacement.program = PathBuf::from("/rescanned/replacement-module");
8444
8445 module
8446 .update_configuration(replacement, HealthConfig::default(), None)
8447 .await
8448 .unwrap();
8449
8450 let after = module.status().unwrap();
8451 assert_eq!(after.pid, before.pid);
8452 assert_eq!(after.spawned_at_ms, before.spawned_at_ms);
8453 assert_eq!(after.spawned_from, before.spawned_from);
8454 drop(module);
8455 }
8456}
8457
8458fn unix_ms_now() -> u64 {
8459 SystemTime::now()
8460 .duration_since(UNIX_EPOCH)
8461 .map(|duration| duration.as_millis().min(u128::from(u64::MAX)) as u64)
8462 .unwrap_or(0)
8463}
8464
8465async fn supervise_loop(
8466 mut spec: ModuleSpec,
8467 mut runtime: SupervisorRuntimeConfig,
8468 registry: Arc<Registry>,
8469 process_liveness: Arc<SupervisorProcessLiveness>,
8470 snapshot: SharedSnapshot,
8471 mut child: Option<SupervisedChild>,
8472 mut commands: mpsc::Receiver<SupervisorCommand>,
8473) {
8474 let mut health_probe = HealthProbeRuntime::default();
8475 let mut registration_changes = match registry.subscribe_module_changes(&spec.module_id) {
8479 Ok(changes) => Some(changes),
8480 Err(err) => {
8481 warn!(module_id = %spec.module_id, error = %err, "health prober could not subscribe to registry");
8484 None
8485 }
8486 };
8487 let mut pending_respawn: Option<PendingRespawn> = None;
8491 let mut requeued: VecDeque<SupervisorCommand> = VecDeque::new();
8494 let mut previous_instance_recheck = tokio::time::interval_at(
8495 Instant::now() + Duration::from_secs(5),
8496 Duration::from_secs(5),
8497 );
8498 previous_instance_recheck.set_missed_tick_behavior(tokio::time::MissedTickBehavior::Skip);
8499 'supervision: loop {
8500 #[cfg(test)]
8501 {
8502 lock_snapshot(&snapshot).unwrap().actor_select = None;
8503 }
8504 #[cfg(target_os = "macos")]
8505 if let Some(active) = child.as_mut() {
8506 active.confirm_privacy_exec().await;
8507 }
8508 if let Some(scheduled) = runtime
8509 .scheduled_respawn
8510 .lock()
8511 .unwrap_or_else(|p| p.into_inner())
8512 .take()
8513 {
8514 runtime.recovery_is_daemon = scheduled.daemon_initiated;
8515 pending_respawn = Some(scheduled);
8516 }
8517 if pending_respawn.is_some() && (child.is_some() || !respawn_still_pending(&snapshot)) {
8518 pending_respawn = None;
8519 cancel_deferred_reload(
8520 &runtime,
8521 &spec.module_id,
8522 "respawn cancelled by a supervisor command",
8523 );
8524 }
8525 if child.is_none()
8526 && pending_respawn.is_none()
8527 && lock_snapshot(&snapshot).is_ok_and(|state| state.waiting_for_previous_pid.is_none())
8528 {
8529 cancel_deferred_reload(
8530 &runtime,
8531 &spec.module_id,
8532 "respawn cancelled before a replacement was spawned",
8533 );
8534 let mut stranded = false;
8535 let _ = update_snapshot(&snapshot, Some(&spec.module_id), |state| {
8536 state.respawn_pending = false;
8537 state.coalesced_restart_pending = false;
8538 if matches!(
8539 state.state,
8540 ModuleState::Restarting
8541 | ModuleState::Starting
8542 | ModuleState::Draining
8543 | ModuleState::Unresponsive
8544 ) {
8545 error!(module_id = %spec.module_id, state = ?state.state,
8546 "supervision operation ended without a child or pending respawn; marking failed so start can retry");
8547 stranded = true;
8548 }
8549 });
8550 if stranded {
8551 fail_snapshot(
8552 &snapshot,
8553 Some(&spec.module_id),
8554 None,
8555 &runtime.failure_context(runtime.recovery_is_daemon),
8556 );
8557 }
8558 }
8559 if child.is_none()
8560 && (runtime.child_roster.is_closed()
8561 || pending_respawn.is_none()
8562 || !lock_snapshot(&snapshot)
8563 .is_ok_and(|state| state.enabled && state.state == ModuleState::Restarting))
8564 {
8565 containment::terminate(&runtime.child_roster.containment).await;
8566 }
8567 if let Some(command) = requeued.pop_front() {
8568 if !handle_supervisor_command(
8569 command,
8570 &mut spec,
8571 &mut runtime,
8572 ®istry,
8573 &process_liveness,
8574 &snapshot,
8575 &mut child,
8576 &mut commands,
8577 &mut requeued,
8578 )
8579 .await
8580 {
8581 containment::terminate(&runtime.child_roster.containment).await;
8582 return;
8583 }
8584 if child.is_some() || !respawn_still_pending(&snapshot) {
8585 pending_respawn = None;
8586 let _ = update_snapshot(&snapshot, Some(&spec.module_id), |state| {
8587 state.respawn_pending = false
8588 });
8589 }
8590 continue;
8591 }
8592 if child.is_some() {
8593 runtime.recovery_is_daemon = true;
8594 #[cfg(test)]
8595 {
8596 lock_snapshot(&snapshot).unwrap().actor_turns += 1;
8597 }
8598 health_probe.refresh_registration(&spec, &runtime, ®istry, &snapshot);
8599 let active = child.as_mut().expect("child checked above");
8600 if active.registered_since_spawn() {
8601 if let Some(delivery) = &runtime.config_delivery {
8602 delivery.queue_registered(
8603 &spec.module_id,
8604 runtime.forwarding.as_ref(),
8605 ®istry,
8606 active,
8607 );
8608 }
8609 }
8610 if runtime.child_roster.is_closed() || !active.awaiting_registration() {
8611 active.registration_deadline = None;
8612 }
8613 let registration_deadline = active.registration_deadline;
8614 let no_checks = runtime.health.http.is_none()
8615 && (running_protocol(&spec, &snapshot) == ModuleProtocol::None
8616 || (health_probe.registered_connection.is_some() && !health_probe.advertised));
8617 let uptime_reset = lock_snapshot(&snapshot).ok().and_then(|mut state| {
8618 state
8619 .failed_retry
8620 .uptime_reset_at(no_checks, runtime.restart_policy.window)
8621 });
8622 let wake_at = match (health_probe.wake_at(), uptime_reset) {
8623 (probe, Some(reset)) => Some(probe.map_or(reset, |probe| probe.min(reset))),
8624 (probe, None) => probe,
8625 };
8626 let wake_at = match (wake_at, registration_deadline) {
8627 (Some(wake), Some(register)) => Some(wake.min(register)),
8628 (wake, register) => wake.or(register),
8629 };
8630 #[cfg(test)]
8631 let wake_after = wake_at.map(|at| at.saturating_duration_since(Instant::now()));
8632 let probe_sleep = supervisor_sleep(wake_at);
8633 tokio::pin!(probe_sleep);
8634 let wire_child = running_protocol(&spec, &snapshot) == ModuleProtocol::Subc;
8639 #[cfg(test)]
8640 {
8641 let registration_pending = wire_child
8642 && registration_changes
8643 .as_ref()
8644 .is_some_and(|changes| changes.has_changed().unwrap());
8645 let mut state = lock_snapshot(&snapshot).unwrap();
8646 state.actor_select = (!registration_pending).then_some(ActorSelectCheckpoint {
8647 generation: state.spawn_generation,
8648 turn: state.actor_turns,
8649 registered_connection: health_probe.registered_connection,
8650 next_probe_at: health_probe.next_probe_at,
8651 wake_after,
8652 });
8653 }
8654 let registration = active
8655 .awaiting_registration()
8656 .then(|| active.registration_spawn.clone())
8657 .flatten();
8658 #[cfg(all(test, target_os = "linux"))]
8659 let registration_before_exit = !runtime.child_roster.admission_event_for_test(
8660 &spec.module_id,
8661 "observe_registration_before_exit",
8662 Some(active.pid),
8663 ) && registration.is_some();
8664 #[cfg(not(all(test, target_os = "linux")))]
8665 let registration_before_exit = false;
8666 #[cfg(all(test, target_os = "linux"))]
8667 let registration_child_is_running = |child: &mut SupervisedChild| {
8668 registration_before_exit || registration_child_is_running(child)
8671 };
8672 let registration_failed = async {
8673 match ®istration {
8674 Some(spawn) => spawn.wait_registration_failure().await,
8675 None => std::future::pending().await,
8676 }
8677 };
8678 let active_child = child.as_mut().expect("child checked above");
8679 tokio::select! {
8680 _ = registration_failed => {
8681 if !active_child.registered_since_spawn() && registration_child_is_running(active_child) {
8682 active_child.registration_deadline = None;
8683 active_child.spawn_failure = Some("registration_channel_closed".into());
8684 if let Err(error) = stop_unregistered_child(active_child).await {
8685 fail_after_wait_error(&spec.module_id, &snapshot, &runtime.terminal_ring,
8686 &runtime.spawn_events, format!("registration_channel_closed: {error}"),
8687 &runtime.failure_context(true));
8688 child = None;
8689 }
8690 }
8691 continue;
8692 }
8693 wait_result = active_child.wait(), if !registration_before_exit => {
8694 if let Ok(mut state) = lock_snapshot(&snapshot) {
8695 state.failed_retry.uptime_reset_at(no_checks, runtime.restart_policy.window);
8696 }
8697 #[cfg(test)]
8698 { lock_snapshot(&snapshot).unwrap().actor_select = None; }
8699 let exit_report = match wait_result {
8708 Ok(status) => classify_reaped_child_exit(&snapshot, active_child, &status),
8709 Err(err) => {
8710 active_child.drain_stderr(&spec.module_id).await;
8711 fail_after_wait_error(
8712 &spec.module_id,
8713 &snapshot,
8714 &runtime.terminal_ring,
8715 &runtime.spawn_events,
8716 format!("failed to wait for supervised module: {err}"),
8717 &runtime.failure_context(true),
8718 );
8719 untrack_if_registration_released(
8720 &process_liveness,
8721 ®istry,
8722 &spec.module_id,
8723 &snapshot,
8724 );
8725 error!(module_id = %spec.module_id, error = %err, "failed to wait for supervised module");
8726 child = None;
8727 continue;
8728 }
8729 };
8730 active_child.drain_stderr(&spec.module_id).await;
8731
8732 let delivery=runtime.config_delivery.clone();
8733 let exited_spawn=active_child.spawned_at;
8734 let spawn=active_child.config_delivery.clone();
8735 let refusal=active_child.spawn_failure.as_deref().filter(|detail|detail.starts_with("program_refused:") && !active_child.live_program_binding.as_ref().is_some_and(|binding|binding.is_refused())).zip(active_child.admission_base.clone()).map(|(detail,base)|(base,subc_config::raw_digest(detail.as_bytes())));
8736 let module_id=spec.module_id.clone();
8737 let observed=async move {
8741 if let Some(delivery)=delivery {
8742 delivery.0.on_blocking(move |owner| {
8743 if let Some(spawn)=spawn { owner.delivery_child_exited(&spawn)?; }
8744 if let Some((base,identity))=refusal {owner.record_program_refusal(&module_id,&base,identity)?;}
8745 Ok(())
8746 }).await
8747 } else {Ok(())}
8748 };
8749 tokio::pin!(observed);
8750 loop {
8751 tokio::select! {
8752 result=&mut observed=>{
8753 if let Err(error)=result {warn!(module_id=%spec.module_id,%error,"configuration exit observation failed");}
8754 break;
8755 }
8756 command=commands.recv()=>{
8757 let Some(command)=command else {containment::terminate(&runtime.child_roster.containment).await;return;};
8758 let deferred=||SuperviseError::ConfigDeliveryDeferred {module_id:spec.module_id.clone()};
8759 let command=match command {
8760 SupervisorCommand::Restart {reply,..}|SupervisorCommand::Reload {reply,..}=>{let _=reply.send(Err(deferred()));continue;}
8761 SupervisorCommand::Swap {reply,..}=>{let _=reply.send(Err(deferred()));continue;}
8762 command @ (SupervisorCommand::ConfigApplyRestart {..}|SupervisorCommand::ConfigPolicyProbe {..}|SupervisorCommand::ConfigDeliveryFailed {..})=>{requeued.push_back(command);continue;}
8763 command=>command,
8764 };
8765 if !handle_supervisor_command(command,&mut spec,&mut runtime,®istry,&process_liveness,&snapshot,&mut child,&mut commands,&mut requeued).await {
8766 containment::terminate(&runtime.child_roster.containment).await;return;
8767 }
8768 if child.as_ref().is_none_or(|current|current.spawned_at!=exited_spawn) {continue 'supervision;}
8769 }
8770 }
8771 }
8772 let active_child=child.as_mut().expect("exit observation retained its child");
8773 let admission_kill = admission_exit::is_admission_kill(active_child.admission_kill_sent, &exit_report);
8777 let deferred_spawn=admission_kill && active_child.spawn_failure.as_deref().is_some_and(|detail|detail.starts_with("config_delivery_deferred:"));
8778 let refused_program = active_child.live_program_binding.as_ref().is_some_and(|binding| binding.is_refused());
8779 let exit_held = !deferred_spawn && (refused_program || runtime.config_delivery.as_ref().is_some_and(|delivery| delivery.0.exit_is_held(&spec.module_id)));
8780 let parked = exit_held && (!refused_program || active_child.admission_base.is_some())
8781 && try_park_config_attempt(&spec, &runtime, &snapshot, &runtime.failure_context(true),
8782 if refused_program { active_child.admission_base.as_ref() } else { None })
8783 .unwrap_or_else(|error| { warn!(module_id=%spec.module_id, %error, "exit hold could not be recorded; using ordinary exit accounting"); false });
8784 let next = if deferred_spawn {
8785 record_terminal_with_detail(&spec.module_id,&runtime.terminal_ring,&runtime.spawn_events,&exit_report,TerminalDisposition::Restarting,active_child.spawn_failure.clone());
8786 let _=update_snapshot(&snapshot,Some(&spec.module_id),|state| {
8787 state.state=ModuleState::Restarting;
8788 clear_current_process_facts(state);
8789 state.last_exit=Some(exit_report.clone());
8790 });
8791 NextAction::Restart {schedule:None}
8792 } else if parked {
8793 record_terminal_with_detail(&spec.module_id, &runtime.terminal_ring, &runtime.spawn_events, &exit_report, TerminalDisposition::Stopped, active_child.spawn_failure.clone().or_else(|| Some("config_gated".into())));
8794 NextAction::Stop { registration_released: registry.get_module(&spec.module_id).ok().flatten().is_none() }
8795 } else { on_child_exit(
8796 &spec,
8797 runtime.restart_policy,
8798 ®istry,
8799 &snapshot,
8800 &runtime.terminal_ring,
8801 &runtime.spawn_events,
8802 &runtime.child_roster,
8803 exit_report,
8804 true,
8805 ).await };
8806 runtime.start_requester = None;
8807 runtime.pending_activity_restart = false;
8808 let carry = matches!(next, NextAction::Restart { .. })
8811 && active_child.spawn_failure.is_none()
8812 && !runtime.child_roster.is_closed();
8813 if let Err(error) = active_child.finish_exit(carry).await {
8814 containment::terminate(&runtime.child_roster.containment).await;
8815 fail_after_wait_error(&spec.module_id, &snapshot, &runtime.terminal_ring,
8816 &runtime.spawn_events, format!("failed to reap classified exit: {error}"),
8817 &runtime.failure_context(true));
8818 child = None;
8819 continue;
8820 }
8821 match next {
8822 NextAction::Stop { registration_released } => {
8823 if registration_released {
8824 process_liveness.untrack_if_current(&spec.module_id, &snapshot);
8825 }
8826 child = None;
8827 }
8828 NextAction::Restart { schedule } => {
8829 let delay = if deferred_spawn {CONFIG_ADMISSION_RETRY_DELAY} else {schedule.map_or(
8830 runtime.restart_policy.delay_for_restart(0),
8831 |schedule| schedule.delay,
8832 )};
8833 if let Some(schedule) = schedule {
8834 log_crash_respawn(&spec.module_id, schedule);
8835 }
8836 child = None;
8844 pending_respawn = Some(PendingRespawn { deadline: Instant::now() + delay, kind: RespawnKind::Spawn, daemon_initiated: true, approval: None });
8845 let _ = update_snapshot(&snapshot, Some(&spec.module_id), |state| state.respawn_pending = true);
8846 }
8847 }
8848 }
8849 command = commands.recv() => {
8850 #[cfg(test)]
8851 { lock_snapshot(&snapshot).unwrap().actor_select = None; }
8852 let Some(command) = command else {
8853 containment::terminate(&runtime.child_roster.containment).await;
8854 return;
8855 };
8856 if !handle_supervisor_command(
8857 command,
8858 &mut spec,
8859 &mut runtime,
8860 ®istry,
8861 &process_liveness,
8862 &snapshot,
8863 &mut child,
8864 &mut commands,
8865 &mut requeued,
8866 ).await {
8867 containment::terminate(&runtime.child_roster.containment).await;
8868 return;
8869 }
8870 }
8871 _ = async {
8872 match registration_changes.as_mut() {
8873 Some(changes) => { let _ = changes.changed().await; }
8874 None => std::future::pending().await,
8875 }
8876 }, if wire_child => {
8877 #[cfg(test)]
8878 { lock_snapshot(&snapshot).unwrap().actor_select = None; }
8879 }
8880 _ = &mut probe_sleep => {
8881 #[cfg(test)]
8882 { lock_snapshot(&snapshot).unwrap().actor_select = None; }
8883 if registration_deadline.is_some_and(|deadline| Instant::now() >= deadline) {
8884 active_child.registration_deadline = None;
8887 if !runtime.child_roster.is_closed()
8888 && !active_child.registered_since_spawn()
8889 && registration_child_is_running(active_child)
8890 {
8891 warn!(module_id = %spec.module_id, pid = active_child.pid,
8892 wait_secs = runtime.register_timeout.as_secs(),
8893 "supervised module registration_timeout: first HELLO deadline expired; stopping child");
8894 active_child.spawn_failure = Some("registration_timeout".into());
8895 if let Err(error) = stop_unregistered_child(active_child).await {
8896 error!(module_id = %spec.module_id, %error, "failed to stop unregistered child");
8897 fail_after_wait_error(&spec.module_id, &snapshot, &runtime.terminal_ring,
8898 &runtime.spawn_events, format!("registration_timeout: {error}"),
8899 &runtime.failure_context(true));
8900 child = None;
8901 }
8902 continue;
8905 }
8906 }
8907 if health_probe.due() {
8908 run_health_probe_cycle(
8909 &spec,
8910 &runtime,
8911 ®istry,
8912 &process_liveness,
8913 &snapshot,
8914 &mut child,
8915 ).await;
8916 if child.is_none() {
8917 runtime.start_requester = None;
8918 runtime.pending_activity_restart = false;
8919 }
8920 if child.is_some() {
8921 health_probe.schedule_next(&spec, runtime.health.cadence);
8922 }
8923 }
8924 }
8925 }
8926 } else if let Some(pending) = pending_respawn.clone() {
8927 tokio::select! {
8928 _ = runtime.child_roster.closed() => {
8929 pending_respawn = None;
8930 let _ = update_snapshot(&snapshot, Some(&spec.module_id), |state| {
8931 state.state = ModuleState::Stopped;
8932 state.respawn_pending = false;
8933 });
8934 containment::terminate(&runtime.child_roster.containment).await;
8935 }
8936 _ = async {
8937 sleep_until(pending.deadline).await;
8938 if let Some(delivery) = &runtime.config_delivery {
8939 delivery.unfenced(&spec.module_id).await;
8940 }
8941 } => {
8942 pending_respawn = None;
8943 let failure = pending.failure_context(&runtime);
8944 let _ = update_snapshot(&snapshot, Some(&spec.module_id), |state| state.respawn_pending = false);
8945 if !respawn_still_pending(&snapshot) {
8949 continue;
8950 }
8951 if runtime.child_roster.is_closed() {
8956 let _ = update_snapshot(&snapshot, Some(&spec.module_id), |state| {
8957 state.state = ModuleState::Stopped;
8958 });
8959 debug!(module_id = %spec.module_id, "crash respawn cancelled by daemon shutdown");
8960 continue;
8961 }
8962 if let Err(err) = release_dead_registration(
8963 ®istry,
8964 runtime.forwarding.as_deref(),
8965 &snapshot,
8966 &spec.module_id,
8967 &failure,
8968 ).await {
8969 record_approved_attempt_failure(&spec, &snapshot, &failure, &err);
8970 error!(module_id = %spec.module_id, error = %err, "registration did not release before restart");
8971 continue;
8972 }
8973
8974 if matches!(pending.kind, RespawnKind::Reload) {
8975 let reply = runtime.deferred_reload_reply.lock().unwrap_or_else(|p| p.into_inner()).take();
8976 let result = finish_reload_child(&spec, &runtime, ®istry, &process_liveness, &snapshot, &mut child).await;
8977 if matches!(result, Err(SuperviseError::ConfigDeliveryDeferred { .. })) {
8978 *runtime.deferred_reload_reply.lock().unwrap_or_else(|p| p.into_inner()) = reply;
8979 continue;
8980 }
8981 if result.is_ok() && runtime.pending_activity_restart {
8982 runtime.record_activity_transition("subc.module_restarted", &spec.module_id, runtime.start_requester.clone());
8983 }
8984 runtime.pending_activity_restart = false;
8985 if let Some(reply) = reply { let _ = reply.send(result); }
8986 continue;
8987 }
8988 process_liveness.track(spec.module_id.clone(), Arc::clone(&snapshot));
8989 match spawn_and_mark_running_async(&spec, &runtime, &snapshot).await {
8990 Ok(next_child) => {
8991 child = Some(next_child);
8992 if runtime.pending_activity_restart {
8993 runtime.record_activity_transition("subc.module_restarted", &spec.module_id, runtime.start_requester.clone());
8994 runtime.pending_activity_restart = false;
8995 }
8996 debug!(module_id = %spec.module_id, "supervised module restarted after crash");
8997 }
8998 Err(SuperviseError::ConfigDeliveryDeferred { .. }) => {
8999 pending_respawn = Some(PendingRespawn { deadline:Instant::now()+CONFIG_ADMISSION_RETRY_DELAY, ..pending });
9000 let _ = update_snapshot(&snapshot, Some(&spec.module_id), |state| state.respawn_pending = true);
9001 }
9002 Err(SuperviseError::ConfigHeld { .. }) => {
9003 let _ = hold_config_spawn(&spec, &runtime, &snapshot, &failure);
9004 runtime.pending_activity_restart = false;
9005 process_liveness.untrack_if_current(&spec.module_id, &snapshot);
9006 }
9007 Err(err) => {
9008 fail_snapshot(&snapshot, Some(&spec.module_id), None, &failure);
9009 record_approved_attempt_failure(&spec, &snapshot, &failure, &err);
9010 process_liveness.untrack_if_current(&spec.module_id, &snapshot);
9011 error!(module_id = %spec.module_id, error = %err, "failed to restart supervised module");
9012 }
9013 }
9014 }
9015 command = commands.recv() => {
9016 let Some(command) = command else {
9017 containment::terminate(&runtime.child_roster.containment).await;
9018 return;
9019 };
9020 if !handle_supervisor_command(
9021 command,
9022 &mut spec,
9023 &mut runtime,
9024 ®istry,
9025 &process_liveness,
9026 &snapshot,
9027 &mut child,
9028 &mut commands,
9029 &mut requeued,
9030 ).await {
9031 containment::terminate(&runtime.child_roster.containment).await;
9032 return;
9033 }
9034 if child.is_some() || !respawn_still_pending(&snapshot) {
9039 pending_respawn = None;
9040 let _ = update_snapshot(&snapshot, Some(&spec.module_id), |state| state.respawn_pending = false);
9041 }
9042 }
9043 }
9044 } else {
9045 let slow_retry = lock_snapshot(&snapshot).ok().and_then(|state| {
9046 (state.enabled && state.state == ModuleState::Failed)
9047 .then_some(state.failed_retry.pending)
9048 .flatten()
9049 });
9050 let command = if lock_snapshot(&snapshot)
9051 .is_ok_and(|state| state.waiting_for_previous_pid.is_some())
9052 {
9053 tokio::select! {
9054 command = commands.recv() => command,
9055 _ = previous_instance_recheck.tick() => {
9056 if runtime.child_roster.previous_instance_pid(&spec.module_id).is_none() {
9057 let enabled = lock_snapshot(&snapshot).is_ok_and(|state| state.enabled);
9058 if enabled && !runtime.child_roster.is_closed() {
9059 process_liveness.track(spec.module_id.clone(), Arc::clone(&snapshot));
9060 match spawn_and_mark_running_async(&spec, &runtime, &snapshot).await {
9061 Ok(next_child) => child = Some(next_child),
9062 Err(SuperviseError::ConfigDeliveryDeferred { .. }) => {
9063 let _ = defer_config_spawn(&spec, &runtime, &snapshot, RespawnKind::Spawn, &runtime.failure_context(true));
9064 }
9065 Err(SuperviseError::ConfigHeld { .. }) => {
9066 let _ = hold_config_spawn(&spec, &runtime, &snapshot, &runtime.failure_context(true));
9067 }
9068 Err(error) => {
9069 fail_snapshot(&snapshot, Some(&spec.module_id), None, &runtime.failure_context(true));
9070 error!(module_id = %spec.module_id, %error, "failed to start module after its previous instance exited");
9071 }
9072 }
9073 } else {
9074 let _ = update_snapshot(&snapshot, Some(&spec.module_id), |state| {
9075 state.waiting_for_previous_pid = None;
9076 state.waiting_for_previous_detail = None;
9077 });
9078 }
9079 }
9080 continue;
9081 }
9082 }
9083 } else {
9084 tokio::select! {
9085 biased;
9086 command = commands.recv() => command,
9087 _ = async {
9088 match slow_retry {
9089 Some(retry) => sleep_until(retry.deadline).await,
9090 None => std::future::pending().await,
9091 }
9092 } => {
9093 let retry = slow_retry.expect("only a scheduled slow retry can wake");
9094 let _ = update_snapshot(&snapshot, Some(&spec.module_id), |state| {
9095 state.failed_retry.pending = None;
9096 });
9097 if runtime.child_roster.is_closed() {
9098 continue;
9099 }
9100 info!(module_id = %spec.module_id, attempt = retry.attempt,
9101 delay_secs = retry.delay.as_secs(), "starting failed module retry");
9102 if let Err(error) = set_child_enabled(
9103 &spec, &runtime, ®istry, &process_liveness, &snapshot, &mut child, true,
9104 &runtime.failure_context(true),
9105 ).await {
9106 error!(module_id = %spec.module_id, %error, "failed module retry could not start");
9107 }
9108 continue;
9109 }
9110 }
9111 };
9112 let Some(command) = command else {
9113 containment::terminate(&runtime.child_roster.containment).await;
9114 return;
9115 };
9116 if !handle_supervisor_command(
9117 command,
9118 &mut spec,
9119 &mut runtime,
9120 ®istry,
9121 &process_liveness,
9122 &snapshot,
9123 &mut child,
9124 &mut commands,
9125 &mut requeued,
9126 )
9127 .await
9128 {
9129 containment::terminate(&runtime.child_roster.containment).await;
9130 return;
9131 }
9132 }
9133 }
9134}
9135
9136async fn supervisor_sleep(wake_at: Option<Instant>) {
9139 match wake_at {
9140 Some(deadline) => sleep_until(deadline).await,
9141 None => std::future::pending().await,
9142 }
9143}
9144
9145fn log_crash_respawn(module_id: &str, schedule: CrashRestartSchedule) {
9146 info!(
9147 module_id,
9148 restart_in_window = schedule.restart_in_window,
9149 delay_ms = schedule.delay.as_millis() as u64,
9150 "respawning after crash"
9151 );
9152}
9153
9154fn respawn_still_pending(snapshot: &SharedSnapshot) -> bool {
9160 matches!(
9161 lock_snapshot(snapshot),
9162 Ok(state) if state.enabled && state.state == ModuleState::Restarting
9163 )
9164}
9165
9166enum NextAction {
9167 Stop {
9168 registration_released: bool,
9169 },
9170 Restart {
9171 schedule: Option<CrashRestartSchedule>,
9172 },
9173}
9174
9175#[allow(clippy::too_many_arguments)]
9176async fn handle_supervisor_command(
9177 command: SupervisorCommand,
9178 spec: &mut ModuleSpec,
9179 runtime: &mut SupervisorRuntimeConfig,
9180 registry: &Arc<Registry>,
9181 process_liveness: &SupervisorProcessLiveness,
9182 snapshot: &SharedSnapshot,
9183 child: &mut Option<SupervisedChild>,
9184 commands: &mut mpsc::Receiver<SupervisorCommand>,
9185 requeued: &mut VecDeque<SupervisorCommand>,
9186) -> bool {
9187 if let SupervisorCommand::SetEnabled {
9188 enabled: true,
9189 reset_budget: false,
9190 reply,
9191 ..
9192 } = command
9193 {
9194 let parked = lock_snapshot(snapshot).ok().and_then(|state| {
9195 (state.config_hold_parked
9196 && state.enabled
9197 && !state.process_alive
9198 && child.is_none()
9199 && state.state == ModuleState::Stopped)
9200 .then(|| {
9201 (
9202 state.config_hold_daemon_initiated,
9203 state.config_hold_approval.clone(),
9204 )
9205 })
9206 });
9207 let result = if let Some((daemon_initiated, approval)) = parked {
9211 runtime.recovery_is_daemon = daemon_initiated;
9212 let failure = FailureContext {
9213 approval,
9214 ..runtime.failure_context(daemon_initiated)
9215 };
9216 let result = set_child_enabled_with_budget_policy(
9217 spec,
9218 runtime,
9219 registry,
9220 process_liveness,
9221 snapshot,
9222 child,
9223 true,
9224 &failure,
9225 false,
9226 )
9227 .await;
9228 if let Err(error) = &result {
9229 record_approved_attempt_failure(spec, snapshot, &failure, error);
9230 }
9231 result
9232 } else {
9233 Ok(false)
9234 };
9235 let _ = reply.send(result);
9236 return true;
9237 }
9238 let blocked = runtime
9239 .config_delivery
9240 .as_ref()
9241 .and_then(|delivery| delivery.0.check_module_hold(&spec.module_id).err())
9242 .filter(|error| error.code != "config_delivery_deferred")
9243 .map(|error| {
9244 config_delivery::gate_hold_error(spec, error, runtime.config_delivery.as_ref())
9245 })
9246 .or_else(|| recovery_block_error(spec, &runtime.child_roster))
9247 .or_else(|| {
9248 runtime
9249 .child_roster
9250 .previous_instance(&spec.module_id)
9251 .map(|(pid, reason)| previous_instance_error(spec, pid, reason))
9252 });
9253 let command = if let Some(error) = blocked {
9254 match command {
9255 SupervisorCommand::Restart { reply, .. } | SupervisorCommand::Reload { reply, .. } => {
9256 let _ = reply.send(Err(error));
9257 return true;
9258 }
9259 SupervisorCommand::Swap { reply, .. } => {
9260 let _ = reply.send(Err(error));
9261 return true;
9262 }
9263 SupervisorCommand::ConfigApplyRestart { .. } => return true,
9264 SupervisorCommand::SetEnabled {
9265 enabled: true,
9266 requester,
9267 reply,
9268 ..
9269 } => {
9270 if matches!(error, SuperviseError::ConfigHeld { .. })
9271 && lock_snapshot(snapshot).is_ok_and(|state| !state.enabled)
9272 && runtime
9273 .config_delivery
9274 .as_ref()
9275 .is_some_and(|delivery| delivery.0.can_stage_unstamp_enable())
9276 {
9277 let mut changed = false;
9281 let result = update_snapshot(snapshot, Some(&spec.module_id), |state| {
9282 changed = !state.enabled;
9283 state.enabled = true;
9284 if child.is_none() && state.waiting_for_previous_pid.is_none() {
9285 state.state = ModuleState::Starting;
9286 state.failed_retry.cancel();
9287 }
9288 })
9289 .and_then(|_| {
9290 if child.is_none()
9291 && lock_snapshot(snapshot)?.waiting_for_previous_pid.is_none()
9292 {
9293 hold_config_spawn(
9294 spec,
9295 runtime,
9296 snapshot,
9297 &runtime.failure_context(false),
9298 )?;
9299 }
9300 Ok(changed)
9301 });
9302 if result.is_ok() && changed {
9303 runtime.start_requester = requester;
9304 }
9305 let _ = reply.send(result);
9306 return true;
9307 }
9308 let _ = reply.send(Err(error));
9309 return true;
9310 }
9311 other => other,
9312 }
9313 } else {
9314 command
9315 };
9316 if matches!(
9319 command,
9320 SupervisorCommand::Drain { .. }
9321 | SupervisorCommand::Retire { .. }
9322 | SupervisorCommand::SetEnabled { .. }
9323 | SupervisorCommand::Restart { .. }
9324 ) {
9325 let _ = update_snapshot(snapshot, Some(&spec.module_id), |state| {
9326 state.failed_retry.cancel();
9327 });
9328 }
9329 match command {
9330 SupervisorCommand::ApprovedConfiguration { approved, reply } => {
9331 let result = async {
9332 let next = approved.module_entry(&spec.module_id).map_err(|error| {
9333 SuperviseError::InvalidSpec { reason: error.to_string() }
9334 })?;
9335 let retry_approved_failure = {
9336 let mut state = lock_snapshot(snapshot)?;
9337 if let Some((generation, detail)) = &state.approved_start_failure {
9338 if *generation == approved.identity() && next.enabled {
9339 return Err(SuperviseError::Spawn {
9340 program: spec.program.clone(),
9341 source: std::io::Error::other(detail.clone()),
9342 cgroup_path: None,
9343 });
9344 }
9345 }
9346 let retry = state.approved_start_failure.is_some();
9347 state.approved_start_failure = None;
9348 retry
9349 };
9350 stage_supervisor_configuration(
9351 spec, runtime, snapshot, next.module_spec(), next.health,
9352 next.drain_timeout_ms,
9353 );
9354 runtime.restart_policy = next.restart;
9355 runtime.register_timeout = next.register_timeout;
9356 if let Some(active) = child.as_mut()
9357 .filter(|child| child.registration_deadline.is_some())
9358 {
9359 active.registration_deadline = Some(active.spawned_at + next.register_timeout);
9360 }
9361 let enabled = lock_snapshot(snapshot)?.enabled;
9362 let retry_gate_stopped = lock_snapshot(snapshot)?
9369 .needs_approved_gate_restart(&approved.identity());
9370 let retry_failed = enabled && child.is_none()
9371 && ((lock_snapshot(snapshot)?.state == ModuleState::Failed
9372 && (retry_approved_failure || approved.changes_entry(&spec.module_id)))
9373 || retry_gate_stopped);
9374 if retry_failed {
9375 update_snapshot(snapshot, Some(&spec.module_id), |state| {
9378 state.failed_retry.cancel();
9379 if retry_gate_stopped {
9380 state.stop_reason = None;
9381 state.config_exit_failed_approval = None;
9382 }
9383 })?;
9384 }
9385 if enabled != next.enabled || retry_failed {
9386 let failure = FailureContext {
9387 approval: next.enabled.then(|| approved.identity()),
9388 ..runtime.failure_context(false)
9389 };
9390 let held = next.enabled && runtime.config_delivery.as_ref()
9391 .is_some_and(|delivery| delivery.0.has_recorded_admission_hold(&spec.module_id)
9392 || delivery.0.check_module_hold(&spec.module_id).is_err_and(|error| error.code == "config_delivery_deferred"));
9393 if held {
9394 reset_restart_count(snapshot, &spec.module_id)?;
9397 update_snapshot(snapshot, Some(&spec.module_id), |state| {
9398 state.enabled = true;
9399 if child.is_none() && state.waiting_for_previous_pid.is_none() {
9400 state.state = ModuleState::Starting;
9401 }
9402 })?;
9403 if child.is_none() && lock_snapshot(snapshot)?.waiting_for_previous_pid.is_none() {
9404 hold_config_spawn(spec, runtime, snapshot, &failure)?;
9405 }
9406 } else {
9407 match set_child_enabled_with_budget_policy(
9408 spec, runtime, registry, process_liveness, snapshot, child,
9409 next.enabled, &failure, true,
9410 ).await {
9411 Ok(_) => {},
9412 Err(SuperviseError::ConfigHeld { .. }) if next.enabled => {},
9413 Err(error) => return Err(error),
9414 }
9415 if !next.enabled {
9416 containment::terminate(&runtime.child_roster.containment).await;
9417 }
9418 }
9419 }
9420 if (child.as_ref().is_some_and(|child| child.registration_spawn.is_some() && child.admission_base.is_none())
9421 || (child.is_none() && lock_snapshot(snapshot)?.state == ModuleState::Restarting
9422 && runtime.config_delivery.as_ref().is_some_and(|delivery| delivery.0.stamp_channel_cutover_pending())))
9423 && runtime.config_delivery.as_ref().is_some_and(|delivery| delivery.0.module_is_gated(&spec.module_id))
9424 {
9425 let failure = FailureContext {
9426 approval: Some(approved.identity()),
9427 ..runtime.failure_context(false)
9428 };
9429 if child.is_some() {
9433 restart_child_with_budget_policy(spec, runtime, registry,
9434 process_liveness, snapshot, child, runtime.drain_timeout, false).await?;
9435 }
9436 update_snapshot(snapshot, Some(&spec.module_id), |state| state.respawn_pending = false)?;
9439 match spawn_and_mark_running_async(spec, runtime, snapshot).await {
9440 Ok(replacement) => *child = Some(replacement),
9441 Err(SuperviseError::ConfigDeliveryDeferred { .. }) => {
9442 defer_config_spawn(spec, runtime, snapshot, RespawnKind::Spawn, &failure)?;
9443 }
9444 Err(SuperviseError::ConfigHeld { .. }) => hold_config_spawn(spec, runtime, snapshot, &failure)?,
9445 Err(error) => {
9446 fail_snapshot(snapshot, Some(&spec.module_id), None, &failure);
9447 return Err(error);
9448 }
9449 }
9450 }
9451 info!(module_id=%spec.module_id, config_generation=approved.generation(), "approved daemon entry reconciled");
9452 Ok(())
9453 }
9454 .await;
9455 if let Err(error) = &result {
9456 let _ = update_snapshot(snapshot, Some(&spec.module_id), |state| {
9457 if state.enabled
9458 && state.state == ModuleState::Failed
9459 && state
9460 .approved_start_failure
9461 .as_ref()
9462 .is_none_or(|(generation, _)| *generation != approved.identity())
9463 {
9464 state.approved_start_failure =
9465 Some((approved.identity(), error.to_string()));
9466 }
9467 });
9468 }
9469 let _ = reply.send(result);
9470 true
9471 }
9472 SupervisorCommand::ConfigPolicyProbe {
9473 connection,
9474 generation,
9475 attempt,
9476 } => {
9477 let Some(delivery) = &runtime.config_delivery else {
9478 return true;
9479 };
9480 let Ok(gate) = delivery.0.module_gate(&spec.module_id) else {
9481 return true;
9482 };
9483 let guard = gate.lock().await;
9484 let pending = delivery
9485 .0
9486 .on_blocking(move |owner| {
9487 owner.policy_probe_pending(connection, generation, attempt)
9488 })
9489 .await
9490 .unwrap_or(false)
9491 && child.is_some()
9492 && registry
9493 .get_module(&spec.module_id)
9494 .ok()
9495 .flatten()
9496 .is_some_and(|registration| registration.connection_id == connection);
9497 drop(guard);
9498 if pending {
9499 run_health_probe_cycle(spec, runtime, registry, process_liveness, snapshot, child)
9500 .await;
9501 }
9502 true
9503 }
9504 SupervisorCommand::ConfigApplyRestart {
9505 generation,
9506 supersedes_swap,
9507 } => {
9508 let Some(owner) = runtime
9509 .config_delivery
9510 .as_ref()
9511 .map(|delivery| delivery.0.clone())
9512 else {
9513 return true;
9514 };
9515 let Ok(gate) = owner.module_gate(&spec.module_id) else {
9516 return true;
9517 };
9518 let _guard = gate.lock().await;
9519 let spawn = child
9520 .as_ref()
9521 .and_then(|child| child.config_delivery.clone());
9522 let module = spec.module_id.clone();
9523 let current = owner
9524 .on_blocking(move |owner| {
9525 Ok((supersedes_swap
9526 || owner
9527 .pending_config_restarts()?
9528 .contains(&(module.clone(), generation)))
9529 && owner.requires_fresher_spawn(&module, spawn.as_ref())?
9530 == Some(generation))
9531 })
9532 .await
9533 .unwrap_or(false);
9534 if !current
9535 || !lock_snapshot(snapshot).is_ok_and(|state| {
9536 state.enabled && !state.respawn_pending && state.process_alive
9537 })
9538 {
9539 return true;
9540 }
9541 drop(_guard);
9542 info!(module_id=%spec.module_id, config_generation=generation, "approved configuration restart preserves the crash budget");
9543 if let Err(error) = restart_child_with_budget_policy(
9544 spec,
9545 runtime,
9546 registry,
9547 process_liveness,
9548 snapshot,
9549 child,
9550 runtime.drain_timeout,
9551 false,
9552 )
9553 .await
9554 {
9555 warn!(module_id=%spec.module_id, %error, "approved configuration restart could not be scheduled");
9556 }
9557 true
9558 }
9559 SupervisorCommand::ConfigExitHold { reply } => {
9560 let idle_start = child.is_none()
9564 && lock_snapshot(snapshot).is_ok_and(|state| {
9565 state.enabled
9566 && state.state == ModuleState::Starting
9567 && !state.process_alive
9568 && !state.respawn_pending
9569 && state.waiting_for_previous_pid.is_none()
9570 && state.waiting_for_previous_detail.is_none()
9571 && state.failed_retry.pending.is_none()
9572 })
9573 && runtime
9574 .scheduled_respawn
9575 .lock()
9576 .unwrap_or_else(|p| p.into_inner())
9577 .is_none()
9578 && runtime
9579 .deferred_reload_reply
9580 .lock()
9581 .unwrap_or_else(|p| p.into_inner())
9582 .is_none();
9583 let result = if idle_start {
9584 try_park_config_attempt(
9585 spec,
9586 runtime,
9587 snapshot,
9588 &runtime.failure_context(runtime.recovery_is_daemon),
9589 None,
9590 )
9591 .map(|_| ())
9592 } else {
9593 Ok(())
9594 };
9595 let _ = reply.send(result);
9596 true
9597 }
9598 SupervisorCommand::ConfigDeliveryFailed { connection, hold } => {
9599 if registry
9600 .get_module(&spec.module_id)
9601 .ok()
9602 .flatten()
9603 .is_none_or(|registration| registration.connection_id != connection)
9604 {
9605 return true;
9606 }
9607 runtime.recovery_is_daemon = true;
9608 if hold {
9612 let notice = begin_forwarding_drain_if_configured(
9613 spec,
9614 runtime,
9615 registry,
9616 snapshot,
9617 Some(true),
9618 RouteCloseReason::Disable,
9619 )
9620 .await;
9621 if let Ok(notice) = notice {
9622 let drained = drain_optional_child(
9623 &spec.module_id,
9624 spec.protocol,
9625 notice,
9626 registry,
9627 runtime.forwarding.as_deref(),
9628 snapshot,
9629 &runtime.terminal_ring,
9630 &runtime.spawn_events,
9631 child,
9632 runtime.drain_timeout,
9633 ModuleState::Failed,
9634 Some(true),
9635 &runtime.failure_context(true),
9636 )
9637 .await;
9638 if drained.is_ok() {
9639 match try_park_config_attempt(
9640 spec,
9641 runtime,
9642 snapshot,
9643 &runtime.failure_context(true),
9644 None,
9645 ) {
9646 Ok(true) => {}
9647 Ok(false) => {
9648 if let Err(error) = budgeted_restart_child(
9649 spec,
9650 runtime,
9651 registry,
9652 process_liveness,
9653 snapshot,
9654 child,
9655 SupervisorHealthStatus::Unknown,
9656 Some("configuration delivery failed after its hold ended"),
9657 unix_ms_now(),
9658 false,
9659 )
9660 .await
9661 {
9662 warn!(module_id=%spec.module_id, %error, "configuration failure restart could not be scheduled");
9663 }
9664 }
9665 Err(error) => {
9666 warn!(module_id=%spec.module_id, %error, "configuration failure could not be parked")
9667 }
9668 }
9669 }
9670 }
9671 } else if let Err(error) = budgeted_restart_child(
9672 spec,
9673 runtime,
9674 registry,
9675 process_liveness,
9676 snapshot,
9677 child,
9678 SupervisorHealthStatus::Unknown,
9679 Some("configuration delivery failed"),
9680 unix_ms_now(),
9681 false,
9682 )
9683 .await
9684 {
9685 warn!(module_id=%spec.module_id, %error, "configuration failure restart could not be scheduled");
9686 }
9687 true
9688 }
9689 SupervisorCommand::Drain { requester, reply } => {
9690 let result = drain_optional_child(
9693 &spec.module_id,
9694 spec.protocol,
9695 StopNotice::NotSent,
9696 registry,
9697 runtime.forwarding.as_deref(),
9698 snapshot,
9699 &runtime.terminal_ring,
9700 &runtime.spawn_events,
9701 child,
9702 runtime.drain_timeout,
9703 ModuleState::Stopped,
9704 None,
9705 &FailureContext::operator(),
9706 )
9707 .await;
9708 containment::terminate(&runtime.child_roster.containment).await;
9709 let registration_released = result.is_ok();
9710 if registration_released {
9711 runtime.record_activity_transition(
9712 "subc.module_stopped",
9713 &spec.module_id,
9714 requester,
9715 );
9716 }
9717 let _ = reply.send(result);
9718 if registration_released {
9719 process_liveness.untrack_if_current(&spec.module_id, snapshot);
9720 }
9721 false
9722 }
9723 SupervisorCommand::Retire { requester, reply } => {
9724 let result = async {
9725 let stop_notice = begin_forwarding_drain_if_configured(
9726 spec,
9727 runtime,
9728 registry,
9729 snapshot,
9730 None,
9731 RouteCloseReason::Disable,
9732 )
9733 .await?;
9734 drain_optional_child(
9735 &spec.module_id,
9736 spec.protocol,
9737 stop_notice,
9738 registry,
9739 runtime.forwarding.as_deref(),
9740 snapshot,
9741 &runtime.terminal_ring,
9742 &runtime.spawn_events,
9743 child,
9744 runtime.drain_timeout,
9745 ModuleState::Stopped,
9746 None,
9747 &FailureContext::operator(),
9748 )
9749 .await
9750 }
9751 .await;
9752 containment::terminate(&runtime.child_roster.containment).await;
9753 let registration_released = result.is_ok();
9754 if registration_released {
9755 runtime.record_activity_transition(
9756 "subc.module_stopped",
9757 &spec.module_id,
9758 requester,
9759 );
9760 }
9761 let _ = reply.send(result);
9762 if registration_released {
9763 process_liveness.untrack_if_current(&spec.module_id, snapshot);
9764 }
9765 false
9766 }
9767 SupervisorCommand::Restart {
9768 requester,
9769 drain_timeout_ms,
9770 received_at_generation,
9771 queued_at,
9772 reply,
9773 } => {
9774 info!(
9778 module_id = %spec.module_id,
9779 queued_ms = u64::try_from(queued_at.elapsed().as_millis()).unwrap_or(u64::MAX),
9780 "restart command dequeued"
9781 );
9782 let validation = match lock_snapshot(snapshot) {
9794 Ok(state) if !state.enabled => Err(SuperviseError::Disabled {
9795 module_id: spec.module_id.clone(),
9796 }),
9797 Ok(_) => Ok(()),
9798 Err(err) => Err(err),
9799 };
9800 let initiated = validation.is_ok();
9801 let _ = reply.send(validation);
9802 let satisfied_by_generation = if initiated
9816 && child
9817 .as_ref()
9818 .is_some_and(|child| child.spawned_at > queued_at)
9819 {
9820 lock_snapshot(snapshot).ok().and_then(|state| {
9821 (!state.configuration_updated_since_spawn).then_some(state.spawn_generation)
9822 })
9823 } else {
9824 None
9825 };
9826 let satisfied_by_pending = initiated
9827 && child.is_none()
9828 && lock_snapshot(snapshot).ok().is_some_and(|mut state| {
9829 let pending = state.respawn_pending && !state.configuration_updated_since_spawn;
9833 if pending {
9834 state.coalesced_restart_pending = true;
9835 }
9836 pending
9837 });
9838 if satisfied_by_pending {
9839 debug!(module_id = %spec.module_id, "restart coalesced into the pending replacement");
9840 } else if let Some(generation) = satisfied_by_generation {
9841 info!(
9842 module_id = %spec.module_id,
9843 received_at_generation,
9844 "restart already satisfied by generation {generation}; not restarting again"
9845 );
9846 } else if initiated {
9847 let drain_timeout = drain_timeout_ms
9850 .map(Duration::from_millis)
9851 .unwrap_or(runtime.drain_timeout);
9852 if let Err(err) = restart_child(
9853 spec,
9854 runtime,
9855 registry,
9856 process_liveness,
9857 snapshot,
9858 child,
9859 drain_timeout,
9860 )
9861 .await
9862 {
9863 warn!(
9864 module_id = %spec.module_id,
9865 error = %err,
9866 "operator restart failed after initiation ack; module state carries the outcome"
9867 );
9868 let _ = update_snapshot(snapshot, Some(&spec.module_id), |state| {
9869 state.state = ModuleState::Failed;
9870 clear_current_process_facts(state);
9871 });
9872 } else {
9873 runtime.start_requester = requester;
9874 runtime.pending_activity_restart = true;
9875 }
9876 }
9877 true
9878 }
9879 SupervisorCommand::Reload { requester, reply } => {
9880 let result =
9881 reload_child(spec, runtime, registry, process_liveness, snapshot, child).await;
9882 if result.is_ok() {
9883 runtime.start_requester = requester;
9884 runtime.pending_activity_restart = true;
9885 }
9886 if result.is_ok()
9887 && runtime
9888 .scheduled_respawn
9889 .lock()
9890 .unwrap_or_else(|p| p.into_inner())
9891 .as_ref()
9892 .is_some_and(|pending| matches!(pending.kind, RespawnKind::Reload))
9893 {
9894 *runtime
9895 .deferred_reload_reply
9896 .lock()
9897 .unwrap_or_else(|p| p.into_inner()) = Some(reply);
9898 } else {
9899 let _ = reply.send(result);
9900 }
9901 true
9902 }
9903 SupervisorCommand::SetEnabled {
9904 enabled,
9905 requester,
9906 reply,
9907 reset_budget,
9908 } => {
9909 let previous_requester = runtime.start_requester.clone();
9910 if enabled {
9911 runtime.start_requester = requester.clone();
9912 }
9913 let result = set_child_enabled_with_budget_policy(
9914 spec,
9915 runtime,
9916 registry,
9917 process_liveness,
9918 snapshot,
9919 child,
9920 enabled,
9921 &runtime.failure_context(false),
9922 reset_budget,
9923 )
9924 .await;
9925 if !enabled {
9926 containment::terminate(&runtime.child_roster.containment).await;
9927 }
9928 if matches!(result, Ok(true)) {
9929 runtime.pending_activity_restart = false;
9930 if enabled {
9931 runtime.start_requester = requester;
9932 } else {
9933 runtime.start_requester = None;
9934 runtime.record_activity_transition(
9935 "subc.module_disabled",
9936 &spec.module_id,
9937 requester,
9938 );
9939 }
9940 } else {
9941 runtime.start_requester = previous_requester;
9942 }
9943 let _ = reply.send(result);
9944 true
9945 }
9946 SupervisorCommand::UpdateConfiguration {
9947 spec: next_spec,
9948 health,
9949 drain_timeout_ms,
9950 reply,
9951 } => {
9952 if runtime
9953 .config_delivery
9954 .as_ref()
9955 .is_some_and(|delivery| delivery.0.file_configuration_is_gated(&spec.module_id))
9956 {
9957 let _ = reply.send(Err(SuperviseError::ConfigHeld {
9958 module_id: spec.module_id.clone(),
9959 code: "config_gated".into(),
9960 detail: "live file reconfiguration cannot replace stamped authority".into(),
9961 }));
9962 return true;
9963 }
9964 stage_supervisor_configuration(
9965 spec,
9966 runtime,
9967 snapshot,
9968 next_spec,
9969 health,
9970 drain_timeout_ms,
9971 );
9972 let _ = update_snapshot(snapshot, Some(&spec.module_id), |state| {
9973 state.configuration_updated_since_spawn = true;
9974 });
9975 let _ = reply.send(Ok(()));
9976 true
9977 }
9978 SupervisorCommand::Swap {
9979 requester,
9980 ready_timeout,
9981 reply,
9982 } => {
9983 if spec.carry_containment
9984 || containment::current(&runtime.child_roster.containment).is_some()
9985 {
9986 let _ = reply.send(Err(SuperviseError::InvalidSpec {
9987 reason: "a carried container cannot overlap a swap candidate".to_owned(),
9988 }));
9989 return true;
9990 }
9991 let mut swap_runtime = runtime.clone();
9992 swap_runtime.start_requester = requester;
9993 let before = lock_snapshot(snapshot)
9994 .ok()
9995 .map(|state| state.spawn_generation);
9996 let end = swap::run_swap(
9997 spec,
9998 &swap_runtime,
9999 registry,
10000 process_liveness,
10001 snapshot,
10002 child,
10003 commands,
10004 ready_timeout.unwrap_or(DEFAULT_SWAP_READY_TIMEOUT),
10005 reply,
10006 )
10007 .await;
10008 if before
10009 != lock_snapshot(snapshot)
10010 .ok()
10011 .map(|state| state.spawn_generation)
10012 {
10013 runtime.start_requester = swap_runtime.start_requester;
10014 }
10015 requeued.extend(end.requeue);
10016 true
10017 }
10018 }
10019}
10020
10021fn stage_supervisor_configuration(
10022 spec: &mut ModuleSpec,
10023 runtime: &mut SupervisorRuntimeConfig,
10024 snapshot: &SharedSnapshot,
10025 next_spec: ModuleSpec,
10026 health: HealthConfig,
10027 drain_timeout_ms: Option<u64>,
10028) {
10029 if let Some(handle) = &runtime.supervisor_handle {
10030 handle.apply_identity_configuration(&next_spec);
10031 }
10032 let launch_changed = *spec != next_spec;
10033 *spec = next_spec;
10034 if launch_changed {
10035 let _ = update_snapshot(snapshot, Some(&spec.module_id), |state| {
10036 state.configuration_updated_since_spawn = true
10037 });
10038 }
10039 let health_changed = runtime.health != health;
10040 runtime.health = health;
10041 if health_changed {
10044 let _ = update_snapshot(snapshot, Some(&spec.module_id), |state| {
10045 state.health = ModuleHealthStatus::default();
10046 state.last_probe_evidence = None;
10047 });
10048 }
10049 runtime.drain_timeout = drain_timeout_ms
10050 .map(Duration::from_millis)
10051 .unwrap_or(runtime.default_drain_timeout);
10052 *runtime
10053 .effective_drain_timeout
10054 .lock()
10055 .unwrap_or_else(|p| p.into_inner()) = runtime.drain_timeout;
10056}
10057
10058async fn restart_child(
10059 spec: &ModuleSpec,
10060 runtime: &SupervisorRuntimeConfig,
10061 registry: &Registry,
10062 process_liveness: &SupervisorProcessLiveness,
10063 snapshot: &SharedSnapshot,
10064 child: &mut Option<SupervisedChild>,
10065 drain_timeout: Duration,
10066) -> Result<(), SuperviseError> {
10067 restart_child_with_budget_policy(
10068 spec,
10069 runtime,
10070 registry,
10071 process_liveness,
10072 snapshot,
10073 child,
10074 drain_timeout,
10075 true,
10076 )
10077 .await
10078}
10079
10080#[allow(clippy::too_many_arguments)]
10081async fn restart_child_with_budget_policy(
10082 spec: &ModuleSpec,
10083 runtime: &SupervisorRuntimeConfig,
10084 registry: &Registry,
10085 process_liveness: &SupervisorProcessLiveness,
10086 snapshot: &SharedSnapshot,
10087 child: &mut Option<SupervisedChild>,
10088 drain_timeout: Duration,
10089 reset_budget: bool,
10090) -> Result<(), SuperviseError> {
10091 if !lock_snapshot(snapshot)?.enabled {
10093 return Err(SuperviseError::Disabled {
10094 module_id: spec.module_id.clone(),
10095 });
10096 }
10097 let stop_notice = begin_forwarding_drain_with_timeout(
10098 spec,
10099 runtime,
10100 registry,
10101 snapshot,
10102 None,
10103 RouteCloseReason::Restart,
10104 drain_timeout,
10105 )
10106 .await?;
10107
10108 if child.is_some() {
10109 drain_optional_child(
10110 &spec.module_id,
10111 spec.protocol,
10112 stop_notice,
10113 registry,
10114 runtime.forwarding.as_deref(),
10115 snapshot,
10116 &runtime.terminal_ring,
10117 &runtime.spawn_events,
10118 child,
10119 drain_timeout,
10120 ModuleState::Restarting,
10121 Some(true),
10122 &FailureContext::operator(),
10123 )
10124 .await?;
10125 } else {
10126 update_snapshot(snapshot, Some(&spec.module_id), |state| {
10127 state.enabled = true;
10128 state.state = ModuleState::Restarting;
10129 clear_current_process_facts(state);
10130 })?;
10131 release_dead_registration(
10132 registry,
10133 runtime.forwarding.as_deref(),
10134 snapshot,
10135 &spec.module_id,
10136 &runtime.failure_context(false),
10137 )
10138 .await?;
10139 }
10140
10141 if reset_budget {
10142 reset_restart_count(snapshot, &spec.module_id)?;
10143 }
10144 process_liveness.track(spec.module_id.clone(), Arc::clone(snapshot));
10145 schedule_respawn(
10146 runtime,
10147 snapshot,
10148 &spec.module_id,
10149 runtime.restart_policy.backoff,
10150 RespawnKind::Spawn,
10151 false,
10152 )
10153}
10154
10155async fn reload_child(
10156 spec: &ModuleSpec,
10157 runtime: &SupervisorRuntimeConfig,
10158 registry: &Registry,
10159 process_liveness: &SupervisorProcessLiveness,
10160 snapshot: &SharedSnapshot,
10161 child: &mut Option<SupervisedChild>,
10162) -> Result<(), SuperviseError> {
10163 if !lock_snapshot(snapshot)?.enabled {
10165 return Err(SuperviseError::Disabled {
10166 module_id: spec.module_id.clone(),
10167 });
10168 }
10169 let stop_notice = begin_forwarding_drain(
10170 spec,
10171 runtime,
10172 registry,
10173 snapshot,
10174 Some(true),
10175 RouteCloseReason::Reload,
10176 )
10177 .await?;
10178
10179 if child.is_some() {
10180 drain_optional_child(
10181 &spec.module_id,
10182 spec.protocol,
10183 stop_notice,
10184 registry,
10185 runtime.forwarding.as_deref(),
10186 snapshot,
10187 &runtime.terminal_ring,
10188 &runtime.spawn_events,
10189 child,
10190 runtime.drain_timeout,
10191 ModuleState::Restarting,
10192 Some(true),
10193 &FailureContext::operator(),
10194 )
10195 .await?;
10196 } else {
10197 update_snapshot(snapshot, Some(&spec.module_id), |state| {
10198 state.enabled = true;
10199 state.state = ModuleState::Restarting;
10200 clear_current_process_facts(state);
10201 })?;
10202 release_dead_registration(
10203 registry,
10204 runtime.forwarding.as_deref(),
10205 snapshot,
10206 &spec.module_id,
10207 &runtime.failure_context(false),
10208 )
10209 .await?;
10210 }
10211
10212 reset_restart_count(snapshot, &spec.module_id)?;
10213 process_liveness.track(spec.module_id.clone(), Arc::clone(snapshot));
10214 schedule_respawn(
10215 runtime,
10216 snapshot,
10217 &spec.module_id,
10218 runtime.restart_policy.backoff,
10219 RespawnKind::Reload,
10220 false,
10221 )
10222}
10223
10224async fn finish_reload_child(
10225 spec: &ModuleSpec,
10226 runtime: &SupervisorRuntimeConfig,
10227 registry: &Registry,
10228 process_liveness: &SupervisorProcessLiveness,
10229 snapshot: &SharedSnapshot,
10230 child: &mut Option<SupervisedChild>,
10231) -> Result<(), SuperviseError> {
10232 process_liveness.track(spec.module_id.clone(), Arc::clone(snapshot));
10233 let next_child = match spawn_and_mark_running_async(spec, runtime, snapshot).await {
10234 Ok(next_child) => next_child,
10235 Err(error @ SuperviseError::ConfigDeliveryDeferred { .. }) => {
10236 defer_config_spawn(
10237 spec,
10238 runtime,
10239 snapshot,
10240 RespawnKind::Reload,
10241 &runtime.failure_context(runtime.recovery_is_daemon),
10242 )?;
10243 return Err(error);
10244 }
10245 Err(error @ SuperviseError::ConfigHeld { .. }) => {
10246 let failure = runtime.failure_context(runtime.recovery_is_daemon);
10247 if try_park_config_attempt(spec, runtime, snapshot, &failure, None)? {
10248 return Err(error);
10249 }
10250 defer_config_spawn(spec, runtime, snapshot, RespawnKind::Reload, &failure)?;
10251 return Err(SuperviseError::ConfigDeliveryDeferred {
10252 module_id: spec.module_id.clone(),
10253 });
10254 }
10255 Err(err) => {
10256 return handle_reload_spawn_failure(
10257 spec,
10258 runtime,
10259 process_liveness,
10260 snapshot,
10261 child,
10262 format!("new child failed to spawn: {err}"),
10263 )
10264 .await;
10265 }
10266 };
10267 *child = Some(next_child);
10268
10269 let wait_outcome = {
10270 let active_child = child.as_mut().expect("new reload child was just stored");
10271 wait_for_registration_after_reload(
10272 registry,
10273 &spec.module_id,
10274 snapshot,
10275 active_child,
10276 REGISTRY_RELEASE_TIMEOUT,
10277 )
10278 .await?
10279 };
10280
10281 match wait_outcome {
10282 RegistrationWaitOutcome::Registered => {
10283 child
10284 .as_mut()
10285 .expect("registered reload child")
10286 .registration_deadline = None;
10287 debug!(module_id = %spec.module_id, "supervised module reloaded and registered");
10288 Ok(())
10289 }
10290 RegistrationWaitOutcome::Exited(exit_report) => {
10291 if let Some(active_child) = child.as_mut() {
10292 active_child.drain_stderr(&spec.module_id).await;
10293 }
10294 let mut exited_child = child.take().expect("exited reload child is still stored");
10297 #[cfg(test)]
10298 if let Some(gate) = &runtime.test_reload_exit_record_gate {
10299 gate.reached.notify_one();
10300 gate.resume.notified().await;
10301 }
10302 let result = handle_reload_child_registration_failure(
10303 spec,
10304 runtime,
10305 registry,
10306 process_liveness,
10307 snapshot,
10308 child,
10309 ReloadRegistrationFailure {
10310 exit_report: registration_failure_exit_report(exit_report),
10311 reason: exited_child
10312 .spawn_failure
10313 .clone()
10314 .unwrap_or_else(|| "new child exited before registering".to_string()),
10315 },
10316 )
10317 .await;
10318 exited_child
10319 .finish_exit(false)
10320 .await
10321 .map_err(|source| SuperviseError::Wait {
10322 module_id: spec.module_id.clone(),
10323 source,
10324 })?;
10325 result
10326 }
10327 RegistrationWaitOutcome::TimedOut => {
10328 let mut timed_out_child = child
10329 .take()
10330 .expect("timed-out reload child is still running");
10331 timed_out_child
10332 .start_kill()
10333 .map_err(|source| SuperviseError::Kill {
10334 module_id: spec.module_id.clone(),
10335 source,
10336 })?;
10337 let status = timed_out_child
10338 .wait()
10339 .await
10340 .map_err(|source| SuperviseError::Wait {
10341 module_id: spec.module_id.clone(),
10342 source,
10343 })?;
10344 timed_out_child.drain_stderr(&spec.module_id).await;
10345 let result = handle_reload_child_registration_failure(
10346 spec,
10347 runtime,
10348 registry,
10349 process_liveness,
10350 snapshot,
10351 child,
10352 ReloadRegistrationFailure {
10353 exit_report: registration_failure_exit_report(classify_reaped_child_exit(
10354 snapshot,
10355 &timed_out_child,
10356 &status,
10357 )),
10358 reason: format!(
10359 "new child did not register within {:?}",
10360 REGISTRY_RELEASE_TIMEOUT
10361 ),
10362 },
10363 )
10364 .await;
10365 timed_out_child
10366 .finish_exit(false)
10367 .await
10368 .map_err(|source| SuperviseError::Wait {
10369 module_id: spec.module_id.clone(),
10370 source,
10371 })?;
10372 result
10373 }
10374 }
10375}
10376
10377#[allow(clippy::too_many_arguments)]
10378async fn set_child_enabled(
10379 spec: &ModuleSpec,
10380 runtime: &SupervisorRuntimeConfig,
10381 registry: &Registry,
10382 process_liveness: &SupervisorProcessLiveness,
10383 snapshot: &SharedSnapshot,
10384 child: &mut Option<SupervisedChild>,
10385 enabled: bool,
10386 failure: &FailureContext,
10387) -> Result<bool, SuperviseError> {
10388 set_child_enabled_with_budget_policy(
10389 spec,
10390 runtime,
10391 registry,
10392 process_liveness,
10393 snapshot,
10394 child,
10395 enabled,
10396 failure,
10397 true,
10398 )
10399 .await
10400}
10401
10402#[allow(clippy::too_many_arguments)]
10403async fn set_child_enabled_with_budget_policy(
10404 spec: &ModuleSpec,
10405 runtime: &SupervisorRuntimeConfig,
10406 registry: &Registry,
10407 process_liveness: &SupervisorProcessLiveness,
10408 snapshot: &SharedSnapshot,
10409 child: &mut Option<SupervisedChild>,
10410 enabled: bool,
10411 failure: &FailureContext,
10412 reset_budget: bool,
10413) -> Result<bool, SuperviseError> {
10414 let (current_enabled, current_state, respawn_pending) = {
10415 let state = lock_snapshot(snapshot)?;
10416 (state.enabled, state.state, state.respawn_pending)
10417 };
10418 let revive_terminal = enabled
10426 && current_enabled
10427 && child.is_none()
10428 && (matches!(current_state, ModuleState::Failed | ModuleState::Stopped)
10429 || (current_state == ModuleState::Restarting && !respawn_pending));
10430 if current_enabled == enabled && !revive_terminal {
10431 return Ok(false);
10432 }
10433
10434 if enabled {
10435 update_snapshot(snapshot, Some(&spec.module_id), |state| {
10436 state.enabled = true;
10437 state.state = ModuleState::Starting;
10438 clear_current_process_facts(state);
10439 })?;
10440 #[cfg(test)]
10441 if runtime.test_seed_stale_facts_before_enable_spawn {
10442 update_snapshot(snapshot, Some(&spec.module_id), |state| {
10443 state.process_alive = true;
10444 state.pid = Some(41);
10445 state.spawned_at_ms = Some(42);
10446 state.spawned_from = Some(PathBuf::from("/spawned/module"));
10447 state.spawned_file_identity = Some(SpawnedFileIdentity {
10448 device: 43,
10449 inode: 44,
10450 #[cfg(windows)]
10451 windows_image: None,
10452 });
10453 })?;
10454 }
10455 release_dead_registration(
10456 registry,
10457 runtime.forwarding.as_deref(),
10458 snapshot,
10459 &spec.module_id,
10460 failure,
10461 )
10462 .await?;
10463 if reset_budget {
10464 reset_restart_count(snapshot, &spec.module_id)?;
10465 }
10466 process_liveness.track(spec.module_id.clone(), Arc::clone(snapshot));
10467 let next_child = match spawn_and_mark_running_async(spec, runtime, snapshot).await {
10468 Ok(next_child) => next_child,
10469 Err(SuperviseError::ConfigDeliveryDeferred { .. }) => {
10470 defer_config_spawn(spec, runtime, snapshot, RespawnKind::Spawn, failure)?;
10471 return Ok(true);
10472 }
10473 Err(error @ SuperviseError::ConfigHeld { .. }) => {
10474 if let Err(parking_error) = hold_config_spawn(spec, runtime, snapshot, failure) {
10475 process_liveness.untrack_if_current(&spec.module_id, snapshot);
10476 return Err(parking_error);
10477 }
10478 return Err(error);
10479 }
10480 Err(err) => {
10481 fail_snapshot(snapshot, Some(&spec.module_id), None, failure);
10482 process_liveness.untrack_if_current(&spec.module_id, snapshot);
10483 return Err(err);
10484 }
10485 };
10486 *child = Some(next_child);
10487 debug!(module_id = %spec.module_id, "supervised module enabled");
10488 Ok(true)
10489 } else {
10490 update_snapshot(snapshot, Some(&spec.module_id), |state| {
10491 state.config_hold_parked = false;
10492 state.config_hold_daemon_initiated = false;
10493 state.config_hold_approval = None;
10494 })?;
10495 let stop_notice = begin_forwarding_drain_if_configured(
10496 spec,
10497 runtime,
10498 registry,
10499 snapshot,
10500 Some(false),
10501 RouteCloseReason::Disable,
10502 )
10503 .await?;
10504 drain_optional_child(
10505 &spec.module_id,
10506 spec.protocol,
10507 stop_notice,
10508 registry,
10509 runtime.forwarding.as_deref(),
10510 snapshot,
10511 &runtime.terminal_ring,
10512 &runtime.spawn_events,
10513 child,
10514 runtime.drain_timeout,
10515 ModuleState::Disabled,
10516 Some(false),
10517 failure,
10518 )
10519 .await?;
10520 debug!(module_id = %spec.module_id, "supervised module disabled");
10521 Ok(true)
10522 }
10523}
10524
10525#[allow(clippy::too_many_arguments)]
10526async fn on_child_exit(
10527 spec: &ModuleSpec,
10528 policy: RestartPolicy,
10529 registry: &Registry,
10530 snapshot: &SharedSnapshot,
10531 terminal_ring: &Arc<Mutex<TerminalRing>>,
10532 spawn_events: &SpawnEventFeed,
10533 roster: &ChildRoster,
10534 exit_report: ExitReport,
10535 daemon_initiated: bool,
10536) -> NextAction {
10537 if roster.is_closed() {
10543 return on_child_exit_during_daemon_shutdown(
10544 spec,
10545 registry,
10546 snapshot,
10547 terminal_ring,
10548 spawn_events,
10549 exit_report,
10550 )
10551 .await;
10552 }
10553 let unrequested_clean_exit_of_protocol_none = exit_report.kind == ExitKind::Clean
10569 && running_protocol(spec, snapshot) == ModuleProtocol::None;
10570 match exit_report.kind {
10571 ExitKind::Clean if !unrequested_clean_exit_of_protocol_none => {
10572 info!(
10573 module_id = %spec.module_id,
10574 exit_code = ?exit_report.code,
10575 exit_signal = ?exit_report.signal,
10576 "supervised module exited cleanly"
10577 );
10578 if let Err(err) = publish_after_terminal_record(
10579 &spec.module_id,
10580 || {
10581 record_terminal(
10582 &spec.module_id,
10583 terminal_ring,
10584 spawn_events,
10585 &exit_report,
10586 TerminalDisposition::Stopped,
10587 )
10588 },
10589 || {
10590 update_snapshot(snapshot, Some(&spec.module_id), |state| {
10591 state.state = ModuleState::Stopped;
10592 clear_current_process_facts(state);
10593 state.last_exit = Some(exit_report.clone());
10594 })
10595 },
10596 ) {
10597 error!(module_id = %spec.module_id, error = %err, "failed to publish clean module exit");
10598 }
10599 let registration_released = match wait_for_registration_release(
10600 registry,
10601 &spec.module_id,
10602 REGISTRY_RELEASE_TIMEOUT,
10603 )
10604 .await
10605 {
10606 Ok(()) => true,
10607 Err(err) => {
10608 warn!(module_id = %spec.module_id, error = %err, "registration still active after clean exit");
10609 false
10610 }
10611 };
10612 NextAction::Stop {
10613 registration_released,
10614 }
10615 }
10616 ExitKind::Clean | ExitKind::Crash => {
10617 if unrequested_clean_exit_of_protocol_none {
10618 warn!(
10619 module_id = %spec.module_id,
10620 exit_code = ?exit_report.code,
10621 exit_signal = ?exit_report.signal,
10622 "protocol-none module exited cleanly without a stop request; handling it as a crash"
10623 );
10624 } else {
10625 warn!(
10626 module_id = %spec.module_id,
10627 exit_code = ?exit_report.code,
10628 exit_signal = ?exit_report.signal,
10629 "supervised module exited abnormally (crash)"
10630 );
10631 }
10632 let mut restart_schedule = None;
10633 let mut disposition = TerminalDisposition::Disabled;
10634 let mut final_state = ModuleState::Disabled;
10635 let mut disposition_detail = lock_snapshot(snapshot)
10639 .ok()
10640 .and_then(|mut state| state.spawn_failure.take());
10641 let now = Instant::now();
10642 if let Err(err) = update_snapshot(snapshot, Some(&spec.module_id), |state| {
10643 if state.enabled {
10644 if let Some(schedule) = state.next_crash_restart(&policy, now) {
10645 state.respawn_pending = true;
10651 final_state = ModuleState::Restarting;
10652 restart_schedule = Some(schedule);
10653 disposition = TerminalDisposition::Restarting;
10654 } else {
10655 final_state = ModuleState::Failed;
10656 disposition = TerminalDisposition::Failed;
10657 let budget = policy.budget_exhausted_detail();
10658 disposition_detail =
10659 Some(disposition_detail.take().map_or_else(
10660 || budget.clone(),
10661 |cause| format!("{cause}; {budget}"),
10662 ));
10663 }
10664 } else {
10665 disposition = TerminalDisposition::Disabled;
10666 }
10667 }) {
10668 error!(module_id = %spec.module_id, error = %err, "failed to record crashed module exit");
10669 return NextAction::Stop {
10670 registration_released: false,
10671 };
10672 }
10673 if disposition == TerminalDisposition::Failed {
10674 error!(
10679 module_id = %spec.module_id,
10680 max_restarts = policy.max_restarts,
10681 window_secs = policy.window.as_secs(),
10682 "module stopped: {}",
10683 policy.budget_exhausted_detail()
10684 );
10685 }
10686 if let Err(err) = publish_after_terminal_record(
10687 &spec.module_id,
10688 || {
10689 record_terminal_with_detail(
10690 &spec.module_id,
10691 terminal_ring,
10692 spawn_events,
10693 &exit_report,
10694 disposition,
10695 disposition_detail,
10696 )
10697 },
10698 || {
10699 update_snapshot(snapshot, Some(&spec.module_id), |state| {
10700 state.state = final_state;
10701 clear_current_process_facts(state);
10702 if final_state == ModuleState::Failed
10703 && daemon_initiated
10704 && !roster.is_closed()
10705 {
10706 state.failed_retry.schedule(&spec.module_id);
10707 }
10708 state.last_exit = Some(exit_report.clone());
10709 })
10710 },
10711 ) {
10712 error!(module_id = %spec.module_id, error = %err, "failed to publish crashed module exit");
10713 }
10714
10715 if let Some(schedule) = restart_schedule {
10716 NextAction::Restart {
10717 schedule: Some(schedule),
10718 }
10719 } else {
10720 let registration_released = match wait_for_registration_release(
10721 registry,
10722 &spec.module_id,
10723 REGISTRY_RELEASE_TIMEOUT,
10724 )
10725 .await
10726 {
10727 Ok(()) => true,
10728 Err(err) => {
10729 warn!(module_id = %spec.module_id, error = %err, "registration still active after failed module");
10730 false
10731 }
10732 };
10733 NextAction::Stop {
10734 registration_released,
10735 }
10736 }
10737 }
10738 ExitKind::DeliberateSeverance => {
10739 warn!(
10740 module_id = %spec.module_id,
10741 exit_code = ?exit_report.code,
10742 exit_signal = ?exit_report.signal,
10743 "supervised module exited after deliberate connection severance"
10744 );
10745 let should_restart = match lock_snapshot(snapshot) {
10746 Ok(state) => state.enabled,
10747 Err(err) => {
10748 error!(module_id = %spec.module_id, error = %err, "failed to read deliberately severed module state");
10749 return NextAction::Stop {
10750 registration_released: false,
10751 };
10752 }
10753 };
10754 let final_state = if should_restart {
10755 ModuleState::Restarting
10756 } else {
10757 ModuleState::Disabled
10758 };
10759 if let Err(err) = publish_after_terminal_record(
10760 &spec.module_id,
10761 || {
10762 record_terminal(
10763 &spec.module_id,
10764 terminal_ring,
10765 spawn_events,
10766 &exit_report,
10767 terminal_disposition(final_state),
10768 )
10769 },
10770 || {
10771 update_snapshot(snapshot, Some(&spec.module_id), |state| {
10772 state.state = final_state;
10773 clear_current_process_facts(state);
10774 state.last_exit = Some(exit_report.clone());
10775 state.lifetime_restarts += 1;
10776 })
10777 },
10778 ) {
10779 error!(module_id = %spec.module_id, error = %err, "failed to publish deliberately severed module exit");
10780 return NextAction::Stop {
10781 registration_released: false,
10782 };
10783 }
10784
10785 if should_restart {
10786 NextAction::Restart { schedule: None }
10787 } else {
10788 let registration_released = match wait_for_registration_release(
10789 registry,
10790 &spec.module_id,
10791 REGISTRY_RELEASE_TIMEOUT,
10792 )
10793 .await
10794 {
10795 Ok(()) => true,
10796 Err(err) => {
10797 warn!(module_id = %spec.module_id, error = %err, "registration still active after deliberately severed module exit");
10798 false
10799 }
10800 };
10801 NextAction::Stop {
10802 registration_released,
10803 }
10804 }
10805 }
10806 }
10807}
10808
10809async fn on_child_exit_during_daemon_shutdown(
10810 spec: &ModuleSpec,
10811 registry: &Registry,
10812 snapshot: &SharedSnapshot,
10813 terminal_ring: &Arc<Mutex<TerminalRing>>,
10814 spawn_events: &SpawnEventFeed,
10815 exit_report: ExitReport,
10816) -> NextAction {
10817 info!(
10818 module_id = %spec.module_id,
10819 exit_code = ?exit_report.code,
10820 exit_signal = ?exit_report.signal,
10821 exit_kind = ?exit_report.kind,
10822 "supervised module exited during daemon shutdown; not restarting it"
10823 );
10824 if let Err(err) = publish_after_terminal_record(
10825 &spec.module_id,
10826 || {
10827 record_terminal(
10828 &spec.module_id,
10829 terminal_ring,
10830 spawn_events,
10831 &exit_report,
10832 TerminalDisposition::DaemonShutdown,
10833 )
10834 },
10835 || {
10836 update_snapshot(snapshot, Some(&spec.module_id), |state| {
10837 state.state = ModuleState::Stopped;
10838 clear_current_process_facts(state);
10839 state.last_exit = Some(exit_report.clone());
10840 })
10841 },
10842 ) {
10843 error!(module_id = %spec.module_id, error = %err, "failed to publish module exit during daemon shutdown");
10844 }
10845 let registration_released =
10846 wait_for_registration_release(registry, &spec.module_id, REGISTRY_RELEASE_TIMEOUT)
10847 .await
10848 .is_ok();
10849 NextAction::Stop {
10850 registration_released,
10851 }
10852}
10853
10854fn publish_after_terminal_record<T>(
10859 module_id: &str,
10860 record: impl FnOnce(),
10861 publish: impl FnOnce() -> T,
10862) -> T {
10863 let recorded = std::panic::catch_unwind(std::panic::AssertUnwindSafe(record));
10864 let published = publish();
10865 if let Err(panic) = recorded {
10866 error!(
10867 module_id,
10868 "terminal recording panicked; published final state before resuming unwind"
10869 );
10870 std::panic::resume_unwind(panic);
10871 }
10872 published
10873}
10874
10875fn fail_after_wait_error(
10876 module_id: &str,
10877 snapshot: &SharedSnapshot,
10878 terminal_ring: &Arc<Mutex<TerminalRing>>,
10879 spawn_events: &SpawnEventFeed,
10880 detail: String,
10881 failure: &FailureContext,
10882) {
10883 let report = wait_error_exit_report();
10884 publish_after_terminal_record(
10885 module_id,
10886 || {
10887 record_terminal_with_detail(
10888 module_id,
10889 terminal_ring,
10890 spawn_events,
10891 &report,
10892 TerminalDisposition::Failed,
10893 Some(detail),
10894 )
10895 },
10896 || fail_snapshot(snapshot, Some(module_id), Some(report.clone()), failure),
10897 );
10898}
10899
10900#[cfg(test)]
10901fn record_wait_error_terminal(
10902 module_id: &str,
10903 terminal_ring: &Arc<Mutex<TerminalRing>>,
10904 spawn_events: &SpawnEventFeed,
10905) {
10906 record_terminal(
10907 module_id,
10908 terminal_ring,
10909 spawn_events,
10910 &wait_error_exit_report(),
10911 TerminalDisposition::Failed,
10912 );
10913}
10914
10915fn record_terminal(
10916 module_id: &str,
10917 terminal_ring: &Arc<Mutex<TerminalRing>>,
10918 spawn_events: &SpawnEventFeed,
10919 exit_report: &ExitReport,
10920 disposition: TerminalDisposition,
10921) {
10922 record_terminal_with_detail(
10923 module_id,
10924 terminal_ring,
10925 spawn_events,
10926 exit_report,
10927 disposition,
10928 None,
10929 );
10930}
10931
10932fn durable_terminal_history_of(
10936 terminal_ring: &Mutex<TerminalRing>,
10937 module_id: &str,
10938) -> subc_control::TerminalHistory {
10939 let read = terminal_ring
10940 .lock()
10941 .unwrap_or_else(|p| p.into_inner())
10942 .capture_durable_history();
10943 read.read(module_id)
10944}
10945
10946fn record_terminal_with_detail(
10947 module_id: &str,
10948 terminal_ring: &Arc<Mutex<TerminalRing>>,
10949 spawn_events: &SpawnEventFeed,
10950 exit_report: &ExitReport,
10951 disposition: TerminalDisposition,
10952 disposition_detail: Option<String>,
10953) {
10954 #[cfg(test)]
10955 terminal_history_tests::before_terminal_record(spawn_events);
10956 spawn_events.emit_exited(module_id, exit_report.code, exit_report.signal);
10957 let record = TerminalRecord {
10958 exit_code: exit_report.code,
10959 exit_signal: exit_report.signal,
10960 at_ms: exit_report.at_ms,
10961 disposition,
10962 exit_kind: exit_report.kind.into(),
10963 disposition_detail,
10964 };
10965 terminal_ring
10966 .lock()
10967 .unwrap_or_else(|poisoned| poisoned.into_inner())
10968 .record_exit(module_id, record);
10969}
10970
10971fn untrack_if_registration_released(
10972 process_liveness: &SupervisorProcessLiveness,
10973 registry: &Registry,
10974 module_id: &str,
10975 snapshot: &SharedSnapshot,
10976) {
10977 match registry.get_module(module_id) {
10978 Ok(None) => process_liveness.untrack_if_current(module_id, snapshot),
10979 Ok(Some(_)) => {}
10980 Err(err) => {
10981 warn!(module_id, error = %err, "could not determine whether supervisor liveness can be untracked");
10982 }
10983 }
10984}
10985
10986#[cfg(test)]
11000fn apply_wire_spawn_args(
11001 command: &mut Command,
11002 spec: &ModuleSpec,
11003 connection_file_path: Option<&std::path::Path>,
11004 handle: Option<&SupervisorHandle>,
11005) -> Result<Option<NonceHandoff>, SuperviseError> {
11006 apply_wire_spawn_args_for_role(
11007 command,
11008 spec,
11009 connection_file_path,
11010 handle,
11011 SpawnRole::Plain,
11012 )
11013}
11014
11015#[cfg(unix)]
11020type NonceHandoff = subc_os::LaunchNonceHandoff;
11021#[cfg(windows)]
11022type NonceHandoff = subc_os::LaunchNoncePipeHandoff;
11023#[cfg(not(any(unix, windows)))]
11024type NonceHandoff = std::convert::Infallible;
11025
11026#[cfg(test)]
11044fn apply_wire_spawn_args_for_role(
11045 command: &mut Command,
11046 spec: &ModuleSpec,
11047 connection_file_path: Option<&std::path::Path>,
11048 handle: Option<&SupervisorHandle>,
11049 role: SpawnRole,
11050) -> Result<Option<NonceHandoff>, SuperviseError> {
11051 apply_wire_spawn_args_for_admission(command, spec, connection_file_path, handle, role, None)
11052}
11053
11054fn apply_wire_spawn_args_for_admission(
11055 command: &mut Command,
11056 spec: &ModuleSpec,
11057 connection_file_path: Option<&std::path::Path>,
11058 handle: Option<&SupervisorHandle>,
11059 role: SpawnRole,
11060 admission: Option<&crate::config_gate::runtime::SpawnAdmission>,
11061) -> Result<Option<NonceHandoff>, SuperviseError> {
11062 command.env(SUBC_MODULE_ID_ENV, &spec.module_id);
11063 command.env_remove(subc_os::LAUNCH_NONCE_FD_ENV);
11068 #[cfg(windows)]
11069 {
11070 command.env_remove(subc_os::LAUNCH_NONCE_PIPE_ENV);
11071 command.env_remove(subc_os::LAUNCH_NONCE_PIPE_FALLBACK_ENV);
11072 }
11073 command.env_remove(SUBC_LAUNCH_NONCE_ENV);
11075 command.env_remove(subc_os::registration_channel::REGISTRATION_FD_ENV);
11076 command.env_remove(subc_os::registration_channel::REGISTRATION_PIPE_ENV);
11077 command.env_remove(subc_os::registration_channel::DAEMON_PID_ENV);
11078 if role == SpawnRole::Plain {
11079 if let Some(handle) = handle {
11080 #[cfg(all(test, unix))]
11081 {
11082 let pause = {
11083 let mut slot = handle.plain_identity_policy_pause.lock().unwrap();
11084 if slot
11085 .as_ref()
11086 .is_some_and(|(module, _)| module == &spec.module_id)
11087 {
11088 slot.take().map(|(_, pause)| pause)
11089 } else {
11090 None
11091 }
11092 };
11093 if let Some(pause) = pause {
11094 pause.reached.send(()).unwrap();
11095 pause.release.recv().unwrap();
11096 }
11097 }
11098 handle.apply_identity_policy(spec, spec.protocol);
11102 }
11103 }
11104 if spec.protocol == ModuleProtocol::None {
11105 return Ok(None);
11106 }
11107 if let Some(connection_file_path) = connection_file_path {
11108 command.arg(SUBC_ARG).arg(connection_file_path);
11109 }
11110
11111 if admission.is_some_and(|admission| admission.launch.is_some()) {
11112 if let Some(handle) = handle {
11113 match role {
11114 SpawnRole::Plain => {
11115 handle
11116 .spawn_nonces
11117 .lock()
11118 .unwrap_or_else(|p| p.into_inner())
11119 .remove(&spec.module_id);
11120 if spec.reserved {
11121 handle
11122 .reserved_nonces
11123 .lock()
11124 .unwrap_or_else(|p| p.into_inner())
11125 .insert(spec.module_id.clone(), None);
11126 }
11127 }
11128 SpawnRole::SwapCandidate => handle.open_swap(&spec.module_id, String::new()),
11131 }
11132 }
11133 return Ok(None);
11134 }
11135 let nonce = generate_launch_nonce()?;
11139 if let Some(handle) = handle {
11140 match role {
11141 SpawnRole::Plain => {
11142 handle.set_spawn_nonce(&spec.module_id, nonce.clone());
11143 if spec.reserved {
11144 handle.set_reserved_nonce(&spec.module_id, nonce.clone());
11145 }
11146 }
11147 SpawnRole::SwapCandidate => handle.open_swap(&spec.module_id, nonce.clone()),
11148 }
11149 }
11150 #[cfg(unix)]
11151 let handoff = {
11152 let handoff =
11153 subc_os::LaunchNonceHandoff::new(&nonce).map_err(|source| SuperviseError::Spawn {
11154 program: spec.program.clone(),
11155 source,
11156 cgroup_path: None,
11157 })?;
11158 command.env(subc_os::LAUNCH_NONCE_FD_ENV, handoff.fd_env_value());
11159 Some(handoff)
11160 };
11161 #[cfg(windows)]
11162 let handoff = prepare_windows_nonce_pipe(command, spec, &nonce);
11163 #[cfg(not(any(unix, windows)))]
11164 let handoff = None;
11165 #[cfg(not(unix))]
11169 command.env(SUBC_LAUNCH_NONCE_ENV, nonce);
11170 Ok(handoff)
11171}
11172
11173fn apply_child_env(command: &mut Command, spec: &ModuleSpec) {
11174 command.env_remove(CK_LOG_ENV);
11175 command.env_remove(SUBC_SPAWN_ROLE_ENV);
11182 for (key, value) in &spec.env {
11183 if matches!(
11187 key.as_str(),
11188 CAPTURE_MAX_FILE_MB_ENV | CAPTURE_KEEP_ENV | CAPTURE_MAX_AGE_DAYS_ENV
11189 ) || key == SUBC_SPAWN_ROLE_ENV
11190 {
11191 continue;
11192 }
11193 command.env(key, value);
11194 }
11195}
11196
11197#[cfg(windows)]
11198fn prepare_windows_registration_pipe(
11199 command: &mut Command,
11200 spec: &ModuleSpec,
11201) -> Result<Option<SpawnRegistrationChannel>, SuperviseError> {
11202 if spec.protocol == ModuleProtocol::None {
11203 return Ok(None);
11204 }
11205 let pipe = subc_os::registration_channel::RegistrationPipe::new().map_err(|source| {
11206 SuperviseError::Spawn {
11207 program: spec.program.clone(),
11208 source,
11209 cgroup_path: None,
11210 }
11211 })?;
11212 command
11213 .env(
11214 subc_os::registration_channel::REGISTRATION_PIPE_ENV,
11215 pipe.name(),
11216 )
11217 .env(
11218 subc_os::registration_channel::DAEMON_PID_ENV,
11219 std::process::id().to_string(),
11220 );
11221 Ok(Some(pipe))
11222}
11223
11224#[cfg(windows)]
11225fn prepare_windows_nonce_pipe(
11226 command: &mut Command,
11227 spec: &ModuleSpec,
11228 nonce: &str,
11229) -> Option<NonceHandoff> {
11230 match subc_os::LaunchNoncePipeHandoff::new(nonce) {
11231 Ok(handoff) => {
11232 command.env(subc_os::LAUNCH_NONCE_PIPE_ENV, handoff.name());
11233 command.env(subc_os::LAUNCH_NONCE_PIPE_FALLBACK_ENV, "env");
11238 Some(handoff)
11239 }
11240 Err(error) => {
11241 warn!(module_id = %spec.module_id, %error, "could not prepare launch nonce pipe; retaining the environment handoff");
11242 None
11243 }
11244 }
11245}
11246
11247#[cfg(windows)]
11248fn start_windows_nonce_pipe(
11249 handoff: Option<NonceHandoff>,
11250 pid: u32,
11251 spec: &ModuleSpec,
11252) -> Option<subc_os::LaunchNoncePipeDelivery> {
11253 handoff.and_then(|handoff| match handoff.serve(pid, REGISTRY_RELEASE_TIMEOUT) {
11257 Ok(delivery) => Some(delivery),
11258 Err(error) => {
11259 warn!(module_id = %spec.module_id, %error, "could not serve launch nonce pipe; retaining the environment handoff");
11260 None
11261 }
11262 })
11263}
11264
11265fn observed_launch_nonce_source(snapshot: &SupervisorSnapshot) -> Option<String> {
11266 #[cfg(windows)]
11267 if snapshot.process_alive && snapshot.spawned_protocol == Some(ModuleProtocol::Subc) {
11268 return Some(
11269 if snapshot
11270 .pipe_consumed
11271 .as_ref()
11272 .is_some_and(|consumed| consumed.get().is_some())
11273 {
11274 "pipe"
11275 } else {
11276 "env"
11277 }
11278 .to_owned(),
11279 );
11280 }
11281 let _ = snapshot;
11282 None
11283}
11284
11285#[derive(Debug, Clone, Copy, PartialEq, Eq)]
11288enum SpawnRole {
11289 Plain,
11290 SwapCandidate,
11291}
11292
11293fn apply_spawn_role(command: &mut Command, role: SpawnRole) {
11296 if role == SpawnRole::SwapCandidate {
11297 command.env(SUBC_SPAWN_ROLE_ENV, SPAWN_ROLE_SWAP_CANDIDATE);
11298 }
11299}
11300
11301fn previous_instance_error(
11302 spec: &ModuleSpec,
11303 pid: u32,
11304 reason: crate::live_children::RetainedReason,
11305) -> SuperviseError {
11306 SuperviseError::Spawn {
11307 program: spec.program.clone(),
11308 source: io::Error::new(io::ErrorKind::WouldBlock, reason.waiting_message(pid)),
11309 cgroup_path: None,
11310 }
11311}
11312
11313fn carries_containment(configured: bool, has_container: bool) -> bool {
11314 configured || has_container
11315}
11316
11317fn recovery_block_error(spec: &ModuleSpec, roster: &ChildRoster) -> Option<SuperviseError> {
11318 if !carries_containment(
11319 spec.carry_containment,
11320 containment::current(&roster.containment).is_some(),
11321 ) {
11322 return None;
11323 }
11324 Some(SuperviseError::Spawn {
11325 program: spec.program.clone(),
11326 source: io::Error::new(io::ErrorKind::WouldBlock, roster.recovery_block()?),
11327 cgroup_path: None,
11328 })
11329}
11330
11331#[allow(clippy::too_many_arguments)]
11332fn spawn_child(
11333 spec: &ModuleSpec,
11334 connection_file_path: Option<&std::path::Path>,
11335 handle: Option<&SupervisorHandle>,
11336 ring: &Arc<Mutex<StderrRing>>,
11337 capture_logs_dir: Option<&std::path::Path>,
11338 roster: &ChildRoster,
11339 config_delivery: Option<config_delivery::SpawnDelivery<'_>>,
11340 #[cfg(target_os = "linux")] cgroup_placement: Option<&subc_cgroup::Placement>,
11341 register_timeout: Duration,
11342) -> Result<SupervisedChild, SuperviseError> {
11343 #[cfg(all(test, unix))]
11344 if let Some(source) = containment_tests::join_failure(spec, roster) {
11345 containment::terminate_blocking(&roster.containment);
11346 return Err(SuperviseError::Spawn {
11347 program: spec.program.clone(),
11348 source,
11349 cgroup_path: None,
11350 });
11351 }
11352 let result = spawn_child_in_slot(
11353 spec,
11354 connection_file_path,
11355 handle,
11356 ring,
11357 capture_logs_dir,
11358 roster,
11359 config_delivery,
11360 #[cfg(target_os = "linux")]
11361 cgroup_placement,
11362 SpawnRole::Plain,
11363 false,
11364 register_timeout,
11365 );
11366 if result.is_err() {
11367 if roster.admission_pending(&spec.module_id) {
11368 containment::terminate_deferred(&roster.containment);
11369 } else {
11370 containment::terminate_blocking(&roster.containment);
11371 }
11372 }
11373 result
11374}
11375
11376#[allow(clippy::too_many_arguments)]
11389fn spawn_child_in_slot(
11390 spec: &ModuleSpec,
11391 connection_file_path: Option<&std::path::Path>,
11392 handle: Option<&SupervisorHandle>,
11393 ring: &Arc<Mutex<StderrRing>>,
11394 capture_logs_dir: Option<&std::path::Path>,
11395 roster: &ChildRoster,
11396 config_delivery: Option<config_delivery::SpawnDelivery<'_>>,
11397 #[cfg(target_os = "linux")] cgroup_placement: Option<&subc_cgroup::Placement>,
11398 role: SpawnRole,
11399 alternate_slot: bool,
11400 register_timeout: Duration,
11401) -> Result<SupervisedChild, SuperviseError> {
11402 let prepared = prepare_child_in_slot(
11403 spec,
11404 connection_file_path,
11405 handle,
11406 capture_logs_dir,
11407 roster,
11408 config_delivery,
11409 #[cfg(target_os = "linux")]
11410 cgroup_placement,
11411 role,
11412 alternate_slot,
11413 false,
11414 )?;
11415 let gate = config_delivery
11416 .map(|delivery| delivery.module_gate(spec))
11417 .transpose()?;
11418 let _gate = gate
11419 .as_ref()
11420 .map(|gate| {
11421 gate.try_lock()
11422 .map_err(|_| SuperviseError::ConfigDeliveryDeferred {
11423 module_id: spec.module_id.clone(),
11424 })
11425 })
11426 .transpose()?;
11427 if let (Some(delivery), Some(admission)) = (config_delivery, &prepared.spawn_admission) {
11428 delivery.check(spec, admission)?;
11429 }
11430 let checked_base = prepared
11431 .spawn_admission
11432 .as_ref()
11433 .and_then(|admission| admission.approved_base())
11434 .cloned();
11435 let mut result = launch_child_in_slot(spec, ring, roster, prepared, register_timeout);
11436 drop(_gate);
11437 if let (Some(delivery), Err(SuperviseError::ConfigHeld { code, detail, .. })) =
11438 (config_delivery, &result)
11439 {
11440 if let Some(base) = checked_base.as_ref().filter(|_| code == "program_refused") {
11441 if !delivery.record_refusal(spec, base, detail)? {
11442 result = Err(spawn_plan_stale(spec));
11443 }
11444 }
11445 }
11446 result
11447}
11448
11449struct AsyncInitialIdentity<'a> {
11450 supervisor: &'a Supervisor,
11451 module: String,
11452 committed: bool,
11453}
11454impl Drop for AsyncInitialIdentity<'_> {
11455 fn drop(&mut self) {
11456 if !self.committed {
11457 self.supervisor.abandon_unrostered(&self.module);
11458 }
11459 }
11460}
11461
11462struct PreparedChild {
11466 registration_spawn: Option<Arc<RegistrationSpawn>>,
11467 registration_handle: Option<SupervisorHandle>,
11468 #[cfg(any(unix, windows))]
11469 registration_channel: Option<SpawnRegistrationChannel>,
11470 config_context: Option<config_delivery::ConfigDelivery>,
11471 binding_publication: Option<ProgramBindingPublication>,
11472 launch_spec: ModuleSpec,
11473 spawn_admission: Option<crate::config_gate::runtime::SpawnAdmission>,
11474 command: Command,
11475 carrying: bool,
11476 carried: Option<Arc<containment::CarriedContainment>>,
11477 #[cfg(unix)]
11478 launch_identity: Option<containment::LaunchIdentity>,
11479 #[cfg(unix)]
11480 group_target: containment::GroupTarget,
11481 #[cfg(target_os = "linux")]
11482 cgroup_placement: Option<subc_cgroup::Placement>,
11483 #[cfg(target_os = "linux")]
11484 cgroup_name: String,
11485 #[cfg(target_os = "linux")]
11486 cgroup_cleanup: PreparedCgroup,
11487 cgroup_path: Option<PathBuf>,
11488 #[cfg(unix)]
11489 admission_gate: Option<admission::Gate>,
11490 output_sink: ChildOutputSink,
11491 #[cfg(target_os = "macos")]
11492 privacy_exec: Option<PrivacyExec>,
11493 #[cfg(windows)]
11494 image_capture: subc_os::ExecutableCapture,
11495 #[cfg(windows)]
11496 nonce_handoff: Option<NonceHandoff>,
11497 spawned_at: Instant,
11498 config_delivery: Option<crate::config_gate::runtime::DeliverySpawn>,
11499 delivery_registration: Option<config_delivery::RegistrationGuard>,
11500 program_binding: Option<SpawnProgramBinding>,
11501}
11502
11503#[cfg(target_os = "linux")]
11504struct PreparedCgroup(Option<(subc_cgroup::Placement, String)>);
11505#[cfg(target_os = "linux")]
11506impl Drop for PreparedCgroup {
11507 fn drop(&mut self) {
11508 if let Some((placement, name)) = &self.0 {
11509 remove_module_cgroup(placement, name);
11510 }
11511 }
11512}
11513
11514pub(crate) struct SpawnProgramBinding {
11515 pub(crate) path: PathBuf,
11516 _file: std::fs::File,
11519 pub(crate) identity: subc_os::RetainedFileIdentity,
11520}
11521
11522impl SpawnProgramBinding {
11523 pub(crate) fn capture(spec: &ModuleSpec) -> Result<Self, SuperviseError> {
11524 Self::capture_after_resolution(spec, |_| {})
11525 }
11526
11527 fn capture_after_resolution(
11528 spec: &ModuleSpec,
11529 before_open: impl FnOnce(&std::path::Path),
11530 ) -> Result<Self, SuperviseError> {
11531 require_program_search_path(spec)?;
11532 #[cfg(windows)]
11533 let windows_capture =
11534 windows_spawn_capture(spec).map_err(|source| SuperviseError::Spawn {
11535 program: spec.program.clone(),
11536 source,
11537 cgroup_path: None,
11538 })?;
11539 #[cfg(windows)]
11540 let path = windows_capture.path().to_path_buf();
11541 #[cfg(not(windows))]
11542 let path = if spec.program.is_absolute() || spec.program.components().count() > 1 {
11543 spec.program.clone()
11544 } else {
11545 let path = spec
11546 .env
11547 .iter()
11548 .rev()
11549 .find(|(key, _)| key == "PATH")
11550 .map(|(_, path)| std::ffi::OsString::from(path))
11551 .or_else(|| std::env::var_os("PATH"))
11552 .unwrap_or_else(|| {
11553 if cfg!(target_os = "macos") {
11554 "/usr/bin:/bin".into()
11557 } else {
11558 Default::default()
11559 }
11560 });
11561 std::env::split_paths(&path)
11562 .map(|dir| dir.join(&spec.program))
11563 .find(|path| {
11564 if !path.is_file() {
11565 return false;
11566 }
11567 #[cfg(unix)]
11568 {
11569 rustix::fs::accessat(
11570 rustix::fs::CWD,
11571 path,
11572 rustix::fs::Access::EXEC_OK,
11573 rustix::fs::AtFlags::EACCESS,
11574 )
11575 .is_ok()
11576 }
11577 #[cfg(not(unix))]
11578 {
11579 true
11580 }
11581 })
11582 .ok_or_else(|| SuperviseError::Spawn {
11583 program: spec.program.clone(),
11584 source: io::Error::new(io::ErrorKind::NotFound, "executable not found in PATH"),
11585 cgroup_path: None,
11586 })?
11587 };
11588 let resolved = std::fs::canonicalize(&path).map_err(|source| SuperviseError::Spawn {
11589 program: spec.program.clone(),
11590 source,
11591 cgroup_path: None,
11592 })?;
11593 before_open(&resolved);
11594 let file =
11597 subc_os::open_regular_nofollow(&resolved).map_err(|source| SuperviseError::Spawn {
11598 program: spec.program.clone(),
11599 source: io::Error::other(source),
11600 cgroup_path: None,
11601 })?;
11602 let identity =
11603 subc_os::retained_file_identity(&file).map_err(|source| SuperviseError::Spawn {
11604 program: spec.program.clone(),
11605 source,
11606 cgroup_path: None,
11607 })?;
11608 Ok(Self {
11609 path,
11610 _file: file,
11611 identity,
11612 })
11613 }
11614 fn current(&self) -> bool {
11615 #[cfg(unix)]
11616 {
11617 subc_os::file_identity(&self.path) == Some(self.identity)
11618 }
11619 #[cfg(windows)]
11620 {
11621 std::fs::canonicalize(&self.path)
11622 .ok()
11623 .and_then(|path| subc_os::open_regular_nofollow(&path).ok())
11624 .and_then(|file| subc_os::retained_file_identity(&file).ok())
11625 == Some(self.identity)
11626 }
11627 }
11628}
11629
11630fn spawn_plan_stale(spec: &ModuleSpec) -> SuperviseError {
11631 SuperviseError::ConfigDeliveryDeferred {
11632 module_id: spec.module_id.clone(),
11633 }
11634}
11635
11636fn require_program_search_path(spec: &ModuleSpec) -> Result<(), SuperviseError> {
11637 if !cfg!(target_os = "macos")
11638 && !spec.program.is_absolute()
11639 && spec.program.components().count() == 1
11640 && !spec.env.iter().any(|(key, _)| {
11641 if cfg!(windows) {
11642 key.eq_ignore_ascii_case("PATH")
11643 } else {
11644 key == "PATH"
11645 }
11646 })
11647 && std::env::var_os("PATH").is_none()
11648 {
11649 return Err(SuperviseError::Spawn {
11652 program: spec.program.clone(),
11653 source: io::Error::new(io::ErrorKind::InvalidInput, "bare program requires PATH"),
11654 cgroup_path: None,
11655 });
11656 }
11657 Ok(())
11658}
11659
11660async fn spawn_child_async(
11661 spec: &ModuleSpec,
11662 runtime: &SupervisorRuntimeConfig,
11663) -> Result<SupervisedChild, SuperviseError> {
11664 #[cfg(all(test, unix))]
11665 if let Some(source) = containment_tests::join_failure(spec, &runtime.child_roster) {
11666 containment::terminate_blocking(&runtime.child_roster.containment);
11667 return Err(SuperviseError::Spawn {
11668 program: spec.program.clone(),
11669 source,
11670 cgroup_path: None,
11671 });
11672 }
11673 let result = spawn_child_in_slot_async(spec, runtime, SpawnRole::Plain, false).await;
11674 if result.is_err() {
11675 if runtime.child_roster.admission_pending(&spec.module_id) {
11676 containment::terminate_deferred(&runtime.child_roster.containment);
11677 } else {
11678 containment::terminate_blocking(&runtime.child_roster.containment);
11679 }
11680 }
11681 result
11682}
11683
11684async fn spawn_child_in_slot_async(
11685 spec: &ModuleSpec,
11686 runtime: &SupervisorRuntimeConfig,
11687 role: SpawnRole,
11688 alternate_slot: bool,
11689) -> Result<SupervisedChild, SuperviseError> {
11690 let gate = runtime
11691 .config_delivery
11692 .as_ref()
11693 .map(|delivery| delivery.0.module_gate(&spec.module_id))
11694 .transpose()
11695 .map_err(|error| SuperviseError::InvalidSpec {
11696 reason: error.to_string(),
11697 })?;
11698 if gate.as_ref().is_some_and(|gate| gate.try_lock().is_err()) {
11699 return Err(SuperviseError::ConfigDeliveryDeferred {
11700 module_id: spec.module_id.clone(),
11701 });
11702 }
11703 let prepared_spec = spec.clone();
11704 let prepared_runtime = runtime.clone();
11705 let prepared = tokio::task::spawn_blocking(move || {
11706 let prepared = prepare_child_in_slot(
11707 &prepared_spec,
11708 prepared_runtime.connection_file_path.as_deref(),
11709 prepared_runtime.supervisor_handle.as_ref(),
11710 prepared_runtime.capture_logs_dir.as_deref(),
11711 &prepared_runtime.child_roster,
11712 prepared_runtime.spawn_delivery(),
11713 #[cfg(target_os = "linux")]
11714 prepared_runtime.cgroup_placement.as_ref(),
11715 role,
11716 alternate_slot,
11717 true,
11718 );
11719 prepared
11720 })
11721 .await
11722 .map_err(|_| SuperviseError::InvalidSpec {
11723 reason: "configuration launch preparation stopped".into(),
11724 })??;
11725 if let Some(registration) = &prepared.delivery_registration {
11726 if !registration.launch_current()? {
11727 return Err(spawn_plan_stale(spec));
11728 }
11729 }
11730 let _gate = match gate {
11731 Some(gate) => {
11732 Some(
11733 gate.try_lock_owned()
11734 .map_err(|_| SuperviseError::ConfigDeliveryDeferred {
11735 module_id: spec.module_id.clone(),
11736 })?,
11737 )
11738 }
11739 None => None,
11740 };
11741 if let (Some(delivery), Some(admission)) = (runtime.spawn_delivery(), &prepared.spawn_admission)
11742 {
11743 delivery.check(spec, admission)?;
11744 }
11745 if prepared
11746 .spawn_admission
11747 .as_ref()
11748 .is_none_or(|admission| admission.launch.is_none())
11749 && prepared
11750 .program_binding
11751 .as_ref()
11752 .is_some_and(|binding| !binding.current())
11753 {
11754 return Err(spawn_plan_stale(spec));
11755 }
11756 let checked_base = prepared
11758 .spawn_admission
11759 .as_ref()
11760 .and_then(|admission| admission.approved_base())
11761 .cloned();
11762 let mut result = launch_child_in_slot(
11763 spec,
11764 &runtime.stderr_ring,
11765 &runtime.child_roster,
11766 prepared,
11767 runtime.register_timeout,
11768 );
11769 drop(_gate);
11770 if let (Some(delivery), Err(SuperviseError::ConfigHeld { code, detail, .. })) =
11771 (runtime.config_delivery.as_ref(), &result)
11772 {
11773 if let Some(base) = checked_base.filter(|_| code == "program_refused") {
11774 let module = spec.module_id.clone();
11775 let identity = subc_config::raw_digest(detail.as_bytes());
11776 let recorded = delivery
11777 .0
11778 .on_blocking(move |owner| owner.record_program_refusal(&module, &base, identity))
11779 .await;
11780 match recorded {
11781 Ok(true) => {}
11782 Ok(false) => result = Err(spawn_plan_stale(spec)),
11783 Err(error) => {
11784 result = Err(config_delivery::gate_hold_error(
11785 spec,
11786 error,
11787 Some(delivery),
11788 ))
11789 }
11790 }
11791 }
11792 }
11793 result
11794}
11795
11796fn validate_spawn_program(
11797 spec: &ModuleSpec,
11798 admission: Option<&crate::config_gate::runtime::SpawnAdmission>,
11799 delivery: Option<config_delivery::SpawnDelivery<'_>>,
11800) -> Result<(), SuperviseError> {
11801 if let Some(admission) = admission {
11802 if let Some(program) = &admission.launch {
11803 if let Err(error) = program.validate_for_spawn() {
11804 if let (Some(context), Some(base)) = (delivery, admission.approved_base()) {
11808 if error.code == "program_refused"
11809 && !context.record_refusal(spec, base, &error.detail)?
11810 {
11811 return Err(spawn_plan_stale(spec));
11812 }
11813 }
11814 return Err(config_delivery::gate_hold_error(
11815 spec,
11816 error,
11817 delivery.map(|context| context.runtime()).as_ref(),
11818 ));
11819 }
11820 }
11821 }
11822 Ok(())
11823}
11824
11825#[allow(clippy::too_many_arguments)]
11826fn prepare_child_in_slot(
11827 spec: &ModuleSpec,
11828 connection_file_path: Option<&std::path::Path>,
11829 handle: Option<&SupervisorHandle>,
11830 capture_logs_dir: Option<&std::path::Path>,
11831 roster: &ChildRoster,
11832 config_delivery: Option<config_delivery::SpawnDelivery<'_>>,
11833 #[cfg(target_os = "linux")] cgroup_placement: Option<&subc_cgroup::Placement>,
11834 role: SpawnRole,
11835 alternate_slot: bool,
11836 asynchronous: bool,
11837) -> Result<PreparedChild, SuperviseError> {
11838 require_program_search_path(spec)?;
11839 let spawn_admission = config_delivery
11840 .map(|delivery| delivery.prepare(spec))
11841 .transpose()?;
11842 #[cfg(target_os = "macos")]
11843 let gated = spawn_admission
11844 .as_ref()
11845 .is_some_and(|admission| admission.approved_base().is_some());
11846 let launch_spec = spawn_admission
11847 .as_ref()
11848 .and_then(|admission| admission.launch.as_ref())
11849 .map(|launch| launch.spec.clone())
11850 .unwrap_or_else(|| spec.clone());
11851 let spec = &launch_spec;
11852 if let Some(error) = recovery_block_error(spec, roster) {
11853 return Err(error);
11854 }
11855 if let Some((pid, reason)) = roster.previous_instance(&spec.module_id) {
11856 return Err(previous_instance_error(spec, pid, reason));
11857 }
11858 if roster.is_closed() {
11859 return Err(SuperviseError::Spawn {
11860 program: spec.program.clone(),
11861 source: io::Error::other("the daemon is shutting down; not starting a new process"),
11862 cgroup_path: None,
11863 });
11864 }
11865 let carried = containment::current(&roster.containment);
11866 let carrying = carries_containment(spec.carry_containment, carried.is_some());
11867 if role != SpawnRole::Plain && carrying {
11868 return Err(SuperviseError::InvalidSpec {
11869 reason: "a carried container cannot overlap a swap candidate".to_owned(),
11870 });
11871 }
11872 #[cfg(unix)]
11873 let launch_identity = if carrying && carried.is_none() {
11874 Some(
11875 containment::LaunchIdentity::capture().map_err(|source| SuperviseError::Spawn {
11876 program: spec.program.clone(),
11877 source,
11878 cgroup_path: None,
11879 })?,
11880 )
11881 } else {
11882 None
11883 };
11884 #[cfg(target_os = "linux")]
11885 let cgroup_placement = match &carried {
11886 Some(container) => container.cgroup().map(|(placement, _)| placement),
11887 None => cgroup_placement,
11888 };
11889 #[cfg(unix)]
11890 let group_target = if let Some(container) = carried
11891 .as_ref()
11892 .filter(|container| container.group().is_some())
11893 {
11894 containment::GroupTarget::Pinned(Arc::clone(container))
11895 } else {
11896 #[cfg(target_os = "linux")]
11897 let cgroup_container = carrying && cgroup_placement.is_some();
11898 #[cfg(not(target_os = "linux"))]
11899 let cgroup_container = false;
11900 if cgroup_container {
11901 containment::GroupTarget::NoGroup
11902 } else {
11903 containment::GroupTarget::OwnLeader
11904 }
11905 };
11906 #[cfg(target_os = "linux")]
11907 let cgroup_name = {
11908 if let Some((_, name)) = carried.as_ref().and_then(|container| container.cgroup()) {
11913 name.to_owned()
11914 } else if cgroup_placement.is_none() {
11915 swap::cgroup_name(&spec.module_id, alternate_slot)
11916 } else {
11917 let nonce = generate_launch_nonce()?;
11918 let suffix = u128::from_str_radix(&nonce[..32], 16).expect("hex launch nonce");
11919 let mut end = spec.module_id.len().min(64);
11922 while !spec.module_id.is_char_boundary(end) {
11923 end -= 1;
11924 }
11925 format!(
11926 "{}_{suffix}",
11927 swap::cgroup_name(&spec.module_id[..end], alternate_slot)
11928 )
11929 }
11930 };
11931 #[cfg(not(target_os = "linux"))]
11932 let _ = alternate_slot;
11933 #[cfg(windows)]
11934 let image_capture = windows_spawn_capture(spec).map_err(|source| SuperviseError::Spawn {
11935 program: spec.program.clone(),
11936 source,
11937 cgroup_path: None,
11938 })?;
11939 #[cfg(windows)]
11940 let program_binding = if asynchronous {
11941 let mut resolved = spec.clone();
11942 resolved.program = image_capture.path().to_path_buf();
11943 Some(SpawnProgramBinding::capture(&resolved)?)
11944 } else {
11945 None
11946 };
11947 #[cfg(not(windows))]
11948 let program_binding = (asynchronous || cfg!(target_os = "macos"))
11949 .then(|| SpawnProgramBinding::capture(spec))
11950 .transpose()?;
11951 #[cfg(target_os = "macos")]
11952 validate_spawn_program(spec, spawn_admission.as_ref(), config_delivery)?;
11953 #[cfg(target_os = "macos")]
11954 let (mut command, privacy_exec, exec_ack) = privacy_command(
11955 spec,
11956 program_binding
11957 .as_ref()
11958 .expect("macOS captures every launch"),
11959 roster,
11960 gated,
11964 )?;
11965 #[cfg(windows)]
11966 let mut command = Command::new(image_capture.path());
11967 #[cfg(not(any(target_os = "macos", windows)))]
11968 let mut command = Command::new(
11969 program_binding
11970 .as_ref()
11971 .map_or(spec.program.as_path(), |binding| binding.path.as_path()),
11972 );
11973 command.args(&spec.args);
11974 apply_child_env(&mut command, spec);
12004 apply_spawn_role(&mut command, role);
12005 let nonce_handoff = apply_wire_spawn_args_for_admission(
12006 &mut command,
12007 spec,
12008 connection_file_path,
12009 handle,
12010 role,
12011 spawn_admission.as_ref(),
12012 )?;
12013 #[cfg(unix)]
12014 let mut slots = subc_os::registration_channel::ChildDescriptorSlots::default();
12015 #[cfg(unix)]
12016 let registration_channel = if spec.protocol == ModuleProtocol::Subc {
12017 let (parent, child, metadata) = subc_os::registration_channel::registration_socketpair()
12018 .map_err(|source| SuperviseError::Spawn {
12019 program: spec.program.clone(),
12020 source,
12021 cgroup_path: None,
12022 })?;
12023 slots
12024 .channel(child)
12025 .map_err(|source| SuperviseError::Spawn {
12026 program: spec.program.clone(),
12027 source,
12028 cgroup_path: None,
12029 })?;
12030 command
12031 .env(subc_os::registration_channel::REGISTRATION_FD_ENV, metadata)
12032 .env(
12033 subc_os::registration_channel::DAEMON_PID_ENV,
12034 std::process::id().to_string(),
12035 );
12036 Some(parent)
12037 } else {
12038 None
12039 };
12040 #[cfg(windows)]
12041 let registration_channel = prepare_windows_registration_pipe(&mut command, spec)?;
12042 let binding_publication = if spawn_admission
12043 .as_ref()
12044 .is_some_and(|admission| admission.launch.is_some())
12045 {
12046 handle.and_then(|handle| handle.begin_program_binding(&spec.module_id, role))
12047 } else {
12048 None
12049 };
12050 let registration_spawn = (spec.protocol == ModuleProtocol::Subc)
12051 .then(|| {
12052 handle.map(|handle| {
12053 RegistrationSpawn::new_with_binding_requirement(
12054 handle,
12055 &spec.module_id,
12056 role,
12057 binding_publication.clone(),
12058 spawn_admission
12059 .as_ref()
12060 .is_some_and(|admission| admission.launch.is_some()),
12061 )
12062 })
12063 })
12064 .flatten();
12065
12066 #[cfg(target_os = "linux")]
12067 let cgroup_path = cgroup_placement
12068 .map(|placement| {
12069 if let Some(container) = &carried {
12070 let path = container
12071 .cgroup_path()
12072 .ok_or_else(|| io::Error::other("carried container has no cgroup path"))?;
12073 placement.existing_module_path(path)?.ok_or_else(|| {
12074 io::Error::new(
12075 io::ErrorKind::NotFound,
12076 "carried cgroup disappeared before replacement",
12077 )
12078 })
12079 } else {
12080 if carrying {
12081 containment::prepare_cgroup_path(placement, &cgroup_name)
12082 } else {
12083 placement.module_path(&cgroup_name)
12084 }
12085 }
12086 })
12087 .transpose()
12088 .map_err(|source| SuperviseError::Cgroup {
12089 module_id: spec.module_id.clone(),
12090 source,
12091 })?;
12092 #[cfg(not(target_os = "linux"))]
12093 let cgroup_path: Option<PathBuf> = None;
12094 #[cfg(target_os = "linux")]
12097 let cgroup_cleanup = PreparedCgroup(
12098 cgroup_placement
12099 .filter(|_| carried.is_none())
12100 .map(|placement| (placement.clone(), cgroup_name.clone())),
12101 );
12102 #[cfg(unix)]
12103 let mut admission_gate = if carrying {
12104 Some(
12105 admission::Gate::install(&mut command, spec, roster).map_err(|source| {
12106 SuperviseError::Spawn {
12107 program: spec.program.clone(),
12108 source,
12109 cgroup_path: cgroup_path.clone(),
12110 }
12111 })?,
12112 )
12113 } else {
12114 None
12115 };
12116 #[cfg(target_os = "linux")]
12117 if let Some(path) = &cgroup_path {
12118 apply_cgroup_placement(&mut command, spec, path)?;
12119 }
12120
12121 let output_sink = if let Some(logs_dir) = capture_logs_dir {
12122 let path = logs_dir.join(format!("{}.stderr.log", spec.module_id));
12123 match ChildOutputSink::open(&path, capture_retention(spec)) {
12124 Ok(sink) => sink,
12125 Err(error) => {
12126 warn!(
12127 module_id = %spec.module_id,
12128 path = %path.display(),
12129 error = %error,
12130 "could not open child output capture file; forwarding to stderr"
12131 );
12132 ChildOutputSink::Stderr
12133 }
12134 }
12135 } else {
12136 ChildOutputSink::Stderr
12137 };
12138
12139 command.stdout(Stdio::piped());
12140 command.stderr(Stdio::piped());
12141 command.kill_on_drop(!carrying);
12142 #[cfg(unix)]
12159 command.process_group(
12160 carried
12161 .as_ref()
12162 .map(|container| container.verified_join_group())
12163 .transpose()
12164 .map_err(|source| SuperviseError::Spawn {
12165 program: spec.program.clone(),
12166 source,
12167 cgroup_path: cgroup_path.clone(),
12168 })?
12169 .flatten()
12170 .unwrap_or(0) as i32,
12171 );
12172 command.stdin(Stdio::null());
12173 #[cfg(unix)]
12174 if let Some(gate) = &mut admission_gate {
12175 gate.install_placement(&mut command);
12176 }
12177 #[cfg(unix)]
12180 {
12181 if let Some(handoff) = nonce_handoff {
12182 slots.nonce(handoff);
12183 }
12184 #[cfg(target_os = "macos")]
12185 slots
12186 .acknowledgement(exec_ack.into_descriptor())
12187 .map_err(|source| SuperviseError::Spawn {
12188 program: spec.program.clone(),
12189 source,
12190 cgroup_path: cgroup_path.clone(),
12191 })?;
12192 slots.install(command.as_std_mut());
12193 }
12194 #[cfg(not(any(unix, windows)))]
12195 let _ = nonce_handoff;
12196
12197 #[cfg(windows)]
12202 {
12203 subc_jobobject::ensure_root_job().map_err(|source| SuperviseError::Spawn {
12204 program: spec.program.clone(),
12205 source,
12206 cgroup_path: None,
12207 })?;
12208 subc_jobobject::suspend_on_create_async(&mut command);
12209 }
12210 let spawned_at = Instant::now();
12214 let delivery_context = config_delivery;
12215 let config_delivery = config_delivery
12216 .map(|delivery| {
12217 delivery.spawned(
12218 spec,
12219 spawn_admission
12220 .as_ref()
12221 .expect("delivery prepares spawn admission"),
12222 spawned_at,
12223 role,
12224 )
12225 })
12226 .transpose()?
12227 .flatten();
12228 let mut delivery_registration =
12229 delivery_context.map(|context| context.registration_guard(config_delivery.as_ref(), role));
12230 if let Some(registration) = &mut delivery_registration {
12231 registration.defer_cleanup();
12232 }
12233 if asynchronous {
12234 #[cfg(test)]
12235 if let Some(context) = delivery_context {
12236 context.pause_preflight_for_test();
12237 }
12238 }
12239 validate_spawn_program(spec, spawn_admission.as_ref(), delivery_context)?;
12240 Ok(PreparedChild {
12241 registration_spawn,
12242 registration_handle: handle.cloned(),
12243 #[cfg(any(unix, windows))]
12244 registration_channel,
12245 config_context: delivery_context.map(|delivery| delivery.runtime()),
12246 binding_publication,
12247 launch_spec,
12248 spawn_admission,
12249 command,
12250 carrying,
12251 carried: carried.clone(),
12252 #[cfg(unix)]
12253 launch_identity,
12254 #[cfg(unix)]
12255 group_target,
12256 #[cfg(target_os = "linux")]
12257 cgroup_placement: cgroup_placement.cloned(),
12258 #[cfg(target_os = "linux")]
12259 cgroup_cleanup,
12260 #[cfg(target_os = "linux")]
12261 cgroup_name,
12262 cgroup_path,
12263 #[cfg(unix)]
12264 admission_gate,
12265 output_sink,
12266 #[cfg(target_os = "macos")]
12267 privacy_exec,
12268 #[cfg(windows)]
12269 image_capture,
12270 #[cfg(windows)]
12271 nonce_handoff,
12272 spawned_at,
12273 config_delivery,
12274 delivery_registration,
12275 program_binding,
12276 })
12277}
12278
12279fn launch_child_in_slot(
12280 _spec: &ModuleSpec,
12281 ring: &Arc<Mutex<StderrRing>>,
12282 roster: &ChildRoster,
12283 prepared: PreparedChild,
12284 register_timeout: Duration,
12285) -> Result<SupervisedChild, SuperviseError> {
12286 let launch_spec = prepared.launch_spec.clone();
12287 let registration_declaration = prepared
12288 .spawn_admission
12289 .as_ref()
12290 .and_then(|admission| admission.registration_declaration(&launch_spec.module_id));
12291 let admission_base = prepared
12292 .spawn_admission
12293 .as_ref()
12294 .and_then(|admission| admission.approved_base())
12295 .cloned();
12296 let spec = &launch_spec;
12297 let expected_image = prepared
12298 .spawn_admission
12299 .as_ref()
12300 .and_then(|admission| admission.launch.as_ref())
12301 .map(|launch| launch.image);
12302 let program_launch = prepared
12303 .spawn_admission
12304 .and_then(|admission| admission.launch);
12305 let config_owner = prepared
12306 .config_context
12307 .as_ref()
12308 .map(|context| context.0.clone());
12309 let binding_publication = prepared.binding_publication;
12310 #[cfg(test)]
12311 if let Some(context) = &prepared.config_context {
12312 context.0.before_spawn_exec_for_test();
12313 }
12314 #[cfg(feature = "test-support")]
12315 if let Some(context) = &prepared.config_context {
12316 context
12317 .0
12318 .pause_spawn_before_exec()
12319 .map_err(|error| config_delivery::gate_hold_error(spec, error, Some(context)))?;
12320 }
12321 let mut command = prepared.command;
12322 let carrying = prepared.carrying;
12323 let carried = prepared.carried;
12324 #[cfg(unix)]
12325 let launch_identity = prepared.launch_identity;
12326 #[cfg(unix)]
12327 let group_target = prepared.group_target;
12328 #[cfg(target_os = "linux")]
12329 let cgroup_placement = prepared.cgroup_placement.as_ref();
12330 #[cfg(target_os = "linux")]
12331 let cgroup_name = prepared.cgroup_name;
12332 #[cfg(target_os = "linux")]
12333 let mut cgroup_cleanup = prepared.cgroup_cleanup;
12334 #[cfg(target_os = "linux")]
12335 let _ = cgroup_cleanup.0.take();
12336 let cgroup_path = prepared.cgroup_path;
12337 #[cfg(unix)]
12338 let admission_gate = prepared.admission_gate;
12339 let output_sink = prepared.output_sink;
12340 #[cfg(target_os = "macos")]
12341 let mut privacy_exec = prepared.privacy_exec;
12342 #[cfg(target_os = "macos")]
12343 if let Some(pending) = &mut privacy_exec {
12344 pending.admission_binding = binding_publication.clone();
12345 if expected_image.is_some() {
12346 pending.expected = expected_image;
12347 }
12348 }
12349 #[cfg(any(unix, windows))]
12350 let registration_channel = prepared.registration_channel;
12351 let registration_spawn = prepared.registration_spawn;
12352 let registration_handle = prepared.registration_handle;
12353 #[cfg(windows)]
12354 let image_capture = prepared.image_capture;
12355 #[cfg(windows)]
12356 let nonce_handoff = prepared.nonce_handoff;
12357 let spawned_at = prepared.spawned_at;
12358 let config_delivery = prepared.config_delivery;
12359 let delivery_registration = prepared.delivery_registration;
12360 let _program_binding = prepared.program_binding;
12361 #[cfg(all(test, unix))]
12362 containment_tests::record_spawn_attempt(spec, &mut command);
12363 #[cfg(unix)]
12364 let spawned = match admission_gate {
12365 Some(gate) => gate
12366 .spawn(
12367 command,
12368 spec,
12369 roster,
12370 launch_identity,
12371 #[cfg(target_os = "linux")]
12372 cgroup_placement
12373 .zip(cgroup_path.as_ref())
12374 .map(|(placement, path)| {
12375 (placement.clone(), cgroup_name.clone(), path.clone())
12376 }),
12377 )
12378 .map(|(child, admitted)| (child, Some(admitted))),
12379 None => {
12380 let spawned =
12381 subc_os::child_spawn::tokio_spawn(&mut command).map(|child| (child, None));
12382 drop(command);
12385 spawned
12386 }
12387 };
12388 #[cfg(not(unix))]
12389 let spawned = subc_os::child_spawn::tokio_spawn(&mut command);
12390 let spawned = match spawned {
12391 Ok(child) => child,
12392 Err(source) => {
12393 if carried
12394 .as_ref()
12395 .and_then(|container| container.group())
12396 .is_some()
12397 {
12398 containment::terminate_blocking(&roster.containment);
12399 }
12400 #[cfg(target_os = "linux")]
12401 if let Some(placement) = cgroup_placement.filter(|_| carried.is_none()) {
12402 remove_module_cgroup(placement, &cgroup_name);
12403 }
12404 return Err(SuperviseError::Spawn {
12405 program: spec.program.clone(),
12406 source,
12407 cgroup_path,
12408 });
12409 }
12410 };
12411 #[cfg(unix)]
12412 let (mut child, mut admitted) = spawned;
12413 #[cfg(unix)]
12414 let group_target = admitted
12415 .as_ref()
12416 .map(|attempt| attempt.group_target())
12417 .unwrap_or(group_target);
12418 #[cfg(not(unix))]
12419 let mut child = spawned;
12420 #[cfg(test)]
12421 spawn_record_tests::after_create(&child, spec);
12422
12423 #[cfg(windows)]
12428 let windows_image = match image_capture.bind_tokio(&child) {
12429 Ok(image) => Arc::new(image),
12430 Err(source) => {
12431 kill_and_reap_rejected_child(&mut child);
12432 return Err(SuperviseError::Spawn {
12433 program: spec.program.clone(),
12434 source,
12435 cgroup_path: None,
12436 });
12437 }
12438 };
12439 #[cfg(windows)]
12442 let job = contain_spawned_child(&mut child, spec, carried.as_ref())?;
12443 let spawned_at_ms = unix_ms_now();
12444 #[cfg(not(windows))]
12445 let spawned_from = spec.program.clone();
12446 #[cfg(windows)]
12447 let spawned_from = windows_image.path().to_path_buf();
12448 #[cfg(not(windows))]
12449 let spawned_file_identity = spawned_file_identity(&spawned_from);
12450 #[cfg(windows)]
12451 let spawned_file_identity = Some(SpawnedFileIdentity {
12452 device: 0,
12453 inode: 0,
12454 windows_image: Some(Arc::clone(&windows_image)),
12455 });
12456 let pid = child.id().ok_or_else(|| SuperviseError::Spawn {
12457 program: spec.program.clone(),
12458 source: io::Error::other("spawned child exposed no live pid"),
12459 cgroup_path: cgroup_path.clone(),
12460 })?;
12461 #[cfg(windows)]
12462 let nonce_delivery = start_windows_nonce_pipe(nonce_handoff, pid, spec);
12463 #[cfg(not(windows))]
12464 let process_start_time = admitted
12465 .as_ref()
12466 .map(|admitted| admitted.birth)
12467 .or_else(|| crate::provenance::process_start_time(pid));
12468 #[cfg(windows)]
12469 let process_start_time = Some(windows_image.start_time());
12470 let process_identity = process_start_time.map(|start_time| ProcessIdentity { pid, start_time });
12471 #[cfg(all(test, target_os = "macos"))]
12472 privacy_exec_boundary_tests::before_image_sample(spec, pid);
12473 #[cfg(all(test, target_os = "linux"))]
12474 assert!(roster.admission_event_for_test(&spec.module_id, "before_image_binding", Some(pid)));
12475 #[cfg(not(windows))]
12480 let recorded_image = observe_spawned_image(pid);
12481 #[cfg(all(test, target_os = "linux"))]
12482 let recorded_image = if roster.admission_event_for_test(
12483 &spec.module_id,
12484 "after_spawn_image_sample",
12485 Some(pid),
12486 ) {
12487 recorded_image
12488 } else {
12489 None
12490 };
12491 #[cfg(target_os = "linux")]
12492 let exited_before_binding = expected_image.is_some()
12495 && recorded_image.and_then(|image| image.executable) != expected_image
12496 && child.try_wait().ok().flatten().is_some();
12497 #[cfg(target_os = "linux")]
12498 let unavailable_initial_image = expected_image.is_some()
12499 && recorded_image.and_then(|image| image.executable).is_none()
12500 && !exited_before_binding;
12501 #[cfg(not(target_os = "linux"))]
12502 let unavailable_initial_image = false;
12503 #[cfg(target_os = "linux")]
12504 if expected_image
12505 .is_some_and(|expected| recorded_image.and_then(|image| image.executable) != Some(expected))
12506 && !exited_before_binding
12507 && !unavailable_initial_image
12508 {
12509 kill_module_cgroup(cgroup_placement, &cgroup_name);
12510 kill_and_reap_rejected_child(&mut child, group_target.clone());
12511 return Err(SuperviseError::ConfigHeld {
12512 module_id: spec.module_id.clone(),
12513 code: "program_refused".into(),
12514 detail: format!(
12515 "spawned executable {:?} does not match hashed program {:?}",
12516 recorded_image.and_then(|image| image.executable),
12517 expected_image
12518 ),
12519 });
12520 }
12521 #[cfg(not(target_os = "linux"))]
12522 let _ = expected_image;
12523 #[cfg(target_os = "macos")]
12524 let binding_ready = privacy_exec.is_none();
12525 #[cfg(target_os = "linux")]
12526 let binding_ready = !exited_before_binding;
12527 #[cfg(not(any(target_os = "macos", target_os = "linux")))]
12528 let binding_ready = cfg!(windows);
12529 #[cfg(target_os = "macos")]
12533 let recorded_image = if privacy_exec.is_some() {
12534 None
12535 } else {
12536 recorded_image
12537 };
12538 #[cfg(target_os = "linux")]
12539 let recorded_cgroup_name = cgroup_path.as_ref().map(|_| cgroup_name.clone());
12540 #[cfg(not(target_os = "linux"))]
12541 let recorded_cgroup_name = None;
12542 #[cfg(unix)]
12543 let preadmitted = admitted
12544 .as_mut()
12545 .and_then(|admitted| admitted.leader.take());
12546 #[cfg(not(unix))]
12547 let preadmitted: Option<crate::child_roster::RosterGuard> = None;
12548 let roster_guard = if let Some(guard) = preadmitted {
12549 Ok(guard)
12550 } else {
12551 roster.admit(
12552 spec.module_id.clone(),
12553 pid,
12554 spec.protocol,
12555 process_start_time,
12556 crate::child_roster::RecordedIdentity {
12557 container: None,
12558 #[cfg(unix)]
12559 carried: None,
12560 #[cfg(not(windows))]
12561 start_time: recorded_image.map(|image| image.start_time),
12562 #[cfg(windows)]
12563 start_time: process_start_time,
12564 #[cfg(not(windows))]
12565 executable: recorded_image
12566 .and_then(|image| image.executable)
12567 .map(crate::live_children::ExecutableIdentity::from),
12568 #[cfg(windows)]
12569 executable: Some(crate::live_children::ExecutableIdentity::from(
12570 windows_image.identity(),
12571 )),
12572 cgroup_name: recorded_cgroup_name,
12573 #[cfg(target_os = "linux")]
12574 cgroup_placement: cgroup_placement.cloned(),
12575 },
12576 )
12577 };
12578 let roster_guard = match roster_guard {
12579 Ok(guard) => guard,
12580 Err(failure) => {
12581 return failure.with_cleanup(|source| {
12582 #[cfg(windows)]
12583 if let Some(job) = &job {
12584 let _ = job.lock().unwrap_or_else(|p| p.into_inner()).terminate();
12585 }
12586 #[cfg(target_os = "linux")]
12587 kill_module_cgroup(cgroup_placement, &cgroup_name);
12588 #[cfg(unix)]
12589 let rejected_group = group_target.clone();
12590 kill_and_reap_rejected_child(
12591 &mut child,
12592 #[cfg(unix)]
12593 rejected_group,
12594 );
12595 #[cfg(target_os = "linux")]
12596 if let Some(placement) = cgroup_placement.filter(|_| carried.is_none()) {
12597 remove_module_cgroup(placement, &cgroup_name);
12598 }
12599 Err(SuperviseError::Spawn {
12600 program: spec.program.clone(),
12601 source: io::Error::other(format!(
12602 "could not durably record module child: {source}"
12603 )),
12604 cgroup_path,
12605 })
12606 });
12607 }
12608 };
12609 #[cfg(unix)]
12610 if admitted.is_some() {
12611 if let Some(image) = recorded_image {
12612 if let Err(source) = roster_guard.confirm_executable(image) {
12613 kill_and_reap_rejected_child(&mut child, group_target.clone());
12614 return Err(SuperviseError::Spawn {
12615 program: spec.program.clone(),
12616 source,
12617 cgroup_path,
12618 });
12619 }
12620 }
12621 }
12622 if roster.is_closed() {
12627 if carrying {
12628 #[cfg(unix)]
12629 let rejected_group = admitted
12630 .as_ref()
12631 .map(|admitted| admitted.rejection_target())
12632 .unwrap_or_else(|| group_target.clone());
12633 #[cfg(windows)]
12634 if let Some(job) = &job {
12635 let _ = job.lock().unwrap_or_else(|p| p.into_inner()).terminate();
12636 }
12637 kill_and_reap_rejected_child(
12638 &mut child,
12639 #[cfg(unix)]
12640 rejected_group,
12641 );
12642 #[cfg(all(test, unix))]
12643 let _ =
12644 roster.admission_event_for_test(&spec.module_id, "after_rejected_child", Some(pid));
12645 drop(roster_guard);
12646 return Err(SuperviseError::Spawn {
12647 program: spec.program.clone(),
12648 source: io::Error::other(
12649 "the daemon began shutting down during carried admission; ended the container",
12650 ),
12651 cgroup_path,
12652 });
12653 }
12654 #[cfg(target_os = "linux")]
12656 kill_module_cgroup(cgroup_placement, &cgroup_name);
12657 if let Err(error) = child.start_kill() {
12658 debug!(module_id = %spec.module_id, pid, %error, "kill of a process spawned during daemon shutdown failed; it may already have exited");
12659 }
12660 #[cfg(target_os = "linux")]
12661 if let Some(placement) = cgroup_placement.filter(|_| carried.is_none()) {
12662 while matches!(child.try_wait(), Ok(None)) {
12666 std::thread::yield_now();
12667 }
12668 if matches!(
12669 subc_cgroup::kill_module(Some(placement), &cgroup_name),
12670 subc_cgroup::KillOutcome::Killed
12671 ) {
12672 if let Ok(path) = placement.module_path(&cgroup_name) {
12673 while std::fs::read_to_string(path.join("cgroup.events"))
12674 .ok()
12675 .is_some_and(|events| events.lines().any(|line| line == "populated 1"))
12676 {
12677 std::thread::yield_now();
12678 }
12679 }
12680 }
12681 remove_module_cgroup(placement, &cgroup_name);
12682 }
12683 drop(roster_guard);
12684 return Err(SuperviseError::Spawn {
12685 program: spec.program.clone(),
12686 source: io::Error::other(
12687 "the daemon began shutting down while this process was starting; ended it",
12688 ),
12689 cgroup_path,
12690 });
12691 }
12692
12693 let stdout_pump = match child.stdout.take() {
12694 Some(stdout) => Some(tokio::spawn(pump_stdout_to(stdout, output_sink.clone()))),
12695 None => {
12696 warn!(
12697 module_id = %spec.module_id,
12698 "spawned child exposed no stdout pipe; file capture will be incomplete"
12699 );
12700 None
12701 }
12702 };
12703 let stderr_pump = match child.stderr.take() {
12704 Some(stderr) => {
12705 let generation = ring
12706 .lock()
12707 .unwrap_or_else(|poisoned| poisoned.into_inner())
12708 .begin_process();
12709 Some(StderrPump {
12710 task: tokio::spawn(pump_stderr_to(
12711 stderr,
12712 Arc::clone(ring),
12713 generation,
12714 output_sink,
12715 )),
12716 generation,
12717 })
12718 }
12719 None => {
12720 ring.lock()
12724 .unwrap_or_else(|poisoned| poisoned.into_inner())
12725 .mark_not_captured("stderr pipe was not available on spawn");
12726 warn!(
12727 module_id = %spec.module_id,
12728 "spawned child exposed no stderr pipe; tail will be unavailable"
12729 );
12730 None
12731 }
12732 };
12733
12734 let exec = Arc::new(crate::activity::ExecOutcome::new());
12735 #[cfg(target_os = "macos")]
12736 let pending_exec = privacy_exec.is_some();
12737 #[cfg(not(target_os = "macos"))]
12738 let pending_exec = false;
12739 if !pending_exec {
12740 exec.finish(crate::activity::ExecState::CommittedExec);
12741 }
12742 #[cfg(windows)]
12743 let subtree_target = job
12744 .as_ref()
12745 .map(|job| crate::subtree_resources::Target::Job(Arc::downgrade(job)))
12746 .unwrap_or(crate::subtree_resources::Target::Unavailable("no_job"));
12747 #[cfg(target_os = "linux")]
12748 let subtree_target = cgroup_path
12749 .map(crate::subtree_resources::Target::Cgroup)
12750 .unwrap_or(crate::subtree_resources::Target::Unavailable("no_cgroup"));
12751 #[cfg(not(any(target_os = "linux", windows)))]
12752 let subtree_target = crate::subtree_resources::Target::Unavailable("unsupported_platform");
12753 let mut supervised = SupervisedChild {
12754 registration_spawn,
12755 #[cfg(any(unix, windows))]
12756 registration_channel,
12757 live_program_binding: None,
12758 registered_config_connection: None,
12759 admission_base,
12760 registration_declaration,
12761 _program_binding: binding_publication,
12762 subtree_target,
12763 child: containment::ProcessChild::plain(child, pid),
12764 exec: crate::activity::ExecOwner(exec),
12765 #[cfg(windows)]
12766 nonce_delivery,
12767 protocol: spec.protocol,
12768 #[cfg(target_os = "linux")]
12769 module_id: cgroup_name,
12770 #[cfg(target_os = "linux")]
12771 cgroup_placement: cgroup_placement.cloned(),
12772 #[cfg(windows)]
12773 job,
12774 stdout_pump,
12775 stderr_pump,
12776 stderr_ring: Arc::clone(ring),
12777 spawned_at,
12778 config_delivery,
12779 registration_deadline: None,
12780 registration_watch: None,
12781 spawned_at_ms,
12782 spawned_from,
12783 launch_spec_digest: LaunchSpecDigest::from_spec(spec),
12784 spawned_file_identity,
12785 process_start_time,
12786 process_identity,
12787 pid,
12788 roster_guard: Some(roster_guard),
12789 #[cfg(target_os = "macos")]
12790 privacy_exec,
12791 #[cfg(target_os = "macos")]
12792 report_ready: Arc::new(OnceLock::new()),
12793 spawn_failure: None,
12794 admission_kill_sent: false,
12795 };
12796 if let Err(error) = containment::adopt(&mut supervised, spec, roster) {
12797 supervised.child.reject(
12798 #[cfg(unix)]
12799 group_target,
12800 );
12801 #[cfg(all(test, unix))]
12802 let _ = roster.admission_event_for_test(&spec.module_id, "after_rejected_child", Some(pid));
12803 return Err(error);
12804 }
12805 if let (Some(launch), Some(owner), Some(base)) = (
12806 program_launch,
12807 config_owner,
12808 supervised.admission_base.clone(),
12809 ) {
12810 supervised.live_program_binding =
12811 content_binding::LiveProgramBinding::new(launch, &supervised, owner, base);
12812 if let (Some(publication), Some(live)) = (
12813 &supervised._program_binding,
12814 &supervised.live_program_binding,
12815 ) {
12816 publication.attach(live);
12817 }
12818 }
12819 #[cfg(unix)]
12820 if let Some(admitted) = admitted.take() {
12821 admitted.disarm();
12822 }
12823 #[cfg(windows)]
12824 if let Err(error) = resume_suspended_child(pid, spec) {
12825 containment::terminate_blocking(&roster.containment);
12826 let _ = supervised.child.start_kill();
12827 return Err(error);
12828 }
12829 if binding_ready {
12830 let checked = if unavailable_initial_image {
12834 Err(crate::config_gate::GateError::new(
12835 "config_delivery_deferred",
12836 "initial process image is unavailable",
12837 ))
12838 } else {
12839 match (
12840 &supervised._program_binding,
12841 &supervised.live_program_binding,
12842 ) {
12843 (Some(binding), _) => binding.verify(),
12844 (None, Some(live)) => live.check(),
12845 _ if supervised.admission_base.is_some() => {
12846 Err(crate::config_gate::GateError::new(
12847 "config_delivery_deferred",
12848 "live program witness is unavailable",
12849 ))
12850 }
12851 _ => Ok(()),
12852 }
12853 };
12854 #[cfg(all(test, target_os = "linux"))]
12855 let checked = if roster.admission_event_for_test(
12856 &spec.module_id,
12857 "after_content_witness_check",
12858 Some(pid),
12859 ) {
12860 checked
12861 } else {
12862 Err(crate::config_gate::GateError::new(
12863 "config_delivery_deferred",
12864 "current process image is unavailable",
12865 ))
12866 };
12867 if let Err(error) = checked {
12868 if let Some(binding) = &mut supervised._program_binding {
12869 binding.reject();
12870 }
12871 if error.code == "program_refused" || supervised.try_wait().ok().flatten().is_none() {
12874 supervised.spawn_failure = Some(format!("{}: {}", error.code, error.detail));
12875 #[cfg(all(test, target_os = "linux"))]
12876 assert!(roster.admission_event_for_test(
12877 &spec.module_id,
12878 if unavailable_initial_image {
12879 "before_initial_image_kill"
12880 } else {
12881 "before_content_witness_kill"
12882 },
12883 Some(pid),
12884 ));
12885 let _ = supervised.start_admission_kill();
12886 #[cfg(all(test, target_os = "linux"))]
12887 if supervised.admission_kill_sent {
12888 assert!(roster.admission_event_for_test(
12889 &spec.module_id,
12890 "after_admission_kill_sent",
12891 Some(pid),
12892 ));
12893 }
12894 }
12895 }
12896 }
12897 if let Some(registration) = delivery_registration {
12898 registration.commit();
12899 }
12900 if let Some(spawn) = &supervised.registration_spawn {
12901 if let Some(identity) = supervised.process_identity {
12902 let _ = spawn.identity.set(identity);
12903 }
12904 if supervised.spawn_failure.is_some() {
12905 spawn.refuse();
12906 } else if binding_ready {
12907 spawn.ready();
12908 }
12909 #[cfg(any(unix, windows))]
12910 if let (Some(handle), Some(channel)) =
12911 (registration_handle, supervised.registration_channel.take())
12912 {
12913 handle.serve_registration(channel, spawn.clone(), pid, spawned_at + register_timeout);
12914 }
12915 }
12916 Ok(supervised)
12917}
12918
12919#[cfg(test)]
12920mod spawn_record_tests {
12921 use super::*;
12922
12923 thread_local! {
12924 static LAST_PID: std::cell::Cell<Option<u32>> = const { std::cell::Cell::new(None) };
12925 }
12926
12927 pub(super) fn after_create(child: &Child, _spec: &ModuleSpec) {
12928 LAST_PID.set(child.id());
12929 #[cfg(windows)]
12930 if let Some((_, address)) = _spec
12931 .env
12932 .iter()
12933 .find(|(key, _)| key == "SUBC_TEST_AFTER_CREATE")
12934 {
12935 use std::io::{Read, Write};
12936 let mut stream = std::net::TcpStream::connect(address).unwrap();
12937 stream
12938 .write_all(&child.id().unwrap().to_ne_bytes())
12939 .unwrap();
12940 let mut release = [0];
12941 stream.read_exact(&mut release).unwrap();
12942 }
12943 }
12944
12945 #[cfg(windows)]
12946 pub(super) fn assignment_failpoint(spec: &ModuleSpec) -> io::Result<()> {
12947 if spec
12948 .env
12949 .iter()
12950 .any(|(key, _)| key == "SUBC_TEST_ASSIGNMENT_FAILURE")
12951 {
12952 Err(io::Error::other("injected module job assignment failure"))
12953 } else {
12954 Ok(())
12955 }
12956 }
12957
12958 fn spec() -> ModuleSpec {
12959 ModuleSpec {
12960 module_id: "spawn-record-worker".into(),
12961 #[cfg(unix)]
12962 program: "/bin/sleep".into(),
12963 #[cfg(windows)]
12964 program: std::env::current_exe().unwrap(),
12965 #[cfg(unix)]
12966 args: vec!["60".into()],
12967 #[cfg(windows)]
12968 args: vec![
12969 "--exact".into(),
12970 "supervise::spawn_record_tests::windows_store_touch_fixture".into(),
12971 "--nocapture".into(),
12972 ],
12973 env: vec![],
12974 reserved: false,
12975 reserved_prefixes: vec![],
12976 protocol: ModuleProtocol::None,
12977 overlap: Default::default(),
12978 carry_containment: false,
12979 }
12980 }
12981
12982 #[tokio::test]
12983 async fn record_sync_failure_fails_spawn_and_reaps_child() {
12984 let home = cortexkit_test_support::ScratchDir::new("record-sync-failure");
12985 let supervisor =
12986 Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default())
12987 .with_live_children_record(home.join("live-children.json"));
12988 crate::live_children::fail_next_file_sync();
12989 let result = supervisor.spawn(spec());
12990 let failed = result.is_err();
12991 if let Ok(module) = result {
12992 module.stop().await.unwrap();
12993 }
12994 assert!(failed, "record failure must fail spawn");
12995 let pid = LAST_PID.get().expect("the real OS child was created");
12996 assert!(
12997 subc_os::Process::open(pid).unwrap().is_none(),
12998 "child {pid} survived rejected admission or was not reaped"
12999 );
13000 }
13001
13002 #[cfg(target_os = "macos")]
13003 #[test]
13004 #[ignore = "manual on-device latency measurement"]
13005 fn durable_record_write_latency_50() {
13006 let home = cortexkit_test_support::ScratchDir::new("record-latency");
13007 let path = home.join("live-children.json");
13008 let children = vec![crate::live_children::LiveChild {
13009 module_id: "latency-worker".into(),
13010 pid: std::process::id(),
13011 protocol: ModuleProtocol::None,
13012 start_time: Some(1),
13013 executable: None,
13014 cgroup_name: None,
13015 container: None,
13016 }];
13017 let mut samples = Vec::new();
13018 for _ in 0..50 {
13019 let start = std::time::Instant::now();
13020 crate::live_children::write_record(&path, &children).unwrap();
13021 samples.push(start.elapsed());
13022 }
13023 samples.sort();
13024 eprintln!(
13025 "durable_record_write_latency_50: n=50 median={:?} p95={:?}",
13026 (samples[24] + samples[25]) / 2,
13027 samples[47]
13028 );
13029 }
13030
13031 #[cfg(windows)]
13032 #[test]
13033 fn windows_store_touch_fixture() {
13034 if let Some(path) = std::env::var_os("SUBC_TEST_STORE_TOUCH") {
13035 std::fs::write(path, b"module ran").unwrap();
13036 }
13037 }
13038
13039 #[cfg(windows)]
13040 #[tokio::test]
13041 async fn windows_daemon_fixture() {
13042 let Ok(address) = std::env::var("SUBC_TEST_DAEMON_BARRIER") else {
13043 return;
13044 };
13045 let mut spec = spec();
13046 spec.env.push(("SUBC_TEST_AFTER_CREATE".into(), address));
13047 let supervisor =
13048 Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default());
13049 let _module = supervisor.spawn(spec).unwrap();
13050 }
13051
13052 #[cfg(windows)]
13053 #[test]
13054 fn windows_daemon_death_before_module_job_assignment_leaves_no_child() {
13055 use std::io::Read;
13056 let listener = std::net::TcpListener::bind("127.0.0.1:0").unwrap();
13057 let mut command = std::process::Command::new(std::env::current_exe().unwrap());
13060 command
13061 .args([
13062 "--exact",
13063 "supervise::spawn_record_tests::windows_daemon_fixture",
13064 "--nocapture",
13065 ])
13066 .env(
13067 "SUBC_TEST_DAEMON_BARRIER",
13068 listener.local_addr().unwrap().to_string(),
13069 );
13070 let mut daemon = subc_jobobject::spawn_contained(&mut command).unwrap();
13071 listener.set_nonblocking(true).unwrap();
13072 let deadline = std::time::Instant::now() + Duration::from_secs(30);
13073 let mut stream = loop {
13074 if let Ok((stream, _)) = listener.accept() {
13075 break stream;
13076 }
13077 assert!(
13078 daemon.child.try_wait().unwrap().is_none(),
13079 "daemon failed before rendezvous"
13080 );
13081 assert!(
13082 std::time::Instant::now() < deadline,
13083 "daemon did not reach CreateProcess failpoint"
13084 );
13085 std::thread::yield_now();
13086 };
13087 stream
13088 .set_read_timeout(Some(Duration::from_secs(30)))
13089 .unwrap();
13090 let mut pid = [0; 4];
13091 stream.read_exact(&mut pid).unwrap();
13092 let pid = u32::from_ne_bytes(pid);
13093 assert!(subc_jobobject::process_exists(pid));
13094 daemon.child.kill().unwrap();
13097 daemon.child.wait().unwrap();
13098 let exited = subc_jobobject::wait_for_process_exit(pid, Duration::from_secs(10));
13099 if !exited {
13100 daemon.job.terminate().unwrap();
13101 }
13102 assert!(exited, "suspended child {pid} outlived hard-killed daemon");
13103 }
13104
13105 #[cfg(windows)]
13106 #[tokio::test]
13107 async fn windows_containment_failure_never_resumes() {
13108 let home = cortexkit_test_support::ScratchDir::new("assignment-refusal");
13109 let sentinel = home.join("store-touched");
13110 let mut spec = spec();
13111 spec.env
13112 .push(("SUBC_TEST_ASSIGNMENT_FAILURE".into(), "1".into()));
13113 spec.env.push((
13114 "SUBC_TEST_STORE_TOUCH".into(),
13115 sentinel.display().to_string(),
13116 ));
13117 let supervisor =
13118 Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default());
13119 let result = supervisor.spawn(spec);
13120 if let Ok(module) = result {
13123 let _ = tokio::time::timeout(Duration::from_secs(10), async {
13124 while !sentinel.exists() && module.state().unwrap() == ModuleState::Running {
13125 tokio::task::yield_now().await;
13126 }
13127 })
13128 .await;
13129 let _ = module.stop().await;
13130 panic!(
13131 "assignment failure resumed a module; store touched={}",
13132 sentinel.exists()
13133 );
13134 }
13135 assert!(
13136 !sentinel.exists(),
13137 "refused module executed store-touch sentinel"
13138 );
13139 assert!(!subc_jobobject::process_exists(LAST_PID.get().unwrap()));
13140 }
13141
13142 #[cfg(windows)]
13143 #[tokio::test]
13144 async fn windows_nested_daemon_fixture() {
13145 let Some(sentinel) = std::env::var_os("SUBC_TEST_NESTED_STORE") else {
13146 return;
13147 };
13148 let mut spec = spec();
13149 spec.env.push((
13150 "SUBC_TEST_STORE_TOUCH".into(),
13151 sentinel.to_str().unwrap().into(),
13152 ));
13153 let supervisor =
13154 Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default());
13155 let module = supervisor
13156 .spawn(spec)
13157 .expect("per-module job must nest below daemon and host jobs");
13158 tokio::time::timeout(Duration::from_secs(10), async {
13159 while !PathBuf::from(&sentinel).exists() {
13160 tokio::task::yield_now().await;
13161 }
13162 })
13163 .await
13164 .unwrap();
13165 module.stop().await.unwrap();
13166 }
13167
13168 #[cfg(windows)]
13169 #[test]
13170 fn windows_module_job_nests_under_host_job() {
13171 let home = cortexkit_test_support::ScratchDir::new("nested-jobs");
13172 let sentinel = home.join("nested-store-touched");
13173 let mut command = std::process::Command::new(std::env::current_exe().unwrap());
13174 command
13175 .args([
13176 "--exact",
13177 "supervise::spawn_record_tests::windows_nested_daemon_fixture",
13178 "--nocapture",
13179 ])
13180 .env("SUBC_TEST_NESTED_STORE", &sentinel);
13181 let mut daemon = subc_jobobject::spawn_contained(&mut command).unwrap();
13182 assert!(daemon.child.wait().unwrap().success());
13183 assert!(sentinel.exists(), "nested protocol:none module never ran");
13184 }
13185}
13186
13187#[cfg(target_os = "linux")]
13188pub(crate) fn kill_module_cgroup(placement: Option<&subc_cgroup::Placement>, module_id: &str) {
13189 use subc_cgroup::KillOutcome;
13190 match subc_cgroup::kill_module(placement, module_id) {
13191 KillOutcome::Killed => {}
13192 KillOutcome::NotPlaced | KillOutcome::Unsupported => {
13193 debug!(
13194 module_id,
13195 "cgroup tree kill unavailable; using direct-child kill"
13196 );
13197 }
13198 KillOutcome::IoError { path, error } => {
13199 warn!(module_id, path = %path.display(), %error, "cgroup tree kill failed; using direct-child kill");
13200 }
13201 }
13202}
13203
13204#[cfg(windows)]
13212fn contain_spawned_child(
13213 child: &mut Child,
13214 spec: &ModuleSpec,
13215 carried: Option<&Arc<containment::CarriedContainment>>,
13216) -> Result<Option<Arc<Mutex<subc_jobobject::JobObject>>>, SuperviseError> {
13217 let assignment = || {
13218 let job = match carried {
13219 Some(container) => container.job(),
13220 None => Arc::new(Mutex::new(subc_jobobject::JobObject::new()?)),
13221 };
13222 #[cfg(test)]
13223 spawn_record_tests::assignment_failpoint(spec)?;
13224 job.lock()
13225 .unwrap_or_else(|p| p.into_inner())
13226 .assign(child)?;
13227 Ok::<_, io::Error>(job)
13228 };
13229 match assignment() {
13230 Ok(job) => Ok(Some(job)),
13231 Err(source) => {
13232 kill_and_reap_rejected_child(child);
13233 Err(SuperviseError::Spawn {
13234 program: spec.program.clone(),
13235 source: io::Error::other(format!(
13236 "Windows module job containment failed: {source}"
13237 )),
13238 cgroup_path: None,
13239 })
13240 }
13241 }
13242}
13243
13244fn kill_and_reap_rejected_child(child: &mut Child, #[cfg(unix)] group: containment::GroupTarget) {
13247 #[cfg(unix)]
13248 group.signal_rejected(child);
13249 let _ = child.start_kill();
13250 while matches!(child.try_wait(), Ok(None)) {
13251 std::thread::yield_now();
13252 }
13253}
13254
13255#[cfg(windows)]
13260fn resume_suspended_child(pid: u32, spec: &ModuleSpec) -> Result<(), SuperviseError> {
13261 if let Err(source) = subc_jobobject::resume_main_thread(pid) {
13262 let _ = subc_os::child_spawn::status(
13266 std::process::Command::new("taskkill.exe")
13267 .args(["/PID", &pid.to_string(), "/T", "/F"])
13268 .stdin(Stdio::null())
13269 .stdout(Stdio::null())
13270 .stderr(Stdio::null()),
13271 );
13272 return Err(SuperviseError::Spawn {
13273 program: spec.program.clone(),
13274 source,
13275 cgroup_path: None,
13276 });
13277 }
13278 Ok(())
13279}
13280
13281#[cfg(target_os = "linux")]
13282fn remove_module_cgroup(placement: &subc_cgroup::Placement, module_id: &str) {
13283 match placement.remove_module(module_id) {
13284 Ok(()) => debug!(module_id, "removed module cgroup after process exit"),
13285 Err(error) => warn!(
13286 module_id,
13287 error = %error,
13288 "could not remove module cgroup after process exit; continuing teardown"
13289 ),
13290 }
13291}
13292
13293#[cfg(target_os = "linux")]
13294const REAPED_CGROUP_TERM_GRACE: Duration = Duration::from_secs(2);
13295#[cfg(target_os = "linux")]
13296const REAPED_CGROUP_FREEZE_TIMEOUT: Duration = Duration::from_millis(100);
13297
13298#[cfg(target_os = "linux")]
13299async fn cleanup_reaped_cgroup(placement: &subc_cgroup::Placement, module_id: &str) {
13300 let path = placement.module_path(module_id).ok();
13301 if path.as_ref().is_some_and(|path| {
13302 std::fs::read_to_string(path.join("cgroup.events"))
13303 .ok()
13304 .is_some_and(|events| events.lines().any(|line| line == "populated 0"))
13305 }) {
13306 remove_module_cgroup(placement, module_id);
13307 return;
13308 }
13309
13310 if terminate_reaped_cgroup(placement, module_id).await {
13313 if let Some(path) = &path {
13314 let deadline = Instant::now() + REAPED_CGROUP_TERM_GRACE;
13315 while cgroup_is_populated(path) && Instant::now() < deadline {
13316 sleep_until((Instant::now() + Duration::from_millis(1)).min(deadline)).await;
13317 }
13318 }
13319 }
13320
13321 if matches!(
13325 subc_cgroup::kill_module(Some(placement), module_id),
13326 subc_cgroup::KillOutcome::Killed
13327 ) {
13328 if let Some(path) = &path {
13329 while cgroup_is_populated(path) {
13330 sleep(Duration::from_millis(1)).await;
13331 }
13332 }
13333 }
13334 remove_module_cgroup(placement, module_id);
13335}
13336
13337#[cfg(target_os = "linux")]
13338fn cgroup_is_populated(path: &std::path::Path) -> bool {
13339 std::fs::read_to_string(path.join("cgroup.events"))
13340 .ok()
13341 .is_some_and(|events| events.lines().any(|line| line == "populated 1"))
13342}
13343
13344#[cfg(target_os = "linux")]
13345async fn terminate_reaped_cgroup(placement: &subc_cgroup::Placement, module_id: &str) -> bool {
13346 let frozen =
13347 match subc_cgroup::freeze_module(Some(placement), module_id, REAPED_CGROUP_FREEZE_TIMEOUT)
13348 .await
13349 {
13350 subc_cgroup::FreezeOutcome::Frozen(frozen) => frozen,
13351 outcome => {
13352 tracing::debug!(
13353 module_id,
13354 ?outcome,
13355 "cgroup freeze failed; skipping SIGTERM grace"
13356 );
13357 return false;
13358 }
13359 };
13360 let listed = match frozen.processes() {
13361 subc_cgroup::ProcsOutcome::Listed(pids) => {
13362 for pid in pids.into_iter().filter(|pid| *pid > 1) {
13363 if let Some(pid) = rustix::process::Pid::from_raw(pid) {
13364 if let Err(error) =
13365 rustix::process::kill_process(pid, rustix::process::Signal::TERM)
13366 {
13367 tracing::debug!(module_id, %pid, %error, "leftover cgroup SIGTERM failed");
13368 }
13369 }
13370 }
13371 true
13372 }
13373 outcome => {
13374 tracing::debug!(
13375 module_id,
13376 ?outcome,
13377 "cgroup process listing failed; skipping SIGTERM grace"
13378 );
13379 false
13380 }
13381 };
13382 match frozen.thaw() {
13383 subc_cgroup::ThawOutcome::Thawed => listed,
13384 outcome => {
13385 tracing::debug!(
13386 module_id,
13387 ?outcome,
13388 "cgroup thaw failed; proceeding to cgroup.kill"
13389 );
13390 false
13391 }
13392 }
13393}
13394
13395#[cfg(target_os = "linux")]
13396fn apply_cgroup_placement(
13397 command: &mut Command,
13398 spec: &ModuleSpec,
13399 path: &std::path::Path,
13400) -> Result<(), SuperviseError> {
13401 subc_cgroup::apply(command, path).map_err(|source| SuperviseError::Cgroup {
13402 module_id: spec.module_id.clone(),
13403 source,
13404 })
13405}
13406
13407fn capture_retention(spec: &ModuleSpec) -> Retention {
13408 let defaults = Retention::default();
13409 let value = |name: &str| {
13410 spec.env
13411 .iter()
13412 .rev()
13413 .find_map(|(key, value)| (key == name).then_some(value.as_str()))
13414 };
13415 Retention {
13416 max_file_mb: value(CAPTURE_MAX_FILE_MB_ENV)
13417 .and_then(|value| value.parse().ok())
13418 .unwrap_or(defaults.max_file_mb),
13419 keep: value(CAPTURE_KEEP_ENV)
13420 .and_then(|value| value.parse().ok())
13421 .unwrap_or(defaults.keep),
13422 max_age_days: value(CAPTURE_MAX_AGE_DAYS_ENV)
13423 .and_then(|value| value.parse().ok())
13424 .unwrap_or(defaults.max_age_days),
13425 }
13426}
13427
13428fn generate_launch_nonce() -> Result<String, SuperviseError> {
13431 let mut bytes = [0u8; 32];
13432 getrandom::getrandom(&mut bytes).map_err(|source| SuperviseError::LaunchNonce {
13433 reason: source.to_string(),
13434 })?;
13435 let mut hex = String::with_capacity(64);
13436 for b in bytes {
13437 use std::fmt::Write;
13438 let _ = write!(hex, "{b:02x}");
13439 }
13440 Ok(hex)
13441}
13442
13443fn constant_time_eq(a: &[u8], b: &[u8]) -> bool {
13446 if a.len() != b.len() {
13447 return false;
13448 }
13449 let mut diff = 0u8;
13450 for (x, y) in a.iter().zip(b.iter()) {
13451 diff |= x ^ y;
13452 }
13453 diff == 0
13454}
13455
13456#[cfg(not(windows))]
13460fn observe_spawned_image(pid: u32) -> Option<subc_os::Observation> {
13461 let process = subc_os::Process::open(pid).ok().flatten()?;
13462 #[cfg(target_os = "linux")]
13463 return sample_after_linux_exec(
13464 || {
13465 let stat = std::fs::read_to_string(format!("/proc/{pid}/stat")).ok()?;
13466 let committed = linux_exec_committed(&stat);
13467 #[cfg(test)]
13468 spawn_image_tests::report_exec_state(committed);
13469 committed
13470 },
13471 || process.observe(),
13472 );
13473 #[cfg(not(target_os = "linux"))]
13474 process.observe()
13475}
13476
13477#[cfg(target_os = "linux")]
13481fn linux_exec_committed(stat: &str) -> Option<bool> {
13482 const PF_FORKNOEXEC: u64 = 0x40;
13483 let flags = stat.rsplit_once(')')?.1.split_whitespace().nth(6)?;
13484 Some(flags.parse::<u64>().ok()? & PF_FORKNOEXEC == 0)
13485}
13486
13487#[cfg(target_os = "linux")]
13488fn sample_after_linux_exec(
13489 mut exec_committed: impl FnMut() -> Option<bool>,
13490 observe: impl FnOnce() -> Option<subc_os::Observation>,
13491) -> Option<subc_os::Observation> {
13492 let deadline = std::time::Instant::now() + Duration::from_secs(1);
13493 while !exec_committed()? {
13494 if std::time::Instant::now() >= deadline {
13498 return None;
13499 }
13500 std::thread::sleep(Duration::from_micros(50));
13501 }
13502 observe()
13503}
13504
13505#[cfg(all(test, target_os = "linux"))]
13506mod spawn_image_tests {
13507 use super::*;
13508 use std::cell::{Cell, RefCell};
13509
13510 enum ExecEvent {
13511 Uncommitted,
13512 Image(Option<subc_os::Observation>),
13513 }
13514
13515 thread_local! {
13516 static EXEC_EVENTS: RefCell<Option<std::sync::mpsc::Sender<ExecEvent>>> = const { RefCell::new(None) };
13517 }
13518
13519 pub(super) fn report_exec_state(committed: Option<bool>) {
13520 if committed == Some(false) {
13521 EXEC_EVENTS.with(|events| {
13522 if let Some(events) = events.borrow_mut().take() {
13523 let _ = events.send(ExecEvent::Uncommitted);
13524 }
13525 });
13526 }
13527 }
13528
13529 #[test]
13530 fn linux_spawn_observation_waits_for_a_paused_fork_to_exec() {
13531 use std::io::{Read, Write};
13532 use std::os::fd::AsRawFd;
13533
13534 let descriptor = std::fs::File::open("/dev/null").unwrap();
13535 let mut command = std::process::Command::new("/bin/sleep");
13536 command.arg("30");
13537 let (mut ready, mut release) =
13538 subc_os::fork_exec_test::pause_before_exec(&mut command, descriptor.as_raw_fd())
13539 .unwrap();
13540 let spawn = std::thread::spawn(move || command.spawn());
13541 let mut message = [0u8; 8];
13542 ready.read_exact(&mut message).unwrap();
13543 let pid = i32::from_ne_bytes(message[..4].try_into().unwrap()) as u32;
13544 let before = std::fs::read_to_string(format!("/proc/{pid}/stat"));
13545 let inherited = subc_os::Process::open(pid).unwrap().unwrap().observe();
13546 let (events, received) = std::sync::mpsc::channel();
13547 let observer = std::thread::spawn(move || {
13548 EXEC_EVENTS.with(|slot| *slot.borrow_mut() = Some(events.clone()));
13549 let image = observe_spawned_image(pid);
13550 let _ = events.send(ExecEvent::Image(image));
13551 });
13552 let first = received.recv_timeout(Duration::from_secs(5));
13556 release.write_all(&[1]).unwrap();
13557 let mut child = spawn.join().unwrap().unwrap();
13558 let after = match first {
13559 Ok(ExecEvent::Uncommitted) => match received.recv_timeout(Duration::from_secs(5)) {
13560 Ok(ExecEvent::Image(image)) => image,
13561 _ => None,
13562 },
13563 Ok(ExecEvent::Image(image)) => image,
13564 Err(_) => None,
13565 };
13566 observer.join().unwrap();
13567 child.kill().unwrap();
13568 child.wait().unwrap();
13569
13570 assert_eq!(linux_exec_committed(&before.unwrap()), Some(false));
13571 assert_eq!(
13572 inherited.unwrap().executable,
13573 subc_os::file_identity(&std::env::current_exe().unwrap())
13574 );
13575 assert_eq!(
13576 after.unwrap().executable,
13577 subc_os::file_identity(std::path::Path::new("/bin/sleep"))
13578 );
13579 }
13580
13581 #[test]
13582 fn spawned_image_is_not_sampled_until_linux_exec_commits() {
13583 let committed = Cell::new(false);
13584 let inherited = subc_os::Observation {
13585 start_time: 23,
13586 executable: Some(subc_os::FileIdentity {
13587 device: 29,
13588 inode: 31,
13589 }),
13590 };
13591 let module = subc_os::Observation {
13592 start_time: 23,
13593 executable: Some(subc_os::FileIdentity {
13594 device: 29,
13595 inode: 37,
13596 }),
13597 };
13598 let polls = Cell::new(0);
13599 let image = sample_after_linux_exec(
13600 || {
13601 polls.set(polls.get() + 1);
13602 committed.set(polls.get() == 2);
13605 Some(committed.get())
13606 },
13607 || Some(if committed.get() { module } else { inherited }),
13608 );
13609 assert_eq!(image, Some(module));
13610 assert_eq!(polls.get(), 2);
13611 }
13612
13613 #[test]
13614 fn linux_exec_flag_is_read_from_flags_after_the_last_parenthesis() {
13615 assert_eq!(
13616 linux_exec_committed("17 (a ) b) R 1 2 3 4 5 4194368"),
13617 Some(false)
13618 );
13619 assert_eq!(
13620 linux_exec_committed("17 (a ) b) R 1 2 3 4 5 4194304"),
13621 Some(true)
13622 );
13623 assert_eq!(linux_exec_committed("17 (a) R 1 2 3 4 5 invalid"), None);
13624 assert_eq!(linux_exec_committed("17 (a) R"), None);
13625 }
13626
13627 #[test]
13628 fn unreadable_exec_state_never_samples_an_inherited_image() {
13629 assert_eq!(
13630 sample_after_linux_exec(
13631 || None,
13632 || panic!("must not sample before exec confirmation")
13633 ),
13634 None
13635 );
13636 }
13637
13638 #[test]
13639 fn uncommitted_exec_times_out_without_sampling_an_inherited_image() {
13640 assert_eq!(
13641 sample_after_linux_exec(
13642 || Some(false),
13643 || panic!("must not sample after an exec timeout")
13644 ),
13645 None
13646 );
13647 }
13648}
13649
13650#[cfg(windows)]
13653pub(crate) fn windows_spawn_capture(spec: &ModuleSpec) -> io::Result<subc_os::ExecutableCapture> {
13654 if let Err(error) = require_program_search_path(spec) {
13655 return Err(match error {
13656 SuperviseError::Spawn { source, .. } => source,
13657 error => io::Error::other(error),
13658 });
13659 }
13660 let mut program = spec.program.clone();
13661 if program.is_absolute() || program.components().count() > 1 {
13662 match subc_os::ExecutableCapture::open(&program) {
13665 Ok(capture) => return Ok(capture),
13666 Err(error)
13667 if error.kind() == io::ErrorKind::NotFound && program.extension().is_none() => {}
13668 Err(error) => return Err(error),
13669 }
13670 program.set_extension("exe");
13671 return subc_os::ExecutableCapture::open(&program);
13672 }
13673 if program.extension().is_none() {
13674 program.set_extension("exe");
13675 }
13676 let custom_path = spec
13677 .env
13678 .iter()
13679 .rev()
13680 .find(|(key, _)| key.eq_ignore_ascii_case("PATH"));
13681 let mut directories = Vec::new();
13682 if let Some(parent) = std::env::current_exe()?.parent() {
13685 directories.push(parent.to_path_buf());
13686 }
13687 directories.push(std::env::current_dir()?);
13688 if let Some(windows) = std::env::var_os("SystemRoot") {
13689 let windows = PathBuf::from(windows);
13690 directories.push(windows.join("System32"));
13691 directories.push(windows.clone());
13692 }
13693 let path = custom_path
13694 .map(|(_, value)| std::ffi::OsString::from(value))
13695 .or_else(|| std::env::var_os("PATH"));
13696 if let Some(path) = path {
13697 directories.extend(std::env::split_paths(&path));
13698 }
13699 for directory in directories {
13700 match subc_os::ExecutableCapture::open(&directory.join(&program)) {
13701 Ok(capture) => return Ok(capture),
13702 Err(error) if error.kind() == io::ErrorKind::NotFound => continue,
13703 Err(error) => return Err(error),
13704 }
13705 }
13706 Err(io::Error::new(
13707 io::ErrorKind::NotFound,
13708 "resolved executable not found",
13709 ))
13710}
13711
13712async fn spawn_and_mark_running_async(
13713 spec: &ModuleSpec,
13714 runtime: &SupervisorRuntimeConfig,
13715 snapshot: &SharedSnapshot,
13716) -> Result<SupervisedChild, SuperviseError> {
13717 let activity_spawn = runtime.begin_activity_spawn(&spec.module_id);
13718 let registration_watch = watch_registration(&runtime.registry, spec);
13719 #[cfg(unix)]
13720 let joining_group = containment::current(&runtime.child_roster.containment)
13721 .is_some_and(|container| container.group().is_some());
13722 let result = spawn_child_async(spec, runtime).await;
13723 #[cfg(unix)]
13724 let result = if result.is_err()
13725 && joining_group
13726 && !runtime.child_roster.is_closed()
13727 && lock_snapshot(snapshot).is_ok_and(|state| state.enabled)
13728 && runtime
13729 .child_roster
13730 .previous_instance(&spec.module_id)
13731 .is_none()
13732 {
13733 warn!(module_id = %spec.module_id, "carried process-group spawn failed; trying one fresh container");
13736 spawn_child_async(spec, runtime).await
13737 } else {
13738 result
13739 };
13740 let mut child = result?;
13741 arm_registration_deadline(&mut child, runtime.register_timeout, registration_watch);
13742 if let Err(error) = set_running(snapshot, &child, &spec.module_id, &runtime.spawn_events) {
13743 containment::terminate_blocking(&runtime.child_roster.containment);
13744 return Err(error);
13745 }
13746 if let Some(spawn) = activity_spawn {
13747 spawn.complete(snapshot);
13748 }
13749 Ok(child)
13750}
13751
13752#[cfg(test)]
13753fn spawn_and_mark_running(
13754 spec: &ModuleSpec,
13755 runtime: &SupervisorRuntimeConfig,
13756 snapshot: &SharedSnapshot,
13757) -> Result<SupervisedChild, SuperviseError> {
13758 let activity_spawn = runtime.begin_activity_spawn(&spec.module_id);
13759 let registration_watch = watch_registration(&runtime.registry, spec);
13760 #[cfg(unix)]
13761 let joining_group = containment::current(&runtime.child_roster.containment)
13762 .is_some_and(|container| container.group().is_some());
13763 let spawn = || {
13764 spawn_child(
13765 spec,
13766 runtime.connection_file_path.as_deref(),
13767 runtime.supervisor_handle.as_ref(),
13768 &runtime.stderr_ring,
13769 runtime.capture_logs_dir.as_deref(),
13770 &runtime.child_roster,
13771 runtime.spawn_delivery(),
13772 #[cfg(target_os = "linux")]
13773 runtime.cgroup_placement.as_ref(),
13774 runtime.register_timeout,
13775 )
13776 };
13777 let result = spawn();
13778 #[cfg(unix)]
13779 let result = if result.is_err()
13780 && joining_group
13781 && !runtime.child_roster.is_closed()
13782 && lock_snapshot(snapshot).is_ok_and(|state| state.enabled)
13783 && runtime
13784 .child_roster
13785 .previous_instance(&spec.module_id)
13786 .is_none()
13787 {
13788 warn!(module_id = %spec.module_id, "carried process-group spawn failed; trying one fresh container");
13792 spawn()
13793 } else {
13794 result
13795 };
13796 let child = result?;
13797 let mut child = child;
13798 arm_registration_deadline(&mut child, runtime.register_timeout, registration_watch);
13799 if let Err(error) = set_running(snapshot, &child, &spec.module_id, &runtime.spawn_events) {
13800 containment::terminate_blocking(&runtime.child_roster.containment);
13801 return Err(error);
13802 }
13803 if let Some(spawn) = activity_spawn {
13804 spawn.complete(snapshot);
13805 }
13806 Ok(child)
13807}
13808
13809struct RegistrationWatch {
13810 changes: watch::Receiver<u64>,
13811 baseline: u64,
13812}
13813
13814fn watch_registration(registry: &Registry, spec: &ModuleSpec) -> Option<RegistrationWatch> {
13818 if spec.protocol != ModuleProtocol::Subc {
13819 return None;
13820 }
13821 let changes = registry.subscribe_module_changes(&spec.module_id).ok()?;
13822 let baseline = *changes.borrow();
13823 Some(RegistrationWatch { changes, baseline })
13824}
13825
13826fn arm_registration_deadline(
13827 child: &mut SupervisedChild,
13828 budget: Duration,
13829 registration_watch: Option<RegistrationWatch>,
13830) {
13831 child.registration_watch = registration_watch;
13832 child.registration_deadline = (child.protocol == ModuleProtocol::Subc
13833 && child.awaiting_registration())
13834 .then_some(child.spawned_at + budget);
13835}
13836
13837async fn stop_unregistered_child(child: &mut SupervisedChild) -> io::Result<()> {
13838 child.retire_registration();
13839 if let Some(container) = child.child.container() {
13840 container.stop_members().await;
13841 child.kill_leader()?;
13842 child.wait().await?;
13843 return Ok(());
13844 }
13845 #[cfg(unix)]
13846 {
13847 let group = unreaped_registration_group(child)?;
13850 let _ = rustix::process::kill_process_group(group, rustix::process::Signal::TERM);
13851 sleep(crate::child_roster::TERM_TO_KILL).await;
13856 let group = unreaped_registration_group(child)?;
13857 let _ = rustix::process::kill_process_group(group, rustix::process::Signal::KILL);
13858 }
13859 child.start_kill()?;
13862 child.wait().await?;
13863 Ok(())
13864}
13865
13866#[cfg(unix)]
13867fn unreaped_registration_group(child: &SupervisedChild) -> io::Result<rustix::process::Pid> {
13868 if let Some(container) = child.child.container() {
13869 return container
13870 .group()
13871 .and_then(|pid| rustix::process::Pid::from_raw(pid as i32))
13872 .ok_or_else(|| io::Error::other("a carried cgroup has no signalable process group"));
13873 }
13874 let group = i32::try_from(child.pid)
13877 .ok()
13878 .filter(|pid| *pid > 1)
13879 .and_then(rustix::process::Pid::from_raw)
13880 .ok_or_else(|| {
13881 io::Error::new(
13882 io::ErrorKind::InvalidInput,
13883 "invalid supervised process group",
13884 )
13885 })?;
13886 if child.child.id() != Some(child.pid) {
13887 return Err(io::Error::new(
13888 io::ErrorKind::InvalidInput,
13889 "refusing process-group signal after supervised child was reaped",
13890 ));
13891 }
13892 Ok(group)
13893}
13894
13895enum RegistrationWaitOutcome {
13896 Registered,
13897 Exited(ExitReport),
13898 TimedOut,
13899}
13900
13901struct ReloadRegistrationFailure {
13902 exit_report: ExitReport,
13903 reason: String,
13904}
13905
13906#[derive(Debug, Clone, Copy, PartialEq, Eq)]
13907enum BusyGaugeObservation {
13908 Quiescent,
13909 Busy,
13910 Omitted,
13911}
13912
13913#[derive(Debug, Clone, PartialEq, Eq)]
13915enum GaugeVerdict {
13916 Zero,
13918 NonZero(u64),
13920 NotACount(String),
13924 Missing,
13926 ProbeFailed(&'static str),
13929}
13930
13931impl GaugeVerdict {
13932 fn reads_quiet(&self) -> bool {
13933 matches!(self, Self::Zero)
13934 }
13935}
13936
13937impl fmt::Display for GaugeVerdict {
13938 fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
13939 match self {
13940 Self::Zero => f.write_str("zero"),
13941 Self::NonZero(value) => write!(f, "non_zero({value})"),
13942 Self::NotACount(value) => write!(f, "not_a_count({value})"),
13943 Self::Missing => f.write_str("missing_from_health_body"),
13944 Self::ProbeFailed(kind) => write!(f, "probe_failed({kind})"),
13945 }
13946 }
13947}
13948
13949const GAUGE_VALUE_TEXT_LIMIT: usize = 32;
13952
13953fn gauge_verdicts(metrics: Option<&Value>, gauges: &[String]) -> Vec<GaugeVerdict> {
13955 let metrics = metrics.and_then(Value::as_object);
13956 gauges
13957 .iter()
13958 .map(
13959 |gauge| match metrics.and_then(|metrics| metrics.get(gauge)) {
13960 None => GaugeVerdict::Missing,
13961 Some(value) => match value.as_u64() {
13962 Some(0) => GaugeVerdict::Zero,
13963 Some(count) => GaugeVerdict::NonZero(count),
13964 None => {
13965 let mut text = value.to_string();
13966 if text.len() > GAUGE_VALUE_TEXT_LIMIT {
13967 let mut end = GAUGE_VALUE_TEXT_LIMIT;
13968 while !text.is_char_boundary(end) {
13969 end -= 1;
13970 }
13971 text.truncate(end);
13972 text.push_str("...");
13973 }
13974 GaugeVerdict::NotACount(text)
13975 }
13976 },
13977 },
13978 )
13979 .collect()
13980}
13981
13982fn busy_gauge_observation(verdicts: &[GaugeVerdict]) -> BusyGaugeObservation {
13985 if verdicts
13986 .iter()
13987 .any(|verdict| matches!(verdict, GaugeVerdict::Missing))
13988 {
13989 BusyGaugeObservation::Omitted
13990 } else if verdicts.iter().all(GaugeVerdict::reads_quiet) {
13991 BusyGaugeObservation::Quiescent
13992 } else {
13993 BusyGaugeObservation::Busy
13994 }
13995}
13996
13997#[derive(Debug, Clone, Default, PartialEq, Eq)]
14002struct GaugeTrace {
14003 latest: Option<(GaugeVerdict, Duration)>,
14004 last_value: Option<(GaugeVerdict, Duration)>,
14005}
14006
14007#[derive(Debug, Clone, Default, PartialEq, Eq)]
14013struct DrainGaugeTrace {
14014 probes: u32,
14016 gauges: Vec<(String, GaugeTrace)>,
14018}
14019
14020impl DrainGaugeTrace {
14021 fn new(gauges: &[String]) -> Self {
14022 Self {
14023 probes: 0,
14024 gauges: gauges
14025 .iter()
14026 .map(|gauge| (gauge.clone(), GaugeTrace::default()))
14027 .collect(),
14028 }
14029 }
14030
14031 fn record(&mut self, verdicts: &[GaugeVerdict], at: Duration) {
14033 self.probes = self.probes.saturating_add(1);
14034 for ((_, trace), verdict) in self.gauges.iter_mut().zip(verdicts) {
14035 trace.latest = Some((verdict.clone(), at));
14036 if matches!(
14037 verdict,
14038 GaugeVerdict::Zero | GaugeVerdict::NonZero(_) | GaugeVerdict::NotACount(_)
14039 ) {
14040 trace.last_value = Some((verdict.clone(), at));
14041 }
14042 }
14043 }
14044}
14045
14046impl fmt::Display for DrainGaugeTrace {
14051 fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
14052 if self.gauges.is_empty() {
14053 return f.write_str("none declared");
14054 }
14055 for (index, (name, trace)) in self.gauges.iter().enumerate() {
14056 if index > 0 {
14057 f.write_str(" | ")?;
14058 }
14059 write!(f, "{name}: ")?;
14060 match &trace.latest {
14061 Some((verdict, at)) => write!(f, "{verdict} @{}ms", at.as_millis())?,
14062 None => f.write_str("never probed")?,
14063 }
14064 match &trace.last_value {
14065 Some((value, at)) => write!(f, "; last value {value} @{}ms", at.as_millis())?,
14066 None => f.write_str("; no value ever read")?,
14067 }
14068 }
14069 Ok(())
14070 }
14071}
14072
14073pub(crate) fn declared_busy_gauges(
14074 registry: &Registry,
14075 module_id: &str,
14076) -> Result<Vec<String>, SuperviseError> {
14077 busy_gauges_of(
14078 registry
14079 .get_module(module_id)
14080 .map_err(SuperviseError::Registry)?,
14081 )
14082}
14083
14084fn declared_busy_gauges_for_connection(
14088 registry: &Registry,
14089 connection_id: ConnectionId,
14090) -> Result<Vec<String>, SuperviseError> {
14091 busy_gauges_of(
14092 registry
14093 .get_module_by_connection(connection_id)
14094 .map_err(SuperviseError::Registry)?,
14095 )
14096}
14097
14098fn busy_gauges_of(
14099 registration: Option<crate::registry::ModuleRegistration>,
14100) -> Result<Vec<String>, SuperviseError> {
14101 let Some(registration) = registration else {
14102 return Ok(Vec::new());
14103 };
14104 let Some(self_signals) = registration.manifest.self_signals else {
14105 return Ok(Vec::new());
14106 };
14107
14108 let mut gauges = Vec::new();
14109 for declaration in self_signals {
14110 if declaration.kind != SelfSignalKind::Busy {
14111 continue;
14112 }
14113 match declaration.anchored_to {
14114 SignalAnchor::HealthGauges { gauges: declared } if !declared.is_empty() => {
14115 gauges.extend(declared)
14116 }
14117 _ => {
14118 gauges.push(String::new());
14121 }
14122 }
14123 }
14124 Ok(gauges)
14125}
14126
14127#[allow(clippy::too_many_arguments)]
14136async fn wait_for_forwarding_quiescence(
14137 forwarding: &ForwardingTable,
14138 module_id: &str,
14139 runtime: &SupervisorRuntimeConfig,
14140 endpoint: crate::ModuleEndpointId,
14141 deadline: Instant,
14142 busy_gauges: &[String],
14143 scope: DrainScope,
14144 trace: &mut DrainGaugeTrace,
14145) -> Result<bool, SuperviseError> {
14146 let wait_started = Instant::now();
14147 let mut gauges_quiescent = busy_gauges.is_empty();
14148 let mut next_probe_at = wait_started;
14149 let mut omission_reported = false;
14150
14151 loop {
14152 let now = Instant::now();
14153 if !busy_gauges.is_empty() && now >= next_probe_at && now < deadline {
14154 let report = match scope {
14155 DrainScope::Active => probe_module_health(module_id, runtime, Some(deadline)).await,
14156 DrainScope::Endpoint(endpoint) => {
14157 probe_endpoint_health(endpoint, runtime, Some(deadline)).await
14158 }
14159 };
14160 let verdicts = match &report {
14161 Ok(report) => gauge_verdicts(report.metrics.as_ref(), busy_gauges),
14162 Err(err) => vec![GaugeVerdict::ProbeFailed(err.label()); busy_gauges.len()],
14163 };
14164 trace.record(&verdicts, wait_started.elapsed());
14165 gauges_quiescent = match report {
14166 Ok(_) => match busy_gauge_observation(&verdicts) {
14167 BusyGaugeObservation::Quiescent => true,
14168 BusyGaugeObservation::Busy => false,
14169 BusyGaugeObservation::Omitted => {
14170 if !omission_reported {
14173 forwarding
14174 .counters()
14175 .increment_drains_with_undeclared_gauge();
14176 let missing = busy_gauges
14177 .iter()
14178 .zip(&verdicts)
14179 .filter(|(_, verdict)| matches!(verdict, GaugeVerdict::Missing))
14180 .map(|(gauge, _)| gauge.as_str())
14181 .collect::<Vec<_>>()
14182 .join(",");
14183 warn!(
14184 module_id,
14185 missing_gauges = %missing,
14186 "drain health.check answer lacks a declared busy gauge; the drain treats the module as busy until it reports it"
14187 );
14188 omission_reported = true;
14189 }
14190 false
14191 }
14192 },
14193 Err(err) => {
14194 warn!(
14195 module_id,
14196 error = %err,
14197 "drain health.check did not produce declared busy gauges; treating module as busy"
14198 );
14199 false
14200 }
14201 };
14202 next_probe_at = Instant::now() + drain_busy_reprobe_interval();
14205 }
14206
14207 let in_flight = forwarding
14208 .endpoint_in_flight_count(endpoint)
14209 .map_err(SuperviseError::Forwarding)?;
14210 if in_flight == 0 && gauges_quiescent {
14211 return Ok(true);
14212 }
14213
14214 let now = Instant::now();
14215 if now >= deadline {
14216 return Ok(false);
14217 }
14218 let mut wait = deadline
14219 .saturating_duration_since(now)
14220 .min(REGISTRY_RELEASE_POLL);
14221 if !busy_gauges.is_empty() {
14222 wait = wait.min(next_probe_at.saturating_duration_since(now));
14223 }
14224 sleep(wait).await;
14225 }
14226}
14227
14228fn drain_busy_reprobe_interval() -> Duration {
14232 DRAIN_BUSY_REPROBE_INTERVAL.max(REGISTRY_RELEASE_POLL)
14233}
14234
14235fn warn_drain_timed_out(
14250 module_id: &str,
14251 drain_timeout: Duration,
14252 reason: RouteCloseReason,
14253 holdouts: &crate::forwarding::DrainHoldouts,
14254 gauges: &DrainGaugeTrace,
14255) {
14256 warn!(
14257 module_id = %module_id,
14258 waited = ?drain_timeout,
14259 ?reason,
14260 held_requests = holdouts.requests,
14261 held_routes = holdouts.routes,
14262 total_routes = holdouts.total_routes,
14263 top_connections = ?holdouts.top_connections,
14264 held = %holdouts
14267 .held
14268 .iter()
14269 .map(|(channel, corr)| format!("{channel}:{corr}"))
14270 .collect::<Vec<_>>()
14271 .join(","),
14272 gauge_probes = gauges.probes,
14273 busy_gauges = %gauges,
14274 "route drain timed out before request quiescence; forcing teardown"
14275 );
14276}
14277
14278fn drained_after_quiescence_wait(wait_result: &Result<bool, SuperviseError>) -> bool {
14286 match wait_result {
14287 Ok(drained) => *drained,
14288 Err(_) => false,
14289 }
14290}
14291
14292fn send_route_goodbyes(forwarding: &ForwardingTable, released_routes: Vec<GoodbyeTarget>) {
14293 for released in released_routes {
14294 let frame = match Frame::build_with_version(
14295 released.negotiated_ver,
14296 FrameType::Goodbye,
14297 control_flags(),
14298 released.channel,
14299 released.epoch,
14300 0,
14301 Vec::new(),
14302 ) {
14303 Ok(frame) => frame,
14304 Err(err) => {
14305 warn!(
14306 route_channel = released.channel,
14307 error = %err,
14308 "failed to build supervisor drain route GOODBYE frame"
14309 );
14310 continue;
14311 }
14312 };
14313 if !released.close_on_delivery_failure() {
14314 crate::forwarding::send_module_route_goodbye(
14315 &forwarding.counters(),
14316 &released.sink,
14317 frame,
14318 released.module_id.as_deref(),
14319 "supervisor drain",
14320 );
14321 continue;
14322 }
14323 if let Err(err) = released.sink.try_send(frame) {
14324 warn!(
14325 target_connection_id = released.connection_id.get(),
14326 route_channel = released.channel,
14327 error = %err,
14328 "supervisor drain route GOODBYE was not delivered to client; closing target connection"
14329 );
14330 let _ = forwarding.escalate_client_delivery_failure(
14331 released.connection_id,
14332 released.channel,
14333 released.epoch,
14334 CloseReason::new(
14335 "route_goodbye_delivery_failed",
14336 format!(
14337 "failed to enqueue supervisor drain route GOODBYE for channel {}: {err}",
14338 released.channel
14339 ),
14340 ),
14341 crate::forwarding::UndeliveredFrame {
14342 module_id: released.module_id.as_deref(),
14343 sink: &released.sink,
14344 },
14345 );
14346 }
14347 }
14348}
14349
14350async fn send_module_draining(
14351 module_id: &str,
14352 reason: RouteCloseReason,
14353 deadline_ms: u64,
14354 target: &ModuleDrainTarget,
14355 _wait_for_flush: bool,
14356) -> bool {
14357 let body = match serde_json::to_vec(&ModuleControlCommand::Draining {
14358 reason,
14359 deadline_ms,
14360 }) {
14361 Ok(body) => body,
14362 Err(err) => {
14363 warn!(
14364 module_id,
14365 error = %err,
14366 "failed to encode module draining command"
14367 );
14368 return false;
14369 }
14370 };
14371 let frame = match Frame::build_with_version(
14372 target.negotiated_ver,
14373 FrameType::Push,
14374 control_flags(),
14375 0,
14376 0,
14377 0,
14378 body,
14379 ) {
14380 Ok(frame) => frame,
14381 Err(err) => {
14382 warn!(
14383 module_id,
14384 error = %err,
14385 "failed to build module draining command frame"
14386 );
14387 return false;
14388 }
14389 };
14390 #[cfg(unix)]
14391 if _wait_for_flush {
14392 return target.sink.send_flushed(frame).await.is_ok();
14393 }
14394 if let Err(err) = target.sink.try_send(frame) {
14395 warn!(
14396 module_id,
14397 target_connection_id = target.endpoint.connection_id.get(),
14398 error = %err,
14399 "module draining command was not delivered to peer"
14400 );
14401 return false;
14402 }
14403 true
14404}
14405
14406fn module_goodbye_frame(module_id: &str, negotiated_ver: u8) -> Option<Frame> {
14408 match Frame::build_with_version(
14409 negotiated_ver,
14410 FrameType::Goodbye,
14411 control_flags(),
14412 0,
14413 0,
14414 0,
14415 Vec::new(),
14416 ) {
14417 Ok(frame) => Some(frame),
14418 Err(err) => {
14419 warn!(
14420 module_id,
14421 error = %err,
14422 "failed to build module GOODBYE frame"
14423 );
14424 None
14425 }
14426 }
14427}
14428
14429#[cfg(unix)]
14443async fn send_module_goodbyes_for_daemon_shutdown(
14444 forwarding: &Arc<ForwardingTable>,
14445 reason: &CloseReason,
14446 wait_for_flush: bool,
14447) {
14448 const GOODBYE_BUDGET: Duration = Duration::from_millis(500);
14449 let targets = match forwarding.module_connections() {
14450 Ok(targets) => targets,
14451 Err(err) => {
14452 warn!(error = %err, "could not list module connections for shutdown GOODBYE");
14453 return;
14454 }
14455 };
14456 let deadline = Instant::now() + GOODBYE_BUDGET;
14457 let mut sends = tokio::task::JoinSet::new();
14458 for target in targets {
14459 let Some(frame) = module_goodbye_frame(&target.module_id, target.negotiated_ver) else {
14460 continue;
14461 };
14462 if !wait_for_flush {
14463 if let Err(err) = target.sink.try_send(frame) {
14464 debug!(
14465 module_id = %target.module_id,
14466 error = %err,
14467 "shutdown module GOODBYE was not queued"
14468 );
14469 }
14470 continue;
14471 }
14472 let forwarding = Arc::clone(forwarding);
14473 let reason = reason.clone();
14474 sends.spawn(async move {
14475 match timeout_at(deadline, target.sink.send_flushed(frame)).await {
14476 Ok(Ok(())) => {}
14477 Ok(Err(err)) => debug!(
14478 module_id = %target.module_id,
14479 error = %err,
14480 "module connection closed before its shutdown GOODBYE was written"
14481 ),
14482 Err(_) => warn!(
14483 module_id = %target.module_id,
14484 budget = ?GOODBYE_BUDGET,
14485 "shutdown module GOODBYE was not written within its budget; closing anyway"
14486 ),
14487 }
14488 forwarding.request_connection_close(target.endpoint.connection_id, reason);
14489 });
14490 }
14491 while sends.join_next().await.is_some() {}
14493}
14494
14495async fn send_module_goodbye(
14496 module_id: &str,
14497 forwarding: &ForwardingTable,
14498 target: &ModuleDrainTarget,
14499 wait_for_flush: bool,
14500) {
14501 let Some(frame) = module_goodbye_frame(module_id, target.negotiated_ver) else {
14502 return;
14503 };
14504 let sent = if wait_for_flush {
14507 timeout(Duration::from_millis(500), target.sink.send_flushed(frame))
14508 .await
14509 .map_err(|_| "module GOODBYE flush exceeded 500ms".to_owned())
14510 .and_then(|result| result.map_err(|err| err.to_string()))
14511 } else {
14512 target.sink.try_send(frame).map_err(|err| err.to_string())
14513 };
14514 if let Err(err) = sent {
14515 warn!(
14516 module_id,
14517 target_connection_id = target.endpoint.connection_id.get(),
14518 error = %err,
14519 "supervisor drain module GOODBYE was not delivered to peer; closing module connection"
14520 );
14521 forwarding.request_connection_close(
14522 target.endpoint.connection_id,
14523 CloseReason::new(
14524 "module_goodbye_delivery_failed",
14525 format!("failed to enqueue supervisor drain module GOODBYE for module '{module_id}': {err}"),
14526 ),
14527 );
14528 }
14529}
14530
14531#[derive(Clone, Copy)]
14532struct ForwardingDrainContext<'a> {
14533 spec: &'a ModuleSpec,
14534 runtime: &'a SupervisorRuntimeConfig,
14535 registry: &'a Registry,
14536 scope: DrainScope,
14537}
14538
14539#[derive(Debug, Clone, Copy, PartialEq, Eq)]
14541enum DrainScope {
14542 Active,
14545 Endpoint(crate::ModuleEndpointId),
14550}
14551
14552#[derive(Debug, Clone, Copy, PartialEq, Eq)]
14560enum StopNotice {
14561 SentOverConnection,
14564 NoConnection,
14568 NotSent,
14572}
14573
14574async fn begin_forwarding_drain(
14575 spec: &ModuleSpec,
14576 runtime: &SupervisorRuntimeConfig,
14577 registry: &Registry,
14578 snapshot: &SharedSnapshot,
14579 enabled: Option<bool>,
14580 reason: RouteCloseReason,
14581) -> Result<StopNotice, SuperviseError> {
14582 let Some(forwarding) = runtime.forwarding.as_ref() else {
14583 return Err(SuperviseError::ReloadUnavailable {
14584 module_id: spec.module_id.clone(),
14585 reason: "supervisor was not configured with a forwarding table".to_string(),
14586 });
14587 };
14588
14589 begin_forwarding_drain_with(
14590 forwarding,
14591 ForwardingDrainContext {
14592 spec,
14593 runtime,
14594 registry,
14595 scope: DrainScope::Active,
14596 },
14597 snapshot,
14598 enabled,
14599 reason,
14600 runtime.drain_timeout,
14601 )
14602 .await
14603}
14604
14605async fn begin_forwarding_drain_if_configured(
14606 spec: &ModuleSpec,
14607 runtime: &SupervisorRuntimeConfig,
14608 registry: &Registry,
14609 snapshot: &SharedSnapshot,
14610 enabled: Option<bool>,
14611 reason: RouteCloseReason,
14612) -> Result<StopNotice, SuperviseError> {
14613 begin_forwarding_drain_with_timeout(
14614 spec,
14615 runtime,
14616 registry,
14617 snapshot,
14618 enabled,
14619 reason,
14620 runtime.drain_timeout,
14621 )
14622 .await
14623}
14624
14625async fn begin_forwarding_drain_with_timeout(
14629 spec: &ModuleSpec,
14630 runtime: &SupervisorRuntimeConfig,
14631 registry: &Registry,
14632 snapshot: &SharedSnapshot,
14633 enabled: Option<bool>,
14634 reason: RouteCloseReason,
14635 drain_timeout: Duration,
14636) -> Result<StopNotice, SuperviseError> {
14637 let Some(forwarding) = runtime.forwarding.as_ref() else {
14638 return Ok(StopNotice::NotSent);
14639 };
14640
14641 begin_forwarding_drain_with(
14642 forwarding,
14643 ForwardingDrainContext {
14644 spec,
14645 runtime,
14646 registry,
14647 scope: DrainScope::Active,
14648 },
14649 snapshot,
14650 enabled,
14651 reason,
14652 drain_timeout,
14653 )
14654 .await
14655}
14656
14657async fn begin_forwarding_drain_with(
14658 forwarding: &ForwardingTable,
14659 context: ForwardingDrainContext<'_>,
14660 snapshot: &SharedSnapshot,
14661 enabled: Option<bool>,
14662 reason: RouteCloseReason,
14663 drain_timeout: Duration,
14664) -> Result<StopNotice, SuperviseError> {
14665 let ForwardingDrainContext {
14666 spec,
14667 runtime,
14668 registry,
14669 scope,
14670 } = context;
14671 debug_assert_ne!(reason, RouteCloseReason::Crash);
14672 let terminal = matches!(reason, RouteCloseReason::Disable);
14673 let drain_started_at = Instant::now();
14674 let drain_deadline = drain_started_at + drain_timeout;
14675 let deadline_ms =
14676 unix_ms_now().saturating_add(u64::try_from(drain_timeout.as_millis()).unwrap_or(u64::MAX));
14677 let busy_gauges = match scope {
14678 DrainScope::Active => declared_busy_gauges(registry, &spec.module_id)?,
14679 DrainScope::Endpoint(endpoint) => {
14680 declared_busy_gauges_for_connection(registry, endpoint.connection_id)?
14681 }
14682 };
14683
14684 let gate_started = Instant::now();
14687 let drain_target = match scope {
14688 DrainScope::Active => forwarding.begin_module_drain(&spec.module_id, reason),
14689 DrainScope::Endpoint(endpoint) => forwarding.begin_endpoint_drain(endpoint, reason),
14690 }
14691 .map_err(SuperviseError::Forwarding)?;
14692 info!(
14697 module_id = %spec.module_id,
14698 ?reason,
14699 gate_ms = u64::try_from(gate_started.elapsed().as_millis()).unwrap_or(u64::MAX),
14700 connected = drain_target.is_some(),
14701 "module drain began; route admission closed"
14702 );
14703 if scope == DrainScope::Active {
14704 update_snapshot(snapshot, Some(&spec.module_id), |state| {
14705 state.state = ModuleState::Draining;
14706 state.draining_to_replace =
14707 matches!(reason, RouteCloseReason::Restart | RouteCloseReason::Reload);
14708 if let Some(enabled) = enabled {
14709 state.enabled = enabled;
14710 }
14711 })?;
14712 }
14713
14714 let Some(target) = drain_target.as_ref() else {
14715 return Ok(StopNotice::NoConnection);
14719 };
14720 let flush_goodbye = runtime.supervisor_handle.as_ref().is_some_and(|handle| {
14721 handle
14722 .registration_spawn(target.endpoint.connection_id)
14723 .is_some()
14724 });
14725 {
14726 send_module_draining(&spec.module_id, reason, deadline_ms, target, false).await;
14727 #[cfg(feature = "test-support")]
14728 forwarding.wait_before_route_closing_for_test().await;
14729 let routes = forwarding
14730 .endpoint_routes(target.endpoint)
14731 .map_err(SuperviseError::Forwarding)?;
14732 let routes_notified = routes.len();
14733 crate::control::send_route_control_pushes(
14734 forwarding,
14735 routes.clone(),
14736 ClientControlPush::RouteClosing {
14737 module_id: spec.module_id.clone(),
14738 channels: Vec::new(),
14739 reason,
14740 },
14741 );
14742 send_route_goodbyes(forwarding, target.abandoned_bindings.clone());
14743
14744 let mut gauge_trace = DrainGaugeTrace::new(&busy_gauges);
14750 let wait_result = wait_for_forwarding_quiescence(
14751 forwarding,
14752 &spec.module_id,
14753 runtime,
14754 target.endpoint,
14755 drain_deadline,
14756 &busy_gauges,
14757 scope,
14758 &mut gauge_trace,
14759 )
14760 .await;
14761 let drained = drained_after_quiescence_wait(&wait_result);
14762 if let Err(err) = &wait_result {
14763 error!(
14764 module_id = %spec.module_id,
14765 ?reason,
14766 error = %err,
14767 "forwarding quiescence wait failed after route.closing; forcing route.closed(drained: false) so the client is not left waiting on an unfulfilled promise"
14768 );
14769 } else if !drained {
14770 let holdouts = forwarding
14772 .endpoint_drain_holdouts(target.endpoint)
14773 .unwrap_or_default();
14774 warn_drain_timed_out(
14775 &spec.module_id,
14776 drain_timeout,
14777 reason,
14778 &holdouts,
14779 &gauge_trace,
14780 );
14781 }
14782 crate::control::send_route_control_pushes(
14783 forwarding,
14784 routes,
14785 ClientControlPush::RouteClosed {
14786 module_id: spec.module_id.clone(),
14787 channels: Vec::new(),
14788 reason,
14789 drained,
14790 abandoned: target.abandoned_bindings.len() as u32,
14791 excluded_subscriptions: target.excluded_subscriptions,
14792 terminal: Some(terminal),
14793 },
14794 );
14795 wait_result?;
14796
14797 let released_routes = match forwarding.release_module_endpoint_routes(target.endpoint) {
14803 Ok(routes) => routes,
14804 Err(err) => {
14805 warn!(
14806 module_id = %spec.module_id,
14807 ?reason,
14808 error = %err,
14809 "failed to release module endpoint routes after route.closed; module GOODBYE will still be sent"
14810 );
14811 send_module_goodbye(&spec.module_id, forwarding, target, flush_goodbye).await;
14812 return Err(SuperviseError::Forwarding(err));
14813 }
14814 };
14815 let route_goodbye_count = released_routes.len();
14816 send_route_goodbyes(forwarding, released_routes);
14817 send_module_goodbye(&spec.module_id, forwarding, target, flush_goodbye).await;
14818
14819 info!(
14825 module_id = %spec.module_id,
14826 ?reason,
14827 routes_notified,
14828 route_goodbyes = route_goodbye_count,
14829 abandoned_reservations = target.abandoned_bindings.len(),
14830 excluded_subscriptions = target.excluded_subscriptions,
14831 drained,
14832 "module drain complete; consumers notified via route.closing/route.closed pushes and per-route GOODBYE frames"
14833 );
14834 }
14835
14836 Ok(StopNotice::SentOverConnection)
14837}
14838
14839async fn wait_for_registration_after_reload(
14842 registry: &Registry,
14843 module_id: &str,
14844 snapshot: &SharedSnapshot,
14845 child: &mut SupervisedChild,
14846 wait: Duration,
14847) -> Result<RegistrationWaitOutcome, SuperviseError> {
14848 wait_for_slot_registration(
14849 registry,
14850 crate::registry::RegistrationSlot::Active(module_id),
14851 module_id,
14852 snapshot,
14853 child,
14854 wait,
14855 )
14856 .await
14857}
14858
14859async fn wait_for_slot_registration(
14867 registry: &Registry,
14868 slot: crate::registry::RegistrationSlot<'_>,
14869 module_id: &str,
14870 snapshot: &SharedSnapshot,
14871 child: &mut SupervisedChild,
14872 wait: Duration,
14873) -> Result<RegistrationWaitOutcome, SuperviseError> {
14874 let deadline = Instant::now() + wait;
14875 loop {
14876 if registry
14877 .registration(slot)
14878 .map_err(SuperviseError::Registry)?
14879 .is_some()
14880 {
14881 return Ok(RegistrationWaitOutcome::Registered);
14882 }
14883
14884 let now = Instant::now();
14885 if now >= deadline {
14886 return Ok(RegistrationWaitOutcome::TimedOut);
14887 }
14888 let remaining = deadline.saturating_duration_since(now);
14889 let poll = remaining.min(REGISTRY_RELEASE_POLL);
14890
14891 tokio::select! {
14892 wait_result = child.wait() => {
14893 let status = wait_result.map_err(|source| SuperviseError::Wait {
14894 module_id: module_id.to_string(),
14895 source,
14896 })?;
14897 return Ok(RegistrationWaitOutcome::Exited(classify_reaped_child_exit(
14898 snapshot,
14899 child,
14900 &status,
14901 )));
14902 }
14903 _ = sleep(poll) => {}
14904 }
14905 }
14906}
14907
14908fn registration_failure_exit_report(mut exit_report: ExitReport) -> ExitReport {
14909 if exit_report.kind != ExitKind::DeliberateSeverance {
14913 exit_report.kind = ExitKind::Crash;
14914 }
14915 exit_report
14916}
14917
14918async fn handle_reload_child_registration_failure(
14919 spec: &ModuleSpec,
14920 runtime: &SupervisorRuntimeConfig,
14921 registry: &Registry,
14922 process_liveness: &SupervisorProcessLiveness,
14923 snapshot: &SharedSnapshot,
14924 _child: &mut Option<SupervisedChild>,
14925 failure: ReloadRegistrationFailure,
14926) -> Result<(), SuperviseError> {
14927 let ReloadRegistrationFailure {
14928 exit_report,
14929 reason,
14930 } = failure;
14931 match on_child_exit(
14932 spec,
14933 runtime.restart_policy,
14934 registry,
14935 snapshot,
14936 &runtime.terminal_ring,
14937 &runtime.spawn_events,
14938 &runtime.child_roster,
14939 exit_report,
14940 runtime.recovery_is_daemon,
14941 )
14942 .await
14943 {
14944 NextAction::Stop {
14945 registration_released,
14946 } => {
14947 if registration_released {
14948 process_liveness.untrack_if_current(&spec.module_id, snapshot);
14949 }
14950 }
14951 NextAction::Restart { schedule } => {
14952 let delay = schedule.map_or(runtime.restart_policy.delay_for_restart(0), |schedule| {
14953 schedule.delay
14954 });
14955 if let Some(schedule) = schedule {
14956 log_crash_respawn(&spec.module_id, schedule);
14957 }
14958 schedule_respawn(
14959 runtime,
14960 snapshot,
14961 &spec.module_id,
14962 delay,
14963 RespawnKind::Spawn,
14964 true,
14965 )?;
14966 }
14967 }
14968 Err(SuperviseError::ReloadFailed {
14969 module_id: spec.module_id.clone(),
14970 reason,
14971 })
14972}
14973
14974async fn handle_reload_spawn_failure(
14975 spec: &ModuleSpec,
14976 runtime: &SupervisorRuntimeConfig,
14977 process_liveness: &SupervisorProcessLiveness,
14978 snapshot: &SharedSnapshot,
14979 _child: &mut Option<SupervisedChild>,
14980 reason: String,
14981) -> Result<(), SuperviseError> {
14982 let now = Instant::now();
14983 let mut schedule = None;
14984 update_snapshot(snapshot, Some(&spec.module_id), |state| {
14985 clear_current_process_facts(state);
14986 if state.enabled {
14987 schedule = state.next_crash_restart(&runtime.restart_policy, now);
14988 state.state = if schedule.is_some() {
14989 ModuleState::Restarting
14990 } else {
14991 if runtime
14992 .failure_context(runtime.recovery_is_daemon)
14993 .may_retry()
14994 {
14995 state.failed_retry.schedule(&spec.module_id);
14996 }
14997 ModuleState::Failed
14998 };
14999 } else {
15000 state.state = ModuleState::Disabled;
15001 }
15002 })?;
15003 if let Some(schedule) = schedule {
15004 schedule_respawn(
15005 runtime,
15006 snapshot,
15007 &spec.module_id,
15008 schedule.delay,
15009 RespawnKind::Spawn,
15010 true,
15011 )?;
15012 } else {
15013 process_liveness.untrack_if_current(&spec.module_id, snapshot);
15014 }
15015 Err(SuperviseError::ReloadFailed {
15016 module_id: spec.module_id.clone(),
15017 reason,
15018 })
15019}
15020
15021fn control_flags() -> Flags {
15022 Flags::new(false, Priority::Passive, false)
15023}
15024
15025#[allow(clippy::too_many_arguments)]
15026async fn drain_optional_child(
15027 module_id: &str,
15028 protocol: ModuleProtocol,
15029 stop_notice: StopNotice,
15030 registry: &Registry,
15031 forwarding: Option<&ForwardingTable>,
15032 snapshot: &SharedSnapshot,
15033 terminal_ring: &Arc<Mutex<TerminalRing>>,
15034 spawn_events: &SpawnEventFeed,
15035 child: &mut Option<SupervisedChild>,
15036 drain_timeout: Duration,
15037 final_state: ModuleState,
15038 enabled: Option<bool>,
15039 failure: &FailureContext,
15040) -> Result<(), SuperviseError> {
15041 if let Some(child) = child.take() {
15042 drain_child_to_state(
15043 module_id,
15044 protocol,
15045 stop_notice,
15046 registry,
15047 forwarding,
15048 snapshot,
15049 terminal_ring,
15050 spawn_events,
15051 child,
15052 drain_timeout,
15053 final_state,
15054 enabled,
15055 failure,
15056 )
15057 .await
15058 } else {
15059 update_snapshot(snapshot, Some(module_id), |state| {
15060 state.state = final_state;
15061 if final_state == ModuleState::Failed {
15062 state.failed_retry.failed_at = Some(Instant::now());
15063 }
15064 if let Some(enabled) = enabled {
15065 state.enabled = enabled;
15066 }
15067 clear_current_process_facts(state);
15068 })?;
15069 release_dead_registration(registry, forwarding, snapshot, module_id, failure).await
15070 }
15071}
15072
15073#[allow(clippy::too_many_arguments)]
15074async fn drain_child_to_state(
15075 module_id: &str,
15076 _protocol: ModuleProtocol,
15077 stop_notice: StopNotice,
15078 registry: &Registry,
15079 forwarding: Option<&ForwardingTable>,
15080 snapshot: &SharedSnapshot,
15081 terminal_ring: &Arc<Mutex<TerminalRing>>,
15082 spawn_events: &SpawnEventFeed,
15083 mut child: SupervisedChild,
15084 drain_timeout: Duration,
15085 final_state: ModuleState,
15086 enabled: Option<bool>,
15087 failure: &FailureContext,
15088) -> Result<(), SuperviseError> {
15089 let protocol = child.protocol;
15090 child.retire_registration();
15091 update_snapshot(snapshot, Some(module_id), |state| {
15092 state.state = ModuleState::Draining;
15093 state.draining_to_replace = final_state == ModuleState::Restarting;
15094 if let Some(enabled) = enabled {
15095 state.enabled = enabled;
15096 }
15097 })?;
15098
15099 if stop_notice != StopNotice::SentOverConnection {
15110 if protocol == ModuleProtocol::Subc && stop_notice == StopNotice::NoConnection {
15111 info!(
15112 module_id,
15113 pid = child.pid,
15114 budget_ms = u64::try_from(drain_timeout.as_millis()).unwrap_or(u64::MAX),
15115 "module has no connection yet; requesting stop by signal"
15116 );
15117 }
15118 request_graceful_stop(module_id, &child);
15119 }
15120
15121 let exit_report = match timeout(drain_timeout, child.wait()).await {
15122 Ok(Ok(status)) => classify_reaped_child_exit(snapshot, &child, &status),
15123 Ok(Err(source)) => {
15124 fail_after_wait_error(
15125 module_id,
15126 snapshot,
15127 terminal_ring,
15128 spawn_events,
15129 format!("failed to wait during drain: {source}"),
15130 failure,
15131 );
15132 return Err(SuperviseError::Wait {
15133 module_id: module_id.to_string(),
15134 source,
15135 });
15136 }
15137 Err(_) => {
15138 warn!(
15151 module_id,
15152 pid = child.pid,
15153 budget_ms = u64::try_from(drain_timeout.as_millis()).unwrap_or(u64::MAX),
15154 reason = ?final_state,
15155 ?stop_notice,
15156 "drain budget expired before the module exited; killing it"
15157 );
15158 let kill =
15159 if final_state == ModuleState::Restarting && child.child.container().is_some() {
15160 child.kill_leader()
15161 } else {
15162 child.start_kill()
15163 };
15164 kill.map_err(|source| {
15165 fail_after_wait_error(
15166 module_id,
15167 snapshot,
15168 terminal_ring,
15169 spawn_events,
15170 format!("failed to kill during drain: {source}"),
15171 failure,
15172 );
15173 SuperviseError::Kill {
15174 module_id: module_id.to_string(),
15175 source,
15176 }
15177 })?;
15178 let status = child.wait().await.map_err(|source| {
15179 fail_after_wait_error(
15180 module_id,
15181 snapshot,
15182 terminal_ring,
15183 spawn_events,
15184 format!("failed to wait after drain kill: {source}"),
15185 failure,
15186 );
15187 SuperviseError::Wait {
15188 module_id: module_id.to_string(),
15189 source,
15190 }
15191 })?;
15192 classify_reaped_child_exit(snapshot, &child, &status)
15193 }
15194 };
15195
15196 let detail = lock_snapshot(snapshot)?.drain_disposition_detail.take();
15197 publish_after_terminal_record(
15198 module_id,
15199 || {
15200 record_terminal_with_detail(
15201 module_id,
15202 terminal_ring,
15203 spawn_events,
15204 &exit_report,
15205 terminal_disposition(final_state),
15206 detail,
15207 )
15208 },
15209 || {
15210 update_snapshot(snapshot, Some(module_id), |state| {
15211 state.state = final_state;
15212 if final_state == ModuleState::Failed {
15213 state.failed_retry.failed_at = Some(Instant::now());
15214 }
15215 if let Some(enabled) = enabled {
15216 state.enabled = enabled;
15217 }
15218 clear_current_process_facts(state);
15219 state.last_exit = Some(exit_report.clone());
15220 if exit_report.kind == ExitKind::DeliberateSeverance {
15221 state.lifetime_restarts += 1;
15222 }
15223 })
15224 },
15225 )?;
15226 child.drain_stderr(module_id).await;
15227
15228 let released =
15229 release_dead_registration(registry, forwarding, snapshot, module_id, failure).await;
15230 child
15231 .finish_exit(
15232 released.is_ok()
15233 && final_state == ModuleState::Restarting
15234 && child.spawn_failure.is_none(),
15235 )
15236 .await
15237 .map_err(|source| SuperviseError::Wait {
15238 module_id: module_id.to_owned(),
15239 source,
15240 })?;
15241 released
15242}
15243
15244#[cfg(unix)]
15264fn request_graceful_stop(module_id: &str, child: &SupervisedChild) {
15265 let Some(pid) = child
15266 .id()
15267 .and_then(|pid| i32::try_from(pid).ok())
15268 .and_then(rustix::process::Pid::from_raw)
15269 else {
15270 debug!(
15271 module_id,
15272 "no pid to signal for teardown; falling through to the drain wait"
15273 );
15274 return;
15275 };
15276 match rustix::process::kill_process(pid, rustix::process::Signal::TERM) {
15277 Ok(()) => debug!(
15278 module_id,
15279 "sent SIGTERM to a module nothing else asked to stop"
15280 ),
15281 Err(err) => debug!(
15282 module_id,
15283 error = %err,
15284 "SIGTERM to module failed; the drain wait and kill still apply"
15285 ),
15286 }
15287}
15288
15289#[cfg(not(unix))]
15297fn request_graceful_stop(module_id: &str, _child: &SupervisedChild) {
15298 debug!(
15299 module_id,
15300 "no graceful stop signal exists on this platform; teardown of a module nothing asked to stop waits, then kills"
15301 );
15302}
15303
15304fn terminal_disposition(final_state: ModuleState) -> TerminalDisposition {
15305 match final_state {
15306 ModuleState::Stopped => TerminalDisposition::Stopped,
15307 ModuleState::Disabled => TerminalDisposition::Disabled,
15308 ModuleState::Restarting => TerminalDisposition::Restarting,
15309 ModuleState::Failed => TerminalDisposition::Failed,
15310 ModuleState::Starting
15311 | ModuleState::Running
15312 | ModuleState::Unresponsive
15313 | ModuleState::Draining => {
15314 unreachable!("terminal exits only finish in terminal or restarting states")
15315 }
15316 }
15317}
15318
15319async fn release_dead_registration(
15327 registry: &Registry,
15328 forwarding: Option<&ForwardingTable>,
15329 snapshot: &SharedSnapshot,
15330 module_id: &str,
15331 failure: &FailureContext,
15332) -> Result<(), SuperviseError> {
15333 let result = async {
15334 let registration = registry
15335 .get_module(module_id)
15336 .map_err(SuperviseError::Registry)?;
15337 match wait_for_registration_release(registry, module_id, REGISTRY_RELEASE_TIMEOUT).await {
15338 Ok(()) => return Ok(()),
15339 Err(SuperviseError::RegistrationStillActive { .. }) => {}
15340 Err(err) => return Err(err),
15341 }
15342 let pid = lock_snapshot(snapshot)?.reaped_pid;
15343 if let (Some(registration), Some(forwarding), Some(pid)) = (registration, forwarding, pid) {
15344 warn!(
15345 module_id,
15346 pid,
15347 connection_id = registration.connection_id.get(),
15348 "reaped module registration outlived release grace; closing dead connection"
15349 );
15350 forwarding.request_connection_close(
15351 registration.connection_id,
15352 CloseReason::new(
15353 "supervised_process_reaped",
15354 format!("module '{module_id}' pid {pid} exited"),
15355 ),
15356 );
15357 wait_for_slot_registration_release(
15358 registry,
15359 crate::registry::RegistrationSlot::Connection(registration.connection_id),
15360 REGISTRY_RELEASE_TIMEOUT,
15361 )
15362 .await?;
15363 }
15364 wait_for_registration_release(registry, module_id, Duration::ZERO).await
15365 }
15366 .await;
15367 if let Err(err) = &result {
15368 fail_snapshot(snapshot, Some(module_id), None, failure);
15369 error!(module_id, error = %err, "registration release failed after child exit; module is failed and start can retry");
15370 }
15371 result
15372}
15373
15374async fn wait_for_registration_release(
15377 registry: &Registry,
15378 module_id: &str,
15379 wait: Duration,
15380) -> Result<(), SuperviseError> {
15381 wait_for_slot_registration_release(
15382 registry,
15383 crate::registry::RegistrationSlot::Active(module_id),
15384 wait,
15385 )
15386 .await
15387}
15388
15389async fn wait_for_slot_registration_release(
15397 registry: &Registry,
15398 slot: crate::registry::RegistrationSlot<'_>,
15399 wait: Duration,
15400) -> Result<(), SuperviseError> {
15401 let deadline = Instant::now() + wait;
15402 let mut release_events = registration_release_events().subscribe();
15403 let still_active = |registration: &crate::registry::ModuleRegistration| {
15404 SuperviseError::RegistrationStillActive {
15405 module_id: registration.manifest.module_id.clone(),
15406 waited: wait,
15407 }
15408 };
15409 loop {
15410 let _observed_generation = *release_events.borrow_and_update();
15411 let Some(registration) = registry
15412 .registration(slot)
15413 .map_err(SuperviseError::Registry)?
15414 else {
15415 return Ok(());
15416 };
15417
15418 let now = Instant::now();
15419 if now >= deadline {
15420 return Err(still_active(®istration));
15421 }
15422
15423 let remaining = deadline.saturating_duration_since(now);
15424 match timeout(remaining, release_events.changed()).await {
15425 Ok(Ok(())) | Ok(Err(_)) => {}
15426 Err(_) => return Err(still_active(®istration)),
15427 }
15428 }
15429}
15430
15431#[cfg(test)]
15432mod slot_registration_wait_tests {
15433 use super::*;
15434 use crate::registry::{ConnectionId, RegistrationSlot};
15435 use subc_protocol::manifest::ModuleManifest;
15436
15437 #[tokio::test]
15438 async fn enable_release_failure_is_failed_and_a_second_enable_retries() {
15439 let registry = Arc::new(Registry::default());
15440 let supervisor = Supervisor::new_for_test(Arc::clone(®istry), RestartPolicy::default());
15441 let runtime = supervisor.runtime_config();
15442 let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::disabled()));
15443 let spec = ModuleSpec {
15444 module_id: "enable-stale-registration".to_string(),
15445 program: PathBuf::from("/missing/enable-retry-test"),
15446 args: Vec::new(),
15447 env: Vec::new(),
15448 reserved: false,
15449 reserved_prefixes: Vec::new(),
15450 protocol: ModuleProtocol::Subc,
15451 overlap: Default::default(),
15452 carry_containment: false,
15453 };
15454 let connection = ConnectionId::new(90);
15455 registry
15456 .register_with_control_ops(
15457 ModuleManifest::builder(&spec.module_id, "0.1.0").build(),
15458 1,
15459 connection,
15460 Vec::new(),
15461 )
15462 .unwrap();
15463 let mut child = None;
15464 let err = set_child_enabled(
15465 &spec,
15466 &runtime,
15467 ®istry,
15468 &supervisor.process_liveness,
15469 &snapshot,
15470 &mut child,
15471 true,
15472 &runtime.failure_context(false),
15473 )
15474 .await
15475 .unwrap_err();
15476 assert!(matches!(
15477 err,
15478 SuperviseError::RegistrationStillActive { .. }
15479 ));
15480 assert_eq!(lock_snapshot(&snapshot).unwrap().state, ModuleState::Failed);
15481 assert!(child.is_none());
15482 registry.deregister_connection(connection).unwrap();
15483 let err = set_child_enabled(
15484 &spec,
15485 &runtime,
15486 ®istry,
15487 &supervisor.process_liveness,
15488 &snapshot,
15489 &mut child,
15490 true,
15491 &runtime.failure_context(false),
15492 )
15493 .await
15494 .unwrap_err();
15495 assert!(
15496 matches!(err, SuperviseError::Spawn { .. }),
15497 "second enable must attempt a spawn: {err}"
15498 );
15499 assert_eq!(lock_snapshot(&snapshot).unwrap().state, ModuleState::Failed);
15500 }
15501
15502 const INCUMBENT: u64 = 1;
15503 const CANDIDATE: u64 = 2;
15504
15505 fn swapped_registry() -> Arc<Registry> {
15506 let registry = Arc::new(Registry::default());
15507 let manifest = ModuleManifest::builder("m", "0.1.0").build();
15508 registry
15509 .register_with_control_ops(
15510 manifest.clone(),
15511 1,
15512 ConnectionId::new(INCUMBENT),
15513 Vec::new(),
15514 )
15515 .unwrap();
15516 registry
15517 .register_candidate_with_control_ops(
15518 manifest,
15519 1,
15520 ConnectionId::new(CANDIDATE),
15521 Vec::new(),
15522 )
15523 .unwrap();
15524 registry
15525 }
15526
15527 #[tokio::test]
15531 async fn incumbent_release_is_awaited_by_connection_not_by_module_id() {
15532 let registry = swapped_registry();
15533 registry.promote_candidate("m").unwrap().unwrap();
15534
15535 assert!(matches!(
15536 wait_for_registration_release(®istry, "m", Duration::from_millis(50)).await,
15537 Err(SuperviseError::RegistrationStillActive { .. })
15538 ));
15539
15540 assert!(matches!(
15542 wait_for_slot_registration_release(
15543 ®istry,
15544 RegistrationSlot::Connection(ConnectionId::new(INCUMBENT)),
15545 Duration::from_millis(50),
15546 )
15547 .await,
15548 Err(SuperviseError::RegistrationStillActive { .. })
15549 ));
15550
15551 let releaser = Arc::clone(®istry);
15552 let release = tokio::spawn(async move {
15553 sleep(Duration::from_millis(20)).await;
15554 releaser
15555 .deregister_connection(ConnectionId::new(INCUMBENT))
15556 .unwrap();
15557 notify_registration_release();
15558 });
15559 wait_for_slot_registration_release(
15560 ®istry,
15561 RegistrationSlot::Connection(ConnectionId::new(INCUMBENT)),
15562 Duration::from_secs(5),
15563 )
15564 .await
15565 .expect("the incumbent's own registration is released");
15566 release.await.unwrap();
15567 assert!(registry.get_module("m").unwrap().is_some());
15568 }
15569
15570 #[tokio::test]
15573 async fn candidate_slot_wait_ignores_the_incumbents_registration() {
15574 let registry = swapped_registry();
15575 assert!(matches!(
15576 wait_for_slot_registration_release(
15577 ®istry,
15578 RegistrationSlot::Candidate("m"),
15579 Duration::from_millis(50),
15580 )
15581 .await,
15582 Err(SuperviseError::RegistrationStillActive { .. })
15583 ));
15584 registry
15585 .deregister_connection(ConnectionId::new(CANDIDATE))
15586 .unwrap();
15587 wait_for_slot_registration_release(
15588 ®istry,
15589 RegistrationSlot::Candidate("m"),
15590 Duration::from_millis(50),
15591 )
15592 .await
15593 .expect("a candidate slot with no candidate is released");
15594 assert!(registry
15595 .registration(RegistrationSlot::Active("m"))
15596 .unwrap()
15597 .is_some());
15598 }
15599}
15600
15601fn classify_exit(status: &ExitStatus) -> ExitReport {
15602 ExitReport {
15603 kind: if status.success() {
15604 ExitKind::Clean
15605 } else {
15606 ExitKind::Crash
15607 },
15608 code: status.code(),
15609 signal: exit_signal(status),
15610 at_ms: unix_ms_now(),
15611 }
15612}
15613
15614fn wait_error_exit_report() -> ExitReport {
15620 ExitReport {
15621 kind: ExitKind::Crash,
15622 code: None,
15623 signal: None,
15624 at_ms: unix_ms_now(),
15625 }
15626}
15627
15628#[cfg(unix)]
15629fn exit_signal(status: &ExitStatus) -> Option<i32> {
15630 use std::os::unix::process::ExitStatusExt;
15631
15632 status.signal()
15633}
15634
15635#[cfg(not(unix))]
15636fn exit_signal(_status: &ExitStatus) -> Option<i32> {
15637 None
15638}
15639
15640fn reset_restart_count(snapshot: &SharedSnapshot, module_id: &str) -> Result<(), SuperviseError> {
15646 update_snapshot(snapshot, Some(module_id), |state| {
15647 state.clear_crash_restarts();
15648 })
15649}
15650
15651fn set_running(
15652 snapshot: &SharedSnapshot,
15653 child: &SupervisedChild,
15654 module_id: &str,
15655 spawn_events: &SpawnEventFeed,
15656) -> Result<(), SuperviseError> {
15657 let mut state = snapshot.lock().map_err(|_| SuperviseError::StatePoisoned {
15658 module_id: Some(module_id.to_string()),
15659 })?;
15660 state.spawn_generation = spawn_events.emit_spawned(module_id, child.pid, child.spawned_at_ms);
15661 if std::mem::take(&mut state.coalesced_restart_pending) {
15662 let generation = state.spawn_generation;
15663 info!(module_id, "restart already satisfied by generation {generation}; coalesced pending request completed");
15664 }
15665 state.drain_disposition_detail = None;
15666 state.spawn_failure = None;
15667 state.admission_refused = None;
15668 state.config_hold_parked = false;
15669 state.config_hold_daemon_initiated = false;
15670 state.config_hold_approval = None;
15671 state.legacy_registration_spawn =
15672 child.registration_spawn.is_some() && child.admission_base.is_none();
15673 state.registration_declaration = child.registration_declaration.clone();
15674 state.stop_reason = None;
15675 state.config_exit_failed_approval = None;
15676 state.waiting_for_previous_pid = None;
15677 state.waiting_for_previous_detail = None;
15678 state.in_alternate_slot = false;
15681 state.configuration_updated_since_spawn = false;
15682 state.spawned_protocol = Some(child.protocol);
15683 state.state = ModuleState::Running;
15684 state.approved_start_failure = None;
15685 state.enabled = true;
15686 state.process_alive = true;
15687 state.pid = child.id();
15688 state.subtree_target = child.subtree_target.clone();
15689 state.health.status = SupervisorHealthStatus::Unknown;
15692 state.last_probe_evidence = None;
15693 state.health.consecutive_failures = 0;
15694 state.health.detail = None;
15695 state.health.metrics = None;
15696 state.health.last_probe_ms = None;
15697 state.failed_retry.healthy_since = None;
15698 state.failed_retry.failed_at = None;
15699 state.failed_retry.running_since = Some(child.spawned_at);
15700 #[cfg(windows)]
15701 {
15702 state.pipe_consumed = child
15703 .nonce_delivery
15704 .as_ref()
15705 .map(|delivery| delivery.consumed());
15706 }
15707 #[cfg(target_os = "macos")]
15708 {
15709 state.report_ready = Some(Arc::clone(&child.report_ready));
15710 }
15711 state.spawned_at_ms = Some(child.spawned_at_ms);
15712 state.exec_outcome = Some(Arc::clone(&child.exec.0));
15713 state.spawned_from = Some(child.spawned_from.clone());
15714 state.launch_spec_digest = Some(child.launch_spec_digest.clone());
15715 state.spawned_file_identity = child.spawned_file_identity.clone();
15716 state.process_start_time = child.process_start_time;
15717 Ok(())
15718}
15719
15720fn clear_current_process_facts(state: &mut SupervisorSnapshot) {
15721 state.legacy_registration_spawn = false;
15722 state.registration_declaration = None;
15723 state.subtree_target = crate::subtree_resources::Target::Unavailable("not_running");
15724 state.failed_retry.healthy_since = None;
15725 state.failed_retry.running_since = None;
15726 state.waiting_for_previous_pid = None;
15727 state.waiting_for_previous_detail = None;
15728 state.process_alive = false;
15729 state.spawned_protocol = None;
15730 state.pid = None;
15731 #[cfg(windows)]
15732 {
15733 state.pipe_consumed = None;
15734 }
15735 #[cfg(target_os = "macos")]
15736 {
15737 state.report_ready = None;
15738 }
15739 state.spawned_at_ms = None;
15740 state.exec_outcome = None;
15741 state.spawned_from = None;
15742 state.launch_spec_digest = None;
15743 state.spawned_file_identity = None;
15744 state.process_start_time = None;
15745 state.deliberate_severance = None;
15746}
15747
15748#[cfg(test)]
15749fn record_deliberate_severance(
15750 snapshot: &SharedSnapshot,
15751 identity: ProcessIdentity,
15752) -> Result<(), SuperviseError> {
15753 update_snapshot(snapshot, None, |state| {
15754 state.deliberate_severance = Some(identity);
15755 })
15756}
15757
15758fn apply_deliberate_severance_marker(
15759 snapshot: &SharedSnapshot,
15760 exited_identity: Option<ProcessIdentity>,
15761 mut exit_report: ExitReport,
15762) -> ExitReport {
15763 let marker = lock_snapshot(snapshot)
15764 .ok()
15765 .and_then(|mut state| state.deliberate_severance.take());
15766 if marker.is_some() && marker == exited_identity {
15767 exit_report.kind = ExitKind::DeliberateSeverance;
15768 }
15769 exit_report
15770}
15771
15772fn classify_reaped_child_exit(
15773 snapshot: &SharedSnapshot,
15774 child: &SupervisedChild,
15775 status: &ExitStatus,
15776) -> ExitReport {
15777 let _ = update_snapshot(snapshot, None, |state| {
15778 state.reaped_pid = Some(child.pid);
15779 state.spawn_failure = child.spawn_failure.clone();
15780 });
15781 if matches!(
15782 child.spawn_failure.as_deref(),
15783 Some("registration_timeout" | "registration_channel_closed")
15784 ) {
15785 registration_failure_exit_report(classify_exit(status))
15786 } else {
15787 apply_deliberate_severance_marker(snapshot, child.process_identity(), classify_exit(status))
15788 }
15789}
15790
15791fn fail_snapshot(
15792 snapshot: &SharedSnapshot,
15793 module_id: Option<&str>,
15794 last_exit: Option<ExitReport>,
15795 failure: &FailureContext,
15796) {
15797 if let Err(err) = update_snapshot(snapshot, module_id, |state| {
15798 state.state = ModuleState::Failed;
15799 state.config_hold_parked = false;
15800 state.config_hold_daemon_initiated = false;
15801 state.config_hold_approval = None;
15802 clear_current_process_facts(state);
15803 if failure.may_retry() && state.enabled {
15804 state.failed_retry.schedule(module_id.unwrap_or("unknown"));
15805 } else if failure.shutdown.is_set() {
15806 state.failed_retry.cancel();
15807 }
15808 if let Some(last_exit) = last_exit {
15809 state.last_exit = Some(last_exit);
15810 }
15811 }) {
15812 error!(error = %err, "failed to mark supervisor state failed");
15813 }
15814}
15815
15816fn update_snapshot(
15817 snapshot: &SharedSnapshot,
15818 module_id: Option<&str>,
15819 update: impl FnOnce(&mut SupervisorSnapshot),
15820) -> Result<(), SuperviseError> {
15821 let mut state = snapshot.lock().map_err(|_| SuperviseError::StatePoisoned {
15822 module_id: module_id.map(ToOwned::to_owned),
15823 })?;
15824 update(&mut state);
15825 Ok(())
15826}
15827
15828const SLOW_SNAPSHOT_LOCK_THRESHOLD: Duration = Duration::from_millis(250);
15829
15830fn lock_snapshot_for_control<'a>(
15831 snapshot: &'a SharedSnapshot,
15832 module_id: &str,
15833 caller: &'static str,
15834) -> Result<std::sync::MutexGuard<'a, SupervisorSnapshot>, SuperviseError> {
15835 let started_at = Instant::now();
15836 let guard = lock_snapshot(snapshot)?;
15837 let waited = started_at.elapsed();
15838 if waited >= SLOW_SNAPSHOT_LOCK_THRESHOLD {
15839 warn!(
15840 module_id = %module_id,
15841 waited_ms = waited.as_millis() as u64,
15842 caller = %caller,
15843 "slow snapshot lock"
15844 );
15845 }
15846 Ok(guard)
15847}
15848
15849fn lock_snapshot(
15850 snapshot: &SharedSnapshot,
15851) -> Result<std::sync::MutexGuard<'_, SupervisorSnapshot>, SuperviseError> {
15852 snapshot
15853 .lock()
15854 .map_err(|_| SuperviseError::StatePoisoned { module_id: None })
15855}
15856
15857#[cfg(test)]
15858mod terminal_history_tests {
15859 include!("supervise_route_open_recovery_tests.rs");
15860 use std::{
15861 path::PathBuf,
15862 sync::Arc,
15863 time::{Duration, Instant},
15864 };
15865
15866 use tokio::{sync::mpsc, time::sleep};
15867
15868 use super::{
15869 apply_deliberate_severance_marker, daemon_will_restart, drain_child_to_state,
15870 drained_after_quiescence_wait, handle_reload_spawn_failure, health_restart_child,
15871 lock_snapshot, on_child_exit, record_deliberate_severance, record_wait_error_terminal,
15872 reset_restart_count, spawn_and_mark_running, update_snapshot, wait_error_exit_report,
15873 ExitKind, ExitReport, ModuleProtocol, ModuleSpec, ModuleState, NextAction, ProcessIdentity,
15874 RestartPolicy, SpawnEventKind, StopNotice, SuperviseError, SupervisedConfiguration,
15875 SupervisedModule, SupervisedModuleInner, Supervisor, SupervisorHandle,
15876 SupervisorHealthStatus, SupervisorSnapshot,
15877 };
15878 use super::Instant as ClockInstant;
15883 use crate::{
15884 registry::Registry,
15885 terminal_ring::{TerminalRing, TerminalRingConfig},
15886 };
15887 use std::sync::Mutex;
15888 use subc_control::TerminalDisposition;
15889
15890 pub(super) fn fake_aft_stub_path() -> PathBuf {
15895 let path = cortexkit_test_support::cargo_profile_bin("fake-aft-stub");
15896 assert!(
15897 path.exists(),
15898 "fake-aft-stub not built at {}: run `cargo test -p subc-core` (which builds \
15899 [[bin]] targets) rather than `cargo test -p subc-core --lib` (which does not)",
15900 path.display()
15901 );
15902 path
15903 }
15904
15905 #[test]
15906 fn reserved_never_spawned_refuses_every_hello() {
15907 let supervisor = SupervisorHandle::default();
15912 supervisor.apply_identity_configuration(&ModuleSpec {
15913 module_id: "never-spawned".to_string(),
15914 program: PathBuf::from("/usr/bin/false"),
15915 args: Vec::new(),
15916 env: Vec::new(),
15917 reserved: true,
15918 reserved_prefixes: Vec::new(),
15919 protocol: ModuleProtocol::Subc,
15920 overlap: Default::default(),
15921 carry_containment: false,
15922 });
15923 assert!(
15924 supervisor
15925 .reserved_hello_verification("never-spawned", Some("any-forged-nonce"))
15926 .is_err(),
15927 "forged nonce must refuse on a reserved never-spawned id"
15928 );
15929 assert!(
15930 supervisor
15931 .reserved_hello_verification("never-spawned", None)
15932 .is_err(),
15933 "absent nonce must refuse on a reserved never-spawned id"
15934 );
15935 supervisor.set_spawn_nonce("never-spawned", "minted".to_string());
15937 supervisor.apply_identity_configuration(&ModuleSpec {
15938 module_id: "never-spawned".to_string(),
15939 program: PathBuf::from("/usr/bin/false"),
15940 args: Vec::new(),
15941 env: Vec::new(),
15942 reserved: true,
15943 reserved_prefixes: Vec::new(),
15944 protocol: ModuleProtocol::Subc,
15945 overlap: Default::default(),
15946 carry_containment: false,
15947 });
15948 assert!(supervisor
15949 .reserved_hello_verification("never-spawned", Some("minted"))
15950 .is_ok());
15951 assert!(supervisor
15952 .reserved_hello_verification("never-spawned", Some("forged"))
15953 .is_err());
15954 }
15955
15956 fn seed_crash_restarts(state: &mut SupervisorSnapshot, count: u32) {
15959 let now = ClockInstant::now();
15960 for _ in 0..count {
15961 state.crash_restarts.push_back(now);
15962 }
15963 }
15964
15965 fn age_oldest_crash_restart_out_of_window(state: &mut SupervisorSnapshot, window: Duration) {
15969 let aged = state
15970 .crash_restarts
15971 .front()
15972 .expect("a crash restart must be recorded before it can be aged")
15973 .checked_sub(window + Duration::from_secs(1))
15974 .expect("the test clock is far enough from its origin to age an instant");
15975 state.crash_restarts[0] = aged;
15976 }
15977
15978 fn snapshot_with_restarts(enabled: bool, count: u32) -> SupervisorSnapshot {
15979 let mut state = SupervisorSnapshot::new(ModuleState::Running, enabled);
15980 seed_crash_restarts(&mut state, count);
15981 state
15982 }
15983
15984 #[test]
15985 fn daemon_owned_recovery_predicate_uses_the_pre_increment_budget() {
15986 let policy = RestartPolicy::new(3, Duration::ZERO);
15987 let now = ClockInstant::now();
15988 assert!(daemon_will_restart(
15989 &mut snapshot_with_restarts(true, 2),
15990 &policy,
15991 now
15992 ));
15993 assert!(!daemon_will_restart(
15994 &mut snapshot_with_restarts(true, 3),
15995 &policy,
15996 now
15997 ));
15998 assert!(!daemon_will_restart(
15999 &mut snapshot_with_restarts(false, 0),
16000 &policy,
16001 now
16002 ));
16003 }
16004
16005 #[test]
16006 fn crash_restart_backoff_escalates_with_in_window_count() {
16007 let policy = RestartPolicy::new(4, Duration::from_millis(100))
16008 .with_max_backoff(Duration::from_secs(30));
16009 let now = ClockInstant::now();
16010 let mut state = SupervisorSnapshot::new(ModuleState::Running, true);
16011 let schedules = (0..4)
16012 .map(|_| {
16013 state
16014 .next_crash_restart(&policy, now)
16015 .expect("the test policy allows four crash restarts")
16016 })
16017 .collect::<Vec<_>>();
16018
16019 assert_eq!(
16020 schedules
16021 .iter()
16022 .map(|schedule| schedule.restart_in_window)
16023 .collect::<Vec<_>>(),
16024 vec![0, 1, 2, 3]
16025 );
16026 assert_eq!(
16027 schedules
16028 .iter()
16029 .map(|schedule| schedule.delay)
16030 .collect::<Vec<_>>(),
16031 vec![
16032 Duration::from_millis(100),
16033 Duration::from_secs(1),
16034 Duration::from_secs(10),
16035 Duration::from_secs(30),
16036 ]
16037 );
16038 }
16039
16040 #[test]
16041 fn crash_restart_backoff_resets_after_ring_clear() {
16042 let policy = RestartPolicy::new(3, Duration::from_millis(100));
16043 let now = ClockInstant::now();
16044 let mut state = SupervisorSnapshot::new(ModuleState::Running, true);
16045 assert_eq!(
16046 state.next_crash_restart(&policy, now).unwrap().delay,
16047 Duration::from_millis(100)
16048 );
16049 assert_eq!(
16050 state.next_crash_restart(&policy, now).unwrap().delay,
16051 Duration::from_secs(1)
16052 );
16053
16054 state.clear_crash_restarts();
16055 let schedule = state
16056 .next_crash_restart(&policy, now)
16057 .expect("a cleared ring must allow another restart");
16058 assert_eq!(schedule.restart_in_window, 0);
16059 assert_eq!(schedule.delay, Duration::from_millis(100));
16060 }
16061
16062 #[test]
16063 fn crash_restart_backoff_ignores_aged_restarts() {
16064 let policy = RestartPolicy::new(3, Duration::from_millis(100));
16065 let now = ClockInstant::now();
16066 let mut state = SupervisorSnapshot::new(ModuleState::Running, true);
16067 state
16068 .next_crash_restart(&policy, now)
16069 .expect("the first restart is allowed");
16070 state
16071 .next_crash_restart(&policy, now)
16072 .expect("the second restart is allowed");
16073 state.crash_restarts[0] = now
16074 .checked_sub(policy.window + Duration::from_secs(1))
16075 .expect("the fake clock can age a restart past the window");
16076
16077 let schedule = state
16078 .next_crash_restart(&policy, now)
16079 .expect("an aged restart must release its slot");
16080 assert_eq!(schedule.restart_in_window, 1);
16081 assert_eq!(schedule.delay, Duration::from_secs(1));
16082 assert_eq!(state.crash_restarts.len(), 2);
16083 }
16084
16085 #[test]
16089 fn a_budget_spent_before_the_window_no_longer_refuses() {
16090 let policy = RestartPolicy::new(3, Duration::ZERO);
16091 let mut state = snapshot_with_restarts(true, 3);
16092 let now = ClockInstant::now();
16093 assert!(!daemon_will_restart(&mut state, &policy, now));
16094
16095 assert!(daemon_will_restart(
16096 &mut state,
16097 &policy,
16098 now + policy.window + Duration::from_secs(1)
16099 ));
16100 assert!(
16101 state.crash_restarts.is_empty(),
16102 "reading the budget must drop the instants that left the window"
16103 );
16104 }
16105
16106 fn module_with_recovery_snapshot(
16107 state: ModuleState,
16108 enabled: bool,
16109 restart_count: u32,
16110 ) -> SupervisedModule {
16111 let registry = Arc::new(Registry::default());
16112 let supervisor =
16113 Supervisor::new_for_test(Arc::clone(®istry), RestartPolicy::new(3, Duration::ZERO));
16114 let runtime = supervisor.runtime_config();
16115 let spec = ModuleSpec {
16116 module_id: "recovery-snapshot".to_string(),
16117 program: fake_aft_stub_path(),
16118 args: Vec::new(),
16119 env: Vec::new(),
16120 reserved: false,
16121 reserved_prefixes: Vec::new(),
16122 protocol: ModuleProtocol::Subc,
16123 overlap: Default::default(),
16124 carry_containment: false,
16125 };
16126 let mut snapshot = SupervisorSnapshot::new(state, enabled);
16127 seed_crash_restarts(&mut snapshot, restart_count);
16128 let (commands, _rx) = mpsc::channel(4);
16131 let configured_enabled = snapshot.enabled;
16132 SupervisedModule {
16133 inner: Arc::new(SupervisedModuleInner {
16134 module_id: spec.module_id.clone(),
16135 registry,
16136 snapshot: Arc::new(Mutex::new(snapshot)),
16137 configuration: Arc::new(Mutex::new(SupervisedConfiguration {
16138 spec,
16139 enabled: configured_enabled,
16140 health: runtime.health,
16141 register_timeout: runtime.register_timeout,
16142 restart_policy: runtime.restart_policy,
16143 })),
16144 stderr_ring: runtime.stderr_ring,
16145 terminal_ring: runtime.terminal_ring,
16146 commands,
16147 monitor: Mutex::new(None),
16148 effective_drain_timeout: runtime.effective_drain_timeout,
16149 provenance_probe: supervisor.provenance_probe.clone(),
16150 retained: runtime.child_roster.retained_view(),
16151 }),
16152 }
16153 }
16154
16155 #[cfg(target_os = "linux")]
16156 #[tokio::test]
16157 async fn no_cgroup_placement_does_not_block_fake_aft_stub_spawn() {
16158 let supervisor =
16159 Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default())
16160 .with_cgroup_placement(None);
16161 let result = supervisor.spawn(ModuleSpec {
16162 module_id: "no-cgroup-placement".to_string(),
16163 program: fake_aft_stub_path(),
16164 args: Vec::new(),
16165 env: Vec::new(),
16166 reserved: false,
16167 reserved_prefixes: Vec::new(),
16168 protocol: ModuleProtocol::Subc,
16169 overlap: Default::default(),
16170 carry_containment: false,
16171 });
16172
16173 assert!(
16174 result.is_ok(),
16175 "no delegation must not turn an otherwise valid spawn into a failure: {result:?}"
16176 );
16177 }
16178
16179 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
16180 async fn undecided_snapshot_uses_shared_restart_predicate() {
16181 assert!(module_with_recovery_snapshot(ModuleState::Running, true, 2)
16182 .will_recover_after_connection_loss()
16183 .unwrap());
16184 assert!(
16185 !module_with_recovery_snapshot(ModuleState::Running, true, 3)
16186 .will_recover_after_connection_loss()
16187 .unwrap()
16188 );
16189 }
16190
16191 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
16192 async fn restarting_snapshot_at_exhausted_budget_is_non_terminal() {
16193 assert!(
16194 module_with_recovery_snapshot(ModuleState::Restarting, true, 3)
16195 .will_recover_after_connection_loss()
16196 .unwrap()
16197 );
16198 }
16199
16200 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
16201 async fn terminal_phase_snapshots_are_terminal_before_budget_exhaustion() {
16202 assert!(!module_with_recovery_snapshot(ModuleState::Failed, true, 0)
16203 .will_recover_after_connection_loss()
16204 .unwrap());
16205 assert!(
16206 !module_with_recovery_snapshot(ModuleState::Disabled, true, 0)
16207 .will_recover_after_connection_loss()
16208 .unwrap()
16209 );
16210 }
16211
16212 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
16213 async fn warming_snapshot_is_limited_to_startup_phases() {
16214 for state in [
16215 ModuleState::Starting,
16216 ModuleState::Running,
16217 ModuleState::Restarting,
16218 ] {
16219 assert!(
16220 module_with_recovery_snapshot(state, true, 0)
16221 .is_warming()
16222 .unwrap(),
16223 "{state:?} should be warming"
16224 );
16225 }
16226 for state in [
16227 ModuleState::Unresponsive,
16228 ModuleState::Draining,
16229 ModuleState::Stopped,
16230 ModuleState::Failed,
16231 ModuleState::Disabled,
16232 ] {
16233 assert!(
16234 !module_with_recovery_snapshot(state, true, 0)
16235 .is_warming()
16236 .unwrap(),
16237 "{state:?} should not be warming"
16238 );
16239 }
16240 }
16241
16242 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
16243 async fn terminal_history_survives_respawn_and_keeps_both_crashes_in_order() {
16244 let registry = Arc::new(Registry::default());
16245 let supervisor =
16246 Supervisor::new_for_test(Arc::clone(®istry), RestartPolicy::new(1, Duration::ZERO));
16247 let module = supervisor
16248 .spawn(ModuleSpec {
16249 module_id: "terminal-history".to_string(),
16250 program: fake_aft_stub_path(),
16251 args: Vec::new(),
16252 env: vec![("FAKE_AFT_EXIT_CODE".to_string(), "23".to_string())],
16253 reserved: false,
16254 reserved_prefixes: Vec::new(),
16255 protocol: ModuleProtocol::Subc,
16256 overlap: Default::default(),
16257 carry_containment: false,
16258 })
16259 .unwrap();
16260
16261 let deadline = Instant::now() + Duration::from_secs(5);
16262 loop {
16263 let history = module.terminal_history();
16264 if history.entries.len() == 2 {
16265 assert_eq!(module.status().unwrap().state, ModuleState::Failed);
16266 assert_eq!(history.dropped, 0);
16267 assert_eq!(
16268 history
16269 .entries
16270 .iter()
16271 .map(|entry| entry.exit_code)
16272 .collect::<Vec<_>>(),
16273 vec![Some(23), Some(23)]
16274 );
16275 assert!(history.entries[0].at_ms <= history.entries[1].at_ms);
16276 return;
16277 }
16278 assert!(
16279 Instant::now() < deadline,
16280 "module did not retain two terminal exits: {history:?}"
16281 );
16282 sleep(Duration::from_millis(10)).await;
16283 }
16284 }
16285
16286 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
16290 async fn disable_during_crash_backoff_cancels_pending_respawn() {
16291 let backoff = Duration::from_secs(2);
16292 let supervisor = Supervisor::new_for_test(
16293 Arc::new(Registry::default()),
16294 RestartPolicy::new(10, backoff),
16295 );
16296 let module = supervisor
16297 .spawn(ModuleSpec {
16298 module_id: "disable-during-backoff".to_string(),
16299 program: fake_aft_stub_path(),
16300 args: Vec::new(),
16301 env: vec![("FAKE_AFT_EXIT_CODE".to_string(), "23".to_string())],
16302 reserved: false,
16303 reserved_prefixes: Vec::new(),
16304 protocol: ModuleProtocol::Subc,
16305 overlap: Default::default(),
16306 carry_containment: false,
16307 })
16308 .unwrap();
16309
16310 let deadline = Instant::now() + Duration::from_secs(5);
16312 loop {
16313 if module.status().unwrap().state == ModuleState::Restarting {
16314 break;
16315 }
16316 assert!(
16317 Instant::now() < deadline,
16318 "module never entered the crash backoff"
16319 );
16320 sleep(Duration::from_millis(10)).await;
16321 }
16322
16323 let started = Instant::now();
16324 module.set_enabled(false).await.unwrap();
16325 let waited = started.elapsed();
16326
16327 assert!(
16328 waited < backoff / 2,
16329 "disable waited {waited:?} behind the {backoff:?} crash backoff; the operator command must preempt the pending respawn"
16330 );
16331 assert_eq!(module.status().unwrap().state, ModuleState::Disabled);
16332
16333 sleep(backoff + Duration::from_millis(500)).await;
16335 let status = module.status().unwrap();
16336 assert_eq!(status.state, ModuleState::Disabled);
16337 assert_eq!(
16338 status.spawn_generation, 1,
16339 "module respawned after the operator disabled it"
16340 );
16341 }
16342
16343 #[cfg(unix)]
16346 fn protocol_none_sigterm_exits_clean_spec(
16347 module_id: &str,
16348 dir: &std::path::Path,
16349 ) -> (ModuleSpec, PathBuf, PathBuf) {
16350 let ready = dir.join("ready");
16351 let marker = dir.join("sigterm");
16352 let spec = ModuleSpec {
16353 module_id: module_id.to_string(),
16354 program: fake_aft_stub_path(),
16355 args: Vec::new(),
16356 env: vec![
16357 ("FAKE_AFT_NEVER_CONNECT".to_string(), "1".to_string()),
16358 (
16359 "FAKE_AFT_SIGTERM_MARKER_PATH".to_string(),
16360 marker.display().to_string(),
16361 ),
16362 (
16363 "FAKE_AFT_NEVER_CONNECT_READY_PATH".to_string(),
16364 ready.display().to_string(),
16365 ),
16366 ],
16367 reserved: false,
16368 reserved_prefixes: Vec::new(),
16369 protocol: ModuleProtocol::None,
16370 overlap: Default::default(),
16371 carry_containment: false,
16372 };
16373 (spec, ready, marker)
16374 }
16375
16376 #[cfg(unix)]
16380 async fn wait_for_file(path: &std::path::Path) {
16381 let deadline = Instant::now() + Duration::from_secs(10);
16382 while !path.exists() {
16383 assert!(
16384 Instant::now() < deadline,
16385 "{} never appeared",
16386 path.display()
16387 );
16388 sleep(Duration::from_millis(10)).await;
16389 }
16390 }
16391
16392 #[cfg(unix)]
16396 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
16397 async fn protocol_none_unrequested_clean_exit_restarts_as_a_crash() {
16398 let dir = cortexkit_test_support::ScratchDir::new("none-unrequested-clean-exit");
16399 let (spec, ready, marker) =
16400 protocol_none_sigterm_exits_clean_spec("none-unrequested-clean-exit", dir.path());
16401 let supervisor = Supervisor::new_for_test(
16402 Arc::new(Registry::default()),
16403 RestartPolicy::new(3, Duration::ZERO),
16404 );
16405 let module = supervisor.spawn(spec).unwrap();
16406 wait_for_file(&ready).await;
16407 let deadline = Instant::now() + Duration::from_secs(10);
16411 let first_pid = loop {
16412 let status = module.status().unwrap();
16413 if status.state == ModuleState::Running {
16414 if let Some(pid) = status.pid {
16415 break pid;
16416 }
16417 }
16418 assert!(
16419 Instant::now() < deadline,
16420 "a running module must report its pid after exec confirmation: {status:?}"
16421 );
16422 sleep(Duration::from_millis(10)).await;
16423 };
16424
16425 rustix::process::kill_process(
16426 rustix::process::Pid::from_raw(i32::try_from(first_pid).unwrap()).unwrap(),
16427 rustix::process::Signal::TERM,
16428 )
16429 .unwrap();
16430
16431 let deadline = Instant::now() + Duration::from_secs(10);
16432 let respawned = loop {
16433 let status = module.status().unwrap();
16434 if status.state == ModuleState::Running
16435 && status.pid.is_some_and(|pid| pid != first_pid)
16436 {
16437 break status;
16438 }
16439 assert!(
16440 Instant::now() < deadline,
16441 "protocol-none module was not respawned after an unrequested clean exit: {status:?}"
16442 );
16443 sleep(Duration::from_millis(10)).await;
16444 };
16445 assert_eq!(respawned.spawn_generation, 2);
16446 assert!(
16447 marker.exists(),
16448 "the child must have exited through its SIGTERM handler (exit 0), or this proves nothing about clean exits"
16449 );
16450
16451 let history = module.terminal_history();
16452 assert_eq!(history.entries.len(), 1, "{history:?}");
16453 let entry = &history.entries[0];
16454 assert_eq!(entry.exit_code, Some(0));
16455 assert_eq!(entry.exit_kind, subc_control::TerminalExitKind::Clean);
16456 assert_eq!(entry.disposition, TerminalDisposition::Restarting);
16457
16458 module.stop().await.unwrap();
16459 }
16460
16461 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
16465 async fn protocol_none_repeated_clean_exits_exhaust_the_restart_budget() {
16466 let supervisor = Supervisor::new_for_test(
16467 Arc::new(Registry::default()),
16468 RestartPolicy::new(1, Duration::ZERO),
16469 );
16470 let module = supervisor
16471 .spawn(ModuleSpec {
16472 module_id: "none-clean-exit-budget".to_string(),
16473 program: fake_aft_stub_path(),
16474 args: Vec::new(),
16475 env: vec![("FAKE_AFT_EXIT_CODE".to_string(), "0".to_string())],
16476 reserved: false,
16477 reserved_prefixes: Vec::new(),
16478 protocol: ModuleProtocol::None,
16479 overlap: Default::default(),
16480 carry_containment: false,
16481 })
16482 .unwrap();
16483
16484 let deadline = Instant::now() + Duration::from_secs(10);
16487 loop {
16488 let status = module.status().unwrap();
16489 if status.state == ModuleState::Failed {
16490 break;
16491 }
16492 assert!(
16493 Instant::now() < deadline,
16494 "module never exhausted its budget: {status:?} {:?}",
16495 module.terminal_history()
16496 );
16497 sleep(Duration::from_millis(10)).await;
16498 }
16499 let history = module.terminal_history();
16500 assert_eq!(
16501 history
16502 .entries
16503 .iter()
16504 .map(|entry| (entry.exit_code, entry.disposition.clone()))
16505 .collect::<Vec<_>>(),
16506 vec![
16507 (Some(0), TerminalDisposition::Restarting),
16508 (Some(0), TerminalDisposition::Failed),
16509 ]
16510 );
16511 let detail = history.entries[1]
16512 .disposition_detail
16513 .as_deref()
16514 .expect("a budget failure names the budget");
16515 assert!(detail.contains("max_restarts=1"), "{detail}");
16516 assert_eq!(module.status().unwrap().spawn_generation, 2);
16517 }
16518
16519 #[test]
16520 fn restart_budget_failure_is_published_after_its_terminal_record() {
16521 use std::io::{Read, Write};
16522 let supervisor = Supervisor::new_for_test(
16523 Arc::new(Registry::default()),
16524 RestartPolicy::new(0, Duration::ZERO),
16525 );
16526 let runtime = supervisor.runtime_config();
16527 let spec = ModuleSpec {
16528 protocol: ModuleProtocol::None,
16529 ..windowed_crash_spec("budget-publication-order")
16530 };
16531 let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::new(
16532 ModuleState::Running,
16533 true,
16534 )));
16535 lock_snapshot(&snapshot).unwrap().process_alive = true;
16536 let reaped_snapshot = snapshot.clone();
16537 let mut barrier = terminal_record_barrier(&runtime.spawn_events);
16538 let history = runtime.terminal_ring.clone();
16539 let reap = std::thread::spawn(move || {
16540 tokio::runtime::Builder::new_current_thread()
16541 .enable_all()
16542 .build()
16543 .unwrap()
16544 .block_on(on_child_exit(
16545 &spec,
16546 runtime.restart_policy,
16547 &supervisor.registry,
16548 &reaped_snapshot,
16549 &runtime.terminal_ring,
16550 &runtime.spawn_events,
16551 &runtime.child_roster,
16552 ExitReport {
16553 kind: ExitKind::Clean,
16554 code: Some(0),
16555 signal: None,
16556 at_ms: 1,
16557 },
16558 true,
16559 ))
16560 });
16561 let mut reached = [0];
16562 barrier.read_exact(&mut reached).unwrap();
16563 assert_eq!(&reached, b"R");
16564 let (before_state, before_alive) = {
16565 let state = lock_snapshot(&snapshot).unwrap();
16566 (state.state, state.process_alive)
16567 };
16568 let before_history = history.lock().unwrap().snapshot();
16569 barrier.write_all(b"X").unwrap();
16570 assert!(matches!(reap.join().unwrap(), NextAction::Stop { .. }));
16571 assert!(before_history.entries.is_empty());
16572 assert_ne!(
16573 before_state,
16574 ModuleState::Failed,
16575 "Failed was visible before its terminal record could be written"
16576 );
16577 assert!(before_alive);
16578 assert_eq!(lock_snapshot(&snapshot).unwrap().state, ModuleState::Failed);
16579 let history = history.lock().unwrap().snapshot();
16580 assert_eq!(history.entries.len(), 1);
16581 assert_eq!(history.entries[0].exit_code, Some(0));
16582 assert_eq!(history.entries[0].disposition, TerminalDisposition::Failed);
16583 }
16584
16585 #[test]
16586 fn crash_restart_at_budget_limit_remains_recoverable_while_recording() {
16587 use std::io::{Read, Write};
16588
16589 let supervisor = Supervisor::new_for_test(
16590 Arc::new(Registry::default()),
16591 RestartPolicy::new(3, Duration::ZERO),
16592 );
16593 let mut runtime = supervisor.runtime_config();
16594 let module = module_with_recovery_snapshot(ModuleState::Running, true, 2);
16595 let snapshot = module.inner.snapshot.clone();
16596 lock_snapshot(&snapshot).unwrap().process_alive = true;
16597 runtime.terminal_ring = module.inner.terminal_ring.clone();
16598 let spec = windowed_crash_spec(module.module_id());
16599 let mut barrier = terminal_record_barrier(&runtime.spawn_events);
16600 let reaped_snapshot = snapshot.clone();
16601 let reap = std::thread::spawn(move || {
16602 tokio::runtime::Builder::new_current_thread()
16603 .enable_all()
16604 .build()
16605 .unwrap()
16606 .block_on(on_child_exit(
16607 &spec,
16608 runtime.restart_policy,
16609 &supervisor.registry,
16610 &reaped_snapshot,
16611 &runtime.terminal_ring,
16612 &runtime.spawn_events,
16613 &runtime.child_roster,
16614 ExitReport {
16615 kind: ExitKind::Crash,
16616 code: Some(2),
16617 signal: None,
16618 at_ms: 1,
16619 },
16620 true,
16621 ))
16622 });
16623
16624 let mut reached = [0];
16628 barrier.read_exact(&mut reached).unwrap();
16629 assert_eq!(&reached, b"R");
16630 let before_status = module.status().unwrap();
16631 let before_history = module.terminal_history();
16632 let before_recovers = module.will_recover_after_connection_loss().unwrap();
16633 barrier.write_all(b"X").unwrap();
16634 assert!(matches!(reap.join().unwrap(), NextAction::Restart { .. }));
16635
16636 assert_eq!(before_status.state, ModuleState::Running);
16637 assert!(before_status.process_alive);
16638 assert_eq!(before_status.restart_count, 3);
16639 assert!(before_history.entries.is_empty());
16640 assert!(
16641 before_recovers,
16642 "connection cleanup would report a terminal close for a scheduled crash restart"
16643 );
16644 let restarting = module.status().unwrap();
16645 assert_eq!(restarting.state, ModuleState::Restarting);
16646 assert_eq!(restarting.restart_count, 3);
16647 assert!(module.will_recover_after_connection_loss().unwrap());
16648 let history = module.terminal_history();
16649 assert_eq!(history.entries.len(), 1);
16650 assert_eq!(
16651 history.entries[0].disposition,
16652 TerminalDisposition::Restarting
16653 );
16654
16655 update_snapshot(&snapshot, Some(module.module_id()), |state| {
16658 state.state = ModuleState::Running;
16659 state.respawn_pending = false;
16660 })
16661 .unwrap();
16662 assert!(!module.will_recover_after_connection_loss().unwrap());
16663 update_snapshot(&snapshot, Some(module.module_id()), |state| {
16664 state.enabled = false;
16665 state.respawn_pending = true;
16666 })
16667 .unwrap();
16668 assert!(!module.will_recover_after_connection_loss().unwrap());
16669 }
16670
16671 fn terminal_record_barrier(events: &super::SpawnEventFeed) -> std::net::TcpStream {
16672 use std::net::{TcpListener, TcpStream};
16673 let listener = TcpListener::bind("127.0.0.1:0").unwrap();
16674 let writer = TcpStream::connect(listener.local_addr().unwrap()).unwrap();
16675 let (barrier, _) = listener.accept().unwrap();
16676 for stream in [&writer, &barrier] {
16677 stream
16678 .set_read_timeout(Some(Duration::from_secs(10)))
16679 .unwrap();
16680 stream
16681 .set_write_timeout(Some(Duration::from_secs(10)))
16682 .unwrap();
16683 }
16684 *events.1.lock().unwrap() = Some(writer);
16685 barrier
16686 }
16687
16688 pub(super) fn before_terminal_record(events: &super::SpawnEventFeed) {
16689 use std::io::{Read, Write};
16690 let barrier = events.1.lock().unwrap().take();
16691 if let Some(mut barrier) = barrier {
16692 barrier.write_all(b"R").unwrap();
16693 let mut release = [0];
16694 barrier.read_exact(&mut release).unwrap();
16695 assert_eq!(&release, b"X");
16696 }
16697 }
16698
16699 #[test]
16700 fn health_restart_budget_failure_is_published_after_its_terminal_record() {
16701 use std::io::{Read, Write};
16702
16703 let supervisor = Supervisor::new_for_test(
16704 Arc::new(Registry::default()),
16705 RestartPolicy::new(0, Duration::ZERO),
16706 );
16707 let mut runtime = supervisor.runtime_config();
16708 let module = module_with_recovery_snapshot(ModuleState::Running, true, 0);
16709 let snapshot = module.inner.snapshot.clone();
16710 runtime.terminal_ring = module.inner.terminal_ring.clone();
16711 let spec = ModuleSpec {
16712 module_id: module.module_id().into(),
16713 program: fake_aft_stub_path(),
16714 protocol: ModuleProtocol::None,
16715 env: vec![("FAKE_AFT_EXIT_CODE".into(), "17".into())],
16716 ..windowed_crash_spec(module.module_id())
16717 };
16718 let mut barrier = terminal_record_barrier(&runtime.spawn_events);
16719 let reap = std::thread::spawn(move || {
16720 tokio::runtime::Builder::new_current_thread()
16721 .enable_all()
16722 .build()
16723 .unwrap()
16724 .block_on(async {
16725 let mut child =
16726 Some(spawn_and_mark_running(&spec, &runtime, &snapshot).unwrap());
16727 health_restart_child(
16728 &spec,
16729 &runtime,
16730 &supervisor.registry,
16731 &supervisor.process_liveness,
16732 &snapshot,
16733 &mut child,
16734 SupervisorHealthStatus::Failing,
16735 Some("health probe did not answer"),
16736 1,
16737 )
16738 .await
16739 .unwrap();
16740 assert!(child.is_none());
16741 });
16742 });
16743 let mut reached = [0];
16748 barrier.read_exact(&mut reached).unwrap();
16749 assert_eq!(&reached, b"R");
16750 let before_status = module.status().unwrap();
16751 let before_history = module.terminal_history();
16752 barrier.write_all(b"X").unwrap();
16753 reap.join().unwrap();
16754 assert!(before_history.entries.is_empty());
16755 assert_ne!(
16756 before_status.state,
16757 ModuleState::Failed,
16758 "Failed was visible before its terminal record could be written"
16759 );
16760 assert!(
16761 before_status.process_alive,
16762 "process death was visible before its terminal record could be written"
16763 );
16764 let failed = module.status().unwrap();
16765 assert_eq!(failed.state, ModuleState::Failed);
16766 assert!(!failed.process_alive);
16767 let history = module.terminal_history();
16768 assert_eq!(history.entries.len(), 1);
16769 assert_eq!(history.entries[0].disposition, TerminalDisposition::Failed);
16770 assert_eq!(
16771 history.entries[0].disposition_detail.as_deref(),
16772 Some("crash budget exhausted: max_restarts=0 within window_secs=600")
16773 );
16774 }
16775
16776 #[test]
16777 fn observed_exits_publish_process_death_after_terminal_history() {
16778 use std::io::{Read, Write};
16779 for (kind, enabled, shutdown, wait_error, expected_state) in [
16780 (ExitKind::Clean, true, false, false, ModuleState::Stopped),
16781 (ExitKind::Crash, true, false, false, ModuleState::Restarting),
16782 (ExitKind::Crash, false, false, false, ModuleState::Disabled),
16783 (
16784 ExitKind::DeliberateSeverance,
16785 true,
16786 false,
16787 false,
16788 ModuleState::Restarting,
16789 ),
16790 #[cfg(unix)]
16791 (ExitKind::Crash, true, true, false, ModuleState::Stopped),
16792 (ExitKind::Crash, true, false, true, ModuleState::Failed),
16793 ] {
16794 let supervisor = Supervisor::new_for_test(
16795 Arc::new(Registry::default()),
16796 RestartPolicy::new(1, Duration::ZERO),
16797 );
16798 let runtime = supervisor.runtime_config();
16799 #[cfg(unix)]
16800 if shutdown {
16801 runtime.child_roster.close();
16802 }
16803 #[cfg(not(unix))]
16804 let _ = shutdown;
16805 let spec = windowed_crash_spec("exit-publication-order");
16806 let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::new(
16807 ModuleState::Running,
16808 enabled,
16809 )));
16810 lock_snapshot(&snapshot).unwrap().process_alive = true;
16811 let reaped_snapshot = snapshot.clone();
16812 let history = runtime.terminal_ring.clone();
16813 let mut barrier = terminal_record_barrier(&runtime.spawn_events);
16814 let reap = std::thread::spawn(move || {
16815 if wait_error {
16816 super::fail_after_wait_error(
16817 &spec.module_id,
16818 &reaped_snapshot,
16819 &runtime.terminal_ring,
16820 &runtime.spawn_events,
16821 "wait failed".into(),
16822 &runtime.failure_context(true),
16823 );
16824 } else {
16825 tokio::runtime::Builder::new_current_thread()
16826 .enable_all()
16827 .build()
16828 .unwrap()
16829 .block_on(on_child_exit(
16830 &spec,
16831 runtime.restart_policy,
16832 &supervisor.registry,
16833 &reaped_snapshot,
16834 &runtime.terminal_ring,
16835 &runtime.spawn_events,
16836 &runtime.child_roster,
16837 ExitReport {
16838 kind,
16839 code: Some(0),
16840 signal: None,
16841 at_ms: 1,
16842 },
16843 true,
16844 ));
16845 }
16846 });
16847 let mut reached = [0];
16848 barrier.read_exact(&mut reached).unwrap();
16849 assert_eq!(&reached, b"R");
16850 let before_alive = lock_snapshot(&snapshot).unwrap().process_alive;
16851 let before_history = history.lock().unwrap().snapshot();
16852 barrier.write_all(b"X").unwrap();
16853 reap.join().unwrap();
16854 assert!(
16855 before_alive,
16856 "{kind:?}: death was published before recording"
16857 );
16858 assert!(before_history.entries.is_empty());
16859 assert_eq!(lock_snapshot(&snapshot).unwrap().state, expected_state);
16860 assert!(!lock_snapshot(&snapshot).unwrap().process_alive);
16861 assert_eq!(history.lock().unwrap().snapshot().entries.len(), 1);
16862 }
16863 }
16864
16865 #[tokio::test]
16866 async fn health_restart_budget_failure_remains_failed_when_journal_append_fails() {
16867 let dir = cortexkit_test_support::ScratchDir::new("health-budget-journal-failure");
16868 let path = dir.join("terminals.jsonl");
16869 std::fs::create_dir(&path).unwrap();
16870 let supervisor = Supervisor::new_for_test(
16871 Arc::new(Registry::default()),
16872 RestartPolicy::new(0, Duration::ZERO),
16873 )
16874 .with_terminal_journal(path, "health-budget-journal-failure".into());
16875 let runtime = supervisor.runtime_config();
16876 let spec = ModuleSpec {
16877 program: fake_aft_stub_path(),
16878 protocol: ModuleProtocol::None,
16879 env: vec![("FAKE_AFT_EXIT_CODE".into(), "17".into())],
16880 ..windowed_crash_spec("health-budget-journal-failure")
16881 };
16882 let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::starting()));
16883 let mut child = Some(spawn_and_mark_running(&spec, &runtime, &snapshot).unwrap());
16884 health_restart_child(
16885 &spec,
16886 &runtime,
16887 &supervisor.registry,
16888 &supervisor.process_liveness,
16889 &snapshot,
16890 &mut child,
16891 SupervisorHealthStatus::Failing,
16892 None,
16893 1,
16894 )
16895 .await
16896 .unwrap();
16897 assert_eq!(lock_snapshot(&snapshot).unwrap().state, ModuleState::Failed);
16898 assert!(!lock_snapshot(&snapshot).unwrap().process_alive);
16899 let history = runtime
16900 .terminal_ring
16901 .lock()
16902 .unwrap()
16903 .durable_history(&spec.module_id);
16904 assert!(history.journal_write_failures > 0);
16905 assert_eq!(history.entries.len(), 1);
16906 assert_eq!(history.entries[0].disposition, TerminalDisposition::Failed);
16907 }
16908
16909 #[test]
16910 fn health_restart_budget_failure_remains_failed_when_exit_recording_panics() {
16911 let supervisor = Supervisor::new_for_test(
16912 Arc::new(Registry::default()),
16913 RestartPolicy::new(0, Duration::ZERO),
16914 );
16915 let runtime = supervisor.runtime_config();
16916 let spec = ModuleSpec {
16917 program: fake_aft_stub_path(),
16918 protocol: ModuleProtocol::None,
16919 env: vec![("FAKE_AFT_EXIT_CODE".into(), "17".into())],
16920 ..windowed_crash_spec("health-budget-recording-panic")
16921 };
16922 let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::starting()));
16923 let result = std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| {
16924 tokio::runtime::Builder::new_current_thread()
16925 .enable_all()
16926 .build()
16927 .unwrap()
16928 .block_on(async {
16929 let mut child =
16930 Some(spawn_and_mark_running(&spec, &runtime, &snapshot).unwrap());
16931 runtime.spawn_events.0.lock().unwrap().seq = u64::MAX;
16932 health_restart_child(
16933 &spec,
16934 &runtime,
16935 &supervisor.registry,
16936 &supervisor.process_liveness,
16937 &snapshot,
16938 &mut child,
16939 SupervisorHealthStatus::Failing,
16940 None,
16941 1,
16942 )
16943 .await
16944 .unwrap();
16945 });
16946 }));
16947 let panic = result.expect_err("recording must still unwind");
16948 assert_eq!(
16949 panic.downcast_ref::<String>().map(String::as_str),
16950 Some("spawn event sequence exhausted")
16951 );
16952 let state = lock_snapshot(&snapshot).unwrap();
16953 assert_eq!(state.state, ModuleState::Failed);
16954 assert!(!state.process_alive);
16955 assert!(state.last_exit.is_some());
16956 }
16957
16958 #[tokio::test]
16959 async fn restart_budget_failure_remains_failed_when_journal_append_fails() {
16960 let dir = cortexkit_test_support::ScratchDir::new("budget-journal-failure");
16961 let path = dir.join("terminals.jsonl");
16962 std::fs::create_dir(&path).unwrap();
16963 let supervisor = Supervisor::new_for_test(
16964 Arc::new(Registry::default()),
16965 RestartPolicy::new(0, Duration::ZERO),
16966 )
16967 .with_terminal_journal(path, "budget-journal-failure".into());
16968 let runtime = supervisor.runtime_config();
16969 let spec = windowed_crash_spec("budget-journal-failure");
16970 let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::starting()));
16971 assert!(matches!(
16972 on_child_exit(
16973 &spec,
16974 runtime.restart_policy,
16975 &supervisor.registry,
16976 &snapshot,
16977 &runtime.terminal_ring,
16978 &runtime.spawn_events,
16979 &runtime.child_roster,
16980 crash_exit_report(1),
16981 true,
16982 )
16983 .await,
16984 NextAction::Stop { .. }
16985 ));
16986 assert_eq!(lock_snapshot(&snapshot).unwrap().state, ModuleState::Failed);
16987 let history = runtime
16988 .terminal_ring
16989 .lock()
16990 .unwrap()
16991 .durable_history(&spec.module_id);
16992 assert!(history.journal_write_failures > 0);
16993 assert_eq!(history.entries.len(), 1);
16994 assert_eq!(history.entries[0].disposition, TerminalDisposition::Failed);
16995 }
16996
16997 #[test]
16998 fn restart_budget_failure_remains_failed_when_exit_recording_panics() {
16999 let supervisor = Supervisor::new_for_test(
17000 Arc::new(Registry::default()),
17001 RestartPolicy::new(0, Duration::ZERO),
17002 );
17003 let runtime = supervisor.runtime_config();
17004 let spec = windowed_crash_spec("budget-recording-panic");
17005 let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::starting()));
17006 runtime.spawn_events.emit_spawned(&spec.module_id, 1, 1);
17007 runtime.spawn_events.0.lock().unwrap().seq = u64::MAX;
17010 let result = std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| {
17011 tokio::runtime::Builder::new_current_thread()
17012 .enable_all()
17013 .build()
17014 .unwrap()
17015 .block_on(on_child_exit(
17016 &spec,
17017 runtime.restart_policy,
17018 &supervisor.registry,
17019 &snapshot,
17020 &runtime.terminal_ring,
17021 &runtime.spawn_events,
17022 &runtime.child_roster,
17023 crash_exit_report(1),
17024 true,
17025 ))
17026 }));
17027 let panic = result.err().expect("recording must still unwind");
17028 assert_eq!(
17029 panic.downcast_ref::<String>().map(String::as_str),
17030 Some("spawn event sequence exhausted")
17031 );
17032 assert_eq!(lock_snapshot(&snapshot).unwrap().state, ModuleState::Failed);
17033 }
17034
17035 #[cfg(unix)]
17038 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
17039 async fn protocol_none_requested_stop_and_disable_do_not_respawn() {
17040 for disable in [false, true] {
17041 let label = if disable {
17042 "none-requested-disable"
17043 } else {
17044 "none-requested-stop"
17045 };
17046 let dir = cortexkit_test_support::ScratchDir::new(label);
17047 let (spec, ready, marker) = protocol_none_sigterm_exits_clean_spec(label, dir.path());
17048 let supervisor = Supervisor::new_for_test(
17049 Arc::new(Registry::default()),
17050 RestartPolicy::new(3, Duration::ZERO),
17051 );
17052 let module = supervisor.spawn(spec).unwrap();
17053 wait_for_file(&ready).await;
17054
17055 if disable {
17056 module.set_enabled(false).await.unwrap();
17057 } else {
17058 module.stop().await.unwrap();
17059 }
17060 assert!(
17061 marker.exists(),
17062 "{label}: the child must have left through its SIGTERM handler with exit 0"
17063 );
17064
17065 sleep(Duration::from_millis(500)).await;
17068 let status = module.status().unwrap();
17069 let expected = if disable {
17070 ModuleState::Disabled
17071 } else {
17072 ModuleState::Stopped
17073 };
17074 assert_eq!(status.state, expected, "{label}");
17075 assert_eq!(
17076 status.spawn_generation, 1,
17077 "{label}: respawned after a requested stop"
17078 );
17079 let history = module.terminal_history();
17080 assert_eq!(history.entries.len(), 1, "{label}: {history:?}");
17081 assert_eq!(history.entries[0].exit_code, Some(0), "{label}");
17082 assert_ne!(
17083 history.entries[0].disposition,
17084 TerminalDisposition::Restarting,
17085 "{label}"
17086 );
17087 }
17088 }
17089
17090 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
17093 async fn subc_wire_clean_exit_is_still_a_stop() {
17094 let supervisor = Supervisor::new_for_test(
17095 Arc::new(Registry::default()),
17096 RestartPolicy::new(3, Duration::ZERO),
17097 );
17098 let module = supervisor
17099 .spawn(ModuleSpec {
17100 module_id: "wire-clean-exit".to_string(),
17101 program: fake_aft_stub_path(),
17102 args: Vec::new(),
17103 env: vec![("FAKE_AFT_EXIT_CODE".to_string(), "0".to_string())],
17104 reserved: false,
17105 reserved_prefixes: Vec::new(),
17106 protocol: ModuleProtocol::Subc,
17107 overlap: Default::default(),
17108 carry_containment: false,
17109 })
17110 .unwrap();
17111
17112 let deadline = Instant::now() + Duration::from_secs(10);
17113 while module.terminal_history().entries.is_empty() {
17114 assert!(Instant::now() < deadline, "module never exited");
17115 sleep(Duration::from_millis(10)).await;
17116 }
17117 sleep(Duration::from_millis(500)).await;
17119 let status = module.status().unwrap();
17120 assert_eq!(status.state, ModuleState::Stopped);
17121 assert_eq!(status.spawn_generation, 1);
17122 let history = module.terminal_history();
17123 assert_eq!(history.entries.len(), 1, "{history:?}");
17124 assert_eq!(history.entries[0].exit_code, Some(0));
17125 assert_eq!(history.entries[0].disposition, TerminalDisposition::Stopped);
17126 }
17127
17128 #[cfg(unix)]
17129 #[tokio::test]
17130 async fn reload_exit_keeps_roster_until_terminal_is_recorded_during_shutdown() {
17131 let dir = cortexkit_test_support::ScratchDir::new("reload-roster-terminal-order");
17132 let record = dir.join("live-children.json");
17133 let supervisor = Supervisor::new_for_test(
17134 Arc::new(Registry::default()),
17135 RestartPolicy::new(0, Duration::ZERO),
17136 );
17137 let mut runtime = supervisor.runtime_config();
17138 runtime.child_roster.record_to(record.clone());
17139 let gate = Arc::new(super::ReloadExitRecordGate::default());
17140 runtime.test_reload_exit_record_gate = Some(Arc::clone(&gate));
17141 let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::starting()));
17142 let spec = ModuleSpec {
17143 module_id: "reload-exit-roster".into(),
17144 program: fake_aft_stub_path(),
17145 args: Vec::new(),
17146 env: vec![("FAKE_AFT_EXIT_CODE".into(), "23".into())],
17147 reserved: false,
17148 reserved_prefixes: Vec::new(),
17149 protocol: ModuleProtocol::Subc,
17150 overlap: Default::default(),
17151 carry_containment: false,
17152 };
17153 let mut child = None;
17154 let reload = super::finish_reload_child(
17155 &spec,
17156 &runtime,
17157 &supervisor.registry,
17158 &supervisor.process_liveness,
17159 &snapshot,
17160 &mut child,
17161 );
17162 tokio::pin!(reload);
17163 tokio::select! {
17164 result = &mut reload => panic!("reload missed the before-record gate: {result:?}"),
17165 _ = gate.reached.notified() => {}
17166 }
17167 assert!(runtime
17168 .terminal_ring
17169 .lock()
17170 .unwrap()
17171 .snapshot()
17172 .entries
17173 .is_empty());
17174 assert_eq!(
17175 crate::live_children::read_record(&record).unwrap().len(),
17176 1,
17177 "shutdown must still wait for the reaped child until its terminal record exists"
17178 );
17179 runtime.child_roster.close();
17180 gate.resume.notify_one();
17181 assert!(reload.await.is_err());
17182 assert!(crate::live_children::read_record(&record)
17183 .unwrap()
17184 .is_empty());
17185 let history = runtime.terminal_ring.lock().unwrap().snapshot();
17186 assert_eq!(history.entries.len(), 1);
17187 assert_eq!(
17188 history.entries[0].disposition,
17189 TerminalDisposition::DaemonShutdown
17190 );
17191 }
17192
17193 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
17197 async fn every_restart_increment_path_advances_lifetime_count() {
17198 let supervisor = Supervisor::new_for_test(
17199 Arc::new(Registry::default()),
17200 RestartPolicy::new(1, Duration::ZERO),
17201 );
17202 let runtime = supervisor.runtime_config();
17203 let spec = ModuleSpec {
17204 module_id: "lifetime-increment-path".to_string(),
17205 program: PathBuf::from("/unused/lifetime-increment-path"),
17206 args: Vec::new(),
17207 env: Vec::new(),
17208 reserved: false,
17209 reserved_prefixes: Vec::new(),
17210 protocol: ModuleProtocol::Subc,
17211 overlap: Default::default(),
17212 carry_containment: false,
17213 };
17214
17215 let crash_snapshot = Arc::new(Mutex::new(SupervisorSnapshot::starting()));
17216 assert!(matches!(
17217 on_child_exit(
17218 &spec,
17219 runtime.restart_policy,
17220 &supervisor.registry,
17221 &crash_snapshot,
17222 &runtime.terminal_ring,
17223 &runtime.spawn_events,
17224 &runtime.child_roster,
17225 ExitReport {
17226 kind: ExitKind::Crash,
17227 code: Some(1),
17228 signal: None,
17229 at_ms: 1,
17230 },
17231 true,
17232 )
17233 .await,
17234 NextAction::Restart { schedule: _ }
17235 ));
17236 let (crash_restarts, crash_lifetime) = {
17237 let state = lock_snapshot(&crash_snapshot).unwrap();
17238 (state.crash_restarts.len(), state.lifetime_restarts)
17239 };
17240 assert_eq!(crash_restarts, 1);
17241 assert_eq!(crash_lifetime, 1);
17242
17243 let health_snapshot = Arc::new(Mutex::new(SupervisorSnapshot::starting()));
17244 let mut health_child = None;
17245 assert!(matches!(
17246 health_restart_child(
17247 &spec,
17248 &runtime,
17249 &supervisor.registry,
17250 &supervisor.process_liveness,
17251 &health_snapshot,
17252 &mut health_child,
17253 SupervisorHealthStatus::Failing,
17254 None,
17255 2,
17256 )
17257 .await,
17258 Ok(())
17259 ));
17260 assert!(health_child.is_none());
17261 assert!(lock_snapshot(&health_snapshot).unwrap().respawn_pending);
17262 assert!(runtime.scheduled_respawn.lock().unwrap().take().is_some());
17263 let (health_restarts, health_lifetime) = {
17264 let state = lock_snapshot(&health_snapshot).unwrap();
17265 (state.crash_restarts.len(), state.lifetime_restarts)
17266 };
17267 assert_eq!(health_restarts, 1);
17268 assert_eq!(health_lifetime, 1);
17269
17270 let reload_snapshot = Arc::new(Mutex::new(SupervisorSnapshot::starting()));
17271 let mut reload_child = None;
17272 assert!(matches!(
17273 handle_reload_spawn_failure(
17274 &spec,
17275 &runtime,
17276 &supervisor.process_liveness,
17277 &reload_snapshot,
17278 &mut reload_child,
17279 "forced reload spawn failure".to_string(),
17280 )
17281 .await,
17282 Err(SuperviseError::ReloadFailed { .. })
17283 ));
17284 let (reload_restarts, reload_lifetime) = {
17285 let state = lock_snapshot(&reload_snapshot).unwrap();
17286 (state.crash_restarts.len(), state.lifetime_restarts)
17287 };
17288 assert_eq!(reload_restarts, 1);
17289 assert_eq!(reload_lifetime, 1);
17290 }
17291
17292 #[tokio::test]
17293 async fn deliberately_severed_live_child_records_lifetime_without_spending_restart_budget() {
17294 let supervisor = Supervisor::new_for_test(
17295 Arc::new(Registry::default()),
17296 RestartPolicy::new(3, Duration::ZERO),
17297 );
17298 let runtime = supervisor.runtime_config();
17299 let spec = ModuleSpec {
17300 module_id: "deliberately-severed".to_string(),
17301 program: PathBuf::from("/unused/deliberately-severed"),
17302 args: Vec::new(),
17303 env: Vec::new(),
17304 reserved: false,
17305 reserved_prefixes: Vec::new(),
17306 protocol: ModuleProtocol::Subc,
17307 overlap: Default::default(),
17308 carry_containment: false,
17309 };
17310 let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::starting()));
17311 let process = ProcessIdentity {
17312 pid: 41,
17313 start_time: 101,
17314 };
17315 record_deliberate_severance(&snapshot, process).unwrap();
17316 let exit_report = apply_deliberate_severance_marker(
17317 &snapshot,
17318 Some(process),
17319 ExitReport {
17320 kind: ExitKind::Crash,
17321 code: Some(1),
17322 signal: None,
17323 at_ms: 1,
17324 },
17325 );
17326 assert_eq!(exit_report.kind, ExitKind::DeliberateSeverance);
17327
17328 assert!(matches!(
17329 on_child_exit(
17330 &spec,
17331 runtime.restart_policy,
17332 &supervisor.registry,
17333 &snapshot,
17334 &runtime.terminal_ring,
17335 &runtime.spawn_events,
17336 &runtime.child_roster,
17337 exit_report,
17338 true,
17339 )
17340 .await,
17341 NextAction::Restart { schedule: _ }
17342 ));
17343 let state = lock_snapshot(&snapshot).unwrap();
17344 assert_eq!(state.lifetime_restarts, 1);
17345 assert_eq!(state.crash_restarts.len(), 0);
17346 }
17347
17348 #[tokio::test]
17349 async fn genuine_crash_spends_restart_budget_and_records_lifetime() {
17350 let supervisor = Supervisor::new_for_test(
17351 Arc::new(Registry::default()),
17352 RestartPolicy::new(3, Duration::ZERO),
17353 );
17354 let runtime = supervisor.runtime_config();
17355 let spec = ModuleSpec {
17356 module_id: "genuine-crash".to_string(),
17357 program: PathBuf::from("/unused/genuine-crash"),
17358 args: Vec::new(),
17359 env: Vec::new(),
17360 reserved: false,
17361 reserved_prefixes: Vec::new(),
17362 protocol: ModuleProtocol::Subc,
17363 overlap: Default::default(),
17364 carry_containment: false,
17365 };
17366 let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::starting()));
17367
17368 assert!(matches!(
17369 on_child_exit(
17370 &spec,
17371 runtime.restart_policy,
17372 &supervisor.registry,
17373 &snapshot,
17374 &runtime.terminal_ring,
17375 &runtime.spawn_events,
17376 &runtime.child_roster,
17377 ExitReport {
17378 kind: ExitKind::Crash,
17379 code: Some(1),
17380 signal: None,
17381 at_ms: 1,
17382 },
17383 true,
17384 )
17385 .await,
17386 NextAction::Restart { schedule: _ }
17387 ));
17388 let state = lock_snapshot(&snapshot).unwrap();
17389 assert_eq!(state.lifetime_restarts, 1);
17390 assert_eq!(state.crash_restarts.len(), 1);
17391 }
17392
17393 fn crash_exit_report(at_ms: u64) -> ExitReport {
17394 ExitReport {
17395 kind: ExitKind::Crash,
17396 code: Some(1),
17397 signal: None,
17398 at_ms,
17399 }
17400 }
17401
17402 fn windowed_crash_spec(module_id: &str) -> ModuleSpec {
17403 ModuleSpec {
17404 module_id: module_id.to_string(),
17405 program: PathBuf::from("/unused").join(module_id),
17406 args: Vec::new(),
17407 env: Vec::new(),
17408 reserved: false,
17409 reserved_prefixes: Vec::new(),
17410 protocol: ModuleProtocol::Subc,
17411 overlap: Default::default(),
17412 carry_containment: false,
17413 }
17414 }
17415
17416 #[tokio::test]
17422 async fn three_crashes_inside_the_window_stop_the_module_and_name_the_window() {
17423 let (logs, _guard) = crate::router::test_log::log_capture(tracing::Level::ERROR);
17424 let supervisor = Supervisor::new_for_test(
17425 Arc::new(Registry::default()),
17426 RestartPolicy::new(2, Duration::ZERO),
17427 );
17428 let runtime = supervisor.runtime_config();
17429 let spec = windowed_crash_spec("crash-loop-in-window");
17430 let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::starting()));
17431
17432 for attempt in 1..=2 {
17433 assert!(
17434 matches!(
17435 on_child_exit(
17436 &spec,
17437 runtime.restart_policy,
17438 &supervisor.registry,
17439 &snapshot,
17440 &runtime.terminal_ring,
17441 &runtime.spawn_events,
17442 &runtime.child_roster,
17443 crash_exit_report(attempt),
17444 true,
17445 )
17446 .await,
17447 NextAction::Restart { schedule: _ }
17448 ),
17449 "crash {attempt} is inside the budget and must respawn"
17450 );
17451 }
17452
17453 assert!(matches!(
17454 on_child_exit(
17455 &spec,
17456 runtime.restart_policy,
17457 &supervisor.registry,
17458 &snapshot,
17459 &runtime.terminal_ring,
17460 &runtime.spawn_events,
17461 &runtime.child_roster,
17462 crash_exit_report(3),
17463 true,
17464 )
17465 .await,
17466 NextAction::Stop { .. }
17467 ));
17468
17469 {
17470 let state = lock_snapshot(&snapshot).unwrap();
17471 assert_eq!(state.state, ModuleState::Failed);
17472 assert_eq!(state.crash_restarts.len(), 2);
17473 assert_eq!(state.lifetime_restarts, 2);
17474 }
17475
17476 let history = runtime
17477 .terminal_ring
17478 .lock()
17479 .expect("terminal ring is not poisoned")
17480 .snapshot();
17481 let last = history
17482 .entries
17483 .last()
17484 .expect("the refused crash is retained");
17485 assert_eq!(last.disposition, TerminalDisposition::Failed);
17486 assert_eq!(
17487 last.disposition_detail.as_deref(),
17488 Some("crash budget exhausted: max_restarts=2 within window_secs=600")
17489 );
17490
17491 let captured = crate::router::test_log::captured_logs(&logs);
17492 assert!(
17493 captured.contains("crash budget exhausted: max_restarts=2 within window_secs=600"),
17494 "the stop must be logged with its window: {captured}"
17495 );
17496 }
17497
17498 #[tokio::test]
17506 async fn a_crash_older_than_the_window_frees_its_slot_for_a_later_crash() {
17507 let supervisor = Supervisor::new_for_test(
17508 Arc::new(Registry::default()),
17509 RestartPolicy::new(2, Duration::ZERO),
17510 );
17511 let runtime = supervisor.runtime_config();
17512 let spec = windowed_crash_spec("crash-across-windows");
17513 let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::starting()));
17514
17515 for attempt in 1..=2 {
17516 assert!(matches!(
17517 on_child_exit(
17518 &spec,
17519 runtime.restart_policy,
17520 &supervisor.registry,
17521 &snapshot,
17522 &runtime.terminal_ring,
17523 &runtime.spawn_events,
17524 &runtime.child_roster,
17525 crash_exit_report(attempt),
17526 true,
17527 )
17528 .await,
17529 NextAction::Restart { schedule: _ }
17530 ));
17531 }
17532
17533 update_snapshot(&snapshot, Some(&spec.module_id), |state| {
17536 age_oldest_crash_restart_out_of_window(state, runtime.restart_policy.window);
17537 })
17538 .unwrap();
17539
17540 assert!(
17541 matches!(
17542 on_child_exit(
17543 &spec,
17544 runtime.restart_policy,
17545 &supervisor.registry,
17546 &snapshot,
17547 &runtime.terminal_ring,
17548 &runtime.spawn_events,
17549 &runtime.child_roster,
17550 crash_exit_report(3),
17551 true,
17552 )
17553 .await,
17554 NextAction::Restart { schedule: _ }
17555 ),
17556 "a crash older than the window must not hold a budget slot"
17557 );
17558
17559 let state = lock_snapshot(&snapshot).unwrap();
17560 assert_eq!(state.state, ModuleState::Restarting);
17561 assert_eq!(
17562 state.crash_restarts.len(),
17563 2,
17564 "the aged instant is dropped and the new one takes its place"
17565 );
17566 assert_eq!(
17567 state.lifetime_restarts, 3,
17568 "the ledger counts every restart, including the ones the window forgot"
17569 );
17570 }
17571
17572 #[tokio::test]
17577 async fn an_operator_restart_clears_the_ring_and_leaves_the_ledger_alone() {
17578 let supervisor = Supervisor::new_for_test(
17579 Arc::new(Registry::default()),
17580 RestartPolicy::new(2, Duration::ZERO),
17581 );
17582 let runtime = supervisor.runtime_config();
17583 let spec = windowed_crash_spec("operator-cleared-budget");
17584 let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::starting()));
17585
17586 for attempt in 1..=2 {
17587 assert!(matches!(
17588 on_child_exit(
17589 &spec,
17590 runtime.restart_policy,
17591 &supervisor.registry,
17592 &snapshot,
17593 &runtime.terminal_ring,
17594 &runtime.spawn_events,
17595 &runtime.child_roster,
17596 crash_exit_report(attempt),
17597 true,
17598 )
17599 .await,
17600 NextAction::Restart { schedule: _ }
17601 ));
17602 }
17603
17604 reset_restart_count(&snapshot, &spec.module_id).unwrap();
17605 {
17606 let state = lock_snapshot(&snapshot).unwrap();
17607 assert!(
17608 state.crash_restarts.is_empty(),
17609 "an operator restart returns the full budget"
17610 );
17611 assert_eq!(
17612 state.lifetime_restarts, 2,
17613 "clearing the budget must not unmake the crashes"
17614 );
17615 }
17616
17617 assert!(
17618 matches!(
17619 on_child_exit(
17620 &spec,
17621 runtime.restart_policy,
17622 &supervisor.registry,
17623 &snapshot,
17624 &runtime.terminal_ring,
17625 &runtime.spawn_events,
17626 &runtime.child_roster,
17627 crash_exit_report(3),
17628 true,
17629 )
17630 .await,
17631 NextAction::Restart { schedule: _ }
17632 ),
17633 "the cleared budget must be spendable again"
17634 );
17635 let state = lock_snapshot(&snapshot).unwrap();
17636 assert_eq!(state.crash_restarts.len(), 1);
17637 assert_eq!(state.lifetime_restarts, 3);
17638 }
17639
17640 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
17641 async fn severance_marker_for_a_dead_child_does_not_label_its_successor() {
17642 let severed = ProcessIdentity {
17643 pid: 41,
17644 start_time: 101,
17645 };
17646 let successor = ProcessIdentity {
17647 pid: 41,
17648 start_time: 202,
17649 };
17650 let module = module_with_recovery_snapshot(ModuleState::Running, true, 0);
17651 update_snapshot(&module.inner.snapshot, Some("recovery-snapshot"), |state| {
17652 state.pid = Some(successor.pid);
17653 state.process_start_time = Some(successor.start_time);
17654 })
17655 .unwrap();
17656 assert!(!module.record_deliberate_severance(severed).unwrap());
17657
17658 let exit_report = apply_deliberate_severance_marker(
17659 &module.inner.snapshot,
17660 Some(successor),
17661 ExitReport {
17662 kind: ExitKind::Crash,
17663 code: Some(1),
17664 signal: None,
17665 at_ms: 1,
17666 },
17667 );
17668
17669 assert_eq!(exit_report.kind, ExitKind::Crash);
17670 }
17671
17672 #[tokio::test]
17673 async fn drain_reap_marks_deliberate_severance_and_records_lifetime_without_budget() {
17674 let registry = Registry::default();
17675 let supervisor = Supervisor::new_for_test(
17676 Arc::new(Registry::default()),
17677 RestartPolicy::new(3, Duration::ZERO),
17678 );
17679 let runtime = supervisor.runtime_config();
17680 let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::starting()));
17681 let spec = ModuleSpec {
17682 module_id: "drain-deliberate-severance".to_string(),
17683 program: fake_aft_stub_path(),
17684 args: Vec::new(),
17685 env: vec![("FAKE_AFT_EXIT_CODE".to_string(), "23".to_string())],
17686 reserved: false,
17687 reserved_prefixes: Vec::new(),
17688 protocol: ModuleProtocol::Subc,
17689 overlap: Default::default(),
17690 carry_containment: false,
17691 };
17692 let mut child = spawn_and_mark_running(&spec, &runtime, &snapshot).unwrap();
17693 let process = ProcessIdentity {
17694 pid: 41,
17695 start_time: 101,
17696 };
17697 child.process_identity = Some(process);
17698 update_snapshot(&snapshot, Some(&spec.module_id), |state| {
17699 state.pid = Some(process.pid);
17700 state.process_start_time = Some(process.start_time);
17701 })
17702 .unwrap();
17703 record_deliberate_severance(&snapshot, process).unwrap();
17704
17705 drain_child_to_state(
17706 &spec.module_id,
17707 spec.protocol,
17708 StopNotice::SentOverConnection,
17711 ®istry,
17712 None,
17713 &snapshot,
17714 &runtime.terminal_ring,
17715 &runtime.spawn_events,
17716 child,
17717 Duration::from_secs(1),
17718 ModuleState::Stopped,
17719 Some(false),
17720 &super::FailureContext::operator(),
17721 )
17722 .await
17723 .unwrap();
17724
17725 let state = lock_snapshot(&snapshot).unwrap();
17726 assert_eq!(
17727 state.last_exit.as_ref().map(|exit| exit.kind),
17728 Some(ExitKind::DeliberateSeverance)
17729 );
17730 assert_eq!(state.lifetime_restarts, 1);
17731 assert_eq!(state.crash_restarts.len(), 0);
17732 drop(state);
17733 let history = runtime.terminal_ring.lock().unwrap().snapshot();
17734 assert_eq!(
17735 history.entries[0].exit_kind,
17736 subc_control::TerminalExitKind::DeliberateSeverance
17737 );
17738 }
17739
17740 #[tokio::test]
17741 async fn ordinary_drain_reap_does_not_record_a_lifetime_restart() {
17742 let registry = Registry::default();
17743 let supervisor = Supervisor::new_for_test(
17744 Arc::new(Registry::default()),
17745 RestartPolicy::new(3, Duration::ZERO),
17746 );
17747 let runtime = supervisor.runtime_config();
17748 let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::starting()));
17749 let spec = ModuleSpec {
17750 module_id: "ordinary-drain".to_string(),
17751 program: fake_aft_stub_path(),
17752 args: Vec::new(),
17753 env: vec![("FAKE_AFT_EXIT_CODE".to_string(), "23".to_string())],
17754 reserved: false,
17755 reserved_prefixes: Vec::new(),
17756 protocol: ModuleProtocol::Subc,
17757 overlap: Default::default(),
17758 carry_containment: false,
17759 };
17760 let child = spawn_and_mark_running(&spec, &runtime, &snapshot).unwrap();
17761
17762 drain_child_to_state(
17763 &spec.module_id,
17764 spec.protocol,
17765 StopNotice::SentOverConnection,
17768 ®istry,
17769 None,
17770 &snapshot,
17771 &runtime.terminal_ring,
17772 &runtime.spawn_events,
17773 child,
17774 Duration::from_secs(1),
17775 ModuleState::Stopped,
17776 Some(false),
17777 &super::FailureContext::operator(),
17778 )
17779 .await
17780 .unwrap();
17781
17782 let state = lock_snapshot(&snapshot).unwrap();
17783 assert_eq!(
17784 state.last_exit.as_ref().map(|exit| exit.kind),
17785 Some(ExitKind::Crash)
17786 );
17787 assert_eq!(state.lifetime_restarts, 0);
17788 assert_eq!(state.crash_restarts.len(), 0);
17789 }
17790
17791 #[test]
17792 fn fatal_connection_teardown_cannot_arm_a_marker_for_a_surviving_process() {
17793 assert!(!include_str!("server.rs")
17799 .contains("router.record_deliberate_connection_severance(ctx.connection_id)"));
17800 }
17801
17802 #[test]
17809 fn drained_after_quiescence_wait_passes_ok_through_and_forces_false_on_err() {
17810 assert!(drained_after_quiescence_wait(&Ok(true)));
17811 assert!(!drained_after_quiescence_wait(&Ok(false)));
17812 assert!(!drained_after_quiescence_wait(&Err(
17813 SuperviseError::StatePoisoned { module_id: None }
17814 )));
17815 }
17816
17817 #[test]
17826 fn wait_error_exit_report_records_a_failed_terminal_with_no_code_or_signal() {
17827 let ring = Arc::new(Mutex::new(TerminalRing::new(
17828 TerminalRingConfig::default(),
17829 0,
17830 )));
17831 record_wait_error_terminal("wait-error", &ring, &super::SpawnEventFeed::default());
17832
17833 let snapshot = ring.lock().unwrap().snapshot();
17834 assert_eq!(snapshot.entries.len(), 1);
17835 let entry = &snapshot.entries[0];
17836 assert_eq!(entry.exit_code, None);
17837 assert_eq!(entry.exit_signal, None);
17838 assert_eq!(entry.disposition, TerminalDisposition::Failed);
17839 }
17840
17841 #[test]
17842 fn wait_error_exit_path_preserves_spawn_event_density() {
17843 let feed = super::SpawnEventFeed::default();
17844 feed.configure_incarnation("wait-error-density".to_string());
17845 feed.emit_spawned("wait-error", 41, 1);
17846 let ring = Arc::new(Mutex::new(TerminalRing::new(
17847 TerminalRingConfig::default(),
17848 0,
17849 )));
17850
17851 record_wait_error_terminal("wait-error", &ring, &feed);
17852 feed.emit_spawned("after-wait-error", 42, 2);
17853
17854 let state = feed.0.lock().unwrap();
17855 let sequences = state
17856 .events
17857 .iter()
17858 .map(|event| event.cursor.seq)
17859 .collect::<Vec<_>>();
17860 assert_eq!(sequences, vec![1, 2, 3]);
17861 assert_eq!(state.events[1].kind, SpawnEventKind::Exited);
17862 assert_eq!(state.events[1].exit_code, None);
17863 assert_eq!(state.events[1].exit_signal, None);
17864 }
17865
17866 #[test]
17870 fn wait_error_exit_report_is_classified_as_a_crash() {
17871 assert_eq!(wait_error_exit_report().kind, ExitKind::Crash);
17872 }
17873}
17874
17875#[cfg(all(test, unix))]
17876mod restart_ordering_tests {
17877 use super::*;
17878 use std::io::Write;
17879 use tokio::io::AsyncReadExt;
17880
17881 #[test]
17882 #[ignore = "child process fixture"]
17883 fn serving_child() {
17884 let Ok(address_file) = std::env::var("RESTART_TEST_ADDRESS_FILE") else {
17885 return;
17886 };
17887 let listener = std::net::TcpListener::bind("127.0.0.1:0").unwrap();
17888 std::fs::write(address_file, listener.local_addr().unwrap().to_string()).unwrap();
17889 for stream in listener.incoming() {
17890 writeln!(stream.unwrap(), "{}", std::process::id()).unwrap();
17891 }
17892 }
17893
17894 async fn restart_before_publication(requests: usize) {
17895 let home = cortexkit_test_support::ScratchDir::new("restart-before-publication");
17896 let address_file = home.path().join("address");
17897 let spec = ModuleSpec {
17898 module_id: "restart-serving-child".into(),
17899 program: std::env::current_exe().unwrap(),
17900 args: vec![
17901 "--exact".into(),
17902 "supervise::restart_ordering_tests::serving_child".into(),
17903 "--ignored".into(),
17904 "--nocapture".into(),
17905 ],
17906 env: vec![(
17907 "RESTART_TEST_ADDRESS_FILE".into(),
17908 address_file.display().to_string(),
17909 )],
17910 reserved: false,
17911 reserved_prefixes: vec![],
17912 protocol: ModuleProtocol::None,
17913 overlap: Default::default(),
17914 carry_containment: false,
17915 };
17916 let mut supervisor =
17917 Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default());
17918 let (spawned_tx, spawned_rx) = oneshot::channel();
17919 let spawned_tx = Mutex::new(Some(spawned_tx));
17920 let (release_tx, release_rx) = std::sync::mpsc::channel();
17921 let release_rx = Mutex::new(release_rx);
17922 supervisor.test_after_first_spawn = AfterFirstSpawnHook(Some(Arc::new(move |_| {
17925 spawned_tx.lock().unwrap().take().unwrap().send(()).unwrap();
17926 release_rx
17927 .lock()
17928 .unwrap()
17929 .recv_timeout(Duration::from_secs(20))
17930 .unwrap();
17931 })));
17932 let registered = supervisor
17933 .register_configured_with_health(
17934 spec.clone(),
17935 true,
17936 HealthConfig::default(),
17937 None,
17938 RestartPolicy::new(3, Duration::ZERO),
17939 DEFAULT_REGISTER_TIMEOUT,
17940 )
17941 .unwrap();
17942 let module = registered.module.clone();
17943 let startup = tokio::task::spawn_blocking(move || {
17944 supervisor.start_registered_configured(registered, None, true)
17945 });
17946 spawned_rx.await.unwrap();
17947 let old_pid = tokio::time::timeout(Duration::from_secs(10), async {
17948 let address = loop {
17949 if let Ok(address) = std::fs::read_to_string(&address_file) {
17950 if !address.is_empty() {
17951 break address;
17952 }
17953 }
17954 tokio::time::sleep(Duration::from_millis(1)).await;
17955 };
17956 let mut stream = tokio::net::TcpStream::connect(address).await.unwrap();
17957 let mut reply = String::new();
17958 stream.read_to_string(&mut reply).await.unwrap();
17959 reply.trim().parse::<u32>().unwrap()
17960 })
17961 .await
17962 .expect("child must serve before generation publication");
17963 assert_eq!(
17964 lock_snapshot(&module.inner.snapshot)
17965 .unwrap()
17966 .spawn_generation,
17967 0
17968 );
17969 let mut replies = Vec::new();
17970 for _ in 0..requests {
17971 let mut restart = Box::pin(module.restart(Some(0)));
17972 std::future::poll_fn(|cx| {
17975 assert!(std::future::Future::poll(restart.as_mut(), cx).is_pending());
17976 std::task::Poll::Ready(())
17977 })
17978 .await;
17979 replies.push(restart);
17980 }
17981 release_tx.send(()).unwrap();
17982 startup.await.unwrap().unwrap();
17983 for reply in replies {
17984 reply.await.unwrap();
17985 }
17986 let replacement = tokio::time::timeout(Duration::from_secs(10), async {
17989 loop {
17990 if lock_snapshot(&module.inner.snapshot)
17991 .unwrap()
17992 .spawn_generation
17993 > 1
17994 {
17995 break;
17996 }
17997 tokio::time::sleep(Duration::from_millis(1)).await;
17998 }
17999 })
18000 .await;
18001 module.update_spec_for_test(spec).await.unwrap();
18005 let (generation, pid) = {
18006 let state = lock_snapshot(&module.inner.snapshot).unwrap();
18007 (state.spawn_generation, state.pid)
18008 };
18009 module.stop().await.unwrap();
18010 replacement.expect("restart must publish a replacement for the already-serving child");
18011 assert_ne!(
18012 pid,
18013 Some(old_pid),
18014 "a serving child predating the restart must be replaced"
18015 );
18016 assert_eq!(generation, 2, "queued restarts must share one replacement");
18017 }
18018
18019 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
18020 async fn restart_received_after_serving_before_generation_publication_replaces_child() {
18021 restart_before_publication(1).await;
18022 }
18023
18024 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
18025 async fn restart_queued_behind_restart_does_not_kill_its_replacement() {
18026 restart_before_publication(2).await;
18027 }
18028
18029 #[tokio::test]
18030 async fn restart_queued_before_replacement_spawn_keeps_the_new_child() {
18031 let supervisor =
18032 Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default());
18033 let registered = supervisor
18034 .register_configured_with_health(
18035 ModuleSpec {
18036 module_id: "queued-before-replacement".into(),
18037 program: "/bin/sleep".into(),
18042 args: vec!["600".into()],
18043 env: vec![],
18044 reserved: false,
18045 reserved_prefixes: vec![],
18046 protocol: ModuleProtocol::None,
18047 overlap: Default::default(),
18048 carry_containment: false,
18049 },
18050 true,
18051 HealthConfig::default(),
18052 None,
18053 RestartPolicy::default(),
18054 DEFAULT_REGISTER_TIMEOUT,
18055 )
18056 .unwrap();
18057 let RegisteredConfiguredModule {
18058 mut spec,
18059 mut runtime,
18060 module,
18061 mut commands,
18062 } = registered;
18063 let snapshot = &module.inner.snapshot;
18064 let mut restart = Box::pin(module.restart(Some(0)));
18065 std::future::poll_fn(|cx| {
18066 assert!(std::future::Future::poll(restart.as_mut(), cx).is_pending());
18067 std::task::Poll::Ready(())
18068 })
18069 .await;
18070 let mut child = Some(spawn_and_mark_running(&spec, &runtime, snapshot).unwrap());
18073 let replacement_pid = child.as_ref().unwrap().id();
18074 let command = commands.recv().await.unwrap();
18075 assert!(
18076 handle_supervisor_command(
18077 command,
18078 &mut spec,
18079 &mut runtime,
18080 &supervisor.registry,
18081 &supervisor.process_liveness,
18082 snapshot,
18083 &mut child,
18084 &mut commands,
18085 &mut VecDeque::new(),
18086 )
18087 .await
18088 );
18089 restart.await.unwrap();
18090 let kept_pid = child.as_ref().and_then(SupervisedChild::id);
18091 let pending = lock_snapshot(snapshot).unwrap().respawn_pending;
18092 if let Some(mut child) = child {
18093 child.start_kill().unwrap();
18094 child.wait().await.unwrap();
18095 }
18096 assert_eq!(
18097 kept_pid, replacement_pid,
18098 "a newer replacement must survive"
18099 );
18100 assert!(
18101 !pending,
18102 "a satisfied restart must not schedule another spawn"
18103 );
18104 }
18105}
18106
18107#[cfg(all(test, unix))]
18108mod health_event_tests {
18109 use super::*;
18110 use cortexkit_test_support::ScratchDir;
18111 use subc_protocol::{manifest::Concurrency, session::ModuleControlResponse};
18112
18113 struct Actor {
18114 module: SupervisedModule,
18115 registry: Arc<Registry>,
18116 forwarding: Arc<ForwardingTable>,
18117 spec: ModuleSpec,
18118 health: HealthConfig,
18119 rx: mpsc::Receiver<crate::router::OutboundFrame>,
18120 roster: ChildRoster,
18121 _home: ScratchDir,
18122 }
18123
18124 async fn wait_for<T>(reason: &str, mut observe: impl FnMut() -> Option<T>) -> T {
18125 let deadline = std::time::Instant::now() + Duration::from_secs(10);
18129 let tick = Instant::now();
18130 loop {
18131 if let Some(value) = observe() {
18132 assert_eq!(Instant::now(), tick, "{reason} must not wait for a timer");
18133 return value;
18134 }
18135 assert!(
18136 std::time::Instant::now() < deadline,
18137 "timed out waiting for {reason}"
18138 );
18139 tokio::task::yield_now().await;
18140 }
18141 }
18142
18143 impl Actor {
18144 async fn start(protocol: ModuleProtocol, cadence: Duration) -> Self {
18145 Self::start_with_policy(
18146 protocol,
18147 cadence,
18148 DEFAULT_REGISTER_TIMEOUT,
18149 RestartPolicy::new(3, Duration::ZERO),
18150 )
18151 .await
18152 }
18153
18154 async fn start_with_policy(
18155 protocol: ModuleProtocol,
18156 cadence: Duration,
18157 register_timeout: Duration,
18158 policy: RestartPolicy,
18159 ) -> Self {
18160 let home = ScratchDir::new("health-events");
18161 let registry = Arc::new(Registry::default());
18162 let forwarding = Arc::new(ForwardingTable::default());
18163 let health = HealthConfig {
18164 cadence,
18165 deadline: Duration::from_secs(3600),
18166 ..HealthConfig::default()
18167 };
18168 let supervisor = Supervisor::new_for_test(registry.clone(), policy)
18169 .with_forwarding(forwarding.clone())
18170 .with_health_config(health.clone());
18171 let spec = ModuleSpec {
18172 module_id: "event-health-child".into(),
18173 program: PathBuf::from("/bin/sleep"),
18181 args: vec!["600".into()],
18184 env: ["XDG_DATA_HOME", "XDG_RUNTIME_DIR", "XDG_CONFIG_HOME"]
18185 .into_iter()
18186 .map(|key| (key.into(), home.path().display().to_string()))
18187 .collect(),
18188 reserved: false,
18189 reserved_prefixes: vec![],
18190 protocol,
18191 overlap: Default::default(),
18192 carry_containment: false,
18193 };
18194 let module = supervisor
18195 .supervise_configured_requested(
18196 spec.clone(),
18197 true,
18198 health.clone(),
18199 Some(0),
18200 policy,
18201 register_timeout,
18202 None,
18203 )
18204 .unwrap();
18205 let (_, rx) = mpsc::channel(8);
18206 let actor = Self {
18207 module,
18208 registry,
18209 forwarding,
18210 spec,
18211 health,
18212 rx,
18213 roster: supervisor.child_roster.clone(),
18214 _home: home,
18215 };
18216 actor
18217 .wait_for_select(|checkpoint| checkpoint.generation > 0)
18218 .await;
18219 actor
18220 }
18221
18222 async fn expire_registration_wait(&self, remaining: Duration) {
18223 tokio::time::advance(remaining).await;
18224 wait_for("registration timeout TERM grace", || {
18225 lock_snapshot(&self.module.inner.snapshot)
18226 .unwrap()
18227 .actor_select
18228 .is_none()
18229 .then_some(())
18230 })
18231 .await;
18232 tokio::time::advance(crate::child_roster::TERM_TO_KILL).await;
18233 }
18234
18235 fn turns(&self) -> u64 {
18236 lock_snapshot(&self.module.inner.snapshot)
18237 .unwrap()
18238 .actor_turns
18239 }
18240
18241 async fn wait_for_select(
18242 &self,
18243 expected: impl Fn(&ActorSelectCheckpoint) -> bool,
18244 ) -> ActorSelectCheckpoint {
18245 wait_for("actor select checkpoint", || {
18246 let state = lock_snapshot(&self.module.inner.snapshot).unwrap();
18247 state.actor_select.filter(|checkpoint| {
18248 checkpoint.generation == state.spawn_generation
18249 && state.state == ModuleState::Running
18250 && state.process_alive
18251 && state.reported_pid().is_some()
18252 && expected(checkpoint)
18253 })
18254 })
18255 .await
18256 }
18257
18258 fn hello(&mut self, connection: u64, health: bool) {
18259 let manifest =
18260 subc_protocol::manifest::ModuleManifest::builder(&self.spec.module_id, "1.0.0")
18261 .protocol_ver(subc_protocol::PROTOCOL_VERSION)
18262 .build();
18263 let connection = ConnectionId::new(connection);
18264 let (tx, rx) = mpsc::channel(8);
18265 self.rx = rx;
18266 self.forwarding
18267 .register_module_connection(
18268 connection,
18269 self.spec.module_id.clone(),
18270 subc_protocol::PROTOCOL_VERSION,
18271 Concurrency::ModuleManaged,
18272 FrameSink::new(tx),
18273 )
18274 .unwrap();
18275 self.registry
18276 .register_with_control_ops(
18277 manifest,
18278 subc_protocol::PROTOCOL_VERSION,
18279 connection,
18280 if health {
18281 vec![MODULE_CONTROL_OP_HEALTH_CHECK.into()]
18282 } else {
18283 vec![]
18284 },
18285 )
18286 .unwrap();
18287 }
18288
18289 async fn probe(&mut self) -> crate::router::OutboundFrame {
18290 let frame = wait_for("outbound health probe", || match self.rx.try_recv() {
18291 Ok(frame) => Some(frame),
18292 Err(mpsc::error::TryRecvError::Empty) => None,
18293 Err(mpsc::error::TryRecvError::Disconnected) => panic!("health peer disconnected"),
18294 })
18295 .await;
18296 let body: Value = serde_json::from_slice(&frame.body).unwrap();
18297 assert_eq!(body["op"], MODULE_CONTROL_OP_HEALTH_CHECK);
18298 frame
18299 }
18300
18301 fn no_probe(&mut self) {
18302 assert!(matches!(
18303 self.rx.try_recv(),
18304 Err(mpsc::error::TryRecvError::Empty)
18305 ));
18306 }
18307
18308 fn answer(&self, connection: u64, frame: crate::router::OutboundFrame) {
18309 self.forwarding
18310 .complete_module_control_rpc(
18311 ConnectionId::new(connection),
18312 frame.header.corr,
18313 Some(MODULE_CONTROL_OP_HEALTH_CHECK),
18314 ModuleControlRpcOutcome::Response(ModuleControlResponse::HealthCheck {
18315 status: HealthStatus::Ok,
18316 detail: None,
18317 metrics: None,
18318 }),
18319 )
18320 .unwrap();
18321 }
18322 }
18323
18324 #[tokio::test(start_paused = true)]
18325 async fn registration_timeout_kills_at_120_seconds_and_respawns() {
18326 let actor = Actor::start(ModuleProtocol::Subc, Duration::from_secs(30)).await;
18327 let initial = actor.module.status().unwrap();
18328 tokio::time::advance(Duration::from_secs(119)).await;
18329 tokio::task::yield_now().await;
18330 assert_eq!(actor.module.status().unwrap().pid, initial.pid);
18331 assert!(actor.module.terminal_history().entries.is_empty());
18332 actor.expire_registration_wait(Duration::from_secs(1)).await;
18333 actor
18334 .wait_for_select(|at| at.generation > initial.spawn_generation)
18335 .await;
18336 let status = actor.module.status().unwrap();
18337 assert_ne!(status.pid, initial.pid);
18338 assert!(!cortexkit_test_support::process_alive(
18339 initial.pid.unwrap() as i32
18340 ));
18341 assert_eq!(status.restart_count, 1);
18342 assert_eq!(status.last_exit.unwrap().kind, ExitKind::Crash);
18343 let entries = actor.module.terminal_history().entries;
18344 assert_eq!(entries.len(), 1);
18345 assert_eq!(
18346 entries[0].disposition_detail.as_deref(),
18347 Some("registration_timeout")
18348 );
18349 actor.module.stop().await.unwrap();
18350 }
18351
18352 #[tokio::test(start_paused = true)]
18353 async fn registration_timeout_is_cancelled_by_hello_at_119_seconds() {
18354 let mut actor = Actor::start(ModuleProtocol::Subc, Duration::from_secs(30)).await;
18355 let pid = actor.module.status().unwrap().pid;
18356 tokio::time::advance(Duration::from_secs(119)).await;
18357 actor.hello(1, false);
18358 let registered = actor
18359 .wait_for_select(|at| at.registered_connection.is_some())
18360 .await;
18361 assert_eq!(
18362 registered.wake_after, None,
18363 "HELLO must cancel the deadline"
18364 );
18365 tokio::time::advance(Duration::from_secs(300)).await;
18366 tokio::task::yield_now().await;
18367 assert_eq!(actor.module.status().unwrap().pid, pid);
18368 assert!(actor.module.terminal_history().entries.is_empty());
18369 actor
18371 .registry
18372 .deregister_connection(ConnectionId::new(1))
18373 .unwrap();
18374 actor
18375 .wait_for_select(|at| at.registered_connection.is_none())
18376 .await;
18377 tokio::time::advance(Duration::from_secs(300)).await;
18378 tokio::task::yield_now().await;
18379 assert_eq!(actor.module.status().unwrap().pid, pid);
18380 assert!(actor.module.terminal_history().entries.is_empty());
18381 actor.module.stop().await.unwrap();
18382
18383 let mut transient = Actor::start(ModuleProtocol::Subc, Duration::from_secs(30)).await;
18384 let pid = transient.module.status().unwrap().pid;
18385 tokio::time::advance(Duration::from_secs(119)).await;
18386 let turns = transient.turns();
18387 transient.hello(1, false);
18390 transient
18391 .registry
18392 .deregister_connection(ConnectionId::new(1))
18393 .unwrap();
18394 let parked = transient.wait_for_select(|at| at.turn > turns).await;
18395 assert_eq!(
18396 parked.wake_after, None,
18397 "transient HELLO must cancel the deadline"
18398 );
18399 tokio::time::advance(Duration::from_secs(300)).await;
18400 tokio::task::yield_now().await;
18401 assert_eq!(transient.module.status().unwrap().pid, pid);
18402 assert!(transient.module.terminal_history().entries.is_empty());
18403 transient.module.stop().await.unwrap();
18404 }
18405
18406 #[tokio::test(start_paused = true)]
18407 async fn registration_timeouts_exhaust_budget_and_schedule_slow_retry() {
18408 let actor = Actor::start_with_policy(
18409 ModuleProtocol::Subc,
18410 Duration::from_secs(30),
18411 Duration::from_secs(10),
18412 RestartPolicy::new(1, Duration::ZERO),
18413 )
18414 .await;
18415 let generation = actor.module.status().unwrap().spawn_generation;
18416 actor
18417 .expire_registration_wait(Duration::from_secs(10))
18418 .await;
18419 actor.wait_for_select(|at| at.generation > generation).await;
18420 actor
18421 .expire_registration_wait(Duration::from_secs(10))
18422 .await;
18423 let retry = wait_for("failed retry schedule", || {
18424 let state = lock_snapshot(&actor.module.inner.snapshot).unwrap();
18425 (state.state == ModuleState::Failed)
18426 .then_some(state.failed_retry.pending)
18427 .flatten()
18428 })
18429 .await;
18430 assert_eq!(retry.delay, Duration::from_secs(600));
18431 assert_eq!(retry.attempt, 1);
18432 let entries = actor.module.terminal_history().entries;
18433 assert_eq!(entries.len(), 2);
18434 assert_eq!(entries[1].disposition, TerminalDisposition::Failed);
18435 assert!(entries[1]
18436 .disposition_detail
18437 .as_deref()
18438 .unwrap()
18439 .contains("registration_timeout"));
18440 tokio::time::advance(Duration::from_secs(599)).await;
18441 tokio::task::yield_now().await;
18442 assert_eq!(actor.module.state().unwrap(), ModuleState::Failed);
18443 tokio::time::advance(Duration::from_secs(1)).await;
18444 actor
18445 .wait_for_select(|at| at.generation > generation + 1)
18446 .await;
18447 actor
18449 .expire_registration_wait(Duration::from_secs(10))
18450 .await;
18451 wait_for("slow retry registration timeout", || {
18452 (actor.module.terminal_history().entries.len() == 3).then_some(())
18453 })
18454 .await;
18455 actor.module.stop().await.unwrap();
18456 }
18457
18458 #[tokio::test(start_paused = true)]
18459 async fn registration_timeout_honours_ten_second_override() {
18460 let actor = Actor::start_with_policy(
18461 ModuleProtocol::Subc,
18462 Duration::from_secs(30),
18463 Duration::from_secs(10),
18464 RestartPolicy::new(3, Duration::ZERO),
18465 )
18466 .await;
18467 let generation = actor.module.status().unwrap().spawn_generation;
18468 tokio::time::advance(Duration::from_secs(9)).await;
18469 tokio::task::yield_now().await;
18470 assert!(actor.module.terminal_history().entries.is_empty());
18471 actor.expire_registration_wait(Duration::from_secs(1)).await;
18472 actor.wait_for_select(|at| at.generation > generation).await;
18473 assert_eq!(actor.module.terminal_history().entries.len(), 1);
18474 actor.module.stop().await.unwrap();
18475 }
18476
18477 #[tokio::test]
18478 async fn registration_timeout_kills_term_ignoring_helper_before_reaping_leader() {
18479 use std::os::unix::process::{CommandExt, ExitStatusExt};
18480
18481 struct Helper(std::process::Child);
18482 impl Drop for Helper {
18483 fn drop(&mut self) {
18484 if matches!(self.0.try_wait(), Ok(None)) {
18487 let _ = self.0.kill();
18488 let _ = self.0.wait();
18489 }
18490 }
18491 }
18492
18493 let home = ScratchDir::new("registration-timeout-helper");
18494 let supervisor = Supervisor::new_for_test(
18495 Arc::new(Registry::default()),
18496 RestartPolicy::new(0, Duration::ZERO),
18497 );
18498 let spec = ModuleSpec {
18499 module_id: "registration-timeout-group".into(),
18500 program: "/bin/sleep".into(),
18501 args: vec!["120".into()],
18502 env: ["XDG_DATA_HOME", "XDG_RUNTIME_DIR", "XDG_CONFIG_HOME"]
18503 .into_iter()
18504 .map(|key| (key.into(), home.path().display().to_string()))
18505 .collect(),
18506 reserved: false,
18507 reserved_prefixes: vec![],
18508 protocol: ModuleProtocol::Subc,
18509 overlap: Default::default(),
18510 carry_containment: false,
18511 };
18512 let module = supervisor
18513 .supervise_configured_requested(
18514 spec,
18515 true,
18516 HealthConfig::default(),
18517 None,
18518 RestartPolicy::new(0, Duration::ZERO),
18519 Duration::from_secs(10),
18520 None,
18521 )
18522 .unwrap();
18523 let leader = timeout(Duration::from_secs(5), async {
18524 loop {
18525 if let Some(pid) = module.status().unwrap().pid {
18526 break pid;
18527 }
18528 sleep(Duration::from_millis(5)).await;
18529 }
18530 })
18531 .await
18532 .expect("supervised group leader must be reported before the deadline");
18533 assert!(leader > 1);
18534 let ready = home.join("helper.ready");
18535 let mut helper = Helper(
18538 std::process::Command::new("/bin/sh")
18539 .args([
18540 "-c",
18541 "trap '' TERM; : > \"$1\"; exec /bin/sleep 120",
18542 "helper",
18543 ])
18544 .arg(&ready)
18545 .process_group(i32::try_from(leader).unwrap())
18546 .stdout(std::process::Stdio::null())
18547 .stderr(std::process::Stdio::null())
18548 .spawn()
18549 .unwrap(),
18550 );
18551 timeout(Duration::from_secs(5), async {
18552 while !ready.exists() {
18553 assert!(helper.0.try_wait().unwrap().is_none());
18554 sleep(Duration::from_millis(5)).await;
18555 }
18556 })
18557 .await
18558 .expect("helper must install TERM ignore before the registration deadline");
18559 let helper_pid =
18560 rustix::process::Pid::from_raw(i32::try_from(helper.0.id()).unwrap()).unwrap();
18561 rustix::process::kill_process(helper_pid, rustix::process::Signal::TERM).unwrap();
18562 sleep(Duration::from_millis(20)).await;
18563 assert!(
18564 helper.0.try_wait().unwrap().is_none(),
18565 "helper must ignore TERM"
18566 );
18567
18568 timeout(Duration::from_secs(20), async {
18569 while module.state().unwrap() != ModuleState::Failed {
18570 sleep(Duration::from_millis(5)).await;
18571 }
18572 })
18573 .await
18574 .expect("registration timeout must stop and reap the group leader");
18575 let helper_exit = timeout(Duration::from_secs(3), async {
18576 loop {
18577 if let Some(status) = helper.0.try_wait().unwrap() {
18578 break status;
18579 }
18580 sleep(Duration::from_millis(5)).await;
18581 }
18582 })
18583 .await
18584 .expect("TERM-ignoring helper survived the registration timeout's group KILL");
18585 assert_eq!(helper_exit.signal(), Some(9));
18586 assert!(!cortexkit_test_support::process_alive(
18587 i32::try_from(leader).unwrap()
18588 ));
18589 assert_eq!(
18590 module.status().unwrap().last_exit.unwrap().kind,
18591 ExitKind::Crash
18592 );
18593 assert!(module.terminal_history().entries[0]
18594 .disposition_detail
18595 .as_deref()
18596 .unwrap()
18597 .contains("registration_timeout"));
18598 module.stop().await.unwrap();
18599 }
18600
18601 #[tokio::test(start_paused = true)]
18602 async fn registration_timeout_does_not_count_an_earlier_exit() {
18603 let actor = Actor::start_with_policy(
18604 ModuleProtocol::Subc,
18605 Duration::from_secs(30),
18606 DEFAULT_REGISTER_TIMEOUT,
18607 RestartPolicy::new(0, Duration::ZERO),
18608 )
18609 .await;
18610 let pid = actor.module.status().unwrap().pid.unwrap();
18611 rustix::process::kill_process(
18612 rustix::process::Pid::from_raw(pid as i32).unwrap(),
18613 rustix::process::Signal::TERM,
18614 )
18615 .unwrap();
18616 wait_for("early exit", || {
18617 (actor.module.state().unwrap() == ModuleState::Failed).then_some(())
18618 })
18619 .await;
18620 tokio::time::advance(Duration::from_secs(121)).await;
18621 tokio::task::yield_now().await;
18622 let entries = actor.module.terminal_history().entries;
18623 assert_eq!(entries.len(), 1);
18624 assert!(!entries[0]
18625 .disposition_detail
18626 .as_deref()
18627 .unwrap()
18628 .contains("registration_timeout"));
18629 assert_eq!(actor.module.status().unwrap().lifetime_restarts, 0);
18630 actor.module.stop().await.unwrap();
18631 }
18632
18633 #[tokio::test(start_paused = true)]
18634 async fn registration_timeout_is_cancelled_by_daemon_shutdown() {
18635 let actor = Actor::start(ModuleProtocol::Subc, Duration::from_secs(30)).await;
18636 let pid = actor.module.status().unwrap().pid;
18637 tokio::time::advance(Duration::from_secs(119)).await;
18638 actor.roster.close();
18639 let turns = actor.turns();
18640 tokio::time::advance(Duration::from_secs(300)).await;
18641 let parked = actor.wait_for_select(|at| at.turn > turns).await;
18642 assert_eq!(parked.wake_after, None);
18643 assert_eq!(actor.module.status().unwrap().pid, pid);
18644 assert!(actor.module.terminal_history().entries.is_empty());
18645 actor.module.stop().await.unwrap();
18646 let status = actor.module.status().unwrap();
18647 assert_eq!(status.restart_count, 0);
18648 assert_eq!(
18649 actor.module.terminal_history().entries[0].disposition,
18650 TerminalDisposition::DaemonShutdown
18651 );
18652 }
18653
18654 #[tokio::test(start_paused = true)]
18655 async fn no_health_children_park_for_sixty_seconds() {
18656 let none = Actor::start(ModuleProtocol::None, Duration::from_secs(30)).await;
18657 let unregistered = Actor::start(ModuleProtocol::Subc, Duration::from_secs(30)).await;
18658 let mut unadvertised = Actor::start(ModuleProtocol::Subc, Duration::from_secs(30)).await;
18659 unadvertised.hello(1, false);
18660 for _ in 0..6000 {
18665 tokio::time::advance(Duration::from_millis(10)).await;
18666 tokio::task::yield_now().await;
18667 }
18668 for actor in [&none, &unregistered, &unadvertised] {
18669 assert!(
18670 actor.turns() <= 5,
18671 "no-health actor woke {} times",
18672 actor.turns()
18673 );
18674 assert_eq!(actor.module.state().unwrap(), ModuleState::Running);
18675 }
18676 }
18677
18678 #[tokio::test(start_paused = true)]
18679 async fn late_hello_starts_a_due_probe_in_the_notification_tick() {
18680 let mut actor = Actor::start(ModuleProtocol::Subc, Duration::ZERO).await;
18684 let tick = Instant::now();
18685 actor.hello(1, true);
18686 actor.probe().await;
18687 assert_eq!(Instant::now(), tick, "HELLO must not wait for a poll");
18688 }
18689
18690 #[tokio::test(start_paused = true)]
18691 async fn hello_keeps_the_jittered_cadence_and_catalog_updates_do_not_reset_it() {
18692 let mut actor = Actor::start(ModuleProtocol::Subc, Duration::from_secs(30)).await;
18693 let hello_tick = Instant::now();
18694 actor.hello(1, true);
18695 let armed = actor
18696 .wait_for_select(|checkpoint| {
18697 checkpoint.registered_connection == Some(ConnectionId::new(1))
18698 && checkpoint.next_probe_at.is_some()
18699 })
18700 .await;
18701 let delay = jittered_health_delay(&actor.spec.module_id, 0, actor.health.cadence);
18702 assert!((Duration::from_secs(30)..Duration::from_secs(33)).contains(&delay));
18703 let first_deadline = hello_tick + delay;
18704 assert_eq!(armed.next_probe_at, Some(first_deadline));
18705 tokio::time::advance(Duration::from_secs(29)).await;
18706 let turns = actor.turns();
18707 actor
18708 .registry
18709 .replace_catalog_for_connection(ConnectionId::new(1), vec![], None, Some(true))
18710 .unwrap();
18711 let updated = actor
18712 .wait_for_select(|checkpoint| checkpoint.turn > turns)
18713 .await;
18714 assert_eq!(
18715 updated.next_probe_at,
18716 Some(first_deadline),
18717 "catalog update must not reset cadence"
18718 );
18719 actor.no_probe();
18720 tokio::time::advance(first_deadline - Instant::now()).await;
18721 let frame = actor.probe().await;
18722 actor.answer(1, frame);
18723 let next = jittered_health_delay(&actor.spec.module_id, 1, actor.health.cadence);
18724 let next_deadline = Instant::now() + next;
18725 let rearmed = actor
18726 .wait_for_select(|checkpoint| {
18727 checkpoint
18728 .next_probe_at
18729 .is_some_and(|deadline| deadline > first_deadline)
18730 })
18731 .await;
18732 assert_eq!(rearmed.next_probe_at, Some(next_deadline));
18733 tokio::time::advance(next - Duration::from_nanos(1)).await;
18734 actor.no_probe();
18735 tokio::time::advance(Duration::from_nanos(1)).await;
18736 actor.probe().await;
18737 }
18738
18739 #[tokio::test(start_paused = true)]
18740 async fn disconnect_disarms_and_reconnect_rearms_health() {
18741 let mut actor = Actor::start(ModuleProtocol::Subc, Duration::from_secs(30)).await;
18742 actor.hello(1, true);
18743 actor
18744 .wait_for_select(|checkpoint| {
18745 checkpoint.registered_connection == Some(ConnectionId::new(1))
18746 && checkpoint.next_probe_at.is_some()
18747 })
18748 .await;
18749 let turns = actor.turns();
18750 let tick = Instant::now();
18751 actor
18752 .registry
18753 .deregister_connection(ConnectionId::new(1))
18754 .unwrap();
18755 let disarmed = actor
18756 .wait_for_select(|checkpoint| {
18757 checkpoint.turn > turns && checkpoint.registered_connection.is_none()
18758 })
18759 .await;
18760 assert_eq!(disarmed.next_probe_at, None);
18761 assert_eq!(disarmed.wake_after, None);
18762 assert_eq!(
18763 Instant::now(),
18764 tick,
18765 "disconnect must disarm in the notification tick"
18766 );
18767 tokio::time::advance(Duration::from_secs(60)).await;
18768 actor.no_probe();
18769 let reconnect_tick = Instant::now();
18770 actor.hello(2, true);
18771 let rearmed = actor
18772 .wait_for_select(|checkpoint| {
18773 checkpoint.registered_connection == Some(ConnectionId::new(2))
18774 && checkpoint.next_probe_at.is_some()
18775 })
18776 .await;
18777 let delay = jittered_health_delay(&actor.spec.module_id, 0, actor.health.cadence);
18778 assert_eq!(rearmed.next_probe_at, Some(reconnect_tick + delay));
18779 tokio::time::advance(delay).await;
18780 actor.probe().await;
18781 }
18782
18783 #[tokio::test(start_paused = true)]
18784 async fn reregistration_without_health_disarms_the_old_schedule() {
18785 let mut actor = Actor::start(ModuleProtocol::Subc, Duration::from_secs(30)).await;
18786 actor.hello(1, true);
18787 actor
18788 .wait_for_select(|checkpoint| {
18789 checkpoint.registered_connection == Some(ConnectionId::new(1))
18790 && checkpoint.next_probe_at.is_some()
18791 })
18792 .await;
18793 actor
18794 .registry
18795 .deregister_connection(ConnectionId::new(1))
18796 .unwrap();
18797 actor.hello(2, false);
18798 let disarmed = actor
18799 .wait_for_select(|checkpoint| {
18800 checkpoint.registered_connection == Some(ConnectionId::new(2))
18801 })
18802 .await;
18803 assert_eq!(disarmed.next_probe_at, None);
18804 assert_eq!(disarmed.wake_after, None);
18805 tokio::time::advance(Duration::from_secs(60)).await;
18806 actor.no_probe();
18807 }
18808
18809 #[tokio::test(start_paused = true)]
18810 async fn restart_waits_for_its_own_hello_before_rearming_health() {
18811 let mut actor = Actor::start(ModuleProtocol::Subc, Duration::from_secs(30)).await;
18812 actor.hello(1, true);
18813 let armed = actor
18814 .wait_for_select(|checkpoint| {
18815 checkpoint.registered_connection == Some(ConnectionId::new(1))
18816 && checkpoint.next_probe_at.is_some()
18817 })
18818 .await;
18819 actor
18823 .registry
18824 .deregister_connection(ConnectionId::new(1))
18825 .unwrap();
18826 actor.module.restart(Some(0)).await.unwrap();
18827 let parked = actor
18828 .wait_for_select(|checkpoint| {
18829 checkpoint.generation > armed.generation
18830 && checkpoint.registered_connection.is_none()
18831 && checkpoint.next_probe_at.is_none()
18832 })
18833 .await;
18834 assert_eq!(parked.wake_after, Some(DEFAULT_REGISTER_TIMEOUT));
18835 actor.rx.close();
18836 while actor.rx.try_recv().is_ok() {}
18839 let turns = actor.turns();
18840 tokio::time::advance(Duration::from_secs(60)).await;
18841 let after = actor.turns();
18842 eprintln!(
18843 "restart park turns: before={turns} after={after} delta={}",
18844 after - turns
18845 );
18846 assert_eq!(after, turns, "replacement must park before HELLO");
18847 let hello_tick = Instant::now();
18848 actor.hello(2, true);
18849 let rearmed = actor
18850 .wait_for_select(|checkpoint| {
18851 checkpoint.registered_connection == Some(ConnectionId::new(2))
18852 && checkpoint.next_probe_at.is_some()
18853 })
18854 .await;
18855 let delay = jittered_health_delay(&actor.spec.module_id, 0, actor.health.cadence);
18856 assert_eq!(rearmed.next_probe_at, Some(hello_tick + delay));
18857 tokio::time::advance(delay).await;
18858 actor.probe().await;
18859 }
18860
18861 #[tokio::test(start_paused = true)]
18862 async fn http_probe_waits_a_full_cadence_for_each_spawn() {
18863 use tokio::{
18864 io::{AsyncReadExt, AsyncWriteExt},
18865 net::TcpListener,
18866 };
18867 let mut actor = Actor::start(ModuleProtocol::None, Duration::from_secs(30)).await;
18868 let listener = TcpListener::bind("127.0.0.1:0").await.unwrap();
18869 actor.health.http = Some(format!("http://{}/healthz", listener.local_addr().unwrap()));
18870 let (started_tx, mut started_rx) = mpsc::channel(8);
18871 let peer = tokio::spawn(async move {
18872 loop {
18873 let (mut socket, _) = listener.accept().await.unwrap();
18874 let mut request = [0; 1024];
18875 assert!(socket.read(&mut request).await.unwrap() > 0);
18876 started_tx.send(Instant::now()).await.unwrap();
18877 socket
18878 .write_all(b"HTTP/1.1 200 OK\r\nContent-Length: 0\r\n\r\n")
18879 .await
18880 .unwrap();
18881 }
18882 });
18883 actor
18884 .module
18885 .update_configuration(actor.spec.clone(), actor.health.clone(), None)
18886 .await
18887 .unwrap();
18888 let first_tick = Instant::now();
18889 let first = actor
18890 .wait_for_select(|checkpoint| checkpoint.next_probe_at.is_some())
18891 .await;
18892 assert!(first.next_probe_at.unwrap() >= first_tick + actor.health.cadence);
18893 assert!(started_rx.try_recv().is_err());
18894
18895 tokio::time::advance(
18898 first.next_probe_at.unwrap() - Instant::now() - Duration::from_secs(1),
18899 )
18900 .await;
18901 actor.module.restart(Some(0)).await.unwrap();
18902 let spawn_tick = Instant::now();
18903 let replacement = actor
18904 .wait_for_select(|checkpoint| checkpoint.generation > first.generation)
18905 .await;
18906 let deadline = replacement.next_probe_at.unwrap();
18907 assert!(
18908 deadline >= spawn_tick + actor.health.cadence,
18909 "a replacement must not inherit its predecessor's probe deadline"
18910 );
18911 tokio::time::advance(actor.health.cadence - Duration::from_nanos(1)).await;
18912 assert!(
18913 started_rx.try_recv().is_err(),
18914 "no HTTP probe during startup grace"
18915 );
18916 tokio::time::advance(deadline - Instant::now()).await;
18917 let landed = wait_for("first replacement HTTP probe", || {
18918 started_rx.try_recv().ok()
18919 })
18920 .await;
18921 assert!(landed >= spawn_tick + actor.health.cadence);
18922 wait_for("healthy replacement", || {
18923 (actor.module.status().unwrap().health.status == SupervisorHealthStatus::Ok)
18924 .then_some(())
18925 })
18926 .await;
18927 peer.abort();
18928 }
18929
18930 #[tokio::test(start_paused = true)]
18931 async fn rescan_adds_and_removes_http_health_without_polling() {
18932 use tokio::{
18933 io::{AsyncReadExt, AsyncWriteExt},
18934 net::TcpListener,
18935 };
18936 let mut actor = Actor::start(ModuleProtocol::None, Duration::from_secs(30)).await;
18937 let listener = TcpListener::bind("127.0.0.1:0").await.unwrap();
18938 actor.health.http = Some(format!("http://{}/health", listener.local_addr().unwrap()));
18939 actor.health.cadence = Duration::from_millis(1);
18941 let (started_tx, mut started_rx) = mpsc::channel(8);
18942 let peer = tokio::spawn(async move {
18943 loop {
18944 let (mut socket, _) = listener.accept().await.unwrap();
18945 let mut request = [0; 1024];
18946 assert!(socket.read(&mut request).await.unwrap() > 0);
18947 started_tx.send(()).await.unwrap();
18948 socket
18949 .write_all(b"HTTP/1.1 200 OK\r\nContent-Length: 0\r\n\r\n")
18950 .await
18951 .unwrap();
18952 }
18953 });
18954 let tick = Instant::now();
18955 let turns = actor.turns();
18956 actor
18957 .module
18958 .update_configuration(actor.spec.clone(), actor.health.clone(), None)
18959 .await
18960 .unwrap();
18961 let armed = actor
18962 .wait_for_select(|checkpoint| {
18963 checkpoint.turn > turns && checkpoint.next_probe_at.is_some()
18964 })
18965 .await;
18966 assert_eq!(armed.next_probe_at, Some(tick + Duration::from_millis(1)));
18967 assert_eq!(Instant::now(), tick);
18968 tokio::time::advance(Duration::from_millis(1)).await;
18969 let mut started = false;
18970 wait_for("successful HTTP probe", || {
18971 started |= started_rx.try_recv().is_ok();
18972 (started && actor.module.status().unwrap().health.status == SupervisorHealthStatus::Ok)
18973 .then_some(())
18974 })
18975 .await;
18976 assert!(started, "added HTTP check must start at its first deadline");
18977 assert_eq!(
18978 actor.module.status().unwrap().health.status,
18979 SupervisorHealthStatus::Ok
18980 );
18981 assert_eq!(Instant::now(), tick + Duration::from_millis(1));
18982 actor.health.http = None;
18983 let turns = actor.turns();
18984 actor
18985 .module
18986 .update_configuration(actor.spec.clone(), actor.health.clone(), None)
18987 .await
18988 .unwrap();
18989 let parked = actor
18990 .wait_for_select(|checkpoint| {
18991 checkpoint.turn > turns && checkpoint.next_probe_at.is_none()
18992 })
18993 .await;
18994 assert_eq!(parked.wake_after, None);
18995 let turns = actor.turns();
18996 tokio::time::advance(Duration::from_secs(60)).await;
18997 assert_eq!(
18998 actor.turns(),
18999 turns,
19000 "removed HTTP check must park the actor"
19001 );
19002 assert!(
19003 started_rx.try_recv().is_err(),
19004 "removed HTTP check must stay disarmed"
19005 );
19006 assert_eq!(
19007 actor.module.status().unwrap().health,
19008 ModuleHealthStatus::default()
19009 );
19010 peer.abort();
19011 }
19012}
19013
19014#[cfg(test)]
19015mod drain_busy_reprobe_tests {
19016 use super::*;
19020 use std::collections::BTreeMap;
19021 use std::sync::Mutex as StdMutex;
19022 use subc_protocol::{manifest::Concurrency, session::ModuleControlResponse};
19023
19024 const MODULE_ID: &str = "drain-reprobe-module";
19025 const CONNECTION: u64 = 7;
19026 const GAUGE: &str = "runs_in_flight";
19027 const CADENCE: Duration = Duration::from_secs(30);
19030 const CEILING: Duration = Duration::from_secs(30);
19031
19032 #[derive(Clone, Copy)]
19034 enum Answer {
19035 BusyUntil { until: Instant, latency: Duration },
19038 OmitsGauge,
19040 Silent,
19042 }
19043
19044 struct Drain {
19045 forwarding: Arc<ForwardingTable>,
19046 runtime: SupervisorRuntimeConfig,
19047 endpoint: crate::ModuleEndpointId,
19048 probes: Arc<StdMutex<Vec<Instant>>>,
19050 responder: tokio::task::JoinHandle<()>,
19051 }
19052
19053 impl Drain {
19054 fn start(answer: Answer) -> Self {
19055 let registry = Arc::new(Registry::default());
19056 let forwarding = Arc::new(ForwardingTable::default());
19057 let health = HealthConfig {
19058 cadence: CADENCE,
19059 ..HealthConfig::default()
19060 };
19061 let runtime = Supervisor::new_for_test(registry, RestartPolicy::new(3, Duration::ZERO))
19062 .with_forwarding(forwarding.clone())
19063 .with_health_config(health)
19064 .runtime_config();
19065 let (tx, mut rx) = mpsc::channel::<crate::router::OutboundFrame>(8);
19066 let endpoint = forwarding
19067 .register_module_connection(
19068 ConnectionId::new(CONNECTION),
19069 MODULE_ID.to_string(),
19070 subc_protocol::PROTOCOL_VERSION,
19071 Concurrency::ModuleManaged,
19072 FrameSink::new(tx),
19073 )
19074 .unwrap();
19075 let probes = Arc::new(StdMutex::new(Vec::new()));
19076 let responder = tokio::spawn({
19077 let forwarding = forwarding.clone();
19078 let probes = probes.clone();
19079 async move {
19080 while let Some(frame) = rx.recv().await {
19081 let body: Value = serde_json::from_slice(&frame.body).unwrap();
19082 assert_eq!(body["op"], MODULE_CONTROL_OP_HEALTH_CHECK);
19083 probes.lock().unwrap().push(Instant::now());
19084 let metrics = match answer {
19085 Answer::BusyUntil { until, latency } => {
19086 sleep(latency).await;
19087 let busy = u64::from(Instant::now() < until);
19088 serde_json::json!({ GAUGE: busy })
19089 }
19090 Answer::OmitsGauge => serde_json::json!({ "queue_depth": 0 }),
19091 Answer::Silent => continue,
19092 };
19093 let _ = forwarding.complete_module_control_rpc(
19096 ConnectionId::new(CONNECTION),
19097 frame.header.corr,
19098 Some(MODULE_CONTROL_OP_HEALTH_CHECK),
19099 ModuleControlRpcOutcome::Response(ModuleControlResponse::HealthCheck {
19100 status: HealthStatus::Ok,
19101 detail: None,
19102 metrics: Some(metrics),
19103 }),
19104 );
19105 }
19106 }
19107 });
19108 Self {
19109 forwarding,
19110 runtime,
19111 endpoint,
19112 probes,
19113 responder,
19114 }
19115 }
19116
19117 async fn run(&self, ceiling: Duration) -> (bool, Duration) {
19120 let (drained, elapsed, _) = self.run_traced(ceiling).await;
19121 (drained, elapsed)
19122 }
19123
19124 async fn run_traced(&self, ceiling: Duration) -> (bool, Duration, DrainGaugeTrace) {
19126 let started = Instant::now();
19127 let mut trace = DrainGaugeTrace::new(&[GAUGE.to_string()]);
19128 let drained = wait_for_forwarding_quiescence(
19129 &self.forwarding,
19130 MODULE_ID,
19131 &self.runtime,
19132 self.endpoint,
19133 started + ceiling,
19134 &[GAUGE.to_string()],
19135 DrainScope::Active,
19136 &mut trace,
19137 )
19138 .await
19139 .unwrap();
19140 (drained, started.elapsed(), trace)
19141 }
19142
19143 fn probes(&self) -> Vec<Instant> {
19144 self.probes.lock().unwrap().clone()
19145 }
19146 }
19147
19148 impl Drop for Drain {
19149 fn drop(&mut self) {
19150 self.responder.abort();
19151 }
19152 }
19153
19154 #[tokio::test(start_paused = true)]
19155 async fn busy_at_first_probe_then_quiet_drains_long_before_the_ceiling() {
19156 let drain = Drain::start(Answer::BusyUntil {
19157 until: Instant::now() + Duration::from_secs(1),
19158 latency: Duration::ZERO,
19159 });
19160 let (drained, elapsed) = drain.run(CEILING).await;
19161 assert!(
19162 drained,
19163 "gauges that read zero during the drain must let it finish drained, \
19164 not wait out the {CADENCE:?} health cadence (elapsed {elapsed:?})"
19165 );
19166 assert!(
19167 elapsed < Duration::from_secs(3),
19168 "drain took {elapsed:?} after the gauges went quiet at 1s"
19169 );
19170 let probes = drain.probes();
19171 assert!(
19172 probes.len() >= 2,
19173 "the module was busy at the first probe, so a quiet verdict needs a second: {probes:?}"
19174 );
19175 }
19176
19177 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
19182 async fn an_instant_busy_first_answer_does_not_stop_later_probes_in_real_time() {
19183 let drain = Drain::start(Answer::BusyUntil {
19184 until: Instant::now() + Duration::from_millis(600),
19185 latency: Duration::ZERO,
19186 });
19187 let (drained, elapsed, trace) = drain.run_traced(CEILING).await;
19188 assert!(drained, "the gauge fell at 600ms: {trace}");
19189 assert!(
19190 elapsed < Duration::from_secs(3),
19191 "drain took {elapsed:?} after the gauge fell at 600ms: {trace}"
19192 );
19193 let probes = drain.probes();
19194 assert!(
19195 probes.len() >= 2 && trace.probes as usize == probes.len(),
19196 "the drain must re-probe after the first busy answer: {probes:?}, {trace}"
19197 );
19198 }
19199
19200 #[tokio::test(start_paused = true)]
19201 async fn busy_throughout_ends_undrained_at_the_ceiling_with_bounded_probes() {
19202 let started = Instant::now();
19203 let drain = Drain::start(Answer::BusyUntil {
19204 until: started + Duration::from_secs(3600),
19205 latency: Duration::ZERO,
19206 });
19207 let (drained, elapsed) = drain.run(CEILING).await;
19208 assert!(!drained, "a module busy throughout must not drain");
19209 assert!(
19210 elapsed >= CEILING && elapsed < CEILING + drain_busy_reprobe_interval(),
19211 "a busy module is held until the ceiling and no longer: {elapsed:?}"
19212 );
19213 let probes = drain.probes();
19216 let max_probes = (CEILING.as_millis() / drain_busy_reprobe_interval().as_millis()) + 1;
19217 assert!(
19218 probes.len() as u128 <= max_probes,
19219 "{} probes in a {CEILING:?} drain exceeds the {max_probes} the interval allows",
19220 probes.len()
19221 );
19222 for pair in probes.windows(2) {
19223 assert!(
19224 pair[1] - pair[0] >= drain_busy_reprobe_interval(),
19225 "consecutive probes {:?} apart",
19226 pair[1] - pair[0]
19227 );
19228 }
19229 }
19230
19231 #[tokio::test(start_paused = true)]
19232 async fn a_slow_probe_is_never_overlapped_by_the_next() {
19233 let latency = Duration::from_secs(1);
19236 let drain = Drain::start(Answer::BusyUntil {
19237 until: Instant::now() + Duration::from_secs(3600),
19238 latency,
19239 });
19240 let (drained, _) = drain.run(Duration::from_secs(5)).await;
19241 assert!(!drained);
19242 let probes = drain.probes();
19243 assert!(probes.len() >= 2, "the drain must re-probe: {probes:?}");
19244 for pair in probes.windows(2) {
19245 assert!(
19246 pair[1] - pair[0] >= latency + drain_busy_reprobe_interval(),
19247 "a probe started {:?} after the previous one, before it could finish",
19248 pair[1] - pair[0]
19249 );
19250 }
19251 }
19252
19253 #[tokio::test(start_paused = true)]
19254 async fn an_unanswered_probe_cannot_hold_the_drain_past_its_ceiling() {
19255 let drain = Drain::start(Answer::Silent);
19256 let ceiling = Duration::from_secs(2);
19257 let (drained, elapsed) = drain.run(ceiling).await;
19258 assert!(!drained, "an unanswered probe counts as busy");
19259 assert!(
19260 elapsed >= ceiling && elapsed < ceiling + drain_busy_reprobe_interval(),
19261 "the probe deadline is capped at the drain ceiling: {elapsed:?}"
19262 );
19263 assert_eq!(
19264 drain.probes().len(),
19265 1,
19266 "the unanswered probe holds the only slot until the ceiling"
19267 );
19268 }
19269
19270 #[derive(Clone, Default)]
19272 struct LogCapture {
19273 events: Arc<StdMutex<Vec<BTreeMap<String, String>>>>,
19274 }
19275
19276 impl<S: tracing::Subscriber> tracing_subscriber::Layer<S> for LogCapture {
19277 fn on_event(
19278 &self,
19279 event: &tracing::Event<'_>,
19280 _context: tracing_subscriber::layer::Context<'_, S>,
19281 ) {
19282 struct Fields(BTreeMap<String, String>);
19283 impl tracing::field::Visit for Fields {
19284 fn record_debug(&mut self, field: &tracing::field::Field, value: &dyn fmt::Debug) {
19285 self.0
19286 .insert(field.name().to_string(), format!("{value:?}"));
19287 }
19288 }
19289 let mut fields = Fields(BTreeMap::new());
19290 event.record(&mut fields);
19291 self.events.lock().unwrap().push(fields.0);
19292 }
19293 }
19294
19295 impl LogCapture {
19296 fn install(&self) -> tracing::subscriber::DefaultGuard {
19297 use tracing_subscriber::layer::SubscriberExt;
19298 tracing::subscriber::set_default(tracing_subscriber::registry().with(self.clone()))
19299 }
19300
19301 fn with_message(&self, needle: &str) -> Vec<BTreeMap<String, String>> {
19302 self.events
19303 .lock()
19304 .unwrap()
19305 .iter()
19306 .filter(|fields| fields.get("message").is_some_and(|m| m.contains(needle)))
19307 .cloned()
19308 .collect()
19309 }
19310 }
19311
19312 #[tokio::test(start_paused = true)]
19317 async fn a_missing_declared_gauge_is_named_on_the_drain_timeout_line() {
19318 let capture = LogCapture::default();
19319 let _guard = capture.install();
19320 let drain = Drain::start(Answer::OmitsGauge);
19321 let (drained, _, trace) = drain.run_traced(Duration::from_secs(2)).await;
19322 assert!(!drained, "a gauge never reported cannot read quiet");
19323 assert!(trace.probes >= 2, "the drain must re-probe: {trace:?}");
19324
19325 let missing = capture.with_message("lacks a declared busy gauge");
19326 assert_eq!(missing.len(), 1, "one warning per drain: {missing:?}");
19327 assert_eq!(
19328 missing[0].get("missing_gauges").map(String::as_str),
19329 Some(GAUGE)
19330 );
19331
19332 warn_drain_timed_out(
19333 MODULE_ID,
19334 Duration::from_secs(2),
19335 RouteCloseReason::Restart,
19336 &Default::default(),
19337 &trace,
19338 );
19339 let timeouts = capture.with_message("route drain timed out");
19340 assert_eq!(timeouts.len(), 1, "{timeouts:?}");
19341 let gauges = &timeouts[0]["busy_gauges"];
19342 assert!(
19343 gauges.starts_with(&format!("{GAUGE}: missing_from_health_body @")),
19344 "the timeout line must name the gauge and why it read busy: {gauges}"
19345 );
19346 assert!(
19347 gauges.ends_with("; no value ever read"),
19348 "no value was ever read for the gauge: {gauges}"
19349 );
19350 assert_eq!(timeouts[0]["gauge_probes"], trace.probes.to_string());
19351 }
19352
19353 #[tokio::test(start_paused = true)]
19357 async fn the_drain_trace_names_a_busy_value_and_an_unanswered_probe() {
19358 let busy = Drain::start(Answer::BusyUntil {
19359 until: Instant::now() + Duration::from_secs(3600),
19360 latency: Duration::ZERO,
19361 });
19362 let (_, _, trace) = busy.run_traced(Duration::from_secs(1)).await;
19363 let rendered = trace.to_string();
19364 assert!(
19365 rendered.starts_with(&format!("{GAUGE}: non_zero(1) @"))
19366 && rendered.contains("; last value non_zero(1) @"),
19367 "{rendered}"
19368 );
19369
19370 let silent = Drain::start(Answer::Silent);
19371 let (_, _, trace) = silent.run_traced(Duration::from_secs(1)).await;
19372 assert_eq!(
19375 trace.to_string(),
19376 format!("{GAUGE}: probe_failed(no-answer) @1000ms; no value ever read")
19377 );
19378 assert_eq!(trace.probes, 1);
19379 }
19380
19381 #[test]
19383 fn a_gauge_that_is_not_a_count_is_busy_and_shown_as_sent() {
19384 let gauges = [GAUGE.to_string()];
19385 let verdicts = gauge_verdicts(Some(&serde_json::json!({ GAUGE: 0.0 })), &gauges);
19386 assert_eq!(verdicts, vec![GaugeVerdict::NotACount("0.0".to_string())]);
19387 assert_eq!(
19388 busy_gauge_observation(&verdicts),
19389 BusyGaugeObservation::Busy
19390 );
19391 let zero = gauge_verdicts(Some(&serde_json::json!({ GAUGE: 0 })), &gauges);
19392 assert_eq!(
19393 busy_gauge_observation(&zero),
19394 BusyGaugeObservation::Quiescent
19395 );
19396 let none = gauge_verdicts(None, &gauges);
19397 assert_eq!(busy_gauge_observation(&none), BusyGaugeObservation::Omitted);
19398 }
19399}
19400
19401#[cfg(test)]
19402mod health_evidence_tests {
19403 use super::{HealthProbeError, HealthProbeEvidence};
19404 use std::collections::HashSet;
19405
19406 #[test]
19414 fn only_a_dead_lane_is_proof_of_death() {
19415 assert!(HealthProbeError::lane_dead("gone").is_proof_of_death());
19416 assert!(!HealthProbeError::no_answer("timed out").is_proof_of_death());
19420 assert!(!HealthProbeError::bad_answer("garbage").is_proof_of_death());
19421 assert!(!HealthProbeError::misconfigured("no table").is_proof_of_death());
19422 }
19423
19424 #[test]
19430 fn every_evidence_class_has_a_distinct_label() {
19431 let labels = [
19432 HealthProbeError::lane_dead("").label(),
19433 HealthProbeError::no_answer("").label(),
19434 HealthProbeError::bad_answer("").label(),
19435 HealthProbeError::misconfigured("").label(),
19436 ];
19437 let unique: HashSet<_> = labels.iter().collect();
19438 assert_eq!(unique.len(), labels.len(), "labels collided: {labels:?}");
19439 }
19440
19441 #[test]
19447 fn classification_preserves_the_original_message() {
19448 let err = HealthProbeError::no_answer("module did not answer within 5s");
19449 assert_eq!(err.to_string(), "module did not answer within 5s");
19450 assert!(matches!(err.evidence, HealthProbeEvidence::NoAnswer));
19451 }
19452}
19453
19454#[cfg(test)]
19455mod health_tombstone_tests {
19456 use std::{path::PathBuf, sync::Arc, time::Duration};
19457
19458 use subc_protocol::{
19459 manifest::Concurrency,
19460 session::{HealthStatus, ModuleControlResponse},
19461 };
19462 use tokio::sync::mpsc;
19463
19464 use super::{
19465 probe_module_health, HealthAction, HealthConfig, HealthProbeEvidence, ModuleProtocol,
19466 ModuleSpec, RestartPolicy, Supervisor, SupervisorRuntimeConfig,
19467 };
19468 use crate::{
19469 control::ControlHandler,
19470 forwarding::{ForwardingTable, ModuleControlRpcCompletion, ModuleControlRpcOutcome},
19471 registry::{ConnectionId, Registry},
19472 router::FrameSink,
19473 };
19474
19475 struct ProbeHarness {
19476 spec: ModuleSpec,
19477 runtime: SupervisorRuntimeConfig,
19478 forwarding: Arc<ForwardingTable>,
19479 module_connection: ConnectionId,
19480 module_rx: mpsc::Receiver<crate::router::OutboundFrame>,
19481 handler: ControlHandler,
19482 module: super::SupervisedModule,
19483 }
19484
19485 fn probe_harness() -> ProbeHarness {
19486 let registry = Arc::new(Registry::default());
19487 let forwarding = Arc::new(ForwardingTable::default());
19488 let supervisor_handle = super::SupervisorHandle::new();
19489 let health = HealthConfig {
19490 http: None,
19491 cadence: Duration::from_secs(30),
19492 deadline: Duration::from_secs(5),
19493 failure_threshold: 3,
19494 on_degraded: HealthAction::Report,
19495 on_failing: HealthAction::Report,
19496 critical: false,
19497 };
19498 let supervisor = Supervisor::new_for_test(Arc::clone(®istry), RestartPolicy::default())
19499 .with_forwarding(Arc::clone(&forwarding))
19500 .with_handle(supervisor_handle.clone())
19501 .with_health_config(health);
19502 let spec = ModuleSpec {
19503 module_id: "late-health-module".to_string(),
19504 program: PathBuf::from("disabled-module"),
19505 args: Vec::new(),
19506 env: Vec::new(),
19507 reserved: false,
19508 reserved_prefixes: Vec::new(),
19509 protocol: ModuleProtocol::Subc,
19510 overlap: Default::default(),
19511 carry_containment: false,
19512 };
19513 let module = supervisor
19514 .supervise_configured(spec.clone(), false)
19515 .unwrap();
19516 let runtime = supervisor.runtime_config();
19517 let handler = ControlHandler::with_forwarding(registry, Arc::clone(&forwarding))
19518 .with_supervisor(supervisor_handle);
19519 let module_connection = ConnectionId::new(700);
19520 let (module_tx, module_rx) = mpsc::channel(8);
19521 forwarding
19522 .register_module_connection(
19523 module_connection,
19524 spec.module_id.clone(),
19525 subc_protocol::PROTOCOL_VERSION,
19526 Concurrency::ModuleManaged,
19527 FrameSink::new(module_tx),
19528 )
19529 .unwrap();
19530
19531 ProbeHarness {
19532 spec,
19533 runtime,
19534 forwarding,
19535 module_connection,
19536 module_rx,
19537 handler,
19538 module,
19539 }
19540 }
19541
19542 async fn finish_after(
19543 harness: &mut ProbeHarness,
19544 stall: Duration,
19545 ) -> ModuleControlRpcCompletion {
19546 assert!(stall > harness.runtime.health.deadline);
19547 let deadline = harness.runtime.health.deadline;
19548 let probe = probe_module_health(&harness.spec.module_id, &harness.runtime, None);
19549 let answer = async {
19550 let frame = harness.module_rx.recv().await.expect("health.check frame");
19551 tokio::time::advance(deadline).await;
19552 tokio::task::yield_now().await;
19553 tokio::time::advance(stall - deadline).await;
19554 harness
19555 .forwarding
19556 .complete_module_control_rpc(
19557 harness.module_connection,
19558 frame.header.corr,
19559 Some("health.check"),
19560 ModuleControlRpcOutcome::Response(ModuleControlResponse::HealthCheck {
19561 status: HealthStatus::Ok,
19562 detail: None,
19563 metrics: None,
19564 }),
19565 )
19566 .unwrap()
19567 };
19568 let (probe_result, completion) = tokio::join!(probe, answer);
19569 let err = probe_result.expect_err("probe must miss its deadline");
19570 assert!(matches!(err.evidence, HealthProbeEvidence::NoAnswer));
19571 completion
19572 }
19573
19574 async fn time_out_without_answer(harness: &mut ProbeHarness) {
19575 let deadline = harness.runtime.health.deadline;
19576 let probe = probe_module_health(&harness.spec.module_id, &harness.runtime, None);
19577 let exhaust_deadline = async {
19578 let _frame = harness.module_rx.recv().await.expect("health.check frame");
19579 tokio::time::advance(deadline).await;
19580 tokio::task::yield_now().await;
19581 };
19582 let (probe_result, ()) = tokio::join!(probe, exhaust_deadline);
19583 let err = probe_result.expect_err("probe must miss its deadline");
19584 assert!(matches!(err.evidence, HealthProbeEvidence::NoAnswer));
19585 }
19586
19587 async fn run_probe_cycle(harness: &mut ProbeHarness, answer: bool) {
19588 let registry = Arc::clone(&harness.module.inner.registry);
19589 let snapshot = Arc::clone(&harness.module.inner.snapshot);
19590 let process_liveness = super::SupervisorProcessLiveness::default();
19591 let mut child = None;
19592 let cycle = super::run_health_probe_cycle(
19593 &harness.spec,
19594 &harness.runtime,
19595 ®istry,
19596 &process_liveness,
19597 &snapshot,
19598 &mut child,
19599 );
19600 let peer = async {
19601 let frame = harness.module_rx.recv().await.expect("health.check frame");
19602 if answer {
19603 harness
19604 .forwarding
19605 .complete_module_control_rpc(
19606 harness.module_connection,
19607 frame.header.corr,
19608 Some("health.check"),
19609 ModuleControlRpcOutcome::Response(ModuleControlResponse::HealthCheck {
19610 status: HealthStatus::Ok,
19611 detail: None,
19612 metrics: Some(serde_json::json!({"ready": true})),
19613 }),
19614 )
19615 .unwrap();
19616 } else {
19617 tokio::time::advance(harness.runtime.health.deadline).await;
19618 tokio::task::yield_now().await;
19619 }
19620 };
19621 tokio::join!(cycle, peer);
19622 }
19623
19624 #[tokio::test(start_paused = true)]
19625 async fn unanswered_probe_is_unknown_until_threshold_and_ok_report_recovers() {
19626 let mut harness = probe_harness();
19627 let monitor = harness.module.inner.monitor.lock().unwrap().take().unwrap();
19631 monitor.abort();
19632 let _ = monitor.await;
19633 super::update_snapshot(&harness.module.inner.snapshot, None, |state| {
19634 state.enabled = true;
19635 state.state = super::ModuleState::Running;
19636 state.process_alive = true;
19637 })
19638 .unwrap();
19639
19640 run_probe_cycle(&mut harness, true).await;
19641 assert_eq!(
19642 harness.module.status().unwrap().health.status,
19643 super::SupervisorHealthStatus::Ok
19644 );
19645
19646 for failures in 1..harness.runtime.health.failure_threshold {
19647 run_probe_cycle(&mut harness, false).await;
19648 let status = harness.module.status().unwrap();
19649 assert_eq!(status.health.status, super::SupervisorHealthStatus::Unknown);
19650 assert_eq!(status.health.consecutive_failures, failures);
19651 assert!(status.health.last_probe_ms.is_some());
19652 assert!(status.health.detail.unwrap().starts_with("[no-answer]"));
19653 assert!(status.health.metrics.is_none());
19654 assert_eq!(status.state, super::ModuleState::Running);
19655 assert!(status.process_alive);
19656 assert_eq!(status.restart_count, 0);
19657 assert_eq!(status.lifetime_restarts, 0);
19658 assert!(status.health.last_action.is_none());
19659 assert!(harness.runtime.scheduled_respawn.lock().unwrap().is_none());
19660 }
19661
19662 run_probe_cycle(&mut harness, true).await;
19663 let recovered = harness.module.status().unwrap();
19664 assert_eq!(recovered.health.status, super::SupervisorHealthStatus::Ok);
19665 assert_eq!(recovered.health.consecutive_failures, 0);
19666 assert!(recovered.health.detail.is_none());
19667 assert_eq!(
19668 recovered.health.metrics,
19669 Some(serde_json::json!({"ready": true}))
19670 );
19671 assert_eq!(recovered.lifetime_restarts, 0);
19672
19673 for failures in 1..=harness.runtime.health.failure_threshold {
19674 run_probe_cycle(&mut harness, false).await;
19675 let status = harness.module.status().unwrap();
19676 assert_eq!(status.health.consecutive_failures, failures);
19677 if failures < harness.runtime.health.failure_threshold {
19678 assert_eq!(status.health.status, super::SupervisorHealthStatus::Unknown);
19679 assert_eq!(status.state, super::ModuleState::Running);
19680 assert_eq!(status.lifetime_restarts, 0);
19681 assert!(harness.runtime.scheduled_respawn.lock().unwrap().is_none());
19682 } else {
19683 assert_eq!(
19684 status.health.status,
19685 super::SupervisorHealthStatus::Unresponsive
19686 );
19687 assert_eq!(status.state, super::ModuleState::Restarting);
19688 assert_eq!(status.restart_count, 1);
19689 assert_eq!(status.lifetime_restarts, 1);
19690 assert_eq!(status.health.last_action.as_deref(), Some("restart"));
19691 assert!(harness.runtime.scheduled_respawn.lock().unwrap().is_some());
19692 }
19693 }
19694 }
19695
19696 #[tokio::test(start_paused = true)]
19697 async fn late_health_answers_record_start_anchored_latency_for_two_stalls() {
19698 let mut harness = probe_harness();
19699
19700 let first = finish_after(&mut harness, Duration::from_secs(8)).await;
19701 let first_latency = match &first {
19702 ModuleControlRpcCompletion::LateHealthAnswer { latency, .. } => *latency,
19703 other => panic!("late answer was not retained: {other:?}"),
19704 };
19705 assert!(harness.handler.observe_module_control_completion(first));
19706
19707 let second = finish_after(&mut harness, Duration::from_secs(11)).await;
19708 let second_latency = match &second {
19709 ModuleControlRpcCompletion::LateHealthAnswer { latency, .. } => *latency,
19710 other => panic!("late answer was not retained: {other:?}"),
19711 };
19712 assert!(harness.handler.observe_module_control_completion(second));
19713
19714 assert_eq!(first_latency, Duration::from_secs(8));
19715 assert_eq!(
19716 second_latency - first_latency,
19717 Duration::from_secs(3),
19718 "latency must grow linearly with the additional stall"
19719 );
19720 let health = harness.module.status().unwrap().health;
19721 assert_eq!(health.late_answer_count, 2);
19722 assert_eq!(health.last_late_answer_latency_ms, Some(11_000));
19723 }
19724
19725 #[tokio::test(start_paused = true)]
19733 async fn late_answer_clears_the_consecutive_failure_streak() {
19734 let mut harness = probe_harness();
19735
19736 time_out_without_answer(&mut harness).await;
19738 harness
19739 .module
19740 .record_health_probe_failure_for_test("[no-answer] test miss")
19741 .unwrap();
19742 assert_eq!(
19743 harness.module.status().unwrap().health.consecutive_failures,
19744 1,
19745 "precondition: the miss must be on the streak before the late answer"
19746 );
19747
19748 let late = finish_after(&mut harness, Duration::from_secs(9)).await;
19750 assert!(matches!(
19751 late,
19752 ModuleControlRpcCompletion::LateHealthAnswer { .. }
19753 ));
19754 assert!(harness.handler.observe_module_control_completion(late));
19755
19756 let health = harness.module.status().unwrap().health;
19757 assert_eq!(
19758 health.consecutive_failures, 0,
19759 "a late answer is an answer: the streak must reset"
19760 );
19761 assert_eq!(health.late_answer_count, 1);
19762 }
19763
19764 #[tokio::test(start_paused = true)]
19765 async fn repeated_serial_probe_cycles_keep_one_tombstone_per_endpoint() {
19766 let mut harness = probe_harness();
19767
19768 for _ in 0..20 {
19769 time_out_without_answer(&mut harness).await;
19770 assert_eq!(
19771 harness.forwarding.health_probe_tombstone_count().unwrap(),
19772 1
19773 );
19774 }
19775 }
19776}
19777
19778#[cfg(test)]
19779mod child_env_tests {
19780 #[cfg(windows)]
19781 use super::{
19782 clear_current_process_facts, observed_launch_nonce_source, ModuleState, SupervisorSnapshot,
19783 };
19784 #[cfg(windows)]
19785 use std::sync::{Arc, OnceLock};
19786 #[cfg(windows)]
19787 #[test]
19788 fn windows_wire_spawn_prepares_pipe_and_keeps_phase_one_env() {
19789 let spec = spec(Vec::new());
19790 let mut command = Command::new("unused");
19791 let handoff = apply_wire_spawn_args(&mut command, &spec, None, None)
19792 .unwrap()
19793 .unwrap();
19794 let name = command
19795 .as_std()
19796 .get_envs()
19797 .find(|(key, _)| *key == OsStr::new(subc_os::launch_nonce::LAUNCH_NONCE_PIPE_ENV))
19798 .and_then(|(_, value)| value)
19799 .unwrap();
19800 assert_eq!(name, OsStr::new(handoff.name()));
19801 assert!(handoff.name().starts_with(r"\\.\pipe\subc-launch-"));
19802 assert!(command.as_std().get_envs().any(|(key, value)| key
19803 == OsStr::new(subc_os::LAUNCH_NONCE_PIPE_FALLBACK_ENV)
19804 && value == Some(OsStr::new("env"))));
19805 assert!(command
19806 .as_std()
19807 .get_envs()
19808 .any(|(key, value)| key == OsStr::new(SUBC_LAUNCH_NONCE_ENV) && value.is_some()));
19809 }
19810
19811 #[cfg(windows)]
19812 #[test]
19813 fn windows_source_reporting_tracks_consumption_and_resets_per_spawn() {
19814 let mut snapshot = SupervisorSnapshot::new(ModuleState::Running, true);
19815 snapshot.process_alive = true;
19816 snapshot.spawned_protocol = Some(ModuleProtocol::Subc);
19817 assert_eq!(
19818 observed_launch_nonce_source(&snapshot).as_deref(),
19819 Some("env")
19820 );
19821 let consumed = Arc::new(OnceLock::new());
19822 snapshot.pipe_consumed = Some(consumed.clone());
19823 assert_eq!(
19824 observed_launch_nonce_source(&snapshot).as_deref(),
19825 Some("env")
19826 );
19827 consumed.set(()).unwrap();
19828 assert_eq!(
19829 observed_launch_nonce_source(&snapshot).as_deref(),
19830 Some("pipe")
19831 );
19832 snapshot.pipe_consumed = Some(Arc::new(OnceLock::new()));
19833 assert_eq!(
19834 observed_launch_nonce_source(&snapshot).as_deref(),
19835 Some("env")
19836 );
19837 clear_current_process_facts(&mut snapshot);
19838 assert_eq!(observed_launch_nonce_source(&snapshot), None);
19839 }
19840 use super::{
19841 apply_child_env, apply_spawn_role, apply_wire_spawn_args, LaunchSpecDigest, ModuleProtocol,
19842 ModuleSpec, SpawnRole, SupervisorHandle, SPAWN_ROLE_SWAP_CANDIDATE, SUBC_ARG,
19843 SUBC_LAUNCH_NONCE_ENV, SUBC_MODULE_ID_ENV, SUBC_SPAWN_ROLE_ENV,
19844 };
19845 use std::{ffi::OsStr, path::PathBuf};
19846 use tokio::process::Command;
19847
19848 fn spec(env: Vec<(String, String)>) -> ModuleSpec {
19849 ModuleSpec {
19850 module_id: "env-plan".to_string(),
19851 program: PathBuf::from("/nonexistent"),
19852 args: Vec::new(),
19853 env,
19854 reserved: false,
19855 reserved_prefixes: Vec::new(),
19856 protocol: ModuleProtocol::Subc,
19857 overlap: Default::default(),
19858 carry_containment: false,
19859 }
19860 }
19861
19862 #[test]
19863 fn configured_launch_spec_digest_is_canonical_and_excludes_injected_inputs() {
19864 let base = spec(vec![("A".into(), "B=C".into()), ("Z".into(), "v".into())]);
19865 let digest = LaunchSpecDigest::from_spec(&base);
19866 let mut reordered = base.clone();
19867 reordered.env.reverse();
19868 reordered.module_id = "different-id".into();
19869 reordered.reserved = true;
19870 reordered.overlap = Default::default();
19871 assert_eq!(digest, LaunchSpecDigest::from_spec(&reordered));
19872 let mut delimiter_collision = base.clone();
19873 delimiter_collision.env[0] = ("A=B".into(), "C".into());
19874 assert_ne!(digest, LaunchSpecDigest::from_spec(&delimiter_collision));
19875 let mut changed = base.clone();
19876 changed.protocol = ModuleProtocol::None;
19877 assert_ne!(digest, LaunchSpecDigest::from_spec(&changed));
19878 changed = base.clone();
19879 changed.args.push("--flag".into());
19880 assert_ne!(digest, LaunchSpecDigest::from_spec(&changed));
19881 changed = base.clone();
19882 changed.program = PathBuf::from("/other");
19883 assert_ne!(digest, LaunchSpecDigest::from_spec(&changed));
19884 changed = base.clone();
19885 changed.env[1].1 = "other".into();
19886 assert_ne!(digest, LaunchSpecDigest::from_spec(&changed));
19887 assert!(!format!("{digest:?}").contains("B=C"));
19888 assert!(!format!("{digest:?}").contains("[u8"));
19889 }
19890
19891 #[cfg(unix)]
19892 #[test]
19893 fn native_program_bytes_do_not_collapse_through_lossy_utf8() {
19894 use std::os::unix::ffi::OsStringExt;
19895 let mut left = spec(Vec::new());
19896 left.program = std::ffi::OsString::from_vec(vec![b'/', 0xff]).into();
19897 let mut right = left.clone();
19898 right.program = std::ffi::OsString::from_vec(vec![b'/', 0xfe]).into();
19899 assert_eq!(
19900 left.program.to_string_lossy(),
19901 right.program.to_string_lossy()
19902 );
19903 assert_ne!(
19904 LaunchSpecDigest::from_spec(&left),
19905 LaunchSpecDigest::from_spec(&right)
19906 );
19907 }
19908
19909 #[test]
19923 fn ambient_ck_log_is_removed_and_a_configured_one_survives() {
19924 let mut command = Command::new("/nonexistent");
19925 apply_child_env(&mut command, &spec(Vec::new()));
19926 let removed = command
19927 .as_std()
19928 .get_envs()
19929 .any(|(key, value)| key == OsStr::new("CK_LOG") && value.is_none());
19930 assert!(
19931 removed,
19932 "ambient CK_LOG must be explicitly removed for an unconfigured module"
19933 );
19934
19935 let mut configured = Command::new("/nonexistent");
19936 apply_child_env(
19937 &mut configured,
19938 &spec(vec![("CK_LOG".to_string(), "debug".to_string())]),
19939 );
19940 let effective = configured
19941 .as_std()
19942 .get_envs()
19943 .filter(|(key, _)| *key == OsStr::new("CK_LOG"))
19944 .last()
19945 .map(|(_, value)| value.map(|v| v.to_string_lossy().into_owned()));
19946 assert_eq!(
19947 effective,
19948 Some(Some("debug".to_string())),
19949 "a module's configured CK_LOG must survive the ambient removal"
19950 );
19951 }
19952
19953 #[test]
19962 fn protocol_none_spawn_carries_no_subc_argument_and_no_nonce() {
19963 let connection_file = std::path::Path::new("/run/subc-connection.json");
19964 let handle = SupervisorHandle::new();
19965
19966 let mut none_spec = spec(Vec::new());
19967 none_spec.protocol = ModuleProtocol::None;
19968 let mut none = Command::new("/nonexistent");
19969 let none_handoff =
19970 apply_wire_spawn_args(&mut none, &none_spec, Some(connection_file), Some(&handle))
19971 .expect("protocol-none spawn args apply");
19972 assert!(
19973 none_handoff.is_none(),
19974 "protocol:none spawn must not receive a nonce descriptor"
19975 );
19976 #[cfg(windows)]
19977 assert!(!none.as_std().get_envs().any(|(key, value)| key
19978 == OsStr::new(subc_os::LAUNCH_NONCE_PIPE_FALLBACK_ENV)
19979 && value.is_some()));
19980 assert!(
19981 !none.as_std().get_envs().any(|(key, value)| key
19982 == OsStr::new(subc_os::LAUNCH_NONCE_FD_ENV)
19983 && value.is_some()),
19984 "protocol:none spawn must not name a nonce descriptor"
19985 );
19986 let none_args: Vec<String> = none
19987 .as_std()
19988 .get_args()
19989 .map(|a| a.to_string_lossy().into_owned())
19990 .collect();
19991 assert!(
19992 !none_args.iter().any(|a| a == SUBC_ARG),
19993 "protocol:none argv must not carry --subc; got {none_args:?}"
19994 );
19995 let none_has_nonce = none
19996 .as_std()
19997 .get_envs()
19998 .any(|(key, value)| key == OsStr::new(SUBC_LAUNCH_NONCE_ENV) && value.is_some());
19999 assert!(
20000 !none_has_nonce,
20001 "protocol:none spawn must not receive a launch nonce"
20002 );
20003 let none_has_module_id = none
20004 .as_std()
20005 .get_envs()
20006 .any(|(key, value)| key == OsStr::new(SUBC_MODULE_ID_ENV) && value.is_some());
20007 assert!(
20008 none_has_module_id,
20009 "SUBC_MODULE_ID is inert and stays on every path"
20010 );
20011 assert!(
20012 handle.spawn_nonce(&none_spec.module_id).is_none(),
20013 "no nonce record for a process that will never present one"
20014 );
20015
20016 let wire_spec = spec(Vec::new());
20018 let mut wire = Command::new("/nonexistent");
20019 let wire_handoff =
20020 apply_wire_spawn_args(&mut wire, &wire_spec, Some(connection_file), Some(&handle))
20021 .expect("subc-wire spawn args apply");
20022 let wire_fd_env = wire
20023 .as_std()
20024 .get_envs()
20025 .find(|(key, _)| *key == OsStr::new(subc_os::LAUNCH_NONCE_FD_ENV))
20026 .and_then(|(_, value)| value.map(|v| v.to_string_lossy().into_owned()));
20027 #[cfg(unix)]
20028 assert_eq!(
20029 wire_fd_env,
20030 Some(wire_handoff.expect("a descriptor handoff").fd_env_value()),
20031 "a subc-wire spawn names the pipe it will receive at descriptor 3"
20032 );
20033 #[cfg(windows)]
20034 assert!(wire_handoff.is_some() && wire_fd_env.is_none());
20035 #[cfg(not(any(unix, windows)))]
20036 assert!(wire_handoff.is_none() && wire_fd_env.is_none());
20037 let wire_args: Vec<String> = wire
20038 .as_std()
20039 .get_args()
20040 .map(|a| a.to_string_lossy().into_owned())
20041 .collect();
20042 assert_eq!(
20043 wire_args,
20044 vec![
20045 SUBC_ARG.to_string(),
20046 connection_file.to_string_lossy().into_owned()
20047 ],
20048 "a subc-wire spawn still carries --subc <path>"
20049 );
20050 assert_eq!(
20051 wire.as_std()
20052 .get_envs()
20053 .any(|(key, value)| key == OsStr::new(SUBC_LAUNCH_NONCE_ENV) && value.is_some()),
20054 !cfg!(unix),
20055 "only Windows supplies the environment nonce"
20056 );
20057 assert!(handle.spawn_nonce(&wire_spec.module_id).is_some());
20058 }
20059
20060 #[test]
20070 fn plain_spawn_removes_the_spawn_role_even_when_the_spec_sets_it() {
20071 let role = |command: &Command| {
20072 command
20073 .as_std()
20074 .get_envs()
20075 .filter(|(key, _)| *key == OsStr::new(SUBC_SPAWN_ROLE_ENV))
20076 .last()
20077 .map(|(_, value)| value.map(|v| v.to_string_lossy().into_owned()))
20078 };
20079 let forged = spec(vec![(
20080 SUBC_SPAWN_ROLE_ENV.to_string(),
20081 SPAWN_ROLE_SWAP_CANDIDATE.to_string(),
20082 )]);
20083
20084 let mut plain = Command::new("/nonexistent");
20085 apply_child_env(&mut plain, &forged);
20086 apply_spawn_role(&mut plain, SpawnRole::Plain);
20087 assert_eq!(
20088 role(&plain),
20089 Some(None),
20090 "a plain spawn must remove SUBC_SPAWN_ROLE, whatever the spec says"
20091 );
20092
20093 let mut candidate = Command::new("/nonexistent");
20094 apply_child_env(&mut candidate, &spec(Vec::new()));
20095 apply_spawn_role(&mut candidate, SpawnRole::SwapCandidate);
20096 assert_eq!(
20097 role(&candidate),
20098 Some(Some(SPAWN_ROLE_SWAP_CANDIDATE.to_string()))
20099 );
20100 }
20101
20102 #[test]
20108 fn daemon_private_capture_keys_are_not_passed_to_the_child() {
20109 let mut command = Command::new("/nonexistent");
20110 apply_child_env(
20111 &mut command,
20112 &spec(vec![
20113 (super::CAPTURE_KEEP_ENV.to_string(), "5".to_string()),
20114 ("KEPT".to_string(), "yes".to_string()),
20115 ]),
20116 );
20117 let keys: Vec<String> = command
20118 .as_std()
20119 .get_envs()
20120 .filter(|(_, value)| value.is_some())
20121 .map(|(key, _)| key.to_string_lossy().into_owned())
20122 .collect();
20123 assert!(keys.contains(&"KEPT".to_string()), "got {keys:?}");
20124 assert!(
20125 !keys.contains(&super::CAPTURE_KEEP_ENV.to_string()),
20126 "daemon-private capture key leaked to the child: {keys:?}"
20127 );
20128 }
20129}
20130
20131#[cfg(test)]
20132mod jitter_tests {
20133 use super::jittered_health_delay;
20134 use std::{collections::HashSet, time::Duration};
20135
20136 const FLEET: [&str; 14] = [
20145 "aft",
20146 "alfonso-core",
20147 "magic-context",
20148 "broca",
20149 "thalamus",
20150 "quota",
20151 "engram",
20152 "plexus",
20153 "cerebellum",
20154 "astrocyte",
20155 "synapse",
20156 "subc-mcp",
20157 "cortexkit-credentials",
20158 "subc-federation",
20159 ];
20160
20161 #[test]
20169 fn probe_delays_disperse_across_the_fleet() {
20170 let cadence = Duration::from_secs(30);
20171 let delays: HashSet<Duration> = FLEET
20172 .iter()
20173 .map(|id| jittered_health_delay(id, 0, cadence))
20174 .collect();
20175 assert_eq!(
20176 delays.len(),
20177 FLEET.len(),
20178 "every supervised module must land on its own probe offset"
20179 );
20180 }
20181
20182 #[test]
20188 fn jitter_only_delays_and_stays_within_one_tenth_of_cadence() {
20189 let cadence = Duration::from_secs(30);
20190 let span = cadence / 10;
20191 for id in FLEET {
20192 for probe_index in 0..8 {
20193 let delay = jittered_health_delay(id, probe_index, cadence);
20194 assert!(
20195 delay >= cadence,
20196 "{id}#{probe_index}: jitter must not shorten the cadence"
20197 );
20198 assert!(
20199 delay < cadence + span,
20200 "{id}#{probe_index}: jitter must stay inside one tenth of the cadence"
20201 );
20202 }
20203 }
20204 }
20205
20206 #[test]
20212 fn a_module_offset_is_stable_across_restarts() {
20213 let cadence = Duration::from_secs(30);
20214 for id in FLEET {
20215 assert_eq!(
20216 jittered_health_delay(id, 0, cadence),
20217 jittered_health_delay(id, 0, cadence),
20218 "{id}: the same module and probe index must produce the same offset"
20219 );
20220 }
20221 }
20222
20223 #[test]
20225 fn zero_cadence_yields_zero_delay() {
20226 assert_eq!(
20227 jittered_health_delay("aft", 0, Duration::ZERO),
20228 Duration::ZERO
20229 );
20230 }
20231}
20232
20233#[cfg(all(test, target_os = "linux"))]
20234mod cgroup_placement_tests {
20235 use super::{
20236 apply_cgroup_placement, remove_module_cgroup, LaunchSpecDigest, ModuleProtocol, ModuleSpec,
20237 SuperviseError, SupervisedChild,
20238 };
20239 use crate::stderr_tail::{StderrRing, StderrTailConfig};
20240 use cortexkit_test_support::ScratchDir;
20241 use std::{
20242 fs, io,
20243 path::{Path, PathBuf},
20244 sync::{Arc, Mutex},
20245 };
20246 use tokio::process::Command;
20247
20248 #[tokio::test]
20249 async fn unique_spawn_cgroups_clean_up_across_restarts_and_shutdown() {
20250 use super::*;
20251 let dir = ScratchDir::new("unique-spawn-cgroups");
20252 let root = PathBuf::from(format!(
20253 "/sys/fs/cgroup/subc-unique-test-{}-{}",
20254 std::process::id(),
20255 unix_ms_now()
20256 ));
20257 if let Err(error) = fs::create_dir(&root) {
20258 assert!(
20259 std::env::var_os("SUBC_REQUIRE_CGROUP_TEST").is_none(),
20260 "required cgroup test cannot execute: {error}"
20261 );
20262 eprintln!("SKIP unique_spawn_cgroups_clean_up_across_restarts_and_shutdown: {error}");
20263 return;
20264 }
20265 let placement = subc_cgroup::prepare_at(&root).unwrap().unwrap();
20266 let group_count = || {
20267 fs::read_dir(root.join("subc-modules"))
20268 .unwrap()
20269 .map(|entry| entry.unwrap().file_type().unwrap())
20270 .filter(|kind| kind.is_dir())
20271 .count()
20272 };
20273 let supervisor =
20274 Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default())
20275 .with_cgroup_placement(Some(placement.clone()));
20276 let runtime = supervisor.runtime_config();
20277 let mut spec = ModuleSpec {
20278 module_id: "unique-spawn".into(),
20279 program: PathBuf::from("/bin/sleep"),
20280 args: vec!["60".into()],
20281 env: ["XDG_DATA_HOME", "XDG_RUNTIME_DIR", "XDG_CONFIG_HOME"]
20282 .into_iter()
20283 .map(|key| (key.into(), dir.to_string_lossy().into_owned()))
20284 .collect(),
20285 reserved: false,
20286 reserved_prefixes: vec![],
20287 protocol: ModuleProtocol::None,
20288 overlap: Default::default(),
20289 carry_containment: false,
20290 };
20291 let spawn = |spec: &ModuleSpec| {
20292 spawn_child(
20293 spec,
20294 None,
20295 None,
20296 &runtime.stderr_ring,
20297 None,
20298 &runtime.child_roster,
20299 runtime.spawn_delivery(),
20300 Some(&placement),
20301 runtime.register_timeout,
20302 )
20303 .unwrap()
20304 };
20305 let mut live = spawn(&spec);
20306 for _ in 0..3 {
20307 let next = spawn(&spec);
20309 assert_ne!(live.module_id, next.module_id);
20310 live.start_kill().unwrap();
20311 live.wait().await.unwrap();
20312 live = next;
20313 assert!(
20314 live.child.try_wait().unwrap().is_none(),
20315 "retiring the old slot must not kill the replacement"
20316 );
20317 assert_eq!(
20318 group_count(),
20319 1,
20320 "only the live spawn's cgroup should remain"
20321 );
20322 }
20323 supervisor.begin_daemon_shutdown();
20324 let reap = tokio::spawn(async move {
20325 live.wait().await.unwrap();
20326 });
20327 supervisor
20328 .end_children_for_daemon_shutdown(false, std::future::pending())
20329 .await;
20330 reap.await.unwrap();
20331 assert_eq!(group_count(), 0);
20332 spec.program = PathBuf::from("/bin/true");
20334 spec.args.clear();
20335 let fresh_roster = ChildRoster::default();
20336 let mut short = spawn_child(
20337 &spec,
20338 None,
20339 None,
20340 &runtime.stderr_ring,
20341 None,
20342 &fresh_roster,
20343 None,
20344 Some(&placement),
20345 runtime.register_timeout,
20346 )
20347 .unwrap();
20348 short.wait().await.unwrap();
20349 assert_eq!(group_count(), 0);
20350 spec.module_id = "_".repeat(255);
20351 let mut long_id = spawn_child(
20352 &spec,
20353 None,
20354 None,
20355 &runtime.stderr_ring,
20356 None,
20357 &fresh_roster,
20358 None,
20359 Some(&placement),
20360 runtime.register_timeout,
20361 )
20362 .unwrap();
20363 long_id.wait().await.unwrap();
20364 assert_eq!(
20365 group_count(),
20366 0,
20367 "valid long module IDs must not exceed cgroup NAME_MAX"
20368 );
20369 fs::remove_dir(root.join("subc-modules")).unwrap();
20370 fs::remove_dir(root).unwrap();
20371 eprintln!("EXECUTED unique_spawn_cgroups_clean_up_across_restarts_and_shutdown");
20372 }
20373
20374 #[test]
20375 fn failed_parent_cgroup_open_is_a_cgroup_supervision_error() {
20376 let path = Path::new("/definitely-missing-subc-cgroup");
20377 let mut command = Command::new("true");
20378 let error = apply_cgroup_placement(
20379 &mut command,
20380 &ModuleSpec {
20381 module_id: "broken-cgroup".to_string(),
20382 program: PathBuf::from("true"),
20383 args: Vec::new(),
20384 env: Vec::new(),
20385 reserved: false,
20386 reserved_prefixes: Vec::new(),
20387 protocol: ModuleProtocol::Subc,
20388 overlap: Default::default(),
20389 carry_containment: false,
20390 },
20391 path,
20392 )
20393 .expect_err("a parent cgroup open failure must reject the supervised spawn");
20394 let reason = error.to_string();
20395
20396 assert!(
20397 matches!(error, SuperviseError::Cgroup { .. }),
20398 "parent cgroup open must be reported as a cgroup supervision error: {reason}"
20399 );
20400 assert!(
20401 reason.contains("/definitely-missing-subc-cgroup/cgroup.procs"),
20402 "parent cgroup open failure must name cgroup.procs: {reason}"
20403 );
20404 }
20405
20406 #[tokio::test]
20407 async fn reaping_a_child_removes_its_empty_module_cgroup() {
20408 let root = ScratchDir::new("supervisor-reap-cgroup");
20409 fs::write(root.join("cgroup.procs"), b"").expect("write scratch cgroup marker");
20410 let placement = subc_cgroup::prepare_at(&root)
20411 .expect("prepare scratch cgroup root")
20412 .expect("scratch root has a cgroup.procs marker");
20413 let module_id = "reaped-module";
20414 let module = placement
20415 .module_path(module_id)
20416 .expect("create scratch module cgroup");
20417 let child = Command::new("true")
20418 .env("XDG_DATA_HOME", root.path())
20419 .env("XDG_RUNTIME_DIR", root.path())
20420 .env("XDG_CONFIG_HOME", root.path())
20421 .spawn()
20422 .expect("spawn short-lived child");
20423 let pid = child.id().expect("spawned child has pid");
20424 let mut child = SupervisedChild {
20425 registration_channel: None,
20426 registration_spawn: None,
20427 live_program_binding: None,
20428 registered_config_connection: None,
20429 admission_base: None,
20430 registration_declaration: None,
20431 _program_binding: None,
20432 subtree_target: crate::subtree_resources::Target::Unavailable("not_running"),
20433 child: crate::supervise::containment::ProcessChild::plain(child, pid),
20434 exec: crate::activity::ExecOwner(Arc::new(crate::activity::ExecOutcome::new())),
20435 protocol: ModuleProtocol::Subc,
20436 module_id: module_id.to_string(),
20437 cgroup_placement: Some(placement),
20438 stdout_pump: None,
20439 stderr_pump: None,
20440 stderr_ring: Arc::new(Mutex::new(StderrRing::new(StderrTailConfig::default()))),
20441 spawned_at: tokio::time::Instant::now(),
20442 config_delivery: None,
20443 registration_deadline: None,
20444 registration_watch: None,
20445 spawned_at_ms: 0,
20446 spawned_from: PathBuf::from("true"),
20447 launch_spec_digest: LaunchSpecDigest([0; 32]),
20448 spawned_file_identity: None,
20449 process_start_time: None,
20450 process_identity: None,
20451 pid,
20452 roster_guard: None,
20453 #[cfg(target_os = "macos")]
20454 privacy_exec: None,
20455 spawn_failure: None,
20456 admission_kill_sent: false,
20457 };
20458
20459 child.wait().await.expect("reap short-lived child");
20460
20461 assert!(
20462 !module.exists(),
20463 "waiting for a non-carried child must remove its empty cgroup before returning"
20464 );
20465 child.finish_exit(false).await.unwrap();
20466 }
20467
20468 #[test]
20469 fn non_empty_cgroup_removal_is_reported_without_blocking_teardown() {
20470 let root = ScratchDir::new("supervisor-non-empty-cgroup");
20471 fs::write(root.join("cgroup.procs"), b"").expect("write scratch cgroup marker");
20472 let placement = subc_cgroup::prepare_at(&root)
20473 .expect("prepare scratch cgroup root")
20474 .expect("scratch root has a cgroup.procs marker");
20475 let module = placement
20476 .module_path("surviving-module")
20477 .expect("create scratch module cgroup");
20478 fs::write(module.join("surviving-process"), b"still present")
20479 .expect("make scratch cgroup non-empty");
20480 let (logs, _guard) = crate::router::test_log::log_capture(tracing::Level::WARN);
20481
20482 remove_module_cgroup(&placement, "surviving-module");
20483
20484 let logs = crate::router::test_log::captured_logs(&logs);
20485 assert!(
20486 module.exists(),
20487 "failed removal must leave the cgroup intact"
20488 );
20489 assert!(
20490 logs.contains("could not remove module cgroup after process exit; continuing teardown")
20491 && logs.contains("surviving-module"),
20492 "best-effort removal must report the failure without returning it: {logs}"
20493 );
20494 }
20495
20496 #[test]
20497 fn cgroup_pre_exec_spawn_failure_names_the_cgroup_path() {
20498 let cgroup_path = PathBuf::from("/sys/fs/cgroup/subc-modules/broken-module");
20499 let reason = SuperviseError::Spawn {
20500 program: PathBuf::from("/bin/true"),
20501 source: io::Error::from_raw_os_error(13),
20502 cgroup_path: Some(cgroup_path.clone()),
20503 }
20504 .to_string();
20505
20506 assert!(
20507 reason.contains(&cgroup_path.display().to_string()),
20508 "a pre_exec spawn failure must name the cgroup path: {reason}"
20509 );
20510 }
20511}
20512
20513#[cfg(test)]
20514mod spawn_subscriber_lag_tests {
20515 use super::*;
20516
20517 #[tokio::test]
20522 async fn lagged_spawn_subscriber_receives_a_terminal_lagged_error_after_its_queued_frames() {
20523 let feed = SpawnEventFeed::default();
20524 feed.configure_incarnation("lag-incarnation".to_string());
20525 let (tx, mut rx) = mpsc::channel(1);
20528 feed.subscribe(ConnectionId::new(1), 7, 1, None, FrameSink::new(tx))
20529 .expect("subscribe");
20530 let emitted = SPAWN_SUBSCRIBER_BUFFER + 16;
20531 for index in 0..emitted {
20532 feed.emit_spawned(&format!("lag-module-{index}"), 1000, 0);
20533 tokio::task::yield_now().await;
20536 }
20537 assert_eq!(
20538 feed.subscriber_count(),
20539 0,
20540 "the lagged subscriber must be removed"
20541 );
20542
20543 let mut data = Vec::new();
20544 let mut last = None;
20545 loop {
20546 let next = tokio::time::timeout(Duration::from_secs(5), rx.recv())
20547 .await
20548 .expect("the forwarder must finish once the subscriber is dropped");
20549 let Some(outbound) = next else { break };
20550 let frame = outbound.frame;
20551 if frame.header.ty == FrameType::StreamData {
20552 assert!(last.is_none(), "no data may follow the terminal frame");
20553 let event: SpawnEvent = serde_json::from_slice(&frame.body).unwrap();
20554 data.push(event.cursor.seq);
20555 } else {
20556 assert!(last.is_none(), "exactly one terminal frame");
20557 last = Some(frame);
20558 }
20559 }
20560 assert!(!data.is_empty(), "queued frames drain before the terminal");
20561 for pair in data.windows(2) {
20562 assert_eq!(
20563 pair[1],
20564 pair[0] + 1,
20565 "queued frames arrive dense and in order"
20566 );
20567 }
20568 let terminal = last.expect("a lagged subscriber must receive a terminal frame");
20569 assert_eq!(terminal.header.ty, FrameType::Error);
20570 assert_eq!(terminal.header.corr, 7);
20571 let body: subc_protocol::ErrorBody = serde_json::from_slice(&terminal.body).unwrap();
20572 assert_eq!(body.code, SPAWN_SUBSCRIBER_LAGGED_CODE);
20573 let detail = body.detail.expect("lagged error carries detail");
20574 assert_eq!(
20575 detail["first_undelivered_cursor"]["seq"],
20576 data.last().unwrap() + 1,
20577 "the named cursor is the first event the subscriber did not receive"
20578 );
20579 assert_eq!(
20580 detail["first_undelivered_cursor"]["daemon_incarnation"],
20581 "lag-incarnation"
20582 );
20583 }
20584}
20585
20586#[cfg(test)]
20587mod terminal_history_read_concurrency_tests {
20588 use super::*;
20589 use crate::terminal_journal::read_pause;
20590 use cortexkit_test_support::ScratchDir;
20591 use std::sync::mpsc as std_mpsc;
20592
20593 fn journaled_ring(
20594 journal: &Arc<crate::terminal_journal::TerminalJournal>,
20595 ) -> Arc<Mutex<TerminalRing>> {
20596 Arc::new(Mutex::new(
20597 TerminalRing::new(TerminalRingConfig::default(), 1)
20598 .with_journal(Some(Arc::clone(journal))),
20599 ))
20600 }
20601
20602 fn crash(at_ms: u64) -> ExitReport {
20603 ExitReport {
20604 kind: ExitKind::Crash,
20605 code: Some(1),
20606 signal: None,
20607 at_ms,
20608 }
20609 }
20610
20611 fn record_within(
20614 module_id: &'static str,
20615 ring: &Arc<Mutex<TerminalRing>>,
20616 at_ms: u64,
20617 bound: Duration,
20618 ) -> bool {
20619 let ring = Arc::clone(ring);
20620 let (done, done_rx) = std_mpsc::channel();
20621 std::thread::spawn(move || {
20622 record_terminal(
20623 module_id,
20624 &ring,
20625 &SpawnEventFeed::default(),
20626 &crash(at_ms),
20627 TerminalDisposition::Restarting,
20628 );
20629 let _ = done.send(());
20630 });
20631 done_rx.recv_timeout(bound).is_ok()
20632 }
20633
20634 #[test]
20639 fn exits_recorded_during_a_paused_history_read_are_not_blocked_or_half_merged() {
20640 let dir = ScratchDir::new("terminal-history-concurrent-read");
20641 let path = dir.join("terminals.jsonl");
20642 let journal = Arc::new(crate::terminal_journal::TerminalJournal::open(
20643 path.clone(),
20644 "daemon".into(),
20645 ));
20646 let reader_ring = journaled_ring(&journal);
20647 let other_ring = journaled_ring(&journal);
20648 assert!(record_within(
20649 "reader-module",
20650 &reader_ring,
20651 10,
20652 Duration::from_secs(5)
20653 ));
20654
20655 let (started, release) = read_pause::install(&path);
20656 let reading = {
20657 let ring = Arc::clone(&reader_ring);
20658 std::thread::spawn(move || durable_terminal_history_of(&ring, "reader-module"))
20659 };
20660 started
20661 .recv_timeout(Duration::from_secs(5))
20662 .expect("the history read reached its pause");
20663
20664 let bound = Duration::from_secs(1);
20665 assert!(
20666 record_within("other-module", &other_ring, 20, bound),
20667 "another module's exit waited on a history read (journal writer held)"
20668 );
20669 assert!(
20670 record_within("reader-module", &reader_ring, 30, bound),
20671 "the read module's own exit waited on its history read (ring held)"
20672 );
20673
20674 drop(release);
20675 let paused = reading.join().unwrap();
20676 assert_eq!(
20677 paused.entries.iter().map(|e| e.at_ms).collect::<Vec<_>>(),
20678 vec![10],
20679 "an exit recorded after the read began lands in neither half of it"
20680 );
20681 assert_eq!(paused.journal_skipped_lines, 0);
20682 assert_eq!(paused.journal_read_errors, 0);
20683
20684 let after = durable_terminal_history_of(&reader_ring, "reader-module");
20685 assert_eq!(
20686 after.entries.iter().map(|e| e.at_ms).collect::<Vec<_>>(),
20687 vec![10, 30],
20688 "the next read merges ring and journal with no duplicate"
20689 );
20690 assert_eq!(after.journal_skipped_lines, 0);
20691 }
20692}
20693
20694#[cfg(test)]
20699mod stderr_settle_tests {
20700 use std::{
20701 future::Future,
20702 io,
20703 pin::Pin,
20704 sync::{Arc, Mutex},
20705 task::{Context, Poll},
20706 time::Duration,
20707 };
20708
20709 use tokio::{
20710 io::{AsyncRead, ReadBuf},
20711 sync::oneshot,
20712 time::Instant,
20713 };
20714
20715 use super::{settle_stderr_pump, settle_stdout_pump, StderrPump};
20716 use crate::stderr_tail::{
20717 pump_stderr_to, pump_stdout_to, split_capture_stamp, untimed, CaptureState,
20718 ChildOutputSink, OutputSink, StderrRing, StderrTailConfig, TailEntry,
20719 };
20720
20721 const BOUND: Duration = Duration::from_millis(250);
20722
20723 struct HeldReader {
20727 before: Option<Vec<u8>>,
20728 gate: Option<oneshot::Receiver<()>>,
20729 after: io::Cursor<Vec<u8>>,
20730 }
20731
20732 impl AsyncRead for HeldReader {
20733 fn poll_read(
20734 mut self: Pin<&mut Self>,
20735 cx: &mut Context<'_>,
20736 buf: &mut ReadBuf<'_>,
20737 ) -> Poll<io::Result<()>> {
20738 if let Some(bytes) = self.before.take() {
20739 buf.put_slice(&bytes);
20740 return Poll::Ready(Ok(()));
20741 }
20742 if let Some(gate) = self.gate.as_mut() {
20743 match Pin::new(gate).poll(cx) {
20744 Poll::Pending => return Poll::Pending,
20745 Poll::Ready(_) => self.gate = None,
20746 }
20747 }
20748 Pin::new(&mut self.after).poll_read(cx, buf)
20749 }
20750 }
20751
20752 struct DiscardSink;
20753
20754 impl OutputSink for DiscardSink {
20755 fn write_line(&mut self, _line: &[u8]) {}
20756 }
20757
20758 fn line(text: &str) -> TailEntry {
20759 TailEntry::Line {
20760 text: text.to_string(),
20761 truncated: false,
20762 at_ms: None,
20763 }
20764 }
20765
20766 fn lock(ring: &Arc<Mutex<StderrRing>>) -> std::sync::MutexGuard<'_, StderrRing> {
20767 ring.lock().unwrap()
20768 }
20769
20770 fn held_pump(
20774 ring: &Arc<Mutex<StderrRing>>,
20775 before: &str,
20776 after: &str,
20777 ) -> (StderrPump, oneshot::Sender<()>) {
20778 let generation = lock(ring).begin_process();
20779 let (release, gate) = oneshot::channel();
20780 let reader = HeldReader {
20781 before: Some(before.as_bytes().to_vec()),
20782 gate: Some(gate),
20783 after: io::Cursor::new(after.as_bytes().to_vec()),
20784 };
20785 let task = tokio::spawn(pump_stderr_to(
20786 reader,
20787 Arc::clone(ring),
20788 generation,
20789 DiscardSink,
20790 ));
20791 (StderrPump { task, generation }, release)
20792 }
20793
20794 async fn wait_until(ring: &Arc<Mutex<StderrRing>>, done: impl Fn(&StderrRing) -> bool) {
20795 for _ in 0..1000 {
20796 if done(&lock(ring)) {
20797 return;
20798 }
20799 tokio::time::sleep(Duration::from_millis(1)).await;
20800 }
20801 panic!(
20802 "ring never reached the expected state: {:?}",
20803 lock(ring).snapshot(None, None)
20804 );
20805 }
20806
20807 #[tokio::test(start_paused = true)]
20808 async fn a_crash_line_the_reader_had_not_reached_by_the_bound_is_kept_before_the_restart() {
20809 let ring = Arc::new(Mutex::new(StderrRing::new(StderrTailConfig::default())));
20810 let (pump, release) = held_pump(&ring, "booting\n", "config error: missing storage\n");
20811
20812 settle_stderr_pump("crasher", &ring, pump, BOUND).await;
20813 let before_release = lock(&ring).snapshot(None, None);
20814 assert!(
20815 matches!(before_release.capture, CaptureState::Incomplete { .. }),
20816 "a reader that has not reached EOF cannot claim a whole tail: {before_release:?}"
20817 );
20818
20819 let next = lock(&ring).begin_process();
20822 lock(&ring).push_line_from(next, "next process booting");
20823 release.send(()).unwrap();
20824 wait_until(&ring, |ring| {
20825 ring.snapshot(None, None).capture == CaptureState::Captured
20826 })
20827 .await;
20828
20829 assert_eq!(
20830 untimed(lock(&ring).snapshot(None, None).entries),
20831 vec![
20832 line("booting"),
20833 line("config error: missing storage"),
20834 TailEntry::ProcessStart,
20835 line("next process booting"),
20836 ],
20837 "the crash's last line must survive a slow reader and stay in the crashed process's section"
20838 );
20839 }
20840
20841 #[tokio::test(start_paused = true)]
20842 async fn a_pipe_held_open_by_a_descendant_reads_incomplete_without_delaying_the_restart_past_the_bound(
20843 ) {
20844 let ring = Arc::new(Mutex::new(StderrRing::new(StderrTailConfig::default())));
20845 let (pump, _held) = held_pump(&ring, "parent exiting\n", "");
20848
20849 let started = Instant::now();
20850 settle_stderr_pump("orphaning", &ring, pump, BOUND).await;
20851 assert_eq!(
20852 started.elapsed(),
20853 BOUND,
20854 "the restart must wait exactly the bound for a pipe that stays open, no longer"
20855 );
20856
20857 let next = lock(&ring).begin_process();
20858 lock(&ring).push_line_from(next, "next process booting");
20859 tokio::time::sleep(Duration::from_secs(60)).await;
20860
20861 let snapshot = lock(&ring).snapshot(None, None);
20862 match &snapshot.capture {
20863 CaptureState::Incomplete { reason } => assert!(
20864 reason.contains("had not reached EOF") && reason.contains("250ms"),
20865 "the reason must say what is missing and after how long: {reason}"
20866 ),
20867 other => panic!("expected Incomplete while the pipe is held open, got {other:?}"),
20868 }
20869 assert_eq!(
20870 untimed(snapshot.entries),
20871 vec![
20872 line("parent exiting"),
20873 TailEntry::ProcessStart,
20874 line("next process booting"),
20875 ]
20876 );
20877 }
20878
20879 #[tokio::test(start_paused = true)]
20880 async fn a_reader_that_reaches_eof_within_the_bound_leaves_the_tail_captured() {
20881 let ring = Arc::new(Mutex::new(StderrRing::new(StderrTailConfig::default())));
20882 let (pump, release) = held_pump(&ring, "one\n", "two\n");
20883 release.send(()).unwrap();
20884
20885 settle_stderr_pump("clean", &ring, pump, BOUND).await;
20886
20887 let snapshot = lock(&ring).snapshot(None, None);
20888 assert_eq!(snapshot.capture, CaptureState::Captured);
20889 assert_eq!(untimed(snapshot.entries), vec![line("one"), line("two")]);
20890 }
20891
20892 #[tokio::test(start_paused = true)]
20893 async fn stdout_buffered_past_the_exit_drain_bound_reaches_the_capture_file() {
20894 const BURST: usize = 150;
20895 let capture = cortexkit_test_support::ScratchDir::new("stdout-exit-drain");
20896 let path = capture.join("two-pipe.stderr.log");
20897 let sink = ChildOutputSink::open(&path, cortexkit_log::Retention::default()).unwrap();
20898 let ring = Arc::new(Mutex::new(StderrRing::new(StderrTailConfig::default())));
20899 let generation = lock(&ring).begin_process();
20900 let stdout = (0..BURST)
20901 .map(|i| format!("out-{i:04}-{}\n", "o".repeat(64)))
20902 .collect::<String>();
20903 let stderr = (0..BURST)
20904 .map(|i| format!("err-{i:04}-{}\n", "e".repeat(64)))
20905 .collect::<String>();
20906 let first_chunk = stdout.find('\n').unwrap() + 1 + "out-0001-".len();
20911 let (release, gate) = oneshot::channel();
20912 let reader = HeldReader {
20913 before: Some(stdout.as_bytes()[..first_chunk].to_vec()),
20914 gate: Some(gate),
20915 after: io::Cursor::new(stdout.as_bytes()[first_chunk..stdout.len() - 1].to_vec()),
20916 };
20917 let (finished, completion) = oneshot::channel();
20918 let stdout_sink = sink.clone();
20919 let pump = tokio::spawn(async move {
20920 pump_stdout_to(reader, stdout_sink).await;
20921 finished.send(()).unwrap();
20922 });
20923 pump_stderr_to(io::Cursor::new(stderr.into_bytes()), ring, generation, sink).await;
20924
20925 let started = Instant::now();
20926 settle_stdout_pump("two-pipe", pump, BOUND).await;
20927 assert_eq!(
20928 started.elapsed(),
20929 BOUND,
20930 "a late reader must not hold up restart"
20931 );
20932 assert_eq!(
20933 std::fs::read_to_string(&path).unwrap().lines().count(),
20934 BURST + 1
20935 );
20936
20937 release
20938 .send(())
20939 .expect("stdout reader must survive the exit drain bound");
20940 tokio::time::timeout(Duration::from_secs(1), completion)
20941 .await
20942 .expect("released stdout reader must reach EOF")
20943 .expect("stdout capture must finish at EOF, not be cancelled");
20944 let contents = std::fs::read_to_string(&path).unwrap();
20945 let mut lines = contents
20946 .lines()
20947 .map(|line| split_capture_stamp(line).expect("capture stamp").1)
20948 .collect::<Vec<_>>();
20949 let mut expected = (0..BURST)
20950 .flat_map(|i| {
20951 [
20952 format!("out-{i:04}-{}", "o".repeat(64)),
20953 format!("err-{i:04}-{}", "e".repeat(64)),
20954 ]
20955 })
20956 .collect::<Vec<_>>();
20957 lines.sort_unstable();
20958 expected.sort_unstable();
20959 assert_eq!(
20960 lines, expected,
20961 "both pipes must retain every complete line exactly once"
20962 );
20963 }
20964}
20965
20966#[cfg(all(test, windows))]
20980mod job_containment_tests {
20981 use super::*;
20982 use cortexkit_test_support::ScratchDir;
20983 use std::{
20984 path::{Path, PathBuf},
20985 sync::{Arc, Mutex},
20986 time::{Duration, Instant},
20987 };
20988
20989 fn stub_path() -> PathBuf {
20995 let path = cortexkit_test_support::cargo_profile_bin("fake-aft-stub");
20996 assert!(
20997 path.exists(),
20998 "fake-aft-stub not built at {}: run `cargo test -p subc-core` (which builds \
20999 [[bin]] targets) rather than `cargo test -p subc-core --lib` (which does not)",
21000 path.display()
21001 );
21002 path
21003 }
21004
21005 fn read_grandchild_pid(path: &Path) -> u32 {
21007 let deadline = Instant::now() + Duration::from_secs(10);
21008 loop {
21009 if let Ok(contents) = std::fs::read_to_string(path) {
21010 if let Ok(pid) = contents.trim().parse() {
21011 return pid;
21012 }
21013 }
21014 assert!(
21015 Instant::now() < deadline,
21016 "the stub never recorded a grandchild pid at {}",
21017 path.display()
21018 );
21019 std::thread::sleep(Duration::from_millis(10));
21020 }
21021 }
21022
21023 struct Fixture {
21026 _dir: ScratchDir,
21027 _roster: ChildRoster,
21028 module_id: String,
21029 grandchild: u32,
21030 child: Option<SupervisedChild>,
21031 registry: Arc<Registry>,
21032 snapshot: Arc<Mutex<SupervisorSnapshot>>,
21033 terminal_ring: Arc<Mutex<TerminalRing>>,
21034 spawn_events: SpawnEventFeed,
21035 }
21036
21037 fn fixture(label: &str, module_id: &str) -> Fixture {
21038 fixture_with_carry(label, module_id, false)
21039 }
21040
21041 fn fixture_with_carry(label: &str, module_id: &str, carry_containment: bool) -> Fixture {
21042 let dir = ScratchDir::new(label);
21043 let pid_file = dir.join("grandchild.pid");
21044 let supervisor = Supervisor::new_for_test(
21045 Arc::new(Registry::default()),
21046 RestartPolicy::new(3, Duration::ZERO),
21047 );
21048 let runtime = supervisor.runtime_config();
21049 let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::starting()));
21050 let spec = ModuleSpec {
21051 module_id: module_id.to_string(),
21052 program: stub_path(),
21053 args: Vec::new(),
21057 env: vec![
21058 ("FAKE_AFT_NEVER_CONNECT".to_string(), "1".to_string()),
21059 (
21060 "FAKE_AFT_GRANDCHILD_PID_FILE".to_string(),
21061 pid_file.display().to_string(),
21062 ),
21063 ],
21064 reserved: false,
21065 reserved_prefixes: Vec::new(),
21066 protocol: ModuleProtocol::Subc,
21067 overlap: Default::default(),
21068 carry_containment,
21069 };
21070 let child = spawn_and_mark_running(&spec, &runtime, &snapshot)
21071 .expect("spawn the supervised fixture");
21072 let grandchild = read_grandchild_pid(&pid_file);
21073 Fixture {
21074 _dir: dir,
21075 _roster: runtime.child_roster.clone(),
21076 module_id: module_id.to_string(),
21077 grandchild,
21078 child: Some(child),
21079 registry: Arc::new(Registry::default()),
21080 snapshot,
21081 terminal_ring: Arc::clone(&runtime.terminal_ring),
21082 spawn_events: SpawnEventFeed::default(),
21083 }
21084 }
21085
21086 async fn assert_admission_kill_reaps_grandchild(carry: bool) {
21087 let label = if carry {
21088 "admission-kill-carried-tree"
21089 } else {
21090 "admission-kill-plain-tree"
21091 };
21092 let mut fixture = fixture_with_carry(label, label, carry);
21093 let descendant = subc_os::Process::open(fixture.grandchild)
21094 .unwrap()
21095 .expect("the stub descendant must be alive before admission kill");
21096 assert!(descendant.observe().is_some());
21097 let child = fixture.child.as_mut().unwrap();
21098 assert_eq!(child.child.container().is_some(), carry);
21099 assert_eq!(child.job.is_some(), !carry);
21100
21101 child.start_admission_kill().unwrap();
21102 assert!(child.admission_kill_sent);
21103 let status = timeout(Duration::from_secs(10), child.wait())
21104 .await
21105 .unwrap()
21106 .unwrap();
21107 assert_eq!(
21108 status.code(),
21109 Some(admission_exit::WINDOWS_ADMISSION_KILL_EXIT_CODE),
21110 "containment teardown must preserve the leader's distinctive status"
21111 );
21112 assert!(
21115 descendant.wait_for_exit(Duration::from_secs(10)).unwrap(),
21116 "admission kill left a descendant running before exit finalization"
21117 );
21118 assert_eq!(child.child.container().is_some(), carry);
21119 assert_eq!(child.job.is_some(), !carry);
21120 child.finish_exit(false).await.unwrap();
21121 }
21122
21123 #[tokio::test]
21124 async fn admission_kill_reaps_grandchild_with_plain_job() {
21125 assert_admission_kill_reaps_grandchild(false).await;
21126 }
21127
21128 #[tokio::test]
21129 async fn admission_kill_reaps_grandchild_with_carried_job() {
21130 assert_admission_kill_reaps_grandchild(true).await;
21131 }
21132
21133 #[tokio::test]
21134 async fn admission_kill_retires_registration_before_wait() {
21135 let mut fixture = fixture("admission-registration", "admission-registration");
21136 let handle = SupervisorHandle::new();
21137 let spawn =
21138 RegistrationSpawn::new(&handle, "admission-registration", SpawnRole::Plain, None);
21139 spawn.ready();
21140 let child = fixture.child.as_mut().unwrap();
21141 child.registration_spawn = Some(spawn.clone());
21142 child.start_admission_kill().unwrap();
21143 assert!(
21144 !spawn.launch_ready(),
21145 "admission kill must retire before its caller can await the child"
21146 );
21147 timeout(Duration::from_secs(10), child.wait())
21148 .await
21149 .unwrap()
21150 .unwrap();
21151 child.finish_exit(false).await.unwrap();
21152 }
21153
21154 impl Fixture {
21155 async fn drain(&mut self) {
21157 let child = self
21158 .child
21159 .take()
21160 .expect("the fixture child is still present");
21161 drain_child_to_state(
21162 &self.module_id,
21163 ModuleProtocol::Subc,
21164 StopNotice::NotSent,
21167 &self.registry,
21168 None,
21169 &self.snapshot,
21170 &self.terminal_ring,
21171 &self.spawn_events,
21172 child,
21173 Duration::from_millis(500),
21174 ModuleState::Stopped,
21175 Some(false),
21176 &FailureContext::operator(),
21177 )
21178 .await
21179 .expect("drain the supervised fixture");
21180 }
21181 }
21182
21183 #[tokio::test]
21189 async fn teardown_reaps_the_grandchild() {
21190 let mut fixture = fixture("teardown-grandchild", "tree-teardown");
21191 let grandchild = fixture.grandchild;
21192
21193 assert!(
21194 subc_jobobject::process_exists(grandchild),
21195 "grandchild {grandchild} must be alive before teardown, or this proves nothing"
21196 );
21197
21198 fixture.drain().await;
21199
21200 assert!(
21201 subc_jobobject::wait_for_process_exit(grandchild, Duration::from_secs(10)),
21202 "grandchild {grandchild} outlived module teardown: the tree was not contained"
21203 );
21204 }
21205
21206 #[test]
21219 fn an_uncontained_grandchild_survives_a_direct_child_kill() {
21220 let dir = ScratchDir::new("teardown-uncontained");
21221 let pid_file = dir.join("grandchild.pid");
21222 let mut child = std::process::Command::new(stub_path())
21223 .env("FAKE_AFT_NEVER_CONNECT", "1")
21224 .env(
21225 "FAKE_AFT_GRANDCHILD_PID_FILE",
21226 pid_file.display().to_string(),
21227 )
21228 .stdin(std::process::Stdio::null())
21229 .stdout(std::process::Stdio::null())
21230 .stderr(std::process::Stdio::null())
21231 .spawn()
21232 .expect("spawn the uncontained fixture");
21233 let grandchild = read_grandchild_pid(&pid_file);
21234
21235 child.kill().expect("kill the direct child");
21237 let _ = child.wait();
21238
21239 assert!(
21240 subc_jobobject::process_exists(grandchild),
21241 "grandchild {grandchild} died with the direct child, so this control no longer \
21242 distinguishes contained from uncontained teardown and the regression test is \
21243 passing vacuously"
21244 );
21245
21246 kill_tree(grandchild);
21249 }
21250
21251 #[tokio::test]
21260 async fn dropping_containment_reaps_the_grandchild() {
21261 let mut fixture = fixture("drop-containment", "tree-drop");
21262 let grandchild = fixture.grandchild;
21263
21264 assert!(subc_jobobject::process_exists(grandchild));
21265
21266 fixture.child.as_mut().expect("child present").job = None;
21268
21269 assert!(
21270 subc_jobobject::wait_for_process_exit(grandchild, Duration::from_secs(10)),
21271 "grandchild {grandchild} survived the containment handle closing, so a daemon \
21272 crash would leave the tree behind"
21273 );
21274 }
21275
21276 fn kill_tree(pid: u32) {
21278 let _ = std::process::Command::new("taskkill.exe")
21279 .args(["/PID", &pid.to_string(), "/T", "/F"])
21280 .stdin(std::process::Stdio::null())
21281 .stdout(std::process::Stdio::null())
21282 .stderr(std::process::Stdio::null())
21283 .status();
21284 assert!(
21285 subc_jobobject::wait_for_process_exit(pid, Duration::from_secs(10)),
21286 "could not clean up grandchild {pid}"
21287 );
21288 }
21289}
21290
21291#[cfg(test)]
21292mod privacy_trampoline_configuration_tests {
21293 #[tokio::test]
21294 #[cfg_attr(not(target_os = "macos"), ignore = "requires macOS privacy trampoline")]
21295 async fn macos_spawn_without_a_configured_trampoline_refuses_by_name() {
21296 #[cfg(target_os = "macos")]
21297 {
21298 let supervisor = super::Supervisor::new(
21299 std::sync::Arc::new(crate::Registry::default()),
21300 super::RestartPolicy::default(),
21301 );
21302 let error = supervisor.spawn(spec()).unwrap_err();
21303 assert!(
21304 error
21305 .to_string()
21306 .contains("no privacy trampoline configured"),
21307 "{error}"
21308 );
21309 }
21310 }
21311
21312 #[tokio::test]
21313 #[cfg_attr(not(target_os = "macos"), ignore = "requires macOS privacy trampoline")]
21314 async fn macos_wrong_trampoline_is_rejected_by_the_startup_probe() {
21315 #[cfg(target_os = "macos")]
21316 {
21317 let supervisor = super::Supervisor::new(
21318 std::sync::Arc::new(crate::Registry::default()),
21319 super::RestartPolicy::default(),
21320 )
21321 .with_privacy_trampoline(std::env::current_exe().unwrap());
21322 let error = supervisor.spawn(spec()).unwrap_err();
21323 assert!(
21324 error
21325 .to_string()
21326 .contains("binary does not implement the privacy trampoline protocol"),
21327 "{error}"
21328 );
21329 }
21330 }
21331
21332 #[cfg(target_os = "macos")]
21333 fn spec() -> super::ModuleSpec {
21334 super::ModuleSpec {
21335 module_id: "privacy-configuration".into(),
21336 program: "/bin/sleep".into(),
21337 args: vec!["30".into()],
21338 env: vec![],
21339 reserved: false,
21340 reserved_prefixes: vec![],
21341 protocol: subc_control::ModuleProtocol::None,
21342 overlap: super::ModuleOverlap::Exclusive,
21343 carry_containment: false,
21344 }
21345 }
21346}
21347
21348#[cfg(test)]
21349mod privacy_exec_boundary_tests {
21350 #[cfg(target_os = "macos")]
21351 use super::*;
21352 #[cfg(target_os = "macos")]
21353 use std::{
21354 io::{Read, Write},
21355 net::{TcpListener, TcpStream},
21356 };
21357
21358 #[cfg(target_os = "macos")]
21361 pub(super) fn before_image_sample(spec: &ModuleSpec, pid: u32) {
21362 if let Some((_, path)) = spec
21363 .env
21364 .iter()
21365 .find(|(key, _)| key == "SUBC_TEST_PRIVACY_SAMPLE_BARRIER")
21366 {
21367 let mut barrier = TcpStream::connect(path).unwrap();
21368 barrier
21369 .set_read_timeout(Some(Duration::from_secs(30)))
21370 .unwrap();
21371 barrier.write_all(&pid.to_ne_bytes()).unwrap();
21372 let mut release = [0];
21373 barrier.read_exact(&mut release).unwrap();
21374 assert_eq!(&release, b"X");
21375 }
21376 }
21377
21378 #[cfg(target_os = "macos")]
21379 fn spec(program: &str, args: &[&str]) -> ModuleSpec {
21380 ModuleSpec {
21381 module_id: "privacy-boundary".into(),
21382 program: program.into(),
21383 args: args.iter().map(|arg| (*arg).into()).collect(),
21384 env: vec![],
21385 reserved: false,
21386 reserved_prefixes: vec![],
21387 protocol: ModuleProtocol::None,
21388 overlap: ModuleOverlap::Exclusive,
21389 carry_containment: false,
21390 }
21391 }
21392
21393 #[cfg(target_os = "macos")]
21394 async fn accept(listener: TcpListener) -> TcpStream {
21395 let listener = tokio::net::TcpListener::from_std({
21397 listener.set_nonblocking(true).unwrap();
21398 listener
21399 })
21400 .unwrap();
21401 let (stream, _) = tokio::time::timeout(Duration::from_secs(30), listener.accept())
21402 .await
21403 .unwrap()
21404 .unwrap();
21405 let stream = stream.into_std().unwrap();
21406 stream.set_nonblocking(false).unwrap();
21407 stream
21408 .set_read_timeout(Some(Duration::from_secs(30)))
21409 .unwrap();
21410 stream
21411 }
21412
21413 #[tokio::test]
21414 #[cfg_attr(not(target_os = "macos"), ignore = "requires macOS privacy trampoline")]
21415 async fn macos_roster_withholds_a_nonnull_trampoline_image_until_exec_confirmation() {
21416 #[cfg(target_os = "macos")]
21417 {
21418 let root = cortexkit_test_support::ScratchDir::new("privacy-roster-barrier");
21419 let exec_listener = TcpListener::bind("127.0.0.1:0").unwrap();
21422 let sample_listener = TcpListener::bind("127.0.0.1:0").unwrap();
21423 let record = root.join("live-children.json");
21424 let supervisor =
21425 Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default())
21426 .with_live_children_record(&record);
21427 let runtime = supervisor.runtime_config();
21428 let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::starting()));
21429 let mut spec = spec("/bin/sleep", &["30"]);
21430 spec.env = vec![
21431 (
21432 "SUBC_TEST_PRIVACY_EXEC_BARRIER".into(),
21433 exec_listener.local_addr().unwrap().to_string(),
21434 ),
21435 (
21436 "SUBC_TEST_PRIVACY_SAMPLE_BARRIER".into(),
21437 sample_listener.local_addr().unwrap().to_string(),
21438 ),
21439 ];
21440 let spawn = tokio::task::spawn_blocking(move || {
21441 spawn_and_mark_running(&spec, &runtime, &snapshot).unwrap()
21442 });
21443 let mut sample = accept(sample_listener).await;
21444 let mut pid = [0; 4];
21445 sample.read_exact(&mut pid).unwrap();
21446 let pid = u32::from_ne_bytes(pid);
21447 let mut exec = accept(exec_listener).await;
21448 let mut ready = [0];
21449 exec.read_exact(&mut ready).unwrap();
21450 assert_eq!(&ready, b"R");
21451 let trampoline = subc_os::file_identity(&test_privacy_trampoline()).unwrap();
21454 assert_eq!(
21455 observe_spawned_image(pid).unwrap().executable,
21456 Some(trampoline)
21457 );
21458 sample.write_all(b"X").unwrap();
21459 let mut child = spawn.await.unwrap();
21460 let early = crate::live_children::read_record(&record).unwrap();
21461 assert_eq!(early.len(), 1);
21462 assert_eq!(early[0].pid, pid);
21463 assert_eq!(
21464 early[0].executable, None,
21465 "unconfirmed trampoline image entered the roster"
21466 );
21467 assert!(child.report_ready.get().is_none());
21468 child.privacy_exec.as_mut().unwrap().deadline =
21471 tokio::time::Instant::now() + Duration::from_secs(30);
21472 exec.write_all(b"X").unwrap();
21473 child.confirm_privacy_exec().await;
21474 assert_eq!(child.spawn_failure, None);
21475 assert!(child.report_ready.get().is_some());
21476 let confirmed = crate::live_children::read_record(&record).unwrap();
21477 let module = subc_os::file_identity(std::path::Path::new("/bin/sleep")).unwrap();
21478 assert_ne!(module, trampoline);
21479 assert_eq!(confirmed[0].executable, Some(module.into()));
21480 child.start_kill().unwrap();
21481 child.wait().await.unwrap();
21482 child.release_roster();
21483 }
21484 }
21485
21486 #[tokio::test]
21487 #[cfg_attr(not(target_os = "macos"), ignore = "requires macOS privacy trampoline")]
21488 async fn macos_unstamped_reexec_records_the_observed_native_image() {
21489 #[cfg(target_os = "macos")]
21490 {
21491 let root = cortexkit_test_support::ScratchDir::new("privacy-reexec");
21492 let record = root.join("live-children.json");
21493 let supervisor =
21494 Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default())
21495 .with_live_children_record(&record);
21496 let runtime = supervisor.runtime_config();
21497 let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::starting()));
21498 let fixture = crate::config_gate::program::native_fixture_for_test();
21499 let mut spec = spec(
21500 "/bin/sh",
21501 &["-c", "exec \"$1\" 600", "reexec", fixture.to_str().unwrap()],
21502 );
21503 spec.env
21504 .push(("SUBC_TEST_PRIVACY_IMAGE_DELAY_MS".into(), "5".into()));
21505 let mut child = spawn_and_mark_running(&spec, &runtime, &snapshot).unwrap();
21506 let native_image = subc_os::file_identity(&fixture).unwrap();
21507 let reexec = tokio::time::timeout(Duration::from_secs(5), async {
21510 while observe_spawned_image(child.pid).and_then(|image| image.executable)
21511 != Some(native_image)
21512 {
21513 tokio::time::sleep(Duration::from_millis(1)).await;
21514 }
21515 })
21516 .await;
21517 child.confirm_privacy_exec().await;
21518 let failure = child.spawn_failure.clone();
21519 let ready = child.report_ready.get().is_some();
21520 let confirmed = crate::live_children::read_record(&record).unwrap();
21521 child.start_kill().unwrap();
21522 tokio::time::timeout(Duration::from_secs(5), child.wait())
21523 .await
21524 .unwrap()
21525 .unwrap();
21526 child.release_roster();
21527
21528 reexec.expect("the shell must re-exec the native fixture");
21529 assert_eq!(
21530 failure, None,
21531 "an unstamped program may re-exec before observation"
21532 );
21533 assert!(ready);
21534 assert_eq!(confirmed.len(), 1);
21535 assert_eq!(confirmed[0].executable, Some(native_image.into()));
21536 }
21537 }
21538
21539 #[tokio::test]
21540 #[cfg_attr(not(target_os = "macos"), ignore = "requires macOS privacy trampoline")]
21541 async fn macos_unstamped_ack_without_exec_still_refuses_the_trampoline() {
21542 #[cfg(target_os = "macos")]
21543 {
21544 let listener = TcpListener::bind("127.0.0.1:0").unwrap();
21545 let supervisor =
21546 Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default());
21547 let runtime = supervisor.runtime_config();
21548 let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::starting()));
21549 let fixture = crate::config_gate::program::native_fixture_for_test();
21550 let mut spec = spec(fixture.to_str().unwrap(), &["600"]);
21551 spec.env.push((
21552 "SUBC_TEST_PRIVACY_EXEC_BARRIER".into(),
21553 listener.local_addr().unwrap().to_string(),
21554 ));
21555 let mut child = spawn_and_mark_running(&spec, &runtime, &snapshot).unwrap();
21556 let mut exec = accept(listener).await;
21557 let mut ready = [0];
21558 exec.read_exact(&mut ready).unwrap();
21559 assert_eq!(&ready, b"R");
21560 assert_eq!(
21561 observe_spawned_image(child.pid).unwrap().executable,
21562 subc_os::file_identity(&test_privacy_trampoline())
21563 );
21564 let (reader, writer) = std::io::pipe().unwrap();
21567 drop(writer);
21568 child.privacy_exec.as_mut().unwrap().reader =
21569 tokio::io::unix::AsyncFd::new(reader).unwrap();
21570 child.confirm_privacy_exec().await;
21571 let failure = child.spawn_failure.clone();
21572 let ready = child.report_ready.get().is_some();
21573 child.start_kill().unwrap();
21574 tokio::time::timeout(Duration::from_secs(5), child.wait())
21575 .await
21576 .unwrap()
21577 .unwrap();
21578 child.release_roster();
21579 assert_eq!(
21580 failure.as_deref(),
21581 Some("privacy identity trampoline did not exec the module")
21582 );
21583 assert!(!ready);
21584 }
21585 }
21586
21587 #[tokio::test]
21588 #[cfg_attr(not(target_os = "macos"), ignore = "requires macOS privacy trampoline")]
21589 async fn macos_resumed_121_is_a_module_exit_not_a_trampoline_refusal() {
21590 #[cfg(target_os = "macos")]
21591 {
21592 let registry = Arc::new(Registry::default());
21593 let policy = RestartPolicy::new(0, Duration::ZERO);
21594 let fixture = crate::config_gate::program::native_fixture_for_test();
21595 let spec = spec(fixture.to_str().unwrap(), &["exit121"]);
21596 let scratch = cortexkit_test_support::ScratchDir::new("resumed-exit");
21597 let supervisor = Supervisor::new_for_test(Arc::clone(®istry), policy)
21598 .with_config_runtime(suspended_spawn_tests::gated_owner(&scratch, &spec, true));
21599 let runtime = supervisor.runtime_config();
21600 let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::starting()));
21601 let mut child = spawn_and_mark_running(&spec, &runtime, &snapshot).unwrap();
21604 child.confirm_privacy_exec().await;
21605 assert_eq!(child.spawn_failure, None);
21606 assert!(child.report_ready.get().is_some());
21607 let pid = child.pid;
21608 tokio::task::spawn_blocking(move || {
21609 subc_os::fork_exec_test::wait_for_child_exit_without_reaping(pid)
21610 })
21611 .await
21612 .unwrap()
21613 .unwrap();
21614 let status = child.wait().await.unwrap();
21615 assert_eq!(status.code(), Some(121));
21616 assert!(child.privacy_exec.is_none());
21617 let report = classify_reaped_child_exit(&snapshot, &child, &status);
21618 on_child_exit(
21619 &spec,
21620 policy,
21621 ®istry,
21622 &snapshot,
21623 &runtime.terminal_ring,
21624 &runtime.spawn_events,
21625 &runtime.child_roster,
21626 report,
21627 true,
21628 )
21629 .await;
21630 let state = lock_snapshot(&snapshot).unwrap();
21631 assert_eq!(state.state, ModuleState::Failed);
21632 assert_eq!(state.last_exit.as_ref().unwrap().code, Some(121));
21633 assert_eq!(state.reported_pid(), None);
21634 drop(state);
21635 let history = runtime.terminal_ring.lock().unwrap().snapshot();
21636 assert_eq!(history.entries.len(), 1);
21637 let terminal = &history.entries[0];
21638 assert_eq!(terminal.exit_code, Some(121));
21639 assert_eq!(terminal.exit_kind, subc_control::TerminalExitKind::Crash);
21640 assert_eq!(terminal.disposition, TerminalDisposition::Failed);
21641 assert_eq!(
21642 terminal.disposition_detail.as_deref(),
21643 Some(policy.budget_exhausted_detail().as_str()),
21644 "module exit 121 was classified as a trampoline refusal: {terminal:?}"
21645 );
21646 assert_eq!(child.spawn_failure, None);
21647 child.release_roster();
21648 }
21649 }
21650}
21651
21652#[cfg(all(test, target_os = "macos"))]
21653#[path = "supervise/suspended_spawn_tests.rs"]
21654mod suspended_spawn_tests;
21655
21656#[cfg(all(test, unix))]
21660mod launch_nonce_descriptor_tests {
21661 use super::{ChildRoster, ModuleProtocol, ModuleSpec, SupervisorHandle};
21662 use crate::stderr_tail::{StderrRing, StderrTailConfig};
21663 use cortexkit_test_support::ScratchDir;
21664 use std::{
21665 path::PathBuf,
21666 sync::{Arc, Mutex},
21667 time::{Duration, Instant},
21668 };
21669
21670 async fn probe(role: super::SpawnRole) {
21671 let scratch = ScratchDir::new("launch-nonce-descriptor");
21672 let fd_copy = scratch.join("from-descriptor");
21673 let env_copy = scratch.join("environment");
21674 let script = format!(
21675 "env > '{env}.tmp' && mv '{env}.tmp' '{env}'; cat <&3 > '{fd}.tmp' && mv '{fd}.tmp' '{fd}'; sleep 30",
21676 fd = fd_copy.display(), env = env_copy.display(),
21677 );
21678 let xdg = |name: &str| (name.to_string(), scratch.join(name).display().to_string());
21679 let spec = ModuleSpec {
21680 module_id: "nonce-descriptor-probe".to_string(),
21681 program: PathBuf::from("/bin/sh"),
21682 args: vec!["-c".to_string(), script],
21683 env: vec![
21684 xdg("XDG_DATA_HOME"),
21685 xdg("XDG_RUNTIME_DIR"),
21686 xdg("XDG_CONFIG_HOME"),
21687 ("SUBC_LAUNCH_NONCE".to_string(), "must-not-leak".to_string()),
21688 ],
21689 reserved: true,
21690 reserved_prefixes: Vec::new(),
21691 protocol: ModuleProtocol::Subc,
21692 overlap: Default::default(),
21693 carry_containment: false,
21694 };
21695 let handle = SupervisorHandle::new();
21696 let ring = Arc::new(Mutex::new(StderrRing::new(StderrTailConfig::default())));
21697 let roster = ChildRoster::default();
21698 #[cfg(target_os = "macos")]
21699 {
21700 let path = super::test_privacy_trampoline();
21701 roster.set_privacy_trampoline(super::probe_privacy_trampoline(&path).map(|()| path));
21702 }
21703 let child = super::spawn_child_in_slot(
21704 &spec,
21705 None,
21706 Some(&handle),
21707 &ring,
21708 None,
21709 &roster,
21710 None,
21711 #[cfg(target_os = "linux")]
21712 None,
21713 role,
21714 matches!(role, super::SpawnRole::SwapCandidate),
21715 super::DEFAULT_REGISTER_TIMEOUT,
21716 )
21717 .expect("spawn probe");
21718 let deadline = Instant::now() + Duration::from_secs(10);
21719 while !(fd_copy.exists() && env_copy.exists()) {
21720 assert!(Instant::now() < deadline, "probe never wrote its copies");
21721 tokio::time::sleep(Duration::from_millis(20)).await;
21722 }
21723 let nonce = std::fs::read_to_string(fd_copy).unwrap();
21724 assert!(!nonce.is_empty());
21725 let environment = std::fs::read_to_string(env_copy).unwrap();
21726 assert!(environment
21727 .lines()
21728 .any(|line| line.starts_with("SUBC_LAUNCH_NONCE_FD=3:")));
21729 let copy = environment
21730 .lines()
21731 .find_map(|line| line.strip_prefix("SUBC_LAUNCH_NONCE="));
21732 assert_eq!(
21733 copy, None,
21734 "Unix children must never receive the environment nonce"
21735 );
21736 if matches!(role, super::SpawnRole::Plain) {
21737 assert_eq!(
21738 handle.spawn_nonce(&spec.module_id).as_deref(),
21739 Some(nonce.as_str())
21740 );
21741 }
21742 drop(child);
21743 }
21744
21745 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
21746 async fn a_spawned_module_receives_its_nonce_only_on_descriptor_3() {
21747 probe(super::SpawnRole::Plain).await;
21748 }
21749
21750 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
21751 async fn swap_candidate_receives_its_nonce_only_on_descriptor_3() {
21752 probe(super::SpawnRole::SwapCandidate).await;
21753 }
21754}
21755
21756#[cfg(all(test, target_os = "linux"))]
21757#[path = "supervise/cgroup_reap_tests.rs"]
21758mod cgroup_reap_tests;
21759
21760#[cfg(all(test, target_os = "linux"))]
21761mod cgroup_containment_tests {
21762 use super::*;
21763 use cortexkit_test_support::ScratchDir;
21764
21765 #[tokio::test]
21766 async fn linux_subtree_memory_covers_module_children() {
21767 let dir = ScratchDir::new("subtree-memory");
21768 let root = PathBuf::from(format!(
21769 "/sys/fs/cgroup/subc-memory-test-{}-{}",
21770 std::process::id(),
21771 unix_ms_now()
21772 ));
21773 if let Err(error) = std::fs::create_dir(&root) {
21774 assert!(
21775 std::env::var_os("SUBC_REQUIRE_CGROUP_TEST").is_none(),
21776 "required cgroup test cannot execute: {error}"
21777 );
21778 eprintln!("SKIP linux_subtree_memory_covers_module_children: {error}");
21779 return;
21780 }
21781 let placement = subc_cgroup::prepare_at(&root)
21782 .unwrap()
21783 .expect("delegated cgroup");
21784 let supervisor = Supervisor::new_for_test(
21785 Arc::new(Registry::default()),
21786 RestartPolicy::new(3, Duration::ZERO),
21787 )
21788 .with_cgroup_placement(Some(placement));
21789 let ready = dir.join("ready");
21790 let module = supervisor
21791 .spawn(ModuleSpec {
21792 module_id: "memory-tree".into(),
21793 program: PathBuf::from("/bin/sh"),
21794 args: vec![
21795 "-c".into(),
21796 "sleep 600 & sleep 600 & sleep 600 & echo ready > \"$1\"; wait".into(),
21797 "fixture".into(),
21798 ready.display().to_string(),
21799 ],
21800 env: ["XDG_DATA_HOME", "XDG_RUNTIME_DIR", "XDG_CONFIG_HOME"]
21801 .into_iter()
21802 .map(|key| (key.into(), dir.display().to_string()))
21803 .collect(),
21804 reserved: false,
21805 reserved_prefixes: Vec::new(),
21806 protocol: ModuleProtocol::None,
21807 overlap: Default::default(),
21808 carry_containment: false,
21809 })
21810 .unwrap();
21811 let deadline = tokio::time::Instant::now() + Duration::from_secs(10);
21812 while !ready.exists() {
21813 assert!(
21814 tokio::time::Instant::now() < deadline,
21815 "children never started"
21816 );
21817 tokio::time::sleep(Duration::from_millis(10)).await;
21818 }
21819 let readings = crate::subtree_resources::read(vec![module.subtree_target()]).await;
21820 let reading = &readings[0];
21821 assert!(
21822 reading.process_count.is_some_and(|count| count >= 4),
21823 "{reading:?}"
21824 );
21825 assert!(
21826 reading.memory_bytes.is_some_and(|bytes| bytes > 0),
21827 "{reading:?}"
21828 );
21829 module.stop().await.unwrap();
21830 drop(module);
21831 std::fs::remove_dir(root.join("subc-modules")).unwrap();
21832 std::fs::remove_dir(&root).unwrap();
21833 eprintln!("EXECUTED linux_subtree_memory_covers_module_children: {reading:?}");
21834 }
21835
21836 fn running(pid: u32) -> bool {
21837 std::fs::read_to_string(format!("/proc/{pid}/stat"))
21839 .ok()
21840 .and_then(|stat| {
21841 stat.rsplit_once(") ")
21842 .map(|(_, rest)| rest.starts_with('Z'))
21843 })
21844 .is_some_and(|zombie| !zombie)
21845 }
21846
21847 #[tokio::test]
21848 async fn linux_teardown_reaps_the_grandchild() {
21849 teardown_tree("linux_teardown_reaps_the_grandchild", false).await;
21850 }
21851
21852 #[tokio::test]
21853 async fn linux_shutdown_straggler_reaps_the_grandchild() {
21854 teardown_tree("linux_shutdown_straggler_reaps_the_grandchild", true).await;
21855 }
21856
21857 async fn teardown_tree(test_name: &str, shutdown: bool) {
21858 let dir = ScratchDir::new(test_name);
21859 let root = PathBuf::from(format!(
21860 "/sys/fs/cgroup/subc-tree-test-{test_name}-{}-{}",
21861 std::process::id(),
21862 unix_ms_now()
21863 ));
21864 if let Err(error) = std::fs::create_dir(&root) {
21865 assert!(
21866 std::env::var_os("SUBC_REQUIRE_CGROUP_TEST").is_none(),
21867 "required cgroup test cannot execute: {error}"
21868 );
21869 eprintln!(
21870 "SKIP {test_name}: no writable delegated cgroup at {}: {error}",
21871 root.display()
21872 );
21873 return;
21874 }
21875 let placement = subc_cgroup::prepare_at(&root)
21876 .expect("prepare isolated kernel cgroup")
21877 .expect("isolated cgroup is delegated");
21878 let module_id = "tree-teardown";
21879 let module = placement
21880 .module_path(module_id)
21881 .expect("create isolated module cgroup");
21882 if !module.join("cgroup.kill").exists() {
21883 std::fs::remove_dir(&module).unwrap();
21884 std::fs::remove_dir(root.join("subc-modules")).unwrap();
21885 std::fs::remove_dir(&root).unwrap();
21886 assert!(
21887 std::env::var_os("SUBC_REQUIRE_CGROUP_TEST").is_none(),
21888 "required cgroup.kill interface unavailable"
21889 );
21890 eprintln!("SKIP {test_name}: cgroup.kill unavailable (kernel < 5.14)");
21891 return;
21892 }
21893 let supervisor = Supervisor::new_for_test(
21894 Arc::new(Registry::default()),
21895 RestartPolicy::new(3, Duration::ZERO),
21896 )
21897 .with_cgroup_placement(Some(placement));
21898 let mut runtime = supervisor.runtime_config();
21899 runtime.child_roster = runtime
21900 .child_roster
21901 .for_module(Arc::new(Mutex::new(Duration::from_millis(100))));
21902 let snapshot = Arc::new(Mutex::new(SupervisorSnapshot::starting()));
21903 let pid_file = dir.join("grandchild.pid");
21904 let spec = ModuleSpec {
21905 module_id: module_id.to_string(),
21906 program: PathBuf::from("/bin/sh"),
21907 args: vec![
21908 "-c".into(),
21909 "trap '' TERM; sleep 600 & echo $! > \"$1\"; wait".into(),
21910 "fixture".into(),
21911 pid_file.display().to_string(),
21912 ],
21913 env: ["XDG_DATA_HOME", "XDG_RUNTIME_DIR", "XDG_CONFIG_HOME"]
21914 .into_iter()
21915 .map(|key| (key.to_string(), dir.display().to_string()))
21916 .collect(),
21917 reserved: false,
21918 reserved_prefixes: Vec::new(),
21919 protocol: ModuleProtocol::None,
21920 overlap: Default::default(),
21921 carry_containment: false,
21922 };
21923 let child =
21924 spawn_and_mark_running(&spec, &runtime, &snapshot).expect("spawn supervised tree");
21925 let deadline = tokio::time::Instant::now() + Duration::from_secs(10);
21926 let grandchild: u32 = loop {
21927 if let Ok(contents) = std::fs::read_to_string(&pid_file) {
21928 if let Ok(pid) = contents.trim().parse() {
21929 break pid;
21930 }
21931 }
21932 assert!(
21933 tokio::time::Instant::now() < deadline,
21934 "grandchild pid was not recorded"
21935 );
21936 tokio::time::sleep(Duration::from_millis(10)).await;
21937 };
21938 assert!(
21939 running(grandchild),
21940 "grandchild must be alive before teardown"
21941 );
21942 if shutdown {
21943 let mut child = child;
21944 crate::child_roster::end_children_for_daemon_shutdown(
21945 &runtime.child_roster,
21946 false,
21947 std::future::pending(),
21948 )
21949 .await;
21950 child.wait().await.expect("reap shutdown straggler");
21951 } else {
21952 drain_child_to_state(
21953 module_id,
21954 ModuleProtocol::None,
21955 StopNotice::NotSent,
21956 &Registry::default(),
21957 None,
21958 &snapshot,
21959 &runtime.terminal_ring,
21960 &SpawnEventFeed::default(),
21961 child,
21962 Duration::from_millis(100),
21963 ModuleState::Stopped,
21964 Some(false),
21965 &FailureContext::operator(),
21966 )
21967 .await
21968 .expect("real supervisor teardown");
21969 }
21970 let deadline = tokio::time::Instant::now() + Duration::from_secs(5);
21971 while running(grandchild) && tokio::time::Instant::now() < deadline {
21972 tokio::time::sleep(Duration::from_millis(10)).await;
21973 }
21974 let survived = running(grandchild);
21975 if survived {
21977 let pid = rustix::process::Pid::from_raw(grandchild as i32).unwrap();
21978 let _ = rustix::process::kill_process(pid, rustix::process::Signal::KILL);
21979 tokio::time::sleep(Duration::from_millis(100)).await;
21980 }
21981 if module.exists() {
21982 std::fs::remove_dir(&module).expect("remove empty module cgroup");
21983 }
21984 std::fs::remove_dir(root.join("subc-modules")).unwrap();
21985 std::fs::remove_dir(&root).unwrap();
21986 assert!(
21987 !survived,
21988 "grandchild {grandchild} outlived module teardown"
21989 );
21990 eprintln!("EXECUTED {test_name}: grandchild {grandchild} killed in isolated cgroup");
21991 }
21992}
21993
21994#[cfg(all(test, unix))]
21997mod record_write_stall_tests {
21998 use super::*;
21999 use std::sync::atomic::{AtomicBool, Ordering};
22000 use subc_transport::{authenticate_client, ConnectionInfo, Endpoint, SCHEMA_VERSION};
22001
22002 const MODULE_AUTH_DEADLINE: Duration = Duration::from_secs(2);
22005 const HELD_RECORD_WRITE: Duration = Duration::from_secs(3);
22009
22010 fn sleeping_spec() -> ModuleSpec {
22022 ModuleSpec {
22023 module_id: "record-write-stall".into(),
22024 program: "/bin/sleep".into(),
22025 args: vec!["600".into()],
22026 env: vec![],
22027 reserved: false,
22028 reserved_prefixes: vec![],
22029 protocol: ModuleProtocol::None,
22030 overlap: Default::default(),
22031 carry_containment: false,
22032 }
22033 }
22034
22035 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
22036 async fn module_auth_completes_while_the_restart_waits_on_the_record_write() {
22037 let home = cortexkit_test_support::ScratchDir::new("record-write-stall");
22038 let supervisor =
22039 Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default())
22040 .with_live_children_record(home.join("live-children.json"));
22041 let module = supervisor.spawn(sleeping_spec()).unwrap();
22042 let old_pid = tokio::time::timeout(Duration::from_secs(10), async {
22047 loop {
22048 if let Some(pid) = lock_snapshot(&module.inner.snapshot)
22049 .unwrap()
22050 .reported_pid()
22051 {
22052 return pid;
22053 }
22054 tokio::time::sleep(Duration::from_millis(10)).await;
22055 }
22056 })
22057 .await
22058 .expect("the first process was admitted as the module");
22059
22060 let key = vec![0x42; 32];
22063 let daemon_id = [0x24; 16];
22064 let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
22065 let port = listener.local_addr().unwrap().port();
22066 let server = tokio::spawn(crate::server::serve_listener(
22067 listener,
22068 Arc::new(crate::router::Router::with_default_self_handler()),
22069 crate::server::ServerAuth::with_limits(
22070 key.clone(),
22071 daemon_id,
22072 "test",
22073 crate::server::DEFAULT_AUTH_DEADLINE,
22074 4,
22075 ),
22076 ));
22077
22078 let (held_tx, held_rx) = std::sync::mpsc::channel();
22081 let armed = AtomicBool::new(true);
22082 supervisor
22083 .child_roster
22084 .set_record_write_hook_for_test(move |live| {
22085 let admitting_replacement = live.values().any(|entry| entry.pid != old_pid);
22086 if admitting_replacement && armed.swap(false, Ordering::SeqCst) {
22087 let _ = held_tx.send(());
22088 std::thread::sleep(HELD_RECORD_WRITE);
22089 }
22090 });
22091
22092 let module_process = std::thread::spawn(move || {
22095 held_rx
22096 .recv_timeout(Duration::from_secs(30))
22097 .expect("the restart reached the replacement's record write");
22098 let conn = ConnectionInfo {
22099 schema: SCHEMA_VERSION,
22100 wire_version: None,
22101 endpoints: vec![Endpoint {
22102 host: "127.0.0.1".to_owned(),
22103 port,
22104 }],
22105 key,
22106 daemon_id,
22107 pid: std::process::id(),
22108 daemon_ver: "test".to_owned(),
22109 };
22110 tokio::runtime::Builder::new_current_thread()
22111 .enable_all()
22112 .build()
22113 .unwrap()
22114 .block_on(async {
22115 let started = Instant::now();
22116 let mut stream = tokio::net::TcpStream::connect(("127.0.0.1", port))
22117 .await
22118 .unwrap();
22119 let result =
22120 authenticate_client(&mut stream, &conn, MODULE_AUTH_DEADLINE).await;
22121 (result, started.elapsed())
22122 })
22123 });
22124
22125 module.restart(Some(0)).await.unwrap();
22126 let (auth, elapsed) = tokio::task::spawn_blocking(move || module_process.join().unwrap())
22127 .await
22128 .unwrap();
22129 assert!(
22130 auth.is_ok(),
22131 "the daemon must answer a module's handshake while a restart waits on the \
22132 live-children record write; got {auth:?} after {elapsed:?}"
22133 );
22134
22135 tokio::time::timeout(Duration::from_secs(20), async {
22136 while lock_snapshot(&module.inner.snapshot)
22137 .unwrap()
22138 .reported_pid()
22139 .is_none_or(|pid| pid == old_pid)
22140 {
22141 tokio::time::sleep(Duration::from_millis(10)).await;
22142 }
22143 })
22144 .await
22145 .expect("the restart published its replacement");
22146 module.stop().await.unwrap();
22147 server.abort();
22148 }
22149}
22150
22151#[cfg(all(test, unix))]
22152mod retained_status_tests {
22153 use super::*;
22154
22155 #[tokio::test]
22156 async fn cached_container_block_is_visible_in_status_and_health_without_rechecking() {
22157 use crate::live_children::{ContainerIdentity, RetainedReason};
22158 for (reason, carried, expected) in [
22159 (
22160 RetainedReason::ContainerSurvivors(vec![4242]),
22161 true,
22162 Some("waiting: container survivors (pids [4242])".to_owned()),
22163 ),
22164 (
22165 RetainedReason::ContainerUnverifiable("member inspection denied".into()),
22166 true,
22167 Some(
22168 "waiting: container survivors could not be verified: member inspection denied"
22169 .to_owned(),
22170 ),
22171 ),
22172 (RetainedReason::Survived, false, None),
22173 ] {
22174 let supervisor =
22175 Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default());
22176 let spec = ModuleSpec {
22177 module_id: "retained-status-fixture".into(),
22178 program: PathBuf::from("/not-spawned"),
22179 args: Vec::new(),
22180 env: Vec::new(),
22181 reserved: false,
22182 reserved_prefixes: Vec::new(),
22183 protocol: ModuleProtocol::None,
22184 overlap: ModuleOverlap::Exclusive,
22185 carry_containment: carried,
22186 };
22187 let module = supervisor
22188 .supervise_configured(spec.clone(), false)
22189 .unwrap();
22190 let guard = supervisor
22194 .child_roster
22195 .admit(
22196 spec.module_id,
22197 0,
22198 ModuleProtocol::None,
22199 None,
22200 crate::child_roster::RecordedIdentity {
22201 container: carried.then_some(ContainerIdentity {
22202 pgid: Some(0),
22203 created_start_time: Some(0),
22204 cgroup_path: None,
22205 ..ContainerIdentity::test_metadata()
22206 }),
22207 ..crate::child_roster::RecordedIdentity::default()
22208 },
22209 )
22210 .unwrap();
22211 guard.retain(reason);
22212 let status = module.status().unwrap();
22213 assert_eq!(status.waiting_for_previous_detail, expected.clone());
22214 assert_eq!(status.health.detail, expected);
22215 assert!(!status.process_alive);
22216 }
22217 }
22218}
22219
22220fn registration_child_is_running(child: &mut SupervisedChild) -> bool {
22221 child.try_wait().is_ok_and(|status| status.is_none())
22222}